mirror of
https://gitea.com/Lydanne/buildx.git
synced 2025-05-18 00:47:48 +08:00
336 lines
8.7 KiB
Go
336 lines
8.7 KiB
Go
package kubernetes
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/docker/buildx/driver"
|
|
"github.com/docker/buildx/driver/bkimage"
|
|
ctxkube "github.com/docker/buildx/driver/kubernetes/context"
|
|
"github.com/docker/buildx/driver/kubernetes/manifest"
|
|
"github.com/docker/buildx/driver/kubernetes/podchooser"
|
|
dockerclient "github.com/docker/docker/client"
|
|
"github.com/pkg/errors"
|
|
"github.com/sirupsen/logrus"
|
|
corev1 "k8s.io/api/core/v1"
|
|
"k8s.io/client-go/kubernetes"
|
|
"k8s.io/client-go/rest"
|
|
)
|
|
|
|
const (
|
|
prioritySupported = 40
|
|
priorityUnsupported = 80
|
|
defaultTimeout = 120 * time.Second
|
|
)
|
|
|
|
type ClientConfig interface {
|
|
ClientConfig() (*rest.Config, error)
|
|
Namespace() (string, bool, error)
|
|
}
|
|
|
|
type ClientConfigInCluster struct{}
|
|
|
|
func (k ClientConfigInCluster) ClientConfig() (*rest.Config, error) {
|
|
return rest.InClusterConfig()
|
|
}
|
|
|
|
func (k ClientConfigInCluster) Namespace() (string, bool, error) {
|
|
namespace, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace")
|
|
if err != nil {
|
|
return "", false, err
|
|
}
|
|
return strings.TrimSpace(string(namespace)), true, nil
|
|
}
|
|
|
|
func init() {
|
|
driver.Register(&factory{})
|
|
}
|
|
|
|
type factory struct {
|
|
cc ClientConfig // used for testing
|
|
}
|
|
|
|
func (*factory) Name() string {
|
|
return DriverName
|
|
}
|
|
|
|
func (*factory) Usage() string {
|
|
return DriverName
|
|
}
|
|
|
|
func (*factory) Priority(ctx context.Context, endpoint string, api dockerclient.APIClient, dialMeta map[string][]string) int {
|
|
if api == nil {
|
|
return priorityUnsupported
|
|
}
|
|
return prioritySupported
|
|
}
|
|
|
|
func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver, error) {
|
|
var err error
|
|
var cc ClientConfig
|
|
if f.cc != nil {
|
|
cc = f.cc
|
|
} else {
|
|
cc, err = ctxkube.ConfigFromEndpoint(cfg.EndpointAddr, cfg.ContextStore)
|
|
if err != nil {
|
|
// err is returned if cfg.EndpointAddr is non-context name like "unix:///var/run/docker.sock".
|
|
// try again with name="default".
|
|
// FIXME(@AkihiroSuda): cfg should retain real context name.
|
|
cc, err = ctxkube.ConfigFromEndpoint("default", cfg.ContextStore)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
}
|
|
tryToUseConfigInCluster := false
|
|
if cc == nil {
|
|
tryToUseConfigInCluster = true
|
|
} else {
|
|
if _, err := cc.ClientConfig(); err != nil {
|
|
tryToUseConfigInCluster = true
|
|
}
|
|
}
|
|
if tryToUseConfigInCluster {
|
|
ccInCluster := ClientConfigInCluster{}
|
|
if _, err := ccInCluster.ClientConfig(); err == nil {
|
|
logrus.Debug("using kube config in cluster")
|
|
cc = ccInCluster
|
|
}
|
|
}
|
|
if cc == nil {
|
|
return nil, errors.Errorf("%s driver requires kubernetes API access", DriverName)
|
|
}
|
|
}
|
|
|
|
deploymentName, err := buildxNameToDeploymentName(cfg.Name)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
namespace, _, err := cc.Namespace()
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "cannot determine Kubernetes namespace, specify manually")
|
|
}
|
|
restClientConfig, err := cc.ClientConfig()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
clientset, err := kubernetes.NewForConfig(restClientConfig)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
d := &Driver{
|
|
factory: f,
|
|
clientConfig: cc,
|
|
InitConfig: cfg,
|
|
clientset: clientset,
|
|
}
|
|
|
|
deploymentOpt, loadbalance, namespace, defaultLoad, timeout, err := f.processDriverOpts(deploymentName, namespace, cfg)
|
|
if nil != err {
|
|
return nil, err
|
|
}
|
|
|
|
d.defaultLoad = defaultLoad
|
|
d.timeout = timeout
|
|
|
|
d.deployment, d.configMaps, err = manifest.NewDeployment(deploymentOpt)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
d.minReplicas = deploymentOpt.Replicas
|
|
|
|
d.deploymentClient = clientset.AppsV1().Deployments(namespace)
|
|
d.podClient = clientset.CoreV1().Pods(namespace)
|
|
d.configMapClient = clientset.CoreV1().ConfigMaps(namespace)
|
|
|
|
switch loadbalance {
|
|
case LoadbalanceSticky:
|
|
d.podChooser = &podchooser.StickyPodChooser{
|
|
Key: cfg.ContextPathHash,
|
|
PodClient: d.podClient,
|
|
Deployment: d.deployment,
|
|
}
|
|
case LoadbalanceRandom:
|
|
d.podChooser = &podchooser.RandomPodChooser{
|
|
PodClient: d.podClient,
|
|
Deployment: d.deployment,
|
|
}
|
|
}
|
|
return d, nil
|
|
}
|
|
|
|
func (f *factory) processDriverOpts(deploymentName string, namespace string, cfg driver.InitConfig) (*manifest.DeploymentOpt, string, string, bool, time.Duration, error) {
|
|
deploymentOpt := &manifest.DeploymentOpt{
|
|
Name: deploymentName,
|
|
Image: bkimage.DefaultImage,
|
|
Replicas: 1,
|
|
BuildkitFlags: cfg.BuildkitdFlags,
|
|
Rootless: false,
|
|
Platforms: cfg.Platforms,
|
|
ConfigFiles: cfg.Files,
|
|
}
|
|
|
|
defaultLoad := false
|
|
timeout := defaultTimeout
|
|
|
|
deploymentOpt.Qemu.Image = bkimage.QemuImage
|
|
|
|
loadbalance := LoadbalanceSticky
|
|
var err error
|
|
|
|
for k, v := range cfg.DriverOpts {
|
|
switch k {
|
|
case "image":
|
|
if v != "" {
|
|
deploymentOpt.Image = v
|
|
}
|
|
case "namespace":
|
|
namespace = v
|
|
case "replicas":
|
|
deploymentOpt.Replicas, err = strconv.Atoi(v)
|
|
if err != nil {
|
|
return nil, "", "", false, 0, err
|
|
}
|
|
case "requests.cpu":
|
|
deploymentOpt.RequestsCPU = v
|
|
case "requests.memory":
|
|
deploymentOpt.RequestsMemory = v
|
|
case "requests.ephemeral-storage":
|
|
deploymentOpt.RequestsEphemeralStorage = v
|
|
case "limits.cpu":
|
|
deploymentOpt.LimitsCPU = v
|
|
case "limits.memory":
|
|
deploymentOpt.LimitsMemory = v
|
|
case "limits.ephemeral-storage":
|
|
deploymentOpt.LimitsEphemeralStorage = v
|
|
case "rootless":
|
|
deploymentOpt.Rootless, err = strconv.ParseBool(v)
|
|
if err != nil {
|
|
return nil, "", "", false, 0, err
|
|
}
|
|
if _, isImage := cfg.DriverOpts["image"]; !isImage {
|
|
deploymentOpt.Image = bkimage.DefaultRootlessImage
|
|
}
|
|
case "schedulername":
|
|
deploymentOpt.SchedulerName = v
|
|
case "serviceaccount":
|
|
deploymentOpt.ServiceAccountName = v
|
|
case "nodeselector":
|
|
deploymentOpt.NodeSelector, err = splitMultiValues(v, ",", "=")
|
|
if err != nil {
|
|
return nil, "", "", false, 0, errors.Wrap(err, "cannot parse node selector")
|
|
}
|
|
case "annotations":
|
|
deploymentOpt.CustomAnnotations, err = splitMultiValues(v, ",", "=")
|
|
if err != nil {
|
|
return nil, "", "", false, 0, errors.Wrap(err, "cannot parse annotations")
|
|
}
|
|
case "labels":
|
|
deploymentOpt.CustomLabels, err = splitMultiValues(v, ",", "=")
|
|
if err != nil {
|
|
return nil, "", "", false, 0, errors.Wrap(err, "cannot parse labels")
|
|
}
|
|
case "tolerations":
|
|
ts := strings.Split(v, ";")
|
|
deploymentOpt.Tolerations = []corev1.Toleration{}
|
|
for i := range ts {
|
|
kvs := strings.Split(ts[i], ",")
|
|
|
|
t := corev1.Toleration{}
|
|
|
|
for j := range kvs {
|
|
kv := strings.Split(kvs[j], "=")
|
|
if len(kv) == 2 {
|
|
switch kv[0] {
|
|
case "key":
|
|
t.Key = kv[1]
|
|
case "operator":
|
|
t.Operator = corev1.TolerationOperator(kv[1])
|
|
case "value":
|
|
t.Value = kv[1]
|
|
case "effect":
|
|
t.Effect = corev1.TaintEffect(kv[1])
|
|
case "tolerationSeconds":
|
|
c, err := strconv.Atoi(kv[1])
|
|
if nil != err {
|
|
return nil, "", "", false, 0, err
|
|
}
|
|
c64 := int64(c)
|
|
t.TolerationSeconds = &c64
|
|
default:
|
|
return nil, "", "", false, 0, errors.Errorf("invalid tolaration %q", v)
|
|
}
|
|
}
|
|
}
|
|
|
|
deploymentOpt.Tolerations = append(deploymentOpt.Tolerations, t)
|
|
}
|
|
case "loadbalance":
|
|
switch v {
|
|
case LoadbalanceSticky:
|
|
case LoadbalanceRandom:
|
|
default:
|
|
return nil, "", "", false, 0, errors.Errorf("invalid loadbalance %q", v)
|
|
}
|
|
loadbalance = v
|
|
case "qemu.install":
|
|
deploymentOpt.Qemu.Install, err = strconv.ParseBool(v)
|
|
if err != nil {
|
|
return nil, "", "", false, 0, err
|
|
}
|
|
case "qemu.image":
|
|
if v != "" {
|
|
deploymentOpt.Qemu.Image = v
|
|
}
|
|
case "default-load":
|
|
defaultLoad, err = strconv.ParseBool(v)
|
|
if err != nil {
|
|
return nil, "", "", false, 0, err
|
|
}
|
|
case "timeout":
|
|
timeout, err = time.ParseDuration(v)
|
|
if err != nil {
|
|
return nil, "", "", false, 0, errors.Wrap(err, "cannot parse timeout")
|
|
}
|
|
default:
|
|
return nil, "", "", false, 0, errors.Errorf("invalid driver option %s for driver %s", k, DriverName)
|
|
}
|
|
}
|
|
|
|
return deploymentOpt, loadbalance, namespace, defaultLoad, timeout, nil
|
|
}
|
|
|
|
func splitMultiValues(in string, itemsep string, kvsep string) (map[string]string, error) {
|
|
kvs := strings.Split(strings.Trim(in, `"`), itemsep)
|
|
s := map[string]string{}
|
|
for i := range kvs {
|
|
kv := strings.Split(kvs[i], kvsep)
|
|
if len(kv) != 2 {
|
|
return nil, errors.Errorf("invalid key-value pair: %s", kvs[i])
|
|
}
|
|
s[kv[0]] = kv[1]
|
|
}
|
|
return s, nil
|
|
}
|
|
|
|
func (f *factory) AllowsInstances() bool {
|
|
return true
|
|
}
|
|
|
|
// buildxNameToDeploymentName converts buildx name to Kubernetes Deployment name.
|
|
//
|
|
// eg. "buildx_buildkit_loving_mendeleev0" -> "loving-mendeleev0"
|
|
func buildxNameToDeploymentName(bx string) (string, error) {
|
|
// TODO: commands.util.go should not pass "buildx_buildkit_" prefix to drivers
|
|
s, err := driver.ParseBuilderName(bx)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
s = strings.ReplaceAll(s, "_", "-")
|
|
return s, nil
|
|
}
|