docker/buildx
1
0
Fork 0
mirror of https://gitea.com/Lydanne/buildx.git synced 2025-05-18 00:47:48 +08:00
Code Issues Packages Projects Releases 1 Wiki Activity

k8s: rename timeout opt and move it out of deployment manifest

Browse Source 
Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com>
This commit is contained in:
CrazyMax 2024-06-03 10:30:06 +02:00
parent 53b7cbc5cb
commit f30e143428
No known key found for this signature in database
GPG Key ID: ADE44D8C9D44FBE4
4 changed files with 99 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
driver/kubernetes/driver.go
View File

@ -14,6 +14,7 @@ import (
"github.com/docker/buildx/store"
"github.com/docker/buildx/util/platformutil"
"github.com/docker/buildx/util/progress"
"github.com/docker/go-units"
"github.com/moby/buildkit/client"
"github.com/pkg/errors"
appsv1 "k8s.io/api/apps/v1"
@ -41,16 +42,16 @@ type Driver struct {
// if you add fields, remember to update docs:
// https://github.com/docker/docs/blob/main/content/build/drivers/kubernetes.md
minReplicas int
deployment *appsv1.Deployment
configMaps []*corev1.ConfigMap
clientset *kubernetes.Clientset
deploymentClient clientappsv1.DeploymentInterface
podClient clientcorev1.PodInterface
configMapClient clientcorev1.ConfigMapInterface
podChooser podchooser.PodChooser
defaultLoad bool
provisioningTimeout time.Duration
minReplicas int
deployment *appsv1.Deployment
configMaps []*corev1.ConfigMap
clientset *kubernetes.Clientset
deploymentClient clientappsv1.DeploymentInterface
podClient clientcorev1.PodInterface
configMapClient clientcorev1.ConfigMapInterface
podChooser podchooser.PodChooser
defaultLoad bool
timeout time.Duration
}
func (d *Driver) IsMobyDriver() bool {
@ -89,7 +90,7 @@ func (d *Driver) Bootstrap(ctx context.Context, l progress.Logger) error {
}
}
return sub.Wrap(
fmt.Sprintf("waiting for %d pods to be ready, timeout: %ds", d.minReplicas, d.provisioningTimeout/time.Second),
fmt.Sprintf("waiting for %d pods to be ready, timeout: %s", d.minReplicas, units.HumanDuration(d.timeout)),
func() error {
return d.wait(ctx)
})
@ -102,19 +103,18 @@ func (d *Driver) wait(ctx context.Context) error {
err error
depl *appsv1.Deployment
)
timeoutChan := time.After(d.provisioningTimeout)
timeoutChan := time.After(d.timeout)
ticker := time.NewTicker(100 * time.Millisecond)
tryCounter := 0
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return ctx.Err()
case <-timeoutChan:
return err
case <-ticker.C:
if tryCounter < 100 {
tryCounter++
}
ticker.Stop()
ticker = time.NewTicker(time.Duration(100+tryCounter*20) * time.Millisecond)
depl, err = d.deploymentClient.Get(ctx, d.deployment.Name, metav1.GetOptions{})
if err == nil {
if depl.Status.ReadyReplicas >= int32(d.minReplicas) {
@ -122,13 +122,8 @@ func (d *Driver) wait(ctx context.Context) error {
}
err = errors.Errorf("expected %d replicas to be ready, got %d", d.minReplicas, depl.Status.ReadyReplicas)
}
case <-timeoutChan:
return errors.Errorf("timeout after %s: last error: %v", d.provisioningTimeout, err)
case <-ctx.Done():
return ctx.Err()
}
}
}
func (d *Driver) Info(ctx context.Context) (*driver.Info, error) {

61
driver/kubernetes/factory.go
View File

@ -17,8 +17,11 @@ import (
"k8s.io/client-go/kubernetes"
)
const prioritySupported = 40
const priorityUnsupported = 80
const (
prioritySupported = 40
priorityUnsupported = 80
defaultTimeout = 120 * time.Second
)
func init() {
driver.Register(&factory{})
@ -69,12 +72,13 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
clientset: clientset,
}
deploymentOpt, loadbalance, namespace, defaultLoad, err := f.processDriverOpts(deploymentName, namespace, cfg)
deploymentOpt, loadbalance, namespace, defaultLoad, timeout, err := f.processDriverOpts(deploymentName, namespace, cfg)
if nil != err {
return nil, err
}
d.defaultLoad = defaultLoad
d.timeout = timeout
d.deployment, d.configMaps, err = manifest.NewDeployment(deploymentOpt)
if err != nil {
@ -83,8 +87,6 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
d.minReplicas = deploymentOpt.Replicas
d.provisioningTimeout = deploymentOpt.ProvisioningTimeout
d.deploymentClient = clientset.AppsV1().Deployments(namespace)
d.podClient = clientset.CoreV1().Pods(namespace)
d.configMapClient = clientset.CoreV1().ConfigMaps(namespace)
@ -105,19 +107,19 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
return d, nil
}
func (f *factory) processDriverOpts(deploymentName string, namespace string, cfg driver.InitConfig) (*manifest.DeploymentOpt, string, string, bool, error) {
func (f *factory) processDriverOpts(deploymentName string, namespace string, cfg driver.InitConfig) (*manifest.DeploymentOpt, string, string, bool, time.Duration, error) {
deploymentOpt := &manifest.DeploymentOpt{
Name: deploymentName,
Image: bkimage.DefaultImage,
Replicas: 1,
ProvisioningTimeout: 120 * time.Second,
BuildkitFlags: cfg.BuildkitdFlags,
Rootless: false,
Platforms: cfg.Platforms,
ConfigFiles: cfg.Files,
Name: deploymentName,
Image: bkimage.DefaultImage,
Replicas: 1,
BuildkitFlags: cfg.BuildkitdFlags,
Rootless: false,
Platforms: cfg.Platforms,
ConfigFiles: cfg.Files,
}
defaultLoad := false
timeout := defaultTimeout
deploymentOpt.Qemu.Image = bkimage.QemuImage
@ -135,7 +137,7 @@ func (f *factory) processDriverOpts(deploymentName string, namespace string, cfg
case "replicas":
deploymentOpt.Replicas, err = strconv.Atoi(v)
if err != nil {
return nil, "", "", false, err
return nil, "", "", false, 0, err
}
case "requests.cpu":
deploymentOpt.RequestsCPU = v
@ -152,7 +154,7 @@ func (f *factory) processDriverOpts(deploymentName string, namespace string, cfg
case "rootless":
deploymentOpt.Rootless, err = strconv.ParseBool(v)
if err != nil {
return nil, "", "", false, err
return nil, "", "", false, 0, err
}
if _, isImage := cfg.DriverOpts["image"]; !isImage {
deploymentOpt.Image = bkimage.DefaultRootlessImage
@ -164,17 +166,17 @@ func (f *factory) processDriverOpts(deploymentName string, namespace string, cfg
case "nodeselector":
deploymentOpt.NodeSelector, err = splitMultiValues(v, ",", "=")
if err != nil {
return nil, "", "", false, errors.Wrap(err, "cannot parse node selector")
return nil, "", "", false, 0, errors.Wrap(err, "cannot parse node selector")
}
case "annotations":
deploymentOpt.CustomAnnotations, err = splitMultiValues(v, ",", "=")
if err != nil {
return nil, "", "", false, errors.Wrap(err, "cannot parse annotations")
return nil, "", "", false, 0, errors.Wrap(err, "cannot parse annotations")
}
case "labels":
deploymentOpt.CustomLabels, err = splitMultiValues(v, ",", "=")
if err != nil {
return nil, "", "", false, errors.Wrap(err, "cannot parse labels")
return nil, "", "", false, 0, errors.Wrap(err, "cannot parse labels")
}
case "tolerations":
ts := strings.Split(v, ";")
@ -199,12 +201,12 @@ func (f *factory) processDriverOpts(deploymentName string, namespace string, cfg
case "tolerationSeconds":
c, err := strconv.Atoi(kv[1])
if nil != err {
return nil, "", "", false, err
return nil, "", "", false, 0, err
}
c64 := int64(c)
t.TolerationSeconds = &c64
default:
return nil, "", "", false, errors.Errorf("invalid tolaration %q", v)
return nil, "", "", false, 0, errors.Errorf("invalid tolaration %q", v)
}
}
}
@ -216,13 +218,13 @@ func (f *factory) processDriverOpts(deploymentName string, namespace string, cfg
case LoadbalanceSticky:
case LoadbalanceRandom:
default:
return nil, "", "", false, errors.Errorf("invalid loadbalance %q", v)
return nil, "", "", false, 0, errors.Errorf("invalid loadbalance %q", v)
}
loadbalance = v
case "qemu.install":
deploymentOpt.Qemu.Install, err = strconv.ParseBool(v)
if err != nil {
return nil, "", "", false, err
return nil, "", "", false, 0, err
}
case "qemu.image":
if v != "" {
@ -231,16 +233,19 @@ func (f *factory) processDriverOpts(deploymentName string, namespace string, cfg
case "default-load":
defaultLoad, err = strconv.ParseBool(v)
if err != nil {
return nil, "", "", false, err
return nil, "", "", false, 0, err
}
case "timeout":
timeout, err = time.ParseDuration(v)
if err != nil {
return nil, "", "", false, 0, errors.Wrap(err, "cannot parse timeout")
}
case "provisioningTimeout":
deploymentOpt.ProvisioningTimeout, err = time.ParseDuration(v)
default:
return nil, "", "", false, errors.Errorf("invalid driver option %s for driver %s", k, DriverName)
return nil, "", "", false, 0, errors.Errorf("invalid driver option %s for driver %s", k, DriverName)
}
}
return deploymentOpt, loadbalance, namespace, defaultLoad, nil
return deploymentOpt, loadbalance, namespace, defaultLoad, timeout, nil
}
func splitMultiValues(in string, itemsep string, kvsep string) (map[string]string, error) {

84
driver/kubernetes/factory_test.go
View File

@ -38,25 +38,25 @@ func TestFactory_processDriverOpts(t *testing.T) {
t.Run(
"ValidOptions", func(t *testing.T) {
cfg.DriverOpts = map[string]string{
"namespace": "test-ns",
"image": "test:latest",
"replicas": "2",
"provisioningTimeout": "300s",
"requests.cpu": "100m",
"requests.memory": "32Mi",
"limits.cpu": "200m",
"limits.memory": "64Mi",
"rootless": "true",
"nodeselector": "selector1=value1,selector2=value2",
"tolerations": "key=tolerationKey1,value=tolerationValue1,operator=Equal,effect=NoSchedule,tolerationSeconds=60;key=tolerationKey2,operator=Exists",
"annotations": "example.com/expires-after=annotation1,example.com/other=annotation2",
"labels": "example.com/owner=label1,example.com/other=label2",
"loadbalance": "random",
"qemu.install": "true",
"qemu.image": "qemu:latest",
"default-load": "true",
"namespace": "test-ns",
"image": "test:latest",
"replicas": "2",
"timeout": "300s",
"requests.cpu": "100m",
"requests.memory": "32Mi",
"limits.cpu": "200m",
"limits.memory": "64Mi",
"rootless": "true",
"nodeselector": "selector1=value1,selector2=value2",
"tolerations": "key=tolerationKey1,value=tolerationValue1,operator=Equal,effect=NoSchedule,tolerationSeconds=60;key=tolerationKey2,operator=Exists",
"annotations": "example.com/expires-after=annotation1,example.com/other=annotation2",
"labels": "example.com/owner=label1,example.com/other=label2",
"loadbalance": "random",
"qemu.install": "true",
"qemu.image": "qemu:latest",
"default-load": "true",
}
r, loadbalance, ns, defaultLoad, err := f.processDriverOpts(cfg.Name, "test", cfg)
r, loadbalance, ns, defaultLoad, timeout, err := f.processDriverOpts(cfg.Name, "test", cfg)
nodeSelectors := map[string]string{
"selector1": "value1",
@ -93,7 +93,6 @@ func TestFactory_processDriverOpts(t *testing.T) {
require.Equal(t, "test-ns", ns)
require.Equal(t, "test:latest", r.Image)
require.Equal(t, 2, r.Replicas)
require.Equal(t, 300*time.Second, r.ProvisioningTimeout)
require.Equal(t, "100m", r.RequestsCPU)
require.Equal(t, "32Mi", r.RequestsMemory)
require.Equal(t, "200m", r.LimitsCPU)
@ -107,6 +106,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
require.True(t, r.Qemu.Install)
require.Equal(t, "qemu:latest", r.Qemu.Image)
require.True(t, defaultLoad)
require.Equal(t, 300*time.Second, timeout)
},
)
@ -114,14 +114,13 @@ func TestFactory_processDriverOpts(t *testing.T) {
"NoOptions", func(t *testing.T) {
cfg.DriverOpts = map[string]string{}
r, loadbalance, ns, defaultLoad, err := f.processDriverOpts(cfg.Name, "test", cfg)
r, loadbalance, ns, defaultLoad, timeout, err := f.processDriverOpts(cfg.Name, "test", cfg)
require.NoError(t, err)
require.Equal(t, "test", ns)
require.Equal(t, bkimage.DefaultImage, r.Image)
require.Equal(t, 1, r.Replicas)
require.Equal(t, 120*time.Second, r.ProvisioningTimeout)
require.Equal(t, "", r.RequestsCPU)
require.Equal(t, "", r.RequestsMemory)
require.Equal(t, "", r.LimitsCPU)
@ -135,6 +134,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
require.False(t, r.Qemu.Install)
require.Equal(t, bkimage.QemuImage, r.Qemu.Image)
require.False(t, defaultLoad)
require.Equal(t, 120*time.Second, timeout)
},
)
@ -145,14 +145,13 @@ func TestFactory_processDriverOpts(t *testing.T) {
"loadbalance": "sticky",
}
r, loadbalance, ns, defaultLoad, err := f.processDriverOpts(cfg.Name, "test", cfg)
r, loadbalance, ns, defaultLoad, timeout, err := f.processDriverOpts(cfg.Name, "test", cfg)
require.NoError(t, err)
require.Equal(t, "test", ns)
require.Equal(t, bkimage.DefaultRootlessImage, r.Image)
require.Equal(t, 1, r.Replicas)
require.Equal(t, 120*time.Second, r.ProvisioningTimeout)
require.Equal(t, "", r.RequestsCPU)
require.Equal(t, "", r.RequestsMemory)
require.Equal(t, "", r.LimitsCPU)
@ -166,6 +165,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
require.False(t, r.Qemu.Install)
require.Equal(t, bkimage.QemuImage, r.Qemu.Image)
require.False(t, defaultLoad)
require.Equal(t, 120*time.Second, timeout)
},
)
@ -174,17 +174,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
cfg.DriverOpts = map[string]string{
"replicas": "invalid",
}
_, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
require.Error(t, err)
},
)
t.Run(
"InvalidProvisioningTimeout", func(t *testing.T) {
cfg.DriverOpts = map[string]string{
"ProvisioningTimeout": "invalid",
}
_, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
_, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
require.Error(t, err)
},
)
@ -194,7 +184,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
cfg.DriverOpts = map[string]string{
"rootless": "invalid",
}
_, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
_, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
require.Error(t, err)
},
)
@ -204,7 +194,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
cfg.DriverOpts = map[string]string{
"tolerations": "key=foo,value=bar,invalid=foo2",
}
_, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
_, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
require.Error(t, err)
},
)
@ -214,7 +204,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
cfg.DriverOpts = map[string]string{
"tolerations": "key=foo,value=bar,tolerationSeconds=invalid",
}
_, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
_, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
require.Error(t, err)
},
)
@ -224,7 +214,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
cfg.DriverOpts = map[string]string{
"annotations": "key,value",
}
_, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
_, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
require.Error(t, err)
},
)
@ -234,7 +224,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
cfg.DriverOpts = map[string]string{
"labels": "key=value=foo",
}
_, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
_, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
require.Error(t, err)
},
)
@ -244,7 +234,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
cfg.DriverOpts = map[string]string{
"loadbalance": "invalid",
}
_, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
_, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
require.Error(t, err)
},
)
@ -254,7 +244,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
cfg.DriverOpts = map[string]string{
"qemu.install": "invalid",
}
_, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
_, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
require.Error(t, err)
},
)
@ -264,7 +254,17 @@ func TestFactory_processDriverOpts(t *testing.T) {
cfg.DriverOpts = map[string]string{
"invalid": "foo",
}
_, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
_, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
require.Error(t, err)
},
)
t.Run(
"InvalidTimeout", func(t *testing.T) {
cfg.DriverOpts = map[string]string{
"timeout": "invalid",
}
_, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
require.Error(t, err)
},
)

14
driver/kubernetes/manifest/manifest.go
View File

@ -4,7 +4,6 @@ import (
"fmt"
"path"
"strings"
"time"
"github.com/docker/buildx/util/platformutil"
v1 "github.com/opencontainers/image-spec/specs-go/v1"
@ -16,13 +15,12 @@ import (
)
type DeploymentOpt struct {
Namespace string
Name string
Image string
Replicas int
ProvisioningTimeout time.Duration
ServiceAccountName string
SchedulerName string
Namespace string
Name string
Image string
Replicas int
ServiceAccountName string
SchedulerName string
// Qemu
Qemu struct {