docker/buildx
1
0
Fork 0
mirror of https://gitea.com/Lydanne/buildx.git synced 2025-05-18 00:47:48 +08:00
Code Issues Packages Projects Releases 1 Wiki Activity

allow custom annotations and labels into kubernetes manifests

Browse Source 
Co-authored-by: Akihiro Suda <suda.kyoto@gmail.com>

Signed-off-by: Jonathan Piché <jpiche@coveo.com>
This commit is contained in:
Jonathan Piché 2023-07-19 07:48:09 -04:00
parent f5719f3017
commit 9c06f383ba
4 changed files with 99 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
docs/reference/buildx_create.md
View File

@ -141,7 +141,7 @@ to achieve that.
Passes additional driver-specific options.
Note: When using quoted values for example for the `nodeselector` or
Note: When using quoted values for the `nodeselector`, `annotations`, `labels` or
`tolerations` options, ensure that quotes are escaped correctly for your shell.
#### `docker` driver
@ -165,6 +165,8 @@ No driver options.
- `limits.memory` - Sets the limit memory value specified in bytes or with a valid suffix. Example `limits.memory=500Mi`, `limits.memory=4G`
- `serviceaccount` - Sets the created pod's service account. Example `serviceaccount=example-sa`
- `"nodeselector=label1=value1,label2=value2"` - Sets the kv of `Pod` nodeSelector. No Defaults. Example `nodeselector=kubernetes.io/arch=arm64`
- `"annotations=domain/thing1=value1,domain/thing2=value2"` - Sets additional annotations on the deployments and pods. No Defaults. Example `annotations=example.com/owner=sarah`
- `"labels=domain/thing1=value1,domain/thing2=value2"` - Sets additional labels on the deployments and pods. No Defaults. Example `labels=example.com/team=rd`
- `"tolerations=key=foo,value=bar;key=foo2,operator=exists;key=foo3,effect=NoSchedule"` - Sets the `Pod` tolerations. Accepts the same values as the kube manifest tolera>tions. Key-value pairs are separated by `,`, tolerations are separated by `;`. No Defaults. Example `tolerations=operator=exists`
- `rootless=(true|false)` - Run the container as a non-root user without `securityContext.privileged`. Needs Kubernetes 1.19 or later. [Using Ubuntu host kernel is recommended](https://github.com/moby/buildkit/blob/master/docs/rootless.md). Defaults to false.
- `loadbalance=(sticky|random)` - Load-balancing strategy. If set to "sticky", the pod is chosen using the hash of the context path. Defaults to "sticky"

34
driver/kubernetes/factory.go
View File

@ -148,15 +148,20 @@ func (f *factory) processDriverOpts(deploymentName string, namespace string, cfg
case "serviceaccount":
deploymentOpt.ServiceAccountName = v
case "nodeselector":
kvs := strings.Split(strings.Trim(v, `"`), ",")
s := map[string]string{}
for i := range kvs {
kv := strings.Split(kvs[i], "=")
if len(kv) == 2 {
s[kv[0]] = kv[1]
}
deploymentOpt.NodeSelector, err = splitMultiValues(v, ",", "=")
if err != nil {
return nil, "", "", errors.Wrap(err, "cannot parse node selector")
}
case "annotations":
deploymentOpt.CustomAnnotations, err = splitMultiValues(v, ",", "=")
if err != nil {
return nil, "", "", errors.Wrap(err, "cannot parse annotations")
}
case "labels":
deploymentOpt.CustomLabels, err = splitMultiValues(v, ",", "=")
if err != nil {
return nil, "", "", errors.Wrap(err, "cannot parse labels")
}
deploymentOpt.NodeSelector = s
case "tolerations":
ts := strings.Split(v, ";")
deploymentOpt.Tolerations = []corev1.Toleration{}
@ -217,6 +222,19 @@ func (f *factory) processDriverOpts(deploymentName string, namespace string, cfg
return deploymentOpt, loadbalance, namespace, nil
}
func splitMultiValues(in string, itemsep string, kvsep string) (map[string]string, error) {
kvs := strings.Split(strings.Trim(in, `"`), itemsep)
s := map[string]string{}
for i := range kvs {
kv := strings.Split(kvs[i], kvsep)
if len(kv) != 2 {
return nil, errors.Errorf("invalid key-value pair: %s", kvs[i])
}
s[kv[0]] = kv[1]
}
return s, nil
}
func (f *factory) AllowsInstances() bool {
return true
}

54
driver/kubernetes/factory_test.go
View File

@ -47,13 +47,13 @@ func TestFactory_processDriverOpts(t *testing.T) {
"rootless": "true",
"nodeselector": "selector1=value1,selector2=value2",
"tolerations": "key=tolerationKey1,value=tolerationValue1,operator=Equal,effect=NoSchedule,tolerationSeconds=60;key=tolerationKey2,operator=Exists",
"annotations": "example.com/expires-after=annotation1,example.com/other=annotation2",
"labels": "example.com/owner=label1,example.com/other=label2",
"loadbalance": "random",
"qemu.install": "true",
"qemu.image": "qemu:latest",
}
ns := "test"
r, loadbalance, ns, err := f.processDriverOpts(cfg.Name, ns, cfg)
r, loadbalance, ns, err := f.processDriverOpts(cfg.Name, "test", cfg)
nodeSelectors := map[string]string{
"selector1": "value1",
@ -75,6 +75,16 @@ func TestFactory_processDriverOpts(t *testing.T) {
},
}
customAnnotations := map[string]string{
"example.com/expires-after": "annotation1",
"example.com/other": "annotation2",
}
customLabels := map[string]string{
"example.com/owner": "label1",
"example.com/other": "label2",
}
require.NoError(t, err)
require.Equal(t, "test-ns", ns)
@ -86,6 +96,8 @@ func TestFactory_processDriverOpts(t *testing.T) {
require.Equal(t, "64Mi", r.LimitsMemory)
require.True(t, r.Rootless)
require.Equal(t, nodeSelectors, r.NodeSelector)
require.Equal(t, customAnnotations, r.CustomAnnotations)
require.Equal(t, customLabels, r.CustomLabels)
require.Equal(t, tolerations, r.Tolerations)
require.Equal(t, LoadbalanceRandom, loadbalance)
require.True(t, r.Qemu.Install)
@ -110,6 +122,8 @@ func TestFactory_processDriverOpts(t *testing.T) {
require.Equal(t, "", r.LimitsMemory)
require.False(t, r.Rootless)
require.Empty(t, r.NodeSelector)
require.Empty(t, r.CustomAnnotations)
require.Empty(t, r.CustomLabels)
require.Empty(t, r.Tolerations)
require.Equal(t, LoadbalanceSticky, loadbalance)
require.False(t, r.Qemu.Install)
@ -137,6 +151,8 @@ func TestFactory_processDriverOpts(t *testing.T) {
require.Equal(t, "", r.LimitsMemory)
require.True(t, r.Rootless)
require.Empty(t, r.NodeSelector)
require.Empty(t, r.CustomAnnotations)
require.Empty(t, r.CustomLabels)
require.Empty(t, r.Tolerations)
require.Equal(t, LoadbalanceSticky, loadbalance)
require.False(t, r.Qemu.Install)
@ -149,9 +165,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
cfg.DriverOpts = map[string]string{
"replicas": "invalid",
}
_, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
require.Error(t, err)
},
)
@ -161,9 +175,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
cfg.DriverOpts = map[string]string{
"rootless": "invalid",
}
_, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
require.Error(t, err)
},
)
@ -173,9 +185,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
cfg.DriverOpts = map[string]string{
"tolerations": "key=foo,value=bar,invalid=foo2",
}
_, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
require.Error(t, err)
},
)
@ -185,9 +195,27 @@ func TestFactory_processDriverOpts(t *testing.T) {
cfg.DriverOpts = map[string]string{
"tolerations": "key=foo,value=bar,tolerationSeconds=invalid",
}
_, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
require.Error(t, err)
},
)
t.Run(
"InvalidCustomAnnotation", func(t *testing.T) {
cfg.DriverOpts = map[string]string{
"annotations": "key,value",
}
_, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
require.Error(t, err)
},
)
t.Run(
"InvalidCustomLabel", func(t *testing.T) {
cfg.DriverOpts = map[string]string{
"labels": "key=value=foo",
}
_, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
require.Error(t, err)
},
)
@ -197,9 +225,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
cfg.DriverOpts = map[string]string{
"loadbalance": "invalid",
}
_, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
require.Error(t, err)
},
)
@ -209,9 +235,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
cfg.DriverOpts = map[string]string{
"qemu.install": "invalid",
}
_, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
require.Error(t, err)
},
)
@ -221,9 +245,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
cfg.DriverOpts = map[string]string{
"invalid": "foo",
}
_, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
require.Error(t, err)
},
)

41
driver/kubernetes/manifest/manifest.go
View File

@ -7,6 +7,7 @@ import (
"github.com/docker/buildx/util/platformutil"
v1 "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
@ -31,24 +32,32 @@ type DeploymentOpt struct {
// files mounted at /etc/buildkitd
ConfigFiles map[string][]byte
Rootless bool
NodeSelector map[string]string
Tolerations []corev1.Toleration
RequestsCPU string
RequestsMemory string
LimitsCPU string
LimitsMemory string
Platforms []v1.Platform
Rootless bool
NodeSelector map[string]string
CustomAnnotations map[string]string
CustomLabels map[string]string
Tolerations []corev1.Toleration
RequestsCPU string
RequestsMemory string
LimitsCPU string
LimitsMemory string
Platforms []v1.Platform
}
const (
containerName = "buildkitd"
AnnotationPlatform = "buildx.docker.com/platform"
LabelApp = "app"
)
var (
ErrReservedAnnotationPlatform = errors.Errorf("the annotation \"%s\" is reserved and cannot be customized", AnnotationPlatform)
ErrReservedLabelApp = errors.Errorf("the label \"%s\" is reserved and cannot be customized", LabelApp)
)
func NewDeployment(opt *DeploymentOpt) (d *appsv1.Deployment, c []*corev1.ConfigMap, err error) {
labels := map[string]string{
"app": opt.Name,
LabelApp: opt.Name,
}
annotations := map[string]string{}
replicas := int32(opt.Replicas)
@ -59,6 +68,20 @@ func NewDeployment(opt *DeploymentOpt) (d *appsv1.Deployment, c []*corev1.Config
annotations[AnnotationPlatform] = strings.Join(platformutil.Format(opt.Platforms), ",")
}
for k, v := range opt.CustomAnnotations {
if k == AnnotationPlatform {
return nil, nil, ErrReservedAnnotationPlatform
}
annotations[k] = v
}
for k, v := range opt.CustomLabels {
if k == LabelApp {
return nil, nil, ErrReservedLabelApp
}
labels[k] = v
}
d = &appsv1.Deployment{
TypeMeta: metav1.TypeMeta{
APIVersion: appsv1.SchemeGroupVersion.String(),