kubernetes: store config files for k8s

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

driver/kubernetes/driver.go

@@ -41,7 +41,7 @@ type Driver struct {
 	factory          driver.Factory
 	minReplicas      int
 	deployment       *appsv1.Deployment
-	configMap        *corev1.ConfigMap
+	configMaps       []*corev1.ConfigMap
 	clientset        *kubernetes.Clientset
 	deploymentClient clientappsv1.DeploymentInterface
 	podClient        clientcorev1.PodInterface
@@ -65,16 +65,16 @@ func (d *Driver) Bootstrap(ctx context.Context, l progress.Logger) error {
 				return errors.Wrapf(err, "error for bootstrap %q", d.deployment.Name)
 			}
 
-			if d.configMap != nil {
+			for _, cfg := range d.configMaps {
 				// create ConfigMap first if exists
-				_, err = d.configMapClient.Create(ctx, d.configMap, metav1.CreateOptions{})
+				_, err = d.configMapClient.Create(ctx, cfg, metav1.CreateOptions{})
 				if err != nil {
 					if !apierrors.IsAlreadyExists(err) {
-						return errors.Wrapf(err, "error while calling configMapClient.Create for %q", d.configMap.Name)
+						return errors.Wrapf(err, "error while calling configMapClient.Create for %q", cfg.Name)
 					}
-					_, err = d.configMapClient.Update(ctx, d.configMap, metav1.UpdateOptions{})
+					_, err = d.configMapClient.Update(ctx, cfg, metav1.UpdateOptions{})
 					if err != nil {
-						return errors.Wrapf(err, "error while calling configMapClient.Update for %q", d.configMap.Name)
+						return errors.Wrapf(err, "error while calling configMapClient.Update for %q", cfg.Name)
 					}
 				}
 			}
@@ -171,10 +171,10 @@ func (d *Driver) Rm(ctx context.Context, force bool, rmVolume bool) error {
 			return errors.Wrapf(err, "error while calling deploymentClient.Delete for %q", d.deployment.Name)
 		}
 	}
-	if d.configMap != nil {
+	for _, cfg := range d.configMaps {
-		if err := d.configMapClient.Delete(ctx, d.configMap.Name, metav1.DeleteOptions{}); err != nil {
+		if err := d.configMapClient.Delete(ctx, cfg.Name, metav1.DeleteOptions{}); err != nil {
 			if !apierrors.IsNotFound(err) {
-				return errors.Wrapf(err, "error while calling configMapClient.Delete for %q", d.configMap.Name)
+				return errors.Wrapf(err, "error while calling configMapClient.Delete for %q", cfg.Name)
 			}
 		}
 	}

driver/kubernetes/factory.go

@@ -73,14 +73,11 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
 		BuildkitFlags: cfg.BuildkitFlags,
 		Rootless:      false,
 		Platforms:     cfg.Platforms,
+		ConfigFiles:   cfg.Files,
 	}
 
 	deploymentOpt.Qemu.Image = bkimage.QemuImage
 
-	if cfg, ok := cfg.Files["buildkitd.toml"]; ok {
-		deploymentOpt.BuildkitConfig = cfg
-	}
-
 	loadbalance := LoadbalanceSticky
 
 	for k, v := range cfg.DriverOpts {
@@ -142,7 +139,7 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
 		}
 	}
 
-	d.deployment, d.configMap, err = manifest.NewDeployment(deploymentOpt)
+	d.deployment, d.configMaps, err = manifest.NewDeployment(deploymentOpt)
 	if err != nil {
 		return nil, err
 	}

driver/kubernetes/manifest/manifest.go

@@ -1,6 +1,8 @@
 package manifest
 
 import (
+	"fmt"
+	"path"
 	"strings"
 
 	"github.com/docker/buildx/util/platformutil"
@@ -25,9 +27,8 @@ type DeploymentOpt struct {
 	}
 
 	BuildkitFlags []string
-	// BuildkitConfig
+	// files mounted at /etc/buildkitd
-	// when not empty, will create configmap with buildkit.toml and mounted
+	ConfigFiles map[string][]byte
-	BuildkitConfig []byte
 
 	Rootless       bool
 	NodeSelector   map[string]string
@@ -43,7 +44,7 @@ const (
 	AnnotationPlatform = "buildx.docker.com/platform"
 )
 
-func NewDeployment(opt *DeploymentOpt) (d *appsv1.Deployment, c *corev1.ConfigMap, err error) {
+func NewDeployment(opt *DeploymentOpt) (d *appsv1.Deployment, c []*corev1.ConfigMap, err error) {
 	labels := map[string]string{
 		"app": opt.Name,
 	}
@@ -103,26 +104,23 @@ func NewDeployment(opt *DeploymentOpt) (d *appsv1.Deployment, c *corev1.ConfigMa
 			},
 		},
 	}
-
+	for _, cfg := range splitConfigFiles(opt.ConfigFiles) {
-	if len(opt.BuildkitConfig) > 0 {
+		cc := &corev1.ConfigMap{
-		c = &corev1.ConfigMap{
 			TypeMeta: metav1.TypeMeta{
 				APIVersion: corev1.SchemeGroupVersion.String(),
 				Kind:       "ConfigMap",
 			},
 			ObjectMeta: metav1.ObjectMeta{
 				Namespace:   opt.Namespace,
-				Name:        opt.Name + "-config",
+				Name:        opt.Name + "-" + cfg.name,
 				Annotations: annotations,
 			},
-			Data: map[string]string{
+			Data: cfg.files,
-				"buildkitd.toml": string(opt.BuildkitConfig),
-			},
 		}
 
 		d.Spec.Template.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{{
-			Name:      "config",
+			Name:      cfg.name,
-			MountPath: "/etc/buildkit",
+			MountPath: path.Join("/etc/buildkit", cfg.path),
 		}}
 
 		d.Spec.Template.Spec.Volumes = []corev1.Volume{{
@@ -130,11 +128,12 @@ func NewDeployment(opt *DeploymentOpt) (d *appsv1.Deployment, c *corev1.ConfigMa
 			VolumeSource: corev1.VolumeSource{
 				ConfigMap: &corev1.ConfigMapVolumeSource{
 					LocalObjectReference: corev1.LocalObjectReference{
-						Name: c.Name,
+						Name: cc.Name,
 					},
 				},
 			},
 		}}
+		c = append(c, cc)
 	}
 
 	if opt.Qemu.Install {
@@ -208,3 +207,35 @@ func toRootless(d *appsv1.Deployment) error {
 	d.Spec.Template.ObjectMeta.Annotations["container.seccomp.security.alpha.kubernetes.io/"+containerName] = "unconfined"
 	return nil
 }
+
+type config struct {
+	name  string
+	path  string
+	files map[string]string
+}
+
+func splitConfigFiles(m map[string][]byte) []config {
+	var c []config
+	idx := map[string]int{}
+	nameIdx := 0
+	for k, v := range m {
+		dir := path.Dir(k)
+		i, ok := idx[dir]
+		if !ok {
+			idx[dir] = len(c)
+			i = len(c)
+			name := "config"
+			if dir != "." {
+				nameIdx++
+				name = fmt.Sprintf("%s-%d", name, nameIdx)
+			}
+			c = append(c, config{
+				path:  dir,
+				name:  name,
+				files: map[string]string{},
+			})
+		}
+		c[i].files[path.Base(k)] = string(v)
+	}
+	return c
+}