Merge pull request #824 from tonistiigi/config-files-store2

store snapshot of config files on create

commands/util.go

@@ -225,7 +225,7 @@ func driversForNodeGroup(ctx context.Context, dockerCli command.Cli, ng *store.N
 					}
 				}
 
-				d, err := driver.GetDriver(ctx, "buildx_buildkit_"+n.Name, f, dockerapi, dockerCli.ConfigFile(), kcc, n.Flags, n.ConfigFile, n.DriverOpts, n.Platforms, contextPathHash)
+				d, err := driver.GetDriver(ctx, "buildx_buildkit_"+n.Name, f, dockerapi, dockerCli.ConfigFile(), kcc, n.Flags, n.Files, n.DriverOpts, n.Platforms, contextPathHash)
 				if err != nil {
 					di.Err = err
 					return nil
@@ -360,7 +360,7 @@ func getDefaultDrivers(ctx context.Context, dockerCli command.Cli, defaultOnly b
 		}
 	}
 
-	d, err := driver.GetDriver(ctx, "buildx_buildkit_default", nil, dockerCli.Client(), dockerCli.ConfigFile(), nil, nil, "", nil, nil, contextPathHash)
+	d, err := driver.GetDriver(ctx, "buildx_buildkit_default", nil, dockerCli.Client(), dockerCli.ConfigFile(), nil, nil, nil, nil, nil, contextPathHash)
 	if err != nil {
 		return nil, err
 	}

driver/docker-container/driver.go

@@ -7,6 +7,8 @@ import (
 	"io/ioutil"
 	"net"
 	"os"
+	"path"
+	"path/filepath"
 	"time"
 
 	"github.com/docker/buildx/driver"
@@ -134,15 +136,8 @@ func (d *Driver) create(ctx context.Context, l progress.SubLogger) error {
 		if err != nil {
 			return err
 		}
-		if f := d.InitConfig.ConfigFile; f != "" {
+		if err := d.copyToContainer(ctx, d.InitConfig.Files); err != nil {
-			configFiles, err := confutil.LoadConfigFiles(f)
+			return err
-			if err != nil {
-				return err
-			}
-			defer os.RemoveAll(configFiles)
-			if err := d.copyToContainer(ctx, configFiles, "/"); err != nil {
-				return err
-			}
 		}
 		if err := d.start(ctx, l); err != nil {
 			return err
@@ -202,7 +197,14 @@ func (d *Driver) copyLogs(ctx context.Context, l progress.SubLogger) error {
 	return rc.Close()
 }
 
-func (d *Driver) copyToContainer(ctx context.Context, srcPath string, dstDir string) error {
+func (d *Driver) copyToContainer(ctx context.Context, files map[string][]byte) error {
+	srcPath, err := writeConfigFiles(files)
+	if err != nil {
+		return err
+	}
+	if srcPath != "" {
+		defer os.RemoveAll(srcPath)
+	}
 	srcArchive, err := dockerarchive.TarWithOptions(srcPath, &dockerarchive.TarOptions{
 		ChownOpts: &idtools.Identity{UID: 0, GID: 0},
 	})
@@ -210,7 +212,7 @@ func (d *Driver) copyToContainer(ctx context.Context, srcPath string, dstDir str
 		return err
 	}
 	defer srcArchive.Close()
-	return d.DockerAPI.CopyToContainer(ctx, d.Name, dstDir, srcArchive, dockertypes.CopyToContainerOptions{})
+	return d.DockerAPI.CopyToContainer(ctx, d.Name, "/", srcArchive, dockertypes.CopyToContainerOptions{})
 }
 
 func (d *Driver) exec(ctx context.Context, cmd []string) (string, net.Conn, error) {
@@ -383,3 +385,27 @@ func (l *logWriter) Write(dt []byte) (int, error) {
 	l.logger.Log(l.stream, dt)
 	return len(dt), nil
 }
+
+func writeConfigFiles(m map[string][]byte) (_ string, err error) {
+	// Temp dir that will be copied to the container
+	tmpDir, err := os.MkdirTemp("", "buildkitd-config")
+	if err != nil {
+		return "", err
+	}
+	defer func() {
+		if err != nil {
+			os.RemoveAll(tmpDir)
+		}
+	}()
+	for f, dt := range m {
+		f = path.Join(confutil.DefaultBuildKitConfigDir, f)
+		p := filepath.Join(tmpDir, f)
+		if err := os.MkdirAll(filepath.Dir(p), 0700); err != nil {
+			return "", err
+		}
+		if err := os.WriteFile(p, dt, 0600); err != nil {
+			return "", err
+		}
+	}
+	return tmpDir, nil
+}

driver/docker/factory.go

@@ -44,7 +44,7 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
 	if cfg.DockerAPI == nil {
 		return nil, errors.Errorf("docker driver requires docker API access")
 	}
-	if cfg.ConfigFile != "" {
+	if len(cfg.Files) > 0 {
 		return nil, errors.Errorf("setting config file is not supported for docker driver, use dockerd configuration file")
 	}
 

driver/kubernetes/driver.go

@@ -41,7 +41,7 @@ type Driver struct {
 	factory          driver.Factory
 	minReplicas      int
 	deployment       *appsv1.Deployment
-	configMap        *corev1.ConfigMap
+	configMaps       []*corev1.ConfigMap
 	clientset        *kubernetes.Clientset
 	deploymentClient clientappsv1.DeploymentInterface
 	podClient        clientcorev1.PodInterface
@@ -65,16 +65,16 @@ func (d *Driver) Bootstrap(ctx context.Context, l progress.Logger) error {
 				return errors.Wrapf(err, "error for bootstrap %q", d.deployment.Name)
 			}
 
-			if d.configMap != nil {
+			for _, cfg := range d.configMaps {
 				// create ConfigMap first if exists
-				_, err = d.configMapClient.Create(ctx, d.configMap, metav1.CreateOptions{})
+				_, err = d.configMapClient.Create(ctx, cfg, metav1.CreateOptions{})
 				if err != nil {
 					if !apierrors.IsAlreadyExists(err) {
-						return errors.Wrapf(err, "error while calling configMapClient.Create for %q", d.configMap.Name)
+						return errors.Wrapf(err, "error while calling configMapClient.Create for %q", cfg.Name)
 					}
-					_, err = d.configMapClient.Update(ctx, d.configMap, metav1.UpdateOptions{})
+					_, err = d.configMapClient.Update(ctx, cfg, metav1.UpdateOptions{})
 					if err != nil {
-						return errors.Wrapf(err, "error while calling configMapClient.Update for %q", d.configMap.Name)
+						return errors.Wrapf(err, "error while calling configMapClient.Update for %q", cfg.Name)
 					}
 				}
 			}
@@ -171,10 +171,10 @@ func (d *Driver) Rm(ctx context.Context, force bool, rmVolume bool) error {
 			return errors.Wrapf(err, "error while calling deploymentClient.Delete for %q", d.deployment.Name)
 		}
 	}
-	if d.configMap != nil {
+	for _, cfg := range d.configMaps {
-		if err := d.configMapClient.Delete(ctx, d.configMap.Name, metav1.DeleteOptions{}); err != nil {
+		if err := d.configMapClient.Delete(ctx, cfg.Name, metav1.DeleteOptions{}); err != nil {
 			if !apierrors.IsNotFound(err) {
-				return errors.Wrapf(err, "error while calling configMapClient.Delete for %q", d.configMap.Name)
+				return errors.Wrapf(err, "error while calling configMapClient.Delete for %q", cfg.Name)
 			}
 		}
 	}

driver/kubernetes/factory.go

@@ -2,7 +2,6 @@ package kubernetes
 
 import (
 	"context"
-	"os"
 	"strconv"
 	"strings"
 
@@ -74,18 +73,11 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
 		BuildkitFlags: cfg.BuildkitFlags,
 		Rootless:      false,
 		Platforms:     cfg.Platforms,
+		ConfigFiles:   cfg.Files,
 	}
 
 	deploymentOpt.Qemu.Image = bkimage.QemuImage
 
-	if cfg.ConfigFile != "" {
-		buildkitConfig, err := os.ReadFile(cfg.ConfigFile)
-		if err != nil {
-			return nil, err
-		}
-		deploymentOpt.BuildkitConfig = buildkitConfig
-	}
-
 	loadbalance := LoadbalanceSticky
 
 	for k, v := range cfg.DriverOpts {
@@ -147,7 +139,7 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
 		}
 	}
 
-	d.deployment, d.configMap, err = manifest.NewDeployment(deploymentOpt)
+	d.deployment, d.configMaps, err = manifest.NewDeployment(deploymentOpt)
 	if err != nil {
 		return nil, err
 	}

driver/kubernetes/manifest/manifest.go

@@ -1,6 +1,8 @@
 package manifest
 
 import (
+	"fmt"
+	"path"
 	"strings"
 
 	"github.com/docker/buildx/util/platformutil"
@@ -25,9 +27,8 @@ type DeploymentOpt struct {
 	}
 
 	BuildkitFlags []string
-	// BuildkitConfig
+	// files mounted at /etc/buildkitd
-	// when not empty, will create configmap with buildkit.toml and mounted
+	ConfigFiles map[string][]byte
-	BuildkitConfig []byte
 
 	Rootless       bool
 	NodeSelector   map[string]string
@@ -43,7 +44,7 @@ const (
 	AnnotationPlatform = "buildx.docker.com/platform"
 )
 
-func NewDeployment(opt *DeploymentOpt) (d *appsv1.Deployment, c *corev1.ConfigMap, err error) {
+func NewDeployment(opt *DeploymentOpt) (d *appsv1.Deployment, c []*corev1.ConfigMap, err error) {
 	labels := map[string]string{
 		"app": opt.Name,
 	}
@@ -103,26 +104,23 @@ func NewDeployment(opt *DeploymentOpt) (d *appsv1.Deployment, c *corev1.ConfigMa
 			},
 		},
 	}
-
+	for _, cfg := range splitConfigFiles(opt.ConfigFiles) {
-	if len(opt.BuildkitConfig) > 0 {
+		cc := &corev1.ConfigMap{
-		c = &corev1.ConfigMap{
 			TypeMeta: metav1.TypeMeta{
 				APIVersion: corev1.SchemeGroupVersion.String(),
 				Kind:       "ConfigMap",
 			},
 			ObjectMeta: metav1.ObjectMeta{
 				Namespace:   opt.Namespace,
-				Name:        opt.Name + "-config",
+				Name:        opt.Name + "-" + cfg.name,
 				Annotations: annotations,
 			},
-			Data: map[string]string{
+			Data: cfg.files,
-				"buildkitd.toml": string(opt.BuildkitConfig),
-			},
 		}
 
 		d.Spec.Template.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{{
-			Name:      "config",
+			Name:      cfg.name,
-			MountPath: "/etc/buildkit",
+			MountPath: path.Join("/etc/buildkit", cfg.path),
 		}}
 
 		d.Spec.Template.Spec.Volumes = []corev1.Volume{{
@@ -130,11 +128,12 @@ func NewDeployment(opt *DeploymentOpt) (d *appsv1.Deployment, c *corev1.ConfigMa
 			VolumeSource: corev1.VolumeSource{
 				ConfigMap: &corev1.ConfigMapVolumeSource{
 					LocalObjectReference: corev1.LocalObjectReference{
-						Name: c.Name,
+						Name: cc.Name,
 					},
 				},
 			},
 		}}
+		c = append(c, cc)
 	}
 
 	if opt.Qemu.Install {
@@ -208,3 +207,35 @@ func toRootless(d *appsv1.Deployment) error {
 	d.Spec.Template.ObjectMeta.Annotations["container.seccomp.security.alpha.kubernetes.io/"+containerName] = "unconfined"
 	return nil
 }
+
+type config struct {
+	name  string
+	path  string
+	files map[string]string
+}
+
+func splitConfigFiles(m map[string][]byte) []config {
+	var c []config
+	idx := map[string]int{}
+	nameIdx := 0
+	for k, v := range m {
+		dir := path.Dir(k)
+		i, ok := idx[dir]
+		if !ok {
+			idx[dir] = len(c)
+			i = len(c)
+			name := "config"
+			if dir != "." {
+				nameIdx++
+				name = fmt.Sprintf("%s-%d", name, nameIdx)
+			}
+			c = append(c, config{
+				path:  dir,
+				name:  name,
+				files: map[string]string{},
+			})
+		}
+		c[i].files[path.Base(k)] = string(v)
+	}
+	return c
+}

driver/manager.go

@@ -53,7 +53,7 @@ type InitConfig struct {
 	DockerAPI        dockerclient.APIClient
 	KubeClientConfig KubeClientConfig
 	BuildkitFlags    []string
-	ConfigFile       string
+	Files            map[string][]byte
 	DriverOpts       map[string]string
 	Auth             Auth
 	Platforms        []specs.Platform
@@ -103,17 +103,17 @@ func GetFactory(name string, instanceRequired bool) Factory {
 	return nil
 }
 
-func GetDriver(ctx context.Context, name string, f Factory, api dockerclient.APIClient, auth Auth, kcc KubeClientConfig, flags []string, config string, do map[string]string, platforms []specs.Platform, contextPathHash string) (Driver, error) {
+func GetDriver(ctx context.Context, name string, f Factory, api dockerclient.APIClient, auth Auth, kcc KubeClientConfig, flags []string, files map[string][]byte, do map[string]string, platforms []specs.Platform, contextPathHash string) (Driver, error) {
 	ic := InitConfig{
 		DockerAPI:        api,
 		KubeClientConfig: kcc,
 		Name:             name,
 		BuildkitFlags:    flags,
-		ConfigFile:       config,
 		DriverOpts:       do,
 		Auth:             auth,
 		Platforms:        platforms,
 		ContextPathHash:  contextPathHash,
+		Files:            files,
 	}
 	if f == nil {
 		var err error

store/nodegroup.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 
 	"github.com/containerd/containerd/platforms"
+	"github.com/docker/buildx/util/confutil"
 	"github.com/docker/buildx/util/platformutil"
 	specs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
@@ -21,8 +22,9 @@ type Node struct {
 	Endpoint   string
 	Platforms  []specs.Platform
 	Flags      []string
-	ConfigFile string
 	DriverOpts map[string]string
+
+	Files map[string][]byte
 }
 
 func (ng *NodeGroup) Leave(name string) error {
@@ -88,10 +90,18 @@ func (ng *NodeGroup) Update(name, endpoint string, platforms []string, endpoints
 		Name:       name,
 		Endpoint:   endpoint,
 		Platforms:  pp,
-		ConfigFile: configFile,
 		Flags:      flags,
 		DriverOpts: do,
 	}
+
+	if configFile != "" {
+		files, err := confutil.LoadConfigFiles(configFile)
+		if err != nil {
+			return err
+		}
+		n.Files = files
+	}
+
 	ng.Nodes = append(ng.Nodes, n)
 
 	if err := ng.validateDuplicates(endpoint, len(ng.Nodes)-1); err != nil {

util/confutil/container.go

@@ -1,6 +1,7 @@
 package confutil
 
 import (
+	"bytes"
 	"io"
 	"os"
 	"path"
@@ -20,31 +21,20 @@ const (
 
 // LoadConfigFiles creates a temp directory with BuildKit config and
 // registry certificates ready to be copied to a container.
-func LoadConfigFiles(bkconfig string) (string, error) {
+func LoadConfigFiles(bkconfig string) (map[string][]byte, error) {
 	if _, err := os.Stat(bkconfig); errors.Is(err, os.ErrNotExist) {
-		return "", errors.Wrapf(err, "buildkit configuration file not found: %s", bkconfig)
+		return nil, errors.Wrapf(err, "buildkit configuration file not found: %s", bkconfig)
 	} else if err != nil {
-		return "", errors.Wrapf(err, "invalid buildkit configuration file: %s", bkconfig)
+		return nil, errors.Wrapf(err, "invalid buildkit configuration file: %s", bkconfig)
 	}
 
 	// Load config tree
 	btoml, err := loadConfigTree(bkconfig)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 
-	// Temp dir that will be copied to the container
+	m := make(map[string][]byte)
-	tmpDir, err := os.MkdirTemp("", "buildkitd-config")
-	if err != nil {
-		return "", err
-	}
-
-	// Create BuildKit config folders
-	tmpBuildKitConfigDir := path.Join(tmpDir, DefaultBuildKitConfigDir)
-	tmpBuildKitCertsDir := path.Join(tmpBuildKitConfigDir, "certs")
-	if err := os.MkdirAll(tmpBuildKitCertsDir, 0700); err != nil {
-		return "", err
-	}
 
 	// Iterate through registry config to copy certs and update
 	// BuildKit config with the underlying certs' path in the container.
@@ -70,19 +60,20 @@ func LoadConfigFiles(bkconfig string) (string, error) {
 			if regConf == nil {
 				continue
 			}
-			regCertsDir := path.Join(tmpBuildKitCertsDir, regName)
+			pfx := path.Join("certs", regName)
-			if err := os.Mkdir(regCertsDir, 0755); err != nil {
-				return "", err
-			}
 			if regConf.Has("ca") {
 				regCAs := regConf.GetArray("ca").([]string)
 				if len(regCAs) > 0 {
 					var cas []string
 					for _, ca := range regCAs {
-						cas = append(cas, path.Join(DefaultBuildKitConfigDir, "certs", regName, path.Base(ca)))
+						fp := path.Join(pfx, path.Base(ca))
-						if err := copyfile(ca, path.Join(regCertsDir, path.Base(ca))); err != nil {
+						cas = append(cas, path.Join(DefaultBuildKitConfigDir, fp))
-							return "", err
+
+						dt, err := readFile(ca)
+						if err != nil {
+							return nil, errors.Wrapf(err, "failed to read CA file: %s", ca)
 						}
+						m[fp] = dt
 					}
 					regConf.Set("ca", cas)
 				}
@@ -98,47 +89,44 @@ func LoadConfigFiles(bkconfig string) (string, error) {
 					}
 					key := kp.Get("key").(string)
 					if len(key) > 0 {
-						kp.Set("key", path.Join(DefaultBuildKitConfigDir, "certs", regName, path.Base(key)))
+						fp := path.Join(pfx, path.Base(key))
-						if err := copyfile(key, path.Join(regCertsDir, path.Base(key))); err != nil {
+						kp.Set("key", path.Join(DefaultBuildKitConfigDir, fp))
-							return "", err
+						dt, err := readFile(key)
+						if err != nil {
+							return nil, errors.Wrapf(err, "failed to read key file: %s", key)
 						}
+						m[fp] = dt
 					}
 					cert := kp.Get("cert").(string)
 					if len(cert) > 0 {
-						kp.Set("cert", path.Join(DefaultBuildKitConfigDir, "certs", regName, path.Base(cert)))
+						fp := path.Join(pfx, path.Base(cert))
-						if err := copyfile(cert, path.Join(regCertsDir, path.Base(cert))); err != nil {
+						kp.Set("cert", path.Join(DefaultBuildKitConfigDir, fp))
-							return "", err
+						dt, err := readFile(cert)
+						if err != nil {
+							return nil, errors.Wrapf(err, "failed to read cert file: %s", cert)
 						}
+						m[fp] = dt
 					}
 				}
 			}
 		}
 	}
 
-	// Write BuildKit config
+	b := bytes.NewBuffer(nil)
-	bkfile, err := os.OpenFile(path.Join(tmpBuildKitConfigDir, "buildkitd.toml"), os.O_CREATE|os.O_WRONLY, 0600)
+	_, err = btoml.WriteTo(b)
 	if err != nil {
-		return "", err
+		return nil, err
-	}
-	_, err = btoml.WriteTo(bkfile)
-	if err != nil {
-		return "", err
 	}
+	m["buildkitd.toml"] = b.Bytes()
 
-	return tmpDir, nil
+	return m, nil
 }
 
-func copyfile(src string, dst string) error {
+func readFile(fp string) ([]byte, error) {
-	sf, err := os.Open(src)
+	sf, err := os.Open(fp)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	defer sf.Close()
-	df, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY, 0600)
+	return io.ReadAll(io.LimitReader(sf, 1024*1024))
-	if err != nil {
-		return err
-	}
-	defer df.Close()
-	_, err = io.Copy(df, sf)
-	return err
 }