store snapshot of config files on create

Files can be reused when container needs to be booted again.

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

driver/docker-container/driver.go

@@ -7,6 +7,8 @@ import (
 	"io/ioutil"
 	"net"
 	"os"
+	"path"
+	"path/filepath"
 	"time"
 
 	"github.com/docker/buildx/driver"
@@ -134,15 +136,8 @@ func (d *Driver) create(ctx context.Context, l progress.SubLogger) error {
 		if err != nil {
 			return err
 		}
-		if f := d.InitConfig.ConfigFile; f != "" {
-			configFiles, err := confutil.LoadConfigFiles(f)
-			if err != nil {
-				return err
-			}
-			defer os.RemoveAll(configFiles)
-			if err := d.copyToContainer(ctx, configFiles, "/"); err != nil {
-				return err
-			}
+		if err := d.copyToContainer(ctx, d.InitConfig.Files); err != nil {
+			return err
 		}
 		if err := d.start(ctx, l); err != nil {
 			return err
@@ -202,7 +197,14 @@ func (d *Driver) copyLogs(ctx context.Context, l progress.SubLogger) error {
 	return rc.Close()
 }
 
-func (d *Driver) copyToContainer(ctx context.Context, srcPath string, dstDir string) error {
+func (d *Driver) copyToContainer(ctx context.Context, files map[string][]byte) error {
+	srcPath, err := writeConfigFiles(files)
+	if err != nil {
+		return err
+	}
+	if srcPath != "" {
+		defer os.RemoveAll(srcPath)
+	}
 	srcArchive, err := dockerarchive.TarWithOptions(srcPath, &dockerarchive.TarOptions{
 		ChownOpts: &idtools.Identity{UID: 0, GID: 0},
 	})
@@ -210,7 +212,7 @@ func (d *Driver) copyToContainer(ctx context.Context, srcPath string, dstDir str
 		return err
 	}
 	defer srcArchive.Close()
-	return d.DockerAPI.CopyToContainer(ctx, d.Name, dstDir, srcArchive, dockertypes.CopyToContainerOptions{})
+	return d.DockerAPI.CopyToContainer(ctx, d.Name, "/", srcArchive, dockertypes.CopyToContainerOptions{})
 }
 
 func (d *Driver) exec(ctx context.Context, cmd []string) (string, net.Conn, error) {
@@ -383,3 +385,27 @@ func (l *logWriter) Write(dt []byte) (int, error) {
 	l.logger.Log(l.stream, dt)
 	return len(dt), nil
 }
+
+func writeConfigFiles(m map[string][]byte) (_ string, err error) {
+	// Temp dir that will be copied to the container
+	tmpDir, err := os.MkdirTemp("", "buildkitd-config")
+	if err != nil {
+		return "", err
+	}
+	defer func() {
+		if err != nil {
+			os.RemoveAll(tmpDir)
+		}
+	}()
+	for f, dt := range m {
+		f = path.Join(confutil.DefaultBuildKitConfigDir, f)
+		p := filepath.Join(tmpDir, f)
+		if err := os.MkdirAll(filepath.Dir(p), 0700); err != nil {
+			return "", err
+		}
+		if err := os.WriteFile(p, dt, 0600); err != nil {
+			return "", err
+		}
+	}
+	return tmpDir, nil
+}

driver/docker/factory.go

@@ -44,7 +44,7 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
 	if cfg.DockerAPI == nil {
 		return nil, errors.Errorf("docker driver requires docker API access")
 	}
-	if cfg.ConfigFile != "" {
+	if len(cfg.Files) > 0 {
 		return nil, errors.Errorf("setting config file is not supported for docker driver, use dockerd configuration file")
 	}
 

driver/kubernetes/factory.go

@@ -2,7 +2,6 @@ package kubernetes
 
 import (
 	"context"
-	"os"
 	"strconv"
 	"strings"
 
@@ -78,12 +77,8 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
 
 	deploymentOpt.Qemu.Image = bkimage.QemuImage
 
-	if cfg.ConfigFile != "" {
-		buildkitConfig, err := os.ReadFile(cfg.ConfigFile)
-		if err != nil {
-			return nil, err
-		}
-		deploymentOpt.BuildkitConfig = buildkitConfig
+	if cfg, ok := cfg.Files["buildkitd.toml"]; ok {
+		deploymentOpt.BuildkitConfig = cfg
 	}
 
 	loadbalance := LoadbalanceSticky

driver/manager.go

@@ -53,7 +53,7 @@ type InitConfig struct {
 	DockerAPI        dockerclient.APIClient
 	KubeClientConfig KubeClientConfig
 	BuildkitFlags    []string
-	ConfigFile       string
+	Files            map[string][]byte
 	DriverOpts       map[string]string
 	Auth             Auth
 	Platforms        []specs.Platform
@@ -103,17 +103,17 @@ func GetFactory(name string, instanceRequired bool) Factory {
 	return nil
 }
 
-func GetDriver(ctx context.Context, name string, f Factory, api dockerclient.APIClient, auth Auth, kcc KubeClientConfig, flags []string, config string, do map[string]string, platforms []specs.Platform, contextPathHash string) (Driver, error) {
+func GetDriver(ctx context.Context, name string, f Factory, api dockerclient.APIClient, auth Auth, kcc KubeClientConfig, flags []string, files map[string][]byte, do map[string]string, platforms []specs.Platform, contextPathHash string) (Driver, error) {
 	ic := InitConfig{
 		DockerAPI:        api,
 		KubeClientConfig: kcc,
 		Name:             name,
 		BuildkitFlags:    flags,
-		ConfigFile:       config,
 		DriverOpts:       do,
 		Auth:             auth,
 		Platforms:        platforms,
 		ContextPathHash:  contextPathHash,
+		Files:            files,
 	}
 	if f == nil {
 		var err error