Merge pull request #2018 from crazy-max/driver-client-meta

driver(docker): opt to set additional dial meta to the client

builder/builder.go

@@ -208,7 +208,7 @@ type driverFactory struct {
 }
 
 // Factory returns the driver factory.
-func (b *Builder) Factory(ctx context.Context) (_ driver.Factory, err error) {
+func (b *Builder) Factory(ctx context.Context, dialMeta map[string][]string) (_ driver.Factory, err error) {
 	b.driverFactory.once.Do(func() {
 		if b.Driver != "" {
 			b.driverFactory.Factory, err = driver.GetFactory(b.Driver, true)
@@ -231,7 +231,7 @@ func (b *Builder) Factory(ctx context.Context) (_ driver.Factory, err error) {
 			if _, err = dockerapi.Ping(ctx); err != nil {
 				return
 			}
-			b.driverFactory.Factory, err = driver.GetDefaultFactory(ctx, ep, dockerapi, false)
+			b.driverFactory.Factory, err = driver.GetDefaultFactory(ctx, ep, dockerapi, false, dialMeta)
 			if err != nil {
 				return
 			}

builder/node.go

@@ -42,9 +42,35 @@ func (b *Builder) Nodes() []Node {
 	return b.nodes
 }
 
+type LoadNodesOption func(*loadNodesOptions)
+
+type loadNodesOptions struct {
+	data     bool
+	dialMeta map[string][]string
+}
+
+func WithData() LoadNodesOption {
+	return func(o *loadNodesOptions) {
+		o.data = true
+	}
+}
+
+func WithDialMeta(dialMeta map[string][]string) LoadNodesOption {
+	return func(o *loadNodesOptions) {
+		o.dialMeta = dialMeta
+	}
+}
+
 // LoadNodes loads and returns nodes for this builder.
 // TODO: this should be a method on a Node object and lazy load data for each driver.
-func (b *Builder) LoadNodes(ctx context.Context, withData bool) (_ []Node, err error) {
+func (b *Builder) LoadNodes(ctx context.Context, opts ...LoadNodesOption) (_ []Node, err error) {
+	lno := loadNodesOptions{
+		data: false,
+	}
+	for _, opt := range opts {
+		opt(&lno)
+	}
+
 	eg, _ := errgroup.WithContext(ctx)
 	b.nodes = make([]Node, len(b.NodeGroup.Nodes))
 
@@ -54,7 +80,7 @@ func (b *Builder) LoadNodes(ctx context.Context, withData bool) (_ []Node, err e
 		}
 	}()
 
-	factory, err := b.Factory(ctx)
+	factory, err := b.Factory(ctx, lno.dialMeta)
 	if err != nil {
 		return nil, err
 	}
@@ -113,7 +139,7 @@ func (b *Builder) LoadNodes(ctx context.Context, withData bool) (_ []Node, err e
 					}
 				}
 
-				d, err := driver.GetDriver(ctx, "buildx_buildkit_"+n.Name, factory, n.Endpoint, dockerapi, imageopt.Auth, kcc, n.Flags, n.Files, n.DriverOpts, n.Platforms, b.opts.contextPathHash)
+				d, err := driver.GetDriver(ctx, "buildx_buildkit_"+n.Name, factory, n.Endpoint, dockerapi, imageopt.Auth, kcc, n.Flags, n.Files, n.DriverOpts, n.Platforms, b.opts.contextPathHash, lno.dialMeta)
 				if err != nil {
 					node.Err = err
 					return nil
@@ -121,7 +147,7 @@ func (b *Builder) LoadNodes(ctx context.Context, withData bool) (_ []Node, err e
 				node.Driver = d
 				node.ImageOpt = imageopt
 
-				if withData {
+				if lno.data {
 					if err := node.loadData(ctx); err != nil {
 						node.Err = err
 					}
@@ -136,7 +162,7 @@ func (b *Builder) LoadNodes(ctx context.Context, withData bool) (_ []Node, err e
 	}
 
 	// TODO: This should be done in the routine loading driver data
-	if withData {
+	if lno.data {
 		kubernetesDriverCount := 0
 		for _, d := range b.nodes {
 			if d.DriverInfo != nil && len(d.DriverInfo.DynamicNodes) > 0 {

commands/bake.go

@@ -114,7 +114,7 @@ func runBake(dockerCli command.Cli, targets []string, in bakeOptions, cFlags com
 		if err = updateLastActivity(dockerCli, b.NodeGroup); err != nil {
 			return errors.Wrapf(err, "failed to update builder last activity time")
 		}
-		nodes, err = b.LoadNodes(ctx, false)
+		nodes, err = b.LoadNodes(ctx)
 		if err != nil {
 			return err
 		}

commands/build.go

@@ -252,7 +252,7 @@ func runBuild(dockerCli command.Cli, options buildOptions) (err error) {
 	if err != nil {
 		return err
 	}
-	_, err = b.LoadNodes(ctx, false)
+	_, err = b.LoadNodes(ctx)
 	if err != nil {
 		return err
 	}

commands/create.go

@@ -123,7 +123,7 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
 			if len(args) > 0 {
 				arg = args[0]
 			}
-			f, err := driver.GetDefaultFactory(ctx, arg, dockerCli.Client(), true)
+			f, err := driver.GetDefaultFactory(ctx, arg, dockerCli.Client(), true, nil)
 			if err != nil {
 				return err
 			}
@@ -270,7 +270,7 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
 	timeoutCtx, cancel := context.WithTimeout(ctx, 20*time.Second)
 	defer cancel()
 
-	nodes, err := b.LoadNodes(timeoutCtx, true)
+	nodes, err := b.LoadNodes(timeoutCtx, builder.WithData())
 	if err != nil {
 		return err
 	}

commands/diskusage.go

@@ -39,7 +39,7 @@ func runDiskUsage(dockerCli command.Cli, opts duOptions) error {
 		return err
 	}
 
-	nodes, err := b.LoadNodes(ctx, false)
+	nodes, err := b.LoadNodes(ctx)
 	if err != nil {
 		return err
 	}

commands/inspect.go

@@ -40,7 +40,7 @@ func runInspect(dockerCli command.Cli, in inspectOptions) error {
 	timeoutCtx, cancel := context.WithTimeout(ctx, 20*time.Second)
 	defer cancel()
 
-	nodes, err := b.LoadNodes(timeoutCtx, true)
+	nodes, err := b.LoadNodes(timeoutCtx, builder.WithData())
 	if in.bootstrap {
 		var ok bool
 		ok, err = b.Boot(ctx)
@@ -48,7 +48,7 @@ func runInspect(dockerCli command.Cli, in inspectOptions) error {
 			return err
 		}
 		if ok {
-			nodes, err = b.LoadNodes(timeoutCtx, true)
+			nodes, err = b.LoadNodes(timeoutCtx, builder.WithData())
 		}
 	}
 

commands/ls.go

@@ -49,7 +49,7 @@ func runLs(dockerCli command.Cli, in lsOptions) error {
 	for _, b := range builders {
 		func(b *builder.Builder) {
 			eg.Go(func() error {
-				_, _ = b.LoadNodes(timeoutCtx, true)
+				_, _ = b.LoadNodes(timeoutCtx, builder.WithData())
 				return nil
 			})
 		}(b)

commands/prune.go

@@ -60,7 +60,7 @@ func runPrune(dockerCli command.Cli, opts pruneOptions) error {
 		return err
 	}
 
-	nodes, err := b.LoadNodes(ctx, false)
+	nodes, err := b.LoadNodes(ctx)
 	if err != nil {
 		return err
 	}

commands/rm.go

@@ -55,7 +55,7 @@ func runRm(dockerCli command.Cli, in rmOptions) error {
 		return err
 	}
 
-	nodes, err := b.LoadNodes(ctx, false)
+	nodes, err := b.LoadNodes(ctx)
 	if err != nil {
 		return err
 	}
@@ -139,7 +139,7 @@ func rmAllInactive(ctx context.Context, txn *store.Txn, dockerCli command.Cli, i
 	for _, b := range builders {
 		func(b *builder.Builder) {
 			eg.Go(func() error {
-				nodes, err := b.LoadNodes(timeoutCtx, true)
+				nodes, err := b.LoadNodes(timeoutCtx, builder.WithData())
 				if err != nil {
 					return errors.Wrapf(err, "cannot load %s", b.Name)
 				}

commands/stop.go

@@ -25,7 +25,7 @@ func runStop(dockerCli command.Cli, in stopOptions) error {
 	if err != nil {
 		return err
 	}
-	nodes, err := b.LoadNodes(ctx, false)
+	nodes, err := b.LoadNodes(ctx)
 	if err != nil {
 		return err
 	}

controller/build/build.go

@@ -171,7 +171,7 @@ func RunBuild(ctx context.Context, dockerCli command.Cli, in controllerapi.Build
 	if err = updateLastActivity(dockerCli, b.NodeGroup); err != nil {
 		return nil, nil, errors.Wrapf(err, "failed to update builder last activity time")
 	}
-	nodes, err := b.LoadNodes(ctx, false)
+	nodes, err := b.LoadNodes(ctx)
 	if err != nil {
 		return nil, nil, err
 	}

driver/docker-container/driver.go

@@ -428,6 +428,10 @@ func (d *Driver) Features(ctx context.Context) map[driver.Feature]bool {
 	}
 }
 
+func (d *Driver) HostGatewayIP(ctx context.Context) (net.IP, error) {
+	return nil, errors.New("host-gateway is not supported by the docker-container driver")
+}
+
 func demuxConn(c net.Conn) net.Conn {
 	pr, pw := io.Pipe()
 	// TODO: rewrite parser with Reader() to avoid goroutine switch

driver/docker-container/factory.go

@@ -29,7 +29,7 @@ func (*factory) Usage() string {
 	return "docker-container"
 }
 
-func (*factory) Priority(ctx context.Context, endpoint string, api dockerclient.APIClient) int {
+func (*factory) Priority(ctx context.Context, endpoint string, api dockerclient.APIClient, dialMeta map[string][]string) int {
 	if api == nil {
 		return priorityUnsupported
 	}

driver/docker/driver.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"net"
 	"strings"
+	"sync"
 
 	"github.com/docker/buildx/driver"
 	"github.com/docker/buildx/util/progress"
@@ -15,6 +16,9 @@ import (
 type Driver struct {
 	factory driver.Factory
 	driver.InitConfig
+
+	features    features
+	hostGateway hostGateway
 }
 
 func (d *Driver) Bootstrap(ctx context.Context, l progress.Logger) error {
@@ -54,7 +58,7 @@ func (d *Driver) Rm(ctx context.Context, force, rmVolume, rmDaemon bool) error {
 func (d *Driver) Client(ctx context.Context) (*client.Client, error) {
 	opts := []client.ClientOpt{
 		client.WithContextDialer(func(context.Context, string) (net.Conn, error) {
-			return d.DockerAPI.DialHijack(ctx, "/grpc", "h2c", nil)
+			return d.DockerAPI.DialHijack(ctx, "/grpc", "h2c", d.DialMeta)
 		}), client.WithSessionDialer(func(ctx context.Context, proto string, meta map[string][]string) (net.Conn, error) {
 			return d.DockerAPI.DialHijack(ctx, "/session", proto, meta)
 		}),
@@ -70,24 +74,67 @@ func (d *Driver) Client(ctx context.Context) (*client.Client, error) {
 	return client.New(ctx, "", opts...)
 }
 
+type features struct {
+	once sync.Once
+	list map[driver.Feature]bool
+}
+
 func (d *Driver) Features(ctx context.Context) map[driver.Feature]bool {
-	var useContainerdSnapshotter bool
+	d.features.once.Do(func() {
-	c, err := d.Client(ctx)
+		var useContainerdSnapshotter bool
-	if err == nil {
+		if c, err := d.Client(ctx); err == nil {
-		workers, _ := c.ListWorkers(ctx)
+			workers, _ := c.ListWorkers(ctx)
+			for _, w := range workers {
+				if _, ok := w.Labels["org.mobyproject.buildkit.worker.snapshotter"]; ok {
+					useContainerdSnapshotter = true
+				}
+			}
+			c.Close()
+		}
+		d.features.list = map[driver.Feature]bool{
+			driver.OCIExporter:    useContainerdSnapshotter,
+			driver.DockerExporter: useContainerdSnapshotter,
+			driver.CacheExport:    useContainerdSnapshotter,
+			driver.MultiPlatform:  useContainerdSnapshotter,
+		}
+	})
+	return d.features.list
+}
+
+type hostGateway struct {
+	once sync.Once
+	ip   net.IP
+	err  error
+}
+
+func (d *Driver) HostGatewayIP(ctx context.Context) (net.IP, error) {
+	d.hostGateway.once.Do(func() {
+		c, err := d.Client(ctx)
+		if err != nil {
+			d.hostGateway.err = err
+			return
+		}
+		defer c.Close()
+		workers, err := c.ListWorkers(ctx)
+		if err != nil {
+			d.hostGateway.err = errors.Wrap(err, "listing workers")
+			return
+		}
 		for _, w := range workers {
-			if _, ok := w.Labels["org.mobyproject.buildkit.worker.snapshotter"]; ok {
+			// should match github.com/docker/docker/builder/builder-next/worker/label.HostGatewayIP const
-				useContainerdSnapshotter = true
+			if v, ok := w.Labels["org.mobyproject.buildkit.worker.moby.host-gateway-ip"]; ok && v != "" {
+				ip := net.ParseIP(v)
+				if ip == nil {
+					d.hostGateway.err = errors.Errorf("failed to parse host-gateway IP: %s", v)
+					return
+				}
+				d.hostGateway.ip = ip
+				return
 			}
 		}
-		c.Close()
+		d.hostGateway.err = errors.New("host-gateway IP not found")
-	}
+	})
-	return map[driver.Feature]bool{
+	return d.hostGateway.ip, d.hostGateway.err
-		driver.OCIExporter:    useContainerdSnapshotter,
-		driver.DockerExporter: useContainerdSnapshotter,
-		driver.CacheExport:    useContainerdSnapshotter,
-		driver.MultiPlatform:  useContainerdSnapshotter,
-	}
 }
 
 func (d *Driver) Factory() driver.Factory {

driver/docker/factory.go

@@ -26,12 +26,12 @@ func (*factory) Usage() string {
 	return "docker"
 }
 
-func (*factory) Priority(ctx context.Context, endpoint string, api dockerclient.APIClient) int {
+func (*factory) Priority(ctx context.Context, endpoint string, api dockerclient.APIClient, dialMeta map[string][]string) int {
 	if api == nil {
 		return priorityUnsupported
 	}
 
-	c, err := api.DialHijack(ctx, "/grpc", "h2c", nil)
+	c, err := api.DialHijack(ctx, "/grpc", "h2c", dialMeta)
 	if err != nil {
 		return priorityUnsupported
 	}

driver/driver.go

@@ -3,6 +3,7 @@ package driver
 import (
 	"context"
 	"io"
+	"net"
 
 	"github.com/docker/buildx/store"
 	"github.com/docker/buildx/util/progress"
@@ -60,6 +61,7 @@ type Driver interface {
 	Rm(ctx context.Context, force, rmVolume, rmDaemon bool) error
 	Client(ctx context.Context) (*client.Client, error)
 	Features(ctx context.Context) map[Feature]bool
+	HostGatewayIP(ctx context.Context) (net.IP, error)
 	IsMobyDriver() bool
 	Config() InitConfig
 }

driver/kubernetes/driver.go

@@ -236,3 +236,7 @@ func (d *Driver) Features(ctx context.Context) map[driver.Feature]bool {
 		driver.MultiPlatform:  true, // Untested (needs multiple Driver instances)
 	}
 }
+
+func (d *Driver) HostGatewayIP(ctx context.Context) (net.IP, error) {
+	return nil, errors.New("host-gateway is not supported by the kubernetes driver")
+}

driver/kubernetes/factory.go

@@ -34,7 +34,7 @@ func (*factory) Usage() string {
 	return DriverName
 }
 
-func (*factory) Priority(ctx context.Context, endpoint string, api dockerclient.APIClient) int {
+func (*factory) Priority(ctx context.Context, endpoint string, api dockerclient.APIClient, dialMeta map[string][]string) int {
 	if api == nil {
 		return priorityUnsupported
 	}

driver/manager.go

@@ -2,7 +2,6 @@ package driver
 
 import (
 	"context"
-	"net"
 	"os"
 	"sort"
 	"strings"
@@ -18,7 +17,7 @@ import (
 type Factory interface {
 	Name() string
 	Usage() string
-	Priority(ctx context.Context, endpoint string, api dockerclient.APIClient) int
+	Priority(ctx context.Context, endpoint string, api dockerclient.APIClient, dialMeta map[string][]string) int
 	New(ctx context.Context, cfg InitConfig) (Driver, error)
 	AllowsInstances() bool
 }
@@ -58,8 +57,8 @@ type InitConfig struct {
 	DriverOpts       map[string]string
 	Auth             Auth
 	Platforms        []specs.Platform
-	// ContextPathHash can be used for determining pods in the driver instance
+	ContextPathHash  string // can be used for determining pods in the driver instance
-	ContextPathHash string
+	DialMeta         map[string][]string
 }
 
 var drivers map[string]Factory
@@ -71,7 +70,7 @@ func Register(f Factory) {
 	drivers[f.Name()] = f
 }
 
-func GetDefaultFactory(ctx context.Context, ep string, c dockerclient.APIClient, instanceRequired bool) (Factory, error) {
+func GetDefaultFactory(ctx context.Context, ep string, c dockerclient.APIClient, instanceRequired bool, dialMeta map[string][]string) (Factory, error) {
 	if len(drivers) == 0 {
 		return nil, errors.Errorf("no drivers available")
 	}
@@ -84,7 +83,7 @@ func GetDefaultFactory(ctx context.Context, ep string, c dockerclient.APIClient,
 		if instanceRequired && !f.AllowsInstances() {
 			continue
 		}
-		dd = append(dd, p{f: f, priority: f.Priority(ctx, ep, c)})
+		dd = append(dd, p{f: f, priority: f.Priority(ctx, ep, c, dialMeta)})
 	}
 	sort.Slice(dd, func(i, j int) bool {
 		return dd[i].priority < dd[j].priority
@@ -104,7 +103,7 @@ func GetFactory(name string, instanceRequired bool) (Factory, error) {
 	return nil, errors.Errorf("failed to find driver %q", name)
 }
 
-func GetDriver(ctx context.Context, name string, f Factory, endpointAddr string, api dockerclient.APIClient, auth Auth, kcc KubeClientConfig, flags []string, files map[string][]byte, do map[string]string, platforms []specs.Platform, contextPathHash string) (*DriverHandle, error) {
+func GetDriver(ctx context.Context, name string, f Factory, endpointAddr string, api dockerclient.APIClient, auth Auth, kcc KubeClientConfig, flags []string, files map[string][]byte, do map[string]string, platforms []specs.Platform, contextPathHash string, dialMeta map[string][]string) (*DriverHandle, error) {
 	ic := InitConfig{
 		EndpointAddr:     endpointAddr,
 		DockerAPI:        api,
@@ -115,11 +114,12 @@ func GetDriver(ctx context.Context, name string, f Factory, endpointAddr string,
 		Auth:             auth,
 		Platforms:        platforms,
 		ContextPathHash:  contextPathHash,
+		DialMeta:         dialMeta,
 		Files:            files,
 	}
 	if f == nil {
 		var err error
-		f, err = GetDefaultFactory(ctx, endpointAddr, api, false)
+		f, err = GetDefaultFactory(ctx, endpointAddr, api, false, dialMeta)
 		if err != nil {
 			return nil, err
 		}
@@ -150,13 +150,8 @@ type DriverHandle struct {
 	client                  *client.Client
 	err                     error
 	once                    sync.Once
-	featuresOnce            sync.Once
-	features                map[Feature]bool
 	historyAPISupportedOnce sync.Once
 	historyAPISupported     bool
-	hostGatewayIPOnce       sync.Once
-	hostGatewayIP           net.IP
-	hostGatewayIPErr        error
 }
 
 func (d *DriverHandle) Client(ctx context.Context) (*client.Client, error) {
@@ -166,13 +161,6 @@ func (d *DriverHandle) Client(ctx context.Context) (*client.Client, error) {
 	return d.client, d.err
 }
 
-func (d *DriverHandle) Features(ctx context.Context) map[Feature]bool {
-	d.featuresOnce.Do(func() {
-		d.features = d.Driver.Features(ctx)
-	})
-	return d.features
-}
-
 func (d *DriverHandle) HistoryAPISupported(ctx context.Context) bool {
 	d.historyAPISupportedOnce.Do(func() {
 		if c, err := d.Client(ctx); err == nil {
@@ -181,36 +169,3 @@ func (d *DriverHandle) HistoryAPISupported(ctx context.Context) bool {
 	})
 	return d.historyAPISupported
 }
-
-func (d *DriverHandle) HostGatewayIP(ctx context.Context) (net.IP, error) {
-	d.hostGatewayIPOnce.Do(func() {
-		if !d.Driver.IsMobyDriver() {
-			d.hostGatewayIPErr = errors.New("host-gateway is only supported with the docker driver")
-			return
-		}
-		c, err := d.Client(ctx)
-		if err != nil {
-			d.hostGatewayIPErr = err
-			return
-		}
-		workers, err := c.ListWorkers(ctx)
-		if err != nil {
-			d.hostGatewayIPErr = errors.Wrap(err, "listing workers")
-			return
-		}
-		for _, w := range workers {
-			// should match github.com/docker/docker/builder/builder-next/worker/label.HostGatewayIP const
-			if v, ok := w.Labels["org.mobyproject.buildkit.worker.moby.host-gateway-ip"]; ok && v != "" {
-				ip := net.ParseIP(v)
-				if ip == nil {
-					d.hostGatewayIPErr = errors.Errorf("failed to parse host-gateway IP: %s", v)
-					return
-				}
-				d.hostGatewayIP = ip
-				return
-			}
-		}
-		d.hostGatewayIPErr = errors.New("host-gateway IP not found")
-	})
-	return d.hostGatewayIP, d.hostGatewayIPErr
-}

driver/remote/driver.go

@@ -2,6 +2,8 @@ package remote
 
 import (
 	"context"
+	"errors"
+	"net"
 
 	"github.com/docker/buildx/driver"
 	"github.com/docker/buildx/util/progress"
@@ -91,6 +93,10 @@ func (d *Driver) Features(ctx context.Context) map[driver.Feature]bool {
 	}
 }
 
+func (d *Driver) HostGatewayIP(ctx context.Context) (net.IP, error) {
+	return nil, errors.New("host-gateway is not supported by the remote driver")
+}
+
 func (d *Driver) Factory() driver.Factory {
 	return d.factory
 }

driver/remote/factory.go

@@ -35,7 +35,7 @@ func (*factory) Usage() string {
 	return "remote"
 }
 
-func (*factory) Priority(ctx context.Context, endpoint string, api dockerclient.APIClient) int {
+func (*factory) Priority(ctx context.Context, endpoint string, api dockerclient.APIClient, dialMeta map[string][]string) int {
 	if util.IsValidEndpoint(endpoint) != nil {
 		return priorityUnsupported
 	}