Merge pull request #28 from tonistiigi/docker-output

build: add docker output for non-moby drivers

build/build.go

@@ -13,12 +13,14 @@ import (
 
 	"github.com/containerd/containerd/platforms"
 	"github.com/docker/distribution/reference"
+	dockerclient "github.com/docker/docker/client"
 	"github.com/docker/docker/pkg/urlutil"
 	"github.com/moby/buildkit/client"
 	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/session/upload/uploadprovider"
 	specs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 	"github.com/tonistiigi/buildx/driver"
 	"github.com/tonistiigi/buildx/util/progress"
 	"golang.org/x/sync/errgroup"
@@ -61,6 +63,10 @@ type DriverInfo struct {
 	Err      error
 }
 
+type DockerAPI interface {
+	DockerAPI(name string) (dockerclient.APIClient, error)
+}
+
 func getFirstDriver(drivers []DriverInfo) (driver.Driver, error) {
 	err := errors.Errorf("no drivers found")
 	for _, di := range drivers {
@@ -74,7 +80,7 @@ func getFirstDriver(drivers []DriverInfo) (driver.Driver, error) {
 	return nil, err
 }
 
-func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, pw progress.Writer) (map[string]*client.SolveResponse, error) {
+func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, docker DockerAPI, pw progress.Writer) (map[string]*client.SolveResponse, error) {
 	if len(drivers) == 0 {
 		return nil, errors.Errorf("driver required for build")
 	}
@@ -83,7 +89,6 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, pw
 		return nil, errors.Errorf("multiple drivers currently not supported")
 	}
 
-	pwOld := pw
 	d, err := getFirstDriver(drivers)
 	if err != nil {
 		return nil, err
@@ -91,10 +96,18 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, pw
 	_, isDefaultMobyDriver := d.(interface {
 		IsDefaultMobyDriver()
 	})
-	c, pw, err := driver.Boot(ctx, d, pw)
+
+	for _, opt := range opt {
+		if !isDefaultMobyDriver && len(opt.Exports) == 0 {
+			logrus.Warnf("No output specified for %s driver. Build result will only remain in the build cache. To push result image into registry use --push or to load image into docker use --load", d.Factory().Name())
+			break
+		}
+	}
+
+	c, err := driver.Boot(ctx, d, pw)
 	if err != nil {
-		close(pwOld.Status())
+		close(pw.Status())
-		<-pwOld.Done()
+		<-pw.Done()
 		return nil, err
 	}
 
@@ -173,10 +186,16 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, pw
 			}
 			if e.Type == "docker" {
 				if e.Output == nil {
-					if !isDefaultMobyDriver {
+					if isDefaultMobyDriver {
-						return nil, errors.Errorf("loading to docker currently not implemented, specify dest file or -")
+						e.Type = "image"
+					} else {
+						w, cancel, err := newDockerLoader(ctx, docker, e.Attrs["context"], mw)
+						if err != nil {
+							return nil, err
+						}
+						defer cancel()
+						opt.Exports[i].Output = w
 					}
-					e.Type = "image"
 				} else if !d.Features()[driver.DockerExporter] {
 					return nil, notSupported(d, driver.DockerExporter)
 				}
@@ -245,6 +264,7 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, pw
 
 		var statusCh chan *client.SolveStatus
 		if pw != nil {
+			pw = progress.ResetTime(pw)
 			statusCh = pw.Status()
 			eg.Go(func() error {
 				<-pw.Done()
@@ -380,5 +400,54 @@ func LoadInputs(inp Inputs, target *client.SolveOpt) (func(), error) {
 }
 
 func notSupported(d driver.Driver, f driver.Feature) error {
-	return errors.Errorf("%s feature is currently not supported for %s driver. Please switch to a different driver (eg. \"docker buildx new\")", f, d.Factory().Name())
+	return errors.Errorf("%s feature is currently not supported for %s driver. Please switch to a different driver (eg. \"docker buildx create\")", f, d.Factory().Name())
+}
+
+func newDockerLoader(ctx context.Context, d DockerAPI, name string, mw *progress.MultiWriter) (io.WriteCloser, func(), error) {
+	c, err := d.DockerAPI(name)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	pr, pw := io.Pipe()
+	started := make(chan struct{})
+	w := &waitingWriter{
+		PipeWriter: pw,
+		f: func() {
+			resp, err := c.ImageLoad(ctx, pr, false)
+			if err != nil {
+				pr.CloseWithError(err)
+				return
+			}
+			prog := mw.WithPrefix("", false)
+			close(started)
+			progress.FromReader(prog, "importing to docker", resp.Body)
+		},
+		started: started,
+	}
+	return w, func() {
+		pr.Close()
+	}, nil
+}
+
+type waitingWriter struct {
+	*io.PipeWriter
+	f       func()
+	once    sync.Once
+	mu      sync.Mutex
+	err     error
+	started chan struct{}
+}
+
+func (w *waitingWriter) Write(dt []byte) (int, error) {
+	w.once.Do(func() {
+		go w.f()
+	})
+	return w.PipeWriter.Write(dt)
+}
+
+func (w *waitingWriter) Close() error {
+	err := w.PipeWriter.Close()
+	<-w.started
+	return err
 }

commands/build.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/docker/cli/cli"
 	"github.com/docker/cli/cli/command"
+	"github.com/moby/buildkit/client"
 	"github.com/moby/buildkit/session/auth/authprovider"
 	"github.com/moby/buildkit/util/appcontext"
 	"github.com/pkg/errors"
@@ -34,6 +35,9 @@ type buildOptions struct {
 	extraHosts  []string
 	networkMode string
 
+	exportPush bool
+	exportLoad bool
+
 	// unimplemented
 	squash bool
 	quiet  bool
@@ -112,6 +116,41 @@ func runBuild(dockerCli command.Cli, in buildOptions) error {
 	if err != nil {
 		return err
 	}
+	if in.exportPush {
+		if in.exportLoad {
+			return errors.Errorf("push and load may not be set together at the moment")
+		}
+		if len(outputs) == 0 {
+			outputs = []client.ExportEntry{{
+				Type: "image",
+				Attrs: map[string]string{
+					"push": "true",
+				},
+			}}
+		} else {
+			switch outputs[0].Type {
+			case "image":
+				outputs[0].Attrs["push"] = "true"
+			default:
+				return errors.Errorf("push and %q output can't be used together", outputs[0].Type)
+			}
+		}
+	}
+	if in.exportLoad {
+		if len(outputs) == 0 {
+			outputs = []client.ExportEntry{{
+				Type:  "docker",
+				Attrs: map[string]string{},
+			}}
+		} else {
+			switch outputs[0].Type {
+			case "docker":
+			default:
+				return errors.Errorf("load and %q output can't be used together", outputs[0].Type)
+			}
+		}
+	}
+
 	opts.Exports = outputs
 
 	return buildTargets(ctx, dockerCli, map[string]build.Options{"default": opts}, in.progress)
@@ -127,7 +166,7 @@ func buildTargets(ctx context.Context, dockerCli command.Cli, opts map[string]bu
 	defer cancel()
 	pw := progress.NewPrinter(ctx2, os.Stderr, progressMode)
 
-	_, err = build.Build(ctx, dis, opts, pw)
+	_, err = build.Build(ctx, dis, opts, dockerAPI(dockerCli), pw)
 	return err
 }
 
@@ -147,6 +186,9 @@ func buildCmd(dockerCli command.Cli) *cobra.Command {
 
 	flags := cmd.Flags()
 
+	flags.BoolVar(&options.exportPush, "push", false, "Shorthand for --output=type=registry")
+	flags.BoolVar(&options.exportLoad, "load", false, "Shorthand for --output=type=docker")
+
 	flags.StringArrayVarP(&options.tags, "tag", "t", []string{}, "Name and optionally a tag in the 'name:tag' format")
 	flags.StringArrayVar(&options.buildArgs, "build-arg", []string{}, "Set build-time variables")
 	flags.StringVarP(&options.dockerfileName, "file", "f", "", "Name of the Dockerfile (Default is 'PATH/Dockerfile')")

commands/inspect.go

@@ -166,7 +166,7 @@ func boot(ctx context.Context, ngi *nginfo) (bool, error) {
 		func(idx int) {
 			eg.Go(func() error {
 				pw := mw.WithPrefix(ngi.ng.Nodes[idx].Name, len(toBoot) > 1)
-				_, _, err := driver.Boot(ctx, ngi.drivers[idx].di.Driver, pw)
+				_, err := driver.Boot(ctx, ngi.drivers[idx].di.Driver, pw)
 				if err != nil {
 					ngi.drivers[idx].err = err
 				}

commands/util.go

@@ -299,3 +299,18 @@ func loadNodeGroupData(ctx context.Context, dockerCli command.Cli, ngi *nginfo)
 
 	return eg.Wait()
 }
+
+func dockerAPI(dockerCli command.Cli) *api {
+	return &api{dockerCli: dockerCli}
+}
+
+type api struct {
+	dockerCli command.Cli
+}
+
+func (a *api) DockerAPI(name string) (dockerclient.APIClient, error) {
+	if name == "" {
+		name = a.dockerCli.CurrentContext()
+	}
+	return clientForEndpoint(a.dockerCli, name)
+}

driver/driver.go

@@ -2,7 +2,6 @@ package driver
 
 import (
 	"context"
-	"time"
 
 	"github.com/moby/buildkit/client"
 	"github.com/pkg/errors"
@@ -52,24 +51,24 @@ type Driver interface {
 	Features() map[Feature]bool
 }
 
-func Boot(ctx context.Context, d Driver, pw progress.Writer) (*client.Client, progress.Writer, error) {
+func Boot(ctx context.Context, d Driver, pw progress.Writer) (*client.Client, error) {
 	try := 0
 	for {
 		info, err := d.Info(ctx)
 		if err != nil {
-			return nil, nil, err
+			return nil, err
 		}
 		try++
 		if info.Status != Running {
 			if try > 2 {
-				return nil, nil, errors.Errorf("failed to bootstrap %T driver in attempts", d)
+				return nil, errors.Errorf("failed to bootstrap %T driver in attempts", d)
 			}
 			if err := d.Bootstrap(ctx, func(s *client.SolveStatus) {
 				if pw != nil {
 					pw.Status() <- s
 				}
 			}); err != nil {
-				return nil, nil, err
+				return nil, err
 			}
 		}
 
@@ -78,72 +77,8 @@ func Boot(ctx context.Context, d Driver, pw progress.Writer) (*client.Client, pr
 			if errors.Cause(err) == ErrNotRunning && try <= 2 {
 				continue
 			}
-			return nil, nil, err
+			return nil, err
 		}
-		return c, newResetWriter(pw), nil
+		return c, nil
 	}
 }
-
-func newResetWriter(in progress.Writer) progress.Writer {
-	w := &pw{Writer: in, status: make(chan *client.SolveStatus), tm: time.Now()}
-	go func() {
-		for {
-			select {
-			case <-in.Done():
-				return
-			case st, ok := <-w.status:
-				if !ok {
-					close(in.Status())
-					return
-				}
-				if w.diff == nil {
-					for _, v := range st.Vertexes {
-						if v.Started != nil {
-							d := v.Started.Sub(w.tm)
-							w.diff = &d
-						}
-					}
-				}
-				if w.diff != nil {
-					for _, v := range st.Vertexes {
-						if v.Started != nil {
-							d := v.Started.Add(-*w.diff)
-							v.Started = &d
-						}
-						if v.Completed != nil {
-							d := v.Completed.Add(-*w.diff)
-							v.Completed = &d
-						}
-					}
-					for _, v := range st.Statuses {
-						if v.Started != nil {
-							d := v.Started.Add(-*w.diff)
-							v.Started = &d
-						}
-						if v.Completed != nil {
-							d := v.Completed.Add(-*w.diff)
-							v.Completed = &d
-						}
-						v.Timestamp = v.Timestamp.Add(-*w.diff)
-					}
-					for _, v := range st.Logs {
-						v.Timestamp = v.Timestamp.Add(-*w.diff)
-					}
-				}
-				in.Status() <- st
-			}
-		}
-	}()
-	return w
-}
-
-type pw struct {
-	progress.Writer
-	tm     time.Time
-	diff   *time.Duration
-	status chan *client.SolveStatus
-}
-
-func (p *pw) Status() chan *client.SolveStatus {
-	return p.status
-}

util/progress/fromreader.go

@@ -0,0 +1,40 @@
+package progress
+
+import (
+	"io"
+	"io/ioutil"
+	"time"
+
+	"github.com/moby/buildkit/client"
+	"github.com/moby/buildkit/identity"
+	"github.com/opencontainers/go-digest"
+)
+
+func FromReader(w Writer, name string, rc io.ReadCloser) {
+	status := w.Status()
+	dgst := digest.FromBytes([]byte(identity.NewID()))
+	tm := time.Now()
+
+	vtx := client.Vertex{
+		Digest:  dgst,
+		Name:    name,
+		Started: &tm,
+	}
+
+	status <- &client.SolveStatus{
+		Vertexes: []*client.Vertex{&vtx},
+	}
+
+	_, err := io.Copy(ioutil.Discard, rc)
+
+	tm2 := time.Now()
+	vtx2 := vtx
+	vtx2.Completed = &tm2
+	if err != nil {
+		vtx2.Error = err.Error()
+	}
+	status <- &client.SolveStatus{
+		Vertexes: []*client.Vertex{&vtx2},
+	}
+	close(status)
+}

util/progress/reset.go

@@ -0,0 +1,71 @@
+package progress
+
+import (
+	"time"
+
+	"github.com/moby/buildkit/client"
+)
+
+func ResetTime(in Writer) Writer {
+	w := &pw{Writer: in, status: make(chan *client.SolveStatus), tm: time.Now()}
+	go func() {
+		for {
+			select {
+			case <-in.Done():
+				return
+			case st, ok := <-w.status:
+				if !ok {
+					close(in.Status())
+					return
+				}
+				if w.diff == nil {
+					for _, v := range st.Vertexes {
+						if v.Started != nil {
+							d := v.Started.Sub(w.tm)
+							w.diff = &d
+						}
+					}
+				}
+				if w.diff != nil {
+					for _, v := range st.Vertexes {
+						if v.Started != nil {
+							d := v.Started.Add(-*w.diff)
+							v.Started = &d
+						}
+						if v.Completed != nil {
+							d := v.Completed.Add(-*w.diff)
+							v.Completed = &d
+						}
+					}
+					for _, v := range st.Statuses {
+						if v.Started != nil {
+							d := v.Started.Add(-*w.diff)
+							v.Started = &d
+						}
+						if v.Completed != nil {
+							d := v.Completed.Add(-*w.diff)
+							v.Completed = &d
+						}
+						v.Timestamp = v.Timestamp.Add(-*w.diff)
+					}
+					for _, v := range st.Logs {
+						v.Timestamp = v.Timestamp.Add(-*w.diff)
+					}
+				}
+				in.Status() <- st
+			}
+		}
+	}()
+	return w
+}
+
+type pw struct {
+	Writer
+	tm     time.Time
+	diff   *time.Duration
+	status chan *client.SolveStatus
+}
+
+func (p *pw) Status() chan *client.SolveStatus {
+	return p.status
+}

vendor/modules.txt

@@ -139,10 +139,10 @@ github.com/docker/compose-on-kubernetes/api/compose/impersonation
 # github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible
 github.com/docker/distribution/reference
 github.com/docker/distribution/digestset
+github.com/docker/distribution/registry/api/errcode
 github.com/docker/distribution/manifest/manifestlist
 github.com/docker/distribution
 github.com/docker/distribution/manifest/schema2
-github.com/docker/distribution/registry/api/errcode
 github.com/docker/distribution/registry/api/v2
 github.com/docker/distribution/registry/client
 github.com/docker/distribution/registry/client/auth
@@ -154,8 +154,8 @@ github.com/docker/distribution/registry/storage/cache/memory
 github.com/docker/distribution/uuid
 github.com/docker/distribution/metrics
 # github.com/docker/docker v1.14.0-0.20190410063227-d9d9eccdc862
-github.com/docker/docker/pkg/urlutil
 github.com/docker/docker/client
+github.com/docker/docker/pkg/urlutil
 github.com/docker/docker/api/types
 github.com/docker/docker/api/types/container
 github.com/docker/docker/api/types/network
@@ -163,21 +163,22 @@ github.com/docker/docker/pkg/stdcopy
 github.com/docker/docker/pkg/namesgenerator
 github.com/docker/docker/api/types/mount
 github.com/docker/docker/api/types/versions
+github.com/docker/docker/api
 github.com/docker/docker/api/types/events
 github.com/docker/docker/api/types/filters
+github.com/docker/docker/api/types/image
 github.com/docker/docker/api/types/registry
+github.com/docker/docker/api/types/swarm
+github.com/docker/docker/api/types/time
+github.com/docker/docker/api/types/volume
+github.com/docker/docker/errdefs
 github.com/docker/docker/pkg/homedir
 github.com/docker/docker/pkg/system
 github.com/docker/docker/pkg/term
 github.com/docker/docker/registry
 github.com/docker/docker/api/types/blkiodev
-github.com/docker/docker/api/types/swarm
-github.com/docker/docker/api
-github.com/docker/docker/api/types/image
-github.com/docker/docker/api/types/time
-github.com/docker/docker/api/types/volume
-github.com/docker/docker/errdefs
 github.com/docker/docker/api/types/strslice
+github.com/docker/docker/api/types/swarm/runtime
 github.com/docker/docker/pkg/jsonmessage
 github.com/docker/docker/pkg/idtools
 github.com/docker/docker/pkg/mount
@@ -186,7 +187,6 @@ github.com/docker/docker/pkg/ioutils
 github.com/docker/docker/pkg/stringid
 github.com/docker/docker/pkg/tarsum
 github.com/docker/docker/registry/resumable
-github.com/docker/docker/api/types/swarm/runtime
 github.com/docker/docker/pkg/fileutils
 github.com/docker/docker/pkg/longpath
 # github.com/docker/docker-credential-helpers v0.6.1
@@ -196,8 +196,8 @@ github.com/docker/docker-credential-helpers/credentials
 github.com/docker/go/canonical/json
 # github.com/docker/go-connections v0.4.0
 github.com/docker/go-connections/nat
-github.com/docker/go-connections/tlsconfig
 github.com/docker/go-connections/sockets
+github.com/docker/go-connections/tlsconfig
 # github.com/docker/go-events v0.0.0-20170721190031-9461782956ad
 github.com/docker/go-events
 # github.com/docker/go-metrics v0.0.0-20170502235133-d466d4f6fd96
@@ -396,10 +396,10 @@ golang.org/x/net/http2
 golang.org/x/net/context
 golang.org/x/net/context/ctxhttp
 golang.org/x/net/trace
+golang.org/x/net/proxy
 golang.org/x/net/http/httpguts
 golang.org/x/net/http2/hpack
 golang.org/x/net/idna
-golang.org/x/net/proxy
 golang.org/x/net/internal/timeseries
 golang.org/x/net/internal/socks
 # golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f