bake: support for remote files

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

bake/bake.go

@@ -5,6 +5,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path"
+	"regexp"
 	"strconv"
 	"strings"
 
@@ -12,14 +13,55 @@ import (
 	"github.com/docker/buildx/util/platformutil"
 	"github.com/docker/docker/pkg/urlutil"
 	hcl "github.com/hashicorp/hcl/v2"
+	"github.com/moby/buildkit/client/llb"
 	"github.com/moby/buildkit/session/auth/authprovider"
 	"github.com/pkg/errors"
 )
 
-func ReadTargets(ctx context.Context, files, targets, overrides []string) (map[string]*Target, error) {
+var httpPrefix = regexp.MustCompile(`^https?://`)
+var gitURLPathWithFragmentSuffix = regexp.MustCompile(`\.git(?:#.+)?$`)
+
+type File struct {
+	Name string
+	Data []byte
+}
+
+func defaultFilenames() []string {
+	return []string{
+		"docker-compose.yml",  // support app
+		"docker-compose.yaml", // support app
+		"docker-bake.json",
+		"docker-bake.override.json",
+		"docker-bake.hcl",
+		"docker-bake.override.hcl",
+	}
+}
+
+func ReadLocalFiles(names []string) ([]File, error) {
+	isDefault := false
+	if len(names) == 0 {
+		isDefault = true
+		names = defaultFilenames()
+	}
+	out := make([]File, 0, len(names))
+
+	for _, n := range names {
+		dt, err := ioutil.ReadFile(n)
+		if err != nil {
+			if isDefault && errors.Is(err, os.ErrNotExist) {
+				continue
+			}
+			return nil, err
+		}
+		out = append(out, File{Name: n, Data: dt})
+	}
+	return out, nil
+}
+
+func ReadTargets(ctx context.Context, files []File, targets, overrides []string) (map[string]*Target, error) {
 	var c Config
 	for _, f := range files {
-		cfg, err := ParseFile(f)
+		cfg, err := ParseFile(f.Data, f.Name)
 		if err != nil {
 			return nil, err
 		}
@@ -44,12 +86,7 @@ func ReadTargets(ctx context.Context, files, targets, overrides []string) (map[s
 	return m, nil
 }
 
-func ParseFile(fn string) (*Config, error) {
+func ParseFile(dt []byte, fn string) (*Config, error) {
-	dt, err := ioutil.ReadFile(fn)
-	if err != nil {
-		return nil, err
-	}
-
 	fnl := strings.ToLower(fn)
 	if strings.HasSuffix(fnl, ".yml") || strings.HasSuffix(fnl, ".yaml") {
 		return ParseCompose(dt)
@@ -350,6 +387,7 @@ type Target struct {
 	Outputs    []string          `json:"output,omitempty" hcl:"output,optional"`
 	Pull       *bool             `json:"pull,omitempty" hcl:"pull,optional"`
 	NoCache    *bool             `json:"no-cache,omitempty" hcl:"no-cache,optional"`
+
 	// IMPORTANT: if you add more fields here, do not forget to update newOverrides and README.
 }
 
@@ -363,10 +401,10 @@ func (t *Target) normalize() {
 	t.Outputs = removeDupes(t.Outputs)
 }
 
-func TargetsToBuildOpt(m map[string]*Target) (map[string]build.Options, error) {
+func TargetsToBuildOpt(m map[string]*Target, inp *Input) (map[string]build.Options, error) {
 	m2 := make(map[string]build.Options, len(m))
 	for k, v := range m {
-		bo, err := toBuildOpt(v)
+		bo, err := toBuildOpt(v, inp)
 		if err != nil {
 			return nil, err
 		}
@@ -375,7 +413,19 @@ func TargetsToBuildOpt(m map[string]*Target) (map[string]build.Options, error) {
 	return m2, nil
 }
 
-func toBuildOpt(t *Target) (*build.Options, error) {
+func updateContext(t *build.Inputs, inp *Input) {
+	if inp == nil || inp.State == nil {
+		return
+	}
+	if t.ContextPath == "." {
+		t.ContextPath = inp.URL
+		return
+	}
+	st := llb.Scratch().File(llb.Copy(*inp.State, t.ContextPath, "/"), llb.WithCustomNamef("set context to %s", t.ContextPath))
+	t.ContextState = &st
+}
+
+func toBuildOpt(t *Target, inp *Input) (*build.Options, error) {
 	if v := t.Context; v != nil && *v == "-" {
 		return nil, errors.Errorf("context from stdin not allowed in bake")
 	}
@@ -387,6 +437,7 @@ func toBuildOpt(t *Target) (*build.Options, error) {
 	if t.Context != nil {
 		contextPath = *t.Context
 	}
+	contextPath = path.Clean(contextPath)
 	dockerfilePath := "Dockerfile"
 	if t.Dockerfile != nil {
 		dockerfilePath = *t.Dockerfile
@@ -405,11 +456,14 @@ func toBuildOpt(t *Target) (*build.Options, error) {
 		pull = *t.Pull
 	}
 
-	bo := &build.Options{
+	bi := build.Inputs{
-		Inputs: build.Inputs{
 		ContextPath:    contextPath,
 		DockerfilePath: dockerfilePath,
-		},
+	}
+	updateContext(&bi, inp)
+
+	bo := &build.Options{
+		Inputs:    bi,
 		Tags:      t.Tags,
 		BuildArgs: t.Args,
 		Labels:    t.Labels,

bake/bake_test.go

@@ -2,9 +2,7 @@ package bake
 
 import (
 	"context"
-	"io/ioutil"
 	"os"
-	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -12,12 +10,10 @@ import (
 
 func TestReadTargets(t *testing.T) {
 	t.Parallel()
-	tmpdir, err := ioutil.TempDir("", "bake")
-	require.NoError(t, err)
-	defer os.RemoveAll(tmpdir)
 
-	fp := filepath.Join(tmpdir, "config.hcl")
+	fp := File{
-	err = ioutil.WriteFile(fp, []byte(`
+		Name: "config.hcl",
+		Data: []byte(`
 target "webDEP" {
 	args = {
 		VAR_INHERITED = "webDEP"
@@ -32,13 +28,13 @@ target "webapp" {
 		VAR_BOTH = "webapp"
 	}
 	inherits = ["webDEP"]
-}`), 0600)
+}`),
-	require.NoError(t, err)
+	}
 
 	ctx := context.TODO()
 
 	t.Run("NoOverrides", func(t *testing.T) {
-		m, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, nil)
+		m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, nil)
 		require.NoError(t, err)
 		require.Equal(t, 1, len(m))
 
@@ -50,7 +46,7 @@ target "webapp" {
 	})
 
 	t.Run("InvalidTargetOverrides", func(t *testing.T) {
-		_, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{"nosuchtarget.context=foo"})
+		_, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"nosuchtarget.context=foo"})
 		require.NotNil(t, err)
 		require.Equal(t, err.Error(), "could not find any target matching 'nosuchtarget'")
 	})
@@ -60,7 +56,7 @@ target "webapp" {
 			os.Setenv("VAR_FROMENV"+t.Name(), "fromEnv")
 			defer os.Unsetenv("VAR_FROM_ENV" + t.Name())
 
-			m, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{
+			m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{
 				"webapp.args.VAR_UNSET",
 				"webapp.args.VAR_EMPTY=",
 				"webapp.args.VAR_SET=bananas",
@@ -89,7 +85,7 @@ target "webapp" {
 
 		// building leaf but overriding parent fields
 		t.Run("parent", func(t *testing.T) {
-			m, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{
+			m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{
 				"webDEP.args.VAR_INHERITED=override",
 				"webDEP.args.VAR_BOTH=override",
 			})
@@ -100,23 +96,23 @@ target "webapp" {
 	})
 
 	t.Run("ContextOverride", func(t *testing.T) {
-		_, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{"webapp.context"})
+		_, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.context"})
 		require.NotNil(t, err)
 
-		m, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{"webapp.context=foo"})
+		m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.context=foo"})
 		require.NoError(t, err)
 
 		require.Equal(t, "foo", *m["webapp"].Context)
 	})
 
 	t.Run("NoCacheOverride", func(t *testing.T) {
-		m, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{"webapp.no-cache=false"})
+		m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.no-cache=false"})
 		require.NoError(t, err)
 		require.Equal(t, false, *m["webapp"].NoCache)
 	})
 
 	t.Run("PullOverride", func(t *testing.T) {
-		m, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{"webapp.pull=false"})
+		m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.pull=false"})
 		require.NoError(t, err)
 		require.Equal(t, false, *m["webapp"].Pull)
 	})
@@ -176,7 +172,7 @@ target "webapp" {
 		}
 		for _, test := range cases {
 			t.Run(test.name, func(t *testing.T) {
-				m, err := ReadTargets(ctx, []string{fp}, test.targets, test.overrides)
+				m, err := ReadTargets(ctx, []File{fp}, test.targets, test.overrides)
 				test.check(t, m, err)
 			})
 		}
@@ -185,14 +181,11 @@ target "webapp" {
 
 func TestReadTargetsCompose(t *testing.T) {
 	t.Parallel()
-	tmpdir, err := ioutil.TempDir("", "bake")
-	require.NoError(t, err)
-	defer os.RemoveAll(tmpdir)
-
-	fp := filepath.Join(tmpdir, "docker-compose.yml")
-	err = ioutil.WriteFile(fp, []byte(`
-version: "3"
 
+	fp := File{
+		Name: "docker-compose.yml",
+		Data: []byte(
+			`version: "3"
 services:
   db:
     build: .
@@ -203,13 +196,13 @@ services:
       dockerfile: Dockerfile.webapp
       args:
         buildno: 1
-`), 0600)
+`),
-	require.NoError(t, err)
+	}
-
-	fp2 := filepath.Join(tmpdir, "docker-compose2.yml")
-	err = ioutil.WriteFile(fp2, []byte(`
-version: "3"
 
+	fp2 := File{
+		Name: "docker-compose2.yml",
+		Data: []byte(
+			`version: "3"
 services:
   newservice:
     build: .
@@ -217,12 +210,12 @@ services:
     build:
       args:
         buildno2: 12
-`), 0600)
+`),
-	require.NoError(t, err)
+	}
 
 	ctx := context.TODO()
 
-	m, err := ReadTargets(ctx, []string{fp, fp2}, []string{"default"}, nil)
+	m, err := ReadTargets(ctx, []File{fp, fp2}, []string{"default"}, nil)
 	require.NoError(t, err)
 
 	require.Equal(t, 3, len(m))

bake/remote.go

@@ -0,0 +1,234 @@
+package bake
+
+import (
+	"archive/tar"
+	"bytes"
+	"context"
+	"strings"
+
+	"github.com/docker/buildx/build"
+	"github.com/docker/buildx/driver"
+	"github.com/docker/buildx/util/progress"
+	"github.com/moby/buildkit/client"
+	"github.com/moby/buildkit/client/llb"
+	gwclient "github.com/moby/buildkit/frontend/gateway/client"
+	"github.com/pkg/errors"
+)
+
+type Input struct {
+	State *llb.State
+	URL   string
+}
+
+func ReadRemoteFiles(ctx context.Context, dis []build.DriverInfo, url string, names []string, pw progress.Writer) ([]File, *Input, error) {
+	st, filename, ok := detectHttpContext(url)
+	if !ok {
+		st, ok = detectGitContext(url)
+		if !ok {
+			return nil, nil, errors.Errorf("not url context")
+		}
+	}
+
+	inp := &Input{State: st, URL: url}
+	var files []File
+
+	var di *build.DriverInfo
+	for _, d := range dis {
+		if d.Err == nil {
+			di = &d
+			continue
+		}
+	}
+	if di == nil {
+		return nil, nil, nil
+	}
+
+	c, err := driver.Boot(ctx, di.Driver, pw)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	_, err = c.Build(ctx, client.SolveOpt{}, "buildx", func(ctx context.Context, c gwclient.Client) (*gwclient.Result, error) {
+		def, err := st.Marshal(ctx)
+		if err != nil {
+			return nil, err
+		}
+		res, err := c.Solve(ctx, gwclient.SolveRequest{
+			Definition: def.ToPB(),
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		ref, err := res.SingleRef()
+		if err != nil {
+			return nil, err
+		}
+
+		if filename != "" {
+			files, err = filesFromURLRef(ctx, c, ref, inp, filename, names)
+		} else {
+			files, err = filesFromRef(ctx, ref, names)
+		}
+		return nil, err
+	}, progress.NewChannel(pw))
+
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return files, inp, nil
+}
+
+func IsRemoteURL(url string) bool {
+	if _, _, ok := detectHttpContext(url); ok {
+		return true
+	}
+	if _, ok := detectGitContext(url); ok {
+		return true
+	}
+	return false
+}
+
+func detectHttpContext(url string) (*llb.State, string, bool) {
+	if httpPrefix.MatchString(url) {
+		httpContext := llb.HTTP(url, llb.Filename("context"), llb.WithCustomName("[internal] load remote build context"))
+		return &httpContext, "context", true
+	}
+	return nil, "", false
+}
+
+func detectGitContext(ref string) (*llb.State, bool) {
+	found := false
+	if httpPrefix.MatchString(ref) && gitURLPathWithFragmentSuffix.MatchString(ref) {
+		found = true
+	}
+
+	for _, prefix := range []string{"git://", "github.com/", "git@"} {
+		if strings.HasPrefix(ref, prefix) {
+			found = true
+			break
+		}
+	}
+	if !found {
+		return nil, false
+	}
+
+	parts := strings.SplitN(ref, "#", 2)
+	branch := ""
+	if len(parts) > 1 {
+		branch = parts[1]
+	}
+	gitOpts := []llb.GitOption{llb.WithCustomName("[internal] load git source " + ref)}
+
+	st := llb.Git(parts[0], branch, gitOpts...)
+	return &st, true
+}
+
+func isArchive(header []byte) bool {
+	for _, m := range [][]byte{
+		{0x42, 0x5A, 0x68},                   // bzip2
+		{0x1F, 0x8B, 0x08},                   // gzip
+		{0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}, // xz
+	} {
+		if len(header) < len(m) {
+			continue
+		}
+		if bytes.Equal(m, header[:len(m)]) {
+			return true
+		}
+	}
+
+	r := tar.NewReader(bytes.NewBuffer(header))
+	_, err := r.Next()
+	return err == nil
+}
+
+func filesFromURLRef(ctx context.Context, c gwclient.Client, ref gwclient.Reference, inp *Input, filename string, names []string) ([]File, error) {
+	stat, err := ref.StatFile(ctx, gwclient.StatRequest{Path: filename})
+	if err != nil {
+		return nil, err
+	}
+
+	dt, err := ref.ReadFile(ctx, gwclient.ReadRequest{
+		Filename: filename,
+		Range: &gwclient.FileRange{
+			Length: 1024,
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	if isArchive(dt) {
+		bc := llb.Scratch().File(llb.Copy(inp.State, filename, "/", &llb.CopyInfo{
+			AttemptUnpack: true,
+		}))
+		inp.State = &bc
+		inp.URL = ""
+		def, err := bc.Marshal(ctx)
+		if err != nil {
+			return nil, err
+		}
+		res, err := c.Solve(ctx, gwclient.SolveRequest{
+			Definition: def.ToPB(),
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		ref, err := res.SingleRef()
+		if err != nil {
+			return nil, err
+		}
+
+		return filesFromRef(ctx, ref, names)
+	}
+
+	inp.State = nil
+	name := inp.URL
+	inp.URL = ""
+
+	if len(dt) > stat.Size() {
+		if stat.Size() > 1024*512 {
+			return nil, errors.Errorf("non-archive definition URL bigger than maximum allowed size")
+		}
+
+		dt, err = ref.ReadFile(ctx, gwclient.ReadRequest{
+			Filename: filename,
+		})
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return []File{{Name: name, Data: dt}}, nil
+}
+
+func filesFromRef(ctx context.Context, ref gwclient.Reference, names []string) ([]File, error) {
+	// TODO: auto-remove parent dir in needed
+	var files []File
+
+	isDefault := false
+	if len(names) == 0 {
+		isDefault = true
+		names = defaultFilenames()
+	}
+
+	for _, name := range names {
+		_, err := ref.StatFile(ctx, gwclient.StatRequest{Path: name})
+		if err != nil {
+			if isDefault {
+				continue
+			}
+			return nil, err
+		}
+		dt, err := ref.ReadFile(ctx, gwclient.ReadRequest{Filename: name})
+		if err != nil {
+			return nil, err
+		}
+		files = append(files, File{Name: name, Data: dt})
+	}
+
+	return files, nil
+}

build/build.go

@@ -22,6 +22,7 @@ import (
 	dockerclient "github.com/docker/docker/client"
 	"github.com/docker/docker/pkg/urlutil"
 	"github.com/moby/buildkit/client"
+	"github.com/moby/buildkit/client/llb"
 	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/session/upload/uploadprovider"
 	"github.com/moby/buildkit/util/entitlements"
@@ -64,6 +65,7 @@ type Inputs struct {
 	ContextPath    string
 	DockerfilePath string
 	InStream       io.Reader
+	ContextState   *llb.State
 }
 
 type DriverInfo struct {
@@ -723,6 +725,12 @@ func LoadInputs(ctx context.Context, d driver.Driver, inp Inputs, pw progress.Wr
 	)
 
 	switch {
+	case inp.ContextState != nil:
+		if target.FrontendInputs == nil {
+			target.FrontendInputs = make(map[string]llb.State)
+		}
+		target.FrontendInputs["context"] = *inp.ContextState
+		target.FrontendInputs["dockerfile"] = *inp.ContextState
 	case inp.ContextPath == "-":
 		if inp.DockerfilePath == "-" {
 			return nil, errStdinConflict
@@ -789,6 +797,7 @@ func LoadInputs(ctx context.Context, d driver.Driver, inp Inputs, pw progress.Wr
 		toRemove = append(toRemove, dockerfileDir)
 		dockerfileName = "Dockerfile"
 		target.FrontendAttrs["dockerfilekey"] = "dockerfile"
+		delete(target.FrontendInputs, "dockerfile")
 	}
 
 	if dockerfileName == "" {

build/url.go

@@ -14,7 +14,7 @@ import (
 )
 
 func createTempDockerfileFromURL(ctx context.Context, d driver.Driver, url string, pw progress.Writer) (string, error) {
-	c, err := d.Client(ctx)
+	c, err := driver.Boot(ctx, d, pw)
 	if err != nil {
 		return "", err
 	}

commands/bake.go

@@ -1,11 +1,14 @@
 package commands
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"os"
 
 	"github.com/docker/buildx/bake"
+	"github.com/docker/buildx/build"
+	"github.com/docker/buildx/util/progress"
 	"github.com/docker/cli/cli/command"
 	"github.com/moby/buildkit/util/appcontext"
 	"github.com/pkg/errors"
@@ -19,18 +22,16 @@ type bakeOptions struct {
 	commonOptions
 }
 
-func runBake(dockerCli command.Cli, targets []string, in bakeOptions) error {
+func runBake(dockerCli command.Cli, targets []string, in bakeOptions) (err error) {
 	ctx := appcontext.Context()
 
-	if len(in.files) == 0 {
+	var url string
-		files, err := defaultFiles()
+
-		if err != nil {
+	if len(targets) > 0 {
-			return err
+		if bake.IsRemoteURL(targets[0]) {
+			url = targets[0]
+			targets = targets[1:]
 		}
-		if len(files) == 0 {
-			return errors.Errorf("no docker-compose.yml or docker-bake.hcl found, specify build file with -f/--file")
-		}
-		in.files = files
 	}
 
 	if len(targets) == 0 {
@@ -52,8 +53,38 @@ func runBake(dockerCli command.Cli, targets []string, in bakeOptions) error {
 	if in.pull != nil {
 		overrides = append(overrides, fmt.Sprintf("*.pull=%t", *in.pull))
 	}
+	contextPathHash, _ := os.Getwd()
 
-	m, err := bake.ReadTargets(ctx, in.files, targets, overrides)
+	ctx2, cancel := context.WithCancel(context.TODO())
+	defer cancel()
+	printer := progress.NewPrinter(ctx2, os.Stderr, in.progress)
+
+	defer func() {
+		if printer != nil {
+			err1 := printer.Wait()
+			if err == nil {
+				err = err1
+			}
+		}
+	}()
+
+	dis, err := getInstanceOrDefault(ctx, dockerCli, in.builder, contextPathHash)
+	if err != nil {
+		return err
+	}
+
+	var files []bake.File
+	var inp *bake.Input
+	if url != "" {
+		files, inp, err = bake.ReadRemoteFiles(ctx, dis, url, in.files, printer)
+	} else {
+		files, err = bake.ReadLocalFiles(in.files)
+	}
+	if err != nil {
+		return err
+	}
+
+	m, err := bake.ReadTargets(ctx, files, targets, overrides)
 	if err != nil {
 		return err
 	}
@@ -63,40 +94,22 @@ func runBake(dockerCli command.Cli, targets []string, in bakeOptions) error {
 		if err != nil {
 			return err
 		}
+		err = printer.Wait()
+		printer = nil
+		if err != nil {
+			return err
+		}
 		fmt.Fprintln(dockerCli.Out(), string(dt))
 		return nil
 	}
 
-	bo, err := bake.TargetsToBuildOpt(m)
+	bo, err := bake.TargetsToBuildOpt(m, inp)
 	if err != nil {
 		return err
 	}
 
-	contextPathHash, _ := os.Getwd()
+	_, err = build.Build(ctx, dis, bo, dockerAPI(dockerCli), dockerCli.ConfigFile(), printer)
-
+	return err
-	return buildTargets(ctx, dockerCli, bo, in.progress, contextPathHash, in.builder)
-}
-
-func defaultFiles() ([]string, error) {
-	fns := []string{
-		"docker-compose.yml",  // support app
-		"docker-compose.yaml", // support app
-		"docker-bake.json",
-		"docker-bake.override.json",
-		"docker-bake.hcl",
-		"docker-bake.override.hcl",
-	}
-	out := make([]string, 0, len(fns))
-	for _, f := range fns {
-		if _, err := os.Stat(f); err != nil {
-			if os.IsNotExist(errors.Cause(err)) {
-				continue
-			}
-			return nil, err
-		}
-		out = append(out, f)
-	}
-	return out, nil
 }
 
 func bakeCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {