docker/buildx
1
0
Fork 0
mirror of https://gitea.com/Lydanne/buildx.git synced 2025-05-28 08:27:42 +08:00
Code Issues Packages Projects Releases 1 Wiki Activity

bake: support for remote files

Browse Source 
Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
This commit is contained in:
Tonis Tiigi 2020-09-16 22:52:40 -07:00
parent 1496ac9b55
commit cffcd57edb
6 changed files with 389 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
bake/bake.go
View File

@ -5,6 +5,7 @@ import (
"io/ioutil"
"os"
"path"
"regexp"
"strconv"
"strings"
@ -12,14 +13,55 @@ import (
"github.com/docker/buildx/util/platformutil"
"github.com/docker/docker/pkg/urlutil"
hcl "github.com/hashicorp/hcl/v2"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/session/auth/authprovider"
"github.com/pkg/errors"
)
func ReadTargets(ctx context.Context, files, targets, overrides []string) (map[string]*Target, error) {
var httpPrefix = regexp.MustCompile(`^https?://`)
var gitURLPathWithFragmentSuffix = regexp.MustCompile(`\.git(?:#.+)?$`)
type File struct {
Name string
Data []byte
}
func defaultFilenames() []string {
return []string{
"docker-compose.yml", // support app
"docker-compose.yaml", // support app
"docker-bake.json",
"docker-bake.override.json",
"docker-bake.hcl",
"docker-bake.override.hcl",
}
}
func ReadLocalFiles(names []string) ([]File, error) {
isDefault := false
if len(names) == 0 {
isDefault = true
names = defaultFilenames()
}
out := make([]File, 0, len(names))
for _, n := range names {
dt, err := ioutil.ReadFile(n)
if err != nil {
if isDefault && errors.Is(err, os.ErrNotExist) {
continue
}
return nil, err
}
out = append(out, File{Name: n, Data: dt})
}
return out, nil
}
func ReadTargets(ctx context.Context, files []File, targets, overrides []string) (map[string]*Target, error) {
var c Config
for _, f := range files {
cfg, err := ParseFile(f)
cfg, err := ParseFile(f.Data, f.Name)
if err != nil {
return nil, err
}
@ -44,12 +86,7 @@ func ReadTargets(ctx context.Context, files, targets, overrides []string) (map[s
return m, nil
}
func ParseFile(fn string) (*Config, error) {
dt, err := ioutil.ReadFile(fn)
if err != nil {
return nil, err
}
func ParseFile(dt []byte, fn string) (*Config, error) {
fnl := strings.ToLower(fn)
if strings.HasSuffix(fnl, ".yml") || strings.HasSuffix(fnl, ".yaml") {
return ParseCompose(dt)
@ -350,6 +387,7 @@ type Target struct {
Outputs []string `json:"output,omitempty" hcl:"output,optional"`
Pull *bool `json:"pull,omitempty" hcl:"pull,optional"`
NoCache *bool `json:"no-cache,omitempty" hcl:"no-cache,optional"`
// IMPORTANT: if you add more fields here, do not forget to update newOverrides and README.
}
@ -363,10 +401,10 @@ func (t *Target) normalize() {
t.Outputs = removeDupes(t.Outputs)
}
func TargetsToBuildOpt(m map[string]*Target) (map[string]build.Options, error) {
func TargetsToBuildOpt(m map[string]*Target, inp *Input) (map[string]build.Options, error) {
m2 := make(map[string]build.Options, len(m))
for k, v := range m {
bo, err := toBuildOpt(v)
bo, err := toBuildOpt(v, inp)
if err != nil {
return nil, err
}
@ -375,7 +413,19 @@ func TargetsToBuildOpt(m map[string]*Target) (map[string]build.Options, error) {
return m2, nil
}
func toBuildOpt(t *Target) (*build.Options, error) {
func updateContext(t *build.Inputs, inp *Input) {
if inp == nil || inp.State == nil {
return
}
if t.ContextPath == "." {
t.ContextPath = inp.URL
return
}
st := llb.Scratch().File(llb.Copy(*inp.State, t.ContextPath, "/"), llb.WithCustomNamef("set context to %s", t.ContextPath))
t.ContextState = &st
}
func toBuildOpt(t *Target, inp *Input) (*build.Options, error) {
if v := t.Context; v != nil && *v == "-" {
return nil, errors.Errorf("context from stdin not allowed in bake")
}
@ -387,6 +437,7 @@ func toBuildOpt(t *Target) (*build.Options, error) {
if t.Context != nil {
contextPath = *t.Context
}
contextPath = path.Clean(contextPath)
dockerfilePath := "Dockerfile"
if t.Dockerfile != nil {
dockerfilePath = *t.Dockerfile
@ -405,11 +456,14 @@ func toBuildOpt(t *Target) (*build.Options, error) {
pull = *t.Pull
}
bi := build.Inputs{
ContextPath: contextPath,
DockerfilePath: dockerfilePath,
}
updateContext(&bi, inp)
bo := &build.Options{
Inputs: build.Inputs{
ContextPath: contextPath,
DockerfilePath: dockerfilePath,
},
Inputs: bi,
Tags: t.Tags,
BuildArgs: t.Args,
Labels: t.Labels,

61
bake/bake_test.go
View File

@ -2,9 +2,7 @@ package bake
import (
"context"
"io/ioutil"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
@ -12,12 +10,10 @@ import (
func TestReadTargets(t *testing.T) {
t.Parallel()
tmpdir, err := ioutil.TempDir("", "bake")
require.NoError(t, err)
defer os.RemoveAll(tmpdir)
fp := filepath.Join(tmpdir, "config.hcl")
err = ioutil.WriteFile(fp, []byte(`
fp := File{
Name: "config.hcl",
Data: []byte(`
target "webDEP" {
args = {
VAR_INHERITED = "webDEP"
@ -32,13 +28,13 @@ target "webapp" {
VAR_BOTH = "webapp"
}
inherits = ["webDEP"]
}`), 0600)
require.NoError(t, err)
}`),
}
ctx := context.TODO()
t.Run("NoOverrides", func(t *testing.T) {
m, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, nil)
m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, nil)
require.NoError(t, err)
require.Equal(t, 1, len(m))
@ -50,7 +46,7 @@ target "webapp" {
})
t.Run("InvalidTargetOverrides", func(t *testing.T) {
_, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{"nosuchtarget.context=foo"})
_, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"nosuchtarget.context=foo"})
require.NotNil(t, err)
require.Equal(t, err.Error(), "could not find any target matching 'nosuchtarget'")
})
@ -60,7 +56,7 @@ target "webapp" {
os.Setenv("VAR_FROMENV"+t.Name(), "fromEnv")
defer os.Unsetenv("VAR_FROM_ENV" + t.Name())
m, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{
m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{
"webapp.args.VAR_UNSET",
"webapp.args.VAR_EMPTY=",
"webapp.args.VAR_SET=bananas",
@ -89,7 +85,7 @@ target "webapp" {
// building leaf but overriding parent fields
t.Run("parent", func(t *testing.T) {
m, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{
m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{
"webDEP.args.VAR_INHERITED=override",
"webDEP.args.VAR_BOTH=override",
})
@ -100,23 +96,23 @@ target "webapp" {
})
t.Run("ContextOverride", func(t *testing.T) {
_, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{"webapp.context"})
_, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.context"})
require.NotNil(t, err)
m, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{"webapp.context=foo"})
m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.context=foo"})
require.NoError(t, err)
require.Equal(t, "foo", *m["webapp"].Context)
})
t.Run("NoCacheOverride", func(t *testing.T) {
m, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{"webapp.no-cache=false"})
m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.no-cache=false"})
require.NoError(t, err)
require.Equal(t, false, *m["webapp"].NoCache)
})
t.Run("PullOverride", func(t *testing.T) {
m, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{"webapp.pull=false"})
m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.pull=false"})
require.NoError(t, err)
require.Equal(t, false, *m["webapp"].Pull)
})
@ -176,7 +172,7 @@ target "webapp" {
}
for _, test := range cases {
t.Run(test.name, func(t *testing.T) {
m, err := ReadTargets(ctx, []string{fp}, test.targets, test.overrides)
m, err := ReadTargets(ctx, []File{fp}, test.targets, test.overrides)
test.check(t, m, err)
})
}
@ -185,14 +181,11 @@ target "webapp" {
func TestReadTargetsCompose(t *testing.T) {
t.Parallel()
tmpdir, err := ioutil.TempDir("", "bake")
require.NoError(t, err)
defer os.RemoveAll(tmpdir)
fp := filepath.Join(tmpdir, "docker-compose.yml")
err = ioutil.WriteFile(fp, []byte(`
version: "3"
fp := File{
Name: "docker-compose.yml",
Data: []byte(
`version: "3"
services:
db:
build: .
@ -203,13 +196,13 @@ services:
dockerfile: Dockerfile.webapp
args:
buildno: 1
`), 0600)
require.NoError(t, err)
fp2 := filepath.Join(tmpdir, "docker-compose2.yml")
err = ioutil.WriteFile(fp2, []byte(`
version: "3"
`),
}
fp2 := File{
Name: "docker-compose2.yml",
Data: []byte(
`version: "3"
services:
newservice:
build: .
@ -217,12 +210,12 @@ services:
build:
args:
buildno2: 12
`), 0600)
require.NoError(t, err)
`),
}
ctx := context.TODO()
m, err := ReadTargets(ctx, []string{fp, fp2}, []string{"default"}, nil)
m, err := ReadTargets(ctx, []File{fp, fp2}, []string{"default"}, nil)
require.NoError(t, err)
require.Equal(t, 3, len(m))

234
bake/remote.go Normal file
View File

@ -0,0 +1,234 @@
package bake
import (
"archive/tar"
"bytes"
"context"
"strings"
"github.com/docker/buildx/build"
"github.com/docker/buildx/driver"
"github.com/docker/buildx/util/progress"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/client/llb"
gwclient "github.com/moby/buildkit/frontend/gateway/client"
"github.com/pkg/errors"
)
type Input struct {
State *llb.State
URL string
}
func ReadRemoteFiles(ctx context.Context, dis []build.DriverInfo, url string, names []string, pw progress.Writer) ([]File, *Input, error) {
st, filename, ok := detectHttpContext(url)
if !ok {
st, ok = detectGitContext(url)
if !ok {
return nil, nil, errors.Errorf("not url context")
}
}
inp := &Input{State: st, URL: url}
var files []File
var di *build.DriverInfo
for _, d := range dis {
if d.Err == nil {
di = &d
continue
}
}
if di == nil {
return nil, nil, nil
}
c, err := driver.Boot(ctx, di.Driver, pw)
if err != nil {
return nil, nil, err
}
_, err = c.Build(ctx, client.SolveOpt{}, "buildx", func(ctx context.Context, c gwclient.Client) (*gwclient.Result, error) {
def, err := st.Marshal(ctx)
if err != nil {
return nil, err
}
res, err := c.Solve(ctx, gwclient.SolveRequest{
Definition: def.ToPB(),
})
if err != nil {
return nil, err
}
ref, err := res.SingleRef()
if err != nil {
return nil, err
}
if filename != "" {
files, err = filesFromURLRef(ctx, c, ref, inp, filename, names)
} else {
files, err = filesFromRef(ctx, ref, names)
}
return nil, err
}, progress.NewChannel(pw))
if err != nil {
return nil, nil, err
}
return files, inp, nil
}
func IsRemoteURL(url string) bool {
if _, _, ok := detectHttpContext(url); ok {
return true
}
if _, ok := detectGitContext(url); ok {
return true
}
return false
}
func detectHttpContext(url string) (*llb.State, string, bool) {
if httpPrefix.MatchString(url) {
httpContext := llb.HTTP(url, llb.Filename("context"), llb.WithCustomName("[internal] load remote build context"))
return &httpContext, "context", true
}
return nil, "", false
}
func detectGitContext(ref string) (*llb.State, bool) {
found := false
if httpPrefix.MatchString(ref) && gitURLPathWithFragmentSuffix.MatchString(ref) {
found = true
}
for _, prefix := range []string{"git://", "github.com/", "git@"} {
if strings.HasPrefix(ref, prefix) {
found = true
break
}
}
if !found {
return nil, false
}
parts := strings.SplitN(ref, "#", 2)
branch := ""
if len(parts) > 1 {
branch = parts[1]
}
gitOpts := []llb.GitOption{llb.WithCustomName("[internal] load git source " + ref)}
st := llb.Git(parts[0], branch, gitOpts...)
return &st, true
}
func isArchive(header []byte) bool {
for _, m := range [][]byte{
{0x42, 0x5A, 0x68}, // bzip2
{0x1F, 0x8B, 0x08}, // gzip
{0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}, // xz
} {
if len(header) < len(m) {
continue
}
if bytes.Equal(m, header[:len(m)]) {
return true
}
}
r := tar.NewReader(bytes.NewBuffer(header))
_, err := r.Next()
return err == nil
}
func filesFromURLRef(ctx context.Context, c gwclient.Client, ref gwclient.Reference, inp *Input, filename string, names []string) ([]File, error) {
stat, err := ref.StatFile(ctx, gwclient.StatRequest{Path: filename})
if err != nil {
return nil, err
}
dt, err := ref.ReadFile(ctx, gwclient.ReadRequest{
Filename: filename,
Range: &gwclient.FileRange{
Length: 1024,
},
})
if err != nil {
return nil, err
}
if isArchive(dt) {
bc := llb.Scratch().File(llb.Copy(inp.State, filename, "/", &llb.CopyInfo{
AttemptUnpack: true,
}))
inp.State = &bc
inp.URL = ""
def, err := bc.Marshal(ctx)
if err != nil {
return nil, err
}
res, err := c.Solve(ctx, gwclient.SolveRequest{
Definition: def.ToPB(),
})
if err != nil {
return nil, err
}
ref, err := res.SingleRef()
if err != nil {
return nil, err
}
return filesFromRef(ctx, ref, names)
}
inp.State = nil
name := inp.URL
inp.URL = ""
if len(dt) > stat.Size() {
if stat.Size() > 1024*512 {
return nil, errors.Errorf("non-archive definition URL bigger than maximum allowed size")
}
dt, err = ref.ReadFile(ctx, gwclient.ReadRequest{
Filename: filename,
})
if err != nil {
return nil, err
}
}
return []File{{Name: name, Data: dt}}, nil
}
func filesFromRef(ctx context.Context, ref gwclient.Reference, names []string) ([]File, error) {
// TODO: auto-remove parent dir in needed
var files []File
isDefault := false
if len(names) == 0 {
isDefault = true
names = defaultFilenames()
}
for _, name := range names {
_, err := ref.StatFile(ctx, gwclient.StatRequest{Path: name})
if err != nil {
if isDefault {
continue
}
return nil, err
}
dt, err := ref.ReadFile(ctx, gwclient.ReadRequest{Filename: name})
if err != nil {
return nil, err
}
files = append(files, File{Name: name, Data: dt})
}
return files, nil
}

9
build/build.go
View File

@ -22,6 +22,7 @@ import (
dockerclient "github.com/docker/docker/client"
"github.com/docker/docker/pkg/urlutil"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/upload/uploadprovider"
"github.com/moby/buildkit/util/entitlements"
@ -64,6 +65,7 @@ type Inputs struct {
ContextPath string
DockerfilePath string
InStream io.Reader
ContextState *llb.State
}
type DriverInfo struct {
@ -723,6 +725,12 @@ func LoadInputs(ctx context.Context, d driver.Driver, inp Inputs, pw progress.Wr
)
switch {
case inp.ContextState != nil:
if target.FrontendInputs == nil {
target.FrontendInputs = make(map[string]llb.State)
}
target.FrontendInputs["context"] = *inp.ContextState
target.FrontendInputs["dockerfile"] = *inp.ContextState
case inp.ContextPath == "-":
if inp.DockerfilePath == "-" {
return nil, errStdinConflict
@ -789,6 +797,7 @@ func LoadInputs(ctx context.Context, d driver.Driver, inp Inputs, pw progress.Wr
toRemove = append(toRemove, dockerfileDir)
dockerfileName = "Dockerfile"
target.FrontendAttrs["dockerfilekey"] = "dockerfile"
delete(target.FrontendInputs, "dockerfile")
}
if dockerfileName == "" {

2
build/url.go
View File

@ -14,7 +14,7 @@ import (
)
func createTempDockerfileFromURL(ctx context.Context, d driver.Driver, url string, pw progress.Writer) (string, error) {
c, err := d.Client(ctx)
c, err := driver.Boot(ctx, d, pw)
if err != nil {
return "", err
}

85
commands/bake.go
View File

@ -1,11 +1,14 @@
package commands
import (
"context"
"encoding/json"
"fmt"
"os"
"github.com/docker/buildx/bake"
"github.com/docker/buildx/build"
"github.com/docker/buildx/util/progress"
"github.com/docker/cli/cli/command"
"github.com/moby/buildkit/util/appcontext"
"github.com/pkg/errors"
@ -19,18 +22,16 @@ type bakeOptions struct {
commonOptions
}
func runBake(dockerCli command.Cli, targets []string, in bakeOptions) error {
func runBake(dockerCli command.Cli, targets []string, in bakeOptions) (err error) {
ctx := appcontext.Context()
if len(in.files) == 0 {
files, err := defaultFiles()
if err != nil {
return err
var url string
if len(targets) > 0 {
if bake.IsRemoteURL(targets[0]) {
url = targets[0]
targets = targets[1:]
}
if len(files) == 0 {
return errors.Errorf("no docker-compose.yml or docker-bake.hcl found, specify build file with -f/--file")
}
in.files = files
}
if len(targets) == 0 {
@ -52,8 +53,38 @@ func runBake(dockerCli command.Cli, targets []string, in bakeOptions) error {
if in.pull != nil {
overrides = append(overrides, fmt.Sprintf("*.pull=%t", *in.pull))
}
contextPathHash, _ := os.Getwd()
m, err := bake.ReadTargets(ctx, in.files, targets, overrides)
ctx2, cancel := context.WithCancel(context.TODO())
defer cancel()
printer := progress.NewPrinter(ctx2, os.Stderr, in.progress)
defer func() {
if printer != nil {
err1 := printer.Wait()
if err == nil {
err = err1
}
}
}()
dis, err := getInstanceOrDefault(ctx, dockerCli, in.builder, contextPathHash)
if err != nil {
return err
}
var files []bake.File
var inp *bake.Input
if url != "" {
files, inp, err = bake.ReadRemoteFiles(ctx, dis, url, in.files, printer)
} else {
files, err = bake.ReadLocalFiles(in.files)
}
if err != nil {
return err
}
m, err := bake.ReadTargets(ctx, files, targets, overrides)
if err != nil {
return err
}
@ -63,40 +94,22 @@ func runBake(dockerCli command.Cli, targets []string, in bakeOptions) error {
if err != nil {
return err
}
err = printer.Wait()
printer = nil
if err != nil {
return err
}
fmt.Fprintln(dockerCli.Out(), string(dt))
return nil
}
bo, err := bake.TargetsToBuildOpt(m)
bo, err := bake.TargetsToBuildOpt(m, inp)
if err != nil {
return err
}
contextPathHash, _ := os.Getwd()
return buildTargets(ctx, dockerCli, bo, in.progress, contextPathHash, in.builder)
}
func defaultFiles() ([]string, error) {
fns := []string{
"docker-compose.yml", // support app
"docker-compose.yaml", // support app
"docker-bake.json",
"docker-bake.override.json",
"docker-bake.hcl",
"docker-bake.override.hcl",
}
out := make([]string, 0, len(fns))
for _, f := range fns {
if _, err := os.Stat(f); err != nil {
if os.IsNotExist(errors.Cause(err)) {
continue
}
return nil, err
}
out = append(out, f)
}
return out, nil
_, err = build.Build(ctx, dis, bo, dockerAPI(dockerCli), dockerCli.ConfigFile(), printer)
return err
}
func bakeCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {