mirror of
https://gitea.com/Lydanne/buildx.git
synced 2025-08-16 16:55:57 +08:00
Compare commits
7 Commits
v0.22.0-rc
...
v0.22.0-rc
Author | SHA1 | Date | |
---|---|---|---|
![]() |
18ccba0720 | ||
![]() |
f5196f1167 | ||
![]() |
ef99381eab | ||
![]() |
00fdcd38ab | ||
![]() |
97f1d47464 | ||
![]() |
337578242d | ||
![]() |
503a8925d2 |
@@ -200,7 +200,7 @@ func (b *Builder) Boot(ctx context.Context) (bool, error) {
|
||||
err = err1
|
||||
}
|
||||
|
||||
if err == nil && len(errCh) == len(toBoot) {
|
||||
if err == nil && len(errCh) > 0 {
|
||||
return false, <-errCh
|
||||
}
|
||||
return true, err
|
||||
|
@@ -66,7 +66,11 @@ type bakeOptions struct {
|
||||
func runBake(ctx context.Context, dockerCli command.Cli, targets []string, in bakeOptions, cFlags commonFlags) (err error) {
|
||||
mp := dockerCli.MeterProvider()
|
||||
|
||||
ctx, end, err := tracing.TraceCurrentCommand(ctx, "bake")
|
||||
ctx, end, err := tracing.TraceCurrentCommand(ctx, append([]string{"bake"}, targets...),
|
||||
attribute.String("builder", in.builder),
|
||||
attribute.StringSlice("targets", targets),
|
||||
attribute.StringSlice("files", in.files),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -283,7 +287,7 @@ func runBake(ctx context.Context, dockerCli command.Cli, targets []string, in ba
|
||||
}
|
||||
}
|
||||
|
||||
if err := saveLocalStateGroup(dockerCli, in, targets, bo, overrides, def); err != nil {
|
||||
if err := saveLocalStateGroup(dockerCli, in, targets, bo); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -488,7 +492,14 @@ func bakeCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {
|
||||
return cmd
|
||||
}
|
||||
|
||||
func saveLocalStateGroup(dockerCli command.Cli, in bakeOptions, targets []string, bo map[string]build.Options, overrides []string, def any) error {
|
||||
func saveLocalStateGroup(dockerCli command.Cli, in bakeOptions, targets []string, bo map[string]build.Options) error {
|
||||
l, err := localstate.New(confutil.NewConfig(dockerCli))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer l.MigrateIfNeeded()
|
||||
|
||||
prm := confutil.MetadataProvenance()
|
||||
if len(in.metadataFile) == 0 {
|
||||
prm = confutil.MetadataProvenanceModeDisabled
|
||||
@@ -508,19 +519,10 @@ func saveLocalStateGroup(dockerCli command.Cli, in bakeOptions, targets []string
|
||||
if len(refs) == 0 {
|
||||
return nil
|
||||
}
|
||||
l, err := localstate.New(confutil.NewConfig(dockerCli))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dtdef, err := json.MarshalIndent(def, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return l.SaveGroup(groupRef, localstate.StateGroup{
|
||||
Definition: dtdef,
|
||||
Targets: targets,
|
||||
Inputs: overrides,
|
||||
Refs: refs,
|
||||
Refs: refs,
|
||||
Targets: targets,
|
||||
})
|
||||
}
|
||||
|
||||
|
@@ -286,7 +286,11 @@ func (o *buildOptionsHash) String() string {
|
||||
func runBuild(ctx context.Context, dockerCli command.Cli, options buildOptions) (err error) {
|
||||
mp := dockerCli.MeterProvider()
|
||||
|
||||
ctx, end, err := tracing.TraceCurrentCommand(ctx, "build")
|
||||
ctx, end, err := tracing.TraceCurrentCommand(ctx, []string{"build", options.contextPath},
|
||||
attribute.String("builder", options.builder),
|
||||
attribute.String("context", options.contextPath),
|
||||
attribute.String("dockerfile", options.dockerfileName),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
@@ -56,6 +56,7 @@ type Driver struct {
|
||||
restartPolicy container.RestartPolicy
|
||||
env []string
|
||||
defaultLoad bool
|
||||
gpus []container.DeviceRequest
|
||||
}
|
||||
|
||||
func (d *Driver) IsMobyDriver() bool {
|
||||
@@ -158,6 +159,9 @@ func (d *Driver) create(ctx context.Context, l progress.SubLogger) error {
|
||||
if d.cpusetMems != "" {
|
||||
hc.Resources.CpusetMems = d.cpusetMems
|
||||
}
|
||||
if len(d.gpus) > 0 && d.hasGPUCapability(ctx, cfg.Image, d.gpus) {
|
||||
hc.Resources.DeviceRequests = d.gpus
|
||||
}
|
||||
if info, err := d.DockerAPI.Info(ctx); err == nil {
|
||||
if info.CgroupDriver == "cgroupfs" {
|
||||
// Place all buildkit containers inside this cgroup by default so limits can be attached
|
||||
@@ -429,6 +433,31 @@ func (d *Driver) HostGatewayIP(ctx context.Context) (net.IP, error) {
|
||||
return nil, errors.New("host-gateway is not supported by the docker-container driver")
|
||||
}
|
||||
|
||||
// hasGPUCapability checks if docker daemon has GPU capability. We need to run
|
||||
// a dummy container with GPU device to check if the daemon has this capability
|
||||
// because there is no API to check it yet.
|
||||
func (d *Driver) hasGPUCapability(ctx context.Context, image string, gpus []container.DeviceRequest) bool {
|
||||
cfg := &container.Config{
|
||||
Image: image,
|
||||
Entrypoint: []string{"/bin/true"},
|
||||
}
|
||||
hc := &container.HostConfig{
|
||||
NetworkMode: container.NetworkMode(container.IPCModeNone),
|
||||
AutoRemove: true,
|
||||
Resources: container.Resources{
|
||||
DeviceRequests: gpus,
|
||||
},
|
||||
}
|
||||
resp, err := d.DockerAPI.ContainerCreate(ctx, cfg, hc, &network.NetworkingConfig{}, nil, "")
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
if err := d.DockerAPI.ContainerStart(ctx, resp.ID, container.StartOptions{}); err != nil {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func demuxConn(c net.Conn) net.Conn {
|
||||
pr, pw := io.Pipe()
|
||||
// TODO: rewrite parser with Reader() to avoid goroutine switch
|
||||
|
@@ -51,6 +51,12 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
|
||||
InitConfig: cfg,
|
||||
restartPolicy: rp,
|
||||
}
|
||||
var gpus dockeropts.GpuOpts
|
||||
if err := gpus.Set("all"); err == nil {
|
||||
if v := gpus.Value(); len(v) > 0 {
|
||||
d.gpus = v
|
||||
}
|
||||
}
|
||||
for k, v := range cfg.DriverOpts {
|
||||
switch {
|
||||
case k == "network":
|
||||
|
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"sync"
|
||||
|
||||
"github.com/docker/buildx/util/confutil"
|
||||
@@ -14,6 +15,7 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
version = 2
|
||||
refsDir = "refs"
|
||||
groupDir = "__group__"
|
||||
)
|
||||
@@ -31,12 +33,8 @@ type State struct {
|
||||
}
|
||||
|
||||
type StateGroup struct {
|
||||
// Definition is the raw representation of the group (bake definition)
|
||||
Definition []byte
|
||||
// Targets are the targets invoked
|
||||
Targets []string `json:",omitempty"`
|
||||
// Inputs are the user inputs (bake overrides)
|
||||
Inputs []string `json:",omitempty"`
|
||||
// Refs are used to track all the refs that belong to the same group
|
||||
Refs []string
|
||||
}
|
||||
@@ -52,9 +50,7 @@ func New(cfg *confutil.Config) (*LocalState, error) {
|
||||
if err := cfg.MkdirAll(refsDir, 0700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &LocalState{
|
||||
cfg: cfg,
|
||||
}, nil
|
||||
return &LocalState{cfg: cfg}, nil
|
||||
}
|
||||
|
||||
func (ls *LocalState) ReadRef(builderName, nodeName, id string) (*State, error) {
|
||||
@@ -87,8 +83,12 @@ func (ls *LocalState) SaveRef(builderName, nodeName, id string, st State) error
|
||||
return ls.cfg.AtomicWriteFile(filepath.Join(refDir, id), dt, 0644)
|
||||
}
|
||||
|
||||
func (ls *LocalState) GroupDir() string {
|
||||
return filepath.Join(ls.cfg.Dir(), refsDir, groupDir)
|
||||
}
|
||||
|
||||
func (ls *LocalState) ReadGroup(id string) (*StateGroup, error) {
|
||||
dt, err := os.ReadFile(filepath.Join(ls.cfg.Dir(), refsDir, groupDir, id))
|
||||
dt, err := os.ReadFile(filepath.Join(ls.GroupDir(), id))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -208,7 +208,7 @@ func (ls *LocalState) removeGroup(id string) error {
|
||||
if id == "" {
|
||||
return errors.Errorf("group ref empty")
|
||||
}
|
||||
f := filepath.Join(ls.cfg.Dir(), refsDir, groupDir, id)
|
||||
f := filepath.Join(ls.GroupDir(), id)
|
||||
if _, err := os.Lstat(f); err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
return err
|
||||
@@ -230,3 +230,16 @@ func (ls *LocalState) validate(builderName, nodeName, id string) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ls *LocalState) readVersion() int {
|
||||
if vdt, err := os.ReadFile(filepath.Join(ls.cfg.Dir(), refsDir, "version")); err == nil {
|
||||
if v, err := strconv.Atoi(string(vdt)); err == nil {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
func (ls *LocalState) writeVersion(version int) error {
|
||||
return ls.cfg.AtomicWriteFile(filepath.Join(refsDir, "version"), []byte(strconv.Itoa(version)), 0600)
|
||||
}
|
||||
|
@@ -68,10 +68,8 @@ var (
|
||||
|
||||
testStateGroupID = "kvqs0sgly2rmitz84r25u9qd0"
|
||||
testStateGroup = StateGroup{
|
||||
Definition: []byte(`{"group":{"default":{"targets":["pre-checkin"]},"pre-checkin":{"targets":["vendor-update","format","build"]}},"target":{"build":{"context":".","dockerfile":"dev.Dockerfile","target":"build-update","platforms":["linux/amd64"],"output":["."]},"format":{"context":".","dockerfile":"dev.Dockerfile","target":"format-update","platforms":["linux/amd64"],"output":["."]},"vendor-update":{"context":".","dockerfile":"dev.Dockerfile","target":"vendor-update","platforms":["linux/amd64"],"output":["."]}}}`),
|
||||
Targets: []string{"pre-checkin"},
|
||||
Inputs: []string{"*.platform=linux/amd64"},
|
||||
Refs: []string{"builder/builder0/hx2qf1w11qvz1x3k471c5i8xw", "builder/builder0/968zj0g03jmlx0s8qslnvh6rl", "builder/builder0/naf44f9i1710lf7y12lv5hb1z"},
|
||||
Targets: []string{"pre-checkin"},
|
||||
Refs: []string{"builder/builder0/hx2qf1w11qvz1x3k471c5i8xw", "builder/builder0/968zj0g03jmlx0s8qslnvh6rl", "builder/builder0/naf44f9i1710lf7y12lv5hb1z"},
|
||||
}
|
||||
|
||||
testStateGroupRef1ID = "hx2qf1w11qvz1x3k471c5i8xw"
|
||||
|
56
localstate/migrate.go
Normal file
56
localstate/migrate.go
Normal file
@@ -0,0 +1,56 @@
|
||||
package localstate
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
func (ls *LocalState) MigrateIfNeeded() error {
|
||||
currentVersion := ls.readVersion()
|
||||
if currentVersion == version {
|
||||
return nil
|
||||
}
|
||||
migrations := map[int]func(*LocalState) error{
|
||||
2: (*LocalState).migration2,
|
||||
}
|
||||
for v := currentVersion + 1; v <= version; v++ {
|
||||
migration, found := migrations[v]
|
||||
if !found {
|
||||
return errors.Errorf("localstate migration v%d not found", v)
|
||||
}
|
||||
if err := migration(ls); err != nil {
|
||||
return errors.Wrapf(err, "localstate migration v%d failed", v)
|
||||
}
|
||||
}
|
||||
return ls.writeVersion(version)
|
||||
}
|
||||
|
||||
func (ls *LocalState) migration2() error {
|
||||
return filepath.Walk(ls.GroupDir(), func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
dt, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var stg StateGroup
|
||||
if err := json.Unmarshal(dt, &stg); err != nil {
|
||||
return err
|
||||
}
|
||||
mdt, err := json.Marshal(stg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(path, mdt, 0600); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
@@ -2,7 +2,6 @@ package tracing
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/moby/buildkit/util/tracing/delegated"
|
||||
@@ -13,7 +12,7 @@ import (
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
func TraceCurrentCommand(ctx context.Context, name string) (context.Context, func(error), error) {
|
||||
func TraceCurrentCommand(ctx context.Context, args []string, attrs ...attribute.KeyValue) (context.Context, func(error), error) {
|
||||
opts := []sdktrace.TracerProviderOption{
|
||||
sdktrace.WithResource(detect.Resource()),
|
||||
sdktrace.WithBatcher(delegated.DefaultExporter),
|
||||
@@ -25,8 +24,8 @@ func TraceCurrentCommand(ctx context.Context, name string) (context.Context, fun
|
||||
}
|
||||
|
||||
tp := sdktrace.NewTracerProvider(opts...)
|
||||
ctx, span := tp.Tracer("").Start(ctx, name, trace.WithAttributes(
|
||||
attribute.String("command", strings.Join(os.Args, " ")),
|
||||
ctx, span := tp.Tracer("").Start(ctx, strings.Join(args, " "), trace.WithAttributes(
|
||||
attrs...,
|
||||
))
|
||||
|
||||
return ctx, func(err error) {
|
||||
|
Reference in New Issue
Block a user