vendor: github.com/moby/buildkit 25bec7145b39 (v0.14.0-dev)

Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com>
This commit is contained in:
CrazyMax
2024-03-28 17:43:43 +01:00
parent 8abef59087
commit de5efcb03b
31 changed files with 402 additions and 232 deletions

View File

@ -101,7 +101,7 @@ func New(ctx context.Context, address string, opts ...ClientOpt) (*Client, error
}
if tracerProvider != nil {
var propagators = propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{})
propagators := propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{})
unary = append(unary, filterInterceptor(otelgrpc.UnaryClientInterceptor(otelgrpc.WithTracerProvider(tracerProvider), otelgrpc.WithPropagators(propagators)))) //nolint:staticcheck // TODO(thaJeztah): ignore SA1019 for deprecated options: see https://github.com/moby/buildkit/issues/4681
stream = append(stream, otelgrpc.StreamClientInterceptor(otelgrpc.WithTracerProvider(tracerProvider), otelgrpc.WithPropagators(propagators))) //nolint:staticcheck // TODO(thaJeztah): ignore SA1019 for deprecated options: see https://github.com/moby/buildkit/issues/4681
}
@ -111,11 +111,17 @@ func New(ctx context.Context, address string, opts ...ClientOpt) (*Client, error
if err != nil {
return nil, err
}
gopts = append(gopts, grpc.WithContextDialer(dialFn))
if dialFn != nil {
gopts = append(gopts, grpc.WithContextDialer(dialFn))
}
}
if address == "" {
address = appdefaults.Address
}
uri, err := url.Parse(address)
if err != nil {
return nil, err
}
// Setting :authority pseudo header
// - HTTP/2 (RFC7540) defines :authority pseudo header includes
@ -130,12 +136,14 @@ func New(ctx context.Context, address string, opts ...ClientOpt) (*Client, error
}
if authority == "" {
// authority as hostname from target address
uri, err := url.Parse(address)
if err != nil {
return nil, err
}
authority = uri.Host
}
if uri.Scheme == "tcp" {
// remove tcp scheme from address, since default dialer doesn't expect that
// name resolution is done by grpc according to the following spec: https://github.com/grpc/grpc/blob/master/doc/naming.md
address = uri.Host
}
gopts = append(gopts, grpc.WithAuthority(authority))
unary = append(unary, grpcerrors.UnaryClientInterceptor)
@ -375,8 +383,7 @@ func resolveDialer(address string) (func(context.Context, string) (net.Conn, err
if ch != nil {
return ch.ContextDialer, nil
}
// basic dialer
return dialer, nil
return nil, nil
}
func filterInterceptor(intercept grpc.UnaryClientInterceptor) grpc.UnaryClientInterceptor {

View File

@ -1,21 +0,0 @@
//go:build !windows
// +build !windows
package client
import (
"context"
"net"
"strings"
"github.com/pkg/errors"
)
func dialer(ctx context.Context, address string) (net.Conn, error) {
addrParts := strings.SplitN(address, "://", 2)
if len(addrParts) != 2 {
return nil, errors.Errorf("invalid address %s", address)
}
var d net.Dialer
return d.DialContext(ctx, addrParts[0], addrParts[1])
}

View File

@ -1,25 +0,0 @@
package client
import (
"context"
"net"
"strings"
winio "github.com/Microsoft/go-winio"
"github.com/pkg/errors"
)
func dialer(ctx context.Context, address string) (net.Conn, error) {
addrParts := strings.SplitN(address, "://", 2)
if len(addrParts) != 2 {
return nil, errors.Errorf("invalid address %s", address)
}
switch addrParts[0] {
case "npipe":
address = strings.Replace(addrParts[1], "/", "\\", -1)
return winio.DialPipeContext(ctx, address)
default:
var d net.Dialer
return d.DialContext(ctx, addrParts[0], addrParts[1])
}
}

View File

@ -0,0 +1,8 @@
// Package npipe provides connhelper for npipe://<address>
package npipe
import "github.com/moby/buildkit/client/connhelper"
func init() {
connhelper.Register("npipe", Helper)
}

View File

@ -0,0 +1,14 @@
//go:build !windows
package npipe
import (
"errors"
"net/url"
"github.com/moby/buildkit/client/connhelper"
)
func Helper(u *url.URL) (*connhelper.ConnectionHelper, error) {
return nil, errors.New("npipe connections are only supported on windows")
}

View File

@ -0,0 +1,28 @@
//go:build windows
package npipe
import (
"context"
"net"
"net/url"
"strings"
"github.com/Microsoft/go-winio"
"github.com/moby/buildkit/client/connhelper"
"github.com/pkg/errors"
)
// Helper returns helper for connecting to a url via npipes.
func Helper(u *url.URL) (*connhelper.ConnectionHelper, error) {
addrParts := strings.SplitN(u.String(), "://", 2)
if len(addrParts) != 2 {
return nil, errors.Errorf("invalid address %s", u)
}
address := strings.Replace(addrParts[1], "/", "\\", -1)
return &connhelper.ConnectionHelper{
ContextDialer: func(ctx context.Context, addr string) (net.Conn, error) {
return winio.DialPipeContext(ctx, address)
},
}, nil
}

View File

@ -438,15 +438,23 @@ func (e *ExecOp) Output() Output {
}
func (e *ExecOp) Inputs() (inputs []Output) {
mm := map[Output]struct{}{}
// make sure mounts are sorted
// the same sort occurs in (*ExecOp).Marshal, and this
// sort must be the same
sort.Slice(e.mounts, func(i int, j int) bool {
return e.mounts[i].target < e.mounts[j].target
})
seen := map[Output]struct{}{}
for _, m := range e.mounts {
if m.source != nil {
mm[m.source] = struct{}{}
if _, ok := seen[m.source]; !ok {
inputs = append(inputs, m.source)
seen[m.source] = struct{}{}
}
}
}
for o := range mm {
inputs = append(inputs, o)
}
return
}

View File

@ -96,24 +96,35 @@ func (fa *FileAction) Copy(input CopyInput, src, dest string, opt ...CopyOption)
return a
}
func (fa *FileAction) allOutputs(m map[Output]struct{}) {
func (fa *FileAction) allOutputs(seen map[Output]struct{}, outputs []Output) []Output {
if fa == nil {
return
return outputs
}
if fa.state != nil && fa.state.Output() != nil {
m[fa.state.Output()] = struct{}{}
if fa.state != nil {
out := fa.state.Output()
if out != nil {
if _, ok := seen[out]; !ok {
outputs = append(outputs, out)
seen[out] = struct{}{}
}
}
}
if a, ok := fa.action.(*fileActionCopy); ok {
if a.state != nil {
if out := a.state.Output(); out != nil {
m[out] = struct{}{}
out := a.state.Output()
if out != nil {
if _, ok := seen[out]; !ok {
outputs = append(outputs, out)
seen[out] = struct{}{}
}
}
} else if a.fas != nil {
a.fas.allOutputs(m)
outputs = a.fas.allOutputs(seen, outputs)
}
}
fa.prev.allOutputs(m)
return fa.prev.allOutputs(seen, outputs)
}
func (fa *FileAction) bind(s State) *FileAction {
@ -806,15 +817,8 @@ func (f *FileOp) Output() Output {
return f.output
}
func (f *FileOp) Inputs() (inputs []Output) {
mm := map[Output]struct{}{}
f.action.allOutputs(mm)
for o := range mm {
inputs = append(inputs, o)
}
return inputs
func (f *FileOp) Inputs() []Output {
return f.action.allOutputs(map[Output]struct{}{}, []Output{})
}
func getIndex(input pb.InputIndex, len int, relative *int) pb.InputIndex {

View File

@ -227,6 +227,11 @@ type ImageInfo struct {
RecordType string
}
const (
GitAuthHeaderKey = "GIT_AUTH_HEADER"
GitAuthTokenKey = "GIT_AUTH_TOKEN"
)
// Git returns a state that represents a git repository.
// Example:
//
@ -267,8 +272,8 @@ func Git(url, ref string, opts ...GitOption) State {
}
gi := &GitInfo{
AuthHeaderSecret: "GIT_AUTH_HEADER",
AuthTokenSecret: "GIT_AUTH_TOKEN",
AuthHeaderSecret: GitAuthHeaderKey,
AuthTokenSecret: GitAuthTokenKey,
}
for _, o := range opts {
o.SetGitOption(gi)

View File

@ -102,6 +102,7 @@ func (s StoreIndex) Put(tag string, desc ocispecs.Descriptor) error {
}
}
setOCIIndexDefaults(&idx)
if err = insertDesc(&idx, desc, tag); err != nil {
return err
}
@ -145,6 +146,19 @@ func (s StoreIndex) GetSingle() (*ocispecs.Descriptor, error) {
return nil, nil
}
// setOCIIndexDefaults updates zero values in index to their default values.
func setOCIIndexDefaults(index *ocispecs.Index) {
if index == nil {
return
}
if index.SchemaVersion == 0 {
index.SchemaVersion = 2
}
if index.MediaType == "" {
index.MediaType = ocispecs.MediaTypeImageIndex
}
}
// insertDesc puts desc to index with tag.
// Existing manifests with the same tag will be removed from the index.
func insertDesc(index *ocispecs.Index, desc ocispecs.Descriptor, tag string) error {
@ -152,9 +166,6 @@ func insertDesc(index *ocispecs.Index, desc ocispecs.Descriptor, tag string) err
return nil
}
if index.SchemaVersion == 0 {
index.SchemaVersion = 2
}
if tag != "" {
if desc.Annotations == nil {
desc.Annotations = make(map[string]string)

View File

@ -51,26 +51,6 @@ func (mp *MultiProvider) SnapshotLabels(descs []ocispecs.Descriptor, index int)
return nil
}
func (mp *MultiProvider) CheckDescriptor(ctx context.Context, desc ocispecs.Descriptor) error {
type checkDescriptor interface {
CheckDescriptor(context.Context, ocispecs.Descriptor) error
}
mp.mu.RLock()
if p, ok := mp.sub[desc.Digest]; ok {
mp.mu.RUnlock()
if cd, ok := p.(checkDescriptor); ok {
return cd.CheckDescriptor(ctx, desc)
}
} else {
mp.mu.RUnlock()
}
if cd, ok := mp.base.(checkDescriptor); ok {
return cd.CheckDescriptor(ctx, desc)
}
return nil
}
// ReaderAt returns a content.ReaderAt
func (mp *MultiProvider) ReaderAt(ctx context.Context, desc ocispecs.Descriptor) (content.ReaderAt, error) {
mp.mu.RLock()

View File

@ -57,7 +57,9 @@ func ParseGitRef(ref string) (*GitRef, error) {
err error
)
if strings.HasPrefix(ref, "github.com/") {
if strings.HasPrefix(ref, "./") || strings.HasPrefix(ref, "../") {
return nil, errdefs.ErrInvalidArgument
} else if strings.HasPrefix(ref, "github.com/") {
res.IndistinguishableFromLocal = true // Deprecated
remote = fromURL(&url.URL{
Scheme: "https",

View File

@ -25,7 +25,7 @@ func (nopLog) Logf(string, ...interface{}) {}
const (
shortLen = 12
defaultDockerdBinary = "dockerd"
DefaultDockerdBinary = "dockerd"
)
type Option func(*Daemon)
@ -43,6 +43,7 @@ type Daemon struct {
pidFile string
sockPath string
args []string
envs []string
}
var sockRoot = filepath.Join(os.TempDir(), "docker-integration")
@ -69,9 +70,10 @@ func NewDaemon(workingDir string, ops ...Option) (*Daemon, error) {
storageDriver: os.Getenv("DOCKER_GRAPHDRIVER"),
// dxr stands for docker-execroot (shortened for avoiding unix(7) path length limitation)
execRoot: filepath.Join(os.TempDir(), "dxr", id),
dockerdBinary: defaultDockerdBinary,
dockerdBinary: DefaultDockerdBinary,
Log: nopLog{},
sockPath: filepath.Join(sockRoot, id+".sock"),
envs: append([]string{}, os.Environ()...),
}
for _, op := range ops {
@ -81,6 +83,18 @@ func NewDaemon(workingDir string, ops ...Option) (*Daemon, error) {
return d, nil
}
func WithBinary(bin string) Option {
return func(d *Daemon) {
d.dockerdBinary = bin
}
}
func WithExtraEnv(envs []string) Option {
return func(d *Daemon) {
d.envs = append(d.envs, envs...)
}
}
func (d *Daemon) Sock() string {
return "unix://" + d.sockPath
}
@ -88,7 +102,7 @@ func (d *Daemon) Sock() string {
func (d *Daemon) StartWithError(daemonLogs map[string]*bytes.Buffer, providedArgs ...string) error {
dockerdBinary, err := exec.LookPath(d.dockerdBinary)
if err != nil {
return errors.Wrapf(err, "[%s] could not find docker binary in $PATH", d.id)
return errors.Wrapf(err, "[%s] could not find dockerd binary %q in $PATH", d.id, d.dockerdBinary)
}
if d.pidFile == "" {
@ -127,7 +141,7 @@ func (d *Daemon) StartWithError(daemonLogs map[string]*bytes.Buffer, providedArg
d.args = append(d.args, providedArgs...)
d.cmd = exec.Command(dockerdBinary, d.args...)
d.cmd.Env = append(os.Environ(), "DOCKER_SERVICE_PREFER_OFFLINE_IMAGE=1", "BUILDKIT_DEBUG_EXEC_OUTPUT=1", "BUILDKIT_DEBUG_PANIC_ON_ERROR=1")
d.cmd.Env = append(d.envs, "DOCKER_SERVICE_PREFER_OFFLINE_IMAGE=1", "BUILDKIT_DEBUG_EXEC_OUTPUT=1", "BUILDKIT_DEBUG_PANIC_ON_ERROR=1")
if daemonLogs != nil {
b := new(bytes.Buffer)

View File

@ -40,6 +40,7 @@ type Backend interface {
Rootless() bool
NetNSDetached() bool
Snapshotter() string
ExtraEnv() []string
Supports(feature string) bool
}

View File

@ -4,6 +4,9 @@ import (
"net"
"github.com/Microsoft/go-winio"
// include npipe connhelper for windows tests
_ "github.com/moby/buildkit/client/connhelper/npipe"
)
var socketScheme = "npipe://"

View File

@ -12,6 +12,7 @@ type backend struct {
rootless bool
netnsDetached bool
snapshotter string
extraEnv []string
unsupportedFeatures []string
isDockerd bool
}
@ -40,6 +41,10 @@ func (b backend) Snapshotter() string {
return b.snapshotter
}
func (b backend) ExtraEnv() []string {
return b.extraEnv
}
func (b backend) Supports(feature string) bool {
if enabledFeatures := os.Getenv("BUILDKIT_TEST_ENABLE_FEATURES"); enabledFeatures != "" {
for _, enabledFeature := range strings.Split(enabledFeatures, ",") {

View File

@ -244,6 +244,7 @@ disabled_plugins = ["cri"]
rootless: rootless,
netnsDetached: false,
snapshotter: c.Snapshotter,
extraEnv: c.ExtraEnv,
}, cl, nil
}

View File

@ -55,12 +55,12 @@ func InitDockerdWorker() {
}
type Moby struct {
ID string
IsRootless bool
ID string
Binary string
IsRootless bool
ContainerdSnapshotter bool
Unsupported []string
Unsupported []string
ExtraEnv []string
}
func (c Moby) Name() string {
@ -137,7 +137,13 @@ func (c Moby) New(ctx context.Context, cfg *integration.BackendConfig) (b integr
return nil, nil, err
}
d, err := dockerd.NewDaemon(workDir)
dockerdOpts := []dockerd.Option{
dockerd.WithExtraEnv(c.ExtraEnv),
}
if c.Binary != "" {
dockerdOpts = append(dockerdOpts, dockerd.WithBinary(c.Binary))
}
d, err := dockerd.NewDaemon(workDir, dockerdOpts...)
if err != nil {
return nil, nil, errors.Errorf("new daemon error: %q, %s", err, integration.FormatLogs(cfg.Logs))
}
@ -164,7 +170,7 @@ func (c Moby) New(ctx context.Context, cfg *integration.BackendConfig) (b integr
deferF.Append(d.StopWithError)
if err := integration.WaitSocket(d.Sock(), 5*time.Second, nil); err != nil {
return nil, nil, errors.Errorf("dockerd did not start up: %q, %s", err, integration.FormatLogs(cfg.Logs))
return nil, nil, errors.Wrapf(err, "dockerd did not start up: %s", integration.FormatLogs(cfg.Logs))
}
dockerAPI, err := client.NewClientWithOpts(client.WithHost(d.Sock()))
@ -229,6 +235,7 @@ func (c Moby) New(ctx context.Context, cfg *integration.BackendConfig) (b integr
dockerAddress: d.Sock(),
rootless: c.IsRootless,
netnsDetached: false,
extraEnv: c.ExtraEnv,
isDockerd: true,
unsupportedFeatures: c.Unsupported,
}, cl, nil

View File

@ -5,7 +5,6 @@ import (
"sync"
"github.com/moby/buildkit/util/tracing/detect"
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
)
@ -14,9 +13,9 @@ const maxBuffer = 256
var exp = &Exporter{}
func init() {
detect.Register("delegated", func() (sdktrace.SpanExporter, sdkmetric.Exporter, error) {
return exp, nil, nil
}, 100)
detect.Register("delegated", detect.TraceExporterDetector(func() (sdktrace.SpanExporter, error) {
return exp, nil
}), 100)
}
type Exporter struct {

View File

@ -21,7 +21,10 @@ import (
"go.opentelemetry.io/otel/trace/noop"
)
type ExporterDetector func() (sdktrace.SpanExporter, sdkmetric.Exporter, error)
type ExporterDetector interface {
DetectTraceExporter() (sdktrace.SpanExporter, error)
DetectMetricExporter() (sdkmetric.Exporter, error)
}
type detector struct {
f ExporterDetector
@ -52,17 +55,45 @@ func Register(name string, exp ExporterDetector, priority int) {
}
}
func detectExporter() (texp sdktrace.SpanExporter, mexp sdkmetric.Exporter, err error) {
if n := os.Getenv("OTEL_TRACES_EXPORTER"); n != "" {
type TraceExporterDetector func() (sdktrace.SpanExporter, error)
func (fn TraceExporterDetector) DetectTraceExporter() (sdktrace.SpanExporter, error) {
return fn()
}
func (fn TraceExporterDetector) DetectMetricExporter() (sdkmetric.Exporter, error) {
return nil, nil
}
func detectExporters() (texp sdktrace.SpanExporter, mexp sdkmetric.Exporter, err error) {
texp, err = detectExporter("OTEL_TRACES_EXPORTER", func(d ExporterDetector) (sdktrace.SpanExporter, bool, error) {
exp, err := d.DetectTraceExporter()
return exp, exp != nil, err
})
if err != nil {
return nil, nil, err
}
mexp, err = detectExporter("OTEL_METRICS_EXPORTER", func(d ExporterDetector) (sdkmetric.Exporter, bool, error) {
exp, err := d.DetectMetricExporter()
return exp, exp != nil, err
})
if err != nil {
return nil, nil, err
}
return texp, mexp, nil
}
func detectExporter[T any](envVar string, fn func(d ExporterDetector) (T, bool, error)) (exp T, err error) {
if n := os.Getenv(envVar); n != "" {
d, ok := detectors[n]
if !ok {
if n == "none" {
return nil, nil, nil
}
return nil, nil, errors.Errorf("unsupported opentelemetry tracer %v", n)
return exp, errors.Errorf("unsupported opentelemetry exporter %v", n)
}
return d.f()
exp, _, err = fn(d.f)
return exp, err
}
arr := make([]detector, 0, len(detectors))
for _, d := range detectors {
arr = append(arr, d)
@ -71,28 +102,22 @@ func detectExporter() (texp sdktrace.SpanExporter, mexp sdkmetric.Exporter, err
return arr[i].priority < arr[j].priority
})
var ok bool
for _, d := range arr {
t, m, err := d.f()
exp, ok, err = fn(d.f)
if err != nil {
return nil, nil, err
}
if texp == nil {
texp = t
}
if mexp == nil {
mexp = m
return exp, err
}
// Found a candidate for both exporters so just return now.
if texp != nil && mexp != nil {
return texp, mexp, nil
if ok {
break
}
}
return texp, mexp, nil
return exp, nil
}
func getExporters() (sdktrace.SpanExporter, sdkmetric.Exporter, error) {
texp, mexp, err := detectExporter()
texp, mexp, err := detectExporters()
if err != nil {
return nil, nil, err
}
@ -253,3 +278,20 @@ func (serviceNameDetector) Detect(ctx context.Context) (*resource.Resource, erro
},
).Detect(ctx)
}
type noneDetector struct{}
func (n noneDetector) DetectTraceExporter() (sdktrace.SpanExporter, error) {
return nil, nil
}
func (n noneDetector) DetectMetricExporter() (sdkmetric.Exporter, error) {
return nil, nil
}
func init() {
// Register a none detector. This will never be chosen if there's another suitable
// exporter that can be detected, but exists to allow telemetry to be explicitly
// disabled.
Register("none", noneDetector{}, 1000)
}

View File

@ -15,24 +15,15 @@ import (
sdktrace "go.opentelemetry.io/otel/sdk/trace"
)
var otlpExporter = otlpExporterDetector{}
func init() {
Register("otlp", otlpExporter, 10)
}
func otlpExporter() (sdktrace.SpanExporter, sdkmetric.Exporter, error) {
texp, err := otlpSpanExporter()
if err != nil {
return nil, nil, err
}
type otlpExporterDetector struct{}
mexp, err := otlpMetricExporter()
if err != nil {
return nil, nil, err
}
return texp, mexp, nil
}
func otlpSpanExporter() (sdktrace.SpanExporter, error) {
func (otlpExporterDetector) DetectTraceExporter() (sdktrace.SpanExporter, error) {
set := os.Getenv("OTEL_TRACES_EXPORTER") == "otlp" || os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT") != "" || os.Getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT") != ""
if !set {
return nil, nil
@ -61,7 +52,7 @@ func otlpSpanExporter() (sdktrace.SpanExporter, error) {
return otlptrace.New(context.Background(), c)
}
func otlpMetricExporter() (sdkmetric.Exporter, error) {
func (otlpExporterDetector) DetectMetricExporter() (sdkmetric.Exporter, error) {
set := os.Getenv("OTEL_METRICS_EXPORTER") == "otlp" || os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT") != "" || os.Getenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT") != ""
if !set {
return nil, nil