vendor: update buildkit to master@31c870e82a48

Signed-off-by: Justin Chadwell <me@jedevc.com>
This commit is contained in:
Justin Chadwell
2023-05-15 18:32:31 +01:00
parent 167cd16acb
commit e61a8cf637
269 changed files with 25798 additions and 3371 deletions

View File

@ -24,7 +24,7 @@ type DefinitionOp struct {
platforms map[digest.Digest]*ocispecs.Platform
dgst digest.Digest
index pb.OutputIndex
inputCache map[digest.Digest][]*DefinitionOp
inputCache *sync.Map // shared and written among DefinitionOps so avoid race on this map using sync.Map
}
// NewDefinitionOp returns a new operation from a marshalled definition.
@ -101,7 +101,7 @@ func NewDefinitionOp(def *pb.Definition) (*DefinitionOp, error) {
platforms: platforms,
dgst: dgst,
index: index,
inputCache: make(map[digest.Digest][]*DefinitionOp),
inputCache: new(sync.Map),
}, nil
}
@ -180,6 +180,18 @@ func (d *DefinitionOp) Output() Output {
}}
}
func (d *DefinitionOp) loadInputCache(dgst digest.Digest) ([]*DefinitionOp, bool) {
a, ok := d.inputCache.Load(dgst.String())
if ok {
return a.([]*DefinitionOp), true
}
return nil, false
}
func (d *DefinitionOp) storeInputCache(dgst digest.Digest, c []*DefinitionOp) {
d.inputCache.Store(dgst.String(), c)
}
func (d *DefinitionOp) Inputs() []Output {
if d.dgst == "" {
return nil
@ -195,7 +207,7 @@ func (d *DefinitionOp) Inputs() []Output {
for _, input := range op.Inputs {
var vtx *DefinitionOp
d.mu.Lock()
if existingIndexes, ok := d.inputCache[input.Digest]; ok {
if existingIndexes, ok := d.loadInputCache(input.Digest); ok {
if int(input.Index) < len(existingIndexes) && existingIndexes[input.Index] != nil {
vtx = existingIndexes[input.Index]
}
@ -211,14 +223,14 @@ func (d *DefinitionOp) Inputs() []Output {
inputCache: d.inputCache,
sources: d.sources,
}
existingIndexes := d.inputCache[input.Digest]
existingIndexes, _ := d.loadInputCache(input.Digest)
indexDiff := int(input.Index) - len(existingIndexes)
if indexDiff >= 0 {
// make room in the slice for the new index being set
existingIndexes = append(existingIndexes, make([]*DefinitionOp, indexDiff+1)...)
}
existingIndexes[input.Index] = vtx
d.inputCache[input.Digest] = existingIndexes
d.storeInputCache(input.Digest, existingIndexes)
}
d.mu.Unlock()

View File

@ -90,6 +90,8 @@ func (m *DiffOp) Inputs() (out []Output) {
return out
}
// Diff returns a state that represents the diff of the lower and upper states.
// The returned State is useful for use with [Merge] where you can merge the lower state with the diff.
func Diff(lower, upper State, opts ...ConstraintsOpt) State {
if lower.Output() == nil {
if upper.Output() == nil {

View File

@ -649,6 +649,7 @@ type SSHInfo struct {
Optional bool
}
// AddSecret is a RunOption that adds a secret to the exec.
func AddSecret(dest string, opts ...SecretOption) RunOption {
return runOptionFunc(func(ei *ExecInfo) {
s := &SecretInfo{ID: dest, Target: dest, Mode: 0400}
@ -696,6 +697,7 @@ func SecretAsEnv(v bool) SecretOption {
})
}
// SecretFileOpt sets the secret's target file uid, gid and permissions.
func SecretFileOpt(uid, gid, mode int) SecretOption {
return secretOptionFunc(func(si *SecretInfo) {
si.UID = uid
@ -704,12 +706,15 @@ func SecretFileOpt(uid, gid, mode int) SecretOption {
})
}
// ReadonlyRootFS sets the execs's root filesystem to be read-only.
func ReadonlyRootFS() RunOption {
return runOptionFunc(func(ei *ExecInfo) {
ei.ReadonlyRootFS = true
})
}
// WithProxy is a RunOption that sets the proxy environment variables in the resulting exec.
// For example `HTTP_PROXY` is a standard environment variable for unix systems that programs may read.
func WithProxy(ps ProxyEnv) RunOption {
return runOptionFunc(func(ei *ExecInfo) {
ei.ProxyEnv = &ps

View File

@ -48,6 +48,7 @@ func NewFileOp(s State, action *FileAction, c Constraints) *FileOp {
}
// CopyInput is either llb.State or *FileActionWithState
// It is used by [Copy] to to specify the source of the copy operation.
type CopyInput interface {
isFileOpCopyInput()
}
@ -131,6 +132,10 @@ type fileActionWithState struct {
func (fas *fileActionWithState) isFileOpCopyInput() {}
// Mkdir creates a FileAction which creates a directory at the given path.
// Example:
//
// llb.Scratch().File(llb.Mkdir("/foo", 0755))
func Mkdir(p string, m os.FileMode, opt ...MkdirOption) *FileAction {
var mi MkdirInfo
for _, o := range opt {
@ -181,6 +186,7 @@ func (fn mkdirOptionFunc) SetMkdirOption(mi *MkdirInfo) {
var _ MkdirOption = &MkdirInfo{}
// WithParents is an option for Mkdir which creates parent directories if they do not exist.
func WithParents(b bool) MkdirOption {
return mkdirOptionFunc(func(mi *MkdirInfo) {
mi.MakeParents = b
@ -282,6 +288,10 @@ func (up *UserOpt) marshal(base pb.InputIndex) *pb.UserOpt {
return &pb.UserOpt{User: &pb.UserOpt_ByID{ByID: uint32(up.UID)}}
}
// Mkfile creates a FileAction which creates a file at the given path with the provided contents.
// Example:
//
// llb.Scratch().File(llb.Mkfile("/foo", 0644, []byte("hello world!")))
func Mkfile(p string, m os.FileMode, dt []byte, opts ...MkfileOption) *FileAction {
var mi MkfileInfo
for _, o := range opts {
@ -332,6 +342,10 @@ func (a *fileActionMkfile) toProtoAction(ctx context.Context, parent string, bas
}, nil
}
// Rm creates a FileAction which removes a file or directory at the given path.
// Example:
//
// llb.Scratch().File(Mkfile("/foo", 0644, []byte("not around for long..."))).File(llb.Rm("/foo"))
func Rm(p string, opts ...RmOption) *FileAction {
var mi RmInfo
for _, o := range opts {
@ -394,6 +408,25 @@ func (a *fileActionRm) toProtoAction(ctx context.Context, parent string, base pb
}, nil
}
// Copy produces a FileAction which copies a file or directory from the source to the destination.
// The "input" parameter is the contents to copy from.
// "src" is the path to copy from within the "input".
// "dest" is the path to copy to within the destination (the state being operated on).
// See [CopyInput] for the valid types of input.
//
// Example:
//
// st := llb.Local(".")
// llb.Scratch().File(llb.Copy(st, "/foo", "/bar"))
//
// The example copies the local (client) directory "./foo" to a new empty directory at /bar.
//
// Note: Copying directories can have different behavior based on if the destination exists or not.
// When the destination already exists, the contents of the source directory is copied underneath the destination, including the directory itself.
// You may need to supply a copy option to copy the dir contents only.
// You may also need to pass in a [CopyOption] which creates parent directories if they do not exist.
//
// See [CopyOption] for more details on what options are available.
func Copy(input CopyInput, src, dest string, opts ...CopyOption) *FileAction {
var state *State
var fas *fileActionWithState

View File

@ -70,6 +70,31 @@ func (m *MergeOp) Inputs() []Output {
return m.inputs
}
// Merge merges multiple states into a single state. This is useful in
// conjunction with [Diff] to create set of patches which are independent of
// each other to a base state without affecting the cache of other merged
// states.
// As an example, lets say you have a rootfs with the following directories:
//
// / /bin /etc /opt /tmp
//
// Now lets say you want to copy a directory /etc/foo from one state and a
// binary /bin/bar from another state.
// [Copy] makes a duplicate of file on top of another directory.
// Merge creates a directory whose contents is an overlay of 2 states on top of each other.
//
// With "Merge" you can do this:
//
// fooState := Diff(rootfs, fooState)
// barState := Diff(rootfs, barState)
//
// Then merge the results with:
//
// Merge(rootfs, fooDiff, barDiff)
//
// The resulting state will have both /etc/foo and /bin/bar, but because Merge
// was used, changing the contents of "fooDiff" does not require copying
// "barDiff" again.
func Merge(inputs []State, opts ...ConstraintsOpt) State {
// filter out any scratch inputs, which have no effect when merged
var filteredInputs []State

View File

@ -91,6 +91,10 @@ func (s *SourceOp) Inputs() []Output {
return nil
}
// Image returns a state that represents a docker image in a registry.
// Example:
//
// st := llb.Image("busybox:latest")
func Image(ref string, opts ...ImageOption) State {
r, err := reference.ParseNormalizedNamed(ref)
if err == nil {
@ -215,6 +219,17 @@ type ImageInfo struct {
RecordType string
}
// Git returns a state that represents a git repository.
// Example:
//
// st := llb.Git("https://github.com/moby/buildkit.git#v0.11.6")
//
// The example fetches the v0.11.6 tag of the buildkit repository.
// You can also use a commit hash or a branch name.
//
// Other URL formats are supported such as "git@github.com:moby/buildkit.git", "git://...", "ssh://..."
// Formats that utilize SSH may need to supply credentials as a [GitOption].
// You may need to check the source code for a full list of supported formats.
func Git(remote, ref string, opts ...GitOption) State {
url := strings.Split(remote, "#")[0]
@ -346,10 +361,12 @@ func MountSSHSock(sshID string) GitOption {
})
}
// Scratch returns a state that represents an empty filesystem.
func Scratch() State {
return NewState(nil)
}
// Local returns a state that represents a directory local to the client.
func Local(name string, opts ...LocalOption) State {
gi := &LocalInfo{}

View File

@ -7,6 +7,16 @@ import (
digest "github.com/opencontainers/go-digest"
)
// SourceMap maps a source file/location to an LLB state/definition.
// SourceMaps are used to provide information for debugging and helpful error messages to the user.
// As an example, lets say you have a Dockerfile with the following content:
//
// FROM alpine
// RUN exit 1
//
// When the "RUN" statement exits with a non-zero exit code buildkit will treat
// it as an error and is able to provide the user with a helpful error message
// pointing to exactly the line in the Dockerfile that caused the error.
type SourceMap struct {
State *State
Definition *Definition

View File

@ -49,6 +49,10 @@ func NewState(o Output) State {
return s
}
// State represents all operations that must be done to produce a given output.
// States are immutable, and all operations return a new state linked to the previous one.
// State is the core type of the LLB API and is used to build a graph of operations.
// The graph is then marshaled into a definition that can be executed by a backend (such as buildkitd).
type State struct {
out Output
prev *State

View File

@ -7,6 +7,7 @@ import (
// Config provides containerd configuration data for the server
type Config struct {
Debug bool `toml:"debug"`
Trace bool `toml:"trace"`
// Root is the path to a directory where buildkit will store persistent data
Root string `toml:"root"`
@ -47,7 +48,7 @@ type TLSConfig struct {
type GCConfig struct {
GC *bool `toml:"gc"`
GCKeepStorage int64 `toml:"gckeepstorage"`
GCKeepStorage DiskSpace `toml:"gckeepstorage"`
GCPolicy []GCPolicy `toml:"gcpolicy"`
}
@ -114,10 +115,10 @@ type ContainerdConfig struct {
}
type GCPolicy struct {
All bool `toml:"all"`
KeepBytes int64 `toml:"keepBytes"`
KeepDuration int64 `toml:"keepDuration"`
Filters []string `toml:"filters"`
All bool `toml:"all"`
KeepBytes DiskSpace `toml:"keepBytes"`
KeepDuration Duration `toml:"keepDuration"`
Filters []string `toml:"filters"`
}
type DNSConfig struct {
@ -127,6 +128,6 @@ type DNSConfig struct {
}
type HistoryConfig struct {
MaxAge int64 `toml:"maxAge"`
MaxEntries int64 `toml:"maxEntries"`
MaxAge Duration `toml:"maxAge"`
MaxEntries int64 `toml:"maxEntries"`
}

View File

@ -1,21 +1,86 @@
package config
import (
"encoding"
"strconv"
"strings"
"time"
"github.com/docker/go-units"
"github.com/pkg/errors"
)
type Duration struct {
time.Duration
}
func (d *Duration) UnmarshalText(textb []byte) error {
text := stripQuotes(string(textb))
if len(text) == 0 {
return nil
}
if duration, err := time.ParseDuration(text); err == nil {
d.Duration = duration
return nil
}
if i, err := strconv.ParseInt(text, 10, 64); err == nil {
d.Duration = time.Duration(i) * time.Second
return nil
}
return errors.Errorf("invalid duration %s", text)
}
var _ encoding.TextUnmarshaler = &Duration{}
type DiskSpace struct {
Bytes int64
Percentage int64
}
var _ encoding.TextUnmarshaler = &DiskSpace{}
func (d *DiskSpace) UnmarshalText(textb []byte) error {
text := stripQuotes(string(textb))
if len(text) == 0 {
return nil
}
if text2 := strings.TrimSuffix(text, "%"); len(text2) < len(text) {
i, err := strconv.ParseInt(text2, 10, 64)
if err != nil {
return err
}
d.Percentage = i
return nil
}
if i, err := units.RAMInBytes(text); err == nil {
d.Bytes = i
return nil
}
return errors.Errorf("invalid disk space %s", text)
}
const defaultCap int64 = 2e9 // 2GB
func DefaultGCPolicy(p string, keep int64) []GCPolicy {
if keep == 0 {
keep = DetectDefaultGCCap(p)
func DefaultGCPolicy(keep DiskSpace) []GCPolicy {
if keep == (DiskSpace{}) {
keep = DetectDefaultGCCap()
}
return []GCPolicy{
// if build cache uses more than 512MB delete the most easily reproducible data after it has not been used for 2 days
{
Filters: []string{"type==source.local,type==exec.cachemount,type==source.git.checkout"},
KeepDuration: 48 * 3600, // 48h
KeepBytes: 512 * 1e6, // 512MB
KeepDuration: Duration{Duration: time.Duration(48) * time.Hour}, // 48h
KeepBytes: DiskSpace{Bytes: 512 * 1e6}, // 512MB
},
// remove any data not used for 60 days
{
KeepDuration: 60 * 24 * 3600, // 60d
KeepDuration: Duration{Duration: time.Duration(60) * 24 * time.Hour}, // 60d
KeepBytes: keep,
},
// keep the unshared build cache under cap
@ -29,3 +94,13 @@ func DefaultGCPolicy(p string, keep int64) []GCPolicy {
},
}
}
func stripQuotes(s string) string {
if len(s) == 0 {
return s
}
if s[0] == '"' && s[len(s)-1] == '"' {
return s[1 : len(s)-1]
}
return s
}

View File

@ -7,12 +7,23 @@ import (
"syscall"
)
func DetectDefaultGCCap(root string) int64 {
func DetectDefaultGCCap() DiskSpace {
return DiskSpace{Percentage: 10}
}
func (d DiskSpace) AsBytes(root string) int64 {
if d.Bytes != 0 {
return d.Bytes
}
if d.Percentage == 0 {
return 0
}
var st syscall.Statfs_t
if err := syscall.Statfs(root, &st); err != nil {
return defaultCap
}
diskSize := int64(st.Bsize) * int64(st.Blocks)
avail := diskSize / 10
avail := diskSize * d.Percentage / 100
return (avail/(1<<30) + 1) * 1e9 // round up
}

View File

@ -3,6 +3,10 @@
package config
func DetectDefaultGCCap(root string) int64 {
return defaultCap
func DetectDefaultGCCap() DiskSpace {
return DiskSpace{Bytes: defaultCap}
}
func (d DiskSpace) AsBytes(root string) int64 {
return d.Bytes
}

View File

@ -33,7 +33,6 @@ type ImageConfig struct {
ocispecs.ImageConfig
Healthcheck *HealthConfig `json:",omitempty"` // Healthcheck describes how to check the container is healthy
ArgsEscaped bool `json:",omitempty"` // True if command is already escaped (Windows specific)
// NetworkDisabled bool `json:",omitempty"` // Is network disabled
// MacAddress string `json:",omitempty"` // Mac Address of the container

View File

@ -129,7 +129,7 @@ func monitorHealth(ctx context.Context, cc *grpc.ClientConn, cancelConn func())
}
}
bklog.G(ctx).WithFields(logFields).Debug("healthcheck completed")
bklog.G(ctx).WithFields(logFields).Trace("healthcheck completed")
}
}
}

View File

@ -24,10 +24,12 @@ func Copy(ctx context.Context, conn io.ReadWriteCloser, stream Stream, closeStre
if err == io.EOF {
// indicates client performed CloseSend, but they may still be
// reading data
if conn, ok := conn.(interface {
if closeWriter, ok := conn.(interface {
CloseWrite() error
}); ok {
conn.CloseWrite()
closeWriter.CloseWrite()
} else {
conn.Close()
}
return nil
}

View File

@ -1,11 +0,0 @@
package attestation
const (
MediaTypeDockerSchema2AttestationType = "application/vnd.in-toto+json"
DockerAnnotationReferenceType = "vnd.docker.reference.type"
DockerAnnotationReferenceDigest = "vnd.docker.reference.digest"
DockerAnnotationReferenceDescription = "vnd.docker.reference.description"
DockerAnnotationReferenceTypeDefault = "attestation-manifest"
)

View File

@ -2,6 +2,7 @@ package bklog
import (
"context"
"runtime/debug"
"github.com/containerd/containerd/log"
"github.com/sirupsen/logrus"
@ -61,3 +62,11 @@ func GetLogger(ctx context.Context) (l *logrus.Entry) {
return l
}
// LazyStackTrace lets you include a stack trace as a field's value in a log but only
// call it when the log level is actually enabled.
type LazyStackTrace struct{}
func (LazyStackTrace) String() string {
return string(debug.Stack())
}

View File

@ -15,6 +15,7 @@ import (
)
func Copy(ctx context.Context, ingester content.Ingester, provider content.Provider, desc ocispecs.Descriptor, ref string, logger func([]byte)) error {
ctx = RegisterContentPayloadTypes(ctx)
if _, err := retryhandler.New(limited.FetchHandler(ingester, &localFetcher{provider}, ref), logger)(ctx, desc); err != nil {
return err
}
@ -60,6 +61,7 @@ func (r *rc) Seek(offset int64, whence int) (int64, error) {
}
func CopyChain(ctx context.Context, ingester content.Ingester, provider content.Provider, desc ocispecs.Descriptor) error {
ctx = RegisterContentPayloadTypes(ctx)
var m sync.Mutex
manifestStack := []ocispecs.Descriptor{}

View File

@ -0,0 +1,15 @@
package contentutil
import (
"context"
"github.com/containerd/containerd/remotes"
intoto "github.com/in-toto/in-toto-golang/in_toto"
)
// RegisterContentPayloadTypes registers content types that are not defined by
// default but that we expect to find in registry images.
func RegisterContentPayloadTypes(ctx context.Context) context.Context {
ctx = remotes.WithMediaTypeKeyPrefix(ctx, intoto.PayloadType, "intoto")
return ctx
}

View File

@ -13,7 +13,7 @@ import (
"github.com/containerd/containerd/reference"
"github.com/containerd/containerd/remotes"
"github.com/containerd/containerd/remotes/docker"
"github.com/moby/buildkit/util/attestation"
intoto "github.com/in-toto/in-toto-golang/in_toto"
"github.com/moby/buildkit/util/contentutil"
"github.com/moby/buildkit/util/leaseutil"
"github.com/moby/buildkit/util/resolver/limited"
@ -174,7 +174,7 @@ func childrenConfigHandler(provider content.Provider, platform platforms.MatchCo
descs = append(descs, index.Manifests...)
}
case images.MediaTypeDockerSchema2Config, ocispecs.MediaTypeImageConfig, docker.LegacyConfigMediaType,
attestation.MediaTypeDockerSchema2AttestationType:
intoto.PayloadType:
// childless data types.
return nil, nil
default:

View File

@ -582,7 +582,7 @@ func (t *trace) update(s *client.SolveStatus, termWidth int) {
} else if sec < 100 {
prec = 2
}
v.logs = append(v.logs, []byte(fmt.Sprintf("#%d %s %s", v.index, fmt.Sprintf("%.[2]*[1]f", sec, prec), dt)))
v.logs = append(v.logs, []byte(fmt.Sprintf("%s %s", fmt.Sprintf("%.[2]*[1]f", sec, prec), dt)))
}
i++
})

View File

@ -146,7 +146,7 @@ func (p *textMux) printVtx(t *trace, dgst digest.Digest) {
if i == 0 {
l = l[v.logsOffset:]
}
fmt.Fprintf(p.w, "%s", []byte(l))
fmt.Fprintf(p.w, "#%d %s", v.index, []byte(l))
if i != len(v.logs)-1 || !v.logsPartial {
fmt.Fprintln(p.w, "")
}

View File

@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.28.1
// protoc-gen-go v1.30.0
// protoc v3.11.4
// source: stack.proto

49
vendor/github.com/moby/buildkit/version/ua.go generated vendored Normal file
View File

@ -0,0 +1,49 @@
package version
import (
"fmt"
"regexp"
"strings"
"sync"
)
var (
reRelease *regexp.Regexp
reDev *regexp.Regexp
reOnce sync.Once
uapCbs map[string]func() string
)
func UserAgent() string {
uaVersion := defaultVersion
reOnce.Do(func() {
reRelease = regexp.MustCompile(`^(v[0-9]+\.[0-9]+)\.[0-9]+$`)
reDev = regexp.MustCompile(`^(v[0-9]+\.[0-9]+)\.[0-9]+`)
})
if matches := reRelease.FindAllStringSubmatch(Version, 1); len(matches) > 0 {
uaVersion = matches[0][1]
} else if matches := reDev.FindAllStringSubmatch(Version, 1); len(matches) > 0 {
uaVersion = matches[0][1] + "-dev"
}
res := &strings.Builder{}
fmt.Fprintf(res, "buildkit/%s", uaVersion)
for pname, pver := range uapCbs {
fmt.Fprintf(res, " %s/%s", pname, pver())
}
return res.String()
}
// SetUserAgentProduct sets a callback to get the version of a product to be
// included in the User-Agent header. The callback is called every time the
// User-Agent header is generated. Caller must ensure that the callback is
// cached if it is expensive to compute.
func SetUserAgentProduct(name string, cb func() (version string)) {
if uapCbs == nil {
uapCbs = make(map[string]func() string)
}
uapCbs[name] = cb
}

View File

@ -17,11 +17,6 @@
package version
import (
"regexp"
"sync"
)
const (
defaultVersion = "v0.0.0+unknown"
)
@ -37,26 +32,3 @@ var (
// the program at linking time.
Revision = ""
)
var (
reRelease *regexp.Regexp
reDev *regexp.Regexp
reOnce sync.Once
)
func UserAgent() string {
uaVersion := defaultVersion
reOnce.Do(func() {
reRelease = regexp.MustCompile(`^(v[0-9]+\.[0-9]+)\.[0-9]+$`)
reDev = regexp.MustCompile(`^(v[0-9]+\.[0-9]+)\.[0-9]+`)
})
if matches := reRelease.FindAllStringSubmatch(Version, 1); len(matches) > 0 {
uaVersion = matches[0][1]
} else if matches := reDev.FindAllStringSubmatch(Version, 1); len(matches) > 0 {
uaVersion = matches[0][1] + "-dev"
}
return "buildkit/" + uaVersion
}