Sebastiaan van Stijn
2024-06-04 11:33:43 +02:00
parent dbdd3601eb
commit 9358f84668
146 changed files with 2661 additions and 1102 deletions

View File

@ -2,6 +2,7 @@ package client
import (
"context"
"maps"
"github.com/moby/buildkit/client/buildid"
gateway "github.com/moby/buildkit/frontend/gateway/client"
@ -42,12 +43,10 @@ func (c *Client) Build(ctx context.Context, opt SolveOpt, product string, buildF
}
cb := func(ref string, s *session.Session, opts map[string]string) error {
for k, v := range opts {
if feOpts == nil {
feOpts = map[string]string{}
}
feOpts[k] = v
if feOpts == nil {
feOpts = map[string]string{}
}
maps.Copy(feOpts, opts)
gwClient := c.gatewayClientForBuild(ref)
g, err := grpcclient.New(ctx, feOpts, s.ID(), product, gwClient, gworkers)
if err != nil {

View File

@ -6,16 +6,12 @@ import (
"github.com/moby/buildkit/solver/pb"
"github.com/moby/buildkit/util/flightcontrol"
digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors"
)
type asyncState struct {
f func(context.Context, State, *Constraints) (State, error)
prev State
target State
set bool
err error
g flightcontrol.Group[State]
f func(context.Context, State, *Constraints) (State, error)
prev State
g flightcontrol.CachedGroup[State]
}
func (as *asyncState) Output() Output {
@ -23,59 +19,33 @@ func (as *asyncState) Output() Output {
}
func (as *asyncState) Vertex(ctx context.Context, c *Constraints) Vertex {
err := as.Do(ctx, c)
target, err := as.Do(ctx, c)
if err != nil {
return &errVertex{err}
}
if as.set {
out := as.target.Output()
if out == nil {
return nil
}
return out.Vertex(ctx, c)
out := target.Output()
if out == nil {
return nil
}
return nil
return out.Vertex(ctx, c)
}
func (as *asyncState) ToInput(ctx context.Context, c *Constraints) (*pb.Input, error) {
err := as.Do(ctx, c)
target, err := as.Do(ctx, c)
if err != nil {
return nil, err
}
if as.set {
out := as.target.Output()
if out == nil {
return nil, nil
}
return out.ToInput(ctx, c)
out := target.Output()
if out == nil {
return nil, nil
}
return nil, nil
return out.ToInput(ctx, c)
}
func (as *asyncState) Do(ctx context.Context, c *Constraints) error {
_, err := as.g.Do(ctx, "", func(ctx context.Context) (State, error) {
if as.set {
return as.target, as.err
}
res, err := as.f(ctx, as.prev, c)
if err != nil {
select {
case <-ctx.Done():
if errors.Is(err, context.Cause(ctx)) {
return res, err
}
default:
}
}
as.target = res
as.err = err
as.set = true
return res, err
func (as *asyncState) Do(ctx context.Context, c *Constraints) (State, error) {
return as.g.Do(ctx, "", func(ctx context.Context) (State, error) {
return as.f(ctx, as.prev, c)
})
if err != nil {
return err
}
return as.err
}
type errVertex struct {

View File

@ -2,6 +2,7 @@ package llb
import (
"io"
"maps"
"github.com/containerd/containerd/platforms"
"github.com/moby/buildkit/solver/pb"
@ -18,24 +19,17 @@ type Definition struct {
}
func (def *Definition) ToPB() *pb.Definition {
md := make(map[digest.Digest]pb.OpMetadata, len(def.Metadata))
for k, v := range def.Metadata {
md[k] = v
}
return &pb.Definition{
Def: def.Def,
Source: def.Source,
Metadata: md,
Metadata: maps.Clone(def.Metadata),
}
}
func (def *Definition) FromPB(x *pb.Definition) {
def.Def = x.Def
def.Source = x.Source
def.Metadata = make(map[digest.Digest]pb.OpMetadata)
for k, v := range x.Metadata {
def.Metadata[k] = v
}
def.Metadata = maps.Clone(x.Metadata)
}
func (def *Definition) Head() (digest.Digest, error) {

View File

@ -5,6 +5,8 @@ import (
"fmt"
"net"
"path"
"slices"
"sync"
"github.com/containerd/containerd/platforms"
"github.com/google/shlex"
@ -111,16 +113,16 @@ func Reset(other State) StateOption {
}
}
func getEnv(s State) func(context.Context, *Constraints) (EnvList, error) {
return func(ctx context.Context, c *Constraints) (EnvList, error) {
func getEnv(s State) func(context.Context, *Constraints) (*EnvList, error) {
return func(ctx context.Context, c *Constraints) (*EnvList, error) {
v, err := s.getValue(keyEnv)(ctx, c)
if err != nil {
return nil, err
}
if v != nil {
return v.(EnvList), nil
return v.(*EnvList), nil
}
return EnvList{}, nil
return &EnvList{}, nil
}
}
@ -346,54 +348,83 @@ func getSecurity(s State) func(context.Context, *Constraints) (pb.SecurityMode,
}
}
type EnvList []KeyValue
type KeyValue struct {
key string
value string
type EnvList struct {
parent *EnvList
key string
value string
del bool
once sync.Once
l int
values map[string]string
keys []string
}
func (e EnvList) AddOrReplace(k, v string) EnvList {
e = e.Delete(k)
e = append(e, KeyValue{key: k, value: v})
return e
func (e *EnvList) AddOrReplace(k, v string) *EnvList {
return &EnvList{
parent: e,
key: k,
value: v,
l: e.l + 1,
}
}
func (e EnvList) SetDefault(k, v string) EnvList {
func (e *EnvList) SetDefault(k, v string) *EnvList {
if _, ok := e.Get(k); !ok {
e = append(e, KeyValue{key: k, value: v})
return e.AddOrReplace(k, v)
}
return e
}
func (e EnvList) Delete(k string) EnvList {
e = append([]KeyValue(nil), e...)
if i, ok := e.Index(k); ok {
return append(e[:i], e[i+1:]...)
func (e *EnvList) Delete(k string) EnvList {
return EnvList{
parent: e,
key: k,
del: true,
l: e.l + 1,
}
return e
}
func (e EnvList) Get(k string) (string, bool) {
if index, ok := e.Index(k); ok {
return e[index].value, true
}
return "", false
func (e *EnvList) makeValues() {
m := make(map[string]string, e.l)
seen := make(map[string]struct{}, e.l)
keys := make([]string, 0, e.l)
e.keys = e.addValue(keys, m, seen)
e.values = m
slices.Reverse(e.keys)
}
func (e EnvList) Index(k string) (int, bool) {
for i, kv := range e {
if kv.key == k {
return i, true
}
func (e *EnvList) addValue(keys []string, vals map[string]string, seen map[string]struct{}) []string {
if e.parent == nil {
return keys
}
return -1, false
if _, ok := seen[e.key]; !e.del && !ok {
vals[e.key] = e.value
keys = append(keys, e.key)
}
seen[e.key] = struct{}{}
if e.parent != nil {
keys = e.parent.addValue(keys, vals, seen)
}
return keys
}
func (e EnvList) ToArray() []string {
out := make([]string, 0, len(e))
for _, kv := range e {
out = append(out, kv.key+"="+kv.value)
func (e *EnvList) Get(k string) (string, bool) {
e.once.Do(e.makeValues)
v, ok := e.values[k]
return v, ok
}
func (e *EnvList) Keys() []string {
e.once.Do(e.makeValues)
return e.keys
}
func (e *EnvList) ToArray() []string {
keys := e.Keys()
out := make([]string, 0, len(keys))
for _, k := range keys {
v, _ := e.Get(k)
out = append(out, k+"="+v)
}
return out
}

View File

@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"maps"
"net"
"strings"
@ -104,11 +105,11 @@ func (s State) getValue(k interface{}) func(context.Context, *Constraints) (inte
}
if s.async != nil {
return func(ctx context.Context, c *Constraints) (interface{}, error) {
err := s.async.Do(ctx, c)
target, err := s.async.Do(ctx, c)
if err != nil {
return nil, err
}
return s.async.target.getValue(k)(ctx, c)
return target.getValue(k)(ctx, c)
}
}
if s.prev == nil {
@ -118,8 +119,13 @@ func (s State) getValue(k interface{}) func(context.Context, *Constraints) (inte
}
func (s State) Async(f func(context.Context, State, *Constraints) (State, error)) State {
as := &asyncState{
f: f,
prev: s,
}
as.g.CacheError = true
s2 := State{
async: &asyncState{f: f, prev: s},
async: as,
}
return s2
}
@ -345,16 +351,12 @@ func (s State) GetEnv(ctx context.Context, key string, co ...ConstraintsOpt) (st
// Env returns a new [State] with the provided environment variable set.
// See [Env]
func (s State) Env(ctx context.Context, co ...ConstraintsOpt) ([]string, error) {
func (s State) Env(ctx context.Context, co ...ConstraintsOpt) (*EnvList, error) {
c := &Constraints{}
for _, f := range co {
f.SetConstraintsOption(c)
}
env, err := getEnv(s)(ctx, c)
if err != nil {
return nil, err
}
return env.ToArray(), nil
return getEnv(s)(ctx, c)
}
// GetDir returns the current working directory for the state.
@ -566,9 +568,7 @@ func mergeMetadata(m1, m2 pb.OpMetadata) pb.OpMetadata {
if m1.Description == nil {
m1.Description = make(map[string]string)
}
for k, v := range m2.Description {
m1.Description[k] = v
}
maps.Copy(m1.Description, m2.Description)
}
if m2.ExportCache != nil {
m1.ExportCache = m2.ExportCache
@ -597,9 +597,7 @@ func WithDescription(m map[string]string) ConstraintsOpt {
if c.Metadata.Description == nil {
c.Metadata.Description = map[string]string{}
}
for k, v := range m {
c.Metadata.Description[k] = v
}
maps.Copy(c.Metadata.Description, m)
})
}

View File

@ -5,6 +5,7 @@ import (
"encoding/base64"
"encoding/json"
"io"
"maps"
"os"
"path/filepath"
"strings"
@ -219,13 +220,8 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
})
}
frontendAttrs := map[string]string{}
for k, v := range opt.FrontendAttrs {
frontendAttrs[k] = v
}
for k, v := range cacheOpt.frontendAttrs {
frontendAttrs[k] = v
}
frontendAttrs := maps.Clone(opt.FrontendAttrs)
maps.Copy(frontendAttrs, cacheOpt.frontendAttrs)
solveCtx, cancelSolve := context.WithCancelCause(ctx)
var res *SolveResponse