bake: initial set of composable bake attributes

This allows using either the csv syntax or object syntax to specify
certain attributes.

This applies to the following fields:
- output
- cache-from
- cache-to
- secret
- ssh

There are still some remaining fields to translate. Specifically
ulimits, annotations, and attest.

Signed-off-by: Jonathan A. Sternberg <jonathan.sternberg@docker.com>
This commit is contained in:
Jonathan A. Sternberg
2024-11-21 12:06:14 -06:00
parent a34c641bc4
commit 3ccbb88e6a
24 changed files with 3661 additions and 373 deletions

View File

@ -2,6 +2,8 @@ package buildflags
import (
"context"
"encoding/json"
"maps"
"os"
"strings"
@ -9,66 +11,154 @@ import (
controllerapi "github.com/docker/buildx/controller/pb"
"github.com/pkg/errors"
"github.com/tonistiigi/go-csvvalue"
"github.com/zclconf/go-cty/cty"
jsoncty "github.com/zclconf/go-cty/cty/json"
)
type CacheOptionsEntry struct {
Type string `json:"type"`
Attrs map[string]string `json:"attrs,omitempty"`
}
func (e *CacheOptionsEntry) Equal(other *CacheOptionsEntry) bool {
if e.Type != other.Type {
return false
}
return maps.Equal(e.Attrs, other.Attrs)
}
func (e *CacheOptionsEntry) String() string {
// Special registry syntax.
if e.Type == "registry" && len(e.Attrs) == 1 {
if ref, ok := e.Attrs["ref"]; ok {
return ref
}
}
var b csvBuilder
if e.Type != "" {
b.Write("type", e.Type)
}
if len(e.Attrs) > 0 {
b.WriteAttributes(e.Attrs)
}
return b.String()
}
func (e *CacheOptionsEntry) ToPB() *controllerapi.CacheOptionsEntry {
return &controllerapi.CacheOptionsEntry{
Type: e.Type,
Attrs: maps.Clone(e.Attrs),
}
}
func (e *CacheOptionsEntry) MarshalJSON() ([]byte, error) {
m := maps.Clone(e.Attrs)
if m == nil {
m = map[string]string{}
}
m["type"] = e.Type
return json.Marshal(m)
}
func (e *CacheOptionsEntry) UnmarshalJSON(data []byte) error {
var m map[string]string
if err := json.Unmarshal(data, &m); err != nil {
return err
}
e.Type = m["type"]
delete(m, "type")
e.Attrs = m
return e.validate(data)
}
func (e *CacheOptionsEntry) IsActive() bool {
// Always active if not gha.
if e.Type != "gha" {
return true
}
return e.Attrs["token"] != "" && e.Attrs["url"] != ""
}
func (e *CacheOptionsEntry) UnmarshalText(text []byte) error {
in := string(text)
fields, err := csvvalue.Fields(in, nil)
if err != nil {
return err
}
if len(fields) == 1 && !strings.Contains(fields[0], "=") {
e.Type = "registry"
e.Attrs = map[string]string{"ref": fields[0]}
return nil
}
e.Type = ""
e.Attrs = map[string]string{}
for _, field := range fields {
parts := strings.SplitN(field, "=", 2)
if len(parts) != 2 {
return errors.Errorf("invalid value %s", field)
}
key := strings.ToLower(parts[0])
value := parts[1]
switch key {
case "type":
e.Type = value
default:
e.Attrs[key] = value
}
}
if e.Type == "" {
return errors.Errorf("type required form> %q", in)
}
addGithubToken(e)
addAwsCredentials(e)
return e.validate(text)
}
func (e *CacheOptionsEntry) validate(gv interface{}) error {
if e.Type == "" {
var text []byte
switch gv := gv.(type) {
case []byte:
text = gv
case string:
text = []byte(gv)
case cty.Value:
text, _ = jsoncty.Marshal(gv, gv.Type())
default:
text, _ = json.Marshal(gv)
}
return errors.Errorf("type required form> %q", string(text))
}
return nil
}
func ParseCacheEntry(in []string) ([]*controllerapi.CacheOptionsEntry, error) {
outs := make([]*controllerapi.CacheOptionsEntry, 0, len(in))
for _, in := range in {
fields, err := csvvalue.Fields(in, nil)
if err != nil {
var out CacheOptionsEntry
if err := out.UnmarshalText([]byte(in)); err != nil {
return nil, err
}
if isRefOnlyFormat(fields) {
for _, field := range fields {
outs = append(outs, &controllerapi.CacheOptionsEntry{
Type: "registry",
Attrs: map[string]string{"ref": field},
})
}
continue
}
out := controllerapi.CacheOptionsEntry{
Attrs: map[string]string{},
}
for _, field := range fields {
parts := strings.SplitN(field, "=", 2)
if len(parts) != 2 {
return nil, errors.Errorf("invalid value %s", field)
}
key := strings.ToLower(parts[0])
value := parts[1]
switch key {
case "type":
out.Type = value
default:
out.Attrs[key] = value
}
}
if out.Type == "" {
return nil, errors.Errorf("type required form> %q", in)
}
if !addGithubToken(&out) {
if !out.IsActive() {
// Skip inactive cache entries.
continue
}
addAwsCredentials(&out)
outs = append(outs, &out)
outs = append(outs, out.ToPB())
}
return outs, nil
}
func isRefOnlyFormat(in []string) bool {
for _, v := range in {
if strings.Contains(v, "=") {
return false
}
}
return true
}
func addGithubToken(ci *controllerapi.CacheOptionsEntry) bool {
func addGithubToken(ci *CacheOptionsEntry) {
if ci.Type != "gha" {
return true
return
}
if _, ok := ci.Attrs["token"]; !ok {
if v, ok := os.LookupEnv("ACTIONS_RUNTIME_TOKEN"); ok {
@ -80,10 +170,9 @@ func addGithubToken(ci *controllerapi.CacheOptionsEntry) bool {
ci.Attrs["url"] = v
}
}
return ci.Attrs["token"] != "" && ci.Attrs["url"] != ""
}
func addAwsCredentials(ci *controllerapi.CacheOptionsEntry) {
func addAwsCredentials(ci *CacheOptionsEntry) {
if ci.Type != "s3" {
return
}

183
util/buildflags/cty.go Normal file
View File

@ -0,0 +1,183 @@
package buildflags
import (
"encoding"
"sync"
"github.com/zclconf/go-cty/cty"
"github.com/zclconf/go-cty/cty/convert"
"github.com/zclconf/go-cty/cty/gocty"
)
func (e *CacheOptionsEntry) FromCtyValue(in cty.Value, p cty.Path) error {
conv, err := convert.Convert(in, cty.Map(cty.String))
if err == nil {
m := conv.AsValueMap()
if err := getAndDelete(m, "type", &e.Type); err != nil {
return err
}
e.Attrs = asMap(m)
return e.validate(in)
}
return unmarshalTextFallback(in, e, err)
}
func (e *CacheOptionsEntry) ToCtyValue() cty.Value {
if e == nil {
return cty.NullVal(cty.Map(cty.String))
}
vals := make(map[string]cty.Value, len(e.Attrs)+1)
for k, v := range e.Attrs {
vals[k] = cty.StringVal(v)
}
vals["type"] = cty.StringVal(e.Type)
return cty.MapVal(vals)
}
func (e *ExportEntry) FromCtyValue(in cty.Value, p cty.Path) error {
conv, err := convert.Convert(in, cty.Map(cty.String))
if err == nil {
m := conv.AsValueMap()
if err := getAndDelete(m, "type", &e.Type); err != nil {
return err
}
if err := getAndDelete(m, "dest", &e.Destination); err != nil {
return err
}
e.Attrs = asMap(m)
return e.validate()
}
return unmarshalTextFallback(in, e, err)
}
func (e *ExportEntry) ToCtyValue() cty.Value {
if e == nil {
return cty.NullVal(cty.Map(cty.String))
}
vals := make(map[string]cty.Value, len(e.Attrs)+2)
for k, v := range e.Attrs {
vals[k] = cty.StringVal(v)
}
vals["type"] = cty.StringVal(e.Type)
vals["dest"] = cty.StringVal(e.Destination)
return cty.MapVal(vals)
}
var secretType = sync.OnceValue(func() cty.Type {
return cty.ObjectWithOptionalAttrs(
map[string]cty.Type{
"id": cty.String,
"src": cty.String,
"env": cty.String,
},
[]string{"id", "src", "env"},
)
})
func (e *Secret) FromCtyValue(in cty.Value, p cty.Path) (err error) {
conv, err := convert.Convert(in, secretType())
if err == nil {
if id := conv.GetAttr("id"); !id.IsNull() {
e.ID = id.AsString()
}
if src := conv.GetAttr("src"); !src.IsNull() {
e.FilePath = src.AsString()
}
if env := conv.GetAttr("env"); !env.IsNull() {
e.Env = env.AsString()
}
return nil
}
return unmarshalTextFallback(in, e, err)
}
func (e *Secret) ToCtyValue() cty.Value {
if e == nil {
return cty.NullVal(secretType())
}
return cty.ObjectVal(map[string]cty.Value{
"id": cty.StringVal(e.ID),
"src": cty.StringVal(e.FilePath),
"env": cty.StringVal(e.Env),
})
}
var sshType = sync.OnceValue(func() cty.Type {
return cty.ObjectWithOptionalAttrs(
map[string]cty.Type{
"id": cty.String,
"paths": cty.List(cty.String),
},
[]string{"id", "paths"},
)
})
func (e *SSH) FromCtyValue(in cty.Value, p cty.Path) (err error) {
conv, err := convert.Convert(in, sshType())
if err == nil {
if id := conv.GetAttr("id"); !id.IsNull() {
e.ID = id.AsString()
}
if paths := conv.GetAttr("paths"); !paths.IsNull() {
if err := gocty.FromCtyValue(paths, &e.Paths); err != nil {
return err
}
}
return nil
}
return unmarshalTextFallback(in, e, err)
}
func (e *SSH) ToCtyValue() cty.Value {
if e == nil {
return cty.NullVal(sshType())
}
var ctyPaths cty.Value
if len(e.Paths) > 0 {
paths := make([]cty.Value, len(e.Paths))
for i, path := range e.Paths {
paths[i] = cty.StringVal(path)
}
ctyPaths = cty.ListVal(paths)
} else {
ctyPaths = cty.ListValEmpty(cty.String)
}
return cty.ObjectVal(map[string]cty.Value{
"id": cty.StringVal(e.ID),
"paths": ctyPaths,
})
}
func getAndDelete(m map[string]cty.Value, attr string, gv interface{}) error {
if v, ok := m[attr]; ok {
delete(m, attr)
return gocty.FromCtyValue(v, gv)
}
return nil
}
func asMap(m map[string]cty.Value) map[string]string {
out := make(map[string]string, len(m))
for k, v := range m {
out[k] = v.AsString()
}
return out
}
func unmarshalTextFallback[V encoding.TextUnmarshaler](in cty.Value, v V, inErr error) (outErr error) {
// Attempt to convert this type to a string.
conv, err := convert.Convert(in, cty.String)
if err != nil {
// Cannot convert. Do not attempt to convert and return the original error.
return inErr
}
// Conversion was successful. Use UnmarshalText on the string and return any
// errors associated with that.
return v.UnmarshalText([]byte(conv.AsString()))
}

View File

@ -1,7 +1,10 @@
package buildflags
import (
"encoding/json"
"maps"
"regexp"
"sort"
"strings"
"github.com/containerd/platforms"
@ -13,67 +16,131 @@ import (
"github.com/tonistiigi/go-csvvalue"
)
type ExportEntry struct {
Type string `json:"type"`
Attrs map[string]string `json:"attrs,omitempty"`
Destination string `json:"dest,omitempty"`
}
func (e *ExportEntry) Equal(other *ExportEntry) bool {
if e.Type != other.Type || e.Destination != other.Destination {
return false
}
return maps.Equal(e.Attrs, other.Attrs)
}
func (e *ExportEntry) String() string {
var b csvBuilder
if e.Type != "" {
b.Write("type", e.Type)
}
if e.Destination != "" {
b.Write("dest", e.Destination)
}
if len(e.Attrs) > 0 {
b.WriteAttributes(e.Attrs)
}
return b.String()
}
func (e *ExportEntry) ToPB() *controllerapi.ExportEntry {
return &controllerapi.ExportEntry{
Type: e.Type,
Attrs: maps.Clone(e.Attrs),
Destination: e.Destination,
}
}
func (e *ExportEntry) MarshalJSON() ([]byte, error) {
m := maps.Clone(e.Attrs)
if m == nil {
m = map[string]string{}
}
m["type"] = e.Type
if e.Destination != "" {
m["dest"] = e.Destination
}
return json.Marshal(m)
}
func (e *ExportEntry) UnmarshalJSON(data []byte) error {
var m map[string]string
if err := json.Unmarshal(data, &m); err != nil {
return err
}
e.Type = m["type"]
delete(m, "type")
e.Destination = m["dest"]
delete(m, "dest")
e.Attrs = m
return e.validate()
}
func (e *ExportEntry) UnmarshalText(text []byte) error {
s := string(text)
fields, err := csvvalue.Fields(s, nil)
if err != nil {
return err
}
// Clear the target entry.
e.Type = ""
e.Attrs = map[string]string{}
e.Destination = ""
if len(fields) == 1 && fields[0] == s && !strings.HasPrefix(s, "type=") {
if s != "-" {
e.Type = client.ExporterLocal
e.Destination = s
return nil
}
e.Type = client.ExporterTar
e.Destination = s
}
if e.Type == "" {
for _, field := range fields {
parts := strings.SplitN(field, "=", 2)
if len(parts) != 2 {
return errors.Errorf("invalid value %s", field)
}
key := strings.TrimSpace(strings.ToLower(parts[0]))
value := parts[1]
switch key {
case "type":
e.Type = value
case "dest":
e.Destination = value
default:
e.Attrs[key] = value
}
}
}
return e.validate()
}
func (e *ExportEntry) validate() error {
if e.Type == "" {
return errors.Errorf("type is required for output")
}
return nil
}
func ParseExports(inp []string) ([]*controllerapi.ExportEntry, error) {
var outs []*controllerapi.ExportEntry
if len(inp) == 0 {
return nil, nil
}
for _, s := range inp {
fields, err := csvvalue.Fields(s, nil)
if err != nil {
var out ExportEntry
if err := out.UnmarshalText([]byte(s)); err != nil {
return nil, err
}
out := controllerapi.ExportEntry{
Attrs: map[string]string{},
}
if len(fields) == 1 && fields[0] == s && !strings.HasPrefix(s, "type=") {
if s != "-" {
outs = append(outs, &controllerapi.ExportEntry{
Type: client.ExporterLocal,
Destination: s,
})
continue
}
out = controllerapi.ExportEntry{
Type: client.ExporterTar,
Destination: s,
}
}
if out.Type == "" {
for _, field := range fields {
parts := strings.SplitN(field, "=", 2)
if len(parts) != 2 {
return nil, errors.Errorf("invalid value %s", field)
}
key := strings.TrimSpace(strings.ToLower(parts[0]))
value := parts[1]
switch key {
case "type":
out.Type = value
default:
out.Attrs[key] = value
}
}
}
if out.Type == "" {
return nil, errors.Errorf("type is required for output")
}
if out.Type == "registry" {
out.Type = client.ExporterImage
if _, ok := out.Attrs["push"]; !ok {
out.Attrs["push"] = "true"
}
}
if dest, ok := out.Attrs["dest"]; ok {
out.Destination = dest
delete(out.Attrs, "dest")
}
outs = append(outs, &out)
outs = append(outs, out.ToPB())
}
return outs, nil
}
@ -141,3 +208,32 @@ func ParseAnnotations(inp []string) (map[exptypes.AnnotationKey]string, error) {
}
return annotations, nil
}
type csvBuilder struct {
sb strings.Builder
}
func (w *csvBuilder) Write(key, value string) {
if w.sb.Len() > 0 {
w.sb.WriteByte(',')
}
w.sb.WriteString(key)
w.sb.WriteByte('=')
w.sb.WriteString(value)
}
func (w *csvBuilder) WriteAttributes(attrs map[string]string) {
keys := make([]string, 0, len(attrs))
for key := range attrs {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
w.Write(key, attrs[key])
}
}
func (w *csvBuilder) String() string {
return w.sb.String()
}

View File

@ -8,6 +8,80 @@ import (
"github.com/tonistiigi/go-csvvalue"
)
type Secret struct {
ID string `json:"id,omitempty"`
FilePath string `json:"src,omitempty"`
Env string `json:"env,omitempty"`
}
func (s *Secret) Equal(other *Secret) bool {
return s.ID == other.ID && s.FilePath == other.FilePath && s.Env == other.Env
}
func (s *Secret) String() string {
var b csvBuilder
if s.ID != "" {
b.Write("id", s.ID)
}
if s.FilePath != "" {
b.Write("src", s.FilePath)
}
if s.Env != "" {
b.Write("env", s.Env)
}
return b.String()
}
func (s *Secret) ToPB() *controllerapi.Secret {
return &controllerapi.Secret{
ID: s.ID,
FilePath: s.FilePath,
Env: s.Env,
}
}
func (s *Secret) UnmarshalText(text []byte) error {
value := string(text)
fields, err := csvvalue.Fields(value, nil)
if err != nil {
return errors.Wrap(err, "failed to parse csv secret")
}
*s = Secret{}
var typ string
for _, field := range fields {
parts := strings.SplitN(field, "=", 2)
key := strings.ToLower(parts[0])
if len(parts) != 2 {
return errors.Errorf("invalid field '%s' must be a key=value pair", field)
}
value := parts[1]
switch key {
case "type":
if value != "file" && value != "env" {
return errors.Errorf("unsupported secret type %q", value)
}
typ = value
case "id":
s.ID = value
case "source", "src":
s.FilePath = value
case "env":
s.Env = value
default:
return errors.Errorf("unexpected key '%s' in '%s'", key, field)
}
}
if typ == "env" && s.Env == "" {
s.Env = s.FilePath
s.FilePath = ""
}
return nil
}
func ParseSecretSpecs(sl []string) ([]*controllerapi.Secret, error) {
fs := make([]*controllerapi.Secret, 0, len(sl))
for _, v := range sl {
@ -21,42 +95,9 @@ func ParseSecretSpecs(sl []string) ([]*controllerapi.Secret, error) {
}
func parseSecret(value string) (*controllerapi.Secret, error) {
fields, err := csvvalue.Fields(value, nil)
if err != nil {
return nil, errors.Wrap(err, "failed to parse csv secret")
var s Secret
if err := s.UnmarshalText([]byte(value)); err != nil {
return nil, err
}
fs := controllerapi.Secret{}
var typ string
for _, field := range fields {
parts := strings.SplitN(field, "=", 2)
key := strings.ToLower(parts[0])
if len(parts) != 2 {
return nil, errors.Errorf("invalid field '%s' must be a key=value pair", field)
}
value := parts[1]
switch key {
case "type":
if value != "file" && value != "env" {
return nil, errors.Errorf("unsupported secret type %q", value)
}
typ = value
case "id":
fs.ID = value
case "source", "src":
fs.FilePath = value
case "env":
fs.Env = value
default:
return nil, errors.Errorf("unexpected key '%s' in '%s'", key, field)
}
}
if typ == "env" && fs.Env == "" {
fs.Env = fs.FilePath
fs.FilePath = ""
}
return &fs, nil
return s.ToPB(), nil
}

View File

@ -1,12 +1,60 @@
package buildflags
import (
"cmp"
"slices"
"strings"
controllerapi "github.com/docker/buildx/controller/pb"
"github.com/moby/buildkit/util/gitutil"
)
type SSH struct {
ID string `json:"id,omitempty" cty:"id"`
Paths []string `json:"paths,omitempty" cty:"paths"`
}
func (s *SSH) Equal(other *SSH) bool {
return s.Less(other) == 0
}
func (s *SSH) Less(other *SSH) int {
if s.ID != other.ID {
return cmp.Compare(s.ID, other.ID)
}
return slices.Compare(s.Paths, other.Paths)
}
func (s *SSH) String() string {
if len(s.Paths) == 0 {
return s.ID
}
var b csvBuilder
paths := strings.Join(s.Paths, ",")
b.Write(s.ID, paths)
return b.String()
}
func (s *SSH) ToPB() *controllerapi.SSH {
return &controllerapi.SSH{
ID: s.ID,
Paths: s.Paths,
}
}
func (s *SSH) UnmarshalText(text []byte) error {
parts := strings.SplitN(string(text), "=", 2)
s.ID = parts[0]
if len(parts) > 1 {
s.Paths = strings.Split(parts[1], ",")
} else {
s.Paths = nil
}
return nil
}
func ParseSSHSpecs(sl []string) ([]*controllerapi.SSH, error) {
var outs []*controllerapi.SSH
if len(sl) == 0 {
@ -14,14 +62,11 @@ func ParseSSHSpecs(sl []string) ([]*controllerapi.SSH, error) {
}
for _, s := range sl {
parts := strings.SplitN(s, "=", 2)
out := controllerapi.SSH{
ID: parts[0],
var out SSH
if err := out.UnmarshalText([]byte(s)); err != nil {
return nil, err
}
if len(parts) > 1 {
out.Paths = strings.Split(parts[1], ",")
}
outs = append(outs, &out)
outs = append(outs, out.ToPB())
}
return outs, nil
}