Merge pull request #1069 from crazy-max/compose-build-secrets

bake: support compose build secrets

bake/compose.go

@@ -74,6 +74,16 @@ func ParseCompose(dt []byte) (*Config, error) {
 				dockerfilePath := s.Build.Dockerfile
 				dockerfilePathP = &dockerfilePath
 			}
+
+			var secrets []string
+			for _, bs := range s.Build.Secrets {
+				secret, err := composeToBuildkitSecret(bs, cfg.Secrets[bs.Source])
+				if err != nil {
+					return nil, err
+				}
+				secrets = append(secrets, secret)
+			}
+
 			g.Targets = append(g.Targets, s.Name)
 			t := &Target{
 				Name:       s.Name,
@@ -89,6 +99,7 @@ func ParseCompose(dt []byte) (*Config, error) {
 				})),
 				CacheFrom:   s.Build.CacheFrom,
 				NetworkMode: &s.Build.Network,
+				Secrets:     secrets,
 			}
 			if err = t.composeExtTarget(s.Build.Extensions); err != nil {
 				return nil, err
@@ -209,3 +220,21 @@ func (t *Target) composeExtTarget(exts map[string]interface{}) error {
 	}
 	return nil
 }
+
+// composeToBuildkitSecret converts secret from compose format to buildkit's
+// csv format.
+func composeToBuildkitSecret(inp compose.ServiceSecretConfig, psecret compose.SecretConfig) (string, error) {
+	if psecret.External.External {
+		return "", errors.Errorf("unsupported external secret %s", psecret.Name)
+	}
+
+	var bkattrs []string
+	if inp.Source != "" {
+		bkattrs = append(bkattrs, "id="+inp.Source)
+	}
+	if psecret.File != "" {
+		bkattrs = append(bkattrs, "src="+psecret.File)
+	}
+
+	return strings.Join(bkattrs, ","), nil
+}

bake/compose_test.go

@@ -23,6 +23,13 @@ services:
         none
       args:
         buildno: 123
+      secrets:
+        - ENV_TOKEN
+        - aws
+secrets:
+  ENV_TOKEN: {}
+  aws:
+    file: /root/.aws/credentials
 `)
 
 	c, err := ParseCompose(dt)
@@ -46,6 +53,10 @@ services:
 	require.Equal(t, 1, len(c.Targets[1].Args))
 	require.Equal(t, "123", c.Targets[1].Args["buildno"])
 	require.Equal(t, "none", *c.Targets[1].NetworkMode)
+	require.Equal(t, []string{
+		"id=ENV_TOKEN",
+		"id=aws,src=/root/.aws/credentials",
+	}, c.Targets[1].Secrets)
 }
 
 func TestNoBuildOutOfTreeService(t *testing.T) {

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/bugsnag/panicwrap v1.2.0 // indirect
 	github.com/cenkalti/backoff v2.1.1+incompatible // indirect
 	github.com/cloudflare/cfssl v0.0.0-20181213083726-b94e044bb51e // indirect
-	github.com/compose-spec/compose-go v1.2.1
+	github.com/compose-spec/compose-go v1.2.4
 	github.com/containerd/console v1.0.3
 	github.com/containerd/containerd v1.6.3-0.20220401172941-5ff8fce1fcc6
 	github.com/docker/cli v20.10.13+incompatible

go.sum

@@ -280,8 +280,8 @@ github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h
 github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA=
 github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
 github.com/codahale/hdrhistogram v0.0.0-20160425231609-f8ad88b59a58/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
-github.com/compose-spec/compose-go v1.2.1 h1:8+DAP7Mt/Ohl5y6YbZdilLMvIhMxvuSZcNZyywjQmJE=
+github.com/compose-spec/compose-go v1.2.4 h1:nzTFqM8+2J7Veao5Pq5U451thinv3U1wChIvcjX59/A=
-github.com/compose-spec/compose-go v1.2.1/go.mod h1:pAy7Mikpeft4pxkFU565/DRHEbDfR84G6AQuiL+Hdg8=
+github.com/compose-spec/compose-go v1.2.4/go.mod h1:pAy7Mikpeft4pxkFU565/DRHEbDfR84G6AQuiL+Hdg8=
 github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE=
 github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU=
 github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU=

vendor/github.com/compose-spec/compose-go/loader/full-example.yml

@@ -15,6 +15,13 @@ services:
         - foo
         - bar
       labels: [FOO=BAR]
+      secrets:
+        - secret1
+        - source: secret2
+          target: my_secret
+          uid: '103'
+          gid: '103'
+          mode: 0440
 
 
     cap_add:

vendor/github.com/compose-spec/compose-go/loader/loader.go

@@ -53,6 +53,8 @@ type Options struct {
 	SkipNormalization bool
 	// Resolve paths
 	ResolvePaths bool
+	// Convert Windows paths
+	ConvertWindowsPaths bool
 	// Skip consistency check
 	SkipConsistencyCheck bool
 	// Skip extends
@@ -489,7 +491,7 @@ func loadServiceWithExtends(filename, name string, servicesDict map[string]inter
 		return nil, fmt.Errorf("cannot extend service %q in %s: service not found", name, filename)
 	}
 
-	serviceConfig, err := LoadService(name, target.(map[string]interface{}), workingDir, lookupEnv, opts.ResolvePaths)
+	serviceConfig, err := LoadService(name, target.(map[string]interface{}), workingDir, lookupEnv, opts.ResolvePaths, opts.ConvertWindowsPaths)
 	if err != nil {
 		return nil, err
 	}
@@ -552,7 +554,7 @@ func loadServiceWithExtends(filename, name string, servicesDict map[string]inter
 
 // LoadService produces a single ServiceConfig from a compose file Dict
 // the serviceDict is not validated if directly used. Use Load() to enable validation
-func LoadService(name string, serviceDict map[string]interface{}, workingDir string, lookupEnv template.Mapping, resolvePaths bool) (*types.ServiceConfig, error) {
+func LoadService(name string, serviceDict map[string]interface{}, workingDir string, lookupEnv template.Mapping, resolvePaths bool, convertPaths bool) (*types.ServiceConfig, error) {
 	serviceConfig := &types.ServiceConfig{
 		Scale: 1,
 	}
@@ -577,11 +579,30 @@ func LoadService(name string, serviceDict map[string]interface{}, workingDir str
 		if resolvePaths {
 			serviceConfig.Volumes[i] = resolveVolumePath(volume, workingDir, lookupEnv)
 		}
+
+		if convertPaths {
+			serviceConfig.Volumes[i] = convertVolumePath(volume)
+		}
 	}
 
 	return serviceConfig, nil
 }
 
+// Windows paths, c:\\my\\path\\shiny, need to be changed to be compatible with
+// the Engine. Volume paths are expected to be linux style /c/my/path/shiny/
+func convertVolumePath(volume types.ServiceVolumeConfig) types.ServiceVolumeConfig {
+	volumeName := strings.ToLower(filepath.VolumeName(volume.Source))
+	if len(volumeName) != 2 {
+		return volume
+	}
+
+	convertedSource := fmt.Sprintf("/%c%s", volumeName[0], volume.Source[len(volumeName):])
+	convertedSource = strings.ReplaceAll(convertedSource, "\\", "/")
+
+	volume.Source = convertedSource
+	return volume
+}
+
 func resolveEnvironment(serviceConfig *types.ServiceConfig, workingDir string, lookupEnv template.Mapping) error {
 	environment := types.MappingWithEquals{}
 
@@ -998,14 +1019,19 @@ var transformSSHConfig TransformerFunc = func(data interface{}) (interface{}, er
 		}
 		return result, nil
 	case string:
+		return ParseShortSSHSyntax(value)
+	}
+	return nil, errors.Errorf("expected a sting, map or a list, got %T: %#v", data, data)
+}
+
+// ParseShortSSHSyntax parse short syntax for SSH authentications
+func ParseShortSSHSyntax(value string) ([]types.SSHKey, error) {
 	if value == "" {
 		value = "default"
 	}
 	key, val := transformValueToMapEntry(value, "=", false)
 	result := []types.SSHKey{{ID: key, Path: val.(string)}}
 	return result, nil
-	}
-	return nil, errors.Errorf("expected a sting, map or a list, got %T: %#v", data, data)
 }
 
 var transformStringOrNumberList TransformerFunc = func(value interface{}) (interface{}, error) {

vendor/github.com/compose-spec/compose-go/loader/validate.go

@@ -55,9 +55,11 @@ func checkConsistency(project *types.Project) error {
 				}
 			}
 		}
-		for _, secret := range s.Secrets {
+		if s.Build != nil {
+			for _, secret := range s.Build.Secrets {
 				if _, ok := project.Secrets[secret.Source]; !ok {
-				return errors.Wrap(errdefs.ErrInvalid, fmt.Sprintf("service %q refers to undefined secret %s", s.Name, secret.Source))
+					return errors.Wrap(errdefs.ErrInvalid, fmt.Sprintf("service %q refers to undefined build secret %s", s.Name, secret.Source))
+				}
 			}
 		}
 		for _, config := range s.Configs {

vendor/github.com/compose-spec/compose-go/schema/compose-spec.json

@@ -101,7 +101,8 @@
                 "target": {"type": "string"},
                 "shm_size": {"type": ["integer", "string"]},
                 "extra_hosts": {"$ref": "#/definitions/list_or_dict"},
-                "isolation": {"type": "string"}
+                "isolation": {"type": "string"},
+                "secrets": {"$ref": "#/definitions/service_config_or_secret"}
               },
               "additionalProperties": false,
               "patternProperties": {"^x-": {}}
@@ -144,26 +145,7 @@
             {"type": "array", "items": {"type": "string"}}
           ]
         },
-        "configs": {
+        "configs": {"$ref": "#/definitions/service_config_or_secret"},
-          "type": "array",
-          "items": {
-            "oneOf": [
-              {"type": "string"},
-              {
-                "type": "object",
-                "properties": {
-                  "source": {"type": "string"},
-                  "target": {"type": "string"},
-                  "uid": {"type": "string"},
-                  "gid": {"type": "string"},
-                  "mode": {"type": "number"}
-                },
-                "additionalProperties": false,
-                "patternProperties": {"^x-": {}}
-              }
-            ]
-          }
-        },
         "container_name": {"type": "string"},
         "cpu_count": {"type": "integer", "minimum": 0},
         "cpu_percent": {"type": "integer", "minimum": 0, "maximum": 100},
@@ -352,26 +334,7 @@
         },
         "security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
         "shm_size": {"type": ["number", "string"]},
-        "secrets": {
+        "secrets": {"$ref": "#/definitions/service_config_or_secret"},
-          "type": "array",
-          "items": {
-            "oneOf": [
-              {"type": "string"},
-              {
-                "type": "object",
-                "properties": {
-                  "source": {"type": "string"},
-                  "target": {"type": "string"},
-                  "uid": {"type": "string"},
-                  "gid": {"type": "string"},
-                  "mode": {"type": "number"}
-                },
-                "additionalProperties": false,
-                "patternProperties": {"^x-": {}}
-              }
-            ]
-          }
-        },
         "sysctls": {"$ref": "#/definitions/list_or_dict"},
         "stdin_open": {"type": "boolean"},
         "stop_grace_period": {"type": "string", "format": "duration"},
@@ -809,6 +772,27 @@
       "additionalProperties": false
     },
 
+    "service_config_or_secret": {
+      "type": "array",
+      "items": {
+        "oneOf": [
+          {"type": "string"},
+          {
+            "type": "object",
+            "properties": {
+              "source": {"type": "string"},
+              "target": {"type": "string"},
+              "uid": {"type": "string"},
+              "gid": {"type": "string"},
+              "mode": {"type": "number"}
+            },
+            "additionalProperties": false,
+            "patternProperties": {"^x-": {}}
+          }
+        ]
+      }
+    },
+
     "constraints": {
       "service": {
         "id": "#/definitions/constraints/service",

vendor/github.com/compose-spec/compose-go/types/project.go

@@ -246,6 +246,11 @@ func (p *Project) WithoutUnnecessaryResources() {
 		for _, v := range s.Secrets {
 			requiredSecrets[v.Source] = struct{}{}
 		}
+		if s.Build != nil {
+			for _, v := range s.Build.Secrets {
+				requiredSecrets[v.Source] = struct{}{}
+			}
+		}
 		for _, v := range s.Configs {
 			requiredConfigs[v.Source] = struct{}{}
 		}

vendor/github.com/compose-spec/compose-go/types/types.go

@@ -304,6 +304,7 @@ type BuildConfig struct {
 	Isolation  string                `yaml:",omitempty" json:"isolation,omitempty"`
 	Network    string                `yaml:",omitempty" json:"network,omitempty"`
 	Target     string                `yaml:",omitempty" json:"target,omitempty"`
+	Secrets    []ServiceSecretConfig `yaml:",omitempty" json:"secrets,omitempty"`
 
 	Extensions map[string]interface{} `yaml:",inline" json:"-"`
 }
@@ -678,6 +679,25 @@ type ServiceVolumeConfig struct {
 	Extensions map[string]interface{} `yaml:",inline" json:"-"`
 }
 
+// String render ServiceVolumeConfig as a volume string, one can parse back using loader.ParseVolume
+func (s ServiceVolumeConfig) String() string {
+	access := "rw"
+	if s.ReadOnly {
+		access = "ro"
+	}
+	options := []string{access}
+	if s.Bind != nil && s.Bind.SELinux != "" {
+		options = append(options, s.Bind.SELinux)
+	}
+	if s.Bind != nil && s.Bind.Propagation != "" {
+		options = append(options, s.Bind.Propagation)
+	}
+	if s.Volume != nil && s.Volume.NoCopy {
+		options = append(options, "nocopy")
+	}
+	return fmt.Sprintf("%s:%s:%s", s.Source, s.Target, strings.Join(options, ","))
+}
+
 const (
 	// VolumeTypeBind is the type for mounting host dir
 	VolumeTypeBind = "bind"

vendor/modules.txt

@@ -32,7 +32,7 @@ github.com/cenkalti/backoff/v4
 github.com/cespare/xxhash/v2
 # github.com/cloudflare/cfssl v0.0.0-20181213083726-b94e044bb51e
 ## explicit
-# github.com/compose-spec/compose-go v1.2.1
+# github.com/compose-spec/compose-go v1.2.4
 ## explicit
 github.com/compose-spec/compose-go/consts
 github.com/compose-spec/compose-go/dotenv