hack: generate vtproto files for buildx

Integrates vtproto into buildx. The generated files dockerfile has been
modified to copy the buildkit equivalent file to ensure files are laid
out in the appropriate way for imports.

An import has also been included to change the grpc codec to the version
in buildkit that supports vtproto. This will allow buildx to utilize the
speed and memory improvements from that.

Also updates the gc control options for prune.

Signed-off-by: Jonathan A. Sternberg <jonathan.sternberg@docker.com>

vendor/github.com/moby/buildkit/cmd/buildkitd/config/config.go

@@ -75,9 +75,13 @@ type OTELConfig struct {
 }
 
 type GCConfig struct {
-	GC            *bool      `toml:"gc"`
-	GCKeepStorage DiskSpace  `toml:"gckeepstorage"`
-	GCPolicy      []GCPolicy `toml:"gcpolicy"`
+	GC *bool `toml:"gc"`
+	// Deprecated: use GCReservedSpace instead
+	GCKeepStorage   DiskSpace  `toml:"gckeepstorage"`
+	GCReservedSpace DiskSpace  `toml:"reservedSpace"`
+	GCMaxUsedSpace  DiskSpace  `toml:"maxUsedSpace"`
+	GCMinFreeSpace  DiskSpace  `toml:"minFreeSpace"`
+	GCPolicy        []GCPolicy `toml:"gcpolicy"`
 }
 
 type NetworkConfig struct {
@@ -163,20 +167,20 @@ type GCPolicy struct {
 	// to consume. Any storage above this mark can be cleared during a gc
 	// sweep.
 	//
-	// Deprecated: use MaxStorage instead
+	// Deprecated: use ReservedSpace instead
 	KeepBytes DiskSpace `toml:"keepBytes"`
 
-	// MinStorage is the minimum amount of storage this policy is always
-	// allowed to consume. Any amount of storage below this mark will not be
-	// cleared by this policy.
-	MinStorage DiskSpace `toml:"minStorage"`
-	// MaxStorage is the maximum amount of storage this policy is ever allowed
-	// to consume. Any storage above this mark can be cleared during a gc
-	// sweep.
-	MaxStorage DiskSpace `toml:"maxStorage"`
-	// Free is the amount of storage the gc will attempt to leave free on the
-	// disk. However, it will never attempt to bring it below MinStorage.
-	Free DiskSpace `toml:"free"`
+	// ReservedSpace is the minimum amount of disk space this policy is guaranteed to retain.
+	// Any usage below this threshold will not be reclaimed during garbage collection.
+	ReservedSpace DiskSpace `toml:"reservedSpace"`
+
+	// MaxUsedSpace is the maximum amount of disk space this policy is allowed to use.
+	// Any usage exceeding this limit will be cleaned up during a garbage collection sweep.
+	MaxUsedSpace DiskSpace `toml:"maxUsedSpace"`
+
+	// MinFreeSpace is the target amount of free disk space the garbage collector will attempt to leave.
+	// However, it will never let the available space fall below ReservedSpace.
+	MinFreeSpace DiskSpace `toml:"minFreeSpace"`
 }
 
 type DNSConfig struct {

vendor/github.com/moby/buildkit/cmd/buildkitd/config/gcpolicy.go

@@ -7,7 +7,6 @@ import (
 	"time"
 
 	"github.com/docker/go-units"
-	"github.com/moby/buildkit/util/bklog"
 	"github.com/moby/buildkit/util/disk"
 	"github.com/pkg/errors"
 )
@@ -69,30 +68,39 @@ func (d *DiskSpace) UnmarshalText(textb []byte) error {
 
 const defaultCap int64 = 2e9 // 2GB
 
-func DefaultGCPolicy(keep DiskSpace) []GCPolicy {
-	if keep == (DiskSpace{}) {
-		keep = DetectDefaultGCCap()
+func DefaultGCPolicy(cfg GCConfig, dstat disk.DiskStat) []GCPolicy {
+	if cfg.IsUnset() {
+		cfg.GCReservedSpace = cfg.GCKeepStorage
+	}
+	if cfg.IsUnset() {
+		cfg = DetectDefaultGCCap(dstat)
 	}
 	return []GCPolicy{
 		// if build cache uses more than 512MB delete the most easily reproducible data after it has not been used for 2 days
 		{
 			Filters:      []string{"type==source.local,type==exec.cachemount,type==source.git.checkout"},
 			KeepDuration: Duration{Duration: time.Duration(48) * time.Hour}, // 48h
-			MaxStorage:   DiskSpace{Bytes: 512 * 1e6},                       // 512MB
+			MaxUsedSpace: DiskSpace{Bytes: 512 * 1e6},                       // 512MB
 		},
 		// remove any data not used for 60 days
 		{
-			KeepDuration: Duration{Duration: time.Duration(60) * 24 * time.Hour}, // 60d
-			MaxStorage:   keep,
+			KeepDuration:  Duration{Duration: time.Duration(60) * 24 * time.Hour}, // 60d
+			MinFreeSpace:  cfg.GCMinFreeSpace,
+			ReservedSpace: cfg.GCReservedSpace,
+			MaxUsedSpace:  cfg.GCMaxUsedSpace,
 		},
 		// keep the unshared build cache under cap
 		{
-			MaxStorage: keep,
+			MinFreeSpace:  cfg.GCMinFreeSpace,
+			ReservedSpace: cfg.GCReservedSpace,
+			MaxUsedSpace:  cfg.GCMaxUsedSpace,
 		},
 		// if previous policies were insufficient start deleting internal data to keep build cache under cap
 		{
-			All:        true,
-			MaxStorage: keep,
+			All:           true,
+			MinFreeSpace:  cfg.GCMinFreeSpace,
+			ReservedSpace: cfg.GCReservedSpace,
+			MaxUsedSpace:  cfg.GCMaxUsedSpace,
 		},
 	}
 }
@@ -107,11 +115,23 @@ func stripQuotes(s string) string {
 	return s
 }
 
-func DetectDefaultGCCap() DiskSpace {
-	return DiskSpace{Percentage: DiskSpacePercentage}
+func DetectDefaultGCCap(dstat disk.DiskStat) GCConfig {
+	reserve := DiskSpace{Percentage: DiskSpaceReservePercentage}
+	if reserve.AsBytes(dstat) > DiskSpaceReserveBytes {
+		reserve = DiskSpace{Bytes: DiskSpaceReserveBytes}
+	}
+	max := DiskSpace{Percentage: DiskSpaceMaxPercentage}
+	if max.AsBytes(dstat) > DiskSpaceMaxBytes {
+		max = DiskSpace{Bytes: DiskSpaceMaxBytes}
+	}
+	return GCConfig{
+		GCReservedSpace: reserve,
+		GCMinFreeSpace:  DiskSpace{Percentage: DiskSpaceFreePercentage},
+		GCMaxUsedSpace:  max,
+	}
 }
 
-func (d DiskSpace) AsBytes(root string) int64 {
+func (d DiskSpace) AsBytes(dstat disk.DiskStat) int64 {
 	if d.Bytes != 0 {
 		return d.Bytes
 	}
@@ -119,12 +139,15 @@ func (d DiskSpace) AsBytes(root string) int64 {
 		return 0
 	}
 
-	dstat, err := disk.GetDiskStat(root)
-	if err != nil {
-		bklog.L.Warnf("failed to get disk size: %v", err)
+	if dstat.Total == 0 {
 		return defaultCap
 	}
+
 	avail := dstat.Total * d.Percentage / 100
 	rounded := (avail/(1<<30) + 1) * 1e9 // round up
 	return rounded
 }
+
+func (cfg *GCConfig) IsUnset() bool {
+	return cfg.GCReservedSpace == DiskSpace{} && cfg.GCMaxUsedSpace == DiskSpace{} && cfg.GCMinFreeSpace == DiskSpace{}
+}

vendor/github.com/moby/buildkit/cmd/buildkitd/config/gcpolicy_unix.go

@@ -3,4 +3,10 @@
 
 package config
 
-var DiskSpacePercentage int64 = 10
+const (
+	DiskSpaceReservePercentage int64 = 10
+	DiskSpaceReserveBytes      int64 = 10 * 1e9 // 10GB
+	DiskSpaceFreePercentage    int64 = 20
+	DiskSpaceMaxPercentage     int64 = 80
+	DiskSpaceMaxBytes          int64 = 100 * 1e9 // 100GB
+)

vendor/github.com/moby/buildkit/cmd/buildkitd/config/gcpolicy_windows.go

@@ -3,6 +3,12 @@
 
 package config
 
-// set as double that for Linux since
-// Windows images are generally larger.
-var DiskSpacePercentage int64 = 20
+const (
+	// Windows images are generally larger.
+	// set as double that for Linux since
+	DiskSpaceReservePercentage int64 = 20
+	DiskSpaceReserveBytes      int64 = 10 * 1e9 // 10GB
+	DiskSpaceFreePercentage    int64 = 20
+	DiskSpaceMaxPercentage     int64 = 80
+	DiskSpaceMaxBytes          int64 = 100 * 1e9 // 100GB
+)