vendor: add buildkit

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
This commit is contained in:
Tonis Tiigi
2019-03-23 22:44:59 -07:00
parent 62faee5f07
commit 8b7c38e61a
364 changed files with 78556 additions and 1007 deletions

View File

@ -0,0 +1,76 @@
package client
import (
"context"
"github.com/moby/buildkit/solver/pb"
"github.com/moby/buildkit/util/apicaps"
digest "github.com/opencontainers/go-digest"
specs "github.com/opencontainers/image-spec/specs-go/v1"
fstypes "github.com/tonistiigi/fsutil/types"
)
type Client interface {
Solve(ctx context.Context, req SolveRequest) (*Result, error)
ResolveImageConfig(ctx context.Context, ref string, opt ResolveImageConfigOpt) (digest.Digest, []byte, error)
BuildOpts() BuildOpts
}
type Reference interface {
ReadFile(ctx context.Context, req ReadRequest) ([]byte, error)
StatFile(ctx context.Context, req StatRequest) (*fstypes.Stat, error)
ReadDir(ctx context.Context, req ReadDirRequest) ([]*fstypes.Stat, error)
}
type ReadRequest struct {
Filename string
Range *FileRange
}
type FileRange struct {
Offset int
Length int
}
type ReadDirRequest struct {
Path string
IncludePattern string
}
type StatRequest struct {
Path string
}
// SolveRequest is same as frontend.SolveRequest but avoiding dependency
type SolveRequest struct {
Definition *pb.Definition
Frontend string
FrontendOpt map[string]string
CacheImports []CacheOptionsEntry
}
type CacheOptionsEntry struct {
Type string
Attrs map[string]string
}
type WorkerInfo struct {
ID string
Labels map[string]string
Platforms []specs.Platform
}
type BuildOpts struct {
Opts map[string]string
SessionID string
Workers []WorkerInfo
Product string
LLBCaps apicaps.CapSet
Caps apicaps.CapSet
}
type ResolveImageConfigOpt struct {
Platform *specs.Platform
ResolveMode string
LogName string
}

View File

@ -0,0 +1,54 @@
package client
import (
"context"
"sync"
"github.com/pkg/errors"
)
type BuildFunc func(context.Context, Client) (*Result, error)
type Result struct {
mu sync.Mutex
Ref Reference
Refs map[string]Reference
Metadata map[string][]byte
}
func NewResult() *Result {
return &Result{}
}
func (r *Result) AddMeta(k string, v []byte) {
r.mu.Lock()
if r.Metadata == nil {
r.Metadata = map[string][]byte{}
}
r.Metadata[k] = v
r.mu.Unlock()
}
func (r *Result) AddRef(k string, ref Reference) {
r.mu.Lock()
if r.Refs == nil {
r.Refs = map[string]Reference{}
}
r.Refs[k] = ref
r.mu.Unlock()
}
func (r *Result) SetRef(ref Reference) {
r.Ref = ref
}
func (r *Result) SingleRef() (Reference, error) {
r.mu.Lock()
defer r.mu.Unlock()
if r.Refs != nil && r.Ref == nil {
return nil, errors.Errorf("invalid map result")
}
return r.Ref, nil
}

View File

@ -0,0 +1,499 @@
package grpcclient
import (
"context"
"encoding/json"
"io"
"net"
"os"
"strings"
"time"
"github.com/gogo/googleapis/google/rpc"
"github.com/moby/buildkit/frontend/gateway/client"
pb "github.com/moby/buildkit/frontend/gateway/pb"
opspb "github.com/moby/buildkit/solver/pb"
"github.com/moby/buildkit/util/apicaps"
digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors"
fstypes "github.com/tonistiigi/fsutil/types"
"google.golang.org/grpc"
"google.golang.org/grpc/status"
)
const frontendPrefix = "BUILDKIT_FRONTEND_OPT_"
type GrpcClient interface {
Run(context.Context, client.BuildFunc) error
}
func New(ctx context.Context, opts map[string]string, session, product string, c pb.LLBBridgeClient, w []client.WorkerInfo) (GrpcClient, error) {
resp, err := c.Ping(ctx, &pb.PingRequest{})
if err != nil {
return nil, err
}
if resp.FrontendAPICaps == nil {
resp.FrontendAPICaps = defaultCaps()
}
if resp.LLBCaps == nil {
resp.LLBCaps = defaultLLBCaps()
}
return &grpcClient{
client: c,
opts: opts,
sessionID: session,
workers: w,
product: product,
caps: pb.Caps.CapSet(resp.FrontendAPICaps),
llbCaps: opspb.Caps.CapSet(resp.LLBCaps),
requests: map[string]*pb.SolveRequest{},
}, nil
}
func current() (GrpcClient, error) {
if ep := product(); ep != "" {
apicaps.ExportedProduct = ep
}
ctx, conn, err := grpcClientConn(context.Background())
if err != nil {
return nil, err
}
return New(ctx, opts(), sessionID(), product(), pb.NewLLBBridgeClient(conn), workers())
}
func convertRef(ref client.Reference) (string, error) {
if ref == nil {
return "", nil
}
r, ok := ref.(*reference)
if !ok {
return "", errors.Errorf("invalid return reference type %T", ref)
}
return r.id, nil
}
func RunFromEnvironment(ctx context.Context, f client.BuildFunc) error {
client, err := current()
if err != nil {
return errors.Wrapf(err, "failed to initialize client from environment")
}
return client.Run(ctx, f)
}
func (c *grpcClient) Run(ctx context.Context, f client.BuildFunc) (retError error) {
export := c.caps.Supports(pb.CapReturnResult) == nil
var (
res *client.Result
err error
)
if export {
defer func() {
req := &pb.ReturnRequest{}
if retError == nil {
if res == nil {
res = &client.Result{}
}
pbRes := &pb.Result{
Metadata: res.Metadata,
}
if res.Refs != nil {
m := map[string]string{}
for k, r := range res.Refs {
id, err := convertRef(r)
if err != nil {
retError = err
continue
}
m[k] = id
}
pbRes.Result = &pb.Result_Refs{Refs: &pb.RefMap{Refs: m}}
} else {
id, err := convertRef(res.Ref)
if err != nil {
retError = err
} else {
pbRes.Result = &pb.Result_Ref{Ref: id}
}
}
if retError == nil {
req.Result = pbRes
}
}
if retError != nil {
st, _ := status.FromError(retError)
stp := st.Proto()
req.Error = &rpc.Status{
Code: stp.Code,
Message: stp.Message,
// Details: stp.Details,
}
}
if _, err := c.client.Return(ctx, req); err != nil && retError == nil {
retError = err
}
}()
}
if res, err = f(ctx, c); err != nil {
return err
}
if err := c.caps.Supports(pb.CapReturnMap); len(res.Refs) > 1 && err != nil {
return err
}
if !export {
exportedAttrBytes, err := json.Marshal(res.Metadata)
if err != nil {
return errors.Wrapf(err, "failed to marshal return metadata")
}
req, err := c.requestForRef(res.Ref)
if err != nil {
return errors.Wrapf(err, "failed to find return ref")
}
req.Final = true
req.ExporterAttr = exportedAttrBytes
if _, err := c.client.Solve(ctx, req); err != nil {
return errors.Wrapf(err, "failed to solve")
}
}
return nil
}
// defaultCaps returns the capabilities that were implemented when capabilities
// support was added. This list is frozen and should never be changed.
func defaultCaps() []apicaps.PBCap {
return []apicaps.PBCap{
{ID: string(pb.CapSolveBase), Enabled: true},
{ID: string(pb.CapSolveInlineReturn), Enabled: true},
{ID: string(pb.CapResolveImage), Enabled: true},
{ID: string(pb.CapReadFile), Enabled: true},
}
}
// defaultLLBCaps returns the LLB capabilities that were implemented when capabilities
// support was added. This list is frozen and should never be changed.
func defaultLLBCaps() []apicaps.PBCap {
return []apicaps.PBCap{
{ID: string(opspb.CapSourceImage), Enabled: true},
{ID: string(opspb.CapSourceLocal), Enabled: true},
{ID: string(opspb.CapSourceLocalUnique), Enabled: true},
{ID: string(opspb.CapSourceLocalSessionID), Enabled: true},
{ID: string(opspb.CapSourceLocalIncludePatterns), Enabled: true},
{ID: string(opspb.CapSourceLocalFollowPaths), Enabled: true},
{ID: string(opspb.CapSourceLocalExcludePatterns), Enabled: true},
{ID: string(opspb.CapSourceLocalSharedKeyHint), Enabled: true},
{ID: string(opspb.CapSourceGit), Enabled: true},
{ID: string(opspb.CapSourceGitKeepDir), Enabled: true},
{ID: string(opspb.CapSourceGitFullURL), Enabled: true},
{ID: string(opspb.CapSourceHTTP), Enabled: true},
{ID: string(opspb.CapSourceHTTPChecksum), Enabled: true},
{ID: string(opspb.CapSourceHTTPPerm), Enabled: true},
{ID: string(opspb.CapSourceHTTPUIDGID), Enabled: true},
{ID: string(opspb.CapBuildOpLLBFileName), Enabled: true},
{ID: string(opspb.CapExecMetaBase), Enabled: true},
{ID: string(opspb.CapExecMetaProxy), Enabled: true},
{ID: string(opspb.CapExecMountBind), Enabled: true},
{ID: string(opspb.CapExecMountCache), Enabled: true},
{ID: string(opspb.CapExecMountCacheSharing), Enabled: true},
{ID: string(opspb.CapExecMountSelector), Enabled: true},
{ID: string(opspb.CapExecMountTmpfs), Enabled: true},
{ID: string(opspb.CapExecMountSecret), Enabled: true},
{ID: string(opspb.CapConstraints), Enabled: true},
{ID: string(opspb.CapPlatform), Enabled: true},
{ID: string(opspb.CapMetaIgnoreCache), Enabled: true},
{ID: string(opspb.CapMetaDescription), Enabled: true},
{ID: string(opspb.CapMetaExportCache), Enabled: true},
}
}
type grpcClient struct {
client pb.LLBBridgeClient
opts map[string]string
sessionID string
product string
workers []client.WorkerInfo
caps apicaps.CapSet
llbCaps apicaps.CapSet
requests map[string]*pb.SolveRequest
}
func (c *grpcClient) requestForRef(ref client.Reference) (*pb.SolveRequest, error) {
emptyReq := &pb.SolveRequest{
Definition: &opspb.Definition{},
}
if ref == nil {
return emptyReq, nil
}
r, ok := ref.(*reference)
if !ok {
return nil, errors.Errorf("return reference has invalid type %T", ref)
}
if r.id == "" {
return emptyReq, nil
}
req, ok := c.requests[r.id]
if !ok {
return nil, errors.Errorf("did not find request for return reference %s", r.id)
}
return req, nil
}
func (c *grpcClient) Solve(ctx context.Context, creq client.SolveRequest) (*client.Result, error) {
if creq.Definition != nil {
for _, md := range creq.Definition.Metadata {
for cap := range md.Caps {
if err := c.llbCaps.Supports(cap); err != nil {
return nil, err
}
}
}
}
var (
// old API
legacyRegistryCacheImports []string
// new API (CapImportCaches)
cacheImports []*pb.CacheOptionsEntry
)
supportCapImportCaches := c.caps.Supports(pb.CapImportCaches) == nil
for _, im := range creq.CacheImports {
if !supportCapImportCaches && im.Type == "registry" {
legacyRegistryCacheImports = append(legacyRegistryCacheImports, im.Attrs["ref"])
} else {
cacheImports = append(cacheImports, &pb.CacheOptionsEntry{
Type: im.Type,
Attrs: im.Attrs,
})
}
}
req := &pb.SolveRequest{
Definition: creq.Definition,
Frontend: creq.Frontend,
FrontendOpt: creq.FrontendOpt,
AllowResultReturn: true,
// old API
ImportCacheRefsDeprecated: legacyRegistryCacheImports,
// new API
CacheImports: cacheImports,
}
// backwards compatibility with inline return
if c.caps.Supports(pb.CapReturnResult) != nil {
req.ExporterAttr = []byte("{}")
}
resp, err := c.client.Solve(ctx, req)
if err != nil {
return nil, err
}
res := &client.Result{}
if resp.Result == nil {
if id := resp.Ref; id != "" {
c.requests[id] = req
}
res.SetRef(&reference{id: resp.Ref, c: c})
} else {
res.Metadata = resp.Result.Metadata
switch pbRes := resp.Result.Result.(type) {
case *pb.Result_Ref:
if id := pbRes.Ref; id != "" {
res.SetRef(&reference{id: id, c: c})
}
case *pb.Result_Refs:
for k, v := range pbRes.Refs.Refs {
ref := &reference{id: v, c: c}
if v == "" {
ref = nil
}
res.AddRef(k, ref)
}
}
}
return res, nil
}
func (c *grpcClient) ResolveImageConfig(ctx context.Context, ref string, opt client.ResolveImageConfigOpt) (digest.Digest, []byte, error) {
var p *opspb.Platform
if platform := opt.Platform; platform != nil {
p = &opspb.Platform{
OS: platform.OS,
Architecture: platform.Architecture,
Variant: platform.Variant,
OSVersion: platform.OSVersion,
OSFeatures: platform.OSFeatures,
}
}
resp, err := c.client.ResolveImageConfig(ctx, &pb.ResolveImageConfigRequest{Ref: ref, Platform: p, ResolveMode: opt.ResolveMode, LogName: opt.LogName})
if err != nil {
return "", nil, err
}
return resp.Digest, resp.Config, nil
}
func (c *grpcClient) BuildOpts() client.BuildOpts {
return client.BuildOpts{
Opts: c.opts,
SessionID: c.sessionID,
Workers: c.workers,
Product: c.product,
LLBCaps: c.llbCaps,
Caps: c.caps,
}
}
type reference struct {
id string
c *grpcClient
}
func (r *reference) ReadFile(ctx context.Context, req client.ReadRequest) ([]byte, error) {
rfr := &pb.ReadFileRequest{FilePath: req.Filename, Ref: r.id}
if r := req.Range; r != nil {
rfr.Range = &pb.FileRange{
Offset: int64(r.Offset),
Length: int64(r.Length),
}
}
resp, err := r.c.client.ReadFile(ctx, rfr)
if err != nil {
return nil, err
}
return resp.Data, nil
}
func (r *reference) ReadDir(ctx context.Context, req client.ReadDirRequest) ([]*fstypes.Stat, error) {
if err := r.c.caps.Supports(pb.CapReadDir); err != nil {
return nil, err
}
rdr := &pb.ReadDirRequest{
DirPath: req.Path,
IncludePattern: req.IncludePattern,
Ref: r.id,
}
resp, err := r.c.client.ReadDir(ctx, rdr)
if err != nil {
return nil, err
}
return resp.Entries, nil
}
func (r *reference) StatFile(ctx context.Context, req client.StatRequest) (*fstypes.Stat, error) {
if err := r.c.caps.Supports(pb.CapStatFile); err != nil {
return nil, err
}
rdr := &pb.StatFileRequest{
Path: req.Path,
Ref: r.id,
}
resp, err := r.c.client.StatFile(ctx, rdr)
if err != nil {
return nil, err
}
return resp.Stat, nil
}
func grpcClientConn(ctx context.Context) (context.Context, *grpc.ClientConn, error) {
dialOpt := grpc.WithDialer(func(addr string, d time.Duration) (net.Conn, error) {
return stdioConn(), nil
})
cc, err := grpc.DialContext(ctx, "", dialOpt, grpc.WithInsecure())
if err != nil {
return nil, nil, errors.Wrap(err, "failed to create grpc client")
}
ctx, cancel := context.WithCancel(ctx)
_ = cancel
// go monitorHealth(ctx, cc, cancel)
return ctx, cc, nil
}
func stdioConn() net.Conn {
return &conn{os.Stdin, os.Stdout, os.Stdout}
}
type conn struct {
io.Reader
io.Writer
io.Closer
}
func (s *conn) LocalAddr() net.Addr {
return dummyAddr{}
}
func (s *conn) RemoteAddr() net.Addr {
return dummyAddr{}
}
func (s *conn) SetDeadline(t time.Time) error {
return nil
}
func (s *conn) SetReadDeadline(t time.Time) error {
return nil
}
func (s *conn) SetWriteDeadline(t time.Time) error {
return nil
}
type dummyAddr struct {
}
func (d dummyAddr) Network() string {
return "pipe"
}
func (d dummyAddr) String() string {
return "localhost"
}
func opts() map[string]string {
opts := map[string]string{}
for _, env := range os.Environ() {
parts := strings.SplitN(env, "=", 2)
k := parts[0]
v := ""
if len(parts) == 2 {
v = parts[1]
}
if !strings.HasPrefix(k, frontendPrefix) {
continue
}
parts = strings.SplitN(v, "=", 2)
v = ""
if len(parts) == 2 {
v = parts[1]
}
opts[parts[0]] = v
}
return opts
}
func sessionID() string {
return os.Getenv("BUILDKIT_SESSION_ID")
}
func workers() []client.WorkerInfo {
var c []client.WorkerInfo
if err := json.Unmarshal([]byte(os.Getenv("BUILDKIT_WORKERS")), &c); err != nil {
return nil
}
return c
}
func product() string {
return os.Getenv("BUILDKIT_EXPORTEDPRODUCT")
}

View File

@ -0,0 +1,95 @@
package moby_buildkit_v1_frontend
import "github.com/moby/buildkit/util/apicaps"
var Caps apicaps.CapList
// Every backwards or forwards non-compatible change needs to add a new capability row.
// By default new capabilities should be experimental. After merge a capability is
// considered immutable. After a capability is marked stable it should not be disabled.
const (
CapSolveBase apicaps.CapID = "solve.base"
CapSolveInlineReturn apicaps.CapID = "solve.inlinereturn"
CapResolveImage apicaps.CapID = "resolveimage"
CapResolveImageResolveMode apicaps.CapID = "resolveimage.resolvemode"
CapReadFile apicaps.CapID = "readfile"
CapReturnResult apicaps.CapID = "return"
CapReturnMap apicaps.CapID = "returnmap"
CapReadDir apicaps.CapID = "readdir"
CapStatFile apicaps.CapID = "statfile"
CapImportCaches apicaps.CapID = "importcaches"
)
func init() {
Caps.Init(apicaps.Cap{
ID: CapSolveBase,
Enabled: true,
Status: apicaps.CapStatusExperimental,
})
Caps.Init(apicaps.Cap{
ID: CapSolveInlineReturn,
Name: "inline return from solve",
Enabled: true,
Deprecated: true,
Status: apicaps.CapStatusExperimental,
})
Caps.Init(apicaps.Cap{
ID: CapResolveImage,
Name: "resolve remote image config",
Enabled: true,
Status: apicaps.CapStatusExperimental,
})
Caps.Init(apicaps.Cap{
ID: CapResolveImageResolveMode,
Name: "resolve remote image config with custom resolvemode",
Enabled: true,
Status: apicaps.CapStatusExperimental,
})
Caps.Init(apicaps.Cap{
ID: CapReadFile,
Name: "read static file",
Enabled: true,
Status: apicaps.CapStatusExperimental,
})
Caps.Init(apicaps.Cap{
ID: CapReturnResult,
Name: "return solve result",
Enabled: true,
Status: apicaps.CapStatusExperimental,
})
Caps.Init(apicaps.Cap{
ID: CapReturnMap,
Name: "return reference map",
Enabled: true,
Status: apicaps.CapStatusExperimental,
})
Caps.Init(apicaps.Cap{
ID: CapReadDir,
Name: "read static directory",
Enabled: true,
Status: apicaps.CapStatusExperimental,
})
Caps.Init(apicaps.Cap{
ID: CapStatFile,
Name: "stat a file",
Enabled: true,
Status: apicaps.CapStatusExperimental,
})
Caps.Init(apicaps.Cap{
ID: CapImportCaches,
Name: "import caches",
Enabled: true,
Status: apicaps.CapStatusExperimental,
})
}

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,138 @@
syntax = "proto3";
package moby.buildkit.v1.frontend;
import "github.com/gogo/protobuf/gogoproto/gogo.proto";
import "github.com/gogo/googleapis/google/rpc/status.proto";
import "github.com/moby/buildkit/solver/pb/ops.proto";
import "github.com/moby/buildkit/api/types/worker.proto";
import "github.com/moby/buildkit/util/apicaps/pb/caps.proto";
import "github.com/tonistiigi/fsutil/types/stat.proto";
option (gogoproto.sizer_all) = true;
option (gogoproto.marshaler_all) = true;
option (gogoproto.unmarshaler_all) = true;
service LLBBridge {
// apicaps:CapResolveImage
rpc ResolveImageConfig(ResolveImageConfigRequest) returns (ResolveImageConfigResponse);
// apicaps:CapSolveBase
rpc Solve(SolveRequest) returns (SolveResponse);
// apicaps:CapReadFile
rpc ReadFile(ReadFileRequest) returns (ReadFileResponse);
// apicaps:CapReadDir
rpc ReadDir(ReadDirRequest) returns (ReadDirResponse);
// apicaps:CapStatFile
rpc StatFile(StatFileRequest) returns (StatFileResponse);
rpc Ping(PingRequest) returns (PongResponse);
rpc Return(ReturnRequest) returns (ReturnResponse);
}
message Result {
oneof result {
string ref = 1;
RefMap refs = 2;
}
map<string, bytes> metadata = 10;
}
message RefMap {
map<string, string> refs = 1;
}
message ReturnRequest {
Result result = 1;
google.rpc.Status error = 2;
}
message ReturnResponse {
}
message ResolveImageConfigRequest {
string Ref = 1;
pb.Platform Platform = 2;
string ResolveMode = 3;
string LogName = 4;
}
message ResolveImageConfigResponse {
string Digest = 1 [(gogoproto.customtype) = "github.com/opencontainers/go-digest.Digest", (gogoproto.nullable) = false];
bytes Config = 2;
}
message SolveRequest {
pb.Definition Definition = 1;
string Frontend = 2;
map<string, string> FrontendOpt = 3;
// ImportCacheRefsDeprecated is deprecated in favor or the new Imports since BuildKit v0.4.0.
// When ImportCacheRefsDeprecated is set, the solver appends
// {.Type = "registry", .Attrs = {"ref": importCacheRef}}
// for each of the ImportCacheRefs entry to CacheImports for compatibility. (planned to be removed)
repeated string ImportCacheRefsDeprecated = 4;
bool allowResultReturn = 5;
// apicaps.CapSolveInlineReturn deprecated
bool Final = 10;
bytes ExporterAttr = 11;
// CacheImports was added in BuildKit v0.4.0.
// apicaps:CapImportCaches
repeated CacheOptionsEntry CacheImports = 12;
}
// CacheOptionsEntry corresponds to the control.CacheOptionsEntry
message CacheOptionsEntry {
string Type = 1;
map<string, string> Attrs = 2;
}
message SolveResponse {
// deprecated
string ref = 1; // can be used by readfile request
// deprecated
/* bytes ExporterAttr = 2;*/
// these fields are returned when allowMapReturn was set
Result result = 3;
}
message ReadFileRequest {
string Ref = 1;
string FilePath = 2;
FileRange Range = 3;
}
message FileRange {
int64 Offset = 1;
int64 Length = 2;
}
message ReadFileResponse {
bytes Data = 1;
}
message ReadDirRequest {
string Ref = 1;
string DirPath = 2;
string IncludePattern = 3;
}
message ReadDirResponse {
repeated fsutil.types.Stat entries = 1;
}
message StatFileRequest {
string Ref = 1;
string Path = 2;
}
message StatFileResponse {
fsutil.types.Stat stat = 1;
}
message PingRequest{
}
message PongResponse{
repeated moby.buildkit.v1.apicaps.APICap FrontendAPICaps = 1 [(gogoproto.nullable) = false];
repeated moby.buildkit.v1.apicaps.APICap LLBCaps = 2 [(gogoproto.nullable) = false];
repeated moby.buildkit.v1.types.WorkerRecord Workers = 3;
}

View File

@ -0,0 +1,3 @@
package moby_buildkit_v1_frontend
//go:generate protoc -I=. -I=../../../vendor/ -I=../../../../../../ --gogo_out=plugins=grpc:. gateway.proto