vendor: update buildkit to master@cbfd4023383d

Signed-off-by: Justin Chadwell <me@jedevc.com>
This commit is contained in:
Justin Chadwell
2023-09-07 12:13:54 +01:00
parent e018f8b6fb
commit e2ebab5f26
60 changed files with 1912 additions and 664 deletions

View File

@ -63,14 +63,13 @@ func GetLogger(ctx context.Context) (l *logrus.Entry) {
return l
}
// LazyStackTrace lets you include a stack trace as a field's value in a log but only
// call it when the log level is actually enabled.
type LazyStackTrace struct{}
func (LazyStackTrace) String() string {
return string(debug.Stack())
}
func (LazyStackTrace) MarshalText() ([]byte, error) {
return debug.Stack(), nil
// TraceLevelOnlyStack returns a stack trace for the current goroutine only if
// trace level logs are enabled; otherwise it returns an empty string. This ensure
// we only pay the cost of generating a stack trace when the log entry will actually
// be emitted.
func TraceLevelOnlyStack() string {
if logrus.GetLevel() == logrus.TraceLevel {
return string(debug.Stack())
}
return ""
}

View File

@ -0,0 +1,24 @@
package contentutil
import (
"context"
"github.com/containerd/containerd/content"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
)
func NewStoreWithProvider(cs content.Store, p content.Provider) content.Store {
return &storeWithProvider{Store: cs, p: p}
}
type storeWithProvider struct {
content.Store
p content.Provider
}
func (cs *storeWithProvider) ReaderAt(ctx context.Context, desc ocispecs.Descriptor) (content.ReaderAt, error) {
if ra, err := cs.p.ReaderAt(ctx, desc); err == nil {
return ra, nil
}
return cs.Store.ReaderAt(ctx, desc)
}

243
vendor/github.com/moby/buildkit/util/gitutil/git_cli.go generated vendored Normal file
View File

@ -0,0 +1,243 @@
package gitutil
import (
"bytes"
"context"
"io"
"os"
"os/exec"
"strings"
"github.com/pkg/errors"
)
// GitCLI carries config to pass to the git cli to make running multiple
// commands less repetitive.
type GitCLI struct {
git string
exec func(context.Context, *exec.Cmd) error
args []string
dir string
streams StreamFunc
workTree string
gitDir string
sshAuthSock string
sshKnownHosts string
}
// Option provides a variadic option for configuring the git client.
type Option func(b *GitCLI)
// WithGitBinary sets the git binary path.
func WithGitBinary(path string) Option {
return func(b *GitCLI) {
b.git = path
}
}
// WithExec sets the command exec function.
func WithExec(exec func(context.Context, *exec.Cmd) error) Option {
return func(b *GitCLI) {
b.exec = exec
}
}
// WithArgs sets extra args.
func WithArgs(args ...string) Option {
return func(b *GitCLI) {
b.args = append(b.args, args...)
}
}
// WithDir sets working directory.
//
// This should be a path to any directory within a standard git repository.
func WithDir(dir string) Option {
return func(b *GitCLI) {
b.dir = dir
}
}
// WithWorkTree sets the --work-tree arg.
//
// This should be the path to the top-level directory of the checkout. When
// setting this, you also likely need to set WithGitDir.
func WithWorkTree(workTree string) Option {
return func(b *GitCLI) {
b.workTree = workTree
}
}
// WithGitDir sets the --git-dir arg.
//
// This should be the path to the .git directory. When setting this, you may
// also need to set WithWorkTree, unless you are working with a bare
// repository.
func WithGitDir(gitDir string) Option {
return func(b *GitCLI) {
b.gitDir = gitDir
}
}
// WithSSHAuthSock sets the ssh auth sock.
func WithSSHAuthSock(sshAuthSock string) Option {
return func(b *GitCLI) {
b.sshAuthSock = sshAuthSock
}
}
// WithSSHKnownHosts sets the known hosts file.
func WithSSHKnownHosts(sshKnownHosts string) Option {
return func(b *GitCLI) {
b.sshKnownHosts = sshKnownHosts
}
}
type StreamFunc func(context.Context) (io.WriteCloser, io.WriteCloser, func())
// WithStreams configures a callback for getting the streams for a command. The
// stream callback will be called once for each command, and both writers will
// be closed after the command has finished.
func WithStreams(streams StreamFunc) Option {
return func(b *GitCLI) {
b.streams = streams
}
}
// New initializes a new git client
func NewGitCLI(opts ...Option) *GitCLI {
c := &GitCLI{}
for _, opt := range opts {
opt(c)
}
return c
}
// New returns a new git client with the same config as the current one, but
// with the given options applied on top.
func (cli *GitCLI) New(opts ...Option) *GitCLI {
c := &GitCLI{
git: cli.git,
dir: cli.dir,
workTree: cli.workTree,
gitDir: cli.gitDir,
args: append([]string{}, cli.args...),
streams: cli.streams,
sshAuthSock: cli.sshAuthSock,
sshKnownHosts: cli.sshKnownHosts,
}
for _, opt := range opts {
opt(c)
}
return c
}
// Run executes a git command with the given args.
func (cli *GitCLI) Run(ctx context.Context, args ...string) (_ []byte, err error) {
gitBinary := "git"
if cli.git != "" {
gitBinary = cli.git
}
for {
var cmd *exec.Cmd
if cli.exec == nil {
cmd = exec.CommandContext(ctx, gitBinary)
} else {
cmd = exec.Command(gitBinary)
}
cmd.Dir = cli.dir
if cmd.Dir == "" {
cmd.Dir = cli.workTree
}
// Block sneaky repositories from using repos from the filesystem as submodules.
cmd.Args = append(cmd.Args, "-c", "protocol.file.allow=user")
if cli.workTree != "" {
cmd.Args = append(cmd.Args, "--work-tree", cli.workTree)
}
if cli.gitDir != "" {
cmd.Args = append(cmd.Args, "--git-dir", cli.gitDir)
}
cmd.Args = append(cmd.Args, cli.args...)
cmd.Args = append(cmd.Args, args...)
buf := bytes.NewBuffer(nil)
errbuf := bytes.NewBuffer(nil)
cmd.Stdin = nil
cmd.Stdout = buf
cmd.Stderr = errbuf
if cli.streams != nil {
stdout, stderr, flush := cli.streams(ctx)
if stdout != nil {
cmd.Stdout = io.MultiWriter(stdout, cmd.Stdout)
}
if stderr != nil {
cmd.Stderr = io.MultiWriter(stderr, cmd.Stderr)
}
defer stdout.Close()
defer stderr.Close()
defer func() {
if err != nil {
flush()
}
}()
}
cmd.Env = []string{
"PATH=" + os.Getenv("PATH"),
"GIT_TERMINAL_PROMPT=0",
"GIT_SSH_COMMAND=" + getGitSSHCommand(cli.sshKnownHosts),
// "GIT_TRACE=1",
"GIT_CONFIG_NOSYSTEM=1", // Disable reading from system gitconfig.
"HOME=/dev/null", // Disable reading from user gitconfig.
"LC_ALL=C", // Ensure consistent output.
}
if cli.sshAuthSock != "" {
cmd.Env = append(cmd.Env, "SSH_AUTH_SOCK="+cli.sshAuthSock)
}
if cli.exec != nil {
// remote git commands spawn helper processes that inherit FDs and don't
// handle parent death signal so exec.CommandContext can't be used
err = cli.exec(ctx, cmd)
} else {
err = cmd.Run()
}
if err != nil {
if strings.Contains(errbuf.String(), "--depth") || strings.Contains(errbuf.String(), "shallow") {
if newArgs := argsNoDepth(args); len(args) > len(newArgs) {
args = newArgs
continue
}
}
return buf.Bytes(), errors.Errorf("git error: %s\nstderr:\n%s", err, errbuf.String())
}
return buf.Bytes(), nil
}
}
func getGitSSHCommand(knownHosts string) string {
gitSSHCommand := "ssh -F /dev/null"
if knownHosts != "" {
gitSSHCommand += " -o UserKnownHostsFile=" + knownHosts
} else {
gitSSHCommand += " -o StrictHostKeyChecking=no"
}
return gitSSHCommand
}
func argsNoDepth(args []string) []string {
out := make([]string, 0, len(args))
for _, a := range args {
if a != "--depth=1" {
out = append(out, a)
}
}
return out
}

View File

@ -0,0 +1,44 @@
package gitutil
import (
"context"
"path/filepath"
"strings"
"github.com/pkg/errors"
)
func (cli *GitCLI) Dir() string {
if cli.dir != "" {
return cli.dir
}
return cli.workTree
}
func (cli *GitCLI) WorkTree(ctx context.Context) (string, error) {
if cli.workTree != "" {
return cli.workTree, nil
}
return cli.clean(cli.Run(ctx, "rev-parse", "--show-toplevel"))
}
func (cli *GitCLI) GitDir(ctx context.Context) (string, error) {
if cli.gitDir != "" {
return cli.gitDir, nil
}
dir, err := cli.WorkTree(ctx)
if err != nil {
return "", err
}
return filepath.Join(dir, ".git"), nil
}
func (cli *GitCLI) clean(dt []byte, err error) (string, error) {
out := string(dt)
out = strings.ReplaceAll(strings.Split(out, "\n")[0], "'", "")
if err != nil {
err = errors.New(strings.TrimSuffix(err.Error(), "\n"))
}
return out, err
}

View File

@ -1,46 +0,0 @@
package gitutil
import (
"strings"
"github.com/moby/buildkit/util/sshutil"
)
const (
HTTPProtocol = iota + 1
HTTPSProtocol
SSHProtocol
GitProtocol
UnknownProtocol
)
// ParseProtocol parses a git URL and returns the remote url and protocol type
func ParseProtocol(remote string) (string, int) {
prefixes := map[string]int{
"http://": HTTPProtocol,
"https://": HTTPSProtocol,
"git://": GitProtocol,
"ssh://": SSHProtocol,
}
protocolType := UnknownProtocol
for prefix, potentialType := range prefixes {
if strings.HasPrefix(remote, prefix) {
remote = strings.TrimPrefix(remote, prefix)
protocolType = potentialType
}
}
if protocolType == UnknownProtocol && sshutil.IsImplicitSSHTransport(remote) {
protocolType = SSHProtocol
}
// remove name from ssh
if protocolType == SSHProtocol {
parts := strings.SplitN(remote, "@", 2)
if len(parts) == 2 {
remote = parts[1]
}
}
return remote, protocolType
}

View File

@ -1,10 +1,11 @@
package gitutil
import (
"regexp"
"net/url"
"strings"
"github.com/containerd/containerd/errdefs"
"github.com/pkg/errors"
)
// GitRef represents a git ref.
@ -51,35 +52,51 @@ type GitRef struct {
func ParseGitRef(ref string) (*GitRef, error) {
res := &GitRef{}
var (
remote *url.URL
err error
)
if strings.HasPrefix(ref, "github.com/") {
res.IndistinguishableFromLocal = true // Deprecated
} else {
_, proto := ParseProtocol(ref)
switch proto {
case UnknownProtocol:
return nil, errdefs.ErrInvalidArgument
remote = &url.URL{
Scheme: "https",
Host: "github.com",
Path: strings.TrimPrefix(ref, "github.com/"),
}
switch proto {
} else {
remote, err = ParseURL(ref)
if errors.Is(err, ErrUnknownProtocol) {
remote, err = ParseURL("https://" + ref)
}
if err != nil {
return nil, err
}
switch remote.Scheme {
case HTTPProtocol, GitProtocol:
res.UnencryptedTCP = true // Discouraged, but not deprecated
}
switch proto {
switch remote.Scheme {
// An HTTP(S) URL is considered to be a valid git ref only when it has the ".git[...]" suffix.
case HTTPProtocol, HTTPSProtocol:
var gitURLPathWithFragmentSuffix = regexp.MustCompile(`\.git(?:#.+)?$`)
if !gitURLPathWithFragmentSuffix.MatchString(ref) {
if !strings.HasSuffix(remote.Path, ".git") {
return nil, errdefs.ErrInvalidArgument
}
}
}
var fragment string
res.Remote, fragment, _ = strings.Cut(ref, "#")
if len(res.Remote) == 0 {
return res, errdefs.ErrInvalidArgument
res.Commit, res.SubDir = SplitGitFragment(remote.Fragment)
remote.Fragment = ""
res.Remote = remote.String()
if res.IndistinguishableFromLocal {
_, res.Remote, _ = strings.Cut(res.Remote, "://")
}
res.Commit, res.SubDir, _ = strings.Cut(fragment, ":")
repoSplitBySlash := strings.Split(res.Remote, "/")
res.ShortName = strings.TrimSuffix(repoSplitBySlash[len(repoSplitBySlash)-1], ".git")
return res, nil
}

View File

@ -0,0 +1,71 @@
package gitutil
import (
"net/url"
"regexp"
"strings"
"github.com/moby/buildkit/util/sshutil"
"github.com/pkg/errors"
)
const (
HTTPProtocol string = "http"
HTTPSProtocol string = "https"
SSHProtocol string = "ssh"
GitProtocol string = "git"
)
var (
ErrUnknownProtocol = errors.New("unknown protocol")
ErrInvalidProtocol = errors.New("invalid protocol")
)
var supportedProtos = map[string]struct{}{
HTTPProtocol: {},
HTTPSProtocol: {},
SSHProtocol: {},
GitProtocol: {},
}
var protoRegexp = regexp.MustCompile(`^[a-zA-Z0-9]+://`)
// ParseURL parses a git URL and returns a parsed URL object.
//
// ParseURL understands implicit ssh URLs such as "git@host:repo", and
// returns the same response as if the URL were "ssh://git@host/repo".
func ParseURL(remote string) (*url.URL, error) {
if proto := protoRegexp.FindString(remote); proto != "" {
proto = strings.ToLower(strings.TrimSuffix(proto, "://"))
if _, ok := supportedProtos[proto]; !ok {
return nil, errors.Wrap(ErrInvalidProtocol, proto)
}
return url.Parse(remote)
}
if sshutil.IsImplicitSSHTransport(remote) {
remote, fragment, _ := strings.Cut(remote, "#")
remote, path, _ := strings.Cut(remote, ":")
user, host, _ := strings.Cut(remote, "@")
if !strings.HasPrefix(path, "/") {
path = "/" + path
}
return &url.URL{
Scheme: SSHProtocol,
User: url.User(user),
Host: host,
Path: path,
Fragment: fragment,
}, nil
}
return nil, ErrUnknownProtocol
}
// SplitGitFragments splits a git URL fragment into its respective git
// reference and subdirectory components.
func SplitGitFragment(fragment string) (ref string, subdir string) {
ref, subdir, _ = strings.Cut(fragment, ":")
return ref, subdir
}

View File

@ -65,7 +65,7 @@ func (ps *MultiWriter) Write(id string, v interface{}) error {
Sys: v,
meta: ps.meta,
}
return ps.WriteRawProgress(p)
return ps.writeRawProgress(p)
}
func (ps *MultiWriter) WriteRawProgress(p *Progress) error {

View File

@ -4,6 +4,7 @@ import (
"bytes"
"container/ring"
"context"
"encoding/json"
"fmt"
"io"
"os"
@ -16,49 +17,67 @@ import (
"github.com/moby/buildkit/client"
"github.com/morikuni/aec"
digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors"
"github.com/tonistiigi/units"
"github.com/tonistiigi/vt100"
"golang.org/x/time/rate"
)
type displaySolveStatusOpts struct {
type displayOpts struct {
phase string
textDesc string
consoleDesc string
}
type DisplaySolveStatusOpt func(b *displaySolveStatusOpts)
func newDisplayOpts(opts ...DisplayOpt) *displayOpts {
dsso := &displayOpts{}
for _, opt := range opts {
opt(dsso)
}
return dsso
}
func WithPhase(phase string) DisplaySolveStatusOpt {
return func(b *displaySolveStatusOpts) {
type DisplayOpt func(b *displayOpts)
func WithPhase(phase string) DisplayOpt {
return func(b *displayOpts) {
b.phase = phase
}
}
func WithDesc(text string, console string) DisplaySolveStatusOpt {
return func(b *displaySolveStatusOpts) {
func WithDesc(text string, console string) DisplayOpt {
return func(b *displayOpts) {
b.textDesc = text
b.consoleDesc = console
}
}
func DisplaySolveStatus(ctx context.Context, c console.Console, w io.Writer, ch chan *client.SolveStatus, opts ...DisplaySolveStatusOpt) ([]client.VertexWarning, error) {
modeConsole := c != nil
type Display struct {
disp display
}
dsso := &displaySolveStatusOpts{}
for _, opt := range opts {
opt(dsso)
}
type display interface {
// init initializes the display and opens any resources
// that are required.
init(displayLimiter *rate.Limiter)
disp := &display{c: c, phase: dsso.phase, desc: dsso.consoleDesc}
printer := &textMux{w: w, desc: dsso.textDesc}
// update sends the signal to update the display.
// Some displays will have buffered output and will not
// display changes for every status update.
update(ss *client.SolveStatus)
if disp.phase == "" {
disp.phase = "Building"
}
// refresh updates the display with the latest state.
// This method only does something with displays that
// have buffered output.
refresh()
t := newTrace(w, modeConsole)
// done is invoked when the display will be closed.
// This method should flush any buffers and close any open
// resources that were opened by init.
done()
}
func (d Display) UpdateFrom(ctx context.Context, ch chan *client.SolveStatus) ([]client.VertexWarning, error) {
tickerTimeout := 150 * time.Millisecond
displayTimeout := 100 * time.Millisecond
@ -69,55 +88,228 @@ func DisplaySolveStatus(ctx context.Context, c console.Console, w io.Writer, ch
}
}
var done bool
ticker := time.NewTicker(tickerTimeout)
// implemented as closure because "ticker" can change
defer func() {
ticker.Stop()
}()
displayLimiter := rate.NewLimiter(rate.Every(displayTimeout), 1)
d.disp.init(displayLimiter)
defer d.disp.done()
var height int
width, _ := disp.getSize()
ticker := time.NewTicker(tickerTimeout)
defer ticker.Stop()
var warnings []client.VertexWarning
for {
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-ticker.C:
d.disp.refresh()
case ss, ok := <-ch:
if ok {
t.update(ss, width)
} else {
done = true
if !ok {
return warnings, nil
}
}
if modeConsole {
width, height = disp.getSize()
if done {
disp.print(t.displayInfo(), width, height, true)
t.printErrorLogs(c)
return t.warnings(), nil
} else if displayLimiter.Allow() {
ticker.Stop()
ticker = time.NewTicker(tickerTimeout)
disp.print(t.displayInfo(), width, height, false)
}
} else {
if done || displayLimiter.Allow() {
printer.print(t)
if done {
t.printErrorLogs(w)
return t.warnings(), nil
}
ticker.Stop()
ticker = time.NewTicker(tickerTimeout)
d.disp.update(ss)
for _, w := range ss.Warnings {
warnings = append(warnings, *w)
}
ticker.Reset(tickerTimeout)
}
}
}
type DisplayMode string
const (
// DefaultMode is the default value for the DisplayMode.
// This is effectively the same as AutoMode.
DefaultMode DisplayMode = ""
// AutoMode will choose TtyMode or PlainMode depending on if the output is
// a tty.
AutoMode DisplayMode = "auto"
// QuietMode discards all output.
QuietMode DisplayMode = "quiet"
// TtyMode enforces the output is a tty and will otherwise cause an error if it isn't.
TtyMode DisplayMode = "tty"
// PlainMode is the human-readable plain text output. This mode is not meant to be read
// by machines.
PlainMode DisplayMode = "plain"
// RawJSONMode is the raw JSON text output. It will marshal the various solve status events
// to JSON to be read by an external program.
RawJSONMode DisplayMode = "rawjson"
)
// NewDisplay constructs a Display that outputs to the given io.Writer with the given DisplayMode.
//
// This method will return an error when the DisplayMode is invalid or if TtyMode is used but the io.Writer
// does not refer to a tty. AutoMode will choose TtyMode or PlainMode depending on if the output is a tty or not.
//
// For TtyMode to work, the io.Writer should also implement console.File.
func NewDisplay(out io.Writer, mode DisplayMode, opts ...DisplayOpt) (Display, error) {
switch mode {
case AutoMode, TtyMode, DefaultMode:
if c, err := consoleFromWriter(out); err == nil {
return newConsoleDisplay(c, opts...), nil
} else if mode == "tty" {
return Display{}, errors.Wrap(err, "failed to get console")
}
fallthrough
case PlainMode:
return newPlainDisplay(out, opts...), nil
case RawJSONMode:
return newRawJSONDisplay(out, opts...), nil
case QuietMode:
return newDiscardDisplay(), nil
default:
return Display{}, errors.Errorf("invalid progress mode %s", mode)
}
}
// consoleFromWriter retrieves a console.Console from an io.Writer.
func consoleFromWriter(out io.Writer) (console.Console, error) {
f, ok := out.(console.File)
if !ok {
return nil, errors.New("output is not a file")
}
return console.ConsoleFromFile(f)
}
type discardDisplay struct{}
func newDiscardDisplay() Display {
return Display{disp: &discardDisplay{}}
}
func (d *discardDisplay) init(displayLimiter *rate.Limiter) {}
func (d *discardDisplay) update(ss *client.SolveStatus) {}
func (d *discardDisplay) refresh() {}
func (d *discardDisplay) done() {}
type consoleDisplay struct {
t *trace
disp *ttyDisplay
width, height int
displayLimiter *rate.Limiter
}
// newConsoleDisplay creates a new Display that prints a TTY
// friendly output.
func newConsoleDisplay(c console.Console, opts ...DisplayOpt) Display {
dsso := newDisplayOpts(opts...)
if dsso.phase == "" {
dsso.phase = "Building"
}
return Display{
disp: &consoleDisplay{
t: newTrace(c, true),
disp: &ttyDisplay{c: c, phase: dsso.phase, desc: dsso.consoleDesc},
},
}
}
func (d *consoleDisplay) init(displayLimiter *rate.Limiter) {
d.displayLimiter = displayLimiter
}
func (d *consoleDisplay) update(ss *client.SolveStatus) {
d.width, d.height = d.disp.getSize()
d.t.update(ss, d.width)
if !d.displayLimiter.Allow() {
// Exit early as we are not allowed to update the display.
return
}
d.refresh()
}
func (d *consoleDisplay) refresh() {
d.disp.print(d.t.displayInfo(), d.width, d.height, false)
}
func (d *consoleDisplay) done() {
d.width, d.height = d.disp.getSize()
d.disp.print(d.t.displayInfo(), d.width, d.height, true)
d.t.printErrorLogs(d.t.w)
}
type plainDisplay struct {
t *trace
printer *textMux
displayLimiter *rate.Limiter
}
// newPlainDisplay creates a new Display that outputs the status
// in a human-readable plain-text format.
func newPlainDisplay(w io.Writer, opts ...DisplayOpt) Display {
dsso := newDisplayOpts(opts...)
return Display{
disp: &plainDisplay{
t: newTrace(w, false),
printer: &textMux{
w: w,
desc: dsso.textDesc,
},
},
}
}
func (d *plainDisplay) init(displayLimiter *rate.Limiter) {
d.displayLimiter = displayLimiter
}
func (d *plainDisplay) update(ss *client.SolveStatus) {
if ss != nil {
d.t.update(ss, 80)
if !d.displayLimiter.Allow() {
// Exit early as we are not allowed to update the display.
return
}
}
d.refresh()
}
func (d *plainDisplay) refresh() {
d.printer.print(d.t)
}
func (d *plainDisplay) done() {
// Force the display to refresh.
d.refresh()
// Print error logs.
d.t.printErrorLogs(d.t.w)
}
type rawJSONDisplay struct {
enc *json.Encoder
w io.Writer
}
// newRawJSONDisplay creates a new Display that outputs an unbuffered
// output of status update events.
func newRawJSONDisplay(w io.Writer, opts ...DisplayOpt) Display {
enc := json.NewEncoder(w)
enc.SetIndent("", " ")
return Display{
disp: &rawJSONDisplay{
enc: enc,
w: w,
},
}
}
func (d *rawJSONDisplay) init(displayLimiter *rate.Limiter) {
// Initialization parameters are ignored for this display.
}
func (d *rawJSONDisplay) update(ss *client.SolveStatus) {
_ = d.enc.Encode(ss)
}
func (d *rawJSONDisplay) refresh() {
// Unbuffered display doesn't have anything to refresh.
}
func (d *rawJSONDisplay) done() {
// No actions needed.
}
const termHeight = 6
const termPad = 10
@ -386,14 +578,6 @@ func newTrace(w io.Writer, modeConsole bool) *trace {
}
}
func (t *trace) warnings() []client.VertexWarning {
var out []client.VertexWarning
for _, v := range t.vertexes {
out = append(out, v.warnings...)
}
return out
}
func (t *trace) triggerVertexEvent(v *client.Vertex) {
if v.Started == nil {
return
@ -734,7 +918,7 @@ func addTime(tm *time.Time, d time.Duration) *time.Time {
return &t
}
type display struct {
type ttyDisplay struct {
c console.Console
phase string
desc string
@ -742,7 +926,7 @@ type display struct {
repeated bool
}
func (disp *display) getSize() (int, int) {
func (disp *ttyDisplay) getSize() (int, int) {
width := 80
height := 10
if disp.c != nil {
@ -789,7 +973,7 @@ func setupTerminals(jobs []*job, height int, all bool) []*job {
return jobs
}
func (disp *display) print(d displayInfo, width, height int, all bool) {
func (disp *ttyDisplay) print(d displayInfo, width, height int, all bool) {
// this output is inspired by Buck
d.jobs = setupTerminals(d.jobs, height, all)
b := aec.EmptyBuilder

View File

@ -7,7 +7,6 @@ import (
"github.com/containerd/console"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/util/progress/progressui"
"github.com/pkg/errors"
)
type printer struct {
@ -70,24 +69,14 @@ func NewPrinter(ctx context.Context, out console.File, mode string) (Writer, err
mode = v
}
var c console.Console
switch mode {
case "auto", "tty", "":
if cons, err := console.ConsoleFromFile(out); err == nil {
c = cons
} else {
if mode == "tty" {
return nil, errors.Wrap(err, "failed to get console")
}
}
case "plain":
default:
return nil, errors.Errorf("invalid progress mode %s", mode)
d, err := progressui.NewDisplay(out, progressui.DisplayMode(mode))
if err != nil {
return nil, err
}
go func() {
// not using shared context to not disrupt display but let is finish reporting errors
_, pw.err = progressui.DisplaySolveStatus(ctx, c, out, statusCh)
_, pw.err = d.UpdateFrom(ctx, statusCh)
close(doneCh)
}()
return pw, nil

View File

@ -10,7 +10,7 @@ import (
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/remotes"
"github.com/docker/distribution/reference"
"github.com/distribution/reference"
"github.com/moby/buildkit/util/bklog"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
"golang.org/x/sync/semaphore"

View File

@ -3,15 +3,15 @@ syntax = "proto3";
package stack;
message Stack {
repeated Frame frames = 1;
repeated string cmdline = 2;
int32 pid = 3;
string version = 4;
string revision = 5;
repeated Frame frames = 1;
repeated string cmdline = 2;
int32 pid = 3;
string version = 4;
string revision = 5;
}
message Frame {
string Name = 1;
string File = 2;
int32 Line = 3;
string Name = 1;
string File = 2;
int32 Line = 3;
}

View File

@ -57,8 +57,8 @@ type Sandbox interface {
// BackendConfig is used to configure backends created by a worker.
type BackendConfig struct {
Logs map[string]*bytes.Buffer
ConfigFile string
Logs map[string]*bytes.Buffer
DaemonConfig []ConfigUpdater
}
type Worker interface {
@ -303,7 +303,7 @@ mirrors=["%s"]
`, in, mc)
}
func writeConfig(updaters []ConfigUpdater) (string, error) {
func WriteConfig(updaters []ConfigUpdater) (string, error) {
tmpdir, err := os.MkdirTemp("", "bktest_config")
if err != nil {
return "", err
@ -320,7 +320,7 @@ func writeConfig(updaters []ConfigUpdater) (string, error) {
if err := os.WriteFile(filepath.Join(tmpdir, buildkitdConfigFile), []byte(s), 0644); err != nil {
return "", err
}
return tmpdir, nil
return filepath.Join(tmpdir, buildkitdConfigFile), nil
}
func runMirror(t *testing.T, mirroredImages map[string]string) (host string, _ func() error, err error) {

View File

@ -7,12 +7,12 @@ import (
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"github.com/google/shlex"
"github.com/moby/buildkit/util/bklog"
"github.com/pkg/errors"
)
const buildkitdConfigFile = "buildkitd.toml"
@ -77,15 +77,14 @@ func newSandbox(ctx context.Context, w Worker, mirror string, mv matrixValue) (s
Logs: make(map[string]*bytes.Buffer),
}
var upt []ConfigUpdater
for _, v := range mv.values {
if u, ok := v.value.(ConfigUpdater); ok {
upt = append(upt, u)
cfg.DaemonConfig = append(cfg.DaemonConfig, u)
}
}
if mirror != "" {
upt = append(upt, withMirrorConfig(mirror))
cfg.DaemonConfig = append(cfg.DaemonConfig, withMirrorConfig(mirror))
}
deferF := &MultiCloser{}
@ -98,17 +97,6 @@ func newSandbox(ctx context.Context, w Worker, mirror string, mv matrixValue) (s
}
}()
if len(upt) > 0 {
dir, err := writeConfig(upt)
if err != nil {
return nil, nil, err
}
deferF.Append(func() error {
return os.RemoveAll(dir)
})
cfg.ConfigFile = filepath.Join(dir, buildkitdConfigFile)
}
b, closer, err := w.New(ctx, cfg)
if err != nil {
return nil, nil, err
@ -126,7 +114,7 @@ func newSandbox(ctx context.Context, w Worker, mirror string, mv matrixValue) (s
}
func RootlessSupported(uid int) bool {
cmd := exec.Command("sudo", "-E", "-u", fmt.Sprintf("#%d", uid), "-i", "--", "exec", "unshare", "-U", "true") //nolint:gosec // test utility
cmd := exec.Command("sudo", "-u", fmt.Sprintf("#%d", uid), "-i", "--", "exec", "unshare", "-U", "true") //nolint:gosec // test utility
b, err := cmd.CombinedOutput()
if err != nil {
bklog.L.Warnf("rootless mode is not supported on this host: %v (%s)", err, string(b))
@ -156,6 +144,13 @@ func FormatLogs(m map[string]*bytes.Buffer) string {
}
func CheckFeatureCompat(t *testing.T, sb Sandbox, features map[string]struct{}, reason ...string) {
t.Helper()
if err := HasFeatureCompat(t, sb, features, reason...); err != nil {
t.Skipf(err.Error())
}
}
func HasFeatureCompat(t *testing.T, sb Sandbox, features map[string]struct{}, reason ...string) error {
t.Helper()
if len(reason) == 0 {
t.Fatal("no reason provided")
@ -172,6 +167,7 @@ func CheckFeatureCompat(t *testing.T, sb Sandbox, features map[string]struct{},
}
}
if len(ereasons) > 0 {
t.Skipf("%s worker can not currently run this test due to missing features (%s)", sb.Name(), strings.Join(ereasons, ", "))
return errors.Errorf("%s worker can not currently run this test due to missing features (%s)", sb.Name(), strings.Join(ereasons, ", "))
}
return nil
}

View File

@ -168,7 +168,7 @@ disabled_plugins = ["cri"]
containerdArgs := []string{c.Containerd, "--config", configFile}
rootlessKitState := filepath.Join(tmpdir, "rootlesskit-containerd")
if rootless {
containerdArgs = append(append([]string{"sudo", "-E", "-u", fmt.Sprintf("#%d", c.UID), "-i",
containerdArgs = append(append([]string{"sudo", "-u", fmt.Sprintf("#%d", c.UID), "-i",
fmt.Sprintf("CONTAINERD_ROOTLESS_ROOTLESSKIT_STATE_DIR=%s", rootlessKitState),
// Integration test requires the access to localhost of the host network namespace.
// TODO: remove these configurations
@ -211,7 +211,7 @@ disabled_plugins = ["cri"]
if err != nil {
return nil, nil, err
}
buildkitdArgs = append([]string{"sudo", "-E", "-u", fmt.Sprintf("#%d", c.UID), "-i", "--", "exec",
buildkitdArgs = append([]string{"sudo", "-u", fmt.Sprintf("#%d", c.UID), "-i", "--", "exec",
"nsenter", "-U", "--preserve-credentials", "-m", "-t", fmt.Sprintf("%d", pid)},
append(buildkitdArgs, "--containerd-worker-snapshotter=native")...)
}

View File

@ -76,9 +76,27 @@ func (c Moby) New(ctx context.Context, cfg *integration.BackendConfig) (b integr
return nil, nil, err
}
bkcfg, err := config.LoadFile(cfg.ConfigFile)
deferF := &integration.MultiCloser{}
cl = deferF.F()
defer func() {
if err != nil {
deferF.F()()
cl = nil
}
}()
cfgFile, err := integration.WriteConfig(cfg.DaemonConfig)
if err != nil {
return nil, nil, errors.Wrapf(err, "failed to load buildkit config file %s", cfg.ConfigFile)
return nil, nil, err
}
deferF.Append(func() error {
return os.RemoveAll(filepath.Dir(cfgFile))
})
bkcfg, err := config.LoadFile(cfgFile)
if err != nil {
return nil, nil, errors.Wrapf(err, "failed to load buildkit config file %s", cfgFile)
}
dcfg := dockerd.Config{
@ -107,16 +125,6 @@ func (c Moby) New(ctx context.Context, cfg *integration.BackendConfig) (b integr
return nil, nil, errors.Wrapf(err, "failed to marshal dockerd config")
}
deferF := &integration.MultiCloser{}
cl = deferF.F()
defer func() {
if err != nil {
deferF.F()()
cl = nil
}
}()
var proxyGroup errgroup.Group
deferF.Append(proxyGroup.Wait)

View File

@ -61,3 +61,7 @@ var features = map[string]struct{}{
func CheckFeatureCompat(t *testing.T, sb integration.Sandbox, reason ...string) {
integration.CheckFeatureCompat(t, sb, features, reason...)
}
func HasFeatureCompat(t *testing.T, sb integration.Sandbox, reason ...string) error {
return integration.HasFeatureCompat(t, sb, features, reason...)
}

View File

@ -66,7 +66,7 @@ func (s *OCI) New(ctx context.Context, cfg *integration.BackendConfig) (integrat
return nil, nil, errors.Errorf("unsupported id pair: uid=%d, gid=%d", s.UID, s.GID)
}
// TODO: make sure the user exists and subuid/subgid are configured.
buildkitdArgs = append([]string{"sudo", "-E", "-u", fmt.Sprintf("#%d", s.UID), "-i", "--", "exec", "rootlesskit"}, buildkitdArgs...)
buildkitdArgs = append([]string{"sudo", "-u", fmt.Sprintf("#%d", s.UID), "-i", "--", "exec", "rootlesskit"}, buildkitdArgs...)
}
var extraEnv []string

View File

@ -4,6 +4,7 @@ import (
"bufio"
"bytes"
"context"
"fmt"
"os"
"os/exec"
"path/filepath"
@ -32,10 +33,6 @@ func runBuildkitd(ctx context.Context, conf *integration.BackendConfig, args []s
}
}()
if conf.ConfigFile != "" {
args = append(args, "--config="+conf.ConfigFile)
}
tmpdir, err := os.MkdirTemp("", "bktest_buildkitd")
if err != nil {
return "", nil, err
@ -49,14 +46,22 @@ func runBuildkitd(ctx context.Context, conf *integration.BackendConfig, args []s
if err := os.Chown(filepath.Join(tmpdir, "tmp"), uid, gid); err != nil {
return "", nil, err
}
deferF.Append(func() error { return os.RemoveAll(tmpdir) })
cfgfile, err := integration.WriteConfig(append(conf.DaemonConfig, withOTELSocketPath(getTraceSocketPath(tmpdir))))
if err != nil {
return "", nil, err
}
deferF.Append(func() error {
return os.RemoveAll(filepath.Dir(cfgfile))
})
args = append(args, "--config="+cfgfile)
address = getBuildkitdAddr(tmpdir)
args = append(args, "--root", tmpdir, "--addr", address, "--debug")
cmd := exec.Command(args[0], args[1:]...) //nolint:gosec // test utility
cmd.Env = append(os.Environ(), "BUILDKIT_DEBUG_EXEC_OUTPUT=1", "BUILDKIT_DEBUG_PANIC_ON_ERROR=1", "BUILDKIT_TRACE_SOCKET="+getTraceSocketPath(tmpdir), "TMPDIR="+filepath.Join(tmpdir, "tmp"))
cmd.Env = append(os.Environ(), "BUILDKIT_DEBUG_EXEC_OUTPUT=1", "BUILDKIT_DEBUG_PANIC_ON_ERROR=1", "TMPDIR="+filepath.Join(tmpdir, "tmp"))
cmd.Env = append(cmd.Env, extraEnv...)
cmd.SysProcAttr = getSysProcAttr()
@ -87,3 +92,17 @@ func runBuildkitd(ctx context.Context, conf *integration.BackendConfig, args []s
return address, cl, err
}
func withOTELSocketPath(socketPath string) integration.ConfigUpdater {
return otelSocketPath(socketPath)
}
type otelSocketPath string
func (osp otelSocketPath) UpdateConfigFile(in string) string {
return fmt.Sprintf(`%s
[otel]
socketPath = %q
`, in, osp)
}