vendor: github.com/moby/buildkit 5ae9b23c40a9 (master / v0.13.0-dev)

full diff:

- 36ef4d8c0d...f098008783
- d5c1d785b0...5ae9b23c40

Signed-off-by: Sebastiaan van Stijn <github@gone.nl>
This commit is contained in:
Sebastiaan van Stijn
2023-11-15 15:59:23 +01:00
parent 898a8eeddf
commit c855277d53
23 changed files with 644 additions and 196 deletions

View File

@ -95,14 +95,18 @@ func MarshalConstraints(base, override *Constraints) (*pb.Op, *pb.OpMetadata) {
c.Platform = &defaultPlatform
}
opPlatform := pb.Platform{
OS: c.Platform.OS,
Architecture: c.Platform.Architecture,
Variant: c.Platform.Variant,
OSVersion: c.Platform.OSVersion,
}
if c.Platform.OSFeatures != nil {
opPlatform.OSFeatures = append([]string{}, c.Platform.OSFeatures...)
}
return &pb.Op{
Platform: &pb.Platform{
OS: c.Platform.OS,
Architecture: c.Platform.Architecture,
Variant: c.Platform.Variant,
OSVersion: c.Platform.OSVersion,
OSFeatures: c.Platform.OSFeatures,
},
Platform: &opPlatform,
Constraints: &pb.WorkerConstraints{
Filter: c.WorkerConstraints,
},

View File

@ -258,11 +258,16 @@ func (s State) WithImageConfig(c []byte) (State, error) {
}
s = s.Dir(img.Config.WorkingDir)
if img.Architecture != "" && img.OS != "" {
s = s.Platform(ocispecs.Platform{
plat := ocispecs.Platform{
OS: img.OS,
Architecture: img.Architecture,
Variant: img.Variant,
})
OSVersion: img.OSVersion,
}
if img.OSFeatures != nil {
plat.OSFeatures = append([]string{}, img.OSFeatures...)
}
s = s.Platform(plat)
}
return s, nil
}

View File

@ -35,7 +35,8 @@ import (
type SolveOpt struct {
Exports []ExportEntry
LocalDirs map[string]string
LocalDirs map[string]string // Deprecated: use LocalMounts
LocalMounts map[string]fsutil.FS
OCIStores map[string]content.Store
SharedKey string
Frontend string
@ -90,7 +91,11 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
return nil, errors.New("invalid with def and cb")
}
syncedDirs, err := prepareSyncedDirs(def, opt.LocalDirs)
mounts, err := prepareMounts(&opt)
if err != nil {
return nil, err
}
syncedDirs, err := prepareSyncedFiles(def, mounts)
if err != nil {
return nil, err
}
@ -361,26 +366,23 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
return res, nil
}
func prepareSyncedDirs(def *llb.Definition, localDirs map[string]string) (filesync.StaticDirSource, error) {
for _, d := range localDirs {
fi, err := os.Stat(d)
if err != nil {
return nil, errors.Wrapf(err, "could not find %s", d)
}
if !fi.IsDir() {
return nil, errors.Errorf("%s not a directory", d)
}
}
func prepareSyncedFiles(def *llb.Definition, localMounts map[string]fsutil.FS) (filesync.StaticDirSource, error) {
resetUIDAndGID := func(p string, st *fstypes.Stat) fsutil.MapResult {
st.Uid = 0
st.Gid = 0
return fsutil.MapResultKeep
}
dirs := make(filesync.StaticDirSource, len(localDirs))
result := make(filesync.StaticDirSource, len(localMounts))
if def == nil {
for name, d := range localDirs {
dirs[name] = filesync.SyncedDir{Dir: d, Map: resetUIDAndGID}
for name, mount := range localMounts {
mount, err := fsutil.NewFilterFS(mount, &fsutil.FilterOpt{
Map: resetUIDAndGID,
})
if err != nil {
return nil, err
}
result[name] = mount
}
} else {
for _, dt := range def.Def {
@ -391,16 +393,22 @@ func prepareSyncedDirs(def *llb.Definition, localDirs map[string]string) (filesy
if src := op.GetSource(); src != nil {
if strings.HasPrefix(src.Identifier, "local://") {
name := strings.TrimPrefix(src.Identifier, "local://")
d, ok := localDirs[name]
mount, ok := localMounts[name]
if !ok {
return nil, errors.Errorf("local directory %s not enabled", name)
}
dirs[name] = filesync.SyncedDir{Dir: d, Map: resetUIDAndGID}
mount, err := fsutil.NewFilterFS(mount, &fsutil.FilterOpt{
Map: resetUIDAndGID,
})
if err != nil {
return nil, err
}
result[name] = mount
}
}
}
}
return dirs, nil
return result, nil
}
func defaultSessionName() string {
@ -523,3 +531,22 @@ func parseCacheOptions(ctx context.Context, isGateway bool, opt SolveOpt) (*cach
}
return &res, nil
}
func prepareMounts(opt *SolveOpt) (map[string]fsutil.FS, error) {
// merge local mounts and fallback local directories together
mounts := make(map[string]fsutil.FS)
for k, mount := range opt.LocalMounts {
mounts[k] = mount
}
for k, dir := range opt.LocalDirs {
mount, err := fsutil.NewFS(dir)
if err != nil {
return nil, err
}
if _, ok := mounts[k]; ok {
return nil, errors.Errorf("local mount %s already exists", k)
}
mounts[k] = mount
}
return mounts, nil
}

View File

@ -57,7 +57,7 @@ func (bc *Client) Build(ctx context.Context, fn BuildFunc) (*ResultBuilder, erro
p.OSVersion = img.OSVersion
}
if p.OSFeatures == nil && len(img.OSFeatures) > 0 {
p.OSFeatures = img.OSFeatures
p.OSFeatures = append([]string{}, img.OSFeatures...)
}
}

View File

@ -47,6 +47,22 @@ type streamWriterCloser struct {
}
func (wc *streamWriterCloser) Write(dt []byte) (int, error) {
// grpc-go has a 4MB limit on messages by default. Split large messages
// so we don't get close to that limit.
const maxChunkSize = 3 * 1024 * 1024
if len(dt) > maxChunkSize {
n1, err := wc.Write(dt[:maxChunkSize])
if err != nil {
return n1, err
}
dt = dt[maxChunkSize:]
var n2 int
if n2, err = wc.Write(dt); err != nil {
return n1 + n2, err
}
return n1 + n2, nil
}
if err := wc.ClientStream.SendMsg(&BytesMessage{Data: dt}); err != nil {
// SendMsg return EOF on remote errors
if errors.Is(err, io.EOF) {

View File

@ -35,20 +35,15 @@ type fsSyncProvider struct {
doneCh chan error
}
type SyncedDir struct {
Dir string
Map func(string, *fstypes.Stat) fsutil.MapResult
}
type DirSource interface {
LookupDir(string) (SyncedDir, bool)
LookupDir(string) (fsutil.FS, bool)
}
type StaticDirSource map[string]SyncedDir
type StaticDirSource map[string]fsutil.FS
var _ DirSource = StaticDirSource{}
func (dirs StaticDirSource) LookupDir(name string) (SyncedDir, bool) {
func (dirs StaticDirSource) LookupDir(name string) (fsutil.FS, bool) {
dir, found := dirs[name]
return dir, found
}
@ -92,15 +87,22 @@ func (sp *fsSyncProvider) handle(method string, stream grpc.ServerStream) (retEr
dirName = name[0]
}
excludes := opts[keyExcludePatterns]
includes := opts[keyIncludePatterns]
followPaths := opts[keyFollowPaths]
dir, ok := sp.dirs.LookupDir(dirName)
if !ok {
return InvalidSessionError{status.Errorf(codes.NotFound, "no access allowed to dir %q", dirName)}
}
excludes := opts[keyExcludePatterns]
includes := opts[keyIncludePatterns]
followPaths := opts[keyFollowPaths]
dir, err := fsutil.NewFilterFS(dir, &fsutil.FilterOpt{
ExcludePatterns: excludes,
IncludePatterns: includes,
FollowPaths: followPaths,
})
if err != nil {
return err
}
var progress progressCb
if sp.p != nil {
@ -113,12 +115,7 @@ func (sp *fsSyncProvider) handle(method string, stream grpc.ServerStream) (retEr
doneCh = sp.doneCh
sp.doneCh = nil
}
err := pr.sendFn(stream, fsutil.NewFS(dir.Dir, &fsutil.WalkOpt{
ExcludePatterns: excludes,
IncludePatterns: includes,
FollowPaths: followPaths,
Map: dir.Map,
}), progress)
err = pr.sendFn(stream, dir, progress)
if doneCh != nil {
if err != nil {
doneCh <- err

View File

@ -7,6 +7,7 @@ import (
"sync/atomic"
"time"
"github.com/containerd/containerd/defaults"
grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
"github.com/moby/buildkit/util/bklog"
"github.com/moby/buildkit/util/grpcerrors"
@ -44,6 +45,8 @@ func grpcClientConn(ctx context.Context, conn net.Conn) (context.Context, *grpc.
dialOpts := []grpc.DialOption{
dialer,
grpc.WithTransportCredentials(insecure.NewCredentials()),
grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(defaults.DefaultMaxRecvMsgSize)),
grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(defaults.DefaultMaxSendMsgSize)),
}
if span := trace.SpanFromContext(ctx); span.SpanContext().IsValid() {

View File

@ -59,6 +59,20 @@ type writer struct {
}
func (w *writer) Write(dt []byte) (int, error) {
// avoid sending too big messages on grpc stream
const maxChunkSize = 3 * 1024 * 1024
if len(dt) > maxChunkSize {
n1, err := w.Write(dt[:maxChunkSize])
if err != nil {
return n1, err
}
dt = dt[maxChunkSize:]
var n2 int
if n2, err := w.Write(dt); err != nil {
return n1 + n2, err
}
return n1 + n2, nil
}
if err := w.SendMsg(&upload.BytesMessage{Data: dt}); err != nil {
return 0, err
}

96
vendor/github.com/moby/buildkit/solver/pb/json.go generated vendored Normal file
View File

@ -0,0 +1,96 @@
package pb
import "encoding/json"
func (m *Op) UnmarshalJSON(data []byte) error {
var v struct {
Inputs []*Input `json:"inputs,omitempty"`
Op struct {
*Op_Exec
*Op_Source
*Op_File
*Op_Build
*Op_Merge
*Op_Diff
}
Platform *Platform `json:"platform,omitempty"`
Constraints *WorkerConstraints `json:"constraints,omitempty"`
}
if err := json.Unmarshal(data, &v); err != nil {
return err
}
m.Inputs = v.Inputs
switch {
case v.Op.Op_Exec != nil:
m.Op = v.Op.Op_Exec
case v.Op.Op_Source != nil:
m.Op = v.Op.Op_Source
case v.Op.Op_File != nil:
m.Op = v.Op.Op_File
case v.Op.Op_Build != nil:
m.Op = v.Op.Op_Build
case v.Op.Op_Merge != nil:
m.Op = v.Op.Op_Merge
case v.Op.Op_Diff != nil:
m.Op = v.Op.Op_Diff
}
m.Platform = v.Platform
m.Constraints = v.Constraints
return nil
}
func (m *FileAction) UnmarshalJSON(data []byte) error {
var v struct {
Input InputIndex `json:"input"`
SecondaryInput InputIndex `json:"secondaryInput"`
Output OutputIndex `json:"output"`
Action struct {
*FileAction_Copy
*FileAction_Mkfile
*FileAction_Mkdir
*FileAction_Rm
}
}
if err := json.Unmarshal(data, &v); err != nil {
return err
}
m.Input = v.Input
m.SecondaryInput = v.SecondaryInput
m.Output = v.Output
switch {
case v.Action.FileAction_Copy != nil:
m.Action = v.Action.FileAction_Copy
case v.Action.FileAction_Mkfile != nil:
m.Action = v.Action.FileAction_Mkfile
case v.Action.FileAction_Mkdir != nil:
m.Action = v.Action.FileAction_Mkdir
case v.Action.FileAction_Rm != nil:
m.Action = v.Action.FileAction_Rm
}
return nil
}
func (m *UserOpt) UnmarshalJSON(data []byte) error {
var v struct {
User struct {
*UserOpt_ByName
*UserOpt_ByID
}
}
if err := json.Unmarshal(data, &v); err != nil {
return err
}
switch {
case v.User.UserOpt_ByName != nil:
m.User = v.User.UserOpt_ByName
case v.User.UserOpt_ByID != nil:
m.User = v.User.UserOpt_ByID
}
return nil
}

View File

@ -151,6 +151,7 @@ func (CacheSharingOpt) EnumDescriptor() ([]byte, []int) {
// Op represents a vertex of the LLB DAG.
type Op struct {
// changes to this structure must be represented in json.go.
// inputs is a set of input edges.
Inputs []*Input `protobuf:"bytes,1,rep,name=inputs,proto3" json:"inputs,omitempty"`
// Types that are valid to be assigned to Op:
@ -1961,6 +1962,7 @@ func (m *FileOp) GetActions() []*FileAction {
}
type FileAction struct {
// changes to this structure must be represented in json.go.
Input InputIndex `protobuf:"varint,1,opt,name=input,proto3,customtype=InputIndex" json:"input"`
SecondaryInput InputIndex `protobuf:"varint,2,opt,name=secondaryInput,proto3,customtype=InputIndex" json:"secondaryInput"`
Output OutputIndex `protobuf:"varint,3,opt,name=output,proto3,customtype=OutputIndex" json:"output"`
@ -2482,6 +2484,8 @@ func (m *ChownOpt) GetGroup() *UserOpt {
}
type UserOpt struct {
// changes to this structure must be represented in json.go.
//
// Types that are valid to be assigned to User:
//
// *UserOpt_ByName

View File

@ -10,6 +10,7 @@ option (gogoproto.stable_marshaler_all) = true;
// Op represents a vertex of the LLB DAG.
message Op {
// changes to this structure must be represented in json.go.
// inputs is a set of input edges.
repeated Input inputs = 1;
oneof op {
@ -29,7 +30,7 @@ message Platform {
string Architecture = 1;
string OS = 2;
string Variant = 3;
string OSVersion = 4; // unused
string OSVersion = 4;
repeated string OSFeatures = 5; // unused
}
@ -288,6 +289,7 @@ message FileOp {
}
message FileAction {
// changes to this structure must be represented in json.go.
int64 input = 1 [(gogoproto.customtype) = "InputIndex", (gogoproto.nullable) = false]; // could be real input or target (target index + max input index)
int64 secondaryInput = 2 [(gogoproto.customtype) = "InputIndex", (gogoproto.nullable) = false]; // --//--
int64 output = 3 [(gogoproto.customtype) = "OutputIndex", (gogoproto.nullable) = false];
@ -373,6 +375,7 @@ message ChownOpt {
}
message UserOpt {
// changes to this structure must be represented in json.go.
oneof user {
NamedUserOpt byName = 1;
uint32 byID = 2;

View File

@ -5,23 +5,29 @@ import (
)
func (p *Platform) Spec() ocispecs.Platform {
return ocispecs.Platform{
result := ocispecs.Platform{
OS: p.OS,
Architecture: p.Architecture,
Variant: p.Variant,
OSVersion: p.OSVersion,
OSFeatures: p.OSFeatures,
}
if p.OSFeatures != nil {
result.OSFeatures = append([]string{}, p.OSFeatures...)
}
return result
}
func PlatformFromSpec(p ocispecs.Platform) Platform {
return Platform{
result := Platform{
OS: p.OS,
Architecture: p.Architecture,
Variant: p.Variant,
OSVersion: p.OSVersion,
OSFeatures: p.OSFeatures,
}
if p.OSFeatures != nil {
result.OSFeatures = append([]string{}, p.OSFeatures...)
}
return result
}
func ToSpecPlatforms(p []Platform) []ocispecs.Platform {

View File

@ -18,6 +18,8 @@ type MultiWriter struct {
meta map[string]interface{}
}
var _ rawProgressWriter = &MultiWriter{}
func NewMultiWriter(opts ...WriterOption) *MultiWriter {
mw := &MultiWriter{
writers: map[rawProgressWriter]struct{}{},

View File

@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.30.0
// protoc-gen-go v1.31.0
// protoc v3.11.4
// source: stack.proto

View File

@ -1,16 +1,17 @@
package system
import (
"fmt"
iofs "io/fs"
"syscall"
"time"
"github.com/pkg/errors"
)
func Atime(st iofs.FileInfo) (time.Time, error) {
stSys, ok := st.Sys().(*syscall.Win32FileAttributeData)
if !ok {
return time.Time{}, fmt.Errorf("expected st.Sys() to be *syscall.Win32FileAttributeData, got %T", st.Sys())
return time.Time{}, errors.Errorf("expected st.Sys() to be *syscall.Win32FileAttributeData, got %T", st.Sys())
}
// ref: https://github.com/golang/go/blob/go1.19.2/src/os/types_windows.go#L230
return time.Unix(0, stSys.LastAccessTime.Nanoseconds()), nil

View File

@ -2,25 +2,35 @@ package fsutil
import (
"context"
"io"
gofs "io/fs"
"os"
"path/filepath"
"strings"
"syscall"
"time"
"github.com/moby/patternmatcher"
"github.com/pkg/errors"
"github.com/tonistiigi/fsutil/types"
)
type WalkOpt struct {
type FilterOpt struct {
// IncludePatterns requires that the path matches at least one of the
// specified patterns.
IncludePatterns []string
// ExcludePatterns requires that the path does not match any of the
// specified patterns.
ExcludePatterns []string
// FollowPaths contains symlinks that are resolved into include patterns
// before performing the fs walk
// FollowPaths contains symlinks that are resolved into IncludePatterns
// at the time of the call to NewFilterFS.
FollowPaths []string
Map MapFunc
// Map is called for each path that is included in the result.
// The function can modify the stat info for each element, while the result
// of the function controls both how Walk continues.
Map MapFunc
}
type MapFunc func(string, *types.Stat) MapResult
@ -43,33 +53,41 @@ const (
MapResultSkipDir
)
func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) error {
root, err := filepath.EvalSymlinks(p)
if err != nil {
return errors.WithStack(&os.PathError{Op: "resolve", Path: root, Err: err})
}
rootFI, err := os.Stat(root)
if err != nil {
return errors.WithStack(err)
}
if !rootFI.IsDir() {
return errors.WithStack(&os.PathError{Op: "walk", Path: root, Err: syscall.ENOTDIR})
type filterFS struct {
fs FS
includeMatcher *patternmatcher.PatternMatcher
excludeMatcher *patternmatcher.PatternMatcher
onlyPrefixIncludes bool
onlyPrefixExcludeExceptions bool
mapFn MapFunc
}
// NewFilterFS creates a new FS that filters the given FS using the given
// FilterOpt.
// The returned FS will not contain any paths that do not match the provided
// include and exclude patterns, or that are are exlcluded using the mapping
// function.
//
// The FS is assumed to be a snapshot of the filesystem at the time of the
// call to NewFilterFS. If the underlying filesystem changes, calls to the
// underlying FS may be inconsistent.
func NewFilterFS(fs FS, opt *FilterOpt) (FS, error) {
if opt == nil {
return fs, nil
}
var (
includePatterns []string
includeMatcher *patternmatcher.PatternMatcher
excludeMatcher *patternmatcher.PatternMatcher
)
if opt != nil && opt.IncludePatterns != nil {
var includePatterns []string
if opt.IncludePatterns != nil {
includePatterns = make([]string, len(opt.IncludePatterns))
copy(includePatterns, opt.IncludePatterns)
}
if opt != nil && opt.FollowPaths != nil {
targets, err := FollowLinks(p, opt.FollowPaths)
if opt.FollowPaths != nil {
targets, err := FollowLinks(fs, opt.FollowPaths)
if err != nil {
return err
return nil, err
}
if targets != nil {
includePatterns = append(includePatterns, targets...)
@ -82,11 +100,18 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
patternChars += `\`
}
onlyPrefixIncludes := true
if len(includePatterns) != 0 {
var (
includeMatcher *patternmatcher.PatternMatcher
excludeMatcher *patternmatcher.PatternMatcher
err error
onlyPrefixIncludes = true
onlyPrefixExcludeExceptions = true
)
if len(includePatterns) > 0 {
includeMatcher, err = patternmatcher.New(includePatterns)
if err != nil {
return errors.Wrapf(err, "invalid includepatterns: %s", opt.IncludePatterns)
return nil, errors.Wrapf(err, "invalid includepatterns: %s", opt.IncludePatterns)
}
for _, p := range includeMatcher.Patterns() {
@ -98,11 +123,10 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
}
onlyPrefixExcludeExceptions := true
if opt != nil && opt.ExcludePatterns != nil {
if len(opt.ExcludePatterns) > 0 {
excludeMatcher, err = patternmatcher.New(opt.ExcludePatterns)
if err != nil {
return errors.Wrapf(err, "invalid excludepatterns: %s", opt.ExcludePatterns)
return nil, errors.Wrapf(err, "invalid excludepatterns: %s", opt.ExcludePatterns)
}
for _, p := range excludeMatcher.Patterns() {
@ -113,10 +137,41 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
}
}
return &filterFS{
fs: fs,
includeMatcher: includeMatcher,
excludeMatcher: excludeMatcher,
onlyPrefixIncludes: onlyPrefixIncludes,
onlyPrefixExcludeExceptions: onlyPrefixExcludeExceptions,
mapFn: opt.Map,
}, nil
}
func (fs *filterFS) Open(p string) (io.ReadCloser, error) {
if fs.includeMatcher != nil {
m, err := fs.includeMatcher.MatchesOrParentMatches(p)
if err != nil {
return nil, err
}
if !m {
return nil, errors.Wrapf(os.ErrNotExist, "open %s", p)
}
}
if fs.excludeMatcher != nil {
m, err := fs.excludeMatcher.MatchesOrParentMatches(p)
if err != nil {
return nil, err
}
if m {
return nil, errors.Wrapf(os.ErrNotExist, "open %s", p)
}
}
return fs.fs.Open(p)
}
func (fs *filterFS) Walk(ctx context.Context, target string, fn gofs.WalkDirFunc) error {
type visitedDir struct {
fi os.FileInfo
path string
origpath string
entry gofs.DirEntry
pathWithSep string
includeMatchInfo patternmatcher.MatchInfo
excludeMatchInfo patternmatcher.MatchInfo
@ -126,34 +181,22 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
// used only for include/exclude handling
var parentDirs []visitedDir
seenFiles := make(map[uint64]string)
return filepath.WalkDir(root, func(path string, dirEntry gofs.DirEntry, walkErr error) (retErr error) {
return fs.fs.Walk(ctx, target, func(path string, dirEntry gofs.DirEntry, walkErr error) (retErr error) {
defer func() {
if retErr != nil && isNotExist(retErr) {
retErr = filepath.SkipDir
}
}()
origpath := path
path, err = filepath.Rel(root, path)
if err != nil {
return err
}
// Skip root
if path == "." {
return nil
}
var (
dir visitedDir
isDir bool
fi gofs.FileInfo
)
if dirEntry != nil {
isDir = dirEntry.IsDir()
}
if includeMatcher != nil || excludeMatcher != nil {
if fs.includeMatcher != nil || fs.excludeMatcher != nil {
for len(parentDirs) != 0 {
lastParentDir := parentDirs[len(parentDirs)-1].pathWithSep
if strings.HasPrefix(path, lastParentDir) {
@ -163,15 +206,8 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
}
if isDir {
fi, err = dirEntry.Info()
if err != nil {
return err
}
dir = visitedDir{
fi: fi,
path: path,
origpath: origpath,
entry: dirEntry,
pathWithSep: path + string(filepath.Separator),
}
}
@ -179,12 +215,12 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
skip := false
if includeMatcher != nil {
if fs.includeMatcher != nil {
var parentIncludeMatchInfo patternmatcher.MatchInfo
if len(parentDirs) != 0 {
parentIncludeMatchInfo = parentDirs[len(parentDirs)-1].includeMatchInfo
}
m, matchInfo, err := includeMatcher.MatchesUsingParentResults(path, parentIncludeMatchInfo)
m, matchInfo, err := fs.includeMatcher.MatchesUsingParentResults(path, parentIncludeMatchInfo)
if err != nil {
return errors.Wrap(err, "failed to match includepatterns")
}
@ -194,11 +230,11 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
}
if !m {
if isDir && onlyPrefixIncludes {
if isDir && fs.onlyPrefixIncludes {
// Optimization: we can skip walking this dir if no include
// patterns could match anything inside it.
dirSlash := path + string(filepath.Separator)
for _, pat := range includeMatcher.Patterns() {
for _, pat := range fs.includeMatcher.Patterns() {
if pat.Exclusion() {
continue
}
@ -214,12 +250,12 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
}
}
if excludeMatcher != nil {
if fs.excludeMatcher != nil {
var parentExcludeMatchInfo patternmatcher.MatchInfo
if len(parentDirs) != 0 {
parentExcludeMatchInfo = parentDirs[len(parentDirs)-1].excludeMatchInfo
}
m, matchInfo, err := excludeMatcher.MatchesUsingParentResults(path, parentExcludeMatchInfo)
m, matchInfo, err := fs.excludeMatcher.MatchesUsingParentResults(path, parentExcludeMatchInfo)
if err != nil {
return errors.Wrap(err, "failed to match excludepatterns")
}
@ -229,16 +265,16 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
}
if m {
if isDir && onlyPrefixExcludeExceptions {
if isDir && fs.onlyPrefixExcludeExceptions {
// Optimization: we can skip walking this dir if no
// exceptions to exclude patterns could match anything
// inside it.
if !excludeMatcher.Exclusions() {
if !fs.excludeMatcher.Exclusions() {
return filepath.SkipDir
}
dirSlash := path + string(filepath.Separator)
for _, pat := range excludeMatcher.Patterns() {
for _, pat := range fs.excludeMatcher.Patterns() {
if !pat.Exclusion() {
continue
}
@ -261,7 +297,7 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
return walkErr
}
if includeMatcher != nil || excludeMatcher != nil {
if fs.includeMatcher != nil || fs.excludeMatcher != nil {
defer func() {
if isDir {
parentDirs = append(parentDirs, dir)
@ -275,25 +311,21 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
dir.calledFn = true
// The FileInfo might have already been read further up.
if fi == nil {
fi, err = dirEntry.Info()
if err != nil {
return err
}
}
stat, err := mkstat(origpath, path, fi, seenFiles)
fi, err := dirEntry.Info()
if err != nil {
return err
}
stat, ok := fi.Sys().(*types.Stat)
if !ok {
return errors.WithStack(&os.PathError{Path: path, Err: syscall.EBADMSG, Op: "fileinfo without stat info"})
}
select {
case <-ctx.Done():
return ctx.Err()
default:
if opt != nil && opt.Map != nil {
result := opt.Map(stat.Path, stat)
if fs.mapFn != nil {
result := fs.mapFn(stat.Path, stat)
if result == MapResultSkipDir {
return filepath.SkipDir
} else if result == MapResultExclude {
@ -304,29 +336,33 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
if parentDir.calledFn {
continue
}
parentStat, err := mkstat(parentDir.origpath, parentDir.path, parentDir.fi, seenFiles)
parentFi, err := parentDir.entry.Info()
if err != nil {
return err
}
parentStat, ok := parentFi.Sys().(*types.Stat)
if !ok {
return errors.WithStack(&os.PathError{Path: path, Err: syscall.EBADMSG, Op: "fileinfo without stat info"})
}
select {
case <-ctx.Done():
return ctx.Err()
default:
}
if opt != nil && opt.Map != nil {
result := opt.Map(parentStat.Path, parentStat)
if fs.mapFn != nil {
result := fs.mapFn(parentStat.Path, parentStat)
if result == MapResultSkipDir || result == MapResultExclude {
continue
}
}
if err := fn(parentStat.Path, &StatInfo{parentStat}, nil); err != nil {
if err := fn(parentStat.Path, &DirEntryInfo{Stat: parentStat}, nil); err != nil {
return err
}
parentDirs[i].calledFn = true
}
if err := fn(stat.Path, &StatInfo{stat}, nil); err != nil {
if err := fn(stat.Path, &DirEntryInfo{Stat: stat}, nil); err != nil {
return err
}
}
@ -334,6 +370,40 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
})
}
func Walk(ctx context.Context, p string, opt *FilterOpt, fn filepath.WalkFunc) error {
f, err := NewFS(p)
if err != nil {
return err
}
f, err = NewFilterFS(f, opt)
if err != nil {
return err
}
return f.Walk(ctx, "/", func(path string, d gofs.DirEntry, err error) error {
var info gofs.FileInfo
if d != nil {
var err2 error
info, err2 = d.Info()
if err == nil {
err = err2
}
}
return fn(path, info, err)
})
}
func WalkDir(ctx context.Context, p string, opt *FilterOpt, fn gofs.WalkDirFunc) error {
f, err := NewFS(p)
if err != nil {
return err
}
f, err = NewFilterFS(f, opt)
if err != nil {
return err
}
return f.Walk(ctx, "/", fn)
}
func patternWithoutTrailingGlob(p *patternmatcher.Pattern) string {
patStr := p.String()
// We use filepath.Separator here because patternmatcher.Pattern patterns
@ -344,29 +414,6 @@ func patternWithoutTrailingGlob(p *patternmatcher.Pattern) string {
return patStr
}
type StatInfo struct {
*types.Stat
}
func (s *StatInfo) Name() string {
return filepath.Base(s.Stat.Path)
}
func (s *StatInfo) Size() int64 {
return s.Stat.Size_
}
func (s *StatInfo) Mode() os.FileMode {
return os.FileMode(s.Stat.Mode)
}
func (s *StatInfo) ModTime() time.Time {
return time.Unix(s.Stat.ModTime/1e9, s.Stat.ModTime%1e9)
}
func (s *StatInfo) IsDir() bool {
return s.Mode().IsDir()
}
func (s *StatInfo) Sys() interface{} {
return s.Stat
}
func isNotExist(err error) bool {
return errors.Is(err, os.ErrNotExist) || errors.Is(err, syscall.ENOTDIR)
}

View File

@ -1,17 +1,21 @@
package fsutil
import (
"context"
gofs "io/fs"
"os"
"path/filepath"
"runtime"
"sort"
strings "strings"
"syscall"
"github.com/pkg/errors"
"github.com/tonistiigi/fsutil/types"
)
func FollowLinks(root string, paths []string) ([]string, error) {
r := &symlinkResolver{root: root, resolved: map[string]struct{}{}}
func FollowLinks(fs FS, paths []string) ([]string, error) {
r := &symlinkResolver{fs: fs, resolved: map[string]struct{}{}}
for _, p := range paths {
if err := r.append(p); err != nil {
return nil, err
@ -26,7 +30,7 @@ func FollowLinks(root string, paths []string) ([]string, error) {
}
type symlinkResolver struct {
root string
fs FS
resolved map[string]struct{}
}
@ -76,10 +80,9 @@ func (r *symlinkResolver) append(p string) error {
}
func (r *symlinkResolver) readSymlink(p string, allowWildcard bool) ([]string, error) {
realPath := filepath.Join(r.root, p)
base := filepath.Base(p)
if allowWildcard && containsWildcards(base) {
fis, err := os.ReadDir(filepath.Dir(realPath))
fis, err := readDir(r.fs, filepath.Dir(p))
if err != nil {
if isNotFound(err) {
return nil, nil
@ -99,21 +102,30 @@ func (r *symlinkResolver) readSymlink(p string, allowWildcard bool) ([]string, e
return out, nil
}
fi, err := os.Lstat(realPath)
entry, err := statFile(r.fs, p)
if err != nil {
if isNotFound(err) {
return nil, nil
}
return nil, errors.WithStack(err)
}
if fi.Mode()&os.ModeSymlink == 0 {
if entry == nil {
return nil, nil
}
link, err := os.Readlink(realPath)
if entry.Type()&os.ModeSymlink == 0 {
return nil, nil
}
fi, err := entry.Info()
if err != nil {
return nil, errors.WithStack(err)
}
link = filepath.Clean(link)
stat, ok := fi.Sys().(*types.Stat)
if !ok {
return nil, errors.WithStack(&os.PathError{Path: p, Err: syscall.EBADMSG, Op: "fileinfo without stat info"})
}
link := filepath.Clean(stat.Linkname)
if filepath.IsAbs(link) {
return []string{link}, nil
}
@ -122,6 +134,76 @@ func (r *symlinkResolver) readSymlink(p string, allowWildcard bool) ([]string, e
}, nil
}
func statFile(fs FS, root string) (os.DirEntry, error) {
var out os.DirEntry
root = filepath.Clean(root)
if root == "/" || root == "." {
return nil, nil
}
err := fs.Walk(context.TODO(), root, func(p string, entry os.DirEntry, err error) error {
if err != nil {
return err
}
if p != root {
return errors.Errorf("expected single entry %q but got %q", root, p)
}
out = entry
if entry.IsDir() {
return filepath.SkipDir
}
return nil
})
if err != nil {
return nil, err
}
if out == nil {
return nil, errors.Wrapf(os.ErrNotExist, "readFile %s", root)
}
return out, nil
}
func readDir(fs FS, root string) ([]os.DirEntry, error) {
var out []os.DirEntry
root = filepath.Clean(root)
if root == "/" || root == "." {
root = "."
out = make([]gofs.DirEntry, 0)
}
err := fs.Walk(context.TODO(), root, func(p string, entry os.DirEntry, err error) error {
if err != nil {
return err
}
if p == root {
if !entry.IsDir() {
return errors.WithStack(&os.PathError{Op: "walk", Path: root, Err: syscall.ENOTDIR})
}
out = make([]gofs.DirEntry, 0)
return nil
}
if out == nil {
return errors.Errorf("expected to read parent entry %q before child %q", root, p)
}
out = append(out, entry)
if entry.IsDir() {
return filepath.SkipDir
}
return nil
})
if err != nil {
return nil, err
}
if out == nil && root != "." {
return nil, errors.Wrapf(os.ErrNotExist, "readDir %s", root)
}
return out, nil
}
func containsWildcards(name string) bool {
isWindows := runtime.GOOS == "windows"
for i := 0; i < len(name); i++ {

View File

@ -3,36 +3,86 @@ package fsutil
import (
"context"
"io"
gofs "io/fs"
"os"
"path"
"path/filepath"
"sort"
"strings"
"syscall"
"time"
"github.com/pkg/errors"
"github.com/tonistiigi/fsutil/types"
)
type FS interface {
Walk(context.Context, filepath.WalkFunc) error
Walk(context.Context, string, gofs.WalkDirFunc) error
Open(string) (io.ReadCloser, error)
}
func NewFS(root string, opt *WalkOpt) FS {
// NewFS creates a new FS from a root directory on the host filesystem.
func NewFS(root string) (FS, error) {
root, err := filepath.EvalSymlinks(root)
if err != nil {
return nil, errors.WithStack(&os.PathError{Op: "resolve", Path: root, Err: err})
}
fi, err := os.Stat(root)
if err != nil {
return nil, err
}
if !fi.IsDir() {
return nil, errors.WithStack(&os.PathError{Op: "stat", Path: root, Err: syscall.ENOTDIR})
}
return &fs{
root: root,
opt: opt,
}
}, nil
}
type fs struct {
root string
opt *WalkOpt
}
func (fs *fs) Walk(ctx context.Context, fn filepath.WalkFunc) error {
return Walk(ctx, fs.root, fs.opt, fn)
func (fs *fs) Walk(ctx context.Context, target string, fn gofs.WalkDirFunc) error {
seenFiles := make(map[uint64]string)
return filepath.WalkDir(filepath.Join(fs.root, target), func(path string, dirEntry gofs.DirEntry, walkErr error) (retErr error) {
defer func() {
if retErr != nil && isNotExist(retErr) {
retErr = filepath.SkipDir
}
}()
origpath := path
path, err := filepath.Rel(fs.root, path)
if err != nil {
return err
}
// Skip root
if path == "." {
return nil
}
var entry gofs.DirEntry
if dirEntry != nil {
entry = &DirEntryInfo{
path: path,
origpath: origpath,
entry: dirEntry,
seenFiles: seenFiles,
}
}
select {
case <-ctx.Done():
return ctx.Err()
default:
if err := fn(path, entry, walkErr); err != nil {
return err
}
}
return nil
})
}
func (fs *fs) Open(p string) (io.ReadCloser, error) {
@ -67,16 +117,31 @@ type subDirFS struct {
dirs []Dir
}
func (fs *subDirFS) Walk(ctx context.Context, fn filepath.WalkFunc) error {
func (fs *subDirFS) Walk(ctx context.Context, target string, fn gofs.WalkDirFunc) error {
first, rest, _ := strings.Cut(target, string(filepath.Separator))
for _, d := range fs.dirs {
fi := &StatInfo{Stat: &d.Stat}
if first != "" && first != d.Stat.Path {
continue
}
fi := &StatInfo{&d.Stat}
if !fi.IsDir() {
return errors.WithStack(&os.PathError{Path: d.Stat.Path, Err: syscall.ENOTDIR, Op: "walk subdir"})
}
if err := fn(d.Stat.Path, fi, nil); err != nil {
dStat := d.Stat
if err := fn(d.Stat.Path, &DirEntryInfo{Stat: &dStat}, nil); err != nil {
return err
}
if err := d.FS.Walk(ctx, func(p string, fi os.FileInfo, err error) error {
if err := d.FS.Walk(ctx, rest, func(p string, entry gofs.DirEntry, err error) error {
if err != nil {
return err
}
fi, err := entry.Info()
if err != nil {
return err
}
stat, ok := fi.Sys().(*types.Stat)
if !ok {
return errors.WithStack(&os.PathError{Path: d.Stat.Path, Err: syscall.EBADMSG, Op: "fileinfo without stat info"})
@ -91,7 +156,7 @@ func (fs *subDirFS) Walk(ctx context.Context, fn filepath.WalkFunc) error {
stat.Linkname = path.Join(d.Stat.Path, stat.Linkname)
}
}
return fn(filepath.Join(d.Stat.Path, p), &StatInfo{stat}, nil)
return fn(filepath.Join(d.Stat.Path, p), &DirEntryInfo{Stat: stat}, nil)
}); err != nil {
return err
}
@ -117,3 +182,70 @@ type emptyReader struct {
func (*emptyReader) Read([]byte) (int, error) {
return 0, io.EOF
}
type StatInfo struct {
*types.Stat
}
func (s *StatInfo) Name() string {
return filepath.Base(s.Stat.Path)
}
func (s *StatInfo) Size() int64 {
return s.Stat.Size_
}
func (s *StatInfo) Mode() os.FileMode {
return os.FileMode(s.Stat.Mode)
}
func (s *StatInfo) ModTime() time.Time {
return time.Unix(s.Stat.ModTime/1e9, s.Stat.ModTime%1e9)
}
func (s *StatInfo) IsDir() bool {
return s.Mode().IsDir()
}
func (s *StatInfo) Sys() interface{} {
return s.Stat
}
type DirEntryInfo struct {
*types.Stat
entry gofs.DirEntry
path string
origpath string
seenFiles map[uint64]string
}
func (s *DirEntryInfo) Name() string {
if s.Stat != nil {
return filepath.Base(s.Stat.Path)
}
return s.entry.Name()
}
func (s *DirEntryInfo) IsDir() bool {
if s.Stat != nil {
return s.Stat.IsDir()
}
return s.entry.IsDir()
}
func (s *DirEntryInfo) Type() gofs.FileMode {
if s.Stat != nil {
return gofs.FileMode(s.Stat.Mode)
}
return s.entry.Type()
}
func (s *DirEntryInfo) Info() (gofs.FileInfo, error) {
if s.Stat == nil {
fi, err := s.entry.Info()
if err != nil {
return nil, err
}
stat, err := mkstat(s.origpath, s.path, fi, s.seenFiles)
if err != nil {
return nil, err
}
s.Stat = stat
}
st := *s.Stat
return &StatInfo{&st}, nil
}

View File

@ -144,7 +144,11 @@ func (s *sender) sendFile(h *sendHandle) error {
func (s *sender) walk(ctx context.Context) error {
var i uint32 = 0
err := s.fs.Walk(ctx, func(path string, fi os.FileInfo, err error) error {
err := s.fs.Walk(ctx, "/", func(path string, entry os.DirEntry, err error) error {
if err != nil {
return err
}
fi, err := entry.Info()
if err != nil {
return err
}

View File

@ -15,10 +15,15 @@ import (
func WriteTar(ctx context.Context, fs FS, w io.Writer) error {
tw := tar.NewWriter(w)
err := fs.Walk(ctx, func(path string, fi os.FileInfo, err error) error {
err := fs.Walk(ctx, "/", func(path string, entry os.DirEntry, err error) error {
if err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
fi, err := entry.Info()
if err != nil {
return err
}
stat, ok := fi.Sys().(*types.Stat)
if !ok {
return errors.WithStack(&os.PathError{Path: path, Err: syscall.EBADMSG, Op: "fileinfo without stat info"})