Bump moby/buildkit

Signed-off-by: ulyssessouza <ulyssessouza@gmail.com>

vendor/github.com/containerd/containerd/remotes/docker/authorizer.go

@@ -40,126 +40,278 @@ type dockerAuthorizer struct {
 	credentials func(string) (string, string, error)
 
 	client *http.Client
+	header http.Header
 	mu     sync.Mutex
 
-	auth map[string]string
+	// indexed by host name
+	handlers map[string]*authHandler
 }
 
 // NewAuthorizer creates a Docker authorizer using the provided function to
 // get credentials for the token server or basic auth.
+// Deprecated: Use NewDockerAuthorizer
 func NewAuthorizer(client *http.Client, f func(string) (string, string, error)) Authorizer {
-	if client == nil {
-		client = http.DefaultClient
-	}
-	return &dockerAuthorizer{
-		credentials: f,
-		client:      client,
-		auth:        map[string]string{},
+	return NewDockerAuthorizer(WithAuthClient(client), WithAuthCreds(f))
+}
+
+type authorizerConfig struct {
+	credentials func(string) (string, string, error)
+	client      *http.Client
+	header      http.Header
+}
+
+// AuthorizerOpt configures an authorizer
+type AuthorizerOpt func(*authorizerConfig)
+
+// WithAuthClient provides the HTTP client for the authorizer
+func WithAuthClient(client *http.Client) AuthorizerOpt {
+	return func(opt *authorizerConfig) {
+		opt.client = client
 	}
 }
 
-func (a *dockerAuthorizer) Authorize(ctx context.Context, req *http.Request) error {
-	// TODO: Lookup matching challenge and scope rather than just host
-	if auth := a.getAuth(req.URL.Host); auth != "" {
-		req.Header.Set("Authorization", auth)
+// WithAuthCreds provides a credential function to the authorizer
+func WithAuthCreds(creds func(string) (string, string, error)) AuthorizerOpt {
+	return func(opt *authorizerConfig) {
+		opt.credentials = creds
+	}
+}
+
+// WithAuthHeader provides HTTP headers for authorization
+func WithAuthHeader(hdr http.Header) AuthorizerOpt {
+	return func(opt *authorizerConfig) {
+		opt.header = hdr
+	}
+}
+
+// NewDockerAuthorizer creates an authorizer using Docker's registry
+// authentication spec.
+// See https://docs.docker.com/registry/spec/auth/
+func NewDockerAuthorizer(opts ...AuthorizerOpt) Authorizer {
+	var ao authorizerConfig
+	for _, opt := range opts {
+		opt(&ao)
 	}
 
+	if ao.client == nil {
+		ao.client = http.DefaultClient
+	}
+
+	return &dockerAuthorizer{
+		credentials: ao.credentials,
+		client:      ao.client,
+		header:      ao.header,
+		handlers:    make(map[string]*authHandler),
+	}
+}
+
+// Authorize handles auth request.
+func (a *dockerAuthorizer) Authorize(ctx context.Context, req *http.Request) error {
+	// skip if there is no auth handler
+	ah := a.getAuthHandler(req.URL.Host)
+	if ah == nil {
+		return nil
+	}
+
+	auth, err := ah.authorize(ctx)
+	if err != nil {
+		return err
+	}
+
+	req.Header.Set("Authorization", auth)
 	return nil
 }
 
+func (a *dockerAuthorizer) getAuthHandler(host string) *authHandler {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+
+	return a.handlers[host]
+}
+
 func (a *dockerAuthorizer) AddResponses(ctx context.Context, responses []*http.Response) error {
 	last := responses[len(responses)-1]
 	host := last.Request.URL.Host
+
+	a.mu.Lock()
+	defer a.mu.Unlock()
 	for _, c := range parseAuthHeader(last.Header) {
 		if c.scheme == bearerAuth {
 			if err := invalidAuthorization(c, responses); err != nil {
-				// TODO: Clear token
-				a.setAuth(host, "")
+				delete(a.handlers, host)
 				return err
 			}
 
-			// TODO(dmcg): Store challenge, not token
-			// Move token fetching to authorize
-			return a.setTokenAuth(ctx, host, c.parameters)
+			// reuse existing handler
+			//
+			// assume that one registry will return the common
+			// challenge information, including realm and service.
+			// and the resource scope is only different part
+			// which can be provided by each request.
+			if _, ok := a.handlers[host]; ok {
+				return nil
+			}
+
+			common, err := a.generateTokenOptions(ctx, host, c)
+			if err != nil {
+				return err
+			}
+
+			a.handlers[host] = newAuthHandler(a.client, a.header, c.scheme, common)
+			return nil
 		} else if c.scheme == basicAuth && a.credentials != nil {
-			// TODO: Resolve credentials on authorize
 			username, secret, err := a.credentials(host)
 			if err != nil {
 				return err
 			}
+
 			if username != "" && secret != "" {
-				auth := username + ":" + secret
-				a.setAuth(host, fmt.Sprintf("Basic %s", base64.StdEncoding.EncodeToString([]byte(auth))))
+				common := tokenOptions{
+					username: username,
+					secret:   secret,
+				}
+
+				a.handlers[host] = newAuthHandler(a.client, a.header, c.scheme, common)
 				return nil
 			}
 		}
 	}
-
 	return errors.Wrap(errdefs.ErrNotImplemented, "failed to find supported auth scheme")
 }
 
-func (a *dockerAuthorizer) getAuth(host string) string {
-	a.mu.Lock()
-	defer a.mu.Unlock()
-
-	return a.auth[host]
-}
-
-func (a *dockerAuthorizer) setAuth(host string, auth string) bool {
-	a.mu.Lock()
-	defer a.mu.Unlock()
-
-	changed := a.auth[host] != auth
-	a.auth[host] = auth
-
-	return changed
-}
-
-func (a *dockerAuthorizer) setTokenAuth(ctx context.Context, host string, params map[string]string) error {
-	realm, ok := params["realm"]
+func (a *dockerAuthorizer) generateTokenOptions(ctx context.Context, host string, c challenge) (tokenOptions, error) {
+	realm, ok := c.parameters["realm"]
 	if !ok {
-		return errors.New("no realm specified for token auth challenge")
+		return tokenOptions{}, errors.New("no realm specified for token auth challenge")
 	}
 
 	realmURL, err := url.Parse(realm)
 	if err != nil {
-		return errors.Wrap(err, "invalid token auth challenge realm")
+		return tokenOptions{}, errors.Wrap(err, "invalid token auth challenge realm")
 	}
 
 	to := tokenOptions{
 		realm:   realmURL.String(),
-		service: params["service"],
+		service: c.parameters["service"],
 	}
 
-	to.scopes = getTokenScopes(ctx, params)
-	if len(to.scopes) == 0 {
-		return errors.Errorf("no scope specified for token auth challenge")
+	scope, ok := c.parameters["scope"]
+	if !ok {
+		return tokenOptions{}, errors.Errorf("no scope specified for token auth challenge")
 	}
+	to.scopes = append(to.scopes, scope)
 
 	if a.credentials != nil {
 		to.username, to.secret, err = a.credentials(host)
 		if err != nil {
-			return err
+			return tokenOptions{}, err
 		}
 	}
+	return to, nil
+}
 
-	var token string
+// authResult is used to control limit rate.
+type authResult struct {
+	sync.WaitGroup
+	token string
+	err   error
+}
+
+// authHandler is used to handle auth request per registry server.
+type authHandler struct {
+	sync.Mutex
+
+	header http.Header
+
+	client *http.Client
+
+	// only support basic and bearer schemes
+	scheme authenticationScheme
+
+	// common contains common challenge answer
+	common tokenOptions
+
+	// scopedTokens caches token indexed by scopes, which used in
+	// bearer auth case
+	scopedTokens map[string]*authResult
+}
+
+func newAuthHandler(client *http.Client, hdr http.Header, scheme authenticationScheme, opts tokenOptions) *authHandler {
+	return &authHandler{
+		header:       hdr,
+		client:       client,
+		scheme:       scheme,
+		common:       opts,
+		scopedTokens: map[string]*authResult{},
+	}
+}
+
+func (ah *authHandler) authorize(ctx context.Context) (string, error) {
+	switch ah.scheme {
+	case basicAuth:
+		return ah.doBasicAuth(ctx)
+	case bearerAuth:
+		return ah.doBearerAuth(ctx)
+	default:
+		return "", errors.Wrap(errdefs.ErrNotImplemented, "failed to find supported auth scheme")
+	}
+}
+
+func (ah *authHandler) doBasicAuth(ctx context.Context) (string, error) {
+	username, secret := ah.common.username, ah.common.secret
+
+	if username == "" || secret == "" {
+		return "", fmt.Errorf("failed to handle basic auth because missing username or secret")
+	}
+
+	auth := base64.StdEncoding.EncodeToString([]byte(username + ":" + secret))
+	return fmt.Sprintf("Basic %s", auth), nil
+}
+
+func (ah *authHandler) doBearerAuth(ctx context.Context) (string, error) {
+	// copy common tokenOptions
+	to := ah.common
+
+	to.scopes = getTokenScopes(ctx, to.scopes)
+	if len(to.scopes) == 0 {
+		return "", errors.Errorf("no scope specified for token auth challenge")
+	}
+
+	// Docs: https://docs.docker.com/registry/spec/auth/scope
+	scoped := strings.Join(to.scopes, " ")
+
+	ah.Lock()
+	if r, exist := ah.scopedTokens[scoped]; exist {
+		ah.Unlock()
+		r.Wait()
+		return r.token, r.err
+	}
+
+	// only one fetch token job
+	r := new(authResult)
+	r.Add(1)
+	ah.scopedTokens[scoped] = r
+	ah.Unlock()
+
+	// fetch token for the resource scope
+	var (
+		token string
+		err   error
+	)
 	if to.secret != "" {
-		// Credential information is provided, use oauth POST endpoint
-		token, err = a.fetchTokenWithOAuth(ctx, to)
-		if err != nil {
-			return errors.Wrap(err, "failed to fetch oauth token")
-		}
+		// credential information is provided, use oauth POST endpoint
+		token, err = ah.fetchTokenWithOAuth(ctx, to)
+		err = errors.Wrap(err, "failed to fetch oauth token")
 	} else {
-		// Do request anonymously
-		token, err = a.fetchToken(ctx, to)
-		if err != nil {
-			return errors.Wrap(err, "failed to fetch anonymous token")
-		}
+		// do request anonymously
+		token, err = ah.fetchToken(ctx, to)
+		err = errors.Wrap(err, "failed to fetch anonymous token")
 	}
-	a.setAuth(host, fmt.Sprintf("Bearer %s", token))
+	token = fmt.Sprintf("Bearer %s", token)
 
-	return nil
+	r.token, r.err = token, err
+	r.Done()
+	return r.token, r.err
 }
 
 type tokenOptions struct {
@@ -178,7 +330,7 @@ type postTokenResponse struct {
 	Scope        string    `json:"scope"`
 }
 
-func (a *dockerAuthorizer) fetchTokenWithOAuth(ctx context.Context, to tokenOptions) (string, error) {
+func (ah *authHandler) fetchTokenWithOAuth(ctx context.Context, to tokenOptions) (string, error) {
 	form := url.Values{}
 	form.Set("scope", strings.Join(to.scopes, " "))
 	form.Set("service", to.service)
@@ -194,11 +346,18 @@ func (a *dockerAuthorizer) fetchTokenWithOAuth(ctx context.Context, to tokenOpti
 		form.Set("password", to.secret)
 	}
 
-	resp, err := ctxhttp.Post(
-		ctx, a.client, to.realm,
-		"application/x-www-form-urlencoded; charset=utf-8",
-		strings.NewReader(form.Encode()),
-	)
+	req, err := http.NewRequest("POST", to.realm, strings.NewReader(form.Encode()))
+	if err != nil {
+		return "", err
+	}
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=utf-8")
+	if ah.header != nil {
+		for k, v := range ah.header {
+			req.Header[k] = append(req.Header[k], v...)
+		}
+	}
+
+	resp, err := ctxhttp.Do(ctx, ah.client, req)
 	if err != nil {
 		return "", err
 	}
@@ -208,7 +367,7 @@ func (a *dockerAuthorizer) fetchTokenWithOAuth(ctx context.Context, to tokenOpti
 	// As of September 2017, GCR is known to return 404.
 	// As of February 2018, JFrog Artifactory is known to return 401.
 	if (resp.StatusCode == 405 && to.username != "") || resp.StatusCode == 404 || resp.StatusCode == 401 {
-		return a.fetchToken(ctx, to)
+		return ah.fetchToken(ctx, to)
 	} else if resp.StatusCode < 200 || resp.StatusCode >= 400 {
 		b, _ := ioutil.ReadAll(io.LimitReader(resp.Body, 64000)) // 64KB
 		log.G(ctx).WithFields(logrus.Fields{
@@ -237,13 +396,19 @@ type getTokenResponse struct {
 	RefreshToken string    `json:"refresh_token"`
 }
 
-// getToken fetches a token using a GET request
-func (a *dockerAuthorizer) fetchToken(ctx context.Context, to tokenOptions) (string, error) {
+// fetchToken fetches a token using a GET request
+func (ah *authHandler) fetchToken(ctx context.Context, to tokenOptions) (string, error) {
 	req, err := http.NewRequest("GET", to.realm, nil)
 	if err != nil {
 		return "", err
 	}
 
+	if ah.header != nil {
+		for k, v := range ah.header {
+			req.Header[k] = append(req.Header[k], v...)
+		}
+	}
+
 	reqParams := req.URL.Query()
 
 	if to.service != "" {
@@ -260,7 +425,7 @@ func (a *dockerAuthorizer) fetchToken(ctx context.Context, to tokenOptions) (str
 
 	req.URL.RawQuery = reqParams.Encode()
 
-	resp, err := ctxhttp.Do(ctx, a.client, req)
+	resp, err := ctxhttp.Do(ctx, ah.client, req)
 	if err != nil {
 		return "", err
 	}

vendor/github.com/containerd/containerd/remotes/docker/errcode.go

@@ -0,0 +1,283 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package docker
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+)
+
+// ErrorCoder is the base interface for ErrorCode and Error allowing
+// users of each to just call ErrorCode to get the real ID of each
+type ErrorCoder interface {
+	ErrorCode() ErrorCode
+}
+
+// ErrorCode represents the error type. The errors are serialized via strings
+// and the integer format may change and should *never* be exported.
+type ErrorCode int
+
+var _ error = ErrorCode(0)
+
+// ErrorCode just returns itself
+func (ec ErrorCode) ErrorCode() ErrorCode {
+	return ec
+}
+
+// Error returns the ID/Value
+func (ec ErrorCode) Error() string {
+	// NOTE(stevvooe): Cannot use message here since it may have unpopulated args.
+	return strings.ToLower(strings.Replace(ec.String(), "_", " ", -1))
+}
+
+// Descriptor returns the descriptor for the error code.
+func (ec ErrorCode) Descriptor() ErrorDescriptor {
+	d, ok := errorCodeToDescriptors[ec]
+
+	if !ok {
+		return ErrorCodeUnknown.Descriptor()
+	}
+
+	return d
+}
+
+// String returns the canonical identifier for this error code.
+func (ec ErrorCode) String() string {
+	return ec.Descriptor().Value
+}
+
+// Message returned the human-readable error message for this error code.
+func (ec ErrorCode) Message() string {
+	return ec.Descriptor().Message
+}
+
+// MarshalText encodes the receiver into UTF-8-encoded text and returns the
+// result.
+func (ec ErrorCode) MarshalText() (text []byte, err error) {
+	return []byte(ec.String()), nil
+}
+
+// UnmarshalText decodes the form generated by MarshalText.
+func (ec *ErrorCode) UnmarshalText(text []byte) error {
+	desc, ok := idToDescriptors[string(text)]
+
+	if !ok {
+		desc = ErrorCodeUnknown.Descriptor()
+	}
+
+	*ec = desc.Code
+
+	return nil
+}
+
+// WithMessage creates a new Error struct based on the passed-in info and
+// overrides the Message property.
+func (ec ErrorCode) WithMessage(message string) Error {
+	return Error{
+		Code:    ec,
+		Message: message,
+	}
+}
+
+// WithDetail creates a new Error struct based on the passed-in info and
+// set the Detail property appropriately
+func (ec ErrorCode) WithDetail(detail interface{}) Error {
+	return Error{
+		Code:    ec,
+		Message: ec.Message(),
+	}.WithDetail(detail)
+}
+
+// WithArgs creates a new Error struct and sets the Args slice
+func (ec ErrorCode) WithArgs(args ...interface{}) Error {
+	return Error{
+		Code:    ec,
+		Message: ec.Message(),
+	}.WithArgs(args...)
+}
+
+// Error provides a wrapper around ErrorCode with extra Details provided.
+type Error struct {
+	Code    ErrorCode   `json:"code"`
+	Message string      `json:"message"`
+	Detail  interface{} `json:"detail,omitempty"`
+
+	// TODO(duglin): See if we need an "args" property so we can do the
+	// variable substitution right before showing the message to the user
+}
+
+var _ error = Error{}
+
+// ErrorCode returns the ID/Value of this Error
+func (e Error) ErrorCode() ErrorCode {
+	return e.Code
+}
+
+// Error returns a human readable representation of the error.
+func (e Error) Error() string {
+	return fmt.Sprintf("%s: %s", e.Code.Error(), e.Message)
+}
+
+// WithDetail will return a new Error, based on the current one, but with
+// some Detail info added
+func (e Error) WithDetail(detail interface{}) Error {
+	return Error{
+		Code:    e.Code,
+		Message: e.Message,
+		Detail:  detail,
+	}
+}
+
+// WithArgs uses the passed-in list of interface{} as the substitution
+// variables in the Error's Message string, but returns a new Error
+func (e Error) WithArgs(args ...interface{}) Error {
+	return Error{
+		Code:    e.Code,
+		Message: fmt.Sprintf(e.Code.Message(), args...),
+		Detail:  e.Detail,
+	}
+}
+
+// ErrorDescriptor provides relevant information about a given error code.
+type ErrorDescriptor struct {
+	// Code is the error code that this descriptor describes.
+	Code ErrorCode
+
+	// Value provides a unique, string key, often captilized with
+	// underscores, to identify the error code. This value is used as the
+	// keyed value when serializing api errors.
+	Value string
+
+	// Message is a short, human readable decription of the error condition
+	// included in API responses.
+	Message string
+
+	// Description provides a complete account of the errors purpose, suitable
+	// for use in documentation.
+	Description string
+
+	// HTTPStatusCode provides the http status code that is associated with
+	// this error condition.
+	HTTPStatusCode int
+}
+
+// ParseErrorCode returns the value by the string error code.
+// `ErrorCodeUnknown` will be returned if the error is not known.
+func ParseErrorCode(value string) ErrorCode {
+	ed, ok := idToDescriptors[value]
+	if ok {
+		return ed.Code
+	}
+
+	return ErrorCodeUnknown
+}
+
+// Errors provides the envelope for multiple errors and a few sugar methods
+// for use within the application.
+type Errors []error
+
+var _ error = Errors{}
+
+func (errs Errors) Error() string {
+	switch len(errs) {
+	case 0:
+		return "<nil>"
+	case 1:
+		return errs[0].Error()
+	default:
+		msg := "errors:\n"
+		for _, err := range errs {
+			msg += err.Error() + "\n"
+		}
+		return msg
+	}
+}
+
+// Len returns the current number of errors.
+func (errs Errors) Len() int {
+	return len(errs)
+}
+
+// MarshalJSON converts slice of error, ErrorCode or Error into a
+// slice of Error - then serializes
+func (errs Errors) MarshalJSON() ([]byte, error) {
+	var tmpErrs struct {
+		Errors []Error `json:"errors,omitempty"`
+	}
+
+	for _, daErr := range errs {
+		var err Error
+
+		switch daErr := daErr.(type) {
+		case ErrorCode:
+			err = daErr.WithDetail(nil)
+		case Error:
+			err = daErr
+		default:
+			err = ErrorCodeUnknown.WithDetail(daErr)
+
+		}
+
+		// If the Error struct was setup and they forgot to set the
+		// Message field (meaning its "") then grab it from the ErrCode
+		msg := err.Message
+		if msg == "" {
+			msg = err.Code.Message()
+		}
+
+		tmpErrs.Errors = append(tmpErrs.Errors, Error{
+			Code:    err.Code,
+			Message: msg,
+			Detail:  err.Detail,
+		})
+	}
+
+	return json.Marshal(tmpErrs)
+}
+
+// UnmarshalJSON deserializes []Error and then converts it into slice of
+// Error or ErrorCode
+func (errs *Errors) UnmarshalJSON(data []byte) error {
+	var tmpErrs struct {
+		Errors []Error
+	}
+
+	if err := json.Unmarshal(data, &tmpErrs); err != nil {
+		return err
+	}
+
+	var newErrs Errors
+	for _, daErr := range tmpErrs.Errors {
+		// If Message is empty or exactly matches the Code's message string
+		// then just use the Code, no need for a full Error struct
+		if daErr.Detail == nil && (daErr.Message == "" || daErr.Message == daErr.Code.Message()) {
+			// Error's w/o details get converted to ErrorCode
+			newErrs = append(newErrs, daErr.Code)
+		} else {
+			// Error's w/ details are untouched
+			newErrs = append(newErrs, Error{
+				Code:    daErr.Code,
+				Message: daErr.Message,
+				Detail:  daErr.Detail,
+			})
+		}
+	}
+
+	*errs = newErrs
+	return nil
+}

vendor/github.com/containerd/containerd/remotes/docker/errdesc.go

@@ -0,0 +1,154 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package docker
+
+import (
+	"fmt"
+	"net/http"
+	"sort"
+	"sync"
+)
+
+var (
+	errorCodeToDescriptors = map[ErrorCode]ErrorDescriptor{}
+	idToDescriptors        = map[string]ErrorDescriptor{}
+	groupToDescriptors     = map[string][]ErrorDescriptor{}
+)
+
+var (
+	// ErrorCodeUnknown is a generic error that can be used as a last
+	// resort if there is no situation-specific error message that can be used
+	ErrorCodeUnknown = Register("errcode", ErrorDescriptor{
+		Value:   "UNKNOWN",
+		Message: "unknown error",
+		Description: `Generic error returned when the error does not have an
+			                                            API classification.`,
+		HTTPStatusCode: http.StatusInternalServerError,
+	})
+
+	// ErrorCodeUnsupported is returned when an operation is not supported.
+	ErrorCodeUnsupported = Register("errcode", ErrorDescriptor{
+		Value:   "UNSUPPORTED",
+		Message: "The operation is unsupported.",
+		Description: `The operation was unsupported due to a missing
+		implementation or invalid set of parameters.`,
+		HTTPStatusCode: http.StatusMethodNotAllowed,
+	})
+
+	// ErrorCodeUnauthorized is returned if a request requires
+	// authentication.
+	ErrorCodeUnauthorized = Register("errcode", ErrorDescriptor{
+		Value:   "UNAUTHORIZED",
+		Message: "authentication required",
+		Description: `The access controller was unable to authenticate
+		the client. Often this will be accompanied by a
+		Www-Authenticate HTTP response header indicating how to
+		authenticate.`,
+		HTTPStatusCode: http.StatusUnauthorized,
+	})
+
+	// ErrorCodeDenied is returned if a client does not have sufficient
+	// permission to perform an action.
+	ErrorCodeDenied = Register("errcode", ErrorDescriptor{
+		Value:   "DENIED",
+		Message: "requested access to the resource is denied",
+		Description: `The access controller denied access for the
+		operation on a resource.`,
+		HTTPStatusCode: http.StatusForbidden,
+	})
+
+	// ErrorCodeUnavailable provides a common error to report unavailability
+	// of a service or endpoint.
+	ErrorCodeUnavailable = Register("errcode", ErrorDescriptor{
+		Value:          "UNAVAILABLE",
+		Message:        "service unavailable",
+		Description:    "Returned when a service is not available",
+		HTTPStatusCode: http.StatusServiceUnavailable,
+	})
+
+	// ErrorCodeTooManyRequests is returned if a client attempts too many
+	// times to contact a service endpoint.
+	ErrorCodeTooManyRequests = Register("errcode", ErrorDescriptor{
+		Value:   "TOOMANYREQUESTS",
+		Message: "too many requests",
+		Description: `Returned when a client attempts to contact a
+		service too many times`,
+		HTTPStatusCode: http.StatusTooManyRequests,
+	})
+)
+
+var nextCode = 1000
+var registerLock sync.Mutex
+
+// Register will make the passed-in error known to the environment and
+// return a new ErrorCode
+func Register(group string, descriptor ErrorDescriptor) ErrorCode {
+	registerLock.Lock()
+	defer registerLock.Unlock()
+
+	descriptor.Code = ErrorCode(nextCode)
+
+	if _, ok := idToDescriptors[descriptor.Value]; ok {
+		panic(fmt.Sprintf("ErrorValue %q is already registered", descriptor.Value))
+	}
+	if _, ok := errorCodeToDescriptors[descriptor.Code]; ok {
+		panic(fmt.Sprintf("ErrorCode %v is already registered", descriptor.Code))
+	}
+
+	groupToDescriptors[group] = append(groupToDescriptors[group], descriptor)
+	errorCodeToDescriptors[descriptor.Code] = descriptor
+	idToDescriptors[descriptor.Value] = descriptor
+
+	nextCode++
+	return descriptor.Code
+}
+
+type byValue []ErrorDescriptor
+
+func (a byValue) Len() int           { return len(a) }
+func (a byValue) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a byValue) Less(i, j int) bool { return a[i].Value < a[j].Value }
+
+// GetGroupNames returns the list of Error group names that are registered
+func GetGroupNames() []string {
+	keys := []string{}
+
+	for k := range groupToDescriptors {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	return keys
+}
+
+// GetErrorCodeGroup returns the named group of error descriptors
+func GetErrorCodeGroup(name string) []ErrorDescriptor {
+	desc := groupToDescriptors[name]
+	sort.Sort(byValue(desc))
+	return desc
+}
+
+// GetErrorAllDescriptors returns a slice of all ErrorDescriptors that are
+// registered, irrespective of what group they're in
+func GetErrorAllDescriptors() []ErrorDescriptor {
+	result := []ErrorDescriptor{}
+
+	for _, group := range GetGroupNames() {
+		result = append(result, GetErrorCodeGroup(group)...)
+	}
+	sort.Sort(byValue(result))
+	return result
+}

vendor/github.com/containerd/containerd/remotes/docker/fetcher.go

@@ -23,16 +23,14 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
-	"path"
+	"net/url"
 	"strings"
 
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/images"
 	"github.com/containerd/containerd/log"
-	"github.com/docker/distribution/registry/api/errcode"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
 )
 
 type dockerFetcher struct {
@@ -40,26 +38,46 @@ type dockerFetcher struct {
 }
 
 func (r dockerFetcher) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser, error) {
-	ctx = log.WithLogger(ctx, log.G(ctx).WithFields(
-		logrus.Fields{
-			"base":   r.base.String(),
-			"digest": desc.Digest,
-		},
-	))
+	ctx = log.WithLogger(ctx, log.G(ctx).WithField("digest", desc.Digest))
 
-	urls, err := r.getV2URLPaths(ctx, desc)
-	if err != nil {
-		return nil, err
+	hosts := r.filterHosts(HostCapabilityPull)
+	if len(hosts) == 0 {
+		return nil, errors.Wrap(errdefs.ErrNotFound, "no pull hosts")
 	}
 
-	ctx, err = contextWithRepositoryScope(ctx, r.refspec, false)
+	ctx, err := contextWithRepositoryScope(ctx, r.refspec, false)
 	if err != nil {
 		return nil, err
 	}
 
 	return newHTTPReadSeeker(desc.Size, func(offset int64) (io.ReadCloser, error) {
-		for _, u := range urls {
-			rc, err := r.open(ctx, u, desc.MediaType, offset)
+		// firstly try fetch via external urls
+		for _, us := range desc.URLs {
+			ctx = log.WithLogger(ctx, log.G(ctx).WithField("url", us))
+
+			u, err := url.Parse(us)
+			if err != nil {
+				log.G(ctx).WithError(err).Debug("failed to parse")
+				continue
+			}
+			log.G(ctx).Debug("trying alternative url")
+
+			// Try this first, parse it
+			host := RegistryHost{
+				Client:       http.DefaultClient,
+				Host:         u.Host,
+				Scheme:       u.Scheme,
+				Path:         u.Path,
+				Capabilities: HostCapabilityPull,
+			}
+			req := r.request(host, http.MethodGet)
+			// Strip namespace from base
+			req.path = u.Path
+			if u.RawQuery != "" {
+				req.path = req.path + "?" + u.RawQuery
+			}
+
+			rc, err := r.open(ctx, req, desc.MediaType, offset)
 			if err != nil {
 				if errdefs.IsNotFound(err) {
 					continue // try one of the other urls.
@@ -71,6 +89,44 @@ func (r dockerFetcher) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.R
 			return rc, nil
 		}
 
+		// Try manifests endpoints for manifests types
+		switch desc.MediaType {
+		case images.MediaTypeDockerSchema2Manifest, images.MediaTypeDockerSchema2ManifestList,
+			images.MediaTypeDockerSchema1Manifest,
+			ocispec.MediaTypeImageManifest, ocispec.MediaTypeImageIndex:
+
+			for _, host := range r.hosts {
+				req := r.request(host, http.MethodGet, "manifests", desc.Digest.String())
+
+				rc, err := r.open(ctx, req, desc.MediaType, offset)
+				if err != nil {
+					if errdefs.IsNotFound(err) {
+						continue // try another host
+					}
+
+					return nil, err
+				}
+
+				return rc, nil
+			}
+		}
+
+		// Finally use blobs endpoints
+		for _, host := range r.hosts {
+			req := r.request(host, http.MethodGet, "blobs", desc.Digest.String())
+
+			rc, err := r.open(ctx, req, desc.MediaType, offset)
+			if err != nil {
+				if errdefs.IsNotFound(err) {
+					continue // try another host
+				}
+
+				return nil, err
+			}
+
+			return rc, nil
+		}
+
 		return nil, errors.Wrapf(errdefs.ErrNotFound,
 			"could not fetch content descriptor %v (%v) from remote",
 			desc.Digest, desc.MediaType)
@@ -78,22 +134,17 @@ func (r dockerFetcher) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.R
 	})
 }
 
-func (r dockerFetcher) open(ctx context.Context, u, mediatype string, offset int64) (io.ReadCloser, error) {
-	req, err := http.NewRequest(http.MethodGet, u, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	req.Header.Set("Accept", strings.Join([]string{mediatype, `*`}, ", "))
+func (r dockerFetcher) open(ctx context.Context, req *request, mediatype string, offset int64) (io.ReadCloser, error) {
+	req.header.Set("Accept", strings.Join([]string{mediatype, `*/*`}, ", "))
 
 	if offset > 0 {
 		// Note: "Accept-Ranges: bytes" cannot be trusted as some endpoints
 		// will return the header without supporting the range. The content
 		// range must always be checked.
-		req.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset))
+		req.header.Set("Range", fmt.Sprintf("bytes=%d-", offset))
 	}
 
-	resp, err := r.doRequestWithRetries(ctx, req, nil)
+	resp, err := req.doWithRetries(ctx, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -106,13 +157,13 @@ func (r dockerFetcher) open(ctx context.Context, u, mediatype string, offset int
 		defer resp.Body.Close()
 
 		if resp.StatusCode == http.StatusNotFound {
-			return nil, errors.Wrapf(errdefs.ErrNotFound, "content at %v not found", u)
+			return nil, errors.Wrapf(errdefs.ErrNotFound, "content at %v not found", req.String())
 		}
-		var registryErr errcode.Errors
+		var registryErr Errors
 		if err := json.NewDecoder(resp.Body).Decode(&registryErr); err != nil || registryErr.Len() < 1 {
-			return nil, errors.Errorf("unexpected status code %v: %v", u, resp.Status)
+			return nil, errors.Errorf("unexpected status code %v: %v", req.String(), resp.Status)
 		}
-		return nil, errors.Errorf("unexpected status code %v: %s - Server message: %s", u, resp.Status, registryErr.Error())
+		return nil, errors.Errorf("unexpected status code %v: %s - Server message: %s", req.String(), resp.Status, registryErr.Error())
 	}
 	if offset > 0 {
 		cr := resp.Header.Get("content-range")
@@ -141,30 +192,3 @@ func (r dockerFetcher) open(ctx context.Context, u, mediatype string, offset int
 
 	return resp.Body, nil
 }
-
-// getV2URLPaths generates the candidate urls paths for the object based on the
-// set of hints and the provided object id. URLs are returned in the order of
-// most to least likely succeed.
-func (r *dockerFetcher) getV2URLPaths(ctx context.Context, desc ocispec.Descriptor) ([]string, error) {
-	var urls []string
-
-	if len(desc.URLs) > 0 {
-		// handle fetch via external urls.
-		for _, u := range desc.URLs {
-			log.G(ctx).WithField("url", u).Debug("adding alternative url")
-			urls = append(urls, u)
-		}
-	}
-
-	switch desc.MediaType {
-	case images.MediaTypeDockerSchema2Manifest, images.MediaTypeDockerSchema2ManifestList,
-		images.MediaTypeDockerSchema1Manifest,
-		ocispec.MediaTypeImageManifest, ocispec.MediaTypeImageIndex:
-		urls = append(urls, r.url(path.Join("manifests", desc.Digest.String())))
-	}
-
-	// always fallback to attempting to get the object out of the blobs store.
-	urls = append(urls, r.url(path.Join("blobs", desc.Digest.String())))
-
-	return urls, nil
-}

vendor/github.com/containerd/containerd/remotes/docker/handler.go

@@ -110,3 +110,45 @@ func appendDistributionSourceLabel(originLabel, repo string) string {
 func distributionSourceLabelKey(source string) string {
 	return fmt.Sprintf("%s.%s", labelDistributionSource, source)
 }
+
+// selectRepositoryMountCandidate will select the repo which has longest
+// common prefix components as the candidate.
+func selectRepositoryMountCandidate(refspec reference.Spec, sources map[string]string) string {
+	u, err := url.Parse("dummy://" + refspec.Locator)
+	if err != nil {
+		// NOTE: basically, it won't be error here
+		return ""
+	}
+
+	source, target := u.Hostname(), strings.TrimPrefix(u.Path, "/")
+	repoLabel, ok := sources[distributionSourceLabelKey(source)]
+	if !ok || repoLabel == "" {
+		return ""
+	}
+
+	n, match := 0, ""
+	components := strings.Split(target, "/")
+	for _, repo := range strings.Split(repoLabel, ",") {
+		// the target repo is not a candidate
+		if repo == target {
+			continue
+		}
+
+		if l := commonPrefixComponents(components, repo); l >= n {
+			n, match = l, repo
+		}
+	}
+	return match
+}
+
+func commonPrefixComponents(components []string, target string) int {
+	targetComponents := strings.Split(target, "/")
+
+	i := 0
+	for ; i < len(components) && i < len(targetComponents); i++ {
+		if components[i] != targetComponents[i] {
+			break
+		}
+	}
+	return i
+}

vendor/github.com/containerd/containerd/remotes/docker/pusher.go

@@ -21,7 +21,7 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
-	"path"
+	"net/url"
 	"strings"
 	"time"
 
@@ -37,7 +37,7 @@ import (
 
 type dockerPusher struct {
 	*dockerBase
-	tag string
+	object string
 
 	// TODO: namespace tracker
 	tracker StatusTracker
@@ -59,31 +59,32 @@ func (p dockerPusher) Push(ctx context.Context, desc ocispec.Descriptor) (conten
 		return nil, errors.Wrap(err, "failed to get status")
 	}
 
+	hosts := p.filterHosts(HostCapabilityPush)
+	if len(hosts) == 0 {
+		return nil, errors.Wrap(errdefs.ErrNotFound, "no push hosts")
+	}
+
 	var (
 		isManifest bool
-		existCheck string
+		existCheck []string
+		host       = hosts[0]
 	)
 
 	switch desc.MediaType {
 	case images.MediaTypeDockerSchema2Manifest, images.MediaTypeDockerSchema2ManifestList,
 		ocispec.MediaTypeImageManifest, ocispec.MediaTypeImageIndex:
 		isManifest = true
-		if p.tag == "" {
-			existCheck = path.Join("manifests", desc.Digest.String())
-		} else {
-			existCheck = path.Join("manifests", p.tag)
-		}
+		existCheck = getManifestPath(p.object, desc.Digest)
 	default:
-		existCheck = path.Join("blobs", desc.Digest.String())
+		existCheck = []string{"blobs", desc.Digest.String()}
 	}
 
-	req, err := http.NewRequest(http.MethodHead, p.url(existCheck), nil)
-	if err != nil {
-		return nil, err
-	}
+	req := p.request(host, http.MethodHead, existCheck...)
+	req.header.Set("Accept", strings.Join([]string{desc.MediaType, `*/*`}, ", "))
 
-	req.Header.Set("Accept", strings.Join([]string{desc.MediaType, `*`}, ", "))
-	resp, err := p.doRequestWithRetries(ctx, req, nil)
+	log.G(ctx).WithField("url", req.String()).Debugf("checking and pushing to")
+
+	resp, err := req.doWithRetries(ctx, nil)
 	if err != nil {
 		if errors.Cause(err) != ErrInvalidAuthorization {
 			return nil, err
@@ -92,7 +93,7 @@ func (p dockerPusher) Push(ctx context.Context, desc ocispec.Descriptor) (conten
 	} else {
 		if resp.StatusCode == http.StatusOK {
 			var exists bool
-			if isManifest && p.tag != "" {
+			if isManifest && existCheck[1] != desc.Digest.String() {
 				dgstHeader := digest.Digest(resp.Header.Get("Docker-Content-Digest"))
 				if dgstHeader == desc.Digest {
 					exists = true
@@ -116,67 +117,94 @@ func (p dockerPusher) Push(ctx context.Context, desc ocispec.Descriptor) (conten
 		}
 	}
 
-	// TODO: Lookup related objects for cross repository push
-
 	if isManifest {
-		var putPath string
-		if p.tag != "" {
-			putPath = path.Join("manifests", p.tag)
-		} else {
-			putPath = path.Join("manifests", desc.Digest.String())
-		}
-
-		req, err = http.NewRequest(http.MethodPut, p.url(putPath), nil)
-		if err != nil {
-			return nil, err
-		}
-		req.Header.Add("Content-Type", desc.MediaType)
+		putPath := getManifestPath(p.object, desc.Digest)
+		req = p.request(host, http.MethodPut, putPath...)
+		req.header.Add("Content-Type", desc.MediaType)
 	} else {
-		// TODO: Do monolithic upload if size is small
-
 		// Start upload request
-		req, err = http.NewRequest(http.MethodPost, p.url("blobs", "uploads")+"/", nil)
-		if err != nil {
-			return nil, err
+		req = p.request(host, http.MethodPost, "blobs", "uploads/")
+
+		var resp *http.Response
+		if fromRepo := selectRepositoryMountCandidate(p.refspec, desc.Annotations); fromRepo != "" {
+			preq := requestWithMountFrom(req, desc.Digest.String(), fromRepo)
+			pctx := contextWithAppendPullRepositoryScope(ctx, fromRepo)
+
+			// NOTE: the fromRepo might be private repo and
+			// auth service still can grant token without error.
+			// but the post request will fail because of 401.
+			//
+			// for the private repo, we should remove mount-from
+			// query and send the request again.
+			resp, err = preq.do(pctx)
+			if err != nil {
+				return nil, err
+			}
+
+			if resp.StatusCode == http.StatusUnauthorized {
+				log.G(ctx).Debugf("failed to mount from repository %s", fromRepo)
+
+				resp.Body.Close()
+				resp = nil
+			}
 		}
 
-		resp, err := p.doRequestWithRetries(ctx, req, nil)
-		if err != nil {
-			return nil, err
+		if resp == nil {
+			resp, err = req.doWithRetries(ctx, nil)
+			if err != nil {
+				return nil, err
+			}
 		}
 
 		switch resp.StatusCode {
 		case http.StatusOK, http.StatusAccepted, http.StatusNoContent:
+		case http.StatusCreated:
+			p.tracker.SetStatus(ref, Status{
+				Status: content.Status{
+					Ref: ref,
+				},
+			})
+			return nil, errors.Wrapf(errdefs.ErrAlreadyExists, "content %v on remote", desc.Digest)
 		default:
 			// TODO: log error
 			return nil, errors.Errorf("unexpected response: %s", resp.Status)
 		}
 
-		location := resp.Header.Get("Location")
+		var (
+			location = resp.Header.Get("Location")
+			lurl     *url.URL
+			lhost    = host
+		)
 		// Support paths without host in location
 		if strings.HasPrefix(location, "/") {
-			// Support location string containing path and query
-			qmIndex := strings.Index(location, "?")
-			if qmIndex > 0 {
-				u := p.base
-				u.Path = location[:qmIndex]
-				u.RawQuery = location[qmIndex+1:]
-				location = u.String()
-			} else {
-				u := p.base
-				u.Path = location
-				location = u.String()
+			lurl, err = url.Parse(lhost.Scheme + "://" + lhost.Host + location)
+			if err != nil {
+				return nil, errors.Wrapf(err, "unable to parse location %v", location)
+			}
+		} else {
+			if !strings.Contains(location, "://") {
+				location = lhost.Scheme + "://" + location
+			}
+			lurl, err = url.Parse(location)
+			if err != nil {
+				return nil, errors.Wrapf(err, "unable to parse location %v", location)
+			}
+
+			if lurl.Host != lhost.Host || lhost.Scheme != lurl.Scheme {
+
+				lhost.Scheme = lurl.Scheme
+				lhost.Host = lurl.Host
+				log.G(ctx).WithField("host", lhost.Host).WithField("scheme", lhost.Scheme).Debug("upload changed destination")
+
+				// Strip authorizer if change to host or scheme
+				lhost.Authorizer = nil
 			}
 		}
-
-		req, err = http.NewRequest(http.MethodPut, location, nil)
-		if err != nil {
-			return nil, err
-		}
-		q := req.URL.Query()
+		q := lurl.Query()
 		q.Add("digest", desc.Digest.String())
-		req.URL.RawQuery = q.Encode()
 
+		req = p.request(lhost, http.MethodPut)
+		req.path = lurl.Path + "?" + q.Encode()
 	}
 	p.tracker.SetStatus(ref, Status{
 		Status: content.Status{
@@ -191,13 +219,22 @@ func (p dockerPusher) Push(ctx context.Context, desc ocispec.Descriptor) (conten
 
 	pr, pw := io.Pipe()
 	respC := make(chan *http.Response, 1)
+	body := ioutil.NopCloser(pr)
 
-	req.Body = ioutil.NopCloser(pr)
-	req.ContentLength = desc.Size
+	req.body = func() (io.ReadCloser, error) {
+		if body == nil {
+			return nil, errors.New("cannot reuse body, request must be retried")
+		}
+		// Only use the body once since pipe cannot be seeked
+		ob := body
+		body = nil
+		return ob, nil
+	}
+	req.size = desc.Size
 
 	go func() {
 		defer close(respC)
-		resp, err = p.doRequest(ctx, req)
+		resp, err = req.do(ctx)
 		if err != nil {
 			pr.CloseWithError(err)
 			return
@@ -223,6 +260,25 @@ func (p dockerPusher) Push(ctx context.Context, desc ocispec.Descriptor) (conten
 	}, nil
 }
 
+func getManifestPath(object string, dgst digest.Digest) []string {
+	if i := strings.IndexByte(object, '@'); i >= 0 {
+		if object[i+1:] != dgst.String() {
+			// use digest, not tag
+			object = ""
+		} else {
+			// strip @<digest> for registry path to make tag
+			object = object[:i]
+		}
+
+	}
+
+	if object == "" {
+		return []string{"manifests", dgst.String()}
+	}
+
+	return []string{"manifests", object}
+}
+
 type pushWriter struct {
 	base *dockerBase
 	ref  string
@@ -296,7 +352,7 @@ func (pw *pushWriter) Commit(ctx context.Context, size int64, expected digest.Di
 	}
 
 	if size > 0 && size != status.Offset {
-		return errors.Errorf("unxpected size %d, expected %d", status.Offset, size)
+		return errors.Errorf("unexpected size %d, expected %d", status.Offset, size)
 	}
 
 	if expected == "" {
@@ -320,3 +376,16 @@ func (pw *pushWriter) Truncate(size int64) error {
 	// TODO: always error on manifest
 	return errors.New("cannot truncate remote upload")
 }
+
+func requestWithMountFrom(req *request, mount, from string) *request {
+	creq := *req
+
+	sep := "?"
+	if strings.Contains(creq.path, sep) {
+		sep = "&"
+	}
+
+	creq.path = creq.path + sep + "mount=" + mount + "&from=" + from
+
+	return &creq
+}

vendor/github.com/containerd/containerd/remotes/docker/registry.go

@@ -0,0 +1,202 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package docker
+
+import (
+	"net/http"
+)
+
+// HostCapabilities represent the capabilities of the registry
+// host. This also represents the set of operations for which
+// the registry host may be trusted to perform.
+//
+// For example pushing is a capability which should only be
+// performed on an upstream source, not a mirror.
+// Resolving (the process of converting a name into a digest)
+// must be considered a trusted operation and only done by
+// a host which is trusted (or more preferably by secure process
+// which can prove the provenance of the mapping). A public
+// mirror should never be trusted to do a resolve action.
+//
+// | Registry Type    | Pull | Resolve | Push |
+// |------------------|------|---------|------|
+// | Public Registry  | yes  | yes     | yes  |
+// | Private Registry | yes  | yes     | yes  |
+// | Public Mirror    | yes  | no      | no   |
+// | Private Mirror   | yes  | yes     | no   |
+type HostCapabilities uint8
+
+const (
+	// HostCapabilityPull represents the capability to fetch manifests
+	// and blobs by digest
+	HostCapabilityPull HostCapabilities = 1 << iota
+
+	// HostCapabilityResolve represents the capability to fetch manifests
+	// by name
+	HostCapabilityResolve
+
+	// HostCapabilityPush represents the capability to push blobs and
+	// manifests
+	HostCapabilityPush
+
+	// Reserved for future capabilities (i.e. search, catalog, remove)
+)
+
+func (c HostCapabilities) Has(t HostCapabilities) bool {
+	return c&t == t
+}
+
+// RegistryHost represents a complete configuration for a registry
+// host, representing the capabilities, authorizations, connection
+// configuration, and location.
+type RegistryHost struct {
+	Client       *http.Client
+	Authorizer   Authorizer
+	Host         string
+	Scheme       string
+	Path         string
+	Capabilities HostCapabilities
+}
+
+// RegistryHosts fetches the registry hosts for a given namespace,
+// provided by the host component of an distribution image reference.
+type RegistryHosts func(string) ([]RegistryHost, error)
+
+// Registries joins multiple registry configuration functions, using the same
+// order as provided within the arguments. When an empty registry configuration
+// is returned with a nil error, the next function will be called.
+// NOTE: This function will not join configurations, as soon as a non-empty
+// configuration is returned from a configuration function, it will be returned
+// to the caller.
+func Registries(registries ...RegistryHosts) RegistryHosts {
+	return func(host string) ([]RegistryHost, error) {
+		for _, registry := range registries {
+			config, err := registry(host)
+			if err != nil {
+				return config, err
+			}
+			if len(config) > 0 {
+				return config, nil
+			}
+		}
+		return nil, nil
+	}
+}
+
+type registryOpts struct {
+	authorizer Authorizer
+	plainHTTP  func(string) (bool, error)
+	host       func(string) (string, error)
+	client     *http.Client
+}
+
+// RegistryOpt defines a registry default option
+type RegistryOpt func(*registryOpts)
+
+// WithPlainHTTP configures registries to use plaintext http scheme
+// for the provided host match function.
+func WithPlainHTTP(f func(string) (bool, error)) RegistryOpt {
+	return func(opts *registryOpts) {
+		opts.plainHTTP = f
+	}
+}
+
+// WithAuthorizer configures the default authorizer for a registry
+func WithAuthorizer(a Authorizer) RegistryOpt {
+	return func(opts *registryOpts) {
+		opts.authorizer = a
+	}
+}
+
+// WithHostTranslator defines the default translator to use for registry hosts
+func WithHostTranslator(h func(string) (string, error)) RegistryOpt {
+	return func(opts *registryOpts) {
+		opts.host = h
+	}
+}
+
+// WithClient configures the default http client for a registry
+func WithClient(c *http.Client) RegistryOpt {
+	return func(opts *registryOpts) {
+		opts.client = c
+	}
+}
+
+// ConfigureDefaultRegistries is used to create a default configuration for
+// registries. For more advanced configurations or per-domain setups,
+// the RegistryHosts interface should be used directly.
+// NOTE: This function will always return a non-empty value or error
+func ConfigureDefaultRegistries(ropts ...RegistryOpt) RegistryHosts {
+	var opts registryOpts
+	for _, opt := range ropts {
+		opt(&opts)
+	}
+
+	return func(host string) ([]RegistryHost, error) {
+		config := RegistryHost{
+			Client:       opts.client,
+			Authorizer:   opts.authorizer,
+			Host:         host,
+			Scheme:       "https",
+			Path:         "/v2",
+			Capabilities: HostCapabilityPull | HostCapabilityResolve | HostCapabilityPush,
+		}
+
+		if config.Client == nil {
+			config.Client = http.DefaultClient
+		}
+
+		if opts.plainHTTP != nil {
+			match, err := opts.plainHTTP(host)
+			if err != nil {
+				return nil, err
+			}
+			if match {
+				config.Scheme = "http"
+			}
+		}
+
+		if opts.host != nil {
+			var err error
+			config.Host, err = opts.host(config.Host)
+			if err != nil {
+				return nil, err
+			}
+		} else if host == "docker.io" {
+			config.Host = "registry-1.docker.io"
+		}
+
+		return []RegistryHost{config}, nil
+	}
+}
+
+// MatchAllHosts is a host match function which is always true.
+func MatchAllHosts(string) (bool, error) {
+	return true, nil
+}
+
+// MatchLocalhost is a host match function which returns true for
+// localhost.
+func MatchLocalhost(host string) (bool, error) {
+	for _, s := range []string{"localhost", "127.0.0.1", "[::1]"} {
+		if len(host) >= len(s) && host[0:len(s)] == s && (len(host) == len(s) || host[len(s)] == ':') {
+			return true, nil
+		}
+	}
+	return host == "::1", nil
+
+}

vendor/github.com/containerd/containerd/remotes/docker/resolver.go

@@ -18,9 +18,10 @@ package docker
 
 import (
 	"context"
+	"fmt"
 	"io"
+	"io/ioutil"
 	"net/http"
-	"net/url"
 	"path"
 	"strings"
 
@@ -46,6 +47,19 @@ var (
 	// ErrInvalidAuthorization is used when credentials are passed to a server but
 	// those credentials are rejected.
 	ErrInvalidAuthorization = errors.New("authorization failed")
+
+	// MaxManifestSize represents the largest size accepted from a registry
+	// during resolution. Larger manifests may be accepted using a
+	// resolution method other than the registry.
+	//
+	// NOTE: The max supported layers by some runtimes is 128 and individual
+	// layers will not contribute more than 256 bytes, making a
+	// reasonable limit for a large image manifests of 32K bytes.
+	// 4M bytes represents a much larger upper bound for images which may
+	// contain large annotations or be non-images. A proper manifest
+	// design puts large metadata in subobjects, as is consistent the
+	// intent of the manifest design.
+	MaxManifestSize int64 = 4 * 1048 * 1048
 )
 
 // Authorizer is used to authorize HTTP requests based on 401 HTTP responses.
@@ -72,31 +86,38 @@ type Authorizer interface {
 
 // ResolverOptions are used to configured a new Docker register resolver
 type ResolverOptions struct {
-	// Authorizer is used to authorize registry requests
-	Authorizer Authorizer
-
-	// Credentials provides username and secret given a host.
-	// If username is empty but a secret is given, that secret
-	// is interpreted as a long lived token.
-	// Deprecated: use Authorizer
-	Credentials func(string) (string, string, error)
-
-	// Host provides the hostname given a namespace.
-	Host func(string) (string, error)
+	// Hosts returns registry host configurations for a namespace.
+	Hosts RegistryHosts
 
 	// Headers are the HTTP request header fields sent by the resolver
 	Headers http.Header
 
-	// PlainHTTP specifies to use plain http and not https
-	PlainHTTP bool
-
-	// Client is the http client to used when making registry requests
-	Client *http.Client
-
 	// Tracker is used to track uploads to the registry. This is used
 	// since the registry does not have upload tracking and the existing
 	// mechanism for getting blob upload status is expensive.
 	Tracker StatusTracker
+
+	// Authorizer is used to authorize registry requests
+	// Deprecated: use Hosts
+	Authorizer Authorizer
+
+	// Credentials provides username and secret given a host.
+	// If username is empty but a secret is given, that secret
+	// is interpreted as a long lived token.
+	// Deprecated: use Hosts
+	Credentials func(string) (string, string, error)
+
+	// Host provides the hostname given a namespace.
+	// Deprecated: use Hosts
+	Host func(string) (string, error)
+
+	// PlainHTTP specifies to use plain http and not https
+	// Deprecated: use Hosts
+	PlainHTTP bool
+
+	// Client is the http client to used when making registry requests
+	// Deprecated: use Hosts
+	Client *http.Client
 }
 
 // DefaultHost is the default host function.
@@ -108,12 +129,10 @@ func DefaultHost(ns string) (string, error) {
 }
 
 type dockerResolver struct {
-	auth      Authorizer
-	host      func(string) (string, error)
-	headers   http.Header
-	plainHTTP bool
-	client    *http.Client
-	tracker   StatusTracker
+	hosts         RegistryHosts
+	header        http.Header
+	resolveHeader http.Header
+	tracker       StatusTracker
 }
 
 // NewResolver returns a new resolver to a Docker registry
@@ -121,33 +140,56 @@ func NewResolver(options ResolverOptions) remotes.Resolver {
 	if options.Tracker == nil {
 		options.Tracker = NewInMemoryTracker()
 	}
-	if options.Host == nil {
-		options.Host = DefaultHost
-	}
+
 	if options.Headers == nil {
 		options.Headers = make(http.Header)
 	}
-	if _, ok := options.Headers["Accept"]; !ok {
-		// set headers for all the types we support for resolution.
-		options.Headers.Set("Accept", strings.Join([]string{
-			images.MediaTypeDockerSchema2Manifest,
-			images.MediaTypeDockerSchema2ManifestList,
-			ocispec.MediaTypeImageManifest,
-			ocispec.MediaTypeImageIndex, "*"}, ", "))
-	}
 	if _, ok := options.Headers["User-Agent"]; !ok {
 		options.Headers.Set("User-Agent", "containerd/"+version.Version)
 	}
-	if options.Authorizer == nil {
-		options.Authorizer = NewAuthorizer(options.Client, options.Credentials)
+
+	resolveHeader := http.Header{}
+	if _, ok := options.Headers["Accept"]; !ok {
+		// set headers for all the types we support for resolution.
+		resolveHeader.Set("Accept", strings.Join([]string{
+			images.MediaTypeDockerSchema2Manifest,
+			images.MediaTypeDockerSchema2ManifestList,
+			ocispec.MediaTypeImageManifest,
+			ocispec.MediaTypeImageIndex, "*/*"}, ", "))
+	} else {
+		resolveHeader["Accept"] = options.Headers["Accept"]
+		delete(options.Headers, "Accept")
+	}
+
+	if options.Hosts == nil {
+		opts := []RegistryOpt{}
+		if options.Host != nil {
+			opts = append(opts, WithHostTranslator(options.Host))
+		}
+
+		if options.Authorizer == nil {
+			options.Authorizer = NewDockerAuthorizer(
+				WithAuthClient(options.Client),
+				WithAuthHeader(options.Headers),
+				WithAuthCreds(options.Credentials))
+		}
+		opts = append(opts, WithAuthorizer(options.Authorizer))
+
+		if options.Client != nil {
+			opts = append(opts, WithClient(options.Client))
+		}
+		if options.PlainHTTP {
+			opts = append(opts, WithPlainHTTP(MatchAllHosts))
+		} else {
+			opts = append(opts, WithPlainHTTP(MatchLocalhost))
+		}
+		options.Hosts = ConfigureDefaultRegistries(opts...)
 	}
 	return &dockerResolver{
-		auth:      options.Authorizer,
-		host:      options.Host,
-		headers:   options.Headers,
-		plainHTTP: options.PlainHTTP,
-		client:    options.Client,
-		tracker:   options.Tracker,
+		hosts:         options.Hosts,
+		header:        options.Headers,
+		resolveHeader: resolveHeader,
+		tracker:       options.Tracker,
 	}
 }
 
@@ -194,13 +236,11 @@ func (r *dockerResolver) Resolve(ctx context.Context, ref string) (string, ocisp
 		return "", ocispec.Descriptor{}, err
 	}
 
-	fetcher := dockerFetcher{
-		dockerBase: base,
-	}
-
 	var (
-		urls []string
-		dgst = refspec.Digest()
+		lastErr error
+		paths   [][]string
+		dgst    = refspec.Digest()
+		caps    = HostCapabilityPull
 	)
 
 	if dgst != "" {
@@ -211,100 +251,130 @@ func (r *dockerResolver) Resolve(ctx context.Context, ref string) (string, ocisp
 		}
 
 		// turns out, we have a valid digest, make a url.
-		urls = append(urls, fetcher.url("manifests", dgst.String()))
+		paths = append(paths, []string{"manifests", dgst.String()})
 
 		// fallback to blobs on not found.
-		urls = append(urls, fetcher.url("blobs", dgst.String()))
+		paths = append(paths, []string{"blobs", dgst.String()})
 	} else {
-		urls = append(urls, fetcher.url("manifests", refspec.Object))
+		// Add
+		paths = append(paths, []string{"manifests", refspec.Object})
+		caps |= HostCapabilityResolve
+	}
+
+	hosts := base.filterHosts(caps)
+	if len(hosts) == 0 {
+		return "", ocispec.Descriptor{}, errors.Wrap(errdefs.ErrNotFound, "no resolve hosts")
 	}
 
 	ctx, err = contextWithRepositoryScope(ctx, refspec, false)
 	if err != nil {
 		return "", ocispec.Descriptor{}, err
 	}
-	for _, u := range urls {
-		req, err := http.NewRequest(http.MethodHead, u, nil)
-		if err != nil {
-			return "", ocispec.Descriptor{}, err
-		}
 
-		req.Header = r.headers
+	for _, u := range paths {
+		for _, host := range hosts {
+			ctx := log.WithLogger(ctx, log.G(ctx).WithField("host", host.Host))
 
-		log.G(ctx).Debug("resolving")
-		resp, err := fetcher.doRequestWithRetries(ctx, req, nil)
-		if err != nil {
-			if errors.Cause(err) == ErrInvalidAuthorization {
-				err = errors.Wrapf(err, "pull access denied, repository does not exist or may require authorization")
+			req := base.request(host, http.MethodHead, u...)
+			for key, value := range r.resolveHeader {
+				req.header[key] = append(req.header[key], value...)
 			}
-			return "", ocispec.Descriptor{}, err
-		}
-		resp.Body.Close() // don't care about body contents.
 
-		if resp.StatusCode > 299 {
-			if resp.StatusCode == http.StatusNotFound {
+			log.G(ctx).Debug("resolving")
+			resp, err := req.doWithRetries(ctx, nil)
+			if err != nil {
+				if errors.Cause(err) == ErrInvalidAuthorization {
+					err = errors.Wrapf(err, "pull access denied, repository does not exist or may require authorization")
+				}
+				return "", ocispec.Descriptor{}, err
+			}
+			resp.Body.Close() // don't care about body contents.
+
+			if resp.StatusCode > 299 {
+				if resp.StatusCode == http.StatusNotFound {
+					continue
+				}
+				return "", ocispec.Descriptor{}, errors.Errorf("unexpected status code %v: %v", u, resp.Status)
+			}
+			size := resp.ContentLength
+			contentType := getManifestMediaType(resp)
+
+			// if no digest was provided, then only a resolve
+			// trusted registry was contacted, in this case use
+			// the digest header (or content from GET)
+			if dgst == "" {
+				// this is the only point at which we trust the registry. we use the
+				// content headers to assemble a descriptor for the name. when this becomes
+				// more robust, we mostly get this information from a secure trust store.
+				dgstHeader := digest.Digest(resp.Header.Get("Docker-Content-Digest"))
+
+				if dgstHeader != "" && size != -1 {
+					if err := dgstHeader.Validate(); err != nil {
+						return "", ocispec.Descriptor{}, errors.Wrapf(err, "%q in header not a valid digest", dgstHeader)
+					}
+					dgst = dgstHeader
+				}
+			}
+			if dgst == "" || size == -1 {
+				log.G(ctx).Debug("no Docker-Content-Digest header, fetching manifest instead")
+
+				req = base.request(host, http.MethodGet, u...)
+				for key, value := range r.resolveHeader {
+					req.header[key] = append(req.header[key], value...)
+				}
+
+				resp, err := req.doWithRetries(ctx, nil)
+				if err != nil {
+					return "", ocispec.Descriptor{}, err
+				}
+				defer resp.Body.Close()
+
+				bodyReader := countingReader{reader: resp.Body}
+
+				contentType = getManifestMediaType(resp)
+				if dgst == "" {
+					if contentType == images.MediaTypeDockerSchema1Manifest {
+						b, err := schema1.ReadStripSignature(&bodyReader)
+						if err != nil {
+							return "", ocispec.Descriptor{}, err
+						}
+
+						dgst = digest.FromBytes(b)
+					} else {
+						dgst, err = digest.FromReader(&bodyReader)
+						if err != nil {
+							return "", ocispec.Descriptor{}, err
+						}
+					}
+				} else if _, err := io.Copy(ioutil.Discard, &bodyReader); err != nil {
+					return "", ocispec.Descriptor{}, err
+				}
+				size = bodyReader.bytesRead
+			}
+			// Prevent resolving to excessively large manifests
+			if size > MaxManifestSize {
+				if lastErr == nil {
+					lastErr = errors.Wrapf(errdefs.ErrNotFound, "rejecting %d byte manifest for %s", size, ref)
+				}
 				continue
 			}
-			return "", ocispec.Descriptor{}, errors.Errorf("unexpected status code %v: %v", u, resp.Status)
+
+			desc := ocispec.Descriptor{
+				Digest:    dgst,
+				MediaType: contentType,
+				Size:      size,
+			}
+
+			log.G(ctx).WithField("desc.digest", desc.Digest).Debug("resolved")
+			return ref, desc, nil
 		}
-		size := resp.ContentLength
-
-		// this is the only point at which we trust the registry. we use the
-		// content headers to assemble a descriptor for the name. when this becomes
-		// more robust, we mostly get this information from a secure trust store.
-		dgstHeader := digest.Digest(resp.Header.Get("Docker-Content-Digest"))
-		contentType := getManifestMediaType(resp)
-
-		if dgstHeader != "" && size != -1 {
-			if err := dgstHeader.Validate(); err != nil {
-				return "", ocispec.Descriptor{}, errors.Wrapf(err, "%q in header not a valid digest", dgstHeader)
-			}
-			dgst = dgstHeader
-		} else {
-			log.G(ctx).Debug("no Docker-Content-Digest header, fetching manifest instead")
-
-			req, err := http.NewRequest(http.MethodGet, u, nil)
-			if err != nil {
-				return "", ocispec.Descriptor{}, err
-			}
-			req.Header = r.headers
-
-			resp, err := fetcher.doRequestWithRetries(ctx, req, nil)
-			if err != nil {
-				return "", ocispec.Descriptor{}, err
-			}
-			defer resp.Body.Close()
-
-			bodyReader := countingReader{reader: resp.Body}
-
-			contentType = getManifestMediaType(resp)
-			if contentType == images.MediaTypeDockerSchema1Manifest {
-				b, err := schema1.ReadStripSignature(&bodyReader)
-				if err != nil {
-					return "", ocispec.Descriptor{}, err
-				}
-
-				dgst = digest.FromBytes(b)
-			} else {
-				dgst, err = digest.FromReader(&bodyReader)
-				if err != nil {
-					return "", ocispec.Descriptor{}, err
-				}
-			}
-			size = bodyReader.bytesRead
-		}
-
-		desc := ocispec.Descriptor{
-			Digest:    dgst,
-			MediaType: contentType,
-			Size:      size,
-		}
-
-		log.G(ctx).WithField("desc.digest", desc.Digest).Debug("resolved")
-		return ref, desc, nil
 	}
 
-	return "", ocispec.Descriptor{}, errors.Errorf("%v not found", ref)
+	if lastErr == nil {
+		lastErr = errors.Wrap(errdefs.ErrNotFound, ref)
+	}
+
+	return "", ocispec.Descriptor{}, lastErr
 }
 
 func (r *dockerResolver) Fetcher(ctx context.Context, ref string) (remotes.Fetcher, error) {
@@ -329,13 +399,6 @@ func (r *dockerResolver) Pusher(ctx context.Context, ref string) (remotes.Pusher
 		return nil, err
 	}
 
-	// Manifests can be pushed by digest like any other object, but the passed in
-	// reference cannot take a digest without the associated content. A tag is allowed
-	// and will be used to tag pushed manifests.
-	if refspec.Object != "" && strings.Contains(refspec.Object, "@") {
-		return nil, errors.New("cannot use digest reference for push locator")
-	}
-
 	base, err := r.base(refspec)
 	if err != nil {
 		return nil, err
@@ -343,60 +406,64 @@ func (r *dockerResolver) Pusher(ctx context.Context, ref string) (remotes.Pusher
 
 	return dockerPusher{
 		dockerBase: base,
-		tag:        refspec.Object,
+		object:     refspec.Object,
 		tracker:    r.tracker,
 	}, nil
 }
 
 type dockerBase struct {
-	refspec reference.Spec
-	base    url.URL
-
-	client *http.Client
-	auth   Authorizer
+	refspec   reference.Spec
+	namespace string
+	hosts     []RegistryHost
+	header    http.Header
 }
 
 func (r *dockerResolver) base(refspec reference.Spec) (*dockerBase, error) {
-	var (
-		err  error
-		base url.URL
-	)
-
 	host := refspec.Hostname()
-	base.Host = host
-	if r.host != nil {
-		base.Host, err = r.host(host)
-		if err != nil {
-			return nil, err
-		}
+	hosts, err := r.hosts(host)
+	if err != nil {
+		return nil, err
 	}
-
-	base.Scheme = "https"
-	if r.plainHTTP || strings.HasPrefix(base.Host, "localhost:") {
-		base.Scheme = "http"
-	}
-
-	prefix := strings.TrimPrefix(refspec.Locator, host+"/")
-	base.Path = path.Join("/v2", prefix)
-
 	return &dockerBase{
-		refspec: refspec,
-		base:    base,
-		client:  r.client,
-		auth:    r.auth,
+		refspec:   refspec,
+		namespace: strings.TrimPrefix(refspec.Locator, host+"/"),
+		hosts:     hosts,
+		header:    r.header,
 	}, nil
 }
 
-func (r *dockerBase) url(ps ...string) string {
-	url := r.base
-	url.Path = path.Join(url.Path, path.Join(ps...))
-	return url.String()
+func (r *dockerBase) filterHosts(caps HostCapabilities) (hosts []RegistryHost) {
+	for _, host := range r.hosts {
+		if host.Capabilities.Has(caps) {
+			hosts = append(hosts, host)
+		}
+	}
+	return
 }
 
-func (r *dockerBase) authorize(ctx context.Context, req *http.Request) error {
+func (r *dockerBase) request(host RegistryHost, method string, ps ...string) *request {
+	header := http.Header{}
+	for key, value := range r.header {
+		header[key] = append(header[key], value...)
+	}
+	parts := append([]string{"/", host.Path, r.namespace}, ps...)
+	p := path.Join(parts...)
+	// Join strips trailing slash, re-add ending "/" if included
+	if len(parts) > 0 && strings.HasSuffix(parts[len(parts)-1], "/") {
+		p = p + "/"
+	}
+	return &request{
+		method: method,
+		path:   p,
+		header: header,
+		host:   host,
+	}
+}
+
+func (r *request) authorize(ctx context.Context, req *http.Request) error {
 	// Check if has header for host
-	if r.auth != nil {
-		if err := r.auth.Authorize(ctx, req); err != nil {
+	if r.host.Authorizer != nil {
+		if err := r.host.Authorizer.Authorize(ctx, req); err != nil {
 			return err
 		}
 	}
@@ -404,80 +471,137 @@ func (r *dockerBase) authorize(ctx context.Context, req *http.Request) error {
 	return nil
 }
 
-func (r *dockerBase) doRequest(ctx context.Context, req *http.Request) (*http.Response, error) {
-	ctx = log.WithLogger(ctx, log.G(ctx).WithField("url", req.URL.String()))
-	log.G(ctx).WithField("request.headers", req.Header).WithField("request.method", req.Method).Debug("do request")
+type request struct {
+	method string
+	path   string
+	header http.Header
+	host   RegistryHost
+	body   func() (io.ReadCloser, error)
+	size   int64
+}
+
+func (r *request) do(ctx context.Context) (*http.Response, error) {
+	u := r.host.Scheme + "://" + r.host.Host + r.path
+	req, err := http.NewRequest(r.method, u, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header = r.header
+	if r.body != nil {
+		body, err := r.body()
+		if err != nil {
+			return nil, err
+		}
+		req.Body = body
+		req.GetBody = r.body
+		if r.size > 0 {
+			req.ContentLength = r.size
+		}
+	}
+
+	ctx = log.WithLogger(ctx, log.G(ctx).WithField("url", u))
+	log.G(ctx).WithFields(requestFields(req)).Debug("do request")
 	if err := r.authorize(ctx, req); err != nil {
 		return nil, errors.Wrap(err, "failed to authorize")
 	}
-	resp, err := ctxhttp.Do(ctx, r.client, req)
+	resp, err := ctxhttp.Do(ctx, r.host.Client, req)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to do request")
 	}
-	log.G(ctx).WithFields(logrus.Fields{
-		"status":           resp.Status,
-		"response.headers": resp.Header,
-	}).Debug("fetch response received")
+	log.G(ctx).WithFields(responseFields(resp)).Debug("fetch response received")
 	return resp, nil
 }
 
-func (r *dockerBase) doRequestWithRetries(ctx context.Context, req *http.Request, responses []*http.Response) (*http.Response, error) {
-	resp, err := r.doRequest(ctx, req)
+func (r *request) doWithRetries(ctx context.Context, responses []*http.Response) (*http.Response, error) {
+	resp, err := r.do(ctx)
 	if err != nil {
 		return nil, err
 	}
 
 	responses = append(responses, resp)
-	req, err = r.retryRequest(ctx, req, responses)
+	retry, err := r.retryRequest(ctx, responses)
 	if err != nil {
 		resp.Body.Close()
 		return nil, err
 	}
-	if req != nil {
+	if retry {
 		resp.Body.Close()
-		return r.doRequestWithRetries(ctx, req, responses)
+		return r.doWithRetries(ctx, responses)
 	}
 	return resp, err
 }
 
-func (r *dockerBase) retryRequest(ctx context.Context, req *http.Request, responses []*http.Response) (*http.Request, error) {
+func (r *request) retryRequest(ctx context.Context, responses []*http.Response) (bool, error) {
 	if len(responses) > 5 {
-		return nil, nil
+		return false, nil
 	}
 	last := responses[len(responses)-1]
-	if last.StatusCode == http.StatusUnauthorized {
+	switch last.StatusCode {
+	case http.StatusUnauthorized:
 		log.G(ctx).WithField("header", last.Header.Get("WWW-Authenticate")).Debug("Unauthorized")
-		if r.auth != nil {
-			if err := r.auth.AddResponses(ctx, responses); err == nil {
-				return copyRequest(req)
+		if r.host.Authorizer != nil {
+			if err := r.host.Authorizer.AddResponses(ctx, responses); err == nil {
+				return true, nil
 			} else if !errdefs.IsNotImplemented(err) {
-				return nil, err
+				return false, err
 			}
 		}
 
-		return nil, nil
-	} else if last.StatusCode == http.StatusMethodNotAllowed && req.Method == http.MethodHead {
+		return false, nil
+	case http.StatusMethodNotAllowed:
 		// Support registries which have not properly implemented the HEAD method for
 		// manifests endpoint
-		if strings.Contains(req.URL.Path, "/manifests/") {
-			// TODO: copy request?
-			req.Method = http.MethodGet
-			return copyRequest(req)
+		if r.method == http.MethodHead && strings.Contains(r.path, "/manifests/") {
+			r.method = http.MethodGet
+			return true, nil
 		}
+	case http.StatusRequestTimeout, http.StatusTooManyRequests:
+		return true, nil
 	}
 
 	// TODO: Handle 50x errors accounting for attempt history
-	return nil, nil
+	return false, nil
 }
 
-func copyRequest(req *http.Request) (*http.Request, error) {
-	ireq := *req
-	if ireq.GetBody != nil {
-		var err error
-		ireq.Body, err = ireq.GetBody()
-		if err != nil {
-			return nil, err
+func (r *request) String() string {
+	return r.host.Scheme + "://" + r.host.Host + r.path
+}
+
+func requestFields(req *http.Request) logrus.Fields {
+	fields := map[string]interface{}{
+		"request.method": req.Method,
+	}
+	for k, vals := range req.Header {
+		k = strings.ToLower(k)
+		if k == "authorization" {
+			continue
+		}
+		for i, v := range vals {
+			field := "request.header." + k
+			if i > 0 {
+				field = fmt.Sprintf("%s.%d", field, i)
+			}
+			fields[field] = v
 		}
 	}
-	return &ireq, nil
+
+	return logrus.Fields(fields)
+}
+
+func responseFields(resp *http.Response) logrus.Fields {
+	fields := map[string]interface{}{
+		"response.status": resp.Status,
+	}
+	for k, vals := range resp.Header {
+		k = strings.ToLower(k)
+		for i, v := range vals {
+			field := "response.header." + k
+			if i > 0 {
+				field = fmt.Sprintf("%s.%d", field, i)
+			}
+			fields[field] = v
+		}
+	}
+
+	return logrus.Fields(fields)
 }

vendor/github.com/containerd/containerd/remotes/docker/schema1/converter.go

@@ -216,12 +216,12 @@ func (c *Converter) Convert(ctx context.Context, opts ...ConvertOpt) (ocispec.De
 
 	ref := remotes.MakeRefKey(ctx, desc)
 	if err := content.WriteBlob(ctx, c.contentStore, ref, bytes.NewReader(mb), desc, content.WithLabels(labels)); err != nil {
-		return ocispec.Descriptor{}, errors.Wrap(err, "failed to write config")
+		return ocispec.Descriptor{}, errors.Wrap(err, "failed to write image manifest")
 	}
 
 	ref = remotes.MakeRefKey(ctx, config)
 	if err := content.WriteBlob(ctx, c.contentStore, ref, bytes.NewReader(b), config); err != nil {
-		return ocispec.Descriptor{}, errors.Wrap(err, "failed to write config")
+		return ocispec.Descriptor{}, errors.Wrap(err, "failed to write image config")
 	}
 
 	return desc, nil

vendor/github.com/containerd/containerd/remotes/docker/scope.go

@@ -18,6 +18,7 @@ package docker
 
 import (
 	"context"
+	"fmt"
 	"net/url"
 	"sort"
 	"strings"
@@ -50,27 +51,47 @@ func contextWithRepositoryScope(ctx context.Context, refspec reference.Spec, pus
 	if err != nil {
 		return nil, err
 	}
-	return context.WithValue(ctx, tokenScopesKey{}, []string{s}), nil
+	return WithScope(ctx, s), nil
 }
 
-// getTokenScopes returns deduplicated and sorted scopes from ctx.Value(tokenScopesKey{}) and params["scope"].
-func getTokenScopes(ctx context.Context, params map[string]string) []string {
+// WithScope appends a custom registry auth scope to the context.
+func WithScope(ctx context.Context, scope string) context.Context {
+	var scopes []string
+	if v := ctx.Value(tokenScopesKey{}); v != nil {
+		scopes = v.([]string)
+		scopes = append(scopes, scope)
+	} else {
+		scopes = []string{scope}
+	}
+	return context.WithValue(ctx, tokenScopesKey{}, scopes)
+}
+
+// contextWithAppendPullRepositoryScope is used to append repository pull
+// scope into existing scopes indexed by the tokenScopesKey{}.
+func contextWithAppendPullRepositoryScope(ctx context.Context, repo string) context.Context {
+	return WithScope(ctx, fmt.Sprintf("repository:%s:pull", repo))
+}
+
+// getTokenScopes returns deduplicated and sorted scopes from ctx.Value(tokenScopesKey{}) and common scopes.
+func getTokenScopes(ctx context.Context, common []string) []string {
 	var scopes []string
 	if x := ctx.Value(tokenScopesKey{}); x != nil {
 		scopes = append(scopes, x.([]string)...)
 	}
-	if scope, ok := params["scope"]; ok {
-		for _, s := range scopes {
-			// Note: this comparison is unaware of the scope grammar (https://docs.docker.com/registry/spec/auth/scope/)
-			// So, "repository:foo/bar:pull,push" != "repository:foo/bar:push,pull", although semantically they are equal.
-			if s == scope {
-				// already appended
-				goto Sort
-			}
-		}
-		scopes = append(scopes, scope)
-	}
-Sort:
+
+	scopes = append(scopes, common...)
 	sort.Strings(scopes)
-	return scopes
+
+	l := 0
+	for idx := 1; idx < len(scopes); idx++ {
+		// Note: this comparison is unaware of the scope grammar (https://docs.docker.com/registry/spec/auth/scope/)
+		// So, "repository:foo/bar:pull,push" != "repository:foo/bar:push,pull", although semantically they are equal.
+		if scopes[l] == scopes[idx] {
+			continue
+		}
+
+		l++
+		scopes[l] = scopes[idx]
+	}
+	return scopes[:l+1]
 }

vendor/github.com/containerd/containerd/remotes/handlers.go

@@ -33,27 +33,46 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
+type refKeyPrefix struct{}
+
+// WithMediaTypeKeyPrefix adds a custom key prefix for a media type which is used when storing
+// data in the content store from the FetchHandler.
+//
+// Used in `MakeRefKey` to determine what the key prefix should be.
+func WithMediaTypeKeyPrefix(ctx context.Context, mediaType, prefix string) context.Context {
+	var values map[string]string
+	if v := ctx.Value(refKeyPrefix{}); v != nil {
+		values = v.(map[string]string)
+	} else {
+		values = make(map[string]string)
+	}
+
+	values[mediaType] = prefix
+	return context.WithValue(ctx, refKeyPrefix{}, values)
+}
+
 // MakeRefKey returns a unique reference for the descriptor. This reference can be
 // used to lookup ongoing processes related to the descriptor. This function
 // may look to the context to namespace the reference appropriately.
 func MakeRefKey(ctx context.Context, desc ocispec.Descriptor) string {
-	// TODO(stevvooe): Need better remote key selection here. Should be a
-	// product of the context, which may include information about the ongoing
-	// fetch process.
-	switch desc.MediaType {
-	case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest:
+	if v := ctx.Value(refKeyPrefix{}); v != nil {
+		values := v.(map[string]string)
+		if prefix := values[desc.MediaType]; prefix != "" {
+			return prefix + "-" + desc.Digest.String()
+		}
+	}
+
+	switch mt := desc.MediaType; {
+	case mt == images.MediaTypeDockerSchema2Manifest || mt == ocispec.MediaTypeImageManifest:
 		return "manifest-" + desc.Digest.String()
-	case images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex:
+	case mt == images.MediaTypeDockerSchema2ManifestList || mt == ocispec.MediaTypeImageIndex:
 		return "index-" + desc.Digest.String()
-	case images.MediaTypeDockerSchema2Layer, images.MediaTypeDockerSchema2LayerGzip,
-		images.MediaTypeDockerSchema2LayerForeign, images.MediaTypeDockerSchema2LayerForeignGzip,
-		ocispec.MediaTypeImageLayer, ocispec.MediaTypeImageLayerGzip,
-		ocispec.MediaTypeImageLayerNonDistributable, ocispec.MediaTypeImageLayerNonDistributableGzip:
+	case images.IsLayerType(mt):
 		return "layer-" + desc.Digest.String()
-	case images.MediaTypeDockerSchema2Config, ocispec.MediaTypeImageConfig:
+	case images.IsKnownConfig(mt):
 		return "config-" + desc.Digest.String()
 	default:
-		log.G(ctx).Warnf("reference for unknown type: %s", desc.MediaType)
+		log.G(ctx).Warnf("reference for unknown type: %s", mt)
 		return "unknown-" + desc.Digest.String()
 	}
 }
@@ -156,7 +175,7 @@ func push(ctx context.Context, provider content.Provider, pusher Pusher, desc oc
 //
 // Base handlers can be provided which will be called before any push specific
 // handlers.
-func PushContent(ctx context.Context, pusher Pusher, desc ocispec.Descriptor, provider content.Provider, platform platforms.MatchComparer, wrapper func(h images.Handler) images.Handler) error {
+func PushContent(ctx context.Context, pusher Pusher, desc ocispec.Descriptor, store content.Store, platform platforms.MatchComparer, wrapper func(h images.Handler) images.Handler) error {
 	var m sync.Mutex
 	manifestStack := []ocispec.Descriptor{}
 
@@ -173,10 +192,14 @@ func PushContent(ctx context.Context, pusher Pusher, desc ocispec.Descriptor, pr
 		}
 	})
 
-	pushHandler := PushHandler(pusher, provider)
+	pushHandler := PushHandler(pusher, store)
+
+	platformFilterhandler := images.FilterPlatforms(images.ChildrenHandler(store), platform)
+
+	annotateHandler := annotateDistributionSourceHandler(platformFilterhandler, store)
 
 	var handler images.Handler = images.Handlers(
-		images.FilterPlatforms(images.ChildrenHandler(provider), platform),
+		annotateHandler,
 		filterHandler,
 		pushHandler,
 	)
@@ -241,3 +264,45 @@ func FilterManifestByPlatformHandler(f images.HandlerFunc, m platforms.Matcher)
 		return descs, nil
 	}
 }
+
+// annotateDistributionSourceHandler add distribution source label into
+// annotation of config or blob descriptor.
+func annotateDistributionSourceHandler(f images.HandlerFunc, manager content.Manager) images.HandlerFunc {
+	return func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
+		children, err := f(ctx, desc)
+		if err != nil {
+			return nil, err
+		}
+
+		// only add distribution source for the config or blob data descriptor
+		switch desc.MediaType {
+		case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest,
+			images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex:
+		default:
+			return children, nil
+		}
+
+		for i := range children {
+			child := children[i]
+
+			info, err := manager.Info(ctx, child.Digest)
+			if err != nil {
+				return nil, err
+			}
+
+			for k, v := range info.Labels {
+				if !strings.HasPrefix(k, "containerd.io/distribution.source.") {
+					continue
+				}
+
+				if child.Annotations == nil {
+					child.Annotations = map[string]string{}
+				}
+				child.Annotations[k] = v
+			}
+
+			children[i] = child
+		}
+		return children, nil
+	}
+}