vendor: update buildkit to master@8b7bcb900d3c

Signed-off-by: Justin Chadwell <me@jedevc.com>
This commit is contained in:
Justin Chadwell
2023-03-29 12:38:36 +01:00
parent c6cdcb02cf
commit 9541457c54
416 changed files with 24398 additions and 16253 deletions

View File

@ -21,12 +21,12 @@ import (
"errors"
"fmt"
"io"
"math/rand"
"sync"
"time"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/log"
"github.com/containerd/containerd/pkg/randutil"
"github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)
@ -59,6 +59,10 @@ func NewReader(ra ReaderAt) io.Reader {
//
// Avoid using this for large blobs, such as layers.
func ReadBlob(ctx context.Context, provider Provider, desc ocispec.Descriptor) ([]byte, error) {
if int64(len(desc.Data)) == desc.Size && digest.FromBytes(desc.Data) == desc.Digest {
return desc.Data, nil
}
ra, err := provider.ReaderAt(ctx, desc)
if err != nil {
return nil, err
@ -119,7 +123,7 @@ func OpenWriter(ctx context.Context, cs Ingester, opts ...WriterOpt) (Writer, er
// error or abort. Requires asserting for an ingest manager
select {
case <-time.After(time.Millisecond * time.Duration(rand.Intn(retry))):
case <-time.After(time.Millisecond * time.Duration(randutil.Intn(retry))):
if retry < 2048 {
retry = retry << 1
}

View File

@ -20,7 +20,6 @@ import (
"context"
"fmt"
"io"
"math/rand"
"os"
"path/filepath"
"strconv"
@ -32,6 +31,7 @@ import (
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/filters"
"github.com/containerd/containerd/log"
"github.com/containerd/containerd/pkg/randutil"
"github.com/sirupsen/logrus"
"github.com/opencontainers/go-digest"
@ -473,7 +473,7 @@ func (s *store) Writer(ctx context.Context, opts ...content.WriterOpt) (content.
lockErr = nil
break
}
time.Sleep(time.Millisecond * time.Duration(rand.Intn(1<<count)))
time.Sleep(time.Millisecond * time.Duration(randutil.Intn(1<<count)))
}
if lockErr != nil {

View File

@ -23,3 +23,7 @@ const LabelUncompressed = "containerd.io/uncompressed"
// LabelSharedNamespace is added to a namespace to allow that namespaces
// contents to be shared.
const LabelSharedNamespace = "containerd.io/namespace.shareable"
// LabelDistributionSource is added to content to indicate its origin.
// e.g., "containerd.io/distribution.source.docker.io=library/redis"
const LabelDistributionSource = "containerd.io/distribution.source"

View File

@ -17,9 +17,9 @@
package leases
import (
"crypto/rand"
"encoding/base64"
"fmt"
"math/rand"
"time"
)

View File

@ -35,6 +35,9 @@ var (
type (
loggerKey struct{}
// Fields type to pass to `WithFields`, alias from `logrus`.
Fields = logrus.Fields
)
const (

View File

@ -0,0 +1,48 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package randutil provides utilities for [cyrpto/rand].
package randutil
import (
"crypto/rand"
"math"
"math/big"
)
// Int63n is similar to [math/rand.Int63n] but uses [crypto/rand.Reader] under the hood.
func Int63n(n int64) int64 {
b, err := rand.Int(rand.Reader, big.NewInt(n))
if err != nil {
panic(err)
}
return b.Int64()
}
// Int63 is similar to [math/rand.Int63] but uses [crypto/rand.Reader] under the hood.
func Int63() int64 {
return Int63n(math.MaxInt64)
}
// Intn is similar to [math/rand.Intn] but uses [crypto/rand.Reader] under the hood.
func Intn(n int) int {
return int(Int63n(int64(n)))
}
// Int is similar to [math/rand.Int] but uses [crypto/rand.Reader] under the hood.
func Int() int {
return int(Int63())
}

View File

@ -14,6 +14,9 @@
limitations under the License.
*/
// Package seed provides an initializer for the global [math/rand] seed.
//
// Deprecated: Do not rely on the global seed.
package seed
import (
@ -23,6 +26,8 @@ import (
// WithTimeAndRand seeds the global math rand generator with nanoseconds
// XOR'ed with a crypto component if available for uniqueness.
//
// Deprecated: Do not rely on the global seed.
func WithTimeAndRand() {
var (
b [4]byte

View File

@ -50,7 +50,10 @@ func (m windowsmatcher) Match(p specs.Platform) bool {
match := m.defaultMatcher.Match(p)
if match && m.OS == "windows" {
return strings.HasPrefix(p.OSVersion, m.osVersionPrefix) && m.defaultMatcher.Match(p)
if strings.HasPrefix(p.OSVersion, m.osVersionPrefix) {
return true
}
return p.OSVersion == ""
}
return match

View File

@ -17,7 +17,7 @@
package protobuf
import (
"github.com/containerd/typeurl"
"github.com/containerd/typeurl/v2"
"google.golang.org/protobuf/types/known/anypb"
)

View File

@ -114,7 +114,7 @@ func FetchTokenWithOAuth(ctx context.Context, client *http.Client, headers http.
form.Set("access_type", "offline")
}
req, err := http.NewRequestWithContext(ctx, "POST", to.Realm, strings.NewReader(form.Encode()))
req, err := http.NewRequestWithContext(ctx, http.MethodPost, to.Realm, strings.NewReader(form.Encode()))
if err != nil {
return nil, err
}
@ -161,7 +161,7 @@ type FetchTokenResponse struct {
// FetchToken fetches a token using a GET request
func FetchToken(ctx context.Context, client *http.Client, headers http.Header, to TokenOptions) (*FetchTokenResponse, error) {
req, err := http.NewRequestWithContext(ctx, "GET", to.Realm, nil)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, to.Realm, nil)
if err != nil {
return nil, err
}

View File

@ -29,7 +29,6 @@ import (
"github.com/containerd/containerd/log"
"github.com/containerd/containerd/remotes/docker/auth"
remoteerrors "github.com/containerd/containerd/remotes/errors"
"github.com/sirupsen/logrus"
)
type dockerAuthorizer struct {
@ -312,7 +311,7 @@ func (ah *authHandler) doBearerAuth(ctx context.Context) (token, refreshToken st
}
return resp.Token, resp.RefreshToken, nil
}
log.G(ctx).WithFields(logrus.Fields{
log.G(ctx).WithFields(log.Fields{
"status": errStatus.Status,
"body": string(errStatus.Body),
}).Debugf("token request failed")

View File

@ -30,11 +30,6 @@ import (
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)
var (
// labelDistributionSource describes the source blob comes from.
labelDistributionSource = "containerd.io/distribution.source"
)
// AppendDistributionSourceLabel updates the label of blob with distribution source.
func AppendDistributionSourceLabel(manager content.Manager, ref string) (images.HandlerFunc, error) {
refspec, err := reference.Parse(ref)
@ -108,7 +103,7 @@ func appendDistributionSourceLabel(originLabel, repo string) string {
}
func distributionSourceLabelKey(source string) string {
return fmt.Sprintf("%s.%s", labelDistributionSource, source)
return fmt.Sprintf("%s.%s", labels.LabelDistributionSource, source)
}
// selectRepositoryMountCandidate will select the repo which has longest

View File

@ -38,7 +38,6 @@ import (
"github.com/containerd/containerd/version"
"github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/sirupsen/logrus"
)
var (
@ -647,7 +646,7 @@ func (r *request) String() string {
return r.host.Scheme + "://" + r.host.Host + r.path
}
func requestFields(req *http.Request) logrus.Fields {
func requestFields(req *http.Request) log.Fields {
fields := map[string]interface{}{
"request.method": req.Method,
}
@ -665,10 +664,10 @@ func requestFields(req *http.Request) logrus.Fields {
}
}
return logrus.Fields(fields)
return log.Fields(fields)
}
func responseFields(resp *http.Response) logrus.Fields {
func responseFields(resp *http.Response) log.Fields {
fields := map[string]interface{}{
"response.status": resp.Status,
}
@ -683,7 +682,7 @@ func responseFields(resp *http.Response) logrus.Fields {
}
}
return logrus.Fields(fields)
return log.Fields(fields)
}
// IsLocalhost checks if the registry host is local.

View File

@ -17,6 +17,7 @@
package remotes
import (
"bytes"
"context"
"errors"
"fmt"
@ -27,10 +28,10 @@ import (
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/labels"
"github.com/containerd/containerd/log"
"github.com/containerd/containerd/platforms"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/sirupsen/logrus"
"golang.org/x/sync/semaphore"
)
@ -90,7 +91,7 @@ func MakeRefKey(ctx context.Context, desc ocispec.Descriptor) string {
// recursive fetch.
func FetchHandler(ingester content.Ingester, fetcher Fetcher) images.HandlerFunc {
return func(ctx context.Context, desc ocispec.Descriptor) (subdescs []ocispec.Descriptor, err error) {
ctx = log.WithLogger(ctx, log.G(ctx).WithFields(logrus.Fields{
ctx = log.WithLogger(ctx, log.G(ctx).WithFields(log.Fields{
"digest": desc.Digest,
"mediatype": desc.MediaType,
"size": desc.Size,
@ -139,6 +140,10 @@ func Fetch(ctx context.Context, ingester content.Ingester, fetcher Fetcher, desc
return err
}
if desc.Size == int64(len(desc.Data)) {
return content.Copy(ctx, cw, bytes.NewReader(desc.Data), desc.Size, desc.Digest)
}
rc, err := fetcher.Fetch(ctx, desc)
if err != nil {
return err
@ -152,7 +157,7 @@ func Fetch(ctx context.Context, ingester content.Ingester, fetcher Fetcher, desc
// using a writer from the pusher.
func PushHandler(pusher Pusher, provider content.Provider) images.HandlerFunc {
return func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
ctx = log.WithLogger(ctx, log.G(ctx).WithFields(logrus.Fields{
ctx = log.WithLogger(ctx, log.G(ctx).WithFields(log.Fields{
"digest": desc.Digest,
"mediatype": desc.MediaType,
"size": desc.Size,
@ -363,7 +368,7 @@ func annotateDistributionSourceHandler(f images.HandlerFunc, manager content.Man
}
for k, v := range info.Labels {
if !strings.HasPrefix(k, "containerd.io/distribution.source.") {
if !strings.HasPrefix(k, labels.LabelDistributionSource+".") {
continue
}

View File

@ -34,7 +34,7 @@ type Resolver interface {
// reference a specific host or be matched against a specific handler.
//
// The returned name should be used to identify the referenced entity.
// Dependending on the remote namespace, this may be immutable or mutable.
// Depending on the remote namespace, this may be immutable or mutable.
// While the name may differ from ref, it should itself be a valid ref.
//
// If the resolution fails, an error will be returned.

View File

@ -30,7 +30,6 @@ import (
ptypes "github.com/containerd/containerd/protobuf/types"
digest "github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/sirupsen/logrus"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
@ -295,7 +294,7 @@ func (s *service) Write(session api.Content_WriteServer) (err error) {
return status.Errorf(codes.InvalidArgument, "first message must have a reference")
}
fields := logrus.Fields{
fields := log.Fields{
"ref": ref,
}
total = req.Total

View File

@ -23,7 +23,8 @@ import (
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
semconv "go.opentelemetry.io/otel/semconv/v1.12.0"
semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
httpconv "go.opentelemetry.io/otel/semconv/v1.17.0/httpconv"
"go.opentelemetry.io/otel/trace"
)
@ -39,8 +40,8 @@ type SpanOpt func(config *StartConfig)
func WithHTTPRequest(request *http.Request) SpanOpt {
return func(config *StartConfig) {
config.spanOpts = append(config.spanOpts,
trace.WithSpanKind(trace.SpanKindClient), // A client making a request to a server
trace.WithAttributes(semconv.HTTPClientAttributesFromHTTPRequest(request)...), // Add HTTP attributes
trace.WithSpanKind(trace.SpanKindClient), // A client making a request to a server
trace.WithAttributes(httpconv.ClientRequest(request)...), // Add HTTP attributes
)
}
}
@ -112,5 +113,5 @@ func Attribute(k string, v interface{}) attribute.KeyValue {
// HTTPStatusCodeAttributes generates attributes of the HTTP namespace as specified by the OpenTelemetry
// specification for a span.
func HTTPStatusCodeAttributes(code int) []attribute.KeyValue {
return semconv.HTTPAttributesFromHTTPStatusCode(code)
return []attribute.KeyValue{semconv.HTTPStatusCodeKey.Int(code)}
}

View File

@ -23,7 +23,7 @@ var (
Package = "github.com/containerd/containerd"
// Version holds the complete version number. Filled in at linking time.
Version = "1.7.0-beta.3+unknown"
Version = "1.7.0+unknown"
// Revision is filled with the VCS (e.g. git) revision being used to build
// the program at linking time.

View File

@ -57,7 +57,7 @@ TESTFLAGS_PARALLEL ?= 8
# Use this to replace `go test` with, for instance, `gotestsum`
GOTEST ?= $(GO) test
.PHONY: clean all AUTHORS build binaries test integration generate protos checkprotos coverage ci check help install vendor install-protobuf install-protobuild
.PHONY: clean all AUTHORS build binaries test integration generate protos check-protos coverage ci check help install vendor install-protobuf install-protobuild
.DEFAULT: default
# Forcibly set the default goal to all, in case an include above brought in a rule definition.
@ -69,7 +69,7 @@ check: proto-fmt ## run all linters
@echo "$(WHALE) $@"
GOGC=75 golangci-lint run
ci: check binaries checkprotos coverage # coverage-integration ## to be used by the CI
ci: check binaries check-protos coverage # coverage-integration ## to be used by the CI
AUTHORS: .mailmap .git/HEAD
git log --format='%aN <%aE>' | sort -fu > $@
@ -145,8 +145,8 @@ install-protobuf:
install-protobuild:
@echo "$(WHALE) $@"
@$(GO) install google.golang.org/protobuf/cmd/protoc-gen-go@v1.27.1
@$(GO) install github.com/containerd/protobuild@7e5ee24bc1f70e9e289fef15e2631eb3491320bf
@$(GO) install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.1
@$(GO) install github.com/containerd/protobuild@14832ccc41429f5c4f81028e5af08aa233a219cf
coverage: ## generate coverprofiles from the unit tests, except tests that require root
@echo "$(WHALE) $@"

View File

@ -23,3 +23,6 @@ generators = ["go"]
# enable ttrpc and disable fieldpath and grpc for the shim
prefixes = ["github.com/containerd/ttrpc/integration/streaming"]
generators = ["go", "go-ttrpc"]
[overrides.parameters.go-ttrpc]
prefix = "TTRPC"

View File

@ -1,7 +1,6 @@
# ttrpc
[![Build Status](https://github.com/containerd/ttrpc/workflows/CI/badge.svg)](https://github.com/containerd/ttrpc/actions?query=workflow%3ACI)
[![codecov](https://codecov.io/gh/containerd/ttrpc/branch/main/graph/badge.svg)](https://codecov.io/gh/containerd/ttrpc)
GRPC for low-memory environments.
@ -30,7 +29,7 @@ Create a gogo vanity binary (see
[`cmd/protoc-gen-gogottrpc/main.go`](cmd/protoc-gen-gogottrpc/main.go) for an
example with the ttrpc plugin enabled.
It's recommended to use [`protobuild`](https://github.com//stevvooe/protobuild)
It's recommended to use [`protobuild`](https://github.com/containerd/protobuild)
to build the protobufs for this project, but this will work with protoc
directly, if required.
@ -41,7 +40,6 @@ directly, if required.
- The client and server interface are identical whereas in GRPC there is a
client and server interface that are different.
- The Go stdlib context package is used instead.
- No support for streams yet.
# Status

View File

@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.27.1
// protoc v3.11.4
// protoc-gen-go v1.28.1
// protoc v3.20.1
// source: github.com/containerd/ttrpc/request.proto
package ttrpc

View File

@ -18,11 +18,13 @@ package ttrpc
import (
"context"
"errors"
"io"
"math/rand"
"net"
"sync"
"sync/atomic"
"syscall"
"time"
"github.com/sirupsen/logrus"
@ -119,12 +121,18 @@ func (s *Server) Serve(ctx context.Context, l net.Listener) error {
approved, handshake, err := handshaker.Handshake(ctx, conn)
if err != nil {
logrus.WithError(err).Errorf("ttrpc: refusing connection after handshake")
logrus.WithError(err).Error("ttrpc: refusing connection after handshake")
conn.Close()
continue
}
sc, err := s.newConn(approved, handshake)
if err != nil {
logrus.WithError(err).Error("ttrpc: create connection failed")
conn.Close()
continue
}
sc := s.newConn(approved, handshake)
go sc.run(ctx)
}
}
@ -143,15 +151,20 @@ func (s *Server) Shutdown(ctx context.Context) error {
ticker := time.NewTicker(200 * time.Millisecond)
defer ticker.Stop()
for {
if s.closeIdleConns() {
return lnerr
s.closeIdleConns()
if s.countConnection() == 0 {
break
}
select {
case <-ctx.Done():
return ctx.Err()
case <-ticker.C:
}
}
return lnerr
}
// Close the server without waiting for active connections.
@ -203,11 +216,18 @@ func (s *Server) closeListeners() error {
return err
}
func (s *Server) addConnection(c *serverConn) {
func (s *Server) addConnection(c *serverConn) error {
s.mu.Lock()
defer s.mu.Unlock()
select {
case <-s.done:
return ErrServerClosed
default:
}
s.connections[c] = struct{}{}
return nil
}
func (s *Server) delConnection(c *serverConn) {
@ -224,20 +244,17 @@ func (s *Server) countConnection() int {
return len(s.connections)
}
func (s *Server) closeIdleConns() bool {
func (s *Server) closeIdleConns() {
s.mu.Lock()
defer s.mu.Unlock()
quiescent := true
for c := range s.connections {
st, ok := c.getState()
if !ok || st != connStateIdle {
quiescent = false
if st, ok := c.getState(); !ok || st == connStateActive {
continue
}
c.close()
delete(s.connections, c)
}
return quiescent
}
type connState int
@ -261,7 +278,7 @@ func (cs connState) String() string {
}
}
func (s *Server) newConn(conn net.Conn, handshake interface{}) *serverConn {
func (s *Server) newConn(conn net.Conn, handshake interface{}) (*serverConn, error) {
c := &serverConn{
server: s,
conn: conn,
@ -269,8 +286,11 @@ func (s *Server) newConn(conn net.Conn, handshake interface{}) *serverConn {
shutdown: make(chan struct{}),
}
c.setState(connStateIdle)
s.addConnection(c)
return c
if err := s.addConnection(c); err != nil {
c.close()
return nil, err
}
return c, nil
}
type serverConn struct {
@ -318,6 +338,7 @@ func (c *serverConn) run(sctx context.Context) {
responses = make(chan response)
recvErr = make(chan error, 1)
done = make(chan struct{})
streams = sync.Map{}
active int32
lastStreamID uint32
)
@ -347,7 +368,6 @@ func (c *serverConn) run(sctx context.Context) {
go func(recvErr chan error) {
defer close(recvErr)
streams := map[uint32]*streamHandler{}
for {
select {
case <-c.shutdown:
@ -383,12 +403,13 @@ func (c *serverConn) run(sctx context.Context) {
}
if mh.Type == messageTypeData {
sh, ok := streams[mh.StreamID]
i, ok := streams.Load(mh.StreamID)
if !ok {
if !sendStatus(mh.StreamID, status.Newf(codes.InvalidArgument, "StreamID is no longer active")) {
return
}
}
sh := i.(*streamHandler)
if mh.Flags&flagNoData != flagNoData {
unmarshal := func(obj interface{}) error {
err := protoUnmarshal(p, obj)
@ -458,7 +479,7 @@ func (c *serverConn) run(sctx context.Context) {
continue
}
streams[id] = sh
streams.Store(id, sh)
atomic.AddInt32(&active, 1)
}
// TODO: else we must ignore this for future compat. log this?
@ -518,6 +539,7 @@ func (c *serverConn) run(sctx context.Context) {
// The ttrpc protocol currently does not support the case where
// the server is localClosed but not remoteClosed. Once the server
// is closing, the whole stream may be considered finished
streams.Delete(response.id)
atomic.AddInt32(&active, -1)
}
case err := <-recvErr:
@ -525,14 +547,12 @@ func (c *serverConn) run(sctx context.Context) {
// branch. Basically, it means that we are no longer receiving
// requests due to a terminal error.
recvErr = nil // connection is now "closing"
if err == io.EOF || err == io.ErrUnexpectedEOF {
if err == io.EOF || err == io.ErrUnexpectedEOF || errors.Is(err, syscall.ECONNRESET) {
// The client went away and we should stop processing
// requests, so that the client connection is closed
return
}
if err != nil {
logrus.WithError(err).Error("error receiving message")
}
logrus.WithError(err).Error("error receiving message")
// else, initiate shutdown
case <-shutdown:
return

View File

@ -46,24 +46,36 @@ var (
ErrNotFound = errors.New("not found")
)
// Any contains an arbitrary protcol buffer message along with its type.
//
// While there is google.golang.org/protobuf/types/known/anypb.Any,
// we'd like to have our own to hide the underlying protocol buffer
// implementations from containerd clients.
//
// https://developers.google.com/protocol-buffers/docs/proto3#any
type Any interface {
// GetTypeUrl returns a URL/resource name that uniquely identifies
// the type of the serialized protocol buffer message.
GetTypeUrl() string
// GetValue returns a valid serialized protocol buffer of the type that
// GetTypeUrl() indicates.
GetValue() []byte
}
type any struct {
type anyType struct {
typeURL string
value []byte
}
func (a *any) GetTypeUrl() string {
func (a *anyType) GetTypeUrl() string {
if a == nil {
return ""
}
return a.typeURL
}
func (a *any) GetValue() []byte {
func (a *anyType) GetValue() []byte {
if a == nil {
return nil
}
@ -150,7 +162,7 @@ func MarshalAny(v interface{}) (Any, error) {
if err != nil {
return nil, err
}
return &any{
return &anyType{
typeURL: url,
value: data,
}, nil