vendor: update buildkit

Signed-off-by: CrazyMax <crazy-max@users.noreply.github.com>
This commit is contained in:
CrazyMax
2023-02-10 18:19:57 +01:00
parent b1949b7388
commit 8311b0963a
433 changed files with 34791 additions and 13411 deletions

File diff suppressed because it is too large Load Diff

View File

@ -18,7 +18,6 @@ syntax = "proto3";
package containerd.services.content.v1;
import weak "gogoproto/gogo.proto";
import "google/protobuf/field_mask.proto";
import "google/protobuf/timestamp.proto";
import "google/protobuf/empty.proto";
@ -92,16 +91,16 @@ service Content {
message Info {
// Digest is the hash identity of the blob.
string digest = 1 [(gogoproto.customtype) = "github.com/opencontainers/go-digest.Digest", (gogoproto.nullable) = false];
string digest = 1;
// Size is the total number of bytes in the blob.
int64 size = 2;
// CreatedAt provides the time at which the blob was committed.
google.protobuf.Timestamp created_at = 3 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
google.protobuf.Timestamp created_at = 3;
// UpdatedAt provides the time the info was last updated.
google.protobuf.Timestamp updated_at = 4 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
google.protobuf.Timestamp updated_at = 4;
// Labels are arbitrary data on snapshots.
//
@ -110,15 +109,15 @@ message Info {
}
message InfoRequest {
string digest = 1 [(gogoproto.customtype) = "github.com/opencontainers/go-digest.Digest", (gogoproto.nullable) = false];
string digest = 1;
}
message InfoResponse {
Info info = 1 [(gogoproto.nullable) = false];
Info info = 1;
}
message UpdateRequest {
Info info = 1 [(gogoproto.nullable) = false];
Info info = 1;
// UpdateMask specifies which fields to perform the update on. If empty,
// the operation applies to all fields.
@ -130,7 +129,7 @@ message UpdateRequest {
}
message UpdateResponse {
Info info = 1 [(gogoproto.nullable) = false];
Info info = 1;
}
message ListContentRequest {
@ -141,26 +140,26 @@ message ListContentRequest {
// filters. Expanded, containers that match the following will be
// returned:
//
// filters[0] or filters[1] or ... or filters[n-1] or filters[n]
// filters[0] or filters[1] or ... or filters[n-1] or filters[n]
//
// If filters is zero-length or nil, all items will be returned.
repeated string filters = 1;
}
message ListContentResponse {
repeated Info info = 1 [(gogoproto.nullable) = false];
repeated Info info = 1;
}
message DeleteContentRequest {
// Digest specifies which content to delete.
string digest = 1 [(gogoproto.customtype) = "github.com/opencontainers/go-digest.Digest", (gogoproto.nullable) = false];
string digest = 1;
}
// ReadContentRequest defines the fields that make up a request to read a portion of
// data from a stored object.
message ReadContentRequest {
// Digest is the hash identity to read.
string digest = 1 [(gogoproto.customtype) = "github.com/opencontainers/go-digest.Digest", (gogoproto.nullable) = false];
string digest = 1;
// Offset specifies the number of bytes from the start at which to begin
// the read. If zero or less, the read will be from the start. This uses
@ -179,12 +178,12 @@ message ReadContentResponse {
}
message Status {
google.protobuf.Timestamp started_at = 1 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
google.protobuf.Timestamp updated_at = 2 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
google.protobuf.Timestamp started_at = 1;
google.protobuf.Timestamp updated_at = 2;
string ref = 3;
int64 offset = 4;
int64 total = 5;
string expected = 6 [(gogoproto.customtype) = "github.com/opencontainers/go-digest.Digest", (gogoproto.nullable) = false];
string expected = 6;
}
@ -201,17 +200,14 @@ message ListStatusesRequest {
}
message ListStatusesResponse {
repeated Status statuses = 1 [(gogoproto.nullable) = false];
repeated Status statuses = 1;
}
// WriteAction defines the behavior of a WriteRequest.
enum WriteAction {
option (gogoproto.goproto_enum_prefix) = false;
option (gogoproto.enum_customname) = "WriteAction";
// WriteActionStat instructs the writer to return the current status while
// holding the lock on the write.
STAT = 0 [(gogoproto.enumvalue_customname) = "WriteActionStat"];
STAT = 0;
// WriteActionWrite sets the action for the write request to write data.
//
@ -219,7 +215,7 @@ enum WriteAction {
// transaction will be left open for further writes.
//
// This is the default.
WRITE = 1 [(gogoproto.enumvalue_customname) = "WriteActionWrite"];
WRITE = 1;
// WriteActionCommit will write any outstanding data in the message and
// commit the write, storing it under the digest.
@ -228,7 +224,7 @@ enum WriteAction {
// commit it.
//
// This action will always terminate the write.
COMMIT = 2 [(gogoproto.enumvalue_customname) = "WriteActionCommit"];
COMMIT = 2;
}
// WriteContentRequest writes data to the request ref at offset.
@ -269,7 +265,7 @@ message WriteContentRequest {
// Only the latest version will be used to check the content against the
// digest. It is only required to include it on a single message, before or
// with the commit action message.
string expected = 4 [(gogoproto.customtype) = "github.com/opencontainers/go-digest.Digest", (gogoproto.nullable) = false];
string expected = 4;
// Offset specifies the number of bytes from the start at which to begin
// the write. For most implementations, this means from the start of the
@ -304,13 +300,13 @@ message WriteContentResponse {
//
// This must be set for stat and commit write actions. All other write
// actions may omit this.
google.protobuf.Timestamp started_at = 2 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
google.protobuf.Timestamp started_at = 2;
// UpdatedAt provides the last time of a successful write.
//
// This must be set for stat and commit write actions. All other write
// actions may omit this.
google.protobuf.Timestamp updated_at = 3 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
google.protobuf.Timestamp updated_at = 3;
// Offset is the current committed size for the write.
int64 offset = 4;
@ -326,7 +322,7 @@ message WriteContentResponse {
// Digest, if present, includes the digest up to the currently committed
// bytes. If action is commit, this field will be set. It is implementation
// defined if this is set for other actions.
string digest = 6 [(gogoproto.customtype) = "github.com/opencontainers/go-digest.Digest", (gogoproto.nullable) = false];
string digest = 6;
}
message AbortRequest {

View File

@ -0,0 +1,569 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions:
// - protoc-gen-go-grpc v1.2.0
// - protoc v3.20.1
// source: github.com/containerd/containerd/api/services/content/v1/content.proto
package content
import (
context "context"
grpc "google.golang.org/grpc"
codes "google.golang.org/grpc/codes"
status "google.golang.org/grpc/status"
emptypb "google.golang.org/protobuf/types/known/emptypb"
)
// This is a compile-time assertion to ensure that this generated file
// is compatible with the grpc package it is being compiled against.
// Requires gRPC-Go v1.32.0 or later.
const _ = grpc.SupportPackageIsVersion7
// ContentClient is the client API for Content service.
//
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
type ContentClient interface {
// Info returns information about a committed object.
//
// This call can be used for getting the size of content and checking for
// existence.
Info(ctx context.Context, in *InfoRequest, opts ...grpc.CallOption) (*InfoResponse, error)
// Update updates content metadata.
//
// This call can be used to manage the mutable content labels. The
// immutable metadata such as digest, size, and committed at cannot
// be updated.
Update(ctx context.Context, in *UpdateRequest, opts ...grpc.CallOption) (*UpdateResponse, error)
// List streams the entire set of content as Info objects and closes the
// stream.
//
// Typically, this will yield a large response, chunked into messages.
// Clients should make provisions to ensure they can handle the entire data
// set.
List(ctx context.Context, in *ListContentRequest, opts ...grpc.CallOption) (Content_ListClient, error)
// Delete will delete the referenced object.
Delete(ctx context.Context, in *DeleteContentRequest, opts ...grpc.CallOption) (*emptypb.Empty, error)
// Read allows one to read an object based on the offset into the content.
//
// The requested data may be returned in one or more messages.
Read(ctx context.Context, in *ReadContentRequest, opts ...grpc.CallOption) (Content_ReadClient, error)
// Status returns the status for a single reference.
Status(ctx context.Context, in *StatusRequest, opts ...grpc.CallOption) (*StatusResponse, error)
// ListStatuses returns the status of ongoing object ingestions, started via
// Write.
//
// Only those matching the regular expression will be provided in the
// response. If the provided regular expression is empty, all ingestions
// will be provided.
ListStatuses(ctx context.Context, in *ListStatusesRequest, opts ...grpc.CallOption) (*ListStatusesResponse, error)
// Write begins or resumes writes to a resource identified by a unique ref.
// Only one active stream may exist at a time for each ref.
//
// Once a write stream has started, it may only write to a single ref, thus
// once a stream is started, the ref may be omitted on subsequent writes.
//
// For any write transaction represented by a ref, only a single write may
// be made to a given offset. If overlapping writes occur, it is an error.
// Writes should be sequential and implementations may throw an error if
// this is required.
//
// If expected_digest is set and already part of the content store, the
// write will fail.
//
// When completed, the commit flag should be set to true. If expected size
// or digest is set, the content will be validated against those values.
Write(ctx context.Context, opts ...grpc.CallOption) (Content_WriteClient, error)
// Abort cancels the ongoing write named in the request. Any resources
// associated with the write will be collected.
Abort(ctx context.Context, in *AbortRequest, opts ...grpc.CallOption) (*emptypb.Empty, error)
}
type contentClient struct {
cc grpc.ClientConnInterface
}
func NewContentClient(cc grpc.ClientConnInterface) ContentClient {
return &contentClient{cc}
}
func (c *contentClient) Info(ctx context.Context, in *InfoRequest, opts ...grpc.CallOption) (*InfoResponse, error) {
out := new(InfoResponse)
err := c.cc.Invoke(ctx, "/containerd.services.content.v1.Content/Info", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *contentClient) Update(ctx context.Context, in *UpdateRequest, opts ...grpc.CallOption) (*UpdateResponse, error) {
out := new(UpdateResponse)
err := c.cc.Invoke(ctx, "/containerd.services.content.v1.Content/Update", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *contentClient) List(ctx context.Context, in *ListContentRequest, opts ...grpc.CallOption) (Content_ListClient, error) {
stream, err := c.cc.NewStream(ctx, &Content_ServiceDesc.Streams[0], "/containerd.services.content.v1.Content/List", opts...)
if err != nil {
return nil, err
}
x := &contentListClient{stream}
if err := x.ClientStream.SendMsg(in); err != nil {
return nil, err
}
if err := x.ClientStream.CloseSend(); err != nil {
return nil, err
}
return x, nil
}
type Content_ListClient interface {
Recv() (*ListContentResponse, error)
grpc.ClientStream
}
type contentListClient struct {
grpc.ClientStream
}
func (x *contentListClient) Recv() (*ListContentResponse, error) {
m := new(ListContentResponse)
if err := x.ClientStream.RecvMsg(m); err != nil {
return nil, err
}
return m, nil
}
func (c *contentClient) Delete(ctx context.Context, in *DeleteContentRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) {
out := new(emptypb.Empty)
err := c.cc.Invoke(ctx, "/containerd.services.content.v1.Content/Delete", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *contentClient) Read(ctx context.Context, in *ReadContentRequest, opts ...grpc.CallOption) (Content_ReadClient, error) {
stream, err := c.cc.NewStream(ctx, &Content_ServiceDesc.Streams[1], "/containerd.services.content.v1.Content/Read", opts...)
if err != nil {
return nil, err
}
x := &contentReadClient{stream}
if err := x.ClientStream.SendMsg(in); err != nil {
return nil, err
}
if err := x.ClientStream.CloseSend(); err != nil {
return nil, err
}
return x, nil
}
type Content_ReadClient interface {
Recv() (*ReadContentResponse, error)
grpc.ClientStream
}
type contentReadClient struct {
grpc.ClientStream
}
func (x *contentReadClient) Recv() (*ReadContentResponse, error) {
m := new(ReadContentResponse)
if err := x.ClientStream.RecvMsg(m); err != nil {
return nil, err
}
return m, nil
}
func (c *contentClient) Status(ctx context.Context, in *StatusRequest, opts ...grpc.CallOption) (*StatusResponse, error) {
out := new(StatusResponse)
err := c.cc.Invoke(ctx, "/containerd.services.content.v1.Content/Status", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *contentClient) ListStatuses(ctx context.Context, in *ListStatusesRequest, opts ...grpc.CallOption) (*ListStatusesResponse, error) {
out := new(ListStatusesResponse)
err := c.cc.Invoke(ctx, "/containerd.services.content.v1.Content/ListStatuses", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *contentClient) Write(ctx context.Context, opts ...grpc.CallOption) (Content_WriteClient, error) {
stream, err := c.cc.NewStream(ctx, &Content_ServiceDesc.Streams[2], "/containerd.services.content.v1.Content/Write", opts...)
if err != nil {
return nil, err
}
x := &contentWriteClient{stream}
return x, nil
}
type Content_WriteClient interface {
Send(*WriteContentRequest) error
Recv() (*WriteContentResponse, error)
grpc.ClientStream
}
type contentWriteClient struct {
grpc.ClientStream
}
func (x *contentWriteClient) Send(m *WriteContentRequest) error {
return x.ClientStream.SendMsg(m)
}
func (x *contentWriteClient) Recv() (*WriteContentResponse, error) {
m := new(WriteContentResponse)
if err := x.ClientStream.RecvMsg(m); err != nil {
return nil, err
}
return m, nil
}
func (c *contentClient) Abort(ctx context.Context, in *AbortRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) {
out := new(emptypb.Empty)
err := c.cc.Invoke(ctx, "/containerd.services.content.v1.Content/Abort", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
// ContentServer is the server API for Content service.
// All implementations must embed UnimplementedContentServer
// for forward compatibility
type ContentServer interface {
// Info returns information about a committed object.
//
// This call can be used for getting the size of content and checking for
// existence.
Info(context.Context, *InfoRequest) (*InfoResponse, error)
// Update updates content metadata.
//
// This call can be used to manage the mutable content labels. The
// immutable metadata such as digest, size, and committed at cannot
// be updated.
Update(context.Context, *UpdateRequest) (*UpdateResponse, error)
// List streams the entire set of content as Info objects and closes the
// stream.
//
// Typically, this will yield a large response, chunked into messages.
// Clients should make provisions to ensure they can handle the entire data
// set.
List(*ListContentRequest, Content_ListServer) error
// Delete will delete the referenced object.
Delete(context.Context, *DeleteContentRequest) (*emptypb.Empty, error)
// Read allows one to read an object based on the offset into the content.
//
// The requested data may be returned in one or more messages.
Read(*ReadContentRequest, Content_ReadServer) error
// Status returns the status for a single reference.
Status(context.Context, *StatusRequest) (*StatusResponse, error)
// ListStatuses returns the status of ongoing object ingestions, started via
// Write.
//
// Only those matching the regular expression will be provided in the
// response. If the provided regular expression is empty, all ingestions
// will be provided.
ListStatuses(context.Context, *ListStatusesRequest) (*ListStatusesResponse, error)
// Write begins or resumes writes to a resource identified by a unique ref.
// Only one active stream may exist at a time for each ref.
//
// Once a write stream has started, it may only write to a single ref, thus
// once a stream is started, the ref may be omitted on subsequent writes.
//
// For any write transaction represented by a ref, only a single write may
// be made to a given offset. If overlapping writes occur, it is an error.
// Writes should be sequential and implementations may throw an error if
// this is required.
//
// If expected_digest is set and already part of the content store, the
// write will fail.
//
// When completed, the commit flag should be set to true. If expected size
// or digest is set, the content will be validated against those values.
Write(Content_WriteServer) error
// Abort cancels the ongoing write named in the request. Any resources
// associated with the write will be collected.
Abort(context.Context, *AbortRequest) (*emptypb.Empty, error)
mustEmbedUnimplementedContentServer()
}
// UnimplementedContentServer must be embedded to have forward compatible implementations.
type UnimplementedContentServer struct {
}
func (UnimplementedContentServer) Info(context.Context, *InfoRequest) (*InfoResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method Info not implemented")
}
func (UnimplementedContentServer) Update(context.Context, *UpdateRequest) (*UpdateResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method Update not implemented")
}
func (UnimplementedContentServer) List(*ListContentRequest, Content_ListServer) error {
return status.Errorf(codes.Unimplemented, "method List not implemented")
}
func (UnimplementedContentServer) Delete(context.Context, *DeleteContentRequest) (*emptypb.Empty, error) {
return nil, status.Errorf(codes.Unimplemented, "method Delete not implemented")
}
func (UnimplementedContentServer) Read(*ReadContentRequest, Content_ReadServer) error {
return status.Errorf(codes.Unimplemented, "method Read not implemented")
}
func (UnimplementedContentServer) Status(context.Context, *StatusRequest) (*StatusResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method Status not implemented")
}
func (UnimplementedContentServer) ListStatuses(context.Context, *ListStatusesRequest) (*ListStatusesResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method ListStatuses not implemented")
}
func (UnimplementedContentServer) Write(Content_WriteServer) error {
return status.Errorf(codes.Unimplemented, "method Write not implemented")
}
func (UnimplementedContentServer) Abort(context.Context, *AbortRequest) (*emptypb.Empty, error) {
return nil, status.Errorf(codes.Unimplemented, "method Abort not implemented")
}
func (UnimplementedContentServer) mustEmbedUnimplementedContentServer() {}
// UnsafeContentServer may be embedded to opt out of forward compatibility for this service.
// Use of this interface is not recommended, as added methods to ContentServer will
// result in compilation errors.
type UnsafeContentServer interface {
mustEmbedUnimplementedContentServer()
}
func RegisterContentServer(s grpc.ServiceRegistrar, srv ContentServer) {
s.RegisterService(&Content_ServiceDesc, srv)
}
func _Content_Info_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(InfoRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(ContentServer).Info(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/containerd.services.content.v1.Content/Info",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ContentServer).Info(ctx, req.(*InfoRequest))
}
return interceptor(ctx, in, info, handler)
}
func _Content_Update_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(UpdateRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(ContentServer).Update(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/containerd.services.content.v1.Content/Update",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ContentServer).Update(ctx, req.(*UpdateRequest))
}
return interceptor(ctx, in, info, handler)
}
func _Content_List_Handler(srv interface{}, stream grpc.ServerStream) error {
m := new(ListContentRequest)
if err := stream.RecvMsg(m); err != nil {
return err
}
return srv.(ContentServer).List(m, &contentListServer{stream})
}
type Content_ListServer interface {
Send(*ListContentResponse) error
grpc.ServerStream
}
type contentListServer struct {
grpc.ServerStream
}
func (x *contentListServer) Send(m *ListContentResponse) error {
return x.ServerStream.SendMsg(m)
}
func _Content_Delete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(DeleteContentRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(ContentServer).Delete(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/containerd.services.content.v1.Content/Delete",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ContentServer).Delete(ctx, req.(*DeleteContentRequest))
}
return interceptor(ctx, in, info, handler)
}
func _Content_Read_Handler(srv interface{}, stream grpc.ServerStream) error {
m := new(ReadContentRequest)
if err := stream.RecvMsg(m); err != nil {
return err
}
return srv.(ContentServer).Read(m, &contentReadServer{stream})
}
type Content_ReadServer interface {
Send(*ReadContentResponse) error
grpc.ServerStream
}
type contentReadServer struct {
grpc.ServerStream
}
func (x *contentReadServer) Send(m *ReadContentResponse) error {
return x.ServerStream.SendMsg(m)
}
func _Content_Status_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(StatusRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(ContentServer).Status(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/containerd.services.content.v1.Content/Status",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ContentServer).Status(ctx, req.(*StatusRequest))
}
return interceptor(ctx, in, info, handler)
}
func _Content_ListStatuses_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ListStatusesRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(ContentServer).ListStatuses(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/containerd.services.content.v1.Content/ListStatuses",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ContentServer).ListStatuses(ctx, req.(*ListStatusesRequest))
}
return interceptor(ctx, in, info, handler)
}
func _Content_Write_Handler(srv interface{}, stream grpc.ServerStream) error {
return srv.(ContentServer).Write(&contentWriteServer{stream})
}
type Content_WriteServer interface {
Send(*WriteContentResponse) error
Recv() (*WriteContentRequest, error)
grpc.ServerStream
}
type contentWriteServer struct {
grpc.ServerStream
}
func (x *contentWriteServer) Send(m *WriteContentResponse) error {
return x.ServerStream.SendMsg(m)
}
func (x *contentWriteServer) Recv() (*WriteContentRequest, error) {
m := new(WriteContentRequest)
if err := x.ServerStream.RecvMsg(m); err != nil {
return nil, err
}
return m, nil
}
func _Content_Abort_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(AbortRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(ContentServer).Abort(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/containerd.services.content.v1.Content/Abort",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ContentServer).Abort(ctx, req.(*AbortRequest))
}
return interceptor(ctx, in, info, handler)
}
// Content_ServiceDesc is the grpc.ServiceDesc for Content service.
// It's only intended for direct use with grpc.RegisterService,
// and not to be introspected or modified (even as a copy)
var Content_ServiceDesc = grpc.ServiceDesc{
ServiceName: "containerd.services.content.v1.Content",
HandlerType: (*ContentServer)(nil),
Methods: []grpc.MethodDesc{
{
MethodName: "Info",
Handler: _Content_Info_Handler,
},
{
MethodName: "Update",
Handler: _Content_Update_Handler,
},
{
MethodName: "Delete",
Handler: _Content_Delete_Handler,
},
{
MethodName: "Status",
Handler: _Content_Status_Handler,
},
{
MethodName: "ListStatuses",
Handler: _Content_ListStatuses_Handler,
},
{
MethodName: "Abort",
Handler: _Content_Abort_Handler,
},
},
Streams: []grpc.StreamDesc{
{
StreamName: "List",
Handler: _Content_List_Handler,
ServerStreams: true,
},
{
StreamName: "Read",
Handler: _Content_Read_Handler,
ServerStreams: true,
},
{
StreamName: "Write",
Handler: _Content_Write_Handler,
ServerStreams: true,
ClientStreams: true,
},
},
Metadata: "github.com/containerd/containerd/api/services/content/v1/content.proto",
}

View File

@ -0,0 +1,28 @@
//go:build gofuzz
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package compression
import (
"bytes"
)
func FuzzDecompressStream(data []byte) int {
_, _ = DecompressStream(bytes.NewReader(data))
return 1
}

View File

@ -25,6 +25,26 @@ import (
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)
// Store combines the methods of content-oriented interfaces into a set that
// are commonly provided by complete implementations.
//
// Overall content lifecycle:
// - Ingester is used to initiate a write operation (aka ingestion)
// - IngestManager is used to manage (e.g. list, abort) active ingestions
// - Once an ingestion is complete (see Writer.Commit), Provider is used to
// query a single piece of content by its digest
// - Manager is used to manage (e.g. list, delete) previously committed content
//
// Note that until ingestion is complete, its content is not visible through
// Provider or Manager. Once ingestion is complete, it is no longer exposed
// through IngestManager.
type Store interface {
Manager
Provider
IngestManager
Ingester
}
// ReaderAt extends the standard io.ReaderAt interface with reporting of Size and io.Closer
type ReaderAt interface {
io.ReaderAt
@ -42,10 +62,30 @@ type Provider interface {
// Ingester writes content
type Ingester interface {
// Some implementations require WithRef to be included in opts.
// Writer initiates a writing operation (aka ingestion). A single ingestion
// is uniquely identified by its ref, provided using a WithRef option.
// Writer can be called multiple times with the same ref to access the same
// ingestion.
// Once all the data is written, use Writer.Commit to complete the ingestion.
Writer(ctx context.Context, opts ...WriterOpt) (Writer, error)
}
// IngestManager provides methods for managing ingestions. An ingestion is a
// not-yet-complete writing operation initiated using Ingester and identified
// by a ref string.
type IngestManager interface {
// Status returns the status of the provided ref.
Status(ctx context.Context, ref string) (Status, error)
// ListStatuses returns the status of any active ingestions whose ref match
// the provided regular expression. If empty, all active ingestions will be
// returned.
ListStatuses(ctx context.Context, filters ...string) ([]Status, error)
// Abort completely cancels the ingest operation targeted by ref.
Abort(ctx context.Context, ref string) error
}
// Info holds content specific information
//
// TODO(stevvooe): Consider a very different name for this struct. Info is way
@ -58,7 +98,7 @@ type Info struct {
Labels map[string]string
}
// Status of a content operation
// Status of a content operation (i.e. an ingestion)
type Status struct {
Ref string
Offset int64
@ -94,21 +134,7 @@ type Manager interface {
Delete(ctx context.Context, dgst digest.Digest) error
}
// IngestManager provides methods for managing ingests.
type IngestManager interface {
// Status returns the status of the provided ref.
Status(ctx context.Context, ref string) (Status, error)
// ListStatuses returns the status of any active ingestions whose ref match the
// provided regular expression. If empty, all active ingestions will be
// returned.
ListStatuses(ctx context.Context, filters ...string) ([]Status, error)
// Abort completely cancels the ingest operation targeted by ref.
Abort(ctx context.Context, ref string) error
}
// Writer handles the write of content into a content store
// Writer handles writing of content into a content store
type Writer interface {
// Close closes the writer, if the writer has not been
// committed this allows resuming or aborting.
@ -131,15 +157,6 @@ type Writer interface {
Truncate(size int64) error
}
// Store combines the methods of content-oriented interfaces into a set that
// are commonly provided by complete implementations.
type Store interface {
Manager
Provider
IngestManager
Ingester
}
// Opt is used to alter the mutable properties of content
type Opt func(*Info) error

View File

@ -43,10 +43,16 @@ var bufPool = sync.Pool{
},
}
type reader interface {
Reader() io.Reader
}
// NewReader returns a io.Reader from a ReaderAt
func NewReader(ra ReaderAt) io.Reader {
rd := io.NewSectionReader(ra, 0, ra.Size())
return rd
if rd, ok := ra.(reader); ok {
return rd.Reader()
}
return io.NewSectionReader(ra, 0, ra.Size())
}
// ReadBlob retrieves the entire contents of the blob from the provider.

View File

@ -0,0 +1,76 @@
//go:build gofuzz
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package local
import (
"bufio"
"bytes"
"context"
_ "crypto/sha256"
"io"
"testing"
"github.com/opencontainers/go-digest"
"github.com/containerd/containerd/content"
)
func FuzzContentStoreWriter(data []byte) int {
t := &testing.T{}
ctx := context.Background()
ctx, _, cs, cleanup := contentStoreEnv(t)
defer cleanup()
cw, err := cs.Writer(ctx, content.WithRef("myref"))
if err != nil {
return 0
}
if err := cw.Close(); err != nil {
return 0
}
// reopen, so we can test things
cw, err = cs.Writer(ctx, content.WithRef("myref"))
if err != nil {
return 0
}
err = checkCopyFuzz(int64(len(data)), cw, bufio.NewReader(io.NopCloser(bytes.NewReader(data))))
if err != nil {
return 0
}
expected := digest.FromBytes(data)
if err = cw.Commit(ctx, int64(len(data)), expected); err != nil {
return 0
}
return 1
}
func checkCopyFuzz(size int64, dst io.Writer, src io.Reader) error {
nn, err := io.Copy(dst, src)
if err != nil {
return err
}
if nn != size {
return err
}
return nil
}

View File

@ -18,6 +18,7 @@ package local
import (
"fmt"
"io"
"os"
"github.com/containerd/containerd/content"
@ -65,3 +66,7 @@ func (ra sizeReaderAt) Size() int64 {
func (ra sizeReaderAt) Close() error {
return ra.fp.Close()
}
func (ra sizeReaderAt) Reader() io.Reader {
return io.LimitReader(ra.fp, ra.size)
}

View File

@ -34,7 +34,7 @@ import (
"github.com/containerd/containerd/log"
"github.com/sirupsen/logrus"
digest "github.com/opencontainers/go-digest"
"github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)
@ -262,7 +262,7 @@ func (s *store) Walk(ctx context.Context, fn content.WalkFunc, fs ...string) err
return nil
}
dgst := digest.NewDigestFromHex(alg.String(), filepath.Base(path))
dgst := digest.NewDigestFromEncoded(alg, filepath.Base(path))
if err := dgst.Validate(); err != nil {
// log error but don't report
log.L.WithError(err).WithField("path", path).Error("invalid digest for blob path")
@ -505,6 +505,7 @@ func (s *store) resumeStatus(ref string, total int64, digester digest.Digester)
return status, fmt.Errorf("provided total differs from status: %v != %v", total, status.Total)
}
//nolint:dupword
// TODO(stevvooe): slow slow slow!!, send to goroutine or use resumable hashes
fp, err := os.Open(data)
if err != nil {
@ -628,14 +629,14 @@ func (s *store) blobPath(dgst digest.Digest) (string, error) {
return "", fmt.Errorf("cannot calculate blob path from invalid digest: %v: %w", err, errdefs.ErrInvalidArgument)
}
return filepath.Join(s.root, "blobs", dgst.Algorithm().String(), dgst.Hex()), nil
return filepath.Join(s.root, "blobs", dgst.Algorithm().String(), dgst.Encoded()), nil
}
func (s *store) ingestRoot(ref string) string {
// we take a digest of the ref to keep the ingest paths constant length.
// Note that this is not the current or potential digest of incoming content.
dgst := digest.FromString(ref)
return filepath.Join(s.root, "ingest", dgst.Hex())
return filepath.Join(s.root, "ingest", dgst.Encoded())
}
// ingestPaths are returned. The paths are the following:

View File

@ -1,5 +1,4 @@
//go:build darwin || freebsd || netbsd
// +build darwin freebsd netbsd
/*
Copyright The containerd Authors.

View File

@ -1,5 +1,4 @@
//go:build openbsd
// +build openbsd
/*
Copyright The containerd Authors.

View File

@ -1,5 +1,4 @@
//go:build linux || solaris
// +build linux solaris
/*
Copyright The containerd Authors.

View File

@ -0,0 +1,38 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package local
import (
"context"
"testing"
"github.com/containerd/containerd/content"
)
func contentStoreEnv(t testing.TB) (context.Context, string, content.Store, func()) {
tmpdir := t.TempDir()
cs, err := NewStore(tmpdir)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
return ctx, tmpdir, cs, func() {
cancel()
}
}

View File

@ -36,9 +36,9 @@ func (ra *remoteReaderAt) Size() int64 {
func (ra *remoteReaderAt) ReadAt(p []byte, off int64) (n int, err error) {
rr := &contentapi.ReadContentRequest{
Digest: ra.digest,
Digest: ra.digest.String(),
Offset: off,
Size_: int64(len(p)),
Size: int64(len(p)),
}
// we need a child context with cancel, or the eventually called
// grpc.NewStream will leak the goroutine until the whole thing is cleared.

View File

@ -23,7 +23,8 @@ import (
contentapi "github.com/containerd/containerd/api/services/content/v1"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/errdefs"
protobuftypes "github.com/gogo/protobuf/types"
"github.com/containerd/containerd/protobuf"
protobuftypes "github.com/containerd/containerd/protobuf/types"
digest "github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)
@ -42,7 +43,7 @@ func NewContentStore(client contentapi.ContentClient) content.Store {
func (pcs *proxyContentStore) Info(ctx context.Context, dgst digest.Digest) (content.Info, error) {
resp, err := pcs.client.Info(ctx, &contentapi.InfoRequest{
Digest: dgst,
Digest: dgst.String(),
})
if err != nil {
return content.Info{}, errdefs.FromGRPC(err)
@ -81,7 +82,7 @@ func (pcs *proxyContentStore) Walk(ctx context.Context, fn content.WalkFunc, fil
func (pcs *proxyContentStore) Delete(ctx context.Context, dgst digest.Digest) error {
if _, err := pcs.client.Delete(ctx, &contentapi.DeleteContentRequest{
Digest: dgst,
Digest: dgst.String(),
}); err != nil {
return errdefs.FromGRPC(err)
}
@ -115,17 +116,17 @@ func (pcs *proxyContentStore) Status(ctx context.Context, ref string) (content.S
status := resp.Status
return content.Status{
Ref: status.Ref,
StartedAt: status.StartedAt,
UpdatedAt: status.UpdatedAt,
StartedAt: protobuf.FromTimestamp(status.StartedAt),
UpdatedAt: protobuf.FromTimestamp(status.UpdatedAt),
Offset: status.Offset,
Total: status.Total,
Expected: status.Expected,
Expected: digest.Digest(status.Expected),
}, nil
}
func (pcs *proxyContentStore) Update(ctx context.Context, info content.Info, fieldpaths ...string) (content.Info, error) {
resp, err := pcs.client.Update(ctx, &contentapi.UpdateRequest{
Info: infoToGRPC(info),
Info: infoToGRPC(&info),
UpdateMask: &protobuftypes.FieldMask{
Paths: fieldpaths,
},
@ -148,11 +149,11 @@ func (pcs *proxyContentStore) ListStatuses(ctx context.Context, filters ...strin
for _, status := range resp.Statuses {
statuses = append(statuses, content.Status{
Ref: status.Ref,
StartedAt: status.StartedAt,
UpdatedAt: status.UpdatedAt,
StartedAt: protobuf.FromTimestamp(status.StartedAt),
UpdatedAt: protobuf.FromTimestamp(status.UpdatedAt),
Offset: status.Offset,
Total: status.Total,
Expected: status.Expected,
Expected: digest.Digest(status.Expected),
})
}
@ -197,10 +198,10 @@ func (pcs *proxyContentStore) negotiate(ctx context.Context, ref string, size in
}
if err := wrclient.Send(&contentapi.WriteContentRequest{
Action: contentapi.WriteActionStat,
Action: contentapi.WriteAction_STAT,
Ref: ref,
Total: size,
Expected: expected,
Expected: expected.String(),
}); err != nil {
return nil, 0, err
}
@ -213,22 +214,22 @@ func (pcs *proxyContentStore) negotiate(ctx context.Context, ref string, size in
return wrclient, resp.Offset, nil
}
func infoToGRPC(info content.Info) contentapi.Info {
return contentapi.Info{
Digest: info.Digest,
Size_: info.Size,
CreatedAt: info.CreatedAt,
UpdatedAt: info.UpdatedAt,
func infoToGRPC(info *content.Info) *contentapi.Info {
return &contentapi.Info{
Digest: info.Digest.String(),
Size: info.Size,
CreatedAt: protobuf.ToTimestamp(info.CreatedAt),
UpdatedAt: protobuf.ToTimestamp(info.UpdatedAt),
Labels: info.Labels,
}
}
func infoFromGRPC(info contentapi.Info) content.Info {
func infoFromGRPC(info *contentapi.Info) content.Info {
return content.Info{
Digest: info.Digest,
Size: info.Size_,
CreatedAt: info.CreatedAt,
UpdatedAt: info.UpdatedAt,
Digest: digest.Digest(info.Digest),
Size: info.Size,
CreatedAt: protobuf.FromTimestamp(info.CreatedAt),
UpdatedAt: protobuf.FromTimestamp(info.UpdatedAt),
Labels: info.Labels,
}
}

View File

@ -24,6 +24,7 @@ import (
contentapi "github.com/containerd/containerd/api/services/content/v1"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/protobuf"
digest "github.com/opencontainers/go-digest"
)
@ -45,7 +46,7 @@ func (rw *remoteWriter) send(req *contentapi.WriteContentRequest) (*contentapi.W
if err == nil {
// try to keep these in sync
if resp.Digest != "" {
rw.digest = resp.Digest
rw.digest = digest.Digest(resp.Digest)
}
}
@ -54,7 +55,7 @@ func (rw *remoteWriter) send(req *contentapi.WriteContentRequest) (*contentapi.W
func (rw *remoteWriter) Status() (content.Status, error) {
resp, err := rw.send(&contentapi.WriteContentRequest{
Action: contentapi.WriteActionStat,
Action: contentapi.WriteAction_STAT,
})
if err != nil {
return content.Status{}, fmt.Errorf("error getting writer status: %w", errdefs.FromGRPC(err))
@ -64,8 +65,8 @@ func (rw *remoteWriter) Status() (content.Status, error) {
Ref: rw.ref,
Offset: resp.Offset,
Total: resp.Total,
StartedAt: resp.StartedAt,
UpdatedAt: resp.UpdatedAt,
StartedAt: protobuf.FromTimestamp(resp.StartedAt),
UpdatedAt: protobuf.FromTimestamp(resp.UpdatedAt),
}, nil
}
@ -77,7 +78,7 @@ func (rw *remoteWriter) Write(p []byte) (n int, err error) {
offset := rw.offset
resp, err := rw.send(&contentapi.WriteContentRequest{
Action: contentapi.WriteActionWrite,
Action: contentapi.WriteAction_WRITE,
Offset: offset,
Data: p,
})
@ -92,7 +93,7 @@ func (rw *remoteWriter) Write(p []byte) (n int, err error) {
rw.offset += int64(n)
if resp.Digest != "" {
rw.digest = resp.Digest
rw.digest = digest.Digest(resp.Digest)
}
return
}
@ -112,10 +113,10 @@ func (rw *remoteWriter) Commit(ctx context.Context, size int64, expected digest.
}
}
resp, err := rw.send(&contentapi.WriteContentRequest{
Action: contentapi.WriteActionCommit,
Action: contentapi.WriteAction_COMMIT,
Total: size,
Offset: rw.offset,
Expected: expected,
Expected: expected.String(),
Labels: base.Labels,
})
if err != nil {
@ -126,11 +127,12 @@ func (rw *remoteWriter) Commit(ctx context.Context, size int64, expected digest.
return fmt.Errorf("unexpected size: %v != %v", resp.Offset, size)
}
if expected != "" && resp.Digest != expected {
actual := digest.Digest(resp.Digest)
if expected != "" && actual != expected {
return fmt.Errorf("unexpected digest: %v != %v", resp.Digest, expected)
}
rw.digest = resp.Digest
rw.digest = actual
rw.offset = resp.Offset
return nil
}

View File

@ -1,5 +1,4 @@
//go:build !windows && !darwin
// +build !windows,!darwin
/*
Copyright The containerd Authors.

View File

@ -138,7 +138,7 @@ type platformManifest struct {
// TODO(stevvooe): This violates the current platform agnostic approach to this
// package by returning a specific manifest type. We'll need to refactor this
// to return a manifest descriptor or decide that we want to bring the API in
// this direction because this abstraction is not needed.`
// this direction because this abstraction is not needed.
func Manifest(ctx context.Context, provider content.Provider, image ocispec.Descriptor, platform platforms.MatchComparer) (ocispec.Manifest, error) {
var (
limit = 1
@ -311,7 +311,7 @@ func Check(ctx context.Context, provider content.Provider, image ocispec.Descrip
return false, nil, nil, nil, fmt.Errorf("failed to check image %v: %w", image.Digest, err)
}
// TODO(stevvooe): It is possible that referenced conponents could have
// TODO(stevvooe): It is possible that referenced components could have
// children, but this is rare. For now, we ignore this and only verify
// that manifest components are present.
required = append([]ocispec.Descriptor{mfst.Config}, mfst.Layers...)

View File

@ -38,7 +38,9 @@ const (
MediaTypeDockerSchema2Config = "application/vnd.docker.container.image.v1+json"
MediaTypeDockerSchema2Manifest = "application/vnd.docker.distribution.manifest.v2+json"
MediaTypeDockerSchema2ManifestList = "application/vnd.docker.distribution.manifest.list.v2+json"
// Checkpoint/Restore Media Types
MediaTypeContainerd1Checkpoint = "application/vnd.containerd.container.criu.checkpoint.criu.tar"
MediaTypeContainerd1CheckpointPreDump = "application/vnd.containerd.container.criu.checkpoint.predump.tar"
MediaTypeContainerd1Resource = "application/vnd.containerd.container.resource.tar"
@ -47,9 +49,12 @@ const (
MediaTypeContainerd1CheckpointOptions = "application/vnd.containerd.container.checkpoint.options.v1+proto"
MediaTypeContainerd1CheckpointRuntimeName = "application/vnd.containerd.container.checkpoint.runtime.name"
MediaTypeContainerd1CheckpointRuntimeOptions = "application/vnd.containerd.container.checkpoint.runtime.options+proto"
// Legacy Docker schema1 manifest
// MediaTypeDockerSchema1Manifest is the legacy Docker schema1 manifest
MediaTypeDockerSchema1Manifest = "application/vnd.docker.distribution.manifest.v1+prettyjws"
// Encypted media types
// Encrypted media types
MediaTypeImageLayerEncrypted = ocispec.MediaTypeImageLayer + "+encrypted"
MediaTypeImageLayerGzipEncrypted = ocispec.MediaTypeImageLayerGzip + "+encrypted"
)
@ -93,16 +98,23 @@ func DiffCompression(ctx context.Context, mediaType string) (string, error) {
// parseMediaTypes splits the media type into the base type and
// an array of sorted extensions
func parseMediaTypes(mt string) (string, []string) {
func parseMediaTypes(mt string) (mediaType string, suffixes []string) {
if mt == "" {
return "", []string{}
}
mediaType, ext, ok := strings.Cut(mt, "+")
if !ok {
return mediaType, []string{}
}
s := strings.Split(mt, "+")
ext := s[1:]
sort.Strings(ext)
return s[0], ext
// Splitting the extensions following the mediatype "(+)gzip+encrypted".
// We expect this to be a limited list, so add an arbitrary limit (50).
//
// Note that DiffCompression is only using the last element, so perhaps we
// should split on the last "+" only.
suffixes = strings.SplitN(ext, "+", 50)
sort.Strings(suffixes)
return mediaType, suffixes
}
// IsNonDistributable returns true if the media type is non-distributable.
@ -118,8 +130,7 @@ func IsLayerType(mt string) bool {
}
// Parse Docker media types, strip off any + suffixes first
base, _ := parseMediaTypes(mt)
switch base {
switch base, _ := parseMediaTypes(mt); base {
case MediaTypeDockerSchema2Layer, MediaTypeDockerSchema2LayerGzip,
MediaTypeDockerSchema2LayerForeign, MediaTypeDockerSchema2LayerForeignGzip:
return true

View File

@ -19,3 +19,7 @@ package labels
// LabelUncompressed is added to compressed layer contents.
// The value is digest of the uncompressed content.
const LabelUncompressed = "containerd.io/uncompressed"
// LabelSharedNamespace is added to a namespace to allow that namespaces
// contents to be shared.
const LabelSharedNamespace = "containerd.io/namespace.shareable"

View File

@ -24,15 +24,18 @@ import (
const (
maxSize = 4096
// maximum length of key portion of error message if len of key + len of value > maxSize
keyMaxLen = 64
)
// Validate a label's key and value are under 4096 bytes
func Validate(k, v string) error {
if (len(k) + len(v)) > maxSize {
if len(k) > 10 {
k = k[:10]
total := len(k) + len(v)
if total > maxSize {
if len(k) > keyMaxLen {
k = k[:keyMaxLen]
}
return fmt.Errorf("label key and value greater than maximum size (%d bytes), key: %s: %w", maxSize, k, errdefs.ErrInvalidArgument)
return fmt.Errorf("label key and value length (%d bytes) greater than maximum size (%d bytes), key: %s: %w", total, maxSize, k, errdefs.ErrInvalidArgument)
}
return nil
}

View File

@ -1,5 +1,4 @@
//go:build !windows
// +build !windows
/*
Copyright The containerd Authors.

View File

@ -1,5 +1,4 @@
//go:build !linux
// +build !linux
/*
Copyright The containerd Authors.

View File

@ -1,5 +1,4 @@
//go:build !linux
// +build !linux
/*
Copyright The containerd Authors.

View File

@ -17,14 +17,9 @@
package platforms
import (
"bufio"
"fmt"
"os"
"runtime"
"strings"
"sync"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/log"
)
@ -37,95 +32,12 @@ var cpuVariantOnce sync.Once
func cpuVariant() string {
cpuVariantOnce.Do(func() {
if isArmArch(runtime.GOARCH) {
cpuVariantValue = getCPUVariant()
var err error
cpuVariantValue, err = getCPUVariant()
if err != nil {
log.L.Errorf("Error getCPUVariant for OS %s: %v", runtime.GOOS, err)
}
}
})
return cpuVariantValue
}
// For Linux, the kernel has already detected the ABI, ISA and Features.
// So we don't need to access the ARM registers to detect platform information
// by ourselves. We can just parse these information from /proc/cpuinfo
func getCPUInfo(pattern string) (info string, err error) {
if !isLinuxOS(runtime.GOOS) {
return "", fmt.Errorf("getCPUInfo for OS %s: %w", runtime.GOOS, errdefs.ErrNotImplemented)
}
cpuinfo, err := os.Open("/proc/cpuinfo")
if err != nil {
return "", err
}
defer cpuinfo.Close()
// Start to Parse the Cpuinfo line by line. For SMP SoC, we parse
// the first core is enough.
scanner := bufio.NewScanner(cpuinfo)
for scanner.Scan() {
newline := scanner.Text()
list := strings.Split(newline, ":")
if len(list) > 1 && strings.EqualFold(strings.TrimSpace(list[0]), pattern) {
return strings.TrimSpace(list[1]), nil
}
}
// Check whether the scanner encountered errors
err = scanner.Err()
if err != nil {
return "", err
}
return "", fmt.Errorf("getCPUInfo for pattern: %s: %w", pattern, errdefs.ErrNotFound)
}
func getCPUVariant() string {
if runtime.GOOS == "windows" || runtime.GOOS == "darwin" {
// Windows/Darwin only supports v7 for ARM32 and v8 for ARM64 and so we can use
// runtime.GOARCH to determine the variants
var variant string
switch runtime.GOARCH {
case "arm64":
variant = "v8"
case "arm":
variant = "v7"
default:
variant = "unknown"
}
return variant
}
variant, err := getCPUInfo("Cpu architecture")
if err != nil {
log.L.WithError(err).Error("failure getting variant")
return ""
}
// handle edge case for Raspberry Pi ARMv6 devices (which due to a kernel quirk, report "CPU architecture: 7")
// https://www.raspberrypi.org/forums/viewtopic.php?t=12614
if runtime.GOARCH == "arm" && variant == "7" {
model, err := getCPUInfo("model name")
if err == nil && strings.HasPrefix(strings.ToLower(model), "armv6-compatible") {
variant = "6"
}
}
switch strings.ToLower(variant) {
case "8", "aarch64":
variant = "v8"
case "7", "7m", "?(12)", "?(13)", "?(14)", "?(15)", "?(16)", "?(17)":
variant = "v7"
case "6", "6tej":
variant = "v6"
case "5", "5t", "5te", "5tej":
variant = "v5"
case "4", "4t":
variant = "v4"
case "3":
variant = "v3"
default:
variant = "unknown"
}
return variant
}

View File

@ -0,0 +1,161 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package platforms
import (
"bufio"
"bytes"
"fmt"
"os"
"runtime"
"strings"
"github.com/containerd/containerd/errdefs"
"golang.org/x/sys/unix"
)
// getMachineArch retrieves the machine architecture through system call
func getMachineArch() (string, error) {
var uname unix.Utsname
err := unix.Uname(&uname)
if err != nil {
return "", err
}
arch := string(uname.Machine[:bytes.IndexByte(uname.Machine[:], 0)])
return arch, nil
}
// For Linux, the kernel has already detected the ABI, ISA and Features.
// So we don't need to access the ARM registers to detect platform information
// by ourselves. We can just parse these information from /proc/cpuinfo
func getCPUInfo(pattern string) (info string, err error) {
cpuinfo, err := os.Open("/proc/cpuinfo")
if err != nil {
return "", err
}
defer cpuinfo.Close()
// Start to Parse the Cpuinfo line by line. For SMP SoC, we parse
// the first core is enough.
scanner := bufio.NewScanner(cpuinfo)
for scanner.Scan() {
newline := scanner.Text()
list := strings.Split(newline, ":")
if len(list) > 1 && strings.EqualFold(strings.TrimSpace(list[0]), pattern) {
return strings.TrimSpace(list[1]), nil
}
}
// Check whether the scanner encountered errors
err = scanner.Err()
if err != nil {
return "", err
}
return "", fmt.Errorf("getCPUInfo for pattern %s: %w", pattern, errdefs.ErrNotFound)
}
// getCPUVariantFromArch get CPU variant from arch through a system call
func getCPUVariantFromArch(arch string) (string, error) {
var variant string
arch = strings.ToLower(arch)
if arch == "aarch64" {
variant = "8"
} else if arch[0:4] == "armv" && len(arch) >= 5 {
//Valid arch format is in form of armvXx
switch arch[3:5] {
case "v8":
variant = "8"
case "v7":
variant = "7"
case "v6":
variant = "6"
case "v5":
variant = "5"
case "v4":
variant = "4"
case "v3":
variant = "3"
default:
variant = "unknown"
}
} else {
return "", fmt.Errorf("getCPUVariantFromArch invalid arch: %s, %w", arch, errdefs.ErrInvalidArgument)
}
return variant, nil
}
// getCPUVariant returns cpu variant for ARM
// We first try reading "Cpu architecture" field from /proc/cpuinfo
// If we can't find it, then fall back using a system call
// This is to cover running ARM in emulated environment on x86 host as this field in /proc/cpuinfo
// was not present.
func getCPUVariant() (string, error) {
variant, err := getCPUInfo("Cpu architecture")
if err != nil {
if errdefs.IsNotFound(err) {
//Let's try getting CPU variant from machine architecture
arch, err := getMachineArch()
if err != nil {
return "", fmt.Errorf("failure getting machine architecture: %v", err)
}
variant, err = getCPUVariantFromArch(arch)
if err != nil {
return "", fmt.Errorf("failure getting CPU variant from machine architecture: %v", err)
}
} else {
return "", fmt.Errorf("failure getting CPU variant: %v", err)
}
}
// handle edge case for Raspberry Pi ARMv6 devices (which due to a kernel quirk, report "CPU architecture: 7")
// https://www.raspberrypi.org/forums/viewtopic.php?t=12614
if runtime.GOARCH == "arm" && variant == "7" {
model, err := getCPUInfo("model name")
if err == nil && strings.HasPrefix(strings.ToLower(model), "armv6-compatible") {
variant = "6"
}
}
switch strings.ToLower(variant) {
case "8", "aarch64":
variant = "v8"
case "7", "7m", "?(12)", "?(13)", "?(14)", "?(15)", "?(16)", "?(17)":
variant = "v7"
case "6", "6tej":
variant = "v6"
case "5", "5t", "5te", "5tej":
variant = "v5"
case "4", "4t":
variant = "v4"
case "3":
variant = "v3"
default:
variant = "unknown"
}
return variant, nil
}

View File

@ -0,0 +1,59 @@
//go:build !linux
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package platforms
import (
"fmt"
"runtime"
"github.com/containerd/containerd/errdefs"
)
func getCPUVariant() (string, error) {
var variant string
if runtime.GOOS == "windows" || runtime.GOOS == "darwin" {
// Windows/Darwin only supports v7 for ARM32 and v8 for ARM64 and so we can use
// runtime.GOARCH to determine the variants
switch runtime.GOARCH {
case "arm64":
variant = "v8"
case "arm":
variant = "v7"
default:
variant = "unknown"
}
} else if runtime.GOOS == "freebsd" {
// FreeBSD supports ARMv6 and ARMv7 as well as ARMv4 and ARMv5 (though deprecated)
// detecting those variants is currently unimplemented
switch runtime.GOARCH {
case "arm64":
variant = "v8"
default:
variant = "unknown"
}
} else {
return "", fmt.Errorf("getCPUVariant for OS %s: %v", runtime.GOOS, errdefs.ErrNotImplemented)
}
return variant, nil
}

View File

@ -21,13 +21,6 @@ import (
"strings"
)
// isLinuxOS returns true if the operating system is Linux.
//
// The OS value should be normalized before calling this function.
func isLinuxOS(os string) bool {
return os == "linux"
}
// These function are generated from https://golang.org/src/go/build/syslist.go.
//
// We use switch statements because they are slightly faster than map lookups

View File

@ -1,5 +1,4 @@
//go:build darwin
// +build darwin
/*
Copyright The containerd Authors.

View File

@ -0,0 +1,43 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package platforms
import (
"runtime"
specs "github.com/opencontainers/image-spec/specs-go/v1"
)
// DefaultSpec returns the current platform's default platform specification.
func DefaultSpec() specs.Platform {
return specs.Platform{
OS: runtime.GOOS,
Architecture: runtime.GOARCH,
// The Variant field will be empty if arch != ARM.
Variant: cpuVariant(),
}
}
// Default returns the default matcher for the platform.
func Default() MatchComparer {
return Ordered(DefaultSpec(), specs.Platform{
OS: "linux",
Architecture: runtime.GOARCH,
// The Variant field will be empty if arch != ARM.
Variant: cpuVariant(),
})
}

View File

@ -1,5 +1,4 @@
//go:build !windows && !darwin
// +build !windows,!darwin
//go:build !windows && !darwin && !freebsd
/*
Copyright The containerd Authors.

View File

@ -22,7 +22,6 @@ import (
"strconv"
"strings"
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
specs "github.com/opencontainers/image-spec/specs-go/v1"
"golang.org/x/sys/windows"
)
@ -39,25 +38,28 @@ func DefaultSpec() specs.Platform {
}
}
type matchComparer struct {
defaults Matcher
type windowsmatcher struct {
specs.Platform
osVersionPrefix string
defaultMatcher Matcher
}
// Match matches platform with the same windows major, minor
// and build version.
func (m matchComparer) Match(p imagespec.Platform) bool {
if m.defaults.Match(p) {
// TODO(windows): Figure out whether OSVersion is deprecated.
return strings.HasPrefix(p.OSVersion, m.osVersionPrefix)
func (m windowsmatcher) Match(p specs.Platform) bool {
match := m.defaultMatcher.Match(p)
if match && m.OS == "windows" {
return strings.HasPrefix(p.OSVersion, m.osVersionPrefix) && m.defaultMatcher.Match(p)
}
return false
return match
}
// Less sorts matched platforms in front of other platforms.
// For matched platforms, it puts platforms with larger revision
// number in front.
func (m matchComparer) Less(p1, p2 imagespec.Platform) bool {
func (m windowsmatcher) Less(p1, p2 specs.Platform) bool {
m1, m2 := m.Match(p1), m.Match(p2)
if m1 && m2 {
r1, r2 := revision(p1.OSVersion), revision(p2.OSVersion)
@ -78,14 +80,15 @@ func revision(v string) int {
return r
}
func prefix(v string) string {
parts := strings.Split(v, ".")
if len(parts) < 4 {
return v
}
return strings.Join(parts[0:3], ".")
}
// Default returns the current platform's default platform specification.
func Default() MatchComparer {
major, minor, build := windows.RtlGetNtVersionNumbers()
return matchComparer{
defaults: Ordered(DefaultSpec(), specs.Platform{
OS: "linux",
Architecture: runtime.GOARCH,
}),
osVersionPrefix: fmt.Sprintf("%d.%d.%d", major, minor, build),
}
return Only(DefaultSpec())
}

View File

@ -114,14 +114,18 @@ import (
"strconv"
"strings"
"github.com/containerd/containerd/errdefs"
specs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/containerd/containerd/errdefs"
)
var (
specifierRe = regexp.MustCompile(`^[A-Za-z0-9_-]+$`)
)
// Platform is a type alias for convenience, so there is no need to import image-spec package everywhere.
type Platform = specs.Platform
// Matcher matches platforms specifications, provided by an image or runtime.
type Matcher interface {
Match(platform specs.Platform) bool
@ -136,9 +140,7 @@ type Matcher interface {
//
// Applications should opt to use `Match` over directly parsing specifiers.
func NewMatcher(platform specs.Platform) Matcher {
return &matcher{
Platform: Normalize(platform),
}
return newDefaultMatcher(platform)
}
type matcher struct {
@ -257,5 +259,6 @@ func Format(platform specs.Platform) string {
func Normalize(platform specs.Platform) specs.Platform {
platform.OS = normalizeOS(platform.OS)
platform.Architecture, platform.Variant = normalizeArch(platform.Architecture, platform.Variant)
return platform
}

View File

@ -0,0 +1,30 @@
//go:build !windows
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package platforms
import (
specs "github.com/opencontainers/image-spec/specs-go/v1"
)
// NewMatcher returns the default Matcher for containerd
func newDefaultMatcher(platform specs.Platform) Matcher {
return &matcher{
Platform: Normalize(platform),
}
}

View File

@ -0,0 +1,34 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package platforms
import (
specs "github.com/opencontainers/image-spec/specs-go/v1"
)
// NewMatcher returns a Windows matcher that will match on osVersionPrefix if
// the platform is Windows otherwise use the default matcher
func newDefaultMatcher(platform specs.Platform) Matcher {
prefix := prefix(platform.OSVersion)
return windowsmatcher{
Platform: platform,
osVersionPrefix: prefix,
defaultMatcher: &matcher{
Platform: Normalize(platform),
},
}
}

View File

@ -0,0 +1,47 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package protobuf
import (
"github.com/containerd/typeurl"
"google.golang.org/protobuf/types/known/anypb"
)
// FromAny converts typeurl.Any to github.com/containerd/containerd/protobuf/types.Any.
func FromAny(from typeurl.Any) *anypb.Any {
if from == nil {
return nil
}
if pbany, ok := from.(*anypb.Any); ok {
return pbany
}
return &anypb.Any{
TypeUrl: from.GetTypeUrl(),
Value: from.GetValue(),
}
}
// FromAny converts an arbitrary interface to github.com/containerd/containerd/protobuf/types.Any.
func MarshalAnyToProto(from interface{}) (*anypb.Any, error) {
any, err := typeurl.MarshalAny(from)
if err != nil {
return nil, err
}
return FromAny(any), nil
}

View File

@ -0,0 +1,41 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package protobuf
import (
"github.com/google/go-cmp/cmp"
"google.golang.org/protobuf/proto"
)
var Compare = cmp.FilterValues(
func(x, y interface{}) bool {
_, xok := x.(proto.Message)
_, yok := y.(proto.Message)
return xok && yok
},
cmp.Comparer(func(x, y interface{}) bool {
vx, ok := x.(proto.Message)
if !ok {
return false
}
vy, ok := y.(proto.Message)
if !ok {
return false
}
return proto.Equal(vx, vy)
}),
)

View File

@ -0,0 +1,36 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package protobuf
import (
"time"
"google.golang.org/protobuf/types/known/timestamppb"
)
// Once we migrate off from gogo/protobuf, we can use the function below, which don't return any errors.
// https://github.com/protocolbuffers/protobuf-go/blob/v1.28.0/types/known/timestamppb/timestamp.pb.go#L200-L208
// ToTimestamp creates protobuf's Timestamp from time.Time.
func ToTimestamp(from time.Time) *timestamppb.Timestamp {
return timestamppb.New(from)
}
// FromTimestamp creates time.Time from protobuf's Timestamp.
func FromTimestamp(from *timestamppb.Timestamp) time.Time {
return from.AsTime()
}

View File

@ -0,0 +1,28 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package types provides convinient aliases that make google.golang.org/protobuf migration easier.
package types
import (
"google.golang.org/genproto/protobuf/field_mask"
"google.golang.org/protobuf/types/known/anypb"
"google.golang.org/protobuf/types/known/emptypb"
)
type Empty = emptypb.Empty
type Any = anypb.Any
type FieldMask = field_mask.FieldMask

View File

@ -0,0 +1,58 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package docker
import "path"
// IsNameOnly returns true if reference only contains a repo name.
func IsNameOnly(ref Named) bool {
if _, ok := ref.(NamedTagged); ok {
return false
}
if _, ok := ref.(Canonical); ok {
return false
}
return true
}
// FamiliarName returns the familiar name string
// for the given named, familiarizing if needed.
func FamiliarName(ref Named) string {
if nn, ok := ref.(normalizedNamed); ok {
return nn.Familiar().Name()
}
return ref.Name()
}
// FamiliarString returns the familiar string representation
// for the given reference, familiarizing if needed.
func FamiliarString(ref Reference) string {
if nn, ok := ref.(normalizedNamed); ok {
return nn.Familiar().String()
}
return ref.String()
}
// FamiliarMatch reports whether ref matches the specified pattern.
// See https://godoc.org/path#Match for supported patterns.
func FamiliarMatch(pattern string, ref Reference) (bool, error) {
matched, err := path.Match(pattern, FamiliarString(ref))
if namedRef, isNamed := ref.(Named); isNamed && !matched {
matched, _ = path.Match(pattern, FamiliarName(namedRef))
}
return matched, err
}

View File

@ -0,0 +1,196 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package docker
import (
"fmt"
"strings"
"github.com/opencontainers/go-digest"
)
var (
legacyDefaultDomain = "index.docker.io"
defaultDomain = "docker.io"
officialRepoName = "library"
defaultTag = "latest"
)
// normalizedNamed represents a name which has been
// normalized and has a familiar form. A familiar name
// is what is used in Docker UI. An example normalized
// name is "docker.io/library/ubuntu" and corresponding
// familiar name of "ubuntu".
type normalizedNamed interface {
Named
Familiar() Named
}
// ParseNormalizedNamed parses a string into a named reference
// transforming a familiar name from Docker UI to a fully
// qualified reference. If the value may be an identifier
// use ParseAnyReference.
func ParseNormalizedNamed(s string) (Named, error) {
if ok := anchoredIdentifierRegexp.MatchString(s); ok {
return nil, fmt.Errorf("invalid repository name (%s), cannot specify 64-byte hexadecimal strings", s)
}
domain, remainder := splitDockerDomain(s)
var remoteName string
if tagSep := strings.IndexRune(remainder, ':'); tagSep > -1 {
remoteName = remainder[:tagSep]
} else {
remoteName = remainder
}
if strings.ToLower(remoteName) != remoteName {
return nil, fmt.Errorf("invalid reference format: repository name (%s) must be lowercase", remoteName)
}
ref, err := Parse(domain + "/" + remainder)
if err != nil {
return nil, err
}
named, isNamed := ref.(Named)
if !isNamed {
return nil, fmt.Errorf("reference %s has no name", ref.String())
}
return named, nil
}
// ParseDockerRef normalizes the image reference following the docker convention. This is added
// mainly for backward compatibility.
// The reference returned can only be either tagged or digested. For reference contains both tag
// and digest, the function returns digested reference, e.g. docker.io/library/busybox:latest@
// sha256:7cc4b5aefd1d0cadf8d97d4350462ba51c694ebca145b08d7d41b41acc8db5aa will be returned as
// docker.io/library/busybox@sha256:7cc4b5aefd1d0cadf8d97d4350462ba51c694ebca145b08d7d41b41acc8db5aa.
func ParseDockerRef(ref string) (Named, error) {
named, err := ParseNormalizedNamed(ref)
if err != nil {
return nil, err
}
if _, ok := named.(NamedTagged); ok {
if canonical, ok := named.(Canonical); ok {
// The reference is both tagged and digested, only
// return digested.
newNamed, err := WithName(canonical.Name())
if err != nil {
return nil, err
}
newCanonical, err := WithDigest(newNamed, canonical.Digest())
if err != nil {
return nil, err
}
return newCanonical, nil
}
}
return TagNameOnly(named), nil
}
// splitDockerDomain splits a repository name to domain and remotename string.
// If no valid domain is found, the default domain is used. Repository name
// needs to be already validated before.
func splitDockerDomain(name string) (domain, remainder string) {
i := strings.IndexRune(name, '/')
if i == -1 || (!strings.ContainsAny(name[:i], ".:") && name[:i] != "localhost" && strings.ToLower(name[:i]) == name[:i]) {
domain, remainder = defaultDomain, name
} else {
domain, remainder = name[:i], name[i+1:]
}
if domain == legacyDefaultDomain {
domain = defaultDomain
}
if domain == defaultDomain && !strings.ContainsRune(remainder, '/') {
remainder = officialRepoName + "/" + remainder
}
return
}
// familiarizeName returns a shortened version of the name familiar
// to the Docker UI. Familiar names have the default domain
// "docker.io" and "library/" repository prefix removed.
// For example, "docker.io/library/redis" will have the familiar
// name "redis" and "docker.io/dmcgowan/myapp" will be "dmcgowan/myapp".
// Returns a familiarized named only reference.
func familiarizeName(named namedRepository) repository {
repo := repository{
domain: named.Domain(),
path: named.Path(),
}
if repo.domain == defaultDomain {
repo.domain = ""
// Handle official repositories which have the pattern "library/<official repo name>"
if split := strings.Split(repo.path, "/"); len(split) == 2 && split[0] == officialRepoName {
repo.path = split[1]
}
}
return repo
}
func (r reference) Familiar() Named {
return reference{
namedRepository: familiarizeName(r.namedRepository),
tag: r.tag,
digest: r.digest,
}
}
func (r repository) Familiar() Named {
return familiarizeName(r)
}
func (t taggedReference) Familiar() Named {
return taggedReference{
namedRepository: familiarizeName(t.namedRepository),
tag: t.tag,
}
}
func (c canonicalReference) Familiar() Named {
return canonicalReference{
namedRepository: familiarizeName(c.namedRepository),
digest: c.digest,
}
}
// TagNameOnly adds the default tag "latest" to a reference if it only has
// a repo name.
func TagNameOnly(ref Named) Named {
if IsNameOnly(ref) {
namedTagged, err := WithTag(ref, defaultTag)
if err != nil {
// Default tag must be valid, to create a NamedTagged
// type with non-validated input the WithTag function
// should be used instead
panic(err)
}
return namedTagged
}
return ref
}
// ParseAnyReference parses a reference string as a possible identifier,
// full digest, or familiar name.
func ParseAnyReference(ref string) (Reference, error) {
if ok := anchoredIdentifierRegexp.MatchString(ref); ok {
return digestReference("sha256:" + ref), nil
}
if dgst, err := digest.Parse(ref); err == nil {
return digestReference(dgst), nil
}
return ParseNormalizedNamed(ref)
}

View File

@ -0,0 +1,453 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package docker provides a general type to represent any way of referencing images within the registry.
// Its main purpose is to abstract tags and digests (content-addressable hash).
//
// Grammar
//
// reference := name [ ":" tag ] [ "@" digest ]
// name := [domain '/'] path-component ['/' path-component]*
// domain := host [':' port-number]
// host := domain-name | IPv4address | \[ IPv6address \] ; rfc3986 appendix-A
// domain-name := domain-component ['.' domain-component]*
// domain-component := /([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])/
// port-number := /[0-9]+/
// path-component := alpha-numeric [separator alpha-numeric]*
// alpha-numeric := /[a-z0-9]+/
// separator := /[_.]|__|[-]*/
//
// tag := /[\w][\w.-]{0,127}/
//
// digest := digest-algorithm ":" digest-hex
// digest-algorithm := digest-algorithm-component [ digest-algorithm-separator digest-algorithm-component ]*
// digest-algorithm-separator := /[+.-_]/
// digest-algorithm-component := /[A-Za-z][A-Za-z0-9]*/
// digest-hex := /[0-9a-fA-F]{32,}/ ; At least 128 bit digest value
//
// identifier := /[a-f0-9]{64}/
// short-identifier := /[a-f0-9]{6,64}/
package docker
import (
"errors"
"fmt"
"strings"
"github.com/opencontainers/go-digest"
)
const (
// NameTotalLengthMax is the maximum total number of characters in a repository name.
NameTotalLengthMax = 255
)
var (
// ErrReferenceInvalidFormat represents an error while trying to parse a string as a reference.
ErrReferenceInvalidFormat = errors.New("invalid reference format")
// ErrTagInvalidFormat represents an error while trying to parse a string as a tag.
ErrTagInvalidFormat = errors.New("invalid tag format")
// ErrDigestInvalidFormat represents an error while trying to parse a string as a tag.
ErrDigestInvalidFormat = errors.New("invalid digest format")
// ErrNameContainsUppercase is returned for invalid repository names that contain uppercase characters.
ErrNameContainsUppercase = errors.New("repository name must be lowercase")
// ErrNameEmpty is returned for empty, invalid repository names.
ErrNameEmpty = errors.New("repository name must have at least one component")
// ErrNameTooLong is returned when a repository name is longer than NameTotalLengthMax.
ErrNameTooLong = fmt.Errorf("repository name must not be more than %v characters", NameTotalLengthMax)
// ErrNameNotCanonical is returned when a name is not canonical.
ErrNameNotCanonical = errors.New("repository name must be canonical")
)
// Reference is an opaque object reference identifier that may include
// modifiers such as a hostname, name, tag, and digest.
type Reference interface {
// String returns the full reference
String() string
}
// Field provides a wrapper type for resolving correct reference types when
// working with encoding.
type Field struct {
reference Reference
}
// AsField wraps a reference in a Field for encoding.
func AsField(reference Reference) Field {
return Field{reference}
}
// Reference unwraps the reference type from the field to
// return the Reference object. This object should be
// of the appropriate type to further check for different
// reference types.
func (f Field) Reference() Reference {
return f.reference
}
// MarshalText serializes the field to byte text which
// is the string of the reference.
func (f Field) MarshalText() (p []byte, err error) {
return []byte(f.reference.String()), nil
}
// UnmarshalText parses text bytes by invoking the
// reference parser to ensure the appropriately
// typed reference object is wrapped by field.
func (f *Field) UnmarshalText(p []byte) error {
r, err := Parse(string(p))
if err != nil {
return err
}
f.reference = r
return nil
}
// Named is an object with a full name
type Named interface {
Reference
Name() string
}
// Tagged is an object which has a tag
type Tagged interface {
Reference
Tag() string
}
// NamedTagged is an object including a name and tag.
type NamedTagged interface {
Named
Tag() string
}
// Digested is an object which has a digest
// in which it can be referenced by
type Digested interface {
Reference
Digest() digest.Digest
}
// Canonical reference is an object with a fully unique
// name including a name with domain and digest
type Canonical interface {
Named
Digest() digest.Digest
}
// namedRepository is a reference to a repository with a name.
// A namedRepository has both domain and path components.
type namedRepository interface {
Named
Domain() string
Path() string
}
// Domain returns the domain part of the Named reference
func Domain(named Named) string {
if r, ok := named.(namedRepository); ok {
return r.Domain()
}
domain, _ := splitDomain(named.Name())
return domain
}
// Path returns the name without the domain part of the Named reference
func Path(named Named) (name string) {
if r, ok := named.(namedRepository); ok {
return r.Path()
}
_, path := splitDomain(named.Name())
return path
}
func splitDomain(name string) (string, string) {
match := anchoredNameRegexp.FindStringSubmatch(name)
if len(match) != 3 {
return "", name
}
return match[1], match[2]
}
// SplitHostname splits a named reference into a
// hostname and name string. If no valid hostname is
// found, the hostname is empty and the full value
// is returned as name
// DEPRECATED: Use Domain or Path
func SplitHostname(named Named) (string, string) {
if r, ok := named.(namedRepository); ok {
return r.Domain(), r.Path()
}
return splitDomain(named.Name())
}
// Parse parses s and returns a syntactically valid Reference.
// If an error was encountered it is returned, along with a nil Reference.
// NOTE: Parse will not handle short digests.
func Parse(s string) (Reference, error) {
matches := ReferenceRegexp.FindStringSubmatch(s)
if matches == nil {
if s == "" {
return nil, ErrNameEmpty
}
if ReferenceRegexp.FindStringSubmatch(strings.ToLower(s)) != nil {
return nil, ErrNameContainsUppercase
}
return nil, ErrReferenceInvalidFormat
}
if len(matches[1]) > NameTotalLengthMax {
return nil, ErrNameTooLong
}
var repo repository
nameMatch := anchoredNameRegexp.FindStringSubmatch(matches[1])
if len(nameMatch) == 3 {
repo.domain = nameMatch[1]
repo.path = nameMatch[2]
} else {
repo.domain = ""
repo.path = matches[1]
}
ref := reference{
namedRepository: repo,
tag: matches[2],
}
if matches[3] != "" {
var err error
ref.digest, err = digest.Parse(matches[3])
if err != nil {
return nil, err
}
}
r := getBestReferenceType(ref)
if r == nil {
return nil, ErrNameEmpty
}
return r, nil
}
// ParseNamed parses s and returns a syntactically valid reference implementing
// the Named interface. The reference must have a name and be in the canonical
// form, otherwise an error is returned.
// If an error was encountered it is returned, along with a nil Reference.
// NOTE: ParseNamed will not handle short digests.
func ParseNamed(s string) (Named, error) {
named, err := ParseNormalizedNamed(s)
if err != nil {
return nil, err
}
if named.String() != s {
return nil, ErrNameNotCanonical
}
return named, nil
}
// WithName returns a named object representing the given string. If the input
// is invalid ErrReferenceInvalidFormat will be returned.
func WithName(name string) (Named, error) {
if len(name) > NameTotalLengthMax {
return nil, ErrNameTooLong
}
match := anchoredNameRegexp.FindStringSubmatch(name)
if match == nil || len(match) != 3 {
return nil, ErrReferenceInvalidFormat
}
return repository{
domain: match[1],
path: match[2],
}, nil
}
// WithTag combines the name from "name" and the tag from "tag" to form a
// reference incorporating both the name and the tag.
func WithTag(name Named, tag string) (NamedTagged, error) {
if !anchoredTagRegexp.MatchString(tag) {
return nil, ErrTagInvalidFormat
}
var repo repository
if r, ok := name.(namedRepository); ok {
repo.domain = r.Domain()
repo.path = r.Path()
} else {
repo.path = name.Name()
}
if canonical, ok := name.(Canonical); ok {
return reference{
namedRepository: repo,
tag: tag,
digest: canonical.Digest(),
}, nil
}
return taggedReference{
namedRepository: repo,
tag: tag,
}, nil
}
// WithDigest combines the name from "name" and the digest from "digest" to form
// a reference incorporating both the name and the digest.
func WithDigest(name Named, digest digest.Digest) (Canonical, error) {
if !anchoredDigestRegexp.MatchString(digest.String()) {
return nil, ErrDigestInvalidFormat
}
var repo repository
if r, ok := name.(namedRepository); ok {
repo.domain = r.Domain()
repo.path = r.Path()
} else {
repo.path = name.Name()
}
if tagged, ok := name.(Tagged); ok {
return reference{
namedRepository: repo,
tag: tagged.Tag(),
digest: digest,
}, nil
}
return canonicalReference{
namedRepository: repo,
digest: digest,
}, nil
}
// TrimNamed removes any tag or digest from the named reference.
func TrimNamed(ref Named) Named {
repo := repository{}
if r, ok := ref.(namedRepository); ok {
repo.domain, repo.path = r.Domain(), r.Path()
} else {
repo.domain, repo.path = splitDomain(ref.Name())
}
return repo
}
func getBestReferenceType(ref reference) Reference {
if ref.Name() == "" {
// Allow digest only references
if ref.digest != "" {
return digestReference(ref.digest)
}
return nil
}
if ref.tag == "" {
if ref.digest != "" {
return canonicalReference{
namedRepository: ref.namedRepository,
digest: ref.digest,
}
}
return ref.namedRepository
}
if ref.digest == "" {
return taggedReference{
namedRepository: ref.namedRepository,
tag: ref.tag,
}
}
return ref
}
type reference struct {
namedRepository
tag string
digest digest.Digest
}
func (r reference) String() string {
return r.Name() + ":" + r.tag + "@" + r.digest.String()
}
func (r reference) Tag() string {
return r.tag
}
func (r reference) Digest() digest.Digest {
return r.digest
}
type repository struct {
domain string
path string
}
func (r repository) String() string {
return r.Name()
}
func (r repository) Name() string {
if r.domain == "" {
return r.path
}
return r.domain + "/" + r.path
}
func (r repository) Domain() string {
return r.domain
}
func (r repository) Path() string {
return r.path
}
type digestReference digest.Digest
func (d digestReference) String() string {
return digest.Digest(d).String()
}
func (d digestReference) Digest() digest.Digest {
return digest.Digest(d)
}
type taggedReference struct {
namedRepository
tag string
}
func (t taggedReference) String() string {
return t.Name() + ":" + t.tag
}
func (t taggedReference) Tag() string {
return t.tag
}
type canonicalReference struct {
namedRepository
digest digest.Digest
}
func (c canonicalReference) String() string {
return c.Name() + "@" + c.digest.String()
}
func (c canonicalReference) Digest() digest.Digest {
return c.digest
}

View File

@ -0,0 +1,191 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package docker
import "regexp"
var (
// alphaNumeric defines the alpha numeric atom, typically a
// component of names. This only allows lower case characters and digits.
alphaNumeric = `[a-z0-9]+`
// separator defines the separators allowed to be embedded in name
// components. This allow one period, one or two underscore and multiple
// dashes. Repeated dashes and underscores are intentionally treated
// differently. In order to support valid hostnames as name components,
// supporting repeated dash was added. Additionally double underscore is
// now allowed as a separator to loosen the restriction for previously
// supported names.
separator = `(?:[._]|__|[-]*)`
// nameComponent restricts registry path component names to start
// with at least one letter or number, with following parts able to be
// separated by one period, one or two underscore and multiple dashes.
nameComponent = expression(
alphaNumeric,
optional(repeated(separator, alphaNumeric)))
// domainNameComponent restricts the registry domain component of a
// repository name to start with a component as defined by DomainRegexp.
domainNameComponent = `(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])`
// ipv6address are enclosed between square brackets and may be represented
// in many ways, see rfc5952. Only IPv6 in compressed or uncompressed format
// are allowed, IPv6 zone identifiers (rfc6874) or Special addresses such as
// IPv4-Mapped are deliberately excluded.
ipv6address = expression(
literal(`[`), `(?:[a-fA-F0-9:]+)`, literal(`]`),
)
// domainName defines the structure of potential domain components
// that may be part of image names. This is purposely a subset of what is
// allowed by DNS to ensure backwards compatibility with Docker image
// names. This includes IPv4 addresses on decimal format.
domainName = expression(
domainNameComponent,
optional(repeated(literal(`.`), domainNameComponent)),
)
// host defines the structure of potential domains based on the URI
// Host subcomponent on rfc3986. It may be a subset of DNS domain name,
// or an IPv4 address in decimal format, or an IPv6 address between square
// brackets (excluding zone identifiers as defined by rfc6874 or special
// addresses such as IPv4-Mapped).
host = `(?:` + domainName + `|` + ipv6address + `)`
// allowed by the URI Host subcomponent on rfc3986 to ensure backwards
// compatibility with Docker image names.
domain = expression(
host,
optional(literal(`:`), `[0-9]+`))
// DomainRegexp defines the structure of potential domain components
// that may be part of image names. This is purposely a subset of what is
// allowed by DNS to ensure backwards compatibility with Docker image
// names.
DomainRegexp = regexp.MustCompile(domain)
tag = `[\w][\w.-]{0,127}`
// TagRegexp matches valid tag names. From docker/docker:graph/tags.go.
TagRegexp = regexp.MustCompile(tag)
anchoredTag = anchored(tag)
// anchoredTagRegexp matches valid tag names, anchored at the start and
// end of the matched string.
anchoredTagRegexp = regexp.MustCompile(anchoredTag)
digestPat = `[A-Za-z][A-Za-z0-9]*(?:[-_+.][A-Za-z][A-Za-z0-9]*)*[:][[:xdigit:]]{32,}`
// DigestRegexp matches valid digests.
DigestRegexp = regexp.MustCompile(digestPat)
anchoredDigest = anchored(digestPat)
// anchoredDigestRegexp matches valid digests, anchored at the start and
// end of the matched string.
anchoredDigestRegexp = regexp.MustCompile(anchoredDigest)
namePat = expression(
optional(domain, literal(`/`)),
nameComponent,
optional(repeated(literal(`/`), nameComponent)))
// NameRegexp is the format for the name component of references. The
// regexp has capturing groups for the domain and name part omitting
// the separating forward slash from either.
NameRegexp = regexp.MustCompile(namePat)
anchoredName = anchored(
optional(capture(domain), literal(`/`)),
capture(nameComponent,
optional(repeated(literal(`/`), nameComponent))))
// anchoredNameRegexp is used to parse a name value, capturing the
// domain and trailing components.
anchoredNameRegexp = regexp.MustCompile(anchoredName)
referencePat = anchored(capture(namePat),
optional(literal(":"), capture(tag)),
optional(literal("@"), capture(digestPat)))
// ReferenceRegexp is the full supported format of a reference. The regexp
// is anchored and has capturing groups for name, tag, and digest
// components.
ReferenceRegexp = regexp.MustCompile(referencePat)
identifier = `([a-f0-9]{64})`
// IdentifierRegexp is the format for string identifier used as a
// content addressable identifier using sha256. These identifiers
// are like digests without the algorithm, since sha256 is used.
IdentifierRegexp = regexp.MustCompile(identifier)
shortIdentifier = `([a-f0-9]{6,64})`
// ShortIdentifierRegexp is the format used to represent a prefix
// of an identifier. A prefix may be used to match a sha256 identifier
// within a list of trusted identifiers.
ShortIdentifierRegexp = regexp.MustCompile(shortIdentifier)
anchoredIdentifier = anchored(identifier)
// anchoredIdentifierRegexp is used to check or match an
// identifier value, anchored at start and end of string.
anchoredIdentifierRegexp = regexp.MustCompile(anchoredIdentifier)
)
// literal compiles s into a literal regular expression, escaping any regexp
// reserved characters.
func literal(s string) string {
re := regexp.MustCompile(regexp.QuoteMeta(s))
if _, complete := re.LiteralPrefix(); !complete {
panic("must be a literal")
}
return re.String()
}
// expression defines a full expression, where each regular expression must
// follow the previous.
func expression(res ...string) string {
var s string
for _, re := range res {
s += re
}
return s
}
// optional wraps the expression in a non-capturing group and makes the
// production optional.
func optional(res ...string) string {
return group(expression(res...)) + `?`
}
// repeated wraps the regexp in a non-capturing group to get one or more
// matches.
func repeated(res ...string) string {
return group(expression(res...)) + `+`
}
// group wraps the regexp in a non-capturing group.
func group(res ...string) string {
return `(?:` + expression(res...) + `)`
}
// capture wraps the expression in a capturing group.
func capture(res ...string) string {
return `(` + expression(res...) + `)`
}
// anchored anchors the regular expression by adding start and end delimiters.
func anchored(res ...string) string {
return `^` + expression(res...) + `$`
}

View File

@ -0,0 +1,73 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package docker
import (
"sort"
)
// Sort sorts string references preferring higher information references
// The precedence is as follows:
// 1. Name + Tag + Digest
// 2. Name + Tag
// 3. Name + Digest
// 4. Name
// 5. Digest
// 6. Parse error
func Sort(references []string) []string {
var prefs []Reference
var bad []string
for _, ref := range references {
pref, err := ParseAnyReference(ref)
if err != nil {
bad = append(bad, ref)
} else {
prefs = append(prefs, pref)
}
}
sort.Slice(prefs, func(a, b int) bool {
ar := refRank(prefs[a])
br := refRank(prefs[b])
if ar == br {
return prefs[a].String() < prefs[b].String()
}
return ar < br
})
sort.Strings(bad)
var refs []string
for _, pref := range prefs {
refs = append(refs, pref.String())
}
return append(refs, bad...)
}
func refRank(ref Reference) uint8 {
if _, ok := ref.(Named); ok {
if _, ok = ref.(Tagged); ok {
if _, ok = ref.(Digested); ok {
return 1
}
return 2
}
if _, ok = ref.(Digested); ok {
return 3
}
return 4
}
return 5
}

View File

@ -29,7 +29,6 @@ import (
"github.com/containerd/containerd/log"
remoteserrors "github.com/containerd/containerd/remotes/errors"
"github.com/containerd/containerd/version"
"golang.org/x/net/context/ctxhttp"
)
var (
@ -115,7 +114,7 @@ func FetchTokenWithOAuth(ctx context.Context, client *http.Client, headers http.
form.Set("access_type", "offline")
}
req, err := http.NewRequest("POST", to.Realm, strings.NewReader(form.Encode()))
req, err := http.NewRequestWithContext(ctx, "POST", to.Realm, strings.NewReader(form.Encode()))
if err != nil {
return nil, err
}
@ -127,7 +126,7 @@ func FetchTokenWithOAuth(ctx context.Context, client *http.Client, headers http.
req.Header.Set("User-Agent", "containerd/"+version.Version)
}
resp, err := ctxhttp.Do(ctx, client, req)
resp, err := client.Do(req)
if err != nil {
return nil, err
}
@ -162,7 +161,7 @@ type FetchTokenResponse struct {
// FetchToken fetches a token using a GET request
func FetchToken(ctx context.Context, client *http.Client, headers http.Header, to TokenOptions) (*FetchTokenResponse, error) {
req, err := http.NewRequest("GET", to.Realm, nil)
req, err := http.NewRequestWithContext(ctx, "GET", to.Realm, nil)
if err != nil {
return nil, err
}
@ -194,7 +193,7 @@ func FetchToken(ctx context.Context, client *http.Client, headers http.Header, t
req.URL.RawQuery = reqParams.Encode()
resp, err := ctxhttp.Do(ctx, client, req)
resp, err := client.Do(req)
if err != nil {
return nil, err
}

View File

@ -45,13 +45,6 @@ type dockerAuthorizer struct {
onFetchRefreshToken OnFetchRefreshToken
}
// NewAuthorizer creates a Docker authorizer using the provided function to
// get credentials for the token server or basic auth.
// Deprecated: Use NewDockerAuthorizer
func NewAuthorizer(client *http.Client, f func(string) (string, string, error)) Authorizer {
return NewDockerAuthorizer(WithAuthClient(client), WithAuthCreds(f))
}
type authorizerConfig struct {
credentials func(string) (string, string, error)
client *http.Client

View File

@ -0,0 +1,55 @@
//go:build gofuzz
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package docker
import (
"context"
"os"
fuzz "github.com/AdaLogics/go-fuzz-headers"
"github.com/containerd/containerd/content/local"
"github.com/containerd/containerd/log"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/sirupsen/logrus"
)
func FuzzConvertManifest(data []byte) int {
ctx := context.Background()
// Do not log the message below
// level=warning msg="do nothing for media type: ..."
log.G(ctx).Logger.SetLevel(logrus.PanicLevel)
f := fuzz.NewConsumer(data)
desc := ocispec.Descriptor{}
err := f.GenerateStruct(&desc)
if err != nil {
return 0
}
tmpdir, err := os.MkdirTemp("", "fuzzing-")
if err != nil {
return 0
}
cs, err := local.NewStore(tmpdir)
if err != nil {
return 0
}
_, _ = ConvertManifest(ctx, cs, desc)
return 1
}

View File

@ -29,6 +29,7 @@ import (
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/log"
digest "github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)
@ -52,18 +53,17 @@ func (r dockerFetcher) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.R
return newHTTPReadSeeker(desc.Size, func(offset int64) (io.ReadCloser, error) {
// firstly try fetch via external urls
for _, us := range desc.URLs {
ctx = log.WithLogger(ctx, log.G(ctx).WithField("url", us))
u, err := url.Parse(us)
if err != nil {
log.G(ctx).WithError(err).Debug("failed to parse")
log.G(ctx).WithError(err).Debugf("failed to parse %q", us)
continue
}
if u.Scheme != "http" && u.Scheme != "https" {
log.G(ctx).Debug("non-http(s) alternative url is unsupported")
continue
}
log.G(ctx).Debug("trying alternative url")
ctx = log.WithLogger(ctx, log.G(ctx).WithField("url", u))
log.G(ctx).Info("request")
// Try this first, parse it
host := RegistryHost{
@ -151,8 +151,106 @@ func (r dockerFetcher) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.R
})
}
func (r dockerFetcher) createGetReq(ctx context.Context, host RegistryHost, ps ...string) (*request, int64, error) {
headReq := r.request(host, http.MethodHead, ps...)
if err := headReq.addNamespace(r.refspec.Hostname()); err != nil {
return nil, 0, err
}
headResp, err := headReq.doWithRetries(ctx, nil)
if err != nil {
return nil, 0, err
}
if headResp.Body != nil {
headResp.Body.Close()
}
if headResp.StatusCode > 299 {
return nil, 0, fmt.Errorf("unexpected HEAD status code %v: %s", headReq.String(), headResp.Status)
}
getReq := r.request(host, http.MethodGet, ps...)
if err := getReq.addNamespace(r.refspec.Hostname()); err != nil {
return nil, 0, err
}
return getReq, headResp.ContentLength, nil
}
func (r dockerFetcher) FetchByDigest(ctx context.Context, dgst digest.Digest) (io.ReadCloser, ocispec.Descriptor, error) {
var desc ocispec.Descriptor
ctx = log.WithLogger(ctx, log.G(ctx).WithField("digest", dgst))
hosts := r.filterHosts(HostCapabilityPull)
if len(hosts) == 0 {
return nil, desc, fmt.Errorf("no pull hosts: %w", errdefs.ErrNotFound)
}
ctx, err := ContextWithRepositoryScope(ctx, r.refspec, false)
if err != nil {
return nil, desc, err
}
var (
getReq *request
sz int64
firstErr error
)
for _, host := range r.hosts {
getReq, sz, err = r.createGetReq(ctx, host, "blobs", dgst.String())
if err == nil {
break
}
// Store the error for referencing later
if firstErr == nil {
firstErr = err
}
}
if getReq == nil {
// Fall back to the "manifests" endpoint
for _, host := range r.hosts {
getReq, sz, err = r.createGetReq(ctx, host, "manifests", dgst.String())
if err == nil {
break
}
// Store the error for referencing later
if firstErr == nil {
firstErr = err
}
}
}
if getReq == nil {
if errdefs.IsNotFound(firstErr) {
firstErr = fmt.Errorf("could not fetch content %v from remote: %w", dgst, errdefs.ErrNotFound)
}
if firstErr == nil {
firstErr = fmt.Errorf("could not fetch content %v from remote: (unknown)", dgst)
}
return nil, desc, firstErr
}
seeker, err := newHTTPReadSeeker(sz, func(offset int64) (io.ReadCloser, error) {
return r.open(ctx, getReq, "", offset)
})
if err != nil {
return nil, desc, err
}
desc = ocispec.Descriptor{
MediaType: "application/octet-stream",
Digest: dgst,
Size: sz,
}
return seeker, desc, nil
}
func (r dockerFetcher) open(ctx context.Context, req *request, mediatype string, offset int64) (_ io.ReadCloser, retErr error) {
req.header.Set("Accept", strings.Join([]string{mediatype, `*/*`}, ", "))
if mediatype == "" {
req.header.Set("Accept", "*/*")
} else {
req.header.Set("Accept", strings.Join([]string{mediatype, `*/*`}, ", "))
}
if offset > 0 {
// Note: "Accept-Ranges: bytes" cannot be trusted as some endpoints

View File

@ -0,0 +1,81 @@
//go:build gofuzz
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package docker
import (
"context"
"fmt"
"io"
"net/http"
"net/http/httptest"
"net/url"
refDocker "github.com/containerd/containerd/reference/docker"
)
func FuzzFetcher(data []byte) int {
dataLen := len(data)
if dataLen == 0 {
return -1
}
s := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
rw.Header().Set("content-range", fmt.Sprintf("bytes %d-%d/%d", 0, dataLen-1, dataLen))
rw.Header().Set("content-length", fmt.Sprintf("%d", dataLen))
rw.Write(data)
}))
defer s.Close()
u, err := url.Parse(s.URL)
if err != nil {
return 0
}
f := dockerFetcher{&dockerBase{
repository: "nonempty",
}}
host := RegistryHost{
Client: s.Client(),
Host: u.Host,
Scheme: u.Scheme,
Path: u.Path,
}
ctx := context.Background()
req := f.request(host, http.MethodGet)
rc, err := f.open(ctx, req, "", 0)
if err != nil {
return 0
}
b, err := io.ReadAll(rc)
if err != nil {
return 0
}
expected := data
if len(b) != len(expected) {
panic("len of request is not equal to len of expected but should be")
}
return 1
}
func FuzzParseDockerRef(data []byte) int {
_, _ = refDocker.ParseDockerRef(string(data))
return 1
}

View File

@ -191,6 +191,9 @@ func (p dockerPusher) push(ctx context.Context, desc ocispec.Descriptor, ref str
if resp == nil {
resp, err = req.doWithRetries(ctx, nil)
if err != nil {
if errors.Is(err, ErrInvalidAuthorization) {
return nil, fmt.Errorf("push access denied, repository does not exist or may require authorization: %w", err)
}
return nil, err
}
}

View File

@ -32,12 +32,13 @@ import (
"github.com/containerd/containerd/log"
"github.com/containerd/containerd/reference"
"github.com/containerd/containerd/remotes"
"github.com/containerd/containerd/remotes/docker/schema1"
"github.com/containerd/containerd/remotes/docker/schema1" //nolint:staticcheck // Ignore SA1019. Need to keep deprecated package for compatibility.
remoteerrors "github.com/containerd/containerd/remotes/errors"
"github.com/containerd/containerd/tracing"
"github.com/containerd/containerd/version"
digest "github.com/opencontainers/go-digest"
"github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/sirupsen/logrus"
"golang.org/x/net/context/ctxhttp"
)
var (
@ -70,6 +71,9 @@ type Authorizer interface {
// unmodified. It may also add an `Authorization` header as
// "bearer <some bearer token>"
// "basic <base64 encoded credentials>"
//
// It may return remotes/errors.ErrUnexpectedStatus, which for example,
// can be used by the caller to find out the status code returned by the registry.
Authorize(context.Context, *http.Request) error
// AddResponses adds a 401 response for the authorizer to consider when
@ -152,7 +156,8 @@ func NewResolver(options ResolverOptions) remotes.Resolver {
images.MediaTypeDockerSchema2Manifest,
images.MediaTypeDockerSchema2ManifestList,
ocispec.MediaTypeImageManifest,
ocispec.MediaTypeImageIndex, "*/*"}, ", "))
ocispec.MediaTypeImageIndex, "*/*",
}, ", "))
} else {
resolveHeader["Accept"] = options.Headers["Accept"]
delete(options.Headers, "Accept")
@ -299,11 +304,11 @@ func (r *dockerResolver) Resolve(ctx context.Context, ref string) (string, ocisp
if resp.StatusCode > 399 {
// Set firstErr when encountering the first non-404 status code.
if firstErr == nil {
firstErr = fmt.Errorf("pulling from host %s failed with status code %v: %v", host.Host, u, resp.Status)
firstErr = remoteerrors.NewUnexpectedStatusErr(resp)
}
continue // try another host
}
return "", ocispec.Descriptor{}, fmt.Errorf("pulling from host %s failed with unexpected status code %v: %v", host.Host, u, resp.Status)
return "", ocispec.Descriptor{}, remoteerrors.NewUnexpectedStatusErr(resp)
}
size := resp.ContentLength
contentType := getManifestMediaType(resp)
@ -340,26 +345,31 @@ func (r *dockerResolver) Resolve(ctx context.Context, ref string) (string, ocisp
if err != nil {
return "", ocispec.Descriptor{}, err
}
defer resp.Body.Close()
bodyReader := countingReader{reader: resp.Body}
contentType = getManifestMediaType(resp)
if dgst == "" {
err = func() error {
defer resp.Body.Close()
if dgst != "" {
_, err = io.Copy(io.Discard, &bodyReader)
return err
}
if contentType == images.MediaTypeDockerSchema1Manifest {
b, err := schema1.ReadStripSignature(&bodyReader)
if err != nil {
return "", ocispec.Descriptor{}, err
return err
}
dgst = digest.FromBytes(b)
} else {
dgst, err = digest.FromReader(&bodyReader)
if err != nil {
return "", ocispec.Descriptor{}, err
}
return nil
}
} else if _, err := io.Copy(io.Discard, &bodyReader); err != nil {
dgst, err = digest.FromReader(&bodyReader)
return err
}()
if err != nil {
return "", ocispec.Descriptor{}, err
}
size = bodyReader.bytesRead
@ -525,7 +535,7 @@ type request struct {
func (r *request) do(ctx context.Context) (*http.Response, error) {
u := r.host.Scheme + "://" + r.host.Host + r.path
req, err := http.NewRequest(r.method, u, nil)
req, err := http.NewRequestWithContext(ctx, r.method, u, nil)
if err != nil {
return nil, err
}
@ -551,7 +561,7 @@ func (r *request) do(ctx context.Context) (*http.Response, error) {
return nil, fmt.Errorf("failed to authorize: %w", err)
}
var client = &http.Client{}
client := &http.Client{}
if r.host.Client != nil {
*client = *r.host.Client
}
@ -566,11 +576,18 @@ func (r *request) do(ctx context.Context) (*http.Response, error) {
return nil
}
}
resp, err := ctxhttp.Do(ctx, client, req)
_, httpSpan := tracing.StartSpan(
ctx,
tracing.Name("remotes.docker.resolver", "HTTPRequest"),
tracing.WithHTTPRequest(req),
)
defer httpSpan.End()
resp, err := client.Do(req)
if err != nil {
httpSpan.SetStatus(err)
return nil, fmt.Errorf("failed to do request: %w", err)
}
httpSpan.SetAttributes(tracing.HTTPStatusCodeAttributes(resp.StatusCode)...)
log.G(ctx).WithFields(responseFields(resp)).Debug("fetch response received")
return resp, nil
}

View File

@ -14,6 +14,9 @@
limitations under the License.
*/
// Package schema1 provides a converter to fetch an image formatted in Docker Image Manifest v2, Schema 1.
//
// Deprecated: use images formatted in Docker Image Manifest v2, Schema 2, or OCI Image Spec v1.
package schema1
import (
@ -33,6 +36,7 @@ import (
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/labels"
"github.com/containerd/containerd/log"
"github.com/containerd/containerd/remotes"
digest "github.com/opencontainers/go-digest"
@ -363,12 +367,12 @@ func (c *Converter) fetchBlob(ctx context.Context, desc ocispec.Descriptor) erro
cinfo := content.Info{
Digest: desc.Digest,
Labels: map[string]string{
"containerd.io/uncompressed": state.diffID.String(),
labels.LabelUncompressed: state.diffID.String(),
labelDockerSchema1EmptyLayer: strconv.FormatBool(state.empty),
},
}
if _, err := c.contentStore.Update(ctx, cinfo, "labels.containerd.io/uncompressed", fmt.Sprintf("labels.%s", labelDockerSchema1EmptyLayer)); err != nil {
if _, err := c.contentStore.Update(ctx, cinfo, "labels."+labels.LabelUncompressed, fmt.Sprintf("labels.%s", labelDockerSchema1EmptyLayer)); err != nil {
return fmt.Errorf("failed to update uncompressed label: %w", err)
}
@ -387,7 +391,7 @@ func (c *Converter) reuseLabelBlobState(ctx context.Context, desc ocispec.Descri
}
desc.Size = cinfo.Size
diffID, ok := cinfo.Labels["containerd.io/uncompressed"]
diffID, ok := cinfo.Labels[labels.LabelUncompressed]
if !ok {
return false, nil
}
@ -406,7 +410,7 @@ func (c *Converter) reuseLabelBlobState(ctx context.Context, desc ocispec.Descri
bState := blobState{empty: isEmpty}
if bState.diffID, err = digest.Parse(diffID); err != nil {
log.G(ctx).WithField("id", desc.Digest).Warnf("failed to parse digest from label containerd.io/uncompressed: %v", diffID)
log.G(ctx).WithField("id", desc.Digest).Warnf("failed to parse digest from label %s: %v", labels.LabelUncompressed, diffID)
return false, nil
}

View File

@ -33,7 +33,7 @@ type ErrUnexpectedStatus struct {
}
func (e ErrUnexpectedStatus) Error() string {
return fmt.Sprintf("unexpected status: %s", e.Status)
return fmt.Sprintf("unexpected status from %s request to %s: %s", e.RequestMethod, e.RequestURL, e.Status)
}
// NewUnexpectedStatusErr creates an ErrUnexpectedStatus from HTTP response

View File

@ -100,20 +100,21 @@ func FetchHandler(ingester content.Ingester, fetcher Fetcher) images.HandlerFunc
case images.MediaTypeDockerSchema1Manifest:
return nil, fmt.Errorf("%v not supported", desc.MediaType)
default:
err := fetch(ctx, ingester, fetcher, desc)
err := Fetch(ctx, ingester, fetcher, desc)
if errdefs.IsAlreadyExists(err) {
return nil, nil
}
return nil, err
}
}
}
func fetch(ctx context.Context, ingester content.Ingester, fetcher Fetcher, desc ocispec.Descriptor) error {
// Fetch fetches the given digest into the provided ingester
func Fetch(ctx context.Context, ingester content.Ingester, fetcher Fetcher, desc ocispec.Descriptor) error {
log.G(ctx).Debug("fetch")
cw, err := content.OpenWriter(ctx, ingester, content.WithRef(MakeRefKey(ctx, desc)), content.WithDescriptor(desc))
if err != nil {
if errdefs.IsAlreadyExists(err) {
return nil
}
return err
}
defer cw.Close()
@ -135,7 +136,7 @@ func fetch(ctx context.Context, ingester content.Ingester, fetcher Fetcher, desc
if err != nil && !errdefs.IsAlreadyExists(err) {
return fmt.Errorf("failed commit on ref %q: %w", ws.Ref, err)
}
return nil
return err
}
rc, err := fetcher.Fetch(ctx, desc)
@ -197,17 +198,25 @@ func push(ctx context.Context, provider content.Provider, pusher Pusher, desc oc
//
// Base handlers can be provided which will be called before any push specific
// handlers.
func PushContent(ctx context.Context, pusher Pusher, desc ocispec.Descriptor, store content.Store, limiter *semaphore.Weighted, platform platforms.MatchComparer, wrapper func(h images.Handler) images.Handler) error {
//
// If the passed in content.Provider is also a content.Manager then this will
// also annotate the distribution sources in the manager.
func PushContent(ctx context.Context, pusher Pusher, desc ocispec.Descriptor, store content.Provider, limiter *semaphore.Weighted, platform platforms.MatchComparer, wrapper func(h images.Handler) images.Handler) error {
var m sync.Mutex
manifestStack := []ocispec.Descriptor{}
manifests := []ocispec.Descriptor{}
indexStack := []ocispec.Descriptor{}
filterHandler := images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
switch desc.MediaType {
case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest,
images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex:
case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest:
m.Lock()
manifestStack = append(manifestStack, desc)
manifests = append(manifests, desc)
m.Unlock()
return nil, images.ErrStopHandler
case images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex:
m.Lock()
indexStack = append(indexStack, desc)
m.Unlock()
return nil, images.ErrStopHandler
default:
@ -219,13 +228,14 @@ func PushContent(ctx context.Context, pusher Pusher, desc ocispec.Descriptor, st
platformFilterhandler := images.FilterPlatforms(images.ChildrenHandler(store), platform)
annotateHandler := annotateDistributionSourceHandler(platformFilterhandler, store)
var handler images.Handler
if m, ok := store.(content.Manager); ok {
annotateHandler := annotateDistributionSourceHandler(platformFilterhandler, m)
handler = images.Handlers(annotateHandler, filterHandler, pushHandler)
} else {
handler = images.Handlers(platformFilterhandler, filterHandler, pushHandler)
}
var handler images.Handler = images.Handlers(
annotateHandler,
filterHandler,
pushHandler,
)
if wrapper != nil {
handler = wrapper(handler)
}
@ -234,16 +244,18 @@ func PushContent(ctx context.Context, pusher Pusher, desc ocispec.Descriptor, st
return err
}
if err := images.Dispatch(ctx, pushHandler, limiter, manifests...); err != nil {
return err
}
// Iterate in reverse order as seen, parent always uploaded after child
for i := len(manifestStack) - 1; i >= 0; i-- {
_, err := pushHandler(ctx, manifestStack[i])
for i := len(indexStack) - 1; i >= 0; i-- {
err := images.Dispatch(ctx, pushHandler, limiter, indexStack[i])
if err != nil {
// TODO(estesp): until we have a more complete method for index push, we need to report
// missing dependencies in an index/manifest list by sensing the "400 Bad Request"
// as a marker for this problem
if (manifestStack[i].MediaType == ocispec.MediaTypeImageIndex ||
manifestStack[i].MediaType == images.MediaTypeDockerSchema2ManifestList) &&
errors.Unwrap(err) != nil && strings.Contains(errors.Unwrap(err).Error(), "400 Bad Request") {
if errors.Unwrap(err) != nil && strings.Contains(errors.Unwrap(err).Error(), "400 Bad Request") {
return fmt.Errorf("manifest list/index references to blobs and/or manifests are missing in your target registry: %w", err)
}
return err

View File

@ -21,6 +21,7 @@ import (
"io"
"github.com/containerd/containerd/content"
"github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)
@ -50,12 +51,23 @@ type Resolver interface {
Pusher(ctx context.Context, ref string) (Pusher, error)
}
// Fetcher fetches content
// Fetcher fetches content.
// A fetcher implementation may implement the FetcherByDigest interface too.
type Fetcher interface {
// Fetch the resource identified by the descriptor.
Fetch(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser, error)
}
// FetcherByDigest fetches content by the digest.
type FetcherByDigest interface {
// FetchByDigest fetches the resource identified by the digest.
//
// FetcherByDigest usually returns an incomplete descriptor.
// Typically, the media type is always set to "application/octet-stream",
// and the annotations are unset.
FetchByDigest(ctx context.Context, dgst digest.Digest) (io.ReadCloser, ocispec.Descriptor, error)
}
// Pusher pushes content
type Pusher interface {
// Push returns a content writer for the given resource identified

View File

@ -26,7 +26,8 @@ import (
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/log"
ptypes "github.com/gogo/protobuf/types"
"github.com/containerd/containerd/protobuf"
ptypes "github.com/containerd/containerd/protobuf/types"
digest "github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/sirupsen/logrus"
@ -37,6 +38,7 @@ import (
type service struct {
store content.Store
api.UnimplementedContentServer
}
var bufPool = sync.Pool{
@ -57,11 +59,12 @@ func (s *service) Register(server *grpc.Server) error {
}
func (s *service) Info(ctx context.Context, req *api.InfoRequest) (*api.InfoResponse, error) {
if err := req.Digest.Validate(); err != nil {
dg, err := digest.Parse(req.Digest)
if err != nil {
return nil, status.Errorf(codes.InvalidArgument, "%q failed validation", req.Digest)
}
bi, err := s.store.Info(ctx, req.Digest)
bi, err := s.store.Info(ctx, dg)
if err != nil {
return nil, errdefs.ToGRPC(err)
}
@ -72,7 +75,8 @@ func (s *service) Info(ctx context.Context, req *api.InfoRequest) (*api.InfoResp
}
func (s *service) Update(ctx context.Context, req *api.UpdateRequest) (*api.UpdateResponse, error) {
if err := req.Info.Digest.Validate(); err != nil {
_, err := digest.Parse(req.Info.Digest)
if err != nil {
return nil, status.Errorf(codes.InvalidArgument, "%q failed validation", req.Info.Digest)
}
@ -88,8 +92,8 @@ func (s *service) Update(ctx context.Context, req *api.UpdateRequest) (*api.Upda
func (s *service) List(req *api.ListContentRequest, session api.Content_ListServer) error {
var (
buffer []api.Info
sendBlock = func(block []api.Info) error {
buffer []*api.Info
sendBlock = func(block []*api.Info) error {
// send last block
return session.Send(&api.ListContentResponse{
Info: block,
@ -98,10 +102,10 @@ func (s *service) List(req *api.ListContentRequest, session api.Content_ListServ
)
if err := s.store.Walk(session.Context(), func(info content.Info) error {
buffer = append(buffer, api.Info{
Digest: info.Digest,
Size_: info.Size,
CreatedAt: info.CreatedAt,
buffer = append(buffer, &api.Info{
Digest: info.Digest.String(),
Size: info.Size,
CreatedAt: protobuf.ToTimestamp(info.CreatedAt),
Labels: info.Labels,
})
@ -130,11 +134,12 @@ func (s *service) List(req *api.ListContentRequest, session api.Content_ListServ
func (s *service) Delete(ctx context.Context, req *api.DeleteContentRequest) (*ptypes.Empty, error) {
log.G(ctx).WithField("digest", req.Digest).Debugf("delete content")
if err := req.Digest.Validate(); err != nil {
dg, err := digest.Parse(req.Digest)
if err != nil {
return nil, status.Errorf(codes.InvalidArgument, err.Error())
}
if err := s.store.Delete(ctx, req.Digest); err != nil {
if err := s.store.Delete(ctx, dg); err != nil {
return nil, errdefs.ToGRPC(err)
}
@ -142,16 +147,17 @@ func (s *service) Delete(ctx context.Context, req *api.DeleteContentRequest) (*p
}
func (s *service) Read(req *api.ReadContentRequest, session api.Content_ReadServer) error {
if err := req.Digest.Validate(); err != nil {
dg, err := digest.Parse(req.Digest)
if err != nil {
return status.Errorf(codes.InvalidArgument, "%v: %v", req.Digest, err)
}
oi, err := s.store.Info(session.Context(), req.Digest)
oi, err := s.store.Info(session.Context(), dg)
if err != nil {
return errdefs.ToGRPC(err)
}
ra, err := s.store.ReaderAt(session.Context(), ocispec.Descriptor{Digest: req.Digest})
ra, err := s.store.ReaderAt(session.Context(), ocispec.Descriptor{Digest: dg})
if err != nil {
return errdefs.ToGRPC(err)
}
@ -161,7 +167,7 @@ func (s *service) Read(req *api.ReadContentRequest, session api.Content_ReadServ
offset = req.Offset
// size is read size, not the expected size of the blob (oi.Size), which the caller might not be aware of.
// offset+size can be larger than oi.Size.
size = req.Size_
size = req.Size
// TODO(stevvooe): Using the global buffer pool. At 32KB, it is probably
// little inefficient for work over a fast network. We can tune this later.
@ -216,12 +222,12 @@ func (s *service) Status(ctx context.Context, req *api.StatusRequest) (*api.Stat
var resp api.StatusResponse
resp.Status = &api.Status{
StartedAt: status.StartedAt,
UpdatedAt: status.UpdatedAt,
StartedAt: protobuf.ToTimestamp(status.StartedAt),
UpdatedAt: protobuf.ToTimestamp(status.UpdatedAt),
Ref: status.Ref,
Offset: status.Offset,
Total: status.Total,
Expected: status.Expected,
Expected: status.Expected.String(),
}
return &resp, nil
@ -235,13 +241,13 @@ func (s *service) ListStatuses(ctx context.Context, req *api.ListStatusesRequest
var resp api.ListStatusesResponse
for _, status := range statuses {
resp.Statuses = append(resp.Statuses, api.Status{
StartedAt: status.StartedAt,
UpdatedAt: status.UpdatedAt,
resp.Statuses = append(resp.Statuses, &api.Status{
StartedAt: protobuf.ToTimestamp(status.StartedAt),
UpdatedAt: protobuf.ToTimestamp(status.UpdatedAt),
Ref: status.Ref,
Offset: status.Offset,
Total: status.Total,
Expected: status.Expected,
Expected: status.Expected.String(),
})
}
@ -293,7 +299,7 @@ func (s *service) Write(session api.Content_WriteServer) (err error) {
"ref": ref,
}
total = req.Total
expected = req.Expected
expected = digest.Digest(req.Expected)
if total > 0 {
fields["total"] = total
}
@ -341,12 +347,13 @@ func (s *service) Write(session api.Content_WriteServer) (err error) {
// Supporting these two paths is quite awkward but it lets both API
// users use the same writer style for each with a minimum of overhead.
if req.Expected != "" {
if expected != "" && expected != req.Expected {
log.G(ctx).Debugf("commit digest differs from writer digest: %v != %v", req.Expected, expected)
dg := digest.Digest(req.Expected)
if expected != "" && expected != dg {
log.G(ctx).Debugf("commit digest differs from writer digest: %v != %v", dg, expected)
}
expected = req.Expected
expected = dg
if _, err := s.store.Info(session.Context(), req.Expected); err == nil {
if _, err := s.store.Info(session.Context(), dg); err == nil {
if err := wr.Close(); err != nil {
log.G(ctx).WithError(err).Error("failed to close writer")
}
@ -368,12 +375,12 @@ func (s *service) Write(session api.Content_WriteServer) (err error) {
}
switch req.Action {
case api.WriteActionStat:
msg.Digest = wr.Digest()
msg.StartedAt = ws.StartedAt
msg.UpdatedAt = ws.UpdatedAt
case api.WriteAction_STAT:
msg.Digest = wr.Digest().String()
msg.StartedAt = protobuf.ToTimestamp(ws.StartedAt)
msg.UpdatedAt = protobuf.ToTimestamp(ws.UpdatedAt)
msg.Total = total
case api.WriteActionWrite, api.WriteActionCommit:
case api.WriteAction_WRITE, api.WriteAction_COMMIT:
if req.Offset > 0 {
// validate the offset if provided
if req.Offset != ws.Offset {
@ -406,7 +413,7 @@ func (s *service) Write(session api.Content_WriteServer) (err error) {
msg.Offset += int64(n)
}
if req.Action == api.WriteActionCommit {
if req.Action == api.WriteAction_COMMIT {
var opts []content.Opt
if req.Labels != nil {
opts = append(opts, content.WithLabels(req.Labels))
@ -416,14 +423,14 @@ func (s *service) Write(session api.Content_WriteServer) (err error) {
}
}
msg.Digest = wr.Digest()
msg.Digest = wr.Digest().String()
}
if err := session.Send(&msg); err != nil {
return err
}
if req.Action == api.WriteActionCommit {
if req.Action == api.WriteAction_COMMIT {
return nil
}
@ -446,22 +453,22 @@ func (s *service) Abort(ctx context.Context, req *api.AbortRequest) (*ptypes.Emp
return &ptypes.Empty{}, nil
}
func infoToGRPC(info content.Info) api.Info {
return api.Info{
Digest: info.Digest,
Size_: info.Size,
CreatedAt: info.CreatedAt,
UpdatedAt: info.UpdatedAt,
func infoToGRPC(info content.Info) *api.Info {
return &api.Info{
Digest: info.Digest.String(),
Size: info.Size,
CreatedAt: protobuf.ToTimestamp(info.CreatedAt),
UpdatedAt: protobuf.ToTimestamp(info.UpdatedAt),
Labels: info.Labels,
}
}
func infoFromGRPC(info api.Info) content.Info {
func infoFromGRPC(info *api.Info) content.Info {
return content.Info{
Digest: info.Digest,
Size: info.Size_,
CreatedAt: info.CreatedAt,
UpdatedAt: info.UpdatedAt,
Digest: digest.Digest(info.Digest),
Size: info.Size,
CreatedAt: protobuf.FromTimestamp(info.CreatedAt),
UpdatedAt: protobuf.FromTimestamp(info.UpdatedAt),
Labels: info.Labels,
}
}

View File

@ -0,0 +1,94 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package tracing
import (
"encoding/json"
"fmt"
"strings"
"go.opentelemetry.io/otel/attribute"
)
const (
spanDelimiter = "."
)
func makeSpanName(names ...string) string {
return strings.Join(names, spanDelimiter)
}
func any(k string, v interface{}) attribute.KeyValue {
if v == nil {
return attribute.String(k, "<nil>")
}
switch typed := v.(type) {
case bool:
return attribute.Bool(k, typed)
case []bool:
return attribute.BoolSlice(k, typed)
case int:
return attribute.Int(k, typed)
case []int:
return attribute.IntSlice(k, typed)
case int8:
return attribute.Int(k, int(typed))
case []int8:
ls := make([]int, 0, len(typed))
for _, i := range typed {
ls = append(ls, int(i))
}
return attribute.IntSlice(k, ls)
case int16:
return attribute.Int(k, int(typed))
case []int16:
ls := make([]int, 0, len(typed))
for _, i := range typed {
ls = append(ls, int(i))
}
return attribute.IntSlice(k, ls)
case int32:
return attribute.Int64(k, int64(typed))
case []int32:
ls := make([]int64, 0, len(typed))
for _, i := range typed {
ls = append(ls, int64(i))
}
return attribute.Int64Slice(k, ls)
case int64:
return attribute.Int64(k, typed)
case []int64:
return attribute.Int64Slice(k, typed)
case float64:
return attribute.Float64(k, typed)
case []float64:
return attribute.Float64Slice(k, typed)
case string:
return attribute.String(k, typed)
case []string:
return attribute.StringSlice(k, typed)
}
if stringer, ok := v.(fmt.Stringer); ok {
return attribute.String(k, stringer.String())
}
if b, err := json.Marshal(v); b != nil && err == nil {
return attribute.String(k, string(b))
}
return attribute.String(k, fmt.Sprintf("%v", v))
}

66
vendor/github.com/containerd/containerd/tracing/log.go generated vendored Normal file
View File

@ -0,0 +1,66 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package tracing
import (
"github.com/sirupsen/logrus"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
)
// NewLogrusHook creates a new logrus hook
func NewLogrusHook() *LogrusHook {
return &LogrusHook{}
}
// LogrusHook is a logrus hook which adds logrus events to active spans.
// If the span is not recording or the span context is invalid, the hook is a no-op.
type LogrusHook struct{}
// Levels returns the logrus levels that this hook is interested in.
func (h *LogrusHook) Levels() []logrus.Level {
return logrus.AllLevels
}
// Fire is called when a log event occurs.
func (h *LogrusHook) Fire(entry *logrus.Entry) error {
span := trace.SpanFromContext(entry.Context)
if span == nil {
return nil
}
if !span.SpanContext().IsValid() || !span.IsRecording() {
return nil
}
span.AddEvent(
entry.Message,
trace.WithAttributes(logrusDataToAttrs(entry.Data)...),
trace.WithAttributes(attribute.String("level", entry.Level.String())),
trace.WithTimestamp(entry.Time),
)
return nil
}
func logrusDataToAttrs(data logrus.Fields) []attribute.KeyValue {
attrs := make([]attribute.KeyValue, 0, len(data))
for k, v := range data {
attrs = append(attrs, any(k, v))
}
return attrs
}

View File

@ -0,0 +1,116 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package tracing
import (
"context"
"net/http"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
semconv "go.opentelemetry.io/otel/semconv/v1.12.0"
"go.opentelemetry.io/otel/trace"
)
// StartConfig defines configuration for a new span object.
type StartConfig struct {
spanOpts []trace.SpanStartOption
}
type SpanOpt func(config *StartConfig)
// WithHTTPRequest marks span as a HTTP request operation from client to server.
// It'll append attributes from the HTTP request object and mark it with `SpanKindClient` type.
func WithHTTPRequest(request *http.Request) SpanOpt {
return func(config *StartConfig) {
config.spanOpts = append(config.spanOpts,
trace.WithSpanKind(trace.SpanKindClient), // A client making a request to a server
trace.WithAttributes(semconv.HTTPClientAttributesFromHTTPRequest(request)...), // Add HTTP attributes
)
}
}
// StartSpan starts child span in a context.
func StartSpan(ctx context.Context, opName string, opts ...SpanOpt) (context.Context, *Span) {
config := StartConfig{}
for _, fn := range opts {
fn(&config)
}
tracer := otel.Tracer("")
if parent := trace.SpanFromContext(ctx); parent != nil && parent.SpanContext().IsValid() {
tracer = parent.TracerProvider().Tracer("")
}
ctx, span := tracer.Start(ctx, opName, config.spanOpts...)
return ctx, &Span{otelSpan: span}
}
// SpanFromContext returns the current Span from the context.
func SpanFromContext(ctx context.Context) *Span {
return &Span{
otelSpan: trace.SpanFromContext(ctx),
}
}
// Span is wrapper around otel trace.Span.
// Span is the individual component of a trace. It represents a
// single named and timed operation of a workflow that is traced.
type Span struct {
otelSpan trace.Span
}
// End completes the span.
func (s *Span) End() {
s.otelSpan.End()
}
// AddEvent adds an event with provided name and options.
func (s *Span) AddEvent(name string, options ...trace.EventOption) {
s.otelSpan.AddEvent(name, options...)
}
// SetStatus sets the status of the current span.
// If an error is encountered, it records the error and sets span status to Error.
func (s *Span) SetStatus(err error) {
if err != nil {
s.otelSpan.RecordError(err)
s.otelSpan.SetStatus(codes.Error, err.Error())
} else {
s.otelSpan.SetStatus(codes.Ok, "")
}
}
// SetAttributes sets kv as attributes of the span.
func (s *Span) SetAttributes(kv ...attribute.KeyValue) {
s.otelSpan.SetAttributes(kv...)
}
// Name sets the span name by joining a list of strings in dot separated format.
func Name(names ...string) string {
return makeSpanName(names...)
}
// Attribute takes a key value pair and returns attribute.KeyValue type.
func Attribute(k string, v interface{}) attribute.KeyValue {
return any(k, v)
}
// HTTPStatusCodeAttributes generates attributes of the HTTP namespace as specified by the OpenTelemetry
// specification for a span.
func HTTPStatusCodeAttributes(code int) []attribute.KeyValue {
return semconv.HTTPAttributesFromHTTPStatusCode(code)
}

View File

@ -23,7 +23,7 @@ var (
Package = "github.com/containerd/containerd"
// Version holds the complete version number. Filled in at linking time.
Version = "1.6.16+unknown"
Version = "1.7.0-beta.3+unknown"
// Revision is filled with the VCS (e.g. git) revision being used to build
// the program at linking time.

1
vendor/github.com/containerd/ttrpc/.gitattributes generated vendored Normal file
View File

@ -0,0 +1 @@
*.go text eol=lf

View File

@ -1,4 +1,5 @@
# Binaries for programs and plugins
/bin/
*.exe
*.dll
*.so
@ -9,3 +10,4 @@
# Output of the go coverage tool, specifically when used with LiteIDE
*.out
coverage.txt

54
vendor/github.com/containerd/ttrpc/.golangci.yml generated vendored Normal file
View File

@ -0,0 +1,54 @@
linters:
enable:
- structcheck
- varcheck
- staticcheck
- unconvert
- gofmt
- goimports
- revive
- ineffassign
- vet
- unused
- misspell
disable:
- errcheck
linters-settings:
revive:
ignore-generated-headers: true
rules:
- name: blank-imports
- name: context-as-argument
- name: context-keys-type
- name: dot-imports
- name: error-return
- name: error-strings
- name: error-naming
- name: exported
- name: if-return
- name: increment-decrement
- name: var-naming
arguments: [["UID", "GID"], []]
- name: var-declaration
- name: package-comments
- name: range
- name: receiver-naming
- name: time-naming
- name: unexported-return
- name: indent-error-flow
- name: errorf
- name: empty-block
- name: superfluous-else
- name: unused-parameter
- name: unreachable-code
- name: redefines-builtin-id
issues:
include:
- EXC0002
run:
timeout: 8m
skip-dirs:
- example

180
vendor/github.com/containerd/ttrpc/Makefile generated vendored Normal file
View File

@ -0,0 +1,180 @@
# Copyright The containerd Authors.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Go command to use for build
GO ?= go
INSTALL ?= install
# Root directory of the project (absolute path).
ROOTDIR=$(dir $(abspath $(lastword $(MAKEFILE_LIST))))
WHALE = "🇩"
ONI = "👹"
# Project binaries.
COMMANDS=protoc-gen-go-ttrpc protoc-gen-gogottrpc
ifdef BUILDTAGS
GO_BUILDTAGS = ${BUILDTAGS}
endif
GO_BUILDTAGS ?=
GO_TAGS=$(if $(GO_BUILDTAGS),-tags "$(strip $(GO_BUILDTAGS))",)
# Project packages.
PACKAGES=$(shell $(GO) list ${GO_TAGS} ./... | grep -v /example)
TESTPACKAGES=$(shell $(GO) list ${GO_TAGS} ./... | grep -v /cmd | grep -v /integration | grep -v /example)
BINPACKAGES=$(addprefix ./cmd/,$(COMMANDS))
#Replaces ":" (*nix), ";" (windows) with newline for easy parsing
GOPATHS=$(shell echo ${GOPATH} | tr ":" "\n" | tr ";" "\n")
TESTFLAGS_RACE=
GO_BUILD_FLAGS=
# See Golang issue re: '-trimpath': https://github.com/golang/go/issues/13809
GO_GCFLAGS=$(shell \
set -- ${GOPATHS}; \
echo "-gcflags=-trimpath=$${1}/src"; \
)
BINARIES=$(addprefix bin/,$(COMMANDS))
# Flags passed to `go test`
TESTFLAGS ?= $(TESTFLAGS_RACE) $(EXTRA_TESTFLAGS)
TESTFLAGS_PARALLEL ?= 8
# Use this to replace `go test` with, for instance, `gotestsum`
GOTEST ?= $(GO) test
.PHONY: clean all AUTHORS build binaries test integration generate protos checkprotos coverage ci check help install vendor install-protobuf install-protobuild
.DEFAULT: default
# Forcibly set the default goal to all, in case an include above brought in a rule definition.
.DEFAULT_GOAL := all
all: binaries
check: proto-fmt ## run all linters
@echo "$(WHALE) $@"
GOGC=75 golangci-lint run
ci: check binaries checkprotos coverage # coverage-integration ## to be used by the CI
AUTHORS: .mailmap .git/HEAD
git log --format='%aN <%aE>' | sort -fu > $@
generate: protos
@echo "$(WHALE) $@"
@PATH="${ROOTDIR}/bin:${PATH}" $(GO) generate -x ${PACKAGES}
protos: bin/protoc-gen-gogottrpc bin/protoc-gen-go-ttrpc ## generate protobuf
@echo "$(WHALE) $@"
@(PATH="${ROOTDIR}/bin:${PATH}" protobuild --quiet ${PACKAGES})
check-protos: protos ## check if protobufs needs to be generated again
@echo "$(WHALE) $@"
@test -z "$$(git status --short | grep ".pb.go" | tee /dev/stderr)" || \
((git diff | cat) && \
(echo "$(ONI) please run 'make protos' when making changes to proto files" && false))
check-api-descriptors: protos ## check that protobuf changes aren't present.
@echo "$(WHALE) $@"
@test -z "$$(git status --short | grep ".pb.txt" | tee /dev/stderr)" || \
((git diff $$(find . -name '*.pb.txt') | cat) && \
(echo "$(ONI) please run 'make protos' when making changes to proto files and check-in the generated descriptor file changes" && false))
proto-fmt: ## check format of proto files
@echo "$(WHALE) $@"
@test -z "$$(find . -name '*.proto' -type f -exec grep -Hn -e "^ " {} \; | tee /dev/stderr)" || \
(echo "$(ONI) please indent proto files with tabs only" && false)
@test -z "$$(find . -name '*.proto' -type f -exec grep -Hn "Meta meta = " {} \; | grep -v '(gogoproto.nullable) = false' | tee /dev/stderr)" || \
(echo "$(ONI) meta fields in proto files must have option (gogoproto.nullable) = false" && false)
build: ## build the go packages
@echo "$(WHALE) $@"
@$(GO) build ${DEBUG_GO_GCFLAGS} ${GO_GCFLAGS} ${GO_BUILD_FLAGS} ${EXTRA_FLAGS} ${PACKAGES}
test: ## run tests, except integration tests and tests that require root
@echo "$(WHALE) $@"
@$(GOTEST) ${TESTFLAGS} ${TESTPACKAGES}
integration: ## run integration tests
@echo "$(WHALE) $@"
@cd "${ROOTDIR}/integration" && $(GOTEST) -v ${TESTFLAGS} -parallel ${TESTFLAGS_PARALLEL} .
benchmark: ## run benchmarks tests
@echo "$(WHALE) $@"
@$(GO) test ${TESTFLAGS} -bench . -run Benchmark
FORCE:
define BUILD_BINARY
@echo "$(WHALE) $@"
@$(GO) build ${DEBUG_GO_GCFLAGS} ${GO_GCFLAGS} ${GO_BUILD_FLAGS} -o $@ ${GO_TAGS} ./$<
endef
# Build a binary from a cmd.
bin/%: cmd/% FORCE
$(call BUILD_BINARY)
binaries: $(BINARIES) ## build binaries
@echo "$(WHALE) $@"
clean: ## clean up binaries
@echo "$(WHALE) $@"
@rm -f $(BINARIES)
install: ## install binaries
@echo "$(WHALE) $@ $(BINPACKAGES)"
@$(GO) install $(BINPACKAGES)
install-protobuf:
@echo "$(WHALE) $@"
@script/install-protobuf
install-protobuild:
@echo "$(WHALE) $@"
@$(GO) install google.golang.org/protobuf/cmd/protoc-gen-go@v1.27.1
@$(GO) install github.com/containerd/protobuild@7e5ee24bc1f70e9e289fef15e2631eb3491320bf
coverage: ## generate coverprofiles from the unit tests, except tests that require root
@echo "$(WHALE) $@"
@rm -f coverage.txt
@$(GO) test -i ${TESTFLAGS} ${TESTPACKAGES} 2> /dev/null
@( for pkg in ${PACKAGES}; do \
$(GO) test ${TESTFLAGS} \
-cover \
-coverprofile=profile.out \
-covermode=atomic $$pkg || exit; \
if [ -f profile.out ]; then \
cat profile.out >> coverage.txt; \
rm profile.out; \
fi; \
done )
vendor: ## ensure all the go.mod/go.sum files are up-to-date
@echo "$(WHALE) $@"
@$(GO) mod tidy
@$(GO) mod verify
verify-vendor: ## verify if all the go.mod/go.sum files are up-to-date
@echo "$(WHALE) $@"
@$(GO) mod tidy
@$(GO) mod verify
@test -z "$$(git status --short | grep "go.sum" | tee /dev/stderr)" || \
((git diff | cat) && \
(echo "$(ONI) make sure to checkin changes after go mod tidy" && false))
help: ## this help
@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) | sort

240
vendor/github.com/containerd/ttrpc/PROTOCOL.md generated vendored Normal file
View File

@ -0,0 +1,240 @@
# Protocol Specification
The ttrpc protocol is client/server protocol to support multiple request streams
over a single connection with lightweight framing. The client represents the
process which initiated the underlying connection and the server is the process
which accepted the connection. The protocol is currently defined as
asymmetrical, with clients sending requests and servers sending responses. Both
clients and servers are able to send stream data. The roles are also used in
determining the stream identifiers, with client initiated streams using odd
number identifiers and server initiated using even number. The protocol may be
extended in the future to support server initiated streams, that is not
supported in the latest version.
## Purpose
The ttrpc protocol is designed to be lightweight and optimized for low latency
and reliable connections between processes on the same host. The protocol does
not include features for handling unreliable connections such as handshakes,
resets, pings, or flow control. The protocol is designed to make low-overhead
implementations as simple as possible. It is not intended as a suitable
replacement for HTTP2/3 over the network.
## Message Frame
Each Message Frame consists of a 10-byte message header followed
by message data. The data length and stream ID are both big-endian
4-byte unsigned integers. The message type is an unsigned 1-byte
integer. The flags are also an unsigned 1-byte integer and
use is defined by the message type.
+---------------------------------------------------------------+
| Data Length (32) |
+---------------------------------------------------------------+
| Stream ID (32) |
+---------------+-----------------------------------------------+
| Msg Type (8) |
+---------------+
| Flags (8) |
+---------------+-----------------------------------------------+
| Data (*) |
+---------------------------------------------------------------+
The Data Length field represents the number of bytes in the Data field. The
total frame size will always be Data Length + 10 bytes. The maximum data length
is 4MB and any larger size should be rejected. Due to the maximum data size
being less than 16MB, the first frame byte should always be zero. This first
byte should be considered reserved for future use.
The Stream ID must be odd for client initiated streams and even for server
initiated streams. Server initiated streams are not currently supported.
## Mesage Types
| Message Type | Name | Description |
|--------------|----------|----------------------------------|
| 0x01 | Request | Initiates stream |
| 0x02 | Response | Final stream data and terminates |
| 0x03 | Data | Stream data |
### Request
The request message is used to initiate stream and send along request data for
properly routing and handling the stream. The stream may indicate unary without
any inbound or outbound stream data with only a response is expected on the
stream. The request may also indicate the stream is still open for more data and
no response is expected until data is finished. If the remote indicates the
stream is closed, the request may be considered non-unary but without anymore
stream data sent. In the case of `remote closed`, the remote still expects to
receive a response or stream data. For compatibility with non streaming clients,
a request with empty flags indicates a unary request.
#### Request Flags
| Flag | Name | Description |
|------|-----------------|--------------------------------------------------|
| 0x01 | `remote closed` | Non-unary, but no more data expected from remote |
| 0x02 | `remote open` | Non-unary, remote is still sending data |
### Response
The response message is used to end a stream with data, an empty response, or
an error. A response message is the only expected message after a unary request.
A non-unary request does not require a response message if the server is sending
back stream data. A non-unary stream may return a single response message but no
other stream data may follow.
#### Response Flags
No response flags are defined at this time, flags should be empty.
### Data
The data message is used to send data on an already initialized stream. Either
client or server may send data. A data message is not allowed on a unary stream.
A data message should not be sent after indicating `remote closed` to the peer.
The last data message on a stream must set the `remote closed` flag.
The `no data` flag is used to indicate that the data message does not include
any data. This is normally used with the `remote closed` flag to indicate the
stream is now closed without transmitting any data. Since ttrpc normally
transmits a single object per message, a zero length data message may be
interpreted as an empty object. For example, transmitting the number zero as a
protobuf message ends up with a data length of zero, but the message is still
considered data and should be processed.
#### Data Flags
| Flag | Name | Description |
|------|-----------------|-----------------------------------|
| 0x01 | `remote closed` | No more data expected from remote |
| 0x04 | `no data` | This message does not have data |
## Streaming
All ttrpc requests use streams to transfer data. Unary streams will only have
two messages sent per stream, a request from a client and a response from the
server. Non-unary streams, however, may send any numbers of messages from the
client and the server. This makes stream management more complicated than unary
streams since both client and server need to track additional state. To keep
this management as simple as possible, ttrpc minimizes the number of states and
uses two flags instead of control frames. Each stream has two states while a
stream is still alive: `local closed` and `remote closed`. Each peer considers
local and remote from their own perspective and sets flags from the other peer's
perspective. For example, if a client sends a data frame with the
`remote closed` flag, that is indicating that the client is now `local closed`
and the server will be `remote closed`. A unary operation does not need to send
these flags since each received message always indicates `remote closed`. Once a
peer is both `local closed` and `remote closed`, the stream is considered
finished and may be cleaned up.
Due to the asymmetric nature of the current protocol, a client should
always be in the `local closed` state before `remote closed` and a server should
always be in the `remote closed` state before `local closed`. This happens
because the client is always initiating requests and a client always expects a
final response back from a server to indicate the initiated request has been
fulfilled. This may mean server sends a final empty response to finish a stream
even after it has already completed sending data before the client.
### Unary State Diagram
+--------+ +--------+
| Client | | Server |
+---+----+ +----+---+
| +---------+ |
local >---------------+ Request +--------------------> remote
closed | +---------+ | closed
| |
| +----------+ |
finished <--------------+ Response +--------------------< finished
| +----------+ |
| |
### Non-Unary State Diagrams
RC: `remote closed` flag
RO: `remote open` flag
+--------+ +--------+
| Client | | Server |
+---+----+ +----+---+
| +--------------+ |
>-------------+ Request [RO] +----------------->
| +--------------+ |
| |
| +------+ |
>-----------------+ Data +--------------------->
| +------+ |
| |
| +-----------+ |
local >---------------+ Data [RC] +------------------> remote
closed | +-----------+ | closed
| |
| +----------+ |
finished <--------------+ Response +--------------------< finished
| +----------+ |
| |
+--------+ +--------+
| Client | | Server |
+---+----+ +----+---+
| +--------------+ |
local >-------------+ Request [RC] +-----------------> remote
closed | +--------------+ | closed
| |
| +------+ |
<-----------------+ Data +---------------------<
| +------+ |
| |
| +-----------+ |
finished <---------------+ Data [RC] +------------------< finished
| +-----------+ |
| |
+--------+ +--------+
| Client | | Server |
+---+----+ +----+---+
| +--------------+ |
>-------------+ Request [RO] +----------------->
| +--------------+ |
| |
| +------+ |
>-----------------+ Data +--------------------->
| +------+ |
| |
| +------+ |
<-----------------+ Data +---------------------<
| +------+ |
| |
| +------+ |
>-----------------+ Data +--------------------->
| +------+ |
| |
| +-----------+ |
local >---------------+ Data [RC] +------------------> remote
closed | +-----------+ | closed
| |
| +------+ |
<-----------------+ Data +---------------------<
| +------+ |
| |
| +-----------+ |
finished <---------------+ Data [RC] +------------------< finished
| +-----------+ |
| |
## RPC
While this protocol is defined primarily to support Remote Procedure Calls, the
protocol does not define the request and response types beyond the messages
defined in the protocol. The implementation provides a default protobuf
definition of request and response which may be used for cross language rpc.
All implementations should at least define a request type which support
routing by procedure name and a response type which supports call status.
## Version History
| Version | Features |
|---------|---------------------|
| 1.0 | Unary requests only |
| 1.2 | Streaming support |

25
vendor/github.com/containerd/ttrpc/Protobuild.toml generated vendored Normal file
View File

@ -0,0 +1,25 @@
version = "2"
generators = ["go"]
# Control protoc include paths. Below are usually some good defaults, but feel
# free to try it without them if it works for your project.
[includes]
# Include paths that will be added before all others. Typically, you want to
# treat the root of the project as an include, but this may not be necessary.
before = ["."]
# Paths that will be added untouched to the end of the includes. We use
# `/usr/local/include` to pickup the common install location of protobuf.
# This is the default.
after = ["/usr/local/include"]
# This section maps protobuf imports to Go packages. These will become
# `-M` directives in the call to the go protobuf generator.
[packages]
"google/protobuf/any.proto" = "github.com/gogo/protobuf/types"
"proto/status.proto" = "google.golang.org/genproto/googleapis/rpc/status"
[[overrides]]
# enable ttrpc and disable fieldpath and grpc for the shim
prefixes = ["github.com/containerd/ttrpc/integration/streaming"]
generators = ["go", "go-ttrpc"]

View File

@ -20,6 +20,10 @@ Please note that while this project supports generating either end of the
protocol, the generated service definitions will be incompatible with regular
GRPC services, as they do not speak the same protocol.
# Protocol
See the [protocol specification](./PROTOCOL.md).
# Usage
Create a gogo vanity binary (see
@ -43,7 +47,6 @@ directly, if required.
TODO:
- [ ] Document protocol layout
- [ ] Add testing under concurrent load to ensure
- [ ] Verify connection error handling

View File

@ -38,6 +38,26 @@ type messageType uint8
const (
messageTypeRequest messageType = 0x1
messageTypeResponse messageType = 0x2
messageTypeData messageType = 0x3
)
func (mt messageType) String() string {
switch mt {
case messageTypeRequest:
return "request"
case messageTypeResponse:
return "response"
case messageTypeData:
return "data"
default:
return "unknown"
}
}
const (
flagRemoteClosed uint8 = 0x1
flagRemoteOpen uint8 = 0x2
flagNoData uint8 = 0x4
)
// messageHeader represents the fixed-length message header of 10 bytes sent
@ -46,7 +66,7 @@ type messageHeader struct {
Length uint32 // length excluding this header. b[:4]
StreamID uint32 // identifies which request stream message is a part of. b[4:8]
Type messageType // message type b[8]
Flags uint8 // reserved b[9]
Flags uint8 // type specific flags b[9]
}
func readMessageHeader(p []byte, r io.Reader) (messageHeader, error) {
@ -111,22 +131,31 @@ func (ch *channel) recv() (messageHeader, []byte, error) {
return mh, nil, status.Errorf(codes.ResourceExhausted, "message length %v exceed maximum message size of %v", mh.Length, messageLengthMax)
}
p := ch.getmbuf(int(mh.Length))
if _, err := io.ReadFull(ch.br, p); err != nil {
return messageHeader{}, nil, fmt.Errorf("failed reading message: %w", err)
var p []byte
if mh.Length > 0 {
p = ch.getmbuf(int(mh.Length))
if _, err := io.ReadFull(ch.br, p); err != nil {
return messageHeader{}, nil, fmt.Errorf("failed reading message: %w", err)
}
}
return mh, p, nil
}
func (ch *channel) send(streamID uint32, t messageType, p []byte) error {
if err := writeMessageHeader(ch.bw, ch.hwbuf[:], messageHeader{Length: uint32(len(p)), StreamID: streamID, Type: t}); err != nil {
func (ch *channel) send(streamID uint32, t messageType, flags uint8, p []byte) error {
// TODO: Error on send rather than on recv
//if len(p) > messageLengthMax {
// return status.Errorf(codes.InvalidArgument, "refusing to send, message length %v exceed maximum message size of %v", len(p), messageLengthMax)
//}
if err := writeMessageHeader(ch.bw, ch.hwbuf[:], messageHeader{Length: uint32(len(p)), StreamID: streamID, Type: t, Flags: flags}); err != nil {
return err
}
_, err := ch.bw.Write(p)
if err != nil {
return err
if len(p) > 0 {
_, err := ch.bw.Write(p)
if err != nil {
return err
}
}
return ch.bw.Flush()

View File

@ -19,30 +19,30 @@ package ttrpc
import (
"context"
"errors"
"fmt"
"io"
"net"
"os"
"strings"
"sync"
"syscall"
"time"
"github.com/gogo/protobuf/proto"
"github.com/sirupsen/logrus"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/proto"
)
// ErrClosed is returned by client methods when the underlying connection is
// closed.
var ErrClosed = errors.New("ttrpc: closed")
// Client for a ttrpc server
type Client struct {
codec codec
conn net.Conn
channel *channel
calls chan *callRequest
streamLock sync.RWMutex
streams map[streamID]*stream
nextStreamID streamID
sendLock sync.Mutex
ctx context.Context
closed func()
@ -51,8 +51,6 @@ type Client struct {
userCloseFunc func()
userCloseWaitCh chan struct{}
errOnce sync.Once
err error
interceptor UnaryClientInterceptor
}
@ -73,13 +71,16 @@ func WithUnaryClientInterceptor(i UnaryClientInterceptor) ClientOpts {
}
}
// NewClient creates a new ttrpc client using the given connection
func NewClient(conn net.Conn, opts ...ClientOpts) *Client {
ctx, cancel := context.WithCancel(context.Background())
channel := newChannel(conn)
c := &Client{
codec: codec{},
conn: conn,
channel: newChannel(conn),
calls: make(chan *callRequest),
channel: channel,
streams: make(map[streamID]*stream),
nextStreamID: 1,
closed: cancel,
ctx: ctx,
userCloseFunc: func() {},
@ -95,13 +96,13 @@ func NewClient(conn net.Conn, opts ...ClientOpts) *Client {
return c
}
type callRequest struct {
ctx context.Context
req *Request
resp *Response // response will be written back here
errs chan error // error written here on completion
func (c *Client) send(sid uint32, mt messageType, flags uint8, b []byte) error {
c.sendLock.Lock()
defer c.sendLock.Unlock()
return c.channel.send(sid, mt, flags, b)
}
// Call makes a unary request and returns with response
func (c *Client) Call(ctx context.Context, service, method string, req, resp interface{}) error {
payload, err := c.codec.Marshal(req)
if err != nil {
@ -113,6 +114,7 @@ func (c *Client) Call(ctx context.Context, service, method string, req, resp int
Service: service,
Method: method,
Payload: payload,
// TODO: metadata from context
}
cresp = &Response{}
@ -123,7 +125,7 @@ func (c *Client) Call(ctx context.Context, service, method string, req, resp int
}
if dl, ok := ctx.Deadline(); ok {
creq.TimeoutNano = dl.Sub(time.Now()).Nanoseconds()
creq.TimeoutNano = time.Until(dl).Nanoseconds()
}
info := &UnaryClientInfo{
@ -143,36 +145,137 @@ func (c *Client) Call(ctx context.Context, service, method string, req, resp int
return nil
}
func (c *Client) dispatch(ctx context.Context, req *Request, resp *Response) error {
errs := make(chan error, 1)
call := &callRequest{
ctx: ctx,
req: req,
resp: resp,
errs: errs,
}
// StreamDesc describes the stream properties, whether the stream has
// a streaming client, a streaming server, or both
type StreamDesc struct {
StreamingClient bool
StreamingServer bool
}
select {
case <-ctx.Done():
return ctx.Err()
case c.calls <- call:
case <-c.ctx.Done():
return c.error()
}
// ClientStream is used to send or recv messages on the underlying stream
type ClientStream interface {
CloseSend() error
SendMsg(m interface{}) error
RecvMsg(m interface{}) error
}
select {
case <-ctx.Done():
return ctx.Err()
case err := <-errs:
type clientStream struct {
ctx context.Context
s *stream
c *Client
desc *StreamDesc
localClosed bool
remoteClosed bool
}
func (cs *clientStream) CloseSend() error {
if !cs.desc.StreamingClient {
return fmt.Errorf("%w: cannot close non-streaming client", ErrProtocol)
}
if cs.localClosed {
return ErrStreamClosed
}
err := cs.s.send(messageTypeData, flagRemoteClosed|flagNoData, nil)
if err != nil {
return filterCloseErr(err)
case <-c.ctx.Done():
return c.error()
}
cs.localClosed = true
return nil
}
func (cs *clientStream) SendMsg(m interface{}) error {
if !cs.desc.StreamingClient {
return fmt.Errorf("%w: cannot send data from non-streaming client", ErrProtocol)
}
if cs.localClosed {
return ErrStreamClosed
}
var (
payload []byte
err error
)
if m != nil {
payload, err = cs.c.codec.Marshal(m)
if err != nil {
return err
}
}
err = cs.s.send(messageTypeData, 0, payload)
if err != nil {
return filterCloseErr(err)
}
return nil
}
func (cs *clientStream) RecvMsg(m interface{}) error {
if cs.remoteClosed {
return io.EOF
}
select {
case <-cs.ctx.Done():
return cs.ctx.Err()
case msg, ok := <-cs.s.recv:
if !ok {
return cs.s.recvErr
}
if msg.header.Type == messageTypeResponse {
resp := &Response{}
err := proto.Unmarshal(msg.payload[:msg.header.Length], resp)
// return the payload buffer for reuse
cs.c.channel.putmbuf(msg.payload)
if err != nil {
return err
}
if err := cs.c.codec.Unmarshal(resp.Payload, m); err != nil {
return err
}
if resp.Status != nil && resp.Status.Code != int32(codes.OK) {
return status.ErrorProto(resp.Status)
}
cs.c.deleteStream(cs.s)
cs.remoteClosed = true
return nil
} else if msg.header.Type == messageTypeData {
if !cs.desc.StreamingServer {
cs.c.deleteStream(cs.s)
cs.remoteClosed = true
return fmt.Errorf("received data from non-streaming server: %w", ErrProtocol)
}
if msg.header.Flags&flagRemoteClosed == flagRemoteClosed {
cs.c.deleteStream(cs.s)
cs.remoteClosed = true
if msg.header.Flags&flagNoData == flagNoData {
return io.EOF
}
}
err := cs.c.codec.Unmarshal(msg.payload[:msg.header.Length], m)
cs.c.channel.putmbuf(msg.payload)
if err != nil {
return err
}
return nil
}
return fmt.Errorf("unexpected %q message received: %w", msg.header.Type, ErrProtocol)
}
}
// Close closes the ttrpc connection and underlying connection
func (c *Client) Close() error {
c.closeOnce.Do(func() {
c.closed()
c.conn.Close()
})
return nil
}
@ -188,194 +291,105 @@ func (c *Client) UserOnCloseWait(ctx context.Context) error {
}
}
type message struct {
messageHeader
p []byte
err error
}
// callMap provides access to a map of active calls, guarded by a mutex.
type callMap struct {
m sync.Mutex
activeCalls map[uint32]*callRequest
closeErr error
}
// newCallMap returns a new callMap with an empty set of active calls.
func newCallMap() *callMap {
return &callMap{
activeCalls: make(map[uint32]*callRequest),
}
}
// set adds a call entry to the map with the given streamID key.
func (cm *callMap) set(streamID uint32, cr *callRequest) error {
cm.m.Lock()
defer cm.m.Unlock()
if cm.closeErr != nil {
return cm.closeErr
}
cm.activeCalls[streamID] = cr
return nil
}
// get looks up the call entry for the given streamID key, then removes it
// from the map and returns it.
func (cm *callMap) get(streamID uint32) (cr *callRequest, ok bool, err error) {
cm.m.Lock()
defer cm.m.Unlock()
if cm.closeErr != nil {
return nil, false, cm.closeErr
}
cr, ok = cm.activeCalls[streamID]
if ok {
delete(cm.activeCalls, streamID)
}
return
}
// abort sends the given error to each active call, and clears the map.
// Once abort has been called, any subsequent calls to the callMap will return the error passed to abort.
func (cm *callMap) abort(err error) error {
cm.m.Lock()
defer cm.m.Unlock()
if cm.closeErr != nil {
return cm.closeErr
}
for streamID, call := range cm.activeCalls {
call.errs <- err
delete(cm.activeCalls, streamID)
}
cm.closeErr = err
return nil
}
func (c *Client) run() {
var (
waiters = newCallMap()
receiverDone = make(chan struct{})
)
err := c.receiveLoop()
c.Close()
c.cleanupStreams(err)
// Sender goroutine
// Receives calls from dispatch, adds them to the set of active calls, and sends them
// to the server.
go func() {
var streamID uint32 = 1
for {
select {
case <-c.ctx.Done():
return
case call := <-c.calls:
id := streamID
streamID += 2 // enforce odd client initiated request ids
if err := waiters.set(id, call); err != nil {
call.errs <- err // errs is buffered so should not block.
continue
}
if err := c.send(id, messageTypeRequest, call.req); err != nil {
call.errs <- err // errs is buffered so should not block.
waiters.get(id) // remove from waiters set
}
}
}
}()
// Receiver goroutine
// Receives responses from the server, looks up the call info in the set of active calls,
// and notifies the caller of the response.
go func() {
defer close(receiverDone)
for {
select {
case <-c.ctx.Done():
c.setError(c.ctx.Err())
return
default:
mh, p, err := c.channel.recv()
if err != nil {
_, ok := status.FromError(err)
if !ok {
// treat all errors that are not an rpc status as terminal.
// all others poison the connection.
c.setError(filterCloseErr(err))
return
}
}
msg := &message{
messageHeader: mh,
p: p[:mh.Length],
err: err,
}
call, ok, err := waiters.get(mh.StreamID)
if err != nil {
logrus.Errorf("ttrpc: failed to look up active call: %s", err)
continue
}
if !ok {
logrus.Errorf("ttrpc: received message for unknown channel %v", mh.StreamID)
continue
}
call.errs <- c.recv(call.resp, msg)
}
}
}()
defer func() {
c.conn.Close()
c.userCloseFunc()
close(c.userCloseWaitCh)
}()
c.userCloseFunc()
close(c.userCloseWaitCh)
}
func (c *Client) receiveLoop() error {
for {
select {
case <-receiverDone:
// The receiver has exited.
// don't return out, let the close of the context trigger the abort of waiters
c.Close()
case <-c.ctx.Done():
// Abort all active calls. This will also prevent any new calls from being added
// to waiters.
waiters.abort(c.error())
return
return ErrClosed
default:
var (
msg = &streamMessage{}
err error
)
msg.header, msg.payload, err = c.channel.recv()
if err != nil {
_, ok := status.FromError(err)
if !ok {
// treat all errors that are not an rpc status as terminal.
// all others poison the connection.
return filterCloseErr(err)
}
}
sid := streamID(msg.header.StreamID)
s := c.getStream(sid)
if s == nil {
logrus.WithField("stream", sid).Errorf("ttrpc: received message on inactive stream")
continue
}
if err != nil {
s.closeWithError(err)
} else {
if err := s.receive(c.ctx, msg); err != nil {
logrus.WithError(err).WithField("stream", sid).Errorf("ttrpc: failed to handle message")
}
}
}
}
}
func (c *Client) error() error {
c.errOnce.Do(func() {
if c.err == nil {
c.err = ErrClosed
}
})
return c.err
}
// createStream creates a new stream and registers it with the client
// Introduce stream types for multiple or single response
func (c *Client) createStream(flags uint8, b []byte) (*stream, error) {
c.streamLock.Lock()
func (c *Client) setError(err error) {
c.errOnce.Do(func() {
c.err = err
})
}
func (c *Client) send(streamID uint32, mtype messageType, msg interface{}) error {
p, err := c.codec.Marshal(msg)
if err != nil {
return err
// Check if closed since lock acquired to prevent adding
// anything after cleanup completes
select {
case <-c.ctx.Done():
c.streamLock.Unlock()
return nil, ErrClosed
default:
}
return c.channel.send(streamID, mtype, p)
// Stream ID should be allocated at same time
s := newStream(c.nextStreamID, c)
c.streams[s.id] = s
c.nextStreamID = c.nextStreamID + 2
c.sendLock.Lock()
defer c.sendLock.Unlock()
c.streamLock.Unlock()
if err := c.channel.send(uint32(s.id), messageTypeRequest, flags, b); err != nil {
return s, filterCloseErr(err)
}
return s, nil
}
func (c *Client) recv(resp *Response, msg *message) error {
if msg.err != nil {
return msg.err
}
func (c *Client) deleteStream(s *stream) {
c.streamLock.Lock()
delete(c.streams, s.id)
c.streamLock.Unlock()
s.closeWithError(nil)
}
if msg.Type != messageTypeResponse {
return errors.New("unknown message type received")
}
func (c *Client) getStream(sid streamID) *stream {
c.streamLock.RLock()
s := c.streams[sid]
c.streamLock.RUnlock()
return s
}
defer c.channel.putmbuf(msg.p)
return proto.Unmarshal(msg.p, resp)
func (c *Client) cleanupStreams(err error) {
c.streamLock.Lock()
defer c.streamLock.Unlock()
for sid, s := range c.streams {
s.closeWithError(err)
delete(c.streams, sid)
}
}
// filterCloseErr rewrites EOF and EPIPE errors to ErrClosed. Use when
@ -388,6 +402,8 @@ func filterCloseErr(err error) error {
return nil
case err == io.EOF:
return ErrClosed
case errors.Is(err, io.ErrClosedPipe):
return ErrClosed
case errors.Is(err, io.EOF):
return ErrClosed
case strings.Contains(err.Error(), "use of closed network connection"):
@ -395,11 +411,9 @@ func filterCloseErr(err error) error {
default:
// if we have an epipe on a write or econnreset on a read , we cast to errclosed
var oerr *net.OpError
if errors.As(err, &oerr) && (oerr.Op == "write" || oerr.Op == "read") {
serr, sok := oerr.Err.(*os.SyscallError)
if sok && ((serr.Err == syscall.EPIPE && oerr.Op == "write") ||
(serr.Err == syscall.ECONNRESET && oerr.Op == "read")) {
if errors.As(err, &oerr) {
if (oerr.Op == "write" && errors.Is(err, syscall.EPIPE)) ||
(oerr.Op == "read" && errors.Is(err, syscall.ECONNRESET)) {
return ErrClosed
}
}
@ -407,3 +421,81 @@ func filterCloseErr(err error) error {
return err
}
// NewStream creates a new stream with the given stream descriptor to the
// specified service and method. If not a streaming client, the request object
// may be provided.
func (c *Client) NewStream(ctx context.Context, desc *StreamDesc, service, method string, req interface{}) (ClientStream, error) {
var payload []byte
if req != nil {
var err error
payload, err = c.codec.Marshal(req)
if err != nil {
return nil, err
}
}
request := &Request{
Service: service,
Method: method,
Payload: payload,
// TODO: metadata from context
}
p, err := c.codec.Marshal(request)
if err != nil {
return nil, err
}
var flags uint8
if desc.StreamingClient {
flags = flagRemoteOpen
} else {
flags = flagRemoteClosed
}
s, err := c.createStream(flags, p)
if err != nil {
return nil, err
}
return &clientStream{
ctx: ctx,
s: s,
c: c,
desc: desc,
}, nil
}
func (c *Client) dispatch(ctx context.Context, req *Request, resp *Response) error {
p, err := c.codec.Marshal(req)
if err != nil {
return err
}
s, err := c.createStream(0, p)
if err != nil {
return err
}
defer c.deleteStream(s)
select {
case <-ctx.Done():
return ctx.Err()
case <-c.ctx.Done():
return ErrClosed
case msg, ok := <-s.recv:
if !ok {
return s.recvErr
}
if msg.header.Type == messageTypeResponse {
err = proto.Unmarshal(msg.payload[:msg.header.Length], resp)
} else {
err = fmt.Errorf("unexpected %q message received: %w", msg.header.Type, ErrProtocol)
}
// return the payload buffer for reuse
c.channel.putmbuf(msg.payload)
return err
}
}

View File

@ -19,7 +19,7 @@ package ttrpc
import (
"fmt"
"github.com/gogo/protobuf/proto"
"google.golang.org/protobuf/proto"
)
type codec struct{}

23
vendor/github.com/containerd/ttrpc/doc.go generated vendored Normal file
View File

@ -0,0 +1,23 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
/*
package ttrpc defines and implements a low level simple transfer protocol
optimized for low latency and reliable connections between processes on the same
host. The protocol uses simple framing for sending requests, responses, and data
using multiple streams.
*/
package ttrpc

34
vendor/github.com/containerd/ttrpc/errors.go generated vendored Normal file
View File

@ -0,0 +1,34 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package ttrpc
import "errors"
var (
// ErrProtocol is a general error in the handling the protocol.
ErrProtocol = errors.New("protocol error")
// ErrClosed is returned by client methods when the underlying connection is
// closed.
ErrClosed = errors.New("ttrpc: closed")
// ErrServerClosed is returned when the Server has closed its connection.
ErrServerClosed = errors.New("ttrpc: server closed")
// ErrStreamClosed is when the streaming connection is closed.
ErrStreamClosed = errors.New("ttrpc: stream closed")
)

View File

@ -45,6 +45,6 @@ func (fn handshakerFunc) Handshake(ctx context.Context, conn net.Conn) (net.Conn
return fn(ctx, conn)
}
func noopHandshake(ctx context.Context, conn net.Conn) (net.Conn, interface{}, error) {
func noopHandshake(_ context.Context, conn net.Conn) (net.Conn, interface{}, error) {
return conn, nil, nil
}

View File

@ -28,6 +28,13 @@ type UnaryClientInfo struct {
FullMethod string
}
// StreamServerInfo provides information about the server request
type StreamServerInfo struct {
FullMethod string
StreamingClient bool
StreamingServer bool
}
// Unmarshaler contains the server request data and allows it to be unmarshaled
// into a concrete type
type Unmarshaler func(interface{}) error
@ -41,10 +48,18 @@ type UnaryServerInterceptor func(context.Context, Unmarshaler, *UnaryServerInfo,
// UnaryClientInterceptor specifies the interceptor function for client request/response
type UnaryClientInterceptor func(context.Context, *Request, *Response, *UnaryClientInfo, Invoker) error
func defaultServerInterceptor(ctx context.Context, unmarshal Unmarshaler, info *UnaryServerInfo, method Method) (interface{}, error) {
func defaultServerInterceptor(ctx context.Context, unmarshal Unmarshaler, _ *UnaryServerInfo, method Method) (interface{}, error) {
return method(ctx, unmarshal)
}
func defaultClientInterceptor(ctx context.Context, req *Request, resp *Response, _ *UnaryClientInfo, invoker Invoker) error {
return invoker(ctx, req, resp)
}
type StreamServerInterceptor func(context.Context, StreamServer, *StreamServerInfo, StreamHandler) (interface{}, error)
func defaultStreamServerInterceptor(ctx context.Context, ss StreamServer, _ *StreamServerInfo, stream StreamHandler) (interface{}, error) {
return stream(ctx, ss)
}
type StreamClientInterceptor func(context.Context)

396
vendor/github.com/containerd/ttrpc/request.pb.go generated vendored Normal file
View File

@ -0,0 +1,396 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.27.1
// protoc v3.11.4
// source: github.com/containerd/ttrpc/request.proto
package ttrpc
import (
status "google.golang.org/genproto/googleapis/rpc/status"
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
reflect "reflect"
sync "sync"
)
const (
// Verify that this generated code is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
// Verify that runtime/protoimpl is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
)
type Request struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Service string `protobuf:"bytes,1,opt,name=service,proto3" json:"service,omitempty"`
Method string `protobuf:"bytes,2,opt,name=method,proto3" json:"method,omitempty"`
Payload []byte `protobuf:"bytes,3,opt,name=payload,proto3" json:"payload,omitempty"`
TimeoutNano int64 `protobuf:"varint,4,opt,name=timeout_nano,json=timeoutNano,proto3" json:"timeout_nano,omitempty"`
Metadata []*KeyValue `protobuf:"bytes,5,rep,name=metadata,proto3" json:"metadata,omitempty"`
}
func (x *Request) Reset() {
*x = Request{}
if protoimpl.UnsafeEnabled {
mi := &file_github_com_containerd_ttrpc_request_proto_msgTypes[0]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *Request) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*Request) ProtoMessage() {}
func (x *Request) ProtoReflect() protoreflect.Message {
mi := &file_github_com_containerd_ttrpc_request_proto_msgTypes[0]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use Request.ProtoReflect.Descriptor instead.
func (*Request) Descriptor() ([]byte, []int) {
return file_github_com_containerd_ttrpc_request_proto_rawDescGZIP(), []int{0}
}
func (x *Request) GetService() string {
if x != nil {
return x.Service
}
return ""
}
func (x *Request) GetMethod() string {
if x != nil {
return x.Method
}
return ""
}
func (x *Request) GetPayload() []byte {
if x != nil {
return x.Payload
}
return nil
}
func (x *Request) GetTimeoutNano() int64 {
if x != nil {
return x.TimeoutNano
}
return 0
}
func (x *Request) GetMetadata() []*KeyValue {
if x != nil {
return x.Metadata
}
return nil
}
type Response struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Status *status.Status `protobuf:"bytes,1,opt,name=status,proto3" json:"status,omitempty"`
Payload []byte `protobuf:"bytes,2,opt,name=payload,proto3" json:"payload,omitempty"`
}
func (x *Response) Reset() {
*x = Response{}
if protoimpl.UnsafeEnabled {
mi := &file_github_com_containerd_ttrpc_request_proto_msgTypes[1]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *Response) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*Response) ProtoMessage() {}
func (x *Response) ProtoReflect() protoreflect.Message {
mi := &file_github_com_containerd_ttrpc_request_proto_msgTypes[1]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use Response.ProtoReflect.Descriptor instead.
func (*Response) Descriptor() ([]byte, []int) {
return file_github_com_containerd_ttrpc_request_proto_rawDescGZIP(), []int{1}
}
func (x *Response) GetStatus() *status.Status {
if x != nil {
return x.Status
}
return nil
}
func (x *Response) GetPayload() []byte {
if x != nil {
return x.Payload
}
return nil
}
type StringList struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
List []string `protobuf:"bytes,1,rep,name=list,proto3" json:"list,omitempty"`
}
func (x *StringList) Reset() {
*x = StringList{}
if protoimpl.UnsafeEnabled {
mi := &file_github_com_containerd_ttrpc_request_proto_msgTypes[2]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *StringList) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*StringList) ProtoMessage() {}
func (x *StringList) ProtoReflect() protoreflect.Message {
mi := &file_github_com_containerd_ttrpc_request_proto_msgTypes[2]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use StringList.ProtoReflect.Descriptor instead.
func (*StringList) Descriptor() ([]byte, []int) {
return file_github_com_containerd_ttrpc_request_proto_rawDescGZIP(), []int{2}
}
func (x *StringList) GetList() []string {
if x != nil {
return x.List
}
return nil
}
type KeyValue struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"`
}
func (x *KeyValue) Reset() {
*x = KeyValue{}
if protoimpl.UnsafeEnabled {
mi := &file_github_com_containerd_ttrpc_request_proto_msgTypes[3]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *KeyValue) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*KeyValue) ProtoMessage() {}
func (x *KeyValue) ProtoReflect() protoreflect.Message {
mi := &file_github_com_containerd_ttrpc_request_proto_msgTypes[3]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use KeyValue.ProtoReflect.Descriptor instead.
func (*KeyValue) Descriptor() ([]byte, []int) {
return file_github_com_containerd_ttrpc_request_proto_rawDescGZIP(), []int{3}
}
func (x *KeyValue) GetKey() string {
if x != nil {
return x.Key
}
return ""
}
func (x *KeyValue) GetValue() string {
if x != nil {
return x.Value
}
return ""
}
var File_github_com_containerd_ttrpc_request_proto protoreflect.FileDescriptor
var file_github_com_containerd_ttrpc_request_proto_rawDesc = []byte{
0x0a, 0x29, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x6e,
0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x74, 0x74, 0x72, 0x70, 0x63, 0x2f, 0x72, 0x65,
0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x05, 0x74, 0x74, 0x72,
0x70, 0x63, 0x1a, 0x12, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73,
0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xa5, 0x01, 0x0a, 0x07, 0x52, 0x65, 0x71, 0x75, 0x65,
0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x18, 0x01, 0x20,
0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x16, 0x0a, 0x06,
0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6d, 0x65,
0x74, 0x68, 0x6f, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18,
0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x21,
0x0a, 0x0c, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x5f, 0x6e, 0x61, 0x6e, 0x6f, 0x18, 0x04,
0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x4e, 0x61, 0x6e,
0x6f, 0x12, 0x2b, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x05, 0x20,
0x03, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x74, 0x72, 0x70, 0x63, 0x2e, 0x4b, 0x65, 0x79, 0x56,
0x61, 0x6c, 0x75, 0x65, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x22, 0x45,
0x0a, 0x08, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x06, 0x73, 0x74,
0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x07, 0x2e, 0x53, 0x74, 0x61,
0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x70,
0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x70, 0x61,
0x79, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x20, 0x0a, 0x0a, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x4c,
0x69, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x73, 0x74, 0x18, 0x01, 0x20, 0x03, 0x28,
0x09, 0x52, 0x04, 0x6c, 0x69, 0x73, 0x74, 0x22, 0x32, 0x0a, 0x08, 0x4b, 0x65, 0x79, 0x56, 0x61,
0x6c, 0x75, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02,
0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x42, 0x1d, 0x5a, 0x1b, 0x67,
0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69,
0x6e, 0x65, 0x72, 0x64, 0x2f, 0x74, 0x74, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
0x6f, 0x33,
}
var (
file_github_com_containerd_ttrpc_request_proto_rawDescOnce sync.Once
file_github_com_containerd_ttrpc_request_proto_rawDescData = file_github_com_containerd_ttrpc_request_proto_rawDesc
)
func file_github_com_containerd_ttrpc_request_proto_rawDescGZIP() []byte {
file_github_com_containerd_ttrpc_request_proto_rawDescOnce.Do(func() {
file_github_com_containerd_ttrpc_request_proto_rawDescData = protoimpl.X.CompressGZIP(file_github_com_containerd_ttrpc_request_proto_rawDescData)
})
return file_github_com_containerd_ttrpc_request_proto_rawDescData
}
var file_github_com_containerd_ttrpc_request_proto_msgTypes = make([]protoimpl.MessageInfo, 4)
var file_github_com_containerd_ttrpc_request_proto_goTypes = []interface{}{
(*Request)(nil), // 0: ttrpc.Request
(*Response)(nil), // 1: ttrpc.Response
(*StringList)(nil), // 2: ttrpc.StringList
(*KeyValue)(nil), // 3: ttrpc.KeyValue
(*status.Status)(nil), // 4: Status
}
var file_github_com_containerd_ttrpc_request_proto_depIdxs = []int32{
3, // 0: ttrpc.Request.metadata:type_name -> ttrpc.KeyValue
4, // 1: ttrpc.Response.status:type_name -> Status
2, // [2:2] is the sub-list for method output_type
2, // [2:2] is the sub-list for method input_type
2, // [2:2] is the sub-list for extension type_name
2, // [2:2] is the sub-list for extension extendee
0, // [0:2] is the sub-list for field type_name
}
func init() { file_github_com_containerd_ttrpc_request_proto_init() }
func file_github_com_containerd_ttrpc_request_proto_init() {
if File_github_com_containerd_ttrpc_request_proto != nil {
return
}
if !protoimpl.UnsafeEnabled {
file_github_com_containerd_ttrpc_request_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*Request); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_github_com_containerd_ttrpc_request_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*Response); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_github_com_containerd_ttrpc_request_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*StringList); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_github_com_containerd_ttrpc_request_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*KeyValue); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_github_com_containerd_ttrpc_request_proto_rawDesc,
NumEnums: 0,
NumMessages: 4,
NumExtensions: 0,
NumServices: 0,
},
GoTypes: file_github_com_containerd_ttrpc_request_proto_goTypes,
DependencyIndexes: file_github_com_containerd_ttrpc_request_proto_depIdxs,
MessageInfos: file_github_com_containerd_ttrpc_request_proto_msgTypes,
}.Build()
File_github_com_containerd_ttrpc_request_proto = out.File
file_github_com_containerd_ttrpc_request_proto_rawDesc = nil
file_github_com_containerd_ttrpc_request_proto_goTypes = nil
file_github_com_containerd_ttrpc_request_proto_depIdxs = nil
}

29
vendor/github.com/containerd/ttrpc/request.proto generated vendored Normal file
View File

@ -0,0 +1,29 @@
syntax = "proto3";
package ttrpc;
import "proto/status.proto";
option go_package = "github.com/containerd/ttrpc";
message Request {
string service = 1;
string method = 2;
bytes payload = 3;
int64 timeout_nano = 4;
repeated KeyValue metadata = 5;
}
message Response {
Status status = 1;
bytes payload = 2;
}
message StringList {
repeated string list = 1;
}
message KeyValue {
string key = 1;
string value = 2;
}

View File

@ -18,7 +18,6 @@ package ttrpc
import (
"context"
"errors"
"io"
"math/rand"
"net"
@ -31,10 +30,6 @@ import (
"google.golang.org/grpc/status"
)
var (
ErrServerClosed = errors.New("ttrpc: server closed")
)
type Server struct {
config *serverConfig
services *serviceSet
@ -66,8 +61,14 @@ func NewServer(opts ...ServerOpt) (*Server, error) {
}, nil
}
// Register registers a map of methods to method handlers
// TODO: Remove in 2.0, does not support streams
func (s *Server) Register(name string, methods map[string]Method) {
s.services.register(name, methods)
s.services.register(name, &ServiceDesc{Methods: methods})
}
func (s *Server) RegisterService(name string, desc *ServiceDesc) {
s.services.register(name, desc)
}
func (s *Server) Serve(ctx context.Context, l net.Listener) error {
@ -301,27 +302,24 @@ func (c *serverConn) close() error {
func (c *serverConn) run(sctx context.Context) {
type (
request struct {
id uint32
req *Request
}
response struct {
id uint32
resp *Response
id uint32
status *status.Status
data []byte
closeStream bool
streaming bool
}
)
var (
ch = newChannel(c.conn)
ctx, cancel = context.WithCancel(sctx)
active int
state connState = connStateIdle
responses = make(chan response)
requests = make(chan request)
recvErr = make(chan error, 1)
shutdown = c.shutdown
done = make(chan struct{})
ch = newChannel(c.conn)
ctx, cancel = context.WithCancel(sctx)
state connState = connStateIdle
responses = make(chan response)
recvErr = make(chan error, 1)
done = make(chan struct{})
active int32
lastStreamID uint32
)
defer c.conn.Close()
@ -329,27 +327,27 @@ func (c *serverConn) run(sctx context.Context) {
defer close(done)
defer c.server.delConnection(c)
sendStatus := func(id uint32, st *status.Status) bool {
select {
case responses <- response{
// even though we've had an invalid stream id, we send it
// back on the same stream id so the client knows which
// stream id was bad.
id: id,
status: st,
closeStream: true,
}:
return true
case <-c.shutdown:
return false
case <-done:
return false
}
}
go func(recvErr chan error) {
defer close(recvErr)
sendImmediate := func(id uint32, st *status.Status) bool {
select {
case responses <- response{
// even though we've had an invalid stream id, we send it
// back on the same stream id so the client knows which
// stream id was bad.
id: id,
resp: &Response{
Status: st.Proto(),
},
}:
return true
case <-c.shutdown:
return false
case <-done:
return false
}
}
streams := map[uint32]*streamHandler{}
for {
select {
case <-c.shutdown:
@ -369,99 +367,159 @@ func (c *serverConn) run(sctx context.Context) {
// in this case, we send an error for that particular message
// when the status is defined.
if !sendImmediate(mh.StreamID, status) {
if !sendStatus(mh.StreamID, status) {
return
}
continue
}
if mh.Type != messageTypeRequest {
// we must ignore this for future compat.
continue
}
var req Request
if err := c.server.codec.Unmarshal(p, &req); err != nil {
ch.putmbuf(p)
if !sendImmediate(mh.StreamID, status.Newf(codes.InvalidArgument, "unmarshal request error: %v", err)) {
return
}
continue
}
ch.putmbuf(p)
if mh.StreamID%2 != 1 {
// enforce odd client initiated identifiers.
if !sendImmediate(mh.StreamID, status.Newf(codes.InvalidArgument, "StreamID must be odd for client initiated streams")) {
if !sendStatus(mh.StreamID, status.Newf(codes.InvalidArgument, "StreamID must be odd for client initiated streams")) {
return
}
continue
}
// Forward the request to the main loop. We don't wait on s.done
// because we have already accepted the client request.
select {
case requests <- request{
id: mh.StreamID,
req: &req,
}:
case <-done:
return
if mh.Type == messageTypeData {
sh, ok := streams[mh.StreamID]
if !ok {
if !sendStatus(mh.StreamID, status.Newf(codes.InvalidArgument, "StreamID is no longer active")) {
return
}
}
if mh.Flags&flagNoData != flagNoData {
unmarshal := func(obj interface{}) error {
err := protoUnmarshal(p, obj)
ch.putmbuf(p)
return err
}
if err := sh.data(unmarshal); err != nil {
if !sendStatus(mh.StreamID, status.Newf(codes.InvalidArgument, "data handling error: %v", err)) {
return
}
}
}
if mh.Flags&flagRemoteClosed == flagRemoteClosed {
sh.closeSend()
if len(p) > 0 {
if !sendStatus(mh.StreamID, status.Newf(codes.InvalidArgument, "data close message cannot include data")) {
return
}
}
}
} else if mh.Type == messageTypeRequest {
if mh.StreamID <= lastStreamID {
// enforce odd client initiated identifiers.
if !sendStatus(mh.StreamID, status.Newf(codes.InvalidArgument, "StreamID cannot be re-used and must increment")) {
return
}
continue
}
lastStreamID = mh.StreamID
// TODO: Make request type configurable
// Unmarshaller which takes in a byte array and returns an interface?
var req Request
if err := c.server.codec.Unmarshal(p, &req); err != nil {
ch.putmbuf(p)
if !sendStatus(mh.StreamID, status.Newf(codes.InvalidArgument, "unmarshal request error: %v", err)) {
return
}
continue
}
ch.putmbuf(p)
id := mh.StreamID
respond := func(status *status.Status, data []byte, streaming, closeStream bool) error {
select {
case responses <- response{
id: id,
status: status,
data: data,
closeStream: closeStream,
streaming: streaming,
}:
case <-done:
return ErrClosed
}
return nil
}
sh, err := c.server.services.handle(ctx, &req, respond)
if err != nil {
status, _ := status.FromError(err)
if !sendStatus(mh.StreamID, status) {
return
}
continue
}
streams[id] = sh
atomic.AddInt32(&active, 1)
}
// TODO: else we must ignore this for future compat. log this?
}
}(recvErr)
for {
newstate := state
switch {
case active > 0:
var (
newstate connState
shutdown chan struct{}
)
activeN := atomic.LoadInt32(&active)
if activeN > 0 {
newstate = connStateActive
shutdown = nil
case active == 0:
} else {
newstate = connStateIdle
shutdown = c.shutdown // only enable this branch in idle mode
}
if newstate != state {
c.setState(newstate)
state = newstate
}
select {
case request := <-requests:
active++
go func(id uint32) {
ctx, cancel := getRequestContext(ctx, request.req)
defer cancel()
p, status := c.server.services.call(ctx, request.req.Service, request.req.Method, request.req.Payload)
resp := &Response{
Status: status.Proto(),
Payload: p,
}
select {
case responses <- response{
id: id,
resp: resp,
}:
case <-done:
}
}(request.id)
case response := <-responses:
p, err := c.server.codec.Marshal(response.resp)
if err != nil {
logrus.WithError(err).Error("failed marshaling response")
return
if !response.streaming || response.status.Code() != codes.OK {
p, err := c.server.codec.Marshal(&Response{
Status: response.status.Proto(),
Payload: response.data,
})
if err != nil {
logrus.WithError(err).Error("failed marshaling response")
return
}
if err := ch.send(response.id, messageTypeResponse, 0, p); err != nil {
logrus.WithError(err).Error("failed sending message on channel")
return
}
} else {
var flags uint8
if response.closeStream {
flags = flagRemoteClosed
}
if response.data == nil {
flags = flags | flagNoData
}
if err := ch.send(response.id, messageTypeData, flags, response.data); err != nil {
logrus.WithError(err).Error("failed sending message on channel")
return
}
}
if err := ch.send(response.id, messageTypeResponse, p); err != nil {
logrus.WithError(err).Error("failed sending message on channel")
return
if response.closeStream {
// The ttrpc protocol currently does not support the case where
// the server is localClosed but not remoteClosed. Once the server
// is closing, the whole stream may be considered finished
atomic.AddInt32(&active, -1)
}
active--
case err := <-recvErr:
// TODO(stevvooe): Not wildly clear what we should do in this
// branch. Basically, it means that we are no longer receiving
@ -475,6 +533,7 @@ func (c *serverConn) run(sctx context.Context) {
if err != nil {
logrus.WithError(err).Error("error receiving message")
}
// else, initiate shutdown
case <-shutdown:
return
}

View File

@ -25,43 +25,62 @@ import (
"path"
"unsafe"
"github.com/gogo/protobuf/proto"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/proto"
)
type Method func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error)
type StreamHandler func(context.Context, StreamServer) (interface{}, error)
type Stream struct {
Handler StreamHandler
StreamingClient bool
StreamingServer bool
}
type ServiceDesc struct {
Methods map[string]Method
// TODO(stevvooe): Add stream support.
Streams map[string]Stream
}
type serviceSet struct {
services map[string]ServiceDesc
interceptor UnaryServerInterceptor
services map[string]*ServiceDesc
unaryInterceptor UnaryServerInterceptor
streamInterceptor StreamServerInterceptor
}
func newServiceSet(interceptor UnaryServerInterceptor) *serviceSet {
return &serviceSet{
services: make(map[string]ServiceDesc),
interceptor: interceptor,
services: make(map[string]*ServiceDesc),
unaryInterceptor: interceptor,
streamInterceptor: defaultStreamServerInterceptor,
}
}
func (s *serviceSet) register(name string, methods map[string]Method) {
func (s *serviceSet) register(name string, desc *ServiceDesc) {
if _, ok := s.services[name]; ok {
panic(fmt.Errorf("duplicate service %v registered", name))
}
s.services[name] = ServiceDesc{
Methods: methods,
}
s.services[name] = desc
}
func (s *serviceSet) call(ctx context.Context, serviceName, methodName string, p []byte) ([]byte, *status.Status) {
p, err := s.dispatch(ctx, serviceName, methodName, p)
func (s *serviceSet) unaryCall(ctx context.Context, method Method, info *UnaryServerInfo, data []byte) (p []byte, st *status.Status) {
unmarshal := func(obj interface{}) error {
return protoUnmarshal(data, obj)
}
resp, err := s.unaryInterceptor(ctx, unmarshal, info, method)
if err == nil {
if isNil(resp) {
err = errors.New("ttrpc: marshal called with nil")
} else {
p, err = protoMarshal(resp)
}
}
st, ok := status.FromError(err)
if !ok {
st = status.New(convertCode(err), err.Error())
@ -70,38 +89,142 @@ func (s *serviceSet) call(ctx context.Context, serviceName, methodName string, p
return p, st
}
func (s *serviceSet) dispatch(ctx context.Context, serviceName, methodName string, p []byte) ([]byte, error) {
method, err := s.resolve(serviceName, methodName)
if err != nil {
return nil, err
func (s *serviceSet) streamCall(ctx context.Context, stream StreamHandler, info *StreamServerInfo, ss StreamServer) (p []byte, st *status.Status) {
resp, err := s.streamInterceptor(ctx, ss, info, stream)
if err == nil {
p, err = protoMarshal(resp)
}
st, ok := status.FromError(err)
if !ok {
st = status.New(convertCode(err), err.Error())
}
return
}
func (s *serviceSet) handle(ctx context.Context, req *Request, respond func(*status.Status, []byte, bool, bool) error) (*streamHandler, error) {
srv, ok := s.services[req.Service]
if !ok {
return nil, status.Errorf(codes.Unimplemented, "service %v", req.Service)
}
unmarshal := func(obj interface{}) error {
switch v := obj.(type) {
case proto.Message:
if err := proto.Unmarshal(p, v); err != nil {
return status.Errorf(codes.Internal, "ttrpc: error unmarshalling payload: %v", err.Error())
if method, ok := srv.Methods[req.Method]; ok {
go func() {
ctx, cancel := getRequestContext(ctx, req)
defer cancel()
info := &UnaryServerInfo{
FullMethod: fullPath(req.Service, req.Method),
}
default:
return status.Errorf(codes.Internal, "ttrpc: error unsupported request type: %T", v)
p, st := s.unaryCall(ctx, method, info, req.Payload)
respond(st, p, false, true)
}()
return nil, nil
}
if stream, ok := srv.Streams[req.Method]; ok {
ctx, cancel := getRequestContext(ctx, req)
info := &StreamServerInfo{
FullMethod: fullPath(req.Service, req.Method),
StreamingClient: stream.StreamingClient,
StreamingServer: stream.StreamingServer,
}
sh := &streamHandler{
ctx: ctx,
respond: respond,
recv: make(chan Unmarshaler, 5),
info: info,
}
go func() {
defer cancel()
p, st := s.streamCall(ctx, stream.Handler, info, sh)
respond(st, p, stream.StreamingServer, true)
}()
if req.Payload != nil {
unmarshal := func(obj interface{}) error {
return protoUnmarshal(req.Payload, obj)
}
if err := sh.data(unmarshal); err != nil {
return nil, err
}
}
return sh, nil
}
return nil, status.Errorf(codes.Unimplemented, "method %v", req.Method)
}
type streamHandler struct {
ctx context.Context
respond func(*status.Status, []byte, bool, bool) error
recv chan Unmarshaler
info *StreamServerInfo
remoteClosed bool
localClosed bool
}
func (s *streamHandler) closeSend() {
if !s.remoteClosed {
s.remoteClosed = true
close(s.recv)
}
}
func (s *streamHandler) data(unmarshal Unmarshaler) error {
if s.remoteClosed {
return ErrStreamClosed
}
select {
case s.recv <- unmarshal:
return nil
case <-s.ctx.Done():
return s.ctx.Err()
}
}
info := &UnaryServerInfo{
FullMethod: fullPath(serviceName, methodName),
func (s *streamHandler) SendMsg(m interface{}) error {
if s.localClosed {
return ErrStreamClosed
}
resp, err := s.interceptor(ctx, unmarshal, info, method)
p, err := protoMarshal(m)
if err != nil {
return nil, err
return err
}
return s.respond(nil, p, true, false)
}
func (s *streamHandler) RecvMsg(m interface{}) error {
select {
case unmarshal, ok := <-s.recv:
if !ok {
return io.EOF
}
return unmarshal(m)
case <-s.ctx.Done():
return s.ctx.Err()
}
}
func protoUnmarshal(p []byte, obj interface{}) error {
switch v := obj.(type) {
case proto.Message:
if err := proto.Unmarshal(p, v); err != nil {
return status.Errorf(codes.Internal, "ttrpc: error unmarshalling payload: %v", err.Error())
}
default:
return status.Errorf(codes.Internal, "ttrpc: error unsupported request type: %T", v)
}
return nil
}
func protoMarshal(obj interface{}) ([]byte, error) {
if obj == nil {
return nil, nil
}
if isNil(resp) {
return nil, errors.New("ttrpc: marshal called with nil")
}
switch v := resp.(type) {
switch v := obj.(type) {
case proto.Message:
r, err := proto.Marshal(v)
if err != nil {
@ -114,20 +237,6 @@ func (s *serviceSet) dispatch(ctx context.Context, serviceName, methodName strin
}
}
func (s *serviceSet) resolve(service, method string) (Method, error) {
srv, ok := s.services[service]
if !ok {
return nil, status.Errorf(codes.Unimplemented, "service %v", service)
}
mthd, ok := srv.Methods[method]
if !ok {
return nil, status.Errorf(codes.Unimplemented, "method %v", method)
}
return mthd, nil
}
// convertCode maps stdlib go errors into grpc space.
//
// This is ripped from the grpc-go code base.

81
vendor/github.com/containerd/ttrpc/stream.go generated vendored Normal file
View File

@ -0,0 +1,81 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package ttrpc
import (
"context"
"sync"
)
type streamID uint32
type streamMessage struct {
header messageHeader
payload []byte
}
type stream struct {
id streamID
sender sender
recv chan *streamMessage
closeOnce sync.Once
recvErr error
}
func newStream(id streamID, send sender) *stream {
return &stream{
id: id,
sender: send,
recv: make(chan *streamMessage, 1),
}
}
func (s *stream) closeWithError(err error) error {
s.closeOnce.Do(func() {
if s.recv != nil {
close(s.recv)
if err != nil {
s.recvErr = err
} else {
s.recvErr = ErrClosed
}
}
})
return nil
}
func (s *stream) send(mt messageType, flags uint8, b []byte) error {
return s.sender.send(uint32(s.id), mt, flags, b)
}
func (s *stream) receive(ctx context.Context, msg *streamMessage) error {
if s.recvErr != nil {
return s.recvErr
}
select {
case s.recv <- msg:
return nil
case <-ctx.Done():
return ctx.Err()
}
}
type sender interface {
send(uint32, messageType, uint8, []byte) error
}

22
vendor/github.com/containerd/ttrpc/stream_server.go generated vendored Normal file
View File

@ -0,0 +1,22 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package ttrpc
type StreamServer interface {
SendMsg(m interface{}) error
RecvMsg(m interface{}) error
}

16
vendor/github.com/containerd/ttrpc/test.proto generated vendored Normal file
View File

@ -0,0 +1,16 @@
syntax = "proto3";
package ttrpc;
option go_package = "github.com/containerd/ttrpc/internal";
message TestPayload {
string foo = 1;
int64 deadline = 2;
string metadata = 3;
}
message EchoPayload {
int64 seq = 1;
string msg = 2;
}

View File

@ -1,63 +0,0 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package ttrpc
import (
"fmt"
spb "google.golang.org/genproto/googleapis/rpc/status"
)
type Request struct {
Service string `protobuf:"bytes,1,opt,name=service,proto3"`
Method string `protobuf:"bytes,2,opt,name=method,proto3"`
Payload []byte `protobuf:"bytes,3,opt,name=payload,proto3"`
TimeoutNano int64 `protobuf:"varint,4,opt,name=timeout_nano,proto3"`
Metadata []*KeyValue `protobuf:"bytes,5,rep,name=metadata,proto3"`
}
func (r *Request) Reset() { *r = Request{} }
func (r *Request) String() string { return fmt.Sprintf("%+#v", r) }
func (r *Request) ProtoMessage() {}
type Response struct {
Status *spb.Status `protobuf:"bytes,1,opt,name=status,proto3"`
Payload []byte `protobuf:"bytes,2,opt,name=payload,proto3"`
}
func (r *Response) Reset() { *r = Response{} }
func (r *Response) String() string { return fmt.Sprintf("%+#v", r) }
func (r *Response) ProtoMessage() {}
type StringList struct {
List []string `protobuf:"bytes,1,rep,name=list,proto3"`
}
func (r *StringList) Reset() { *r = StringList{} }
func (r *StringList) String() string { return fmt.Sprintf("%+#v", r) }
func (r *StringList) ProtoMessage() {}
func makeStringList(item ...string) StringList { return StringList{List: item} }
type KeyValue struct {
Key string `protobuf:"bytes,1,opt,name=key,proto3"`
Value string `protobuf:"bytes,2,opt,name=value,proto3"`
}
func (m *KeyValue) Reset() { *m = KeyValue{} }
func (*KeyValue) ProtoMessage() {}
func (m *KeyValue) String() string { return fmt.Sprintf("%+#v", m) }

View File

@ -29,7 +29,7 @@ import (
type UnixCredentialsFunc func(*unix.Ucred) error
func (fn UnixCredentialsFunc) Handshake(ctx context.Context, conn net.Conn) (net.Conn, interface{}, error) {
func (fn UnixCredentialsFunc) Handshake(_ context.Context, conn net.Conn) (net.Conn, interface{}, error) {
uc, err := requireUnixSocket(conn)
if err != nil {
return nil, nil, fmt.Errorf("ttrpc.UnixCredentialsFunc: require unix socket: %w", err)
@ -50,7 +50,7 @@ func (fn UnixCredentialsFunc) Handshake(ctx context.Context, conn net.Conn) (net
}
if ucredErr != nil {
return nil, nil, fmt.Errorf("ttrpc.UnixCredentialsFunc: failed to retrieve socket peer credentials: %w", err)
return nil, nil, fmt.Errorf("ttrpc.UnixCredentialsFunc: failed to retrieve socket peer credentials: %w", ucredErr)
}
if err := fn(ucred); err != nil {
@ -88,10 +88,6 @@ func UnixSocketRequireSameUser() UnixCredentialsFunc {
return UnixSocketRequireUidGid(euid, egid)
}
func requireRoot(ucred *unix.Ucred) error {
return requireUidGid(ucred, 0, 0)
}
func requireUidGid(ucred *unix.Ucred, uid, gid int) error {
if (uid != -1 && uint32(uid) != ucred.Uid) || (gid != -1 && uint32(gid) != ucred.Gid) {
return fmt.Errorf("ttrpc: invalid credentials: %v", syscall.EPERM)

View File

@ -7,7 +7,7 @@
A Go package for managing the registration, marshaling, and unmarshaling of encoded types.
This package helps when types are sent over a GRPC API and marshaled as a [protobuf.Any](https://github.com/gogo/protobuf/blob/master/protobuf/google/protobuf/any.proto).
This package helps when types are sent over a ttrpc/GRPC API and marshaled as a protobuf [Any](https://pkg.go.dev/google.golang.org/protobuf@v1.27.1/types/known/anypb#Any)
## Project details

View File

@ -18,13 +18,15 @@ package typeurl
import (
"encoding/json"
"errors"
"fmt"
"path"
"reflect"
"sync"
"github.com/gogo/protobuf/proto"
"github.com/gogo/protobuf/types"
"github.com/pkg/errors"
gogoproto "github.com/gogo/protobuf/proto"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/reflect/protoregistry"
)
var (
@ -39,10 +41,35 @@ var (
//
// To detect an error class, use errors.Is() functions to tell whether an
// error is of this type.
var (
ErrNotFound = errors.New("not found")
)
type Any interface {
GetTypeUrl() string
GetValue() []byte
}
type any struct {
typeURL string
value []byte
}
func (a *any) GetTypeUrl() string {
if a == nil {
return ""
}
return a.typeURL
}
func (a *any) GetValue() []byte {
if a == nil {
return nil
}
return a.value
}
// Register a type with a base URL for JSON marshaling. When the MarshalAny and
// UnmarshalAny functions are called they will treat the Any type value as JSON.
// To use protocol buffers for handling the Any value the proto.Register
@ -56,7 +83,7 @@ func Register(v interface{}, args ...string) {
defer mu.Unlock()
if et, ok := registry[t]; ok {
if et != p {
panic(errors.Errorf("type registered with alternate path %q != %q", et, p))
panic(fmt.Errorf("type registered with alternate path %q != %q", et, p))
}
return
}
@ -69,41 +96,47 @@ func TypeURL(v interface{}) (string, error) {
u, ok := registry[tryDereference(v)]
mu.RUnlock()
if !ok {
// fallback to the proto registry if it is a proto message
pb, ok := v.(proto.Message)
if !ok {
return "", errors.Wrapf(ErrNotFound, "type %s", reflect.TypeOf(v))
switch t := v.(type) {
case proto.Message:
return string(t.ProtoReflect().Descriptor().FullName()), nil
case gogoproto.Message:
return gogoproto.MessageName(t), nil
default:
return "", fmt.Errorf("type %s: %w", reflect.TypeOf(v), ErrNotFound)
}
return proto.MessageName(pb), nil
}
return u, nil
}
// Is returns true if the type of the Any is the same as v.
func Is(any *types.Any, v interface{}) bool {
func Is(any Any, v interface{}) bool {
// call to check that v is a pointer
tryDereference(v)
url, err := TypeURL(v)
if err != nil {
return false
}
return any.TypeUrl == url
return any.GetTypeUrl() == url
}
// MarshalAny marshals the value v into an any with the correct TypeUrl.
// If the provided object is already a proto.Any message, then it will be
// returned verbatim. If it is of type proto.Message, it will be marshaled as a
// protocol buffer. Otherwise, the object will be marshaled to json.
func MarshalAny(v interface{}) (*types.Any, error) {
func MarshalAny(v interface{}) (Any, error) {
var marshal func(v interface{}) ([]byte, error)
switch t := v.(type) {
case *types.Any:
case Any:
// avoid reserializing the type if we have an any.
return t, nil
case proto.Message:
marshal = func(v interface{}) ([]byte, error) {
return proto.Marshal(t)
}
case gogoproto.Message:
marshal = func(v interface{}) ([]byte, error) {
return gogoproto.Marshal(t)
}
default:
marshal = json.Marshal
}
@ -117,15 +150,15 @@ func MarshalAny(v interface{}) (*types.Any, error) {
if err != nil {
return nil, err
}
return &types.Any{
TypeUrl: url,
Value: data,
return &any{
typeURL: url,
value: data,
}, nil
}
// UnmarshalAny unmarshals the any type into a concrete type.
func UnmarshalAny(any *types.Any) (interface{}, error) {
return UnmarshalByTypeURL(any.TypeUrl, any.Value)
func UnmarshalAny(any Any) (interface{}, error) {
return UnmarshalByTypeURL(any.GetTypeUrl(), any.GetValue())
}
// UnmarshalByTypeURL unmarshals the given type and value to into a concrete type.
@ -136,11 +169,11 @@ func UnmarshalByTypeURL(typeURL string, value []byte) (interface{}, error) {
// UnmarshalTo unmarshals the any type into a concrete type passed in the out
// argument. It is identical to UnmarshalAny, but lets clients provide a
// destination type through the out argument.
func UnmarshalTo(any *types.Any, out interface{}) error {
return UnmarshalToByTypeURL(any.TypeUrl, any.Value, out)
func UnmarshalTo(any Any, out interface{}) error {
return UnmarshalToByTypeURL(any.GetTypeUrl(), any.GetValue(), out)
}
// UnmarshalTo unmarshals the given type and value into a concrete type passed
// UnmarshalToByTypeURL unmarshals the given type and value into a concrete type passed
// in the out argument. It is identical to UnmarshalByTypeURL, but lets clients
// provide a destination type through the out argument.
func UnmarshalToByTypeURL(typeURL string, value []byte, out interface{}) error {
@ -149,6 +182,10 @@ func UnmarshalToByTypeURL(typeURL string, value []byte, out interface{}) error {
}
func unmarshal(typeURL string, value []byte, v interface{}) (interface{}, error) {
if value == nil {
return nil, nil
}
t, err := getTypeByUrl(typeURL)
if err != nil {
return nil, err
@ -163,12 +200,17 @@ func unmarshal(typeURL string, value []byte, v interface{}) (interface{}, error)
return nil, err
}
if typeURL != vURL {
return nil, errors.Errorf("can't unmarshal type %q to output %q", typeURL, vURL)
return nil, fmt.Errorf("can't unmarshal type %q to output %q", typeURL, vURL)
}
}
if t.isProto {
err = proto.Unmarshal(value, v.(proto.Message))
switch t := v.(type) {
case proto.Message:
err = proto.Unmarshal(value, t)
case gogoproto.Message:
err = gogoproto.Unmarshal(value, t)
}
} else {
err = json.Unmarshal(value, v)
}
@ -193,7 +235,7 @@ func getTypeByUrl(url string) (urlType, error) {
}
mu.RUnlock()
// fallback to proto registry
t := proto.MessageType(url)
t := gogoproto.MessageType(url)
if t != nil {
return urlType{
// get the underlying Elem because proto returns a pointer to the type
@ -201,7 +243,12 @@ func getTypeByUrl(url string) (urlType, error) {
isProto: true,
}, nil
}
return urlType{}, errors.Wrapf(ErrNotFound, "type with url %s", url)
mt, err := protoregistry.GlobalTypes.FindMessageByURL(url)
if err != nil {
return urlType{}, fmt.Errorf("type with url %s: %w", url, ErrNotFound)
}
empty := mt.New().Interface()
return urlType{t: reflect.TypeOf(empty).Elem(), isProto: true}, nil
}
func tryDereference(v interface{}) reflect.Type {