From 025646d75847ab9d7f5a33630ecadeec1108a8d1 Mon Sep 17 00:00:00 2001 From: Evgenii Baidakov Date: Wed, 26 Mar 2025 12:28:49 +0400 Subject: [PATCH 1/5] *: Migrate to aws-sdk-v2 Closes #1028. Signed-off-by: Evgenii Baidakov --- CHANGELOG.md | 1 + api/auth/center.go | 76 +++++++++++++++++++++++++----- api/auth/center_test.go | 28 +++++++---- api/auth/signer/v4/chunk_signer.go | 13 ++--- api/auth/signer/v4/stream.go | 17 ++----- go.mod | 5 +- go.sum | 16 +++---- 7 files changed, 101 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 48bd3795..c4ff6535 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ This document outlines major changes between releases. ### Added ### Changed +- AWS SDK migrated to V2 (#1028) ### Fixed diff --git a/api/auth/center.go b/api/auth/center.go index f496b814..7c591944 100644 --- a/api/auth/center.go +++ b/api/auth/center.go @@ -9,12 +9,13 @@ import ( "io" "mime/multipart" "net/http" + "net/url" "regexp" "strings" "time" - "github.com/aws/aws-sdk-go/aws/credentials" - v4amz "github.com/aws/aws-sdk-go/aws/signer/v4" + v4amz "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + "github.com/aws/aws-sdk-go-v2/credentials" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" v4 "github.com/nspcc-dev/neofs-s3-gw/api/auth/signer/v4" "github.com/nspcc-dev/neofs-s3-gw/api/cache" @@ -61,6 +62,7 @@ type ( Date string IsPresigned bool Expiration time.Duration + PayloadHash string } ) @@ -83,6 +85,10 @@ const ( ContentEncodingChunked = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" timeFormatISO8601 = "20060102T150405Z" + + // emptyStringSHA256 is a SHA256 of an empty string. + emptyStringSHA256 = `e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855` + UnsignedPayload = "UNSIGNED-PAYLOAD" ) // ErrNoAuthorizationHeader is returned for unauthenticated requests. @@ -108,7 +114,7 @@ func New(neoFS tokens.NeoFS, key *keys.PrivateKey, prefixes []string, config *ca } } -func (c *center) parseAuthHeader(header string) (*authHeader, error) { +func (c *center) parseAuthHeader(header, amzContentSha256Header string) (*authHeader, error) { submatches := c.reg.GetSubmatches(header) if len(submatches) != authHeaderPartsNum { return nil, s3errors.GetAPIError(s3errors.ErrCredMalformed) @@ -128,6 +134,7 @@ func (c *center) parseAuthHeader(header string) (*authHeader, error) { SignatureV4: submatches["v4_signature"], SignedFields: signedFields, Date: submatches["date"], + PayloadHash: amzContentSha256Header, }, nil } @@ -161,6 +168,11 @@ func (c *center) Authenticate(r *http.Request) (*Box, error) { SignedFields: queryValues[AmzSignedHeaders], Date: creds[1], IsPresigned: true, + PayloadHash: r.Header.Get(AmzContentSha256), + } + + if authHdr.PayloadHash == "" { + authHdr.PayloadHash = UnsignedPayload } authHdr.Expiration, err = time.ParseDuration(queryValues.Get(AmzExpires) + "s") if err != nil { @@ -180,7 +192,7 @@ func (c *center) Authenticate(r *http.Request) (*Box, error) { } return nil, ErrNoAuthorizationHeader } - authHdr, err = c.parseAuthHeader(authHeaderField[0]) + authHdr, err = c.parseAuthHeader(authHeaderField[0], r.Header.Get(AmzContentSha256)) if err != nil { return nil, err } @@ -220,8 +232,13 @@ func (c *center) Authenticate(r *http.Request) (*Box, error) { return nil, fmt.Errorf("DecodeString: %w", err) } - awsCreds := credentials.NewStaticCredentials(authHdr.AccessKeyID, box.Gate.AccessKey, "") - streamSigner := v4.NewChunkSigner(authHdr.Region, authHdr.Service, sig, signatureDateTime, awsCreds) + appCreds := credentials.NewStaticCredentialsProvider(authHdr.AccessKeyID, box.Gate.AccessKey, "") + value, err := appCreds.Retrieve(r.Context()) + if err != nil { + return nil, fmt.Errorf("get credentials: %w", err) + } + + streamSigner := v4.NewChunkSigner(authHdr.Region, authHdr.Service, sig, signatureDateTime, value) r.Body = v4.NewChunkedReader(r.Body, streamSigner) } @@ -314,25 +331,58 @@ func cloneRequest(r *http.Request, authHeader *authHeader) *http.Request { } func (c *center) checkSign(authHeader *authHeader, box *accessbox.Box, request *http.Request, signatureDateTime time.Time) error { - awsCreds := credentials.NewStaticCredentials(authHeader.AccessKeyID, box.Gate.AccessKey, "") - signer := v4amz.NewSigner(awsCreds) - signer.DisableURIPathEscaping = true + credProvider := credentials.NewStaticCredentialsProvider(authHeader.AccessKeyID, box.Gate.AccessKey, "") + awsCreds, err := credProvider.Retrieve(request.Context()) + if err != nil { + return fmt.Errorf("get credentials: %w", err) + } + + signer := v4amz.NewSigner(func(signer *v4amz.SignerOptions) { + signer.DisableURIPathEscaping = true + }) + + if authHeader.PayloadHash == "" { + authHeader.PayloadHash = emptyStringSHA256 + } + + var hasContentLength bool + for _, h := range authHeader.SignedFields { + if strings.ToLower(h) == "content-length" { + hasContentLength = true + break + } + } + + // Final content length is unknown, request.ContentLength == -1. + if !hasContentLength { + request.ContentLength = 0 + } var signature string if authHeader.IsPresigned { - now := time.Now() + var ( + now = time.Now() + signedURI string + ) if signatureDateTime.Add(authHeader.Expiration).Before(now) { return s3errors.GetAPIError(s3errors.ErrExpiredPresignRequest) } if now.Before(signatureDateTime) { return s3errors.GetAPIError(s3errors.ErrBadRequest) } - if _, err := signer.Presign(request, nil, authHeader.Service, authHeader.Region, authHeader.Expiration, signatureDateTime); err != nil { + + signedURI, _, err = signer.PresignHTTP(request.Context(), awsCreds, request, authHeader.PayloadHash, authHeader.Service, authHeader.Region, signatureDateTime) + if err != nil { return fmt.Errorf("failed to pre-sign temporary HTTP request: %w", err) } - signature = request.URL.Query().Get(AmzSignature) + + u, err := url.ParseRequestURI(signedURI) + if err != nil { + return fmt.Errorf("parse signed uri: %w", err) + } + signature = u.Query().Get(AmzSignature) } else { - if _, err := signer.Sign(request, nil, authHeader.Service, authHeader.Region, signatureDateTime); err != nil { + if err = signer.SignHTTP(request.Context(), awsCreds, request, authHeader.PayloadHash, authHeader.Service, authHeader.Region, signatureDateTime); err != nil { return fmt.Errorf("failed to sign temporary HTTP request: %w", err) } signature = c.reg.GetSubmatches(request.Header.Get(AuthorizationHdr))["v4_signature"] diff --git a/api/auth/center_test.go b/api/auth/center_test.go index 073c6df7..7640c9f4 100644 --- a/api/auth/center_test.go +++ b/api/auth/center_test.go @@ -2,6 +2,8 @@ package auth import ( "bytes" + "context" + "crypto/sha256" "encoding/hex" "fmt" "io" @@ -12,8 +14,8 @@ import ( "testing" "time" - "github.com/aws/aws-sdk-go/aws/credentials" - v4aws "github.com/aws/aws-sdk-go/aws/signer/v4" + v4amz "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + "github.com/aws/aws-sdk-go-v2/credentials" v4 "github.com/nspcc-dev/neofs-s3-gw/api/auth/signer/v4" "github.com/nspcc-dev/neofs-s3-gw/api/s3errors" "github.com/stretchr/testify/require" @@ -54,7 +56,7 @@ func TestAuthHeaderParse(t *testing.T) { expected: nil, }, } { - authHeader, err := center.parseAuthHeader(tc.header) + authHeader, err := center.parseAuthHeader(tc.header, "") require.Equal(t, tc.err, err, tc.header) require.Equal(t, tc.expected, authHeader, tc.header) } @@ -118,7 +120,9 @@ func TestAwsEncodedChunkReader(t *testing.T) { } chunkOneBody := append([]byte("10000;chunk-signature=ad80c730a21e5b8d04586a2213dd63b9a0e99e0e2307b0ade35a65485a288648\n"), chunkOnePayload...) - awsCreds := credentials.NewStaticCredentials("AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "") + appCreds := credentials.NewStaticCredentialsProvider("AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "") + awsCreds, err := appCreds.Retrieve(context.Background()) + require.NoError(t, err) ts, err := time.Parse(timeFormatISO8601, "20130524T000000Z") require.NoError(t, err) @@ -354,6 +358,7 @@ func TestAwsEncodedWithRequest(t *testing.T) { t.Skipf("Only for manual launch") ts := time.Now() + ctx := context.Background() host := "http://localhost:19080" bucketName := "heh1701422026" @@ -376,15 +381,22 @@ func TestAwsEncodedWithRequest(t *testing.T) { req.Header.Set("content-encoding", "aws-chunked") req.Header.Set("x-amz-decoded-content-length", strconv.Itoa(totalPayloadLength)) - awsCreds := credentials.NewStaticCredentials( + appCreds := credentials.NewStaticCredentialsProvider( "6cpBf2jzHdD2MJHsjwLuVYYDAPJcfsJ5oufJWnHhrSBQ0FPjWXxmLmvKDAyhr1SEwnfKLJq3twKzuWG7f24qfyWcD", // access_key_id "79488f248493cb5175ea079a12a3e08015021d9c710a064017e1da6a2b0ae111", // secret_access_key "") - signer := v4aws.NewSigner(awsCreds) + awsCreds, err := appCreds.Retrieve(ctx) + require.NoError(t, err) + + signer := v4amz.NewSigner(func(signer *v4amz.SignerOptions) { + signer.DisableURIPathEscaping = true + }) + + h := sha256.New() + h.Write(payload) - signer.DisableURIPathEscaping = true - _, err = signer.Sign(req, nil, "s3", "us-east-1", ts) + err = signer.SignHTTP(ctx, awsCreds, req, hex.EncodeToString(h.Sum(nil)), "s3", "us-east-1", ts) require.NoError(t, err) reg := NewRegexpMatcher(authorizationFieldRegexp) diff --git a/api/auth/signer/v4/chunk_signer.go b/api/auth/signer/v4/chunk_signer.go index 99834794..b6a585c7 100644 --- a/api/auth/signer/v4/chunk_signer.go +++ b/api/auth/signer/v4/chunk_signer.go @@ -6,7 +6,7 @@ import ( "strings" "time" - "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go-v2/aws" ) const ( @@ -19,14 +19,14 @@ type ChunkSigner struct { region string service string - credentials credentialValueProvider + credentials aws.Credentials prevSig []byte seedDate time.Time } // NewChunkSigner creates a SigV4 signer used to sign Event Stream encoded messages. -func NewChunkSigner(region, service string, seedSignature []byte, seedDate time.Time, credentials *credentials.Credentials) *ChunkSigner { +func NewChunkSigner(region, service string, seedSignature []byte, seedDate time.Time, credentials aws.Credentials) *ChunkSigner { return &ChunkSigner{ region: region, service: service, @@ -47,12 +47,7 @@ func (s *ChunkSigner) GetSignatureByHash(payloadHash hash.Hash) ([]byte, error) } func (s *ChunkSigner) getSignature(payloadHash []byte) ([]byte, error) { - credValue, err := s.credentials.Get() - if err != nil { - return nil, err - } - - sigKey := deriveSigningKey(s.region, s.service, credValue.SecretAccessKey, s.seedDate) + sigKey := deriveSigningKey(s.region, s.service, s.credentials.SecretAccessKey, s.seedDate) keyPath := buildSigningScope(s.region, s.service, s.seedDate) diff --git a/api/auth/signer/v4/stream.go b/api/auth/signer/v4/stream.go index eb579a8a..4469cc25 100644 --- a/api/auth/signer/v4/stream.go +++ b/api/auth/signer/v4/stream.go @@ -5,25 +5,21 @@ import ( "strings" "time" - "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go-v2/aws" ) -type credentialValueProvider interface { - Get() (credentials.Value, error) -} - // StreamSigner implements signing of event stream encoded payloads. type StreamSigner struct { region string service string - credentials credentialValueProvider + credentials aws.Credentials prevSig []byte } // NewStreamSigner creates a SigV4 signer used to sign Event Stream encoded messages. -func NewStreamSigner(region, service string, seedSignature []byte, credentials *credentials.Credentials) *StreamSigner { +func NewStreamSigner(region, service string, seedSignature []byte, credentials aws.Credentials) *StreamSigner { return &StreamSigner{ region: region, service: service, @@ -34,12 +30,7 @@ func NewStreamSigner(region, service string, seedSignature []byte, credentials * // GetSignature takes an event stream encoded headers and payload and returns a signature. func (s *StreamSigner) GetSignature(headers, payload []byte, date time.Time) ([]byte, error) { - credValue, err := s.credentials.Get() - if err != nil { - return nil, err - } - - sigKey := deriveSigningKey(s.region, s.service, credValue.SecretAccessKey, date) + sigKey := deriveSigningKey(s.region, s.service, s.credentials.SecretAccessKey, date) keyPath := buildSigningScope(s.region, s.service, date) diff --git a/go.mod b/go.mod index 8cf18c28..2dbe3d9b 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,8 @@ module github.com/nspcc-dev/neofs-s3-gw go 1.23 require ( - github.com/aws/aws-sdk-go v1.55.6 + github.com/aws/aws-sdk-go-v2 v1.36.3 + github.com/aws/aws-sdk-go-v2/credentials v1.17.64 github.com/bluele/gcache v0.0.2 github.com/cenkalti/backoff/v5 v5.0.2 github.com/google/uuid v1.6.0 @@ -28,6 +29,7 @@ require ( require ( github.com/antlr4-go/antlr/v4 v4.13.1 // indirect + github.com/aws/smithy-go v1.22.2 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect github.com/golang/snappy v0.0.1 // indirect github.com/gorilla/websocket v1.5.3 // indirect @@ -58,7 +60,6 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect diff --git a/go.sum b/go.sum index af0a5c46..61a4fda0 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,12 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= -github.com/aws/aws-sdk-go v1.55.6 h1:cSg4pvZ3m8dgYcgqB97MrcdjUmZ1BeMYKUxMMB89IPk= -github.com/aws/aws-sdk-go v1.55.6/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= +github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM= +github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= +github.com/aws/aws-sdk-go-v2/credentials v1.17.64 h1:NH4RAQJEXBDQDUudTqMNHdyyEVa5CvMn0tQicqv48jo= +github.com/aws/aws-sdk-go-v2/credentials v1.17.64/go.mod h1:tUoJfj79lzEcalHDbyNkpnZZTRg/2ayYOK/iYnRfPbo= +github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ= +github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bits-and-blooms/bitset v1.14.2 h1:YXVoyPndbdvcEVcseEovVfp0qjJp7S+i5+xgp/Nfbdc= @@ -34,7 +38,6 @@ github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GK github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -93,10 +96,6 @@ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T github.com/holiman/uint256 v1.3.1 h1:JfTzmih28bittyHM8z360dCjIA9dbPIBlcTI6lmctQs= github.com/holiman/uint256 v1.3.1/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -172,7 +171,6 @@ github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9F github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= @@ -209,7 +207,6 @@ github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= @@ -313,7 +310,6 @@ gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= From 492df913d3cef6dcb443967f182d28785335ed14 Mon Sep 17 00:00:00 2001 From: Evgenii Baidakov Date: Wed, 2 Apr 2025 11:53:06 +0400 Subject: [PATCH 2/5] *: Remove MD5 header check on delete multiple objects The documentation says the header is required, but even new aws cli (aws-cli/2.25.3) don't send this header any more. Moreover, we even don't check it. Signed-off-by: Evgenii Baidakov --- api/handler/delete.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/api/handler/delete.go b/api/handler/delete.go index a2e93254..fbbe95cc 100644 --- a/api/handler/delete.go +++ b/api/handler/delete.go @@ -159,13 +159,6 @@ func isErrObjectLocked(err error) bool { func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) { reqInfo := api.GetReqInfo(r.Context()) - // Content-Md5 is required and should be set - // http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html - if _, ok := r.Header[api.ContentMD5]; !ok { - h.logAndSendError(w, "missing Content-MD5", reqInfo, s3errors.GetAPIError(s3errors.ErrMissingContentMD5)) - return - } - // Content-Length is required and should be non-zero // http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html if r.ContentLength <= 0 { From ae34f70604eb2be4e97a8c5e0523b43541d9d892 Mon Sep 17 00:00:00 2001 From: Evgenii Baidakov Date: Wed, 2 Apr 2025 11:54:45 +0400 Subject: [PATCH 3/5] layer: Check error properly Signed-off-by: Evgenii Baidakov --- api/layer/detector.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/layer/detector.go b/api/layer/detector.go index 81ec75b7..0744371b 100644 --- a/api/layer/detector.go +++ b/api/layer/detector.go @@ -1,6 +1,7 @@ package layer import ( + "errors" "io" "net/http" ) @@ -45,7 +46,7 @@ func newDetector(reader io.Reader) *detector { func (d *detector) Detect() (string, error) { n, err := d.Reader.Read(d.data) - if err != nil && err != io.EOF { + if err != nil && !errors.Is(err, io.EOF) { d.err = err return "", err } From c5f758f53855bac450cdc548fefb4cff67844d17 Mon Sep 17 00:00:00 2001 From: Evgenii Baidakov Date: Fri, 28 Mar 2025 12:34:54 +0400 Subject: [PATCH 4/5] *: Refactor chunk reader Using state machine simplifies reader logic. It makes possible to extend its logic in the future in a more simple and predictable way. Signed-off-by: Evgenii Baidakov --- api/auth/center_test.go | 2 +- api/auth/signer/v4/chunked_reader.go | 222 +++++++++++++-------------- 2 files changed, 108 insertions(+), 116 deletions(-) diff --git a/api/auth/center_test.go b/api/auth/center_test.go index 7640c9f4..28a365c0 100644 --- a/api/auth/center_test.go +++ b/api/auth/center_test.go @@ -214,7 +214,7 @@ func TestAwsEncodedChunkReader(t *testing.T) { payload := bytes.NewBuffer(nil) _, err = io.CopyBuffer(payload, chunkedReader, chunk) - require.ErrorIs(t, err, v4.ErrMissingSeparator) + require.ErrorIs(t, err, v4.ErrInvalidByteInChunkLength) }) t.Run("err missing equality byte", func(t *testing.T) { diff --git a/api/auth/signer/v4/chunked_reader.go b/api/auth/signer/v4/chunked_reader.go index ce1c1acb..395d5a3a 100644 --- a/api/auth/signer/v4/chunked_reader.go +++ b/api/auth/signer/v4/chunked_reader.go @@ -24,12 +24,22 @@ var ( // ErrInvalidChunkSignature appears if passed chunk signature differs from calculated. ErrInvalidChunkSignature = errors.New("invalid chunk signature") - // ErrMissingSeparator appears if chunk header doesn't contain ';' separator. - ErrMissingSeparator = errors.New("missing header separator") - // ErrNoChunksSeparator appears if chunks not properly separated between each other. // They should be divided with \r\n bytes. ErrNoChunksSeparator = errors.New("no chunk separator") + + // ErrInvalidByteInChunkLength appears if chunk header has invalid encoding. + ErrInvalidByteInChunkLength = errors.New("invalid byte in chunk length") +) + +type readerState int + +const ( + readChunkHeader readerState = iota + readChunkPayload + verifyChunkSignature + readChunkCRLF + exit ) // NewChunkedReader returns a new chunkedReader that translates the data read from r @@ -41,6 +51,8 @@ func NewChunkedReader(r io.ReadCloser, streamSigner *ChunkSigner) io.ReadCloser // bufio.Reader can't be closed, thus left link to the original reader to close it later. origReader: r, streamSigner: streamSigner, + chunkHash: sha256.New(), + nextState: readChunkHeader, } } @@ -51,9 +63,9 @@ type chunkedReader struct { origReader io.ReadCloser n uint64 // unread bytes in chunk err error - buf [2]byte - checkEnd bool // whether need to check for \r\n chunk footer streamSigner *ChunkSigner + nextState readerState + lastChunk bool } // Close implements [io.ReadCloser]. @@ -63,41 +75,31 @@ func (cr *chunkedReader) Close() (err error) { func (cr *chunkedReader) beginChunk() { // chunk-size CRLF - var line, chunkSignature []byte - line, chunkSignature, cr.err = readChunkLine(cr.r) - if cr.err != nil { - return - } - cr.n, cr.err = parseHexUint(line) + var line []byte + line, cr.err = readChunkLine(cr.r) if cr.err != nil { return } - if err := cr.validatePreviousChunkData(); err != nil { - cr.err = err + hexSize, signaturePart := removeChunkExtension(line) + + cr.n, cr.err = parseHexUint(hexSize) + if cr.err != nil { return } - // creating instance here to avoid validating non-existent chunk in the first validatePreviousChunkData call. - if cr.chunkHash == nil { - cr.chunkHash = sha256.New() - } else { - cr.chunkHash.Reset() + if signaturePart != nil { + cr.chunkSignature = string(signaturePart) } - cr.chunkSignature = string(chunkSignature) + cr.chunkHash.Reset() if cr.n == 0 { - if err := cr.validatePreviousChunkData(); err != nil { - cr.err = err - return - } - cr.err = io.EOF } } -func (cr *chunkedReader) validatePreviousChunkData() error { +func (cr *chunkedReader) validateChunkData() error { if cr.chunkHash != nil { calculatedSignature, err := cr.streamSigner.GetSignatureByHash(cr.chunkHash) if err != nil { @@ -112,82 +114,91 @@ func (cr *chunkedReader) validatePreviousChunkData() error { return nil } -func (cr *chunkedReader) chunkHeaderAvailable() bool { - n := cr.r.Buffered() - if n > 0 { - peek, _ := cr.r.Peek(n) - return bytes.IndexByte(peek, '\n') >= 0 - } - return false -} - // Read gets data from reader. Implements [io.ReadCloser]. func (cr *chunkedReader) Read(b []uint8) (n int, err error) { - for cr.err == nil { - if cr.checkEnd { - if n > 0 && cr.r.Buffered() < 2 { - // We have some data. Return early (per the io.Reader - // contract) instead of potentially blocking while - // reading more. - break + for { + switch cr.nextState { + case readChunkHeader: + cr.beginChunk() + + if cr.n == 0 && errors.Is(cr.err, io.EOF) { + cr.nextState = readChunkCRLF + cr.lastChunk = true + continue } - if _, cr.err = io.ReadFull(cr.r, cr.buf[:2]); cr.err == nil { - if string(cr.buf[:]) != "\r\n" { - cr.err = ErrNoChunksSeparator - break - } - } else { - if errors.Is(cr.err, io.EOF) { - cr.err = io.ErrUnexpectedEOF - } - break + + if cr.err != nil { + return 0, cr.err } - cr.checkEnd = false - } - if cr.n == 0 { - if n > 0 && !cr.chunkHeaderAvailable() { - // We've read enough. Don't potentially block - // reading a new chunk header. - break + cr.nextState = readChunkPayload + case readChunkPayload: + // The incoming buffer is fulfilled. + if len(b) == 0 { + return n, nil } - cr.beginChunk() - continue - } - if len(b) == 0 { - break - } - rbuf := b - if uint64(len(rbuf)) > cr.n { - rbuf = rbuf[:cr.n] - } - var n0 int - n0, cr.err = cr.r.Read(rbuf) - n += n0 - b = b[n0:] - cr.n -= uint64(n0) - // Hashing chunk data to calculate the signature. - // rbuf may contain payload and empty bytes, taking only payload. - if _, err = cr.chunkHash.Write(rbuf[:n0]); err != nil { - cr.err = err - break + + rbuf := b + if uint64(len(rbuf)) > cr.n { + rbuf = rbuf[:cr.n] + } + var n0 int + n0, cr.err = cr.r.Read(rbuf) + n += n0 + b = b[n0:] + cr.n -= uint64(n0) + // Hashing chunk data to calculate the signature. + // rbuf may contain payload and empty bytes, taking only payload. + if _, err = cr.chunkHash.Write(rbuf[:n0]); err != nil { + return 0, err + } + // If we're at the end of a chunk. + if cr.n == 0 { + cr.nextState = readChunkCRLF + } + case readChunkCRLF: + cr.err = readCRLF(cr.r) + if cr.err != nil { + return 0, cr.err + } + cr.nextState = verifyChunkSignature + case verifyChunkSignature: + if err = cr.validateChunkData(); err != nil { + return 0, err + } + + if cr.lastChunk { + cr.nextState = exit + } else { + cr.nextState = readChunkHeader + } + case exit: + return n, io.EOF } + } +} - // If we're at the end of a chunk, read the next two - // bytes to verify they are "\r\n". - if cr.n == 0 && cr.err == nil { - cr.checkEnd = true - } else if errors.Is(cr.err, io.EOF) { - cr.err = io.ErrUnexpectedEOF +func readCRLF(reader io.Reader) error { + buf := make([]byte, 2) + if _, err := io.ReadFull(reader, buf[:2]); err != nil { + if errors.Is(err, io.EOF) { + return io.ErrUnexpectedEOF } + + return err } - return n, cr.err + + if string(buf[:]) != "\r\n" { + return ErrNoChunksSeparator + } + + return nil } // Read a line of bytes (up to \n) from b. // Give up if the line exceeds maxLineLength. // The returned bytes are owned by the bufio.Reader // so they are only valid until the next bufio read. -func readChunkLine(b *bufio.Reader) ([]byte, []byte, error) { +func readChunkLine(b *bufio.Reader) ([]byte, error) { p, err := b.ReadSlice('\n') if err != nil { // We always know when EOF is coming. @@ -197,27 +208,13 @@ func readChunkLine(b *bufio.Reader) ([]byte, []byte, error) { } else if errors.Is(err, bufio.ErrBufferFull) { err = ErrLineTooLong } - return nil, nil, err + return nil, err } if len(p) >= maxLineLength { - return nil, nil, ErrLineTooLong - } - - var signaturePart []byte - - p = trimTrailingWhitespace(p) - p, signaturePart, err = removeChunkExtension(p) - if err != nil { - return nil, nil, err - } - - pos := bytes.IndexByte(signaturePart, '=') - if pos == -1 { - return nil, nil, errors.New("chunk header is malformed") + return nil, ErrLineTooLong } - // even if '=' is the latest symbol, the new slice will be just empty - return p, signaturePart[pos+1:], nil + return p, nil } func trimTrailingWhitespace(b []byte) []byte { @@ -231,8 +228,6 @@ func isASCIISpace(b byte) bool { return b == ' ' || b == '\t' || b == '\n' || b == '\r' } -var semi = []byte(";") - // removeChunkExtension removes any chunk-extension from p. // For example, // @@ -240,17 +235,14 @@ var semi = []byte(";") // "0;chunk-signature" => "0" // "0;chunk-signature=val" => "0" // `0;chunk-signature="quoted string"` => "0" -func removeChunkExtension(p []byte) ([]byte, []byte, error) { - var ( - chunkSignature []byte - found bool - ) - p, chunkSignature, found = bytes.Cut(p, semi) - if !found { - return nil, nil, ErrMissingSeparator +func removeChunkExtension(p []byte) ([]byte, []byte) { + p = trimTrailingWhitespace(p) + pos := bytes.SplitN(p, []byte(";chunk-signature="), 2) + if len(pos) == 1 { + return pos[0], nil } - return p, chunkSignature, nil + return pos[0], pos[1] } func parseHexUint(v []byte) (n uint64, err error) { @@ -263,7 +255,7 @@ func parseHexUint(v []byte) (n uint64, err error) { case 'A' <= b && b <= 'F': b = b - 'A' + 10 default: - return 0, errors.New("invalid byte in chunk length") + return 0, ErrInvalidByteInChunkLength } if i == 16 { return 0, errors.New("http chunk length too large") From 11a66f082b37cad02821ba43a469cbcc43eca044 Mon Sep 17 00:00:00 2001 From: Evgenii Baidakov Date: Fri, 28 Mar 2025 15:32:30 +0400 Subject: [PATCH 5/5] *: Process trailing chunks Closes #1059. Signed-off-by: Evgenii Baidakov --- CHANGELOG.md | 1 + api/auth/center.go | 48 +++++--- api/auth/center_test.go | 48 +++++++- api/auth/signer/v4/checksum_algorithm.go | 78 +++++++++++++ api/auth/signer/v4/chunked_reader.go | 133 ++++++++++++++++++++--- api/handler/multipart_upload.go | 8 +- api/layer/multipart_upload.go | 2 +- go.mod | 2 + go.sum | 4 + 9 files changed, 290 insertions(+), 34 deletions(-) create mode 100644 api/auth/signer/v4/checksum_algorithm.go diff --git a/CHANGELOG.md b/CHANGELOG.md index c4ff6535..43117a02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ This document outlines major changes between releases. ## [Unreleased] ### Added +- Support of `x-amz-content-sha256` header with `STREAMING-UNSIGNED-PAYLOAD-TRAILER` value (#1028) ### Changed - AWS SDK migrated to V2 (#1028) diff --git a/api/auth/center.go b/api/auth/center.go index 7c591944..5708e40f 100644 --- a/api/auth/center.go +++ b/api/auth/center.go @@ -71,18 +71,18 @@ const ( authHeaderPartsNum = 6 maxFormSizeMemory = 50 * 1048576 // 50 MB - AmzAlgorithm = "X-Amz-Algorithm" - AmzCredential = "X-Amz-Credential" - AmzSignature = "X-Amz-Signature" - AmzSignedHeaders = "X-Amz-SignedHeaders" - AmzExpires = "X-Amz-Expires" - AmzDate = "X-Amz-Date" - AmzContentSha256 = "X-Amz-Content-Sha256" - AuthorizationHdr = "Authorization" - ContentTypeHdr = "Content-Type" - ContentEncodingHdr = "Content-Encoding" - ContentEncodingAwsChunked = "aws-chunked" - ContentEncodingChunked = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" + AmzAlgorithm = "X-Amz-Algorithm" + AmzCredential = "X-Amz-Credential" + AmzSignature = "X-Amz-Signature" + AmzSignedHeaders = "X-Amz-SignedHeaders" + AmzExpires = "X-Amz-Expires" + AmzDate = "X-Amz-Date" + AmzContentSha256 = "X-Amz-Content-Sha256" + AuthorizationHdr = "Authorization" + AmzTrailer = "x-amz-trailer" + ContentTypeHdr = "Content-Type" + ContentEncodingChunked = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" + UnsignedPayloadMultipleChunks = "STREAMING-UNSIGNED-PAYLOAD-TRAILER" timeFormatISO8601 = "20060102T150405Z" @@ -190,6 +190,14 @@ func (c *center) Authenticate(r *http.Request) (*Box, error) { if strings.HasPrefix(r.Header.Get(ContentTypeHdr), "multipart/form-data") { return c.checkFormData(r) } + + if r.Header.Get(AmzContentSha256) == UnsignedPayloadMultipleChunks { + r.Body, err = v4.NewChunkedReaderWithTrail(r.Body, r.Header.Get(AmzTrailer)) + if err != nil { + return nil, err + } + } + return nil, ErrNoAuthorizationHeader } authHdr, err = c.parseAuthHeader(authHeaderField[0], r.Header.Get(AmzContentSha256)) @@ -226,20 +234,26 @@ func (c *center) Authenticate(r *http.Request) (*Box, error) { amzContent := r.Header.Get(AmzContentSha256) - if contentEncodingHdr := r.Header.Get(ContentEncodingHdr); contentEncodingHdr == ContentEncodingAwsChunked || amzContent == ContentEncodingChunked { + switch amzContent { + case ContentEncodingChunked: sig, err := hex.DecodeString(authHdr.SignatureV4) if err != nil { - return nil, fmt.Errorf("DecodeString: %w", err) + return nil, fmt.Errorf("decode auth header signature: %w", err) } appCreds := credentials.NewStaticCredentialsProvider(authHdr.AccessKeyID, box.Gate.AccessKey, "") value, err := appCreds.Retrieve(r.Context()) if err != nil { - return nil, fmt.Errorf("get credentials: %w", err) + return nil, fmt.Errorf("retrieve aws credentials: %w", err) } - streamSigner := v4.NewChunkSigner(authHdr.Region, authHdr.Service, sig, signatureDateTime, value) - r.Body = v4.NewChunkedReader(r.Body, streamSigner) + chunkSigner := v4.NewChunkSigner(authHdr.Region, authHdr.Service, sig, signatureDateTime, value) + r.Body = v4.NewChunkedReader(r.Body, chunkSigner) + case UnsignedPayloadMultipleChunks: + r.Body, err = v4.NewChunkedReaderWithTrail(r.Body, clonedRequest.Header.Get(AmzTrailer)) + if err != nil { + return nil, err + } } result := &Box{AccessBox: box} diff --git a/api/auth/center_test.go b/api/auth/center_test.go index 28a365c0..97054a31 100644 --- a/api/auth/center_test.go +++ b/api/auth/center_test.go @@ -4,8 +4,10 @@ import ( "bytes" "context" "crypto/sha256" + "encoding/base64" "encoding/hex" "fmt" + "hash/crc32" "io" "net/http" "slices" @@ -318,7 +320,7 @@ func TestAwsEncodedChunkReader(t *testing.T) { payload := bytes.NewBuffer(nil) _, err = io.CopyBuffer(payload, chunkedReader, chunk) - require.ErrorIs(t, err, v4.ErrNoChunksSeparator) + require.ErrorIs(t, err, v4.ErrInvalidChunkSignature) }) t.Run("err chunk header too long", func(t *testing.T) { @@ -459,3 +461,47 @@ func chunkSlice(payload []byte, chunkSize int) [][]byte { return result } + +func TestAwsEncodedChunkReaderWithTrailer(t *testing.T) { + chunk1 := "2000\r\n" + strings.Repeat("a", 8192) + "\r\n" + chunk2 := "2000\r\n" + strings.Repeat("a", 8192) + "\r\n" + chunk3 := "400\r\n" + strings.Repeat("a", 1024) + "\r\n" + chunk4 := "0\r\n" + + var ( + objectPayload = strings.Repeat("a", 17408) + writer = crc32.NewIEEE() + checksumType = "x-amz-checksum-crc32" + ) + + _, err := writer.Write([]byte(objectPayload)) + require.NoError(t, err) + + var ( + checksum = writer.Sum(nil) + base64EncodedChecksum = base64.StdEncoding.EncodeToString(checksum) + trailer = checksumType + ":" + base64EncodedChecksum + "\n\r\n\r\n\r\n" + requestPayload = chunk1 + chunk2 + chunk3 + chunk4 + trailer + ) + + t.Run("correct signature", func(t *testing.T) { + buf := bytes.NewBuffer(nil) + + _, err = buf.Write([]byte(requestPayload)) + require.NoError(t, err) + + chunkedReader, err := v4.NewChunkedReaderWithTrail(io.NopCloser(buf), checksumType) + require.NoError(t, err) + + defer func() { + _ = chunkedReader.Close() + }() + + chunk := make([]byte, 4096) + payload2 := bytes.NewBuffer(nil) + _, err = io.CopyBuffer(payload2, chunkedReader, chunk) + require.NoError(t, err) + + require.Equal(t, []byte(objectPayload), payload2.Bytes()) + }) +} diff --git a/api/auth/signer/v4/checksum_algorithm.go b/api/auth/signer/v4/checksum_algorithm.go new file mode 100644 index 00000000..df0da9cd --- /dev/null +++ b/api/auth/signer/v4/checksum_algorithm.go @@ -0,0 +1,78 @@ +package v4 + +import ( + "crypto/sha1" + "crypto/sha256" + "fmt" + "hash" + "hash/crc32" + + "github.com/minio/crc64nvme" +) + +type ( + checksumType int +) + +const ( + checksumNone checksumType = iota + checksumCRC32 + checksumCRC32C + checksumSHA1 + checksumSHA256 + checksumCRC64NVMe +) + +func (ca checksumType) String() string { + switch ca { + case checksumCRC32: + return "CRC32" + case checksumCRC32C: + return "CRC32C" + case checksumCRC64NVMe: + return "CRC64NVMe" + case checksumSHA1: + return "SHA1" + case checksumSHA256: + return "SHA256" + case checksumNone: + return "" + } + return "" +} + +func checksumWriter(algo checksumType) hash.Hash { + switch algo { + case checksumCRC32: + return crc32.NewIEEE() + case checksumCRC32C: + return crc32.New(crc32.MakeTable(crc32.Castagnoli)) + case checksumCRC64NVMe: + return crc64nvme.New() + case checksumSHA1: + return sha1.New() + case checksumSHA256: + return sha256.New() + default: + return nil + } +} + +func detectChecksumType(amzTrailerHeader string) (checksumType, error) { + switch amzTrailerHeader { + case "x-amz-checksum-crc32": + return checksumCRC32, nil + case "x-amz-checksum-crc32c": + return checksumCRC32C, nil + case "x-amz-checksum-crc64nvme": + return checksumCRC64NVMe, nil + case "x-amz-checksum-sha1": + return checksumSHA1, nil + case "x-amz-checksum-sha256": + return checksumSHA256, nil + case "": + return checksumNone, nil + default: + return checksumNone, fmt.Errorf("unsupported: %s", amzTrailerHeader) + } +} diff --git a/api/auth/signer/v4/chunked_reader.go b/api/auth/signer/v4/chunked_reader.go index 395d5a3a..bbd5af44 100644 --- a/api/auth/signer/v4/chunked_reader.go +++ b/api/auth/signer/v4/chunked_reader.go @@ -8,6 +8,7 @@ import ( "bufio" "bytes" "crypto/sha256" + "encoding/base64" "encoding/hex" "errors" "fmt" @@ -30,6 +31,8 @@ var ( // ErrInvalidByteInChunkLength appears if chunk header has invalid encoding. ErrInvalidByteInChunkLength = errors.New("invalid byte in chunk length") + + errInvalidChunkEncoding = errors.New("invalid chunk encoding") ) type readerState int @@ -40,6 +43,7 @@ const ( verifyChunkSignature readChunkCRLF exit + readTrailerChunk ) // NewChunkedReader returns a new chunkedReader that translates the data read from r @@ -56,16 +60,38 @@ func NewChunkedReader(r io.ReadCloser, streamSigner *ChunkSigner) io.ReadCloser } } +// NewChunkedReaderWithTrail returns a new chunkedReader that translates the data read from r +// out of HTTP "chunked" format before returning it. It uses trailing chunk to verify data consistency. +// The chunkedReader returns io.EOF when the final 0-length chunk is read. +func NewChunkedReaderWithTrail(r io.ReadCloser, amzTrailerHeader string) (io.ReadCloser, error) { + checksumAlgorithm, err := detectChecksumType(amzTrailerHeader) + if err != nil { + return nil, err + } + + return &chunkedReader{ + r: bufio.NewReader(r), + // bufio.Reader can't be closed, thus left link to the original reader to close it later. + origReader: r, + chunkHash: sha256.New(), + nextState: readChunkHeader, + checkSumAlgorithm: checksumAlgorithm.String(), + checkSumWriter: checksumWriter(checksumAlgorithm), + }, nil +} + type chunkedReader struct { - chunkHash hash.Hash - chunkSignature string - r *bufio.Reader - origReader io.ReadCloser - n uint64 // unread bytes in chunk - err error - streamSigner *ChunkSigner - nextState readerState - lastChunk bool + chunkHash hash.Hash + chunkSignature string + r *bufio.Reader + origReader io.ReadCloser + n uint64 // unread bytes in chunk + err error + streamSigner *ChunkSigner + nextState readerState + lastChunk bool + checkSumAlgorithm string + checkSumWriter hash.Hash } // Close implements [io.ReadCloser]. @@ -100,7 +126,7 @@ func (cr *chunkedReader) beginChunk() { } func (cr *chunkedReader) validateChunkData() error { - if cr.chunkHash != nil { + if cr.chunkHash != nil && cr.streamSigner != nil { calculatedSignature, err := cr.streamSigner.GetSignatureByHash(cr.chunkHash) if err != nil { return fmt.Errorf("GetSignature: %w", err) @@ -151,16 +177,34 @@ func (cr *chunkedReader) Read(b []uint8) (n int, err error) { if _, err = cr.chunkHash.Write(rbuf[:n0]); err != nil { return 0, err } + + if cr.checkSumWriter != nil { + cr.checkSumWriter.Write(rbuf[:n0]) + } + // If we're at the end of a chunk. if cr.n == 0 { cr.nextState = readChunkCRLF } case readChunkCRLF: - cr.err = readCRLF(cr.r) - if cr.err != nil { - return 0, cr.err + err = peekCRLF(cr.r) + isTrailingChunk := cr.n == 0 && cr.lastChunk + + if !isTrailingChunk { + cr.err = readCRLF(cr.r) + } else if err != nil && !errors.Is(err, errInvalidChunkEncoding) { + cr.err = err + return 0, errInvalidChunkEncoding + } + + // Unsigned streaming upload. + if cr.chunkSignature != "" { + cr.nextState = verifyChunkSignature + } else if cr.lastChunk { + cr.nextState = readTrailerChunk + } else { + cr.nextState = readChunkHeader } - cr.nextState = verifyChunkSignature case verifyChunkSignature: if err = cr.validateChunkData(); err != nil { return 0, err @@ -171,6 +215,25 @@ func (cr *chunkedReader) Read(b []uint8) (n int, err error) { } else { cr.nextState = readChunkHeader } + case readTrailerChunk: + extractedCheckSumAlgorithm, extractedChecksum := parseChunkChecksum(cr.r) + if extractedCheckSumAlgorithm.String() != cr.checkSumAlgorithm { + cr.err = fmt.Errorf("request header and trailed chunk checksum algorithm mismatch. %s vs %s", extractedCheckSumAlgorithm.String(), cr.checkSumAlgorithm) + return 0, cr.err + } + + base64Checksum := base64.StdEncoding.EncodeToString(cr.checkSumWriter.Sum(nil)) + if string(extractedChecksum) != base64Checksum { + cr.err = errors.New("payload checksum does not match") + return 0, cr.err + } + + // Reading remaining CRLF. + for range 2 { + cr.err = readCRLF(cr.r) + } + + cr.nextState = exit case exit: return n, io.EOF } @@ -194,6 +257,24 @@ func readCRLF(reader io.Reader) error { return nil } +func peekCRLF(reader *bufio.Reader) error { + peeked, err := reader.Peek(2) + if err != nil { + return err + } + if err = checkCRLF(peeked); err != nil { + return err + } + return nil +} + +func checkCRLF(buf []byte) error { + if string(buf[:]) != "\r\n" { + return errInvalidChunkEncoding + } + return nil +} + // Read a line of bytes (up to \n) from b. // Give up if the line exceeds maxLineLength. // The returned bytes are owned by the bufio.Reader @@ -265,3 +346,27 @@ func parseHexUint(v []byte) (n uint64, err error) { } return } + +func parseChunkChecksum(b *bufio.Reader) (checksumType, []byte) { + bytesRead, err := readChunkLine(b) + if err != nil { + return checksumNone, nil + } + + parts := bytes.SplitN(bytesRead, []byte(":"), 2) + if len(parts) != 2 { + return checksumNone, nil + } + + var ( + checksumKey = string(parts[0]) + checksumValue = trimTrailingWhitespace(parts[1]) + ) + + extractedAlgorithm, err := detectChecksumType(checksumKey) + if err != nil { + return checksumNone, nil + } + + return extractedAlgorithm, checksumValue +} diff --git a/api/handler/multipart_upload.go b/api/handler/multipart_upload.go index 65e43516..1950b687 100644 --- a/api/handler/multipart_upload.go +++ b/api/handler/multipart_upload.go @@ -235,6 +235,12 @@ func (h *handler) UploadPartHandler(w http.ResponseWriter, r *http.Request) { return } + contentLength, err := contentLengthFromRequest(r) + if err != nil { + h.logAndSendError(w, "content length parse failed", reqInfo, err) + return + } + p := &layer.UploadPartParams{ Info: &layer.UploadInfoParams{ UploadID: uploadID, @@ -242,7 +248,7 @@ func (h *handler) UploadPartHandler(w http.ResponseWriter, r *http.Request) { Key: reqInfo.ObjectName, }, PartNumber: partNumber, - Size: r.ContentLength, + Size: contentLength, Reader: r.Body, } diff --git a/api/layer/multipart_upload.go b/api/layer/multipart_upload.go index aa601aae..f8c15ec1 100644 --- a/api/layer/multipart_upload.go +++ b/api/layer/multipart_upload.go @@ -356,7 +356,7 @@ func (n *layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf isReturnToPool bool ) - if p.Size > n.neoFS.MaxObjectSize()/2 { + if p.Size > n.neoFS.MaxObjectSize()/2 || p.Size == -1 { chunk = n.buffers.Get().(*[]byte) isReturnToPool = true } else { diff --git a/go.mod b/go.mod index 2dbe3d9b..5f33f3a9 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/cenkalti/backoff/v5 v5.0.2 github.com/google/uuid v1.6.0 github.com/gorilla/mux v1.8.1 + github.com/minio/crc64nvme v1.0.1 github.com/minio/sio v0.4.1 github.com/nats-io/nats.go v1.37.0 github.com/nspcc-dev/neo-go v0.108.1 @@ -36,6 +37,7 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/holiman/uint256 v1.3.1 // indirect github.com/klauspost/compress v1.17.11 // indirect + github.com/klauspost/cpuid/v2 v2.2.9 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/nspcc-dev/go-ordered-json v0.0.0-20240830112754-291b000d1f3b // indirect github.com/nspcc-dev/hrw/v2 v2.0.3 // indirect diff --git a/go.sum b/go.sum index 61a4fda0..7df39247 100644 --- a/go.sum +++ b/go.sum @@ -98,6 +98,8 @@ github.com/holiman/uint256 v1.3.1/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXei github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= +github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -108,6 +110,8 @@ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/minio/crc64nvme v1.0.1 h1:DHQPrYPdqK7jQG/Ls5CTBZWeex/2FMS3G5XGkycuFrY= +github.com/minio/crc64nvme v1.0.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= github.com/minio/sio v0.4.1 h1:EMe3YBC1nf+sRQia65Rutxi+Z554XPV0dt8BIBA+a/0= github.com/minio/sio v0.4.1/go.mod h1:oBSjJeGbBdRMZZwna07sX9EFzZy+ywu5aofRiV1g79I= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=