diff --git a/api/handler/attributes.go b/api/handler/attributes.go index b678fd05..c3504e47 100644 --- a/api/handler/attributes.go +++ b/api/handler/attributes.go @@ -10,6 +10,7 @@ import ( "github.com/nspcc-dev/neofs-s3-gw/api/data" "github.com/nspcc-dev/neofs-s3-gw/api/layer" "github.com/nspcc-dev/neofs-s3-gw/api/s3errors" + "github.com/nspcc-dev/neofs-s3-gw/api/s3headers" "go.uber.org/zap" ) @@ -206,7 +207,7 @@ func encodeToObjectAttributesResponse(info *data.ObjectInfo, p *GetObjectAttribu } func formUploadAttributes(info *data.ObjectInfo, maxParts, marker int) (*ObjectParts, error) { - completedParts, ok := info.Headers[layer.UploadCompletedParts] + completedParts, ok := info.Headers[s3headers.UploadCompletedParts] if !ok { return nil, nil } diff --git a/api/handler/get.go b/api/handler/get.go index cd95dfe7..c47ed445 100644 --- a/api/handler/get.go +++ b/api/handler/get.go @@ -13,6 +13,7 @@ import ( "github.com/nspcc-dev/neofs-s3-gw/api/data" "github.com/nspcc-dev/neofs-s3-gw/api/layer" "github.com/nspcc-dev/neofs-s3-gw/api/s3errors" + "github.com/nspcc-dev/neofs-s3-gw/api/s3headers" "go.uber.org/zap" ) @@ -88,8 +89,8 @@ func writeHeaders(h http.Header, requestHeader http.Header, info *data.ObjectInf } h.Set(api.LastModified, info.Created.UTC().Format(http.TimeFormat)) - if len(info.Headers[layer.AttributeEncryptionAlgorithm]) > 0 { - h.Set(api.ContentLength, info.Headers[layer.AttributeDecryptedSize]) + if len(info.Headers[s3headers.AttributeEncryptionAlgorithm]) > 0 { + h.Set(api.ContentLength, info.Headers[s3headers.AttributeDecryptedSize]) addSSECHeaders(h, requestHeader) } else { h.Set(api.ContentLength, strconv.FormatInt(info.Size, 10)) @@ -186,7 +187,7 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) { fullSize := info.Size if encryptionParams.Enabled() { - if fullSize, err = strconv.ParseInt(info.Headers[layer.AttributeDecryptedSize], 10, 64); err != nil { + if fullSize, err = strconv.ParseInt(info.Headers[s3headers.AttributeDecryptedSize], 10, 64); err != nil { h.logAndSendError(w, "invalid decrypted size header", reqInfo, s3errors.GetAPIError(s3errors.ErrBadRequest)) return } diff --git a/api/headers.go b/api/headers.go index 8437ac65..996ad06e 100644 --- a/api/headers.go +++ b/api/headers.go @@ -2,16 +2,16 @@ package api // Standard S3 HTTP request/response constants. const ( - MetadataPrefix = "X-Amz-Meta-" - NeoFSSystemMetadataPrefix = "S3-" - AmzMetadataDirective = "X-Amz-Metadata-Directive" - AmzTaggingDirective = "X-Amz-Tagging-Directive" - AmzVersionID = "X-Amz-Version-Id" - AmzTaggingCount = "X-Amz-Tagging-Count" - AmzTagging = "X-Amz-Tagging" - AmzDeleteMarker = "X-Amz-Delete-Marker" - AmzCopySource = "X-Amz-Copy-Source" - AmzCopySourceRange = "X-Amz-Copy-Source-Range" + MetadataPrefix = "X-Amz-Meta-" + + AmzMetadataDirective = "X-Amz-Metadata-Directive" + AmzTaggingDirective = "X-Amz-Tagging-Directive" + AmzVersionID = "X-Amz-Version-Id" + AmzTaggingCount = "X-Amz-Tagging-Count" + AmzTagging = "X-Amz-Tagging" + AmzDeleteMarker = "X-Amz-Delete-Marker" + AmzCopySource = "X-Amz-Copy-Source" + AmzCopySourceRange = "X-Amz-Copy-Source-Range" LastModified = "Last-Modified" Date = "Date" diff --git a/api/layer/layer.go b/api/layer/layer.go index 252b43bf..588ec6e6 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -278,13 +278,8 @@ type ( const ( tagPrefix = "S3-Tag-" - AESEncryptionAlgorithm = "AES256" - AESKeySize = 32 - AttributeEncryptionAlgorithm = api.NeoFSSystemMetadataPrefix + "Algorithm" - AttributeDecryptedSize = api.NeoFSSystemMetadataPrefix + "Decrypted-Size" - AttributeHMACSalt = api.NeoFSSystemMetadataPrefix + "HMAC-Salt" - AttributeHMACKey = api.NeoFSSystemMetadataPrefix + "HMAC-Key" - + AESEncryptionAlgorithm = "AES256" + AESKeySize = 32 AttributeNeofsCopiesNumber = "neofs-copies-number" // such format to match X-Amz-Meta-Neofs-Copies-Number header ) @@ -536,12 +531,12 @@ func getDecrypter(p *GetObjectParams) (*encryption.Decrypter, error) { encRange = &encryption.Range{Start: p.Range.Start, End: p.Range.End} } - header := p.ObjectInfo.Headers[UploadCompletedParts] + header := p.ObjectInfo.Headers[s3headers.UploadCompletedParts] if len(header) == 0 { return encryption.NewDecrypter(p.Encryption, uint64(p.ObjectInfo.Size), encRange) } - decryptedObjectSize, err := strconv.ParseUint(p.ObjectInfo.Headers[AttributeDecryptedSize], 10, 64) + decryptedObjectSize, err := strconv.ParseUint(p.ObjectInfo.Headers[s3headers.AttributeDecryptedSize], 10, 64) if err != nil { return nil, fmt.Errorf("parse decrypted size: %w", err) } @@ -773,7 +768,7 @@ func (n *layer) GetIDForVersioningContainer(ctx context.Context, p *ShortInfoPar object.AttributeFilePath, object.FilterCreationEpoch, object.AttributeTimestamp, - attrS3DeleteMarker, + s3headers.AttributeDeleteMarker, } opts client.SearchObjectsOptions @@ -788,7 +783,7 @@ func (n *layer) GetIDForVersioningContainer(ctx context.Context, p *ShortInfoPar filters.AddFilter(s3headers.MetaType, "", object.MatchNotPresent) if !p.FindNullVersion { - filters.AddFilter(attrS3VersioningState, data.VersioningEnabled, object.MatchStringEqual) + filters.AddFilter(s3headers.AttributeVersioningState, data.VersioningEnabled, object.MatchStringEqual) } ids, err := n.neoFS.SearchObjectsV2(ctx, p.CID, filters, returningAttributes, opts) @@ -1072,7 +1067,7 @@ func (n *layer) putDeleteMarker(ctx context.Context, bktInfo *data.BucketInfo, o Object: objectName, Reader: bytes.NewReader(nil), Header: map[string]string{ - attrS3DeleteMarker: ts, + s3headers.AttributeDeleteMarker: ts, }, } ) diff --git a/api/layer/multipart_upload.go b/api/layer/multipart_upload.go index 43e1b90c..aa601aae 100644 --- a/api/layer/multipart_upload.go +++ b/api/layer/multipart_upload.go @@ -36,8 +36,6 @@ import ( ) const ( - UploadCompletedParts = "S3-Completed-Parts" - metaPrefix = "meta-" aclPrefix = "acl-" @@ -261,7 +259,7 @@ func (n *layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf if err != nil { return nil, fmt.Errorf("failed to create ecnrypted reader: %w", err) } - attributes[AttributeDecryptedSize] = strconv.FormatInt(p.Size, 10) + attributes[s3headers.AttributeDecryptedSize] = strconv.FormatInt(p.Size, 10) payloadReader = r p.Size = int64(encSize) } @@ -429,7 +427,7 @@ func (n *layer) uploadZeroPart(ctx context.Context, multipartInfo *data.Multipar ) if p.Encryption.Enabled() { - attributes[AttributeDecryptedSize] = "0" + attributes[s3headers.AttributeDecryptedSize] = "0" } if n.neoFS.IsHomomorphicHashingEnabled() { @@ -446,9 +444,9 @@ func (n *layer) uploadZeroPart(ctx context.Context, multipartInfo *data.Multipar } if encInfo.Enabled { - attrs = append(attrs, *object.NewAttribute(AttributeEncryptionAlgorithm, encInfo.Algorithm)) - attrs = append(attrs, *object.NewAttribute(AttributeHMACKey, encInfo.HMACKey)) - attrs = append(attrs, *object.NewAttribute(AttributeHMACSalt, encInfo.HMACSalt)) + attrs = append(attrs, *object.NewAttribute(s3headers.AttributeEncryptionAlgorithm, encInfo.Algorithm)) + attrs = append(attrs, *object.NewAttribute(s3headers.AttributeHMACKey, encInfo.HMACKey)) + attrs = append(attrs, *object.NewAttribute(s3headers.AttributeHMACSalt, encInfo.HMACSalt)) } var hashlessHeaderObject object.Object @@ -948,10 +946,10 @@ func (n *layer) multipartMetaGetParts(ctx context.Context, bktInfo *data.BucketI return nil, fmt.Errorf("convert multipart %s MultipartTotalSize %s: %w", uploadID, element.Attributes[s3headers.MultipartTotalSize], err) } - if decSize, ok := element.Attributes[AttributeDecryptedSize]; ok && decSize != "" { + if decSize, ok := element.Attributes[s3headers.AttributeDecryptedSize]; ok && decSize != "" { element.TotalSize, err = strconv.ParseInt(decSize, 10, 64) if err != nil { - return nil, fmt.Errorf("convert multipart %s AttributeDecryptedSize %s: %w", uploadID, element.Attributes[AttributeDecryptedSize], err) + return nil, fmt.Errorf("convert multipart %s s3headers.AttributeDecryptedSize %s: %w", uploadID, element.Attributes[s3headers.AttributeDecryptedSize], err) } } @@ -1029,7 +1027,7 @@ func (n *layer) multipartGetPartsList(ctx context.Context, bktInfo *data.BucketI s3headers.MultipartPartHash, s3headers.MultipartElementID, s3headers.MultipartTotalSize, - AttributeDecryptedSize, + s3headers.AttributeDecryptedSize, } ) @@ -1517,7 +1515,7 @@ func (n *layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipar } initMetadata := make(map[string]string, len(multipartInfo.Meta)+1) - initMetadata[UploadCompletedParts] = completedPartsHeader.String() + initMetadata[s3headers.UploadCompletedParts] = completedPartsHeader.String() uploadData := &UploadData{ TagSet: make(map[string]string), @@ -1534,10 +1532,10 @@ func (n *layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipar } if encInfo.Enabled { - initMetadata[AttributeEncryptionAlgorithm] = encInfo.Algorithm - initMetadata[AttributeHMACKey] = encInfo.HMACKey - initMetadata[AttributeHMACSalt] = encInfo.HMACSalt - initMetadata[AttributeDecryptedSize] = strconv.FormatInt(multipartObjetSize, 10) + initMetadata[s3headers.AttributeEncryptionAlgorithm] = encInfo.Algorithm + initMetadata[s3headers.AttributeHMACKey] = encInfo.HMACKey + initMetadata[s3headers.AttributeHMACSalt] = encInfo.HMACSalt + initMetadata[s3headers.AttributeDecryptedSize] = strconv.FormatInt(multipartObjetSize, 10) n.log.Debug("big object", zap.Int64("size", multipartObjetSize), zap.Uint64("enc_size", encMultipartObjectSize)) multipartObjetSize = int64(encMultipartObjectSize) diff --git a/api/layer/neofs_mock.go b/api/layer/neofs_mock.go index 25cd7bc2..30702ea4 100644 --- a/api/layer/neofs_mock.go +++ b/api/layer/neofs_mock.go @@ -19,6 +19,7 @@ import ( "time" "github.com/nspcc-dev/neofs-s3-gw/api" + "github.com/nspcc-dev/neofs-s3-gw/api/s3headers" "github.com/nspcc-dev/neofs-s3-gw/creds/accessbox" "github.com/nspcc-dev/neofs-sdk-go/checksum" "github.com/nspcc-dev/neofs-sdk-go/client" @@ -48,8 +49,7 @@ type searchedItem struct { } const ( - objectNonceSize = 8 - objectNonceAttribute = "__NEOFS__NONCE" + objectNonceSize = 8 ) func NewTestNeoFS(signer neofscrypto.Signer) *TestNeoFS { @@ -296,7 +296,7 @@ func (t *TestNeoFS) CreateObject(_ context.Context, prm PrmObjectCreate) (oid.ID if _, err := rand.Read(nonce); err != nil { return oid.ID{}, fmt.Errorf("object nonce: %w", err) } - objectNonceAttr := object.NewAttribute(objectNonceAttribute, base64.StdEncoding.EncodeToString(nonce)) + objectNonceAttr := object.NewAttribute(s3headers.AttributeObjectNonce, base64.StdEncoding.EncodeToString(nonce)) attrs = append(attrs, *objectNonceAttr) obj := object.New() diff --git a/api/layer/object.go b/api/layer/object.go index bb2e0089..7abf88db 100644 --- a/api/layer/object.go +++ b/api/layer/object.go @@ -112,11 +112,6 @@ type ( } ) -const ( - attrS3VersioningState = "S3-versioning-state" - attrS3DeleteMarker = "S3-delete-marker" -) - // objectHead returns all object's headers. func (n *layer) objectHead(ctx context.Context, bktInfo *data.BucketInfo, idObj oid.ID) (*object.Object, error) { prm := PrmObjectRead{ @@ -237,7 +232,7 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend r := p.Reader if p.Encryption.Enabled() { - p.Header[AttributeDecryptedSize] = strconv.FormatInt(p.Size, 10) + p.Header[s3headers.AttributeDecryptedSize] = strconv.FormatInt(p.Size, 10) if err = addEncryptionHeaders(p.Header, p.Encryption); err != nil { return nil, fmt.Errorf("add encryption header: %w", err) } @@ -281,7 +276,7 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend } if bktSettings.VersioningEnabled() { - prm.Attributes[attrS3VersioningState] = data.VersioningEnabled + prm.Attributes[s3headers.AttributeVersioningState] = data.VersioningEnabled } id, hash, err := n.objectPutAndHash(ctx, prm, p.BktInfo) @@ -351,7 +346,7 @@ func (n *layer) prepareMultipartHeadObject(ctx context.Context, p *PutObjectPara ) if p.Encryption.Enabled() { - p.Header[AttributeDecryptedSize] = strconv.FormatInt(p.Size, 10) + p.Header[s3headers.AttributeDecryptedSize] = strconv.FormatInt(p.Size, 10) if err = addEncryptionHeaders(p.Header, p.Encryption); err != nil { return nil, fmt.Errorf("add encryption header: %w", err) } @@ -391,7 +386,7 @@ func (n *layer) prepareMultipartHeadObject(ctx context.Context, p *PutObjectPara } if versioningEnabled { - attributes = append(attributes, *object.NewAttribute(attrS3VersioningState, data.VersioningEnabled)) + attributes = append(attributes, *object.NewAttribute(s3headers.AttributeVersioningState, data.VersioningEnabled)) } headerObject.SetAttributes(attributes...) @@ -414,9 +409,9 @@ func (n *layer) searchAllVersionsInNeoFS(ctx context.Context, bkt *data.BucketIn object.AttributeFilePath, object.FilterCreationEpoch, object.AttributeTimestamp, - attrS3VersioningState, + s3headers.AttributeVersioningState, object.FilterPayloadSize, - attrS3DeleteMarker, + s3headers.AttributeDeleteMarker, } opts client.SearchObjectsOptions @@ -436,7 +431,7 @@ func (n *layer) searchAllVersionsInNeoFS(ctx context.Context, bkt *data.BucketIn filters.AddFilter(s3headers.MetaType, "", object.MatchNotPresent) if onlyUnversioned { - filters.AddFilter(attrS3VersioningState, data.VersioningUnversioned, object.MatchNotPresent) + filters.AddFilter(s3headers.AttributeVersioningState, data.VersioningUnversioned, object.MatchNotPresent) } searchResultItems, err := n.neoFS.SearchObjectsV2(ctx, bkt.CID, filters, returningAttributes, opts) @@ -517,9 +512,9 @@ func (n *layer) comprehensiveSearchAllVersionsInNeoFS(ctx context.Context, bkt * object.AttributeFilePath, object.FilterCreationEpoch, object.AttributeTimestamp, - attrS3VersioningState, + s3headers.AttributeVersioningState, object.FilterPayloadSize, - attrS3DeleteMarker, + s3headers.AttributeDeleteMarker, s3headers.MetaType, } @@ -537,7 +532,7 @@ func (n *layer) comprehensiveSearchAllVersionsInNeoFS(ctx context.Context, bkt * } if onlyUnversioned { - filters.AddFilter(attrS3VersioningState, data.VersioningUnversioned, object.MatchNotPresent) + filters.AddFilter(s3headers.AttributeVersioningState, data.VersioningUnversioned, object.MatchNotPresent) } searchResultItems, err := n.neoFS.SearchObjectsV2(ctx, bkt.CID, filters, returningAttributes, opts) @@ -646,8 +641,8 @@ func (n *layer) searchTagsAndLocksInNeoFS(ctx context.Context, bkt *data.BucketI filters.AddFilter(object.AttributeFilePath, "", object.MatchCommonPrefix) } - filters.AddFilter(attrS3VersioningState, data.VersioningEnabled, object.MatchStringEqual) - filters.AddFilter(AttributeObjectVersion, objectVersion, object.MatchStringEqual) + filters.AddFilter(s3headers.AttributeVersioningState, data.VersioningEnabled, object.MatchStringEqual) + filters.AddFilter(s3headers.AttributeObjectVersion, objectVersion, object.MatchStringEqual) filters.AddFilter(s3headers.MetaType, "", object.MatchStringNotEqual) @@ -701,8 +696,8 @@ func (n *layer) searchAllVersionsInNeoFSByPrefix(ctx context.Context, bkt *data. object.AttributeFilePath, object.FilterCreationEpoch, object.AttributeTimestamp, - attrS3DeleteMarker, - AttributeDecryptedSize, + s3headers.AttributeDeleteMarker, + s3headers.AttributeDecryptedSize, object.FilterPayloadSize, object.FilterPayloadChecksum, } @@ -725,7 +720,7 @@ func (n *layer) searchAllVersionsInNeoFSByPrefix(ctx context.Context, bkt *data. filters.AddFilter(s3headers.MetaType, "", object.MatchNotPresent) if onlyUnversioned { - filters.AddFilter(attrS3VersioningState, "", object.MatchNotPresent) + filters.AddFilter(s3headers.AttributeVersioningState, "", object.MatchNotPresent) } searchResultItems, nextCursor, err := n.neoFS.SearchObjectsV2WithCursor(ctx, bkt.CID, filters, returningAttributes, cursor, opts) @@ -1108,7 +1103,7 @@ func (n *layer) getAllObjectsVersions(ctx context.Context, bkt *data.BucketInfo, func IsSystemHeader(key string) bool { _, ok := api.SystemMetadata[key] - return ok || strings.HasPrefix(key, api.NeoFSSystemMetadataPrefix) + return ok || strings.HasPrefix(key, s3headers.NeoFSSystemMetadataPrefix) } func shouldSkip(result prefixSearchResult, p allObjectParams, existed map[string]struct{}) bool { diff --git a/api/layer/system_object.go b/api/layer/system_object.go index 9366b0d2..736c14cd 100644 --- a/api/layer/system_object.go +++ b/api/layer/system_object.go @@ -21,12 +21,6 @@ import ( oid "github.com/nspcc-dev/neofs-sdk-go/object/id" ) -const ( - AttributeComplianceMode = ".s3-compliance-mode" - AttributeRetentionUntilMode = ".s3-retention-until" - AttributeObjectVersion = ".s3-object-version" -) - type PutLockInfoParams struct { ObjVersion *ObjectVersion NewLock *data.ObjectLock @@ -116,8 +110,8 @@ func (n *layer) getLockDataFromObjects(ctx context.Context, bkt *data.BucketInfo object.FilterCreationEpoch, object.AttributeTimestamp, object.AttributeExpirationEpoch, - AttributeComplianceMode, - AttributeRetentionUntilMode, + s3headers.AttributeComplianceMode, + s3headers.AttributeRetentionUntilMode, } opts client.SearchObjectsOptions @@ -131,7 +125,7 @@ func (n *layer) getLockDataFromObjects(ctx context.Context, bkt *data.BucketInfo filters.AddFilter(s3headers.MetaType, s3headers.TypeLock, object.MatchStringEqual) filters.AddTypeFilter(object.MatchStringEqual, object.TypeLock) if version != "" { - filters.AddFilter(AttributeObjectVersion, version, object.MatchStringEqual) + filters.AddFilter(s3headers.AttributeObjectVersion, version, object.MatchStringEqual) } searchResultItems, err := n.neoFS.SearchObjectsV2(ctx, bkt.CID, filters, returningAttributes, opts) @@ -238,8 +232,8 @@ func (n *layer) putLockObject(ctx context.Context, bktInfo *data.BucketInfo, obj } if objectVersion != "" { - prm.Attributes[AttributeObjectVersion] = objectVersion - prm.Attributes[attrS3VersioningState] = data.VersioningEnabled + prm.Attributes[s3headers.AttributeObjectVersion] = objectVersion + prm.Attributes[s3headers.AttributeVersioningState] = data.VersioningEnabled } prm.Attributes[s3headers.MetaType] = s3headers.TypeLock @@ -361,10 +355,10 @@ func (n *layer) attributesFromLock(ctx context.Context, lock *data.ObjectLock) ( return nil, fmt.Errorf("fetch time to epoch: %w", err) } - result[AttributeRetentionUntilMode] = lock.Retention.Until.UTC().Format(time.RFC3339) + result[s3headers.AttributeRetentionUntilMode] = lock.Retention.Until.UTC().Format(time.RFC3339) if lock.Retention.IsCompliance { - result[AttributeComplianceMode] = "true" + result[s3headers.AttributeComplianceMode] = "true" } } diff --git a/api/layer/tagging.go b/api/layer/tagging.go index 51b52487..bbcc26e6 100644 --- a/api/layer/tagging.go +++ b/api/layer/tagging.go @@ -56,8 +56,8 @@ func (n *layer) PutObjectTagging(ctx context.Context, p *PutObjectTaggingParams) } if p.ObjectVersion.VersionID != "" { - prm.Attributes[AttributeObjectVersion] = p.ObjectVersion.VersionID - prm.Attributes[attrS3VersioningState] = data.VersioningEnabled + prm.Attributes[s3headers.AttributeObjectVersion] = p.ObjectVersion.VersionID + prm.Attributes[s3headers.AttributeVersioningState] = data.VersioningEnabled } prm.Attributes[s3headers.MetaType] = s3headers.TypeTags @@ -100,7 +100,7 @@ func (n *layer) GetObjectTagging(ctx context.Context, p *GetObjectTaggingParams) filters.AddFilter(s3headers.MetaType, s3headers.TypeTags, object.MatchStringEqual) filters.AddTypeFilter(object.MatchStringEqual, object.TypeRegular) if p.ObjectVersion.VersionID != "" { - filters.AddFilter(AttributeObjectVersion, p.ObjectVersion.VersionID, object.MatchStringEqual) + filters.AddFilter(s3headers.AttributeObjectVersion, p.ObjectVersion.VersionID, object.MatchStringEqual) } searchResultItems, err := n.neoFS.SearchObjectsV2(ctx, p.ObjectVersion.BktInfo.CID, filters, returningAttributes, opts) @@ -195,7 +195,7 @@ func (n *layer) DeleteObjectTagging(ctx context.Context, p *ObjectVersion) error fs.AddFilter(s3headers.MetaType, s3headers.TypeTags, object.MatchStringEqual) fs.AddTypeFilter(object.MatchStringEqual, object.TypeRegular) if p.VersionID != "" { - fs.AddFilter(AttributeObjectVersion, p.VersionID, object.MatchStringEqual) + fs.AddFilter(s3headers.AttributeObjectVersion, p.VersionID, object.MatchStringEqual) } var opts client.SearchObjectsOptions diff --git a/api/layer/util.go b/api/layer/util.go index 5a5d98f6..edd94dd5 100644 --- a/api/layer/util.go +++ b/api/layer/util.go @@ -10,6 +10,7 @@ import ( "github.com/nspcc-dev/neofs-s3-gw/api" "github.com/nspcc-dev/neofs-s3-gw/api/data" "github.com/nspcc-dev/neofs-s3-gw/api/layer/encryption" + "github.com/nspcc-dev/neofs-s3-gw/api/s3headers" "github.com/nspcc-dev/neofs-s3-gw/creds/accessbox" "github.com/nspcc-dev/neofs-sdk-go/object" ) @@ -64,7 +65,7 @@ func extractHeaders(headers map[string]string) (map[string]string, string, time. ) delete(headers, object.AttributeFilePath) - delete(headers, objectNonceAttribute) + delete(headers, s3headers.AttributeObjectNonce) if contentType, ok := headers[object.AttributeContentType]; ok { mimeType = contentType @@ -89,7 +90,7 @@ func objectInfoFromMeta(bkt *data.BucketInfo, meta *object.Object) *data.ObjectI payloadChecksum, _ := meta.PayloadChecksum() var versionID = data.UnversionedObjectVersionID - if _, ok := attributes[attrS3VersioningState]; ok { + if _, ok := attributes[s3headers.AttributeVersioningState]; ok { versionID = objID.EncodeToString() } @@ -112,23 +113,23 @@ func objectInfoFromMeta(bkt *data.BucketInfo, meta *object.Object) *data.ObjectI } func FormEncryptionInfo(headers map[string]string) encryption.ObjectEncryption { - algorithm := headers[AttributeEncryptionAlgorithm] + algorithm := headers[s3headers.AttributeEncryptionAlgorithm] return encryption.ObjectEncryption{ Enabled: len(algorithm) > 0, Algorithm: algorithm, - HMACKey: headers[AttributeHMACKey], - HMACSalt: headers[AttributeHMACSalt], + HMACKey: headers[s3headers.AttributeHMACKey], + HMACSalt: headers[s3headers.AttributeHMACSalt], } } func addEncryptionHeaders(meta map[string]string, enc encryption.Params) error { - meta[AttributeEncryptionAlgorithm] = AESEncryptionAlgorithm + meta[s3headers.AttributeEncryptionAlgorithm] = AESEncryptionAlgorithm hmacKey, hmacSalt, err := enc.HMAC() if err != nil { return fmt.Errorf("get hmac: %w", err) } - meta[AttributeHMACKey] = hex.EncodeToString(hmacKey) - meta[AttributeHMACSalt] = hex.EncodeToString(hmacSalt) + meta[s3headers.AttributeHMACKey] = hex.EncodeToString(hmacKey) + meta[s3headers.AttributeHMACSalt] = hex.EncodeToString(hmacSalt) return nil } diff --git a/api/s3headers/headers.go b/api/s3headers/headers.go index 08d81b8d..8509ae7f 100644 --- a/api/s3headers/headers.go +++ b/api/s3headers/headers.go @@ -57,4 +57,19 @@ const ( const ( // BucketSettingsVersioning contains versioning setting for bucket. BucketSettingsVersioning = "s3bsVersioning" + + AttributeComplianceMode = ".s3-compliance-mode" + AttributeRetentionUntilMode = ".s3-retention-until" + AttributeObjectVersion = ".s3-object-version" + AttributeObjectNonce = "__NEOFS__NONCE" + + NeoFSSystemMetadataPrefix = "S3-" + + AttributeEncryptionAlgorithm = NeoFSSystemMetadataPrefix + "Algorithm" + AttributeDecryptedSize = NeoFSSystemMetadataPrefix + "Decrypted-Size" + AttributeHMACSalt = NeoFSSystemMetadataPrefix + "HMAC-Salt" + AttributeHMACKey = NeoFSSystemMetadataPrefix + "HMAC-Key" + AttributeVersioningState = NeoFSSystemMetadataPrefix + "versioning-state" + AttributeDeleteMarker = NeoFSSystemMetadataPrefix + "delete-marker" + UploadCompletedParts = NeoFSSystemMetadataPrefix + "Completed-Parts" ) diff --git a/docs/attribute_mapping.md b/docs/attribute_mapping.md new file mode 100644 index 00000000..12faebf2 --- /dev/null +++ b/docs/attribute_mapping.md @@ -0,0 +1,292 @@ +# Attributes + +Each uploaded object includes a set of attributes. + +## Common Attributes + +### `S3-versioning-state` + +If bucket versioning is enabled, each uploaded object will have the attribute `S3-versioning-state: Enabled`. +> It means storing multiple versions of an object within the same bucket. Each version can be retrieved from the +> versioning container. If you attempt to retrieve an object by name without specifying a version, +> the latest uploaded version will be returned. + +If bucket versioning is disabled, this attribute will not be present. +> It means each object has only one visible version. When retrieving an object by name, the latest uploaded version +> is returned. In fact, every object uploaded with the same name is preserved in NeoFS, but only the most recent version +> is accessible to the user. + +**Possible values:** + +- `Enabled` + +### `__NEOFS__NONCE` + +In NeoFS, the `ObjectID` is calculated as a `SHA256` hash of the marshalled object header. The object's creation time +is stored in the `Timestamp` attribute within the header. This means that two objects with identical attributes, +created within the same second, will result in the same `ObjectID`. To ensure uniqueness, each object has +`__NEOFS__NONCE` +header, which makes each object distinct. + +### `Timestamp` + +A NeoFS SDK attribute containing the UNIX timestamp of object creation. + +### `FilePath` + +A NeoFS SDK attribute representing the object name. This is used by end-users to interact with the gateway. + +### `S3-delete-marker` + +Contains the server timestamp of object removal for versioned container. It is used for special object called +`delete marker`. +A delete marker is a marker for a versioned object that was specified in a simple DELETE request. +GET request for a delete marker doesn't retrieve anything because a delete marker has no data. +GET request doesn't specify a `versionId` get a 404 (Not Found) error. + +More details in the [AWS Documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/DeleteMarker.html). + +--- + +## Multipart Uploads + +### Multipart Info Object + +This object is required for listing multipart uploads. [AWS Documentation](https://docs.aws.amazon.com/cli/latest/reference/s3api/list-multipart-uploads.html) + +Includes the following attributes: + +- `s3MetaMultipartType: info` +- `s3mpObjectKey` +- `s3mpUpload` +- `s3mpMeta`, contains base64 encoded map with: `s3mpObjectKey`, `s3mpOwner`, `s3mpCreated`, `s3mpCopiesNumber` + +### Multipart Part Object + +This object is required for listing part for multipart upload. [AWS Documentation](https://docs.aws.amazon.com/cli/latest/reference/s3api/list-multipart-uploads.html) +It also required to complete multipart upload. + +Includes the following attributes: + +- `s3MetaMultipartType: part` +- `s3mpPartNumber` +- `s3mpElementId` +- `s3mpTotalSize` +- `s3mpHash` +- `s3mpHomoHash` +- `s3mpIsArbitrary` +- `s3mpUpload` + +--- + +The following attributes are used in the context of multipart upload logic. + +### `s3MetaMultipartType` + +Indicates the type of object within a multipart upload. + +**Possible values:** + +- `info` – Stores multipart upload metadata +- `part` – Stores information about an individual part + +### `s3mpPartNumber` + +Stores the part number for multipart uploads. +Note: Per [AWS documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/mpuoverview.html), the first valid +part number is `1`. +An internal part `0` exist but is hidden from users. + +### `s3mpElementId` + +Since each part can be up to 5 GB (and NeoFS atomic max object size smaller), this attribute tracks internal slices that +form a full part. + +### `s3mpTotalSize` + +Describes the total size of all sliced elements that comprise a part. + +### `s3mpHash` + +Stores the intermediate hash state used to compute the final object hash. + +### `s3mpHomoHash` + +Stores the intermediate state for computing a homomorphic hash, if enabled. + +### `s3mpIsArbitrary` + +Indicates whether a part was uploaded out of sequence. + +**Possible values:** + +- `true` + +**Example:** + +- Upload part 1 +- Upload part 3 → This will be marked with `s3mpIsArbitrary: true` because part 2 is missing. + +### `s3mpObjectKey` + +Represents the object key for the multipart upload. +This is stored separately from the `object.AttributeFilePath` attribute and ensures that multipart-uploaded object +remain inaccessible +to users until `CompleteMultipartUpload` is called. + +### `s3mpUpload` + +Stores the multipart upload ID. + +### `s3mpPartHash` + +Stores the hash of the entire part (including slices). + +### `s3mpOwner` + +Stores the owner of the uploading object. + +### `s3mpCopiesNumber` + +Indicates the CopiesNumber setting for the object. + +### `s3mpMeta` + +Contains the original object's attributes. + +### `s3mpCreated` + +Contains the final creation timestamp of the object. + +### `s3bsVersioning` + +Indicates whether bucket versioning is enabled. + +### `.s3-object-version` + +If bucket versioning is enabled, this attribute indicates which version the object belongs to. + +### `.s3-compliance-mode` + +Indicates whether the object is under a compliance-mode retention lock. + +**Possible values:** + +- `true` + +### `.s3-retention-until` + +Contains the retention expiration timestamp in `time.RFC3339` format. + +### S3-Algorithm + +Contains encryption algorith for object payload, if encryption is enabled. + +### S3-Decrypted-Size + +Contains decrypted size for object payload, if encryption is enabled. + +### S3-HMAC-Salt + +Contains salt for encryption algorith, if encryption is enabled. + +### S3-HMAC-Key + +Contains key for encryption algorith, if encryption is enabled. + +### S3-Completed-Parts + +Contains info about all multipart parts for object. It is stored in the final object attributes. + +--- + +### `s3MetaType` + +Describes special metadata types used for S3 features. + +**Possible values:** + +- `lock` – Stores object lock info +- `tags` – Stores object tags +- `bucketTags` – Stores bucket tags +- `bucketNotifConf` – Stores bucket notification configuration +- `bucketCORS` – Stores bucket CORS configuration +- `bucketSettings` – Stores bucket settings + +--- + +## Special Meta Objects + +These objects are used to implement specific S3 features. + +### Lock Object + +Stores lock data for an object. + +**Attributes:** + +- `s3MetaType: lock` +- `.s3-object-version` +- `S3-versioning-state` + +### Tags Object + +Stores tag data for an object. + +**Attributes:** + +- `s3MetaType: tags` +- `.s3-object-version` +- `S3-versioning-state` + +### Bucket Tags Object + +Stores tag data for a bucket. + +**Attributes:** + +- `s3MetaType: bucketTags` + +### Bucket Notifications Object + +Stores bucket notification configuration. + +**Attributes:** + +- `s3MetaType: bucketNotifConf` + +The object payload contains the configuration data. + +[Format](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-s3-bucket-notificationconfiguration.html) +> Note `QueueConfigurations` only supported. + +### Bucket CORS Object + +Stores bucket CORS configuration. + +**Attributes:** + +- `s3MetaType: bucketCORS` + +The object payload contains the configuration data in XML format. +[Format](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ManageCorsUsing.html) + +### Bucket Settings Object + +Stores settings configuration for a bucket. + +**Attributes:** + +- `s3MetaType: bucketSettings` +- `s3bsVersioning` + +The object payload contains the settings data. + +It canbe empty string or `%s,%d,%s,%d` string. + +* `string`. Is object lock enabled. Possible values: `` (empty string), `Enabled`. +* `int64`. Retention days for bucket objects. +* `string`. Retention mode. Possible values: `GOVERNANCE`, `COMPLIANCE`. +* `int64`. Retention years for bucket objects. + +`Days` and `Years` represent durations and cannot be used simultaneously. Only one of them can be specified at a time. diff --git a/internal/neofs/neofs.go b/internal/neofs/neofs.go index b964bfa4..5c94e75e 100644 --- a/internal/neofs/neofs.go +++ b/internal/neofs/neofs.go @@ -19,6 +19,7 @@ import ( "github.com/nspcc-dev/neofs-s3-gw/api" "github.com/nspcc-dev/neofs-s3-gw/api/layer" + "github.com/nspcc-dev/neofs-s3-gw/api/s3headers" "github.com/nspcc-dev/neofs-s3-gw/authmate" "github.com/nspcc-dev/neofs-s3-gw/creds/tokens" "github.com/nspcc-dev/neofs-sdk-go/checksum" @@ -61,8 +62,7 @@ type NeoFS struct { } const ( - objectNonceSize = 8 - objectNonceAttribute = "__NEOFS__NONCE" + objectNonceSize = 8 containerMetaDataPolicyAttribute = "__NEOFS__METAINFO_CONSISTENCY" ContainerMetaDataPolicyStrict = "strict" @@ -289,7 +289,7 @@ func (x *NeoFS) CreateObject(ctx context.Context, prm layer.PrmObjectCreate) (oi } uniqAttributes[object.AttributeTimestamp] = strconv.FormatInt(creationTime.Unix(), 10) - uniqAttributes[objectNonceAttribute] = base64.StdEncoding.EncodeToString(nonce) + uniqAttributes[s3headers.AttributeObjectNonce] = base64.StdEncoding.EncodeToString(nonce) if prm.Filepath != "" { uniqAttributes[object.AttributeFilePath] = prm.Filepath diff --git a/internal/neofs/neofs_test.go b/internal/neofs/neofs_test.go index 59ccbe94..31264f07 100644 --- a/internal/neofs/neofs_test.go +++ b/internal/neofs/neofs_test.go @@ -15,6 +15,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neofs-s3-gw/api/layer" + "github.com/nspcc-dev/neofs-s3-gw/api/s3headers" "github.com/nspcc-dev/neofs-sdk-go/client" apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status" "github.com/nspcc-dev/neofs-sdk-go/container" @@ -265,7 +266,7 @@ func TestObjectNonce(t *testing.T) { obj.SetPayload(payload) var ( - attr = object.NewAttribute(objectNonceAttribute, base64.StdEncoding.EncodeToString(nonce)) + attr = object.NewAttribute(s3headers.AttributeObjectNonce, base64.StdEncoding.EncodeToString(nonce)) attrs = []object.Attribute{*attrTS, *attr} )