Skip to content

Optimize attributes #1146

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
May 5, 2025
Merged

Optimize attributes #1146

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
47c5b67
s3headers: Remove MetaMultipartType attribute
smallhive May 5, 2025
c957d90
s3headers: Rename meta version attribute
smallhive May 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 13 additions & 13 deletions api/layer/multipart_upload.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -467,7 +467,7 @@ func (n *layer) uploadZeroPart(ctx context.Context, multipartInfo *data.Multipar
return nil, fmt.Errorf("marshalBinary: %w", err)
}

attributes[s3headers.MetaMultipartType] = s3headers.TypeMultipartPart
attributes[s3headers.MetaType] = s3headers.TypeMultipartPart
attributes[s3headers.MultipartPartNumber] = "0"
attributes[s3headers.MultipartElementID] = "0"
attributes[s3headers.MultipartTotalSize] = "0"
Expand Down Expand Up @@ -581,7 +581,7 @@ func (n *layer) multipartMetaGetPartByNumber(ctx context.Context, bktInfo *data.

filters.AddFilter(s3headers.MultipartUpload, uploadID, object.MatchStringEqual)
filters.AddFilter(s3headers.MultipartPartNumber, strconv.Itoa(number), object.MatchStringEqual)
filters.AddFilter(s3headers.MetaMultipartType, s3headers.TypeMultipartPart, object.MatchStringEqual)
filters.AddFilter(s3headers.MetaType, s3headers.TypeMultipartPart, object.MatchStringEqual)

if bt := bearerTokenFromContext(ctx, bktInfo.Owner); bt != nil {
opts.WithBearerToken(*bt)
Expand Down Expand Up @@ -704,7 +704,7 @@ func (n *layer) createMultipartInfoObject(ctx context.Context, bktInfo *data.Buc

attributes[s3headers.MultipartUpload] = info.UploadID
attributes[s3headers.MultipartObjectKey] = info.Key
attributes[s3headers.MetaMultipartType] = s3headers.TypeMultipartInfo
attributes[s3headers.MetaType] = s3headers.TypeMultipartInfo

prm := PrmObjectCreate{
Container: bktInfo.CID,
Expand Down Expand Up @@ -737,7 +737,7 @@ func (n *layer) getMultipartInfoObject(ctx context.Context, bktInfo *data.Bucket

filters.AddFilter(s3headers.MultipartUpload, uploadID, object.MatchStringEqual)
filters.AddFilter(s3headers.MultipartObjectKey, objectName, object.MatchStringEqual)
filters.AddFilter(s3headers.MetaMultipartType, s3headers.TypeMultipartInfo, object.MatchStringEqual)
filters.AddFilter(s3headers.MetaType, s3headers.TypeMultipartInfo, object.MatchStringEqual)

if bt := bearerTokenFromContext(ctx, bktInfo.Owner); bt != nil {
opts.WithBearerToken(*bt)
Expand Down Expand Up @@ -778,7 +778,7 @@ func (n *layer) getMultipartInfoByPrefix(ctx context.Context, bktInfo *data.Buck
opts client.SearchObjectsOptions

returningAttributes = []string{
s3headers.MetaMultipartType,
s3headers.MetaType,
object.AttributeTimestamp,
s3headers.MultipartUpload,
}
Expand All @@ -794,7 +794,7 @@ func (n *layer) getMultipartInfoByPrefix(ctx context.Context, bktInfo *data.Buck

opts.SetCount(uint32(maxKeys))

filters.AddFilter(s3headers.MetaMultipartType, s3headers.TypeMultipartInfo, object.MatchStringEqual)
filters.AddFilter(s3headers.MetaType, s3headers.TypeMultipartInfo, object.MatchStringEqual)
filters.AddFilter(s3headers.MultipartObjectKey, prefix, object.MatchCommonPrefix)

res, nextCursor, err := n.neoFS.SearchObjectsV2WithCursor(ctx, bktInfo.CID, filters, returningAttributes, cursor, opts)
Expand Down Expand Up @@ -892,7 +892,7 @@ func (n *layer) multipartMetaGetParts(ctx context.Context, bktInfo *data.BucketI
)

filters.AddFilter(s3headers.MultipartUpload, uploadID, object.MatchStringEqual)
filters.AddFilter(s3headers.MetaMultipartType, s3headers.TypeMultipartPart, object.MatchStringEqual)
filters.AddFilter(s3headers.MetaType, s3headers.TypeMultipartPart, object.MatchStringEqual)

if bt := bearerTokenFromContext(ctx, bktInfo.Owner); bt != nil {
opts.WithBearerToken(*bt)
Expand Down Expand Up @@ -987,7 +987,7 @@ func (n *layer) multipartMetaGetOIDsForAbort(ctx context.Context, bktInfo *data.
)

filters.AddFilter(s3headers.MultipartUpload, uploadID, object.MatchStringEqual)
filters.AddFilter(s3headers.MetaMultipartType, s3headers.TypeMultipartPart, object.MatchStringEqual)
filters.AddFilter(s3headers.MetaType, s3headers.TypeMultipartPart, object.MatchStringEqual)

if bt := bearerTokenFromContext(ctx, bktInfo.Owner); bt != nil {
opts.WithBearerToken(*bt)
Expand Down Expand Up @@ -1032,7 +1032,7 @@ func (n *layer) multipartGetPartsList(ctx context.Context, bktInfo *data.BucketI
)

filters.AddFilter(s3headers.MultipartUpload, uploadID, object.MatchStringEqual)
filters.AddFilter(s3headers.MetaMultipartType, s3headers.TypeMultipartPart, object.MatchStringEqual)
filters.AddFilter(s3headers.MetaType, s3headers.TypeMultipartPart, object.MatchStringEqual)

if bt := bearerTokenFromContext(ctx, bktInfo.Owner); bt != nil {
opts.WithBearerToken(*bt)
Expand Down Expand Up @@ -1118,7 +1118,7 @@ func (n *layer) multipartMetaGetPartsAfter(ctx context.Context, bktInfo *data.Bu
)

filters.AddFilter(s3headers.MultipartUpload, uploadID, object.MatchStringEqual)
filters.AddFilter(s3headers.MetaMultipartType, s3headers.TypeMultipartPart, object.MatchStringEqual)
filters.AddFilter(s3headers.MetaType, s3headers.TypeMultipartPart, object.MatchStringEqual)
filters.AddFilter(s3headers.MultipartPartNumber, strconv.Itoa(partID), object.MatchNumGT)

if bt := bearerTokenFromContext(ctx, bktInfo.Owner); bt != nil {
Expand Down Expand Up @@ -1888,7 +1888,7 @@ func (n *layer) manualSlice(ctx context.Context, bktInfo *data.BucketInfo, prm P
stateBytes []byte
)

prm.Attributes[s3headers.MetaMultipartType] = s3headers.TypeMultipartPart
prm.Attributes[s3headers.MetaType] = s3headers.TypeMultipartPart
prm.Attributes[s3headers.MultipartUpload] = p.Info.UploadID
prm.Attributes[s3headers.MultipartPartNumber] = strconv.Itoa(p.PartNumber)

Expand Down Expand Up @@ -2009,7 +2009,7 @@ func (n *layer) uploadPartAsSlot(ctx context.Context, params uploadPartAsSlotPar

params.attributes[s3headers.MultipartUpload] = params.multipartInfo.UploadID
params.attributes[s3headers.MultipartPartNumber] = strconv.FormatInt(int64(params.uploadPartParams.PartNumber), 10)
params.attributes[s3headers.MetaMultipartType] = s3headers.TypeMultipartPart
params.attributes[s3headers.MetaType] = s3headers.TypeMultipartPart
params.attributes[s3headers.MultipartIsArbitraryPart] = "true"
params.attributes[s3headers.MultipartElementID] = "0"
params.attributes[s3headers.MultipartTotalSize] = strconv.FormatInt(params.decSize, 10)
Expand Down Expand Up @@ -2056,7 +2056,7 @@ func (n *layer) uploadPartAsSlot(ctx context.Context, params uploadPartAsSlotPar
func (n *layer) getFirstArbitraryPart(ctx context.Context, uploadID string, bucketInfo *data.BucketInfo) (int64, error) {
var filters object.SearchFilters
filters.AddFilter(s3headers.MultipartUpload, uploadID, object.MatchStringEqual)
filters.AddFilter(s3headers.MetaMultipartType, s3headers.TypeMultipartPart, object.MatchStringEqual)
filters.AddFilter(s3headers.MetaType, s3headers.TypeMultipartPart, object.MatchStringEqual)
filters.AddFilter(s3headers.MultipartIsArbitraryPart, "true", object.MatchStringEqual)

var opts client.SearchObjectsOptions
Expand Down
6 changes: 3 additions & 3 deletions api/layer/system_object.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -320,7 +320,7 @@ func decodeBucketSettings(settingsObj *object.Object) (*data.BucketSettings, err

for _, attr := range settingsObj.Attributes() {
switch attr.Key() {
case s3headers.BucketSettingsObjectVersion:
case s3headers.BucketSettingsMetaVersion:
settingsVersion = attr.Value()
case s3headers.BucketSettingsVersioning:
versioning = attr.Value()
Expand Down Expand Up @@ -360,8 +360,8 @@ func (n *layer) PutBucketSettings(ctx context.Context, p *PutSettingsParams) err
CreationTime: TimeNow(ctx),
CopiesNumber: p.CopiesNumber,
Attributes: map[string]string{
s3headers.MetaType: s3headers.TypeBucketSettings,
s3headers.BucketSettingsObjectVersion: data.BucketSettingsV1,
s3headers.MetaType: s3headers.TypeBucketSettings,
s3headers.BucketSettingsMetaVersion: data.BucketSettingsV1,
},
Payload: bytes.NewReader(payload),
PayloadSize: uint64(len(payload)),
Expand Down
15 changes: 4 additions & 11 deletions api/s3headers/headers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,15 +52,8 @@ const (
TypeBucketNotifConfig = "bucketNotifConf"
TypeBucketCORS = "bucketCORS"
TypeBucketSettings = "bucketSettings"
)

const (
// MetaMultipartType is a header name to identify multipart meta for objects.
// Result: S3-MP-Type.
MetaMultipartType = multipartPrefix + "Type"

TypeMultipartInfo = "info"
TypeMultipartPart = "part"
TypeMultipartInfo = "multipartInfo"
TypeMultipartPart = "multipartPart"
)

const (
Expand All @@ -70,8 +63,8 @@ const (
BucketSettingsVersioning = bucketSettingsPrefix + "Versioning"
AttributeComplianceMode = bucketSettingsPrefix + "ComplianceMode"
AttributeRetentionUntilMode = bucketSettingsPrefix + "RetentionUntil"
// BucketSettingsVersioning contains version of bucket settings file.
BucketSettingsObjectVersion = bucketSettingsPrefix + "ObjVersion"
// BucketSettingsMetaVersion contains version of bucket settings file.
BucketSettingsMetaVersion = bucketSettingsPrefix + "MetaVersion"

AttributeObjectVersion = attributePrefix + "ObjectVersion"
AttributeObjectNonce = "__NEOFS__NONCE"
Expand Down
17 changes: 5 additions & 12 deletions docs/attribute_mapping.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ uploads. [AWS Documentation](https://docs.aws.amazon.com/cli/latest/reference/s3

Includes the following attributes:

- `S3-MP-Type: info`
- `S3-MetaType: multipartInfo`
- `S3-MP-ObjectKey`
- `S3-MP-Upload`
- `S3-MP-Meta`, contains base64 encoded map with: `S3-MP-ObjectKey`, `S3-MP-Owner`, `S3-MP-Created`,
Expand All @@ -71,7 +71,7 @@ It also required to complete multipart upload.

Includes the following attributes:

- `S3-MP-Type: part`
- `S3-MetaType: multipartPart`
- `S3-MP-PartNumber`
- `S3-MP-ElementId`
- `S3-MP-TotalSize`
Expand All @@ -84,15 +84,6 @@ Includes the following attributes:

The following attributes are used in the context of multipart upload logic.

### `S3-MP-Type`

Indicates the type of object within a multipart upload.

**Possible values:**

- `info` – Stores multipart upload metadata
- `part` – Stores information about an individual part

### `S3-MP-PartNumber`

Stores the part number for multipart uploads.
Expand Down Expand Up @@ -165,7 +156,7 @@ Contains the final creation timestamp of the object.

Indicates whether bucket versioning is enabled.

### `S3-BucketSettings-ObjVersion`
### `S3-BucketSettings-MetaVersion`

Specifies the version of the bucket settings object's file structure

Expand Down Expand Up @@ -219,6 +210,8 @@ Describes special metadata types used for S3 features.
- `bucketNotifConf` – Stores bucket notification configuration
- `bucketCORS` – Stores bucket CORS configuration
- `bucketSettings` – Stores bucket settings
- `multipartInfo` – Stores multipart upload metadata
- `multipartPart` – Stores information about an individual part

---

Expand Down
Loading