diff --git a/.gitignore b/.gitignore index 711a08df..b9f86bdf 100644 --- a/.gitignore +++ b/.gitignore @@ -2,9 +2,6 @@ .idea .vscode -# Tree service -internal/neofs/services/tree/ - # Vendoring vendor diff --git a/README.md b/README.md index e9accfd6..d40fa89d 100644 --- a/README.md +++ b/README.md @@ -45,9 +45,6 @@ $ S3_GW_PEERS_0_ADDRESS=grpcs://192.168.130.72:8080 \ neofs-s3-gw ``` -Notice that currently S3 gateway can't be used for public networks like mainnet -or testnet because of experimental tree service extension that is required for it. - ## Domains By default, s3-gw enable only `path-style access`. @@ -79,7 +76,6 @@ Also, you can configure domains using `.env` variables or `yaml` file. - [Configuration](./docs/configuration.md) - [NeoFS S3 AuthMate](./docs/authmate.md) -- [NeoFS Tree service](./docs/tree_service.md) - [AWS CLI basic usage](./docs/aws_cli.md) - [AWS S3 API compatibility](./docs/aws_s3_compat.md) - [AWS S3 Compatibility test results](./docs/s3_test_results.md) diff --git a/api/data/tree.go b/api/data/meta.go similarity index 96% rename from api/data/tree.go rename to api/data/meta.go index 5f219e29..786d800b 100644 --- a/api/data/tree.go +++ b/api/data/meta.go @@ -13,7 +13,7 @@ const ( UnversionedObjectVersionID = "null" ) -// NodeVersion represent node from tree service. +// NodeVersion represent basic object metadata. type NodeVersion struct { OID oid.ID Timestamp uint64 @@ -93,7 +93,7 @@ func SortPartInfo(a, b *PartInfo) int { return cmp.Compare(a.Number, b.Number) } -// LockInfo is lock information to create appropriate tree node. +// LockInfo is lock information for a particular object. type LockInfo struct { legalHoldOID oid.ID setLegalHold bool diff --git a/api/layer/layer.go b/api/layer/layer.go index 588ec6e6..ab1ab7f5 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -270,7 +270,7 @@ type ( // Compound methods for optimizations - // GetObjectTaggingAndLock unifies GetObjectTagging and GetLock methods in single tree service invocation. + // GetObjectTaggingAndLock unifies GetObjectTagging and GetLock methods in a single search invocation. GetObjectTaggingAndLock(ctx context.Context, p *ObjectVersion) (map[string]string, *data.LockInfo, error) } ) @@ -288,10 +288,10 @@ var ( ) var ( - // ErrNodeNotFound is returned from Tree service in case of not found error. + // ErrNodeNotFound is returned in case of not found error. ErrNodeNotFound = errors.New("not found") - // ErrNodeAccessDenied is returned from Tree service in case of access denied error. + // ErrNodeAccessDenied is returned in case of access denied error. ErrNodeAccessDenied = errors.New("access denied") // ErrPartListIsEmpty is returned if no parts available for the upload. @@ -564,7 +564,7 @@ func (n *layer) GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.O return extendedObjectInfo.ObjectInfo, nil } -// GetExtendedObjectInfo returns meta information and corresponding info from the tree service about the object. +// GetExtendedObjectInfo returns meta information and corresponding info about the object. func (n *layer) GetExtendedObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.ExtendedObjectInfo, error) { var ( id oid.ID diff --git a/api/layer/multipart_upload.go b/api/layer/multipart_upload.go index 1313c185..05199344 100644 --- a/api/layer/multipart_upload.go +++ b/api/layer/multipart_upload.go @@ -459,7 +459,7 @@ func (n *layer) uploadZeroPart(ctx context.Context, multipartInfo *data.Multipar currentVersion := version.Current() hashlessHeaderObject.SetVersion(¤tVersion) - // encoding hash.Hash state to save it in tree service. + // encoding hash.Hash state to save it in the object metadata. // the required interface is guaranteed according to the docs, so just cast without checks. binaryMarshaler := multipartHash.(encoding.BinaryMarshaler) stateBytes, err := binaryMarshaler.MarshalBinary() @@ -1985,7 +1985,7 @@ func (n *layer) uploadPartAsSlot(ctx context.Context, params uploadPartAsSlotPar mpHashBytes, homoHashBytes []byte ) - // encoding hash.Hash state to save it in tree service. + // encoding hash.Hash state to save it the object metadata. // the required interface is guaranteed according to the docs, so just cast without checks. binaryMarshaler := multipartHash.(encoding.BinaryMarshaler) mpHashBytes, err := binaryMarshaler.MarshalBinary() diff --git a/docs/aws_s3_compat.md b/docs/aws_s3_compat.md index e2286520..f0234235 100644 --- a/docs/aws_s3_compat.md +++ b/docs/aws_s3_compat.md @@ -39,7 +39,7 @@ Reference: ## ACL -For now there are some limitations: +There are some limitations: * [Bucket policy](https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucket-policies.html) supports only one `Principal` per `Statement`. Principal must be `"AWS": "*"` or `"*"` (to refer all users) or `"CanonicalUser": "NiskPF9pfRMzg7V7PeB4d6ogLzu74a1L2Q"` (base58 encoded address of desired user). ```json @@ -114,6 +114,9 @@ Principal must be `"AWS": "*"` or `"*"` (to refer all users) or `"CanonicalUser" "Grants": [] } ``` +Given that ACLs are disabled by default now and users are expected to use +alternative mechanisms to manage access improving ACL support is not +a priority. | | Method | Comments | |----|--------------|-----------------| @@ -137,8 +140,6 @@ For now there are some limitations: ## Multipart -Should be supported soon. - | | Method | Comments | |----|-------------------------|----------| | 🟢 | AbortMultipartUpload | | @@ -164,7 +165,7 @@ See also `GetObject` and other method parameters. | | Method | Comments | |----|--------------------|--------------------------| | 🟢 | ListObjectVersions | ListBucketObjectVersions | -| 🔵 | RestoreObject | | +| 🔵 | RestoreObject | Related to lifecycles and tiering. | ## Bucket @@ -176,7 +177,7 @@ See also `GetObject` and other method parameters. | 🟢 | HeadBucket | | | 🟢 | ListBuckets | | | 🔵 | PutPublicAccessBlock | | -| 🔵 | GetPublicAccessBlock | | +| 🔵 | GetPublicAccessBlock | Related to ACL management, not a priority. | * `CreateBucket` method allows you to select a placement policy using the `LocationConstraint` parameter in the AWS CLI. The policy name should be passed as a value. * Policies mapping can be defined via: @@ -203,6 +204,8 @@ See [Ownership](./aws_s3_compat.md#ownership-controls) section for details. ## Analytics +Requires a separate entity to generate/upload data daily. + | | Method | Comments | |----|------------------------------------|----------| | 🔵 | DeleteBucketAnalyticsConfiguration | | @@ -220,6 +223,8 @@ See [Ownership](./aws_s3_compat.md#ownership-controls) section for details. ## Encryption +Methods below are related to AWS SSE-S3 and SSE-KMS encryption. S3 gateway supports SSE-C only for now. + | | Method | Comments | |----|------------------------|----------| | 🔵 | DeleteBucketEncryption | | @@ -228,6 +233,8 @@ See [Ownership](./aws_s3_compat.md#ownership-controls) section for details. ## Inventory +Requires a separate entity to generate/upload data regularly. + | | Method | Comments | |----|------------------------------------|----------| | 🔵 | DeleteBucketInventoryConfiguration | | @@ -237,6 +244,9 @@ See [Ownership](./aws_s3_compat.md#ownership-controls) section for details. ## Lifecycle +In general requires a separate entity to manage uploaded objects. Without it +can be only partially supported. + | | Method | Comments | |----|---------------------------------|------------------------------------------------------| | 🔵 | DeleteBucketLifecycle | | @@ -247,6 +257,8 @@ See [Ownership](./aws_s3_compat.md#ownership-controls) section for details. ## Logging +Doesn't make much sense in non-AWS case. + | | Method | Comments | |----|------------------|----------| | 🔵 | GetBucketLogging | | @@ -254,6 +266,8 @@ See [Ownership](./aws_s3_compat.md#ownership-controls) section for details. ## Metrics +Doesn't make much sense in non-AWS case. + | | Method | Comments | |----|----------------------------------|----------| | 🔵 | DeleteBucketMetricsConfiguration | | @@ -263,6 +277,9 @@ See [Ownership](./aws_s3_compat.md#ownership-controls) section for details. ## Notifications +Can't be exactly the same as in AWS. AMQP/NATS/etc can be supported, but it's +not exactly the gateway job. It also requires metadata synchronization. + | | Method | Comments | |----|------------------------------------|---------------| | 🔵 | GetBucketNotification | | @@ -309,7 +326,7 @@ Pay attention to the fact that object owner in NeoFS is bucket owner in any case | 🔵 | GetBucketReplication | | | 🟢 | PostPolicyBucket | Upload file using POST form | | 🟡 | PutBucketPolicy | See ACL limitations | -| 🔵 | PutBucketReplication | | +| 🔵 | PutBucketReplication | Hardly applicable to NeoFS | You may set requiring the `bucket-owner-full-control` canned ACL for Amazon S3 PUT operations ([bucket owner preferred](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ensure-object-ownership.html#ensure-object-ownership-bucket-policy)): ```shell @@ -362,6 +379,8 @@ policy.json: ## Tiering +Tightly related to lifecycle support. + | | Method | Comments | |----|---------------------------------------------|----------| | 🔵 | DeleteBucketIntelligentTieringConfiguration | | diff --git a/docs/configuration.md b/docs/configuration.md index 4304cdb4..e173876a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -158,7 +158,6 @@ There are some custom types used for brevity: | `placement_policy` | [Placement policy configuration](#placement_policy-section) | | `server` | [Server configuration](#server-section) | | `logger` | [Logger configuration](#logger-section) | -| `tree` | [Tree configuration](#tree-section) | | `cache` | [Cache configuration](#cache-section) | | `nats` | [NATS configuration](#nats-section) | | `cors` | [CORS configuration](#cors-section) | @@ -327,17 +326,6 @@ logger: | `encoding` | `string` | | `console` | Encoding type.
Possible values: `console`, `json`. | | `timestamp` | `bool` | | `false` | Flag to enable timestamps. If the parameter is not set, they will be enabled when you run with tty. | -### `tree` section - -```yaml -tree: - service: s01.neofs.devenv:8080 -``` - -| Parameter | Type | Default value | Description | -|-----------|----------|---------------|------------------------------------------------------------------------------------------------------------| -| `service` | `string` | | Endpoint of the tree service. Must be provided. Can be one of the node address (from the `peers` section). | - ### `cache` section ```yaml diff --git a/docs/labels.md b/docs/labels.md index 4b2180ec..ba440e0c 100644 --- a/docs/labels.md +++ b/docs/labels.md @@ -6,3 +6,4 @@ Currently only these ones are used, but the list can be extended in future: - auth-mate - tree-service + (outdated, was used for issues related to the tree service) diff --git a/docs/tree_service.md b/docs/tree_service.md deleted file mode 100644 index 84018d07..00000000 --- a/docs/tree_service.md +++ /dev/null @@ -1,18 +0,0 @@ -# Tree service - -To get objects' metadata and system information, the S3 GW makes requests to the Tree service. -This is a service in NeoFS storage that keeps different information as a tree structure. - -Each node keeps one of the types of data as a set of **key-value pairs**: -* Bucket settings: lock configuration and versioning mode -* Bucket tagging -* Object tagging -* Object metadata: OID, name, creation time, system metadata -* Object locking settings -* Active multipart upload info - -Some data takes up a lot of memory, so we store it in NeoFS nodes as an object with payload. -But we keep these objects' metadata in the Tree service too: -* Notification configuration -* CORS -* Metadata of parts of active multipart uploads