From 80d4fc216067983597c207841c0fddbfcf5a15de Mon Sep 17 00:00:00 2001 From: naveenpaul1 Date: Fri, 6 Jun 2025 14:58:54 +0530 Subject: [PATCH 1/2] SDK | Upgrade AWS SDK to v3 - Block Store S3 Signed-off-by: naveenpaul1 --- package-lock.json | 142 ++++++++++++++++++ package.json | 2 + .../block_store_services/block_store_s3.js | 65 ++++---- src/test/unit_tests/test_cloud_utils.js | 50 +++--- src/util/cloud_utils.js | 37 +++-- 5 files changed, 219 insertions(+), 77 deletions(-) diff --git a/package-lock.json b/package-lock.json index fb136885d9..824dadb79d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,8 @@ "dependencies": { "@aws-sdk/client-s3": "3.826.0", "@aws-sdk/client-sts": "3.826.0", + "@aws-sdk/credential-providers": "3.826.0", + "@aws-sdk/s3-request-presigner": "3.826.0", "@azure/identity": "4.10.0", "@azure/monitor-query": "1.3.2", "@azure/storage-blob": "12.27.0", @@ -300,6 +302,70 @@ "node": ">=14.0.0" } }, + "node_modules/@aws-sdk/client-cognito-identity": { + "version": "3.826.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-cognito-identity/-/client-cognito-identity-3.826.0.tgz", + "integrity": "sha512-3SStkRCHoB7a2Q4aGA++Zvlps8qxJhtCgSRs9cOSL9ldoYGCjOATkMCYmQD9iySge/WULPDEPf5qYlAyGfoRCg==", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.826.0", + "@aws-sdk/credential-provider-node": "3.826.0", + "@aws-sdk/middleware-host-header": "3.821.0", + "@aws-sdk/middleware-logger": "3.821.0", + "@aws-sdk/middleware-recursion-detection": "3.821.0", + "@aws-sdk/middleware-user-agent": "3.826.0", + "@aws-sdk/region-config-resolver": "3.821.0", + "@aws-sdk/types": "3.821.0", + "@aws-sdk/util-endpoints": "3.821.0", + "@aws-sdk/util-user-agent-browser": "3.821.0", + "@aws-sdk/util-user-agent-node": "3.826.0", + "@smithy/config-resolver": "^4.1.4", + "@smithy/core": "^3.5.3", + "@smithy/fetch-http-handler": "^5.0.4", + "@smithy/hash-node": "^4.0.4", + "@smithy/invalid-dependency": "^4.0.4", + "@smithy/middleware-content-length": "^4.0.4", + "@smithy/middleware-endpoint": "^4.1.11", + "@smithy/middleware-retry": "^4.1.12", + "@smithy/middleware-serde": "^4.0.8", + "@smithy/middleware-stack": "^4.0.4", + "@smithy/node-config-provider": "^4.1.3", + "@smithy/node-http-handler": "^4.0.6", + "@smithy/protocol-http": "^5.1.2", + "@smithy/smithy-client": "^4.4.3", + "@smithy/types": "^4.3.1", + "@smithy/url-parser": "^4.0.4", + "@smithy/util-base64": "^4.0.0", + "@smithy/util-body-length-browser": "^4.0.0", + "@smithy/util-body-length-node": "^4.0.0", + "@smithy/util-defaults-mode-browser": "^4.0.19", + "@smithy/util-defaults-mode-node": "^4.0.19", + "@smithy/util-endpoints": "^3.0.6", + "@smithy/util-middleware": "^4.0.4", + "@smithy/util-retry": "^4.0.5", + "@smithy/util-utf8": "^4.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-cognito-identity/node_modules/@smithy/node-http-handler": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.0.6.tgz", + "integrity": "sha512-NqbmSz7AW2rvw4kXhKGrYTiJVDHnMsFnX4i+/FzcZAfbOBauPYs2ekuECkSbtqaxETLLTu9Rl/ex6+I2BKErPA==", + "dependencies": { + "@smithy/abort-controller": "^4.0.4", + "@smithy/protocol-http": "^5.1.2", + "@smithy/querystring-builder": "^4.0.4", + "@smithy/types": "^4.3.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@aws-sdk/client-iam": { "version": "3.826.0", "resolved": "https://registry.npmjs.org/@aws-sdk/client-iam/-/client-iam-3.826.0.tgz", @@ -609,6 +675,21 @@ "node": ">=18.0.0" } }, + "node_modules/@aws-sdk/credential-provider-cognito-identity": { + "version": "3.826.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-cognito-identity/-/credential-provider-cognito-identity-3.826.0.tgz", + "integrity": "sha512-WnHLJD1iy0Gqyv0S7PXI+c1c9xBY8L5XnESOzYM78Sx1zlVHw557ZtQM1lKtHa6DiGMN4apYZr0lmalgmtHpQQ==", + "dependencies": { + "@aws-sdk/client-cognito-identity": "3.826.0", + "@aws-sdk/types": "3.821.0", + "@smithy/property-provider": "^4.0.4", + "@smithy/types": "^4.3.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@aws-sdk/credential-provider-env": { "version": "3.826.0", "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.826.0.tgz", @@ -762,6 +843,35 @@ "node": ">=18.0.0" } }, + "node_modules/@aws-sdk/credential-providers": { + "version": "3.826.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-providers/-/credential-providers-3.826.0.tgz", + "integrity": "sha512-2wuJFyI5I1LTN3dAF9AzyPDX5zn8/3hWxE8+M3Q1iAP/JH4dBB3SFyvE88Ts01iQ/n4Hu8cAlU5Ke1dnrCxUnw==", + "dependencies": { + "@aws-sdk/client-cognito-identity": "3.826.0", + "@aws-sdk/core": "3.826.0", + "@aws-sdk/credential-provider-cognito-identity": "3.826.0", + "@aws-sdk/credential-provider-env": "3.826.0", + "@aws-sdk/credential-provider-http": "3.826.0", + "@aws-sdk/credential-provider-ini": "3.826.0", + "@aws-sdk/credential-provider-node": "3.826.0", + "@aws-sdk/credential-provider-process": "3.826.0", + "@aws-sdk/credential-provider-sso": "3.826.0", + "@aws-sdk/credential-provider-web-identity": "3.826.0", + "@aws-sdk/nested-clients": "3.826.0", + "@aws-sdk/types": "3.821.0", + "@smithy/config-resolver": "^4.1.4", + "@smithy/core": "^3.5.3", + "@smithy/credential-provider-imds": "^4.0.6", + "@smithy/node-config-provider": "^4.1.3", + "@smithy/property-provider": "^4.0.4", + "@smithy/types": "^4.3.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@aws-sdk/lib-storage": { "version": "3.826.0", "resolved": "https://registry.npmjs.org/@aws-sdk/lib-storage/-/lib-storage-3.826.0.tgz", @@ -1038,6 +1148,24 @@ "node": ">=18.0.0" } }, + "node_modules/@aws-sdk/s3-request-presigner": { + "version": "3.826.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/s3-request-presigner/-/s3-request-presigner-3.826.0.tgz", + "integrity": "sha512-47IcILH3CfVzUmGwJhwuZQyuZ5zXNsFyvtpQWR2s2dkoT7TJCMAKY0MtWE+y2T99b20OGbUhQHz/9qlx7dR3zw==", + "dependencies": { + "@aws-sdk/signature-v4-multi-region": "3.826.0", + "@aws-sdk/types": "3.821.0", + "@aws-sdk/util-format-url": "3.821.0", + "@smithy/middleware-endpoint": "^4.1.11", + "@smithy/protocol-http": "^5.1.2", + "@smithy/smithy-client": "^4.4.3", + "@smithy/types": "^4.3.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@aws-sdk/signature-v4-multi-region": { "version": "3.826.0", "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.826.0.tgz", @@ -1113,6 +1241,20 @@ "node": ">=18.0.0" } }, + "node_modules/@aws-sdk/util-format-url": { + "version": "3.821.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-format-url/-/util-format-url-3.821.0.tgz", + "integrity": "sha512-h+xqmPToxDrZ0a7rxE1a8Oh4zpWfZe9oiQUphGtfiGFA6j75UiURH5J3MmGHa/G4t15I3iLLbYtUXxvb1i7evg==", + "dependencies": { + "@aws-sdk/types": "3.821.0", + "@smithy/querystring-builder": "^4.0.4", + "@smithy/types": "^4.3.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@aws-sdk/util-locate-window": { "version": "3.804.0", "resolved": "https://registry.npmjs.org/@aws-sdk/util-locate-window/-/util-locate-window-3.804.0.tgz", diff --git a/package.json b/package.json index ed4cc46add..8bbc201986 100644 --- a/package.json +++ b/package.json @@ -74,6 +74,8 @@ "dependencies": { "@aws-sdk/client-s3": "3.826.0", "@aws-sdk/client-sts": "3.826.0", + "@aws-sdk/credential-providers": "3.826.0", + "@aws-sdk/s3-request-presigner": "3.826.0", "@azure/identity": "4.10.0", "@azure/monitor-query": "1.3.2", "@azure/storage-blob": "12.27.0", diff --git a/src/agent/block_store_services/block_store_s3.js b/src/agent/block_store_services/block_store_s3.js index 3d60ad84cc..5cbd452c78 100644 --- a/src/agent/block_store_services/block_store_s3.js +++ b/src/agent/block_store_services/block_store_s3.js @@ -2,7 +2,7 @@ 'use strict'; const _ = require('lodash'); -const AWS = require('aws-sdk'); +const { S3 } = require('@aws-sdk/client-s3'); const config = require('../../../config'); const P = require('../../util/promise'); @@ -12,6 +12,7 @@ const cloud_utils = require('../../util/cloud_utils'); const size_utils = require('../../util/size_utils'); const BlockStoreBase = require('./block_store_base').BlockStoreBase; const { RpcError } = require('../../rpc'); +const { NodeHttpHandler } = require("@smithy/node-http-handler"); const DEFAULT_REGION = 'us-east-1'; @@ -39,16 +40,17 @@ class BlockStoreS3 extends BlockStoreBase { RoleSessionName: 'block_store_operations' }; } else { - this.s3cloud = new AWS.S3({ + this.s3cloud = new S3({ endpoint: endpoint, - accessKeyId: this.cloud_info.access_keys.access_key.unwrap(), - secretAccessKey: this.cloud_info.access_keys.secret_key.unwrap(), - s3ForcePathStyle: true, - signatureVersion: cloud_utils.get_s3_endpoint_signature_ver(endpoint, this.cloud_info.auth_method), + credentials: { + accessKeyId: this.cloud_info.access_keys.access_key.unwrap(), + secretAccessKey: this.cloud_info.access_keys.secret_key.unwrap(), + }, + forcePathStyle: true, region: DEFAULT_REGION, - httpOptions: { - agent: http_utils.get_default_agent(endpoint) - } + requestHandler: new NodeHttpHandler({ + httpsAgent: http_utils.get_default_agent(endpoint) + }), }); } } else { @@ -56,16 +58,17 @@ class BlockStoreS3 extends BlockStoreBase { config.EXPERIMENTAL_DISABLE_S3_COMPATIBLE_DELEGATION.DEFAULT; this.disable_metadata = config.EXPERIMENTAL_DISABLE_S3_COMPATIBLE_METADATA[this.cloud_info.endpoint_type] || config.EXPERIMENTAL_DISABLE_S3_COMPATIBLE_METADATA.DEFAULT; - this.s3cloud = new AWS.S3({ + this.s3cloud = new S3({ endpoint: endpoint, - s3ForcePathStyle: true, - accessKeyId: this.cloud_info.access_keys.access_key.unwrap(), - secretAccessKey: this.cloud_info.access_keys.secret_key.unwrap(), - signatureVersion: cloud_utils.get_s3_endpoint_signature_ver(endpoint, this.cloud_info.auth_method), - s3DisableBodySigning: cloud_utils.disable_s3_compatible_bodysigning(endpoint), - httpOptions: { - agent: http_utils.get_unsecured_agent(endpoint) - } + forcePathStyle: true, + credentials: { + accessKeyId: this.cloud_info.access_keys.access_key.unwrap(), + secretAccessKey: this.cloud_info.access_keys.secret_key.unwrap(), + }, + applyChecksum: cloud_utils.disable_s3_compatible_bodysigning(endpoint), + requestHandler: new NodeHttpHandler({ + httpsAgent: http_utils.get_unsecured_agent(endpoint) + }), }); } @@ -79,7 +82,7 @@ class BlockStoreS3 extends BlockStoreBase { const res = await this.s3cloud.getObject({ Bucket: this.cloud_info.target_bucket, Key: this.usage_path, - }).promise(); + }); const usage_data = this.disable_metadata ? res.Body.toString() : @@ -106,7 +109,7 @@ class BlockStoreS3 extends BlockStoreBase { const res = await this.s3cloud.headObject({ Bucket: this.cloud_info.target_bucket, Key: this._block_key(block_md.id), - }).promise(); + }); return { block_md: this._get_store_block_md(block_md, res), store_md5: res.ETag.toUpperCase(), @@ -164,7 +167,7 @@ class BlockStoreS3 extends BlockStoreBase { const res = await this.s3cloud.getObject({ Bucket: this.cloud_info.target_bucket, Key: this._block_key(block_md.id), - }).promise(); + }); return { data: res.Body, block_md: this._get_store_block_md(block_md, res), @@ -193,7 +196,7 @@ class BlockStoreS3 extends BlockStoreBase { Key: block_key, Body: data, Metadata: this.disable_metadata ? undefined : { noobaablockmd: encoded_md }, - }).promise(); + }); if (options && options.ignore_usage) return; // return usage count for the object return this._update_usage({ @@ -242,7 +245,7 @@ class BlockStoreS3 extends BlockStoreBase { Metadata: this.disable_metadata ? undefined : { [this.usage_md_key]: usage_data }, - }).promise(); + }); // if our target bucket returns version ids that means versioning is enabled // and for the usage file that we keep replacing we want to keep only the latest // so we delete the past versions of the usage file. @@ -262,12 +265,12 @@ class BlockStoreS3 extends BlockStoreBase { await this.s3cloud.deleteObjectTagging({ Bucket: this.cloud_info.target_bucket, Key: block_key - }).promise(); + }); } else { await this.s3cloud.deleteObject({ Bucket: this.cloud_info.target_bucket, Key: block_key - }).promise(); + }); } } catch (err) { // NoSuchKey is expected @@ -309,7 +312,7 @@ class BlockStoreS3 extends BlockStoreBase { Delimiter: '/', KeyMarker: key_marker, VersionIdMarker: version_marker, - }).promise(); + }); is_truncated = res.IsTruncated; key_marker = res.NextKeyMarker; version_marker = res.NextVersionIdMarker; @@ -322,7 +325,7 @@ class BlockStoreS3 extends BlockStoreBase { await this.s3cloud.deleteObjects({ Bucket: this.cloud_info.target_bucket, Delete: { Objects: delete_list }, - }).promise(); + }); } } } @@ -344,7 +347,7 @@ class BlockStoreS3 extends BlockStoreBase { Bucket: this.cloud_info.target_bucket, KeyMarker: key_marker, VersionIdMarker: version_marker - }).promise(); + }); const del_objs = list_res.Versions.map(ver => ({ Key: ver.Key, VersionId: ver.VersionId })); if (del_objs.length > 0) { await this.s3cloud.deleteObjects({ @@ -352,7 +355,7 @@ class BlockStoreS3 extends BlockStoreBase { Delete: { Objects: del_objs, } - }).promise(); + }); total += del_objs.length; } @@ -390,7 +393,7 @@ class BlockStoreS3 extends BlockStoreBase { Key: this._block_key(block_id) })) } - }).promise(); + }); if (res.Errors) { for (const delete_error of res.Errors) { const block_id = this._block_id_from_key(delete_error.Key); @@ -421,7 +424,7 @@ class BlockStoreS3 extends BlockStoreBase { const res = await this.s3cloud.headObject({ Bucket: this.cloud_info.target_bucket, Key: this._block_key(block_id), - }).promise(); + }); const noobaablockmd = res.Metadata.noobaablockmd || res.Metadata.noobaa_block_md; const md_size = (noobaablockmd && noobaablockmd.length) || 0; usage.size += Number(res.ContentLength) + md_size; diff --git a/src/test/unit_tests/test_cloud_utils.js b/src/test/unit_tests/test_cloud_utils.js index 7e069516bc..318944c9a8 100644 --- a/src/test/unit_tests/test_cloud_utils.js +++ b/src/test/unit_tests/test_cloud_utils.js @@ -4,7 +4,7 @@ const mocha = require('mocha'); const assert = require('assert'); const sinon = require('sinon'); -const AWS = require('aws-sdk'); +const { STSClient } = require('@aws-sdk/client-sts'); const cloud_utils = require('../../util/cloud_utils'); const dbg = require('../../util/debug_module')(__filename); const fs = require("fs"); @@ -13,50 +13,40 @@ const fakeAccessKeyId = "fakeAccessKeyId"; const fakeSecretAccessKey = "fakeSecretAccessKey"; const fakeSessionToken = "fakeSessionToken"; const roleArn = "arn:aws:iam::261532230807:role/noobaa_s3_sts"; -const defaultSTSCredsValidity = 3600; -const expectedParams = [{ - RoleArn: roleArn, - RoleSessionName: 'testSession', - WebIdentityToken: 'web-identity-token', - DurationSeconds: defaultSTSCredsValidity, -}]; +const REGION = "us-east-1"; + mocha.describe('AWS STS tests', function() { let STSStub; - let stsFake; mocha.before('Creating STS stub', function() { sinon.stub(fs.promises, "readFile") .withArgs(projectedServiceAccountToken) .returns("web-identity-token"); - stsFake = { - assumeRoleWithWebIdentity: sinon.stub().returnsThis(), - promise: sinon.stub() - .resolves({ - Credentials: { - AccessKeyId: fakeAccessKeyId, - SecretAccessKey: fakeSecretAccessKey, - SessionToken: fakeSessionToken - } - }), - }; - STSStub = sinon.stub(AWS, 'STS') - .callsFake(() => stsFake); + + STSStub = sinon.stub(STSClient.prototype, 'send') + .callsFake(() => Promise.resolve({ + Credentials: { + AccessKeyId: fakeAccessKeyId, + SecretAccessKey: fakeSecretAccessKey, + SessionToken: fakeSessionToken + } + })); }); mocha.after('Restoring STS stub', function() { STSStub.restore(); }); mocha.it('should generate aws sts creds', async function() { const params = { - aws_sts_arn: roleArn + aws_sts_arn: roleArn, + region: REGION, }; const roleSessionName = "testSession"; const json = await cloud_utils.generate_aws_sts_creds(params, roleSessionName); sinon.assert.calledOnce(STSStub); - sinon.assert.calledWith(stsFake.assumeRoleWithWebIdentity, ...expectedParams); assert.equal(json.accessKeyId, fakeAccessKeyId); assert.equal(json.secretAccessKey, fakeSecretAccessKey); assert.equal(json.sessionToken, fakeSessionToken); dbg.log0('test.aws.sts.assumeRoleWithWebIdentity: ', json); - }); + }, 50000); mocha.it('should generate an STS S3 client', async function() { const params = { aws_sts_arn: roleArn, @@ -66,10 +56,10 @@ mocha.describe('AWS STS tests', function() { RoleSessionName: 'testSession' }; const s3 = await cloud_utils.createSTSS3Client(params, additionalParams); - dbg.log0('test.aws.sts.createSTSS3Client: ', s3); - assert.equal(s3.config.credentials.accessKeyId, fakeAccessKeyId); - assert.equal(s3.config.credentials.secretAccessKey, fakeSecretAccessKey); - assert.equal(s3.config.credentials.sessionToken, fakeSessionToken); - assert.equal(s3.config.region, 'us-east-1'); + dbg.log0('test.aws.sts.createSTSS3Client: ', (await s3.config.credentials())); + assert.equal((await s3.config.credentials()).accessKeyId, fakeAccessKeyId); + assert.equal((await s3.config.credentials()).secretAccessKey, fakeSecretAccessKey); + assert.equal((await s3.config.credentials()).sessionToken, fakeSessionToken); + assert.equal((await s3.config.region()), 'us-east-1'); }); }); diff --git a/src/util/cloud_utils.js b/src/util/cloud_utils.js index ec6b5c465f..dd03f4b1e1 100644 --- a/src/util/cloud_utils.js +++ b/src/util/cloud_utils.js @@ -10,6 +10,8 @@ const { S3 } = require('@aws-sdk/client-s3'); const url = require('url'); const _ = require('lodash'); const SensitiveString = require('./sensitive_string'); +const { STSClient, AssumeRoleWithWebIdentityCommand } = require('@aws-sdk/client-sts'); +const { NodeHttpHandler } = require("@smithy/node-http-handler"); const config = require('../../config'); const noobaa_s3_client = require('../sdk/noobaa_s3_client/noobaa_s3_client'); @@ -31,34 +33,37 @@ function find_cloud_connection(account, conn_name) { async function createSTSS3Client(params, additionalParams) { const creds = await generate_aws_sts_creds(params, additionalParams.RoleSessionName); - return new AWS.S3({ + return new S3({ credentials: creds, - region: params.region, + region: params.region || config.DEFAULT_REGION, endpoint: additionalParams.endpoint, - signatureVersion: additionalParams.signatureVersion, - s3DisableBodySigning: additionalParams.s3DisableBodySigning, - httpOptions: additionalParams.httpOptions, - s3ForcePathStyle: additionalParams.s3ForcePathStyle + requestHandler: new NodeHttpHandler({ + httpsAgent: additionalParams.httpOptions + }), + forcePathStyle: additionalParams.s3ForcePathStyle }); } async function generate_aws_sts_creds(params, roleSessionName) { - const sts = new AWS.STS(); - const creds = await (sts.assumeRoleWithWebIdentity({ + + const sts_client = new STSClient({ region: params.region || config.DEFAULT_REGION }); + const input = { + DurationSeconds: defaultSTSCredsValidity, RoleArn: params.aws_sts_arn, RoleSessionName: roleSessionName || defaultRoleSessionName, WebIdentityToken: (await fs.promises.readFile(projectedServiceAccountToken)).toString(), - DurationSeconds: defaultSTSCredsValidity - }).promise()); - if (_.isEmpty(creds.Credentials)) { + }; + const command = new AssumeRoleWithWebIdentityCommand(input); + const response = await sts_client.send(command); + if (_.isEmpty(response) || _.isEmpty(response.Credentials)) { dbg.error(`AWS STS empty creds ${params.RoleArn}, RolesessionName: ${params.RoleSessionName},Projected service Account Token Path : ${projectedServiceAccountToken}`); throw new RpcError('AWS_STS_ERROR', 'Empty AWS STS creds retrieved for Role "' + params.RoleArn + '"'); } - return new AWS.Credentials( - creds.Credentials.AccessKeyId, - creds.Credentials.SecretAccessKey, - creds.Credentials.SessionToken - ); + return { + accessKeyId: response.Credentials.AccessKeyId, + secretAccessKey: response.Credentials.SecretAccessKey, + sessionToken: response.Credentials.SessionToken, + }; } function get_signed_url(params, expiry = 604800, custom_operation = 'getObject') { From 430919b2f338a5f895fcc431b250ce8636f0693f Mon Sep 17 00:00:00 2001 From: naveenpaul1 Date: Mon, 30 Jun 2025 10:11:49 +0530 Subject: [PATCH 2/2] namespace fs --- src/endpoint/s3/s3_utils.js | 26 +++--- src/sdk/namespace_s3.js | 159 +++++++++++++++++------------------- src/sdk/object_sdk.js | 10 ++- 3 files changed, 98 insertions(+), 97 deletions(-) diff --git a/src/endpoint/s3/s3_utils.js b/src/endpoint/s3/s3_utils.js index ffd2c0c276..06277eef95 100644 --- a/src/endpoint/s3/s3_utils.js +++ b/src/endpoint/s3/s3_utils.js @@ -666,16 +666,22 @@ function parse_body_logging_xml(req) { return logging; } -function get_http_response_date(res) { - const r = get_http_response_from_resp(res); - if (!r.httpResponse.headers.date) throw new Error("date not found in response header"); - return r.httpResponse.headers.date; -} - -function get_http_response_from_resp(res) { - const r = res.$response; - if (!r) throw new Error("no $response in s3 returned object"); - return r; +async function get_http_response_date(res, s3_client, bucket, key) { + const r = await get_http_response_from_resp(res, s3_client, bucket, key); + if (!r.LastModified) throw new Error("Date not found in response header"); + return r.LastModified; +} + +async function get_http_response_from_resp(res, s3_client, bucket, key) { + const status_code = res.$metadata.httpStatusCode; + if (status_code >= 300) throw new Error(`PutObject S3 request failed for bucket ${bucket}, Key: ${key} with status code :`, status_code); + const head_res = await s3_client.headObject({ + Bucket: bucket, + Key: key, + }); + const head_status_code = res.$metadata.httpStatusCode; + if (head_status_code >= 300) throw new Error(`headObject S3 request failed for bucket ${bucket}, Key: ${key} with status code :`, status_code); + return head_res; } function get_response_field_encoder(req) { diff --git a/src/sdk/namespace_s3.js b/src/sdk/namespace_s3.js index 834e2c966b..13ec2d360f 100644 --- a/src/sdk/namespace_s3.js +++ b/src/sdk/namespace_s3.js @@ -2,7 +2,6 @@ 'use strict'; const _ = require('lodash'); -const AWS = require('aws-sdk'); const util = require('util'); const dbg = require('../util/debug_module')(__filename); @@ -12,6 +11,7 @@ const cloud_utils = require('../util/cloud_utils'); const stream_utils = require('../util/stream_utils'); const blob_translator = require('./blob_translator'); const S3Error = require('../endpoint/s3/s3_errors').S3Error; +const noobaa_s3_client = require('../sdk/noobaa_s3_client/noobaa_s3_client'); /** * @implements {nb.Namespace} @@ -21,20 +21,22 @@ class NamespaceS3 { /** * @param {{ * namespace_resource_id: any, - * s3_params: AWS.S3.ClientConfiguration & { + * s3_params: Object & { * access_mode?: string, * aws_sts_arn?: string, * }, + * bucket?: string, * stats: import('./endpoint_stats_collector').EndpointStatsCollector, * }} args */ - constructor({ namespace_resource_id, s3_params, stats }) { + constructor({ namespace_resource_id, s3_params, bucket, stats }) { this.namespace_resource_id = namespace_resource_id; this.s3_params = s3_params; this.access_key = s3_params.accessKeyId; this.endpoint = s3_params.endpoint; - this.s3 = new AWS.S3(s3_params); - this.bucket = String(this.s3.config.params.Bucket); + //this.s3 = new S3(s3_params) + this.s3 = noobaa_s3_client.get_s3_client_v3_params(s3_params); + this.bucket = String(bucket); this.access_mode = s3_params.access_mode; this.stats = stats; } @@ -85,7 +87,7 @@ class NamespaceS3 { Delimiter: params.delimiter, Marker: params.key_marker, MaxKeys: params.limit, - }).promise(); + }); dbg.log0('NamespaceS3.list_objects:', this.bucket, inspect(params), 'list', inspect(res)); @@ -117,7 +119,7 @@ class NamespaceS3 { KeyMarker: params.key_marker, UploadIdMarker: params.upload_id_marker, MaxUploads: params.limit, - }).promise(); + }); dbg.log0('NamespaceS3.list_uploads:', this.bucket, inspect(params), 'list', inspect(res)); @@ -142,7 +144,7 @@ class NamespaceS3 { KeyMarker: params.key_marker, VersionIdMarker: params.version_id_marker, MaxKeys: params.limit, - }).promise(); + }); dbg.log0('NamespaceS3.list_object_versions:', this.bucket, inspect(params), 'list', inspect(res)); @@ -183,7 +185,7 @@ class NamespaceS3 { dbg.log0('NamespaceS3.read_object_md:', this.bucket, inspect(params)); await this._prepare_sts_client(); - /** @type {AWS.S3.HeadObjectRequest | AWS.S3.GetObjectRequest} */ + /** @type {import("@aws-sdk/client-s3").HeadObjectRequest | import("@aws-sdk/client-s3").GetObjectRequest} */ const request = { Bucket: this.bucket, Key: params.key, @@ -203,16 +205,16 @@ class NamespaceS3 { let res; try { res = can_use_get_inline ? - await this.s3.getObject(request).promise() : - await this.s3.headObject(request).promise(); + await this.s3.getObject(request) : + await this.s3.headObject(request); } catch (err) { // catch invalid range error for objects of size 0 and trying head object instead if (err.code !== 'InvalidRange') { throw err; } - res = await this.s3.headObject({ ...request, Range: undefined }).promise(); + res = await this.s3.headObject({ ...request, Range: undefined }); } - dbg.log0('NamespaceS3.read_object_md:', this.bucket, inspect(params), 'res', inspect(res)); + dbg.log0('NamespaceS3.read_object_md:', this.bucket, inspect(params), 'metadata', inspect(res.$metadata)); return this._get_s3_object_info(res, params.bucket, params.part_number); } catch (err) { this._translate_error_code(params, err); @@ -240,51 +242,42 @@ class NamespaceS3 { dbg.log0('NamespaceS3.read_object_stream:', this.bucket, inspect(_.omit(params, 'object_md.ns'))); await this._prepare_sts_client(); - return new Promise((resolve, reject) => { - /** @type {AWS.S3.HeadObjectRequest & AWS.S3.GetObjectRequest | AWS.S3.CopyObjectRequest} */ - const request = { - Bucket: this.bucket, - Key: params.key, - Range: params.end ? `bytes=${params.start}-${params.end - 1}` : undefined, - PartNumber: params.part_number, - }; - this._set_md_conditions(params, request); - this._assign_encryption_to_request(params, request); - const req = this.s3.getObject(request) - .on('error', err => { - this._translate_error_code(params, err); - dbg.warn('NamespaceS3.read_object_stream:', inspect(err)); - reject(err); - }) - .on('httpHeaders', (statusCode, headers, res) => { - dbg.log0('NamespaceS3.read_object_stream:', + /** @type {import("@aws-sdk/client-s3").HeadObjectRequest & import("@aws-sdk/client-s3").GetObjectRequest | import("@aws-sdk/client-s3").CopyObjectRequest} */ + const request = { + Bucket: this.bucket, + Key: params.key, + Range: params.end ? `bytes=${params.start}-${params.end - 1}` : undefined, + PartNumber: params.part_number, + }; + this._set_md_conditions(params, request); + this._assign_encryption_to_request(params, request); + try { + const obj_out = await this.s3.getObject(request); + dbg.log0('NamespaceS3.read_object_stream:', this.bucket, inspect(_.omit(params, 'object_md.ns')), - 'statusCode', statusCode, - 'headers', headers ); - if (statusCode >= 300) return; // will be handled by error event - req.removeListener('httpData', AWS.EventListeners.Core.HTTP_DATA); - req.removeListener('httpError', AWS.EventListeners.Core.HTTP_ERROR); - let count = 1; - // on s3 read_object_md might not return x-amz-tagging-count header, so we get it here - params.tag_count = headers['x-amz-tagging-count']; - const count_stream = stream_utils.get_tap_stream(data => { - this.stats?.update_namespace_read_stats({ - namespace_resource_id: this.namespace_resource_id, - bucket_name: params.bucket, - size: data.length, - count - }); - // clear count for next updates - count = 0; - }); - const read_stream = /** @type {import('stream').Readable} */ - (res.httpResponse.createUnbufferedStream()); - return resolve(read_stream.pipe(count_stream)); + if (obj_out.$metadata.httpStatusCode >= 300) return; // will be handled by error event + let count = 1; + // on s3 read_object_md might not return x-amz-tagging-count header, so we get it here + //params.tag_count = headers['x-amz-tagging-count']; + const count_stream = stream_utils.get_tap_stream(data => { + this.stats?.update_namespace_read_stats({ + namespace_resource_id: this.namespace_resource_id, + bucket_name: params.bucket, + size: data.length, + count }); - req.send(); - }); + // clear count for next updates + count = 0; + }); + const read_stream = /** @type {import('stream').Readable} **/ obj_out.Body.transformToWebStream(); + await stream_utils.pipeline([read_stream, count_stream], true); + return count_stream; + } catch (err) { + this._translate_error_code(params, err); + dbg.warn('NamespaceS3.read_object_stream:', inspect(err)); + } } @@ -305,7 +298,7 @@ class NamespaceS3 { throw new Error('NamespaceS3.upload_object: CopySourceRange not supported by s3.copyObject()'); } - /** @type {AWS.S3.CopyObjectRequest} */ + /** @type {import("@aws-sdk/client-s3").CopyObjectRequest} */ const request = { Bucket: this.bucket, Key: params.key, @@ -320,7 +313,7 @@ class NamespaceS3 { this._assign_encryption_to_request(params, request); - res = await this.s3.copyObject(request).promise(); + res = await this.s3.copyObject(request); } else { let count = 1; const count_stream = stream_utils.get_tap_stream(data => { @@ -334,7 +327,7 @@ class NamespaceS3 { count = 0; }); - /** @type {AWS.S3.PutObjectRequest} */ + /** @type {import("@aws-sdk/client-s3").PutObjectRequest} */ const request = { Bucket: this.bucket, Key: params.key, @@ -348,7 +341,7 @@ class NamespaceS3 { this._assign_encryption_to_request(params, request); try { - res = await this.s3.putObject(request).promise(); + res = await this.s3.putObject(request); } catch (err) { object_sdk.rpc_client.pool.update_issues_report({ namespace_resource_id: this.namespace_resource_id, @@ -360,7 +353,7 @@ class NamespaceS3 { } dbg.log0('NamespaceS3.upload_object:', this.bucket, inspect(params), 'res', inspect(res)); const etag = s3_utils.parse_etag(res.ETag); - const last_modified_time = s3_utils.get_http_response_date(res); + const last_modified_time = await s3_utils.get_http_response_date(res, this.s3, this.bucket, params.key); return { etag, version_id: res.VersionId, last_modified_time }; } @@ -389,7 +382,7 @@ class NamespaceS3 { await this._prepare_sts_client(); const Tagging = params.tagging && params.tagging.map(tag => tag.key + '=' + tag.value).join('&'); - /** @type {AWS.S3.CreateMultipartUploadRequest} */ + /** @type {import("@aws-sdk/client-s3").CreateMultipartUploadRequest} */ const request = { Bucket: this.bucket, Key: params.key, @@ -399,7 +392,7 @@ class NamespaceS3 { Tagging }; this._assign_encryption_to_request(params, request); - const res = await this.s3.createMultipartUpload(request).promise(); + const res = await this.s3.createMultipartUpload(request); dbg.log0('NamespaceS3.create_object_upload:', this.bucket, inspect(params), 'res', inspect(res)); return { obj_id: res.UploadId }; @@ -413,7 +406,7 @@ class NamespaceS3 { if (params.copy_source) { const { copy_source, copy_source_range } = s3_utils.format_copy_source(params.copy_source); - /** @type {AWS.S3.UploadPartCopyRequest} */ + /** @type {import("@aws-sdk/client-s3").UploadPartCopyRequest} */ const request = { Bucket: this.bucket, Key: params.key, @@ -425,7 +418,7 @@ class NamespaceS3 { this._assign_encryption_to_request(params, request); - res = await this.s3.uploadPartCopy(request).promise(); + res = await this.s3.uploadPartCopy(request); } else { let count = 1; const count_stream = stream_utils.get_tap_stream(data => { @@ -438,7 +431,7 @@ class NamespaceS3 { count = 0; }); - /** @type {AWS.S3.UploadPartRequest} */ + /** @type {import("@aws-sdk/client-s3").UploadPartRequest} */ const request = { Bucket: this.bucket, Key: params.key, @@ -451,7 +444,7 @@ class NamespaceS3 { this._assign_encryption_to_request(params, request); try { - res = await this.s3.uploadPart(request).promise(); + res = await this.s3.uploadPart(request); } catch (err) { object_sdk.rpc_client.pool.update_issues_report({ namespace_resource_id: this.namespace_resource_id, @@ -476,7 +469,7 @@ class NamespaceS3 { UploadId: params.obj_id, MaxParts: params.max, PartNumberMarker: params.num_marker, - }).promise(); + }); dbg.log0('NamespaceS3.list_multiparts:', this.bucket, inspect(params), 'res', inspect(res)); return { @@ -505,7 +498,7 @@ class NamespaceS3 { ETag: `"${p.etag}"`, })) } - }).promise(); + }); dbg.log0('NamespaceS3.complete_object_upload:', this.bucket, inspect(params), 'res', inspect(res)); const etag = s3_utils.parse_etag(res.ETag); @@ -520,7 +513,7 @@ class NamespaceS3 { Bucket: this.bucket, Key: params.key, UploadId: params.obj_id, - }).promise(); + }); dbg.log0('NamespaceS3.abort_object_upload:', this.bucket, inspect(params), 'res', inspect(res)); } @@ -543,7 +536,7 @@ class NamespaceS3 { Key: params.key, VersionId: params.version_id, Tagging: { TagSet } - }).promise(); + }); dbg.log0('NamespaceS3.put_object_tagging:', this.bucket, inspect(params), 'res', inspect(res)); @@ -560,7 +553,7 @@ class NamespaceS3 { Bucket: this.bucket, Key: params.key, VersionId: params.version_id - }).promise(); + }); dbg.log0('NamespaceS3.delete_object_tagging:', this.bucket, inspect(params), 'res', inspect(res)); @@ -577,7 +570,7 @@ class NamespaceS3 { Bucket: this.bucket, Key: params.key, VersionId: params.version_id - }).promise(); + }); dbg.log0('NamespaceS3.get_object_tagging:', this.bucket, inspect(params), 'res', inspect(res)); @@ -604,7 +597,7 @@ class NamespaceS3 { Bucket: this.bucket, Key: params.key, VersionId: params.version_id - }).promise(); + }); dbg.log0('NamespaceS3.get_object_acl:', this.bucket, inspect(params), 'res', inspect(res)); @@ -623,7 +616,7 @@ class NamespaceS3 { Key: params.key, VersionId: params.version_id, ACL: params.acl - }).promise(); + }); dbg.log0('NamespaceS3.put_object_acl:', this.bucket, inspect(params), 'res', inspect(res)); } @@ -640,7 +633,7 @@ class NamespaceS3 { Bucket: this.bucket, Key: params.key, VersionId: params.version_id, - }).promise(); + }); dbg.log0('NamespaceS3.delete_object:', this.bucket, @@ -672,7 +665,7 @@ class NamespaceS3 { VersionId: obj.version_id, })) } - }).promise(); + }); dbg.log0('NamespaceS3.delete_multiple_objects:', this.bucket, @@ -753,7 +746,7 @@ class NamespaceS3 { dbg.log0('NamespaceS3.get_object_attributes:', this.bucket, inspect(params)); await this._prepare_sts_client(); - /** @type {AWS.S3.GetObjectAttributesRequest} */ + /** @type {import("@aws-sdk/client-s3").GetObjectAttributesRequest} */ const request = { Bucket: this.bucket, Key: params.key, @@ -763,7 +756,7 @@ class NamespaceS3 { this._set_md_conditions(params, request); this._assign_encryption_to_request(params, request); try { - const res = await this.s3.getObjectAttributes(request).promise(); + const res = await this.s3.getObjectAttributes(request); dbg.log0('NamespaceS3.get_object_attributes:', this.bucket, inspect(params), 'res', inspect(res)); return this._get_s3_object_info(res, params.bucket); } catch (err) { @@ -789,12 +782,12 @@ class NamespaceS3 { /** * * @param {Omit, 'ChecksumAlgorithm'>} res * @param {string} bucket * @param {number} [part_number] diff --git a/src/sdk/object_sdk.js b/src/sdk/object_sdk.js index 8d65186a39..d5916f4e60 100644 --- a/src/sdk/object_sdk.js +++ b/src/sdk/object_sdk.js @@ -467,15 +467,17 @@ class ObjectSDK { params: { Bucket: r.target_bucket }, endpoint: r.endpoint, aws_sts_arn: r.aws_sts_arn, - accessKeyId: r.access_key.unwrap(), - secretAccessKey: r.secret_key.unwrap(), - // region: 'us-east-1', // TODO needed? - signatureVersion: cloud_utils.get_s3_endpoint_signature_ver(r.endpoint, r.auth_method), + credentials: { + accessKeyId: r.access_key.unwrap(), + secretAccessKey: r.secret_key.unwrap(), + }, + region: config.DEFAULT_REGION, // SDKv3 needs region s3ForcePathStyle: true, // computeChecksums: false, // disabled by default for performance httpOptions: { agent }, access_mode: r.access_mode }, + bucket: r.target_bucket, stats: this.stats, }); }