Skip to content

SDK | Upgrade AWS SDK to v3 - Block Store S3 #9067

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 2 commits into
base: master
Choose a base branch
from
Draft

SDK | Upgrade AWS SDK to v3 - Block Store S3 #9067

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
80d4fc2
SDK | Upgrade AWS SDK to v3 - Block Store S3
naveenpaul1 Jun 6, 2025
430919b
namespace fs
naveenpaul1 Jun 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
142 changes: 142 additions & 0 deletions package-lock.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,8 @@
"dependencies": {
"@aws-sdk/client-s3": "3.826.0",
"@aws-sdk/client-sts": "3.826.0",
"@aws-sdk/credential-providers": "3.826.0",
"@aws-sdk/s3-request-presigner": "3.826.0",
"@azure/identity": "4.10.0",
"@azure/monitor-query": "1.3.2",
"@azure/storage-blob": "12.27.0",
Expand Down
65 changes: 34 additions & 31 deletions src/agent/block_store_services/block_store_s3.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
'use strict';

const _ = require('lodash');
const AWS = require('aws-sdk');
const { S3 } = require('@aws-sdk/client-s3');

const config = require('../../../config');
const P = require('../../util/promise');
Expand All @@ -12,6 +12,7 @@ const cloud_utils = require('../../util/cloud_utils');
const size_utils = require('../../util/size_utils');
const BlockStoreBase = require('./block_store_base').BlockStoreBase;
const { RpcError } = require('../../rpc');
const { NodeHttpHandler } = require("@smithy/node-http-handler");


const DEFAULT_REGION = 'us-east-1';
Expand Down Expand Up @@ -39,33 +40,35 @@ class BlockStoreS3 extends BlockStoreBase {
RoleSessionName: 'block_store_operations'
};
} else {
this.s3cloud = new AWS.S3({
this.s3cloud = new S3({
endpoint: endpoint,
accessKeyId: this.cloud_info.access_keys.access_key.unwrap(),
secretAccessKey: this.cloud_info.access_keys.secret_key.unwrap(),
s3ForcePathStyle: true,
signatureVersion: cloud_utils.get_s3_endpoint_signature_ver(endpoint, this.cloud_info.auth_method),
credentials: {
accessKeyId: this.cloud_info.access_keys.access_key.unwrap(),
secretAccessKey: this.cloud_info.access_keys.secret_key.unwrap(),
},
forcePathStyle: true,
region: DEFAULT_REGION,
httpOptions: {
agent: http_utils.get_default_agent(endpoint)
}
requestHandler: new NodeHttpHandler({
httpsAgent: http_utils.get_default_agent(endpoint)
}),
});
}
} else {
this.disable_delegation = config.EXPERIMENTAL_DISABLE_S3_COMPATIBLE_DELEGATION[this.cloud_info.endpoint_type] ||
config.EXPERIMENTAL_DISABLE_S3_COMPATIBLE_DELEGATION.DEFAULT;
this.disable_metadata = config.EXPERIMENTAL_DISABLE_S3_COMPATIBLE_METADATA[this.cloud_info.endpoint_type] ||
config.EXPERIMENTAL_DISABLE_S3_COMPATIBLE_METADATA.DEFAULT;
this.s3cloud = new AWS.S3({
this.s3cloud = new S3({
endpoint: endpoint,
s3ForcePathStyle: true,
accessKeyId: this.cloud_info.access_keys.access_key.unwrap(),
secretAccessKey: this.cloud_info.access_keys.secret_key.unwrap(),
signatureVersion: cloud_utils.get_s3_endpoint_signature_ver(endpoint, this.cloud_info.auth_method),
s3DisableBodySigning: cloud_utils.disable_s3_compatible_bodysigning(endpoint),
httpOptions: {
agent: http_utils.get_unsecured_agent(endpoint)
}
forcePathStyle: true,
credentials: {
accessKeyId: this.cloud_info.access_keys.access_key.unwrap(),
secretAccessKey: this.cloud_info.access_keys.secret_key.unwrap(),
},
applyChecksum: cloud_utils.disable_s3_compatible_bodysigning(endpoint),
requestHandler: new NodeHttpHandler({
httpsAgent: http_utils.get_unsecured_agent(endpoint)
}),
});
}

Expand All @@ -79,7 +82,7 @@ class BlockStoreS3 extends BlockStoreBase {
const res = await this.s3cloud.getObject({
Bucket: this.cloud_info.target_bucket,
Key: this.usage_path,
}).promise();
});

const usage_data = this.disable_metadata ?
res.Body.toString() :
Expand All @@ -106,7 +109,7 @@ class BlockStoreS3 extends BlockStoreBase {
const res = await this.s3cloud.headObject({
Bucket: this.cloud_info.target_bucket,
Key: this._block_key(block_md.id),
}).promise();
});
return {
block_md: this._get_store_block_md(block_md, res),
store_md5: res.ETag.toUpperCase(),
Expand Down Expand Up @@ -164,7 +167,7 @@ class BlockStoreS3 extends BlockStoreBase {
const res = await this.s3cloud.getObject({
Bucket: this.cloud_info.target_bucket,
Key: this._block_key(block_md.id),
}).promise();
});
return {
data: res.Body,
block_md: this._get_store_block_md(block_md, res),
Expand Down Expand Up @@ -193,7 +196,7 @@ class BlockStoreS3 extends BlockStoreBase {
Key: block_key,
Body: data,
Metadata: this.disable_metadata ? undefined : { noobaablockmd: encoded_md },
}).promise();
});
if (options && options.ignore_usage) return;
// return usage count for the object
return this._update_usage({
Expand Down Expand Up @@ -242,7 +245,7 @@ class BlockStoreS3 extends BlockStoreBase {
Metadata: this.disable_metadata ? undefined : {
[this.usage_md_key]: usage_data
},
}).promise();
});
// if our target bucket returns version ids that means versioning is enabled
// and for the usage file that we keep replacing we want to keep only the latest
// so we delete the past versions of the usage file.
Expand All @@ -262,12 +265,12 @@ class BlockStoreS3 extends BlockStoreBase {
await this.s3cloud.deleteObjectTagging({
Bucket: this.cloud_info.target_bucket,
Key: block_key
}).promise();
});
} else {
await this.s3cloud.deleteObject({
Bucket: this.cloud_info.target_bucket,
Key: block_key
}).promise();
});
}
} catch (err) {
// NoSuchKey is expected
Expand Down Expand Up @@ -309,7 +312,7 @@ class BlockStoreS3 extends BlockStoreBase {
Delimiter: '/',
KeyMarker: key_marker,
VersionIdMarker: version_marker,
}).promise();
});
is_truncated = res.IsTruncated;
key_marker = res.NextKeyMarker;
version_marker = res.NextVersionIdMarker;
Expand All @@ -322,7 +325,7 @@ class BlockStoreS3 extends BlockStoreBase {
await this.s3cloud.deleteObjects({
Bucket: this.cloud_info.target_bucket,
Delete: { Objects: delete_list },
}).promise();
});
}
}
}
Expand All @@ -344,15 +347,15 @@ class BlockStoreS3 extends BlockStoreBase {
Bucket: this.cloud_info.target_bucket,
KeyMarker: key_marker,
VersionIdMarker: version_marker
}).promise();
});
const del_objs = list_res.Versions.map(ver => ({ Key: ver.Key, VersionId: ver.VersionId }));
if (del_objs.length > 0) {
await this.s3cloud.deleteObjects({
Bucket: this.cloud_info.target_bucket,
Delete: {
Objects: del_objs,
}
}).promise();
});
total += del_objs.length;
}

Expand Down Expand Up @@ -390,7 +393,7 @@ class BlockStoreS3 extends BlockStoreBase {
Key: this._block_key(block_id)
}))
}
}).promise();
});
if (res.Errors) {
for (const delete_error of res.Errors) {
const block_id = this._block_id_from_key(delete_error.Key);
Expand Down Expand Up @@ -421,7 +424,7 @@ class BlockStoreS3 extends BlockStoreBase {
const res = await this.s3cloud.headObject({
Bucket: this.cloud_info.target_bucket,
Key: this._block_key(block_id),
}).promise();
});
const noobaablockmd = res.Metadata.noobaablockmd || res.Metadata.noobaa_block_md;
const md_size = (noobaablockmd && noobaablockmd.length) || 0;
usage.size += Number(res.ContentLength) + md_size;
Expand Down
Loading
Loading