Skip to content

Commit 80d4fc2

Browse files
naveenpaul1naveenpaul1
committed
SDK | Upgrade AWS SDK to v3 - Block Store S3
Signed-off-by: naveenpaul1 <napaul@redhat.com>
1 parent bb57299 commit 80d4fc2

File tree

5 files changed

+219
-77
lines changed

5 files changed

+219
-77
lines changed

‎package-lock.json

Lines changed: 142 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,8 @@
7474
"dependencies": {
7575
"@aws-sdk/client-s3": "3.826.0",
7676
"@aws-sdk/client-sts": "3.826.0",
77+
"@aws-sdk/credential-providers": "3.826.0",
78+
"@aws-sdk/s3-request-presigner": "3.826.0",
7779
"@azure/identity": "4.10.0",
7880
"@azure/monitor-query": "1.3.2",
7981
"@azure/storage-blob": "12.27.0",

‎src/agent/block_store_services/block_store_s3.js

Lines changed: 34 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
'use strict';
33

44
const _ = require('lodash');
5-
const AWS = require('aws-sdk');
5+
const { S3 } = require('@aws-sdk/client-s3');
66

77
const config = require('../../../config');
88
const P = require('../../util/promise');
@@ -12,6 +12,7 @@ const cloud_utils = require('../../util/cloud_utils');
1212
const size_utils = require('../../util/size_utils');
1313
const BlockStoreBase = require('./block_store_base').BlockStoreBase;
1414
const { RpcError } = require('../../rpc');
15+
const { NodeHttpHandler } = require("@smithy/node-http-handler");
1516

1617

1718
const DEFAULT_REGION = 'us-east-1';
@@ -39,33 +40,35 @@ class BlockStoreS3 extends BlockStoreBase {
3940
RoleSessionName: 'block_store_operations'
4041
};
4142
} else {
42-
this.s3cloud = new AWS.S3({
43+
this.s3cloud = new S3({
4344
endpoint: endpoint,
44-
accessKeyId: this.cloud_info.access_keys.access_key.unwrap(),
45-
secretAccessKey: this.cloud_info.access_keys.secret_key.unwrap(),
46-
s3ForcePathStyle: true,
47-
signatureVersion: cloud_utils.get_s3_endpoint_signature_ver(endpoint, this.cloud_info.auth_method),
45+
credentials: {
46+
accessKeyId: this.cloud_info.access_keys.access_key.unwrap(),
47+
secretAccessKey: this.cloud_info.access_keys.secret_key.unwrap(),
48+
},
49+
forcePathStyle: true,
4850
region: DEFAULT_REGION,
49-
httpOptions: {
50-
agent: http_utils.get_default_agent(endpoint)
51-
}
51+
requestHandler: new NodeHttpHandler({
52+
httpsAgent: http_utils.get_default_agent(endpoint)
53+
}),
5254
});
5355
}
5456
} else {
5557
this.disable_delegation = config.EXPERIMENTAL_DISABLE_S3_COMPATIBLE_DELEGATION[this.cloud_info.endpoint_type] ||
5658
config.EXPERIMENTAL_DISABLE_S3_COMPATIBLE_DELEGATION.DEFAULT;
5759
this.disable_metadata = config.EXPERIMENTAL_DISABLE_S3_COMPATIBLE_METADATA[this.cloud_info.endpoint_type] ||
5860
config.EXPERIMENTAL_DISABLE_S3_COMPATIBLE_METADATA.DEFAULT;
59-
this.s3cloud = new AWS.S3({
61+
this.s3cloud = new S3({
6062
endpoint: endpoint,
61-
s3ForcePathStyle: true,
62-
accessKeyId: this.cloud_info.access_keys.access_key.unwrap(),
63-
secretAccessKey: this.cloud_info.access_keys.secret_key.unwrap(),
64-
signatureVersion: cloud_utils.get_s3_endpoint_signature_ver(endpoint, this.cloud_info.auth_method),
65-
s3DisableBodySigning: cloud_utils.disable_s3_compatible_bodysigning(endpoint),
66-
httpOptions: {
67-
agent: http_utils.get_unsecured_agent(endpoint)
68-
}
63+
forcePathStyle: true,
64+
credentials: {
65+
accessKeyId: this.cloud_info.access_keys.access_key.unwrap(),
66+
secretAccessKey: this.cloud_info.access_keys.secret_key.unwrap(),
67+
},
68+
applyChecksum: cloud_utils.disable_s3_compatible_bodysigning(endpoint),
69+
requestHandler: new NodeHttpHandler({
70+
httpsAgent: http_utils.get_unsecured_agent(endpoint)
71+
}),
6972
});
7073
}
7174

@@ -79,7 +82,7 @@ class BlockStoreS3 extends BlockStoreBase {
7982
const res = await this.s3cloud.getObject({
8083
Bucket: this.cloud_info.target_bucket,
8184
Key: this.usage_path,
82-
}).promise();
85+
});
8386

8487
const usage_data = this.disable_metadata ?
8588
res.Body.toString() :
@@ -106,7 +109,7 @@ class BlockStoreS3 extends BlockStoreBase {
106109
const res = await this.s3cloud.headObject({
107110
Bucket: this.cloud_info.target_bucket,
108111
Key: this._block_key(block_md.id),
109-
}).promise();
112+
});
110113
return {
111114
block_md: this._get_store_block_md(block_md, res),
112115
store_md5: res.ETag.toUpperCase(),
@@ -164,7 +167,7 @@ class BlockStoreS3 extends BlockStoreBase {
164167
const res = await this.s3cloud.getObject({
165168
Bucket: this.cloud_info.target_bucket,
166169
Key: this._block_key(block_md.id),
167-
}).promise();
170+
});
168171
return {
169172
data: res.Body,
170173
block_md: this._get_store_block_md(block_md, res),
@@ -193,7 +196,7 @@ class BlockStoreS3 extends BlockStoreBase {
193196
Key: block_key,
194197
Body: data,
195198
Metadata: this.disable_metadata ? undefined : { noobaablockmd: encoded_md },
196-
}).promise();
199+
});
197200
if (options && options.ignore_usage) return;
198201
// return usage count for the object
199202
return this._update_usage({
@@ -242,7 +245,7 @@ class BlockStoreS3 extends BlockStoreBase {
242245
Metadata: this.disable_metadata ? undefined : {
243246
[this.usage_md_key]: usage_data
244247
},
245-
}).promise();
248+
});
246249
// if our target bucket returns version ids that means versioning is enabled
247250
// and for the usage file that we keep replacing we want to keep only the latest
248251
// so we delete the past versions of the usage file.
@@ -262,12 +265,12 @@ class BlockStoreS3 extends BlockStoreBase {
262265
await this.s3cloud.deleteObjectTagging({
263266
Bucket: this.cloud_info.target_bucket,
264267
Key: block_key
265-
}).promise();
268+
});
266269
} else {
267270
await this.s3cloud.deleteObject({
268271
Bucket: this.cloud_info.target_bucket,
269272
Key: block_key
270-
}).promise();
273+
});
271274
}
272275
} catch (err) {
273276
// NoSuchKey is expected
@@ -309,7 +312,7 @@ class BlockStoreS3 extends BlockStoreBase {
309312
Delimiter: '/',
310313
KeyMarker: key_marker,
311314
VersionIdMarker: version_marker,
312-
}).promise();
315+
});
313316
is_truncated = res.IsTruncated;
314317
key_marker = res.NextKeyMarker;
315318
version_marker = res.NextVersionIdMarker;
@@ -322,7 +325,7 @@ class BlockStoreS3 extends BlockStoreBase {
322325
await this.s3cloud.deleteObjects({
323326
Bucket: this.cloud_info.target_bucket,
324327
Delete: { Objects: delete_list },
325-
}).promise();
328+
});
326329
}
327330
}
328331
}
@@ -344,15 +347,15 @@ class BlockStoreS3 extends BlockStoreBase {
344347
Bucket: this.cloud_info.target_bucket,
345348
KeyMarker: key_marker,
346349
VersionIdMarker: version_marker
347-
}).promise();
350+
});
348351
const del_objs = list_res.Versions.map(ver => ({ Key: ver.Key, VersionId: ver.VersionId }));
349352
if (del_objs.length > 0) {
350353
await this.s3cloud.deleteObjects({
351354
Bucket: this.cloud_info.target_bucket,
352355
Delete: {
353356
Objects: del_objs,
354357
}
355-
}).promise();
358+
});
356359
total += del_objs.length;
357360
}
358361

@@ -390,7 +393,7 @@ class BlockStoreS3 extends BlockStoreBase {
390393
Key: this._block_key(block_id)
391394
}))
392395
}
393-
}).promise();
396+
});
394397
if (res.Errors) {
395398
for (const delete_error of res.Errors) {
396399
const block_id = this._block_id_from_key(delete_error.Key);
@@ -421,7 +424,7 @@ class BlockStoreS3 extends BlockStoreBase {
421424
const res = await this.s3cloud.headObject({
422425
Bucket: this.cloud_info.target_bucket,
423426
Key: this._block_key(block_id),
424-
}).promise();
427+
});
425428
const noobaablockmd = res.Metadata.noobaablockmd || res.Metadata.noobaa_block_md;
426429
const md_size = (noobaablockmd && noobaablockmd.length) || 0;
427430
usage.size += Number(res.ContentLength) + md_size;

0 commit comments

Comments
 (0)