Merge pull request #8475 from aspandey/list-bucket-pagination

aspandey · web-flow · commit d37c0fb60367 · 2024-11-03T17:51:35.000+05:30
bucket_api: Pagination support for list_bucket API
diff --git a/1 b/1
diff --git a/src/api/bucket_api.js b/src/api/bucket_api.js
@@ -441,6 +441,17 @@ module.exports = {
 
         list_buckets: {
             method: 'GET',
+            params: {
+                type: 'object',
+                properties: {
+                    continuation_token: { $ref: 'common_api#/definitions/continuation_token' },
+                    max_buckets: {
+                        type: 'integer',
+                        minimum: 1,
+                        maximum: 1000
+                    }
+                }
+            },
             reply: {
                 type: 'object',
                 required: ['buckets'],
@@ -454,10 +465,11 @@ module.exports = {
                                 name: { $ref: 'common_api#/definitions/bucket_name' },
                                 creation_date: {
                                     idate: true
-                                },
+                                }
                             }
                         }
-                    }
+                    },
+                    continuation_token: { $ref: 'common_api#/definitions/continuation_token' }
                 }
             },
             auth: {
diff --git a/src/api/common_api.js b/src/api/common_api.js
@@ -1210,6 +1210,10 @@ module.exports = {
             wrapper: SensitiveString,
         },
 
+        continuation_token: {
+            wrapper: SensitiveString,
+        },
+
         port: {
             type: 'integer',
             minimum: 0,
diff --git a/src/endpoint/s3/ops/s3_get_bucket.js b/src/endpoint/s3/ops/s3_get_bucket.js
@@ -43,7 +43,7 @@ async function get_bucket(req) {
         delimiter: req.query.delimiter,
         limit: Math.min(max_keys_received, 1000),
         key_marker: list_type === '2' ?
-            (cont_tok_to_key_marker(cont_tok) || start_after) : req.query.marker,
+            (s3_utils.cont_tok_to_key_marker(cont_tok) || start_after) : req.query.marker,
     };
 
     if (req.query.get_from_cache) {
@@ -66,7 +66,7 @@ async function get_bucket(req) {
                 ContinuationToken: cont_tok,
                 StartAfter: field_encoder(start_after),
                 KeyCount: reply.objects.length + reply.common_prefixes.length,
-                NextContinuationToken: key_marker_to_cont_tok(
+                NextContinuationToken: s3_utils.key_marker_to_cont_tok(
                     reply.next_marker, reply.objects, reply.is_truncated),
             } : { // list_type v1
                 Marker: req.query.marker || '',
@@ -96,25 +96,6 @@ async function get_bucket(req) {
     };
 }
 
-function cont_tok_to_key_marker(cont_tok) {
-    if (!cont_tok) return;
-    try {
-        const b = Buffer.from(cont_tok, 'base64');
-        const j = JSON.parse(b.toString());
-        return j.key;
-    } catch (err) {
-        throw new S3Error(S3Error.InvalidArgument);
-    }
-}
-
-function key_marker_to_cont_tok(key_marker, objects_arr, is_truncated) {
-    if (!key_marker && !is_truncated) return;
-    // next marker is the key marker we got or the key of the last item in the objects list.
-    const next_marker = key_marker || objects_arr[objects_arr.length - 1].key;
-    const j = JSON.stringify({ key: next_marker });
-    return Buffer.from(j).toString('base64');
-}
-
 function get_object_restore_status(obj, restore_status_requested) {
     if (!restore_status_requested || !obj.restore_status) {
         return;
diff --git a/src/endpoint/s3/ops/s3_get_service.js b/src/endpoint/s3/ops/s3_get_service.js
@@ -7,8 +7,17 @@ const s3_utils = require('../s3_utils');
  * http://docs.aws.amazon.com/AmazonS3/latest/API/RESTServiceGET.html
  */
 async function list_buckets(req) {
-    const reply = await req.object_sdk.list_buckets();
+
+    const params = {
+        continuation_token: s3_utils.cont_tok_to_key_marker(req.query['continuation-token']),
+        max_buckets: req.query['max-buckets'] ? Number(req.query['max-buckets']) : undefined
+    };
+
+    const reply = await req.object_sdk.list_buckets(params);
     const date = s3_utils.format_s3_xml_date(new Date());
+    const bucket_cont_token = s3_utils.key_marker_to_cont_tok(reply.continuation_token, null, false);
+    // const bucket_cont_token = bucket_name_to_cont_token(reply.continuation_token);
+
     return {
         ListAllMyBucketsResult: {
             Owner: s3_utils.DEFAULT_S3_USER,
@@ -17,7 +26,8 @@ async function list_buckets(req) {
                     Name: bucket.name.unwrap(),
                     CreationDate: bucket.creation_date ? s3_utils.format_s3_xml_date(bucket.creation_date) : date,
                 }
-            }))
+            })),
+            ContinuationToken: bucket_cont_token,
         }
     };
 }
diff --git a/src/endpoint/s3/s3_utils.js b/src/endpoint/s3/s3_utils.js
@@ -724,6 +724,42 @@ function parse_restore_request_days(req) {
     return days;
 }
 
+/**
+ * cont_tok_to_key_marker takes an encoded string and decodes it.
+ * cont_tok is the token which represents the next item in
+ * the list which some API returns to user in parts.
+ * @param {string} cont_tok
+ * @returns {string}
+ */
+function cont_tok_to_key_marker(cont_tok) {
+    if (!cont_tok) return;
+    try {
+        const b = Buffer.from(cont_tok, 'base64');
+        const j = JSON.parse(b.toString());
+        return j.key;
+    } catch (err) {
+        throw new S3Error(S3Error.InvalidArgument);
+    }
+}
+
+/**
+ * key_marker_to_cont_tok takes a string and returns an encoded
+ * string. key_marker is the token which represents the next item in
+ * the list which some API returns to user in parts.
+ * @param {string} key_marker
+ * @param {array} objects_arr
+ * @param {boolean} is_truncated
+ * @returns {string}
+ */
+
+function key_marker_to_cont_tok(key_marker, objects_arr, is_truncated) {
+    if (!key_marker && !is_truncated) return;
+    // next marker is the key marker we got or the key of the last item in the objects list.
+    const next_marker = key_marker || (objects_arr && objects_arr.length > 0 ? objects_arr[objects_arr.length - 1].key : undefined);
+    const j = JSON.stringify({ key: next_marker });
+    return Buffer.from(j).toString('base64');
+}
+
 exports.STORAGE_CLASS_STANDARD = STORAGE_CLASS_STANDARD;
 exports.STORAGE_CLASS_GLACIER = STORAGE_CLASS_GLACIER;
 exports.STORAGE_CLASS_GLACIER_IR = STORAGE_CLASS_GLACIER_IR;
@@ -763,3 +799,5 @@ exports.parse_version_id = parse_version_id;
 exports.get_object_owner = get_object_owner;
 exports.get_default_object_owner = get_default_object_owner;
 exports.set_response_supported_storage_classes = set_response_supported_storage_classes;
+exports.cont_tok_to_key_marker = cont_tok_to_key_marker;
+exports.key_marker_to_cont_tok = key_marker_to_cont_tok;
diff --git a/src/sdk/bucketspace_fs.js b/src/sdk/bucketspace_fs.js
@@ -192,7 +192,7 @@ class BucketSpaceFS extends BucketSpaceSimpleFS {
      * @param {nb.ObjectSDK} object_sdk
      * @returns {Promise<object>}
      */
-    async list_buckets(object_sdk) {
+    async list_buckets(params, object_sdk) {
         let bucket_names;
         try {
             bucket_names = await this.config_fs.list_buckets();
diff --git a/src/sdk/bucketspace_nb.js b/src/sdk/bucketspace_nb.js
@@ -32,8 +32,9 @@ class BucketSpaceNB {
     // BUCKET //
     ////////////
 
-    async list_buckets(object_sdk) {
-        const { buckets } = (await this.rpc_client.bucket.list_buckets());
+    async list_buckets(params, object_sdk) {
+        const { buckets, continuation_token} = (await this.rpc_client.bucket.list_buckets(params));
+
         const has_access_buckets = (await P.all(_.map(
             buckets,
             async bucket => {
@@ -43,7 +44,7 @@ class BucketSpaceNB {
                     object_sdk.has_non_nsfs_bucket_access(object_sdk.requesting_account, ns);
                 return has_access_to_bucket && bucket;
             }))).filter(bucket => bucket);
-        return { buckets: has_access_buckets };
+        return { buckets: has_access_buckets, continuation_token};
     }
 
     async read_bucket(params) {
diff --git a/src/sdk/bucketspace_s3.js b/src/sdk/bucketspace_s3.js
@@ -27,7 +27,7 @@ class BucketSpaceS3 {
     // BUCKET //
     ////////////
 
-    async list_buckets() {
+    async list_buckets(params) {
         try {
             console.log(`bss3: list_buckets`);
             const res = await this.s3.listBuckets({});
diff --git a/src/sdk/bucketspace_simple_fs.js b/src/sdk/bucketspace_simple_fs.js
@@ -40,7 +40,7 @@ class BucketSpaceSimpleFS {
     * @param {nb.ObjectSDK} object_sdk
     * @returns {Promise<object>}
     */
-    async list_buckets(object_sdk) {
+    async list_buckets(params, object_sdk) {
         try {
             const entries = await nb_native().fs.readdir(this.fs_context, this.fs_root);
             const dirs_only = entries.filter(entree => native_fs_utils.isDirectory(entree));
diff --git a/src/sdk/nb.d.ts b/src/sdk/nb.d.ts
@@ -821,7 +821,7 @@ interface BucketSpace {
     read_account_by_access_key({ access_key: string }): Promise<any>;
     read_bucket_sdk_info({ name: string }): Promise<any>;
 
-    list_buckets(object_sdk: ObjectSDK): Promise<any>;
+    list_buckets(params: object, object_sdk: ObjectSDK): Promise<any>;
     read_bucket(params: object): Promise<any>;
     create_bucket(params: object, object_sdk: ObjectSDK): Promise<any>;
     delete_bucket(params: object, object_sdk: ObjectSDK): Promise<any>;
@@ -1099,7 +1099,7 @@ interface X509Name {
     O: string;
 }
 
-type select_input_format =  'CSV' | 'JSON' | 'Parquet';
+type select_input_format = 'CSV' | 'JSON' | 'Parquet';
 interface S3SelectOptions {
     query: string;
     input_format: select_input_format;
@@ -1125,7 +1125,7 @@ type NodeCallback<T = void> = (err: Error | null, res?: T) => void;
 type RestoreState = 'CAN_RESTORE' | 'ONGOING' | 'RESTORED';
 
 interface RestoreStatus {
-  state: nb.RestoreState;
-  ongoing?: boolean;
-  expiry_time?: Date;
+    state: nb.RestoreState;
+    ongoing?: boolean;
+    expiry_time?: Date;
 }
diff --git a/src/sdk/object_sdk.js b/src/sdk/object_sdk.js
@@ -877,12 +877,12 @@ class ObjectSDK {
     // BUCKET //
     ////////////
 
-    async list_buckets() {
+    async list_buckets(params = {}) {
         return this._call_op_and_update_stats({
             op_name: 'list_buckets',
             op_func: async () => {
                 const bs = this._get_bucketspace();
-                return bs.list_buckets(this);
+                return bs.list_buckets(params, this);
             },
         });
     }
diff --git a/src/server/system_services/bucket_server.js b/src/server/system_services/bucket_server.js
@@ -999,19 +999,60 @@ async function delete_bucket_lifecycle(req) {
  * LIST_BUCKETS
  *
  */
+
 async function list_buckets(req) {
-    const buckets_by_name = _.filter(
-        req.system.buckets_by_name,
+
+    let next_index = 0;
+    let is_truncated = false;
+
+    let continuation_token = req.rpc_params?.continuation_token;
+    const max_buckets = req.rpc_params?.max_buckets;
+
+    const accessible_bucket_list = system_store.data.buckets.filter(
         async bucket => await req.has_s3_bucket_permission(bucket, "s3:ListBucket", req) && !bucket.deleting
     );
-    return {
-        buckets: _.map(buckets_by_name, function(bucket) {
-            return {
-                name: bucket.name,
-                creation_date: bucket._id.getTimestamp().getTime()
-            };
-        })
-    };
+
+    accessible_bucket_list.sort((a, b) => a.name.unwrap().localeCompare(b.name.unwrap()));
+
+    if (!max_buckets) {
+        const buckets = accessible_bucket_list.map(b => ({
+            name: b.name,
+            creation_date: b._id.getTimestamp().getTime()
+        }));
+        return {
+            buckets,
+        };
+    }
+
+    if (continuation_token) {
+        const index = accessible_bucket_list.findIndex(
+            bucket => bucket.name.unwrap().localeCompare(continuation_token.unwrap()) >= 0
+        );
+        if (index !== -1) {
+            next_index = index + 1;
+        }
+    }
+
+    const paged_bucket_list = accessible_bucket_list.slice(next_index, next_index + max_buckets);
+
+    const buckets = paged_bucket_list.map(bucket => ({
+        name: bucket.name,
+        creation_date: bucket._id.getTimestamp().getTime()
+    }));
+
+    is_truncated = accessible_bucket_list.length > next_index + max_buckets;
+    continuation_token = is_truncated ? accessible_bucket_list[next_index + max_buckets - 1].name : undefined;
+
+    if (continuation_token) {
+        return {
+            buckets,
+            continuation_token,
+        };
+    } else {
+        return {
+            buckets,
+        };
+    }
 }
 
 
diff --git a/src/test/unit_tests/test_bucketspace_fs.js b/src/test/unit_tests/test_bucketspace_fs.js
diff --git a/src/test/unit_tests/test_s3_list_buckets.js b/src/test/unit_tests/test_s3_list_buckets.js
diff --git a/src/test/unit_tests/test_system_servers.js b/src/test/unit_tests/test_system_servers.js
