Merge pull request #7987 from naveenpaul1/nsfs-multipart-upload-list

NSFS | Add support for list multiple uploads in namespace_fs, pagination not handled

src/sdk/namespace_fs.js

@@ -1505,10 +1505,43 @@ class NamespaceFS {
     //////////////////////
 
     async list_uploads(params, object_sdk) {
-        // TODO for now we do not support listing of multipart uploads
+        // TODO: Need to support pagination.
+        const fs_context = this.prepare_fs_context(object_sdk);
+        await this._load_bucket(params, fs_context);
+        const mpu_root_path = this._mpu_root_path();
+        await this._check_path_in_bucket_boundaries(fs_context, mpu_root_path);
+        const multipart_upload_dirs = await nb_native().fs.readdir(fs_context, mpu_root_path);
+        const common_prefixes_set = new Set();
+        const multipart_uploads = await P.map(multipart_upload_dirs, async obj => {
+            const create_path = path.join(mpu_root_path, obj.name, 'create_object_upload');
+            const { data: create_params_buffer } = await nb_native().fs.readFile(
+                fs_context,
+                create_path
+            );
+            const create_params_parsed = JSON.parse(create_params_buffer.toString());
+
+            // dont add keys that dont start with the prefix
+            if (params.prefix && !create_params_parsed.key.startsWith(params.prefix)) {
+                return undefined;
+            }
+
+            // common_prefix contains (if there are any) prefixes between the provide prefix
+            // and the next occurrence of the string specified by the delimiter
+            if (params.delimiter) {
+                const start_idx = params.prefix ? params.prefix.length : 0;
+                const delimiter_idx = create_params_parsed.key.indexOf(params.delimiter, start_idx);
+                if (delimiter_idx > 0) {
+                    common_prefixes_set.add(create_params_parsed.key.substring(0, delimiter_idx + 1));
+                    // if key has common prefix it should not be returned as an upload object 
+                    return undefined;
+                }
+            }
+            const stat = await nb_native().fs.stat(fs_context, create_path);
+            return this._get_mpu_info(create_params_parsed, stat);
+        });
         return {
-            objects: [],
-            common_prefixes: [],
+            objects: _.compact(multipart_uploads),
+            common_prefixes: [...common_prefixes_set],
             is_truncated: false,
             next_marker: undefined,
             next_upload_id_marker: undefined,
@@ -2248,7 +2281,7 @@ class NamespaceFS {
      * @param {nb.NativeFSStats} stat 
      * @returns {nb.ObjectInfo}
      */
-     _get_object_info(bucket, key, stat, return_version_id, isDir, is_latest = true) {
+    _get_object_info(bucket, key, stat, return_version_id, isDir, is_latest = true) {
         const etag = this._get_etag(stat);
         const create_time = stat.mtime.getTime();
         const encryption = this._get_encryption_info(stat);
@@ -2336,11 +2369,28 @@ class NamespaceFS {
         }
     }
 
-    _mpu_path(params) {
+    _get_mpu_info(create_params, stat) {
+        return {
+            obj_id: create_params.obj_id,
+            bucket: create_params.bucket,
+            key: create_params.key,
+            storage_class: create_params.storage_class,
+            create_time: stat.mtime.getTime(),
+            upload_started: stat.mtime.getTime(),
+            content_type: create_params.content_type,
+        };
+    }
+
+    _mpu_root_path() {
         return path.join(
             this.bucket_path,
             this.get_bucket_tmpdir(),
-            'multipart-uploads',
+            'multipart-uploads');
+    }
+
+    _mpu_path(params) {
+        return path.join(
+            this._mpu_root_path(),
             params.obj_id
         );
     }

src/test/unit_tests/test_namespace_fs.js

@@ -294,12 +294,12 @@ mocha.describe('namespace_fs', function() {
                 console.log('list_multiparts response', inspect(list_parts_res));
             }
 
-            const list1_res = await ns_src.list_uploads({
+            const list1_res = await ns_tmp.list_uploads({
                 bucket: mpu_bkt,
             }, dummy_object_sdk);
             console.log('list_uploads response', inspect(list1_res));
-            // TODO list_uploads is not implemented
-            assert.deepStrictEqual(list1_res.objects, []);
+            assert.strictEqual(list1_res.objects.length, 1);
+            assert.strictEqual(list1_res.objects[0].key, mpu_key);
 
             const complete_res = await ns_tmp.complete_object_upload({
                 obj_id,
@@ -311,7 +311,7 @@ mocha.describe('namespace_fs', function() {
             if (config.NSFS_CALCULATE_MD5 ||
                 ns_tmp.force_md5_etag || dummy_object_sdk.requesting_account.force_md5_etag) xattr[XATTR_MD5_KEY] = complete_res.etag;
 
-            const list2_res = await ns_src.list_uploads({
+            const list2_res = await ns_tmp.list_uploads({
                 bucket: mpu_bkt,
             }, dummy_object_sdk);
             console.log('list_uploads response', inspect(list2_res));
@@ -341,6 +341,75 @@ mocha.describe('namespace_fs', function() {
         });
     });
 
+    mocha.describe('list multipart upload', function() {
+        const mpu_key = 'mpu_upload';
+        const mpu_key2 = 'multipart/mpu_upload';
+        const mpu_key3 = 'multipart/tmp/mpu_upload';
+        const prefix = 'multipart/';
+        const delimiter = '/';
+        let create_res1;
+        let create_res2;
+        let create_res3;
+
+        const xattr = { key: 'value', key2: 'value2' };
+
+        mocha.before(async function() {
+            create_res1 = await ns_tmp.create_object_upload({
+                bucket: mpu_bkt,
+                key: mpu_key,
+                xattr,
+            }, dummy_object_sdk);
+
+            create_res2 = await ns_tmp.create_object_upload({
+                bucket: mpu_bkt,
+                key: mpu_key2,
+                xattr,
+            }, dummy_object_sdk);
+
+            create_res3 = await ns_tmp.create_object_upload({
+                bucket: mpu_bkt,
+                key: mpu_key3,
+                xattr,
+            }, dummy_object_sdk);
+        });
+
+        mocha.after(async function() {
+            await ns_tmp.abort_object_upload({bucket: mpu_bkt, obj_id: create_res1.obj_id}, dummy_object_sdk);
+            await ns_tmp.abort_object_upload({bucket: mpu_bkt, obj_id: create_res2.obj_id}, dummy_object_sdk);
+            await ns_tmp.abort_object_upload({bucket: mpu_bkt, obj_id: create_res3.obj_id}, dummy_object_sdk);
+        });
+
+        mocha.it('multipartUploadList without prefix or delimiter', async function() {
+            const res = await ns_tmp.list_uploads({bucket: mpu_bkt}, dummy_object_sdk);
+            //should return all three items
+            assert.strictEqual(res.objects.length, 3);
+        });
+
+        mocha.it('multipartUploadList with prefix', async function() {
+            const res = await ns_tmp.list_uploads({bucket: mpu_bkt, prefix}, dummy_object_sdk);
+            //should only include keys that start with prefix
+            assert.strictEqual(res.objects.length, 2);
+        });
+
+        mocha.it('multipartUploadList with delimiter', async function() {
+            const res = await ns_tmp.list_uploads({bucket: mpu_bkt, delimiter}, dummy_object_sdk);
+            assert.strictEqual(res.objects.length, 1);
+            // should combine both items with key that starts with multipart/ into common_prefixes
+            assert.strictEqual(res.common_prefixes.length, 1);
+            assert.strictEqual(res.common_prefixes[0], prefix);
+        });
+
+        mocha.it('multipartUploadList with prefix and delimiter', async function() {
+            const res = await ns_tmp.list_uploads({bucket: mpu_bkt, prefix, delimiter}, dummy_object_sdk);
+            // multipart/mpu_upload doesnt have delimiter after prefix
+            assert.strictEqual(res.objects.length, 1);
+            assert.strictEqual(res.objects[0].key, mpu_key2);
+            //first delimiter after prifix
+            assert.strictEqual(res.common_prefixes.length, 1);
+            assert.strictEqual(res.common_prefixes[0], 'multipart/tmp/');
+        });
+    });
+
     mocha.describe('delete_object', function() {
 
         const dir_1 = '/a/b/c/';