feat: add function to allow user to set destination in transfer manager (#2497)

ddelgrosso1 · web-flow · commit dc1e488b50dd · 2024-07-15T10:06:36.000-04:00
* feat: add function to allow user to set destination in transfer manager

* lint
diff --git a/src/transfer-manager.ts b/src/transfer-manager.ts
@@ -94,6 +94,10 @@ const GCCL_GCS_CMD_FEATURE = {
 
 export interface UploadManyFilesOptions {
   concurrencyLimit?: number;
+  customDestinationBuilder?(
+    path: string,
+    options: UploadManyFilesOptions
+  ): string;
   skipIfExists?: boolean;
   prefix?: string;
   passthroughOptions?: Omit<UploadOptions, 'destination'>;
@@ -411,6 +415,8 @@ export class TransferManager {
    * @typedef {object} UploadManyFilesOptions
    * @property {number} [concurrencyLimit] The number of concurrently executing promises
    * to use when uploading the files.
+   * @property {Function} [customDestinationBuilder] A fuction that will take the current path of a local file
+   * and return a string representing a custom path to be used to upload the file to GCS.
    * @property {boolean} [skipIfExists] Do not upload the file if it already exists in
    * the bucket. This will set the precondition ifGenerationMatch = 0.
    * @property {string} [prefix] A prefix to append to all of the uploaded files.
@@ -490,9 +496,9 @@ export class TransferManager {
         [GCCL_GCS_CMD_KEY]: GCCL_GCS_CMD_FEATURE.UPLOAD_MANY,
       };
 
-      passThroughOptionsCopy.destination = filePath
-        .split(path.sep)
-        .join(path.posix.sep);
+      passThroughOptionsCopy.destination = options.customDestinationBuilder
+        ? options.customDestinationBuilder(filePath, options)
+        : filePath.split(path.sep).join(path.posix.sep);
       if (options.prefix) {
         passThroughOptionsCopy.destination = path.posix.join(
           ...options.prefix.split(path.sep),
diff --git a/test/transfer-manager.ts b/test/transfer-manager.ts
@@ -26,6 +26,7 @@ import {
   MultiPartUploadError,
   MultiPartUploadHelper,
   UploadOptions,
+  UploadManyFilesOptions,
   TransferManager,
   Storage,
   DownloadResponse,
@@ -173,6 +174,24 @@ describe('Transfer Manager', () => {
 
       await transferManager.uploadManyFiles([filePath]);
     });
+
+    it('allows the user to apply a custom destination transformation when supplied a custom function', async () => {
+      const paths = ['a', 'b', 'foo/bar', 'bar.txt'];
+      const expected = ['foo/a', 'b/bar', 'foo/foo/bar', 'bar.txt/bar'];
+      sandbox.stub(bucket, 'upload').callsFake((path, options) => {
+        const uploadOpts = options as UploadOptions;
+        assert(expected.includes(uploadOpts.destination as string));
+      });
+
+      let callCount = 0;
+      const transformationFunc = (path: string) => {
+        assert.strictEqual(path, paths[callCount]);
+        return expected[callCount++];
+      };
+      await transferManager.uploadManyFiles(paths, {
+        customDestinationBuilder: transformationFunc,
+      });
+    });
   });
 
   describe('downloadManyFiles', () => {
