feat!: create-manifest allows flatten as optional (#159)

In order to be able to not flatten by default the the target path when
creating a manifest, a new argument `--flatten` had been added to the
`create-manifest` command.

Co-authored-by: Paul Fouquet <pfouquet@linz.govt.nz>
Co-authored-by: Megan Davidson <mdavidson@linz.govt.nz>

package.json

@@ -12,6 +12,7 @@
     "organization": true
   },
   "devDependencies": {
+    "@chunkd/source-memory": "^10.0.0",
     "@types/node": "^18.7.11",
     "@types/ospec": "^4.0.4",
     "esno": "^0.16.3",

src/commands/create-manifest/__test__/create-manifest.test.ts

@@ -0,0 +1,49 @@
+import { fsa } from '@chunkd/fs';
+import { FsMemory } from '@chunkd/source-memory';
+import o from 'ospec';
+import { createManifest } from '../create-manifest.js';
+
+o.spec('argoLocation', () => {
+  o.beforeEach(() => {
+    memory.files.clear();
+  });
+  const memory = new FsMemory();
+  fsa.register('memory://', memory);
+  o.only('should copy to the target location', async () => {
+    await Promise.all([
+      fsa.write('memory://source/topographic.json', Buffer.from(JSON.stringify({ test: true }))),
+      fsa.write('memory://source/foo/bar/topographic.png', Buffer.from('test')),
+    ]);
+
+    const outputFiles = await createManifest('memory://source/', 'memory://target/', { flatten: true });
+    o(outputFiles[0]).deepEquals([
+      {
+        source: 'memory://source/topographic.json',
+        target: 'memory://target/topographic.json',
+      },
+      {
+        source: 'memory://source/foo/bar/topographic.png',
+        target: 'memory://target/topographic.png',
+      },
+    ]);
+  });
+
+  o.only('should copy to the target location without flattening', async () => {
+    await Promise.all([
+      fsa.write('memory://source/topographic.json', Buffer.from(JSON.stringify({ test: true }))),
+      fsa.write('memory://source/foo/bar/topographic.png', Buffer.from('test')),
+    ]);
+
+    const outputFiles = await createManifest('memory://source/', 'memory://target/sub/', { flatten: false });
+    o(outputFiles[0]).deepEquals([
+      {
+        source: 'memory://source/topographic.json',
+        target: 'memory://target/sub/topographic.json',
+      },
+      {
+        source: 'memory://source/foo/bar/topographic.png',
+        target: 'memory://target/sub/foo/bar/topographic.png',
+      },
+    ]);
+  });
+});

src/commands/create-manifest/create-manifest.ts

@@ -1,11 +1,11 @@
 import { fsa } from '@chunkd/fs';
-import { command, number, option, optional, restPositionals, string } from 'cmd-ts';
+import { command, flag, number, option, optional, restPositionals, string } from 'cmd-ts';
 import { createHash } from 'crypto';
 import path from 'path';
 import { gzipSync } from 'zlib';
 import { getActionLocation } from '../../utils/action.storage.js';
-import { getFiles } from '../../utils/chunk.js';
 import { ActionCopy } from '../../utils/actions.js';
+import { FileFilter, getFiles } from '../../utils/chunk.js';
 import { config, registerCli, verbose } from '../common.js';
 
 export const commandCreateManifest = command({
@@ -14,6 +14,7 @@ export const commandCreateManifest = command({
   args: {
     config,
     verbose,
+    flatten: flag({ long: 'flatten', description: 'Flatten the files in the target location' }),
     include: option({ type: optional(string), long: 'include', description: 'Include files eg ".*.tiff?$"' }),
     exclude: option({ type: optional(string), long: 'exclude', description: 'Exclude files eg ".*.prj$"' }),
     groupSize: option({
@@ -38,24 +39,14 @@ export const commandCreateManifest = command({
   },
   handler: async (args) => {
     registerCli(args);
-    const actionLocation = getActionLocation();
 
     const outputCopy: string[] = [];
 
     const targetPath: string = args.target;
-
+    const actionLocation = getActionLocation();
     for (const source of args.source) {
-      const outputFiles = await getFiles([source], args);
-
-      for (const chunk of outputFiles) {
-        const current: { source: string; target: string }[] = [];
-
-        for (const filePath of chunk) {
-          const baseFile = path.basename(filePath);
-          const target = fsa.joinAll(targetPath, baseFile);
-          current.push({ source: filePath, target });
-        }
-
+      const outputFiles = await createManifest(source, targetPath, args);
+      for (const current of outputFiles) {
         const outBuf = Buffer.from(JSON.stringify(current));
         const targetHash = createHash('sha256').update(outBuf).digest('base64url');
 
@@ -70,7 +61,32 @@ export const commandCreateManifest = command({
         }
       }
     }
-
     await fsa.write(args.output, JSON.stringify(outputCopy));
   },
 });
+
+export type SourceTarget = { source: string; target: string };
+export type ManifestFilter = FileFilter & { flatten: boolean };
+
+export async function createManifest(
+  source: string,
+  targetPath: string,
+  args: ManifestFilter,
+): Promise<SourceTarget[][]> {
+  const outputFiles = await getFiles([source], args);
+  const outputCopy: SourceTarget[][] = [];
+
+  for (const chunk of outputFiles) {
+    const current: SourceTarget[] = [];
+
+    for (const filePath of chunk) {
+      const baseFile = args.flatten ? path.basename(filePath) : filePath.slice(source.length);
+      const target = fsa.joinAll(targetPath, baseFile);
+
+      current.push({ source: filePath, target });
+    }
+    outputCopy.push(current);
+  }
+
+  return outputCopy;
+}

src/utils/chunk.ts

@@ -40,11 +40,8 @@ export function chunkFiles(values: FileSizeInfo[], count: number, size: number):
   if (current.length > 0) output.push(current);
   return output;
 }
-
-export async function getFiles(
-  paths: string[],
-  args: { include?: string; exclude?: string; limit?: number; group?: number; groupSize?: string },
-): Promise<string[][]> {
+export type FileFilter = { include?: string; exclude?: string; limit?: number; group?: number; groupSize?: string };
+export async function getFiles(paths: string[], args: FileFilter): Promise<string[][]> {
   const limit = args.limit ?? -1; // no limit by default
   const maxSize = parseSize(args.groupSize ?? '-1');
   const maxLength = args.group ?? -1;

yarn.lock

@@ -56,6 +56,13 @@
     "@chunkd/core" "^10.0.0"
     node-fetch "^3.1.0"
 
+"@chunkd/source-memory@^10.0.0":
+  version "10.0.0"
+  resolved "https://registry.yarnpkg.com/@chunkd/source-memory/-/source-memory-10.0.0.tgz#17880ca2161bd3fc5af042d6b392cbcf42e36b0a"
+  integrity sha512-0s+rCuFpeJ2cO3Zjrhsxr08H0ujeVDnXl6VMWaOX97jDGZUGR4Nbiy3Azx3tfvL8X5LjeMuxXrs0e1gm70UpMw==
+  dependencies:
+    "@chunkd/core" "^10.0.0"
+
 "@cogeotiff/core@^7.2.1":
   version "7.2.1"
   resolved "https://registry.yarnpkg.com/@cogeotiff/core/-/core-7.2.1.tgz#96e62532613a5ae46781bb614a8e652b1750cecb"