diff --git a/client/dive-common/apispec.ts b/client/dive-common/apispec.ts index 3323fa63d..abac8415a 100644 --- a/client/dive-common/apispec.ts +++ b/client/dive-common/apispec.ts @@ -161,7 +161,7 @@ interface Api { saveAttributeTrackFilters(datasetId: string, args: SaveAttributeTrackFilterArgs): Promise; // Non-Endpoint shared functions - openFromDisk(datasetType: DatasetType | 'calibration' | 'annotation' | 'text' | 'zip', directory?: boolean): + openFromDisk(datasetType: DatasetType | 'bulk' | 'calibration' | 'annotation' | 'text' | 'zip', directory?: boolean): Promise<{canceled?: boolean; filePaths: string[]; fileList?: File[]; root?: string}>; getTiles?(itemId: string, projection?: string): Promise; // eslint-disable-next-line @typescript-eslint/no-explicit-any diff --git a/client/dive-common/components/ImportButton.vue b/client/dive-common/components/ImportButton.vue index dc1c026ea..739fe2640 100644 --- a/client/dive-common/components/ImportButton.vue +++ b/client/dive-common/components/ImportButton.vue @@ -20,7 +20,7 @@ export default defineComponent({ required: true, }, openType: { - type: String as PropType, + type: String as PropType, required: true, }, multiCamImport: { //TODO: Temporarily used to hide the stereo settings from users diff --git a/client/platform/desktop/backend/ipcService.ts b/client/platform/desktop/backend/ipcService.ts index b26666b2c..f9ca09dc9 100644 --- a/client/platform/desktop/backend/ipcService.ts +++ b/client/platform/desktop/backend/ipcService.ts @@ -70,6 +70,11 @@ export default function register() { return defaults; }); + ipcMain.handle('bulk-import-media', async (event, { path }: { path: string }) => { + const results = await common.bulkMediaImport(path); + return results; + }); + ipcMain.handle('import-media', async (event, { path }: { path: string }) => { const ret = await common.beginMediaImport(path); return ret; diff --git a/client/platform/desktop/backend/native/common.spec.ts b/client/platform/desktop/backend/native/common.spec.ts index 4acadeaff..a84f6e56a 100644 --- a/client/platform/desktop/backend/native/common.spec.ts +++ b/client/platform/desktop/backend/native/common.spec.ts @@ -448,7 +448,7 @@ describe('native.common', () => { name: 'myproject1_name', createdAt: (new Date()).toString(), originalBasePath: '/foo/bar/baz', - id: 'myproject1', + id: 'myproject1_name_tktfgyv2g9', originalImageFiles: [], transcodedImageFiles: [], originalVideoFile: '', @@ -461,7 +461,7 @@ describe('native.common', () => { const contents = fs.readdirSync(result); expect(stat.isDirectory()).toBe(true); expect(contents).toEqual([]); - expect(result).toMatch(/DIVE_Jobs\/myproject1_name_mypipeline\.pipe_/); + expect(result).toMatch(/DIVE_Jobs\/myproject1_name_tktfgyv2g9_mypipeline\.pipe_/); }); it('beginMediaImport image sequence success', async () => { diff --git a/client/platform/desktop/backend/native/common.ts b/client/platform/desktop/backend/native/common.ts index f2c3bdb70..858bfa805 100644 --- a/client/platform/desktop/backend/native/common.ts +++ b/client/platform/desktop/backend/native/common.ts @@ -801,6 +801,59 @@ async function findTrackandMetaFileinFolder(path: string) { return { trackFileAbsPath, metaFileAbsPath }; } +/** + * Attempt a media import on the provided path, which may or may not be a valid dataset. + */ +async function attemptMediaImport(path: string) { + try { + // Must await here, as otherwise the try/catch isn't correctly executed. + return await beginMediaImport(path); + } catch (e) { + console.warn( + `*** Failed to import at path "${path}", with message: "${(e as Error).message}".` + + ' This is expected if this file or directory does not contain a dataset.', + ); + } + + return undefined; +} + +/** + * Recursively import all datasets in this directory, using a "breadth-first" approach. + * This function only recurses into a directory if the import of that directory fails. + */ +async function bulkMediaImport(path: string): Promise { + const children = await fs.readdir(path, { withFileTypes: true }); + const results: {path: fs.Dirent, result: DesktopMediaImportResponse | undefined}[] = []; + + // Use a for-of loop, to run imports sequentially. If run concurrently, they can fail behind the scenes. + // eslint-disable-next-line no-restricted-syntax + for (const dirent of children) { + // eslint-disable-next-line no-await-in-loop + const result = await attemptMediaImport(npath.resolve(path, dirent.name)); + results.push({ + path: dirent, + result, + }); + } + + // Filter successful imports + const importResults = results.filter((r) => r.result !== undefined).map((r) => r.result as DesktopMediaImportResponse); + + // If the result was undefined and was a directory, recurse. + const toRecurse = results.filter((r) => r.result === undefined && r.path.isDirectory()); + + // Use a for-of loop, to run imports sequentially. If run concurrently, they can fail behind the scenes. + // eslint-disable-next-line no-restricted-syntax + for (const r of toRecurse) { + // eslint-disable-next-line no-await-in-loop + const results = await bulkMediaImport(npath.resolve(path, r.path.name)); + importResults.push(...results); + } + + return importResults; +} + /** * Begin a dataset import. */ @@ -1133,6 +1186,7 @@ export { ProjectsFolderName, JobsFolderName, autodiscoverData, + bulkMediaImport, beginMediaImport, dataFileImport, deleteDataset, diff --git a/client/platform/desktop/backend/native/utils.ts b/client/platform/desktop/backend/native/utils.ts index e67636dd6..32be97e5a 100644 --- a/client/platform/desktop/backend/native/utils.ts +++ b/client/platform/desktop/backend/native/utils.ts @@ -91,7 +91,7 @@ async function createWorkingDirectory(settings: Settings, jsonMetaList: JsonMeta const jobFolderPath = path.join(settings.dataPath, JobsFolderName); // eslint won't recognize \. as valid escape // eslint-disable-next-line no-useless-escape - const safeDatasetName = jsonMetaList[0].name.replace(/[\.\s/]+/g, '_'); + const safeDatasetName = jsonMetaList[0].id.replace(/[\.\s/]+/g, '_'); const runFolderName = moment().format(`[${safeDatasetName}_${pipeline}]_MM-DD-yy_hh-mm-ss.SSS`); const runFolderPath = path.join(jobFolderPath, runFolderName); if (!fs.existsSync(jobFolderPath)) { diff --git a/client/platform/desktop/frontend/api.ts b/client/platform/desktop/frontend/api.ts index eb54bbb6e..dc98d2b62 100644 --- a/client/platform/desktop/frontend/api.ts +++ b/client/platform/desktop/frontend/api.ts @@ -26,7 +26,7 @@ import { * Native functions that run entirely in the renderer */ -async function openFromDisk(datasetType: DatasetType | 'calibration' | 'annotation' | 'text', directory = false) { +async function openFromDisk(datasetType: DatasetType | 'bulk' | 'calibration' | 'annotation' | 'text', directory = false) { let filters: FileFilter[] = []; const allFiles = { name: 'All Files', extensions: ['*'] }; if (datasetType === 'video') { @@ -53,7 +53,7 @@ async function openFromDisk(datasetType: DatasetType | 'calibration' | 'annotati allFiles, ]; } - const props = (datasetType === 'image-sequence' || directory) ? 'openDirectory' : 'openFile'; + const props = (['image-sequence', 'bulk'].includes(datasetType) || directory) ? 'openDirectory' : 'openFile'; const results = await dialog.showOpenDialog({ properties: [props], filters, @@ -117,6 +117,10 @@ function importMedia(path: string): Promise { return ipcRenderer.invoke('import-media', { path }); } +function bulkImportMedia(path: string): Promise { + return ipcRenderer.invoke('bulk-import-media', { path }); +} + function deleteDataset(datasetId: string): Promise { return ipcRenderer.invoke('delete-dataset', { datasetId }); } @@ -228,6 +232,7 @@ export { exportConfiguration, finalizeImport, importMedia, + bulkImportMedia, deleteDataset, checkDataset, importAnnotationFile, diff --git a/client/platform/desktop/frontend/components/BulkImportDialog.vue b/client/platform/desktop/frontend/components/BulkImportDialog.vue new file mode 100644 index 000000000..7317d7c99 --- /dev/null +++ b/client/platform/desktop/frontend/components/BulkImportDialog.vue @@ -0,0 +1,193 @@ + + + + + diff --git a/client/platform/desktop/frontend/components/ImportDialog.vue b/client/platform/desktop/frontend/components/ImportDialog.vue index c827c134d..a7b6882ea 100644 --- a/client/platform/desktop/frontend/components/ImportDialog.vue +++ b/client/platform/desktop/frontend/components/ImportDialog.vue @@ -22,6 +22,11 @@ export default defineComponent({ type: Boolean, default: false, }, + // If being embedded into the bulk import dialog + embedded: { + type: Boolean, + default: false, + }, }, setup(props) { const argCopy = ref(cloneDeep(props.importData)); @@ -321,7 +326,7 @@ export default defineComponent({ :disabled="!ready || disabled" @click="$emit('finalize-import', argCopy)" > - Finish Import + {{ embedded ? "Save" : "Finish Import" }} diff --git a/client/platform/desktop/frontend/components/Recent.vue b/client/platform/desktop/frontend/components/Recent.vue index 8e18e5191..28b4c8355 100644 --- a/client/platform/desktop/frontend/components/Recent.vue +++ b/client/platform/desktop/frontend/components/Recent.vue @@ -30,12 +30,14 @@ import { setOrGetConversionJob } from '../store/jobs'; import BrowserLink from './BrowserLink.vue'; import NavigationBar from './NavigationBar.vue'; import ImportDialog from './ImportDialog.vue'; +import BulkImportDialog from './BulkImportDialog.vue'; export default defineComponent({ components: { BrowserLink, ImportButton, ImportDialog, + BulkImportDialog, NavigationBar, ImportMultiCamDialog, TooltipBtn, @@ -44,7 +46,8 @@ export default defineComponent({ setup() { const router = useRouter(); const importMultiCamDialog = ref(false); - const pendingImportPayload: Ref = ref(null); + const pendingImportPayload: Ref = ref(null); + const bulkImport = ref(false); const searchText: Ref = ref(''); const stereo = ref(false); const multiCamOpenType: Ref<'image-sequence'|'video'> = ref('image-sequence'); @@ -54,11 +57,43 @@ export default defineComponent({ error, loading: checkingMedia, request, reset: resetError, } = useRequest(); - async function open(dstype: DatasetType | 'text', directory = false) { + async function open(dstype: DatasetType | 'bulk' | 'text', directory = false) { + bulkImport.value = false; + const ret = await api.openFromDisk(dstype, directory); - if (!ret.canceled) { - pendingImportPayload.value = await request(() => api.importMedia(ret.filePaths[0])); + if (ret.canceled) { + return; + } + + if (dstype !== 'bulk') { + pendingImportPayload.value = [await request(() => api.importMedia(ret.filePaths[0]))]; + return; + } + + bulkImport.value = true; + const foundImports = await request(() => api.bulkImportMedia(ret.filePaths[0])); + if (!foundImports.length) { + prompt({ title: 'No datasets found', text: 'Please check that your import path is correct and try again.', positiveButton: 'Okay' }); + pendingImportPayload.value = null; + return; } + + pendingImportPayload.value = foundImports; + } + + /** Accept args from the dialog, as it may have modified some parts */ + async function finalizeBulkImport(argsArray: DesktopMediaImportResponse[]) { + importing.value = true; + + const imports = await request(async () => Promise.all(argsArray.map((args) => api.finalizeImport(args)))); + pendingImportPayload.value = null; + + imports.forEach(async (jsonMeta) => { + const recentsMeta = await api.loadMetadata(jsonMeta.id); + setRecents(recentsMeta); + }); + + importing.value = false; } /** Accept args from the dialog, as it may have modified some parts */ @@ -89,7 +124,7 @@ export default defineComponent({ async function multiCamImport(args: MultiCamImportArgs) { importMultiCamDialog.value = false; - pendingImportPayload.value = await request(() => api.importMultiCam(args)); + pendingImportPayload.value = [await request(() => api.importMultiCam(args))]; } async function confirmDeleteDataset(datasetId: string, datasetName: string) { @@ -180,6 +215,7 @@ export default defineComponent({ // methods acknowledgeVersion, open, + finalizeBulkImport, finalizeImport, multiCamImport, join, @@ -196,6 +232,7 @@ export default defineComponent({ stereo, filteredRecents, pendingImportPayload, + bulkImport, searchText, error, importing, @@ -215,19 +252,21 @@ export default defineComponent({