Skip to content

Commit

Permalink
Implement bulk import (#1457)
Browse files Browse the repository at this point in the history
* Implement bulk import

* Name jobs based on dataset ID, instead of name

* Fix test against pipeline run working directory name

* Wrap text for dataset name and import path

* Remove unnecessary eslint rule ignore

* Display number of currently selected datasets in bulk import dialog

* Handle bulk import with no results

* Fix sporatic failures in bulk import

Run imports sequentially, to prevent ffprobe from failing. ffprobe seems
to fail if run with sufficiently many concurrent processes.

* Split up import dialogs

* Prevent v-data-table overflow on smaller screen sizes
  • Loading branch information
jjnesbitt authored Nov 15, 2024
1 parent 347cb5e commit 6b66a28
Show file tree
Hide file tree
Showing 10 changed files with 353 additions and 34 deletions.
2 changes: 1 addition & 1 deletion client/dive-common/apispec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ interface Api {
saveAttributeTrackFilters(datasetId: string,
args: SaveAttributeTrackFilterArgs): Promise<unknown>;
// Non-Endpoint shared functions
openFromDisk(datasetType: DatasetType | 'calibration' | 'annotation' | 'text' | 'zip', directory?: boolean):
openFromDisk(datasetType: DatasetType | 'bulk' | 'calibration' | 'annotation' | 'text' | 'zip', directory?: boolean):
Promise<{canceled?: boolean; filePaths: string[]; fileList?: File[]; root?: string}>;
getTiles?(itemId: string, projection?: string): Promise<StringKeyObject>;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
Expand Down
2 changes: 1 addition & 1 deletion client/dive-common/components/ImportButton.vue
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ export default defineComponent({
required: true,
},
openType: {
type: String as PropType<DatasetType | 'zip'>,
type: String as PropType<DatasetType | 'zip' | 'bulk'>,
required: true,
},
multiCamImport: { //TODO: Temporarily used to hide the stereo settings from users
Expand Down
5 changes: 5 additions & 0 deletions client/platform/desktop/backend/ipcService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,11 @@ export default function register() {
return defaults;
});

ipcMain.handle('bulk-import-media', async (event, { path }: { path: string }) => {
const results = await common.bulkMediaImport(path);
return results;
});

ipcMain.handle('import-media', async (event, { path }: { path: string }) => {
const ret = await common.beginMediaImport(path);
return ret;
Expand Down
4 changes: 2 additions & 2 deletions client/platform/desktop/backend/native/common.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -448,7 +448,7 @@ describe('native.common', () => {
name: 'myproject1_name',
createdAt: (new Date()).toString(),
originalBasePath: '/foo/bar/baz',
id: 'myproject1',
id: 'myproject1_name_tktfgyv2g9',
originalImageFiles: [],
transcodedImageFiles: [],
originalVideoFile: '',
Expand All @@ -461,7 +461,7 @@ describe('native.common', () => {
const contents = fs.readdirSync(result);
expect(stat.isDirectory()).toBe(true);
expect(contents).toEqual([]);
expect(result).toMatch(/DIVE_Jobs\/myproject1_name_mypipeline\.pipe_/);
expect(result).toMatch(/DIVE_Jobs\/myproject1_name_tktfgyv2g9_mypipeline\.pipe_/);
});

it('beginMediaImport image sequence success', async () => {
Expand Down
54 changes: 54 additions & 0 deletions client/platform/desktop/backend/native/common.ts
Original file line number Diff line number Diff line change
Expand Up @@ -801,6 +801,59 @@ async function findTrackandMetaFileinFolder(path: string) {
return { trackFileAbsPath, metaFileAbsPath };
}

/**
* Attempt a media import on the provided path, which may or may not be a valid dataset.
*/
async function attemptMediaImport(path: string) {
try {
// Must await here, as otherwise the try/catch isn't correctly executed.
return await beginMediaImport(path);
} catch (e) {
console.warn(
`*** Failed to import at path "${path}", with message: "${(e as Error).message}".`
+ ' This is expected if this file or directory does not contain a dataset.',
);
}

return undefined;
}

/**
* Recursively import all datasets in this directory, using a "breadth-first" approach.
* This function only recurses into a directory if the import of that directory fails.
*/
async function bulkMediaImport(path: string): Promise<DesktopMediaImportResponse[]> {
const children = await fs.readdir(path, { withFileTypes: true });
const results: {path: fs.Dirent, result: DesktopMediaImportResponse | undefined}[] = [];

// Use a for-of loop, to run imports sequentially. If run concurrently, they can fail behind the scenes.
// eslint-disable-next-line no-restricted-syntax
for (const dirent of children) {
// eslint-disable-next-line no-await-in-loop
const result = await attemptMediaImport(npath.resolve(path, dirent.name));
results.push({
path: dirent,
result,
});
}

// Filter successful imports
const importResults = results.filter((r) => r.result !== undefined).map((r) => r.result as DesktopMediaImportResponse);

// If the result was undefined and was a directory, recurse.
const toRecurse = results.filter((r) => r.result === undefined && r.path.isDirectory());

// Use a for-of loop, to run imports sequentially. If run concurrently, they can fail behind the scenes.
// eslint-disable-next-line no-restricted-syntax
for (const r of toRecurse) {
// eslint-disable-next-line no-await-in-loop
const results = await bulkMediaImport(npath.resolve(path, r.path.name));
importResults.push(...results);
}

return importResults;
}

/**
* Begin a dataset import.
*/
Expand Down Expand Up @@ -1133,6 +1186,7 @@ export {
ProjectsFolderName,
JobsFolderName,
autodiscoverData,
bulkMediaImport,
beginMediaImport,
dataFileImport,
deleteDataset,
Expand Down
2 changes: 1 addition & 1 deletion client/platform/desktop/backend/native/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ async function createWorkingDirectory(settings: Settings, jsonMetaList: JsonMeta
const jobFolderPath = path.join(settings.dataPath, JobsFolderName);
// eslint won't recognize \. as valid escape
// eslint-disable-next-line no-useless-escape
const safeDatasetName = jsonMetaList[0].name.replace(/[\.\s/]+/g, '_');
const safeDatasetName = jsonMetaList[0].id.replace(/[\.\s/]+/g, '_');
const runFolderName = moment().format(`[${safeDatasetName}_${pipeline}]_MM-DD-yy_hh-mm-ss.SSS`);
const runFolderPath = path.join(jobFolderPath, runFolderName);
if (!fs.existsSync(jobFolderPath)) {
Expand Down
9 changes: 7 additions & 2 deletions client/platform/desktop/frontend/api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import {
* Native functions that run entirely in the renderer
*/

async function openFromDisk(datasetType: DatasetType | 'calibration' | 'annotation' | 'text', directory = false) {
async function openFromDisk(datasetType: DatasetType | 'bulk' | 'calibration' | 'annotation' | 'text', directory = false) {
let filters: FileFilter[] = [];
const allFiles = { name: 'All Files', extensions: ['*'] };
if (datasetType === 'video') {
Expand All @@ -53,7 +53,7 @@ async function openFromDisk(datasetType: DatasetType | 'calibration' | 'annotati
allFiles,
];
}
const props = (datasetType === 'image-sequence' || directory) ? 'openDirectory' : 'openFile';
const props = (['image-sequence', 'bulk'].includes(datasetType) || directory) ? 'openDirectory' : 'openFile';
const results = await dialog.showOpenDialog({
properties: [props],
filters,
Expand Down Expand Up @@ -117,6 +117,10 @@ function importMedia(path: string): Promise<DesktopMediaImportResponse> {
return ipcRenderer.invoke('import-media', { path });
}

function bulkImportMedia(path: string): Promise<DesktopMediaImportResponse[]> {
return ipcRenderer.invoke('bulk-import-media', { path });
}

function deleteDataset(datasetId: string): Promise<boolean> {
return ipcRenderer.invoke('delete-dataset', { datasetId });
}
Expand Down Expand Up @@ -228,6 +232,7 @@ export {
exportConfiguration,
finalizeImport,
importMedia,
bulkImportMedia,
deleteDataset,
checkDataset,
importAnnotationFile,
Expand Down
193 changes: 193 additions & 0 deletions client/platform/desktop/frontend/components/BulkImportDialog.vue
Original file line number Diff line number Diff line change
@@ -0,0 +1,193 @@
<script lang="ts">
import {
defineComponent, ref, PropType,
computed,
} from 'vue';
import { DesktopMediaImportResponse } from 'platform/desktop/constants';
import { clone, cloneDeep } from 'lodash';
import ImportDialog from './ImportDialog.vue';
const headers = [
{
text: 'Dataset Name',
align: 'start',
sortable: false,
value: 'name',
},
{
text: 'Path',
align: 'start',
sortable: false,
value: 'path',
},
{
text: 'Dataset Type',
align: 'start',
sortable: false,
value: 'jsonMeta.type',
width: '150',
},
{
text: 'Config',
align: 'end',
sortable: false,
value: 'config',
},
];
export default defineComponent({
name: 'BulkImportDialog',
components: {
ImportDialog,
},
props: {
importData: {
type: Array as PropType<DesktopMediaImportResponse[]>,
required: true,
},
},
setup(props, ctx) {
// Map imports to include generated "id" field, used in rendering.
const imports = ref(props.importData.map((im) => {
const cloned = cloneDeep(im);
cloned.jsonMeta.id = (Math.random() + 1).toString(36).substring(2);
return cloned;
}));
// The dataset import currently being configured
const currentImport = ref<DesktopMediaImportResponse>();
// Selected state and selected imports maintain the ordering of `imports`
const selectedState = ref(imports.value.map(() => true));
const selectedImports = computed(() => imports.value.filter((_, index) => selectedState.value[index]));
function updateSelected(val: DesktopMediaImportResponse[]) {
selectedState.value = imports.value.map((im) => val.includes(im));
}
function stripId(item: DesktopMediaImportResponse) {
const cloned = cloneDeep(item);
cloned.jsonMeta.id = '';
return item;
}
function finalizeImport() {
const finalImports = imports.value.filter((im) => selectedImports.value.includes(im)).map((im) => stripId(im));
ctx.emit('finalize-import', finalImports);
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
function updateImportConfig(oldItem: DesktopMediaImportResponse, newItem: DesktopMediaImportResponse) {
const itemIndex = imports.value.indexOf(oldItem);
// Need to modify the imports array ref to contain the new item
const newArray = clone(imports.value);
newArray.splice(itemIndex, 1, newItem);
imports.value = newArray;
currentImport.value = undefined;
}
function formatPath(item: DesktopMediaImportResponse) {
let path = item.jsonMeta.originalBasePath;
if (item.jsonMeta.originalVideoFile !== '') {
path = `${path}/${item.jsonMeta.originalVideoFile}`;
}
return path;
}
return {
updateSelected,
updateImportConfig,
formatPath,
imports,
headers,
selectedImports,
currentImport,
finalizeImport,
};
},
});
</script>

<template>
<v-card outlined class="import-card" style="overflow-x: hidden;">
<v-card-title class="text-h5">
Bulk Import (Selecting {{ selectedImports.length }} of {{ imports.length }})
</v-card-title>

<v-dialog :value="currentImport !== undefined" width="800">
<ImportDialog
v-if="currentImport !== undefined"
:import-data="currentImport"
:embedded="true"
@abort="currentImport = undefined"
@finalize-import="updateImportConfig(currentImport, $event)"
/>
</v-dialog>

<v-data-table
:value="selectedImports"
:items="imports"
item-key="jsonMeta.id"
:headers="headers"
show-select
disable-sort
@input="updateSelected"
>
<template #item.name="{ item }">
<div class="text-wrap" style="word-break: break-word;">
{{ item.jsonMeta.name }}
</div>
</template>

<template #item.path="{ item }">
<div class="text-wrap" style="word-break: break-word;">
{{ formatPath(item) }}
</div>
</template>

<template #item.config="{ item }">
<v-btn
icon
:disabled="!selectedImports.includes(item)"
@click="currentImport = item"
>
<v-icon>
mdi-cog
</v-icon>
</v-btn>
</template>
</v-data-table>

<v-card-text>
<div class="d-flex flex-row mt-4">
<v-spacer />
<v-btn
text
outlined
class="mr-5"
@click="$emit('abort')"
>
Cancel
</v-btn>
<v-btn
color="primary"
@click="finalizeImport"
>
Import Selected
</v-btn>
</div>
</v-card-text>
</v-card>
</template>

<style lang="scss">
@import 'dive-common/components/styles/KeyValueTable.scss';
.v-data-table__selected {
background: unset !important;
}
</style>
7 changes: 6 additions & 1 deletion client/platform/desktop/frontend/components/ImportDialog.vue
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,11 @@ export default defineComponent({
type: Boolean,
default: false,
},
// If being embedded into the bulk import dialog
embedded: {
type: Boolean,
default: false,
},
},
setup(props) {
const argCopy = ref(cloneDeep(props.importData));
Expand Down Expand Up @@ -321,7 +326,7 @@ export default defineComponent({
:disabled="!ready || disabled"
@click="$emit('finalize-import', argCopy)"
>
Finish Import
{{ embedded ? "Save" : "Finish Import" }}
</v-btn>
</div>
</v-card-text>
Expand Down
Loading

0 comments on commit 6b66a28

Please sign in to comment.