Aggregate updates

client/dive-common/components/Viewer.vue

@@ -598,6 +598,7 @@
           progress.total = tracks.length + groups.length;
           const trackStore = cameraStore.camMap.value.get(camera)?.trackStore;
           const groupStore = cameraStore.camMap.value.get(camera)?.groupStore;
+          console.log(`Loading Camera: ${camera}`);
           if (trackStore && groupStore) {
             // We can start sorting if our total tracks are less than 20000
             // If greater we do one sort at the end instead to speed loading.

client/dive-common/constants.ts

@@ -13,7 +13,7 @@ const MediaTypes: Record<DatasetType, string> = {
   [LargeImageType]: 'tiled image',
 };
 
-const DefaultVideoFPS = 10;
+const DefaultVideoFPS = -1;
 const FPSOptions = [
   { text: 1, value: 1 },
   { text: 5, value: 5 },

client/platform/desktop/backend/native/common.ts

@@ -430,9 +430,9 @@ async function getPipelineList(settings: Settings): Promise<Pipelines> {
  */
 async function getTrainingConfigs(settings: Settings): Promise<TrainingConfigs> {
   const pipelinePath = npath.join(settings.viamePath, 'configs/pipelines');
-  const defaultTrainingConfiguration = 'train_detector_default.viame_csv.conf';
-  const allowedPatterns = /\.viame_csv\.conf$/;
-  const disallowedPatterns = /.*(_nf|\.continue)\.viame_csv\.conf$/;
+  const defaultTrainingConfiguration = 'train_detector_default.conf';
+  const allowedPatterns = /train_.*\.conf$/;
+  const disallowedPatterns = /.*(_nf|\.continue)\.viame_csv\.conf$|.*\.kw18\.conf$|.*\.habcam\.conf$|.*\.continue\.conf$/;
   const exists = await fs.pathExists(pipelinePath);
   if (!exists) {
     throw new Error(`Path does not exist: ${pipelinePath}`);

client/platform/desktop/backend/serializers/viame.ts

@@ -281,8 +281,11 @@ async function parse(input: Readable, imageMap?: Map<string, number>): Promise<A
           const k = i + 1;
           if (k < filteredImages.length) {
             if (filteredImages[i].csvFrame + 1 !== filteredImages[k].csvFrame || filteredImages[i].frame + 1 !== filteredImages[k].frame) {
-            // We have misaligned video sequences so we error out
-              error = new Error('A subsampling of images were used with the CSV but they were not sequential');
+            // We have misaligned image sequences so we error out
+              error = new Error(`A subsampling of images were used with the CSV but they were not sequential\n 
+                ${filteredImages[i].csvFrame + 1} !== ${filteredImages[k].csvFrame} || ${filteredImages[i].frame + 1} !== ${filteredImages[k].frame}\n
+                image1: ${filteredImages[i].image} image2: ${filteredImages[k].image} - these should be sequential in the CSV
+              \n`);
             }
           }
           frameMapper[filteredImages[i].csvFrame] = i;

client/platform/desktop/frontend/components/Export.vue

@@ -210,7 +210,7 @@ export default defineComponent({
             block
             @click="doExport({ type: 'dataset' })"
           >
-            <span>export detections</span>
+            <span>VIAME CSV</span>
           </v-btn>
         </v-card-actions>
         <v-card-text class="pb-0">

client/platform/desktop/frontend/components/NavigationBar.vue

@@ -21,7 +21,7 @@ export default defineComponent({
       color="accent"
     >
       <v-tab :to="{ name: 'recent' }">
-        Recents
+        Library
         <v-icon>mdi-folder-open</v-icon>
       </v-tab>
       <job-tab />

client/platform/desktop/frontend/components/Settings.vue

@@ -282,7 +282,7 @@ export default defineComponent({
           >
             mdi-check-circle
           </v-icon>
-          Sync recents with Project Data
+          Sync library with Project Data
         </v-btn>
 
         <v-card-text>

client/platform/desktop/frontend/components/ViewerLoader.vue

@@ -129,7 +129,7 @@ export default defineComponent({
         style="flex-basis:0; flex-grow:0;"
       >
         <v-tab :to="{ name: 'recent' }">
-          Recents
+          Library
           <v-icon>mdi-folder-open</v-icon>
         </v-tab>
         <job-tab />

client/platform/web-girder/views/Export.vue

@@ -298,13 +298,8 @@ export default defineComponent({
           </v-card-text>
 
           <v-card-actions>
-            <v-menu
-              offset-y
-              offset-x
-              nudge-left="180"
-              max-width="180"
-            >
-              <template v-slot:activator="{ on }">
+            <v-row>
+              <v-col>
                 <v-btn
                   depressed
                   block
@@ -313,35 +308,25 @@ export default defineComponent({
                 >
                   <span
                     v-if="exportUrls.exportDetectionsUrl"
-                    class="col-11"
-                  >annotations</span>
+                  >VIAME CSV</span>
                   <span
                     v-else
-                    class="col-11"
                   >detections unavailable</span>
-                  <v-icon
-                    v-if="exportUrls.exportDetectionsUrl"
-                    class="button-dropdown col-1"
-                    v-on="on"
-                  >
-                    mdi-chevron-down
-                  </v-icon>
                 </v-btn>
-              </template>
-              <v-card outlined>
-                <v-list dense>
-                  <v-list-item
-                    style="align-items':'center"
-                    @click="doExport({ url: exportUrls
-                      && exportUrls.exportDetectionsUrlTrackJSON })"
-                  >
-                    <v-list-item-content>
-                      <v-list-item-title>TrackJSON</v-list-item-title>
-                    </v-list-item-content>
-                  </v-list-item>
-                </v-list>
-              </v-card>
-            </v-menu>
+                <v-btn
+                  depressed
+                  block
+                  class="mt-2"
+                  :disabled="!exportUrls.exportDetectionsUrl"
+                  @click="doExport({ url: exportUrls
+                      && exportUrls.exportDetectionsUrlTrackJSON })"                >
+                  <span
+                    v-if="exportUrls.exportDetectionsUrl"
+                  >DIVE TrackJSON</span>
+                  <span
+                    v-else
+                  >detections unavailable</span>
+                      </v-btn>
             <!-- <v-btn
               depressed
               block
@@ -351,6 +336,8 @@ export default defineComponent({
               <span v-if="exportUrls.exportDetectionsUrl">annotations</span>
               <span v-else>detections unavailable</span>
             </v-btn> -->
+            </v-col>
+            </v-row>
           </v-card-actions>
 
           <v-card-text class="pb-0">

client/src/components/LayerManager.vue

@@ -278,7 +278,11 @@ export default defineComponent({
       } else {
         tailLayer.disable();
       }
-      pointLayer.changeData(frameData);
+      if (visibleModes.includes('LineString')) {
+        pointLayer.changeData(frameData);
+      } else {
+        pointLayer.disable();
+      }
       if (visibleModes.includes('text')) {
         textLayer.changeData(frameData);
         attributeBoxLayer.changeData(frameData);

client/src/track.ts

@@ -517,6 +517,9 @@
       };
     });
     // accept either number or string, convert to number
+    if (json.id === null) {
+      console.log(json);
+    }
     const intTrackId = parseInt(json.id.toString(), 10);
     const track = new Track(intTrackId, {
       features: sparseFeatures,

docs/Dive-Desktop.md

@@ -73,7 +73,7 @@ DIVE Desktop requires a local installation of the VIAME toolkit to run ML pipeli
     * Use ==Choose :material-folder:== to choose the base installation path, then click ==:material-content-save: Save==.
 * **Project Data Storage Path** defaults to a subfolder in your user workspace and should generally not be changed.
 * **Read only mode** disables the ability to save when using the annotator.
-* **Synchronize Recents** - The ==:material-sync: Synchronize Recents with Project Data== button is useful if data in the Project Data Storage Path gets out of sync with what appears in the ==:material-folder-open: Recents== list.
+* **Synchronize Library** - The ==:material-sync: Synchronize Library with Project Data== button is useful if data in the Project Data Storage Path gets out of sync with what appears in the ==:material-folder-open: Library== list.
 
 ### Data Storage Path
 

docs/Pipeline-Documentation.md

@@ -112,15 +112,15 @@ By default, training runs include all frames from the chosen input datasets, and
 | Configuration | Availability | Use Case |
 | ------------- | ------------ | -------- |
 | detector_default | both | alias: train detector netharn cfrnn |
-| [detector_netharn_cfrnn](https://github.com/VIAME/VIAME/blob/master/configs/pipelines/train_detector_netharn_cfrnn.viame_csv.conf) | both | |
-| [detector_netharn_mask_rcnn](https://github.com/VIAME/VIAME/blob/master/configs/pipelines/train_detector_netharn_mask_rcnn.viame_csv.conf) | both | |
-| [detector_svm_over_generic_detections](https://github.com/VIAME/VIAME/blob/master/configs/pipelines/train_detector_svm_over_generic_detections.viame_csv.conf) | both | general purpose svm |
-| [detector_svm_over_fish_detections](https://github.com/VIAME/VIAME/blob/master/configs/pipelines/train_detector_svm_over_fish_detections.viame_csv.conf) | both | fish svm |
+| [detector_netharn_cfrnn](https://github.com/VIAME/VIAME/blob/master/configs/pipelines/train_detector_netharn_cfrnn.conf) | both | |
+| [detector_netharn_mask_rcnn](https://github.com/VIAME/VIAME/blob/master/configs/pipelines/train_detector_netharn_mask_rcnn.conf) | both | |
+| [detector_svm_over_generic_detections](https://github.com/VIAME/VIAME/blob/master/configs/pipelines/train_detector_svm_over_generic_detections.conf) | both | general purpose svm |
+| [detector_svm_over_fish_detections](https://github.com/VIAME/VIAME/blob/master/configs/pipelines/train_detector_svm_over_fish_detections.conf) | both | fish svm |
 | frame_classifier_default | both | alias: frame classifier netharn resnet |
-| [frame_classifier_netharn_resnet](https://github.com/VIAME/VIAME/blob/master/configs/pipelines/train_frame_classifier_netharn_resnet.viame_csv.conf) | both | full-frame |
-| [frame_classifier_svm_overn_resnet](https://github.com/VIAME/VIAME/blob/master/configs/pipelines/train_frame_classifier_svm_over_resnet.viame_csv.conf) | both | full-frame |
+| [frame_classifier_netharn_resnet](https://github.com/VIAME/VIAME/blob/master/configs/pipelines/train_frame_classifier_netharn_resnet.conf) | both | full-frame |
+| [frame_classifier_svm_overn_resnet](https://github.com/VIAME/VIAME/blob/master/configs/pipelines/train_frame_classifier_svm_over_resnet.conf) | both | full-frame |
 | object_classifier_default | both | alias: netharn resnet object classifier |
-| [object_classifier_netharn_resnet](https://github.com/VIAME/VIAME/blob/master/configs/pipelines/train_object_classifier_netharn_resnet.viame_csv.conf) | both | |
+| [object_classifier_netharn_resnet](https://github.com/VIAME/VIAME/blob/master/configs/pipelines/train_object_classifier_netharn_resnet.conf) | both | |
 | yolo | desktop only | can train, but resulting model **is not runnable with desktop** yet |
 
 ## Pipeline Import and Export

samples/rename.py

@@ -0,0 +1,47 @@
+import json
+import os
+
+import click
+import girder_client
+import csv
+from os.path import basename, splitext
+
+
+apiURL = "viame.kitware.com"
+rootFolder = "65a19f18cf5a99794ea99cdb"  # Sample folder girder Id
+limit = 10  # for testing purposes keep lower then increase
+
+
+# Login to the girder client, interactive means it will prompt for username and password
+def login():
+    gc = girder_client.GirderClient(apiURL, port=443, apiRoot="girder/api/v1", )
+    gc.authenticate(interactive=True)
+    return gc
+
+
+def get_dive_datasets(gc: girder_client.GirderClient, rootFolderId: str,):
+    folders = gc.listFolder(rootFolderId)
+    dive_datasets = {}
+    for item in folders:
+        if item.get('meta', {}).get('annotate', False) is True:
+            # then it's a dive Dataset
+            dive_datasets[item['name'].replace('Video ', '').replace('.mp4', '')] = item
+        if 'Video ' in item['name']:
+            print(f'Renaming: {item["name"]}')
+            newname = item['name'].replace('Video ', '').replace('.mp4', '')
+            gc.put(f"folder/{item['_id']}", data={'name': newname})
+    items = gc.listItem(rootFolderId)
+    for item in items:
+        if item.get('name').endswith('csv'):
+            print(f'Deleting: {item["name"]}')
+            gc.delete(f"item/{item['_id']}")
+
+
+@click.command(name="process NoCopy Imports", help="Load in ")
+def load_data():
+    gc = login()
+    get_dive_datasets(gc, rootFolder)
+
+
+if __name__ == "__main__":
+    load_data()

server/dive_tasks/pipeline_discovery.py

@@ -9,9 +9,9 @@
     TrainingConfigurationSummary,
 )
 
-DefaultTrainingConfiguration = "train_detector_default.viame_csv.conf"
-AllowedTrainingConfigs = r".*\.viame_csv\.conf$"
-DisallowedTrainingConfigs = r".*(_nf|\.continue)\.viame_csv\.conf$"
+DefaultTrainingConfiguration = "train_detector_default.conf"
+AllowedTrainingConfigs = r"train_.*\.conf$"
+DisallowedTrainingConfigs = r".*(_nf|\.continue)\.viame_csv\.conf$|.*\.continue\.conf$|.*\.habcam\.conf$|.*\.kw18\.conf$"
 AllowedStaticPipelines = r"^detector_.+|^tracker_.+|^utility_.+|^generate_.+"
 DisallowedStaticPipelines = (
     # Remove utilities pipes which hold no meaning in web

server/dive_tasks/utils.py

@@ -1,6 +1,7 @@
 from datetime import datetime, timedelta
 import json
 import os
+import time
 from pathlib import Path
 import shutil
 import signal
@@ -91,6 +92,8 @@ def stream_subprocess(
         if process.stdout is None:
             raise RuntimeError("Stdout must not be none")
 
+        last_refresh_time = time.time()
+
         # call readline until it returns empty bytes
         for line in iter(process.stdout.readline, b''):
             line_str = line.decode('utf-8')
@@ -99,8 +102,13 @@ def stream_subprocess(
                 stdout += line_str
 
             # Cancel the subprocess if the status is cancelling
-            # note this only checks when there is stdout from the subprocess
-            manager.refreshStatus()
+            # note this only checks when there is stdout from the subprocess every 5 minutes
+            # refreshStatus I believe is an expensive tas
+            current_time = time.time()
+            if current_time - last_refresh_time >= 300:
+                last_refresh_time = current_time
+                manager.refreshStatus()
+
             if check_canceled(task, context, force=False) or manager.status == JobStatus.CANCELING:
                 # Can never be sure what signal a process will respond to.
                 process.send_signal(signal.SIGTERM)

server/dive_utils/constants.py

@@ -7,7 +7,7 @@
 ImageSequenceType = "image-sequence"
 VideoType = "video"
 LargeImageType = "large-image"
-DefaultVideoFPS = 10
+DefaultVideoFPS = -1
 JsonMetaCurrentVersion = 1
 SettingsCurrentVersion = 1
 AnnotationsCurrentVersion = 2