Evaluating pipeline for openscene

Readme.md

@@ -8,6 +8,7 @@
 - OVIR-3D (Todo)
 - Semantic Abstraction (Todo)
 - PLA (Todo)
+- Openscene
 
 ## Concept-graphs:
 
@@ -80,48 +81,14 @@ Change "room1_mesh" with your dataset's name. Then open in browser: http://0.0.0
 make build-openscene
 ```
 
-### Transform dataset
-```
-python3 ./datasets/scripts/transformations/transform_to_openscene_format.py \
-    /path/to/original/scene/folder \
-    /path/to/output/folder
-```
-
 ### RUN
 ```bash
 make run-openscene
 ```
 
-### Get feature fusion
-Inside the docker run the command from /opt/src/
-```bash
-python ./scripts/feature_fusion/replica_openseg.py \
-    --data_dir /path/to/transformed/dataset \
-    --scene name_of_the_scene \
-    --output_dir /path/to/output/dir \
-    --openseg_model /path/to/openseg/model
-```
 
 ### Specific settings 
 Inside the docker you can edit the export files in export folder, and then run one of them
 ```bash
-bash /export/export_openscene_replica.sh
-```
-
-### Visualize
-Inside the docker you can run the following to visualize the results
-```bash
-cd ./demo/gaps
-make
-
-cd pkgs/RNNets
-make
-
-cd ../../apps/osview
-make
-
-/opt/src/demo/run_demo
+bash /export/export_openscene_replica_none.sh
 ```
-To change the inputs you can open run_demo
-
-In the opened window you can type queries!

export/export_openscene_replica.sh

@@ -1,6 +1,133 @@
 #!/bin/bash
 
+export OPENSEG_PATH=/assets/openseg_exported_clip
+export SCENE_NAME=v3_sc0_staging_00
+export CONFIG_NAME=baseline
+export FEATURE_DIR_PREFIX=/data/openscene/replica/fusion
+export DATASET_DIR_PATH=/data/openscene/replica
+export EXPORT_PATH=/export/output/openscene
+export CONFIG_ROOT=./config/replica
+export DISTILLED_MODEL_PATH=/assets/openscene/distill_models/replica
+export STRIDE=5
+export METRICS_DIR=/data/openscene/replica/metrics
+
+cd /opt/src
+
+######
+# Point clouds generation (gt and colored)
+######
+
+python ./scripts/run_slam.py \
+        --dataset_root "${DATASET_DIR_PATH}/${CONFIG_NAME}/" \
+        --scene_id ${SCENE_NAME} \
+        --stride 5 \
+        --downsample_rate 10 \
+        --load_semseg
+
+######
+# Dataset transformation 
+######
+
+python3 ./scripts/preprocess/preprocess_replica.py \
+    $DATASET_DIR_PATH/$CONFIG_NAME/$SCENE_NAME \
+    $DATASET_DIR_PATH/$CONFIG_NAME/$SCENE_NAME/rgb_cloud \
+    $DATASET_DIR_PATH \
+    $STRIDE
+
+######
+# The following command extracts 2D features from images using OpenSeg and pair them to the point cloud
+######
+
+python ./scripts/feature_fusion/replica_openseg.py \
+    --data_dir $DATASET_DIR_PATH \
+    --scene $SCENE_NAME \
+    --config $CONFIG_NAME \
+    --output_dir $FEATURE_DIR_PREFIX/$CONFIG_NAME \
+    --split test \
+    --openseg_model $OPENSEG_PATH
+
+python ./scripts/feature_fusion/replica_openseg.py \
+    --data_dir $DATASET_DIR_PATH \
+    --scene $SCENE_NAME \
+    --output_dir $FEATURE_DIR_PREFIX'_train' \
+    --split train \
+    --openseg_model $OPENSEG_PATH
+
+
+######
+# To add validation set use this
+######
+
+# python ./scripts/feature_fusion/replica_openseg.py \
+#     --data_dir $DATASET_DIR_PATH \
+#     --scene $SCENE_NAME \
+#     --output_dir $FEATURE_DIR_PREFIX'_val' \
+#     --split val \
+#     --openseg_model $OPENSEG_PATH
+
+
+######
+# Now we can evaluate on our dataset and visualize the results
+# CHECK config to be sure everything is correct
+# P.S. you should change vis_gt, vis_pred and vis_input to True if you want to see the visualization
+######
+
+# sh ./run/eval.sh $EXPORT_PATH $CONFIG_ROOT/replica_openseg_pretrained.yaml $METRICS_DIR/$CONFIG_NAME/$SCENE_NAME fusion 
+
+# python ./scripts/visualization/semantic_visualization.py $EXPORT_PATH 0 fusion # second argument is the scene number
+
+
+
+######
+# To improve semantic segmentation we can use distillation 
+# First, let's try using pretrained (on Matterport3D) model
+# P.S. you can also change 'ensemble' to 'distill' to see how it works without 2D semantics
+######
+
+sh ./run/eval.sh $EXPORT_PATH $CONFIG_ROOT/replica_openseg_pretrained.yaml $METRICS_DIR/$CONFIG_NAME/$SCENE_NAME ensemble
+
+python ./scripts/visualization/semantic_visualization.py $EXPORT_PATH 0 ensemble
+
+
+
+######
+# Now we will try to distill the model by ourselves
+# P.S. if you want to evaluate on validation set change 'aug' and 'evaluate' to True in the config file
+######
+
+# sh ./run/distill.sh $DISTILLED_MODEL_PATH $CONFIG_ROOT/replica_openseg.yaml
+
+
+
+######
+# And evaluate using our model
+# In config file change 'model_path' to "/assets/openscene/distill_models/replica/model/model_best.pth.tar" if you are evaluating on a validation set
+######
+
+# sh ./run/eval.sh $EXPORT_PATH $CONFIG_ROOT/replica_openseg_pretrained.yaml ensemble
+
+# python ./scripts/visualization/semantic_visualization.py $EXPORT_PATH 0 ensemble
+
+
+######
+# These commands are for testing queries
+# To run this you should have .npy files (save_feature_as_numpy: True in config) and input point cloud (vis_input: True in config)
+######
+
+# cd ./demo/gaps
+# make
+
+# cd pkgs/RNNets
+# make
+
+# cd ../../apps/osview
+# make
+
+# cd /opt/src
+
+# python3 ./demo/clip_server.py --out_dir tmp&
+
+# demo/gaps/bin/x86_64/osview $EXPORT_PATH/result_eval/0_input.ply $EXPORT_PATH/result_eval/saved_feature/*.npy -v -tcp
+
+
 
-### YOU CAN USE YOUR OWN CONFIG FILE
-### ALSO PATHS TO DATA CAN BE MODIFIED FROM THE CONFIG
-sh run/eval.sh /export/output/openscene config/replica/replica_openseg.yaml ensemble

export/export_openscene_replica_all_scenes.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+export OPENSEG_PATH=/assets/openseg_exported_clip
+export FEATURE_DIR_PREFIX=/data/openscene/replica/fusion
+export DATASET_DIR_PATH=/data/openscene/replica
+export EXPORT_PATH=/export/output/openscene
+export CONFIG_ROOT=./config/replica
+export DISTILLED_MODEL_PATH=/assets/openscene/distill_models/replica
+export STRIDE=5
+export METRICS_DIR=/data/openscene/replica/metrics
+
+export SCENE_NAMES=(
+    v3_sc0_staging_00
+    v3_sc0_staging_12
+    v3_sc0_staging_16
+    v3_sc0_staging_19
+    v3_sc0_staging_20
+    v3_sc1_staging_00
+    v3_sc1_staging_06
+    v3_sc1_staging_12
+    v3_sc1_staging_19
+    v3_sc1_staging_20
+    v3_sc2_staging_00
+    v3_sc2_staging_11
+    v3_sc2_staging_13
+    v3_sc2_staging_19
+    v3_sc2_staging_20
+    v3_sc3_staging_03
+    v3_sc3_staging_04
+    v3_sc3_staging_08
+    v3_sc3_staging_15
+    v3_sc3_staging_20
+)
+
+export CONFIG_NAME=baseline # don't forget to change config
+
+cd /opt/src
+
+for SCENE_NAME in ${SCENE_NAMES[*]}
+do
+
+    printf "Running scene:   %s\n" "$CONFIG_NAME/${SCENE_NAME}"
+
+    python ./scripts/run_slam.py \
+        --dataset_root "${DATASET_DIR_PATH}/${CONFIG_NAME}/" \
+        --scene_id ${SCENE_NAME} \
+        --stride 5 \
+        --downsample_rate 15 \
+        --load_semseg
+
+
+    python3 ./scripts/preprocess/preprocess_replica.py \
+    $DATASET_DIR_PATH/$CONFIG_NAME/$SCENE_NAME \
+    $DATASET_DIR_PATH/$CONFIG_NAME/$SCENE_NAME/rgb_cloud \
+    $DATASET_DIR_PATH \
+    $STRIDE 
+
+    python ./scripts/feature_fusion/replica_openseg.py \
+    --data_dir $DATASET_DIR_PATH \
+    --scene $SCENE_NAME \
+    --config $CONFIG_NAME \
+    --output_dir $FEATURE_DIR_PREFIX/$CONFIG_NAME \
+    --split test \
+    --openseg_model $OPENSEG_PATH
+
+    sh ./run/eval.sh $EXPORT_PATH $CONFIG_ROOT/replica_openseg_pretrained.yaml $METRICS_DIR/$CONFIG_NAME/$SCENE_NAME fusion 
+
+
+
+done