From 18400e9db9e02af7593815006d151afe08d687f7 Mon Sep 17 00:00:00 2001 From: maxkazmsft Date: Mon, 13 Apr 2020 14:58:05 -0400 Subject: [PATCH 01/20] correctness branch setup (#251) * created correctnes branch, trimmed experiments to Dutch F3 only * trivial change to re-trigger build * dummy PR to re-trigger malfunctioning builds --- README.md | 4 - .../interpretation/penobscot/README.md | 0 .../penobscot/local/configs/hrnet.yaml | 0 .../local/configs/seresnet_unet.yaml | 0 .../interpretation/penobscot/local/default.py | 0 .../penobscot/local/logging.conf | 0 .../interpretation/penobscot/local/test.py | 0 .../interpretation/penobscot/local/test.sh | 0 .../interpretation/penobscot/local/train.py | 0 .../interpretation/penobscot/local/train.sh | 0 tests/cicd/main_build.yml | 157 +----------------- 11 files changed, 3 insertions(+), 158 deletions(-) rename {experiments => contrib/experiments}/interpretation/penobscot/README.md (100%) rename {experiments => contrib/experiments}/interpretation/penobscot/local/configs/hrnet.yaml (100%) rename {experiments => contrib/experiments}/interpretation/penobscot/local/configs/seresnet_unet.yaml (100%) rename {experiments => contrib/experiments}/interpretation/penobscot/local/default.py (100%) rename {experiments => contrib/experiments}/interpretation/penobscot/local/logging.conf (100%) rename {experiments => contrib/experiments}/interpretation/penobscot/local/test.py (100%) rename {experiments => contrib/experiments}/interpretation/penobscot/local/test.sh (100%) rename {experiments => contrib/experiments}/interpretation/penobscot/local/train.py (100%) rename {experiments => contrib/experiments}/interpretation/penobscot/local/train.sh (100%) diff --git a/README.md b/README.md index 369e2143..cf8886a7 100644 --- a/README.md +++ b/README.md @@ -419,7 +419,3 @@ which will indicate that anaconda folder is __/anaconda__. We'll refer to this l 5. Navigate back to the Virtual Machine view in Step 2 and click the Start button to start the virtual machine. - - - - diff --git a/experiments/interpretation/penobscot/README.md b/contrib/experiments/interpretation/penobscot/README.md similarity index 100% rename from experiments/interpretation/penobscot/README.md rename to contrib/experiments/interpretation/penobscot/README.md diff --git a/experiments/interpretation/penobscot/local/configs/hrnet.yaml b/contrib/experiments/interpretation/penobscot/local/configs/hrnet.yaml similarity index 100% rename from experiments/interpretation/penobscot/local/configs/hrnet.yaml rename to contrib/experiments/interpretation/penobscot/local/configs/hrnet.yaml diff --git a/experiments/interpretation/penobscot/local/configs/seresnet_unet.yaml b/contrib/experiments/interpretation/penobscot/local/configs/seresnet_unet.yaml similarity index 100% rename from experiments/interpretation/penobscot/local/configs/seresnet_unet.yaml rename to contrib/experiments/interpretation/penobscot/local/configs/seresnet_unet.yaml diff --git a/experiments/interpretation/penobscot/local/default.py b/contrib/experiments/interpretation/penobscot/local/default.py similarity index 100% rename from experiments/interpretation/penobscot/local/default.py rename to contrib/experiments/interpretation/penobscot/local/default.py diff --git a/experiments/interpretation/penobscot/local/logging.conf b/contrib/experiments/interpretation/penobscot/local/logging.conf similarity index 100% rename from experiments/interpretation/penobscot/local/logging.conf rename to contrib/experiments/interpretation/penobscot/local/logging.conf diff --git a/experiments/interpretation/penobscot/local/test.py b/contrib/experiments/interpretation/penobscot/local/test.py similarity index 100% rename from experiments/interpretation/penobscot/local/test.py rename to contrib/experiments/interpretation/penobscot/local/test.py diff --git a/experiments/interpretation/penobscot/local/test.sh b/contrib/experiments/interpretation/penobscot/local/test.sh similarity index 100% rename from experiments/interpretation/penobscot/local/test.sh rename to contrib/experiments/interpretation/penobscot/local/test.sh diff --git a/experiments/interpretation/penobscot/local/train.py b/contrib/experiments/interpretation/penobscot/local/train.py similarity index 100% rename from experiments/interpretation/penobscot/local/train.py rename to contrib/experiments/interpretation/penobscot/local/train.py diff --git a/experiments/interpretation/penobscot/local/train.sh b/contrib/experiments/interpretation/penobscot/local/train.sh similarity index 100% rename from experiments/interpretation/penobscot/local/train.sh rename to contrib/experiments/interpretation/penobscot/local/train.sh diff --git a/tests/cicd/main_build.yml b/tests/cicd/main_build.yml index b0d641f1..f4372ce3 100644 --- a/tests/cicd/main_build.yml +++ b/tests/cicd/main_build.yml @@ -6,17 +6,19 @@ pr: - master - staging - contrib +- correctness # Any commit to this branch will trigger the build. trigger: - master - staging - contrib +- correctness ################################################################################################### # The pre-requisite for these jobs is to have 4 GPUs on your virtual machine (K80 or better) # Jobs are daisy-chained by stages - more relevant stages come first (the ones we're currently -# working on): +# working on): # - if they fail no point in running anything else # - each stage _can_ have parallel jobs but that's not always needed for fast execution ################################################################################################### @@ -299,156 +301,3 @@ jobs: --cfg=configs/hrnet.yaml --debug echo "PASSED" - -################################################################################################### -# Stage 5: Docker container -################################################################################################### - -- job: docker_image_build_job - dependsOn: dutchf3_patch_dist - timeoutInMinutes: 15 - displayName: Docker image build - pool: - name: deepseismicagentpool - steps: - - bash: | - # terminate as soon as any internal script fails - set -e - - echo "cleaning up old Docker image" - if [[ "$(docker images -q seismic-deeplearning:latest 2> /dev/null)" == "" ]]; then - echo "no previous image found" - else - echo "removing previous image" - docker rmi seismic-deeplearning - fi - # test building the image - cd docker - docker build -t seismic-deeplearning . - -################################################################################################### -# Stage 6: remaining code - Penobscot, section and deconvnet_skip model for F3 -################################################################################################### - -- job: penobscot_et_al - dependsOn: docker_image_build_job - timeoutInMinutes: 20 - displayName: Penobscot et al - pool: - name: deepseismicagentpool - steps: - - bash: | - - source activate seismic-interpretation - - # disable auto error handling as we flag it manually - set +e - - cd experiments/interpretation/penobscot/local - - # Create a temporary directory to store the statuses - dir=$(mktemp -d) - - pids= - export CUDA_VISIBLE_DEVICES=0 - { python train.py 'DATASET.ROOT' '/home/alfred/data_dynamic/penobscot' 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \ - 'TRAIN.DEPTH' 'section' \ - 'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'section_depth' \ - --cfg=configs/seresnet_unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } & - pids+=" $!" - - export CUDA_VISIBLE_DEVICES=1 - { python train.py 'DATASET.ROOT' '/home/alfred/data_dynamic/penobscot' 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \ - 'TRAIN.DEPTH' 'section' \ - 'MODEL.PRETRAINED' '/home/alfred/models/hrnetv2_w48_imagenet_pretrained.pth' \ - 'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'section_depth' \ - --cfg=configs/hrnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } & - pids+=" $!" - - export CUDA_VISIBLE_DEVICES=2 - cd ../../../../experiments/interpretation/dutchf3_patch/local - { python train.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \ - 'TRAIN.DEPTH' 'none' \ - 'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'no_depth' \ - --cfg=configs/patch_deconvnet_skip.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } & - pids+=" $!" - - export CUDA_VISIBLE_DEVICES=3 - cd ../../../../experiments/interpretation/dutchf3_section/local - { python train.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \ - 'TRAIN.DEPTH' 'none' \ - 'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'no_depth' \ - --cfg=configs/section_deconvnet_skip.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } & - pids+=" $!" - cd ../../../../ - - wait $pids || exit 1 - - # check if any of the models had an error during execution - # Get return information for each pid - for file in "$dir"/*; do - printf 'PID %d returned %d\n' "${file##*/}" "$(<"$file")" - [[ "$(<"$file")" -ne "0" ]] && exit 1 || echo "pass" - done - - # Remove the temporary directory - rm -r "$dir" - - echo "Models finished training" - - # Create a temporary directory to store the statuses - dir=$(mktemp -d) - - cd experiments/interpretation/penobscot/local - pids= - export CUDA_VISIBLE_DEVICES=0 - # find the latest model which we just trained - model=$(ls -td output/seresnet_unet/section_depth/* | head -1) - # try running the test script - { python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/penobscot' \ - 'TEST.MODEL_PATH' ${model}/resnet_unet_running_model_1.pth \ - --cfg=configs/seresnet_unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } & - pids+=" $!" - export CUDA_VISIBLE_DEVICES=1 - # find the latest model which we just trained - model=$(ls -td output/hrnet/section_depth/* | head -1) - echo ${model} - # # try running the test script - { python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/penobscot' \ - 'MODEL.PRETRAINED' '/home/alfred/models/hrnetv2_w48_imagenet_pretrained.pth' \ - 'TEST.MODEL_PATH' ${model}/seg_hrnet_running_model_1.pth \ - --cfg=configs/hrnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } & - pids+=" $!" - export CUDA_VISIBLE_DEVICES=2 - cd ../../../../experiments/interpretation/dutchf3_patch/local - # find the latest model which we just trained - model=$(ls -td output/patch_deconvnet_skip/no_depth/* | head -1) - # try running the test script - { python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' \ - 'TEST.MODEL_PATH' ${model}/patch_deconvnet_skip_running_model_1.pth \ - --cfg=configs/patch_deconvnet_skip.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } & - pids+=" $!" - export CUDA_VISIBLE_DEVICES=3 - cd ../../../../experiments/interpretation/dutchf3_section/local - # find the latest model which we just trained - model=$(ls -td output/section_deconvnet_skip/no_depth/* | head -1) - echo ${model} - # try running the test script - { python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' \ - 'TEST.MODEL_PATH' ${model}/section_deconvnet_skip_running_model_1.pth \ - --cfg=configs/section_deconvnet_skip.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } & - pids+=" $!" - cd ../../../../ - wait $pids || exit 1 - - # check if any of the models had an error during execution - # Get return information for each pid - for file in "$dir"/*; do - printf 'PID %d returned %d\n' "${file##*/}" "$(<"$file")" - [[ "$(<"$file")" -ne "0" ]] && exit 1 || echo "pass" - done - - # Remove the temporary directory - rm -r "$dir" - - echo "PASSED" From 913510f5836f864d3ff14d72956ec3fd09d75fd0 Mon Sep 17 00:00:00 2001 From: maxkazmsft Date: Thu, 16 Apr 2020 14:26:12 -0400 Subject: [PATCH 02/20] reducing scope further (#258) * created correctnes branch, trimmed experiments to Dutch F3 only * trivial change to re-trigger build * dummy PR to re-trigger malfunctioning builds * reducing scope of the correctness branch further * added branch triggers --- .../distributed/configs/hrnet.yaml | 0 .../distributed/configs/patch_deconvnet.yaml | 0 .../configs/patch_deconvnet_skip.yaml | 0 .../distributed/configs/seresnet_unet.yaml | 0 .../distributed/configs/unet.yaml | 0 .../dutchf3_patch/distributed/default.py | 0 .../dutchf3_patch/distributed/logging.conf | 0 .../dutchf3_patch/distributed/run.sh | 0 .../dutchf3_patch/distributed/train.py | 0 .../dutchf3_patch/distributed/train.sh | 0 .../interpretation/dutchf3_section/README.md | 0 .../local/configs/section_deconvnet_skip.yaml | 0 .../dutchf3_section/local/default.py | 0 .../dutchf3_section/local/logging.conf | 0 .../dutchf3_section/local/test.py | 0 .../dutchf3_section/local/train.py | 0 tests/cicd/component_governance.yml | 2 + tests/cicd/main_build.yml | 96 ------------------- tests/cicd/notebooks_build.yml | 2 + 19 files changed, 4 insertions(+), 96 deletions(-) rename {experiments => contrib/experiments}/interpretation/dutchf3_patch/distributed/configs/hrnet.yaml (100%) rename {experiments => contrib/experiments}/interpretation/dutchf3_patch/distributed/configs/patch_deconvnet.yaml (100%) rename {experiments => contrib/experiments}/interpretation/dutchf3_patch/distributed/configs/patch_deconvnet_skip.yaml (100%) rename {experiments => contrib/experiments}/interpretation/dutchf3_patch/distributed/configs/seresnet_unet.yaml (100%) rename {experiments => contrib/experiments}/interpretation/dutchf3_patch/distributed/configs/unet.yaml (100%) rename {experiments => contrib/experiments}/interpretation/dutchf3_patch/distributed/default.py (100%) rename {experiments => contrib/experiments}/interpretation/dutchf3_patch/distributed/logging.conf (100%) rename {experiments => contrib/experiments}/interpretation/dutchf3_patch/distributed/run.sh (100%) rename {experiments => contrib/experiments}/interpretation/dutchf3_patch/distributed/train.py (100%) rename {experiments => contrib/experiments}/interpretation/dutchf3_patch/distributed/train.sh (100%) rename {experiments => contrib/experiments}/interpretation/dutchf3_section/README.md (100%) rename {experiments => contrib/experiments}/interpretation/dutchf3_section/local/configs/section_deconvnet_skip.yaml (100%) rename {experiments => contrib/experiments}/interpretation/dutchf3_section/local/default.py (100%) rename {experiments => contrib/experiments}/interpretation/dutchf3_section/local/logging.conf (100%) rename {experiments => contrib/experiments}/interpretation/dutchf3_section/local/test.py (100%) rename {experiments => contrib/experiments}/interpretation/dutchf3_section/local/train.py (100%) diff --git a/experiments/interpretation/dutchf3_patch/distributed/configs/hrnet.yaml b/contrib/experiments/interpretation/dutchf3_patch/distributed/configs/hrnet.yaml similarity index 100% rename from experiments/interpretation/dutchf3_patch/distributed/configs/hrnet.yaml rename to contrib/experiments/interpretation/dutchf3_patch/distributed/configs/hrnet.yaml diff --git a/experiments/interpretation/dutchf3_patch/distributed/configs/patch_deconvnet.yaml b/contrib/experiments/interpretation/dutchf3_patch/distributed/configs/patch_deconvnet.yaml similarity index 100% rename from experiments/interpretation/dutchf3_patch/distributed/configs/patch_deconvnet.yaml rename to contrib/experiments/interpretation/dutchf3_patch/distributed/configs/patch_deconvnet.yaml diff --git a/experiments/interpretation/dutchf3_patch/distributed/configs/patch_deconvnet_skip.yaml b/contrib/experiments/interpretation/dutchf3_patch/distributed/configs/patch_deconvnet_skip.yaml similarity index 100% rename from experiments/interpretation/dutchf3_patch/distributed/configs/patch_deconvnet_skip.yaml rename to contrib/experiments/interpretation/dutchf3_patch/distributed/configs/patch_deconvnet_skip.yaml diff --git a/experiments/interpretation/dutchf3_patch/distributed/configs/seresnet_unet.yaml b/contrib/experiments/interpretation/dutchf3_patch/distributed/configs/seresnet_unet.yaml similarity index 100% rename from experiments/interpretation/dutchf3_patch/distributed/configs/seresnet_unet.yaml rename to contrib/experiments/interpretation/dutchf3_patch/distributed/configs/seresnet_unet.yaml diff --git a/experiments/interpretation/dutchf3_patch/distributed/configs/unet.yaml b/contrib/experiments/interpretation/dutchf3_patch/distributed/configs/unet.yaml similarity index 100% rename from experiments/interpretation/dutchf3_patch/distributed/configs/unet.yaml rename to contrib/experiments/interpretation/dutchf3_patch/distributed/configs/unet.yaml diff --git a/experiments/interpretation/dutchf3_patch/distributed/default.py b/contrib/experiments/interpretation/dutchf3_patch/distributed/default.py similarity index 100% rename from experiments/interpretation/dutchf3_patch/distributed/default.py rename to contrib/experiments/interpretation/dutchf3_patch/distributed/default.py diff --git a/experiments/interpretation/dutchf3_patch/distributed/logging.conf b/contrib/experiments/interpretation/dutchf3_patch/distributed/logging.conf similarity index 100% rename from experiments/interpretation/dutchf3_patch/distributed/logging.conf rename to contrib/experiments/interpretation/dutchf3_patch/distributed/logging.conf diff --git a/experiments/interpretation/dutchf3_patch/distributed/run.sh b/contrib/experiments/interpretation/dutchf3_patch/distributed/run.sh similarity index 100% rename from experiments/interpretation/dutchf3_patch/distributed/run.sh rename to contrib/experiments/interpretation/dutchf3_patch/distributed/run.sh diff --git a/experiments/interpretation/dutchf3_patch/distributed/train.py b/contrib/experiments/interpretation/dutchf3_patch/distributed/train.py similarity index 100% rename from experiments/interpretation/dutchf3_patch/distributed/train.py rename to contrib/experiments/interpretation/dutchf3_patch/distributed/train.py diff --git a/experiments/interpretation/dutchf3_patch/distributed/train.sh b/contrib/experiments/interpretation/dutchf3_patch/distributed/train.sh similarity index 100% rename from experiments/interpretation/dutchf3_patch/distributed/train.sh rename to contrib/experiments/interpretation/dutchf3_patch/distributed/train.sh diff --git a/experiments/interpretation/dutchf3_section/README.md b/contrib/experiments/interpretation/dutchf3_section/README.md similarity index 100% rename from experiments/interpretation/dutchf3_section/README.md rename to contrib/experiments/interpretation/dutchf3_section/README.md diff --git a/experiments/interpretation/dutchf3_section/local/configs/section_deconvnet_skip.yaml b/contrib/experiments/interpretation/dutchf3_section/local/configs/section_deconvnet_skip.yaml similarity index 100% rename from experiments/interpretation/dutchf3_section/local/configs/section_deconvnet_skip.yaml rename to contrib/experiments/interpretation/dutchf3_section/local/configs/section_deconvnet_skip.yaml diff --git a/experiments/interpretation/dutchf3_section/local/default.py b/contrib/experiments/interpretation/dutchf3_section/local/default.py similarity index 100% rename from experiments/interpretation/dutchf3_section/local/default.py rename to contrib/experiments/interpretation/dutchf3_section/local/default.py diff --git a/experiments/interpretation/dutchf3_section/local/logging.conf b/contrib/experiments/interpretation/dutchf3_section/local/logging.conf similarity index 100% rename from experiments/interpretation/dutchf3_section/local/logging.conf rename to contrib/experiments/interpretation/dutchf3_section/local/logging.conf diff --git a/experiments/interpretation/dutchf3_section/local/test.py b/contrib/experiments/interpretation/dutchf3_section/local/test.py similarity index 100% rename from experiments/interpretation/dutchf3_section/local/test.py rename to contrib/experiments/interpretation/dutchf3_section/local/test.py diff --git a/experiments/interpretation/dutchf3_section/local/train.py b/contrib/experiments/interpretation/dutchf3_section/local/train.py similarity index 100% rename from experiments/interpretation/dutchf3_section/local/train.py rename to contrib/experiments/interpretation/dutchf3_section/local/train.py diff --git a/tests/cicd/component_governance.yml b/tests/cicd/component_governance.yml index 0d3f9158..959bd056 100644 --- a/tests/cicd/component_governance.yml +++ b/tests/cicd/component_governance.yml @@ -11,11 +11,13 @@ pr: - master - staging - contrib +- correctness trigger: - master - staging - contrib +- correctness pool: name: deepseismicagentpool diff --git a/tests/cicd/main_build.yml b/tests/cicd/main_build.yml index f4372ce3..bb65fb15 100644 --- a/tests/cicd/main_build.yml +++ b/tests/cicd/main_build.yml @@ -205,99 +205,3 @@ jobs: rm -r "$dir" echo "PASSED" - -################################################################################################### -# Stage 4: DISTRIBUTED PATCH JOBS (in pairs, 2 GPUs at a time per model) -################################################################################################### - -- job: dutchf3_patch_dist - dependsOn: dutchf3_patch - timeoutInMinutes: 25 - displayName: Dutch F3 patch distributed - pool: - name: deepseismicagentpool - steps: - - bash: | - - source activate seismic-interpretation - - # we're manually collecting and re-throwing process run exit codes - set +e - - # number of GPUs to test each model on, except last one - NGPU=2 - - # run the tests - cd experiments/interpretation/dutchf3_patch/distributed - - # Create a temporary directory to store the statuses - dir=$(mktemp -d) - - pids= - export CUDA_VISIBLE_DEVICES=0,1 - { python -m torch.distributed.launch --master_port 6000 --nproc_per_node=$NGPU train.py \ - 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \ - 'TRAIN.DEPTH' 'none' \ - 'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'no_depth' \ - --cfg=configs/patch_deconvnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } & - pids+=" $!" - - export CUDA_VISIBLE_DEVICES=2,3 - { python -m torch.distributed.launch --master_port 7000 --nproc_per_node=$NGPU train.py \ - 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \ - 'TRAIN.DEPTH' 'none' \ - 'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'no_depth' \ - --cfg=configs/patch_deconvnet_skip.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } & - pids+=" $!" - - wait $pids || exit 1 - - # check if any of the models had an error during execution - # Get return information for each pid - for file in "$dir"/*; do - printf 'PID %d returned %d\n' "${file##*/}" "$(<"$file")" - [[ "$(<"$file")" -ne "0" ]] && exit 1 || echo "pass" - done - - pids= - export CUDA_VISIBLE_DEVICES=0,1 - { python -m torch.distributed.launch --master_port 6000 --nproc_per_node=$NGPU train.py \ - 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \ - 'TRAIN.DEPTH' 'section' \ - 'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'section_depth' \ - --cfg=configs/unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } & - pids+=" $!" - - export CUDA_VISIBLE_DEVICES=2,3 - { python -m torch.distributed.launch --master_port 7000 --nproc_per_node=$NGPU train.py \ - 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \ - 'TRAIN.DEPTH' 'section' \ - 'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'section_depth' \ - --cfg=configs/seresnet_unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } & - pids+=" $!" - - wait $pids || exit 1 - - # check if any of the models had an error during execution - # Get return information for each pid - for file in "$dir"/*; do - printf 'PID %d returned %d\n' "${file##*/}" "$(<"$file")" - [[ "$(<"$file")" -ne "0" ]] && exit 1 || echo "pass" - done - - # Remove the temporary directory - rm -r "$dir" - - # we only have 5 models, so just let the last most important one use all remaining GPUs - # re-enable error code flagging - set -e - NGPU=4 - export CUDA_VISIBLE_DEVICES=0,1,2,3 - python -m torch.distributed.launch --master_port 6000 --nproc_per_node=$NGPU train.py \ - 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \ - 'TRAIN.DEPTH' 'section' \ - 'MODEL.PRETRAINED' '/home/alfred/models/hrnetv2_w48_imagenet_pretrained.pth' \ - 'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'section_depth' \ - --cfg=configs/hrnet.yaml --debug - - echo "PASSED" diff --git a/tests/cicd/notebooks_build.yml b/tests/cicd/notebooks_build.yml index 90387e6c..3cf7d1ef 100644 --- a/tests/cicd/notebooks_build.yml +++ b/tests/cicd/notebooks_build.yml @@ -6,12 +6,14 @@ pr: - master - staging - contrib +- correctness # Any commit to this branch will trigger the build. trigger: - master - staging - contrib +- correctness jobs: From affbc5d0342f74fbbd9d09ffd26a44385aa9d7c2 Mon Sep 17 00:00:00 2001 From: maxkazmsft Date: Fri, 17 Apr 2020 10:55:57 -0400 Subject: [PATCH 03/20] 214 Ignite 0.3.0 upgrade (#261) * upgraded to Ignite 0.3.0 and fixed upgrade compatibility * added seeds and modified notebook for ignite 0.3.0 * updated code and tests to work with ignite 0.3.0 * made code consistent with Ignite 0.3.0 as much as possible * fixed iterator epoch_length bug by subsetting validation set * applied same fix to the notebook * bugfix in distributed train.py * increased distributed tests to 2 batched - hoping for one batch per GPU * resolved rebase conflict * added seeds and modified notebook for ignite 0.3.0 * updated code and tests to work with ignite 0.3.0 * made code consistent with Ignite 0.3.0 as much as possible * fixed iterator epoch_length bug by subsetting validation set * applied same fix to the notebook * bugfix in distributed train.py * increased distributed tests to 2 batched - hoping for one batch per GPU --- .../dutchf3_patch/distributed/train.py | 23 ++++---- .../dutchf3_section/local/train.py | 15 +++-- .../penobscot/local/configs/hrnet.yaml | 2 +- .../local/configs/seresnet_unet.yaml | 2 +- .../interpretation/penobscot/local/test.py | 7 +-- .../interpretation/penobscot/local/train.py | 18 +++--- environment/anaconda/local/environment.yml | 2 +- ..._patch_model_training_and_evaluation.ipynb | 58 ++++++++++++------- .../dutchf3_patch/local/test.py | 1 - .../dutchf3_patch/local/train.py | 23 ++++---- tests/cicd/main_build.yml | 8 +-- 11 files changed, 82 insertions(+), 77 deletions(-) diff --git a/contrib/experiments/interpretation/dutchf3_patch/distributed/train.py b/contrib/experiments/interpretation/dutchf3_patch/distributed/train.py index d5259937..9d1a2631 100644 --- a/contrib/experiments/interpretation/dutchf3_patch/distributed/train.py +++ b/contrib/experiments/interpretation/dutchf3_patch/distributed/train.py @@ -72,7 +72,6 @@ from ignite.utils import convert_tensor from toolz import compose, curry from torch.utils import data -from toolz import take def prepare_batch(batch, device=None, non_blocking=False): @@ -179,6 +178,10 @@ def run(*options, cfg=None, local_rank=0, debug=False): patch_size=config.TRAIN.PATCH_SIZE, augmentations=val_aug, ) + + if debug: + val_set = data.Subset(val_set, range(3)) + logger.info(f"Validation examples {len(val_set)}") n_classes = train_set.n_classes @@ -258,16 +261,13 @@ def _select_pred_and_mask(model_out_dict): device=device, ) - # Set the validation run to start on the epoch completion of the training run - if debug: - logger.info("Running Validation in Debug/Test mode") - val_loader = take(3, val_loader) + # Set the validation run to start on the epoch completion of the training run trainer.add_event_handler(Events.EPOCH_COMPLETED, Evaluator(evaluator, val_loader)) if local_rank == 0: # Run only on master process trainer.add_event_handler( - Events.ITERATION_COMPLETED, logging_handlers.log_training_output(log_interval=config.PRINT_FREQ), + Events.ITERATION_COMPLETED, logging_handlers.log_training_output(log_interval=config.TRAIN.BATCH_SIZE_PER_GPU), ) trainer.add_event_handler(Events.EPOCH_STARTED, logging_handlers.log_lr(optimizer)) @@ -340,12 +340,11 @@ def snapshot_function(): evaluator.add_event_handler(Events.EPOCH_COMPLETED, checkpoint_handler, {"model": model}) logger.info("Starting training") - - if debug: - logger.info("Running Training in Debug/Test mode") - train_loader = take(3, train_loader) - - trainer.run(train_loader, max_epochs=config.TRAIN.END_EPOCH) + + if debug: + trainer.run(train_loader, max_epochs=config.TRAIN.END_EPOCH, epoch_length = config.TRAIN.BATCH_SIZE_PER_GPU*2, seed = config.SEED) + else: + trainer.run(train_loader, max_epochs=config.TRAIN.END_EPOCH, epoch_length = len(train_loader), seed = config.SEED) if __name__ == "__main__": diff --git a/contrib/experiments/interpretation/dutchf3_section/local/train.py b/contrib/experiments/interpretation/dutchf3_section/local/train.py index dd8c5b44..b216268e 100644 --- a/contrib/experiments/interpretation/dutchf3_section/local/train.py +++ b/contrib/experiments/interpretation/dutchf3_section/local/train.py @@ -50,7 +50,6 @@ from ignite.metrics import Loss from toolz import compose from torch.utils import data -from toolz import take def prepare_batch(batch, device="cuda", non_blocking=False): @@ -132,6 +131,9 @@ def __len__(self): shuffle=False, ) + if debug: + val_set = data.Subset(val_set, range(3)) + val_loader = data.DataLoader( val_set, batch_size=config.VALIDATION.BATCH_SIZE_PER_GPU, @@ -176,7 +178,7 @@ def __len__(self): trainer.add_event_handler(Events.ITERATION_STARTED, scheduler) trainer.add_event_handler( - Events.ITERATION_COMPLETED, logging_handlers.log_training_output(log_interval=config.PRINT_FREQ), + Events.ITERATION_COMPLETED, logging_handlers.log_training_output(log_interval=config.TRAIN.BATCH_SIZE_PER_GPU), ) trainer.add_event_handler(Events.EPOCH_STARTED, logging_handlers.log_lr(optimizer)) @@ -206,9 +208,6 @@ def _select_pred_and_mask(model_out_dict): device=device, ) - if debug: - logger.info("Running Validation in Debug/Test mode") - val_loader = take(3, val_loader) trainer.add_event_handler(Events.EPOCH_COMPLETED, Evaluator(evaluator, val_loader)) evaluator.add_event_handler( @@ -279,9 +278,9 @@ def snapshot_function(): logger.info("Starting training") if debug: - logger.info("Running Validation in Debug/Test mode") - train_loader = take(3, train_loader) - trainer.run(train_loader, max_epochs=config.TRAIN.END_EPOCH) + trainer.run(train_loader, max_epochs=config.TRAIN.END_EPOCH, epoch_length = config.TRAIN.BATCH_SIZE_PER_GPU, seed = config.SEED) + else: + trainer.run(train_loader, max_epochs=config.TRAIN.END_EPOCH, epoch_length = len(train_loader), seed = config.SEED) if __name__ == "__main__": diff --git a/contrib/experiments/interpretation/penobscot/local/configs/hrnet.yaml b/contrib/experiments/interpretation/penobscot/local/configs/hrnet.yaml index 06d2d765..ba4b3967 100644 --- a/contrib/experiments/interpretation/penobscot/local/configs/hrnet.yaml +++ b/contrib/experiments/interpretation/penobscot/local/configs/hrnet.yaml @@ -92,7 +92,7 @@ TRAIN: VALIDATION: - BATCH_SIZE_PER_GPU: 128 + BATCH_SIZE_PER_GPU: 32 COMPLETE_PATCHES_ONLY: True TEST: diff --git a/contrib/experiments/interpretation/penobscot/local/configs/seresnet_unet.yaml b/contrib/experiments/interpretation/penobscot/local/configs/seresnet_unet.yaml index 29c61936..3ba4d807 100644 --- a/contrib/experiments/interpretation/penobscot/local/configs/seresnet_unet.yaml +++ b/contrib/experiments/interpretation/penobscot/local/configs/seresnet_unet.yaml @@ -49,7 +49,7 @@ TRAIN: VALIDATION: - BATCH_SIZE_PER_GPU: 32 + BATCH_SIZE_PER_GPU: 16 COMPLETE_PATCHES_ONLY: True TEST: diff --git a/contrib/experiments/interpretation/penobscot/local/test.py b/contrib/experiments/interpretation/penobscot/local/test.py index b8928c1e..8687085a 100644 --- a/contrib/experiments/interpretation/penobscot/local/test.py +++ b/contrib/experiments/interpretation/penobscot/local/test.py @@ -283,10 +283,9 @@ def _tensor_to_numpy(pred_tensor): logger.info("Starting training") if debug: - logger.info("Running in Debug/Test mode") - test_loader = take(3, test_loader) - - evaluator.run(test_loader, max_epochs=1) + evaluator.run(test_loader, max_epochs=1, epoch_length = 1) + else: + evaluator.run(test_loader, max_epochs=1, epoch_length = len(test_loader)) # Log top N and bottom N inlines in terms of IoU to tensorboard inline_ious = inline_mean_iou.iou_per_inline() diff --git a/contrib/experiments/interpretation/penobscot/local/train.py b/contrib/experiments/interpretation/penobscot/local/train.py index eea5361b..22394b53 100644 --- a/contrib/experiments/interpretation/penobscot/local/train.py +++ b/contrib/experiments/interpretation/penobscot/local/train.py @@ -64,7 +64,6 @@ from default import _C as config from default import update_config -from toolz import take mask_value = 255 @@ -180,7 +179,10 @@ def run(*options, cfg=None, debug=False): train_set, batch_size=config.TRAIN.BATCH_SIZE_PER_GPU, num_workers=config.WORKERS, shuffle=True, ) - val_loader = data.DataLoader(val_set, batch_size=config.VALIDATION.BATCH_SIZE_PER_GPU, num_workers=config.WORKERS,) + if debug: + val_set = data.Subset(val_set, range(3)) + + val_loader = data.DataLoader(val_set, batch_size=config.VALIDATION.BATCH_SIZE_PER_GPU, num_workers=config.WORKERS) model = getattr(models, config.MODEL.NAME).get_seg_model(config) @@ -215,7 +217,7 @@ def run(*options, cfg=None, debug=False): trainer.add_event_handler(Events.ITERATION_STARTED, scheduler) trainer.add_event_handler( - Events.ITERATION_COMPLETED, logging_handlers.log_training_output(log_interval=config.PRINT_FREQ), + Events.ITERATION_COMPLETED, logging_handlers.log_training_output(log_interval=config.TRAIN.BATCH_SIZE_PER_GPU), ) trainer.add_event_handler(Events.EPOCH_STARTED, logging_handlers.log_lr(optimizer)) trainer.add_event_handler( @@ -243,9 +245,6 @@ def _select_pred_and_mask(model_out_dict): ) # Set the validation run to start on the epoch completion of the training run - if debug: - logger.info("Running Validation in Debug/Test mode") - val_loader = take(3, val_loader) trainer.add_event_handler(Events.EPOCH_COMPLETED, Evaluator(evaluator, val_loader)) evaluator.add_event_handler( @@ -307,10 +306,9 @@ def snapshot_function(): logger.info("Starting training") if debug: - logger.info("Running Training in Debug/Test mode") - train_loader = take(3, train_loader) - trainer.run(train_loader, max_epochs=config.TRAIN.END_EPOCH) - + trainer.run(train_loader, max_epochs=config.TRAIN.END_EPOCH, epoch_length = config.TRAIN.BATCH_SIZE_PER_GPU, seed = config.SEED) + else: + trainer.run(train_loader, max_epochs=config.TRAIN.END_EPOCH, epoch_length = len(train_loader), seed = config.SEED) if __name__ == "__main__": fire.Fire(run) diff --git a/environment/anaconda/local/environment.yml b/environment/anaconda/local/environment.yml index 22f2f114..de0d65af 100644 --- a/environment/anaconda/local/environment.yml +++ b/environment/anaconda/local/environment.yml @@ -21,7 +21,7 @@ dependencies: - papermill>=1.0.1 - pip: - segyio==1.8.8 - - pytorch-ignite==0.3.0.dev20191105 # pre-release until stable available + - pytorch-ignite==0.3.0 - fire==0.2.1 - toolz==0.10.0 - tabulate==0.8.2 diff --git a/examples/interpretation/notebooks/Dutch_F3_patch_model_training_and_evaluation.ipynb b/examples/interpretation/notebooks/Dutch_F3_patch_model_training_and_evaluation.ipynb index 66fc0dba..de4e8542 100644 --- a/examples/interpretation/notebooks/Dutch_F3_patch_model_training_and_evaluation.ipynb +++ b/examples/interpretation/notebooks/Dutch_F3_patch_model_training_and_evaluation.ipynb @@ -136,7 +136,7 @@ "from ignite.engine import Events\n", "from ignite.metrics import Loss\n", "from ignite.utils import convert_tensor\n", - "from toolz import compose, take\n", + "from toolz import compose\n", "from torch.utils import data\n", "\n", "from cv_lib.utils import load_log_configuration\n", @@ -533,18 +533,14 @@ " patch_size=config.TRAIN.PATCH_SIZE,\n", " augmentations=val_aug,\n", ")\n", - "logger.info(val_set)\n", "\n", - "snapshot_duration = scheduler_step * len(train_set)\n", + "# TODO: workaround for Ignite 0.3.0 bug as epoch_lengh in trainer.run method below doesn't apply to validation set\n", + "if papermill:\n", + " val_set = data.Subset(val_set, range(3))\n", + "elif DEMO:\n", + " val_set = data.Subset(val_set, range(config.VALIDATION.BATCH_SIZE_PER_GPU))\n", "\n", - "# use the following code when this error\n", - "# https://stackoverflow.com/questions/53810497/indexerror-when-iterating-my-dataset-using-dataloader-in-pytorch\n", - "# has been fixed and remove the toolz.take workaround at the bottom of this cell\n", - "# if DEMO:\n", - "# count = config.TRAIN.BATCH_SIZE_PER_GPU * 10\n", - "# train_set = data.Subset(train_set, list(range(count)))\n", - "# val_set = data.Subset(val_set, list(range(config.VALIDATION.BATCH_SIZE_PER_GPU)))\n", - "# snapshot_duration = scheduler_step * count\n", + "logger.info(val_set)\n", "\n", "train_loader = data.DataLoader(\n", " train_set,\n", @@ -556,15 +552,33 @@ " val_set,\n", " batch_size=config.VALIDATION.BATCH_SIZE_PER_GPU,\n", " num_workers=config.WORKERS,\n", - ")\n", + ")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The following code defines the snapshot duration in batches over which we snapshot training models to disk. Variable `scheduler_step` defines how many epochs we have in a snapshot and multiplying that by the number of data points per epoch gives us the number of datapoints which we have per snapshot." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# if we're running in test mode, just run 2 batches\n", + "if papermill:\n", + " train_len = config.TRAIN.BATCH_SIZE_PER_GPU*2 \n", + "# if we're running in demo mode, just run 10 batches to fine-tune the model\n", + "elif DEMO:\n", + " train_len = config.TRAIN.BATCH_SIZE_PER_GPU*10 \n", + "# if we're not in test or demo modes, run the entire loop\n", + "else:\n", + " train_len = len(train_loader)\n", "\n", - "# the commented out data.Subset step above causes the Ignite engine to fail, so this \n", - "# is the workaround for now which sets batch size to 10\n", - "if DEMO:\n", - " count = max(20 if papermill else 100, config.TRAIN.BATCH_SIZE_PER_GPU)\n", - " train_loader = take(count, train_loader)\n", - " val_loader = take(config.VALIDATION.BATCH_SIZE_PER_GPU, val_loader)\n", - " snapshot_duration = scheduler_step * count" + "snapshot_duration = scheduler_step * train_len" ] }, { @@ -700,7 +714,7 @@ "# add logging of training output\n", "trainer.add_event_handler(\n", " Events.ITERATION_COMPLETED,\n", - " logging_handlers.log_training_output(log_interval=config.PRINT_FREQ),\n", + " logging_handlers.log_training_output(log_interval=config.TRAIN.BATCH_SIZE_PER_GPU),\n", ")\n", "\n", "# add logging of learning rate\n", @@ -862,7 +876,7 @@ "metadata": {}, "outputs": [], "source": [ - "trainer.run(train_loader, max_epochs=config.TRAIN.END_EPOCH)" + "trainer.run(train_loader, max_epochs=config.TRAIN.END_EPOCH, epoch_length=train_len, seed = config.SEED)" ] }, { @@ -924,7 +938,7 @@ "outputs": [], "source": [ "# use the model which we just fine-tuned\n", - "opts = [\"TEST.MODEL_PATH\", path.join(output_dir, \"model_f3_nb_seg_hrnet_1.pth\")]\n", + "opts = [\"TEST.MODEL_PATH\", path.join(output_dir, f\"model_f3_nb_seg_hrnet_{train_len}.pth\")]\n", "# uncomment the line below to use the pre-trained model instead\n", "# opts = [\"TEST.MODEL_PATH\", config.MODEL.PRETRAINED]\n", "config.merge_from_list(opts)" diff --git a/experiments/interpretation/dutchf3_patch/local/test.py b/experiments/interpretation/dutchf3_patch/local/test.py index 7696518b..3d6eebc7 100644 --- a/experiments/interpretation/dutchf3_patch/local/test.py +++ b/experiments/interpretation/dutchf3_patch/local/test.py @@ -56,7 +56,6 @@ "zechstein", ] - class runningScore(object): def __init__(self, n_classes): self.n_classes = n_classes diff --git a/experiments/interpretation/dutchf3_patch/local/train.py b/experiments/interpretation/dutchf3_patch/local/train.py index e4567826..9c77d713 100644 --- a/experiments/interpretation/dutchf3_patch/local/train.py +++ b/experiments/interpretation/dutchf3_patch/local/train.py @@ -65,7 +65,6 @@ from default import _C as config from default import update_config -from toolz import take def prepare_batch(batch, device=None, non_blocking=False): @@ -167,6 +166,10 @@ def run(*options, cfg=None, debug=False): num_workers=config.WORKERS, shuffle=True, ) + + if debug: + val_set = data.Subset(val_set, range(3)) + val_loader = data.DataLoader( val_set, batch_size=config.VALIDATION.BATCH_SIZE_PER_GPU, num_workers=config.WORKERS, ) @@ -224,7 +227,7 @@ def run(*options, cfg=None, debug=False): # log all training output trainer.add_event_handler( Events.ITERATION_COMPLETED, - logging_handlers.log_training_output(log_interval=config.PRINT_FREQ), + logging_handlers.log_training_output(log_interval=config.TRAIN.BATCH_SIZE_PER_GPU), ) # add logging of learning rate @@ -269,12 +272,7 @@ def snapshot_function(): }, device=device, ) - - # Set the validation run to start on the epoch completion of the training run - if debug: - logger.info("Running Validation in Debug/Test mode") - val_loader = take(3, val_loader) - + trainer.add_event_handler(Events.EPOCH_COMPLETED, Evaluator(evaluator, val_loader)) evaluator.add_event_handler( @@ -330,12 +328,11 @@ def snapshot_function(): ) evaluator.add_event_handler(Events.EPOCH_COMPLETED, checkpoint_handler, {"model": model}) - logger.info("Starting training") + logger.info("Starting training") if debug: - logger.info("Running Training in Debug/Test mode") - train_loader = take(3, train_loader) - - trainer.run(train_loader, max_epochs=config.TRAIN.END_EPOCH) + trainer.run(train_loader, max_epochs=config.TRAIN.END_EPOCH, epoch_length = config.TRAIN.BATCH_SIZE_PER_GPU, seed = config.SEED) + else: + trainer.run(train_loader, max_epochs=config.TRAIN.END_EPOCH, epoch_length = len(train_loader), seed = config.SEED) if __name__ == "__main__": diff --git a/tests/cicd/main_build.yml b/tests/cicd/main_build.yml index bb65fb15..3a343233 100644 --- a/tests/cicd/main_build.yml +++ b/tests/cicd/main_build.yml @@ -162,7 +162,7 @@ jobs: model=$(ls -td output/patch_deconvnet/no_depth/* | head -1) # try running the test script { python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' \ - 'TEST.MODEL_PATH' ${model}/patch_deconvnet_running_model_1.pth \ + 'TEST.MODEL_PATH' ${model}/patch_deconvnet_running_model_0.*.pth \ --cfg=configs/patch_deconvnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } & pids+=" $!" export CUDA_VISIBLE_DEVICES=1 @@ -170,7 +170,7 @@ jobs: model=$(ls -td output/unet/section_depth/* | head -1) # try running the test script { python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' \ - 'TEST.MODEL_PATH' ${model}/resnet_unet_running_model_1.pth \ + 'TEST.MODEL_PATH' ${model}/resnet_unet_running_model_0.*.pth \ --cfg=configs/unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } & pids+=" $!" export CUDA_VISIBLE_DEVICES=2 @@ -178,7 +178,7 @@ jobs: model=$(ls -td output/seresnet_unet/section_depth/* | head -1) # try running the test script { python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' \ - 'TEST.MODEL_PATH' ${model}/resnet_unet_running_model_1.pth \ + 'TEST.MODEL_PATH' ${model}/resnet_unet_running_model_0.*.pth \ --cfg=configs/seresnet_unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } & pids+=" $!" export CUDA_VISIBLE_DEVICES=3 @@ -187,7 +187,7 @@ jobs: # try running the test script { python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' \ 'MODEL.PRETRAINED' '/home/alfred/models/hrnetv2_w48_imagenet_pretrained.pth' \ - 'TEST.MODEL_PATH' ${model}/seg_hrnet_running_model_1.pth \ + 'TEST.MODEL_PATH' ${model}/seg_hrnet_running_model_0.*.pth \ --cfg=configs/hrnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } & pids+=" $!" From 122b5614ac60739c0deaf2e24acd3f68fa60a76f Mon Sep 17 00:00:00 2001 From: yalaudah Date: Sat, 18 Apr 2020 01:06:29 +0000 Subject: [PATCH 04/20] update docker readme (#262) Co-authored-by: maxkazmsft --- docker/README.md | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/docker/README.md b/docker/README.md index 00d13fe4..e2ccfacb 100644 --- a/docker/README.md +++ b/docker/README.md @@ -8,20 +8,8 @@ If you are using an Azure Virtual Machine to run this code, you can download the ```bash scp hrnetv2_w48_imagenet_pretrained.pth @:/home//seismic-deeplearning/docker/hrnetv2_w48_imagenet_pretrained.pth ``` - Once you have the model downloaded (ideally under the `docker` directory), you can process to build the Docker image. -# Install `nvidia-docker` -In order to give the Docker container access to the GPU, we need to install [`nvidia-docker`](https://github.com/NVIDIA/nvidia-docker). Please follow the instructions [here](https://github.com/NVIDIA/nvidia-docker#quickstart). For an Ubuntu 16.04/18.04 based system, run the following commands: - -```bash -curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - -curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list - -sudo apt-get update && sudo apt-get install -y nvidia-container-toolkit -sudo systemctl restart docker -``` - # Build the Docker image: In the `docker` directory, run the following command to build the Docker image and tag it as `seismic-deeplearning`: @@ -34,8 +22,7 @@ This process will take a few minutes to complete. # Run the Docker image: Once the Docker image is built, you can run it anytime using the following command: ```bash -sudo nvidia-docker run --rm -it -p 9000:9000 -p 9001:9001 --shm-size 11G --mount type=bind,source=$PWD/hrnetv2_w48_imagenet_pretrained.pth,target=/home/username/models/hrnetv2_w48_imagenet_pretrained.pth seismic-deeplearning +sudo docker run --rm -it -p 9000:9000 -p 9001:9001 --gpus=all --shm-size 11G --mount type=bind,source=$PWD/hrnetv2_w48_imagenet_pretrained.pth,target=/home/username/models/hrnetv2_w48_imagenet_pretrained.pth seismic-deeplearning ``` - If you have saved the pretrained model in a different directory, make sure you replace `$PWD/hrnetv2_w48_imagenet_pretrained.pth` with the **absolute** path to the pretrained HRNet model. The command above will run a jupyter notebook server that you can access by clicking on the link in your terminal. You can then navigate to the notebook that you would like to run. From 0fd62d8722b4b99e000942432541d7e557a29b81 Mon Sep 17 00:00:00 2001 From: maxkazmsft Date: Tue, 21 Apr 2020 08:57:02 -0400 Subject: [PATCH 05/20] tagged all TODOs with issues on github (and created issues) (#278) * created correctnes branch, trimmed experiments to Dutch F3 only * trivial change to re-trigger build * dummy PR to re-trigger malfunctioning builds * resolved merge conflict * flagged all non-contrib TODO with github issues * resolved rebase conflict * resolved merge conflict * cleaned up archaic voxel code --- .../dutchf3_patch/distributed/train.py | 13 +- .../dutchf3_section/local/test.py | 1 + .../interpretation/notebooks/utilities.py | 3 + .../dutchf3_patch/local/default.py | 1 + .../dutchf3_patch/local/test.py | 1 + .../dutchf3/data.py | 268 +----------------- .../dutchf3/utils/batch.py | 114 -------- .../models/texture_net.py | 1 + .../penobscot/metrics.py | 4 +- scripts/prepare_dutchf3.py | 1 - 10 files changed, 27 insertions(+), 380 deletions(-) diff --git a/contrib/experiments/interpretation/dutchf3_patch/distributed/train.py b/contrib/experiments/interpretation/dutchf3_patch/distributed/train.py index 9d1a2631..bc28249a 100644 --- a/contrib/experiments/interpretation/dutchf3_patch/distributed/train.py +++ b/contrib/experiments/interpretation/dutchf3_patch/distributed/train.py @@ -24,6 +24,7 @@ import cv2 import fire import numpy as np +import toolz import torch from albumentations import Compose, HorizontalFlip, Normalize, Resize, PadIfNeeded from cv_lib.utils import load_log_configuration @@ -167,8 +168,7 @@ def run(*options, cfg=None, local_rank=0, debug=False): stride=config.TRAIN.STRIDE, patch_size=config.TRAIN.PATCH_SIZE, augmentations=train_aug, - ) - logger.info(f"Training examples {len(train_set)}") + ) val_set = TrainPatchLoader( config.DATASET.ROOT, @@ -185,6 +185,13 @@ def run(*options, cfg=None, local_rank=0, debug=False): logger.info(f"Validation examples {len(val_set)}") n_classes = train_set.n_classes + #if debug: + #val_set = data.Subset(val_set, range(config.VALIDATION.BATCH_SIZE_PER_GPU)) + #train_set = data.Subset(train_set, range(config.TRAIN.BATCH_SIZE_PER_GPU*2)) + + logger.info(f"Training examples {len(train_set)}") + logger.info(f"Validation examples {len(val_set)}") + train_sampler = torch.utils.data.distributed.DistributedSampler(train_set, num_replicas=world_size, rank=local_rank) train_loader = data.DataLoader( @@ -220,6 +227,8 @@ def run(*options, cfg=None, local_rank=0, debug=False): model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[device], find_unused_parameters=True) snapshot_duration = scheduler_step * len(train_loader) + if debug: + snapshot_duration = 2 warmup_duration = 5 * len(train_loader) warmup_scheduler = LinearCyclicalScheduler( optimizer, diff --git a/contrib/experiments/interpretation/dutchf3_section/local/test.py b/contrib/experiments/interpretation/dutchf3_section/local/test.py index 5b4d6858..384ac64b 100644 --- a/contrib/experiments/interpretation/dutchf3_section/local/test.py +++ b/contrib/experiments/interpretation/dutchf3_section/local/test.py @@ -7,6 +7,7 @@ Modified version of the Alaudah testing script # TODO: Needs to be improved. Needs to be able to run across multiple GPUs and better # factoring around the loader +# issue: https://github.com/microsoft/seismic-deeplearning/issues/268 """ import logging diff --git a/examples/interpretation/notebooks/utilities.py b/examples/interpretation/notebooks/utilities.py index cc91523a..f0d3b9e3 100644 --- a/examples/interpretation/notebooks/utilities.py +++ b/examples/interpretation/notebooks/utilities.py @@ -266,6 +266,7 @@ def plot_aline(aline, labels, xlabel, ylabel="depth"): def validate_config_paths(config): """Checks that all paths in the config file are valid""" # TODO: this is currently hardcoded, in the future, its better to have a more generic solution. + # issue https://github.com/microsoft/seismic-deeplearning/issues/265 # Make sure DATASET.ROOT directory exist: assert os.path.isdir(config.DATASET.ROOT), ( @@ -351,6 +352,7 @@ def download_pretrained_model(config): if dataset == "penobscot": if model == "hrnet": # TODO: the code should check if the model uses patches or sections. + # issue: https://github.com/microsoft/seismic-deeplearning/issues/266 url = "https://deepseismicsharedstore.blob.core.windows.net/master-public-models/penobscot_hrnet_patch_section_depth.pth" else: raise NotImplementedError( @@ -421,6 +423,7 @@ def download_pretrained_model(config): # Update config MODEL.PRETRAINED # TODO: Only HRNet uses a pretrained model currently. + # issue https://github.com/microsoft/seismic-deeplearning/issues/267 opts = [ "MODEL.PRETRAINED", pretrained_model_path, diff --git a/experiments/interpretation/dutchf3_patch/local/default.py b/experiments/interpretation/dutchf3_patch/local/default.py index e34627a8..aac539ea 100644 --- a/experiments/interpretation/dutchf3_patch/local/default.py +++ b/experiments/interpretation/dutchf3_patch/local/default.py @@ -58,6 +58,7 @@ _C.TRAIN.PATCH_SIZE = 99 _C.TRAIN.MEAN = 0.0009997 # 0.0009996710808862074 _C.TRAIN.STD = 0.20977 # 0.20976548783479299 # TODO: Should we apply std scaling? +# issue: https://github.com/microsoft/seismic-deeplearning/issues/269 _C.TRAIN.DEPTH = "no" # Options are None, Patch and Section # None adds no depth information and the num of channels remains at 1 # Patch adds depth per patch so is simply the height of that patch from 0 to 1, channels=3 diff --git a/experiments/interpretation/dutchf3_patch/local/test.py b/experiments/interpretation/dutchf3_patch/local/test.py index 3d6eebc7..ee4c0b09 100644 --- a/experiments/interpretation/dutchf3_patch/local/test.py +++ b/experiments/interpretation/dutchf3_patch/local/test.py @@ -420,6 +420,7 @@ def test(*options, cfg=None, debug=False): ) # TODO: make sure that this is consistent with how normalization and agumentation for train.py + # issue: https://github.com/microsoft/seismic-deeplearning/issues/270 patch_aug = Compose( [ Resize( diff --git a/interpretation/deepseismic_interpretation/dutchf3/data.py b/interpretation/deepseismic_interpretation/dutchf3/data.py index 4cd710ad..e11dd059 100644 --- a/interpretation/deepseismic_interpretation/dutchf3/data.py +++ b/interpretation/deepseismic_interpretation/dutchf3/data.py @@ -18,11 +18,7 @@ interpolate_to_fit_data, parse_labels_in_image, get_coordinates_for_slice, - get_grid, - augment_flip, - augment_rot_xy, - augment_rot_z, - augment_stretch, + get_grid, rand_int, trilinear_interpolation, ) @@ -52,46 +48,6 @@ def _test2_labels_for(data_dir): return path.join(data_dir, "test_once", "test2_labels.npy") -def readSEGY(filename): - """[summary] - Read the segy file and return the data as a numpy array and a dictionary describing what has been read in. - - Arguments: - filename {str} -- .segy file location. - - Returns: - [type] -- 3D segy data as numy array and a dictionary with metadata information - """ - - # TODO: we really need to add logging to this repo - print("Loading data cube from", filename, "with:") - - # Read full data cube - data = segyio.tools.cube(filename) - - # Put temporal axis first - data = np.moveaxis(data, -1, 0) - - # Make data cube fast to acess - data = np.ascontiguousarray(data, "float32") - - # Read meta data - segyfile = segyio.open(filename, "r") - print(" Crosslines: ", segyfile.xlines[0], ":", segyfile.xlines[-1]) - print(" Inlines: ", segyfile.ilines[0], ":", segyfile.ilines[-1]) - print(" Timeslices: ", "1", ":", data.shape[0]) - - # Make dict with cube-info - data_info = {} - data_info["crossline_start"] = segyfile.xlines[0] - data_info["inline_start"] = segyfile.ilines[0] - data_info["timeslice_start"] = 1 # Todo: read this from segy - data_info["shape"] = data.shape - # Read dt and other params needed to do create a new - - return data, data_info - - def read_labels(fname, data_info): """ Read labels from an image. @@ -156,95 +112,6 @@ def read_labels(fname, data_info): return label_imgs, label_coordinates - -def get_random_batch( - data_cube, - label_coordinates, - im_size, - batch_size, - index, - random_flip=False, - random_stretch=None, - random_rot_xy=None, - random_rot_z=None, -): - """ - Returns a batch of augmented samples with center pixels randomly drawn from label_coordinates - - Args: - data_cube: 3D numpy array with floating point velocity values - label_coordinates: 3D coordinates of the labeled training slice - im_size: size of the 3D voxel which we're cutting out around each label_coordinate - batch_size: size of the batch - index: element index of this element in a batch - random_flip: bool to perform random voxel flip - random_stretch: bool to enable random stretch - random_rot_xy: bool to enable random rotation of the voxel around dim-0 and dim-1 - random_rot_z: bool to enable random rotation around dim-2 - - Returns: - a tuple of batch numpy array array of data with dimension - (batch, 1, data_cube.shape[0], data_cube.shape[1], data_cube.shape[2]) and the associated labels as an array - of size (batch). - """ - - # always generate only one datapoint - batch_size controls class balance - num_batch_size = 1 - - # Make 3 im_size elements - if isinstance(im_size, int): - im_size = [im_size, im_size, im_size] - - # Output arrays - batch = np.zeros([num_batch_size, 1, im_size[0], im_size[1], im_size[2]]) - ret_labels = np.zeros([num_batch_size]) - - class_keys = list(label_coordinates) - n_classes = len(class_keys) - - # We seek to have a balanced batch with equally many samples from each class. - # get total number of samples per class - samples_per_class = batch_size // n_classes - # figure out index relative to zero (not sequentially counting points) - index = index - batch_size * (index // batch_size) - # figure out which class to sample for this datapoint - class_ind = index // samples_per_class - - # Start by getting a grid centered around (0,0,0) - grid = get_grid(im_size) - - # Apply random flip - if random_flip: - grid = augment_flip(grid) - - # Apply random rotations - if random_rot_xy: - grid = augment_rot_xy(grid, random_rot_xy) - if random_rot_z: - grid = augment_rot_z(grid, random_rot_z) - - # Apply random stretch - if random_stretch: - grid = augment_stretch(grid, random_stretch) - - # Pick random location from the label_coordinates for this class: - coords_for_class = label_coordinates[class_keys[class_ind]] - random_index = rand_int(0, coords_for_class.shape[1]) - coord = coords_for_class[:, random_index : random_index + 1] - - # Move grid to be centered around this location - grid += coord - - # Interpolate samples at grid from the data: - sample = trilinear_interpolation(data_cube, grid) - - # Insert in output arrays - ret_labels[0] = class_ind - batch[0, 0, :, :, :] = np.reshape(sample, (im_size[0], im_size[1], im_size[2])) - - return batch, ret_labels - - class SectionLoader(data.Dataset): """ Base class for section data loader @@ -298,72 +165,6 @@ def transform(self, img, lbl): return torch.from_numpy(img).float(), torch.from_numpy(lbl).long() -class VoxelLoader(data.Dataset): - def __init__( - self, root_path, filename, window_size=65, split="train", n_classes=2, gen_coord_list=False, len=None, - ): - - assert split == "train" or split == "val" - - # location of the file - self.root_path = root_path - self.split = split - self.n_classes = n_classes - self.window_size = window_size - self.coord_list = None - self.filename = filename - self.full_filename = path.join(root_path, filename) - - # Read 3D cube - # NOTE: we cannot pass this data manually as serialization of data into each python process is costly, - # so each worker has to load the data on its own. - self.data, self.data_info = readSEGY(self.full_filename) - if len: - self.len = len - else: - self.len = self.data.size - self.labels = None - - if gen_coord_list: - # generate a list of coordinates to index the entire voxel - # memory footprint of this isn't large yet, so not need to wrap as a generator - nx, ny, nz = self.data.shape - x_list = range(self.window_size, nx - self.window_size) - y_list = range(self.window_size, ny - self.window_size) - z_list = range(self.window_size, nz - self.window_size) - - print("-- generating coord list --") - # TODO: is there any way to use a generator with pyTorch data loader? - self.coord_list = list(itertools.product(x_list, y_list, z_list)) - - def __len__(self): - return self.len - - def __getitem__(self, index): - - # TODO: can we specify a pixel mathematically by index? - pixel = self.coord_list[index] - x, y, z = pixel - # TODO: current bottleneck - can we slice out voxels any faster - small_cube = self.data[ - x - self.window : x + self.window + 1, - y - self.window : y + self.window + 1, - z - self.window : z + self.window + 1, - ] - - return small_cube[np.newaxis, :, :, :], pixel - - # TODO: do we need a transformer for voxels? - """ - def transform(self, img, lbl): - # to be in the BxCxHxW that PyTorch uses: - lbl = np.expand_dims(lbl, 0) - if len(img.shape) == 2: - img = np.expand_dims(img, 0) - return torch.from_numpy(img).float(), torch.from_numpy(lbl).long() - """ - - class TrainSectionLoader(SectionLoader): """ Training data loader for sections @@ -447,47 +248,6 @@ def __getitem__(self, index): return im, lbl -class TrainVoxelWaldelandLoader(VoxelLoader): - def __init__( - self, root_path, filename, split="train", window_size=65, batch_size=None, len=None, - ): - super(TrainVoxelWaldelandLoader, self).__init__( - root_path, filename, split=split, window_size=window_size, len=len - ) - - label_fname = None - if split == "train": - label_fname = path.join(self.root_path, "inline_339.png") - elif split == "val": - label_fname = path.join(self.root_path, "inline_405.png") - else: - raise Exception("undefined split") - - self.class_imgs, self.coordinates = read_labels(label_fname, self.data_info) - - self.batch_size = batch_size if batch_size else 1 - - def __getitem__(self, index): - # print(index) - batch, labels = get_random_batch( - self.data, - self.coordinates, - self.window_size, - self.batch_size, - index, - random_flip=True, - random_stretch=0.2, - random_rot_xy=180, - random_rot_z=15, - ) - - return batch, labels - - -# TODO: write TrainVoxelLoaderWithDepth -TrainVoxelLoaderWithDepth = TrainVoxelWaldelandLoader - - class TestSectionLoader(SectionLoader): """ Test data loader for sections @@ -574,15 +334,6 @@ def __getitem__(self, index): return im, lbl -class TestVoxelWaldelandLoader(VoxelLoader): - def __init__(self, data_dir, split="test"): - super(TestVoxelWaldelandLoader, self).__init__(data_dir, split=split) - - -# TODO: write TestVoxelLoaderWithDepth -TestVoxelLoaderWithDepth = TestVoxelWaldelandLoader - - def _transform_WH_to_HW(numpy_array): assert len(numpy_array.shape) >= 2, "This method needs at least 2D arrays" return np.swapaxes(numpy_array, -2, -1) @@ -627,6 +378,7 @@ def __getitem__(self, index): # Shift offsets the padding that is added in training # shift = self.patch_size if "test" not in self.split else 0 # TODO: Remember we are cancelling the shift since we no longer pad + # issue: https://github.com/microsoft/seismic-deeplearning/issues/273 shift = 0 idx, xdx, ddx = int(idx) + shift, int(xdx) + shift, int(ddx) + shift @@ -680,6 +432,7 @@ def __init__(self, data_dir, stride=30, patch_size=99, is_transform=True, augmen # loading txt file and create patches. if not txt_path: self.split = "test1" # TODO: Fix this can also be test2 + # issue: https://github.com/microsoft/seismic-deeplearning/issues/274 txt_path = path.join(self.data_dir, "splits", "patch_" + self.split + ".txt") patch_list = tuple(open(txt_path, "r")) patch_list = [id_.rstrip() for id_ in patch_list] @@ -760,6 +513,7 @@ def __getitem__(self, index): # Shift offsets the padding that is added in training # shift = self.patch_size if "test" not in self.split else 0 # TODO: Remember we are cancelling the shift since we no longer pad + # issue https://github.com/microsoft/seismic-deeplearning/issues/273 shift = 0 idx, xdx, ddx = int(idx) + shift, int(xdx) + shift, int(ddx) + shift @@ -772,6 +526,7 @@ def __getitem__(self, index): im, lbl = _transform_WH_to_HW(im), _transform_WH_to_HW(lbl) # TODO: Add check for rotation augmentations and raise warning if found + # issue: https://github.com/microsoft/seismic-deeplearning/issues/275 if self.augmentations is not None: augmented_dict = self.augmentations(image=im, mask=lbl) im, lbl = augmented_dict["image"], augmented_dict["mask"] @@ -827,6 +582,7 @@ def __getitem__(self, index): # Shift offsets the padding that is added in training # shift = self.patch_size if "test" not in self.split else 0 # TODO: Remember we are cancelling the shift since we no longer pad + # issue: https://github.com/microsoft/seismic-deeplearning/issues/273 shift = 0 idx, xdx, ddx = int(idx) + shift, int(xdx) + shift, int(ddx) + shift if direction == "i": @@ -862,9 +618,6 @@ def __repr__(self): _TRAIN_SECTION_LOADERS = {"section": TrainSectionLoaderWithDepth} -_TRAIN_VOXEL_LOADERS = {"voxel": TrainVoxelLoaderWithDepth} - - def get_patch_loader(cfg): assert cfg.TRAIN.DEPTH in [ "section", @@ -884,15 +637,6 @@ def get_section_loader(cfg): return _TRAIN_SECTION_LOADERS.get(cfg.TRAIN.DEPTH, TrainSectionLoader) -def get_voxel_loader(cfg): - assert cfg.TRAIN.DEPTH in [ - "voxel", - "none", - ], f"Depth {cfg.TRAIN.DEPTH} not supported for section data. \ - Valid values: voxel, none." - return _TRAIN_SECTION_LOADERS.get(cfg.TRAIN.DEPTH, TrainVoxelWaldelandLoader) - - _TEST_LOADERS = {"section": TestSectionLoaderWithDepth} diff --git a/interpretation/deepseismic_interpretation/dutchf3/utils/batch.py b/interpretation/deepseismic_interpretation/dutchf3/utils/batch.py index 8ebc6790..110abaf5 100644 --- a/interpretation/deepseismic_interpretation/dutchf3/utils/batch.py +++ b/interpretation/deepseismic_interpretation/dutchf3/utils/batch.py @@ -182,63 +182,6 @@ def augment_flip(grid): return grid -def augment_stretch(grid, stretch_factor): - """ - Random stretch/scale - - Args: - grid: 3D coordinate grid of the voxel - stretch_factor: this is actually a boolean which triggers stretching - TODO: change this to just call the function and not do -1,1 in rand_float - - Returns: - stretched grid coordinates - """ - stretch = rand_float(-stretch_factor, stretch_factor) - grid *= 1 + stretch - return grid - - -def augment_rot_xy(grid, random_rot_xy): - """ - Random rotation - - Args: - grid: coordinate grid list of 3D points - random_rot_xy: this is actually a boolean which triggers rotation - TODO: change this to just call the function and not do -1,1 in rand_float - - Returns: - randomly rotated grid - """ - theta = np.deg2rad(rand_float(-random_rot_xy, random_rot_xy)) - x = grid[2, :] * np.cos(theta) - grid[1, :] * np.sin(theta) - y = grid[2, :] * np.sin(theta) + grid[1, :] * np.cos(theta) - grid[1, :] = x - grid[2, :] = y - return grid - - -def augment_rot_z(grid, random_rot_z): - """ - Random tilt around z-axis (dim-2) - - Args: - grid: coordinate grid list of 3D points - random_rot_z: this is actually a boolean which triggers rotation - TODO: change this to just call the function and not do -1,1 in rand_float - - Returns: - randomly tilted coordinate grid - """ - theta = np.deg2rad(rand_float(-random_rot_z, random_rot_z)) - z = grid[0, :] * np.cos(theta) - grid[1, :] * np.sin(theta) - x = grid[0, :] * np.sin(theta) + grid[1, :] * np.cos(theta) - grid[0, :] = z - grid[1, :] = x - return grid - - def trilinear_interpolation(input_array, indices): """ Linear interpolation @@ -343,63 +286,6 @@ def rand_bool(): return bool(np.random.randint(0, 2)) -def augment_stretch(grid, stretch_factor): - """ - Random stretch/scale - - Args: - grid: 3D coordinate grid of the voxel - stretch_factor: this is actually a boolean which triggers stretching - TODO: change this to just call the function and not do -1,1 in rand_float - - Returns: - stretched grid coordinates - """ - stretch = rand_float(-stretch_factor, stretch_factor) - grid *= 1 + stretch - return grid - - -def augment_rot_xy(grid, random_rot_xy): - """ - Random rotation - - Args: - grid: coordinate grid list of 3D points - random_rot_xy: this is actually a boolean which triggers rotation - TODO: change this to just call the function and not do -1,1 in rand_float - - Returns: - randomly rotated grid - """ - theta = np.deg2rad(rand_float(-random_rot_xy, random_rot_xy)) - x = grid[2, :] * np.cos(theta) - grid[1, :] * np.sin(theta) - y = grid[2, :] * np.sin(theta) + grid[1, :] * np.cos(theta) - grid[1, :] = x - grid[2, :] = y - return grid - - -def augment_rot_z(grid, random_rot_z): - """ - Random tilt around z-axis (dim-2) - - Args: - grid: coordinate grid list of 3D points - random_rot_z: this is actually a boolean which triggers rotation - TODO: change this to just call the function and not do -1,1 in rand_float - - Returns: - randomly tilted coordinate grid - """ - theta = np.deg2rad(rand_float(-random_rot_z, random_rot_z)) - z = grid[0, :] * np.cos(theta) - grid[1, :] * np.sin(theta) - x = grid[0, :] * np.sin(theta) + grid[1, :] * np.cos(theta) - grid[0, :] = z - grid[1, :] = x - return grid - - def trilinear_interpolation(input_array, indices): """ Linear interpolation diff --git a/interpretation/deepseismic_interpretation/models/texture_net.py b/interpretation/deepseismic_interpretation/models/texture_net.py index da5371d5..3bc3b1da 100644 --- a/interpretation/deepseismic_interpretation/models/texture_net.py +++ b/interpretation/deepseismic_interpretation/models/texture_net.py @@ -7,6 +7,7 @@ from torch import nn # TODO; set chanels from yaml config file +# issue: https://github.com/microsoft/seismic-deeplearning/issues/277 class TextureNet(nn.Module): def __init__(self, n_classes=2): super(TextureNet, self).__init__() diff --git a/interpretation/deepseismic_interpretation/penobscot/metrics.py b/interpretation/deepseismic_interpretation/penobscot/metrics.py index 846faacc..3411e145 100644 --- a/interpretation/deepseismic_interpretation/penobscot/metrics.py +++ b/interpretation/deepseismic_interpretation/penobscot/metrics.py @@ -18,7 +18,7 @@ def _torch_hist(label_true, label_pred, n_class): Returns: [type]: [description] """ - # TODO Add exceptions + assert len(label_true.shape) == 1, "Labels need to be 1D" assert len(label_pred.shape) == 1, "Predictions need to be 1D" mask = (label_true >= 0) & (label_true < n_class) @@ -34,6 +34,7 @@ def _default_tensor(image_height, image_width, pad_value=255): # TODO: make output transform unpad and scale down mask # scale up y_pred and remove padding +# issue: https://github.com/microsoft/seismic-deeplearning/issues/276 class InlineMeanIoU(Metric): """Compute Mean IoU for Inline @@ -95,6 +96,7 @@ def reset(self): def update(self, output): y_pred, y, ids, patch_locations = output # TODO: Make assertion exception + # issue: https://github.com/microsoft/seismic-deeplearning/issues/276 max_prediction = y_pred.max(1)[1].squeeze() assert y.shape == max_prediction.shape, "Shape not the same" diff --git a/scripts/prepare_dutchf3.py b/scripts/prepare_dutchf3.py index 5c5a4e08..33209074 100644 --- a/scripts/prepare_dutchf3.py +++ b/scripts/prepare_dutchf3.py @@ -331,7 +331,6 @@ def split_alaudah_et_al_19(data_dir, stride, patch_size, fraction_validation=0.2 _write_split_files(splits_path, train_list, val_list, loader_type) -# TODO: Try https://github.com/Chilipp/docrep for doscstring reuse class SplitTrainValCLI(object): def section(self, data_dir, label_file, per_val=0.2, log_config="logging.conf", output_dir=None, From a3d59848d4e990f39f989212a98e00abb1ee2013 Mon Sep 17 00:00:00 2001 From: yalaudah Date: Wed, 22 Apr 2020 14:42:43 +0000 Subject: [PATCH 06/20] Refactoring train.py, removing OpenCV, adding training results to Tensborboard, bug fixes (#264) I think moving forward, we'll use smaller PRs. But here are the changes in this one: Fixes issue #236 that involves rewriting a big portion of train.py such that: All the tensorboard event handlers are organized in tensorboard_handlers.py and only called in train.py to log training and validation results in Tensorboard The code logs the same results for training and validation. Also, it adds the class IoU score as well. All single-use functions (e.g. _select_max, _tensor_to_numpy, _select_pred_and_mask) are lambda functions now The code is organized into more meaningful "chunks".. e.g. all the optimizer-related code should be together if possible, same thing for logging, configuration, loaders, tensorboard, ..etc. In addition: Fixed a visualization bug where the seismic images where not normalized correctly. This solves Issue #217. Fixed a visualization bug where the predictions where not masked where the input image was padded. This improves the ability to visually inspect and evaluate the results. This solves Issue #230. Fixes a potential issue where Tensorboard can crash when a large training batchsize is used. Now the number of images visualized in Tensorboard from every batch has an upper limit. Completely removed OpenCV as a dependency from the DeepSeismic Repo. It was only used in a small part of the code where it wasn't really necessary, and OpenCV is a huge library. Fixes Issue #218 where the epoch number for the images in Tensorboard was always logged as 1 (therefore, not allowing use to see the epoch number of the different results in Tensorboard. Removes the HorovodLRScheduler class since its no longer used Removes toolz.take from Debug mode, and uses PyTorch's native Subset() dataset class Changes default patch size for the HRNet model to 256 In addition to several other minor changes Co-authored-by: Yazeed Alaudah Co-authored-by: Ubuntu Co-authored-by: Max Kaznady --- AUTHORS.md | 3 +- README.md | 10 +- .../distributed/configs/hrnet.yaml | 3 +- .../distributed/configs/patch_deconvnet.yaml | 2 +- .../configs/patch_deconvnet_skip.yaml | 2 +- .../distributed/configs/seresnet_unet.yaml | 2 +- .../distributed/configs/unet.yaml | 2 +- .../dutchf3_patch/distributed/default.py | 3 +- .../dutchf3_patch/distributed/train.py | 118 ++++---- .../dutchf3_section/local/default.py | 3 +- .../dutchf3_section/local/train.py | 6 +- .../dutchf3_voxel/configs/texture_net.yaml | 2 +- .../interpretation/dutchf3_voxel/default.py | 4 +- .../interpretation/dutchf3_voxel/train.py | 2 +- .../penobscot/local/configs/hrnet.yaml | 3 +- .../local/configs/seresnet_unet.yaml | 2 +- .../interpretation/penobscot/local/default.py | 3 +- .../interpretation/penobscot/local/test.py | 41 +-- .../interpretation/penobscot/local/train.py | 63 ++--- .../cv_lib/event_handlers/logging_handlers.py | 43 +-- .../event_handlers/tensorboard_handlers.py | 76 ++++-- environment/anaconda/local/environment.yml | 1 - environment/docker/apex/dockerfile | 2 +- environment/docker/horovod/dockerfile | 2 +- ..._patch_model_training_and_evaluation.ipynb | 9 +- .../dutchf3_patch/local/configs/hrnet.yaml | 11 +- .../local/configs/patch_deconvnet.yaml | 2 +- .../local/configs/patch_deconvnet_skip.yaml | 2 +- .../local/configs/seresnet_unet.yaml | 2 +- .../dutchf3_patch/local/configs/unet.yaml | 2 +- .../dutchf3_patch/local/default.py | 6 +- .../dutchf3_patch/local/test.py | 95 ++----- .../dutchf3_patch/local/train.py | 253 ++++++------------ .../dutchf3/data.py | 14 +- tests/cicd/main_build.yml | 25 +- 35 files changed, 325 insertions(+), 494 deletions(-) diff --git a/AUTHORS.md b/AUTHORS.md index cb2995fa..b903ddb4 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -9,14 +9,15 @@ Contributors (sorted alphabetically) ------------------------------------- To contributors: please add your name to the list when you submit a patch to the project. +* Yazeed Alaudah * Ashish Bhatia +* Sharat Chikkerur * Daniel Ciborowski * George Iordanescu * Ilia Karmanov * Max Kaznady * Vanja Paunic * Mathew Salvaris -* Sharat Chikkerur * Wee Hyong Tok ## How to be a contributor to the repository diff --git a/README.md b/README.md index cf8886a7..44d60b5a 100644 --- a/README.md +++ b/README.md @@ -287,7 +287,7 @@ for the Penobscot dataset follow the same instructions but navigate to the [peno ## Contributing -This project welcomes contributions and suggestions. Most contributions require you to agree to a Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com. +This project welcomes contributions and suggestions. Most contributions require you to agree to a Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us the rights to use your contribution. For details, visit [https://cla.opensource.microsoft.com](https://cla.opensource.microsoft.com). ### Submitting a Pull Request @@ -321,7 +321,7 @@ A typical output will be: someusername@somevm:/projects/DeepSeismic$ which python /anaconda/envs/py35/bin/python ``` -which will indicate that anaconda folder is __/anaconda__. We'll refer to this location in the instructions below, but you should update the commands according to your local anaconda folder. +which will indicate that anaconda folder is `__/anaconda__`. We'll refer to this location in the instructions below, but you should update the commands according to your local anaconda folder.
Data Science Virtual Machine conda package installation errors @@ -339,7 +339,7 @@ which will indicate that anaconda folder is __/anaconda__. We'll refer to this l
Data Science Virtual Machine conda package installation warnings - It could happen that while creating the conda environment defined by environment/anaconda/local/environment.yml on an Ubuntu DSVM, one can get multiple warnings like so: + It could happen that while creating the conda environment defined by `environment/anaconda/local/environment.yml` on an Ubuntu DSVM, one can get multiple warnings like so: ``` WARNING conda.gateways.disk.delete:unlink_or_rename_to_trash(140): Could not remove or rename /anaconda/pkgs/ipywidgets-7.5.1-py_0/site-packages/ipywidgets-7.5.1.dist-info/LICENSE. Please remove this file manually (you may need to reboot to free file handles) ``` @@ -350,7 +350,7 @@ which will indicate that anaconda folder is __/anaconda__. We'll refer to this l sudo chown -R $USER /anaconda ``` - After these command completes, try creating the conda environment in __environment/anaconda/local/environment.yml__ again. + After these command completes, try creating the conda environment in `__environment/anaconda/local/environment.yml__` again.
@@ -395,7 +395,7 @@ which will indicate that anaconda folder is __/anaconda__. We'll refer to this l
GPU out of memory errors - You should be able to see how much GPU memory your process is using by running + You should be able to see how much GPU memory your process is using by running: ```bash nvidia-smi ``` diff --git a/contrib/experiments/interpretation/dutchf3_patch/distributed/configs/hrnet.yaml b/contrib/experiments/interpretation/dutchf3_patch/distributed/configs/hrnet.yaml index 04ad6479..fe3995f6 100644 --- a/contrib/experiments/interpretation/dutchf3_patch/distributed/configs/hrnet.yaml +++ b/contrib/experiments/interpretation/dutchf3_patch/distributed/configs/hrnet.yaml @@ -9,6 +9,7 @@ WORKERS: 4 PRINT_FREQ: 10 LOG_CONFIG: logging.conf SEED: 2019 +OPENCV_BORDER_CONSTANT: 0 DATASET: @@ -73,7 +74,7 @@ TRAIN: WEIGHT_DECAY: 0.0001 SNAPSHOTS: 5 AUGMENTATION: True - DEPTH: "section" #"patch" # Options are No, Patch and Section + DEPTH: "section" #"patch" # Options are none, patch, and section STRIDE: 50 PATCH_SIZE: 100 AUGMENTATIONS: diff --git a/contrib/experiments/interpretation/dutchf3_patch/distributed/configs/patch_deconvnet.yaml b/contrib/experiments/interpretation/dutchf3_patch/distributed/configs/patch_deconvnet.yaml index eb89ff00..fa1d6add 100644 --- a/contrib/experiments/interpretation/dutchf3_patch/distributed/configs/patch_deconvnet.yaml +++ b/contrib/experiments/interpretation/dutchf3_patch/distributed/configs/patch_deconvnet.yaml @@ -30,7 +30,7 @@ TRAIN: WEIGHT_DECAY: 0.0001 SNAPSHOTS: 5 AUGMENTATION: True - DEPTH: "none" #"patch" # Options are None, Patch and Section + DEPTH: "none" #"patch" # Options are none, patch, and section STRIDE: 50 PATCH_SIZE: 99 AUGMENTATIONS: diff --git a/contrib/experiments/interpretation/dutchf3_patch/distributed/configs/patch_deconvnet_skip.yaml b/contrib/experiments/interpretation/dutchf3_patch/distributed/configs/patch_deconvnet_skip.yaml index eb89ff00..fa1d6add 100644 --- a/contrib/experiments/interpretation/dutchf3_patch/distributed/configs/patch_deconvnet_skip.yaml +++ b/contrib/experiments/interpretation/dutchf3_patch/distributed/configs/patch_deconvnet_skip.yaml @@ -30,7 +30,7 @@ TRAIN: WEIGHT_DECAY: 0.0001 SNAPSHOTS: 5 AUGMENTATION: True - DEPTH: "none" #"patch" # Options are None, Patch and Section + DEPTH: "none" #"patch" # Options are none, patch, and section STRIDE: 50 PATCH_SIZE: 99 AUGMENTATIONS: diff --git a/contrib/experiments/interpretation/dutchf3_patch/distributed/configs/seresnet_unet.yaml b/contrib/experiments/interpretation/dutchf3_patch/distributed/configs/seresnet_unet.yaml index d0b8126f..9bc10d34 100644 --- a/contrib/experiments/interpretation/dutchf3_patch/distributed/configs/seresnet_unet.yaml +++ b/contrib/experiments/interpretation/dutchf3_patch/distributed/configs/seresnet_unet.yaml @@ -30,7 +30,7 @@ TRAIN: WEIGHT_DECAY: 0.0001 SNAPSHOTS: 5 AUGMENTATION: True - DEPTH: "section" # Options are No, Patch and Section + DEPTH: "section" # Options are none, patch, and section STRIDE: 50 PATCH_SIZE: 100 AUGMENTATIONS: diff --git a/contrib/experiments/interpretation/dutchf3_patch/distributed/configs/unet.yaml b/contrib/experiments/interpretation/dutchf3_patch/distributed/configs/unet.yaml index 2843e62c..3fe5f439 100644 --- a/contrib/experiments/interpretation/dutchf3_patch/distributed/configs/unet.yaml +++ b/contrib/experiments/interpretation/dutchf3_patch/distributed/configs/unet.yaml @@ -33,7 +33,7 @@ TRAIN: WEIGHT_DECAY: 0.0001 SNAPSHOTS: 5 AUGMENTATION: True - DEPTH: "section" # Options are No, Patch and Section + DEPTH: "section" # Options are none, patch, and section STRIDE: 50 PATCH_SIZE: 100 AUGMENTATIONS: diff --git a/contrib/experiments/interpretation/dutchf3_patch/distributed/default.py b/contrib/experiments/interpretation/dutchf3_patch/distributed/default.py index bf23527b..34d3c4d3 100644 --- a/contrib/experiments/interpretation/dutchf3_patch/distributed/default.py +++ b/contrib/experiments/interpretation/dutchf3_patch/distributed/default.py @@ -20,6 +20,7 @@ _C.PIN_MEMORY = True _C.LOG_CONFIG = "logging.conf" _C.SEED = 42 +_C.OPENCV_BORDER_CONSTANT = 0 # Cudnn related params _C.CUDNN = CN() @@ -58,7 +59,7 @@ _C.TRAIN.PATCH_SIZE = 99 _C.TRAIN.MEAN = 0.0009997 # 0.0009996710808862074 _C.TRAIN.STD = 0.21 # 0.20976548783479299 -_C.TRAIN.DEPTH = "None" # Options are None, Patch and Section +_C.TRAIN.DEPTH = "none" # Options are: none, patch, and section # None adds no depth information and the num of channels remains at 1 # Patch adds depth per patch so is simply the height of that patch from 0 to 1, channels=3 # Section adds depth per section so contains depth information for the whole section, channels=3 diff --git a/contrib/experiments/interpretation/dutchf3_patch/distributed/train.py b/contrib/experiments/interpretation/dutchf3_patch/distributed/train.py index bc28249a..33bb0045 100644 --- a/contrib/experiments/interpretation/dutchf3_patch/distributed/train.py +++ b/contrib/experiments/interpretation/dutchf3_patch/distributed/train.py @@ -21,59 +21,30 @@ import os from os import path -import cv2 import fire import numpy as np import toolz import torch -from albumentations import Compose, HorizontalFlip, Normalize, Resize, PadIfNeeded -from cv_lib.utils import load_log_configuration -from cv_lib.event_handlers import ( - SnapshotHandler, - logging_handlers, - tensorboard_handlers, -) -from cv_lib.event_handlers.logging_handlers import Evaluator -from cv_lib.event_handlers.tensorboard_handlers import ( - create_image_writer, - create_summary_writer, -) -from cv_lib.segmentation import models -from cv_lib.segmentation import extract_metric_from -from deepseismic_interpretation.dutchf3.data import get_patch_loader, decode_segmap -from cv_lib.segmentation.dutchf3.engine import ( - create_supervised_evaluator, - create_supervised_trainer, -) - -from ignite.metrics import Loss -from cv_lib.segmentation.metrics import ( - pixelwise_accuracy, - class_accuracy, - mean_class_accuracy, - class_iou, - mean_iou, -) - -from cv_lib.segmentation.dutchf3.utils import ( - current_datetime, - generate_path, - git_branch, - git_hash, - np_to_tb, -) -from default import _C as config -from default import update_config -from ignite.contrib.handlers import ( - ConcatScheduler, - CosineAnnealingScheduler, - LinearCyclicalScheduler, -) +from albumentations import Compose, HorizontalFlip, Normalize, PadIfNeeded, Resize +from ignite.contrib.handlers import ConcatScheduler, CosineAnnealingScheduler, LinearCyclicalScheduler from ignite.engine import Events +from ignite.metrics import Loss from ignite.utils import convert_tensor from toolz import compose, curry from torch.utils import data +from cv_lib.event_handlers import SnapshotHandler, logging_handlers, tensorboard_handlers +from cv_lib.event_handlers.logging_handlers import Evaluator +from cv_lib.event_handlers.tensorboard_handlers import create_image_writer, create_summary_writer +from cv_lib.segmentation import extract_metric_from, models +from cv_lib.segmentation.dutchf3.engine import create_supervised_evaluator, create_supervised_trainer +from cv_lib.segmentation.dutchf3.utils import current_datetime, generate_path, git_branch, git_hash, np_to_tb +from cv_lib.segmentation.metrics import class_accuracy, class_iou, mean_class_accuracy, mean_iou, pixelwise_accuracy +from cv_lib.utils import load_log_configuration +from deepseismic_interpretation.dutchf3.data import decode_segmap, get_patch_loader +from default import _C as config +from default import update_config + def prepare_batch(batch, device=None, non_blocking=False): x, y = batch @@ -123,7 +94,7 @@ def run(*options, cfg=None, local_rank=0, debug=False): # provide environment variables, and requires that you use init_method=`env://`. torch.distributed.init_process_group(backend="nccl", init_method="env://") - scheduler_step = config.TRAIN.END_EPOCH // config.TRAIN.SNAPSHOTS + epochs_per_cycle = config.TRAIN.END_EPOCH // config.TRAIN.SNAPSHOTS torch.backends.cudnn.benchmark = config.CUDNN.BENCHMARK torch.manual_seed(config.SEED) @@ -137,7 +108,7 @@ def run(*options, cfg=None, local_rank=0, debug=False): PadIfNeeded( min_height=config.TRAIN.PATCH_SIZE, min_width=config.TRAIN.PATCH_SIZE, - border_mode=cv2.BORDER_CONSTANT, + border_mode=config.OPENCV_BORDER_CONSTANT, always_apply=True, mask_value=255, ), @@ -147,7 +118,7 @@ def run(*options, cfg=None, local_rank=0, debug=False): PadIfNeeded( min_height=config.TRAIN.AUGMENTATIONS.PAD.HEIGHT, min_width=config.TRAIN.AUGMENTATIONS.PAD.WIDTH, - border_mode=cv2.BORDER_CONSTANT, + border_mode=config.OPENCV_BORDER_CONSTANT, always_apply=True, mask_value=255, ), @@ -185,15 +156,16 @@ def run(*options, cfg=None, local_rank=0, debug=False): logger.info(f"Validation examples {len(val_set)}") n_classes = train_set.n_classes - #if debug: - #val_set = data.Subset(val_set, range(config.VALIDATION.BATCH_SIZE_PER_GPU)) - #train_set = data.Subset(train_set, range(config.TRAIN.BATCH_SIZE_PER_GPU*2)) + if debug: + logger.info("Running in debug mode..") + train_set = data.Subset(train_set, list(range(4))) + val_set = data.Subset(val_set, list(range(4))) logger.info(f"Training examples {len(train_set)}") logger.info(f"Validation examples {len(val_set)}") - train_sampler = torch.utils.data.distributed.DistributedSampler(train_set, num_replicas=world_size, rank=local_rank) + train_sampler = torch.utils.data.distributed.DistributedSampler(train_set, num_replicas=world_size, rank=local_rank) train_loader = data.DataLoader( train_set, batch_size=config.TRAIN.BATCH_SIZE_PER_GPU, num_workers=config.WORKERS, sampler=train_sampler, ) @@ -226,9 +198,7 @@ def run(*options, cfg=None, local_rank=0, debug=False): model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[device], find_unused_parameters=True) - snapshot_duration = scheduler_step * len(train_loader) - if debug: - snapshot_duration = 2 + snapshot_duration = epochs_per_cycle * len(train_loader) if not debug else 2*len(train_loader) warmup_duration = 5 * len(train_loader) warmup_scheduler = LinearCyclicalScheduler( optimizer, @@ -238,7 +208,7 @@ def run(*options, cfg=None, local_rank=0, debug=False): cycle_size=10 * len(train_loader), ) cosine_scheduler = CosineAnnealingScheduler( - optimizer, "lr", config.TRAIN.MAX_LR * world_size, config.TRAIN.MIN_LR * world_size, snapshot_duration, + optimizer, "lr", config.TRAIN.MAX_LR * world_size, config.TRAIN.MIN_LR * world_size, cycle_size=snapshot_duration, ) scheduler = ConcatScheduler(schedulers=[warmup_scheduler, cosine_scheduler], durations=[warmup_duration]) @@ -270,18 +240,27 @@ def _select_pred_and_mask(model_out_dict): device=device, ) - # Set the validation run to start on the epoch completion of the training run + # Set the validation run to start on the epoch completion of the training run + trainer.add_event_handler(Events.EPOCH_COMPLETED, Evaluator(evaluator, val_loader)) if local_rank == 0: # Run only on master process trainer.add_event_handler( - Events.ITERATION_COMPLETED, logging_handlers.log_training_output(log_interval=config.TRAIN.BATCH_SIZE_PER_GPU), + Events.ITERATION_COMPLETED, + logging_handlers.log_training_output(log_interval=config.TRAIN.BATCH_SIZE_PER_GPU), ) - trainer.add_event_handler(Events.EPOCH_STARTED, logging_handlers.log_lr(optimizer)) + trainer.add_event_handler(Events.EPOCH_STARTED, logging_handlers.log_lr(optimizer)) try: - output_dir = generate_path(config.OUTPUT_DIR, git_branch(), git_hash(), config_file_name, config.TRAIN.MODEL_DIR, current_datetime(),) + output_dir = generate_path( + config.OUTPUT_DIR, + git_branch(), + git_hash(), + config_file_name, + config.TRAIN.MODEL_DIR, + current_datetime(), + ) except TypeError: output_dir = generate_path(config.OUTPUT_DIR, config_file_name, config.TRAIN.MODEL_DIR, current_datetime(),) @@ -322,9 +301,7 @@ def _tensor_to_numpy(pred_tensor): return pred_tensor.squeeze().cpu().numpy() transform_func = compose(np_to_tb, decode_segmap(n_classes=n_classes), _tensor_to_numpy) - transform_pred = compose(transform_func, _select_max) - evaluator.add_event_handler( Events.EPOCH_COMPLETED, create_image_writer(summary_writer, "Validation/Image", "image"), ) @@ -341,19 +318,22 @@ def snapshot_function(): return (trainer.state.iteration % snapshot_duration) == 0 checkpoint_handler = SnapshotHandler( - output_dir, - config.MODEL.NAME, - extract_metric_from("mIoU"), - snapshot_function, + output_dir, config.MODEL.NAME, extract_metric_from("mIoU"), snapshot_function, ) evaluator.add_event_handler(Events.EPOCH_COMPLETED, checkpoint_handler, {"model": model}) - logger.info("Starting training") - + if debug: - trainer.run(train_loader, max_epochs=config.TRAIN.END_EPOCH, epoch_length = config.TRAIN.BATCH_SIZE_PER_GPU*2, seed = config.SEED) + trainer.run( + train_loader, + max_epochs=config.TRAIN.END_EPOCH, + epoch_length=config.TRAIN.BATCH_SIZE_PER_GPU * 2, + seed=config.SEED, + ) else: - trainer.run(train_loader, max_epochs=config.TRAIN.END_EPOCH, epoch_length = len(train_loader), seed = config.SEED) + trainer.run( + train_loader, max_epochs=config.TRAIN.END_EPOCH, epoch_length=len(train_loader), seed=config.SEED + ) if __name__ == "__main__": diff --git a/contrib/experiments/interpretation/dutchf3_section/local/default.py b/contrib/experiments/interpretation/dutchf3_section/local/default.py index 5e296295..2b4888d2 100644 --- a/contrib/experiments/interpretation/dutchf3_section/local/default.py +++ b/contrib/experiments/interpretation/dutchf3_section/local/default.py @@ -21,6 +21,7 @@ _C.PIN_MEMORY = True _C.LOG_CONFIG = "./logging.conf" # Logging config file relative to the experiment _C.SEED = 42 +_C.OPENCV_BORDER_CONSTANT = 0 # Cudnn related params _C.CUDNN = CN() @@ -55,7 +56,7 @@ _C.TRAIN.AUGMENTATION = True _C.TRAIN.MEAN = 0.0009997 # 0.0009996710808862074 _C.TRAIN.STD = 0.20977 # 0.20976548783479299 -_C.TRAIN.DEPTH = "none" # Options are 'none', 'patch' and 'section' +_C.TRAIN.DEPTH = "none" # Options are: none, patch, and section # None adds no depth information and the num of channels remains at 1 # Patch adds depth per patch so is simply the height of that patch from 0 to 1, channels=3 # Section adds depth per section so contains depth information for the whole section, channels=3 diff --git a/contrib/experiments/interpretation/dutchf3_section/local/train.py b/contrib/experiments/interpretation/dutchf3_section/local/train.py index b216268e..5a9b4900 100644 --- a/contrib/experiments/interpretation/dutchf3_section/local/train.py +++ b/contrib/experiments/interpretation/dutchf3_section/local/train.py @@ -84,7 +84,7 @@ def run(*options, cfg=None, debug=False): load_log_configuration(config.LOG_CONFIG) logger = logging.getLogger(__name__) logger.debug(config.WORKERS) - scheduler_step = config.TRAIN.END_EPOCH // config.TRAIN.SNAPSHOTS + epochs_per_cycle = config.TRAIN.END_EPOCH // config.TRAIN.SNAPSHOTS torch.backends.cudnn.benchmark = config.CUDNN.BENCHMARK torch.manual_seed(config.SEED) @@ -164,8 +164,8 @@ def __len__(self): summary_writer = create_summary_writer(log_dir=path.join(output_dir, config.LOG_DIR)) - snapshot_duration = scheduler_step * len(train_loader) - scheduler = CosineAnnealingScheduler(optimizer, "lr", config.TRAIN.MAX_LR, config.TRAIN.MIN_LR, snapshot_duration) + snapshot_duration = epochs_per_cycle * len(train_loader) if not debug else 2*len(train_loader) + scheduler = CosineAnnealingScheduler(optimizer, "lr", config.TRAIN.MAX_LR, config.TRAIN.MIN_LR, cycle_size=snapshot_duration) # weights are inversely proportional to the frequency of the classes in # the training set diff --git a/contrib/experiments/interpretation/dutchf3_voxel/configs/texture_net.yaml b/contrib/experiments/interpretation/dutchf3_voxel/configs/texture_net.yaml index aeeffb86..3ff72dca 100644 --- a/contrib/experiments/interpretation/dutchf3_voxel/configs/texture_net.yaml +++ b/contrib/experiments/interpretation/dutchf3_voxel/configs/texture_net.yaml @@ -29,7 +29,7 @@ TRAIN: LR: 0.02 MOMENTUM: 0.9 WEIGHT_DECAY: 0.0001 - DEPTH: "voxel" # Options are No, Patch, Section and Voxel + DEPTH: "voxel" # Options are none, patch, section and voxel MODEL_DIR: "models" VALIDATION: diff --git a/contrib/experiments/interpretation/dutchf3_voxel/default.py b/contrib/experiments/interpretation/dutchf3_voxel/default.py index 100da598..bcf84731 100644 --- a/contrib/experiments/interpretation/dutchf3_voxel/default.py +++ b/contrib/experiments/interpretation/dutchf3_voxel/default.py @@ -24,6 +24,8 @@ _C.PRINT_FREQ = 20 _C.LOG_CONFIG = "logging.conf" _C.SEED = 42 +_C.OPENCV_BORDER_CONSTANT = 0 + # size of voxel cube: WINDOW_SIZE x WINDOW_SIZE x WINDOW_SIZE; used for 3D models only _C.WINDOW_SIZE = 65 @@ -50,7 +52,7 @@ _C.TRAIN.LR = 0.01 _C.TRAIN.MOMENTUM = 0.9 _C.TRAIN.WEIGHT_DECAY = 0.0001 -_C.TRAIN.DEPTH = "voxel" # Options are None, Patch and Section +_C.TRAIN.DEPTH = "voxel" # Options are none, patch and section _C.TRAIN.MODEL_DIR = "models" # This will be a subdirectory inside OUTPUT_DIR # validation diff --git a/contrib/experiments/interpretation/dutchf3_voxel/train.py b/contrib/experiments/interpretation/dutchf3_voxel/train.py index bd8cdf4b..3864e38f 100644 --- a/contrib/experiments/interpretation/dutchf3_voxel/train.py +++ b/contrib/experiments/interpretation/dutchf3_voxel/train.py @@ -208,7 +208,7 @@ def _select_pred_and_mask(model_out): summary_writer = create_summary_writer(log_dir=path.join(output_dir, config.LOG_DIR)) - snapshot_duration = 1 + snapshot_duration = 2 def snapshot_function(): return (trainer.state.iteration % snapshot_duration) == 0 diff --git a/contrib/experiments/interpretation/penobscot/local/configs/hrnet.yaml b/contrib/experiments/interpretation/penobscot/local/configs/hrnet.yaml index ba4b3967..7c711177 100644 --- a/contrib/experiments/interpretation/penobscot/local/configs/hrnet.yaml +++ b/contrib/experiments/interpretation/penobscot/local/configs/hrnet.yaml @@ -9,6 +9,7 @@ WORKERS: 4 PRINT_FREQ: 10 LOG_CONFIG: logging.conf SEED: 2019 +OPENCV_BORDER_CONSTANT: 0 DATASET: @@ -75,7 +76,7 @@ TRAIN: WEIGHT_DECAY: 0.0001 SNAPSHOTS: 5 AUGMENTATION: True - DEPTH: "patch" # Options are none, patch and section + DEPTH: "patch" # Options are none, patch, and section STRIDE: 64 PATCH_SIZE: 128 AUGMENTATIONS: diff --git a/contrib/experiments/interpretation/penobscot/local/configs/seresnet_unet.yaml b/contrib/experiments/interpretation/penobscot/local/configs/seresnet_unet.yaml index 3ba4d807..800cf4ce 100644 --- a/contrib/experiments/interpretation/penobscot/local/configs/seresnet_unet.yaml +++ b/contrib/experiments/interpretation/penobscot/local/configs/seresnet_unet.yaml @@ -32,7 +32,7 @@ TRAIN: WEIGHT_DECAY: 0.0001 SNAPSHOTS: 5 AUGMENTATION: True - DEPTH: "patch" # Options are none, patch and section + DEPTH: "patch" # Options are none, patch, and section STRIDE: 64 PATCH_SIZE: 128 AUGMENTATIONS: diff --git a/contrib/experiments/interpretation/penobscot/local/default.py b/contrib/experiments/interpretation/penobscot/local/default.py index fa8e540e..d72946ce 100644 --- a/contrib/experiments/interpretation/penobscot/local/default.py +++ b/contrib/experiments/interpretation/penobscot/local/default.py @@ -21,6 +21,7 @@ _C.PIN_MEMORY = True _C.LOG_CONFIG = "logging.conf" _C.SEED = 42 +_C.OPENCV_BORDER_CONSTANT = 0 # size of voxel cube: WINDOW_SIZE x WINDOW_SIZE x WINDOW_SIZE; used for 3D models only _C.WINDOW_SIZE = 65 @@ -72,7 +73,7 @@ _C.TRAIN.MEAN = [-0.0001777, 0.49, -0.0000688] # 0.0009996710808862074 _C.TRAIN.STD = [0.14076, 0.2717, 0.06286] # 0.20976548783479299 _C.TRAIN.MAX = 1 -_C.TRAIN.DEPTH = "patch" # Options are none, patch and section +_C.TRAIN.DEPTH = "patch" # Options are none, patch, and section # None adds no depth information and the num of channels remains at 1 # Patch adds depth per patch so is simply the height of that patch from 0 to 1, channels=3 # Section adds depth per section so contains depth information for the whole section, channels=3 diff --git a/contrib/experiments/interpretation/penobscot/local/test.py b/contrib/experiments/interpretation/penobscot/local/test.py index 8687085a..073c72f1 100644 --- a/contrib/experiments/interpretation/penobscot/local/test.py +++ b/contrib/experiments/interpretation/penobscot/local/test.py @@ -18,45 +18,30 @@ from itertools import chain from os import path -import cv2 import fire import numpy as np import torch import torchvision from albumentations import Compose, Normalize, PadIfNeeded, Resize -from cv_lib.utils import load_log_configuration +from ignite.engine import Events +from ignite.metrics import Loss +from ignite.utils import convert_tensor +from toolz import compose, tail, take +from toolz.sandbox.core import unzip +from torch.utils import data + from cv_lib.event_handlers import logging_handlers, tensorboard_handlers -from cv_lib.event_handlers.tensorboard_handlers import ( - create_image_writer, - create_summary_writer, -) +from cv_lib.event_handlers.tensorboard_handlers import create_image_writer, create_summary_writer from cv_lib.segmentation import models -from cv_lib.segmentation.metrics import ( - pixelwise_accuracy, - class_accuracy, - mean_class_accuracy, - class_iou, - mean_iou, -) -from cv_lib.segmentation.dutchf3.utils import ( - current_datetime, - generate_path, - git_branch, - git_hash, - np_to_tb, -) +from cv_lib.segmentation.dutchf3.utils import current_datetime, generate_path, git_branch, git_hash, np_to_tb +from cv_lib.segmentation.metrics import class_accuracy, class_iou, mean_class_accuracy, mean_iou, pixelwise_accuracy from cv_lib.segmentation.penobscot.engine import create_supervised_evaluator +from cv_lib.utils import load_log_configuration from deepseismic_interpretation.dutchf3.data import decode_segmap from deepseismic_interpretation.penobscot.data import get_patch_dataset from deepseismic_interpretation.penobscot.metrics import InlineMeanIoU from default import _C as config from default import update_config -from ignite.engine import Events -from ignite.metrics import Loss -from ignite.utils import convert_tensor -from toolz import compose, tail, take -from toolz.sandbox.core import unzip -from torch.utils import data def _prepare_batch(batch, device=None, non_blocking=False): @@ -139,7 +124,7 @@ def run(*options, cfg=None, debug=False): PadIfNeeded( min_height=config.TRAIN.PATCH_SIZE, min_width=config.TRAIN.PATCH_SIZE, - border_mode=cv2.BORDER_CONSTANT, + border_mode=config.OPENCV_BORDER_CONSTANT, always_apply=True, mask_value=mask_value, value=0, @@ -150,7 +135,7 @@ def run(*options, cfg=None, debug=False): PadIfNeeded( min_height=config.TRAIN.AUGMENTATIONS.PAD.HEIGHT, min_width=config.TRAIN.AUGMENTATIONS.PAD.WIDTH, - border_mode=cv2.BORDER_CONSTANT, + border_mode=config.OPENCV_BORDER_CONSTANT, always_apply=True, mask_value=mask_value, value=0, diff --git a/contrib/experiments/interpretation/penobscot/local/train.py b/contrib/experiments/interpretation/penobscot/local/train.py index 22394b53..04f563b7 100644 --- a/contrib/experiments/interpretation/penobscot/local/train.py +++ b/contrib/experiments/interpretation/penobscot/local/train.py @@ -17,7 +17,6 @@ import logging.config from os import path -import cv2 import fire import numpy as np import torch @@ -29,43 +28,19 @@ from toolz import compose from torch.utils import data +from cv_lib.event_handlers import SnapshotHandler, logging_handlers, tensorboard_handlers +from cv_lib.event_handlers.logging_handlers import Evaluator +from cv_lib.event_handlers.tensorboard_handlers import create_image_writer, create_summary_writer +from cv_lib.segmentation import extract_metric_from, models +from cv_lib.segmentation.dutchf3.utils import current_datetime, generate_path, git_branch, git_hash, np_to_tb +from cv_lib.segmentation.metrics import class_accuracy, class_iou, mean_class_accuracy, mean_iou, pixelwise_accuracy +from cv_lib.segmentation.penobscot.engine import create_supervised_evaluator, create_supervised_trainer +from cv_lib.utils import load_log_configuration from deepseismic_interpretation.dutchf3.data import decode_segmap from deepseismic_interpretation.penobscot.data import get_patch_dataset -from cv_lib.utils import load_log_configuration -from cv_lib.event_handlers import ( - SnapshotHandler, - logging_handlers, - tensorboard_handlers, -) -from cv_lib.event_handlers.logging_handlers import Evaluator -from cv_lib.event_handlers.tensorboard_handlers import ( - create_image_writer, - create_summary_writer, -) -from cv_lib.segmentation import models, extract_metric_from -from cv_lib.segmentation.penobscot.engine import ( - create_supervised_evaluator, - create_supervised_trainer, -) -from cv_lib.segmentation.metrics import ( - pixelwise_accuracy, - class_accuracy, - mean_class_accuracy, - class_iou, - mean_iou, -) -from cv_lib.segmentation.dutchf3.utils import ( - current_datetime, - generate_path, - git_branch, - git_hash, - np_to_tb, -) - from default import _C as config from default import update_config - mask_value = 255 _SEG_COLOURS = np.asarray( [[241, 238, 246], [208, 209, 230], [166, 189, 219], [116, 169, 207], [54, 144, 192], [5, 112, 176], [3, 78, 123],] @@ -107,7 +82,7 @@ def run(*options, cfg=None, debug=False): load_log_configuration(config.LOG_CONFIG) logger = logging.getLogger(__name__) logger.debug(config.WORKERS) - scheduler_step = config.TRAIN.END_EPOCH // config.TRAIN.SNAPSHOTS + epochs_per_cycle = config.TRAIN.END_EPOCH // config.TRAIN.SNAPSHOTS torch.backends.cudnn.benchmark = config.CUDNN.BENCHMARK torch.manual_seed(config.SEED) @@ -126,7 +101,7 @@ def run(*options, cfg=None, debug=False): PadIfNeeded( min_height=config.TRAIN.PATCH_SIZE, min_width=config.TRAIN.PATCH_SIZE, - border_mode=cv2.BORDER_CONSTANT, + border_mode=config.OPENCV_BORDER_CONSTANT, always_apply=True, mask_value=mask_value, value=0, @@ -137,7 +112,7 @@ def run(*options, cfg=None, debug=False): PadIfNeeded( min_height=config.TRAIN.AUGMENTATIONS.PAD.HEIGHT, min_width=config.TRAIN.AUGMENTATIONS.PAD.WIDTH, - border_mode=cv2.BORDER_CONSTANT, + border_mode=config.OPENCV_BORDER_CONSTANT, always_apply=True, mask_value=mask_value, value=0, @@ -182,7 +157,7 @@ def run(*options, cfg=None, debug=False): if debug: val_set = data.Subset(val_set, range(3)) - val_loader = data.DataLoader(val_set, batch_size=config.VALIDATION.BATCH_SIZE_PER_GPU, num_workers=config.WORKERS) + val_loader = data.DataLoader(val_set, batch_size=config.VALIDATION.BATCH_SIZE_PER_GPU, num_workers=config.WORKERS) model = getattr(models, config.MODEL.NAME).get_seg_model(config) @@ -203,8 +178,8 @@ def run(*options, cfg=None, debug=False): output_dir = generate_path(config.OUTPUT_DIR, config_file_name, config.TRAIN.MODEL_DIR, current_datetime(),) summary_writer = create_summary_writer(log_dir=path.join(output_dir, config.LOG_DIR)) - snapshot_duration = scheduler_step * len(train_loader) - scheduler = CosineAnnealingScheduler(optimizer, "lr", config.TRAIN.MAX_LR, config.TRAIN.MIN_LR, snapshot_duration) + snapshot_duration = epochs_per_cycle * len(train_loader) if not debug else 2*len(train_loader) + scheduler = CosineAnnealingScheduler(optimizer, "lr", config.TRAIN.MAX_LR, config.TRAIN.MIN_LR, cycle_size=snapshot_duration) # weights are inversely proportional to the frequency of the classes in # the training set @@ -306,9 +281,15 @@ def snapshot_function(): logger.info("Starting training") if debug: - trainer.run(train_loader, max_epochs=config.TRAIN.END_EPOCH, epoch_length = config.TRAIN.BATCH_SIZE_PER_GPU, seed = config.SEED) + trainer.run( + train_loader, + max_epochs=config.TRAIN.END_EPOCH, + epoch_length=config.TRAIN.BATCH_SIZE_PER_GPU, + seed=config.SEED, + ) else: - trainer.run(train_loader, max_epochs=config.TRAIN.END_EPOCH, epoch_length = len(train_loader), seed = config.SEED) + trainer.run(train_loader, max_epochs=config.TRAIN.END_EPOCH, epoch_length=len(train_loader), seed=config.SEED) + if __name__ == "__main__": fire.Fire(run) diff --git a/cv_lib/cv_lib/event_handlers/logging_handlers.py b/cv_lib/cv_lib/event_handlers/logging_handlers.py index b7c41651..ea883a36 100644 --- a/cv_lib/cv_lib/event_handlers/logging_handlers.py +++ b/cv_lib/cv_lib/event_handlers/logging_handlers.py @@ -25,11 +25,8 @@ def log_lr(optimizer, engine): logger.info(f"lr - {lr}") -_DEFAULT_METRICS = {"pixacc": "Avg accuracy :", "nll": "Avg loss :"} - - @curry -def log_metrics(log_msg, engine, metrics_dict=_DEFAULT_METRICS): +def log_metrics(log_msg, engine, metrics_dict={"pixacc": "Avg accuracy :", "nll": "Avg loss :"}): logger = logging.getLogger(__name__) metrics = engine.state.metrics metrics_msg = " ".join([f"{metrics_dict[k]} {metrics[k]:.2f}" for k in metrics_dict]) @@ -44,6 +41,7 @@ def log_class_metrics(log_msg, engine, metrics_dict): logger.info(f"{log_msg} - Epoch {engine.state.epoch} [{engine.state.max_epochs}]\n" + metrics_msg) +# TODO: remove Evaluator once other train.py scripts are updated class Evaluator: def __init__(self, evaluation_engine, data_loader): self._evaluation_engine = evaluation_engine @@ -51,40 +49,3 @@ def __init__(self, evaluation_engine, data_loader): def __call__(self, engine): self._evaluation_engine.run(self._data_loader) - - -class HorovodLRScheduler: - """ - Horovod: using `lr = base_lr * hvd.size()` from the very beginning leads to worse final - accuracy. Scale the learning rate `lr = base_lr` ---> `lr = base_lr * hvd.size()` during - the first five epochs. See https://arxiv.org/abs/1706.02677 for details. - After the warmup reduce learning rate by 10 on the 30th, 60th and 80th epochs. - """ - - def __init__( - self, base_lr, warmup_epochs, cluster_size, data_loader, optimizer, batches_per_allreduce, - ): - self._warmup_epochs = warmup_epochs - self._cluster_size = cluster_size - self._data_loader = data_loader - self._optimizer = optimizer - self._base_lr = base_lr - self._batches_per_allreduce = batches_per_allreduce - self._logger = logging.getLogger(__name__) - - def __call__(self, engine): - epoch = engine.state.epoch - if epoch < self._warmup_epochs: - epoch += float(engine.state.iteration + 1) / len(self._data_loader) - lr_adj = 1.0 / self._cluster_size * (epoch * (self._cluster_size - 1) / self._warmup_epochs + 1) - elif epoch < 30: - lr_adj = 1.0 - elif epoch < 60: - lr_adj = 1e-1 - elif epoch < 80: - lr_adj = 1e-2 - else: - lr_adj = 1e-3 - for param_group in self._optimizer.param_groups: - param_group["lr"] = self._base_lr * self._cluster_size * self._batches_per_allreduce * lr_adj - self._logger.debug(f"Adjust learning rate {param_group['lr']}") diff --git a/cv_lib/cv_lib/event_handlers/tensorboard_handlers.py b/cv_lib/cv_lib/event_handlers/tensorboard_handlers.py index a9ba5f4c..30cb5fc4 100644 --- a/cv_lib/cv_lib/event_handlers/tensorboard_handlers.py +++ b/cv_lib/cv_lib/event_handlers/tensorboard_handlers.py @@ -1,12 +1,14 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -from toolz import curry import torchvision +from tensorboardX import SummaryWriter import logging import logging.config +from toolz import curry -from tensorboardX import SummaryWriter +from cv_lib.segmentation.dutchf3.utils import np_to_tb +from deepseismic_interpretation.dutchf3.data import decode_segmap def create_summary_writer(log_dir): @@ -14,18 +16,29 @@ def create_summary_writer(log_dir): return writer +def _transform_image(output_tensor): + output_tensor = output_tensor.cpu() + return torchvision.utils.make_grid(output_tensor, normalize=True, scale_each=True) + + +def _transform_pred(output_tensor, n_classes): + output_tensor = output_tensor.squeeze().cpu().numpy() + decoded = decode_segmap(output_tensor, n_classes) + return torchvision.utils.make_grid(np_to_tb(decoded), normalize=False, scale_each=False) + + def _log_model_output(log_label, summary_writer, engine): summary_writer.add_scalar(log_label, engine.state.output["loss"], engine.state.iteration) @curry def log_training_output(summary_writer, engine): - _log_model_output("training/loss", summary_writer, engine) + _log_model_output("Training/loss", summary_writer, engine) @curry def log_validation_output(summary_writer, engine): - _log_model_output("validation/loss", summary_writer, engine) + _log_model_output("Validation/loss", summary_writer, engine) @curry @@ -42,31 +55,60 @@ def log_lr(summary_writer, optimizer, log_interval, engine): summary_writer.add_scalar("lr", lr[0], getattr(engine.state, log_interval)) -_DEFAULT_METRICS = {"accuracy": "Avg accuracy :", "nll": "Avg loss :"} - - +# TODO: This is deprecated, and will be removed in the future. @curry -def log_metrics(summary_writer, train_engine, log_interval, engine, metrics_dict=_DEFAULT_METRICS): +def log_metrics(summary_writer, train_engine, log_interval, engine, metrics_dict={"pixacc": "Avg accuracy :", "nll": "Avg loss :"}): metrics = engine.state.metrics for m in metrics_dict: - summary_writer.add_scalar( - metrics_dict[m], metrics[m], getattr(train_engine.state, log_interval) - ) + summary_writer.add_scalar(metrics_dict[m], metrics[m], getattr(train_engine.state, log_interval)) -def create_image_writer( - summary_writer, label, output_variable, normalize=False, transform_func=lambda x: x -): +# TODO: This is deprecated, and will be removed in the future. +def create_image_writer(summary_writer, label, output_variable, normalize=False, transform_func=lambda x: x): logger = logging.getLogger(__name__) + logger.warning( + "create_image_writer() in tensorboard_handlers.py is deprecated, and will be removed in a future update." + ) def write_to(engine): try: data_tensor = transform_func(engine.state.output[output_variable]) - image_grid = torchvision.utils.make_grid( - data_tensor, normalize=normalize, scale_each=True - ) + image_grid = torchvision.utils.make_grid(data_tensor, normalize=normalize, scale_each=True) summary_writer.add_image(label, image_grid, engine.state.epoch) except KeyError: logger.warning("Predictions and or ground truth labels not available to report") return write_to + + +def log_results(engine, evaluator, summary_writer, n_classes, stage): + epoch = engine.state.epoch + metrics = evaluator.state.metrics + outputs = evaluator.state.output + + # Log Metrics: + summary_writer.add_scalar(f"{stage}/mIoU", metrics["mIoU"], epoch) + summary_writer.add_scalar(f"{stage}/nll", metrics["nll"], epoch) + summary_writer.add_scalar(f"{stage}/mca", metrics["mca"], epoch) + summary_writer.add_scalar(f"{stage}/pixacc", metrics["pixacc"], epoch) + + for i in range(n_classes): + summary_writer.add_scalar(f"{stage}/IoU_class_" + str(i), metrics["ciou"][i], epoch) + + # Log Images: + image = outputs["image"] + mask = outputs["mask"] + y_pred = outputs["y_pred"].max(1, keepdim=True)[1] + VISUALIZATION_LIMIT = 8 + + if evaluator.state.batch[0].shape[0] > VISUALIZATION_LIMIT: + image = image[:VISUALIZATION_LIMIT] + mask = mask[:VISUALIZATION_LIMIT] + y_pred = y_pred[:VISUALIZATION_LIMIT] + + # Mask out the region in y_pred where padding exists in the mask: + y_pred[mask == 255] = 255 + + summary_writer.add_image(f"{stage}/Image", _transform_image(image), epoch) + summary_writer.add_image(f"{stage}/Mask", _transform_pred(mask, n_classes), epoch) + summary_writer.add_image(f"{stage}/Pred", _transform_pred(y_pred, n_classes), epoch) diff --git a/environment/anaconda/local/environment.yml b/environment/anaconda/local/environment.yml index de0d65af..40b28a34 100644 --- a/environment/anaconda/local/environment.yml +++ b/environment/anaconda/local/environment.yml @@ -11,7 +11,6 @@ dependencies: - ipykernel - torchvision>=0.5.0 - pandas==0.25.3 - - opencv==4.1.2 - scikit-learn==0.21.3 - tensorflow==2.0 - opt-einsum>=2.3.2 diff --git a/environment/docker/apex/dockerfile b/environment/docker/apex/dockerfile index 3becd3c4..9dcf5615 100644 --- a/environment/docker/apex/dockerfile +++ b/environment/docker/apex/dockerfile @@ -10,7 +10,7 @@ RUN git clone https://github.com/NVIDIA/apex && \ cd apex && \ pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" ./ -RUN pip install toolz pytorch-ignite torchvision pandas opencv-python fire tensorboardx scikit-learn yacs +RUN pip install toolz pytorch-ignite torchvision pandas fire tensorboardx scikit-learn yacs WORKDIR /workspace CMD /bin/bash \ No newline at end of file diff --git a/environment/docker/horovod/dockerfile b/environment/docker/horovod/dockerfile index 0e12f455..04ed2f67 100644 --- a/environment/docker/horovod/dockerfile +++ b/environment/docker/horovod/dockerfile @@ -60,7 +60,7 @@ RUN pip install future typing RUN pip install numpy RUN pip install https://download.pytorch.org/whl/cu100/torch-${PYTORCH_VERSION}-$(python -c "import wheel.pep425tags as w; print('-'.join(w.get_supported()[0]))").whl \ https://download.pytorch.org/whl/cu100/torchvision-${TORCHVISION_VERSION}-$(python -c "import wheel.pep425tags as w; print('-'.join(w.get_supported()[0]))").whl -RUN pip install --no-cache-dir torchvision h5py toolz pytorch-ignite pandas opencv-python fire tensorboardx scikit-learn tqdm yacs albumentations gitpython +RUN pip install --no-cache-dir torchvision h5py toolz pytorch-ignite pandas fire tensorboardx scikit-learn tqdm yacs albumentations gitpython COPY ComputerVision_fork/contrib /contrib RUN pip install -e /contrib COPY DeepSeismic /DeepSeismic diff --git a/examples/interpretation/notebooks/Dutch_F3_patch_model_training_and_evaluation.ipynb b/examples/interpretation/notebooks/Dutch_F3_patch_model_training_and_evaluation.ipynb index de4e8542..6f830701 100644 --- a/examples/interpretation/notebooks/Dutch_F3_patch_model_training_and_evaluation.ipynb +++ b/examples/interpretation/notebooks/Dutch_F3_patch_model_training_and_evaluation.ipynb @@ -473,7 +473,7 @@ " PadIfNeeded(\n", " min_height=config.TRAIN.AUGMENTATIONS.PAD.HEIGHT,\n", " min_width=config.TRAIN.AUGMENTATIONS.PAD.WIDTH,\n", - " border_mode=cv2.BORDER_CONSTANT,\n", + " border_mode=config.OPENCV_BORDER_CONSTANT,\n", " always_apply=True,\n", " mask_value=255,\n", " ),\n", @@ -534,7 +534,6 @@ " augmentations=val_aug,\n", ")\n", "\n", - "# TODO: workaround for Ignite 0.3.0 bug as epoch_lengh in trainer.run method below doesn't apply to validation set\n", "if papermill:\n", " val_set = data.Subset(val_set, range(3))\n", "elif DEMO:\n", @@ -578,7 +577,7 @@ "else:\n", " train_len = len(train_loader)\n", "\n", - "snapshot_duration = scheduler_step * train_len" + "snapshot_duration = scheduler_step * train_len if not papermill else 2*len(train_loader)" ] }, { @@ -643,7 +642,7 @@ "\n", "# learning rate scheduler\n", "scheduler = CosineAnnealingScheduler(\n", - " optimizer, \"lr\", config.TRAIN.MAX_LR, config.TRAIN.MIN_LR, snapshot_duration\n", + " optimizer, \"lr\", config.TRAIN.MAX_LR, config.TRAIN.MIN_LR, cycle_size=snapshot_duration\n", ")\n", "\n", "# weights are inversely proportional to the frequency of the classes in the training set\n", @@ -984,7 +983,7 @@ " PadIfNeeded(\n", " min_height=config.TRAIN.AUGMENTATIONS.PAD.HEIGHT,\n", " min_width=config.TRAIN.AUGMENTATIONS.PAD.WIDTH,\n", - " border_mode=cv2.BORDER_CONSTANT,\n", + " border_mode=config.OPENCV_BORDER_CONSTANT,\n", " always_apply=True,\n", " mask_value=255,\n", " ),\n", diff --git a/experiments/interpretation/dutchf3_patch/local/configs/hrnet.yaml b/experiments/interpretation/dutchf3_patch/local/configs/hrnet.yaml index 9d705df2..52263bbf 100644 --- a/experiments/interpretation/dutchf3_patch/local/configs/hrnet.yaml +++ b/experiments/interpretation/dutchf3_patch/local/configs/hrnet.yaml @@ -9,6 +9,7 @@ WORKERS: 4 PRINT_FREQ: 10 LOG_CONFIG: logging.conf SEED: 2019 +OPENCV_BORDER_CONSTANT: 0 DATASET: @@ -73,7 +74,7 @@ TRAIN: WEIGHT_DECAY: 0.0001 SNAPSHOTS: 5 AUGMENTATION: True - DEPTH: "section" #"patch" # Options are No, Patch and Section + DEPTH: "section" # Options are: none, patch, and section STRIDE: 50 PATCH_SIZE: 100 AUGMENTATIONS: @@ -82,7 +83,7 @@ TRAIN: WIDTH: 200 PAD: HEIGHT: 256 - WIDTH: 256 + WIDTH: 256 MEAN: 0.0009997 # 0.0009996710808862074 STD: 0.20977 # 0.20976548783479299 MODEL_DIR: "models" @@ -91,12 +92,12 @@ TRAIN: VALIDATION: BATCH_SIZE_PER_GPU: 128 -TEST: +TEST: MODEL_PATH: "/data/home/mat/repos/DeepSeismic/experiments/interpretation/dutchf3_patch/local/output/staging/0d1d2bbf9685995a0515ca1d9de90f9bcec0db90/seg_hrnet/Dec20_233535/models/seg_hrnet_running_model_33.pth" TEST_STRIDE: 10 SPLIT: 'Both' # Can be Both, Test1, Test2 INLINE: True CROSSLINE: True - POST_PROCESSING: - SIZE: 128 # + POST_PROCESSING: + SIZE: 128 # CROP_PIXELS: 14 # Number of pixels to crop top, bottom, left and right diff --git a/experiments/interpretation/dutchf3_patch/local/configs/patch_deconvnet.yaml b/experiments/interpretation/dutchf3_patch/local/configs/patch_deconvnet.yaml index 7c695f96..35787b95 100644 --- a/experiments/interpretation/dutchf3_patch/local/configs/patch_deconvnet.yaml +++ b/experiments/interpretation/dutchf3_patch/local/configs/patch_deconvnet.yaml @@ -29,7 +29,7 @@ TRAIN: WEIGHT_DECAY: 0.0001 SNAPSHOTS: 5 AUGMENTATION: True - DEPTH: "none" # Options are None, Patch and Section + DEPTH: "none" # Options are none, patch, and section STRIDE: 50 PATCH_SIZE: 99 AUGMENTATIONS: diff --git a/experiments/interpretation/dutchf3_patch/local/configs/patch_deconvnet_skip.yaml b/experiments/interpretation/dutchf3_patch/local/configs/patch_deconvnet_skip.yaml index d14ea134..46fab9f6 100644 --- a/experiments/interpretation/dutchf3_patch/local/configs/patch_deconvnet_skip.yaml +++ b/experiments/interpretation/dutchf3_patch/local/configs/patch_deconvnet_skip.yaml @@ -29,7 +29,7 @@ TRAIN: WEIGHT_DECAY: 0.0001 SNAPSHOTS: 5 AUGMENTATION: True - DEPTH: "none" #"patch" # Options are None, Patch and Section + DEPTH: "none" #"patch" # Options are none, patch, and section STRIDE: 50 PATCH_SIZE: 99 AUGMENTATIONS: diff --git a/experiments/interpretation/dutchf3_patch/local/configs/seresnet_unet.yaml b/experiments/interpretation/dutchf3_patch/local/configs/seresnet_unet.yaml index d0b8126f..9bc10d34 100644 --- a/experiments/interpretation/dutchf3_patch/local/configs/seresnet_unet.yaml +++ b/experiments/interpretation/dutchf3_patch/local/configs/seresnet_unet.yaml @@ -30,7 +30,7 @@ TRAIN: WEIGHT_DECAY: 0.0001 SNAPSHOTS: 5 AUGMENTATION: True - DEPTH: "section" # Options are No, Patch and Section + DEPTH: "section" # Options are none, patch, and section STRIDE: 50 PATCH_SIZE: 100 AUGMENTATIONS: diff --git a/experiments/interpretation/dutchf3_patch/local/configs/unet.yaml b/experiments/interpretation/dutchf3_patch/local/configs/unet.yaml index c31157bf..3a8ee71a 100644 --- a/experiments/interpretation/dutchf3_patch/local/configs/unet.yaml +++ b/experiments/interpretation/dutchf3_patch/local/configs/unet.yaml @@ -33,7 +33,7 @@ TRAIN: WEIGHT_DECAY: 0.0001 SNAPSHOTS: 5 AUGMENTATION: True - DEPTH: "section" # Options are No, Patch and Section + DEPTH: "section" # Options are none, patch, and section STRIDE: 50 PATCH_SIZE: 100 AUGMENTATIONS: diff --git a/experiments/interpretation/dutchf3_patch/local/default.py b/experiments/interpretation/dutchf3_patch/local/default.py index aac539ea..f2cadfc1 100644 --- a/experiments/interpretation/dutchf3_patch/local/default.py +++ b/experiments/interpretation/dutchf3_patch/local/default.py @@ -20,7 +20,7 @@ _C.PIN_MEMORY = True _C.LOG_CONFIG = "logging.conf" _C.SEED = 42 - +_C.OPENCV_BORDER_CONSTANT = 0 # Cudnn related params _C.CUDNN = CN() @@ -58,8 +58,8 @@ _C.TRAIN.PATCH_SIZE = 99 _C.TRAIN.MEAN = 0.0009997 # 0.0009996710808862074 _C.TRAIN.STD = 0.20977 # 0.20976548783479299 # TODO: Should we apply std scaling? -# issue: https://github.com/microsoft/seismic-deeplearning/issues/269 -_C.TRAIN.DEPTH = "no" # Options are None, Patch and Section +_C.TRAIN.DEPTH = "none" # Options are: none, patch, and section + # None adds no depth information and the num of channels remains at 1 # Patch adds depth per patch so is simply the height of that patch from 0 to 1, channels=3 # Section adds depth per section so contains depth information for the whole section, channels=3 diff --git a/experiments/interpretation/dutchf3_patch/local/test.py b/experiments/interpretation/dutchf3_patch/local/test.py index ee4c0b09..631fd4f4 100644 --- a/experiments/interpretation/dutchf3_patch/local/test.py +++ b/experiments/interpretation/dutchf3_patch/local/test.py @@ -18,34 +18,22 @@ import os from os import path -import cv2 import fire import numpy as np import torch import torch.nn.functional as F -from PIL import Image from albumentations import Compose, Normalize, PadIfNeeded, Resize -from cv_lib.utils import load_log_configuration +from matplotlib import cm +from PIL import Image +from toolz import compose, curry, itertoolz, pipe, take +from torch.utils import data + from cv_lib.segmentation import models -from cv_lib.segmentation.dutchf3.utils import ( - current_datetime, - generate_path, - git_branch, - git_hash, -) -from deepseismic_interpretation.dutchf3.data import ( - add_patch_depth_channels, - get_seismic_labels, - get_test_loader, -) +from cv_lib.segmentation.dutchf3.utils import current_datetime, generate_path, git_branch, git_hash +from cv_lib.utils import load_log_configuration +from deepseismic_interpretation.dutchf3.data import add_patch_depth_channels, get_seismic_labels, get_test_loader from default import _C as config from default import update_config -from toolz import compose, curry, itertoolz, pipe -from torch.utils import data -from toolz import take - -from matplotlib import cm - _CLASS_NAMES = [ "upper_ns", @@ -63,9 +51,9 @@ def __init__(self, n_classes): def _fast_hist(self, label_true, label_pred, n_class): mask = (label_true >= 0) & (label_true < n_class) - hist = np.bincount( - n_class * label_true[mask].astype(int) + label_pred[mask], minlength=n_class ** 2, - ).reshape(n_class, n_class) + hist = np.bincount(n_class * label_true[mask].astype(int) + label_pred[mask], minlength=n_class ** 2,).reshape( + n_class, n_class + ) return hist def update(self, label_trues, label_preds): @@ -201,9 +189,7 @@ def _compose_processing_pipeline(depth, aug=None): def _generate_batches(h, w, ps, patch_size, stride, batch_size=64): - hdc_wdx_generator = itertools.product( - range(0, h - patch_size + ps, stride), range(0, w - patch_size + ps, stride), - ) + hdc_wdx_generator = itertools.product(range(0, h - patch_size + ps, stride), range(0, w - patch_size + ps, stride),) for batch_indexes in itertoolz.partition_all(batch_size, hdc_wdx_generator): yield batch_indexes @@ -214,9 +200,7 @@ def _output_processing_pipeline(config, output): _, _, h, w = output.shape if config.TEST.POST_PROCESSING.SIZE != h or config.TEST.POST_PROCESSING.SIZE != w: output = F.interpolate( - output, - size=(config.TEST.POST_PROCESSING.SIZE, config.TEST.POST_PROCESSING.SIZE,), - mode="bilinear", + output, size=(config.TEST.POST_PROCESSING.SIZE, config.TEST.POST_PROCESSING.SIZE,), mode="bilinear", ) if config.TEST.POST_PROCESSING.CROP_PIXELS > 0: @@ -231,15 +215,7 @@ def _output_processing_pipeline(config, output): def _patch_label_2d( - model, - img, - pre_processing, - output_processing, - patch_size, - stride, - batch_size, - device, - num_classes, + model, img, pre_processing, output_processing, patch_size, stride, batch_size, device, num_classes, ): """Processes a whole section """ @@ -254,19 +230,14 @@ def _patch_label_2d( # generate output: for batch_indexes in _generate_batches(h, w, ps, patch_size, stride, batch_size=batch_size): batch = torch.stack( - [ - pipe(img_p, _extract_patch(hdx, wdx, ps, patch_size), pre_processing,) - for hdx, wdx in batch_indexes - ], + [pipe(img_p, _extract_patch(hdx, wdx, ps, patch_size), pre_processing,) for hdx, wdx in batch_indexes], dim=0, ) model_output = model(batch.to(device)) for (hdx, wdx), output in zip(batch_indexes, model_output.detach().cpu()): output = output_processing(output) - output_p[ - :, :, hdx + ps : hdx + ps + patch_size, wdx + ps : wdx + ps + patch_size, - ] += output + output_p[:, :, hdx + ps : hdx + ps + patch_size, wdx + ps : wdx + ps + patch_size,] += output # crop the output_p in the middle output = output_p[:, :, ps:-ps, ps:-ps] @@ -291,22 +262,12 @@ def to_image(label_mask, n_classes=6): def _evaluate_split( - split, - section_aug, - model, - pre_processing, - output_processing, - device, - running_metrics_overall, - config, - debug=False, + split, section_aug, model, pre_processing, output_processing, device, running_metrics_overall, config, debug=False, ): logger = logging.getLogger(__name__) TestSectionLoader = get_test_loader(config) - test_set = TestSectionLoader( - config.DATASET.ROOT, split=split, is_transform=True, augmentations=section_aug, - ) + test_set = TestSectionLoader(config.DATASET.ROOT, split=split, is_transform=True, augmentations=section_aug,) n_classes = test_set.n_classes @@ -318,16 +279,10 @@ def _evaluate_split( try: output_dir = generate_path( - config.OUTPUT_DIR + "_test", - git_branch(), - git_hash(), - config.MODEL.NAME, - current_datetime(), + config.OUTPUT_DIR + "_test", git_branch(), git_hash(), config.MODEL.NAME, current_datetime(), ) except TypeError: - output_dir = generate_path( - config.OUTPUT_DIR + "_test", config.MODEL.NAME, current_datetime(), - ) + output_dir = generate_path(config.OUTPUT_DIR + "_test", config.MODEL.NAME, current_datetime(),) running_metrics_split = runningScore(n_classes) @@ -415,23 +370,19 @@ def test(*options, cfg=None, debug=False): running_metrics_overall = runningScore(n_classes) # Augmentation - section_aug = Compose( - [Normalize(mean=(config.TRAIN.MEAN,), std=(config.TRAIN.STD,), max_pixel_value=1,)] - ) + section_aug = Compose([Normalize(mean=(config.TRAIN.MEAN,), std=(config.TRAIN.STD,), max_pixel_value=1,)]) # TODO: make sure that this is consistent with how normalization and agumentation for train.py # issue: https://github.com/microsoft/seismic-deeplearning/issues/270 patch_aug = Compose( [ Resize( - config.TRAIN.AUGMENTATIONS.RESIZE.HEIGHT, - config.TRAIN.AUGMENTATIONS.RESIZE.WIDTH, - always_apply=True, + config.TRAIN.AUGMENTATIONS.RESIZE.HEIGHT, config.TRAIN.AUGMENTATIONS.RESIZE.WIDTH, always_apply=True, ), PadIfNeeded( min_height=config.TRAIN.AUGMENTATIONS.PAD.HEIGHT, min_width=config.TRAIN.AUGMENTATIONS.PAD.WIDTH, - border_mode=cv2.BORDER_CONSTANT, + border_mode=config.OPENCV_BORDER_CONSTANT, always_apply=True, mask_value=255, ), diff --git a/experiments/interpretation/dutchf3_patch/local/train.py b/experiments/interpretation/dutchf3_patch/local/train.py index 9c77d713..14dcf726 100644 --- a/experiments/interpretation/dutchf3_patch/local/train.py +++ b/experiments/interpretation/dutchf3_patch/local/train.py @@ -17,52 +17,24 @@ import logging.config from os import path -import cv2 import fire import numpy as np import torch +from torch.utils import data from albumentations import Compose, HorizontalFlip, Normalize, PadIfNeeded, Resize from ignite.contrib.handlers import CosineAnnealingScheduler from ignite.engine import Events from ignite.metrics import Loss from ignite.utils import convert_tensor -from toolz import compose -from torch.utils import data -from deepseismic_interpretation.dutchf3.data import get_patch_loader, decode_segmap +from cv_lib.event_handlers import SnapshotHandler, logging_handlers, tensorboard_handlers +from cv_lib.event_handlers.tensorboard_handlers import create_summary_writer, log_results +from cv_lib.segmentation import extract_metric_from, models +from cv_lib.segmentation.dutchf3.engine import create_supervised_evaluator, create_supervised_trainer +from cv_lib.segmentation.dutchf3.utils import current_datetime, generate_path, git_branch, git_hash +from cv_lib.segmentation.metrics import class_accuracy, class_iou, mean_class_accuracy, mean_iou, pixelwise_accuracy from cv_lib.utils import load_log_configuration -from cv_lib.event_handlers import ( - SnapshotHandler, - logging_handlers, - tensorboard_handlers, -) -from cv_lib.event_handlers.logging_handlers import Evaluator -from cv_lib.event_handlers.tensorboard_handlers import ( - create_image_writer, - create_summary_writer, -) -from cv_lib.segmentation import models, extract_metric_from -from cv_lib.segmentation.dutchf3.engine import ( - create_supervised_evaluator, - create_supervised_trainer, -) - -from cv_lib.segmentation.metrics import ( - pixelwise_accuracy, - class_accuracy, - mean_class_accuracy, - class_iou, - mean_iou, -) - -from cv_lib.segmentation.dutchf3.utils import ( - current_datetime, - generate_path, - git_branch, - git_hash, - np_to_tb, -) - +from deepseismic_interpretation.dutchf3.data import get_patch_loader from default import _C as config from default import update_config @@ -90,44 +62,50 @@ def run(*options, cfg=None, debug=False): cfg (str, optional): Location of config file to load. Defaults to None. debug (bool): Places scripts in debug/test mode and only executes a few iterations """ - + # Configuration: update_config(config, options=options, config_file=cfg) - - # we will write the model under outputs / config_file_name / model_dir + # The model will be saved under: outputs// config_file_name = "default_config" if not cfg else cfg.split("/")[-1].split(".")[0] + try: + output_dir = generate_path( + config.OUTPUT_DIR, git_branch(), git_hash(), config_file_name, config.TRAIN.MODEL_DIR, current_datetime(), + ) + except TypeError: + output_dir = generate_path(config.OUTPUT_DIR, config_file_name, config.TRAIN.MODEL_DIR, current_datetime(),) - # Start logging + # Logging: load_log_configuration(config.LOG_CONFIG) logger = logging.getLogger(__name__) logger.debug(config.WORKERS) + + # Set CUDNN benchmark mode: torch.backends.cudnn.benchmark = config.CUDNN.BENCHMARK + # Fix random seeds: torch.manual_seed(config.SEED) if torch.cuda.is_available(): torch.cuda.manual_seed_all(config.SEED) np.random.seed(seed=config.SEED) - # Setup Augmentations + # Augmentation: basic_aug = Compose( [ Normalize(mean=(config.TRAIN.MEAN,), std=(config.TRAIN.STD,), max_pixel_value=1), PadIfNeeded( min_height=config.TRAIN.PATCH_SIZE, min_width=config.TRAIN.PATCH_SIZE, - border_mode=0, + border_mode=config.OPENCV_BORDER_CONSTANT, always_apply=True, mask_value=255, value=0, ), Resize( - config.TRAIN.AUGMENTATIONS.RESIZE.HEIGHT, - config.TRAIN.AUGMENTATIONS.RESIZE.WIDTH, - always_apply=True, + config.TRAIN.AUGMENTATIONS.RESIZE.HEIGHT, config.TRAIN.AUGMENTATIONS.RESIZE.WIDTH, always_apply=True, ), PadIfNeeded( min_height=config.TRAIN.AUGMENTATIONS.PAD.HEIGHT, min_width=config.TRAIN.AUGMENTATIONS.PAD.WIDTH, - border_mode=cv2.BORDER_CONSTANT, + border_mode=config.OPENCV_BORDER_CONSTANT, always_apply=True, mask_value=255, ), @@ -139,8 +117,8 @@ def run(*options, cfg=None, debug=False): else: train_aug = val_aug = basic_aug + # Training and Validation Loaders: TrainPatchLoader = get_patch_loader(config) - train_set = TrainPatchLoader( config.DATASET.ROOT, split="train", @@ -150,6 +128,7 @@ def run(*options, cfg=None, debug=False): augmentations=train_aug, ) logger.info(train_set) + n_classes = train_set.n_classes val_set = TrainPatchLoader( config.DATASET.ROOT, split="val", @@ -160,27 +139,22 @@ def run(*options, cfg=None, debug=False): ) logger.info(val_set) - train_loader = data.DataLoader( - train_set, - batch_size=config.TRAIN.BATCH_SIZE_PER_GPU, - num_workers=config.WORKERS, - shuffle=True, - ) - if debug: - val_set = data.Subset(val_set, range(3)) + logger.info("Running in debug mode..") + train_set = data.Subset(train_set, list(range(4))) + val_set = data.Subset(val_set, list(range(4))) - val_loader = data.DataLoader( - val_set, batch_size=config.VALIDATION.BATCH_SIZE_PER_GPU, num_workers=config.WORKERS, + train_loader = data.DataLoader( + train_set, batch_size=config.TRAIN.BATCH_SIZE_PER_GPU, num_workers=config.WORKERS, shuffle=True ) + val_loader = data.DataLoader(val_set, batch_size=config.VALIDATION.BATCH_SIZE_PER_GPU, num_workers=config.WORKERS) + # Model: model = getattr(models, config.MODEL.NAME).get_seg_model(config) + device = "cuda" if torch.cuda.is_available() else "cpu" + model = model.to(device) - device = "cpu" - if torch.cuda.is_available(): - device = "cuda" - model = model.to(device) # Send to GPU - + # Optimizer and LR Scheduler: optimizer = torch.optim.SGD( model.parameters(), lr=config.TRAIN.MAX_LR, @@ -188,92 +162,44 @@ def run(*options, cfg=None, debug=False): weight_decay=config.TRAIN.WEIGHT_DECAY, ) - # learning rate scheduler - scheduler_step = config.TRAIN.END_EPOCH // config.TRAIN.SNAPSHOTS - snapshot_duration = scheduler_step * len(train_loader) + epochs_per_cycle = config.TRAIN.END_EPOCH // config.TRAIN.SNAPSHOTS + snapshot_duration = epochs_per_cycle * len(train_loader) if not debug else 2*len(train_loader) scheduler = CosineAnnealingScheduler( - optimizer, "lr", config.TRAIN.MAX_LR, config.TRAIN.MIN_LR, snapshot_duration + optimizer, "lr", config.TRAIN.MAX_LR, config.TRAIN.MIN_LR, cycle_size=snapshot_duration ) - # weights are inversely proportional to the frequency of the classes in the - # training set + # Tensorboard writer: + summary_writer = create_summary_writer(log_dir=path.join(output_dir, "logs")) + + # class weights are inversely proportional to the frequency of the classes in the training set class_weights = torch.tensor(config.DATASET.CLASS_WEIGHTS, device=device, requires_grad=False) + # Loss: criterion = torch.nn.CrossEntropyLoss(weight=class_weights, ignore_index=255, reduction="mean") + # Ignite trainer and evaluator: trainer = create_supervised_trainer(model, optimizer, criterion, prepare_batch, device=device) - - trainer.add_event_handler(Events.ITERATION_STARTED, scheduler) - - ######################### - # Logging setup below - - try: - output_dir = generate_path( - config.OUTPUT_DIR, - git_branch(), - git_hash(), - config_file_name, - config.TRAIN.MODEL_DIR, - current_datetime(), - ) - except TypeError: - output_dir = generate_path( - config.OUTPUT_DIR, config_file_name, config.TRAIN.MODEL_DIR, current_datetime(), - ) - - summary_writer = create_summary_writer(log_dir=path.join(output_dir, config.LOG_DIR)) - - # log all training output - trainer.add_event_handler( - Events.ITERATION_COMPLETED, - logging_handlers.log_training_output(log_interval=config.TRAIN.BATCH_SIZE_PER_GPU), - ) - - # add logging of learning rate - trainer.add_event_handler(Events.EPOCH_STARTED, logging_handlers.log_lr(optimizer)) - - # log LR to tensorboard - trainer.add_event_handler( - Events.EPOCH_STARTED, tensorboard_handlers.log_lr(summary_writer, optimizer, "epoch"), - ) - - # log training summary to tensorboard as well - trainer.add_event_handler( - Events.ITERATION_COMPLETED, tensorboard_handlers.log_training_output(summary_writer), - ) - - def _select_pred_and_mask(model_out_dict): - return (model_out_dict["y_pred"].squeeze(), model_out_dict["mask"].squeeze()) - - def _select_max(pred_tensor): - return pred_tensor.max(1)[1] - - def _tensor_to_numpy(pred_tensor): - return pred_tensor.squeeze().cpu().numpy() - - def snapshot_function(): - return (trainer.state.iteration % snapshot_duration) == 0 - - n_classes = train_set.n_classes - + transform_fn = lambda output_dict: (output_dict["y_pred"].squeeze(), output_dict["mask"].squeeze()) evaluator = create_supervised_evaluator( model, prepare_batch, metrics={ - "nll": Loss(criterion, output_transform=_select_pred_and_mask), - "pixacc": pixelwise_accuracy( - n_classes, output_transform=_select_pred_and_mask, device=device - ), - "cacc": class_accuracy(n_classes, output_transform=_select_pred_and_mask), - "mca": mean_class_accuracy(n_classes, output_transform=_select_pred_and_mask), - "ciou": class_iou(n_classes, output_transform=_select_pred_and_mask), - "mIoU": mean_iou(n_classes, output_transform=_select_pred_and_mask), + "nll": Loss(criterion, output_transform=transform_fn), + "pixacc": pixelwise_accuracy(n_classes, output_transform=transform_fn, device=device), + "cacc": class_accuracy(n_classes, output_transform=transform_fn), + "mca": mean_class_accuracy(n_classes, output_transform=transform_fn), + "ciou": class_iou(n_classes, output_transform=transform_fn), + "mIoU": mean_iou(n_classes, output_transform=transform_fn), }, device=device, ) - - trainer.add_event_handler(Events.EPOCH_COMPLETED, Evaluator(evaluator, val_loader)) + trainer.add_event_handler(Events.ITERATION_STARTED, scheduler) + + # Logging: + trainer.add_event_handler( + Events.ITERATION_COMPLETED, logging_handlers.log_training_output(log_interval=config.TRAIN.BATCH_SIZE_PER_GPU), + ) + trainer.add_event_handler(Events.EPOCH_COMPLETED, logging_handlers.log_lr(optimizer)) evaluator.add_event_handler( Events.EPOCH_COMPLETED, @@ -288,51 +214,40 @@ def snapshot_function(): ), ) - evaluator.add_event_handler( - Events.EPOCH_COMPLETED, - tensorboard_handlers.log_metrics( - summary_writer, - trainer, - "epoch", - metrics_dict={ - "mIoU": "Validation/mIoU", - "nll": "Validation/Loss", - "mca": "Validation/MCA", - "pixacc": "Validation/Pixel_Acc", - }, - ), - ) - - transform_func = compose(np_to_tb, decode_segmap(n_classes=n_classes), _tensor_to_numpy) + # Tensorboard and Logging: + trainer.add_event_handler(Events.ITERATION_COMPLETED, tensorboard_handlers.log_training_output(summary_writer)) + trainer.add_event_handler(Events.ITERATION_COMPLETED, tensorboard_handlers.log_validation_output(summary_writer)) - transform_pred = compose(transform_func, _select_max) + @trainer.on(Events.EPOCH_COMPLETED) + def log_training_results(engine): + evaluator.run(train_loader) + log_results(engine, evaluator, summary_writer, n_classes, stage="Training") - evaluator.add_event_handler( - Events.EPOCH_COMPLETED, create_image_writer(summary_writer, "Validation/Image", "image"), - ) - evaluator.add_event_handler( - Events.EPOCH_COMPLETED, - create_image_writer( - summary_writer, "Validation/Mask", "mask", transform_func=transform_func - ), - ) - evaluator.add_event_handler( - Events.EPOCH_COMPLETED, - create_image_writer( - summary_writer, "Validation/Pred", "y_pred", transform_func=transform_pred - ), - ) + @trainer.on(Events.EPOCH_COMPLETED) + def log_validation_results(engine): + evaluator.run(val_loader) + log_results(engine, evaluator, summary_writer, n_classes, stage="Validation") + # Checkpointing: checkpoint_handler = SnapshotHandler( - output_dir, config.MODEL.NAME, extract_metric_from("mIoU"), snapshot_function, + output_dir, + config.MODEL.NAME, + extract_metric_from("mIoU"), + lambda: (trainer.state.iteration % snapshot_duration) == 0, ) evaluator.add_event_handler(Events.EPOCH_COMPLETED, checkpoint_handler, {"model": model}) - logger.info("Starting training") + logger.info("Starting training") if debug: - trainer.run(train_loader, max_epochs=config.TRAIN.END_EPOCH, epoch_length = config.TRAIN.BATCH_SIZE_PER_GPU, seed = config.SEED) + trainer.run( + train_loader, + max_epochs=config.TRAIN.END_EPOCH, + epoch_length=config.TRAIN.BATCH_SIZE_PER_GPU, + seed=config.SEED, + ) else: - trainer.run(train_loader, max_epochs=config.TRAIN.END_EPOCH, epoch_length = len(train_loader), seed = config.SEED) + trainer.run(train_loader, max_epochs=config.TRAIN.END_EPOCH, epoch_length=len(train_loader), seed=config.SEED) + summary_writer.close() if __name__ == "__main__": diff --git a/interpretation/deepseismic_interpretation/dutchf3/data.py b/interpretation/deepseismic_interpretation/dutchf3/data.py index e11dd059..4517f510 100644 --- a/interpretation/deepseismic_interpretation/dutchf3/data.py +++ b/interpretation/deepseismic_interpretation/dutchf3/data.py @@ -579,12 +579,14 @@ def __getitem__(self, index): patch_name = self.patches[index] direction, idx, xdx, ddx = patch_name.split(sep="_") + # Shift offsets the padding that is added in training # shift = self.patch_size if "test" not in self.split else 0 # TODO: Remember we are cancelling the shift since we no longer pad # issue: https://github.com/microsoft/seismic-deeplearning/issues/273 shift = 0 idx, xdx, ddx = int(idx) + shift, int(xdx) + shift, int(ddx) + shift + if direction == "i": im = self.seismic[idx, :, xdx : xdx + self.patch_size, ddx : ddx + self.patch_size] lbl = self.labels[idx, xdx : xdx + self.patch_size, ddx : ddx + self.patch_size] @@ -604,11 +606,11 @@ def __getitem__(self, index): if self.is_transform: im, lbl = self.transform(im, lbl) return im, lbl - + def __repr__(self): unique, counts = np.unique(self.labels, return_counts=True) - ratio = counts/np.sum(counts) - return "\n".join(f"{lbl}: {cnt} [{rat}]"for lbl, cnt, rat in zip(unique, counts, ratio)) + ratio = counts / np.sum(counts) + return "\n".join(f"{lbl}: {cnt} [{rat}]" for lbl, cnt, rat in zip(unique, counts, ratio)) _TRAIN_PATCH_LOADERS = { @@ -619,7 +621,7 @@ def __repr__(self): _TRAIN_SECTION_LOADERS = {"section": TrainSectionLoaderWithDepth} def get_patch_loader(cfg): - assert cfg.TRAIN.DEPTH in [ + assert str(cfg.TRAIN.DEPTH).lower() in [ "section", "patch", "none", @@ -629,7 +631,7 @@ def get_patch_loader(cfg): def get_section_loader(cfg): - assert cfg.TRAIN.DEPTH in [ + assert str(cfg.TRAIN.DEPTH).lower() in [ "section", "none", ], f"Depth {cfg.TRAIN.DEPTH} not supported for section data. \ @@ -693,7 +695,7 @@ def get_seismic_labels(): @curry -def decode_segmap(label_mask, n_classes=6, label_colours=get_seismic_labels()): +def decode_segmap(label_mask, n_classes, label_colours=get_seismic_labels()): """Decode segmentation class labels into a colour image Args: label_mask (np.ndarray): an (N,H,W) array of integer values denoting diff --git a/tests/cicd/main_build.yml b/tests/cicd/main_build.yml index 3a343233..bfc9b026 100644 --- a/tests/cicd/main_build.yml +++ b/tests/cicd/main_build.yml @@ -159,35 +159,42 @@ jobs: pids= export CUDA_VISIBLE_DEVICES=0 # find the latest model which we just trained - model=$(ls -td output/patch_deconvnet/no_depth/* | head -1) + model_dir=$(ls -td output/patch_deconvnet/no_depth/* | head -1) + model=$(ls -t ${model_dir}/*.pth | head -1) # try running the test script { python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' \ - 'TEST.MODEL_PATH' ${model}/patch_deconvnet_running_model_0.*.pth \ + 'TEST.MODEL_PATH' ${model} \ --cfg=configs/patch_deconvnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } & pids+=" $!" export CUDA_VISIBLE_DEVICES=1 # find the latest model which we just trained - model=$(ls -td output/unet/section_depth/* | head -1) + model_dir=$(ls -td output/unet/section_depth/* | head -1) + model=$(ls -t ${model_dir}/*.pth | head -1) + # try running the test script { python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' \ - 'TEST.MODEL_PATH' ${model}/resnet_unet_running_model_0.*.pth \ + 'TEST.MODEL_PATH' ${model} \ --cfg=configs/unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } & pids+=" $!" export CUDA_VISIBLE_DEVICES=2 # find the latest model which we just trained - model=$(ls -td output/seresnet_unet/section_depth/* | head -1) + model_dir=$(ls -td output/seresnet_unet/section_depth/* | head -1) + model=$(ls -t ${model_dir}/*.pth | head -1) + # try running the test script { python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' \ - 'TEST.MODEL_PATH' ${model}/resnet_unet_running_model_0.*.pth \ + 'TEST.MODEL_PATH' ${model} \ --cfg=configs/seresnet_unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } & pids+=" $!" export CUDA_VISIBLE_DEVICES=3 # find the latest model which we just trained - model=$(ls -td output/hrnet/section_depth/* | head -1) + model_dir=$(ls -td output/hrnet/section_depth/* | head -1) + model=$(ls -t ${model_dir}/*.pth | head -1) + # try running the test script { python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' \ 'MODEL.PRETRAINED' '/home/alfred/models/hrnetv2_w48_imagenet_pretrained.pth' \ - 'TEST.MODEL_PATH' ${model}/seg_hrnet_running_model_0.*.pth \ + 'TEST.MODEL_PATH' ${model} \ --cfg=configs/hrnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } & pids+=" $!" @@ -204,4 +211,4 @@ jobs: # Remove the temporary directory rm -r "$dir" - echo "PASSED" + echo "PASSED" \ No newline at end of file From 4f30cb1f4afc489e4ad157525ae316cd226b0a27 Mon Sep 17 00:00:00 2001 From: yalaudah Date: Tue, 28 Apr 2020 15:18:15 +0000 Subject: [PATCH 07/20] Fixes training/validation overlap #143, #233, #253, and #259 (#282) --- NOTICE.txt | 2 +- README.md | 48 +- cgmanifest.json | 2 +- .../dutchf3_patch/distributed/train.py | 15 +- .../dutchf3_section/local/test.py | 2 +- .../dutchf3_section/local/train.py | 15 +- .../interpretation/penobscot/local/test.py | 4 +- .../interpretation/penobscot/local/train.py | 6 +- contrib/scripts/ablation.sh | 8 +- .../event_handlers/tensorboard_handlers.py | 4 +- docker/Dockerfile | 4 +- ..._patch_model_training_and_evaluation.ipynb | 2 +- .../dutchf3_patch/local/default.py | 1 - .../dutchf3_patch/local/test.py | 3 +- .../dutchf3_patch/local/train.py | 2 +- .../dutchf3/data.py | 139 ++++-- scripts/prepare_dutchf3.py | 412 +++++++++--------- scripts/prepare_penobscot.py | 2 +- tests/cicd/src/scripts/get_data_for_builds.sh | 4 +- tests/test_prepare_dutchf3.py | 385 ++++++++++------ 20 files changed, 620 insertions(+), 440 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index 6dc34351..fe7884f1 100755 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -1949,7 +1949,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ------------------------------------------------------------------- -olivesgatech/facies_classification_benchmark 12102683a1ae78f8fbc953823c35a43b151194b3 - MIT +yalaudah/facies_classification_benchmark 12102683a1ae78f8fbc953823c35a43b151194b3 - MIT Copyright (c) 2017 Meet Pragnesh Shah Copyright (c) 2010-2018 Benjamin Peterson diff --git a/README.md b/README.md index 44d60b5a..a4559c20 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ from the root of DeepSeismic repo. ### Dataset download and preparation -This repository provides examples on how to run seismic interpretation on two publicly available annotated seismic datasets: [Penobscot](https://zenodo.org/record/1341774) and [F3 Netherlands](https://github.com/olivesgatech/facies_classification_benchmark). Their respective sizes (uncompressed on disk in your folder after downloading and pre-processing) are: +This repository provides examples on how to run seismic interpretation on two publicly available annotated seismic datasets: [Penobscot](https://zenodo.org/record/1341774) and [F3 Netherlands](https://github.com/yalaudah/facies_classification_benchmark). Their respective sizes (uncompressed on disk in your folder after downloading and pre-processing) are: - **Penobscot**: 7.9 GB - **Dutch F3**: 2.2 GB @@ -142,12 +142,12 @@ To prepare the data for the experiments (e.g. split into train/val/test), please cd scripts # For section-based experiments -python prepare_dutchf3.py split_train_val section --data_dir=${data_dir} --label_file=train/train_labels.npy --output_dir=splits +python prepare_dutchf3.py split_train_val section --data_dir=${data_dir} --label_file=train/train_labels.npy --output_dir=splits --split_direction=both # For patch-based experiments python prepare_dutchf3.py split_train_val patch --data_dir=${data_dir} --label_file=train/train_labels.npy --output_dir=splits \ ---stride=50 --patch_size=100 +--stride=50 --patch_size=100 --split_direction=both # go back to repo root cd .. @@ -244,24 +244,24 @@ Below are the results from the models contained in this repo. To run them check #### Netherlands F3 -| Source | Experiment | PA | FW IoU | MCA | V100 (16GB) training time | -|------------------|-----------------------------------|-------------|--------------|------------|-----------------------------| -| Alaudah et al.| Section-based | 0.905 | 0.817 | .832 | N/A | -| | Patch-based | 0.852 | 0.743 | .689 | N/A | -| DeepSeismic | Patch-based+fixed | .875 | .784 | .740 | 08h 54min | -| | SEResNet UNet+section depth | .910 | .841 | .809 | 55h 02min | -| | HRNet(patch)+patch_depth | .884 | .795 | .739 | 67h 41min | -| | HRNet(patch)+section_depth | .900 | .820 | .767 | 55h 08min | +| Source | Experiment | PA | FW IoU | MCA | V100 (16GB) training time | +| -------------- | --------------------------- | ----- | ------ | ---- | ------------------------- | +| Alaudah et al. | Section-based | 0.905 | 0.817 | .832 | N/A | +| | Patch-based | 0.852 | 0.743 | .689 | N/A | +| DeepSeismic | Patch-based+fixed | .875 | .784 | .740 | 08h 54min | +| | SEResNet UNet+section depth | .910 | .841 | .809 | 55h 02min | +| | HRNet(patch)+patch_depth | .884 | .795 | .739 | 67h 41min | +| | HRNet(patch)+section_depth | .900 | .820 | .767 | 55h 08min | #### Penobscot Trained and tested on the full dataset. Inlines with artifacts were left in for training, validation and testing. The dataset was split 70% training, 10% validation and 20% test. The results below are from the test set -| Source | Experiment | PA | mIoU | MCA | V100 (16GB) training time | -|------------------|-------------------------------------|-------------|--------------|------------|-----------------------------| -| DeepSeismic | SEResNet UNet + section depth | 0.72 | .35 | .47 | 92h 59min | -| | HRNet(patch) + section depth | 0.91 | .75 | .85 | 80h 50min | +| Source | Experiment | PA | mIoU | MCA | V100 (16GB) training time | +| ----------- | ----------------------------- | ---- | ---- | --- | ------------------------- | +| DeepSeismic | SEResNet UNet + section depth | 0.72 | .35 | .47 | 92h 59min | +| | HRNet(patch) + section depth | 0.91 | .75 | .85 | 80h 50min | ![Best Penobscot SEResNet](assets/penobscot_seresnet_best.png "Best performing inlines, Mask and Predictions from SEResNet") ![Worst Penobscot SEResNet](assets/penobscot_seresnet_worst.png "Worst performing inlines Mask and Predictions from SEResNet") @@ -298,16 +298,16 @@ When you submit a pull request, a CLA bot will automatically determine whether y This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. ## Build Status -| Build | Branch | Status | -| --- | --- | --- | +| Build | Branch | Status | +| -------------------- | ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | **Legal Compliance** | staging | [![Build Status](https://dev.azure.com/best-practices/deepseismic/_apis/build/status/microsoft.ComponentGovernance%20(seismic-deeplearning)?branchName=staging)](https://dev.azure.com/best-practices/deepseismic/_build/latest?definitionId=124&branchName=staging) | -| **Legal Compliance** | master | [![Build Status](https://dev.azure.com/best-practices/deepseismic/_apis/build/status/microsoft.ComponentGovernance%20(seismic-deeplearning)?branchName=master)](https://dev.azure.com/best-practices/deepseismic/_build/latest?definitionId=124&branchName=master) | -| **Core Tests** | staging | [![Build Status](https://dev.azure.com/best-practices/deepseismic/_apis/build/status/microsoft.Tests%20(seismic-deeplearning)?branchName=staging)](https://dev.azure.com/best-practices/deepseismic/_build/latest?definitionId=126&branchName=staging) | -| **Core Tests** | master | [![Build Status](https://dev.azure.com/best-practices/deepseismic/_apis/build/status/microsoft.Tests%20(seismic-deeplearning)?branchName=master)](https://dev.azure.com/best-practices/deepseismic/_build/latest?definitionId=126&branchName=master) | -| **Notebook Tests** | staging | [![Build Status](https://dev.azure.com/best-practices/deepseismic/_apis/build/status/microsoft.Notebooks%20(seismic-deeplearning)?branchName=staging)](https://dev.azure.com/best-practices/deepseismic/_build/latest?definitionId=125&branchName=staging) | -| **Notebook Tests** | master | [![Build Status](https://dev.azure.com/best-practices/deepseismic/_apis/build/status/microsoft.Notebooks%20(seismic-deeplearning)?branchName=master)](https://dev.azure.com/best-practices/deepseismic/_build/latest?definitionId=125&branchName=master) | -| **Azure ML Tests** | staging | TODO add badge link | -| **Azure ML Tests** | master | TODO add badge link | +| **Legal Compliance** | master | [![Build Status](https://dev.azure.com/best-practices/deepseismic/_apis/build/status/microsoft.ComponentGovernance%20(seismic-deeplearning)?branchName=master)](https://dev.azure.com/best-practices/deepseismic/_build/latest?definitionId=124&branchName=master) | +| **Core Tests** | staging | [![Build Status](https://dev.azure.com/best-practices/deepseismic/_apis/build/status/microsoft.Tests%20(seismic-deeplearning)?branchName=staging)](https://dev.azure.com/best-practices/deepseismic/_build/latest?definitionId=126&branchName=staging) | +| **Core Tests** | master | [![Build Status](https://dev.azure.com/best-practices/deepseismic/_apis/build/status/microsoft.Tests%20(seismic-deeplearning)?branchName=master)](https://dev.azure.com/best-practices/deepseismic/_build/latest?definitionId=126&branchName=master) | +| **Notebook Tests** | staging | [![Build Status](https://dev.azure.com/best-practices/deepseismic/_apis/build/status/microsoft.Notebooks%20(seismic-deeplearning)?branchName=staging)](https://dev.azure.com/best-practices/deepseismic/_build/latest?definitionId=125&branchName=staging) | +| **Notebook Tests** | master | [![Build Status](https://dev.azure.com/best-practices/deepseismic/_apis/build/status/microsoft.Notebooks%20(seismic-deeplearning)?branchName=master)](https://dev.azure.com/best-practices/deepseismic/_build/latest?definitionId=125&branchName=master) | +| **Azure ML Tests** | staging | TODO add badge link | +| **Azure ML Tests** | master | TODO add badge link | # Troubleshooting diff --git a/cgmanifest.json b/cgmanifest.json index d647c543..d83c6bfd 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -3,7 +3,7 @@ "component": { "type": "git", "git": { - "repositoryUrl": "https://github.com/olivesgatech/facies_classification_benchmark", + "repositoryUrl": "https://github.com/yalaudah/facies_classification_benchmark", "commitHash": "12102683a1ae78f8fbc953823c35a43b151194b3" } }, diff --git a/contrib/experiments/interpretation/dutchf3_patch/distributed/train.py b/contrib/experiments/interpretation/dutchf3_patch/distributed/train.py index 33bb0045..e52970ed 100644 --- a/contrib/experiments/interpretation/dutchf3_patch/distributed/train.py +++ b/contrib/experiments/interpretation/dutchf3_patch/distributed/train.py @@ -139,7 +139,7 @@ def run(*options, cfg=None, local_rank=0, debug=False): stride=config.TRAIN.STRIDE, patch_size=config.TRAIN.PATCH_SIZE, augmentations=train_aug, - ) + ) val_set = TrainPatchLoader( config.DATASET.ROOT, @@ -160,10 +160,9 @@ def run(*options, cfg=None, local_rank=0, debug=False): logger.info("Running in debug mode..") train_set = data.Subset(train_set, list(range(4))) val_set = data.Subset(val_set, list(range(4))) - - logger.info(f"Training examples {len(train_set)}") - logger.info(f"Validation examples {len(val_set)}") + logger.info(f"Training examples {len(train_set)}") + logger.info(f"Validation examples {len(val_set)}") train_sampler = torch.utils.data.distributed.DistributedSampler(train_set, num_replicas=world_size, rank=local_rank) train_loader = data.DataLoader( @@ -198,7 +197,7 @@ def run(*options, cfg=None, local_rank=0, debug=False): model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[device], find_unused_parameters=True) - snapshot_duration = epochs_per_cycle * len(train_loader) if not debug else 2*len(train_loader) + snapshot_duration = epochs_per_cycle * len(train_loader) if not debug else 2 * len(train_loader) warmup_duration = 5 * len(train_loader) warmup_scheduler = LinearCyclicalScheduler( optimizer, @@ -208,7 +207,11 @@ def run(*options, cfg=None, local_rank=0, debug=False): cycle_size=10 * len(train_loader), ) cosine_scheduler = CosineAnnealingScheduler( - optimizer, "lr", config.TRAIN.MAX_LR * world_size, config.TRAIN.MIN_LR * world_size, cycle_size=snapshot_duration, + optimizer, + "lr", + config.TRAIN.MAX_LR * world_size, + config.TRAIN.MIN_LR * world_size, + cycle_size=snapshot_duration, ) scheduler = ConcatScheduler(schedulers=[warmup_scheduler, cosine_scheduler], durations=[warmup_duration]) diff --git a/contrib/experiments/interpretation/dutchf3_section/local/test.py b/contrib/experiments/interpretation/dutchf3_section/local/test.py index 384ac64b..3bc4cabb 100644 --- a/contrib/experiments/interpretation/dutchf3_section/local/test.py +++ b/contrib/experiments/interpretation/dutchf3_section/local/test.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # commitHash: c76bf579a0d5090ebd32426907d051d499f3e847 -# url: https://github.com/olivesgatech/facies_classification_benchmark +# url: https://github.com/yalaudah/facies_classification_benchmark """ Modified version of the Alaudah testing script diff --git a/contrib/experiments/interpretation/dutchf3_section/local/train.py b/contrib/experiments/interpretation/dutchf3_section/local/train.py index 5a9b4900..484bbb4f 100644 --- a/contrib/experiments/interpretation/dutchf3_section/local/train.py +++ b/contrib/experiments/interpretation/dutchf3_section/local/train.py @@ -164,8 +164,10 @@ def __len__(self): summary_writer = create_summary_writer(log_dir=path.join(output_dir, config.LOG_DIR)) - snapshot_duration = epochs_per_cycle * len(train_loader) if not debug else 2*len(train_loader) - scheduler = CosineAnnealingScheduler(optimizer, "lr", config.TRAIN.MAX_LR, config.TRAIN.MIN_LR, cycle_size=snapshot_duration) + snapshot_duration = epochs_per_cycle * len(train_loader) if not debug else 2 * len(train_loader) + scheduler = CosineAnnealingScheduler( + optimizer, "lr", config.TRAIN.MAX_LR, config.TRAIN.MIN_LR, cycle_size=snapshot_duration + ) # weights are inversely proportional to the frequency of the classes in # the training set @@ -278,9 +280,14 @@ def snapshot_function(): logger.info("Starting training") if debug: - trainer.run(train_loader, max_epochs=config.TRAIN.END_EPOCH, epoch_length = config.TRAIN.BATCH_SIZE_PER_GPU, seed = config.SEED) + trainer.run( + train_loader, + max_epochs=config.TRAIN.END_EPOCH, + epoch_length=config.TRAIN.BATCH_SIZE_PER_GPU, + seed=config.SEED, + ) else: - trainer.run(train_loader, max_epochs=config.TRAIN.END_EPOCH, epoch_length = len(train_loader), seed = config.SEED) + trainer.run(train_loader, max_epochs=config.TRAIN.END_EPOCH, epoch_length=len(train_loader), seed=config.SEED) if __name__ == "__main__": diff --git a/contrib/experiments/interpretation/penobscot/local/test.py b/contrib/experiments/interpretation/penobscot/local/test.py index 073c72f1..0bf11667 100644 --- a/contrib/experiments/interpretation/penobscot/local/test.py +++ b/contrib/experiments/interpretation/penobscot/local/test.py @@ -268,9 +268,9 @@ def _tensor_to_numpy(pred_tensor): logger.info("Starting training") if debug: - evaluator.run(test_loader, max_epochs=1, epoch_length = 1) + evaluator.run(test_loader, max_epochs=1, epoch_length=1) else: - evaluator.run(test_loader, max_epochs=1, epoch_length = len(test_loader)) + evaluator.run(test_loader, max_epochs=1, epoch_length=len(test_loader)) # Log top N and bottom N inlines in terms of IoU to tensorboard inline_ious = inline_mean_iou.iou_per_inline() diff --git a/contrib/experiments/interpretation/penobscot/local/train.py b/contrib/experiments/interpretation/penobscot/local/train.py index 04f563b7..b48de2ad 100644 --- a/contrib/experiments/interpretation/penobscot/local/train.py +++ b/contrib/experiments/interpretation/penobscot/local/train.py @@ -178,8 +178,10 @@ def run(*options, cfg=None, debug=False): output_dir = generate_path(config.OUTPUT_DIR, config_file_name, config.TRAIN.MODEL_DIR, current_datetime(),) summary_writer = create_summary_writer(log_dir=path.join(output_dir, config.LOG_DIR)) - snapshot_duration = epochs_per_cycle * len(train_loader) if not debug else 2*len(train_loader) - scheduler = CosineAnnealingScheduler(optimizer, "lr", config.TRAIN.MAX_LR, config.TRAIN.MIN_LR, cycle_size=snapshot_duration) + snapshot_duration = epochs_per_cycle * len(train_loader) if not debug else 2 * len(train_loader) + scheduler = CosineAnnealingScheduler( + optimizer, "lr", config.TRAIN.MAX_LR, config.TRAIN.MIN_LR, cycle_size=snapshot_duration + ) # weights are inversely proportional to the frequency of the classes in # the training set diff --git a/contrib/scripts/ablation.sh b/contrib/scripts/ablation.sh index 6d5a0245..3e57a6e5 100755 --- a/contrib/scripts/ablation.sh +++ b/contrib/scripts/ablation.sh @@ -3,22 +3,22 @@ source activate seismic-interpretation # Patch_Size 100: Patch vs Section Depth -python scripts/prepare_dutchf3.py split_train_val patch --data_dir=/mnt/dutch --stride=50 --patch_size=100 +python scripts/prepare_dutchf3.py split_train_val patch --data_dir=/mnt/dutch --stride=50 --patch_size=100 --split_direction=both python train.py OUTPUT_DIR /data/output/hrnet_patch TRAIN.DEPTH patch TRAIN.PATCH_SIZE 100 --cfg 'configs/hrnet.yaml' python train.py OUTPUT_DIR /data/output/hrnet_section TRAIN.DEPTH section TRAIN.PATCH_SIZE 100 --cfg 'configs/hrnet.yaml' # Patch_Size 150: Patch vs Section Depth -python scripts/prepare_dutchf3.py split_train_val patch --data_dir=/mnt/dutch --stride=50 --patch_size=150 +python scripts/prepare_dutchf3.py split_train_val patch --data_dir=/mnt/dutch --stride=50 --patch_size=150 --split_direction=both python train.py OUTPUT_DIR /data/output/hrnet_patch TRAIN.DEPTH patch TRAIN.PATCH_SIZE 150 --cfg 'configs/hrnet.yaml' python train.py OUTPUT_DIR /data/output/hrnet_section TRAIN.DEPTH section TRAIN.PATCH_SIZE 150 --cfg 'configs/hrnet.yaml' # Patch_Size 200: Patch vs Section Depth -python scripts/prepare_dutchf3.py split_train_val patch --data_dir=/mnt/dutch --stride=50 --patch_size=200 +python scripts/prepare_dutchf3.py split_train_val patch --data_dir=/mnt/dutch --stride=50 --patch_size=200 --split_direction=both python train.py OUTPUT_DIR /data/output/hrnet_patch TRAIN.DEPTH patch TRAIN.PATCH_SIZE 200 --cfg 'configs/hrnet.yaml' python train.py OUTPUT_DIR /data/output/hrnet_section TRAIN.DEPTH section TRAIN.PATCH_SIZE 200 --cfg 'configs/hrnet.yaml' # Patch_Size 250: Patch vs Section Depth -python scripts/prepare_dutchf3.py split_train_val patch --data_dir=/mnt/dutch --stride=50 --patch_size=250 +python scripts/prepare_dutchf3.py split_train_val patch --data_dir=/mnt/dutch --stride=50 --patch_size=250 --split_direction=both python train.py OUTPUT_DIR /data/output/hrnet_patch TRAIN.DEPTH patch TRAIN.PATCH_SIZE 250 TRAIN.AUGMENTATIONS.RESIZE.HEIGHT 250 TRAIN.AUGMENTATIONS.RESIZE.WIDTH 250 --cfg 'configs/hrnet.yaml' python train.py OUTPUT_DIR /data/output/hrnet_section TRAIN.DEPTH section TRAIN.PATCH_SIZE 250 TRAIN.AUGMENTATIONS.RESIZE.HEIGHT 250 TRAIN.AUGMENTATIONS.RESIZE.WIDTH 250 --cfg 'configs/hrnet.yaml' diff --git a/cv_lib/cv_lib/event_handlers/tensorboard_handlers.py b/cv_lib/cv_lib/event_handlers/tensorboard_handlers.py index 30cb5fc4..625e11d1 100644 --- a/cv_lib/cv_lib/event_handlers/tensorboard_handlers.py +++ b/cv_lib/cv_lib/event_handlers/tensorboard_handlers.py @@ -57,7 +57,9 @@ def log_lr(summary_writer, optimizer, log_interval, engine): # TODO: This is deprecated, and will be removed in the future. @curry -def log_metrics(summary_writer, train_engine, log_interval, engine, metrics_dict={"pixacc": "Avg accuracy :", "nll": "Avg loss :"}): +def log_metrics( + summary_writer, train_engine, log_interval, engine, metrics_dict={"pixacc": "Avg accuracy :", "nll": "Avg loss :"} +): metrics = engine.state.metrics for m in metrics_dict: summary_writer.add_scalar(metrics_dict[m], metrics[m], getattr(train_engine.state, log_interval)) diff --git a/docker/Dockerfile b/docker/Dockerfile index 9a7414c9..f81cf19f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -47,8 +47,8 @@ RUN data_dir="/home/username/data/dutch" && \ ./scripts/download_dutch_f3.sh "$data_dir" && \ cd scripts && \ source activate seismic-interpretation && \ - python prepare_dutchf3.py split_train_val section --data-dir=${data_dir}/data --label_file=train/train_labels.npy --output_dir=splits && \ - python prepare_dutchf3.py split_train_val patch --data-dir=${data_dir}/data --label_file=train/train_labels.npy --output_dir=splits --stride=50 --patch_size=100 && \ + python prepare_dutchf3.py split_train_val section --data-dir=${data_dir}/data --label_file=train/train_labels.npy --output_dir=splits --split_direction=both && \ + python prepare_dutchf3.py split_train_val patch --data-dir=${data_dir}/data --label_file=train/train_labels.npy --output_dir=splits --stride=50 --patch_size=100 --split_direction=both && \ cd .. # Run notebook diff --git a/examples/interpretation/notebooks/Dutch_F3_patch_model_training_and_evaluation.ipynb b/examples/interpretation/notebooks/Dutch_F3_patch_model_training_and_evaluation.ipynb index 6f830701..c11014c5 100644 --- a/examples/interpretation/notebooks/Dutch_F3_patch_model_training_and_evaluation.ipynb +++ b/examples/interpretation/notebooks/Dutch_F3_patch_model_training_and_evaluation.ipynb @@ -22,7 +22,7 @@ "source": [ "Seismic interpretation, also referred to as facies classification, is a task of determining types of rock in the earth’s subsurface, given seismic data. Seismic interpretation is used as a standard approach for determining precise locations of oil deposits for drilling, therefore reducing risks and potential losses. In recent years, there has been a great interest in using fully-supervised deep learning models for seismic interpretation. \n", "\n", - "In this notebook, we demonstrate how to train a deep neural network for facies prediction using F3 Netherlands dataset. The F3 block is located in the North Sea off the shores of Netherlands. The dataset contains 6 classes (facies or lithostratigraphic units), all of which are of varying thickness (class imbalance). Processed data is available in numpy format as a `401 x 701 x 255` array. The processed F3 data is made available by [Alaudah et al. 2019](https://github.com/olivesgatech/facies_classification_benchmark).\n", + "In this notebook, we demonstrate how to train a deep neural network for facies prediction using F3 Netherlands dataset. The F3 block is located in the North Sea off the shores of Netherlands. The dataset contains 6 classes (facies or lithostratigraphic units), all of which are of varying thickness (class imbalance). Processed data is available in numpy format as a `401 x 701 x 255` array. The processed F3 data is made available by [Alaudah et al. 2019](https://github.com/yalaudah/facies_classification_benchmark).\n", "\n", "We specifically demonstrate a patch-based model approach, where we process a patch of an inline or crossline slice, instead of the entire slice." ] diff --git a/experiments/interpretation/dutchf3_patch/local/default.py b/experiments/interpretation/dutchf3_patch/local/default.py index f2cadfc1..4a4c74af 100644 --- a/experiments/interpretation/dutchf3_patch/local/default.py +++ b/experiments/interpretation/dutchf3_patch/local/default.py @@ -59,7 +59,6 @@ _C.TRAIN.MEAN = 0.0009997 # 0.0009996710808862074 _C.TRAIN.STD = 0.20977 # 0.20976548783479299 # TODO: Should we apply std scaling? _C.TRAIN.DEPTH = "none" # Options are: none, patch, and section - # None adds no depth information and the num of channels remains at 1 # Patch adds depth per patch so is simply the height of that patch from 0 to 1, channels=3 # Section adds depth per section so contains depth information for the whole section, channels=3 diff --git a/experiments/interpretation/dutchf3_patch/local/test.py b/experiments/interpretation/dutchf3_patch/local/test.py index 631fd4f4..f56162cb 100644 --- a/experiments/interpretation/dutchf3_patch/local/test.py +++ b/experiments/interpretation/dutchf3_patch/local/test.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. # commitHash: c76bf579a0d5090ebd32426907d051d499f3e847 -# url: https://github.com/olivesgatech/facies_classification_benchmark +# url: https://github.com/yalaudah/facies_classification_benchmark # # To Test: # python test.py TRAIN.END_EPOCH 1 TRAIN.SNAPSHOTS 1 --cfg "configs/hrnet.yaml" --debug @@ -44,6 +44,7 @@ "zechstein", ] + class runningScore(object): def __init__(self, n_classes): self.n_classes = n_classes diff --git a/experiments/interpretation/dutchf3_patch/local/train.py b/experiments/interpretation/dutchf3_patch/local/train.py index 14dcf726..bb51b1d9 100644 --- a/experiments/interpretation/dutchf3_patch/local/train.py +++ b/experiments/interpretation/dutchf3_patch/local/train.py @@ -163,7 +163,7 @@ def run(*options, cfg=None, debug=False): ) epochs_per_cycle = config.TRAIN.END_EPOCH // config.TRAIN.SNAPSHOTS - snapshot_duration = epochs_per_cycle * len(train_loader) if not debug else 2*len(train_loader) + snapshot_duration = epochs_per_cycle * len(train_loader) if not debug else 2 * len(train_loader) scheduler = CosineAnnealingScheduler( optimizer, "lr", config.TRAIN.MAX_LR, config.TRAIN.MIN_LR, cycle_size=snapshot_duration ) diff --git a/interpretation/deepseismic_interpretation/dutchf3/data.py b/interpretation/deepseismic_interpretation/dutchf3/data.py index 4517f510..c76b29ef 100644 --- a/interpretation/deepseismic_interpretation/dutchf3/data.py +++ b/interpretation/deepseismic_interpretation/dutchf3/data.py @@ -18,7 +18,7 @@ interpolate_to_fit_data, parse_labels_in_image, get_coordinates_for_slice, - get_grid, + get_grid, rand_int, trilinear_interpolation, ) @@ -112,6 +112,7 @@ def read_labels(fname, data_info): return label_imgs, label_coordinates + class SectionLoader(data.Dataset): """ Base class for section data loader @@ -122,8 +123,10 @@ class SectionLoader(data.Dataset): :param str seismic_path: Override file path for seismic data :param str label_path: Override file path for label data """ - def __init__(self, data_dir, split="train", is_transform=True, augmentations=None, - seismic_path=None, label_path=None): + + def __init__( + self, data_dir, split="train", is_transform=True, augmentations=None, seismic_path=None, label_path=None + ): self.split = split self.data_dir = data_dir self.is_transform = is_transform @@ -175,11 +178,17 @@ class TrainSectionLoader(SectionLoader): :param str seismic_path: Override file path for seismic data :param str label_path: Override file path for label data """ - def __init__(self, data_dir, split="train", is_transform=True, augmentations=None, - seismic_path=None, label_path=None): + + def __init__( + self, data_dir, split="train", is_transform=True, augmentations=None, seismic_path=None, label_path=None + ): super(TrainSectionLoader, self).__init__( - data_dir, split=split, is_transform=is_transform, augmentations=augmentations, - seismic_path=seismic_path, label_path=label_path + data_dir, + split=split, + is_transform=is_transform, + augmentations=augmentations, + seismic_path=seismic_path, + label_path=label_path, ) if seismic_path is not None and label_path is not None: @@ -212,11 +221,17 @@ class TrainSectionLoaderWithDepth(TrainSectionLoader): :param str seismic_path: Override file path for seismic data :param str label_path: Override file path for label data """ - def __init__(self, data_dir, split="train", is_transform=True, augmentations=None, - seismic_path=None, label_path=None): + + def __init__( + self, data_dir, split="train", is_transform=True, augmentations=None, seismic_path=None, label_path=None + ): super(TrainSectionLoaderWithDepth, self).__init__( - data_dir, split=split, is_transform=is_transform, augmentations=augmentations, - seismic_path=seismic_path, label_path=label_path + data_dir, + split=split, + is_transform=is_transform, + augmentations=augmentations, + seismic_path=seismic_path, + label_path=label_path, ) self.seismic = add_section_depth_channels(self.seismic) # NCWH @@ -258,10 +273,12 @@ class TestSectionLoader(SectionLoader): :param str seismic_path: Override file path for seismic data :param str label_path: Override file path for label data """ - def __init__(self, data_dir, split = "test1", is_transform = True, augmentations = None, - seismic_path = None, label_path = None): + + def __init__( + self, data_dir, split="test1", is_transform=True, augmentations=None, seismic_path=None, label_path=None + ): super(TestSectionLoader, self).__init__( - data_dir, split=split, is_transform=is_transform, augmentations=augmentations, + data_dir, split=split, is_transform=is_transform, augmentations=augmentations, ) if "test1" in self.split: @@ -298,11 +315,17 @@ class TestSectionLoaderWithDepth(TestSectionLoader): :param str seismic_path: Override file path for seismic data :param str label_path: Override file path for label data """ - def __init__(self, data_dir, split="test1", is_transform=True, augmentations=None, - seismic_path = None, label_path = None): + + def __init__( + self, data_dir, split="test1", is_transform=True, augmentations=None, seismic_path=None, label_path=None + ): super(TestSectionLoaderWithDepth, self).__init__( - data_dir, split=split, is_transform=is_transform, augmentations=augmentations, - seismic_path = seismic_path, label_path = label_path + data_dir, + split=split, + is_transform=is_transform, + augmentations=augmentations, + seismic_path=seismic_path, + label_path=label_path, ) self.seismic = add_section_depth_channels(self.seismic) # NCWH @@ -351,8 +374,17 @@ class PatchLoader(data.Dataset): :param str seismic_path: Override file path for seismic data :param str label_path: Override file path for label data """ - def __init__(self, data_dir, stride=30, patch_size=99, is_transform=True, augmentations=None, - seismic_path=None, label_path=None): + + def __init__( + self, + data_dir, + stride=30, + patch_size=99, + is_transform=True, + augmentations=None, + seismic_path=None, + label_path=None, + ): self.data_dir = data_dir self.is_transform = is_transform self.augmentations = augmentations @@ -416,8 +448,8 @@ class TestPatchLoader(PatchLoader): :param bool is_transform: Transform patch to dimensions expected by PyTorch :param list augmentations: Data augmentations to apply to patches """ - def __init__(self, data_dir, stride=30, patch_size=99, is_transform=True, augmentations=None, - txt_path=None): + + def __init__(self, data_dir, stride=30, patch_size=99, is_transform=True, augmentations=None, txt_path=None): super(TestPatchLoader, self).__init__( data_dir, stride=stride, patch_size=patch_size, is_transform=is_transform, augmentations=augmentations, ) @@ -428,7 +460,7 @@ def __init__(self, data_dir, stride=30, patch_size=99, is_transform=True, augmen # We are in test mode. Only read the given split. The other one might not # be available. - # If txt_path is not provided, it will be assumed as below. Otherwise, provided path will be used for + # If txt_path is not provided, it will be assumed as below. Otherwise, provided path will be used for # loading txt file and create patches. if not txt_path: self.split = "test1" # TODO: Fix this can also be test2 @@ -451,13 +483,26 @@ class TrainPatchLoader(PatchLoader): :param str seismic_path: Override file path for seismic data :param str label_path: Override file path for label data """ + def __init__( - self, data_dir, split="train", stride=30, patch_size=99, is_transform=True, augmentations=None, - seismic_path=None, label_path=None + self, + data_dir, + split="train", + stride=30, + patch_size=99, + is_transform=True, + augmentations=None, + seismic_path=None, + label_path=None, ): super(TrainPatchLoader, self).__init__( - data_dir, stride=stride, patch_size=patch_size, is_transform=is_transform, augmentations=augmentations, - seismic_path=seismic_path, label_path=label_path + data_dir, + stride=stride, + patch_size=patch_size, + is_transform=is_transform, + augmentations=augmentations, + seismic_path=seismic_path, + label_path=label_path, ) # self.seismic = self.pad_volume(np.load(seismic_path)) # self.labels = self.pad_volume(np.load(labels_path)) @@ -496,13 +541,27 @@ class TrainPatchLoaderWithDepth(TrainPatchLoader): :param str seismic_path: Override file path for seismic data :param str label_path: Override file path for label data """ + def __init__( - self, data_dir, split="train", stride=30, patch_size=99, is_transform=True, augmentations=None, - seismic_path=None, label_path=None + self, + data_dir, + split="train", + stride=30, + patch_size=99, + is_transform=True, + augmentations=None, + seismic_path=None, + label_path=None, ): super(TrainPatchLoaderWithDepth, self).__init__( - data_dir, split=split, stride=stride, patch_size=patch_size, is_transform=is_transform, augmentations=augmentations, - seismic_path=seismic_path, label_path=label_path + data_dir, + split=split, + stride=stride, + patch_size=patch_size, + is_transform=is_transform, + augmentations=augmentations, + seismic_path=seismic_path, + label_path=label_path, ) def __getitem__(self, index): @@ -558,9 +617,17 @@ class TrainPatchLoaderWithSectionDepth(TrainPatchLoader): :param str seismic_path: Override file path for seismic data :param str label_path: Override file path for label data """ + def __init__( - self, data_dir, split="train", stride=30, patch_size=99, is_transform=True, augmentations=None, - seismic_path=None, label_path=None + self, + data_dir, + split="train", + stride=30, + patch_size=99, + is_transform=True, + augmentations=None, + seismic_path=None, + label_path=None, ): super(TrainPatchLoaderWithSectionDepth, self).__init__( data_dir, @@ -569,8 +636,8 @@ def __init__( patch_size=patch_size, is_transform=is_transform, augmentations=augmentations, - seismic_path = seismic_path, - label_path = label_path + seismic_path=seismic_path, + label_path=label_path, ) self.seismic = add_section_depth_channels(self.seismic) @@ -579,7 +646,6 @@ def __getitem__(self, index): patch_name = self.patches[index] direction, idx, xdx, ddx = patch_name.split(sep="_") - # Shift offsets the padding that is added in training # shift = self.patch_size if "test" not in self.split else 0 # TODO: Remember we are cancelling the shift since we no longer pad @@ -620,6 +686,7 @@ def __repr__(self): _TRAIN_SECTION_LOADERS = {"section": TrainSectionLoaderWithDepth} + def get_patch_loader(cfg): assert str(cfg.TRAIN.DEPTH).lower() in [ "section", diff --git a/scripts/prepare_dutchf3.py b/scripts/prepare_dutchf3.py index 33209074..7157214d 100644 --- a/scripts/prepare_dutchf3.py +++ b/scripts/prepare_dutchf3.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. # commitHash: c76bf579a0d5090ebd32426907d051d499f3e847 -# url: https://github.com/olivesgatech/facies_classification_benchmark +# url: https://github.com/yalaudah/facies_classification_benchmark """Script to generate train and validation sets for Netherlands F3 dataset """ import itertools @@ -16,244 +16,188 @@ from sklearn.model_selection import train_test_split +def _get_splits_path(data_dir): + return path.join(data_dir, "splits") + + +def _get_labels_path(data_dir): + return path.join(data_dir, "train", "train_labels.npy") + + +def get_split_function(loader_type): + return _LOADER_TYPES.get(loader_type, split_patch_train_val) + + +def run_split_func(loader_type, *args, **kwargs): + split_func = get_split_function(loader_type) + split_func(*args, **kwargs) + + def _write_split_files(splits_path, train_list, val_list, loader_type): if not path.isdir(splits_path): mkdir(splits_path) - file_object = open(path.join(splits_path, - loader_type + "_train_val.txt"), "w") + file_object = open(path.join(splits_path, loader_type + "_train_val.txt"), "w") file_object.write("\n".join(train_list + val_list)) file_object.close() - file_object = open(path.join(splits_path, - loader_type + "_train.txt"), "w") + file_object = open(path.join(splits_path, loader_type + "_train.txt"), "w") file_object.write("\n".join(train_list)) file_object.close() - file_object = open(path.join(splits_path, - loader_type + "_val.txt"), "w") + file_object = open(path.join(splits_path, loader_type + "_val.txt"), "w") file_object.write("\n".join(val_list)) file_object.close() -def _get_aline_range(aline, per_val, slice_steps): +def _get_aline_range(aline, per_val, section_stride=1): + """ + Args: + aline (int): number of seismic sections in the inline or + crossline directions + per_val (float): the fraction of the volume to use for + validation. Defaults to 0.2. + section_stride (int): the stride of the sections in the training data. + If greater than 1, this function will skip (section_stride-1) between each section + Defaults to 1, do not skip any section. + """ try: - if slice_steps < 1: - raise ValueError('slice_steps cannot be zero or a negative number') - # Inline and Crossline sections + if section_stride < 1: + raise ValueError("section_stride cannot be zero or a negative number") + + if per_val < 0 or per_val >= 1: + raise ValueError("Validation percentage (per_val) should be a number in the range [0,1).") + val_aline = math.floor(aline * per_val / 2) - val_aline_range = itertools.chain(range(0, val_aline), - range(aline - val_aline, aline)) - train_aline_range = range(val_aline, aline - val_aline, slice_steps) + val_range = itertools.chain(range(0, val_aline), range(aline - val_aline, aline)) + train_range = range(val_aline, aline - val_aline, section_stride) - print("aline: ", aline) - print("val_aline: ", val_aline) - return train_aline_range, val_aline_range + return train_range, val_range except (Exception, ValueError): raise -def split_section_train_val(data_dir, output_dir, label_file, per_val=0.2, - log_config=None, slice_steps=1): +def split_section_train_val(label_file, split_direction, per_val=0.2, log_config=None, section_stride=1): """Generate train and validation files for Netherlands F3 dataset. Args: - data_dir (str): data directory path - output_dir (str): directory under data_dir to store the split files label_file (str): npy files with labels. Stored in data_dir + split_direction (str): Direction in which to split the data into + train & val. Use "inline" or "crossline". per_val (float, optional): the fraction of the volume to use for validation. Defaults to 0.2. log_config (str): path to log configurations - slice_steps (int): increment to the slices count. - If slice_steps > 1 the function will skip: - slice_steps - 1 slice. - Defaults to 1, do not skip any slice. + section_stride (int): the stride of the sections in the training data. + If greater than 1, this function will skip (section_stride-1) between each section + Defaults to 1, do not skip any section. """ if log_config is not None: logging.config.fileConfig(log_config) logger = logging.getLogger(__name__) - logger.info("Splitting data into sections .... ") - logger.info(f"Reading data from {data_dir}") - logger.info(f"Loading {label_file}") + labels = np.load(label_file) logger.debug(f"Data shape [iline|xline|depth] {labels.shape}") + iline, xline, _ = labels.shape # TODO: Must make sure in the future, all new datasets conform to this order. - iline, xline, _ = labels.shape - # Inline sections - train_iline_range, val_iline_range = _get_aline_range(iline, - per_val, - slice_steps) - train_i_list = ["i_" + str(i) for i in train_iline_range] - val_i_list = ["i_" + str(i) for i in val_iline_range] + logger.info(f"Splitting in {split_direction} direction.. ") + if split_direction.lower() == "inline": + num_sections = iline + index = "i" + elif split_direction.lower() == "crossline": + num_sections = xline + index = "x" + else: + raise ValueError(f"Unknown split_direction {split_direction}") - # Xline sections - train_xline_range, val_xline_range = _get_aline_range(xline, - per_val, - slice_steps) - train_x_list = ["x_" + str(x) for x in train_xline_range] - val_x_list = ["x_" + str(x) for x in val_xline_range] + train_range, val_range = _get_aline_range(num_sections, per_val, section_stride) + train_list = [f"{index}_" + str(section) for section in train_range] + val_list = [f"{index}_" + str(section) for section in val_range] - train_list = train_x_list + train_i_list - val_list = val_x_list + val_i_list + return train_list, val_list - # write to files to disk - logger.info(f"Writing {output_dir}") - _write_split_files(output_dir, train_list, val_list, "section") - -def split_patch_train_val(data_dir, output_dir, label_file, stride, patch_size, - slice_steps=1, per_val=0.2, log_config=None): +def split_patch_train_val( + label_file, patch_stride, patch_size, split_direction, section_stride=1, per_val=0.2, log_config=None, +): """Generate train and validation files for Netherlands F3 dataset. Args: - data_dir (str): data directory path - output_dir (str): directory under data_dir to store the split files label_file (str): npy files with labels. Stored in data_dir - stride (int): stride to use when sectioning of the volume + patch_stride (int): stride to use when sampling patches patch_size (int): size of patch to extract + split_direction (str): Direction in which to split the data into + train & val. Use "inline" or "crossline". + section_stride (int): increment to the slices count. + If section_stride > 1 the function will skip: + section_stride - 1 sections in the training data. + Defaults to 1, do not skip any slice. per_val (float, optional): the fraction of the volume to use for validation. Defaults to 0.2. log_config (str): path to log configurations - slice_steps (int): increment to the slices count. - If slice_steps > 1 the function will skip: - slice_steps - 1 slice. - Defaults to 1, do not skip any slice. """ if log_config is not None: logging.config.fileConfig(log_config) logger = logging.getLogger(__name__) - - logger.info("Splitting data into patches .... ") - logger.info(f"Reading data from {data_dir}") - + logger.info(f"Splitting data into patches along {split_direction} direction .. ") logger.info(f"Loading {label_file}") labels = np.load(label_file) logger.debug(f"Data shape [iline|xline|depth] {labels.shape}") iline, xline, depth = labels.shape - # Inline sections - train_iline_range, val_iline_range = _get_aline_range(iline, - per_val, - slice_steps) - - # Xline sections - train_xline_range, val_xline_range = _get_aline_range(xline, - per_val, - slice_steps) - - # Generate patches from sections - # Vertical locations is common to all patches processed - vert_locations = range(0, depth - patch_size, patch_size) - logger.debug(vert_locations) - - # Process inlines - def _i_extract_patches(iline_range, horz_locations, vert_locations): - for i in iline_range: - locations = ([j, k] for j in horz_locations - for k in vert_locations) - for j, k in locations: - yield "i_" + str(i) + "_" + str(j) + "_" + str(k) - - # Process inlines - train - logger.debug("Generating Inline patches") - logger.debug("Generating Inline patches - Train") - # iline = xline x depth - val_iline = math.floor(xline * per_val / 2) - logger.debug(val_iline) - - # Process ilines - train - horz_locations_train = range(val_iline, xline - val_iline, max(1,patch_size)) - logger.debug(horz_locations_train) - train_i_list = list(_i_extract_patches(train_iline_range, - horz_locations_train, - vert_locations)) - - # val_iline - define size of the validation set for the fist part - val_iline_range = list(val_iline_range) - - # Process inlines - validation - horz_locations_val = itertools.chain(range(0, val_iline, max(1,patch_size)), - range(xline - val_iline, xline, max(1,patch_size))) - val_iline_range = list(val_iline_range) - val_i_list = list(_i_extract_patches(val_iline_range, - horz_locations_val, - vert_locations)) - - logger.debug(train_iline_range) - logger.debug(val_iline_range) - - # Process crosslines - def _x_extract_patches(xline_range, horz_locations, vert_locations): - for j in xline_range: - locations = ([i, k] for i in horz_locations - for k in vert_locations) - for i, k in locations: - yield "x_" + str(i) + "_" + str(j) + "_" + str(k) - - logger.debug("Generating Crossline patches") - logger.debug("Generating Crossline patches - Train") - # xline = iline x depth - val_xline = math.floor(iline * per_val / 2) - logger.debug(val_xline) - - # Process xlines - train - horz_locations_train = range(val_xline, iline - val_xline, max(1,patch_size)) - logger.debug(horz_locations_train) - train_x_list = list(_x_extract_patches(train_xline_range, - horz_locations_train, - vert_locations)) - - # val_xline - define size of the validation set for the fist part - val_xline_range = list(val_xline_range) - - # Process xlines - validation - horz_locations_val = itertools.chain(range(0, val_xline, max(1,patch_size)), - range(iline - val_xline, iline, max(1,patch_size))) - val_xline_range = list(val_xline_range) - val_x_list = list(_x_extract_patches(val_xline_range, - horz_locations_val, - vert_locations)) - - logger.debug(train_xline_range) - logger.debug(val_xline_range) - - train_list = train_x_list + train_i_list - val_list = val_x_list + val_i_list + split_direction = split_direction.lower() + if split_direction == "inline": + num_sections, section_length = iline, xline + elif split_direction == "crossline": + num_sections, section_length = xline, iline + else: + raise ValueError(f"Unknown split_direction: {split_direction}") + + train_range, val_range = _get_aline_range(num_sections, per_val, section_stride) + vert_locations = range(0, depth, patch_stride) + horz_locations = range(0, section_length, patch_stride) + logger.debug(vert_locations) + logger.debug(horz_locations) + + # Process sections: + def _extract_patches(sections_range, direction, horz_locations, vert_locations): + locations = itertools.product(sections_range, horz_locations, vert_locations) + if direction == "inline": + idx, xdx, ddx = 0, 1, 2 + dir = "i" + elif direction == "crossline": + idx, xdx, ddx = 1, 0, 2 + dir = "x" + + for loc in locations: # iline xline depth + yield f"{dir}_" + str(loc[idx]) + "_" + str(loc[xdx]) + "_" + str(loc[ddx]) + + # Process sections - train + logger.debug("Generating patches..") + train_list = list(_extract_patches(train_range, split_direction, horz_locations, vert_locations)) + val_list = list(_extract_patches(val_range, split_direction, horz_locations, vert_locations)) + + logger.debug(train_range) + logger.debug(val_range) logger.debug(train_list) logger.debug(val_list) - - # write to files to disk: - # NOTE: This isn't quite right we should calculate the patches - # again for the whole volume - logger.info(f"Writing {output_dir}") - _write_split_files(output_dir, train_list, val_list, "patch") - -_LOADER_TYPES = {"section": split_section_train_val, - "patch": split_patch_train_val} + return train_list, val_list -def get_split_function(loader_type): - return _LOADER_TYPES.get(loader_type, split_patch_train_val) - - -def run_split_func(loader_type, *args, **kwargs): - split_func = get_split_function(loader_type) - split_func(*args, **kwargs) - - -def split_alaudah_et_al_19(data_dir, stride, patch_size, fraction_validation=0.2, loader_type="patch", log_config=None): +def split_alaudah_et_al_19( + data_dir, patch_stride, patch_size, fraction_validation=0.2, loader_type="patch", log_config=None +): """Generate train and validation files (with overlap) for Netherlands F3 dataset. - The original split method from https://github.com/olivesgatech/facies_classification_benchmark + The original split method from https://github.com/yalaudah/facies_classification_benchmark DON'T USE, SEE NOTES BELOW Args: data_dir (str): data directory path - stride (int): stride to use when sectioning of the volume + patch_stride (int): stride to use when sampling patches patch_size (int): size of patch to extract fraction_validation (float, optional): the fraction of the volume to use for validation. Defaults to 0.2. @@ -293,8 +237,8 @@ def split_alaudah_et_al_19(data_dir, stride, patch_size, fraction_validation=0.2 x_list = ["x_" + str(x) for x in range(xline)] elif loader_type == "patch": i_list = [] - horz_locations = range(0, xline - patch_size + 1, stride) - vert_locations = range(0, depth - patch_size + 1, stride) + horz_locations = range(0, xline - patch_size + 1, patch_stride) + vert_locations = range(0, depth - patch_size + 1, patch_stride) logger.debug("Generating Inline patches") logger.debug(horz_locations) logger.debug(vert_locations) @@ -309,8 +253,8 @@ def split_alaudah_et_al_19(data_dir, stride, patch_size, fraction_validation=0.2 i_list = list(itertools.chain(*i_list)) x_list = [] - horz_locations = range(0, iline - patch_size + 1, stride) - vert_locations = range(0, depth - patch_size + 1, stride) + horz_locations = range(0, iline - patch_size + 1, patch_stride) + vert_locations = range(0, depth - patch_size + 1, patch_stride) for j in range(xline): # for every xline: # images are references by top-left corner: @@ -332,9 +276,16 @@ def split_alaudah_et_al_19(data_dir, stride, patch_size, fraction_validation=0.2 class SplitTrainValCLI(object): - def section(self, data_dir, label_file, per_val=0.2, - log_config="logging.conf", output_dir=None, - slice_steps=1): + def section( + self, + data_dir, + label_file, + split_direction, + per_val=0.2, + log_config="logging.conf", + output_dir=None, + section_stride=1, + ): """Generate section based train and validation files for Netherlands F3 dataset. @@ -342,69 +293,108 @@ def section(self, data_dir, label_file, per_val=0.2, data_dir (str): data directory path output_dir (str): directory under data_dir to store the split files label_file (str): npy files with labels. Stored in data_dir + split_direction (int): Direction in which to split the data into + train & val. Use "inline" or "crossline", or "both". per_val (float, optional): the fraction of the volume to use for validation. Defaults to 0.2. log_config (str): path to log configurations - slice_steps (int): increment to the slices count. - If slice_steps > 1 the function will skip: - slice_steps - 1 slice. - Defaults to 1, do not skip any slice. + section_stride (int): the stride of the sections in the training data. + If greater than 1, this function will skip (section_stride-1) between each section + Defaults to 1, do not skip any section. """ if data_dir is not None: label_file = path.join(data_dir, label_file) - output_dir = path.join(data_dir, output_dir) - return split_section_train_val(data_dir=data_dir, - output_dir=output_dir, - label_file=label_file, - slice_steps=slice_steps, - per_val=per_val, - log_config=log_config) - - def patch(self, label_file, stride, patch_size, - per_val=0.2, log_config="logging.conf", - data_dir=None, output_dir=None, slice_steps=1): + output_dir = path.join(data_dir, output_dir) + + if split_direction.lower() == "both": + train_list_i, val_list_i = split_section_train_val( + label_file, "inline", per_val, log_config, section_stride + ) + train_list_x, val_list_x = split_section_train_val( + label_file, "crossline", per_val, log_config, section_stride + ) + # concatenate the two lists: + train_list = train_list_i + train_list_x + val_list = val_list_i + val_list_x + elif split_direction.lower() in ["inline", "crossline"]: + train_list, val_list = split_section_train_val( + label_file, split_direction, per_val, log_config, section_stride + ) + else: + raise ValueError(f"Unknown split_direction: {split_direction}") + # write to files to disk + _write_split_files(output_dir, train_list, val_list, "section") + + def patch( + self, + label_file, + stride, + patch_size, + split_direction, + per_val=0.2, + log_config="logging.conf", + data_dir=None, + output_dir=None, + section_stride=1, + ): """Generate train and validation files for Netherlands F3 dataset. Args: data_dir (str): data directory path output_dir (str): directory under data_dir to store the split files label_file (str): npy files with labels. Stored in data_dir - stride (int): stride to use when sectioning of the volume + stride (int): stride to use when sampling patches patch_size (int): size of patch to extract per_val (float, optional): the fraction of the volume to use for validation. Defaults to 0.2. log_config (str): path to log configurations - slice_steps (int): increment to the slices count. - If slice_steps > 1 the function will skip: - slice_steps - 1 slice. - Defaults to 1, do not skip any slice. + split_direction (int): Direction in which to split the data into + train & val. Use "inline" or "crossline", or "both". + section_stride (int): the stride of the sections in the training data. + If greater than 1, this function will skip (section_stride-1) between each section + Defaults to 1, do not skip any section. """ if data_dir is not None: label_file = path.join(data_dir, label_file) - output_dir = path.join(data_dir, output_dir) + output_dir = path.join(data_dir, output_dir) + + if split_direction.lower() == "both": + train_list_i, val_list_i = split_patch_train_val( + label_file, stride, patch_size, "inline", section_stride, per_val, log_config + ) + + train_list_x, val_list_x = split_patch_train_val( + label_file, stride, patch_size, "crossline", section_stride, per_val, log_config + ) + # concatenate the two lists: + train_list = train_list_i + train_list_x + val_list = val_list_i + val_list_x + elif split_direction.lower() in ["inline", "crossline"]: + train_list, val_list = split_patch_train_val( + label_file, stride, patch_size, split_direction, section_stride, per_val, log_config + ) + else: + raise ValueError(f"Unknown split_direction: {split_direction}") + + # write to files to disk: + _write_split_files(output_dir, train_list, val_list, "patch") + print(f"Successfully created the splits files in {output_dir}") - return split_patch_train_val(data_dir=data_dir, - output_dir=output_dir, - label_file=label_file, - stride=stride, - patch_size=patch_size, - slice_steps=slice_steps, - per_val=per_val, - log_config=log_config) +_LOADER_TYPES = {"section": split_section_train_val, "patch": split_patch_train_val} if __name__ == "__main__": """Example: python prepare_data.py split_train_val section --data_dir=data \ - --label_file=label_file.npy --output_dir=splits --slice_steps=2 + --label_file=label_file.npy --output_dir=splits --split_direction=both --section_stride=2 or python prepare_dutchf3.py split_train_val patch --data_dir=data \ --label_file=label_file.npy --output_dir=splits --stride=50 \ - --patch_size=100 --slice_steps=2 + --patch_size=100 --split_direction=both --section_stride=2 """ fire.Fire( {"split_train_val": SplitTrainValCLI} - # commenting the following line as this was not updated with - # the new parameters names - # "split_alaudah_et_al_19": split_alaudah_et_al_19} - ) \ No newline at end of file + # commenting the following line as this was not updated with + # the new parameters names + # "split_alaudah_et_al_19": split_alaudah_et_al_19} + ) diff --git a/scripts/prepare_penobscot.py b/scripts/prepare_penobscot.py index 754993be..8f164d82 100644 --- a/scripts/prepare_penobscot.py +++ b/scripts/prepare_penobscot.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. # commitHash: c76bf579a0d5090ebd32426907d051d499f3e847 -# url: https://github.com/olivesgatech/facies_classification_benchmark +# url: https://github.com/yalaudah/facies_classification_benchmark """Script to generate train and validation sets for Netherlands F3 dataset """ import itertools diff --git a/tests/cicd/src/scripts/get_data_for_builds.sh b/tests/cicd/src/scripts/get_data_for_builds.sh index e9302e14..f3610fa9 100755 --- a/tests/cicd/src/scripts/get_data_for_builds.sh +++ b/tests/cicd/src/scripts/get_data_for_builds.sh @@ -39,5 +39,5 @@ DATA_F3="${DATA_F3}/data" # test preprocessing scripts cd scripts python prepare_penobscot.py split_inline --data-dir=${DATA_PENOBSCOT} --val-ratio=.1 --test-ratio=.2 -python prepare_dutchf3.py split_train_val section --data_dir=${DATA_F3} --label_file=train/train_labels.npy --output_dir=splits -python prepare_dutchf3.py split_train_val patch --data_dir=${DATA_F3} --label_file=train/train_labels.npy --output_dir=splits --stride=50 --patch_size=100 +python prepare_dutchf3.py split_train_val section --data_dir=${DATA_F3} --label_file=train/train_labels.npy --output_dir=splits --split_direction=both +python prepare_dutchf3.py split_train_val patch --data_dir=${DATA_F3} --label_file=train/train_labels.npy --output_dir=splits --stride=50 --patch_size=100 --split_direction=both diff --git a/tests/test_prepare_dutchf3.py b/tests/test_prepare_dutchf3.py index 1484ee49..34349e29 100644 --- a/tests/test_prepare_dutchf3.py +++ b/tests/test_prepare_dutchf3.py @@ -2,12 +2,15 @@ # Licensed under the MIT License. """Test the extract functions against a variety of SEGY files and trace_header scenarioes """ -import pytest +import math +import os.path as path +import tempfile + import numpy as np import pandas as pd -import tempfile +import pytest + import scripts.prepare_dutchf3 as prep_dutchf3 -import math # Setup OUTPUT = None @@ -15,8 +18,8 @@ XLINE = 1008 DEPTH = 351 ALINE = np.zeros((ILINE, XLINE, DEPTH)) -STRIDE = 100 -PATCH = 50 +STRIDE = 50 +PATCH = 100 PER_VAL = 0.2 LOG_CONFIG = None @@ -25,26 +28,26 @@ def test_get_aline_range_step_one(): """check if it includes the step in the range if step = 1 """ - SLICE_STEPS = 1 + SECTION_STRIDE = 1 # Test - output_iline = prep_dutchf3._get_aline_range(ILINE, PER_VAL, SLICE_STEPS) - output_xline = prep_dutchf3._get_aline_range(XLINE, PER_VAL, SLICE_STEPS) + output_iline = prep_dutchf3._get_aline_range(ILINE, PER_VAL, SECTION_STRIDE) + output_xline = prep_dutchf3._get_aline_range(XLINE, PER_VAL, SECTION_STRIDE) - assert str(output_iline[0].step) == str(SLICE_STEPS) - assert str(output_xline[0].step) == str(SLICE_STEPS) + assert str(output_iline[0].step) == str(SECTION_STRIDE) + assert str(output_xline[0].step) == str(SECTION_STRIDE) def test_get_aline_range_step_zero(): - """check if a ValueError exception is raised when slice_steps = 0 + """check if a ValueError exception is raised when section_stride = 0 """ - with pytest.raises(ValueError, match=r'slice_steps cannot be zero or a negative number'): - SLICE_STEPS = 0 + with pytest.raises(ValueError, match="section_stride cannot be zero or a negative number"): + SECTION_STRIDE = 0 # Test - output_iline = prep_dutchf3._get_aline_range(ILINE, PER_VAL, SLICE_STEPS) - output_xline = prep_dutchf3._get_aline_range(XLINE, PER_VAL, SLICE_STEPS) + output_iline = prep_dutchf3._get_aline_range(ILINE, PER_VAL, SECTION_STRIDE) + output_xline = prep_dutchf3._get_aline_range(XLINE, PER_VAL, SECTION_STRIDE) assert output_iline assert output_xline @@ -52,14 +55,14 @@ def test_get_aline_range_step_zero(): def test_get_aline_range_negative_step(): - """check if a ValueError exception is raised when slice_steps = -1 + """check if a ValueError exception is raised when section_stride = -1 """ - with pytest.raises(ValueError, match='slice_steps cannot be zero or a negative number'): - SLICE_STEPS = -1 + with pytest.raises(ValueError, match="section_stride cannot be zero or a negative number"): + SECTION_STRIDE = -1 # Test - output_iline = prep_dutchf3._get_aline_range(ILINE, PER_VAL, SLICE_STEPS) - output_xline = prep_dutchf3._get_aline_range(XLINE, PER_VAL, SLICE_STEPS) + output_iline = prep_dutchf3._get_aline_range(ILINE, PER_VAL, SECTION_STRIDE) + output_xline = prep_dutchf3._get_aline_range(XLINE, PER_VAL, SECTION_STRIDE) assert output_iline assert output_xline @@ -67,14 +70,14 @@ def test_get_aline_range_negative_step(): def test_get_aline_range_float_step(): - """check if a ValueError exception is raised when slice_steps = 1.1 + """check if a ValueError exception is raised when section_stride = 1.1 """ with pytest.raises(TypeError, match="'float' object cannot be interpreted as an integer"): - SLICE_STEPS = 1. + SECTION_STRIDE = 1.0 # Test - output_iline = prep_dutchf3._get_aline_range(ILINE, PER_VAL, SLICE_STEPS) - output_xline = prep_dutchf3._get_aline_range(XLINE, PER_VAL, SLICE_STEPS) + output_iline = prep_dutchf3._get_aline_range(ILINE, PER_VAL, SECTION_STRIDE) + output_xline = prep_dutchf3._get_aline_range(XLINE, PER_VAL, SECTION_STRIDE) assert output_iline assert output_xline @@ -84,235 +87,341 @@ def test_get_aline_range_single_digit_step(): """check if it includes the step in the range if 1 < step < 10 """ - SLICE_STEPS = 1 + SECTION_STRIDE = 1 # Test - output_iline = prep_dutchf3._get_aline_range(ILINE, PER_VAL, SLICE_STEPS) - output_xline = prep_dutchf3._get_aline_range(XLINE, PER_VAL, SLICE_STEPS) + output_iline = prep_dutchf3._get_aline_range(ILINE, PER_VAL, SECTION_STRIDE) + output_xline = prep_dutchf3._get_aline_range(XLINE, PER_VAL, SECTION_STRIDE) - assert str(output_iline[0].step) == str(SLICE_STEPS) - assert str(output_xline[0].step) == str(SLICE_STEPS) + assert str(output_iline[0].step) == str(SECTION_STRIDE) + assert str(output_xline[0].step) == str(SECTION_STRIDE) def test_get_aline_range_double_digit_step(): """check if it includes the step in the range if step > 10 """ - SLICE_STEPS = 17 + SECTION_STRIDE = 17 # Test - output_iline = prep_dutchf3._get_aline_range(ILINE, PER_VAL, SLICE_STEPS) - output_xline = prep_dutchf3._get_aline_range(XLINE, PER_VAL, SLICE_STEPS) + output_iline = prep_dutchf3._get_aline_range(ILINE, PER_VAL, SECTION_STRIDE) + output_xline = prep_dutchf3._get_aline_range(XLINE, PER_VAL, SECTION_STRIDE) - assert str(output_iline[0].step) == str(SLICE_STEPS) - assert str(output_xline[0].step) == str(SLICE_STEPS) + assert str(output_iline[0].step) == str(SECTION_STRIDE) + assert str(output_xline[0].step) == str(SECTION_STRIDE) def test_prepare_dutchf3_patch_step_1(): """check a complete run for the script in case further changes are needed """ - # setting a value to SLICE_STEPS as needed to test the values - SLICE_STEPS = 1 + # setting a value to SECTION_STRIDE as needed to test the values + SECTION_STRIDE = 1 + DIRECTION = "inline" # use a temp dir that will be discarded at the end of the execution with tempfile.TemporaryDirectory() as tmpdirname: # saving the file to be used by the script - label_file = tmpdirname + '/label_file.npy' + label_file = tmpdirname + "/label_file.npy" np.save(label_file, ALINE) # stting the output directory to be used by the script - output = tmpdirname + '/split' - - # calling the main function of the script without SLICE_STEPS, to check default value - prep_dutchf3.split_patch_train_val(data_dir=tmpdirname, output_dir=output, label_file=label_file, - slice_steps=SLICE_STEPS, stride=STRIDE, patch_size=PATCH, per_val=PER_VAL,log_config=LOG_CONFIG) + output = tmpdirname + "/split" + + # calling the main function of the script without SECTION_STRIDE, to check default value + train_list, val_list = prep_dutchf3.split_patch_train_val( + label_file=label_file, + section_stride=SECTION_STRIDE, + patch_stride=STRIDE, + split_direction=DIRECTION, + patch_size=PATCH, + per_val=PER_VAL, + log_config=LOG_CONFIG, + ) + prep_dutchf3._write_split_files(output, train_list, val_list, "patch") # reading the file and splitting the data - patch_train = pd.read_csv(output + '/patch_train.txt', header=None, names=['row', 'a', 'b']) - patch_train = pd.DataFrame(patch_train.row.str.split('_').tolist(), columns=['aline', 'x', 'y', 'z']) + patch_train = pd.read_csv(output + "/patch_train.txt", header=None, names=["row"]) + patch_train = pd.DataFrame(patch_train.row.str.split("_").tolist(), columns=["dir", "i", "x", "d"]) - # test patch_train and slice_steps=2 - y = list(sorted(set(patch_train.y.astype(int)))) + # test patch_train and section_stride=2 x = list(sorted(set(patch_train.x.astype(int)))) - assert (int(y[1]) - int(y[0])) == SLICE_STEPS - assert (int(x[1]) - int(x[0])) == SLICE_STEPS + i = list(sorted(set(patch_train.i.astype(int)))) + + if DIRECTION == "crossline": + assert x[1] - x[0] == SECTION_STRIDE + assert i[1] - i[0] == STRIDE + elif DIRECTION == "inline": + assert x[1] - x[0] == STRIDE + assert i[1] - i[0] == SECTION_STRIDE # reading the file and splitting the data - patch_val = pd.read_csv(output + '/patch_val.txt', header=None, names=['row', 'a', 'b']) - patch_val = pd.DataFrame(patch_val.row.str.split('_').tolist(), columns=['aline', 'x', 'y', 'z']) + patch_val = pd.read_csv(output + "/patch_val.txt", header=None, names=["row"]) + patch_val = pd.DataFrame(patch_val.row.str.split("_").tolist(), columns=["dir", "i", "x", "d"]) - # test patch_val and slice_steps=2 - y = list(sorted(set(patch_val.y.astype(int)))) + # test patch_val and section_stride=2 x = list(sorted(set(patch_val.x.astype(int)))) - assert (int(y[1]) - int(y[0])) != SLICE_STEPS - assert (int(x[1]) - int(x[0])) != SLICE_STEPS + i = list(sorted(set(patch_val.i.astype(int)))) + + if DIRECTION == "crossline": + assert x[1] - x[0] == 1 # SECTION_STRIDE is only used in training. + assert i[1] - i[0] == STRIDE + elif DIRECTION == "inline": + assert x[1] - x[0] == STRIDE + assert i[1] - i[0] == 1 # test validation set is, at least, PER_VAL - PER_VAL_CHK = len(set(patch_train.y))/(len(set(patch_train.y))+len(set(patch_val.y))) * 100 - assert round(PER_VAL_CHK,0) >= int(PER_VAL * 100) - PER_VAL_CHK = len(set(patch_train.x))/(len(set(patch_train.x))+len(set(patch_val.x))) * 100 - assert round(PER_VAL_CHK,0) >= int(PER_VAL * 100) + PER_VAL_CHK = len(set(patch_train.x)) / (len(set(patch_train.x)) + len(set(patch_val.x))) * 100 + assert round(PER_VAL_CHK, 0) >= int(PER_VAL * 100) + PER_VAL_CHK = len(set(patch_train.i)) / (len(set(patch_train.i)) + len(set(patch_val.i))) * 100 + assert round(PER_VAL_CHK, 0) >= int(PER_VAL * 100) def test_prepare_dutchf3_patch_step_2(): """check a complete run for the script in case further changes are needed """ - # setting a value to SLICE_STEPS as needed to test the values - SLICE_STEPS = 2 + # setting a value to SECTION_STRIDE as needed to test the values + SECTION_STRIDE = 2 + DIRECTION = "crossline" # use a temp dir that will be discarded at the end of the execution with tempfile.TemporaryDirectory() as tmpdirname: # saving the file to be used by the script - label_file = tmpdirname + '/label_file.npy' + label_file = tmpdirname + "/label_file.npy" np.save(label_file, ALINE) # stting the output directory to be used by the script - output = tmpdirname + '/split' - - # calling the main function of the script without SLICE_STEPS, to check default value - prep_dutchf3.split_patch_train_val(data_dir=tmpdirname, output_dir=output, label_file=label_file, - slice_steps=SLICE_STEPS, stride=STRIDE, patch_size=PATCH, per_val=PER_VAL,log_config=LOG_CONFIG) + output = tmpdirname + "/split" + + # calling the main function of the script without SECTION_STRIDE, to check default value + train_list, val_list = prep_dutchf3.split_patch_train_val( + label_file=label_file, + section_stride=SECTION_STRIDE, + patch_stride=STRIDE, + split_direction=DIRECTION, + patch_size=PATCH, + per_val=PER_VAL, + log_config=LOG_CONFIG, + ) + prep_dutchf3._write_split_files(output, train_list, val_list, "patch") # reading the file and splitting the data - patch_train = pd.read_csv(output + '/patch_train.txt', header=None, names=['row', 'a', 'b']) - patch_train = pd.DataFrame(patch_train.row.str.split('_').tolist(), columns=['aline', 'x', 'y', 'z']) + patch_train = pd.read_csv(output + "/patch_train.txt", header=None, names=["row"]) + patch_train = pd.DataFrame(patch_train.row.str.split("_").tolist(), columns=["dir", "i", "x", "d"]) - # test patch_train and slice_steps=2 - y = list(sorted(set(patch_train.y.astype(int)))) + # test patch_train and section_stride=2 x = list(sorted(set(patch_train.x.astype(int)))) - assert (int(y[1]) - int(y[0])) == SLICE_STEPS - assert (int(x[1]) - int(x[0])) == SLICE_STEPS + i = list(sorted(set(patch_train.i.astype(int)))) + + if DIRECTION == "crossline": + assert x[1] - x[0] == SECTION_STRIDE + assert i[1] - i[0] == STRIDE + elif DIRECTION == "inline": + assert x[1] - x[0] == STRIDE + assert i[1] - i[0] == SECTION_STRIDE # reading the file and splitting the data - patch_val = pd.read_csv(output + '/patch_val.txt', header=None, names=['row', 'a', 'b']) - patch_val = pd.DataFrame(patch_val.row.str.split('_').tolist(), columns=['aline', 'x', 'y', 'z']) + patch_val = pd.read_csv(output + "/patch_val.txt", header=None, names=["row"]) + patch_val = pd.DataFrame(patch_val.row.str.split("_").tolist(), columns=["dir", "i", "x", "d"]) - # test patch_val and slice_steps=2 - y = list(sorted(set(patch_val.y.astype(int)))) + # test patch_val and section_stride=2 x = list(sorted(set(patch_val.x.astype(int)))) - assert (int(y[1]) - int(y[0])) != SLICE_STEPS - assert (int(x[1]) - int(x[0])) != SLICE_STEPS + i = list(sorted(set(patch_val.i.astype(int)))) + + if DIRECTION == "crossline": + assert x[1] - x[0] == 1 # SECTION_STRIDE is only used in training. + assert i[1] - i[0] == STRIDE + elif DIRECTION == "inline": + assert x[1] - x[0] == STRIDE + assert i[1] - i[0] == 1 # test validation set is, at least, PER_VAL - PER_VAL_CHK = len(set(patch_train.y))/(len(set(patch_train.y))+len(set(patch_val.y))) * 100 - assert round(PER_VAL_CHK,0) >= int(PER_VAL * 100) - PER_VAL_CHK = len(set(patch_train.x))/(len(set(patch_train.x))+len(set(patch_val.x))) * 100 - assert round(PER_VAL_CHK,0) >= int(PER_VAL * 100) + PER_VAL_CHK = len(set(patch_train.x)) / (len(set(patch_train.x)) + len(set(patch_val.x))) * 100 + assert round(PER_VAL_CHK, 0) >= int(PER_VAL * 100) + PER_VAL_CHK = len(set(patch_train.i)) / (len(set(patch_train.i)) + len(set(patch_val.i))) * 100 + assert round(PER_VAL_CHK, 0) >= int(PER_VAL * 100) -def test_prepare_dutchf3_patch_train_and_test_sets(): +def test_prepare_dutchf3_patch_train_and_test_sets_inline(): """check a complete run for the script in case further changes are needed """ - # setting a value to SLICE_STEPS as needed to test the values - SLICE_STEPS = 1 + # setting a value to SECTION_STRIDE as needed to test the values + SECTION_STRIDE = 1 + DIRECTION = "inline" # use a temp dir that will be discarded at the end of the execution with tempfile.TemporaryDirectory() as tmpdirname: # saving the file to be used by the script - label_file = tmpdirname + '/label_file.npy' + label_file = tmpdirname + "/label_file.npy" np.save(label_file, ALINE) # stting the output directory to be used by the script - output = tmpdirname + '/split' + output = tmpdirname + "/split" + + # calling the main function of the script without SECTION_STRIDE, to check default value + train_list, val_list = prep_dutchf3.split_patch_train_val( + label_file=label_file, + section_stride=SECTION_STRIDE, + patch_stride=STRIDE, + split_direction=DIRECTION, + patch_size=PATCH, + per_val=PER_VAL, + log_config=LOG_CONFIG, + ) + prep_dutchf3._write_split_files(output, train_list, val_list, "patch") - # calling the main function of the script without SLICE_STEPS, to check default value - prep_dutchf3.split_patch_train_val(data_dir=tmpdirname, output_dir=output, label_file=label_file, - slice_steps=SLICE_STEPS, stride=STRIDE, patch_size=PATCH, per_val=PER_VAL,log_config=LOG_CONFIG) + # reading the file and splitting the data + patch_train = pd.read_csv(output + "/patch_train.txt", header=None, names=["row"]) + patch_train = patch_train.row.tolist() # reading the file and splitting the data - patch_train = pd.read_csv(output + '/patch_train.txt', header=None, names=['row', 'a', 'b']) - patch_train = pd.DataFrame(patch_train.row.str.split('_').tolist(), columns=['aline', 'x', 'y', 'z']) + patch_val = pd.read_csv(output + "/patch_val.txt", header=None, names=["row"]) + patch_val = patch_val.row.tolist() + + # assert patches are unique + assert set(patch_train) & set(patch_val) == set() + + # test validation set is, at least, PER_VAL + PER_VAL_CHK = 100 * len(patch_train) / (len(patch_train) + len(patch_val)) + assert round(PER_VAL_CHK, 0) >= int(PER_VAL * 100) + PER_VAL_CHK = 100 * len(patch_train) / (len(patch_train) + len(patch_val)) + assert round(PER_VAL_CHK, 0) >= int(PER_VAL * 100) + + +def test_prepare_dutchf3_patch_train_and_test_sets_crossline(): + + """check a complete run for the script in case further changes are needed + """ + # setting a value to SECTION_STRIDE as needed to test the values + SECTION_STRIDE = 1 + DIRECTION = "crossline" + + # use a temp dir that will be discarded at the end of the execution + with tempfile.TemporaryDirectory() as tmpdirname: + + # saving the file to be used by the script + label_file = tmpdirname + "/label_file.npy" + np.save(label_file, ALINE) + + # stting the output directory to be used by the script + output = tmpdirname + "/split" + + # calling the main function of the script without SECTION_STRIDE, to check default value + train_list, val_list = prep_dutchf3.split_patch_train_val( + label_file=label_file, + section_stride=SECTION_STRIDE, + patch_stride=STRIDE, + split_direction=DIRECTION, + patch_size=PATCH, + per_val=PER_VAL, + log_config=LOG_CONFIG, + ) + prep_dutchf3._write_split_files(output, train_list, val_list, "patch") # reading the file and splitting the data - patch_val = pd.read_csv(output + '/patch_val.txt', header=None, names=['row', 'a', 'b']) - patch_val = pd.DataFrame(patch_val.row.str.split('_').tolist(), columns=['aline', 'x', 'y', 'z']) + patch_train = pd.read_csv(output + "/patch_train.txt", header=None, names=["row"]) + patch_train = patch_train.row.tolist() - y_train = set(patch_train.y) - x_train = set(patch_train.x) - y_val = set(patch_val.y) - x_val = set(patch_val.x) + # reading the file and splitting the data + patch_val = pd.read_csv(output + "/patch_val.txt", header=None, names=["row"]) + patch_val = patch_val.row.tolist() - # The sets must not contain values common to both - assert y_train & y_val == set() - assert x_train & x_val == set() + # assert patches are unique + assert set(patch_train) & set(patch_val) == set() # test validation set is, at least, PER_VAL - PER_VAL_CHK = len(set(patch_train.y))/(len(set(patch_train.y))+len(set(patch_val.y))) * 100 - assert round(PER_VAL_CHK,0) >= int(PER_VAL * 100) - PER_VAL_CHK = len(set(patch_train.x))/(len(set(patch_train.x))+len(set(patch_val.x))) * 100 - assert round(PER_VAL_CHK,0) >= int(PER_VAL * 100) + PER_VAL_CHK = 100 * len(patch_train) / (len(patch_train) + len(patch_val)) + assert round(PER_VAL_CHK, 0) >= int(PER_VAL * 100) + PER_VAL_CHK = 100 * len(patch_train) / (len(patch_train) + len(patch_val)) + assert round(PER_VAL_CHK, 0) >= int(PER_VAL * 100) -def test_prepare_dutchf3_section_step_1(): + +def test_prepare_dutchf3_section_step_1_crossline(): """check a complete run for the script in case further changes are needed """ - # setting a value to SLICE_STEPS as needed to test the values - SLICE_STEPS = 1 + # setting a value to SECTION_STRIDE as needed to test the values + SECTION_STRIDE = 2 + DIRECTION = "crossline" # use a temp dir that will be discarded at the end of the execution with tempfile.TemporaryDirectory() as tmpdirname: # saving the file to be used by the script - label_file = tmpdirname + '/label_file.npy' + label_file = tmpdirname + "/label_file.npy" np.save(label_file, ALINE) # stting the output directory to be used by the script - output = tmpdirname + '/split' - - # calling the main function of the script without SLICE_STEPS, to check default value - prep_dutchf3.split_section_train_val(data_dir=tmpdirname, output_dir=output, label_file=label_file,slice_steps=SLICE_STEPS, per_val=PER_VAL, log_config=LOG_CONFIG) + output = tmpdirname + "/split" + + # calling the main function of the script without SECTION_STRIDE, to check default value + train_list, val_list = prep_dutchf3.split_section_train_val( + label_file=label_file, + section_stride=SECTION_STRIDE, + split_direction=DIRECTION, + per_val=PER_VAL, + log_config=LOG_CONFIG, + ) + prep_dutchf3._write_split_files(output, train_list, val_list, "section") # reading the file and splitting the data - section_train = pd.read_csv(output + '/section_train.txt', header=None, names=['row']) - section_train = pd.DataFrame(section_train.row.str.split('_').tolist(), columns=['aline', 'section']) + section_train = pd.read_csv(output + "/section_train.txt", header=None, names=["row"]) + section_train = pd.DataFrame(section_train.row.str.split("_").tolist(), columns=["dir", "section"]) - section_val = pd.read_csv(output + '/section_val.txt', header=None, names=['row']) - section_val = pd.DataFrame(section_val.row.str.split('_').tolist(), columns=['aline', 'section']) + section_val = pd.read_csv(output + "/section_val.txt", header=None, names=["row"]) + section_val = pd.DataFrame(section_val.row.str.split("_").tolist(), columns=["dir", "section"]) # test - assert (float(section_train.section[1]) - float(section_train.section[0])) % float(SLICE_STEPS) == 0.0 - assert (float(section_val.section[1]) - float(section_val.section[0])) % float(SLICE_STEPS) == 0.0 + assert (float(section_train.section[1]) - float(section_train.section[0])) % float(SECTION_STRIDE) == 0.0 + assert (float(section_val.section[1]) - float(section_val.section[0])) % float(SECTION_STRIDE) > 0.0 # test validation set is, at least, PER_VAL - PER_VAL_CHK = len(section_val)/(len(section_val)+len(section_train)) * 100 - assert round(PER_VAL_CHK,0) >= int(PER_VAL * 100) + PER_VAL_CHK = len(section_val) / (len(section_val) + len(section_train)) * 100 + assert round(PER_VAL_CHK, 0) >= int(PER_VAL * 100) -def test_prepare_dutchf3_section_step_2(): + +def test_prepare_dutchf3_section_step_2_inline(): """check a complete run for the script in case further changes are needed """ - # setting a value to SLICE_STEPS as needed to test the values - SLICE_STEPS = 2 + # setting a value to SECTION_STRIDE as needed to test the values + SECTION_STRIDE = 1 + DIRECTION = "inline" # use a temp dir that will be discarded at the end of the execution with tempfile.TemporaryDirectory() as tmpdirname: # saving the file to be used by the script - label_file = tmpdirname + '/label_file.npy' + label_file = tmpdirname + "/label_file.npy" np.save(label_file, ALINE) # stting the output directory to be used by the script - output = tmpdirname + '/split' - - # calling the main function of the script without SLICE_STEPS, to check default value - prep_dutchf3.split_section_train_val(data_dir=tmpdirname, output_dir=output, label_file=label_file, - slice_steps=SLICE_STEPS, per_val=PER_VAL, log_config=LOG_CONFIG) + output = tmpdirname + "/split" + + # calling the main function of the script without SECTION_STRIDE, to check default value + train_list, val_list = prep_dutchf3.split_section_train_val( + label_file=label_file, + section_stride=SECTION_STRIDE, + split_direction=DIRECTION, + per_val=PER_VAL, + log_config=LOG_CONFIG, + ) + prep_dutchf3._write_split_files(output, train_list, val_list, "section") # reading the file and splitting the data - section_train = pd.read_csv(output + '/section_train.txt', header=None, names=['row']) - section_train = pd.DataFrame(section_train.row.str.split('_').tolist(), columns=['aline', 'section']) + section_train = pd.read_csv(output + "/section_train.txt", header=None, names=["row"]) + section_train = pd.DataFrame(section_train.row.str.split("_").tolist(), columns=["dir", "section"]) - section_val = pd.read_csv(output + '/section_val.txt', header=None, names=['row']) - section_val = pd.DataFrame(section_val.row.str.split('_').tolist(), columns=['aline', 'section']) + section_val = pd.read_csv(output + "/section_val.txt", header=None, names=["row"]) + section_val = pd.DataFrame(section_val.row.str.split("_").tolist(), columns=["dir", "section"]) # test - assert (float(section_train.section[1]) - float(section_train.section[0])) % float(SLICE_STEPS) == 0.0 - assert (float(section_val.section[1]) - float(section_val.section[0])) % float(SLICE_STEPS) > 0.0 + assert (float(section_train.section[1]) - float(section_train.section[0])) % float(SECTION_STRIDE) == 0.0 + assert (float(section_val.section[1]) - float(section_val.section[0])) % float(SECTION_STRIDE) == 0.0 # test validation set is, at least, PER_VAL - PER_VAL_CHK = len(section_val)/(len(section_val)+len(section_train)) * 100 - assert round(PER_VAL_CHK,0) >= int(PER_VAL * 100) \ No newline at end of file + PER_VAL_CHK = len(section_val) / (len(section_val) + len(section_train)) * 100 + assert round(PER_VAL_CHK, 0) >= int(PER_VAL * 100) From 21cedd882468961d8a737d8bf4bd6d45a044e1d5 Mon Sep 17 00:00:00 2001 From: maxkazmsft Date: Wed, 29 Apr 2020 11:43:24 -0400 Subject: [PATCH 08/20] Correctness single GPU switch (#290) * resolved rebase conflict * resolved merge conflict * resolved rebase conflict * resolved merge conflict * reverted multi-GPU builds to run on single GPU --- .../penobscot/local/logging.conf | 34 ------------------- tests/cicd/main_build.yml | 33 +++++++++--------- 2 files changed, 17 insertions(+), 50 deletions(-) delete mode 100644 contrib/experiments/interpretation/penobscot/local/logging.conf diff --git a/contrib/experiments/interpretation/penobscot/local/logging.conf b/contrib/experiments/interpretation/penobscot/local/logging.conf deleted file mode 100644 index 56334fc4..00000000 --- a/contrib/experiments/interpretation/penobscot/local/logging.conf +++ /dev/null @@ -1,34 +0,0 @@ -[loggers] -keys=root,__main__,event_handlers - -[handlers] -keys=consoleHandler - -[formatters] -keys=simpleFormatter - -[logger_root] -level=INFO -handlers=consoleHandler - -[logger___main__] -level=INFO -handlers=consoleHandler -qualname=__main__ -propagate=0 - -[logger_event_handlers] -level=INFO -handlers=consoleHandler -qualname=event_handlers -propagate=0 - -[handler_consoleHandler] -class=StreamHandler -level=INFO -formatter=simpleFormatter -args=(sys.stdout,) - -[formatter_simpleFormatter] -format=%(asctime)s - %(name)s - %(levelname)s - %(message)s - diff --git a/tests/cicd/main_build.yml b/tests/cicd/main_build.yml index bfc9b026..f1a33f61 100644 --- a/tests/cicd/main_build.yml +++ b/tests/cicd/main_build.yml @@ -91,6 +91,7 @@ jobs: ################################################################################################### # Stage 3: Dutch F3 patch models: deconvnet, unet, HRNet patch depth, HRNet section depth +# CAUTION: reverted these builds to single-GPU leaving new multi-GPU code in to be reverted later ################################################################################################### - job: dutchf3_patch @@ -113,30 +114,30 @@ jobs: dir=$(mktemp -d) pids= - export CUDA_VISIBLE_DEVICES=0 + # export CUDA_VISIBLE_DEVICES=0 { python train.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \ 'TRAIN.DEPTH' 'none' \ 'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'no_depth' \ - --cfg=configs/patch_deconvnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } & + --cfg=configs/patch_deconvnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } pids+=" $!" - export CUDA_VISIBLE_DEVICES=1 + # export CUDA_VISIBLE_DEVICES=1 { python train.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \ 'TRAIN.DEPTH' 'section' \ 'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'section_depth' \ - --cfg=configs/unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } & + --cfg=configs/unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } pids+=" $!" - export CUDA_VISIBLE_DEVICES=2 + # export CUDA_VISIBLE_DEVICES=2 { python train.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \ 'TRAIN.DEPTH' 'section' \ 'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'section_depth' \ - --cfg=configs/seresnet_unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } & + --cfg=configs/seresnet_unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } pids+=" $!" - export CUDA_VISIBLE_DEVICES=3 + # export CUDA_VISIBLE_DEVICES=3 { python train.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \ 'TRAIN.DEPTH' 'section' \ 'MODEL.PRETRAINED' '/home/alfred/models/hrnetv2_w48_imagenet_pretrained.pth' \ 'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'section_depth' \ - --cfg=configs/hrnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } & + --cfg=configs/hrnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } pids+=" $!" wait $pids || exit 1 @@ -157,16 +158,16 @@ jobs: dir=$(mktemp -d) pids= - export CUDA_VISIBLE_DEVICES=0 + # export CUDA_VISIBLE_DEVICES=0 # find the latest model which we just trained model_dir=$(ls -td output/patch_deconvnet/no_depth/* | head -1) model=$(ls -t ${model_dir}/*.pth | head -1) # try running the test script { python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' \ 'TEST.MODEL_PATH' ${model} \ - --cfg=configs/patch_deconvnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } & + --cfg=configs/patch_deconvnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } pids+=" $!" - export CUDA_VISIBLE_DEVICES=1 + # export CUDA_VISIBLE_DEVICES=1 # find the latest model which we just trained model_dir=$(ls -td output/unet/section_depth/* | head -1) model=$(ls -t ${model_dir}/*.pth | head -1) @@ -174,9 +175,9 @@ jobs: # try running the test script { python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' \ 'TEST.MODEL_PATH' ${model} \ - --cfg=configs/unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } & + --cfg=configs/unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } pids+=" $!" - export CUDA_VISIBLE_DEVICES=2 + # export CUDA_VISIBLE_DEVICES=2 # find the latest model which we just trained model_dir=$(ls -td output/seresnet_unet/section_depth/* | head -1) model=$(ls -t ${model_dir}/*.pth | head -1) @@ -184,9 +185,9 @@ jobs: # try running the test script { python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' \ 'TEST.MODEL_PATH' ${model} \ - --cfg=configs/seresnet_unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } & + --cfg=configs/seresnet_unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } pids+=" $!" - export CUDA_VISIBLE_DEVICES=3 + # export CUDA_VISIBLE_DEVICES=3 # find the latest model which we just trained model_dir=$(ls -td output/hrnet/section_depth/* | head -1) model=$(ls -t ${model_dir}/*.pth | head -1) @@ -195,7 +196,7 @@ jobs: { python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' \ 'MODEL.PRETRAINED' '/home/alfred/models/hrnetv2_w48_imagenet_pretrained.pth' \ 'TEST.MODEL_PATH' ${model} \ - --cfg=configs/hrnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } & + --cfg=configs/hrnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } pids+=" $!" # wait for completion From 13c01e91808a92362b7401f5a245bfbc6f37b26f Mon Sep 17 00:00:00 2001 From: maxkazmsft Date: Wed, 29 Apr 2020 17:28:20 -0400 Subject: [PATCH 09/20] 249r3 (#283) * resolved rebase conflict * resolved merge conflict * resolved rebase conflict * resolved merge conflict * wrote the bulk of checkerboard example * finished checkerboard generator * resolved merge conflict * resolved rebase conflict * got binary dataset to run * finished first implementation mockup - commit before rebase * made sure rebase went well manually * added new files * resolved PR comments and made tests work * fixed build error * fixed build VM errors * more fixes to get the test to pass * fixed n_classes issue in data.py * fixed notebook as well * cleared notebook run cell * trivial commit to restart builds * addressed PR comments * moved notebook tests to main build pipeline * fixed checkerboard label precision * relaxed performance tests for now * resolved merge conflict * resolved merge conflict * fixed build error * resolved merge conflicts * fixed another merge mistake --- README.md | 1 + conftest.py | 0 .../dutchf3_patch/distributed/train.py | 19 +- .../penobscot/local/logging.conf | 34 +++ .../cv_lib/event_handlers/logging_handlers.py | 10 +- ..._patch_model_training_and_evaluation.ipynb | 6 +- .../dutchf3_patch/local/test.py | 49 +++- .../dutchf3_patch/local/train.py | 16 +- .../dutchf3/data.py | 93 ++++++-- scripts/gen_checkerboard.py | 197 ++++++++++++++++ tests/cicd/main_build.yml | 217 ++++++++++++++++-- tests/cicd/notebooks_build.yml | 59 ----- tests/cicd/src/check_performance.py | 97 ++++++++ tests/cicd/src/scripts/get_data_for_builds.sh | 18 +- 14 files changed, 674 insertions(+), 142 deletions(-) create mode 100644 conftest.py create mode 100644 contrib/experiments/interpretation/penobscot/local/logging.conf create mode 100644 scripts/gen_checkerboard.py delete mode 100644 tests/cicd/notebooks_build.yml create mode 100644 tests/cicd/src/check_performance.py diff --git a/README.md b/README.md index a4559c20..81f6920f 100644 --- a/README.md +++ b/README.md @@ -419,3 +419,4 @@ which will indicate that anaconda folder is `__/anaconda__`. We'll refer to this 5. Navigate back to the Virtual Machine view in Step 2 and click the Start button to start the virtual machine.
+ diff --git a/conftest.py b/conftest.py new file mode 100644 index 00000000..e69de29b diff --git a/contrib/experiments/interpretation/dutchf3_patch/distributed/train.py b/contrib/experiments/interpretation/dutchf3_patch/distributed/train.py index e52970ed..7f3a136e 100644 --- a/contrib/experiments/interpretation/dutchf3_patch/distributed/train.py +++ b/contrib/experiments/interpretation/dutchf3_patch/distributed/train.py @@ -139,7 +139,7 @@ def run(*options, cfg=None, local_rank=0, debug=False): stride=config.TRAIN.STRIDE, patch_size=config.TRAIN.PATCH_SIZE, augmentations=train_aug, - ) + ) val_set = TrainPatchLoader( config.DATASET.ROOT, @@ -150,21 +150,18 @@ def run(*options, cfg=None, local_rank=0, debug=False): augmentations=val_aug, ) - if debug: - val_set = data.Subset(val_set, range(3)) - logger.info(f"Validation examples {len(val_set)}") n_classes = train_set.n_classes if debug: - logger.info("Running in debug mode..") - train_set = data.Subset(train_set, list(range(4))) - val_set = data.Subset(val_set, list(range(4))) - + val_set = data.Subset(val_set, range(config.VALIDATION.BATCH_SIZE_PER_GPU)) + train_set = data.Subset(train_set, range(config.TRAIN.BATCH_SIZE_PER_GPU*2)) + logger.info(f"Training examples {len(train_set)}") - logger.info(f"Validation examples {len(val_set)}") + logger.info(f"Validation examples {len(val_set)}") train_sampler = torch.utils.data.distributed.DistributedSampler(train_set, num_replicas=world_size, rank=local_rank) + train_loader = data.DataLoader( train_set, batch_size=config.TRAIN.BATCH_SIZE_PER_GPU, num_workers=config.WORKERS, sampler=train_sampler, ) @@ -197,8 +194,10 @@ def run(*options, cfg=None, local_rank=0, debug=False): model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[device], find_unused_parameters=True) - snapshot_duration = epochs_per_cycle * len(train_loader) if not debug else 2 * len(train_loader) + snapshot_duration = epochs_per_cycle * len(train_loader) if not debug else 2*len(train_loader) + warmup_duration = 5 * len(train_loader) + warmup_scheduler = LinearCyclicalScheduler( optimizer, "lr", diff --git a/contrib/experiments/interpretation/penobscot/local/logging.conf b/contrib/experiments/interpretation/penobscot/local/logging.conf new file mode 100644 index 00000000..56334fc4 --- /dev/null +++ b/contrib/experiments/interpretation/penobscot/local/logging.conf @@ -0,0 +1,34 @@ +[loggers] +keys=root,__main__,event_handlers + +[handlers] +keys=consoleHandler + +[formatters] +keys=simpleFormatter + +[logger_root] +level=INFO +handlers=consoleHandler + +[logger___main__] +level=INFO +handlers=consoleHandler +qualname=__main__ +propagate=0 + +[logger_event_handlers] +level=INFO +handlers=consoleHandler +qualname=event_handlers +propagate=0 + +[handler_consoleHandler] +class=StreamHandler +level=INFO +formatter=simpleFormatter +args=(sys.stdout,) + +[formatter_simpleFormatter] +format=%(asctime)s - %(name)s - %(levelname)s - %(message)s + diff --git a/cv_lib/cv_lib/event_handlers/logging_handlers.py b/cv_lib/cv_lib/event_handlers/logging_handlers.py index ea883a36..97f382bc 100644 --- a/cv_lib/cv_lib/event_handlers/logging_handlers.py +++ b/cv_lib/cv_lib/event_handlers/logging_handlers.py @@ -1,6 +1,6 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. - +import json import logging import logging.config from toolz import curry @@ -26,10 +26,14 @@ def log_lr(optimizer, engine): @curry -def log_metrics(log_msg, engine, metrics_dict={"pixacc": "Avg accuracy :", "nll": "Avg loss :"}): +def log_metrics(log_msg, engine, metrics_dict={"pixacc": "Avg accuracy :", "nll": "Avg loss :"}, fname=None): logger = logging.getLogger(__name__) metrics = engine.state.metrics - metrics_msg = " ".join([f"{metrics_dict[k]} {metrics[k]:.2f}" for k in metrics_dict]) + metrics_msg = " ".join([f"{metrics_dict[k]} {metrics[k]:.4f}" for k in metrics_dict]) + if fname: + with open(fname, "w") as fid: + output_dict = {metrics_dict[k]: float(metrics[k]) for k in metrics_dict} + json.dump(output_dict, fid) logger.info(f"{log_msg} - Epoch {engine.state.epoch} [{engine.state.max_epochs}] " + metrics_msg) diff --git a/examples/interpretation/notebooks/Dutch_F3_patch_model_training_and_evaluation.ipynb b/examples/interpretation/notebooks/Dutch_F3_patch_model_training_and_evaluation.ipynb index c11014c5..1748e45c 100644 --- a/examples/interpretation/notebooks/Dutch_F3_patch_model_training_and_evaluation.ipynb +++ b/examples/interpretation/notebooks/Dutch_F3_patch_model_training_and_evaluation.ipynb @@ -517,6 +517,7 @@ "\n", "train_set = TrainPatchLoader(\n", " config.DATASET.ROOT,\n", + " config.DATASET.NUM_CLASSES,\n", " split=\"train\",\n", " is_transform=True,\n", " stride=config.TRAIN.STRIDE,\n", @@ -527,6 +528,7 @@ "logger.info(train_set)\n", "val_set = TrainPatchLoader(\n", " config.DATASET.ROOT,\n", + " config.DATASET.NUM_CLASSES,\n", " split=\"val\",\n", " is_transform=True,\n", " stride=config.TRAIN.STRIDE,\n", @@ -551,7 +553,7 @@ " val_set,\n", " batch_size=config.VALIDATION.BATCH_SIZE_PER_GPU,\n", " num_workers=config.WORKERS,\n", - ")\n" + ")" ] }, { @@ -1004,7 +1006,7 @@ "# Load test data\n", "TestSectionLoader = get_test_loader(config)\n", "test_set = TestSectionLoader(\n", - " config.DATASET.ROOT, split=split, is_transform=True, augmentations=section_aug\n", + " config.DATASET.ROOT, config.DATASET.NUM_CLASSES, split=split, is_transform=True, augmentations=section_aug\n", ")\n", "# needed to fix this bug in pytorch https://github.com/pytorch/pytorch/issues/973\n", "# one of the workers will quit prematurely\n", diff --git a/experiments/interpretation/dutchf3_patch/local/test.py b/experiments/interpretation/dutchf3_patch/local/test.py index f56162cb..4162d0d4 100644 --- a/experiments/interpretation/dutchf3_patch/local/test.py +++ b/experiments/interpretation/dutchf3_patch/local/test.py @@ -13,6 +13,7 @@ """ import itertools +import json import logging import logging.config import os @@ -268,7 +269,13 @@ def _evaluate_split( logger = logging.getLogger(__name__) TestSectionLoader = get_test_loader(config) - test_set = TestSectionLoader(config.DATASET.ROOT, split=split, is_transform=True, augmentations=section_aug,) + test_set = TestSectionLoader( + config.DATASET.ROOT, + config.DATASET.NUM_CLASSES, + split=split, + is_transform=True, + augmentations=section_aug + ) n_classes = test_set.n_classes @@ -276,7 +283,7 @@ def _evaluate_split( if debug: logger.info("Running in Debug/Test mode") - test_loader = take(1, test_loader) + test_loader = take(2, test_loader) try: output_dir = generate_path( @@ -321,8 +328,12 @@ def _evaluate_split( # Log split results logger.info(f'Pixel Acc: {score["Pixel Acc: "]:.3f}') - for cdx, class_name in enumerate(_CLASS_NAMES): - logger.info(f' {class_name}_accuracy {score["Class Accuracy: "][cdx]:.3f}') + if debug: + for cdx in range(n_classes): + logger.info(f' Class_{cdx}_accuracy {score["Class Accuracy: "][cdx]:.3f}') + else: + for cdx, class_name in enumerate(_CLASS_NAMES): + logger.info(f' {class_name}_accuracy {score["Class Accuracy: "][cdx]:.3f}') logger.info(f'Mean Class Acc: {score["Mean Class Acc: "]:.3f}') logger.info(f'Freq Weighted IoU: {score["Freq Weighted IoU: "]:.3f}') @@ -414,17 +425,35 @@ def test(*options, cfg=None, debug=False): score, class_iou = running_metrics_overall.get_scores() logger.info("--------------- FINAL RESULTS -----------------") - logger.info(f'Pixel Acc: {score["Pixel Acc: "]:.3f}') - for cdx, class_name in enumerate(_CLASS_NAMES): - logger.info(f' {class_name}_accuracy {score["Class Accuracy: "][cdx]:.3f}') - logger.info(f'Mean Class Acc: {score["Mean Class Acc: "]:.3f}') - logger.info(f'Freq Weighted IoU: {score["Freq Weighted IoU: "]:.3f}') - logger.info(f'Mean IoU: {score["Mean IoU: "]:0.3f}') + logger.info(f'Pixel Acc: {score["Pixel Acc: "]:.4f}') + + if debug: + for cdx in range(n_classes): + logger.info(f' Class_{cdx}_accuracy {score["Class Accuracy: "][cdx]:.3f}') + else: + for cdx, class_name in enumerate(_CLASS_NAMES): + logger.info(f' {class_name}_accuracy {score["Class Accuracy: "][cdx]:.4f}') + + logger.info(f'Mean Class Acc: {score["Mean Class Acc: "]:.4f}') + logger.info(f'Freq Weighted IoU: {score["Freq Weighted IoU: "]:.4f}') + logger.info(f'Mean IoU: {score["Mean IoU: "]:0.4f}') # Save confusion matrix: confusion = score["confusion_matrix"] np.savetxt(path.join(log_dir, "confusion.csv"), confusion, delimiter=" ") + if debug: + config_file_name = "default_config" if not cfg else cfg.split("/")[-1].split(".")[0] + fname = f"metrics_test_{config_file_name}_{config.TRAIN.MODEL_DIR}.json" + with open(fname, "w") as fid: + json.dump( + { + metric: score[metric] + for metric in ["Pixel Acc: ", "Mean Class Acc: ", "Freq Weighted IoU: ", "Mean IoU: "] + }, + fid, + ) + if __name__ == "__main__": fire.Fire(test) diff --git a/experiments/interpretation/dutchf3_patch/local/train.py b/experiments/interpretation/dutchf3_patch/local/train.py index bb51b1d9..9e26f7a2 100644 --- a/experiments/interpretation/dutchf3_patch/local/train.py +++ b/experiments/interpretation/dutchf3_patch/local/train.py @@ -59,11 +59,12 @@ def run(*options, cfg=None, debug=False): *options (str,int ,optional): Options used to overide what is loaded from the config. To see what options are available consult default.py - cfg (str, optional): Location of config file to load. Defaults to None. + cfg (str, optional): Location of config file to load. Defaults to None. debug (bool): Places scripts in debug/test mode and only executes a few iterations """ # Configuration: - update_config(config, options=options, config_file=cfg) + update_config(config, options=options, config_file=cfg) + # The model will be saved under: outputs// config_file_name = "default_config" if not cfg else cfg.split("/")[-1].split(".")[0] try: @@ -71,7 +72,7 @@ def run(*options, cfg=None, debug=False): config.OUTPUT_DIR, git_branch(), git_hash(), config_file_name, config.TRAIN.MODEL_DIR, current_datetime(), ) except TypeError: - output_dir = generate_path(config.OUTPUT_DIR, config_file_name, config.TRAIN.MODEL_DIR, current_datetime(),) + output_dir = generate_path(config.OUTPUT_DIR, config_file_name, config.TRAIN.MODEL_DIR, current_datetime(),) # Logging: load_log_configuration(config.LOG_CONFIG) @@ -81,6 +82,9 @@ def run(*options, cfg=None, debug=False): # Set CUDNN benchmark mode: torch.backends.cudnn.benchmark = config.CUDNN.BENCHMARK + # we will write the model under outputs / config_file_name / model_dir + config_file_name = "default_config" if not cfg else cfg.split("/")[-1].split(".")[0] + # Fix random seeds: torch.manual_seed(config.SEED) if torch.cuda.is_available(): @@ -121,16 +125,18 @@ def run(*options, cfg=None, debug=False): TrainPatchLoader = get_patch_loader(config) train_set = TrainPatchLoader( config.DATASET.ROOT, + config.DATASET.NUM_CLASSES, split="train", is_transform=True, stride=config.TRAIN.STRIDE, patch_size=config.TRAIN.PATCH_SIZE, - augmentations=train_aug, + augmentations=train_aug ) logger.info(train_set) n_classes = train_set.n_classes val_set = TrainPatchLoader( config.DATASET.ROOT, + config.DATASET.NUM_CLASSES, split="val", is_transform=True, stride=config.TRAIN.STRIDE, @@ -201,6 +207,7 @@ def run(*options, cfg=None, debug=False): ) trainer.add_event_handler(Events.EPOCH_COMPLETED, logging_handlers.log_lr(optimizer)) + fname = f"metrics_{config_file_name}_{config.TRAIN.MODEL_DIR}.json" if debug else None evaluator.add_event_handler( Events.EPOCH_COMPLETED, logging_handlers.log_metrics( @@ -211,6 +218,7 @@ def run(*options, cfg=None, debug=False): "mca": "Avg Class Accuracy :", "mIoU": "Avg Class IoU :", }, + fname=fname, ), ) diff --git a/interpretation/deepseismic_interpretation/dutchf3/data.py b/interpretation/deepseismic_interpretation/dutchf3/data.py index c76b29ef..01f42f14 100644 --- a/interpretation/deepseismic_interpretation/dutchf3/data.py +++ b/interpretation/deepseismic_interpretation/dutchf3/data.py @@ -17,10 +17,7 @@ from deepseismic_interpretation.dutchf3.utils.batch import ( interpolate_to_fit_data, parse_labels_in_image, - get_coordinates_for_slice, - get_grid, - rand_int, - trilinear_interpolation, + get_coordinates_for_slice ) @@ -117,21 +114,25 @@ class SectionLoader(data.Dataset): """ Base class for section data loader :param str data_dir: Root directory for training/test data + :param str n_classes: number of segmentation mask classes :param str split: split file to use for loading patches :param bool is_transform: Transform patch to dimensions expected by PyTorch :param list augmentations: Data augmentations to apply to patches - :param str seismic_path: Override file path for seismic data - :param str label_path: Override file path for label data """ def __init__( - self, data_dir, split="train", is_transform=True, augmentations=None, seismic_path=None, label_path=None + self, + data_dir, + n_classes, + split="train", + is_transform=True, + augmentations=None ): self.split = split self.data_dir = data_dir self.is_transform = is_transform self.augmentations = augmentations - self.n_classes = 6 + self.n_classes = n_classes self.sections = list() def __len__(self): @@ -172,6 +173,7 @@ class TrainSectionLoader(SectionLoader): """ Training data loader for sections :param str data_dir: Root directory for training/test data + :param str n_classes: number of segmentation mask classes :param str split: split file to use for loading patches :param bool is_transform: Transform patch to dimensions expected by PyTorch :param list augmentations: Data augmentations to apply to patches @@ -180,10 +182,18 @@ class TrainSectionLoader(SectionLoader): """ def __init__( - self, data_dir, split="train", is_transform=True, augmentations=None, seismic_path=None, label_path=None + self, + data_dir, + n_classes, + split="train", + is_transform=True, + augmentations=None, + seismic_path=None, + label_path=None, ): super(TrainSectionLoader, self).__init__( data_dir, + n_classes, split=split, is_transform=is_transform, augmentations=augmentations, @@ -215,6 +225,7 @@ class TrainSectionLoaderWithDepth(TrainSectionLoader): """ Section data loader that includes additional channel for depth :param str data_dir: Root directory for training/test data + :param str n_classes: number of segmentation mask classes :param str split: split file to use for loading patches :param bool is_transform: Transform patch to dimensions expected by PyTorch :param list augmentations: Data augmentations to apply to patches @@ -223,10 +234,18 @@ class TrainSectionLoaderWithDepth(TrainSectionLoader): """ def __init__( - self, data_dir, split="train", is_transform=True, augmentations=None, seismic_path=None, label_path=None + self, + data_dir, + n_classes, + split="train", + is_transform=True, + augmentations=None, + seismic_path=None, + label_path=None, ): super(TrainSectionLoaderWithDepth, self).__init__( data_dir, + n_classes, split=split, is_transform=is_transform, augmentations=augmentations, @@ -267,6 +286,7 @@ class TestSectionLoader(SectionLoader): """ Test data loader for sections :param str data_dir: Root directory for training/test data + :param str n_classes: number of segmentation mask classes :param str split: split file to use for loading patches :param bool is_transform: Transform patch to dimensions expected by PyTorch :param list augmentations: Data augmentations to apply to patches @@ -275,10 +295,17 @@ class TestSectionLoader(SectionLoader): """ def __init__( - self, data_dir, split="test1", is_transform=True, augmentations=None, seismic_path=None, label_path=None + self, + data_dir, + n_classes, + split="test1", + is_transform=True, + augmentations=None, + seismic_path=None, + label_path=None, ): super(TestSectionLoader, self).__init__( - data_dir, split=split, is_transform=is_transform, augmentations=augmentations, + data_dir, n_classes, split=split, is_transform=is_transform, augmentations=augmentations ) if "test1" in self.split: @@ -309,6 +336,7 @@ class TestSectionLoaderWithDepth(TestSectionLoader): """ Test data loader for sections that includes additional channel for depth :param str data_dir: Root directory for training/test data + :param str n_classes: number of segmentation mask classes :param str split: split file to use for loading patches :param bool is_transform: Transform patch to dimensions expected by PyTorch :param list augmentations: Data augmentations to apply to patches @@ -317,10 +345,18 @@ class TestSectionLoaderWithDepth(TestSectionLoader): """ def __init__( - self, data_dir, split="test1", is_transform=True, augmentations=None, seismic_path=None, label_path=None + self, + data_dir, + n_classes, + split="test1", + is_transform=True, + augmentations=None, + seismic_path=None, + label_path=None, ): super(TestSectionLoaderWithDepth, self).__init__( data_dir, + n_classes, split=split, is_transform=is_transform, augmentations=augmentations, @@ -366,6 +402,7 @@ class PatchLoader(data.Dataset): """ Base Data loader for the patch-based deconvnet :param str data_dir: Root directory for training/test data + :param str n_classes: number of segmentation mask classes :param int stride: training data stride :param int patch_size: Size of patch for training :param str split: split file to use for loading patches @@ -378,17 +415,16 @@ class PatchLoader(data.Dataset): def __init__( self, data_dir, + n_classes, stride=30, patch_size=99, is_transform=True, - augmentations=None, - seismic_path=None, - label_path=None, + augmentations=None ): self.data_dir = data_dir self.is_transform = is_transform self.augmentations = augmentations - self.n_classes = 6 + self.n_classes = n_classes self.patches = list() self.patch_size = patch_size self.stride = stride @@ -443,15 +479,21 @@ class TestPatchLoader(PatchLoader): """ Test Data loader for the patch-based deconvnet :param str data_dir: Root directory for training/test data + :param str n_classes: number of segmentation mask classes :param int stride: training data stride :param int patch_size: Size of patch for training :param bool is_transform: Transform patch to dimensions expected by PyTorch :param list augmentations: Data augmentations to apply to patches """ - def __init__(self, data_dir, stride=30, patch_size=99, is_transform=True, augmentations=None, txt_path=None): + def __init__(self, data_dir, n_classes, stride=30, patch_size=99, is_transform=True, augmentations=None): super(TestPatchLoader, self).__init__( - data_dir, stride=stride, patch_size=patch_size, is_transform=is_transform, augmentations=augmentations, + data_dir, + n_classes, + stride=stride, + patch_size=patch_size, + is_transform=is_transform, + augmentations=augmentations, ) ## Warning: this is not used or tested raise NotImplementedError("This class is not correctly implemented.") @@ -487,6 +529,7 @@ class TrainPatchLoader(PatchLoader): def __init__( self, data_dir, + n_classes, split="train", stride=30, patch_size=99, @@ -497,15 +540,13 @@ def __init__( ): super(TrainPatchLoader, self).__init__( data_dir, + n_classes, stride=stride, patch_size=patch_size, is_transform=is_transform, - augmentations=augmentations, - seismic_path=seismic_path, - label_path=label_path, + augmentations=augmentations ) - # self.seismic = self.pad_volume(np.load(seismic_path)) - # self.labels = self.pad_volume(np.load(labels_path)) + warnings.warn("This no longer pads the volume") if seismic_path is not None and label_path is not None: # Load npy files (seismc and corresponding labels) from provided @@ -621,6 +662,7 @@ class TrainPatchLoaderWithSectionDepth(TrainPatchLoader): def __init__( self, data_dir, + n_classes, split="train", stride=30, patch_size=99, @@ -631,6 +673,7 @@ def __init__( ): super(TrainPatchLoaderWithSectionDepth, self).__init__( data_dir, + n_classes, split=split, stride=stride, patch_size=patch_size, @@ -762,7 +805,7 @@ def get_seismic_labels(): @curry -def decode_segmap(label_mask, n_classes, label_colours=get_seismic_labels()): +def decode_segmap(label_mask, n_classes=None, label_colours=get_seismic_labels()): """Decode segmentation class labels into a colour image Args: label_mask (np.ndarray): an (N,H,W) array of integer values denoting diff --git a/scripts/gen_checkerboard.py b/scripts/gen_checkerboard.py new file mode 100644 index 00000000..3e0d0349 --- /dev/null +++ b/scripts/gen_checkerboard.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 +""" Please see the def main() function for code description.""" + +""" libraries """ + +import numpy as np +import sys +import os + +np.set_printoptions(linewidth=200) +import logging + +# toggle to WARNING when running in production, or use CLI +logging.getLogger().setLevel(logging.DEBUG) +# logging.getLogger().setLevel(logging.WARNING) +import argparse + +parser = argparse.ArgumentParser() + +""" useful information when running from a GIT folder.""" +myname = os.path.realpath(__file__) +mypath = os.path.dirname(myname) +myname = os.path.basename(myname) + + +def make_box(n_inlines, n_crosslines, n_depth, box_size): + """ + Makes a 3D box in checkerboard pattern. + + :param n_inlines: dim x + :param n_crosslines: dim y + :param n_depth: dim z + :return: numpy array + """ + # inline by crossline by depth + zero_patch = np.ones((box_size, box_size)) * WHITE + one_patch = np.ones((box_size, box_size)) * BLACK + + stride = np.hstack((zero_patch, one_patch)) + + # come up with a 2D inline image + nx, ny = stride.shape + + step_col = int(np.ceil(n_crosslines / float(ny))) + step_row = int(np.ceil(n_inlines / float(nx) / 2)) + + # move in the crossline direction + crossline_band = np.hstack((stride,) * step_col) + # multiplying by negative one flips the sign + neg_crossline_band = -1 * crossline_band + + checker_stripe = np.vstack((crossline_band, neg_crossline_band)) + + # move down a section + checker_image = np.vstack((checker_stripe,) * step_row) + + # trim excess + checker_image = checker_image[0:n_inlines, 0:n_crosslines] + + # now make a box with alternating checkers + checker_box = np.ones((n_inlines, n_crosslines, box_size * 2)) + checker_box[:, :, 0:box_size] = checker_image[:, :, np.newaxis] + # now invert the colors + checker_box[:, :, box_size:] = -1 * checker_image[:, :, np.newaxis] + + # stack boxes depth wise + step_depth = int(np.ceil(n_depth / float(box_size) / 2)) + final_box = np.concatenate((checker_box,) * step_depth, axis=2) + + # trim excess again + return final_box[0:n_inlines, 0:n_crosslines, 0:n_depth] + + +def mkdir(path): + """ + Create a directory helper function + """ + if not os.path.isdir(path): + os.mkdir(path) + + +def main(args): + """ + + Generates checkerboard dataset based on Dutch F3 in Alaudah format. + + Pre-requisite: valid Dutch F3 dataset in Alaudah format. + + """ + + logging.info("loading data") + + train_seismic = np.load(os.path.join(args.dataroot, "train", "train_seismic.npy")) + train_labels = np.load(os.path.join(args.dataroot, "train", "train_labels.npy")) + test1_seismic = np.load(os.path.join(args.dataroot, "test_once", "test1_seismic.npy")) + test1_labels = np.load(os.path.join(args.dataroot, "test_once", "test1_labels.npy")) + test2_seismic = np.load(os.path.join(args.dataroot, "test_once", "test2_seismic.npy")) + test2_labels = np.load(os.path.join(args.dataroot, "test_once", "test2_labels.npy")) + + assert train_seismic.shape == train_labels.shape + assert train_seismic.min() == WHITE + assert train_seismic.max() == BLACK + assert train_labels.min() == 0 + # this is the number of classes in Alaudah's Dutch F3 dataset + assert train_labels.max() == 5 + + assert test1_seismic.shape == test1_labels.shape + assert test1_seismic.min() == WHITE + assert test1_seismic.max() == BLACK + assert test1_labels.min() == 0 + # this is the number of classes in Alaudah's Dutch F3 dataset + assert test1_labels.max() == 5 + + assert test2_seismic.shape == test2_labels.shape + assert test2_seismic.min() == WHITE + assert test2_seismic.max() == BLACK + assert test2_labels.min() == 0 + # this is the number of classes in Alaudah's Dutch F3 dataset + assert test2_labels.max() == 5 + + logging.info("train checkerbox") + n_inlines, n_crosslines, n_depth = train_seismic.shape + checkerboard_train_seismic = make_box(n_inlines, n_crosslines, n_depth, args.box_size) + checkerboard_train_seismic = checkerboard_train_seismic.astype(train_seismic.dtype) + checkerboard_train_labels = checkerboard_train_seismic.astype(train_labels.dtype) + # labels are integers and start from zero + checkerboard_train_labels[checkerboard_train_seismic < WHITE_LABEL] = WHITE_LABEL + + # create checkerbox + logging.info("test1 checkerbox") + n_inlines, n_crosslines, n_depth = test1_seismic.shape + checkerboard_test1_seismic = make_box(n_inlines, n_crosslines, n_depth, args.box_size) + checkerboard_test1_seismic = checkerboard_test1_seismic.astype(test1_seismic.dtype) + checkerboard_test1_labels = checkerboard_test1_seismic.astype(test1_labels.dtype) + # labels are integers and start from zero + checkerboard_test1_labels[checkerboard_test1_seismic < WHITE_LABEL] = WHITE_LABEL + + logging.info("test2 checkerbox") + n_inlines, n_crosslines, n_depth = test2_seismic.shape + checkerboard_test2_seismic = make_box(n_inlines, n_crosslines, n_depth, args.box_size) + checkerboard_test2_seismic = checkerboard_test2_seismic.astype(test2_seismic.dtype) + checkerboard_test2_labels = checkerboard_test2_seismic.astype(test2_labels.dtype) + # labels are integers and start from zero + checkerboard_test2_labels[checkerboard_test2_seismic < WHITE_LABEL] = WHITE_LABEL + + logging.info("writing data to disk") + mkdir(args.dataout) + mkdir(os.path.join(args.dataout, "data")) + mkdir(os.path.join(args.dataout, "data", "splits")) + mkdir(os.path.join(args.dataout, "data", "train")) + mkdir(os.path.join(args.dataout, "data", "test_once")) + + np.save(os.path.join(args.dataout, "data", "train", "train_seismic.npy"), checkerboard_train_seismic) + np.save(os.path.join(args.dataout, "data", "train", "train_labels.npy"), checkerboard_train_labels) + + np.save(os.path.join(args.dataout, "data", "test_once", "test1_seismic.npy"), checkerboard_test1_seismic) + np.save(os.path.join(args.dataout, "data", "test_once", "test1_labels.npy"), checkerboard_test1_labels) + + np.save(os.path.join(args.dataout, "data", "test_once", "test2_seismic.npy"), checkerboard_test2_seismic) + np.save(os.path.join(args.dataout, "data", "test_once", "test2_labels.npy"), checkerboard_test2_labels) + + logging.info("all done") + + +""" GLOBAL VARIABLES """ +WHITE = -1 +BLACK = 1 +WHITE_LABEL = 0 + +parser.add_argument("--dataroot", help="Root location of the input data", type=str, required=True) +parser.add_argument("--dataout", help="Root location of the output data", type=str, required=True) +parser.add_argument("--box_size", help="Size of the bounding box", type=int, required=False, default=100) +parser.add_argument("--debug", help="Turn on debug mode", type=bool, required=False, default=False) + +""" main wrapper with profiler """ +if __name__ == "__main__": + main(parser.parse_args()) + +# pretty printing of the stack +""" + try: + logging.info('before main') + main(parser.parse_args()) + logging.info('after main') + except: + for frame in traceback.extract_tb(sys.exc_info()[2]): + fname,lineno,fn,text = frame + print ("Error in %s on line %d" % (fname, lineno)) +""" +# optionally enable profiling information +# import cProfile +# name = +# cProfile.run('main.run()', name + '.prof') +# import pstats +# p = pstats.Stats(name + '.prof') +# p.sort_stats('cumulative').print_stats(10) +# p.sort_stats('time').print_stats() diff --git a/tests/cicd/main_build.yml b/tests/cicd/main_build.yml index f1a33f61..875c1c05 100644 --- a/tests/cicd/main_build.yml +++ b/tests/cicd/main_build.yml @@ -88,6 +88,151 @@ jobs: pytest --durations=0 cv_lib/tests/ echo "cv_lib unit test job passed" +################################################################################################### +# Stage 3: Dutch F3 patch models on checkerboard test set: +# deconvnet, unet, HRNet patch depth, HRNet section depth +# CAUTION: reverted these builds to single-GPU leaving new multi-GPU code in to be reverted later +################################################################################################### + +- job: checkerboard_dutchf3_patch + dependsOn: cv_lib_unit_tests_job + timeoutInMinutes: 20 + displayName: Checkerboard Dutch F3 patch local + pool: + name: deepseismicagentpool + steps: + - bash: | + + source activate seismic-interpretation + + # disable auto error handling as we flag it manually + set +e + + cd experiments/interpretation/dutchf3_patch/local + + # Create a temporary directory to store the statuses + dir=$(mktemp -d) + + pids= + # export CUDA_VISIBLE_DEVICES=0 + { python train.py 'DATASET.ROOT' '/home/alfred/data_dynamic/checkerboard/data' 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \ + 'DATASET.NUM_CLASSES' 2 'DATASET.CLASS_WEIGHTS' '[1.0, 1.0]' \ + 'TRAIN.DEPTH' 'none' \ + 'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'no_depth' \ + --cfg=configs/patch_deconvnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } + pids+=" $!" + # export CUDA_VISIBLE_DEVICES=1 + { python train.py 'DATASET.ROOT' '/home/alfred/data_dynamic/checkerboard/data' 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \ + 'DATASET.NUM_CLASSES' 2 'DATASET.CLASS_WEIGHTS' '[1.0, 1.0]' \ + 'TRAIN.DEPTH' 'section' \ + 'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'section_depth' \ + --cfg=configs/unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } + pids+=" $!" + # export CUDA_VISIBLE_DEVICES=2 + { python train.py 'DATASET.ROOT' '/home/alfred/data_dynamic/checkerboard/data' 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \ + 'DATASET.NUM_CLASSES' 2 'DATASET.CLASS_WEIGHTS' '[1.0, 1.0]' \ + 'TRAIN.DEPTH' 'section' \ + 'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'section_depth' \ + --cfg=configs/seresnet_unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } + pids+=" $!" + # export CUDA_VISIBLE_DEVICES=3 + { python train.py 'DATASET.ROOT' '/home/alfred/data_dynamic/checkerboard/data' 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \ + 'DATASET.NUM_CLASSES' 2 'DATASET.CLASS_WEIGHTS' '[1.0, 1.0]' \ + 'TRAIN.DEPTH' 'section' \ + 'MODEL.PRETRAINED' '/home/alfred/models/hrnetv2_w48_imagenet_pretrained.pth' \ + 'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'section_depth' \ + --cfg=configs/hrnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } + pids+=" $!" + + wait $pids || exit 1 + + # check if any of the models had an error during execution + # Get return information for each pid + for file in "$dir"/*; do + printf 'PID %d returned %d\n' "${file##*/}" "$(<"$file")" + [[ "$(<"$file")" -ne "0" ]] && exit 1 || echo "pass" + done + + # Remove the temporary directory + rm -r "$dir" + + # check validation set performance + set -e + python ../../../../tests/cicd/src/check_performance.py --infile metrics_patch_deconvnet_no_depth.json + python ../../../../tests/cicd/src/check_performance.py --infile metrics_unet_section_depth.json + python ../../../../tests/cicd/src/check_performance.py --infile metrics_seresnet_unet_section_depth.json + python ../../../../tests/cicd/src/check_performance.py --infile metrics_hrnet_section_depth.json + set +e + echo "All models finished training - start scoring" + + # Create a temporary directory to store the statuses + dir=$(mktemp -d) + + pids= + # export CUDA_VISIBLE_DEVICES=0 + # find the latest model which we just trained + model=$(ls -td output/patch_deconvnet/no_depth/* | head -1) + # try running the test script + { python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/checkerboard/data' \ + 'TEST.SPLIT' 'Both' 'TRAIN.MODEL_DIR' 'no_depth' \ + 'DATASET.NUM_CLASSES' 2 'DATASET.CLASS_WEIGHTS' '[1.0, 1.0]' \ + 'TEST.MODEL_PATH' ${model}/patch_deconvnet_running_model_0.*.pth \ + --cfg=configs/patch_deconvnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } + pids+=" $!" + # export CUDA_VISIBLE_DEVICES=1 + # find the latest model which we just trained + model=$(ls -td output/unet/section_depth/* | head -1) + # try running the test script + { python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/checkerboard/data' \ + 'TEST.SPLIT' 'Both' 'TRAIN.MODEL_DIR' 'section_depth' \ + 'DATASET.NUM_CLASSES' 2 'DATASET.CLASS_WEIGHTS' '[1.0, 1.0]' \ + 'TEST.MODEL_PATH' ${model}/resnet_unet_running_model_0.*.pth \ + --cfg=configs/unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } + pids+=" $!" + # export CUDA_VISIBLE_DEVICES=2 + # find the latest model which we just trained + model=$(ls -td output/seresnet_unet/section_depth/* | head -1) + # try running the test script + { python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/checkerboard/data' \ + 'DATASET.NUM_CLASSES' 2 'DATASET.CLASS_WEIGHTS' '[1.0, 1.0]' \ + 'TEST.SPLIT' 'Both' 'TRAIN.MODEL_DIR' 'section_depth' \ + 'TEST.MODEL_PATH' ${model}/resnet_unet_running_model_0.*.pth \ + --cfg=configs/seresnet_unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } + pids+=" $!" + # export CUDA_VISIBLE_DEVICES=3 + # find the latest model which we just trained + model=$(ls -td output/hrnet/section_depth/* | head -1) + # try running the test script + { python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/checkerboard/data' \ + 'DATASET.NUM_CLASSES' 2 'DATASET.CLASS_WEIGHTS' '[1.0, 1.0]' \ + 'TEST.SPLIT' 'Both' 'TRAIN.MODEL_DIR' 'section_depth' \ + 'MODEL.PRETRAINED' '/home/alfred/models/hrnetv2_w48_imagenet_pretrained.pth' \ + 'TEST.MODEL_PATH' ${model}/seg_hrnet_running_model_0.*.pth \ + --cfg=configs/hrnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } + pids+=" $!" + + # wait for completion + wait $pids || exit 1 + + # check if any of the models had an error during execution + # Get return information for each pid + for file in "$dir"/*; do + printf 'PID %d returned %d\n' "${file##*/}" "$(<"$file")" + [[ "$(<"$file")" -ne "0" ]] && exit 1 || echo "pass" + done + + # Remove the temporary directory + rm -r "$dir" + + # check test set performance + set -e + python ../../../../tests/cicd/src/check_performance.py --infile metrics_test_patch_deconvnet_no_depth.json --test + python ../../../../tests/cicd/src/check_performance.py --infile metrics_test_unet_section_depth.json --test + python ../../../../tests/cicd/src/check_performance.py --infile metrics_test_seresnet_unet_section_depth.json --test + python ../../../../tests/cicd/src/check_performance.py --infile metrics_test_hrnet_section_depth.json --test + + echo "PASSED" + ################################################################################################### # Stage 3: Dutch F3 patch models: deconvnet, unet, HRNet patch depth, HRNet section depth @@ -95,7 +240,7 @@ jobs: ################################################################################################### - job: dutchf3_patch - dependsOn: cv_lib_unit_tests_job + dependsOn: checkerboard_dutchf3_patch timeoutInMinutes: 20 displayName: Dutch F3 patch local pool: @@ -116,28 +261,28 @@ jobs: pids= # export CUDA_VISIBLE_DEVICES=0 { python train.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \ - 'TRAIN.DEPTH' 'none' \ - 'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'no_depth' \ - --cfg=configs/patch_deconvnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } + 'TRAIN.DEPTH' 'none' \ + 'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'no_depth' \ + --cfg=configs/patch_deconvnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } pids+=" $!" # export CUDA_VISIBLE_DEVICES=1 { python train.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \ - 'TRAIN.DEPTH' 'section' \ - 'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'section_depth' \ - --cfg=configs/unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } + 'TRAIN.DEPTH' 'section' \ + 'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'section_depth' \ + --cfg=configs/unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } pids+=" $!" # export CUDA_VISIBLE_DEVICES=2 { python train.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \ - 'TRAIN.DEPTH' 'section' \ - 'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'section_depth' \ - --cfg=configs/seresnet_unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } + 'TRAIN.DEPTH' 'section' \ + 'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'section_depth' \ + --cfg=configs/seresnet_unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } pids+=" $!" # export CUDA_VISIBLE_DEVICES=3 { python train.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \ - 'TRAIN.DEPTH' 'section' \ - 'MODEL.PRETRAINED' '/home/alfred/models/hrnetv2_w48_imagenet_pretrained.pth' \ - 'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'section_depth' \ - --cfg=configs/hrnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } + 'TRAIN.DEPTH' 'section' \ + 'MODEL.PRETRAINED' '/home/alfred/models/hrnetv2_w48_imagenet_pretrained.pth' \ + 'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'section_depth' \ + --cfg=configs/hrnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } pids+=" $!" wait $pids || exit 1 @@ -164,8 +309,9 @@ jobs: model=$(ls -t ${model_dir}/*.pth | head -1) # try running the test script { python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' \ - 'TEST.MODEL_PATH' ${model} \ - --cfg=configs/patch_deconvnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } + 'TEST.SPLIT' 'Both' 'TRAIN.MODEL_DIR' 'no_depth' \ + 'TEST.MODEL_PATH' ${model} \ + --cfg=configs/patch_deconvnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } pids+=" $!" # export CUDA_VISIBLE_DEVICES=1 # find the latest model which we just trained @@ -173,9 +319,10 @@ jobs: model=$(ls -t ${model_dir}/*.pth | head -1) # try running the test script - { python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' \ - 'TEST.MODEL_PATH' ${model} \ - --cfg=configs/unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } + { python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' \ + 'TEST.SPLIT' 'Both' 'TRAIN.MODEL_DIR' 'section_depth' \ + 'TEST.MODEL_PATH' ${model} \ + --cfg=configs/unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } pids+=" $!" # export CUDA_VISIBLE_DEVICES=2 # find the latest model which we just trained @@ -184,8 +331,9 @@ jobs: # try running the test script { python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' \ - 'TEST.MODEL_PATH' ${model} \ - --cfg=configs/seresnet_unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } + 'TEST.SPLIT' 'Both' 'TRAIN.MODEL_DIR' 'section_depth' \ + 'TEST.MODEL_PATH' ${model} \ + --cfg=configs/seresnet_unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } pids+=" $!" # export CUDA_VISIBLE_DEVICES=3 # find the latest model which we just trained @@ -194,9 +342,10 @@ jobs: # try running the test script { python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' \ - 'MODEL.PRETRAINED' '/home/alfred/models/hrnetv2_w48_imagenet_pretrained.pth' \ - 'TEST.MODEL_PATH' ${model} \ - --cfg=configs/hrnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } + 'TEST.SPLIT' 'Both' 'TRAIN.MODEL_DIR' 'section_depth' \ + 'MODEL.PRETRAINED' '/home/alfred/models/hrnetv2_w48_imagenet_pretrained.pth' \ + 'TEST.MODEL_PATH' ${model} \ + --cfg=configs/hrnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } pids+=" $!" # wait for completion @@ -212,4 +361,22 @@ jobs: # Remove the temporary directory rm -r "$dir" - echo "PASSED" \ No newline at end of file + echo "PASSED" + +################################################################################################### +# Stage 5: Notebook tests +################################################################################################### + +- job: F3_block_training_and_evaluation_local_notebook + dependsOn: dutchf3_patch + timeoutInMinutes: 5 + displayName: F3 block training and evaluation local notebook + pool: + name: deepseismicagentpool + steps: + - bash: | + source activate seismic-interpretation + pytest -s tests/cicd/src/notebook_integration_tests.py \ + --nbname examples/interpretation/notebooks/Dutch_F3_patch_model_training_and_evaluation.ipynb \ + --dataset_root /home/alfred/data_dynamic/dutch_f3/data \ + --model_pretrained download diff --git a/tests/cicd/notebooks_build.yml b/tests/cicd/notebooks_build.yml deleted file mode 100644 index 3cf7d1ef..00000000 --- a/tests/cicd/notebooks_build.yml +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -# Pull request against these branches will trigger this build -pr: -- master -- staging -- contrib -- correctness - -# Any commit to this branch will trigger the build. -trigger: -- master -- staging -- contrib -- correctness - -jobs: - -- job: setup - timeoutInMinutes: 10 - displayName: Setup - pool: - name: deepseismicagentpool - - steps: - - bash: | - # terminate as soon as any internal script fails - set -e - - echo "Running setup..." - pwd - ls - git branch - uname -ra - - ./scripts/env_reinstall.sh - - # use hardcoded root for now because not sure how env changes under ADO policy - DATA_ROOT="/home/alfred/data_dynamic" - - ./tests/cicd/src/scripts/get_data_for_builds.sh ${DATA_ROOT} - - # copy your model files like so - using dummy file to illustrate - azcopy --quiet --source:https://$(storagename).blob.core.windows.net/models/model --source-key $(storagekey) --destination /home/alfred/models/your_model_name - -- job: F3_block_training_and_evaluation_local - dependsOn: setup - timeoutInMinutes: 5 - displayName: F3 block training and evaluation local - pool: - name: deepseismicagentpool - steps: - - bash: | - source activate seismic-interpretation - pytest -s tests/cicd/src/notebook_integration_tests.py \ - --nbname examples/interpretation/notebooks/Dutch_F3_patch_model_training_and_evaluation.ipynb \ - --dataset_root /home/alfred/data_dynamic/dutch_f3/data \ - --model_pretrained download diff --git a/tests/cicd/src/check_performance.py b/tests/cicd/src/check_performance.py new file mode 100644 index 00000000..5df69dda --- /dev/null +++ b/tests/cicd/src/check_performance.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +""" Please see the def main() function for code description.""" +import json + +""" libraries """ + +import numpy as np +import sys +import os + +np.set_printoptions(linewidth=200) +import logging + +# toggle to WARNING when running in production, or use CLI +logging.getLogger().setLevel(logging.DEBUG) +# logging.getLogger().setLevel(logging.WARNING) +import argparse + +parser = argparse.ArgumentParser() + +""" useful information when running from a GIT folder.""" +myname = os.path.realpath(__file__) +mypath = os.path.dirname(myname) +myname = os.path.basename(myname) + + +def main(args): + """ + + Check to see whether performance metrics are within range on both validation + and test sets. + + """ + + logging.info("loading data") + + with open(args.infile, 'r') as fp: + data = json.load(fp) + + if args.test: + # process training set results + assert data["Pixel Acc: "] > 0.0 + assert data["Pixel Acc: "] <= 1.0 + # TODO make these into proper tests + # assert data["Pixel Acc: "] == 1.0 + # TODO: add more tests as we fix performance + # assert data["Mean Class Acc: "] == 1.0 + # assert data["Freq Weighted IoU: "] == 1.0 + # assert data["Mean IoU: "] == 1.0 + + else: + # process validation results + assert data['Pixelwise Accuracy :'] > 0.0 + assert data['Pixelwise Accuracy :'] <= 1.0 + # TODO make these into proper tests + # assert data['Pixelwise Accuracy :'] == 1.0 + # TODO: add more tests as we fix performance + # assert data['Avg loss :'] < 1e-3 + + + logging.info("all done") + + +""" GLOBAL VARIABLES """ + + +""" cmd-line arguments """ +parser.add_argument("--infile", help="Location of the file which has the metrics", type=str, required=True) +parser.add_argument( + "--test", + help="Flag to indicate that these are test set results - validation by default", + action="store_true" +) + +""" main wrapper with profiler """ +if __name__ == "__main__": + main(parser.parse_args()) + +# pretty printing of the stack +""" + try: + logging.info('before main') + main(parser.parse_args()) + logging.info('after main') + except: + for frame in traceback.extract_tb(sys.exc_info()[2]): + fname,lineno,fn,text = frame + print ("Error in %s on line %d" % (fname, lineno)) +""" +# optionally enable profiling information +# import cProfile +# name = +# cProfile.run('main.run()', name + '.prof') +# import pstats +# p = pstats.Stats(name + '.prof') +# p.sort_stats('cumulative').print_stats(10) +# p.sort_stats('time').print_stats() diff --git a/tests/cicd/src/scripts/get_data_for_builds.sh b/tests/cicd/src/scripts/get_data_for_builds.sh index f3610fa9..46dc1e95 100755 --- a/tests/cicd/src/scripts/get_data_for_builds.sh +++ b/tests/cicd/src/scripts/get_data_for_builds.sh @@ -16,6 +16,7 @@ source activate seismic-interpretation # these have to match the rest of the build jobs unless we want to # define this in ADO pipelines +DATA_CHECKERBOARD="${DATA_ROOT}/checkerboard" DATA_F3="${DATA_ROOT}/dutch_f3" DATA_PENOBSCOT="${DATA_ROOT}/penobscot" @@ -28,16 +29,25 @@ mkdir -p "${DATA_F3}" mkdir -p "${DATA_PENOBSCOT}" # test download scripts in parallel -./scripts/download_penobscot.sh "${DATA_PENOBSCOT}" & +./scripts/download_penobscot.sh "${DATA_PENOBSCOT}" & ./scripts/download_dutch_f3.sh "${DATA_F3}" & - wait # change imposed by download script DATA_F3="${DATA_F3}/data" -# test preprocessing scripts cd scripts + +python gen_checkerboard.py --dataroot ${DATA_F3} --dataout ${DATA_CHECKERBOARD} + +# finished data download and generation + +# test preprocessing scripts python prepare_penobscot.py split_inline --data-dir=${DATA_PENOBSCOT} --val-ratio=.1 --test-ratio=.2 python prepare_dutchf3.py split_train_val section --data_dir=${DATA_F3} --label_file=train/train_labels.npy --output_dir=splits --split_direction=both -python prepare_dutchf3.py split_train_val patch --data_dir=${DATA_F3} --label_file=train/train_labels.npy --output_dir=splits --stride=50 --patch_size=100 --split_direction=both +python prepare_dutchf3.py split_train_val patch --data_dir=${DATA_F3} --label_file=train/train_labels.npy --output_dir=splits --stride=50 --patch_size=100 --split_direction=both + +DATA_CHECKERBOARD="${DATA_CHECKERBOARD}/data" +# repeat for checkerboard dataset +python prepare_dutchf3.py split_train_val section --data_dir=${DATA_CHECKERBOARD} --label_file=train/train_labels.npy --output_dir=splits --split_direction=both +python prepare_dutchf3.py split_train_val patch --data_dir=${DATA_CHECKERBOARD} --label_file=train/train_labels.npy --output_dir=splits --stride=50 --patch_size=100 --split_direction=both From 806b33763a53b6204fe26b746f16b93cc4f65f51 Mon Sep 17 00:00:00 2001 From: yalaudah Date: Mon, 4 May 2020 16:40:18 +0000 Subject: [PATCH 10/20] enabling development on docker (#291) --- docker/Dockerfile | 15 +++++++++------ docker/README.md | 4 ++-- environment/anaconda/local/environment.yml | 1 + 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index f81cf19f..7e05f919 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -13,6 +13,8 @@ ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 ENV PATH /opt/conda/bin:$PATH SHELL ["/bin/bash", "-c"] +WORKDIR /home/username + # Install Anaconda and download the seismic-deeplearning repo RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh && \ /bin/bash ~/miniconda.sh -b -p /opt/conda && \ @@ -23,9 +25,8 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86 wget --quiet https://github.com/microsoft/seismic-deeplearning/archive/staging.zip -O staging.zip && \ unzip staging.zip && rm staging.zip -WORKDIR seismic-deeplearning-staging - -RUN conda env create -n seismic-interpretation --file environment/anaconda/local/environment.yml && \ +RUN cd seismic-deeplearning-staging && \ + conda env create -n seismic-interpretation --file environment/anaconda/local/environment.yml && \ source activate seismic-interpretation && \ python -m ipykernel install --user --name seismic-interpretation && \ pip install -e interpretation && \ @@ -33,7 +34,8 @@ RUN conda env create -n seismic-interpretation --file environment/anaconda/local # TODO: add back in later when Penobscot notebook is available # Download Penobscot dataset: -# RUN data_dir="/home/username/data/penobscot" && \ +# RUN cd seismic-deeplearning-staging && \ +# data_dir="/home/username/data/penobscot" && \ # mkdir -p "$data_dir" && \ # ./scripts/download_penobscot.sh "$data_dir" && \ # cd scripts && \ @@ -42,7 +44,8 @@ RUN conda env create -n seismic-interpretation --file environment/anaconda/local # cd .. # Download F3 dataset: -RUN data_dir="/home/username/data/dutch" && \ +RUN cd seismic-deeplearning-staging && \ + data_dir="/home/username/data/dutch" && \ mkdir -p "$data_dir" && \ ./scripts/download_dutch_f3.sh "$data_dir" && \ cd scripts && \ @@ -57,4 +60,4 @@ EXPOSE 9000/tcp EXPOSE 9001/tcp CMD source activate seismic-interpretation && \ - jupyter notebook --allow-root --ip 0.0.0.0 --port 9000 + jupyter lab --allow-root --ip 0.0.0.0 --port 9000 diff --git a/docker/README.md b/docker/README.md index e2ccfacb..ce5bb37e 100644 --- a/docker/README.md +++ b/docker/README.md @@ -22,7 +22,7 @@ This process will take a few minutes to complete. # Run the Docker image: Once the Docker image is built, you can run it anytime using the following command: ```bash -sudo docker run --rm -it -p 9000:9000 -p 9001:9001 --gpus=all --shm-size 11G --mount type=bind,source=$PWD/hrnetv2_w48_imagenet_pretrained.pth,target=/home/username/models/hrnetv2_w48_imagenet_pretrained.pth seismic-deeplearning +sudo docker run --rm -it -p 9000:9000 -p 9001:9001 --gpus=all --shm-size 11G --mount type=bind,source=$PWD/hrnetv2_w48_imagenet_pretrained.pth,target=/home/models/hrnetv2_w48_imagenet_pretrained.pth -v ~/:/home/username seismic-deeplearning ``` -If you have saved the pretrained model in a different directory, make sure you replace `$PWD/hrnetv2_w48_imagenet_pretrained.pth` with the **absolute** path to the pretrained HRNet model. The command above will run a jupyter notebook server that you can access by clicking on the link in your terminal. You can then navigate to the notebook that you would like to run. +If you have saved the pretrained model in a different directory, make sure you replace `$PWD/hrnetv2_w48_imagenet_pretrained.pth` with the **absolute** path to the pretrained HRNet model. The command above will run a Jupyter Lab instance that you can access by clicking on the link in your terminal. You can then navigate to the notebook that you would like to run. By default, running the command above would mount your home directory to the Docker container, allowing you to access your files and data from within Jupyter Lab. diff --git a/environment/anaconda/local/environment.yml b/environment/anaconda/local/environment.yml index 40b28a34..10ad22f4 100644 --- a/environment/anaconda/local/environment.yml +++ b/environment/anaconda/local/environment.yml @@ -18,6 +18,7 @@ dependencies: - itkwidgets==0.23.1 - pytest - papermill>=1.0.1 + - jupyterlab - pip: - segyio==1.8.8 - pytorch-ignite==0.3.0 From a1278a4dabccc7080fa2453e700c5670e1dd088e Mon Sep 17 00:00:00 2001 From: maxkazmsft Date: Mon, 4 May 2020 21:55:02 -0400 Subject: [PATCH 11/20] 289: correctness metrics and tighter tests (#293) * resolved rebase conflict * resolved merge conflict * resolved rebase conflict * resolved merge conflict * wrote the bulk of checkerboard example * finished checkerboard generator * resolved merge conflict * resolved rebase conflict * got binary dataset to run * finished first implementation mockup - commit before rebase * made sure rebase went well manually * added new files * resolved PR comments and made tests work * fixed build error * fixed build VM errors * more fixes to get the test to pass * fixed n_classes issue in data.py * fixed notebook as well * cleared notebook run cell * trivial commit to restart builds * addressed PR comments * moved notebook tests to main build pipeline * fixed checkerboard label precision * relaxed performance tests for now * resolved merge conflict * resolved merge conflict * fixed build error * resolved merge conflicts * fixed another merge mistake * resolved rebase conflict * resolved rebase 2 * resolved merge conflict * resolved merge conflict * adding new logging * added better logging - cleaner - debugged metrics on checkerboard dataset * resolved rebase conflict * resolved merge conflict * resolved merge conflict * resolved merge conflict * resolved rebase 2 * resolved merge conflict * updated notebook with the changes * addressed PR comments * addressed another PR comment --- .../dutchf3_patch/distributed/train.py | 1 - cv_lib/cv_lib/event_handlers/__init__.py | 3 +- .../cv_lib/event_handlers/logging_handlers.py | 20 +- ..._patch_model_training_and_evaluation.ipynb | 188 ++++++------------ .../dutchf3_patch/local/default.py | 8 +- .../dutchf3_patch/local/train.py | 64 +++--- tests/cicd/main_build.yml | 19 +- 7 files changed, 122 insertions(+), 181 deletions(-) diff --git a/contrib/experiments/interpretation/dutchf3_patch/distributed/train.py b/contrib/experiments/interpretation/dutchf3_patch/distributed/train.py index 7f3a136e..34f1157a 100644 --- a/contrib/experiments/interpretation/dutchf3_patch/distributed/train.py +++ b/contrib/experiments/interpretation/dutchf3_patch/distributed/train.py @@ -23,7 +23,6 @@ import fire import numpy as np -import toolz import torch from albumentations import Compose, HorizontalFlip, Normalize, PadIfNeeded, Resize from ignite.contrib.handlers import ConcatScheduler, CosineAnnealingScheduler, LinearCyclicalScheduler diff --git a/cv_lib/cv_lib/event_handlers/__init__.py b/cv_lib/cv_lib/event_handlers/__init__.py index 589bbd86..8bd8567f 100644 --- a/cv_lib/cv_lib/event_handlers/__init__.py +++ b/cv_lib/cv_lib/event_handlers/__init__.py @@ -31,8 +31,7 @@ def _create_checkpoint_handler(self): def __call__(self, engine, to_save): self._checkpoint_handler(engine, to_save) if self._snapshot_function(): - files = glob.glob(os.path.join(self._model_save_location, self._running_model_prefix + "*")) - print(files) + files = glob.glob(os.path.join(self._model_save_location, self._running_model_prefix + "*")) name_postfix = os.path.basename(files[0]).lstrip(self._running_model_prefix) copyfile( files[0], diff --git a/cv_lib/cv_lib/event_handlers/logging_handlers.py b/cv_lib/cv_lib/event_handlers/logging_handlers.py index 97f382bc..de354760 100644 --- a/cv_lib/cv_lib/event_handlers/logging_handlers.py +++ b/cv_lib/cv_lib/event_handlers/logging_handlers.py @@ -26,15 +26,21 @@ def log_lr(optimizer, engine): @curry -def log_metrics(log_msg, engine, metrics_dict={"pixacc": "Avg accuracy :", "nll": "Avg loss :"}, fname=None): +def log_metrics( + engine, + evaluator, + metrics_dict={ + "nll": "Avg loss :", + "pixacc": "Pixelwise Accuracy :", + "mca": "Avg Class Accuracy :", + "mIoU": "Avg Class IoU :", + }, + stage="undefined", +): logger = logging.getLogger(__name__) - metrics = engine.state.metrics + metrics = evaluator.state.metrics metrics_msg = " ".join([f"{metrics_dict[k]} {metrics[k]:.4f}" for k in metrics_dict]) - if fname: - with open(fname, "w") as fid: - output_dict = {metrics_dict[k]: float(metrics[k]) for k in metrics_dict} - json.dump(output_dict, fid) - logger.info(f"{log_msg} - Epoch {engine.state.epoch} [{engine.state.max_epochs}] " + metrics_msg) + logger.info(f"{stage} - Epoch {engine.state.epoch} [{engine.state.max_epochs}] " + metrics_msg) @curry diff --git a/examples/interpretation/notebooks/Dutch_F3_patch_model_training_and_evaluation.ipynb b/examples/interpretation/notebooks/Dutch_F3_patch_model_training_and_evaluation.ipynb index 1748e45c..28351947 100644 --- a/examples/interpretation/notebooks/Dutch_F3_patch_model_training_and_evaluation.ipynb +++ b/examples/interpretation/notebooks/Dutch_F3_patch_model_training_and_evaluation.ipynb @@ -143,10 +143,7 @@ "from cv_lib.event_handlers import SnapshotHandler, logging_handlers\n", "from cv_lib.event_handlers.logging_handlers import Evaluator\n", "from cv_lib.event_handlers import tensorboard_handlers\n", - "from cv_lib.event_handlers.tensorboard_handlers import (\n", - " create_image_writer,\n", - " create_summary_writer, \n", - ")\n", + "from cv_lib.event_handlers.tensorboard_handlers import create_summary_writer\n", "from cv_lib.segmentation import models\n", "from cv_lib.segmentation.dutchf3.engine import (\n", " create_supervised_evaluator,\n", @@ -537,6 +534,7 @@ ")\n", "\n", "if papermill:\n", + " train_set = data.Subset(train_set, range(3))\n", " val_set = data.Subset(val_set, range(3))\n", "elif DEMO:\n", " val_set = data.Subset(val_set, range(config.VALIDATION.BATCH_SIZE_PER_GPU))\n", @@ -571,15 +569,15 @@ "source": [ "# if we're running in test mode, just run 2 batches\n", "if papermill:\n", - " train_len = config.TRAIN.BATCH_SIZE_PER_GPU*2 \n", - "# if we're running in demo mode, just run 10 batches to fine-tune the model\n", + " train_len = 2\n", + "# if we're running in demo mode, just run 20 batches to fine-tune the model\n", "elif DEMO:\n", - " train_len = config.TRAIN.BATCH_SIZE_PER_GPU*10 \n", + " train_len = 20\n", "# if we're not in test or demo modes, run the entire loop\n", "else:\n", " train_len = len(train_loader)\n", "\n", - "snapshot_duration = scheduler_step * train_len if not papermill else 2*len(train_loader)" + "snapshot_duration = scheduler_step * train_len if not papermill else train_len" ] }, { @@ -678,10 +676,7 @@ "# create training engine\n", "trainer = create_supervised_trainer(\n", " model, optimizer, criterion, prepare_batch, device=device\n", - ")\n", - "\n", - "# add learning rate scheduler\n", - "trainer.add_event_handler(Events.ITERATION_STARTED, scheduler)" + ")" ] }, { @@ -710,35 +705,16 @@ "generate_path(output_dir)\n", "\n", "# define main summary writer which logs all model summaries\n", - "summary_writer = create_summary_writer(log_dir=path.join(output_dir, config.LOG_DIR))\n", - "\n", - "# add logging of training output\n", - "trainer.add_event_handler(\n", - " Events.ITERATION_COMPLETED,\n", - " logging_handlers.log_training_output(log_interval=config.TRAIN.BATCH_SIZE_PER_GPU),\n", - ")\n", - "\n", - "# add logging of learning rate\n", - "trainer.add_event_handler(Events.EPOCH_STARTED, logging_handlers.log_lr(optimizer))\n", - "\n", - "# log learning rate to tensorboard\n", - "trainer.add_event_handler(\n", - " Events.EPOCH_STARTED,\n", - " tensorboard_handlers.log_lr(summary_writer, optimizer, \"epoch\"),\n", - ")\n", - "\n", - "# log training summary to tensorboard as well\n", - "trainer.add_event_handler(\n", - " Events.ITERATION_COMPLETED,\n", - " tensorboard_handlers.log_training_output(summary_writer),\n", - ")" + "summary_writer = create_summary_writer(log_dir=path.join(output_dir, config.LOG_DIR))\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "We also checkpoint models and snapshot them to disk with every training epoch." + "Next we need to score the model on validation set as it's training. To do this we need to add helper functions to manipulate data into the required shape just as we've done to prepare each batch for training at the beginning of this notebook.\n", + "\n", + "We also set up evaluation metrics which we want to record on the training set." ] }, { @@ -747,28 +723,52 @@ "metadata": {}, "outputs": [], "source": [ - "# add model checkpointing\n", - "checkpoint_handler = ModelCheckpoint(\n", - " output_dir,\n", - " \"model_f3_nb\",\n", - " save_interval=1,\n", - " n_saved=1,\n", - " create_dir=True,\n", - " require_empty=False,\n", + "transform_fn = lambda output_dict: (output_dict[\"y_pred\"].squeeze(), output_dict[\"mask\"].squeeze())\n", + "evaluator = create_supervised_evaluator(\n", + " model,\n", + " prepare_batch,\n", + " metrics={\n", + " \"nll\": Loss(criterion, output_transform=transform_fn),\n", + " \"pixacc\": pixelwise_accuracy(n_classes, output_transform=transform_fn, device=device),\n", + " \"cacc\": class_accuracy(n_classes, output_transform=transform_fn),\n", + " \"mca\": mean_class_accuracy(n_classes, output_transform=transform_fn),\n", + " \"ciou\": class_iou(n_classes, output_transform=transform_fn),\n", + " \"mIoU\": mean_iou(n_classes, output_transform=transform_fn),\n", + " },\n", + " device=device,\n", ")\n", + "trainer.add_event_handler(Events.ITERATION_STARTED, scheduler)\n", "\n", + "# Logging:\n", "trainer.add_event_handler(\n", - " Events.EPOCH_COMPLETED, checkpoint_handler, {config.MODEL.NAME: model}\n", - ")" + " Events.ITERATION_COMPLETED, logging_handlers.log_training_output(log_interval=config.PRINT_FREQ),\n", + ")\n", + "trainer.add_event_handler(Events.EPOCH_COMPLETED, logging_handlers.log_lr(optimizer))\n", + "\n", + "# Tensorboard and Logging:\n", + "trainer.add_event_handler(Events.ITERATION_COMPLETED, tensorboard_handlers.log_training_output(summary_writer))\n", + "trainer.add_event_handler(Events.ITERATION_COMPLETED, tensorboard_handlers.log_validation_output(summary_writer))\n", + "\n", + "# add specific logger which also triggers printed metrics on test set\n", + "@trainer.on(Events.EPOCH_COMPLETED)\n", + "def log_training_results(engine):\n", + " evaluator.run(train_loader)\n", + " tensorboard_handlers.log_results(engine, evaluator, summary_writer, n_classes, stage=\"Training\")\n", + " logging_handlers.log_metrics(engine, evaluator, stage=\"Training\")\n", + "\n", + "# add specific logger which also triggers printed metrics on validation set\n", + "@trainer.on(Events.EPOCH_COMPLETED)\n", + "def log_validation_results(engine):\n", + " evaluator.run(val_loader)\n", + " tensorboard_handlers.log_results(engine, evaluator, summary_writer, n_classes, stage=\"Validation\")\n", + " logging_handlers.log_metrics(engine, evaluator, stage=\"Validation\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Next we need to score the model on validation set as it's training. To do this we need to add helper functions to manipulate data into the required shape just as we've done to prepare each batch for training at the beginning of this notebook.\n", - "\n", - "We also set up evaluation metrics which we want to record on the training set." + "We also checkpoint models and snapshot them to disk with every training epoch." ] }, { @@ -777,90 +777,18 @@ "metadata": {}, "outputs": [], "source": [ - "# helper function for\n", - "def _select_pred_and_mask(model_out_dict):\n", - " return (model_out_dict[\"y_pred\"].squeeze(), model_out_dict[\"mask\"].squeeze())\n", - "\n", - "\n", - "def _select_max(pred_tensor):\n", - " return pred_tensor.max(1)[1]\n", - "\n", - "\n", - "def _tensor_to_numpy(pred_tensor):\n", - " return pred_tensor.squeeze().cpu().numpy()\n", - "\n", - "\n", - "def snapshot_function():\n", - " return (trainer.state.iteration % snapshot_duration) == 0\n", - "\n", - "evaluator = create_supervised_evaluator(\n", - " model,\n", - " prepare_batch,\n", - " metrics={\n", - " \"nll\": Loss(criterion, output_transform=_select_pred_and_mask),\n", - " \"pixacc\": pixelwise_accuracy(\n", - " n_classes, output_transform=_select_pred_and_mask, device=device\n", - " ),\n", - " \"cacc\": class_accuracy(n_classes, output_transform=_select_pred_and_mask),\n", - " \"mca\": mean_class_accuracy(n_classes, output_transform=_select_pred_and_mask),\n", - " \"ciou\": class_iou(n_classes, output_transform=_select_pred_and_mask),\n", - " \"mIoU\": mean_iou(n_classes, output_transform=_select_pred_and_mask),\n", - " },\n", - " device=device,\n", - ")\n", - "\n", - "trainer.add_event_handler(Events.EPOCH_COMPLETED, Evaluator(evaluator, val_loader))\n", - "\n", - "evaluator.add_event_handler(\n", - " Events.EPOCH_COMPLETED,\n", - " logging_handlers.log_metrics(\n", - " \"Validation results\",\n", - " metrics_dict={\n", - " \"nll\": \"Avg loss :\",\n", - " \"pixacc\": \"Pixelwise Accuracy :\",\n", - " \"mca\": \"Avg Class Accuracy :\",\n", - " \"mIoU\": \"Avg Class IoU :\",\n", - " },\n", - " ),\n", - ")\n", - "\n", - "evaluator.add_event_handler(\n", - " Events.EPOCH_COMPLETED,\n", - " tensorboard_handlers.log_metrics(\n", - " summary_writer,\n", - " trainer,\n", - " \"epoch\",\n", - " metrics_dict={\n", - " \"mIoU\": \"Validation/mIoU\",\n", - " \"nll\": \"Validation/Loss\",\n", - " \"mca\": \"Validation/MCA\",\n", - " \"pixacc\": \"Validation/Pixel_Acc\",\n", - " },\n", - " ),\n", - ")\n", - "\n", - "\n", - "transform_func = compose(np_to_tb, decode_segmap(n_classes=n_classes), _tensor_to_numpy)\n", - "\n", - "transform_pred = compose(transform_func, _select_max)\n", - "\n", - "evaluator.add_event_handler(\n", - " Events.EPOCH_COMPLETED,\n", - " create_image_writer(summary_writer, \"Validation/Image\", \"image\"),\n", - ")\n", - "\n", - "evaluator.add_event_handler(\n", - " Events.EPOCH_COMPLETED,\n", - " create_image_writer(\n", - " summary_writer, \"Validation/Mask\", \"mask\", transform_func=transform_func\n", - " ),\n", + "# add model checkpointing\n", + "checkpoint_handler = ModelCheckpoint(\n", + " output_dir,\n", + " \"model_f3_nb\",\n", + " save_interval=1,\n", + " n_saved=1,\n", + " create_dir=True,\n", + " require_empty=False,\n", ")\n", "\n", - "evaluator.add_event_handler(\n", - " Events.EPOCH_COMPLETED,\n", - " create_image_writer(\n", - " summary_writer, \"Validation/Pred\", \"y_pred\", transform_func=transform_pred\n", - " ),\n", + "trainer.add_event_handler(\n", + " Events.EPOCH_COMPLETED, checkpoint_handler, {config.MODEL.NAME: model}\n", ")" ] }, diff --git a/experiments/interpretation/dutchf3_patch/local/default.py b/experiments/interpretation/dutchf3_patch/local/default.py index 4a4c74af..0322d5b1 100644 --- a/experiments/interpretation/dutchf3_patch/local/default.py +++ b/experiments/interpretation/dutchf3_patch/local/default.py @@ -11,8 +11,10 @@ _C = CN() -_C.OUTPUT_DIR = "output" # This will be the base directory for all output, such as logs and saved models -_C.LOG_DIR = "" # This will be a subdirectory inside OUTPUT_DIR +# This will be the base directory for all output, such as logs and saved models +_C.OUTPUT_DIR = "output" +# This will be a subdirectory inside OUTPUT_DIR +_C.LOG_DIR = "" _C.GPUS = (0,) _C.WORKERS = 4 _C.PRINT_FREQ = 20 @@ -21,6 +23,8 @@ _C.LOG_CONFIG = "logging.conf" _C.SEED = 42 _C.OPENCV_BORDER_CONSTANT = 0 +# number of batches to use in test/debug mode +_C.NUM_DEBUG_BATCHES = 1 # Cudnn related params _C.CUDNN = CN() diff --git a/experiments/interpretation/dutchf3_patch/local/train.py b/experiments/interpretation/dutchf3_patch/local/train.py index 9e26f7a2..1d4e3348 100644 --- a/experiments/interpretation/dutchf3_patch/local/train.py +++ b/experiments/interpretation/dutchf3_patch/local/train.py @@ -12,7 +12,7 @@ Time to run on single V100 for 300 epochs: 4.5 days """ - +import json import logging import logging.config from os import path @@ -28,7 +28,7 @@ from ignite.utils import convert_tensor from cv_lib.event_handlers import SnapshotHandler, logging_handlers, tensorboard_handlers -from cv_lib.event_handlers.tensorboard_handlers import create_summary_writer, log_results +from cv_lib.event_handlers.tensorboard_handlers import create_summary_writer from cv_lib.segmentation import extract_metric_from, models from cv_lib.segmentation.dutchf3.engine import create_supervised_evaluator, create_supervised_trainer from cv_lib.segmentation.dutchf3.utils import current_datetime, generate_path, git_branch, git_hash @@ -53,6 +53,7 @@ def run(*options, cfg=None, debug=False): Notes: Options can be passed in via the options argument and loaded from the cfg file Options from default.py will be overridden by options loaded from cfg file + Options from default.py will be overridden by options loaded from cfg file Options passed in via options argument will override option loaded from cfg file Args: @@ -63,7 +64,7 @@ def run(*options, cfg=None, debug=False): debug (bool): Places scripts in debug/test mode and only executes a few iterations """ # Configuration: - update_config(config, options=options, config_file=cfg) + update_config(config, options=options, config_file=cfg) # The model will be saved under: outputs// config_file_name = "default_config" if not cfg else cfg.split("/")[-1].split(".")[0] @@ -147,13 +148,15 @@ def run(*options, cfg=None, debug=False): if debug: logger.info("Running in debug mode..") - train_set = data.Subset(train_set, list(range(4))) - val_set = data.Subset(val_set, list(range(4))) + train_set = data.Subset(train_set, range(config.TRAIN.BATCH_SIZE_PER_GPU*config.NUM_DEBUG_BATCHES)) + val_set = data.Subset(val_set, range(config.VALIDATION.BATCH_SIZE_PER_GPU)) train_loader = data.DataLoader( train_set, batch_size=config.TRAIN.BATCH_SIZE_PER_GPU, num_workers=config.WORKERS, shuffle=True ) - val_loader = data.DataLoader(val_set, batch_size=config.VALIDATION.BATCH_SIZE_PER_GPU, num_workers=config.WORKERS) + val_loader = data.DataLoader( + val_set, batch_size=config.VALIDATION.BATCH_SIZE_PER_GPU, num_workers=1 + ) # config.WORKERS) # Model: model = getattr(models, config.MODEL.NAME).get_seg_model(config) @@ -203,40 +206,38 @@ def run(*options, cfg=None, debug=False): # Logging: trainer.add_event_handler( - Events.ITERATION_COMPLETED, logging_handlers.log_training_output(log_interval=config.TRAIN.BATCH_SIZE_PER_GPU), + Events.ITERATION_COMPLETED, logging_handlers.log_training_output(log_interval=config.PRINT_FREQ), ) trainer.add_event_handler(Events.EPOCH_COMPLETED, logging_handlers.log_lr(optimizer)) - fname = f"metrics_{config_file_name}_{config.TRAIN.MODEL_DIR}.json" if debug else None - evaluator.add_event_handler( - Events.EPOCH_COMPLETED, - logging_handlers.log_metrics( - "Validation results", - metrics_dict={ - "nll": "Avg loss :", - "pixacc": "Pixelwise Accuracy :", - "mca": "Avg Class Accuracy :", - "mIoU": "Avg Class IoU :", - }, - fname=fname, - ), - ) - # Tensorboard and Logging: trainer.add_event_handler(Events.ITERATION_COMPLETED, tensorboard_handlers.log_training_output(summary_writer)) trainer.add_event_handler(Events.ITERATION_COMPLETED, tensorboard_handlers.log_validation_output(summary_writer)) + # add specific logger which also triggers printed metrics on training set @trainer.on(Events.EPOCH_COMPLETED) def log_training_results(engine): evaluator.run(train_loader) - log_results(engine, evaluator, summary_writer, n_classes, stage="Training") + tensorboard_handlers.log_results(engine, evaluator, summary_writer, n_classes, stage="Training") + logging_handlers.log_metrics(engine, evaluator, stage="Training") + # add specific logger which also triggers printed metrics on validation set @trainer.on(Events.EPOCH_COMPLETED) def log_validation_results(engine): evaluator.run(val_loader) - log_results(engine, evaluator, summary_writer, n_classes, stage="Validation") - - # Checkpointing: + tensorboard_handlers.log_results(engine, evaluator, summary_writer, n_classes, stage="Validation") + logging_handlers.log_metrics(engine, evaluator, stage="Validation") + # dump validation set metrics at the very end for debugging purposes + if engine.state.epoch == config.TRAIN.END_EPOCH and debug: + fname = f"metrics_test_{config_file_name}_{config.TRAIN.MODEL_DIR}.json" + metrics = evaluator.state.metrics + out_dict = {x: metrics[x] for x in ["nll", "pixacc", "mca", "mIoU"]} + with open(fname, "w") as fid: + json.dump(out_dict, fid) + log_msg = " ".join(f"{k}: {out_dict[k]}" for k in out_dict.keys()) + logging.info(log_msg) + + # Checkpointing: snapshotting trained models to disk checkpoint_handler = SnapshotHandler( output_dir, config.MODEL.NAME, @@ -246,15 +247,8 @@ def log_validation_results(engine): evaluator.add_event_handler(Events.EPOCH_COMPLETED, checkpoint_handler, {"model": model}) logger.info("Starting training") - if debug: - trainer.run( - train_loader, - max_epochs=config.TRAIN.END_EPOCH, - epoch_length=config.TRAIN.BATCH_SIZE_PER_GPU, - seed=config.SEED, - ) - else: - trainer.run(train_loader, max_epochs=config.TRAIN.END_EPOCH, epoch_length=len(train_loader), seed=config.SEED) + trainer.run(train_loader, max_epochs=config.TRAIN.END_EPOCH, epoch_length=len(train_loader), seed=config.SEED) + summary_writer.close() diff --git a/tests/cicd/main_build.yml b/tests/cicd/main_build.yml index 875c1c05..4a0b8f2b 100644 --- a/tests/cicd/main_build.yml +++ b/tests/cicd/main_build.yml @@ -113,30 +113,37 @@ jobs: # Create a temporary directory to store the statuses dir=$(mktemp -d) + # we are running a single batch in debug mode through, so increase the + # number of epochs to obtain a representative set of results + pids= # export CUDA_VISIBLE_DEVICES=0 - { python train.py 'DATASET.ROOT' '/home/alfred/data_dynamic/checkerboard/data' 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \ + { python train.py 'DATASET.ROOT' '/home/alfred/data_dynamic/checkerboard/data' \ + 'NUM_DEBUG_BATCHES' 5 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \ 'DATASET.NUM_CLASSES' 2 'DATASET.CLASS_WEIGHTS' '[1.0, 1.0]' \ 'TRAIN.DEPTH' 'none' \ 'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'no_depth' \ --cfg=configs/patch_deconvnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } pids+=" $!" # export CUDA_VISIBLE_DEVICES=1 - { python train.py 'DATASET.ROOT' '/home/alfred/data_dynamic/checkerboard/data' 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \ + { python train.py 'DATASET.ROOT' '/home/alfred/data_dynamic/checkerboard/data' \ + 'NUM_DEBUG_BATCHES' 5 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \ 'DATASET.NUM_CLASSES' 2 'DATASET.CLASS_WEIGHTS' '[1.0, 1.0]' \ 'TRAIN.DEPTH' 'section' \ 'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'section_depth' \ --cfg=configs/unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } pids+=" $!" # export CUDA_VISIBLE_DEVICES=2 - { python train.py 'DATASET.ROOT' '/home/alfred/data_dynamic/checkerboard/data' 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \ + { python train.py 'DATASET.ROOT' '/home/alfred/data_dynamic/checkerboard/data' \ + 'NUM_DEBUG_BATCHES' 5 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \ 'DATASET.NUM_CLASSES' 2 'DATASET.CLASS_WEIGHTS' '[1.0, 1.0]' \ 'TRAIN.DEPTH' 'section' \ 'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'section_depth' \ --cfg=configs/seresnet_unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } pids+=" $!" # export CUDA_VISIBLE_DEVICES=3 - { python train.py 'DATASET.ROOT' '/home/alfred/data_dynamic/checkerboard/data' 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \ + { python train.py 'DATASET.ROOT' '/home/alfred/data_dynamic/checkerboard/data' \ + 'NUM_DEBUG_BATCHES' 5 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \ 'DATASET.NUM_CLASSES' 2 'DATASET.CLASS_WEIGHTS' '[1.0, 1.0]' \ 'TRAIN.DEPTH' 'section' \ 'MODEL.PRETRAINED' '/home/alfred/models/hrnetv2_w48_imagenet_pretrained.pth' \ @@ -262,24 +269,28 @@ jobs: # export CUDA_VISIBLE_DEVICES=0 { python train.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \ 'TRAIN.DEPTH' 'none' \ + 'TRAIN.BATCH_SIZE_PER_GPU' 2 'VALIDATION.BATCH_SIZE_PER_GPU' 2 \ 'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'no_depth' \ --cfg=configs/patch_deconvnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } pids+=" $!" # export CUDA_VISIBLE_DEVICES=1 { python train.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \ 'TRAIN.DEPTH' 'section' \ + 'TRAIN.BATCH_SIZE_PER_GPU' 2 'VALIDATION.BATCH_SIZE_PER_GPU' 2 \ 'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'section_depth' \ --cfg=configs/unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } pids+=" $!" # export CUDA_VISIBLE_DEVICES=2 { python train.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \ 'TRAIN.DEPTH' 'section' \ + 'TRAIN.BATCH_SIZE_PER_GPU' 2 'VALIDATION.BATCH_SIZE_PER_GPU' 2 \ 'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'section_depth' \ --cfg=configs/seresnet_unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } pids+=" $!" # export CUDA_VISIBLE_DEVICES=3 { python train.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' 'TRAIN.END_EPOCH' 1 'TRAIN.SNAPSHOTS' 1 \ 'TRAIN.DEPTH' 'section' \ + 'TRAIN.BATCH_SIZE_PER_GPU' 2 'VALIDATION.BATCH_SIZE_PER_GPU' 2 \ 'MODEL.PRETRAINED' '/home/alfred/models/hrnetv2_w48_imagenet_pretrained.pth' \ 'OUTPUT_DIR' 'output' 'TRAIN.MODEL_DIR' 'section_depth' \ --cfg=configs/hrnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } From 4c5c65dab0623654c4ab3559af1882dac6ebaf18 Mon Sep 17 00:00:00 2001 From: maxkazmsft Date: Fri, 8 May 2020 16:32:40 -0400 Subject: [PATCH 12/20] uniform colormap and correctness tests (#295) * correctness code good for PR review * addressed PR comments --- .../interpretation/penobscot/local/test.py | 7 +- .../interpretation/penobscot/local/train.py | 6 +- .../event_handlers/tensorboard_handlers.py | 11 +- cv_lib/cv_lib/segmentation/dutchf3/utils.py | 6 - cv_lib/cv_lib/segmentation/utils.py | 30 ----- cv_lib/cv_lib/utils.py | 53 ++++++++ ..._patch_model_training_and_evaluation.ipynb | 10 +- .../dutchf3_patch/local/test.py | 42 +----- .../dutchf3_patch/local/train.py | 13 +- .../dutchf3/data.py | 124 ++++++++---------- tests/cicd/src/check_performance.py | 8 +- 11 files changed, 136 insertions(+), 174 deletions(-) diff --git a/contrib/experiments/interpretation/penobscot/local/test.py b/contrib/experiments/interpretation/penobscot/local/test.py index 0bf11667..e56b82de 100644 --- a/contrib/experiments/interpretation/penobscot/local/test.py +++ b/contrib/experiments/interpretation/penobscot/local/test.py @@ -74,11 +74,6 @@ def _scale_from(config): return int(scale_height) -_SEG_COLOURS = np.asarray( - [[241, 238, 246], [208, 209, 230], [166, 189, 219], [116, 169, 207], [54, 144, 192], [5, 112, 176], [3, 78, 123],] -) - - def _log_tensor_to_tensorboard(images_tensor, identifier, summary_writer, evaluator): image_grid = torchvision.utils.make_grid(images_tensor, normalize=False, scale_each=False, nrow=2) summary_writer.add_image(identifier, image_grid, evaluator.state.epoch) @@ -250,7 +245,7 @@ def _tensor_to_numpy(pred_tensor): return pred_tensor.squeeze().cpu().numpy() transform_func = compose( - np_to_tb, decode_segmap(n_classes=n_classes, label_colours=_SEG_COLOURS), _tensor_to_numpy, + np_to_tb, decode_segmap, _tensor_to_numpy, ) transform_pred = compose(transform_func, _select_max) diff --git a/contrib/experiments/interpretation/penobscot/local/train.py b/contrib/experiments/interpretation/penobscot/local/train.py index b48de2ad..eeeff141 100644 --- a/contrib/experiments/interpretation/penobscot/local/train.py +++ b/contrib/experiments/interpretation/penobscot/local/train.py @@ -42,10 +42,6 @@ from default import update_config mask_value = 255 -_SEG_COLOURS = np.asarray( - [[241, 238, 246], [208, 209, 230], [166, 189, 219], [116, 169, 207], [54, 144, 192], [5, 112, 176], [3, 78, 123],] -) - def _prepare_batch(batch, device=None, non_blocking=False): x, y, ids, patch_locations = batch @@ -258,7 +254,7 @@ def _tensor_to_numpy(pred_tensor): return pred_tensor.squeeze().cpu().numpy() transform_func = compose( - np_to_tb, decode_segmap(n_classes=n_classes, label_colours=_SEG_COLOURS), _tensor_to_numpy, + np_to_tb, decode_segmap, _tensor_to_numpy, ) transform_pred = compose(transform_func, _select_max) diff --git a/cv_lib/cv_lib/event_handlers/tensorboard_handlers.py b/cv_lib/cv_lib/event_handlers/tensorboard_handlers.py index 625e11d1..d3df7f31 100644 --- a/cv_lib/cv_lib/event_handlers/tensorboard_handlers.py +++ b/cv_lib/cv_lib/event_handlers/tensorboard_handlers.py @@ -8,8 +8,7 @@ from toolz import curry from cv_lib.segmentation.dutchf3.utils import np_to_tb -from deepseismic_interpretation.dutchf3.data import decode_segmap - +from cv_lib.utils import decode_segmap def create_summary_writer(log_dir): writer = SummaryWriter(logdir=log_dir) @@ -21,9 +20,9 @@ def _transform_image(output_tensor): return torchvision.utils.make_grid(output_tensor, normalize=True, scale_each=True) -def _transform_pred(output_tensor, n_classes): +def _transform_pred(output_tensor): output_tensor = output_tensor.squeeze().cpu().numpy() - decoded = decode_segmap(output_tensor, n_classes) + decoded = decode_segmap(output_tensor) return torchvision.utils.make_grid(np_to_tb(decoded), normalize=False, scale_each=False) @@ -112,5 +111,5 @@ def log_results(engine, evaluator, summary_writer, n_classes, stage): y_pred[mask == 255] = 255 summary_writer.add_image(f"{stage}/Image", _transform_image(image), epoch) - summary_writer.add_image(f"{stage}/Mask", _transform_pred(mask, n_classes), epoch) - summary_writer.add_image(f"{stage}/Pred", _transform_pred(y_pred, n_classes), epoch) + summary_writer.add_image(f"{stage}/Mask", _transform_pred(mask), epoch) + summary_writer.add_image(f"{stage}/Pred", _transform_pred(y_pred), epoch) diff --git a/cv_lib/cv_lib/segmentation/dutchf3/utils.py b/cv_lib/cv_lib/segmentation/dutchf3/utils.py index adad1e97..f00cae8c 100644 --- a/cv_lib/cv_lib/segmentation/dutchf3/utils.py +++ b/cv_lib/cv_lib/segmentation/dutchf3/utils.py @@ -38,9 +38,3 @@ def git_hash(): repo = Repo(search_parent_directories=True) return repo.active_branch.commit.hexsha - -def generate_path(base_path, *directories): - path = os.path.join(base_path, *directories) - if not os.path.exists(path): - os.makedirs(path) - return path diff --git a/cv_lib/cv_lib/segmentation/utils.py b/cv_lib/cv_lib/segmentation/utils.py index 07951e88..9c68d398 100644 --- a/cv_lib/cv_lib/segmentation/utils.py +++ b/cv_lib/cv_lib/segmentation/utils.py @@ -2,38 +2,8 @@ # Licensed under the MIT License. import numpy as np -from deepseismic_interpretation.dutchf3.data import decode_segmap -from os import path -from PIL import Image -from toolz import pipe - def _chw_to_hwc(image_array_numpy): return np.moveaxis(image_array_numpy, 0, -1) -def save_images(pred_dict, output_dir, num_classes, colours, extra_identifier=""): - for id in pred_dict: - save_image( - pred_dict[id].unsqueeze(0).cpu().numpy(), - output_dir, - num_classes, - colours, - extra_identifier=extra_identifier, - ) - - -def save_image(image_numpy_array, output_dir, num_classes, colours, extra_identifier=""): - """Save segmentation map as image - - Args: - image_numpy_array (numpy.Array): numpy array that represents an image - output_dir ([type]): - num_classes ([type]): [description] - colours ([type]): [description] - extra_identifier (str, optional): [description]. Defaults to "". - """ - im_array = decode_segmap(image_numpy_array, n_classes=num_classes, label_colours=colours,) - im = pipe((im_array * 255).astype(np.uint8).squeeze(), _chw_to_hwc, Image.fromarray,) - filename = path.join(output_dir, f"{id}_{extra_identifier}.png") - im.save(filename) diff --git a/cv_lib/cv_lib/utils.py b/cv_lib/cv_lib/utils.py index d3e41aeb..8af56d16 100644 --- a/cv_lib/cv_lib/utils.py +++ b/cv_lib/cv_lib/utils.py @@ -1,6 +1,52 @@ + +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + import os import logging +from PIL import Image +import numpy as np +from matplotlib import pyplot as plt + +def normalize(array): + """ + Normalizes a segmentation mask array to be in [0,1] range + for use with PIL.Image + """ + min = array.min() + return (array - min) / (array.max() - min) +def mask_to_disk(mask, fname, cmap_name="Paired"): + """ + write segmentation mask to disk using a particular colormap + """ + cmap = plt.get_cmap(cmap_name) + Image.fromarray(cmap(normalize(mask), bytes=True)).save(fname) + +def image_to_disk(mask, fname, cmap_name="seismic"): + """ + write segmentation image to disk using a particular colormap + """ + cmap = plt.get_cmap(cmap_name) + Image.fromarray(cmap(normalize(mask), bytes=True)).save(fname) + +def decode_segmap(label_mask, colormap_name="Paired"): + """ + Decode segmentation class labels into a colour image + Args: + label_mask (np.ndarray): an (N,H,W) array of integer values denoting + the class label at each spatial location. + Returns: + (np.ndarray): the resulting decoded color image (NCHW). + """ + out = np.zeros((label_mask.shape[0], 3, label_mask.shape[1], label_mask.shape[2])) + cmap = plt.get_cmap(colormap_name) + # loop over the batch + for i in range(label_mask.shape[0]): + im = Image.fromarray(cmap(normalize(label_mask[i, :, :]), bytes=True)).convert("RGB") + out[i, :, :, :] = np.array(im).swapaxes(0, 2).swapaxes(1, 2) + + return out def load_log_configuration(log_config_file): """ @@ -17,3 +63,10 @@ def load_log_configuration(log_config_file): logging.getLogger(__name__).error("Failed to load configuration from %s!", log_config_file) logging.getLogger(__name__).debug(str(e), exc_info=True) raise e + + +def generate_path(base_path, *directories): + path = os.path.join(base_path, *directories) + if not os.path.exists(path): + os.makedirs(path) + return path diff --git a/examples/interpretation/notebooks/Dutch_F3_patch_model_training_and_evaluation.ipynb b/examples/interpretation/notebooks/Dutch_F3_patch_model_training_and_evaluation.ipynb index 28351947..bffbe756 100644 --- a/examples/interpretation/notebooks/Dutch_F3_patch_model_training_and_evaluation.ipynb +++ b/examples/interpretation/notebooks/Dutch_F3_patch_model_training_and_evaluation.ipynb @@ -159,16 +159,16 @@ ")\n", "\n", "from cv_lib.segmentation.dutchf3.utils import (\n", - " current_datetime,\n", - " generate_path,\n", + " current_datetime, \n", " git_branch,\n", " git_hash,\n", " np_to_tb,\n", ")\n", "\n", + "from cv_lib.utils import generate_path\n", + "\n", "from deepseismic_interpretation.dutchf3.data import (\n", - " get_patch_loader,\n", - " decode_segmap,\n", + " get_patch_loader, \n", " get_test_loader,\n", ")\n", "\n", @@ -1100,4 +1100,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} +} \ No newline at end of file diff --git a/experiments/interpretation/dutchf3_patch/local/test.py b/experiments/interpretation/dutchf3_patch/local/test.py index 4162d0d4..6aa68062 100644 --- a/experiments/interpretation/dutchf3_patch/local/test.py +++ b/experiments/interpretation/dutchf3_patch/local/test.py @@ -24,15 +24,14 @@ import torch import torch.nn.functional as F from albumentations import Compose, Normalize, PadIfNeeded, Resize -from matplotlib import cm -from PIL import Image from toolz import compose, curry, itertoolz, pipe, take from torch.utils import data from cv_lib.segmentation import models -from cv_lib.segmentation.dutchf3.utils import current_datetime, generate_path, git_branch, git_hash -from cv_lib.utils import load_log_configuration -from deepseismic_interpretation.dutchf3.data import add_patch_depth_channels, get_seismic_labels, get_test_loader +from cv_lib.segmentation.dutchf3.utils import current_datetime, git_branch, git_hash + +from cv_lib.utils import load_log_configuration, mask_to_disk, generate_path +from deepseismic_interpretation.dutchf3.data import add_patch_depth_channels, get_test_loader from default import _C as config from default import update_config @@ -95,21 +94,6 @@ def reset(self): self.confusion_matrix = np.zeros((self.n_classes, self.n_classes)) -def normalize(array): - """ - Normalizes a segmentation mask array to be in [0,1] range - """ - min = array.min() - return (array - min) / (array.max() - min) - - -def mask_to_disk(mask, fname): - """ - write segmentation mask to disk using a particular colormap - """ - Image.fromarray(cm.gist_earth(normalize(mask), bytes=True)).save(fname) - - def _transform_CHW_to_HWC(numpy_array): return np.moveaxis(numpy_array, 0, -1) @@ -245,24 +229,6 @@ def _patch_label_2d( output = output_p[:, :, ps:-ps, ps:-ps] return output - -@curry -def to_image(label_mask, n_classes=6): - label_colours = get_seismic_labels() - r = label_mask.copy() - g = label_mask.copy() - b = label_mask.copy() - for ll in range(0, n_classes): - r[label_mask == ll] = label_colours[ll, 0] - g[label_mask == ll] = label_colours[ll, 1] - b[label_mask == ll] = label_colours[ll, 2] - rgb = np.zeros((label_mask.shape[0], label_mask.shape[1], label_mask.shape[2], 3)) - rgb[:, :, :, 0] = r - rgb[:, :, :, 1] = g - rgb[:, :, :, 2] = b - return rgb - - def _evaluate_split( split, section_aug, model, pre_processing, output_processing, device, running_metrics_overall, config, debug=False, ): diff --git a/experiments/interpretation/dutchf3_patch/local/train.py b/experiments/interpretation/dutchf3_patch/local/train.py index 1d4e3348..c180f2c5 100644 --- a/experiments/interpretation/dutchf3_patch/local/train.py +++ b/experiments/interpretation/dutchf3_patch/local/train.py @@ -31,9 +31,9 @@ from cv_lib.event_handlers.tensorboard_handlers import create_summary_writer from cv_lib.segmentation import extract_metric_from, models from cv_lib.segmentation.dutchf3.engine import create_supervised_evaluator, create_supervised_trainer -from cv_lib.segmentation.dutchf3.utils import current_datetime, generate_path, git_branch, git_hash +from cv_lib.segmentation.dutchf3.utils import current_datetime, git_branch, git_hash from cv_lib.segmentation.metrics import class_accuracy, class_iou, mean_class_accuracy, mean_iou, pixelwise_accuracy -from cv_lib.utils import load_log_configuration +from cv_lib.utils import load_log_configuration, generate_path from deepseismic_interpretation.dutchf3.data import get_patch_loader from default import _C as config from default import update_config @@ -124,6 +124,7 @@ def run(*options, cfg=None, debug=False): # Training and Validation Loaders: TrainPatchLoader = get_patch_loader(config) + logging.info(f"Using {TrainPatchLoader}") train_set = TrainPatchLoader( config.DATASET.ROOT, config.DATASET.NUM_CLASSES, @@ -131,7 +132,9 @@ def run(*options, cfg=None, debug=False): is_transform=True, stride=config.TRAIN.STRIDE, patch_size=config.TRAIN.PATCH_SIZE, - augmentations=train_aug + augmentations=train_aug, + #augmentations=Resize(config.TRAIN.AUGMENTATIONS.RESIZE.HEIGHT, config.TRAIN.AUGMENTATIONS.RESIZE.WIDTH, always_apply=True), + debug=True ) logger.info(train_set) n_classes = train_set.n_classes @@ -143,6 +146,8 @@ def run(*options, cfg=None, debug=False): stride=config.TRAIN.STRIDE, patch_size=config.TRAIN.PATCH_SIZE, augmentations=val_aug, + #augmentations=Resize(config.TRAIN.AUGMENTATIONS.RESIZE.HEIGHT, config.TRAIN.AUGMENTATIONS.RESIZE.WIDTH, always_apply=True), + debug=True ) logger.info(val_set) @@ -229,7 +234,7 @@ def log_validation_results(engine): logging_handlers.log_metrics(engine, evaluator, stage="Validation") # dump validation set metrics at the very end for debugging purposes if engine.state.epoch == config.TRAIN.END_EPOCH and debug: - fname = f"metrics_test_{config_file_name}_{config.TRAIN.MODEL_DIR}.json" + fname = f"metrics_{config_file_name}_{config.TRAIN.MODEL_DIR}.json" metrics = evaluator.state.metrics out_dict = {x: metrics[x] for x in ["nll", "pixacc", "mca", "mIoU"]} with open(fname, "w") as fid: diff --git a/interpretation/deepseismic_interpretation/dutchf3/data.py b/interpretation/deepseismic_interpretation/dutchf3/data.py index 01f42f14..e580b4bb 100644 --- a/interpretation/deepseismic_interpretation/dutchf3/data.py +++ b/interpretation/deepseismic_interpretation/dutchf3/data.py @@ -6,6 +6,10 @@ import segyio from os import path import scipy +from cv_lib.utils import generate_path, mask_to_disk, image_to_disk + +from matplotlib import pyplot as plt +from PIL import Image # bugfix for scipy imports import scipy.misc @@ -17,7 +21,7 @@ from deepseismic_interpretation.dutchf3.utils.batch import ( interpolate_to_fit_data, parse_labels_in_image, - get_coordinates_for_slice + get_coordinates_for_slice, ) @@ -118,16 +122,10 @@ class SectionLoader(data.Dataset): :param str split: split file to use for loading patches :param bool is_transform: Transform patch to dimensions expected by PyTorch :param list augmentations: Data augmentations to apply to patches + :param bool debug: enable debugging output """ - def __init__( - self, - data_dir, - n_classes, - split="train", - is_transform=True, - augmentations=None - ): + def __init__(self, data_dir, n_classes, split="train", is_transform=True, augmentations=None, debug=False): self.split = split self.data_dir = data_dir self.is_transform = is_transform @@ -179,6 +177,7 @@ class TrainSectionLoader(SectionLoader): :param list augmentations: Data augmentations to apply to patches :param str seismic_path: Override file path for seismic data :param str label_path: Override file path for label data + :param bool debug: enable debugging output """ def __init__( @@ -190,6 +189,7 @@ def __init__( augmentations=None, seismic_path=None, label_path=None, + debug=False, ): super(TrainSectionLoader, self).__init__( data_dir, @@ -199,6 +199,7 @@ def __init__( augmentations=augmentations, seismic_path=seismic_path, label_path=label_path, + debug=debug, ) if seismic_path is not None and label_path is not None: @@ -231,6 +232,7 @@ class TrainSectionLoaderWithDepth(TrainSectionLoader): :param list augmentations: Data augmentations to apply to patches :param str seismic_path: Override file path for seismic data :param str label_path: Override file path for label data + :param bool debug: enable debugging output """ def __init__( @@ -242,6 +244,7 @@ def __init__( augmentations=None, seismic_path=None, label_path=None, + debug=False, ): super(TrainSectionLoaderWithDepth, self).__init__( data_dir, @@ -251,6 +254,7 @@ def __init__( augmentations=augmentations, seismic_path=seismic_path, label_path=label_path, + debug=debug, ) self.seismic = add_section_depth_channels(self.seismic) # NCWH @@ -292,6 +296,7 @@ class TestSectionLoader(SectionLoader): :param list augmentations: Data augmentations to apply to patches :param str seismic_path: Override file path for seismic data :param str label_path: Override file path for label data + :param bool debug: enable debugging output """ def __init__( @@ -303,9 +308,10 @@ def __init__( augmentations=None, seismic_path=None, label_path=None, + debug=False, ): super(TestSectionLoader, self).__init__( - data_dir, n_classes, split=split, is_transform=is_transform, augmentations=augmentations + data_dir, n_classes, split=split, is_transform=is_transform, augmentations=augmentations, debug=debug, ) if "test1" in self.split: @@ -342,6 +348,7 @@ class TestSectionLoaderWithDepth(TestSectionLoader): :param list augmentations: Data augmentations to apply to patches :param str seismic_path: Override file path for seismic data :param str label_path: Override file path for label data + :param bool debug: enable debugging output """ def __init__( @@ -353,6 +360,7 @@ def __init__( augmentations=None, seismic_path=None, label_path=None, + debug=False, ): super(TestSectionLoaderWithDepth, self).__init__( data_dir, @@ -362,6 +370,7 @@ def __init__( augmentations=augmentations, seismic_path=seismic_path, label_path=label_path, + debug=debug, ) self.seismic = add_section_depth_channels(self.seismic) # NCWH @@ -408,18 +417,11 @@ class PatchLoader(data.Dataset): :param str split: split file to use for loading patches :param bool is_transform: Transform patch to dimensions expected by PyTorch :param list augmentations: Data augmentations to apply to patches - :param str seismic_path: Override file path for seismic data - :param str label_path: Override file path for label data + :param bool debug: enable debugging output """ def __init__( - self, - data_dir, - n_classes, - stride=30, - patch_size=99, - is_transform=True, - augmentations=None + self, data_dir, n_classes, stride=30, patch_size=99, is_transform=True, augmentations=None, debug=False, ): self.data_dir = data_dir self.is_transform = is_transform @@ -428,6 +430,7 @@ def __init__( self.patches = list() self.patch_size = patch_size self.stride = stride + self.debug=debug def pad_volume(self, volume): """ @@ -445,8 +448,7 @@ def __getitem__(self, index): # Shift offsets the padding that is added in training # shift = self.patch_size if "test" not in self.split else 0 - # TODO: Remember we are cancelling the shift since we no longer pad - # issue: https://github.com/microsoft/seismic-deeplearning/issues/273 + # Remember we are cancelling the shift since we no longer pad shift = 0 idx, xdx, ddx = int(idx) + shift, int(xdx) + shift, int(ddx) + shift @@ -463,6 +465,13 @@ def __getitem__(self, index): augmented_dict = self.augmentations(image=im, mask=lbl) im, lbl = augmented_dict["image"], augmented_dict["mask"] + # dump images and labels to disk + if self.debug: + outdir = f"patchLoader_{self.split}_{'aug' if self.augmentations is not None else 'noaug'}" + generate_path(outdir) + image_to_disk(im, f"{outdir}/{index}_img.png") + mask_to_disk(lbl, f"{outdir}/{index}_lbl.png") + if self.is_transform: im, lbl = self.transform(im, lbl) return im, lbl @@ -484,9 +493,12 @@ class TestPatchLoader(PatchLoader): :param int patch_size: Size of patch for training :param bool is_transform: Transform patch to dimensions expected by PyTorch :param list augmentations: Data augmentations to apply to patches + :param bool debug: enable debugging output """ - def __init__(self, data_dir, n_classes, stride=30, patch_size=99, is_transform=True, augmentations=None): + def __init__( + self, data_dir, n_classes, stride=30, patch_size=99, is_transform=True, augmentations=None, debug=False + ): super(TestPatchLoader, self).__init__( data_dir, n_classes, @@ -494,20 +506,13 @@ def __init__(self, data_dir, n_classes, stride=30, patch_size=99, is_transform=T patch_size=patch_size, is_transform=is_transform, augmentations=augmentations, + debug=debug, ) ## Warning: this is not used or tested raise NotImplementedError("This class is not correctly implemented.") self.seismic = np.load(_train_data_for(self.data_dir)) self.labels = np.load(_train_labels_for(self.data_dir)) - # We are in test mode. Only read the given split. The other one might not - # be available. - # If txt_path is not provided, it will be assumed as below. Otherwise, provided path will be used for - # loading txt file and create patches. - if not txt_path: - self.split = "test1" # TODO: Fix this can also be test2 - # issue: https://github.com/microsoft/seismic-deeplearning/issues/274 - txt_path = path.join(self.data_dir, "splits", "patch_" + self.split + ".txt") patch_list = tuple(open(txt_path, "r")) patch_list = [id_.rstrip() for id_ in patch_list] self.patches = patch_list @@ -522,8 +527,7 @@ class TrainPatchLoader(PatchLoader): :param str split: split file to use for loading patches :param bool is_transform: Transform patch to dimensions expected by PyTorch :param list augmentations: Data augmentations to apply to patches - :param str seismic_path: Override file path for seismic data - :param str label_path: Override file path for label data + :param bool debug: enable debugging output """ def __init__( @@ -537,6 +541,7 @@ def __init__( augmentations=None, seismic_path=None, label_path=None, + debug=False, ): super(TrainPatchLoader, self).__init__( data_dir, @@ -544,7 +549,8 @@ def __init__( stride=stride, patch_size=patch_size, is_transform=is_transform, - augmentations=augmentations + augmentations=augmentations, + debug=debug, ) warnings.warn("This no longer pads the volume") @@ -579,8 +585,7 @@ class TrainPatchLoaderWithDepth(TrainPatchLoader): :param str split: split file to use for loading patches :param bool is_transform: Transform patch to dimensions expected by PyTorch :param list augmentations: Data augmentations to apply to patches - :param str seismic_path: Override file path for seismic data - :param str label_path: Override file path for label data + :param bool debug: enable debugging output """ def __init__( @@ -593,6 +598,7 @@ def __init__( augmentations=None, seismic_path=None, label_path=None, + debug=False, ): super(TrainPatchLoaderWithDepth, self).__init__( data_dir, @@ -603,6 +609,7 @@ def __init__( augmentations=augmentations, seismic_path=seismic_path, label_path=label_path, + debug=debug, ) def __getitem__(self, index): @@ -612,8 +619,7 @@ def __getitem__(self, index): # Shift offsets the padding that is added in training # shift = self.patch_size if "test" not in self.split else 0 - # TODO: Remember we are cancelling the shift since we no longer pad - # issue https://github.com/microsoft/seismic-deeplearning/issues/273 + # Remember we are cancelling the shift since we no longer pad shift = 0 idx, xdx, ddx = int(idx) + shift, int(xdx) + shift, int(ddx) + shift @@ -625,8 +631,6 @@ def __getitem__(self, index): lbl = self.labels[idx : idx + self.patch_size, xdx, ddx : ddx + self.patch_size] im, lbl = _transform_WH_to_HW(im), _transform_WH_to_HW(lbl) - # TODO: Add check for rotation augmentations and raise warning if found - # issue: https://github.com/microsoft/seismic-deeplearning/issues/275 if self.augmentations is not None: augmented_dict = self.augmentations(image=im, mask=lbl) im, lbl = augmented_dict["image"], augmented_dict["mask"] @@ -657,6 +661,7 @@ class TrainPatchLoaderWithSectionDepth(TrainPatchLoader): :param list augmentations: Data augmentations to apply to patches :param str seismic_path: Override file path for seismic data :param str label_path: Override file path for label data + :param bool debug: enable debugging output """ def __init__( @@ -670,6 +675,7 @@ def __init__( augmentations=None, seismic_path=None, label_path=None, + debug=False, ): super(TrainPatchLoaderWithSectionDepth, self).__init__( data_dir, @@ -681,6 +687,7 @@ def __init__( augmentations=augmentations, seismic_path=seismic_path, label_path=label_path, + debug=debug, ) self.seismic = add_section_depth_channels(self.seismic) @@ -691,8 +698,7 @@ def __getitem__(self, index): # Shift offsets the padding that is added in training # shift = self.patch_size if "test" not in self.split else 0 - # TODO: Remember we are cancelling the shift since we no longer pad - # issue: https://github.com/microsoft/seismic-deeplearning/issues/273 + # Remember we are cancelling the shift since we no longer pad shift = 0 idx, xdx, ddx = int(idx) + shift, int(xdx) + shift, int(ddx) + shift @@ -712,6 +718,13 @@ def __getitem__(self, index): im, lbl = augmented_dict["image"], augmented_dict["mask"] im = _transform_HWC_to_CHW(im) + # dump images and labels to disk + if self.debug: + outdir = f"patchLoaderWithSectionDepth_{self.split}_{'aug' if self.augmentations is not None else 'noaug'}" + generate_path(outdir) + image_to_disk(im[0,:,:], f"{outdir}/{index}_img.png") + mask_to_disk(lbl, f"{outdir}/{index}_lbl.png") + if self.is_transform: im, lbl = self.transform(im, lbl) return im, lbl @@ -796,32 +809,3 @@ def add_section_depth_channels(sections_numpy): image[1, :, :, row] = const image[2] = image[0] * image[1] return np.swapaxes(image, 0, 1) - - -def get_seismic_labels(): - return np.asarray( - [[69, 117, 180], [145, 191, 219], [224, 243, 248], [254, 224, 144], [252, 141, 89], [215, 48, 39]] - ) - - -@curry -def decode_segmap(label_mask, n_classes=None, label_colours=get_seismic_labels()): - """Decode segmentation class labels into a colour image - Args: - label_mask (np.ndarray): an (N,H,W) array of integer values denoting - the class label at each spatial location. - Returns: - (np.ndarray): the resulting decoded color image (NCHW). - """ - r = label_mask.copy() - g = label_mask.copy() - b = label_mask.copy() - for ll in range(0, n_classes): - r[label_mask == ll] = label_colours[ll, 0] - g[label_mask == ll] = label_colours[ll, 1] - b[label_mask == ll] = label_colours[ll, 2] - rgb = np.zeros((label_mask.shape[0], label_mask.shape[1], label_mask.shape[2], 3)) - rgb[:, :, :, 0] = r / 255.0 - rgb[:, :, :, 1] = g / 255.0 - rgb[:, :, :, 2] = b / 255.0 - return np.transpose(rgb, (0, 3, 1, 2)) diff --git a/tests/cicd/src/check_performance.py b/tests/cicd/src/check_performance.py index 5df69dda..925e6ad6 100644 --- a/tests/cicd/src/check_performance.py +++ b/tests/cicd/src/check_performance.py @@ -50,12 +50,12 @@ def main(args): else: # process validation results - assert data['Pixelwise Accuracy :'] > 0.0 - assert data['Pixelwise Accuracy :'] <= 1.0 + assert data['pixacc'] > 0.0 + assert data['pixacc'] <= 1.0 # TODO make these into proper tests - # assert data['Pixelwise Accuracy :'] == 1.0 + # assert data['pixacc'] == 1.0 # TODO: add more tests as we fix performance - # assert data['Avg loss :'] < 1e-3 + # assert data['mIoU'] < 1e-3 logging.info("all done") From 5484702cd079d0ca115ffcad6448dd314e4b19a0 Mon Sep 17 00:00:00 2001 From: Max Kaznady Date: Tue, 12 May 2020 20:02:07 +0000 Subject: [PATCH 13/20] added data dumps to the code --- .../dutchf3_patch/local/test.py | 33 ++++++++-- .../dutchf3/data.py | 61 ++++++++++++++++--- scripts/gen_checkerboard.py | 8 +++ 3 files changed, 88 insertions(+), 14 deletions(-) diff --git a/experiments/interpretation/dutchf3_patch/local/test.py b/experiments/interpretation/dutchf3_patch/local/test.py index 6aa68062..67352454 100644 --- a/experiments/interpretation/dutchf3_patch/local/test.py +++ b/experiments/interpretation/dutchf3_patch/local/test.py @@ -30,7 +30,7 @@ from cv_lib.segmentation import models from cv_lib.segmentation.dutchf3.utils import current_datetime, git_branch, git_hash -from cv_lib.utils import load_log_configuration, mask_to_disk, generate_path +from cv_lib.utils import load_log_configuration, mask_to_disk, generate_path, image_to_disk from deepseismic_interpretation.dutchf3.data import add_patch_depth_channels, get_test_loader from default import _C as config from default import update_config @@ -201,7 +201,7 @@ def _output_processing_pipeline(config, output): def _patch_label_2d( - model, img, pre_processing, output_processing, patch_size, stride, batch_size, device, num_classes, + model, img, pre_processing, output_processing, patch_size, stride, batch_size, device, num_classes, split, debug ): """Processes a whole section """ @@ -220,11 +220,30 @@ def _patch_label_2d( dim=0, ) + # dump the data right before it's being put into the model + if debug: + outdir = f"batch_{split}" + generate_path(outdir) + for i in range(batch.shape[0]): + image_to_disk(np.array(batch[i,0,:,:]), f"{outdir}/{batch_indexes[i][0]}_{batch_indexes[i][1]}_img.png") + model_output = model(batch.to(device)) + for (hdx, wdx), output in zip(batch_indexes, model_output.detach().cpu()): output = output_processing(output) output_p[:, :, hdx + ps : hdx + ps + patch_size, wdx + ps : wdx + ps + patch_size,] += output + # dump the data right before it's being put into the model + if debug: + outdir = f"batch_{split}" + generate_path(outdir) + for i in range(batch.shape[0]): + image_to_disk(np.array(batch[i, 0, :, :]), + f"{outdir}/{batch_indexes[i][0]}_{batch_indexes[i][1]}_img.png") + # now dump model predictions + for nclass in range(num_classes): + mask_to_disk(np.array(model_output[i,nclass,:,:].detach().cpu()), f"{outdir}/{batch_indexes[i][0]}_{batch_indexes[i][1]}_class_{nclass}_pred.png") + # crop the output_p in the middle output = output_p[:, :, ps:-ps, ps:-ps] return output @@ -235,12 +254,14 @@ def _evaluate_split( logger = logging.getLogger(__name__) TestSectionLoader = get_test_loader(config) + print(TestSectionLoader) test_set = TestSectionLoader( config.DATASET.ROOT, config.DATASET.NUM_CLASSES, split=split, is_transform=True, - augmentations=section_aug + augmentations=section_aug, + debug=debug ) n_classes = test_set.n_classes @@ -253,10 +274,10 @@ def _evaluate_split( try: output_dir = generate_path( - config.OUTPUT_DIR + "_test", git_branch(), git_hash(), config.MODEL.NAME, current_datetime(), + f"{config.OUTPUT_DIR}_test_{split}", git_branch(), git_hash(), config.MODEL.NAME, current_datetime(), ) except TypeError: - output_dir = generate_path(config.OUTPUT_DIR + "_test", config.MODEL.NAME, current_datetime(),) + output_dir = generate_path(f"{config.OUTPUT_DIR}_test_{split}", config.MODEL.NAME, current_datetime(),) running_metrics_split = runningScore(n_classes) @@ -278,6 +299,8 @@ def _evaluate_split( config.VALIDATION.BATCH_SIZE_PER_GPU, device, n_classes, + split, + debug ) pred = outputs.detach().max(1)[1].numpy() diff --git a/interpretation/deepseismic_interpretation/dutchf3/data.py b/interpretation/deepseismic_interpretation/dutchf3/data.py index e580b4bb..5bb4abce 100644 --- a/interpretation/deepseismic_interpretation/dutchf3/data.py +++ b/interpretation/deepseismic_interpretation/dutchf3/data.py @@ -132,6 +132,7 @@ def __init__(self, data_dir, n_classes, split="train", is_transform=True, augmen self.augmentations = augmentations self.n_classes = n_classes self.sections = list() + self.debug=debug def __len__(self): return len(self.sections) @@ -150,6 +151,12 @@ def __getitem__(self, index): im, lbl = _transform_WH_to_HW(im), _transform_WH_to_HW(lbl) + if self.debug and "test" in self.split: + outdir = f"sectionLoader_{self.split}_raw" + generate_path(outdir) + image_to_disk(im, f"{outdir}/index_{index}_section_{section_name}_img.png") + mask_to_disk(lbl, f"{outdir}/index_{index}_section_{section_name}_lbl.png") + if self.augmentations is not None: augmented_dict = self.augmentations(image=im, mask=lbl) im, lbl = augmented_dict["image"], augmented_dict["mask"] @@ -157,6 +164,12 @@ def __getitem__(self, index): if self.is_transform: im, lbl = self.transform(im, lbl) + if self.debug and "test" in self.split: + outdir = f"sectionLoader_{self.split}_{'aug' if self.augmentations is not None else 'noaug'}" + generate_path(outdir) + image_to_disk(np.array(im[0]), f"{outdir}/index_{index}_section_{section_name}_img.png") + mask_to_disk(np.array(lbl[0]), f"{outdir}/index_{index}_section_{section_name}_lbl.png") + return im, lbl def transform(self, img, lbl): @@ -390,6 +403,13 @@ def __getitem__(self, index): im, lbl = _transform_WH_to_HW(im), _transform_WH_to_HW(lbl) + # dump images before augmentation + if self.debug: + outdir = f"testSectionLoaderWithDepth_{self.split}_raw" + generate_path(outdir) + image_to_disk(im[0, :, :], f"{outdir}/index_{index}_section_{section_name}_img.png") + mask_to_disk(lbl, f"{outdir}/index_{index}_section_{section_name}_lbl.png") + if self.augmentations is not None: im = _transform_CHW_to_HWC(im) augmented_dict = self.augmentations(image=im, mask=lbl) @@ -399,6 +419,13 @@ def __getitem__(self, index): if self.is_transform: im, lbl = self.transform(im, lbl) + # dump images and labels to disk after augmentation + if self.debug: + outdir = f"testSectionLoaderWithDepth_{self.split}_{'aug' if self.augmentations is not None else 'noaug'}" + generate_path(outdir) + image_to_disk(im[0, :, :], f"{outdir}/index_{index}_section_{section_name}_img.png") + mask_to_disk(lbl, f"{outdir}/index_{index}_section_{section_name}_lbl.png") + return im, lbl @@ -461,19 +488,27 @@ def __getitem__(self, index): im, lbl = _transform_WH_to_HW(im), _transform_WH_to_HW(lbl) + # dump raw images before augmentation + if self.debug: + outdir = f"patchLoader_{self.split}_raw" + generate_path(outdir) + image_to_disk(im, f"{outdir}/index_{index}_section_{patch_name}_img.png") + mask_to_disk(lbl, f"{outdir}/index_{index}_section_{patch_name}_lbl.png") + if self.augmentations is not None: augmented_dict = self.augmentations(image=im, mask=lbl) im, lbl = augmented_dict["image"], augmented_dict["mask"] + if self.is_transform: + im, lbl = self.transform(im, lbl) + # dump images and labels to disk if self.debug: outdir = f"patchLoader_{self.split}_{'aug' if self.augmentations is not None else 'noaug'}" generate_path(outdir) - image_to_disk(im, f"{outdir}/{index}_img.png") - mask_to_disk(lbl, f"{outdir}/{index}_lbl.png") + image_to_disk(im, f"{outdir}/index_{index}_section_{patch_name}_img.png") + mask_to_disk(lbl, f"{outdir}/index_{index}_section_{patch_name}_lbl.png") - if self.is_transform: - im, lbl = self.transform(im, lbl) return im, lbl def transform(self, img, lbl): @@ -712,21 +747,29 @@ def __getitem__(self, index): im, lbl = _transform_WH_to_HW(im), _transform_WH_to_HW(lbl) + # dump images before augmentation + if self.debug: + outdir = f"patchLoaderWithSectionDepth_{self.split}_raw" + generate_path(outdir) + image_to_disk(im[0,:,:], f"{outdir}/index_{index}_section_{patch_name}_img.png") + mask_to_disk(lbl, f"{outdir}/index_{index}_section_{patch_name}_lbl.png") + if self.augmentations is not None: im = _transform_CHW_to_HWC(im) augmented_dict = self.augmentations(image=im, mask=lbl) im, lbl = augmented_dict["image"], augmented_dict["mask"] im = _transform_HWC_to_CHW(im) - # dump images and labels to disk + if self.is_transform: + im, lbl = self.transform(im, lbl) + + # dump images and labels to disk after augmentation if self.debug: outdir = f"patchLoaderWithSectionDepth_{self.split}_{'aug' if self.augmentations is not None else 'noaug'}" generate_path(outdir) - image_to_disk(im[0,:,:], f"{outdir}/{index}_img.png") - mask_to_disk(lbl, f"{outdir}/{index}_lbl.png") + image_to_disk(im[0,:,:], f"{outdir}/index_{index}_section_{patch_name}_img.png") + mask_to_disk(lbl, f"{outdir}/index_{index}_section_{patch_name}_lbl.png") - if self.is_transform: - im, lbl = self.transform(im, lbl) return im, lbl def __repr__(self): diff --git a/scripts/gen_checkerboard.py b/scripts/gen_checkerboard.py index 3e0d0349..9663041c 100644 --- a/scripts/gen_checkerboard.py +++ b/scripts/gen_checkerboard.py @@ -170,6 +170,14 @@ def main(args): parser.add_argument("--dataroot", help="Root location of the input data", type=str, required=True) parser.add_argument("--dataout", help="Root location of the output data", type=str, required=True) parser.add_argument("--box_size", help="Size of the bounding box", type=int, required=False, default=100) +parser.add_argument( + "--type", + help="Type of data to generate", + type=str, + required=False, + choices=["checkerboard", "gradient", "binary"], + default="checkerboard", +) parser.add_argument("--debug", help="Turn on debug mode", type=bool, required=False, default=False) """ main wrapper with profiler """ From 124bcb62f31571800110a5b6add50d3fb5eb2f3f Mon Sep 17 00:00:00 2001 From: Max Kaznady Date: Tue, 12 May 2020 21:32:30 +0000 Subject: [PATCH 14/20] all dumps work properly now --- .../interpretation/dutchf3_patch/local/test.py | 10 ++-------- .../deepseismic_interpretation/dutchf3/data.py | 12 ++++++------ tests/cicd/main_build.yml | 8 ++++++++ 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/experiments/interpretation/dutchf3_patch/local/test.py b/experiments/interpretation/dutchf3_patch/local/test.py index 67352454..5bbf7ebc 100644 --- a/experiments/interpretation/dutchf3_patch/local/test.py +++ b/experiments/interpretation/dutchf3_patch/local/test.py @@ -220,20 +220,13 @@ def _patch_label_2d( dim=0, ) - # dump the data right before it's being put into the model - if debug: - outdir = f"batch_{split}" - generate_path(outdir) - for i in range(batch.shape[0]): - image_to_disk(np.array(batch[i,0,:,:]), f"{outdir}/{batch_indexes[i][0]}_{batch_indexes[i][1]}_img.png") - model_output = model(batch.to(device)) for (hdx, wdx), output in zip(batch_indexes, model_output.detach().cpu()): output = output_processing(output) output_p[:, :, hdx + ps : hdx + ps + patch_size, wdx + ps : wdx + ps + patch_size,] += output - # dump the data right before it's being put into the model + # dump the data right before it's being put into the model and after scoring if debug: outdir = f"batch_{split}" generate_path(outdir) @@ -354,6 +347,7 @@ def _write_section_file(labels, section_file): def test(*options, cfg=None, debug=False): + update_config(config, options=options, config_file=cfg) n_classes = config.DATASET.NUM_CLASSES diff --git a/interpretation/deepseismic_interpretation/dutchf3/data.py b/interpretation/deepseismic_interpretation/dutchf3/data.py index 5bb4abce..d03ef367 100644 --- a/interpretation/deepseismic_interpretation/dutchf3/data.py +++ b/interpretation/deepseismic_interpretation/dutchf3/data.py @@ -423,8 +423,8 @@ def __getitem__(self, index): if self.debug: outdir = f"testSectionLoaderWithDepth_{self.split}_{'aug' if self.augmentations is not None else 'noaug'}" generate_path(outdir) - image_to_disk(im[0, :, :], f"{outdir}/index_{index}_section_{section_name}_img.png") - mask_to_disk(lbl, f"{outdir}/index_{index}_section_{section_name}_lbl.png") + image_to_disk(np.array(im[0, :, :]), f"{outdir}/index_{index}_section_{section_name}_img.png") + mask_to_disk(np.array(lbl[0, :, :]), f"{outdir}/index_{index}_section_{section_name}_lbl.png") return im, lbl @@ -506,8 +506,8 @@ def __getitem__(self, index): if self.debug: outdir = f"patchLoader_{self.split}_{'aug' if self.augmentations is not None else 'noaug'}" generate_path(outdir) - image_to_disk(im, f"{outdir}/index_{index}_section_{patch_name}_img.png") - mask_to_disk(lbl, f"{outdir}/index_{index}_section_{patch_name}_lbl.png") + image_to_disk(np.array(im[0,:,:]), f"{outdir}/index_{index}_section_{patch_name}_img.png") + mask_to_disk(np.array(lbl[0,:,:]), f"{outdir}/index_{index}_section_{patch_name}_lbl.png") return im, lbl @@ -767,8 +767,8 @@ def __getitem__(self, index): if self.debug: outdir = f"patchLoaderWithSectionDepth_{self.split}_{'aug' if self.augmentations is not None else 'noaug'}" generate_path(outdir) - image_to_disk(im[0,:,:], f"{outdir}/index_{index}_section_{patch_name}_img.png") - mask_to_disk(lbl, f"{outdir}/index_{index}_section_{patch_name}_lbl.png") + image_to_disk(np.array(im[0,:,:]), f"{outdir}/index_{index}_section_{patch_name}_img.png") + mask_to_disk(np.array(lbl[0,:,:]), f"{outdir}/index_{index}_section_{patch_name}_lbl.png") return im, lbl diff --git a/tests/cicd/main_build.yml b/tests/cicd/main_build.yml index 4a0b8f2b..0d867c9f 100644 --- a/tests/cicd/main_build.yml +++ b/tests/cicd/main_build.yml @@ -184,6 +184,7 @@ jobs: 'TEST.SPLIT' 'Both' 'TRAIN.MODEL_DIR' 'no_depth' \ 'DATASET.NUM_CLASSES' 2 'DATASET.CLASS_WEIGHTS' '[1.0, 1.0]' \ 'TEST.MODEL_PATH' ${model}/patch_deconvnet_running_model_0.*.pth \ + 'WORKERS' 1 \ --cfg=configs/patch_deconvnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } pids+=" $!" # export CUDA_VISIBLE_DEVICES=1 @@ -194,6 +195,7 @@ jobs: 'TEST.SPLIT' 'Both' 'TRAIN.MODEL_DIR' 'section_depth' \ 'DATASET.NUM_CLASSES' 2 'DATASET.CLASS_WEIGHTS' '[1.0, 1.0]' \ 'TEST.MODEL_PATH' ${model}/resnet_unet_running_model_0.*.pth \ + 'WORKERS' 1 \ --cfg=configs/unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } pids+=" $!" # export CUDA_VISIBLE_DEVICES=2 @@ -204,6 +206,7 @@ jobs: 'DATASET.NUM_CLASSES' 2 'DATASET.CLASS_WEIGHTS' '[1.0, 1.0]' \ 'TEST.SPLIT' 'Both' 'TRAIN.MODEL_DIR' 'section_depth' \ 'TEST.MODEL_PATH' ${model}/resnet_unet_running_model_0.*.pth \ + 'WORKERS' 1 \ --cfg=configs/seresnet_unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } pids+=" $!" # export CUDA_VISIBLE_DEVICES=3 @@ -215,6 +218,7 @@ jobs: 'TEST.SPLIT' 'Both' 'TRAIN.MODEL_DIR' 'section_depth' \ 'MODEL.PRETRAINED' '/home/alfred/models/hrnetv2_w48_imagenet_pretrained.pth' \ 'TEST.MODEL_PATH' ${model}/seg_hrnet_running_model_0.*.pth \ + 'WORKERS' 1 \ --cfg=configs/hrnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } pids+=" $!" @@ -322,6 +326,7 @@ jobs: { python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' \ 'TEST.SPLIT' 'Both' 'TRAIN.MODEL_DIR' 'no_depth' \ 'TEST.MODEL_PATH' ${model} \ + 'WORKERS' 1 \ --cfg=configs/patch_deconvnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } pids+=" $!" # export CUDA_VISIBLE_DEVICES=1 @@ -333,6 +338,7 @@ jobs: { python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' \ 'TEST.SPLIT' 'Both' 'TRAIN.MODEL_DIR' 'section_depth' \ 'TEST.MODEL_PATH' ${model} \ + 'WORKERS' 1 \ --cfg=configs/unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } pids+=" $!" # export CUDA_VISIBLE_DEVICES=2 @@ -344,6 +350,7 @@ jobs: { python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' \ 'TEST.SPLIT' 'Both' 'TRAIN.MODEL_DIR' 'section_depth' \ 'TEST.MODEL_PATH' ${model} \ + 'WORKERS' 1 \ --cfg=configs/seresnet_unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } pids+=" $!" # export CUDA_VISIBLE_DEVICES=3 @@ -356,6 +363,7 @@ jobs: 'TEST.SPLIT' 'Both' 'TRAIN.MODEL_DIR' 'section_depth' \ 'MODEL.PRETRAINED' '/home/alfred/models/hrnetv2_w48_imagenet_pretrained.pth' \ 'TEST.MODEL_PATH' ${model} \ + 'WORKERS' 1 \ --cfg=configs/hrnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } pids+=" $!" From 6cc58a7a0640f2cef1afd41db16e906c3e805423 Mon Sep 17 00:00:00 2001 From: Max Kaznady Date: Wed, 13 May 2020 16:49:50 +0000 Subject: [PATCH 15/20] fixed build error, added binary dataset --- .../local/configs/patch_deconvnet.yaml | 12 +-- .../dutchf3_patch/local/test.py | 23 +++--- .../dutchf3_patch/local/train.py | 12 +-- .../dutchf3/data.py | 41 +++++----- scripts/gen_checkerboard.py | 77 +++++++++++-------- 5 files changed, 96 insertions(+), 69 deletions(-) diff --git a/experiments/interpretation/dutchf3_patch/local/configs/patch_deconvnet.yaml b/experiments/interpretation/dutchf3_patch/local/configs/patch_deconvnet.yaml index 35787b95..e8bbb295 100644 --- a/experiments/interpretation/dutchf3_patch/local/configs/patch_deconvnet.yaml +++ b/experiments/interpretation/dutchf3_patch/local/configs/patch_deconvnet.yaml @@ -31,14 +31,14 @@ TRAIN: AUGMENTATION: True DEPTH: "none" # Options are none, patch, and section STRIDE: 50 - PATCH_SIZE: 99 + PATCH_SIZE: 100 AUGMENTATIONS: RESIZE: - HEIGHT: 99 - WIDTH: 99 + HEIGHT: 100 + WIDTH: 100 PAD: - HEIGHT: 99 - WIDTH: 99 + HEIGHT: 100 + WIDTH: 100 MEAN: 0.0009997 # 0.0009996710808862074 STD: 0.20977 # 0.20976548783479299 MODEL_DIR: "models" @@ -53,5 +53,5 @@ TEST: INLINE: True CROSSLINE: True POST_PROCESSING: - SIZE: 99 + SIZE: 100 CROP_PIXELS: 0 # Number of pixels to crop top, bottom, left and right diff --git a/experiments/interpretation/dutchf3_patch/local/test.py b/experiments/interpretation/dutchf3_patch/local/test.py index 5bbf7ebc..4b74a68c 100644 --- a/experiments/interpretation/dutchf3_patch/local/test.py +++ b/experiments/interpretation/dutchf3_patch/local/test.py @@ -228,33 +228,38 @@ def _patch_label_2d( # dump the data right before it's being put into the model and after scoring if debug: - outdir = f"batch_{split}" + outdir = f"debug/batch_{split}" generate_path(outdir) for i in range(batch.shape[0]): - image_to_disk(np.array(batch[i, 0, :, :]), - f"{outdir}/{batch_indexes[i][0]}_{batch_indexes[i][1]}_img.png") + image_to_disk( + np.array(batch[i, 0, :, :]), f"{outdir}/{batch_indexes[i][0]}_{batch_indexes[i][1]}_img.png" + ) # now dump model predictions for nclass in range(num_classes): - mask_to_disk(np.array(model_output[i,nclass,:,:].detach().cpu()), f"{outdir}/{batch_indexes[i][0]}_{batch_indexes[i][1]}_class_{nclass}_pred.png") + mask_to_disk( + np.array(model_output[i, nclass, :, :].detach().cpu()), + f"{outdir}/{batch_indexes[i][0]}_{batch_indexes[i][1]}_class_{nclass}_pred.png", + ) # crop the output_p in the middle output = output_p[:, :, ps:-ps, ps:-ps] return output + def _evaluate_split( split, section_aug, model, pre_processing, output_processing, device, running_metrics_overall, config, debug=False, ): logger = logging.getLogger(__name__) TestSectionLoader = get_test_loader(config) - print(TestSectionLoader) + test_set = TestSectionLoader( config.DATASET.ROOT, config.DATASET.NUM_CLASSES, split=split, is_transform=True, augmentations=section_aug, - debug=debug + debug=debug, ) n_classes = test_set.n_classes @@ -267,10 +272,10 @@ def _evaluate_split( try: output_dir = generate_path( - f"{config.OUTPUT_DIR}_test_{split}", git_branch(), git_hash(), config.MODEL.NAME, current_datetime(), + f"debug/{config.OUTPUT_DIR}_test_{split}", git_branch(), git_hash(), config.MODEL.NAME, current_datetime(), ) except TypeError: - output_dir = generate_path(f"{config.OUTPUT_DIR}_test_{split}", config.MODEL.NAME, current_datetime(),) + output_dir = generate_path(f"debug/{config.OUTPUT_DIR}_test_{split}", config.MODEL.NAME, current_datetime(),) running_metrics_split = runningScore(n_classes) @@ -293,7 +298,7 @@ def _evaluate_split( device, n_classes, split, - debug + debug, ) pred = outputs.detach().max(1)[1].numpy() diff --git a/experiments/interpretation/dutchf3_patch/local/train.py b/experiments/interpretation/dutchf3_patch/local/train.py index c180f2c5..ef7c094a 100644 --- a/experiments/interpretation/dutchf3_patch/local/train.py +++ b/experiments/interpretation/dutchf3_patch/local/train.py @@ -73,7 +73,7 @@ def run(*options, cfg=None, debug=False): config.OUTPUT_DIR, git_branch(), git_hash(), config_file_name, config.TRAIN.MODEL_DIR, current_datetime(), ) except TypeError: - output_dir = generate_path(config.OUTPUT_DIR, config_file_name, config.TRAIN.MODEL_DIR, current_datetime(),) + output_dir = generate_path(config.OUTPUT_DIR, config_file_name, config.TRAIN.MODEL_DIR, current_datetime(),) # Logging: load_log_configuration(config.LOG_CONFIG) @@ -133,8 +133,8 @@ def run(*options, cfg=None, debug=False): stride=config.TRAIN.STRIDE, patch_size=config.TRAIN.PATCH_SIZE, augmentations=train_aug, - #augmentations=Resize(config.TRAIN.AUGMENTATIONS.RESIZE.HEIGHT, config.TRAIN.AUGMENTATIONS.RESIZE.WIDTH, always_apply=True), - debug=True + # augmentations=Resize(config.TRAIN.AUGMENTATIONS.RESIZE.HEIGHT, config.TRAIN.AUGMENTATIONS.RESIZE.WIDTH, always_apply=True), + debug=True, ) logger.info(train_set) n_classes = train_set.n_classes @@ -146,14 +146,14 @@ def run(*options, cfg=None, debug=False): stride=config.TRAIN.STRIDE, patch_size=config.TRAIN.PATCH_SIZE, augmentations=val_aug, - #augmentations=Resize(config.TRAIN.AUGMENTATIONS.RESIZE.HEIGHT, config.TRAIN.AUGMENTATIONS.RESIZE.WIDTH, always_apply=True), - debug=True + # augmentations=Resize(config.TRAIN.AUGMENTATIONS.RESIZE.HEIGHT, config.TRAIN.AUGMENTATIONS.RESIZE.WIDTH, always_apply=True), + debug=True, ) logger.info(val_set) if debug: logger.info("Running in debug mode..") - train_set = data.Subset(train_set, range(config.TRAIN.BATCH_SIZE_PER_GPU*config.NUM_DEBUG_BATCHES)) + train_set = data.Subset(train_set, range(config.TRAIN.BATCH_SIZE_PER_GPU * config.NUM_DEBUG_BATCHES)) val_set = data.Subset(val_set, range(config.VALIDATION.BATCH_SIZE_PER_GPU)) train_loader = data.DataLoader( diff --git a/interpretation/deepseismic_interpretation/dutchf3/data.py b/interpretation/deepseismic_interpretation/dutchf3/data.py index d03ef367..70ae12b9 100644 --- a/interpretation/deepseismic_interpretation/dutchf3/data.py +++ b/interpretation/deepseismic_interpretation/dutchf3/data.py @@ -132,7 +132,7 @@ def __init__(self, data_dir, n_classes, split="train", is_transform=True, augmen self.augmentations = augmentations self.n_classes = n_classes self.sections = list() - self.debug=debug + self.debug = debug def __len__(self): return len(self.sections) @@ -152,7 +152,7 @@ def __getitem__(self, index): im, lbl = _transform_WH_to_HW(im), _transform_WH_to_HW(lbl) if self.debug and "test" in self.split: - outdir = f"sectionLoader_{self.split}_raw" + outdir = f"debug/sectionLoader_{self.split}_raw" generate_path(outdir) image_to_disk(im, f"{outdir}/index_{index}_section_{section_name}_img.png") mask_to_disk(lbl, f"{outdir}/index_{index}_section_{section_name}_lbl.png") @@ -165,7 +165,7 @@ def __getitem__(self, index): im, lbl = self.transform(im, lbl) if self.debug and "test" in self.split: - outdir = f"sectionLoader_{self.split}_{'aug' if self.augmentations is not None else 'noaug'}" + outdir = f"debug/sectionLoader_{self.split}_{'aug' if self.augmentations is not None else 'noaug'}" generate_path(outdir) image_to_disk(np.array(im[0]), f"{outdir}/index_{index}_section_{section_name}_img.png") mask_to_disk(np.array(lbl[0]), f"{outdir}/index_{index}_section_{section_name}_lbl.png") @@ -405,8 +405,9 @@ def __getitem__(self, index): # dump images before augmentation if self.debug: - outdir = f"testSectionLoaderWithDepth_{self.split}_raw" + outdir = f"debug/testSectionLoaderWithDepth_{self.split}_raw" generate_path(outdir) + # this needs to take the first dimension of image (no depth) but lbl only has 1 dim image_to_disk(im[0, :, :], f"{outdir}/index_{index}_section_{section_name}_img.png") mask_to_disk(lbl, f"{outdir}/index_{index}_section_{section_name}_lbl.png") @@ -421,7 +422,9 @@ def __getitem__(self, index): # dump images and labels to disk after augmentation if self.debug: - outdir = f"testSectionLoaderWithDepth_{self.split}_{'aug' if self.augmentations is not None else 'noaug'}" + outdir = ( + f"debug/testSectionLoaderWithDepth_{self.split}_{'aug' if self.augmentations is not None else 'noaug'}" + ) generate_path(outdir) image_to_disk(np.array(im[0, :, :]), f"{outdir}/index_{index}_section_{section_name}_img.png") mask_to_disk(np.array(lbl[0, :, :]), f"{outdir}/index_{index}_section_{section_name}_lbl.png") @@ -457,7 +460,7 @@ def __init__( self.patches = list() self.patch_size = patch_size self.stride = stride - self.debug=debug + self.debug = debug def pad_volume(self, volume): """ @@ -475,7 +478,7 @@ def __getitem__(self, index): # Shift offsets the padding that is added in training # shift = self.patch_size if "test" not in self.split else 0 - # Remember we are cancelling the shift since we no longer pad + # Remember we are cancelling the shift since we no longer pad shift = 0 idx, xdx, ddx = int(idx) + shift, int(xdx) + shift, int(ddx) + shift @@ -490,7 +493,7 @@ def __getitem__(self, index): # dump raw images before augmentation if self.debug: - outdir = f"patchLoader_{self.split}_raw" + outdir = f"debug/patchLoader_{self.split}_raw" generate_path(outdir) image_to_disk(im, f"{outdir}/index_{index}_section_{patch_name}_img.png") mask_to_disk(lbl, f"{outdir}/index_{index}_section_{patch_name}_lbl.png") @@ -504,10 +507,10 @@ def __getitem__(self, index): # dump images and labels to disk if self.debug: - outdir = f"patchLoader_{self.split}_{'aug' if self.augmentations is not None else 'noaug'}" + outdir = f"debug/patchLoader_{self.split}_{'aug' if self.augmentations is not None else 'noaug'}" generate_path(outdir) - image_to_disk(np.array(im[0,:,:]), f"{outdir}/index_{index}_section_{patch_name}_img.png") - mask_to_disk(np.array(lbl[0,:,:]), f"{outdir}/index_{index}_section_{patch_name}_lbl.png") + image_to_disk(np.array(im[0, :, :]), f"{outdir}/index_{index}_section_{patch_name}_img.png") + mask_to_disk(np.array(lbl[0, :, :]), f"{outdir}/index_{index}_section_{patch_name}_lbl.png") return im, lbl @@ -654,7 +657,7 @@ def __getitem__(self, index): # Shift offsets the padding that is added in training # shift = self.patch_size if "test" not in self.split else 0 - # Remember we are cancelling the shift since we no longer pad + # Remember we are cancelling the shift since we no longer pad shift = 0 idx, xdx, ddx = int(idx) + shift, int(xdx) + shift, int(ddx) + shift @@ -733,7 +736,7 @@ def __getitem__(self, index): # Shift offsets the padding that is added in training # shift = self.patch_size if "test" not in self.split else 0 - # Remember we are cancelling the shift since we no longer pad + # Remember we are cancelling the shift since we no longer pad shift = 0 idx, xdx, ddx = int(idx) + shift, int(xdx) + shift, int(ddx) + shift @@ -749,9 +752,9 @@ def __getitem__(self, index): # dump images before augmentation if self.debug: - outdir = f"patchLoaderWithSectionDepth_{self.split}_raw" + outdir = f"debug/patchLoaderWithSectionDepth_{self.split}_raw" generate_path(outdir) - image_to_disk(im[0,:,:], f"{outdir}/index_{index}_section_{patch_name}_img.png") + image_to_disk(im[0, :, :], f"{outdir}/index_{index}_section_{patch_name}_img.png") mask_to_disk(lbl, f"{outdir}/index_{index}_section_{patch_name}_lbl.png") if self.augmentations is not None: @@ -765,10 +768,12 @@ def __getitem__(self, index): # dump images and labels to disk after augmentation if self.debug: - outdir = f"patchLoaderWithSectionDepth_{self.split}_{'aug' if self.augmentations is not None else 'noaug'}" + outdir = ( + f"debug/patchLoaderWithSectionDepth_{self.split}_{'aug' if self.augmentations is not None else 'noaug'}" + ) generate_path(outdir) - image_to_disk(np.array(im[0,:,:]), f"{outdir}/index_{index}_section_{patch_name}_img.png") - mask_to_disk(np.array(lbl[0,:,:]), f"{outdir}/index_{index}_section_{patch_name}_lbl.png") + image_to_disk(np.array(im[0, :, :]), f"{outdir}/index_{index}_section_{patch_name}_img.png") + mask_to_disk(np.array(lbl[0, :, :]), f"{outdir}/index_{index}_section_{patch_name}_lbl.png") return im, lbl diff --git a/scripts/gen_checkerboard.py b/scripts/gen_checkerboard.py index 9663041c..616589bc 100644 --- a/scripts/gen_checkerboard.py +++ b/scripts/gen_checkerboard.py @@ -118,30 +118,50 @@ def main(args): # this is the number of classes in Alaudah's Dutch F3 dataset assert test2_labels.max() == 5 - logging.info("train checkerbox") - n_inlines, n_crosslines, n_depth = train_seismic.shape - checkerboard_train_seismic = make_box(n_inlines, n_crosslines, n_depth, args.box_size) - checkerboard_train_seismic = checkerboard_train_seismic.astype(train_seismic.dtype) - checkerboard_train_labels = checkerboard_train_seismic.astype(train_labels.dtype) - # labels are integers and start from zero - checkerboard_train_labels[checkerboard_train_seismic < WHITE_LABEL] = WHITE_LABEL - - # create checkerbox - logging.info("test1 checkerbox") - n_inlines, n_crosslines, n_depth = test1_seismic.shape - checkerboard_test1_seismic = make_box(n_inlines, n_crosslines, n_depth, args.box_size) - checkerboard_test1_seismic = checkerboard_test1_seismic.astype(test1_seismic.dtype) - checkerboard_test1_labels = checkerboard_test1_seismic.astype(test1_labels.dtype) - # labels are integers and start from zero - checkerboard_test1_labels[checkerboard_test1_seismic < WHITE_LABEL] = WHITE_LABEL - - logging.info("test2 checkerbox") - n_inlines, n_crosslines, n_depth = test2_seismic.shape - checkerboard_test2_seismic = make_box(n_inlines, n_crosslines, n_depth, args.box_size) - checkerboard_test2_seismic = checkerboard_test2_seismic.astype(test2_seismic.dtype) - checkerboard_test2_labels = checkerboard_test2_seismic.astype(test2_labels.dtype) - # labels are integers and start from zero - checkerboard_test2_labels[checkerboard_test2_seismic < WHITE_LABEL] = WHITE_LABEL + if args.type == "checkerboard": + logging.info("train checkerbox") + n_inlines, n_crosslines, n_depth = train_seismic.shape + checkerboard_train_seismic = make_box(n_inlines, n_crosslines, n_depth, args.box_size) + checkerboard_train_seismic = checkerboard_train_seismic.astype(train_seismic.dtype) + checkerboard_train_labels = checkerboard_train_seismic.astype(train_labels.dtype) + # labels are integers and start from zero + checkerboard_train_labels[checkerboard_train_seismic < WHITE_LABEL] = WHITE_LABEL + + # create checkerbox + logging.info("test1 checkerbox") + n_inlines, n_crosslines, n_depth = test1_seismic.shape + checkerboard_test1_seismic = make_box(n_inlines, n_crosslines, n_depth, args.box_size) + checkerboard_test1_seismic = checkerboard_test1_seismic.astype(test1_seismic.dtype) + checkerboard_test1_labels = checkerboard_test1_seismic.astype(test1_labels.dtype) + # labels are integers and start from zero + checkerboard_test1_labels[checkerboard_test1_seismic < WHITE_LABEL] = WHITE_LABEL + + logging.info("test2 checkerbox") + n_inlines, n_crosslines, n_depth = test2_seismic.shape + checkerboard_test2_seismic = make_box(n_inlines, n_crosslines, n_depth, args.box_size) + checkerboard_test2_seismic = checkerboard_test2_seismic.astype(test2_seismic.dtype) + checkerboard_test2_labels = checkerboard_test2_seismic.astype(test2_labels.dtype) + # labels are integers and start from zero + checkerboard_test2_labels[checkerboard_test2_seismic < WHITE_LABEL] = WHITE_LABEL + + # substitute gradient dataset instead of checkerboard + elif args.type == "gradient": + pass + # substitute binary dataset instead of checkerboard + elif args.type == "binary": + + logging.info("train binary") + checkerboard_train_seismic = train_seismic * 0 + WHITE + checkerboard_train_labels = train_labels * 0 + WHITE_LABEL + + # create checkerbox + logging.info("test1 binary") + checkerboard_test1_seismic = test1_seismic * 0 + BLACK + checkerboard_test1_labels = test1_labels * 0 + BLACK_LABEL + + logging.info("test2 binary") + checkerboard_test2_seismic = test2_seismic * 0 + BLACK + checkerboard_test2_labels = test2_labels * 0 + BLACK_LABEL logging.info("writing data to disk") mkdir(args.dataout) @@ -166,17 +186,14 @@ def main(args): WHITE = -1 BLACK = 1 WHITE_LABEL = 0 +BLACK_LABEL = BLACK +TYPES = ["checkerboard", "gradient", "binary"] parser.add_argument("--dataroot", help="Root location of the input data", type=str, required=True) parser.add_argument("--dataout", help="Root location of the output data", type=str, required=True) parser.add_argument("--box_size", help="Size of the bounding box", type=int, required=False, default=100) parser.add_argument( - "--type", - help="Type of data to generate", - type=str, - required=False, - choices=["checkerboard", "gradient", "binary"], - default="checkerboard", + "--type", help="Type of data to generate", type=str, required=False, choices=TYPES, default="checkerboard", ) parser.add_argument("--debug", help="Turn on debug mode", type=bool, required=False, default=False) From 88e09bf40a17845762e8a2b2034a1b8aca67ac64 Mon Sep 17 00:00:00 2001 From: Max Kaznady Date: Wed, 13 May 2020 20:49:04 +0000 Subject: [PATCH 16/20] done - now need to test --- .../dutchf3_patch/local/train.py | 2 - scripts/gen_checkerboard.py | 55 ++++++++++++++++++- 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/experiments/interpretation/dutchf3_patch/local/train.py b/experiments/interpretation/dutchf3_patch/local/train.py index ef7c094a..c91f8333 100644 --- a/experiments/interpretation/dutchf3_patch/local/train.py +++ b/experiments/interpretation/dutchf3_patch/local/train.py @@ -133,7 +133,6 @@ def run(*options, cfg=None, debug=False): stride=config.TRAIN.STRIDE, patch_size=config.TRAIN.PATCH_SIZE, augmentations=train_aug, - # augmentations=Resize(config.TRAIN.AUGMENTATIONS.RESIZE.HEIGHT, config.TRAIN.AUGMENTATIONS.RESIZE.WIDTH, always_apply=True), debug=True, ) logger.info(train_set) @@ -146,7 +145,6 @@ def run(*options, cfg=None, debug=False): stride=config.TRAIN.STRIDE, patch_size=config.TRAIN.PATCH_SIZE, augmentations=val_aug, - # augmentations=Resize(config.TRAIN.AUGMENTATIONS.RESIZE.HEIGHT, config.TRAIN.AUGMENTATIONS.RESIZE.WIDTH, always_apply=True), debug=True, ) logger.info(val_set) diff --git a/scripts/gen_checkerboard.py b/scripts/gen_checkerboard.py index 616589bc..6b1e79f8 100644 --- a/scripts/gen_checkerboard.py +++ b/scripts/gen_checkerboard.py @@ -30,6 +30,7 @@ def make_box(n_inlines, n_crosslines, n_depth, box_size): :param n_inlines: dim x :param n_crosslines: dim y :param n_depth: dim z + :param box_size: size of each checkerbox :return: numpy array """ # inline by crossline by depth @@ -70,6 +71,35 @@ def make_box(n_inlines, n_crosslines, n_depth, box_size): # trim excess again return final_box[0:n_inlines, 0:n_crosslines, 0:n_depth] +def make_gradient(n_inlines, n_crosslines, n_depth, box_size, dir="inline"): + """ + Makes a 3D box gradient pattern in a particula direction + + :param n_inlines: dim x + :param n_crosslines: dim y + :param n_depth: dim z + :param box_size: thichkness of gradient box + :param dir: direction of the gradient - can be crossline, inline or depth + :return: numpy array + """ + + axis = ["inline", "crossline", "depth"].index(dir) + n_points = (n_inlines, n_crosslines, n_depth)[axis] + n_classes = np.ceil(float(n_points)/box_size) + logging.info(f"GRADIENT: we will output {n_classes} classes in the {dir} direction") + + output = np.ones((n_inlines, n_crosslines, n_depth)) + start, finish = 0, box_size + for i in range(n_classes): + sl = [slice(None)] * output.ndim + sl[axis] = range(start, finish) + # set all values equal to class number, starting from 0 + output[tuple(sl)] = i + start += box_size + finish = max(n_points, finish+box_size) + + return output + def mkdir(path): """ @@ -146,7 +176,30 @@ def main(args): # substitute gradient dataset instead of checkerboard elif args.type == "gradient": - pass + + logging.info("train gradient") + n_inlines, n_crosslines, n_depth = train_seismic.shape + checkerboard_train_seismic = make_gradient(n_inlines, n_crosslines, n_depth, args.box_size) + checkerboard_train_seismic = checkerboard_train_seismic.astype(train_seismic.dtype) + checkerboard_train_labels = checkerboard_train_seismic.astype(train_labels.dtype) + + # create checkerbox + logging.info("test1 gradient") + n_inlines, n_crosslines, n_depth = test1_seismic.shape + checkerboard_test1_seismic = make_gradient(n_inlines, n_crosslines, n_depth, args.box_size) + checkerboard_test1_seismic = checkerboard_test1_seismic.astype(test1_seismic.dtype) + checkerboard_test1_labels = checkerboard_test1_seismic.astype(test1_labels.dtype) + # labels are integers and start from zero + checkerboard_test1_labels[checkerboard_test1_seismic < WHITE_LABEL] = WHITE_LABEL + + logging.info("test2 gradient") + n_inlines, n_crosslines, n_depth = test2_seismic.shape + checkerboard_test2_seismic = make_gradient(n_inlines, n_crosslines, n_depth, args.box_size) + checkerboard_test2_seismic = checkerboard_test2_seismic.astype(test2_seismic.dtype) + checkerboard_test2_labels = checkerboard_test2_seismic.astype(test2_labels.dtype) + # labels are integers and start from zero + checkerboard_test2_labels[checkerboard_test2_seismic < WHITE_LABEL] = WHITE_LABEL + # substitute binary dataset instead of checkerboard elif args.type == "binary": From 03f84aafee37a06aa404b6d1e025b335f1d80d6e Mon Sep 17 00:00:00 2001 From: Max Kaznady Date: Tue, 19 May 2020 17:45:18 +0000 Subject: [PATCH 17/20] finished dev build script --- environment/anaconda/local/environment.yml | 2 + scripts/dev_build.py | 120 +++++++++++++++++++++ 2 files changed, 122 insertions(+) create mode 100644 scripts/dev_build.py diff --git a/environment/anaconda/local/environment.yml b/environment/anaconda/local/environment.yml index 10ad22f4..d2d202fb 100644 --- a/environment/anaconda/local/environment.yml +++ b/environment/anaconda/local/environment.yml @@ -37,3 +37,5 @@ dependencies: - scipy==1.1.0 - jupytext==1.3.0 - validators + - pyyaml + diff --git a/scripts/dev_build.py b/scripts/dev_build.py new file mode 100644 index 00000000..302d3588 --- /dev/null +++ b/scripts/dev_build.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +""" Please see the def main() function for code description.""" + +""" libraries """ + +import numpy as np +import os +import sys +import yaml +import subprocess + +np.set_printoptions(linewidth=200) +import logging + +# toggle to WARNING when running in production, or use CLI +logging.getLogger().setLevel(logging.DEBUG) +# logging.getLogger().setLevel(logging.WARNING) +import argparse + +parser = argparse.ArgumentParser() + +""" useful information when running from a GIT folder.""" +myname = os.path.realpath(__file__) +mypath = os.path.dirname(myname) +myname = os.path.basename(myname) + +def main(args): + """ + + Runs main build jobs on your local VM. By default setup job is not run, + add --setup to run it (destroys existing environment and creates a new one, along with all the data) + + """ + + logging.info("loading data") + + with open(args.file) as file: + + list = yaml.load(file, Loader=yaml.FullLoader) + logging.info(f"Loaded {file}") + + # run single job + job_names = [x["job"] for x in list["jobs"]] if not args.job else args.job.split(',') + + if not args.setup and "setup" in job_names: + job_names.remove("setup") + + job_list = list["jobs"] + + # copy existing environment + current_env = os.environ.copy() + # modify for conda to work + # TODO: not sure why on DS VM this does not get picked up from the standard environment + current_env["PATH"] = PATH_PREFIX+":"+current_env["PATH"] + + for job in job_list: + job_name = job["job"] + if job_name not in job_names: + continue + + bash = job["steps"][0]["bash"] + + logging.info(f"Running job {job_name}") + + try: + completed = subprocess.run( + # 'set -e && source activate seismic-interpretation && which python && pytest --durations=0 cv_lib/tests/', + bash, + check=True, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + executable=current_env["SHELL"], + env=current_env, + cwd=os.getcwd() + ) + except subprocess.CalledProcessError as err: + logging.info(f'ERROR: \n{err}') + logging.info(f"Have {len(err.stdout)} output bytes: \n{err.stdout.decode('utf-8')}") + sys.exit() + else: + logging.info(f"returncode: {completed.returncode}") + logging.info(f"Have {len(completed.stdout)} output bytes: \n{completed.stdout.decode('utf-8')}") + + logging.info(f"Everything ran! You can try running the same jobs {job_names} on the build VM now") + +""" GLOBAL VARIABLES """ +PATH_PREFIX = "/data/anaconda/envs/seismic-interpretation/bin:/data/anaconda/bin" + +parser.add_argument( + "--file", help="Which yaml file you'd like to read which specifies build info", type=str, required=True +) +parser.add_argument( + "--job", help="CVS list of the job names which you would like to run", type=str, default=None, required=False +) +parser.add_argument("--setup", help="Add setup job", action="store_true") + +""" main wrapper with profiler """ +if __name__ == "__main__": + main(parser.parse_args()) + +# pretty printing of the stack +""" + try: + logging.info('before main') + main(parser.parse_args()) + logging.info('after main') + except: + for frame in traceback.extract_tb(sys.exc_info()[2]): + fname,lineno,fn,text = frame + print ("Error in %s on line %d" % (fname, lineno)) +""" +# optionally enable profiling information +# import cProfile +# name = +# cProfile.run('main.run()', name + '.prof') +# import pstats +# p = pstats.Stats(name + '.prof') +# p.sort_stats('cumulative').print_stats(10) +# p.sort_stats('time').print_stats() From 8c4cb7c340851df059ef3fd34e36215038c18037 Mon Sep 17 00:00:00 2001 From: Max Kaznady Date: Wed, 20 May 2020 19:53:57 +0000 Subject: [PATCH 18/20] updates to tests to run on local machine as well we build --- scripts/dev_build.py | 6 ++++- scripts/gen_checkerboard.py | 2 -- tests/cicd/main_build.yml | 47 ++++++++++++++++++++++++++++--------- 3 files changed, 41 insertions(+), 14 deletions(-) diff --git a/scripts/dev_build.py b/scripts/dev_build.py index 302d3588..76b1d7e0 100644 --- a/scripts/dev_build.py +++ b/scripts/dev_build.py @@ -76,7 +76,11 @@ def main(args): ) except subprocess.CalledProcessError as err: logging.info(f'ERROR: \n{err}') - logging.info(f"Have {len(err.stdout)} output bytes: \n{err.stdout.decode('utf-8')}") + decoded_stdout = err.stdout.decode('utf-8') + log_file = "dev_build.latest_error.log" + logging.info(f"Have {len(err.stdout)} output bytes in {log_file}") + with open(log_file, 'w') as log_file: + log_file.write(decoded_stdout) sys.exit() else: logging.info(f"returncode: {completed.returncode}") diff --git a/scripts/gen_checkerboard.py b/scripts/gen_checkerboard.py index 6b1e79f8..13c9e76c 100644 --- a/scripts/gen_checkerboard.py +++ b/scripts/gen_checkerboard.py @@ -4,7 +4,6 @@ """ libraries """ import numpy as np -import sys import os np.set_printoptions(linewidth=200) @@ -22,7 +21,6 @@ mypath = os.path.dirname(myname) myname = os.path.basename(myname) - def make_box(n_inlines, n_crosslines, n_depth, box_size): """ Makes a 3D box in checkerboard pattern. diff --git a/tests/cicd/main_build.yml b/tests/cicd/main_build.yml index 0d867c9f..37bcfb26 100644 --- a/tests/cicd/main_build.yml +++ b/tests/cicd/main_build.yml @@ -178,46 +178,62 @@ jobs: pids= # export CUDA_VISIBLE_DEVICES=0 # find the latest model which we just trained - model=$(ls -td output/patch_deconvnet/no_depth/* | head -1) + # if we're running on a build VM + model_dir=$(ls -td output/patch_deconvnet/no_depth/* | head -1) + # if we're running in a checked out git repo + [[ -z ${model_dir} ]] && model_dir=$(ls -td output/*/*/patch_deconvnet/no_depth/* | head -1) + model=$(ls -t ${model_dir}/*.pth | head -1) # try running the test script { python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/checkerboard/data' \ 'TEST.SPLIT' 'Both' 'TRAIN.MODEL_DIR' 'no_depth' \ 'DATASET.NUM_CLASSES' 2 'DATASET.CLASS_WEIGHTS' '[1.0, 1.0]' \ - 'TEST.MODEL_PATH' ${model}/patch_deconvnet_running_model_0.*.pth \ + 'TEST.MODEL_PATH' ${model} \ 'WORKERS' 1 \ --cfg=configs/patch_deconvnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } pids+=" $!" # export CUDA_VISIBLE_DEVICES=1 # find the latest model which we just trained - model=$(ls -td output/unet/section_depth/* | head -1) + # if we're running on a build VM + model_dir=$(ls -td output/unet/section_depth/* | head -1) + # if we're running in a checked out git repo + [[ -z ${model_dir} ]] && model_dir=$(ls -td output/*/*/unet/section_depth/* | head -1) + model=$(ls -t ${model_dir}/*.pth | head -1) # try running the test script { python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/checkerboard/data' \ 'TEST.SPLIT' 'Both' 'TRAIN.MODEL_DIR' 'section_depth' \ 'DATASET.NUM_CLASSES' 2 'DATASET.CLASS_WEIGHTS' '[1.0, 1.0]' \ - 'TEST.MODEL_PATH' ${model}/resnet_unet_running_model_0.*.pth \ + 'TEST.MODEL_PATH' ${model} \ 'WORKERS' 1 \ --cfg=configs/unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } pids+=" $!" # export CUDA_VISIBLE_DEVICES=2 # find the latest model which we just trained - model=$(ls -td output/seresnet_unet/section_depth/* | head -1) + # if we're running on a build VM + model_dir=$(ls -td output/seresnet_unet/section_depth/* | head -1) + # if we're running in a checked out git repo + [[ -z ${model_dir} ]] && model_dir=$(ls -td output/*/*/seresnet_unet/section_depth/* | head -1) + model=$(ls -t ${model_dir}/*.pth | head -1) # try running the test script { python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/checkerboard/data' \ 'DATASET.NUM_CLASSES' 2 'DATASET.CLASS_WEIGHTS' '[1.0, 1.0]' \ 'TEST.SPLIT' 'Both' 'TRAIN.MODEL_DIR' 'section_depth' \ - 'TEST.MODEL_PATH' ${model}/resnet_unet_running_model_0.*.pth \ + 'TEST.MODEL_PATH' ${model} \ 'WORKERS' 1 \ --cfg=configs/seresnet_unet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } pids+=" $!" # export CUDA_VISIBLE_DEVICES=3 # find the latest model which we just trained - model=$(ls -td output/hrnet/section_depth/* | head -1) + # if we're running on a build VM + model_dir=$(ls -td output/hrnet/section_depth/* | head -1) + # if we're running in a checked out git repo + [[ -z ${model_dir} ]] && model_dir=$(ls -td output/*/*/hrnet/section_depth/* | head -1) + model=$(ls -t ${model_dir}/*.pth | head -1) # try running the test script { python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/checkerboard/data' \ 'DATASET.NUM_CLASSES' 2 'DATASET.CLASS_WEIGHTS' '[1.0, 1.0]' \ 'TEST.SPLIT' 'Both' 'TRAIN.MODEL_DIR' 'section_depth' \ 'MODEL.PRETRAINED' '/home/alfred/models/hrnetv2_w48_imagenet_pretrained.pth' \ - 'TEST.MODEL_PATH' ${model}/seg_hrnet_running_model_0.*.pth \ + 'TEST.MODEL_PATH' ${model} \ 'WORKERS' 1 \ --cfg=configs/hrnet.yaml --debug ; echo "$?" > "$dir/$BASHPID"; } pids+=" $!" @@ -320,7 +336,10 @@ jobs: pids= # export CUDA_VISIBLE_DEVICES=0 # find the latest model which we just trained + # if we're running on a build VM model_dir=$(ls -td output/patch_deconvnet/no_depth/* | head -1) + # if we're running in a checked out git repo + [[ -z ${model_dir} ]] && model_dir=$(ls -td output/*/*/patch_deconvnet/no_depth/* | head -1) model=$(ls -t ${model_dir}/*.pth | head -1) # try running the test script { python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' \ @@ -331,9 +350,11 @@ jobs: pids+=" $!" # export CUDA_VISIBLE_DEVICES=1 # find the latest model which we just trained + # if we're running on a build VM model_dir=$(ls -td output/unet/section_depth/* | head -1) + # if we're running in a checked out git repo + [[ -z ${model_dir} ]] && model_dir=$(ls -td output/*/*/unet/section_depth/* | head -1) model=$(ls -t ${model_dir}/*.pth | head -1) - # try running the test script { python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' \ 'TEST.SPLIT' 'Both' 'TRAIN.MODEL_DIR' 'section_depth' \ @@ -343,9 +364,11 @@ jobs: pids+=" $!" # export CUDA_VISIBLE_DEVICES=2 # find the latest model which we just trained + # if we're running on a build VM model_dir=$(ls -td output/seresnet_unet/section_depth/* | head -1) + # if we're running in a checked out git repo + [[ -z ${model_dir} ]] && model_dir=$(ls -td output/*/*/seresnet_unet/section_depth/* | head -1) model=$(ls -t ${model_dir}/*.pth | head -1) - # try running the test script { python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' \ 'TEST.SPLIT' 'Both' 'TRAIN.MODEL_DIR' 'section_depth' \ @@ -355,9 +378,11 @@ jobs: pids+=" $!" # export CUDA_VISIBLE_DEVICES=3 # find the latest model which we just trained + # if we're running on a build VM model_dir=$(ls -td output/hrnet/section_depth/* | head -1) + # if we're running in a checked out git repo + [[ -z ${model_dir} ]] && model_dir=$(ls -td output/*/*/hrnet/section_depth/* | head -1) model=$(ls -t ${model_dir}/*.pth | head -1) - # try running the test script { python test.py 'DATASET.ROOT' '/home/alfred/data_dynamic/dutch_f3/data' \ 'TEST.SPLIT' 'Both' 'TRAIN.MODEL_DIR' 'section_depth' \ From 2c5000e9816b4be7164f1064687d8d3ee3a9a845 Mon Sep 17 00:00:00 2001 From: Max Kaznady Date: Wed, 20 May 2020 20:41:35 +0000 Subject: [PATCH 19/20] updated gradient direction in gen_checkerboard --- scripts/gen_checkerboard.py | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/scripts/gen_checkerboard.py b/scripts/gen_checkerboard.py index 13c9e76c..b4796186 100644 --- a/scripts/gen_checkerboard.py +++ b/scripts/gen_checkerboard.py @@ -21,6 +21,7 @@ mypath = os.path.dirname(myname) myname = os.path.basename(myname) + def make_box(n_inlines, n_crosslines, n_depth, box_size): """ Makes a 3D box in checkerboard pattern. @@ -69,6 +70,7 @@ def make_box(n_inlines, n_crosslines, n_depth, box_size): # trim excess again return final_box[0:n_inlines, 0:n_crosslines, 0:n_depth] + def make_gradient(n_inlines, n_crosslines, n_depth, box_size, dir="inline"): """ Makes a 3D box gradient pattern in a particula direction @@ -81,9 +83,9 @@ def make_gradient(n_inlines, n_crosslines, n_depth, box_size, dir="inline"): :return: numpy array """ - axis = ["inline", "crossline", "depth"].index(dir) + axis = GRADIENT_DIR.index(dir) n_points = (n_inlines, n_crosslines, n_depth)[axis] - n_classes = np.ceil(float(n_points)/box_size) + n_classes = int(np.ceil(float(n_points) / box_size)) logging.info(f"GRADIENT: we will output {n_classes} classes in the {dir} direction") output = np.ones((n_inlines, n_crosslines, n_depth)) @@ -94,7 +96,7 @@ def make_gradient(n_inlines, n_crosslines, n_depth, box_size, dir="inline"): # set all values equal to class number, starting from 0 output[tuple(sl)] = i start += box_size - finish = max(n_points, finish+box_size) + finish = min(n_points, finish + box_size) return output @@ -177,14 +179,18 @@ def main(args): logging.info("train gradient") n_inlines, n_crosslines, n_depth = train_seismic.shape - checkerboard_train_seismic = make_gradient(n_inlines, n_crosslines, n_depth, args.box_size) + checkerboard_train_seismic = make_gradient( + n_inlines, n_crosslines, n_depth, args.box_size, dir=args.gradient_dir + ) checkerboard_train_seismic = checkerboard_train_seismic.astype(train_seismic.dtype) checkerboard_train_labels = checkerboard_train_seismic.astype(train_labels.dtype) # create checkerbox logging.info("test1 gradient") n_inlines, n_crosslines, n_depth = test1_seismic.shape - checkerboard_test1_seismic = make_gradient(n_inlines, n_crosslines, n_depth, args.box_size) + checkerboard_test1_seismic = make_gradient( + n_inlines, n_crosslines, n_depth, args.box_size, dir=args.gradient_dir + ) checkerboard_test1_seismic = checkerboard_test1_seismic.astype(test1_seismic.dtype) checkerboard_test1_labels = checkerboard_test1_seismic.astype(test1_labels.dtype) # labels are integers and start from zero @@ -192,7 +198,9 @@ def main(args): logging.info("test2 gradient") n_inlines, n_crosslines, n_depth = test2_seismic.shape - checkerboard_test2_seismic = make_gradient(n_inlines, n_crosslines, n_depth, args.box_size) + checkerboard_test2_seismic = make_gradient( + n_inlines, n_crosslines, n_depth, args.box_size, dir=args.gradient_dir + ) checkerboard_test2_seismic = checkerboard_test2_seismic.astype(test2_seismic.dtype) checkerboard_test2_labels = checkerboard_test2_seismic.astype(test2_labels.dtype) # labels are integers and start from zero @@ -239,6 +247,7 @@ def main(args): WHITE_LABEL = 0 BLACK_LABEL = BLACK TYPES = ["checkerboard", "gradient", "binary"] +GRADIENT_DIR = ["inline", "crossline", "depth"] parser.add_argument("--dataroot", help="Root location of the input data", type=str, required=True) parser.add_argument("--dataout", help="Root location of the output data", type=str, required=True) @@ -246,6 +255,14 @@ def main(args): parser.add_argument( "--type", help="Type of data to generate", type=str, required=False, choices=TYPES, default="checkerboard", ) +parser.add_argument( + "--gradient_dir", + help="Direction in which to build the gradient", + type=str, + required=False, + choices=GRADIENT_DIR, + default="inline", +) parser.add_argument("--debug", help="Turn on debug mode", type=bool, required=False, default=False) """ main wrapper with profiler """ From b844bb1c09d1e2a35e3c0b6df1e9fb7006ba712b Mon Sep 17 00:00:00 2001 From: Max Kaznady Date: Thu, 21 May 2020 01:45:17 +0000 Subject: [PATCH 20/20] increased Dutch F3 timeout --- tests/cicd/main_build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cicd/main_build.yml b/tests/cicd/main_build.yml index 37bcfb26..60c8f86f 100644 --- a/tests/cicd/main_build.yml +++ b/tests/cicd/main_build.yml @@ -268,7 +268,7 @@ jobs: - job: dutchf3_patch dependsOn: checkerboard_dutchf3_patch - timeoutInMinutes: 20 + timeoutInMinutes: 25 displayName: Dutch F3 patch local pool: name: deepseismicagentpool