Merge pull request #34 from deephealthproject/development

New built-in parallel batch loader
deephealthproject · Jun 9, 2021 · c4fdc36 · c4fdc36
2 parents a9cd3e8 + 83d324a
commit c4fdc36
Show file tree

Hide file tree

Showing 26 changed files with 1,455 additions and 346 deletions.
diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
@@ -4,8 +4,8 @@ on: [push, pull_request]
 
 env:
   BUILD_TYPE: "Release"
-  OPENCV_VERSION: "3.4.13"
-  EDDL_VERSION: "v0.8.3a"
+  OPENCV_VERSION: "3.4.14"
+  EDDL_VERSION: "v0.9.2b"
   PROC: 2
 
 jobs:
@@ -14,8 +14,8 @@ jobs:
     strategy:
       matrix:
         cfg:
-          - { os: ubuntu-18.04, c-version: gcc-6, cxx-version: g++-6, generator: "Unix Makefiles" }
-          - { os: ubuntu-18.04, c-version: gcc-10, cxx-version: g++-10, generator: "Unix Makefiles" }
+          - { os: ubuntu-18.04, c-version: gcc-7, cxx-version: g++-7, generator: "Unix Makefiles" }
+          - { os: ubuntu-18.04, c-version: gcc-11, cxx-version: g++-11, generator: "Unix Makefiles" }
           - { os: ubuntu-18.04, c-version: clang-5.0, cxx-version: clang++-5.0, generator: "Unix Makefiles" }
           - { os: ubuntu-18.04, c-version: clang-10, cxx-version: clang++-10, generator: "Unix Makefiles" }
     steps:

diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml
@@ -4,8 +4,8 @@ on: [push, pull_request]
 
 env:
   BUILD_TYPE: "Release"
-  OPENCV_VERSION: "3.4.13"
-  EDDL_VERSION: "v0.8.3a"
+  OPENCV_VERSION: "3.4.14"
+  EDDL_VERSION: "v0.9.2b"
   PROC: 2
 
 jobs:

diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
@@ -4,8 +4,8 @@ on: [push, pull_request]
 
 env:
   BUILD_TYPE: "Release"
-  OPENCV_VERSION: "3.4.13"
-  EDDL_VERSION: "v0.8.3a"
+  OPENCV_VERSION: "3.4.14"
+  EDDL_VERSION: "v0.9.2b"
   PROC: 2
 
 jobs:

diff --git a/3rdparty/dcmtk/CMakeLists.txt b/3rdparty/dcmtk/CMakeLists.txt
@@ -6,6 +6,9 @@ FetchContent_Declare(
 
 if(ECVL_WITH_DICOM)
     if(ECVL_BUILD_DEPS)
+        if(POLICY CMP0115)
+            set(CMAKE_POLICY_DEFAULT_CMP0115 OLD)
+        endif()
         FetchContent_GetProperties(dcmtk)
         if(NOT dcmtk_POPULATED)
             FetchContent_Populate(dcmtk)

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -22,6 +22,7 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/")
 set_property(GLOBAL PROPERTY USE_FOLDERS ON)
 set_property(GLOBAL PROPERTY PREDEFINED_TARGETS_FOLDER "")
 set(CMAKE_POSITION_INDEPENDENT_CODE ON) # To always generate position independent code
+set(CMAKE_VERBOSE_MAKEFILE ON)
 if (WIN32)
     set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
 endif()

diff --git a/Jenkinsfile b/Jenkinsfile
@@ -15,7 +15,7 @@ pipeline {
                             steps {
                                 timeout(60) {
                                     echo 'Building..'
-                                    cmakeBuild buildDir: 'build', cmakeArgs: '-DECVL_TESTS=ON -DECVL_BUILD_EDDL=ON -DECVL_DATASET=ON -DECVL_WITH_DICOM=ON -DECVL_WITH_OPENSLIDE=ON -DECVL_GPU=OFF', installation: 'InSearchPath', sourceDir: '.', cleanBuild: true, steps: [
+                                    cmakeBuild buildDir: 'build', buildType: 'Release', cmakeArgs: '-DECVL_TESTS=ON -DECVL_BUILD_EDDL=ON -DECVL_DATASET=ON -DECVL_WITH_DICOM=ON -DECVL_WITH_OPENSLIDE=ON -DECVL_GPU=OFF', installation: 'InSearchPath', sourceDir: '.', cleanBuild: true, steps: [
                                         [args: '--parallel 4', withCmake: true]
                                     ]
                                 }
@@ -25,7 +25,7 @@ pipeline {
                             steps {
                                 timeout(15) {
                                     echo 'Testing..'
-                                    ctest arguments: '-C Debug -VV', installation: 'InSearchPath', workingDir: 'build'
+                                    ctest arguments: '-C Release -VV', installation: 'InSearchPath', workingDir: 'build'
                                 }
                             }
                         }
@@ -46,8 +46,8 @@ pipeline {
                                 timeout(60) {
                                     echo 'Building..'
                                     bat 'powershell ../../ecvl_dependencies/ecvl_dependencies.ps1'
-                                    cmakeBuild buildDir: 'build', cmakeArgs: '-DECVL_TESTS=ON -DECVL_BUILD_EDDL=ON -DECVL_DATASET=ON -DECVL_WITH_DICOM=ON -DECVL_WITH_OPENSLIDE=ON -DOPENSLIDE_LIBRARIES=C:/Library/openslide-win32-20171122/lib/libopenslide.lib', installation: 'InSearchPath', sourceDir: '.', cleanBuild: true, steps: [
-                                        [args: '--parallel 4', withCmake: true]
+                                    cmakeBuild buildDir: 'build', buildType: 'Release', cmakeArgs: '-DECVL_TESTS=ON -DECVL_BUILD_EDDL=ON -DECVL_DATASET=ON -DECVL_WITH_DICOM=ON -DECVL_WITH_OPENSLIDE=ON -DOPENSLIDE_LIBRARIES=C:/Library/openslide-win32-20171122/lib/libopenslide.lib', installation: 'InSearchPath', sourceDir: '.', cleanBuild: true, steps: [
+                                        [args: '--config Release --parallel 4', withCmake: true]
                                     ]
                                 }
                             }
@@ -56,15 +56,15 @@ pipeline {
                             steps {
                                 timeout(15) {
                                     echo 'Testing..'
-                                    bat 'cd build && ctest -C Debug -VV'
+                                    bat 'cd build && ctest -C Release -VV'
                                 }
                             }
                         }
                         stage('Coverage') {
                             steps {
                                 timeout(15) {
                                     echo 'Calculating coverage..'
-                                    bat '"C:/Program Files/OpenCppCoverage/OpenCppCoverage.exe" --source %cd% --export_type=cobertura --excluded_sources=3rdparty -- "build/bin/Debug/ECVL_TESTS.exe"'
+                                    bat '"C:/Program Files/OpenCppCoverage/OpenCppCoverage.exe" --source %cd% --export_type=cobertura --excluded_sources=3rdparty -- "build/bin/Release/ECVL_TESTS.exe"'
                                     cobertura coberturaReportFile: 'ECVL_TESTSCoverage.xml'
                                     bat 'codecov -f ECVL_TESTSCoverage.xml -t 7635bd2e-51cf-461e-bb1b-fc7ba9fb26d1'
                                 }

diff --git a/README.md b/README.md
@@ -16,7 +16,7 @@ The ECVL documentation is available [here](https://deephealthproject.github.io/e
 
 ## Requirements
 - CMake 3.13 or later
-- C++ Compiler with C++17 support (e.g. GCC 6 or later, Clang 5.0 or later, Visual Studio 2017 or later)
+- C++ Compiler with C++17 support (e.g. GCC 7 or later, Clang 5.0 or later, Visual Studio 2017 or later)
 - [OpenCV](https://opencv.org) 3.0 or later (modules required: `core`, `imgproc`, `imgcodecs`, `photo`, [`calib3d` since OpenCV 4.0 only. Note that `calib3d` depends on `features2d` and `flann`])
 
 ### Optional
@@ -193,16 +193,16 @@ Contributions of any kind are welcome!
   <tr>
     <td style="text-align:center">Windows Server 2016</td>
     <td style="text-align:center">VS 2017 15.9.28307</td>
-    <td style="text-align:center">3.4.13</td>
-    <td style="text-align:center">0.8.3</td>
+    <td style="text-align:center">3.4.14</td>
+    <td style="text-align:center">0.9.2b</td>
     <td style="text-align:center">GitHub Actions</td>
     <td style="text-align:center" rowspan="2"><a href="https://github.com/deephealthproject/ecvl/actions?query=workflow%3Awindows"><img src="https://github.com/deephealthproject/ecvl/workflows/windows/badge.svg" alt="Workflow status badge" loading="lazy" height="20"></a></td>
   </tr>
   <tr>
     <td style="text-align:center">Windows Server 2019</td>
-    <td style="text-align:center">VS 2019 16.8.30804</td>
-    <td style="text-align:center">3.4.13</td>
-    <td style="text-align:center">0.8.3</td>
+    <td style="text-align:center">VS 2019 16.9.31229</td>
+    <td style="text-align:center">3.4.14</td>
+    <td style="text-align:center">0.9.2b</td>
     <td style="text-align:center">GitHub Actions</td>
   </tr>
 </table>
@@ -228,31 +228,31 @@ Contributions of any kind are welcome!
   </tr>
   <tr>
     <td style="text-align:center">Ubuntu 18.04.5</td>
-    <td style="text-align:center">GCC 6.5.0</td>
-    <td style="text-align:center">3.4.13</td>
-    <td style="text-align:center">0.8.3</td>
+    <td style="text-align:center">GCC 7.5.0</td>
+    <td style="text-align:center">3.4.14</td>
+    <td style="text-align:center">0.9.2b</td>
     <td style="text-align:center">GitHub Actions</td>
     <td style="text-align:center" rowspan="4"><a href="https://github.com/deephealthproject/ecvl/actions?query=workflow%3Alinux"><img src="https://github.com/deephealthproject/ecvl/workflows/linux/badge.svg" alt="Workflow status badge" loading="lazy" height="20"></a></td>
   </tr>
   <tr>
     <td style="text-align:center">Ubuntu 18.04.5</td>
-    <td style="text-align:center">GCC 10.1.0</td>
-    <td style="text-align:center">3.4.13</td>
-    <td style="text-align:center">0.8.3</td>
+    <td style="text-align:center">GCC 11.1.0</td>
+    <td style="text-align:center">3.4.14</td>
+    <td style="text-align:center">0.9.2b</td>
     <td style="text-align:center">GitHub Actions</td>
   </tr>
   <tr>
     <td style="text-align:center">Ubuntu 18.04.5</td>
     <td style="text-align:center">Clang 5.0.1</td>
-    <td style="text-align:center">3.4.13</td>
-    <td style="text-align:center">0.8.3</td>
+    <td style="text-align:center">3.4.14</td>
+    <td style="text-align:center">0.9.2b</td>
     <td style="text-align:center">GitHub Actions</td>
   </tr>
   <tr>
     <td style="text-align:center">Ubuntu 18.04.5</td>
     <td style="text-align:center">Clang 10.0.0</td>
-    <td style="text-align:center">3.4.13</td>
-    <td style="text-align:center">0.8.3</td>
+    <td style="text-align:center">3.4.14</td>
+    <td style="text-align:center">0.9.2b</td>
     <td style="text-align:center">GitHub Actions</td>
   </tr>
 </table>
@@ -273,8 +273,8 @@ Contributions of any kind are welcome!
   <tr>
     <td style="text-align:center">macOS 10.15</td>
     <td style="text-align:center">Apple Clang 12.0.0</td>
-    <td style="text-align:center">3.4.13</td>
-    <td style="text-align:center">0.8.3</td>
+    <td style="text-align:center">3.4.14</td>
+    <td style="text-align:center">0.9.2b</td>
     <td style="text-align:center">GitHub Actions</td>
     <td style="text-align:center" rowspan="4"><a href="https://github.com/deephealthproject/ecvl/actions?query=workflow%3Amacos"><img src="https://github.com/deephealthproject/ecvl/workflows/macos/badge.svg" alt="Workflow status badge" loading="lazy" height="20"></a></td>
   </tr>

diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
@@ -38,8 +38,11 @@ if(ECVL_DATASET)
 endif()
 if(ECVL_BUILD_EDDL AND eddl_FOUND)
     add_executable(example_ecvl_eddl "examples/example_ecvl_eddl.cpp")
+    add_executable(example_pipeline "examples/example_pipeline.cpp")
     set_target_properties(example_ecvl_eddl PROPERTIES FOLDER "Examples")
     target_link_libraries(example_ecvl_eddl ${ECVL_MODULES})
+    set_target_properties(example_pipeline PROPERTIES FOLDER "Examples")
+    target_link_libraries(example_pipeline ${ECVL_MODULES})
 endif()
 if(ECVL_BUILD_GUI AND wxWidgets_FOUND)
     add_executable(example_ecvl_gui "examples/example_ecvl_gui.cpp")

diff --git a/examples/example_dataset_generator.cpp b/examples/example_dataset_generator.cpp
@@ -45,17 +45,18 @@ int main()
     vector<int> mask;
     vector<int> black;
 
-    for (auto& index : d_segmentation.split_.training_) {
-        if (d_segmentation.samples_[index].label_path_.value().filename().compare("black.png") == 0) {
-            black.emplace_back(index);
+    auto& training = d_segmentation.GetSplit("training");
+    for (auto& sample_index : training) {
+        if (d_segmentation.samples_[sample_index].label_path_.value().filename().compare("black.png") == 0) {
+            black.emplace_back(sample_index);
         }
         else {
-            mask.emplace_back(index);
+            mask.emplace_back(sample_index);
         }
     }
 
-    d_segmentation.split_.training_.clear();
-    d_segmentation.split_.training_.insert(d_segmentation.split_.training_.end(), mask.begin(), mask.end());
+    training.clear();
+    training.insert(training.end(), mask.begin(), mask.end());
 
     // Dump the Dataset on file
     d_segmentation.Dump(dateset_root_folder_segmentation / path(dateset_root_folder_segmentation.stem().string() + ".yml"));

diff --git a/examples/example_ecvl_eddl.cpp b/examples/example_ecvl_eddl.cpp
@@ -13,7 +13,6 @@
 
 #include <iostream>
 #include <sstream>
-#include <unordered_map>
 
 #include "ecvl/core.h"
 #include "ecvl/support_eddl.h"
@@ -34,6 +33,7 @@ int main()
 
     // Create an augmentation sequence to be applied to the image
     auto augs = make_shared<SequentialAugmentationContainer>(
+        AugCenterCrop(), // Make image squared
         AugRotate({ -5, 5 }),
         AugMirror(.5),
         AugFlip(.5),
@@ -71,6 +71,7 @@ int main()
     cout << "Executing TensorToView" << endl;
     TensorToView(t, view);
 
+    // Create an augmentation sequence from stream
     stringstream ss(
         "SequentialAugmentationContainer\n"
         "    AugRotate angle=[-5,5] center=(0,0) interp=\"linear\"\n"
@@ -85,21 +86,28 @@ int main()
     auto newdeal_augs = AugmentationFactory::create(ss);
     newdeal_augs->Apply(tmp);
 
+    /*--------------------------------------------------------------------------------------------*/
+
     // Create the augmentations to be applied to the dataset images during training and test.
-    // nullptr is given as augmentation for validation because this split doesn't exist in the mnist dataset.
     auto training_augs = make_shared<SequentialAugmentationContainer>(
         AugRotate({ -5, 5 }),
         AugAdditiveLaplaceNoise({ 0, 0.2 * 255 }),
         AugCoarseDropout({ 0, 0.55 }, { 0.02,0.1 }, 0),
         AugAdditivePoissonNoise({ 0, 40 }),
-        AugResizeDim({ 30, 30 })
+        AugResizeDim({ 30, 30 }),
+        AugToFloat32(255),
+        AugNormalize({ 0.449 }, { 0.226 }) // mean of imagenet stats
         );
 
     auto test_augs = make_shared<SequentialAugmentationContainer>(
-        AugResizeDim({ 30, 30 })
+        AugResizeDim({ 30, 30 }),
+        AugToFloat32(255),
+        AugNormalize({ 0.449 }, { 0.226 }) // mean of imagenet stats
         );
 
-    DatasetAugmentations dataset_augmentations{ {training_augs, nullptr, test_augs } };
+    // OLD version: now the number of augmentations must match the number of splits in the yml file
+    // DatasetAugmentations dataset_augmentations{ {training_augs, nullptr, test_augs } };
+    DatasetAugmentations dataset_augmentations{ {training_augs, test_augs } };
 
     int batch_size = 64;
     cout << "Creating a DLDataset" << endl;
@@ -128,6 +136,10 @@ int main()
     d.SetSplit(SplitType::test);
     d.LoadBatch(x, y);
 
+    // Save some input images
+    ImWrite("mnist_batch.png", MakeGrid(x, 8, false));
+    ImWrite("mnist_batch_normalized.png", MakeGrid(x, 8, true));
+
     delete x;
     delete y;
     delete t;