Pull request to merge

.azure-pipelines/build-docker.yaml

@@ -0,0 +1,21 @@
+parameters:
+  displayName: 'Docker'
+
+jobs:
+- job: docker_build
+  displayName: '${{ parameters.displayName }}'
+  pool:
+    vmImage: 'ubuntu-latest'
+  variables:
+    DOCKER_BUILDKIT: 1
+  steps:
+  - bash: |
+      echo "Building git commit: $(Build.SourceVersion)"
+    displayName: "Log git commit"
+
+  - task: Docker@2
+    displayName: Build
+    inputs:
+      command: build
+      arguments: --build-arg COLMAP_GIT_COMMIT=$(Build.SourceVersion) --build-arg CUDA_ARCHITECTURES=50
+      Dockerfile: docker/Dockerfile

.azure-pipelines/build-mac.yaml

@@ -1,27 +1,42 @@
 parameters:
-  displayName: 'Mac 10.15'
+  displayName: Mac 10.15
   macVersion: '10.15'
 
 jobs:
 - job: mac_build_${{ replace(parameters.macVersion, '.', '') }}
-  displayName: '${{ parameters.displayName }}'
+  displayName: ${{ parameters.displayName }}
   pool:
     vmImage: 'macOS-${{ parameters.macVersion }}'
+  variables:
+    CCACHE_DIR: $(Pipeline.Workspace)/ccache
+    CCACHE_BASEDIR: $(Build.SourcesDirectory)
+    CCACHE_VERSION: 1
   steps:
   - script: |
       brew install \
         cmake \
         ninja \
         boost \
         eigen \
+        flann \
         freeimage \
+        metis \
         glog \
-        gflags \
+        googletest \
         ceres-solver \
         qt5 \
         glew \
-        cgal
-    displayName: 'Install dependencies'
+        cgal \
+        sqlite3 \
+        ccache
+    displayName: Install dependencies
+
+  - task: Cache@2
+    inputs:
+      key: ccache | "$(CCACHE_VERSION)" | "$(Agent.OS)" | "${{ parameters.displayName }}" | "$(Build.BuildNumber)"
+      restoreKeys: ccache | "$(CCACHE_VERSION)" | "$(Agent.OS)" | "${{ parameters.displayName }}"
+      path: $(CCACHE_DIR)
+    displayName: Restore CCache
 
   - script: |
       export PATH="/usr/local/opt/qt@5/bin:$PATH"
@@ -33,7 +48,7 @@ jobs:
         -DTESTS_ENABLED=ON \
         -DQt5_DIR=/usr/local/opt/qt/lib/cmake/Qt5
       ninja
-    displayName: 'Configure and build'
+    displayName: Configure and build
 
   - script: |
       cd build
@@ -44,4 +59,11 @@ jobs:
           ctest --rerun-failed --output-on-failure
       fi
       exit $tests_pass
-    displayName: 'Run tests'
+    displayName: Run tests
+
+  - script: |
+      set -x
+      ccache --show-stats
+      ccache --evict-older-than 1d
+      ccache --show-stats
+    displayName: Cleanup ccache

.azure-pipelines/build-ubuntu.yaml

@@ -1,13 +1,30 @@
 parameters:
-  displayName: 'Ubuntu 18.04'
-  ubuntuVersion: '18.04'
+  displayName: Ubuntu 20.04
+  ubuntuVersion: '20.04'
   cudaEnabled: false
+  asanEnabled: false
+  e2eTestsEnabled: false
+  cmakeBuildType: Release
 
 jobs:
-- job: ubuntu_build_${{ replace(parameters.ubuntuVersion, '.', '') }}_cuda_${{ parameters.cudaEnabled }}
-  displayName: '${{ parameters.displayName }}'
+- job: ubuntu_build_${{ replace(parameters.ubuntuVersion, '.', '') }}_cuda_${{ parameters.cudaEnabled }}_asan_${{ parameters.asanEnabled }}_${{ parameters.cmakeBuildType }}_e2e_${{ parameters.e2eTestsEnabled }}
+  displayName: ${{ parameters.displayName }}
   pool:
     vmImage: 'ubuntu-${{ parameters.ubuntuVersion }}'
+  variables:
+    CCACHE_DIR: $(Pipeline.Workspace)/ccache
+    CCACHE_BASEDIR: $(Build.SourcesDirectory)
+    CCACHE_VERSION: 1
+    ${{ if eq(parameters.asanEnabled, false) }}:
+      ctestExclusions: "(mvs/colmap_mvs_gpu_mat_test)"
+    ${{ if eq(parameters.asanEnabled, true) }}:
+      ctestExclusions: "(feature/colmap_feature_sift_test)|(util/colmap_util_opengl_utils_test)|(mvs/colmap_mvs_gpu_mat_test)"
+    ${{ if eq(parameters.cmakeBuildType, 'ClangTidy') }}:
+      # Do not abort compilation in ClangTidy mode, so we see issues
+      # for all targets logged out to the console.
+      numAllowedNinjaErrors: 1000
+    ${{ if ne(parameters.cmakeBuildType, 'ClangTidy') }}:
+      numAllowedNinjaErrors: 1
   steps:
   - script: |
       sudo apt-get update && sudo apt-get install -y \
@@ -17,52 +34,144 @@ jobs:
         libboost-filesystem-dev \
         libboost-graph-dev \
         libboost-system-dev \
-        libboost-test-dev \
         libeigen3-dev \
         libceres-dev \
+        libflann-dev \
         libfreeimage-dev \
+        libmetis-dev \
         libgoogle-glog-dev \
-        libgflags-dev \
+        libgtest-dev \
+        libsqlite3-dev \
         libglew-dev \
         qtbase5-dev \
         libqt5opengl5-dev \
         libcgal-dev \
         libcgal-qt5-dev \
         libgl1-mesa-dri \
+        libunwind-dev \
         xvfb
-    displayName: 'Install dependencies'
+    displayName: Install dependencies
 
   - ${{ if eq(parameters.cudaEnabled, true) }}:
+    - ${{ if eq(parameters.ubuntuVersion, '22.04') }}:
+      - script: |
+          sudo apt-get install -y \
+            nvidia-cuda-toolkit \
+            nvidia-cuda-toolkit-gcc \
+            gcc-10 g++-10
+          echo '##vso[task.setvariable variable=CC]/usr/bin/gcc-10'
+          echo '##vso[task.setvariable variable=CXX]/usr/bin/g++-10'
+          echo '##vso[task.setvariable variable=CUDAHOSTCXX]/usr/bin/g++-10'
+        displayName: Install CUDA
+    - ${{ if ne(parameters.ubuntuVersion, '22.04') }}:
+      - script: |
+          sudo apt-get install -y \
+            nvidia-cuda-toolkit \
+            nvidia-cuda-toolkit-gcc
+          echo '##vso[task.setvariable variable=CC]/usr/bin/cuda-gcc'
+          echo '##vso[task.setvariable variable=CXX]/usr/bin/cuda-g++'
+        displayName: Install CUDA
+
+  - ${{ if eq(parameters.asanEnabled, true) }}:
+    - script: |
+        sudo apt-get install -y clang-15 libomp-15-dev
+        echo '##vso[task.setvariable variable=CC]/usr/bin/clang-15'
+        echo '##vso[task.setvariable variable=CXX]/usr/bin/clang++-15'
+      displayName: Install Clang
+
+  - ${{ if eq(parameters.cmakeBuildType, 'ClangTidy') }}:
     - script: |
-        sudo apt-get install -y \
-          nvidia-cuda-toolkit \
-          nvidia-cuda-toolkit-gcc
-        echo '##vso[task.setvariable variable=CC]/usr/bin/cuda-gcc'
-        echo '##vso[task.setvariable variable=CXX]/usr/bin/cuda-g++'
-      displayName: 'Install CUDA'
+        sudo apt-get install -y clang-15 clang-tidy-15 libomp-15-dev
+        echo '##vso[task.setvariable variable=CC]/usr/bin/clang-15'
+        echo '##vso[task.setvariable variable=CXX]/usr/bin/clang++-15'
+      displayName: Install Clang
+
+  - task: Cache@2
+    inputs:
+      key: ccache | "$(CCACHE_VERSION)" | "$(Agent.OS)" | "${{ parameters.displayName }}" | "$(Build.BuildNumber)"
+      restoreKeys: ccache | "$(CCACHE_VERSION)" | "$(Agent.OS)" | "${{ parameters.displayName }}"
+      path: $(CCACHE_DIR)
+      cacheHitVar: CCACHE_RESTORED
+    displayName: Restore CCache
 
   - script: |
+      set -x
+      wget https://github.com/ccache/ccache/releases/download/v4.8.2/ccache-4.8.2-linux-x86_64.tar.xz
+      echo "0b33f39766fe9db67f40418aed6a5b3d7b2f4f7fab025a8213264b77a2d0e1b1  ccache-4.8.2-linux-x86_64.tar.xz" | sha256sum --check
+      tar xfv ccache-4.8.2-linux-x86_64.tar.xz
+      mkdir -p "$(CCACHE_DIR)/bin"
+      mv ./ccache-4.8.2-linux-x86_64/ccache "$(CCACHE_DIR)/bin"
+      echo "##vso[task.prependpath]$(CCACHE_DIR)/bin"
+    displayName: Install CCache
+    condition: ne(variables.CCACHE_RESTORED, 'true')
+
+  - script: |
+      set -x
       cmake --version
       mkdir build
       cd build
       cmake .. \
         -GNinja \
+        -DCMAKE_BUILD_TYPE=${{ parameters.cmakeBuildType }} \
+        -DCMAKE_INSTALL_PREFIX=./install \
         -DTESTS_ENABLED=ON \
-        -DCUDA_ARCHS=6.0
-      ninja
-    displayName: 'Configure and build'
+        -DCMAKE_CUDA_ARCHITECTURES=50 \
+        -DASAN_ENABLED=${{ parameters.asanEnabled }}
+      ninja -k $(numAllowedNinjaErrors)
+    displayName: Configure and build colmap
+
+  - ${{ if and(eq(parameters.cmakeBuildType, 'Release'), eq(parameters.asanEnabled, false)) }}:
+    - script: |
+        set -x
+        cd build
+        ninja install
+        cd ../doc/sample-project
+        mkdir build
+        cd build
+        colmap_DIR=$(Build.SourcesDirectory)/build/install/share/colmap cmake .. \
+          -GNinja \
+          -DCMAKE_CUDA_ARCHITECTURES=50
+        ninja
+        ./hello_world --message "world"
+      displayName: Install colmap and build sample
+
+  - ${{ if ne(parameters.cmakeBuildType, 'ClangTidy') }}:
+    - script: |
+        export DISPLAY=":99.0"
+        export QT_QPA_PLATFORM="offscreen"
+        Xvfb :99 &
+        sleep 3
+        cd build
+        ctest -E "$(ctestExclusions)"
+        tests_pass=$?
+        if [ $tests_pass -ne 0 ]; then
+            echo "Tests failed, rerunning with verbose output"
+            ctest --rerun-failed --output-on-failure
+        fi
+        exit $tests_pass
+      displayName: Run tests
+
+  - ${{ if eq(parameters.e2eTestsEnabled, true) }}:
+    - script: |
+        export DISPLAY=":99.0"
+        export QT_QPA_PLATFORM="offscreen"
+        Xvfb :99 &
+        sleep 3
+        sudo apt install 7zip
+        mkdir eth3d_benchmark
+        # Error thresholds in degrees and meters,
+        # as the ETH3D groundtruth has metric scale.
+        python ./scripts/python/benchmark_eth3d.py \
+            --workspace_path ./eth3d_benchmark \
+            --colmap_path ./build/src/colmap/exe/colmap \
+            --dataset_names boulders,door \
+            --max_rotation_error 1.0 \
+            --max_proj_center_error 0.05
+      displayName: Run E2E tests
 
   - script: |
-      export DISPLAY=":99.0"
-      export QT_QPA_PLATFORM="offscreen"
-      Xvfb :99 &
-      sleep 3
-      cd build
-      ctest -E "(feature/sift_test)|(util/opengl_utils_test)|(mvs/gpu_mat_test)"
-      tests_pass=$?
-      if [ $tests_pass -ne 0 ]; then
-          echo "Tests failed, rerunning with verbose output"
-          ctest --rerun-failed --output-on-failure
-      fi
-      exit $tests_pass
-    displayName: 'Run tests'
+      set -x
+      ccache --show-stats
+      ccache --evict-older-than 1d
+      ccache --show-stats
+    displayName: Cleanup ccache

.azure-pipelines/build-windows-vcpkg.txt

@@ -5,12 +5,15 @@ boost-graph
 boost-program-options
 boost-regex
 boost-system
-boost-test
 ceres[lapack,suitesparse]
 cgal
 eigen3
+flann
 freeimage
+metis
 gflags
 glew
 glog
-qt5-base
+gtest
+qt5-base
+sqlite3