Merge pull request #321 from hpc4cmb/mapmaker_pipeline_tools

rough implementation of the mapmaker pipeline tools. Includes mappin…
hpc4cmb · Mar 20, 2020 · 0300a4e · 0300a4e
2 parents 951f692 + d4806f4
commit 0300a4e
Show file tree

Hide file tree

Showing 7 changed files with 389 additions and 26 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -10,7 +10,8 @@ language: python
 
 matrix:
   include:
-    # GCC 4.9 - this is the oldest version working well with libelemental.
+    # GCC 4.9 - this is the oldest version working well with C++11
+    # Serial test
     - os: linux
       dist: trusty
       sudo: false
@@ -36,7 +37,36 @@ matrix:
       env:
         - MATRIX_EVAL="export CC=$(which gcc-4.9) && export CXX=$(which g++-4.9) && export FC=$(which gfortran-4.9) && export DEPGCC=-4.9"
         - DEPSURL="https://www.dropbox.com/s/dhpj6vb1fch6xb5/travis_14.04_gcc-4.9_python3.6.tar.bz2"
+        - TRAVIS_TYPE=serial
+    # MPI test
+    - os: linux
+      dist: trusty
+      sudo: false
+      addons:
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test
+          packages:
+            - build-essential
+            - git
+            - pkg-config
+            - locales
+            - libgl1-mesa-glx
+            - xvfb
+            - libopenblas-dev
+            - liblapack-dev
+            - libfftw3-dev
+            - libhdf5-dev
+            - libcfitsio3-dev
+            - gcc-4.9
+            - g++-4.9
+            - gfortran-4.9
+      env:
+        - MATRIX_EVAL="export CC=$(which gcc-4.9) && export CXX=$(which g++-4.9) && export FC=$(which gfortran-4.9) && export DEPGCC=-4.9"
+        - DEPSURL="https://www.dropbox.com/s/dhpj6vb1fch6xb5/travis_14.04_gcc-4.9_python3.6.tar.bz2"
+        - TRAVIS_TYPE=mpi
     # GCC 5 - default version for ubuntu 16.04 LTS
+    # Serial test
     - os: linux
       dist: trusty
       sudo: false
@@ -62,7 +92,63 @@ matrix:
       env:
         - MATRIX_EVAL="export CC=$(which gcc-5) && export CXX=$(which g++-5) && export FC=$(which gfortran-5) && export DEPGCC=-5"
         - DEPSURL="https://www.dropbox.com/s/yoqu6z1e7eto03b/travis_14.04_gcc-5_python3.6.tar.bz2"
+        - TRAVIS_TYPE=serial
+    # MPI test
+    - os: linux
+      dist: trusty
+      sudo: false
+      addons:
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test
+          packages:
+            - build-essential
+            - git
+            - pkg-config
+            - locales
+            - libgl1-mesa-glx
+            - xvfb
+            - libopenblas-dev
+            - liblapack-dev
+            - libfftw3-dev
+            - libhdf5-dev
+            - libcfitsio3-dev
+            - gcc-5
+            - g++-5
+            - gfortran-5
+      env:
+        - MATRIX_EVAL="export CC=$(which gcc-5) && export CXX=$(which g++-5) && export FC=$(which gfortran-5) && export DEPGCC=-5"
+        - DEPSURL="https://www.dropbox.com/s/yoqu6z1e7eto03b/travis_14.04_gcc-5_python3.6.tar.bz2"
+        - TRAVIS_TYPE=mpi
     # GCC 7 - default version for ubuntu 18.04 LTS
+    # Serial test
+    - os: linux
+      dist: trusty
+      sudo: false
+      addons:
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test
+          packages:
+            - build-essential
+            - git
+            - pkg-config
+            - locales
+            - libgl1-mesa-glx
+            - xvfb
+            - libopenblas-dev
+            - liblapack-dev
+            - libfftw3-dev
+            - libhdf5-dev
+            - libcfitsio3-dev
+            - gcc-7
+            - g++-7
+            - gfortran-7
+      env:
+        - MATRIX_EVAL="export CC=$(which gcc-7) && export CXX=$(which g++-7) && export FC=$(which gfortran-7) && export DEPGCC=-7"
+        - DEPSURL="https://www.dropbox.com/s/diy5gchpwh8og6v/travis_14.04_gcc-7_python3.6.tar.bz2"
+        - TRAVIS_TYPE=serial
+    # MPI test
     - os: linux
       dist: trusty
       sudo: false
@@ -88,6 +174,7 @@ matrix:
       env:
         - MATRIX_EVAL="export CC=$(which gcc-7) && export CXX=$(which g++-7) && export FC=$(which gfortran-7) && export DEPGCC=-7"
         - DEPSURL="https://www.dropbox.com/s/diy5gchpwh8og6v/travis_14.04_gcc-7_python3.6.tar.bz2"
+        - TRAVIS_TYPE=mpi
 
 # The python versions to test.
 python:
@@ -149,7 +236,7 @@ install:
 # Run tests
 
 script:
-    - ./travis/travis_run.sh
+    - ./travis/travis_run_${TRAVIS_TYPE}.sh
 
 # Slack integration
 

diff --git a/src/toast/pipeline_tools/CMakeLists.txt b/src/toast/pipeline_tools/CMakeLists.txt
@@ -12,6 +12,7 @@ install(FILES
   export.py
   filters.py
   gain.py
+  mapmaker.py
   madam.py
   noise.py
   pointing.py

diff --git a/src/toast/pipeline_tools/__init__.py b/src/toast/pipeline_tools/__init__.py
@@ -26,6 +26,7 @@
     apply_groundfilter,
 )
 from .gain import add_gainscrambler_args, scramble_gains
+from .mapmaker import add_mapmaker_args, apply_mapmaker
 from .madam import add_madam_args, setup_madam, apply_madam
 from .noise import add_noise_args, simulate_noise, get_analytic_noise
 from .pointing import add_pointing_args, expand_pointing

diff --git a/src/toast/pipeline_tools/madam.py b/src/toast/pipeline_tools/madam.py
@@ -20,21 +20,27 @@ def add_madam_args(parser):
     """
 
     parser.add_argument(
-        "--madam-prefix", required=False, default="toast", help="Output map prefix"
+        "--madam-prefix",
+        required=False,
+        default="toast",
+        help="Output map prefix",
+        dest="mapmaker_prefix",
     )
     parser.add_argument(
         "--madam-iter-max",
         required=False,
         default=1000,
         type=np.int,
         help="Maximum number of CG iterations in Madam",
+        dest="mapmaker_iter_max",
     )
     parser.add_argument(
         "--madam-precond-width",
         required=False,
         default=100,
         type=np.int,
         help="Width of the Madam band preconditioner",
+        dest="mapmaker_precond_width",
     )
     parser.add_argument(
         "--madam-precond-width-min",
@@ -54,6 +60,7 @@ def add_madam_args(parser):
         default=10000.0,
         type=np.float,
         help="Destriping baseline length (seconds)",
+        dest="mapmaker_baseline_length",
     )
     parser.add_argument(
         "--madam-baseline-order",
@@ -68,6 +75,7 @@ def add_madam_args(parser):
         default=False,
         action="store_true",
         help="Destripe with the noise filter enabled",
+        dest="mapmaker_noisefilter",
     )
     parser.add_argument(
         "--madam-parfile", required=False, default=None, help="Madam parameter file"
@@ -308,25 +316,25 @@ def setup_madam(args):
                         key, value = result.group(1), result.group(2)
                         pars[key] = value
 
-    pars["base_first"] = args.madam_baseline_length
+    pars["base_first"] = args.mapmaker_baseline_length
     pars["basis_order"] = args.madam_baseline_order
     # Adaptive preconditioner width
     width_min = args.madam_precond_width_min
     width_max = args.madam_precond_width_max
     if width_min is None:
         # madam-precond-width has a default value
-        width_min = args.madam_precond_width
+        width_min = args.mapmaker_precond_width
     if width_max is None:
         # madam-precond-width has a default value
-        width_max = args.madam_precond_width
+        width_max = args.mapmaker_precond_width
     if width_min > width_max:
         # it is not an error for these two to match
         width_min = width_max
     pars["precond_width_min"] = width_min
     pars["precond_width_max"] = width_max
     #
     pars["nside_map"] = args.nside
-    if args.madam_noisefilter:
+    if args.mapmaker_noisefilter:
         if args.madam_baseline_order != 0:
             raise RuntimeError(
                 "Madam cannot build a noise filter when baseline"
@@ -336,8 +344,8 @@ def setup_madam(args):
     else:
         pars["kfilter"] = False
     pars["fsample"] = args.sample_rate
-    pars["iter_max"] = args.madam_iter_max
-    pars["file_root"] = args.madam_prefix
+    pars["iter_max"] = args.mapmaker_iter_max
+    pars["file_root"] = args.mapmaker_prefix
 
     # Translate boolean values.  Madam knows how to do this but it
     # simplifies pipeline_tools/madam.py
@@ -431,7 +439,7 @@ def apply_madam(
             log.info("No Madam outputs requested.  Skipping.")
         return
 
-    if args.madam_noisefilter or not pars["kfirst"]:
+    if args.mapmaker_noisefilter or not pars["kfirst"]:
         # With the noise filter enabled, we want to enforce continuity
         # across the Observation.  Otherwise we fit each interval
         # separately.