From 32fe713391e82d9c7570827b8f1e74e8e0f74b9e Mon Sep 17 00:00:00 2001 From: Samuel Farrens Date: Wed, 12 Oct 2022 11:47:20 +0200 Subject: [PATCH] Patch fixes for v1.0.1 (#589) * added CD GitHub action * fixed bug in RunLog class (#548) * 800deg2 (#544) * added config files for matching 800deg2 survey * Added Xavier as contributor; match external catalogue fix * match 800 * match_ext: output distance * match_ext output_dist def value * Tutorial documentation (#543) * updating job script and tutorial * tutorial; fix config file * tutorial finished * no change * psfex update (minor bug) * psfex running * updated MCCD config files * comment in config file * config file fix for MCCD * config file fix for MCCD * removed unused option in usage str * Update docs/source/pipeline_tutorial.md Co-authored-by: Samuel Farrens * improved tutorial according to PR review * Update docs/source/pipeline_tutorial.md Co-authored-by: Samuel Farrens * tutorial clean-up * post_proc cleaned up; tutorial added output dir names * changes to tutorial, post-proc and mccd plot doc * get_images error msg testing * Update docs/source/pipeline_tutorial.md Co-authored-by: Samuel Farrens * Update docs/source/pipeline_tutorial.md Co-authored-by: Samuel Farrens * Update docs/source/pipeline_tutorial.md Co-authored-by: Samuel Farrens * Update docs/source/pipeline_tutorial.md Co-authored-by: Samuel Farrens * removed prompt from bash example * Update docs/source/post_processing.md Co-authored-by: Samuel Farrens * PR review changes to doc * PR review changes to doc * Update docs/source/post_processing.md Co-authored-by: Samuel Farrens Co-authored-by: Samuel Farrens * added contribution guidelines (#550) * added contribution guidelines * Update CODE_OF_CONDUCT.md * tutorial bugfixes (#552) Co-authored-by: Lucie Baumont * removed pep8 error exceptions (#554) * Understanding the API docs (#553) * added md for understanding the api docs * updates following reviwer comments * Update package versions (#539) * updated doc-tests * updated package versions * pin documentation packages * updated ShapePipe version and cleaned up * Config file clean up (#556) * Removed unused config files in example/cfis * Removed obsolete config dirs tutorial and ps3pi_cfis * removed old unions config files * Update docs/source/configuration.md Co-authored-by: Martin Kilbinger * final clean up * removed obsolete distance-based overlap region flagging * removed errornous mccd_interp_runner call * Added Tobias' MCCD plot config file to validate joint cat * Added Tobias' MCCD plot config file to validate joint cat * removed duplicate config file * Random catalogue improvements (#570) * merge_final script: added option for additional tile ID path to select tiles; clean-up * typo in doc string * Updated docs and references (#579) * updated docs and references * Update docs/source/module_develop.md Co-authored-by: Martin Kilbinger Co-authored-by: Martin Kilbinger * Random runner updates (#573) * merge_final script: added option for additional tile ID path to select tiles; clean-up * typo in doc string * started to implement healpix mask output * healpix options for random/mask * Update shapepipe/modules/random_cat_package/random_cat.py Co-authored-by: Samuel Farrens * Update shapepipe/modules/random_cat_package/random_cat.py Co-authored-by: Samuel Farrens * Update shapepipe/modules/random_cat_package/random_cat.py Co-authored-by: Samuel Farrens * Update shapepipe/modules/random_cat_package/random_cat.py Co-authored-by: Samuel Farrens * Modifications for PR review * comments from PR review Co-authored-by: Samuel Farrens * added masking for NGC objects (#575) * added masking for NGC objects * debugging ngc np binary file * Fixed bugs in new mask handling (input FITS files) * debug info to log file for checking existing images * mask: combined dso masks into dictionary * PR edits * Update shapepipe/modules/mask_package/mask.py Co-authored-by: Samuel Farrens * Update shapepipe/modules/mask_package/mask.py Co-authored-by: Samuel Farrens * Update shapepipe/modules/mask_package/mask.py Co-authored-by: Samuel Farrens * Update shapepipe/modules/mask_package/mask.py Co-authored-by: Samuel Farrens * added symlink * Update shapepipe/pipeline/file_io.py Co-authored-by: Samuel Farrens * Update shapepipe/modules/mask_package/mask.py Co-authored-by: Samuel Farrens * Update shapepipe/pipeline/file_io.py Co-authored-by: Samuel Farrens * Update shapepipe/pipeline/file_io.py Co-authored-by: Samuel Farrens * Update shapepipe/pipeline/file_io.py Co-authored-by: Samuel Farrens * Update shapepipe/pipeline/file_io.py Co-authored-by: Samuel Farrens * Update shapepipe/modules/mask_package/mask.py Co-authored-by: Samuel Farrens * Update shapepipe/modules/mask_package/mask.py Co-authored-by: Samuel Farrens * Update shapepipe/modules/mask_package/mask.py Co-authored-by: Samuel Farrens * Update shapepipe/pipeline/file_io.py Co-authored-by: Samuel Farrens * PR suggestions Co-authored-by: Samuel Farrens * update patch version * random runner: hp masks working; doc updated (#580) * random runner: hp masks working; doc updated * random runner config file updated * Update shapepipe/modules/random_cat_package/random_cat.py Co-authored-by: Samuel Farrens * passing tests * Update shapepipe/modules/random_cat_package/__init__.py Co-authored-by: Samuel Farrens * Update shapepipe/modules/random_cat_package/__init__.py Co-authored-by: Samuel Farrens Co-authored-by: Samuel Farrens * Bug fix(es) for v1.0.1 (#586) * Summer 2022 Patch Release v1.0.1 (#581) * added CD GitHub action * fixed bug in RunLog class (#548) * 800deg2 (#544) * added config files for matching 800deg2 survey * Added Xavier as contributor; match external catalogue fix * match 800 * match_ext: output distance * match_ext output_dist def value * Tutorial documentation (#543) * updating job script and tutorial * tutorial; fix config file * tutorial finished * no change * psfex update (minor bug) * psfex running * updated MCCD config files * comment in config file * config file fix for MCCD * config file fix for MCCD * removed unused option in usage str * Update docs/source/pipeline_tutorial.md Co-authored-by: Samuel Farrens * improved tutorial according to PR review * Update docs/source/pipeline_tutorial.md Co-authored-by: Samuel Farrens * tutorial clean-up * post_proc cleaned up; tutorial added output dir names * changes to tutorial, post-proc and mccd plot doc * get_images error msg testing * Update docs/source/pipeline_tutorial.md Co-authored-by: Samuel Farrens * Update docs/source/pipeline_tutorial.md Co-authored-by: Samuel Farrens * Update docs/source/pipeline_tutorial.md Co-authored-by: Samuel Farrens * Update docs/source/pipeline_tutorial.md Co-authored-by: Samuel Farrens * removed prompt from bash example * Update docs/source/post_processing.md Co-authored-by: Samuel Farrens * PR review changes to doc * PR review changes to doc * Update docs/source/post_processing.md Co-authored-by: Samuel Farrens Co-authored-by: Samuel Farrens * added contribution guidelines (#550) * added contribution guidelines * Update CODE_OF_CONDUCT.md * tutorial bugfixes (#552) Co-authored-by: Lucie Baumont * removed pep8 error exceptions (#554) * Understanding the API docs (#553) * added md for understanding the api docs * updates following reviwer comments * Update package versions (#539) * updated doc-tests * updated package versions * pin documentation packages * updated ShapePipe version and cleaned up * Config file clean up (#556) * Removed unused config files in example/cfis * Removed obsolete config dirs tutorial and ps3pi_cfis * removed old unions config files * Update docs/source/configuration.md Co-authored-by: Martin Kilbinger * final clean up * removed obsolete distance-based overlap region flagging * removed errornous mccd_interp_runner call * Added Tobias' MCCD plot config file to validate joint cat * Added Tobias' MCCD plot config file to validate joint cat * removed duplicate config file * Random catalogue improvements (#570) * merge_final script: added option for additional tile ID path to select tiles; clean-up * typo in doc string * Updated docs and references (#579) * updated docs and references * Update docs/source/module_develop.md Co-authored-by: Martin Kilbinger Co-authored-by: Martin Kilbinger * Random runner updates (#573) * merge_final script: added option for additional tile ID path to select tiles; clean-up * typo in doc string * started to implement healpix mask output * healpix options for random/mask * Update shapepipe/modules/random_cat_package/random_cat.py Co-authored-by: Samuel Farrens * Update shapepipe/modules/random_cat_package/random_cat.py Co-authored-by: Samuel Farrens * Update shapepipe/modules/random_cat_package/random_cat.py Co-authored-by: Samuel Farrens * Update shapepipe/modules/random_cat_package/random_cat.py Co-authored-by: Samuel Farrens * Modifications for PR review * comments from PR review Co-authored-by: Samuel Farrens * added masking for NGC objects (#575) * added masking for NGC objects * debugging ngc np binary file * Fixed bugs in new mask handling (input FITS files) * debug info to log file for checking existing images * mask: combined dso masks into dictionary * PR edits * Update shapepipe/modules/mask_package/mask.py Co-authored-by: Samuel Farrens * Update shapepipe/modules/mask_package/mask.py Co-authored-by: Samuel Farrens * Update shapepipe/modules/mask_package/mask.py Co-authored-by: Samuel Farrens * Update shapepipe/modules/mask_package/mask.py Co-authored-by: Samuel Farrens * added symlink * Update shapepipe/pipeline/file_io.py Co-authored-by: Samuel Farrens * Update shapepipe/modules/mask_package/mask.py Co-authored-by: Samuel Farrens * Update shapepipe/pipeline/file_io.py Co-authored-by: Samuel Farrens * Update shapepipe/pipeline/file_io.py Co-authored-by: Samuel Farrens * Update shapepipe/pipeline/file_io.py Co-authored-by: Samuel Farrens * Update shapepipe/pipeline/file_io.py Co-authored-by: Samuel Farrens * Update shapepipe/modules/mask_package/mask.py Co-authored-by: Samuel Farrens * Update shapepipe/modules/mask_package/mask.py Co-authored-by: Samuel Farrens * Update shapepipe/modules/mask_package/mask.py Co-authored-by: Samuel Farrens * Update shapepipe/pipeline/file_io.py Co-authored-by: Samuel Farrens * PR suggestions Co-authored-by: Samuel Farrens * update patch version Co-authored-by: Martin Kilbinger Co-authored-by: Lucie Baumont Co-authored-by: Lucie Baumont Co-authored-by: Tobias Liaudat * fixed typo in variable name * config file vignet runner output fixed * typo in make_cat config ini file fixed Co-authored-by: Samuel Farrens Co-authored-by: Lucie Baumont Co-authored-by: Lucie Baumont Co-authored-by: Tobias Liaudat * fixed blank lines Co-authored-by: Martin Kilbinger Co-authored-by: Lucie Baumont Co-authored-by: Lucie Baumont Co-authored-by: Tobias Liaudat --- docs/source/installation.md | 2 +- example/cfis/config_Rc.ini | 9 ++++++ example/cfis/config_make_cat_mccd.ini | 2 +- example/cfis/config_tile_MiViSmVi.ini | 4 +-- shapepipe/modules/mask_package/mask.py | 10 ++++-- .../modules/random_cat_package/__init__.py | 9 ++++-- .../modules/random_cat_package/random_cat.py | 32 +++++++++++-------- shapepipe/modules/random_cat_runner.py | 2 +- 8 files changed, 47 insertions(+), 23 deletions(-) diff --git a/docs/source/installation.md b/docs/source/installation.md index bfe4e8096..63f030c17 100644 --- a/docs/source/installation.md +++ b/docs/source/installation.md @@ -33,7 +33,7 @@ cd shapepipe where `` is a [tagged release](https://github.com/CosmoStat/shapepipe/releases) of ShapePipe -(e.g. `v1.0.0`). It is recommend to use the +(e.g. `v1.0.1`). It is recommend to use the [latest release](https://github.com/CosmoStat/shapepipe/releases/latest) unless you want to reproduce an older set of results. diff --git a/example/cfis/config_Rc.ini b/example/cfis/config_Rc.ini index 8c24d1e81..2fec0a0fc 100644 --- a/example/cfis/config_Rc.ini +++ b/example/cfis/config_Rc.ini @@ -65,3 +65,12 @@ N_RANDOM = 50000 # N_RANDOM is per square degrees if True DENSITY = True + +# Output healpix mask if True +SAVE_MASK_AS_HEALPIX = True + +# Healpix mask file base name (used if SAVE_MASK_AS_HEALPIX is True) +HEALPIX_OUT_FILE_BASE = mask_hp + +# Healpix mask nside (used if SAVE_MASK_AS_HEALPIX is True) +HEALPIX_OUT_NSIDE = 1024 diff --git a/example/cfis/config_make_cat_mccd.ini b/example/cfis/config_make_cat_mccd.ini index b4d6d886a..5aa33d272 100644 --- a/example/cfis/config_make_cat_mccd.ini +++ b/example/cfis/config_make_cat_mccd.ini @@ -70,7 +70,7 @@ FILE_EXT = .fits, .fits, .sqlite, .fits NUMBERING_SCHEME = -000-000 SM_DO_CLASSIFICATION = True -SM_STAR_STRESH = 0.003 +SM_STAR_THRESH = 0.003 SM_GAL_THRESH = 0.01 SHAPE_MEASUREMENT_TYPE = ngmix diff --git a/example/cfis/config_tile_MiViSmVi.ini b/example/cfis/config_tile_MiViSmVi.ini index ddddbebcf..6198c324c 100644 --- a/example/cfis/config_tile_MiViSmVi.ini +++ b/example/cfis/config_tile_MiViSmVi.ini @@ -90,7 +90,7 @@ ME_LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite [VIGNETMAKER_RUNNER_RUN_1] -# Create vignets for tiles weights +# Create vignets for tile weights INPUT_DIR = last:sextractor_runner_run_1, last:uncompress_fits_runner @@ -177,5 +177,5 @@ PREFIX = # Additional parameters for path and file pattern corresponding to single-exposure # run outputs ME_IMAGE_DIR = split_exp_runner, split_exp_runner, split_exp_runner, sextractor_runner_run_2 -ME_IMAGE_PATTERN = flag, image, weight, sexcat_background +ME_IMAGE_PATTERN = flag, image, weight, background ME_LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite diff --git a/shapepipe/modules/mask_package/mask.py b/shapepipe/modules/mask_package/mask.py index 572f91bad..17e61b847 100644 --- a/shapepipe/modules/mask_package/mask.py +++ b/shapepipe/modules/mask_package/mask.py @@ -364,7 +364,7 @@ def make_mask(self): self._config[_type]['cat_path'], size_plus=self._config[_type]['size_plus'], flag_value=self._config[_type]['flag'], - typ=_type, + obj_type=_type, ) if not self._err: @@ -535,7 +535,13 @@ def mask_border(self, width=100, flag_value=4): return flag - def mask_dso(self, cat_path, size_plus=0.1, flag_value=8, typ='Messier'): + def mask_dso( + self, + cat_path, + size_plus=0.1, + flag_value=8, + obj_type='Messier', + ): """Mask DSO. Create a circular patch for deep-sky objects (DSOs), e.g. diff --git a/shapepipe/modules/random_cat_package/__init__.py b/shapepipe/modules/random_cat_package/__init__.py index 2dda4943c..b3a1cf21e 100644 --- a/shapepipe/modules/random_cat_package/__init__.py +++ b/shapepipe/modules/random_cat_package/__init__.py @@ -24,8 +24,13 @@ DENSITY : bool, optional Option to interpret the number of random objects per square degree; the default is ``False`` -TILE_LIST : str, optional - Path to tile IDs for overlap flagging +SAVE_MASK_AS_HEALPIX : bool + Output healpix mask if ``True`` +HEALPIX_OUT_FILE_BASE : str, optional + Output halpix mask file base name; used only if SAVE_MASK_AS_HEALPIX is + ``True`` +HEALPIX_OUT_NSIDE : int, optional + Output healpix mask nside; used only if SAVE_MASK_AS_HEALPIX is ``True`` """ diff --git a/shapepipe/modules/random_cat_package/random_cat.py b/shapepipe/modules/random_cat_package/random_cat.py index 2f24a3970..3bdb6ba6d 100644 --- a/shapepipe/modules/random_cat_package/random_cat.py +++ b/shapepipe/modules/random_cat_package/random_cat.py @@ -72,15 +72,16 @@ def __init__( self._w_log = w_log self._healpix_options = healpix_options - def save_as_healpix(self, hdu_mask, header): + def save_as_healpix(self, mask, header): """Save As Healpix. Save mask as healpix FITS file. Parameters ---------- - hdu_mask : class HDUList - HDU with 2D pixel mask image + mask : numpy.ndarray + 2D pixel mask image + header : class Header Image header with WCS information @@ -88,22 +89,25 @@ def save_as_healpix(self, hdu_mask, header): if not self._healpix_options: return + # Tranform config entry from str to int + nside = int(self._healpix_options['NSIDE']) + mask_1d, footprint = reproject_to_healpix( - (hdu_mask, header), + (mask, header), 'galactic', - nside=self._healpix_options['OUT_NSIDE'] + nside=nside, ) t = Table() t['flux'] = mask_1d t.meta['ORDERING'] = 'RING' t.meta['COORDSYS'] = 'G' - t.meta['NSIDE'] = self._healpix_options['OUT_NSIDE'] + t.meta['NSIDE'] = nside t.meta['INDXSCHM'] = 'IMPLICIT' output_path = ( - f'{output_dir}/{self._healpix_options["FILE_BASE"]}-' - + f'{file_number_string}.fits' + f'{self._output_dir}/{self._healpix_options["FILE_BASE"]}-' + + f'{self._file_number_string}.fits' ) t.write(output_path) @@ -135,7 +139,7 @@ def process(self): mask = hdu_mask[0].data # Save mask in healpix format (if option is set) - self._save_as_healpix(hdu_mask, header) + self.save_as_healpix(mask, header) # Number of pixels n_pix_x = mask.data.shape[0] @@ -143,7 +147,7 @@ def process(self): n_pix = n_pix_x * n_pix_y # Number of non-masked pixels - n_unmasked = len(np.where(hdu_mask[0].data == 0)[0]) + n_unmasked = len(np.where(mask == 0)[0]) # Compute various areas @@ -209,10 +213,10 @@ def process(self): # Tile ID output_path = ( - f'{output_dir}/{output_file_pattern}-' - + f'{file_number_string}.fits' + f'{self._output_dir}/{self._output_file_pattern}-' + + f'{self._file_number_string}.fits' ) - file_name = os.path.split(self._output_path)[1] + file_name = os.path.split(output_path)[1] file_base = os.path.splitext(file_name)[0] tile_ID_str = re.split('-', file_base)[1:] tile_id = float('.'.join(tile_ID_str)) @@ -230,7 +234,7 @@ def process(self): # TODO: Add units to header output = file_io.FITSCatalogue( - self._output_path, + output_path, open_mode=file_io.BaseCatalogue.OpenMode.ReadWrite ) output.save_as_fits(cat_out, names=column_names) diff --git a/shapepipe/modules/random_cat_runner.py b/shapepipe/modules/random_cat_runner.py index 7aad692a6..5595a53ad 100644 --- a/shapepipe/modules/random_cat_runner.py +++ b/shapepipe/modules/random_cat_runner.py @@ -56,7 +56,7 @@ def random_cat_runner( ) if save_mask_as_healpix: healpix_options = {} - for option_trunc in ['FILE_BASE', 'OUT_NSIDE']: + for option_trunc in ['FILE_BASE', 'NSIDE']: option = f'HEALPIX_OUT_{option_trunc}' healpix_options[option_trunc] = config.get( module_config_sec,