From 37ec72d50e488e5840920e3bca823d8159cb34b4 Mon Sep 17 00:00:00 2001 From: Garrett 'Karto' Keating Date: Sat, 11 Dec 2021 18:02:33 -0500 Subject: [PATCH 01/26] Adding methods to check/fix autos for imag components --- pyuvdata/uvdata/uvdata.py | 54 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/pyuvdata/uvdata/uvdata.py b/pyuvdata/uvdata/uvdata.py index 8f92160112..27e41c916c 100644 --- a/pyuvdata/uvdata/uvdata.py +++ b/pyuvdata/uvdata/uvdata.py @@ -2549,6 +2549,25 @@ def _calc_nants_data(self): """Calculate the number of antennas from ant_1_array and ant_2_array arrays.""" return int(np.union1d(self.ant_1_array, self.ant_2_array).size) + def _fix_autos(self): + """Remove imaginary component of auto-correlations.""" + auto_screen = self.ant_1_array == self.ant_2_array + auto_pol_list = ["xx", "yy", "rr", "ll", "pI", "pQ", "pQ", "pV"] + pol_screen = np.array( + [ + uvutils.POL_NUM2STR_DICT[pol] in auto_pol_list + for pol in self.polarization_array + ] + ) + + auto_data = self.data_array[auto_screen] + if self.future_array_shapes: + auto_data[:, :, pol_screen] = np.abs(auto_data[:, :, pol_screen]) + else: + auto_data[:, :, :, pol_screen] = np.abs(auto_data[:, :, :, pol_screen]) + + self.data_array[auto_screen] = auto_data + def check( self, check_extra=True, @@ -2556,6 +2575,8 @@ def check( check_freq_spacing=False, strict_uvw_antpos_check=False, allow_flip_conj=False, + check_autos=False, + fix_autos=False, ): """ Add some extra checks on top of checks on UVBase class. @@ -2727,6 +2748,39 @@ def check( raise ValueError( "Some auto-correlations have non-zero uvw_array coordinates." ) + if (self.data_array is not None and np.any(autos)) and check_autos: + # Verify here that the autos do not have any imaginary components + auto_pol_list = ["xx", "yy", "rr", "ll", "pI", "pQ", "pQ", "pV"] + pol_screen = np.array( + [ + uvutils.POL_NUM2STR_DICT[pol] in auto_pol_list + for pol in self.polarization_array + ] + ) + # There's no relevant pols to check, just skip the rest of this + if not np.any(pol_screen): + return + + # Check autos if they have imag component -- doing iscomplex first and + # then pol select was faster in every case checked in test files + auto_imag = np.iscomplex(self.data_array[autos]) + if np.all(pol_screen): + auto_imag = np.any(auto_imag) + elif self.future_array_shapes: + auto_imag = np.any(auto_imag[:, :, pol_screen]) + else: + auto_imag = np.any(auto_imag[:, :, :, pol_screen]) + if auto_imag: + if fix_autos: + warnings.warn( + "Fixing auto-correlations to be be real-only, after some " + "imaginary values were detected in data_array." + ) + self._fix_autos() + else: + raise ValueError( + "Some auto-correlations have non-real values in data_array." + ) if np.any( np.isclose( # this line used to use np.linalg.norm but it turns out From 5fe370b2110565e341f893f07fa69fec47a2c3d2 Mon Sep 17 00:00:00 2001 From: Garrett 'Karto' Keating Date: Sat, 11 Dec 2021 18:17:01 -0500 Subject: [PATCH 02/26] Adding auto check defaults on read/write --- pyuvdata/uvdata/uvdata.py | 74 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/pyuvdata/uvdata/uvdata.py b/pyuvdata/uvdata/uvdata.py index 27e41c916c..16b3a5dcc1 100644 --- a/pyuvdata/uvdata/uvdata.py +++ b/pyuvdata/uvdata/uvdata.py @@ -10204,6 +10204,8 @@ def read_fhd( check_extra=True, run_check_acceptability=True, strict_uvw_antpos_check=False, + check_autos=True, + fix_autos=True, ): """ Read in data from a list of FHD files. @@ -10274,6 +10276,8 @@ def read_fhd( check_extra=check_extra, run_check_acceptability=run_check_acceptability, strict_uvw_antpos_check=strict_uvw_antpos_check, + check_autos=check_autos, + fix_autos=fix_autos, ) self._convert_from_filetype(fhd_obj) del fhd_obj @@ -10291,6 +10295,8 @@ def read_mir( run_check_acceptability=True, strict_uvw_antpos_check=False, allow_flex_pol=True, + check_autos=True, + fix_autos=True, ): """ Read in data from an SMA MIR file. @@ -10352,6 +10358,8 @@ def read_mir( run_check_acceptability=run_check_acceptability, strict_uvw_antpos_check=strict_uvw_antpos_check, allow_flex_pol=allow_flex_pol, + check_autos=check_autos, + fix_autos=fix_autos, ) self._convert_from_filetype(mir_obj) del mir_obj @@ -10376,6 +10384,8 @@ def read_miriad( calc_lst=True, fix_old_proj=False, fix_use_ant_pos=True, + check_autos=True, + fix_autos=True, ): """ Read in data from a miriad file. @@ -10501,6 +10511,8 @@ def read_miriad( calc_lst=calc_lst, fix_old_proj=fix_old_proj, fix_use_ant_pos=fix_use_ant_pos, + check_autos=check_autos, + fix_autos=fix_autos, ) self._convert_from_filetype(miriad_obj) del miriad_obj @@ -10520,6 +10532,8 @@ def read_ms( raise_error=True, read_weights=True, allow_flex_pol=True, + check_autos=True, + fix_autos=True, ): """ Read in data from a measurement set. @@ -10617,6 +10631,8 @@ def read_ms( raise_error=raise_error, read_weights=read_weights, allow_flex_pol=allow_flex_pol, + check_autos=check_autos, + fix_autos=fix_autos, ) self._convert_from_filetype(ms_obj) del ms_obj @@ -10650,6 +10666,8 @@ def read_mwa_corr_fits( check_extra=True, run_check_acceptability=True, strict_uvw_antpos_check=False, + check_autos=True, + fix_autos=True, ): """ Read in MWA correlator gpu box files. @@ -10823,6 +10841,8 @@ def read_mwa_corr_fits( check_extra=check_extra, run_check_acceptability=run_check_acceptability, strict_uvw_antpos_check=strict_uvw_antpos_check, + check_autos=check_autos, + fix_autos=fix_autos, ) self._convert_from_filetype(corr_obj) del corr_obj @@ -10852,6 +10872,8 @@ def read_uvfits( strict_uvw_antpos_check=False, fix_old_proj=None, fix_use_ant_pos=True, + check_autos=True, + fix_autos=True, ): """ Read in header, metadata and data from a single uvfits file. @@ -11011,6 +11033,8 @@ def read_uvfits( strict_uvw_antpos_check=strict_uvw_antpos_check, fix_old_proj=fix_old_proj, fix_use_ant_pos=fix_use_ant_pos, + check_autos=check_autos, + fix_autos=fix_autos, ) self._convert_from_filetype(uvfits_obj) del uvfits_obj @@ -11042,6 +11066,8 @@ def read_uvh5( strict_uvw_antpos_check=False, fix_old_proj=None, fix_use_ant_pos=True, + check_autos=True, + fix_autos=True, ): """ Read a UVH5 file. @@ -11213,6 +11239,8 @@ def read_uvh5( strict_uvw_antpos_check=strict_uvw_antpos_check, fix_old_proj=fix_old_proj, fix_use_ant_pos=fix_use_ant_pos, + check_autos=check_autos, + fix_autos=fix_autos, ) self._convert_from_filetype(uvh5_obj) del uvh5_obj @@ -11285,6 +11313,8 @@ def read( make_multi_phase=False, ignore_name=False, allow_flex_pol=True, + check_autos=True, + fix_autos=True, ): """ Read a generic file into a UVData object. @@ -11708,6 +11738,8 @@ def read( fix_use_ant_pos=fix_use_ant_pos, make_multi_phase=make_multi_phase, allow_flex_pol=allow_flex_pol, + check_autos=check_autos, + fix_autos=fix_autos, ) unread = False except KeyError as err: @@ -11792,6 +11824,8 @@ def read( fix_use_ant_pos=fix_use_ant_pos, make_multi_phase=make_multi_phase, allow_flex_pol=allow_flex_pol, + check_autos=check_autos, + fix_autos=fix_autos, ) uv_list.append(uv2) except KeyError as err: @@ -11979,6 +12013,8 @@ def read( strict_uvw_antpos_check=strict_uvw_antpos_check, fix_old_proj=fix_old_proj, fix_use_ant_pos=fix_use_ant_pos, + check_autos=check_autos, + fix_autos=fix_autos, ) elif file_type == "mir": @@ -11994,6 +12030,8 @@ def read( run_check_acceptability=run_check_acceptability, strict_uvw_antpos_check=strict_uvw_antpos_check, allow_flex_pol=allow_flex_pol, + check_autos=check_autos, + fix_autos=fix_autos, ) select = False @@ -12016,6 +12054,8 @@ def read( calc_lst=calc_lst, fix_old_proj=fix_old_proj, fix_use_ant_pos=fix_use_ant_pos, + check_autos=check_autos, + fix_autos=fix_autos, ) elif file_type == "mwa_corr_fits": @@ -12046,6 +12086,8 @@ def read( check_extra=check_extra, run_check_acceptability=run_check_acceptability, strict_uvw_antpos_check=strict_uvw_antpos_check, + check_autos=check_autos, + fix_autos=fix_autos, ) elif file_type == "fhd": @@ -12058,6 +12100,8 @@ def read( check_extra=check_extra, run_check_acceptability=run_check_acceptability, strict_uvw_antpos_check=strict_uvw_antpos_check, + check_autos=check_autos, + fix_autos=fix_autos, ) elif file_type == "ms": @@ -12071,6 +12115,8 @@ def read( run_check_acceptability=run_check_acceptability, strict_uvw_antpos_check=strict_uvw_antpos_check, allow_flex_pol=allow_flex_pol, + check_autos=check_autos, + fix_autos=fix_autos, ) elif file_type == "uvh5": @@ -12099,6 +12145,8 @@ def read( strict_uvw_antpos_check=strict_uvw_antpos_check, fix_old_proj=fix_old_proj, fix_use_ant_pos=fix_use_ant_pos, + check_autos=check_autos, + fix_autos=fix_autos, ) select = False @@ -12119,6 +12167,8 @@ def read( check_extra=check_extra, run_check_acceptability=run_check_acceptability, strict_uvw_antpos_check=strict_uvw_antpos_check, + check_autos=check_autos, + fix_autos=fix_autos, ) if make_multi_phase: @@ -12234,6 +12284,8 @@ def from_file( make_multi_phase=False, ignore_name=False, allow_flex_pol=True, + check_autos=True, + fix_autos=True, ): """ Initialize a new UVData object by reading the input file. @@ -12602,6 +12654,8 @@ def from_file( make_multi_phase=make_multi_phase, ignore_name=ignore_name, allow_flex_pol=allow_flex_pol, + check_autos=check_autos, + fix_autos=fix_autos, ) return uvd @@ -12615,6 +12669,8 @@ def write_miriad( strict_uvw_antpos_check=False, no_antnums=False, calc_lst=False, + check_autos=True, + fix_autos=False, ): """ Write the data to a miriad file. @@ -12681,6 +12737,8 @@ def write_miriad( strict_uvw_antpos_check=strict_uvw_antpos_check, no_antnums=no_antnums, calc_lst=calc_lst, + check_autos=check_autos, + fix_autos=fix_autos, ) del miriad_obj @@ -12719,6 +12777,8 @@ def write_ms( check_extra=True, run_check_acceptability=True, strict_uvw_antpos_check=False, + check_autos=True, + fix_autos=False, ): """ Write a CASA measurement set (MS). @@ -12765,6 +12825,8 @@ def write_ms( check_extra=check_extra, run_check_acceptability=run_check_acceptability, strict_uvw_antpos_check=strict_uvw_antpos_check, + check_autos=check_autos, + fix_autos=fix_autos, ) del ms_obj @@ -12778,6 +12840,8 @@ def write_uvfits( check_extra=True, run_check_acceptability=True, strict_uvw_antpos_check=False, + check_autos=True, + fix_autos=False, ): """ Write the data to a uvfits file. @@ -12853,6 +12917,8 @@ def write_uvfits( check_extra=check_extra, run_check_acceptability=run_check_acceptability, strict_uvw_antpos_check=strict_uvw_antpos_check, + check_autos=check_autos, + fix_autos=fix_autos, ) del uvfits_obj @@ -12869,6 +12935,8 @@ def write_uvh5( check_extra=True, run_check_acceptability=True, strict_uvw_antpos_check=False, + check_autos=True, + fix_autos=False, ): """ Write a completely in-memory UVData object to a UVH5 file. @@ -12937,6 +13005,8 @@ def write_uvh5( check_extra=check_extra, run_check_acceptability=run_check_acceptability, strict_uvw_antpos_check=strict_uvw_antpos_check, + check_autos=check_autos, + fix_autos=fix_autos, ) del uvh5_obj @@ -13018,6 +13088,8 @@ def write_uvh5_part( blt_inds=None, add_to_history=None, run_check_acceptability=True, + check_autos=True, + fix_autos=False, ): """ Write data to a UVH5 file that has already been initialized. @@ -13131,5 +13203,7 @@ def write_uvh5_part( blt_inds=blt_inds, add_to_history=add_to_history, run_check_acceptability=run_check_acceptability, + check_autos=check_autos, + fix_autos=fix_autos, ) del uvh5_obj From efa45086f0d07a35255104376a2070b5d8488987 Mon Sep 17 00:00:00 2001 From: Garrett 'Karto' Keating Date: Sat, 11 Dec 2021 22:42:52 -0500 Subject: [PATCH 03/26] Fixing a couple of bugs with argument passing inside of uvdata --- pyuvdata/uvdata/uvdata.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pyuvdata/uvdata/uvdata.py b/pyuvdata/uvdata/uvdata.py index 16b3a5dcc1..ff12304ce7 100644 --- a/pyuvdata/uvdata/uvdata.py +++ b/pyuvdata/uvdata/uvdata.py @@ -12167,8 +12167,6 @@ def read( check_extra=check_extra, run_check_acceptability=run_check_acceptability, strict_uvw_antpos_check=strict_uvw_antpos_check, - check_autos=check_autos, - fix_autos=fix_autos, ) if make_multi_phase: @@ -13088,8 +13086,6 @@ def write_uvh5_part( blt_inds=None, add_to_history=None, run_check_acceptability=True, - check_autos=True, - fix_autos=False, ): """ Write data to a UVH5 file that has already been initialized. @@ -13203,7 +13199,5 @@ def write_uvh5_part( blt_inds=blt_inds, add_to_history=add_to_history, run_check_acceptability=run_check_acceptability, - check_autos=check_autos, - fix_autos=fix_autos, ) del uvh5_obj From c8257589e88ba92740fbe908efb93f7e2ff862ed Mon Sep 17 00:00:00 2001 From: Garrett 'Karto' Keating Date: Sat, 11 Dec 2021 22:43:14 -0500 Subject: [PATCH 04/26] Adding auto-fix/check to FHD --- pyuvdata/uvdata/fhd.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pyuvdata/uvdata/fhd.py b/pyuvdata/uvdata/fhd.py index a5bf1565ea..324cfdee05 100644 --- a/pyuvdata/uvdata/fhd.py +++ b/pyuvdata/uvdata/fhd.py @@ -331,6 +331,8 @@ def read_fhd( check_extra=True, run_check_acceptability=True, strict_uvw_antpos_check=False, + check_autos=True, + fix_autos=True, ): """ Read in data from a list of FHD files. @@ -369,6 +371,12 @@ def read_fhd( strict_uvw_antpos_check : bool Option to raise an error rather than a warning if the check that uvws match antenna positions does not pass. + check_autos : bool + Check whether any auto-correlations have imaginary values in them (which + should not mathematically exist). Default is True. + fix_autos : bool + If auto-correlations with imaginary values are found, fix those values so + that they are real-only. Default is False. Raises ------ @@ -711,4 +719,6 @@ def read_fhd( run_check_acceptability=run_check_acceptability, strict_uvw_antpos_check=strict_uvw_antpos_check, allow_flip_conj=True, + check_autos=check_autos, + fix_autos=fix_autos, ) From 749d979cbc15e92837c1f985e4bfb34ccb6c4d83 Mon Sep 17 00:00:00 2001 From: Garrett 'Karto' Keating Date: Sat, 11 Dec 2021 22:43:27 -0500 Subject: [PATCH 05/26] Adding auto fix/check to MIRIAD --- pyuvdata/uvdata/miriad.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/pyuvdata/uvdata/miriad.py b/pyuvdata/uvdata/miriad.py index ba1a71723a..3559cd41a7 100644 --- a/pyuvdata/uvdata/miriad.py +++ b/pyuvdata/uvdata/miriad.py @@ -730,6 +730,8 @@ def read_miriad( calc_lst=True, fix_old_proj=False, fix_use_ant_pos=True, + check_autos=True, + fix_autos=True, ): """ Read in data from a miriad file. @@ -808,6 +810,12 @@ def read_miriad( If setting `fix_old_proj` to True, use the antenna positions to derive the correct uvw-coordinates rather than using the baseline vectors. Default is True. + check_autos : bool + Check whether any auto-correlations have imaginary values in them (which + should not mathematically exist). Default is True. + fix_autos : bool + If auto-correlations with imaginary values are found, fix those values so + that they are real-only. Default is False. Raises ------ @@ -1569,6 +1577,8 @@ def read_miriad( run_check_acceptability=run_check_acceptability, strict_uvw_antpos_check=strict_uvw_antpos_check, allow_flip_conj=True, + check_autos=check_autos, + fix_autos=fix_autos, ) def write_miriad( @@ -1581,6 +1591,8 @@ def write_miriad( strict_uvw_antpos_check=False, no_antnums=False, calc_lst=False, + check_autos=True, + fix_autos=False, ): """ Write the data to a miriad file. @@ -1615,6 +1627,12 @@ def write_miriad( marks the midpoint). Default is False, which instead uses a simple formula for correcting the LSTs, expected to be accurate to approximately 0.1 µsec precision. + check_autos : bool + Check whether any auto-correlations have imaginary values in them (which + should not mathematically exist). Default is True. + fix_autos : bool + If auto-correlations with imaginary values are found, fix those values so + that they are real-only. Default is False. Raises ------ @@ -1651,6 +1669,8 @@ def write_miriad( run_check_acceptability=run_check_acceptability, check_freq_spacing=True, strict_uvw_antpos_check=strict_uvw_antpos_check, + check_autos=check_autos, + fix_autos=fix_autos, ) if os.path.exists(filepath): From 78131e8d95c6e1159e46da04d4359e1412135cd9 Mon Sep 17 00:00:00 2001 From: Garrett 'Karto' Keating Date: Sat, 11 Dec 2021 22:43:38 -0500 Subject: [PATCH 06/26] Adding auto-fix/check to MS --- pyuvdata/uvdata/ms.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/pyuvdata/uvdata/ms.py b/pyuvdata/uvdata/ms.py index d29457ad46..4854c103f3 100644 --- a/pyuvdata/uvdata/ms.py +++ b/pyuvdata/uvdata/ms.py @@ -1054,6 +1054,8 @@ def write_ms( check_extra=True, run_check_acceptability=True, strict_uvw_antpos_check=False, + check_autos=True, + fix_autos=False, ): """ Write a CASA measurement set (MS). @@ -1078,7 +1080,12 @@ def write_ms( strict_uvw_antpos_check : bool Option to raise an error rather than a warning if the check that uvws match antenna positions does not pass. - + check_autos : bool + Check whether any auto-correlations have imaginary values in them (which + should not mathematically exist). Default is True. + fix_autos : bool + If auto-correlations with imaginary values are found, fix those values so + that they are real-only. Default is False. """ if not casa_present: # pragma: no cover raise ImportError(no_casa_message) from casa_error @@ -1088,6 +1095,8 @@ def write_ms( check_extra=check_extra, run_check_acceptability=run_check_acceptability, strict_uvw_antpos_check=strict_uvw_antpos_check, + check_autos=check_autos, + fix_autos=fix_autos, ) if os.path.exists(filepath): @@ -1876,6 +1885,8 @@ def read_ms( raise_error=True, read_weights=True, allow_flex_pol=False, + check_autos=True, + fix_autos=True, ): """ Read in a casa measurement set. @@ -1931,6 +1942,12 @@ def read_ms( "flexible polarization", which compresses the polarization-axis of various attributes to be of length 1, sets the `flex_spw_polarization_array` attribute to define the polarization per spectral window. Default is True. + check_autos : bool + Check whether any auto-correlations have imaginary values in them (which + should not mathematically exist). Default is True. + fix_autos : bool + If auto-correlations with imaginary values are found, fix those values so + that they are real-only. Default is True. Raises ------ @@ -2273,4 +2290,6 @@ def read_ms( run_check_acceptability=run_check_acceptability, strict_uvw_antpos_check=strict_uvw_antpos_check, allow_flip_conj=True, + check_autos=check_autos, + fix_autos=fix_autos, ) From 00dcc8706f3dc8969acef8fee8186e0dcb8f66d0 Mon Sep 17 00:00:00 2001 From: Garrett 'Karto' Keating Date: Sat, 11 Dec 2021 22:44:07 -0500 Subject: [PATCH 07/26] Adding checking/fixing of autos to MWA corr_fits --- pyuvdata/uvdata/mwa_corr_fits.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pyuvdata/uvdata/mwa_corr_fits.py b/pyuvdata/uvdata/mwa_corr_fits.py index 0cede82c43..3783cc6fea 100644 --- a/pyuvdata/uvdata/mwa_corr_fits.py +++ b/pyuvdata/uvdata/mwa_corr_fits.py @@ -1083,6 +1083,8 @@ def read_mwa_corr_fits( check_extra=True, run_check_acceptability=True, strict_uvw_antpos_check=False, + check_autos=True, + fix_autos=True, ): """ Read in MWA correlator gpu box files. @@ -1189,6 +1191,12 @@ def read_mwa_corr_fits( strict_uvw_antpos_check : bool Option to raise an error rather than a warning if the check that uvws match antenna positions does not pass. + check_autos : bool + Check whether any auto-correlations have imaginary values in them (which + should not mathematically exist). Default is True. + fix_autos : bool + If auto-correlations with imaginary values are found, fix those values so + that they are real-only. Default is True. Raises @@ -1820,4 +1828,6 @@ def read_mwa_corr_fits( run_check_acceptability=run_check_acceptability, strict_uvw_antpos_check=strict_uvw_antpos_check, allow_flip_conj=True, + check_autos=check_autos, + fix_autos=fix_autos, ) From ebdb807fb08e9b49ea6467d3c9c11a9ac4b1de5e Mon Sep 17 00:00:00 2001 From: Garrett 'Karto' Keating Date: Sat, 11 Dec 2021 22:54:33 -0500 Subject: [PATCH 08/26] Add checking and fixing of autos to uvfits --- pyuvdata/uvdata/uvfits.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/pyuvdata/uvdata/uvfits.py b/pyuvdata/uvdata/uvfits.py index 0fbcba467f..e9d31e85ba 100644 --- a/pyuvdata/uvdata/uvfits.py +++ b/pyuvdata/uvdata/uvfits.py @@ -231,6 +231,8 @@ def _get_data( strict_uvw_antpos_check, fix_old_proj, fix_use_ant_pos, + check_autos, + fix_autos, ): """ Read just the visibility and flag data of the uvfits file. @@ -365,6 +367,8 @@ def _get_data( run_check_acceptability=run_check_acceptability, strict_uvw_antpos_check=strict_uvw_antpos_check, allow_flip_conj=True, + check_autos=check_autos, + fix_autos=fix_autos, ) def read_uvfits( @@ -391,6 +395,8 @@ def read_uvfits( strict_uvw_antpos_check=False, fix_old_proj=False, fix_use_ant_pos=True, + check_autos=True, + fix_autos=True, ): """ Read in header, metadata and data from a uvfits file. @@ -496,7 +502,12 @@ def read_uvfits( If setting `fix_old_proj` to True, use the antenna positions to derive the correct uvw-coordinates rather than using the baseline vectors. Default is True. - + check_autos : bool + Check whether any auto-correlations have imaginary values in them (which + should not mathematically exist). Default is True. + fix_autos : bool + If auto-correlations with imaginary values are found, fix those values so + that they are real-only. Default is True. Raises ------ @@ -853,6 +864,8 @@ def read_uvfits( strict_uvw_antpos_check, fix_old_proj, fix_use_ant_pos, + check_autos, + fix_autos, ) def write_uvfits( @@ -865,6 +878,8 @@ def write_uvfits( check_extra=True, run_check_acceptability=True, strict_uvw_antpos_check=False, + check_autos=True, + fix_autos=False, ): """ Write the data to a uvfits file. @@ -898,6 +913,12 @@ def write_uvfits( strict_uvw_antpos_check : bool Option to raise an error rather than a warning if the check that uvws match antenna positions does not pass. + check_autos : bool + Check whether any auto-correlations have imaginary values in them (which + should not mathematically exist). Default is True. + fix_autos : bool + If auto-correlations with imaginary values are found, fix those values so + that they are real-only. Default is False. Raises ------ @@ -920,6 +941,8 @@ def write_uvfits( run_check_acceptability=run_check_acceptability, check_freq_spacing=True, strict_uvw_antpos_check=strict_uvw_antpos_check, + check_autos=check_autos, + fix_autos=fix_autos, ) if self.phase_type == "phased": From 0401f7697704680751e74b1514fe17ff881d8213 Mon Sep 17 00:00:00 2001 From: Garrett 'Karto' Keating Date: Sat, 11 Dec 2021 22:55:11 -0500 Subject: [PATCH 09/26] Adding fixing/checking of autos to UVH5 --- pyuvdata/uvdata/uvh5.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/pyuvdata/uvdata/uvh5.py b/pyuvdata/uvdata/uvh5.py index 32d1db4209..4073714cd8 100644 --- a/pyuvdata/uvdata/uvh5.py +++ b/pyuvdata/uvdata/uvh5.py @@ -423,6 +423,8 @@ def _get_data( strict_uvw_antpos_check, fix_old_proj, fix_use_ant_pos, + check_autos, + fix_autos, ): """ Read the data-size arrays (data, flags, nsamples) from a file. @@ -737,6 +739,8 @@ def _get_data( run_check_acceptability=run_check_acceptability, strict_uvw_antpos_check=strict_uvw_antpos_check, allow_flip_conj=True, + check_autos=check_autos, + fix_autos=fix_autos, ) return @@ -767,6 +771,8 @@ def read_uvh5( strict_uvw_antpos_check=False, fix_old_proj=None, fix_use_ant_pos=True, + check_autos=True, + fix_autos=True, ): """ Read in data from a UVH5 file. @@ -883,6 +889,12 @@ def read_uvh5( If setting `fix_old_proj` to True, use the antenna positions to derive the correct uvw-coordinates rather than using the baseline vectors. Default is True. + check_autos : bool + Check whether any auto-correlations have imaginary values in them (which + should not mathematically exist). Default is True. + fix_autos : bool + If auto-correlations with imaginary values are found, fix those values so + that they are real-only. Default is True. Returns ------- @@ -957,6 +969,8 @@ def read_uvh5( strict_uvw_antpos_check, fix_old_proj, fix_use_ant_pos, + check_autos, + fix_autos, ) # For now, always use current shapes when data is read in, even if the file @@ -1112,6 +1126,8 @@ def write_uvh5( check_extra=True, run_check_acceptability=True, strict_uvw_antpos_check=False, + check_autos=True, + fix_autos=False, ): """ Write an in-memory UVData object to a UVH5 file. @@ -1152,6 +1168,12 @@ def write_uvh5( strict_uvw_antpos_check : bool Option to raise an error rather than a warning if the check that uvws match antenna positions does not pass. + check_autos : bool + Check whether any auto-correlations have imaginary values in them (which + should not mathematically exist). Default is True. + fix_autos : bool + If auto-correlations with imaginary values are found, fix those values so + that they are real-only. Default is False. Returns ------- @@ -1184,6 +1206,8 @@ def write_uvh5( check_extra=check_extra, run_check_acceptability=run_check_acceptability, strict_uvw_antpos_check=strict_uvw_antpos_check, + check_autos=check_autos, + fix_autos=fix_autos, ) if os.path.exists(filename): From 1bff4fddb783d090d1de91de4679259c7ce6c24f Mon Sep 17 00:00:00 2001 From: Garrett 'Karto' Keating Date: Sat, 11 Dec 2021 22:55:38 -0500 Subject: [PATCH 10/26] Add check to mir read --- pyuvdata/uvdata/mir.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pyuvdata/uvdata/mir.py b/pyuvdata/uvdata/mir.py index 4ea448387f..005bb1ea5c 100644 --- a/pyuvdata/uvdata/mir.py +++ b/pyuvdata/uvdata/mir.py @@ -38,6 +38,8 @@ def read_mir( run_check_acceptability=True, strict_uvw_antpos_check=False, allow_flex_pol=True, + check_autos=True, + fix_autos=False, ): """ Read in data from an SMA MIR file, and map to the UVData model. @@ -80,6 +82,12 @@ def read_mir( "flexible polarization", which compresses the polarization-axis of various attributes to be of length 1, sets the `flex_spw_polarization_array` attribute to define the polarization per spectral window. Default is True. + check_autos : bool + Check whether any auto-correlations have imaginary values in them (which + should not mathematically exist). Default is True. + fix_autos : bool + If auto-correlations with imaginary values are found, fix those values so + that they are real-only. Default is False. """ # Use the mir_parser to read in metadata, which can be used to select data. mir_data = mir_parser.MirParser(filepath) @@ -612,6 +620,16 @@ def _init_from_mir_parser(self, mir_data, allow_flex_pol=True): self.flag_array = np.zeros(self.data_array.shape, dtype=bool) self.nsample_array = np.ones(self.data_array.shape, dtype=np.float32) + if run_check: + self.check( + check_extra=check_extra, + run_check_acceptability=run_check_acceptability, + strict_uvw_antpos_check=strict_uvw_antpos_check, + allow_flip_conj=True, + check_autos=check_autos, + fix_autos=fix_autos, + ) + def write_mir(self, filename): """ Write out the SMA MIR files. From 2908b581a5a0cf0da5667172c680e52ba694dba0 Mon Sep 17 00:00:00 2001 From: Garrett 'Karto' Keating Date: Sat, 11 Dec 2021 23:05:01 -0500 Subject: [PATCH 11/26] Filitering out new warning for known tests --- pyuvdata/uvdata/tests/test_miriad.py | 2 ++ pyuvdata/uvdata/tests/test_mwa_corr_fits.py | 1 + pyuvdata/uvdata/tests/test_uvh5.py | 1 + pyuvdata/uvflag/tests/test_uvflag.py | 1 + 4 files changed, 5 insertions(+) diff --git a/pyuvdata/uvdata/tests/test_miriad.py b/pyuvdata/uvdata/tests/test_miriad.py index 83b56bcf0f..6fd6071e37 100644 --- a/pyuvdata/uvdata/tests/test_miriad.py +++ b/pyuvdata/uvdata/tests/test_miriad.py @@ -1030,6 +1030,7 @@ def test_miriad_antenna_diameters(uv_in_paper): assert uv_in == uv_out +@pytest.mark.filterwarnings("ignore:Fixing auto-correlations to be be real-only,") @pytest.mark.filterwarnings("ignore:The uvw_array does not match the expected values") def test_miriad_write_read_diameters(tmp_path): uv_in = UVData() @@ -1660,6 +1661,7 @@ def test_antpos_units(casa_uvfits, tmp_path): assert np.allclose(aantpos, uv.antenna_positions) +@pytest.mark.filterwarnings("ignore:Fixing auto-correlations to be be real-only,") @pytest.mark.filterwarnings("ignore:The uvw_array does not match the expected values") def test_readmiriad_write_miriad_check_time_format(tmp_path): """ diff --git a/pyuvdata/uvdata/tests/test_mwa_corr_fits.py b/pyuvdata/uvdata/tests/test_mwa_corr_fits.py index 18f2c01187..c23e388576 100644 --- a/pyuvdata/uvdata/tests/test_mwa_corr_fits.py +++ b/pyuvdata/uvdata/tests/test_mwa_corr_fits.py @@ -1000,6 +1000,7 @@ def test_van_vleck_interp(tmp_path): ] messages = messages * 10 messages.append("some coarse channel files were not submitted") + messages.append("Fixing auto-correlations to be be real-only,") uv = UVData() with uvtest.check_warnings(UserWarning, messages): uv.read( diff --git a/pyuvdata/uvdata/tests/test_uvh5.py b/pyuvdata/uvdata/tests/test_uvh5.py index 9aecfbab7b..5d39ae8c5b 100644 --- a/pyuvdata/uvdata/tests/test_uvh5.py +++ b/pyuvdata/uvdata/tests/test_uvh5.py @@ -3212,6 +3212,7 @@ def test_read_metadata(casa_uvfits, tmp_path): os.remove(testfile) +@pytest.mark.filterwarnings("ignore:Fixing auto-correlations to be be real-only,") @pytest.mark.filterwarnings("ignore:The original `phase` method is deprecated") def test_fix_phase(tmp_path): """Test that the fix phase method works""" diff --git a/pyuvdata/uvflag/tests/test_uvflag.py b/pyuvdata/uvflag/tests/test_uvflag.py index 1c6265edaa..fa281e2c33 100644 --- a/pyuvdata/uvflag/tests/test_uvflag.py +++ b/pyuvdata/uvflag/tests/test_uvflag.py @@ -147,6 +147,7 @@ def test_outfile(tmp_path): yield str(tmp_path / "outtest_uvflag.h5") +@pytest.mark.filterwarnings("ignore:Fixing auto-correlations to be be real-only,") @pytest.mark.filterwarnings("ignore:The uvw_array does not match the expected values") def test_check_flag_array(uvdata_obj): uvf = UVFlag() From 03f909480e440247677670ae815dae60443a3f50 Mon Sep 17 00:00:00 2001 From: Garrett 'Karto' Keating Date: Sat, 11 Dec 2021 23:06:02 -0500 Subject: [PATCH 12/26] Filtering new warning, and slightly changing one test due to linter complaint --- pyuvdata/tests/test_utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pyuvdata/tests/test_utils.py b/pyuvdata/tests/test_utils.py index 893011c3dc..fe643aaa6e 100644 --- a/pyuvdata/tests/test_utils.py +++ b/pyuvdata/tests/test_utils.py @@ -2671,7 +2671,7 @@ def test_mean_weights_and_weights_square(): assert np.allclose(wso, np.sum(1.0 / (np.arange(data.shape[1]) + 1) ** 2)) # Zero weights - w = np.ones_like(w) + w = np.ones_like(data) w[0, :] = 0 w[:, 0] = 0 out, wo = uvutils.mean_collapse(data, weights=w, axis=0, return_weights=True) @@ -2810,6 +2810,7 @@ def test_and_collapse_errors(): pytest.raises(ValueError, uvutils.and_collapse, data) +@pytest.mark.filterwarnings("ignore:Fixing auto-correlations to be be real-only,") @pytest.mark.filterwarnings("ignore:The uvw_array does not match the expected values") def test_uvcalibrate_apply_gains_oldfiles(): # read data @@ -2863,6 +2864,7 @@ def test_uvcalibrate_apply_gains_oldfiles(): @pytest.mark.filterwarnings("ignore:When converting a delay-style cal to future array") +@pytest.mark.filterwarnings("ignore:Fixing auto-correlations to be be real-only,") @pytest.mark.filterwarnings("ignore:The uvw_array does not match the expected values") @pytest.mark.parametrize("uvd_future_shapes", [True, False]) @pytest.mark.parametrize("uvc_future_shapes", [True, False]) @@ -2918,6 +2920,7 @@ def test_uvcalibrate_delay_oldfiles(uvd_future_shapes, uvc_future_shapes): @pytest.mark.parametrize("uvc_future_shapes", [True, False]) @pytest.mark.parametrize("uvd_future_shapes", [True, False]) +@pytest.mark.filterwarnings("ignore:Fixing auto-correlations to be be real-only,") @pytest.mark.parametrize("flip_gain_conj", [False, True]) @pytest.mark.parametrize("gain_convention", ["divide", "multiply"]) def test_uvcalibrate( @@ -3401,6 +3404,7 @@ def test_uvcalibrate_delay_multispw(uvcalibrate_data): uvutils.uvcalibrate(uvd, uvc, inplace=False) +@pytest.mark.filterwarnings("ignore:Fixing auto-correlations to be be real-only,") @pytest.mark.filterwarnings("ignore:The uvw_array does not match the expected values") @pytest.mark.parametrize("future_shapes", [True, False]) def test_apply_uvflag(future_shapes): From fd78b98a49dc40877890e89354dde7dd8ba8307b Mon Sep 17 00:00:00 2001 From: Garrett 'Karto' Keating Date: Sat, 11 Dec 2021 23:06:27 -0500 Subject: [PATCH 13/26] Adding sma-mir dataset to conftest, since its used in multiple places --- pyuvdata/uvdata/tests/conftest.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/pyuvdata/uvdata/tests/conftest.py b/pyuvdata/uvdata/tests/conftest.py index 24aec20e96..2f514b2796 100644 --- a/pyuvdata/uvdata/tests/conftest.py +++ b/pyuvdata/uvdata/tests/conftest.py @@ -129,3 +129,20 @@ def mir_data_object(request): # cleanup del mir_data + + +def sma_mir_main(): + # read in test file for the resampling in time functions + uv_object = UVData() + testfile = os.path.join(DATA_PATH, "sma_test.mir") + uv_object.read(testfile) + + yield uv_object + + +@pytest.fixture(scope="function") +def sma_mir(sma_mir_main): + # read in test file for the resampling in time functions + uv_object = sma_mir_main.copy() + + yield uv_object From 6bdb495413e592f1b6e05daae64afef5c709baf8 Mon Sep 17 00:00:00 2001 From: Garrett 'Karto' Keating Date: Sat, 11 Dec 2021 23:06:59 -0500 Subject: [PATCH 14/26] Cleaning up tests following moving fixtures --- pyuvdata/uvdata/tests/test_mir.py | 180 +++++++++------------------ pyuvdata/uvdata/tests/test_uvfits.py | 19 +-- 2 files changed, 63 insertions(+), 136 deletions(-) diff --git a/pyuvdata/uvdata/tests/test_mir.py b/pyuvdata/uvdata/tests/test_mir.py index 981a336410..2fd9c0d446 100644 --- a/pyuvdata/uvdata/tests/test_mir.py +++ b/pyuvdata/uvdata/tests/test_mir.py @@ -22,109 +22,55 @@ @pytest.fixture(scope="session") -def sma_mir_main(): - # read in test file for the resampling in time functions - uv_object = UVData() +def mir_data(): testfile = os.path.join(DATA_PATH, "sma_test.mir") - uv_object.read(testfile) - - yield uv_object - - -@pytest.fixture(scope="function") -def sma_mir(sma_mir_main): - # read in test file for the resampling in time functions - uv_object = sma_mir_main.copy() - - yield uv_object - - -@pytest.fixture(scope="session") -def sma_mir_filt_main(): - # read in test file for the resampling in time functions - uv_object = UVData() - testfile = os.path.join(DATA_PATH, "sma_test.mir") - uv_object.read(testfile, pseudo_cont=True, corrchunk=0) - - uv_object.flag_array[:, :, : uv_object.Nfreqs // 2, 0] = True - uv_object.flag_array[:, :, uv_object.Nfreqs // 2 :, 1] = True - uv_object.set_lsts_from_time_array() - uv_object._set_app_coords_helper() - - yield uv_object - - -@pytest.fixture(scope="function") -def sma_mir_filt(sma_mir_filt_main): - # read in test file for the resampling in time functions - uv_object = sma_mir_filt_main.copy() + mir_data = MirParser( + testfile, load_vis=True, load_raw=True, load_auto=True, + ) - yield uv_object + yield mir_data @pytest.fixture def uv_in_ms(tmp_path): - uv_in = UVData() - testfile = os.path.join(DATA_PATH, "sma_test.mir") write_file = os.path.join(tmp_path, "outtest_mir.ms") - - # Currently only one source is supported. - uv_in.read(testfile) uv_out = UVData() - yield uv_in, uv_out, write_file - - # cleanup - del uv_in, uv_out + yield uv_out, write_file @pytest.fixture def uv_in_uvfits(tmp_path): - uv_in = UVData() - testfile = os.path.join(DATA_PATH, "sma_test.mir/") write_file = os.path.join(tmp_path, "outtest_mir.uvfits") - # Currently only one source is supported. - uv_in.read(testfile, pseudo_cont=False) uv_out = UVData() - yield uv_in, uv_out, write_file - - # cleanup - del uv_in, uv_out + yield uv_out, write_file @pytest.fixture def uv_in_uvh5(tmp_path): - uv_in = UVData() - testfile = os.path.join(DATA_PATH, "sma_test.mir") write_file = os.path.join(tmp_path, "outtest_mir.uvh5") - - # Currently only one source is supported. - uv_in.read(testfile) uv_out = UVData() - yield uv_in, uv_out, write_file - - # cleanup - del uv_in, uv_out + yield uv_out, write_file @pytest.mark.filterwarnings("ignore:LST values stored in this file are not ") @pytest.mark.parametrize("future_shapes", [True, False]) -def test_read_mir_write_uvfits(uv_in_uvfits, future_shapes): +def test_read_mir_write_uvfits(sma_mir, uv_in_uvfits, future_shapes): """ Mir to uvfits loopback test. Read in Mir files, write out as uvfits, read back in and check for object equality. """ - mir_uv, uvfits_uv, testfile = uv_in_uvfits + uvfits_uv, testfile = uv_in_uvfits if future_shapes: - mir_uv.use_future_array_shapes() + sma_mir.use_future_array_shapes() - mir_uv.write_uvfits(testfile, spoof_nonessential=True) + sma_mir.write_uvfits(testfile, spoof_nonessential=True) uvfits_uv.read_uvfits(testfile) if future_shapes: @@ -132,72 +78,72 @@ def test_read_mir_write_uvfits(uv_in_uvfits, future_shapes): # UVFITS doesn't allow for numbering of spectral windows like MIR does, so # we need an extra bit of handling here - assert len(np.unique(mir_uv.spw_array)) == len(np.unique(uvfits_uv.spw_array)) + assert len(np.unique(sma_mir.spw_array)) == len(np.unique(uvfits_uv.spw_array)) - spw_dict = {idx: jdx for idx, jdx in zip(uvfits_uv.spw_array, mir_uv.spw_array)} + spw_dict = {idx: jdx for idx, jdx in zip(uvfits_uv.spw_array, sma_mir.spw_array)} assert np.all( [ idx == spw_dict[jdx] - for idx, jdx in zip(mir_uv.flex_spw_id_array, uvfits_uv.flex_spw_id_array,) + for idx, jdx in zip(sma_mir.flex_spw_id_array, uvfits_uv.flex_spw_id_array,) ] ) # Now that we've checked, set this things as equivalent - uvfits_uv.spw_array = mir_uv.spw_array - uvfits_uv.flex_spw_id_array = mir_uv.flex_spw_id_array + uvfits_uv.spw_array = sma_mir.spw_array + uvfits_uv.flex_spw_id_array = sma_mir.flex_spw_id_array # Check the history first via find assert 0 == uvfits_uv.history.find( - mir_uv.history + " Read/written with pyuvdata version:" + sma_mir.history + " Read/written with pyuvdata version:" ) - mir_uv.history = uvfits_uv.history + sma_mir.history = uvfits_uv.history # We have to do a bit of special handling for the phase_center_catalog, because # _very_ small errors (like last bit in the mantissa) creep in when passing through # the util function transform_sidereal_coords (for mutli-phase-ctr datasets). Verify # the two match up in terms of their coordinates - for cat_name in mir_uv.phase_center_catalog.keys(): + for cat_name in sma_mir.phase_center_catalog.keys(): assert np.isclose( - mir_uv.phase_center_catalog[cat_name]["cat_lat"], + sma_mir.phase_center_catalog[cat_name]["cat_lat"], uvfits_uv.phase_center_catalog[cat_name]["cat_lat"], ) assert np.isclose( - mir_uv.phase_center_catalog[cat_name]["cat_lon"], + sma_mir.phase_center_catalog[cat_name]["cat_lon"], uvfits_uv.phase_center_catalog[cat_name]["cat_lon"], ) - uvfits_uv.phase_center_catalog = mir_uv.phase_center_catalog + uvfits_uv.phase_center_catalog = sma_mir.phase_center_catalog # There's a minor difference between what SMA calculates online for app coords # and what pyuvdata calculates, to the tune of ~1 arcsec. Check those values here, # then set them equal to one another. assert np.all( - np.abs(mir_uv.phase_center_app_ra - uvfits_uv.phase_center_app_ra) < 1e-5 + np.abs(sma_mir.phase_center_app_ra - uvfits_uv.phase_center_app_ra) < 1e-5 ) assert np.all( - np.abs(mir_uv.phase_center_app_dec - uvfits_uv.phase_center_app_dec) < 1e-5 + np.abs(sma_mir.phase_center_app_dec - uvfits_uv.phase_center_app_dec) < 1e-5 ) - mir_uv._set_app_coords_helper() + sma_mir._set_app_coords_helper() uvfits_uv._set_app_coords_helper() # make sure filenames are what we expect - assert mir_uv.filename == ["sma_test.mir"] + assert sma_mir.filename == ["sma_test.mir"] assert uvfits_uv.filename == ["outtest_mir.uvfits"] - mir_uv.filename = uvfits_uv.filename - assert mir_uv == uvfits_uv + sma_mir.filename = uvfits_uv.filename + assert sma_mir == uvfits_uv # Since mir is mutli-phase-ctr by default, this should effectively be a no-op - mir_uv._set_multi_phase_center() + sma_mir._set_multi_phase_center() - assert mir_uv == uvfits_uv + assert sma_mir == uvfits_uv @pytest.mark.filterwarnings("ignore:Writing in the MS file that the units of the data") @pytest.mark.filterwarnings("ignore:LST values stored in this file are not ") @pytest.mark.parametrize("future_shapes", [True, False]) -def test_read_mir_write_ms(uv_in_ms, future_shapes): +def test_read_mir_write_ms(sma_mir, uv_in_ms, future_shapes): """ Mir to uvfits loopback test. @@ -205,12 +151,12 @@ def test_read_mir_write_ms(uv_in_ms, future_shapes): object equality. """ pytest.importorskip("casacore") - mir_uv, ms_uv, testfile = uv_in_ms + ms_uv, testfile = uv_in_ms if future_shapes: - mir_uv.use_future_array_shapes() + sma_mir.use_future_array_shapes() - mir_uv.write_ms(testfile, clobber=True) + sma_mir.write_ms(testfile, clobber=True) ms_uv.read(testfile) # Single integration with 1 phase center = single scan number @@ -223,79 +169,78 @@ def test_read_mir_write_ms(uv_in_ms, future_shapes): # There are some minor differences between the values stored by MIR and that # calculated by UVData. Since MS format requires these to be calculated on the fly, # we calculate them here just to verify that everything is looking okay. - mir_uv.set_lsts_from_time_array() - mir_uv._set_app_coords_helper() + sma_mir.set_lsts_from_time_array() + sma_mir._set_app_coords_helper() # These reorderings just make sure that data from the two formats are lined up # correctly. - mir_uv.reorder_freqs(spw_order="number") + sma_mir.reorder_freqs(spw_order="number") ms_uv.reorder_blts() # MS doesn't have the concept of an "instrument" name like FITS does, and instead # defaults to the telescope name. Make sure that checks out here. - assert mir_uv.instrument == "SWARM" + assert sma_mir.instrument == "SWARM" assert ms_uv.instrument == "SMA" - mir_uv.instrument = ms_uv.instrument + sma_mir.instrument = ms_uv.instrument # Quick check for history here - assert ms_uv.history != mir_uv.history - ms_uv.history = mir_uv.history + assert ms_uv.history != sma_mir.history + ms_uv.history = sma_mir.history # Only MS has extra keywords, verify those look as expected. assert ms_uv.extra_keywords == {"DATA_COL": "DATA", "observer": "SMA"} - assert mir_uv.extra_keywords == {} - mir_uv.extra_keywords = ms_uv.extra_keywords + assert sma_mir.extra_keywords == {} + sma_mir.extra_keywords = ms_uv.extra_keywords # Make sure the filenames line up as expected. - assert mir_uv.filename == ["sma_test.mir"] + assert sma_mir.filename == ["sma_test.mir"] assert ms_uv.filename == ["outtest_mir.ms"] - mir_uv.filename = ms_uv.filename = None + sma_mir.filename = ms_uv.filename = None # Finally, with all exceptions handled, check for equality. - assert ms_uv.__eq__(mir_uv, allowed_failures=["filename"]) + assert ms_uv.__eq__(sma_mir, allowed_failures=["filename"]) @pytest.mark.filterwarnings("ignore:LST values stored ") -def test_read_mir_write_uvh5(uv_in_uvh5): +def test_read_mir_write_uvh5(sma_mir, uv_in_uvh5): """ Mir to uvfits loopback test. Read in Mir files, write out as uvfits, read back in and check for object equality. """ - mir_uv, uvh5_uv, testfile = uv_in_uvh5 + uvh5_uv, testfile = uv_in_uvh5 - mir_uv.write_uvh5(testfile) + sma_mir.write_uvh5(testfile) uvh5_uv.read_uvh5(testfile) # Check the history first via find assert 0 == uvh5_uv.history.find( - mir_uv.history + " Read/written with pyuvdata version:" + sma_mir.history + " Read/written with pyuvdata version:" ) # test fails because of updated history, so this is our workaround for now. - mir_uv.history = uvh5_uv.history + sma_mir.history = uvh5_uv.history # make sure filenames are what we expect - assert mir_uv.filename == ["sma_test.mir"] + assert sma_mir.filename == ["sma_test.mir"] assert uvh5_uv.filename == ["outtest_mir.uvh5"] - mir_uv.filename = uvh5_uv.filename + sma_mir.filename = uvh5_uv.filename - assert mir_uv == uvh5_uv + assert sma_mir == uvh5_uv -def test_write_mir(uv_in_uvfits, err_type=NotImplementedError): +def test_write_mir(hera_uvh5, err_type=NotImplementedError): """ Mir writer test Check and make sure that attempts to use the writer return a 'not implemented' error. """ - mir_uv, uvfits_uv, testfile = uv_in_uvfits # Check and see if the correct error is raised with pytest.raises(err_type): - mir_uv.write_mir("dummy.mir") + hera_uvh5.write_mir("dummy.mir") def test_multi_nchan_spw_read(tmp_path): @@ -335,7 +280,7 @@ def test_read_mir_no_records(): uv_in.read_mir(testfile, corrchunk=999) -def test_read_mir_sideband_select(): +def test_read_mir_sideband_select(sma_mir): """ Mir sideband read check @@ -343,12 +288,11 @@ def test_read_mir_sideband_select(): stitch them back together as though they were read together from the start. """ testfile = os.path.join(DATA_PATH, "sma_test.mir") - mir_dsb = UVData() - mir_dsb.read(testfile) + # Re-order here so that we can more easily compare the two - mir_dsb.reorder_freqs(channel_order="freq", spw_order="freq") + sma_mir.reorder_freqs(channel_order="freq", spw_order="freq") # Drop the history - mir_dsb.history = "" + sma_mir.history = "" mir_lsb = UVData() mir_lsb.read(testfile, isb=[0]) @@ -362,7 +306,7 @@ def test_read_mir_sideband_select(): # Drop the history mir_recomb.history = "" - assert mir_dsb == mir_recomb + assert sma_mir == mir_recomb def test_mir_auto_read( diff --git a/pyuvdata/uvdata/tests/test_uvfits.py b/pyuvdata/uvdata/tests/test_uvfits.py index 308d75e347..ff4fe6a191 100644 --- a/pyuvdata/uvdata/tests/test_uvfits.py +++ b/pyuvdata/uvdata/tests/test_uvfits.py @@ -40,24 +40,6 @@ def uvfits_nospw(uvfits_nospw_main): return uvfits_nospw_main.copy() -@pytest.fixture(scope="session") -def sma_mir_main(): - # read in test file for the resampling in time functions - uv_object = UVData() - testfile = os.path.join(DATA_PATH, "sma_test.mir") - uv_object.read(testfile) - - yield uv_object - - -@pytest.fixture(scope="function") -def sma_mir(sma_mir_main): - # read in test file for the resampling in time functions - uv_object = sma_mir_main.copy() - - yield uv_object - - @pytest.mark.filterwarnings("ignore:The uvw_array does not match the expected values") @pytest.mark.filterwarnings("ignore:Telescope EVLA is not") def test_read_nrao(casa_uvfits): @@ -77,6 +59,7 @@ def test_read_nrao(casa_uvfits): assert uvobj2 == uvobj3 +@pytest.mark.filterwarnings("ignore:Fixing auto-correlations to be be real-only,") @pytest.mark.filterwarnings("ignore:ITRF coordinate frame detected") @pytest.mark.filterwarnings("ignore:Telescope OVRO_MMA is not") @pytest.mark.filterwarnings("ignore:The uvw_array does not match the expected values") From a914686e5c5390fe8da8668a14b7e4b8ea328591 Mon Sep 17 00:00:00 2001 From: Garrett 'Karto' Keating Date: Sat, 11 Dec 2021 23:07:53 -0500 Subject: [PATCH 15/26] Adding new tests for coverage, fixing things up after moving fixtures --- pyuvdata/uvdata/tests/test_uvdata.py | 73 +++++++++++++++++++++------- 1 file changed, 55 insertions(+), 18 deletions(-) diff --git a/pyuvdata/uvdata/tests/test_uvdata.py b/pyuvdata/uvdata/tests/test_uvdata.py index 3401ecee24..31c717a058 100644 --- a/pyuvdata/uvdata/tests/test_uvdata.py +++ b/pyuvdata/uvdata/tests/test_uvdata.py @@ -203,24 +203,6 @@ def hera_uvh5_xx(hera_uvh5_xx_main): return -@pytest.fixture(scope="session") -def sma_mir_main(): - # read in test file for the resampling in time functions - uv_object = UVData() - testfile = os.path.join(DATA_PATH, "sma_test.mir") - uv_object.read(testfile) - - yield uv_object - - -@pytest.fixture(scope="function") -def sma_mir(sma_mir_main): - # read in test file for the resampling in time functions - uv_object = sma_mir_main.copy() - - yield uv_object - - @pytest.fixture(scope="session") def sma_mir_catalog(sma_mir_main): catalog_dict = sma_mir_main.phase_center_catalog @@ -623,6 +605,7 @@ def test_check_strict_uvw(casa_uvfits): uvobj.check(strict_uvw_antpos_check=True) +@pytest.mark.filterwarnings("ignore:Fixing auto-correlations to be be real-only,") @pytest.mark.filterwarnings("ignore:The uvw_array does not match the expected values") def test_check_autos_only(hera_uvh5_xx): """ @@ -1089,6 +1072,7 @@ def test_phase_unphase_hera_bad_frame(uv1_2_set_uvws): assert str(cm.value).startswith("phase_frame can only be set to icrs or gcrs.") +@pytest.mark.filterwarnings("ignore:Fixing auto-correlations to be be real-only,") @pytest.mark.parametrize("future_shapes", [True, False]) @pytest.mark.parametrize("use_ant_pos1", [True, False]) @pytest.mark.parametrize("use_ant_pos2", [True, False]) @@ -1221,6 +1205,7 @@ def test_phasing_fix_old_proj(hera_uvh5, future_shapes): # We're using the old phase method here since these values were all derived using that # method, so we'll just filter out those warnings now. @pytest.mark.filterwarnings("ignore:The original `phase` method is deprecated") +@pytest.mark.filterwarnings("ignore:Fixing auto-correlations to be be real-only,") @pytest.mark.parametrize("future_shapes", [True, False]) def test_old_phasing(future_shapes): """Use MWA files phased to 2 different places to test phasing.""" @@ -11236,6 +11221,7 @@ def test_set_nsamples_wrong_shape_error(hera_uvh5): return +@pytest.mark.filterwarnings("ignore:Fixing auto-correlations to be be real-only,") @pytest.mark.filterwarnings("ignore:Altitude is not present in Miriad file,") @pytest.mark.filterwarnings("ignore:using known location values for SZA.") @pytest.mark.filterwarnings("ignore:The uvw_array does not match the expected") @@ -11405,3 +11391,54 @@ def test_make_flex_pol_errs(sma_mir, err_msg, param, param_val): with uvtest.check_warnings(None): sma_mir._make_flex_pol(False, False) assert sma_copy == sma_mir + + +@pytest.mark.parametrize("dataset", ["hera", "casa"]) +@pytest.mark.parametrize("future_array_shapes", [True, False]) +def test_auto_check( + hera_uvh5_main, casa_uvh5_main, future_array_shapes, dataset, tmp_path +): + """ + Checks that checking/fixing the autos works correctly, both with dual-pol data + (supplied by hera_uvh5) and full-pol data (supplied by casa_uvfits). + """ + if dataset == "hera": + uv = hera_uvh5_main.copy() + elif dataset == "casa": + uv = casa_uvh5_main.copy() + + if future_array_shapes: + uv.use_future_array_shapes() + + out_file = os.path.join(tmp_path, "auto_check.uvh5") + + # Corrupt the auto data + auto_screen = uv.ant_1_array == uv.ant_2_array + uv.data_array[auto_screen] *= 1j + + with pytest.raises( + ValueError, match="Some auto-correlations have non-real values in data_array." + ): + uv.write_uvh5(out_file, clobber=True) + + uv.write_uvh5(out_file, check_autos=False, clobber=True) + + with pytest.raises( + ValueError, match="Some auto-correlations have non-real values in data_array." + ): + uv1 = uv.from_file(out_file, fix_autos=False) + + with uvtest.check_warnings(UserWarning, "Fixing auto-correlations to be be real"): + uv1 = uv.from_file(out_file) + + with uvtest.check_warnings(UserWarning, "Fixing auto-correlations to be be real"): + uv.write_uvh5(out_file, fix_autos=True, clobber=True) + + uv2 = uv.from_file(out_file) + + assert uv1 == uv2 + + if future_array_shapes: + uv1.use_future_array_shapes() + + assert uv == uv1 From 3e16dd327fa87a54ff795ead111ed13cd361cf11 Mon Sep 17 00:00:00 2001 From: Garrett 'Karto' Keating Date: Sat, 11 Dec 2021 23:31:21 -0500 Subject: [PATCH 16/26] Fixing new auto test, since original test file didnt actually have any autos to test. --- pyuvdata/uvdata/tests/test_uvdata.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/pyuvdata/uvdata/tests/test_uvdata.py b/pyuvdata/uvdata/tests/test_uvdata.py index 31c717a058..167b6b9825 100644 --- a/pyuvdata/uvdata/tests/test_uvdata.py +++ b/pyuvdata/uvdata/tests/test_uvdata.py @@ -11393,19 +11393,18 @@ def test_make_flex_pol_errs(sma_mir, err_msg, param, param_val): assert sma_copy == sma_mir -@pytest.mark.parametrize("dataset", ["hera", "casa"]) +@pytest.mark.parametrize("dataset", ["hera", "mwa"]) @pytest.mark.parametrize("future_array_shapes", [True, False]) -def test_auto_check( - hera_uvh5_main, casa_uvh5_main, future_array_shapes, dataset, tmp_path -): +@pytest.mark.filterwarnings("ignore:The uvw_array does not match the expected values") +def test_auto_check(hera_uvh5, uv_phase_comp, future_array_shapes, dataset, tmp_path): """ Checks that checking/fixing the autos works correctly, both with dual-pol data (supplied by hera_uvh5) and full-pol data (supplied by casa_uvfits). """ if dataset == "hera": - uv = hera_uvh5_main.copy() - elif dataset == "casa": - uv = casa_uvh5_main.copy() + uv = hera_uvh5 + elif dataset == "mwa": + uv, _ = uv_phase_comp if future_array_shapes: uv.use_future_array_shapes() From e23f3518356de1850415408c6f255d678311259d Mon Sep 17 00:00:00 2001 From: Garrett 'Karto' Keating Date: Sat, 11 Dec 2021 23:35:22 -0500 Subject: [PATCH 17/26] Adding message on how to fix non-real autos in check() --- pyuvdata/uvdata/uvdata.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyuvdata/uvdata/uvdata.py b/pyuvdata/uvdata/uvdata.py index ff12304ce7..fda5c78783 100644 --- a/pyuvdata/uvdata/uvdata.py +++ b/pyuvdata/uvdata/uvdata.py @@ -2780,6 +2780,7 @@ def check( else: raise ValueError( "Some auto-correlations have non-real values in data_array." + " You can attempt to fix this by setting fix_autos=True." ) if np.any( np.isclose( From 125ca59fb4d5a1cc72c47a29bd080f4375a977c7 Mon Sep 17 00:00:00 2001 From: Garrett 'Karto' Keating Date: Sat, 11 Dec 2021 23:48:04 -0500 Subject: [PATCH 18/26] Doc string updates --- pyuvdata/uvdata/fhd.py | 6 +- pyuvdata/uvdata/mir.py | 6 +- pyuvdata/uvdata/miriad.py | 12 ++-- pyuvdata/uvdata/ms.py | 12 ++-- pyuvdata/uvdata/mwa_corr_fits.py | 6 +- pyuvdata/uvdata/uvdata.py | 95 ++++++++++++++++++++++++++++++-- pyuvdata/uvdata/uvfits.py | 12 ++-- pyuvdata/uvdata/uvh5.py | 12 ++-- 8 files changed, 122 insertions(+), 39 deletions(-) diff --git a/pyuvdata/uvdata/fhd.py b/pyuvdata/uvdata/fhd.py index 324cfdee05..b20a61a2b3 100644 --- a/pyuvdata/uvdata/fhd.py +++ b/pyuvdata/uvdata/fhd.py @@ -372,11 +372,11 @@ def read_fhd( Option to raise an error rather than a warning if the check that uvws match antenna positions does not pass. check_autos : bool - Check whether any auto-correlations have imaginary values in them (which - should not mathematically exist). Default is True. + Check whether any auto-correlations have non-zero imaginary values in + data_array (which should not mathematically exist). Default is True. fix_autos : bool If auto-correlations with imaginary values are found, fix those values so - that they are real-only. Default is False. + that they are real-only in data_array. Default is False. Raises ------ diff --git a/pyuvdata/uvdata/mir.py b/pyuvdata/uvdata/mir.py index 005bb1ea5c..91d56fecb7 100644 --- a/pyuvdata/uvdata/mir.py +++ b/pyuvdata/uvdata/mir.py @@ -83,11 +83,11 @@ def read_mir( attributes to be of length 1, sets the `flex_spw_polarization_array` attribute to define the polarization per spectral window. Default is True. check_autos : bool - Check whether any auto-correlations have imaginary values in them (which - should not mathematically exist). Default is True. + Check whether any auto-correlations have non-zero imaginary values in + data_array (which should not mathematically exist). Default is True. fix_autos : bool If auto-correlations with imaginary values are found, fix those values so - that they are real-only. Default is False. + that they are real-only in data_array. Default is False. """ # Use the mir_parser to read in metadata, which can be used to select data. mir_data = mir_parser.MirParser(filepath) diff --git a/pyuvdata/uvdata/miriad.py b/pyuvdata/uvdata/miriad.py index 3559cd41a7..ea7c0c71f0 100644 --- a/pyuvdata/uvdata/miriad.py +++ b/pyuvdata/uvdata/miriad.py @@ -811,11 +811,11 @@ def read_miriad( correct uvw-coordinates rather than using the baseline vectors. Default is True. check_autos : bool - Check whether any auto-correlations have imaginary values in them (which - should not mathematically exist). Default is True. + Check whether any auto-correlations have non-zero imaginary values in + data_array (which should not mathematically exist). Default is True. fix_autos : bool If auto-correlations with imaginary values are found, fix those values so - that they are real-only. Default is False. + that they are real-only in data_array. Default is False. Raises ------ @@ -1628,11 +1628,11 @@ def write_miriad( for correcting the LSTs, expected to be accurate to approximately 0.1 µsec precision. check_autos : bool - Check whether any auto-correlations have imaginary values in them (which - should not mathematically exist). Default is True. + Check whether any auto-correlations have non-zero imaginary values in + data_array (which should not mathematically exist). Default is True. fix_autos : bool If auto-correlations with imaginary values are found, fix those values so - that they are real-only. Default is False. + that they are real-only in data_array. Default is False. Raises ------ diff --git a/pyuvdata/uvdata/ms.py b/pyuvdata/uvdata/ms.py index 4854c103f3..486d04c4a1 100644 --- a/pyuvdata/uvdata/ms.py +++ b/pyuvdata/uvdata/ms.py @@ -1081,11 +1081,11 @@ def write_ms( Option to raise an error rather than a warning if the check that uvws match antenna positions does not pass. check_autos : bool - Check whether any auto-correlations have imaginary values in them (which - should not mathematically exist). Default is True. + Check whether any auto-correlations have non-zero imaginary values in + data_array (which should not mathematically exist). Default is True. fix_autos : bool If auto-correlations with imaginary values are found, fix those values so - that they are real-only. Default is False. + that they are real-only in data_array. Default is False. """ if not casa_present: # pragma: no cover raise ImportError(no_casa_message) from casa_error @@ -1943,11 +1943,11 @@ def read_ms( attributes to be of length 1, sets the `flex_spw_polarization_array` attribute to define the polarization per spectral window. Default is True. check_autos : bool - Check whether any auto-correlations have imaginary values in them (which - should not mathematically exist). Default is True. + Check whether any auto-correlations have non-zero imaginary values in + data_array (which should not mathematically exist). Default is True. fix_autos : bool If auto-correlations with imaginary values are found, fix those values so - that they are real-only. Default is True. + that they are real-only in data_array. Default is True. Raises ------ diff --git a/pyuvdata/uvdata/mwa_corr_fits.py b/pyuvdata/uvdata/mwa_corr_fits.py index 3783cc6fea..f6615501a0 100644 --- a/pyuvdata/uvdata/mwa_corr_fits.py +++ b/pyuvdata/uvdata/mwa_corr_fits.py @@ -1192,11 +1192,11 @@ def read_mwa_corr_fits( Option to raise an error rather than a warning if the check that uvws match antenna positions does not pass. check_autos : bool - Check whether any auto-correlations have imaginary values in them (which - should not mathematically exist). Default is True. + Check whether any auto-correlations have non-zero imaginary values in + data_array (which should not mathematically exist). Default is True. fix_autos : bool If auto-correlations with imaginary values are found, fix those values so - that they are real-only. Default is True. + that they are real-only in data_array. Default is True. Raises diff --git a/pyuvdata/uvdata/uvdata.py b/pyuvdata/uvdata/uvdata.py index fda5c78783..bead571a47 100644 --- a/pyuvdata/uvdata/uvdata.py +++ b/pyuvdata/uvdata/uvdata.py @@ -2551,7 +2551,11 @@ def _calc_nants_data(self): def _fix_autos(self): """Remove imaginary component of auto-correlations.""" + # Select out the autos auto_screen = self.ant_1_array == self.ant_2_array + + # Only these pols have "true" auto-correlations, that we'd expect + # to be real only. Select on only them auto_pol_list = ["xx", "yy", "rr", "ll", "pI", "pQ", "pQ", "pV"] pol_screen = np.array( [ @@ -2560,13 +2564,20 @@ def _fix_autos(self): ] ) - auto_data = self.data_array[auto_screen] - if self.future_array_shapes: - auto_data[:, :, pol_screen] = np.abs(auto_data[:, :, pol_screen]) - else: - auto_data[:, :, :, pol_screen] = np.abs(auto_data[:, :, :, pol_screen]) + # Make sure we actually have work to do here, otherwise skip all of this + if np.any(pol_screen) and np.any(auto_screen): + # Select out the relevant data. Need to do this because we have two + # complex slices we need to do + auto_data = self.data_array[auto_screen] + + # Set the autos to be real-only by taking the absolute value + if self.future_array_shapes: + auto_data[:, :, pol_screen] = np.abs(auto_data[:, :, pol_screen]) + else: + auto_data[:, :, :, pol_screen] = np.abs(auto_data[:, :, :, pol_screen]) - self.data_array[auto_screen] = auto_data + # Finally, plug the modified values back into data_array + self.data_array[auto_screen] = auto_data def check( self, @@ -2603,6 +2614,12 @@ def check( the UVWs by -1) resolves the apparent discrepancy -- and if it does, fix the apparent conjugation error in `uvw_array` and `data_array`. Default is False. + check_autos : bool + Check whether any auto-correlations have non-zero imaginary values in + data_array (which should not mathematically exist). Default is False. + fix_autos : bool + If auto-correlations with imaginary values are found, fix those values so + that they are real-only in data_array. Default is False. Returns ------- @@ -10249,6 +10266,12 @@ def read_fhd( strict_uvw_antpos_check : bool Option to raise an error rather than a warning if the check that uvws match antenna positions does not pass. + check_autos : bool + Check whether any auto-correlations have non-zero imaginary values in + data_array (which should not mathematically exist). Default is True. + fix_autos : bool + If auto-correlations with imaginary values are found, fix those values so + that they are real-only in data_array. Default is True. Raises ------ @@ -10343,6 +10366,12 @@ def read_mir( "flexible polarization", which compresses the polarization-axis of various attributes to be of length 1, sets the `flex_spw_polarization_array` attribute to define the polarization per spectral window. Default is True. + check_autos : bool + Check whether any auto-correlations have non-zero imaginary values in + data_array (which should not mathematically exist). Default is True. + fix_autos : bool + If auto-correlations with imaginary values are found, fix those values so + that they are real-only in data_array. Default is True. """ from . import mir @@ -10470,6 +10499,12 @@ def read_miriad( If setting `fix_old_proj` to True, use the antenna positions to derive the correct uvw-coordinates rather than using the baseline vectors. Default is True. + check_autos : bool + Check whether any auto-correlations have non-zero imaginary values in + data_array (which should not mathematically exist). Default is True. + fix_autos : bool + If auto-correlations with imaginary values are found, fix those values so + that they are real-only in data_array. Default is True. Raises ------ @@ -10597,6 +10632,12 @@ def read_ms( "flexible polarization", which compresses the polarization-axis of various attributes to be of length 1, sets the `flex_spw_polarization_array` attribute to define the polarization per spectral window. Default is True. + check_autos : bool + Check whether any auto-correlations have non-zero imaginary values in + data_array (which should not mathematically exist). Default is True. + fix_autos : bool + If auto-correlations with imaginary values are found, fix those values so + that they are real-only in data_array. Default is True. Raises ------ @@ -10779,6 +10820,12 @@ def read_mwa_corr_fits( strict_uvw_antpos_check : bool Option to raise an error rather than a warning if the check that uvws match antenna positions does not pass. + check_autos : bool + Check whether any auto-correlations have non-zero imaginary values in + data_array (which should not mathematically exist). Default is True. + fix_autos : bool + If auto-correlations with imaginary values are found, fix those values so + that they are real-only in data_array. Default is True. Raises ------ @@ -10987,6 +11034,12 @@ def read_uvfits( If setting `fix_old_proj` to True, use the antenna positions to derive the correct uvw-coordinates rather than using the baseline vectors. Default is True. + check_autos : bool + Check whether any auto-correlations have non-zero imaginary values in + data_array (which should not mathematically exist). Default is True. + fix_autos : bool + If auto-correlations with imaginary values are found, fix those values so + that they are real-only in data_array. Default is True. Raises ------ @@ -11193,6 +11246,12 @@ def read_uvh5( If setting `fix_old_proj` to True, use the antenna positions to derive the correct uvw-coordinates rather than using the baseline vectors. Default is True. + check_autos : bool + Check whether any auto-correlations have non-zero imaginary values in + data_array (which should not mathematically exist). Default is True. + fix_autos : bool + If auto-correlations with imaginary values are found, fix those values so + that they are real-only in data_array. Default is True. Raises ------ @@ -11614,6 +11673,12 @@ def read( If the data are multi source or have multiple spectral windows. If phase_center_radec is not None and is not length 2. + check_autos : bool + Check whether any auto-correlations have non-zero imaginary values in + data_array (which should not mathematically exist). Default is True. + fix_autos : bool + If auto-correlations with imaginary values are found, fix those values so + that they are real-only in data_array. Default is True. """ if isinstance(filename, (list, tuple, np.ndarray)): @@ -12803,6 +12868,12 @@ def write_ms( strict_uvw_antpos_check : bool Option to raise an error rather than a warning if the check that uvws match antenna positions does not pass. + check_autos : bool + Check whether any auto-correlations have non-zero imaginary values in + data_array (which should not mathematically exist). Default is True. + fix_autos : bool + If auto-correlations with imaginary values are found, fix those values so + that they are real-only in data_array. Default is False. Raises ------ @@ -12877,6 +12948,12 @@ def write_uvfits( strict_uvw_antpos_check : bool Option to raise an error rather than a warning if the check that uvws match antenna positions does not pass. + check_autos : bool + Check whether any auto-correlations have non-zero imaginary values in + data_array (which should not mathematically exist). Default is True. + fix_autos : bool + If auto-correlations with imaginary values are found, fix those values so + that they are real-only in data_array. Default is False. Raises ------ @@ -12977,6 +13054,12 @@ def write_uvh5( strict_uvw_antpos_check : bool Option to raise an error rather than a warning if the check that uvws match antenna positions does not pass. + check_autos : bool + Check whether any auto-correlations have non-zero imaginary values in + data_array (which should not mathematically exist). Default is True. + fix_autos : bool + If auto-correlations with imaginary values are found, fix those values so + that they are real-only in data_array. Default is False. Raises ------ diff --git a/pyuvdata/uvdata/uvfits.py b/pyuvdata/uvdata/uvfits.py index e9d31e85ba..dd2c7160cc 100644 --- a/pyuvdata/uvdata/uvfits.py +++ b/pyuvdata/uvdata/uvfits.py @@ -503,11 +503,11 @@ def read_uvfits( correct uvw-coordinates rather than using the baseline vectors. Default is True. check_autos : bool - Check whether any auto-correlations have imaginary values in them (which - should not mathematically exist). Default is True. + Check whether any auto-correlations have non-zero imaginary values in + data_array (which should not mathematically exist). Default is True. fix_autos : bool If auto-correlations with imaginary values are found, fix those values so - that they are real-only. Default is True. + that they are real-only in data_array. Default is True. Raises ------ @@ -914,11 +914,11 @@ def write_uvfits( Option to raise an error rather than a warning if the check that uvws match antenna positions does not pass. check_autos : bool - Check whether any auto-correlations have imaginary values in them (which - should not mathematically exist). Default is True. + Check whether any auto-correlations have non-zero imaginary values in + data_array (which should not mathematically exist). Default is True. fix_autos : bool If auto-correlations with imaginary values are found, fix those values so - that they are real-only. Default is False. + that they are real-only in data_array. Default is False. Raises ------ diff --git a/pyuvdata/uvdata/uvh5.py b/pyuvdata/uvdata/uvh5.py index 4073714cd8..d7739bbbae 100644 --- a/pyuvdata/uvdata/uvh5.py +++ b/pyuvdata/uvdata/uvh5.py @@ -890,11 +890,11 @@ def read_uvh5( correct uvw-coordinates rather than using the baseline vectors. Default is True. check_autos : bool - Check whether any auto-correlations have imaginary values in them (which - should not mathematically exist). Default is True. + Check whether any auto-correlations have non-zero imaginary values in + data_array (which should not mathematically exist). Default is True. fix_autos : bool If auto-correlations with imaginary values are found, fix those values so - that they are real-only. Default is True. + that they are real-only in data_array. Default is True. Returns ------- @@ -1169,11 +1169,11 @@ def write_uvh5( Option to raise an error rather than a warning if the check that uvws match antenna positions does not pass. check_autos : bool - Check whether any auto-correlations have imaginary values in them (which - should not mathematically exist). Default is True. + Check whether any auto-correlations have non-zero imaginary values in + data_array (which should not mathematically exist). Default is True. fix_autos : bool If auto-correlations with imaginary values are found, fix those values so - that they are real-only. Default is False. + that they are real-only in data_array. Default is False. Returns ------- From f3771d6963840151535fa56f52df742df07c3b7f Mon Sep 17 00:00:00 2001 From: Garrett 'Karto' Keating Date: Sat, 11 Dec 2021 23:56:59 -0500 Subject: [PATCH 19/26] More docstring fixes, making sure to pass check arguments to MIR reader --- pyuvdata/uvdata/uvdata.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pyuvdata/uvdata/uvdata.py b/pyuvdata/uvdata/uvdata.py index bead571a47..2049af8ee4 100644 --- a/pyuvdata/uvdata/uvdata.py +++ b/pyuvdata/uvdata/uvdata.py @@ -12637,6 +12637,12 @@ def from_file( attribute to define the polarization per spectral window. Only applicable for MIR and MS filetypes, otherwise this argument is ignored. Default is True. + check_autos : bool + Check whether any auto-correlations have non-zero imaginary values in + data_array (which should not mathematically exist). Default is True. + fix_autos : bool + If auto-correlations with imaginary values are found, fix those values so + that they are real-only in data_array. Default is True. Raises ------ @@ -12769,6 +12775,12 @@ def write_miriad( marks the midpoint). Default is False, which instead uses a simple formula for correcting the LSTs, expected to be accurate to approximately 0.1 µsec precision. + check_autos : bool + Check whether any auto-correlations have non-zero imaginary values in + data_array (which should not mathematically exist). Default is True. + fix_autos : bool + If auto-correlations with imaginary values are found, fix those values so + that they are real-only in data_array. Default is False. Raises ------ From d3b1208d22114f93034fcf35b03c0248c4a40d66 Mon Sep 17 00:00:00 2001 From: Garrett 'Karto' Keating Date: Sun, 12 Dec 2021 00:03:05 -0500 Subject: [PATCH 20/26] Fix to a small bug potentially caused parts of check to be skipped if checking autos --- pyuvdata/uvdata/uvdata.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/pyuvdata/uvdata/uvdata.py b/pyuvdata/uvdata/uvdata.py index 2049af8ee4..e3d2808088 100644 --- a/pyuvdata/uvdata/uvdata.py +++ b/pyuvdata/uvdata/uvdata.py @@ -2767,6 +2767,8 @@ def check( ) if (self.data_array is not None and np.any(autos)) and check_autos: # Verify here that the autos do not have any imaginary components + # Only these pols have "true" auto-correlations, that we'd expect + # to be real only. Select on only them auto_pol_list = ["xx", "yy", "rr", "ll", "pI", "pQ", "pQ", "pV"] pol_screen = np.array( [ @@ -2774,19 +2776,20 @@ def check( for pol in self.polarization_array ] ) - # There's no relevant pols to check, just skip the rest of this - if not np.any(pol_screen): - return # Check autos if they have imag component -- doing iscomplex first and - # then pol select was faster in every case checked in test files - auto_imag = np.iscomplex(self.data_array[autos]) - if np.all(pol_screen): - auto_imag = np.any(auto_imag) - elif self.future_array_shapes: - auto_imag = np.any(auto_imag[:, :, pol_screen]) + # then pol select was faster in every case checked in test files. + if not np.any(pol_screen): + # There's no relevant pols to check, just skip the rest of this + auto_imag = False else: - auto_imag = np.any(auto_imag[:, :, :, pol_screen]) + auto_imag = np.iscomplex(self.data_array[autos]) + if np.all(pol_screen): + auto_imag = np.any(auto_imag) + elif self.future_array_shapes: + auto_imag = np.any(auto_imag[:, :, pol_screen]) + else: + auto_imag = np.any(auto_imag[:, :, :, pol_screen]) if auto_imag: if fix_autos: warnings.warn( From c07d2b36eef3e8e34f75415a1b8a8ea6b2eb2345 Mon Sep 17 00:00:00 2001 From: Garrett 'Karto' Keating Date: Sun, 12 Dec 2021 00:30:57 -0500 Subject: [PATCH 21/26] Updating CHANGELOG --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bbb51ab60d..68a3680e1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +### Added +- Added new functionality to `UVData.check` to verify that auto-correlations are real-only, +along with an option to force them to be real-only if non-zero imaginary components are detected. + ## [2.2.6] - 2022-01-12 ### Added From ac35cc75e6da507e35bd83fa341714171882d21a Mon Sep 17 00:00:00 2001 From: Garrett 'Karto' Keating Date: Sun, 12 Dec 2021 10:08:07 -0500 Subject: [PATCH 22/26] Small tweak to tests to improve coverage --- pyuvdata/uvdata/tests/test_mir.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pyuvdata/uvdata/tests/test_mir.py b/pyuvdata/uvdata/tests/test_mir.py index 2fd9c0d446..bf7e8ac45d 100644 --- a/pyuvdata/uvdata/tests/test_mir.py +++ b/pyuvdata/uvdata/tests/test_mir.py @@ -280,7 +280,8 @@ def test_read_mir_no_records(): uv_in.read_mir(testfile, corrchunk=999) -def test_read_mir_sideband_select(sma_mir): +@pytest.mark.parametrize("pseudo_cont", [True, False]) +def test_read_mir_sideband_select(sma_mir, pseudo_cont): """ Mir sideband read check @@ -288,6 +289,8 @@ def test_read_mir_sideband_select(sma_mir): stitch them back together as though they were read together from the start. """ testfile = os.path.join(DATA_PATH, "sma_test.mir") + if pseudo_cont: + sma_mir.read(testfile, pseudo_cont=pseudo_cont) # Re-order here so that we can more easily compare the two sma_mir.reorder_freqs(channel_order="freq", spw_order="freq") @@ -295,10 +298,10 @@ def test_read_mir_sideband_select(sma_mir): sma_mir.history = "" mir_lsb = UVData() - mir_lsb.read(testfile, isb=[0]) + mir_lsb.read(testfile, isb=[0], pseudo_cont=pseudo_cont) mir_usb = UVData() - mir_usb.read(testfile, isb=[1]) + mir_usb.read(testfile, isb=[1], pseudo_cont=pseudo_cont) mir_recomb = mir_lsb + mir_usb # Re-order here so that we can more easily compare the two From 4ac9c9ac94b9a806d7cbd2ffab0b255ab4ecf24d Mon Sep 17 00:00:00 2001 From: Garrett 'Karto' Keating Date: Mon, 13 Dec 2021 16:36:10 -0500 Subject: [PATCH 23/26] Small test fix to failing test post rebase --- pyuvdata/uvdata/tests/test_mwa_corr_fits.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyuvdata/uvdata/tests/test_mwa_corr_fits.py b/pyuvdata/uvdata/tests/test_mwa_corr_fits.py index c23e388576..ee9d7a0174 100644 --- a/pyuvdata/uvdata/tests/test_mwa_corr_fits.py +++ b/pyuvdata/uvdata/tests/test_mwa_corr_fits.py @@ -131,6 +131,7 @@ def test_read_mwax_write_uvfits(tmp_path): mwax_uv = UVData() uvfits_uv = UVData() messages = [ + "Fixing auto-correlations to be be real-only, after some imaginary values", "some coarse channel files were not submitted", ] with uvtest.check_warnings(UserWarning, messages): From 858522cd774bd3494d9b7813c65be214201f49ab Mon Sep 17 00:00:00 2001 From: Garrett 'Karto' Keating Date: Thu, 16 Dec 2021 22:28:34 -0500 Subject: [PATCH 24/26] Adding fix_autos to write_uvh5_part, adding test coverage --- pyuvdata/uvdata/tests/test_uvh5.py | 41 ++++++++++++++++++++++++++++++ pyuvdata/uvdata/uvdata.py | 20 ++++++++++++++- 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/pyuvdata/uvdata/tests/test_uvh5.py b/pyuvdata/uvdata/tests/test_uvh5.py index 5d39ae8c5b..a83c6864d9 100644 --- a/pyuvdata/uvdata/tests/test_uvh5.py +++ b/pyuvdata/uvdata/tests/test_uvh5.py @@ -3314,3 +3314,44 @@ def test_none_extra_keywords(uv_uvh5, tmp_path): assert h5f["Header/extra_keywords/foo"].shape is None return + + +def test_write_uvh5_part_fix_autos(uv_uvh5, tmp_path): + """Test that fix_autos works correctly on partial UVH5 wrute""" + test_uvh5 = UVData() + testfile = os.path.join(tmp_path, "write_uvh5_part_fix_autos.uvh5") + + # Select out the relevant data (where the 0 and 1 indicies of the pol array + # correspond to xx and yy polarization data), and corrupt it accordingly + auto_data = uv_uvh5.data_array[uv_uvh5.ant_1_array == uv_uvh5.ant_2_array] + auto_data[:, :, :, [0, 1]] *= 1j + uv_uvh5.data_array[uv_uvh5.ant_1_array == uv_uvh5.ant_2_array] = auto_data + + # Create and write out the data, with fix_autos set to operate + initialize_with_zeros_ints(uv_uvh5, testfile) + uv_uvh5.write_uvh5_part( + testfile, + uv_uvh5.data_array, + uv_uvh5.flag_array, + uv_uvh5.nsample_array, + fix_autos=True, + ) + + # Fix the autos we corrupted earlier, and plug the data back in to data_array + auto_data[:, :, :, [0, 1]] *= -1j + uv_uvh5.data_array[uv_uvh5.ant_1_array == uv_uvh5.ant_2_array] = auto_data + + # Read in the data on disk, make sure it looks like our manually repaired data + test_uvh5.read(testfile) + + assert uv_uvh5 == test_uvh5 + + +def test_fix_autos_no_op(): + """Test that a no-op with _fix_autos returns a warning""" + uvd = UVData() + + with uvtest.check_warnings( + UserWarning, "Cannot use _fix_autos if ant_1_array, ant_2_array, or " + ): + uvd._fix_autos() diff --git a/pyuvdata/uvdata/uvdata.py b/pyuvdata/uvdata/uvdata.py index e3d2808088..99a36ac1ea 100644 --- a/pyuvdata/uvdata/uvdata.py +++ b/pyuvdata/uvdata/uvdata.py @@ -2551,6 +2551,15 @@ def _calc_nants_data(self): def _fix_autos(self): """Remove imaginary component of auto-correlations.""" + if self.polarization_array is None or ( + self.ant_1_array is None or self.ant_2_array is None + ): + warnings.warn( + "Cannot use _fix_autos if ant_1_array, ant_2_array, or " + "polarization_array are None. Leaving data_array untouched." + ) + return + # Select out the autos auto_screen = self.ant_1_array == self.ant_2_array @@ -2565,7 +2574,9 @@ def _fix_autos(self): ) # Make sure we actually have work to do here, otherwise skip all of this - if np.any(pol_screen) and np.any(auto_screen): + if (np.any(pol_screen) and np.any(auto_screen)) and not ( + pol_screen is None or auto_screen is None + ): # Select out the relevant data. Need to do this because we have two # complex slices we need to do auto_data = self.data_array[auto_screen] @@ -13185,6 +13196,7 @@ def write_uvh5_part( blt_inds=None, add_to_history=None, run_check_acceptability=True, + fix_autos=False, ): """ Write data to a UVH5 file that has already been initialized. @@ -13275,8 +13287,14 @@ def write_uvh5_part( strict_uvw_antpos_check : bool Option to raise an error rather than a warning if the check that uvws match antenna positions does not pass. + fix_autos : bool + Force the auto-correlations to be real-only values in data_array. + Default is False. """ + if fix_autos: + self._fix_autos() + uvh5_obj = self._convert_to_filetype("uvh5") uvh5_obj.write_uvh5_part( filename, From 994c8d850b70e64d16aad88fb2e2b8938024aec3 Mon Sep 17 00:00:00 2001 From: Garrett 'Karto' Keating Date: Thu, 30 Dec 2021 12:15:41 -0500 Subject: [PATCH 25/26] Fixing naming conventions in test fixtures to be a bit more uniform --- pyuvdata/uvdata/tests/test_uvdata.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pyuvdata/uvdata/tests/test_uvdata.py b/pyuvdata/uvdata/tests/test_uvdata.py index 167b6b9825..71dbcc1194 100644 --- a/pyuvdata/uvdata/tests/test_uvdata.py +++ b/pyuvdata/uvdata/tests/test_uvdata.py @@ -11257,10 +11257,10 @@ def test_from_file(filename): @pytest.mark.parametrize("add_type", ["blt", "freq", "pol"]) @pytest.mark.parametrize("sort_type", ["blt", "freq", "pol"]) -@pytest.mark.parametrize("future_array_shapes", [True, False]) +@pytest.mark.parametrize("future_shapes", [True, False]) @pytest.mark.filterwarnings("ignore:The uvw_array does not match the expected values") -def test_add_pol_sorting_bl(casa_uvfits, add_type, sort_type, future_array_shapes): - if future_array_shapes: +def test_add_pol_sorting_bl(casa_uvfits, add_type, sort_type, future_shapes): + if future_shapes: casa_uvfits.use_future_array_shapes() if add_type == "pol": @@ -11394,9 +11394,9 @@ def test_make_flex_pol_errs(sma_mir, err_msg, param, param_val): @pytest.mark.parametrize("dataset", ["hera", "mwa"]) -@pytest.mark.parametrize("future_array_shapes", [True, False]) +@pytest.mark.parametrize("future_shapes", [True, False]) @pytest.mark.filterwarnings("ignore:The uvw_array does not match the expected values") -def test_auto_check(hera_uvh5, uv_phase_comp, future_array_shapes, dataset, tmp_path): +def test_auto_check(hera_uvh5, uv_phase_comp, future_shapes, dataset, tmp_path): """ Checks that checking/fixing the autos works correctly, both with dual-pol data (supplied by hera_uvh5) and full-pol data (supplied by casa_uvfits). @@ -11406,7 +11406,7 @@ def test_auto_check(hera_uvh5, uv_phase_comp, future_array_shapes, dataset, tmp_ elif dataset == "mwa": uv, _ = uv_phase_comp - if future_array_shapes: + if future_shapes: uv.use_future_array_shapes() out_file = os.path.join(tmp_path, "auto_check.uvh5") @@ -11437,7 +11437,7 @@ def test_auto_check(hera_uvh5, uv_phase_comp, future_array_shapes, dataset, tmp_ assert uv1 == uv2 - if future_array_shapes: + if future_shapes: uv1.use_future_array_shapes() assert uv == uv1 From 3b772614e942d988f1dfec9afd33eccb781b3130 Mon Sep 17 00:00:00 2001 From: Garrett 'Karto' Keating Date: Wed, 12 Jan 2022 22:40:32 -0500 Subject: [PATCH 26/26] Cleaning up post-rebase --- pyuvdata/tests/test_utils.py | 1 + pyuvdata/uvdata/mir.py | 10 ---- pyuvdata/uvdata/tests/conftest.py | 50 +++++++------------ pyuvdata/uvdata/tests/test_mir.py | 63 ++++++++++-------------- pyuvdata/uvdata/tests/test_mir_parser.py | 1 - 5 files changed, 44 insertions(+), 81 deletions(-) diff --git a/pyuvdata/tests/test_utils.py b/pyuvdata/tests/test_utils.py index fe643aaa6e..25956aedbb 100644 --- a/pyuvdata/tests/test_utils.py +++ b/pyuvdata/tests/test_utils.py @@ -3366,6 +3366,7 @@ def test_uvcalibrate_wideband_gain(uvcalibrate_data): uvutils.uvcalibrate(uvd, uvc, inplace=False) +@pytest.mark.filterwarnings("ignore:Fixing auto-correlations to be be real-only") @pytest.mark.filterwarnings("ignore:The uvw_array does not match the expected values") @pytest.mark.filterwarnings("ignore:When converting a delay-style cal to future array") @pytest.mark.filterwarnings("ignore:Nfreqs will be required to be 1 for wide_band cals") diff --git a/pyuvdata/uvdata/mir.py b/pyuvdata/uvdata/mir.py index 91d56fecb7..4536b9b91a 100644 --- a/pyuvdata/uvdata/mir.py +++ b/pyuvdata/uvdata/mir.py @@ -620,16 +620,6 @@ def _init_from_mir_parser(self, mir_data, allow_flex_pol=True): self.flag_array = np.zeros(self.data_array.shape, dtype=bool) self.nsample_array = np.ones(self.data_array.shape, dtype=np.float32) - if run_check: - self.check( - check_extra=check_extra, - run_check_acceptability=run_check_acceptability, - strict_uvw_antpos_check=strict_uvw_antpos_check, - allow_flip_conj=True, - check_autos=check_autos, - fix_autos=fix_autos, - ) - def write_mir(self, filename): """ Write out the SMA MIR files. diff --git a/pyuvdata/uvdata/tests/conftest.py b/pyuvdata/uvdata/tests/conftest.py index 2f514b2796..4a24adbbc1 100644 --- a/pyuvdata/uvdata/tests/conftest.py +++ b/pyuvdata/uvdata/tests/conftest.py @@ -49,23 +49,6 @@ def casa_uvfits(casa_uvfits_main): return -@pytest.fixture(scope="session") -def mir_data_main(): - testfile = os.path.join(DATA_PATH, "sma_test.mir") - mir_data = MirParser( - testfile, load_vis=True, load_raw=True, load_auto=True, has_auto=True, - ) - - yield mir_data - - -@pytest.fixture(scope="function") -def mir_data(mir_data_main): - mir_data = mir_data_main.copy() - - yield mir_data - - @pytest.fixture(scope="session") def hera_uvh5_main(): # read in test file for the resampling in time functions @@ -116,21 +99,7 @@ def paper_miriad(paper_miriad_main): del uv_in -@pytest.fixture(params=[True, False]) -def mir_data_object(request): - """Make MIR data object for tests. Param to read autocorr data.""" - has_auto = request.param - testfile = os.path.join(DATA_PATH, "sma_test.mir") - mir_data = MirParser( - testfile, load_vis=True, load_raw=True, load_auto=True, has_auto=has_auto - ) - - yield mir_data - - # cleanup - del mir_data - - +@pytest.fixture(scope="session") def sma_mir_main(): # read in test file for the resampling in time functions uv_object = UVData() @@ -146,3 +115,20 @@ def sma_mir(sma_mir_main): uv_object = sma_mir_main.copy() yield uv_object + + +@pytest.fixture(scope="session") +def mir_data_main(): + testfile = os.path.join(DATA_PATH, "sma_test.mir") + mir_data = MirParser( + testfile, load_vis=True, load_raw=True, load_auto=True, has_auto=True, + ) + + yield mir_data + + +@pytest.fixture(scope="function") +def mir_data(mir_data_main): + mir_data = mir_data_main.copy() + + yield mir_data diff --git a/pyuvdata/uvdata/tests/test_mir.py b/pyuvdata/uvdata/tests/test_mir.py index bf7e8ac45d..c70ff23a1d 100644 --- a/pyuvdata/uvdata/tests/test_mir.py +++ b/pyuvdata/uvdata/tests/test_mir.py @@ -22,50 +22,39 @@ @pytest.fixture(scope="session") -def mir_data(): +def sma_mir_filt_main(): + # read in test file for the resampling in time functions + uv_object = UVData() testfile = os.path.join(DATA_PATH, "sma_test.mir") - mir_data = MirParser( - testfile, load_vis=True, load_raw=True, load_auto=True, - ) - - yield mir_data - - -@pytest.fixture -def uv_in_ms(tmp_path): - write_file = os.path.join(tmp_path, "outtest_mir.ms") - uv_out = UVData() - - yield uv_out, write_file + uv_object.read(testfile, pseudo_cont=True, corrchunk=0) + uv_object.flag_array[:, :, : uv_object.Nfreqs // 2, 0] = True + uv_object.flag_array[:, :, uv_object.Nfreqs // 2 :, 1] = True + uv_object.set_lsts_from_time_array() + uv_object._set_app_coords_helper() -@pytest.fixture -def uv_in_uvfits(tmp_path): - write_file = os.path.join(tmp_path, "outtest_mir.uvfits") + yield uv_object - uv_out = UVData() - yield uv_out, write_file +@pytest.fixture(scope="function") +def sma_mir_filt(sma_mir_filt_main): + # read in test file for the resampling in time functions + uv_object = sma_mir_filt_main.copy() - -@pytest.fixture -def uv_in_uvh5(tmp_path): - write_file = os.path.join(tmp_path, "outtest_mir.uvh5") - uv_out = UVData() - - yield uv_out, write_file + yield uv_object @pytest.mark.filterwarnings("ignore:LST values stored in this file are not ") @pytest.mark.parametrize("future_shapes", [True, False]) -def test_read_mir_write_uvfits(sma_mir, uv_in_uvfits, future_shapes): +def test_read_mir_write_uvfits(sma_mir, tmp_path, future_shapes): """ Mir to uvfits loopback test. Read in Mir files, write out as uvfits, read back in and check for object equality. """ - uvfits_uv, testfile = uv_in_uvfits + testfile = os.path.join(tmp_path, "outtest_mir.uvfits") + uvfits_uv = UVData() if future_shapes: sma_mir.use_future_array_shapes() @@ -143,7 +132,7 @@ def test_read_mir_write_uvfits(sma_mir, uv_in_uvfits, future_shapes): @pytest.mark.filterwarnings("ignore:Writing in the MS file that the units of the data") @pytest.mark.filterwarnings("ignore:LST values stored in this file are not ") @pytest.mark.parametrize("future_shapes", [True, False]) -def test_read_mir_write_ms(sma_mir, uv_in_ms, future_shapes): +def test_read_mir_write_ms(sma_mir, tmp_path, future_shapes): """ Mir to uvfits loopback test. @@ -151,7 +140,8 @@ def test_read_mir_write_ms(sma_mir, uv_in_ms, future_shapes): object equality. """ pytest.importorskip("casacore") - ms_uv, testfile = uv_in_ms + testfile = os.path.join(tmp_path, "outtest_mir.ms") + ms_uv = UVData() if future_shapes: sma_mir.use_future_array_shapes() @@ -202,14 +192,15 @@ def test_read_mir_write_ms(sma_mir, uv_in_ms, future_shapes): @pytest.mark.filterwarnings("ignore:LST values stored ") -def test_read_mir_write_uvh5(sma_mir, uv_in_uvh5): +def test_read_mir_write_uvh5(sma_mir, tmp_path): """ Mir to uvfits loopback test. Read in Mir files, write out as uvfits, read back in and check for object equality. """ - uvh5_uv, testfile = uv_in_uvh5 + testfile = os.path.join(tmp_path, "outtest_mir.uvh5") + uvh5_uv = UVData() sma_mir.write_uvh5(testfile) uvh5_uv.read_uvh5(testfile) @@ -401,12 +392,10 @@ def test_read_mir_write_ms_flex_pol(mir_data, tmp_path): assert ms_uv.__eq__(mir_uv, allowed_failures=["filename"]) -def test_inconsistent_sp_records(mir_data, uv_in_ms): +def test_inconsistent_sp_records(mir_data, sma_mir): """ Test that the MIR object does the right thing w/ inconsistent meta-data. """ - sma_mir, _, _ = uv_in_ms - mir_data.use_sp = mir_data.sp_read["iband"] != 0 mir_data.sp_read["ipq"][1] = 0 mir_data.load_data() @@ -420,12 +409,10 @@ def test_inconsistent_sp_records(mir_data, uv_in_ms): assert mir_uv == sma_mir -def test_inconsistent_bl_records(mir_data, uv_in_ms): +def test_inconsistent_bl_records(mir_data, sma_mir): """ Test that the MIR object does the right thing w/ inconsistent meta-data. """ - sma_mir, _, _ = uv_in_ms - mir_data.use_sp = mir_data.sp_read["iband"] != 0 mir_data.bl_read["u"][0] = 0.0 mir_data.load_data() diff --git a/pyuvdata/uvdata/tests/test_mir_parser.py b/pyuvdata/uvdata/tests/test_mir_parser.py index a334c70b78..e66819c877 100644 --- a/pyuvdata/uvdata/tests/test_mir_parser.py +++ b/pyuvdata/uvdata/tests/test_mir_parser.py @@ -52,7 +52,6 @@ def test_mir_parser_index_linked(mir_data): inhid_set = set(np.unique(mir_data.in_read["inhid"])) # Should not exist is has_auto=False - # See `mir_data_object` above. if mir_data.ac_read is not None: assert set(np.unique(mir_data.ac_read["inhid"])).issubset(inhid_set) else: