Merge remote-tracking branch 'origin/order_sync' into spatflex_options

# Conflicts:
#	pypeit/core/flexure.py
#	pypeit/wavecalib.py

doc/releases/1.16.1dev.rst

@@ -21,9 +21,18 @@ Dependency Changes
 Functionality/Performance Improvements and Additions
 ----------------------------------------------------
 
+- Added the ``max_overlap`` parameter, which limits the set of new order traces
+  added, to compensate for orders missed during automated edge-tracing, to those
+  that have less than a given fractional overlap with adjacent orders.
+- Added the ``order_fitrej`` and ``order_outlier`` parameters used to set the
+  sigma-clipping threshold used when fitting Legendre functions to the order
+  widths and gaps.
 - Added the possibility to decide if the extracted standard star spectrum should be
   used as a crutch for tracing the object in the science frame (before it was done as default).
   This is done by setting the parameter ``use_std_trace`` in FindObjPar.
+- Now PypeIt can handle the case where "Standard star trace does not match the
+  number of orders in the echelle data" both in `run_pypeit` and in
+  `pypeit_coadd_1dspec`.
 - Now PypeIt can handle the case where "Standard star trace does not match the number of orders in the echelle data"
   both in `run_pypeit` and in `pypeit_coadd_1dspec`.
 - The spatial flexure can now take a constant value for every slit, independent values for each slit,
@@ -112,6 +121,14 @@ Under-the-hood Improvements
 - Introduced :class:`~pypeit.pypeitdata.PypeItDataPaths` to handle all
   interactions with the ``pypeit/data`` directory, which provides a unified
   interface for accessing on-disk and cached files.
+- When adding missing orders, the full syncing procedure is no longer performed.
+  The code now only checks that the edges are still synced after the missed
+  orders are added.
+- When detecting overlapping orders/slits, the code now forces each edge used to
+  have been directly detected; i.e., if an edge is inserted, the fact that the
+  resulting slit is abnormally short should not trigger the overlap detection.
+- Improved the QA plot resulting from fitting order widths and gaps as a
+  function of spatial position.
 - Updated general raw image reader so that it correctly accounts for
   spectrographs that read the data and overscan sections directly from the file
   headers.

pypeit/core/datacube.py

@@ -223,7 +223,7 @@ def extract_point_source(wave, flxcube, ivarcube, bpmcube, wcscube, exptime,
 
     Returns
     -------
-    sobjs : :class:`pypeit.specobjs.SpecObjs`
+    sobjs : :class:`~pypeit.specobjs.SpecObjs`
         SpecObjs object containing the extracted spectrum
     """
     if whitelight_range is None:

pypeit/core/flexure.py

@@ -37,7 +37,7 @@
 from pypeit.datamodel import DataContainer
 from pypeit.images.detector_container import DetectorContainer
 from pypeit.images.mosaic import Mosaic
-from pypeit import specobj, specobjs, spec2dobj
+from pypeit import specobj, specobjs
 from pypeit import wavemodel
 
 from IPython import embed

pypeit/core/skysub.py

@@ -1090,7 +1090,7 @@ def ech_local_skysub_extract(sciimg, sciivar, fullmask, tilts, waveimg,
     Parameters
     ----------
     sciimg : `numpy.ndarray`_
-        science image, usually with a global sky subtracted.
+        Science image, usually with a global sky subtracted.
         shape = (nspec, nspat)
     sciivar : `numpy.ndarray`_
         inverse variance of science image.
@@ -1181,9 +1181,9 @@ def ech_local_skysub_extract(sciimg, sciivar, fullmask, tilts, waveimg,
             fullbkpt = bset.breakpoints
 
     force_gauss : bool, default = False
-        If True, a Gaussian profile will always be assumed for the
-        optimal extraction using the FWHM determined from object finding (or provided by the user) for the spatial
-        profile.
+        If True, a Gaussian profile will always be assumed for the optimal
+        extraction using the FWHM determined from object finding (or provided by
+        the user) for the spatial profile.
     sn_gauss : int or float, default = 4.0
         The signal to noise threshold above which optimal extraction
         with non-parametric b-spline fits to the objects spatial

pypeit/core/trace.py

@@ -1647,7 +1647,41 @@ def predicted_center_difference(lower_spat, spat, width_fit, gap_fit):
     return np.absolute(spat - test_spat)
 
 
-def extrapolate_orders(cen, width_fit, gap_fit, min_spat, max_spat, tol=0.01):
+def extrapolate_prev_order(cen, width_fit, gap_fit, tol=0.01):
+    """
+    Predict the location of the order to the spatial left (lower spatial pixel
+    numbers) of the order center provided.
+
+    This is largely a support function for :func:`extrapolate_orders`.
+
+    Args:
+        cen (:obj:`float`):
+            Center of the known order.
+        width_fit (:class:`~pypeit.core.fitting.PypeItFit`):
+            Model of the order width as a function of the order center.
+        gap_fit (:class:`~pypeit.core.fitting.PypeItFit`):
+            Model of the order gap *after* each order as a function of the order
+            center.
+        tol (:obj:`float`, optional):
+            Tolerance used when optimizing the order locations predicted toward
+            lower spatial pixels.
+
+    Returns:
+        :obj:`float`: Predicted center of the previous order.
+    """
+    # Guess the position of the previous order
+    w = width_fit.eval(cen)
+    guess = np.array([cen - w - gap_fit.eval(cen)])
+    # Set the bounds based on this guess and the expected order width
+    bounds = optimize.Bounds(lb=guess - w/2, ub=guess + w/2)
+    # Optimize the spatial position
+    res = optimize.minimize(predicted_center_difference, guess,
+                            args=(cen, width_fit, gap_fit),
+                            method='trust-constr', jac='2-point', bounds=bounds, tol=tol)    
+    return res.x[0]
+
+
+def extrapolate_orders(cen, width_fit, gap_fit, min_spat, max_spat, tol=0.01, bracket=False):
     """
     Predict the locations of additional orders by extrapolation.
 
@@ -1676,6 +1710,9 @@ def extrapolate_orders(cen, width_fit, gap_fit, min_spat, max_spat, tol=0.01):
         tol (:obj:`float`, optional):
             Tolerance used when optimizing the order locations predicted toward
             lower spatial pixels.
+        bracket (:obj:`bool`, optional):
+            Bracket the added orders with one additional order on either side.
+            This can be useful for dealing with predicted overlap.
 
     Returns:
         :obj:`tuple`: Two arrays with orders centers (1) below the first and (2)
@@ -1686,25 +1723,20 @@ def extrapolate_orders(cen, width_fit, gap_fit, min_spat, max_spat, tol=0.01):
     # Extrapolate toward lower spatial positions
     lower_spat = [cen[0]]
     while lower_spat[-1] > min_spat:
-        # Guess the position of the previous order
-        l = width_fit.eval(lower_spat[-1])
-        guess = np.array([lower_spat[-1] - l - gap_fit.eval(lower_spat[-1])])
-        # Set the bounds based on this guess and the expected order width
-        bounds = optimize.Bounds(lb=guess - l/2, ub=guess + l/2)
-        # Optimize the spatial position
-        res = optimize.minimize(predicted_center_difference, guess,
-                                args=(lower_spat[-1], width_fit, gap_fit),
-                                method='trust-constr', jac='2-point', bounds=bounds, tol=tol)
-        lower_spat += [res.x[0]]
+        lower_spat += [extrapolate_prev_order(lower_spat[-1], width_fit, gap_fit, tol=tol)]
 
     # Extrapolate toward larger spatial positions
     upper_spat = [cen[-1]]
     while upper_spat[-1] < max_spat:
         upper_spat += [upper_spat[-1] + width_fit.eval(upper_spat[-1]) 
                         + gap_fit.eval(upper_spat[-1])]
 
-    # Return arrays after removing the first and last spatial position (which
-    # are either repeats of values in `cen` or outside the spatial range)
+    # Return arrays after removing the first spatial position because it is a
+    # repeat of the input.  If not bracketting, also remove the last point
+    # because it will be outside the minimum or maximum spatial position (set by
+    # min_spat, max_spat).
+    if bracket:
+        return np.array(lower_spat[-1:0:-1]), np.array(upper_spat[1:])
     return np.array(lower_spat[-2:0:-1]), np.array(upper_spat[1:-1])