From 65f1a9e500658722cd8f325168821ca10e6bd21a Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Mon, 5 Mar 2018 09:40:33 -0800 Subject: [PATCH 01/36] test commit --- pypit/arproc.py | 73 +++++++++++++++- pypit/artrace.py | 213 ++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 283 insertions(+), 3 deletions(-) diff --git a/pypit/arproc.py b/pypit/arproc.py index 679881bb35..61119f5aa5 100644 --- a/pypit/arproc.py +++ b/pypit/arproc.py @@ -22,6 +22,9 @@ # Logging msgs = armsgs.get_logger() +#KBW TESTING +import time + def background_subtraction(slf, sciframe, varframe, slitn, det, refine=0.0): """ Generate a frame containing the background sky spectrum @@ -1710,7 +1713,35 @@ def lacosmic(slf, fitsdict, det, sciframe, scidx, maxiter=1, grow=1.5, maskval=- filt = ndimage.sobel(sciframe, axis=1, mode='constant') filty = ndimage.sobel(filt/np.sqrt(np.abs(sciframe)), axis=0, mode='constant') filty[np.where(np.isnan(filty))]=0.0 - sigimg = arcyproc.cr_screen(filty,0.0) + +# t = time.clock() +# sigimg = arcyproc.cr_screen(filty,0.0) +# print('Old cr_screen: {0} seconds'.format(time.clock() - t)) +# +# t = time.clock() +# new_sigimg = new_cr_screen(filty,0.0) +# print('New cr_screen: {0} seconds'.format(time.clock() - t)) +# sigimg = arcyproc.cr_screen(filty,0.0) + sigimg = new_cr_screen(filty) + +# print(sigimg.shape) +# print(new_sigimg.shape) +# print(np.ma.mean(np.ma.log10(sigimg))) +# print(np.ma.mean(np.ma.log10(new_sigimg))) +# print(np.ma.mean(np.ma.log10(sigimg)-np.ma.log10(new_sigimg))) +# +# plt.imshow(np.ma.log10(sigimg), origin='lower', interpolation='nearest', aspect='auto') +# plt.colorbar() +# plt.show() +# +# plt.imshow(np.ma.log10(new_sigimg), origin='lower', interpolation='nearest', aspect='auto') +# plt.colorbar() +# plt.show() +# +# plt.imshow(np.ma.log10(new_sigimg)-np.ma.log10(sigimg), origin='lower', interpolation='nearest', aspect='auto') +# plt.colorbar() +# plt.show() + sigsmth = ndimage.filters.gaussian_filter(sigimg,1.5) sigsmth[np.where(np.isnan(sigsmth))]=0.0 sigmask = np.cast['bool'](np.zeros(sciframe.shape)) @@ -1721,6 +1752,46 @@ def lacosmic(slf, fitsdict, det, sciframe, scidx, maxiter=1, grow=1.5, maskval=- return crmask +def new_cr_screen(a, mask_value=0.0, spatial_axis=1): + r""" + Calculate the significance of pixel deviations from the median along + the spatial direction. + + No type checking is performed of the input array; however, the + function assumes floating point values. + + Args: + a (numpy.ndarray): Input 2D array + mask_value (float): (**Optional**) Values to ignore during the + calculation of the median. Default is 0.0. + spatial_axis (int): (**Optional**) Axis along which to calculate + the median. Default is 1. + + Returns: + numpy.ndarray: Returns a map of :math:`|\Delta_{i,j}|/\sigma_j`, + where :math:`\Delta_{i,j}` is the difference between the pixel + value and the median along axis :math:`i` and :math:`\sigma_j` + is robustly determined using the median absolute deviation, + :math:`sigma_j = 1.4826 MAD`. + """ + # Check input + if len(a.shape) != 2: + msgs.error('Input array must be two-dimensional.') + if spatial_axis not in [0,1]: + msgs.error('Spatial axis must be 0 or 1.') + + # Mask the pixels equal to mask value: should use np.isclose() + _a = np.ma.MaskedArray(a, mask=(a==mask_value)) + # Get the median along the spatial axis + meda = np.ma.median(_a, axis=spatial_axis) + # Get a robust measure of the standard deviation using the median + # absolute deviation; 1.4826 factor is the ratio of sigma/MAD + d = np.absolute(_a - meda[:,None]) + mada = 1.4826*np.ma.median(d, axis=spatial_axis) + # Return the ratio of the difference to the standard deviation + return np.ma.divide(d, mada[:,None]).filled(0.0) + + def gain_frame(slf, det): """ Generate a gain image from the spect dict diff --git a/pypit/artrace.py b/pypit/artrace.py index 4eb8a3a012..498722ee54 100644 --- a/pypit/artrace.py +++ b/pypit/artrace.py @@ -1,5 +1,6 @@ from __future__ import (print_function, absolute_import, division, unicode_literals) +import time import numpy as np import copy from pypit import arqa @@ -121,7 +122,17 @@ def assign_slits(binarr, edgearr, ednum=100000, lor=-1): # After pruning, there are no more peaks break pks = wpk+2 # Shifted by 2 because of the peak finding algorithm above + print('calling find_peak_limits') + + t = time.clock() pedges = arcytrace.find_peak_limits(smedgehist, pks) + print('Old find_peak_limits: {0} seconds'.format(time.clock() - t)) + print(pedges.shape) + print(pedges[:,0]) + t = time.clock() + pedges = new_find_peak_limits(smedgehist, pks) + print('New find_peak_limits: {0} seconds'.format(time.clock() - t)) + exit() if np.all(pedges[:, 1]-pedges[:, 0] == 0): # Remaining peaks have no width break @@ -567,7 +578,13 @@ def trace_object(slf, det, sciframe, varframe, crmask, trim=2, trcprof2 = np.mean(rec_sciframe, axis=0) objl, objr, bckl, bckr = find_obj_minima(trcprof2, triml=triml, trimr=trimr, nsmooth=nsmooth) elif settings.argflag['trace']['object']['find'] == 'standard': - objl, objr, bckl, bckr = arcytrace.find_objects(trcprof, bgreg, mad) +# print('calling find_objects') +# t = time.clock() +# objl, objr, bckl, bckr = arcytrace.find_objects(trcprof, bgreg, mad) +# print('Old find_objects: {0} seconds'.format(time.clock() - t)) +# t = time.clock() + objl, objr, bckl, bckr = new_find_objects(trcprof, bgreg, mad) +# print('New find_objects: {0} seconds'.format(time.clock() - t)) else: msgs.error("Bad object identification algorithm!!") if msgs._debug['trace_obj']: @@ -698,6 +715,175 @@ def trace_object(slf, det, sciframe, varframe, crmask, trim=2, return tracedict +def new_find_objects(profile, bgreg, stddev): + """ + Find significantly detected objects in the profile array + For all objects found, the background regions will be defined. + """ + # Input profile must be a vector + if len(profile.shape) != 1: + raise ValueError('Input profile array must be 1D.') + # Get the length of the array + sz_x = profile.size + # Copy it to a masked array for processing + # TODO: Assumes input is NOT a masked array + _profile = np.ma.MaskedArray(profile.copy()) + + # Peaks are found as having flux at > 5 sigma and background is + # where the flux is not greater than 3 sigma + gt_5_sigma = profile > 5*stddev + not_gt_3_sigma = np.invert(profile > 3*stddev) + + # Define the object centroids array + objl = np.zeros(sz_x, dtype=int) + objr = np.zeros(sz_x, dtype=int) + has_obj = np.zeros(sz_x, dtype=bool) + + obj = 0 + while np.any(gt_5_sigma & np.invert(_profile.mask)): + # Find next maximum flux point + imax = np.ma.argmax(_profile) + maxv = _profile[imax] + # Find the valid source pixels around the peak + f = np.arange(sz_x)[np.roll(not_gt_3_sigma, -imax)] + objl[obj] = imax-sz_x+f[-1] + objr[obj] = f[0]+imax + # Mask source pixels and increment for next iteration + has_obj[objl[obj]:objr[obj]+1] = True + _profile[objl[obj]:objr[obj]+1] = np.ma.masked + obj += 1 + + # The background is the region away from sources up to the provided + # region size. Starting pixel for the left limit... + s = objl[:obj]-bgreg + s[s < 0] = 0 + # ... and ending pixel for the right limit. + e = objr[:obj]+1+bgreg + e[e > sz_x] = sz_x + # Flag the possible background regions + bgl = np.zeros((sz_x,obj), dtype=bool) + bgr = np.zeros((sz_x,obj), dtype=bool) + for i in range(obj): + bgl[s[i]:objl[i],i] = True + bgr[objl[i]+1:e[i],i] = True + + # Return source region limits and background regions that do not + # have sources + return objl[:obj], objr[:obj], (bgl & np.invert(has_obj)[:,None]).astype(int), \ + (bgr & np.invert(has_obj)[:,None]).astype(int) + + +def new_find_peak_limits(hist, pks): + """ + Find all values between the zeros of hist + + hist and pks are expected to be 1d vectors + + """ + + sz_i = pks.shape[0] + sz_h = hist.shape[0] + + edges = np.zeros((sz_i,2), dtype=int) + + indx = np.ma.MaskedArray(np.array([np.arange(sz_h)]*sz_i), + mask=(hist == 0)[None,:] | (np.arange(sz_h)[None,:] > pks[:,None])) + + plt.imshow(indx, origin='lower', interpolation='nearest', aspect='auto') + plt.show() + + print(hist) + + print(pks[0]) + print(indx[0,np.invert(indx.mask[0,:])]) + + print(pks[1]) + print(indx[1,np.invert(indx.mask[1,:])]) + exit() + print(sz_i, sz_h) + print(pks) + print(indx.shape) + edges[:,0] = np.ma.amin(indx, axis=1) + print(edges[:,0]) + exit() + + hi_indx = hist_ne_0[None,:] & (np.arange(sz_h)[None,:] > pks[:,None]) + + + for ii in range(0, sz_i): + + indx = (hist != 0) & (np.arange(sz_h) <= pks[ii]) + + # Search below the peak + lim = pks[ii] + while True: + if lim < 0: + break + if hist[lim] == 0: + break + lim -= 1 + # Store the limit + edges[ii, 0] = lim + # Search above the peak + lim = pks[ii] + while True: + if lim > sz_h-1: + break + if hist[lim] == 0: + break + lim += 1 + # Store the limit + edges[ii, 1] = lim + return edges + + +def new_ignore_orders(edgdet, fracpix, lmin, lmax, rmin, rmax): + + sz_x = edgdet.shape[0] + sz_y = edgdet.shape[1] + + lsize = lmax-lmin+1 + larr = np.zeros((2,lsize), dtype=int) + larr[0,:] = sz_x + + rsize = rmax-rmin+1 + rarr = np.zeros((2,rsize), dtype=int) + rarr[0,:] = sz_x + + # TODO: Can I remove the loop? Or maybe just iterate through the + # smallest dimension of edgdet? + for x in range(sz_x): + indx = edgdet[x,:] < 0 + if np.any(indx): + larr[0,-edgdet[x,indx]-lmin] = np.clip(larr[0,-edgdet[x,indx]-lmin], None, x) + larr[1,-edgdet[x,indx]-lmin] = np.clip(larr[1,-edgdet[x,indx]-lmin], x, None) + indx = edgdet[x,:] > 0 + if np.any(indx): + rarr[0,edgdet[x,indx]-rmin] = np.clip(rarr[0,edgdet[x,indx]-rmin], None, x) + rarr[1,edgdet[x,indx]-rmin] = np.clip(rarr[1,edgdet[x,indx]-rmin], x, None) + + # Go through the array once more to remove pixels that do not cover fracpix + edgdet = edgdet.ravel() + lt_zero = np.arange(edgdet.size)[edgdet < 0] + if len(lt_zero) > 0: + edgdet[lt_zero[larr[1,-edgdet[lt_zero]-lmin]-larr[0,-edgdet[lt_zero]-lmin] < fracpix]] = 0 + gt_zero = np.arange(edgdet.size)[edgdet > 0] + if len(gt_zero) > 0: + edgdet[gt_zero[rarr[1,edgdet[gt_zero]-rmin]-rarr[0,edgdet[gt_zero]-rmin] < fracpix]] = 0 + edgdet = edgdet.reshape(sz_x,sz_y) + + # Check if lmin, lmax, rmin, and rmax need to be changed + lindx = np.arange(lsize)[larr[1,:]-larr[0,:] > fracpix] + lnc = lindx[0] + lxc = lsize-1-lindx[-1] + + rindx = np.arange(rsize)[rarr[1,:]-rarr[0,:] > fracpix] + rnc = rindx[0] + rxc = rsize-1-rindx[-1] + + return lnc, lxc, rnc, rxc, larr, rarr + + def trace_slits(slf, mstrace, det, pcadesc="", maskBadRows=False, min_sqm=30.): """ This routine traces the locations of the slit edges @@ -907,6 +1093,8 @@ def trace_slits(slf, mstrace, det, pcadesc="", maskBadRows=False, min_sqm=30.): assign_slits(binarr, edgearrcp, lor=+1) if settings.argflag['trace']['slits']['maxgap'] is not None: vals = np.sort(np.unique(edgearrcp[np.where(edgearrcp != 0)])) + print('calling close_edges') + exit() hasedge = arcytrace.close_edges(edgearrcp, vals, int(settings.argflag['trace']['slits']['maxgap'])) # Find all duplicate edges edgedup = vals[np.where(hasedge == 1)] @@ -991,10 +1179,14 @@ def trace_slits(slf, mstrace, det, pcadesc="", maskBadRows=False, min_sqm=30.): wvla = np.unique(edgearr[wdup][wdda]) wvlb = np.unique(edgearr[wdup][wddb]) # Now generate the dual edge + print('calling dual_edge') + exit() arcytrace.dual_edge(edgearr, edgearrcp, wdup[0], wdup[1], wvla, wvlb, shadj, int(settings.argflag['trace']['slits']['maxgap']), edgedup[jj]) # Now introduce new edge locations vals = np.sort(np.unique(edgearrcp[np.where(edgearrcp != 0)])) + print('calling close_slits') + exit() edgearrcp = arcytrace.close_slits(binarr, edgearrcp, vals, int(settings.argflag['trace']['slits']['maxgap'])) # Update edgearr edgearr = edgearrcp.copy() @@ -1057,7 +1249,16 @@ def trace_slits(slf, mstrace, det, pcadesc="", maskBadRows=False, min_sqm=30.): #msgs.info("Ignoring any slit that spans < {0:3.2f}x{1:d} pixels on the detector".format(settings.argflag['trace']['slits']['fracignore'], int(edgearr.shape[0]*binby))) msgs.info("Ignoring any slit that spans < {0:3.2f}x{1:d} pixels on the detector".format(settings.argflag['trace']['slits']['fracignore'], int(edgearr.shape[0]))) fracpix = int(settings.argflag['trace']['slits']['fracignore']*edgearr.shape[0]) - lnc, lxc, rnc, rxc, ldarr, rdarr = arcytrace.ignore_orders(edgearr, fracpix, lmin, lmax, rmin, rmax) + +# print('calling ignore_orders') +# t = time.clock() +# lnc, lxc, rnc, rxc, ldarr, rdarr = arcytrace.ignore_orders(edgearr, fracpix, lmin, lmax, rmin, rmax) +# print('Old ignore_orders: {0} seconds'.format(time.clock() - t)) +# t = time.clock() + lnc, lxc, rnc, rxc, ldarr, rdarr = new_ignore_orders(edgearr, fracpix, lmin, lmax, rmin, + rmax) +# print('New ignore_orders: {0} seconds'.format(time.clock() - t)) + lmin += lnc rmin += rnc lmax -= lxc @@ -1200,6 +1401,8 @@ def trace_slits(slf, mstrace, det, pcadesc="", maskBadRows=False, min_sqm=30.): if mnvalp > mnvalm: lvp = (arutils.func_val(lcoeff[:, lval+1-lmin], xv, settings.argflag['trace']['slits']['function'], minv=minvf, maxv=maxvf)+0.5).astype(np.int) + print('calling find_between') + exit() edgbtwn = arcytrace.find_between(edgearr, lv, lvp, 1) # edgbtwn is a 3 element array that determines what is between two adjacent left edges # edgbtwn[0] is the next right order along, from left order lval @@ -1537,6 +1740,8 @@ def refine_traces(binarr, outpar, extrap_cent, extrap_diff, extord, orders, lopos = phys_to_pix(extfit[:,-i]-srchz, locations, 1) # The pixel indices for the bottom of the search window numsrch = np.int(np.max(np.round(2.0*srchz-extrap_diff[:,-i]))) diffarr = np.round(extrap_diff[:,-i]).astype(np.int) + print('calling find_shift') + exit() shift = arcytrace.find_shift(binarr, minarr, lopos, diffarr, numsrch) relshift = np.mean(shift+extrap_diff[:,-i]/2-srchz) if shift == -1: @@ -1564,6 +1769,8 @@ def refine_traces(binarr, outpar, extrap_cent, extrap_diff, extord, orders, lopos = phys_to_pix(extfit[:,i-1]-srchz, locations, 1) numsrch = np.int(np.max(np.round(2.0*srchz-extrap_diff[:,i-1]))) diffarr = np.round(extrap_diff[:,i-1]).astype(np.int) + print('calling find_shift') + exit() shift = arcytrace.find_shift(binarr, minarr, lopos, diffarr, numsrch) relshift = np.mean(shift+extrap_diff[:,i-1]/2-srchz) if shift == -1: @@ -2510,6 +2717,8 @@ def get_censpec(slf, frame, det, gen_satmask=False): if gen_satmask: msgs.info("Generating a mask of arc line saturation streaks") satmask = arcyarc.saturation_mask(frame, settings.spect[dnum]['saturation']*settings.spect[dnum]['nonlinear']) + print('calling order saturation') + exit() satsnd = arcyarc.order_saturation(satmask, (ordcen+0.5).astype(np.int), (ordwid+0.5).astype(np.int)) # Extract a rough spectrum of the arc in each slit msgs.info("Extracting an approximate arc spectrum at the centre of each slit") From ea299d6ec7667ee6ba97492e9db2c0877abad90b Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Sat, 10 Mar 2018 12:15:55 -0800 Subject: [PATCH 02/36] removed cython functions touched by pypit_test --- pypit/ararc.py | 89 +++++- pypit/arcomb.py | 204 +++++++++++--- pypit/arextract.py | 107 +++++++- pypit/arproc.py | 180 ++++++++++-- pypit/artrace.py | 666 ++++++++++++++++++++++++++++++++++++++------- pypit/arutils.py | 4 + pypit/filter.py | 553 +++++++++++++++++++++++++++++++++++++ 7 files changed, 1658 insertions(+), 145 deletions(-) create mode 100644 pypit/filter.py diff --git a/pypit/ararc.py b/pypit/ararc.py index 178db4a726..e957b902c8 100644 --- a/pypit/ararc.py +++ b/pypit/ararc.py @@ -10,6 +10,7 @@ from pypit import arqa from matplotlib import pyplot as plt import os +import time from pypit import ardebug as debugger @@ -76,8 +77,23 @@ def detect_lines(slf, det, msarc, censpec=None, MK_SATMASK=False): if MK_SATMASK: ordwid = 0.5*np.abs(slf._lordloc[det-1] - slf._rordloc[det-1]) msgs.info("Generating a mask of arc line saturation streaks") - satmask = arcyarc.saturation_mask(msarc, slf._nonlinear[det-1]) - satsnd = arcyarc.order_saturation(satmask, ordcen, (ordwid+0.5).astype(np.int)) + print('calling saturation_mask') + t = time.clock() + _satmask = arcyarc.saturation_mask(msarc, slf._nonlinear[det-1]) + print('Old saturation_mask: {0} seconds'.format(time.clock() - t)) + t = time.clock() + satmask = new_saturation_mask(msarc, slf._nonlinear[det-1]) + print('New saturation_mask: {0} seconds'.format(time.clock() - t)) + assert np.sum(_satmask != satmask) == 0, 'Difference between old and new saturation_mask' + + print('calling order_saturation') + t = time.clock() + _satsnd = arcyarc.order_saturation(satmask, ordcen, (ordwid+0.5).astype(np.int)) + print('Old order_saturation: {0} seconds'.format(time.clock() - t)) + t = time.clock() + satsnd = new_order_saturation(satmask, ordcen, (ordwid+0.5).astype(np.int)) + print('New order_saturation: {0} seconds'.format(time.clock() - t)) + assert np.sum(_satsnd != satsnd) == 0, 'Difference between old and new order_saturation' else: satsnd = np.zeros_like(ordcen) # Detect the location of the arc lines @@ -576,3 +592,72 @@ def calib_with_arclines(slf, det, get_poly=False, use_method="general"): arqa.arc_fit_qa(slf, final_fit) # return final_fit + + + +def new_order_saturation(satmask, ordcen, ordwid): + + sz_y, sz_x = satmask.shape + sz_o = ordcen.shape[1] + + xmin = ordcen - ordwid + xmax = ordcen + ordwid + 1 + xmin[xmin < 0] = 0 + xmax[xmax >= sz_x] = sz_x + + ordsat = np.zeros((sz_y, sz_o), dtype=int) + for o in range(sz_o): + for y in range(sz_y): + ordsat[y,o] = (xmax[y,o] > xmin[y,o]) & np.any(satmask[y,xmin[y,o]:xmax[y,o]] == 1) + + return ordsat + + +def search_for_saturation_edge(a, x, y, sy, dx, satdown, satlevel, mask): + sx = dx + localx = a[x+sx,y+sy] + while True: + mask[x+sx,y+sy] = True + sx += dx + if x+sx > a.shape[0]-1 or x+sx < 0: + break + if a[x+sx,y+sy] >= localx/satdown and a[x+sx,y+sy] a.shape[1]-1 or y+sy < 0: + return mask + if a[x,y+sy] >= localy/satdown and a[x,y+sy] < satlevel: + return mask + localy = a[x,y+sy] + + +def new_saturation_mask(a, satlevel): + + mask = np.zeros(a.shape, dtype=bool) + a_is_saturated = a >= satlevel + if not np.any(a_is_saturated): + return mask.astype(int) + + satdown = 1.001 + sz_x, sz_y = a.shape + + for y in range (0,sz_y): + for x in range(0,sz_x): + if a_is_saturated[x,y] and not mask[x,y]: + mask[x,y] = True + mask = determine_saturation_region(a, x, y, 0, 1, satdown, satlevel, mask) + mask = determine_saturation_region(a, x, y, -1, -1, satdown, satlevel, mask) + + return mask.astype(int) + diff --git a/pypit/arcomb.py b/pypit/arcomb.py index c1e3966155..009fa201f6 100644 --- a/pypit/arcomb.py +++ b/pypit/arcomb.py @@ -1,16 +1,24 @@ from __future__ import (print_function, absolute_import, division, unicode_literals) +import time import numpy as np from pypit import armsgs from pypit import arparse as settings +from matplotlib import pyplot as plt # Logging msgs = armsgs.get_logger() - def comb_frames(frames_arr, det, frametype, weights=None, maskvalue=1048577, printtype=None): """ Combine several frames + .. todo:: + - I've just replaced the arcycomb calls, but this function + should probably be rewritten so that it makes better use of + np.ma.MaskedArray objects throughout. + + - Some of the replacement code still needs to be tested! + Parameters ---------- frames_arr : ndarray (3D) @@ -18,18 +26,23 @@ def comb_frames(frames_arr, det, frametype, weights=None, maskvalue=1048577, pri det : int Detector index frametype : str - What is the type of frame being combining? (only used for screen printout) + What is the type of frame being combining? (only used for screen + printout) weights : str, or None (optional) - How should the frame combination by weighted (not currently implemented) + How should the frame combination by weighted (not currently + implemented) maskvalue : int (optional) - What should the masked values be set to (should be greater than the detector's saturation value -- Default = 1 + 2**20) + What should the masked values be set to (should be greater than + the detector's saturation value -- Default = 1 + 2**20) printtype : str (optional) - The frame type string that should be printed by armsgs. If None, frametype will be used + The frame type string that should be printed by armsgs. If None, + frametype will be used """ from pypit import arcycomb dnum = settings.get_dnum(det) reject = settings.argflag[frametype]['combine']['reject'] method = settings.argflag[frametype]['combine']['method'] + ########### # FIRST DO SOME CHECKS ON THE INPUT ########### @@ -46,59 +59,111 @@ def comb_frames(frames_arr, det, frametype, weights=None, maskvalue=1048577, pri return frames_arr[:, :, 0] else: msgs.info("Combining {0:d} {1:s} frames".format(num_frames, printtype)) - # Check if the user has allowed the combination of long and short frames (e.g. different exposure times) + + # Check if the user has allowed the combination of long and short + # frames (e.g. different exposure times) msgs.work("lscomb feature has not been included here yet...") # Check the user hasn't requested to reject more frames than available if reject['lowhigh'][0] > 0 and reject['lowhigh'][1] > 0 and \ reject['lowhigh'][0] + reject['lowhigh'][1] >= num_frames: - msgs.error("You cannot reject more frames than is available with 'reject lowhigh'." + msgs.newline() + - "There are {0:d} frames and reject lowhigh will reject {1:d} low and {2:d} high".format(num_frames, reject['lowhigh'][0], reject['lowhigh'][1])) + msgs.error('You cannot reject more frames than is available with \'reject lowhigh\'.' + + msgs.newline() + 'There are {0:d} frames '.format(num_frames) + + 'and reject lowhigh will reject {0:d} low '.format(reject['lowhigh'][0]) + + 'and {0:d} high'.format(reject['lowhigh'][1])) + + saturation = settings.spect[dnum]['saturation']*settings.spect[dnum]['nonlinear'] + # Check that some information on the frames was supplied if settings.spect is None: - msgs.error("When combining the {0:s} frames, spectrograph information".format(printtype)+msgs.newline()+"was not provided") + msgs.error('When combining the {0:s} frames, spectrograph information'.format(printtype) + + msgs.newline() + 'was not provided.') # Calculate the values to be used if all frames are rejected in some pixels if reject['replace'] == 'min': - allrej_arr = arcycomb.minmax(frames_arr, 0) +# allrej_arr = arcycomb.minmax(frames_arr, 0) + allrej_arr = np.amin(frames_arr, axis=2) elif reject['replace'] == 'max': - allrej_arr = arcycomb.minmax(frames_arr, 1) +# allrej_arr = arcycomb.minmax(frames_arr, 1) + allrej_arr = np.amax(frames_arr, axis=2) elif reject['replace'] == 'mean': - allrej_arr = arcycomb.mean(frames_arr) +# allrej_arr = arcycomb.mean(frames_arr) + allrej_arr = np.mean(frames_arr, axis=2) elif reject['replace'] == 'median': - allrej_arr = arcycomb.median(frames_arr) +# allrej_arr = arcycomb.median(frames_arr) + allrej_arr = np.median(frames_arr, axis=2) elif reject['replace'] == 'weightmean': msgs.work("No weights are implemented yet") - allrej_arr = arcycomb.masked_weightmean(frames_arr, maskvalue) + print('calling masked_weightmean') + _allrej_arr = frames_arr.copy() + t = time.clock() + _allrej_arr = arcycomb.masked_weightmean(_allrej_arr, maskvalue) + print('Old masked_weightmean: {0} seconds'.format(time.clock() - t)) + __allrej_arr = frames_arr.copy() + t = time.clock() + __allrej_arr = new_masked_weightmean(__allrej_arr, maskvalue) + print('New masked_weightmean: {0} seconds'.format(time.clock() - t)) + print(__allrej_arr.shape) + assert np.sum(__allrej_arr != _allrej_arr) == 0, \ + 'Difference between old and new masked_weightmean' + allrej_arr = __allrej_arr +## allrej_arr = arcycomb.masked_weightmean(frames_arr, maskvalue) +# allrej_arr = new_masked_weightmean(frames_arr, maskvalue) elif reject['replace'] == 'maxnonsat': - allrej_arr = arcycomb.maxnonsat(frames_arr, settings.spect[dnum]['saturation']*settings.spect[dnum]['nonlinear']) + print('calling maxnonsat') + _allrej_arr = frames_arr.copy() + t = time.clock() + _allrej_arr = arcycomb.maxnonsat(_allrej_arr, saturation) + print('Old maxnonsat: {0} seconds'.format(time.clock() - t)) + __allrej_arr = frames_arr.copy() + t = time.clock() + __allrej_arr = new_maxnonsat(__allrej_arr, saturation) + print('New maxnonsat: {0} seconds'.format(time.clock() - t)) +# Bug in arcycomb.maxnonsat ? +# assert np.sum(__allrej_arr != _allrej_arr) == 0, 'Difference between old and new maxnonsat' + allrej_arr = __allrej_arr +## allrej_arr = arcycomb.maxnonsat(frames_arr, settings.spect[dnum]['saturation']*settings.spect[dnum]['nonlinear']) +# allrej_arr = new_maxnonsat(frames_arr, saturation) else: msgs.error("You must specify what to do in case all pixels are rejected") + ################ # Saturated Pixels msgs.info("Finding saturated and non-linear pixels") if settings.argflag[frametype]['combine']['satpix'] == 'force': - # If a saturated pixel is in one of the frames, force them to all have saturated pixels + # If a saturated pixel is in one of the frames, force them to + # all have saturated pixels # satw = np.zeros_like(frames_arr) # satw[np.where(frames_arr > settings.spect['det']['saturation']*settings.spect['det']['nonlinear'])] = 1.0 # satw = np.any(satw,axis=2) - setsat = arcycomb.masked_limitget(frames_arr, settings.spect[dnum]['saturation']*settings.spect[dnum]['nonlinear'], 2) # del satw +# setsat = arcycomb.masked_limitget(frames_arr, settings.spect[dnum]['saturation']*settings.spect[dnum]['nonlinear'], 2) + setsat = np.zeros_like(frames_arr) + setsat[frames_arr > saturation] = 1 elif settings.argflag[frametype]['combine']['satpix'] == 'reject': # Ignore saturated pixels in frames if possible - frames_arr = arcycomb.masked_limitset(frames_arr, settings.spect[dnum]['saturation']*settings.spect[dnum]['nonlinear'], 2, maskvalue) +# frames_arr = arcycomb.masked_limitset(frames_arr, settings.spect[dnum]['saturation']*settings.spect[dnum]['nonlinear'], 2, maskvalue) + frames_arr[frames_arr > saturation] = maskvalue elif settings.argflag[frametype]['combine']['satpix'] == 'nothing': - # Don't do anything special for saturated pixels (Hopefully the user has specified how to deal with them below!) + # Don't do anything special for saturated pixels (Hopefully the + # user has specified how to deal with them below!) pass else: - msgs.error("Option '{0:s}' for dealing with saturated pixels was not recognised".format(settings.argflag[frametype]['combine']['satpix'])) + msgs.error('Option \'{0}\' '.format(settings.argflag[frametype]['combine']['satpix']) + + 'for dealing with saturated pixels was not recognised.') # Delete unecessary arrays # None! ################ # Cosmic Rays if reject['cosmics'] > 0.0: msgs.info("Rejecting cosmic rays") # Use a robust statistic - medarr = arcycomb.masked_median(frames_arr, maskvalue) - stdarr = 1.4826*arcycomb.masked_median(np.abs(frames_arr-medarr[:, :, np.newaxis]), maskvalue) - frames_arr = arcycomb.masked_limitsetarr(frames_arr, (medarr + reject['cosmics']*stdarr), 2, maskvalue) + masked_fa = np.ma.MaskedArray(frames_arr, mask=frames_arr==maskvalue) + medarr = np.ma.median(masked_fa, axis=2) + stdarr = 1.4826*np.ma.median(np.ma.absolute(masked_fa - medarr[:,:,None]), axis=2) + indx = (frames_arr != maskvalue) \ + & (frames_arr > (medarr.data + reject['cosmics']*stdarr.data)[:,:,None]) + frames_arr[indx] = maskvalue +# medarr = arcycomb.masked_median(frames_arr, maskvalue) +# stdarr = 1.4826*arcycomb.masked_median(np.abs(frames_arr-medarr[:, :, np.newaxis]), maskvalue) +# frames_arr = arcycomb.masked_limitsetarr(frames_arr, (medarr + reject['cosmics']*stdarr), 2, maskvalue) # Delete unecessary arrays del medarr, stdarr else: @@ -136,10 +201,19 @@ def comb_frames(frames_arr, det, frametype, weights=None, maskvalue=1048577, pri # Deviant Pixels if reject['level'][0] > 0.0 or reject['level'][1] > 0.0: msgs.info("Rejecting deviant pixels") # Use a robust statistic - medarr = arcycomb.masked_median(frames_arr, maskvalue) - stdarr = 1.4826*arcycomb.masked_median(np.abs(frames_arr-medarr[:, :, np.newaxis]), maskvalue) - frames_arr = arcycomb.masked_limitsetarr(frames_arr, (medarr - reject['level'][0]*stdarr), -2, maskvalue) - frames_arr = arcycomb.masked_limitsetarr(frames_arr, (medarr + reject['level'][1]*stdarr), 2, maskvalue) + + masked_fa = np.ma.MaskedArray(frames_arr, mask=frames_arr==maskvalue) + medarr = np.ma.median(masked_fa, axis=2) + stdarr = 1.4826*np.ma.median(np.ma.absolute(masked_fa - medarr[:,:,None]), axis=2) + indx = (frames_arr != maskvalue) \ + & ( (frames_arr > (medarr.data + reject['cosmics']*stdarr.data)[:,:,None]) + | (frames_arr < (medarr.data - reject['cosmics']*stdarr.data)[:,:,None])) + frames_arr[indx] = maskvalue + +# medarr = arcycomb.masked_median(frames_arr, maskvalue) +# stdarr = 1.4826*arcycomb.masked_median(np.abs(frames_arr-medarr[:, :, np.newaxis]), maskvalue) +# frames_arr = arcycomb.masked_limitsetarr(frames_arr, (medarr - reject['level'][0]*stdarr), -2, maskvalue) +# frames_arr = arcycomb.masked_limitsetarr(frames_arr, (medarr + reject['level'][1]*stdarr), 2, maskvalue) # Delete unecessary arrays del medarr, stdarr else: @@ -148,17 +222,46 @@ def comb_frames(frames_arr, det, frametype, weights=None, maskvalue=1048577, pri # Combine the arrays msgs.info("Combining frames with a {0:s} operation".format(method)) if method == 'mean': - frames_arr = arcycomb.masked_mean(frames_arr, maskvalue) +# frames_arr = arcycomb.masked_mean(frames_arr, maskvalue) + frames_arr = np.ma.mean(np.ma.MaskedArray(frames_arr, mask=frames_arr==maskvalue), axis=2) elif method == 'median': - frames_arr = arcycomb.masked_median(frames_arr, maskvalue) +# frames_arr = arcycomb.masked_median(frames_arr, maskvalue) + frames_arr = np.ma.median(np.ma.MaskedArray(frames_arr, mask=frames_arr==maskvalue), axis=2) elif method == 'weightmean': - frames_arr = arcycomb.masked_weightmean(frames_arr, maskvalue) + print('calling masked_weightmean') + _frames_arr = frames_arr.copy() + t = time.clock() + _frames_arr = arcycomb.masked_weightmean(_frames_arr, maskvalue) + print('Old masked_weightmean: {0} seconds'.format(time.clock() - t)) + __frames_arr = frames_arr.copy() + t = time.clock() + __frames_arr = new_masked_weightmean(__frames_arr, maskvalue) + print('New masked_weightmean: {0} seconds'.format(time.clock() - t)) + print(__frames_arr.shape) + + if np.sum(np.absolute(__frames_arr-_frames_arr) > 1e-10) != 0: + print(np.sum(np.absolute(__frames_arr-_frames_arr) > 1e-10)) + plt.imshow(_frames_arr, origin='lower', interpolation='nearest', aspect='auto') + plt.show() + plt.imshow(__frames_arr, origin='lower', interpolation='nearest', aspect='auto') + plt.show() + plt.imshow(_frames_arr - __frames_arr, origin='lower', interpolation='nearest', aspect='auto') + plt.show() + plt.imshow(np.ma.divide(_frames_arr,__frames_arr) - 1, origin='lower', interpolation='nearest', aspect='auto') + plt.colorbar() + plt.show() + + assert np.sum( np.absolute(__frames_arr-_frames_arr) > 1e-10 ) == 0, \ + 'Difference between old and new masked_weightmean' + frames_arr = __frames_arr else: msgs.error("Combination type '{0:s}' is unknown".format(method)) ############## # If any pixels are completely masked, apply user-specified function msgs.info("Replacing completely masked pixels with the {0:s} value of the input frames".format(reject['replace'])) - frames_arr = arcycomb.masked_replace(frames_arr, allrej_arr, maskvalue) +# frames_arr = arcycomb.masked_replace(frames_arr, allrej_arr, maskvalue) + indx = frames_arr == maskvalue + frames_arr[indx] = allrej_arr[indx] # Delete unecessary arrays del allrej_arr ############## @@ -172,3 +275,42 @@ def comb_frames(frames_arr, det, frametype, weights=None, maskvalue=1048577, pri # Make sure the returned array is the correct type frames_arr = np.array(frames_arr, dtype=np.float) return frames_arr + + +def new_masked_weightmean(a, maskvalue): + num = np.ma.MaskedArray(a.copy(), mask=(a==maskvalue)) + num[np.invert(num.mask) & (num <= 1.0)] = 0.0 + num = np.ma.sum(np.ma.sqrt(num)*num, axis=2) + den = np.ma.MaskedArray(a.copy(), mask=(a==maskvalue)) + den[np.invert(den.mask) & (den <= 1.0)] = 1.0 + den = np.ma.sum(np.sqrt(den), axis=2) + return np.ma.divide(num, den).filled(maskvalue) + + +def new_maxnonsat(array, saturated): + """ + sz_x, sz_y, nfr = array.shape + mmarr = np.zeros((sz_x,sz_y), dtype=float) + d = 0 + for x in range(sz_x): + for y in range(sz_y): + # Sort the array + temp = 0.0 + minv = saturated + for n in range(nfr): + if array[x,y,n] > temp and (BUG: temp) < saturated: + temp = array[x,y,n] + if array[x,y,n] < minv: + minv = array[x,y,n] + if temp == 0.0: + mmarr[x,y] = minv + else: + mmarr[x,y] = temp + return mmarr + """ + minimum = np.amin(np.clip(array, None, saturated), axis=2) + _array = np.ma.MaskedArray(array, mask=np.invert((array > 0.0) & (array 2: + msgs.error('y cannot have more than 2 dimensions.') + + _y = np.atleast_2d(y) + ny, npix = _y.shape + + # Set the x coordinates + if x is None: + _x = np.linspace(-1,1,npix) + else: + if x.ndim != 1: + msgs.error('x must be a vector') + if x.size != npix: + msgs.error('Input x must match y vector or column length.') + _x = x.copy() + + # Generate the Vandermonde matrix + vand = np.polynomial.polynomial.polyvander(_x, order) + + # Fit with appropriate weighting + if w is None: + # Fit without weights + c = np.linalg.lstsq(vand, _y.T)[0] + ym = np.sum(c[:,:,None] * vand.T[:,None,:], axis=0) + elif w.ndim == 1: + # Fit with the same weight for each vector + _vand = w[:,None] * vand + __y = w[None,:] * _y + c = np.linalg.lstsq(_vand, __y.T)[0] + ym = np.sum(c[:,:,None] * vand.T[:,None,:], axis=0) + else: + # Fit with different weights for each vector + if w.shape != y.shape: + msgs.error('Input w must match y axis length or y shape.') + # Prep the output model + ym = np.empty(_y.shape, dtype=float) + __y = w * _y + for i in range(ny): + # Fit with the same weight for each vector + _vand = w[i,:,None] * vand + c = np.linalg.lstsq(_vand, __y[i,:])[0] + ym[i,:] = np.sum(c[:,None] * vand.T[:,:], axis=0) + + # Return the model with the appropriate shape + return ym if y.ndim == 2 else ym[0,:] + diff --git a/pypit/arproc.py b/pypit/arproc.py index 61119f5aa5..acd1cd81ab 100644 --- a/pypit/arproc.py +++ b/pypit/arproc.py @@ -196,9 +196,20 @@ def bg_subtraction(slf, det, sciframe, varframe, crpix, tracemask=None, errframe = np.sqrt(varframe) # Find which pixels are within the order edges msgs.info("Identifying pixels within each order") - ordpix = arcyutils.order_pixels(slf._pixlocn[det-1], - slf._lordloc[det-1]*0.95+slf._rordloc[det-1]*0.05, - slf._lordloc[det-1]*0.05+slf._rordloc[det-1]*0.95) + print('calling order_pixels') + t = time.clock() + _ordpix = arcyutils.order_pixels(slf._pixlocn[det-1], + slf._lordloc[det-1]*0.95+slf._rordloc[det-1]*0.05, + slf._lordloc[det-1]*0.05+slf._rordloc[det-1]*0.95) + print('Old order_pixels: {0} seconds'.format(time.clock() - t)) + t = time.clock() + ordpix = new_order_pixels(slf._pixlocn[det-1], + slf._lordloc[det-1]*0.95+slf._rordloc[det-1]*0.05, + slf._lordloc[det-1]*0.05+slf._rordloc[det-1]*0.95) + print('New order_pixels: {0} seconds'.format(time.clock() - t)) + assert np.sum(_ordpix != ordpix) == 0, \ + 'Difference between old and new order_pixels' + msgs.info("Applying bad pixel mask") ordpix *= (1-slf._bpix[det-1].astype(np.int)) * (1-crpix.astype(np.int)) if tracemask is not None: ordpix *= (1-tracemask.astype(np.int)) @@ -439,7 +450,17 @@ def flatnorm(slf, det, msflat, maskval=-999999.9, overpix=6, plotdesc=""): #xint = slf._pixlocn[det-1][:,0,0] # Find which pixels are within the order edges msgs.info("Identifying pixels within each order") - ordpix = arcyutils.order_pixels(slf._pixlocn[det-1], slf._lordloc[det-1], slf._rordloc[det-1]) + + print('calling order_pixels') + t = time.clock() + _ordpix = arcyutils.order_pixels(slf._pixlocn[det-1], slf._lordloc[det-1], slf._rordloc[det-1]) + print('Old order_pixels: {0} seconds'.format(time.clock() - t)) + t = time.clock() + ordpix = new_order_pixels(slf._pixlocn[det-1], slf._lordloc[det-1], slf._rordloc[det-1]) + print('New order_pixels: {0} seconds'.format(time.clock() - t)) + assert np.sum(_ordpix != ordpix) == 0, \ + 'Difference between old and new order_pixels' + msgs.info("Applying bad pixel mask") ordpix *= (1-slf._bpix[det-1].astype(np.int)) mskord = np.zeros(msflat.shape) @@ -1165,8 +1186,17 @@ def slit_pixels(slf, frameshape, det): for o in range(nslits): lordloc = slf._lordloc[det - 1][:, o] rordloc = slf._rordloc[det - 1][:, o] - ordloc = arcytrace.locate_order(lordloc, rordloc, frameshape[0], frameshape[1], - settings.argflag['trace']['slits']['pad']) + print('calling locate_order') + t = time.clock() + _ordloc = arcytrace.locate_order(lordloc, rordloc, frameshape[0], frameshape[1], + settings.argflag['trace']['slits']['pad']) + print('Old locate_order: {0} seconds'.format(time.clock() - t)) + t = time.clock() + ordloc = new_locate_order(lordloc, rordloc, frameshape[0], frameshape[1], + settings.argflag['trace']['slits']['pad']) + print('New locate_order: {0} seconds'.format(time.clock() - t)) + assert np.sum(_ordloc != ordloc) == 0, \ + 'Difference between old and new locate_order' word = np.where(ordloc != 0) if word[0].size == 0: msgs.warn("There are no pixels in slit {0:d}".format(o + 1)) @@ -1175,6 +1205,47 @@ def slit_pixels(slf, frameshape, det): return msordloc +def new_locate_order(lordloc, rordloc, sz_x, sz_y, pad): + """ Generate a boolean image that identifies which pixels + belong to the slit associated with the supplied left and + right slit edges. + + Parameters + ---------- + lordloc : ndarray + Location of the left slit edges of 1 slit + rordloc : ndarray + Location of the right slit edges of 1 slit + sz_x : int + The size of an image in the spectral (0th) dimension + sz_y : int + The size of an image in the spatial (1st) dimension + pad : int + Additional pixels to pad the left and right slit edges + + Returns + ------- + orderloc : ndarray + An image the same size as the input frame, containing values from 0-1. + 0 = pixel is not in the specified slit + 1 = pixel is in the specified slit + """ + ow = (rordloc-lordloc)/2.0 + oc = (rordloc+lordloc)/2.0 + ymin = (oc-ow).astype(int)-pad + ymax = (oc+ow).astype(int)+1+pad + indx = np.invert((ymax < 0) | (ymin >= sz_y)) + ymin[ymin < 0] = 0 + ymax[ymax > sz_y-1] = sz_y-1 + indx &= (ymax > ymin) + + orderloc = np.zeros((sz_x,sz_y), dtype=int) + for x in np.arange(sz_x)[indx]: + orderloc[x,ymin[x]:ymax[x]] = 1 + return orderloc + + + def slit_profile(slf, mstrace, det, ntcky=None): """ Generate an image of the spatial slit profile. @@ -1714,15 +1785,32 @@ def lacosmic(slf, fitsdict, det, sciframe, scidx, maxiter=1, grow=1.5, maskval=- filty = ndimage.sobel(filt/np.sqrt(np.abs(sciframe)), axis=0, mode='constant') filty[np.where(np.isnan(filty))]=0.0 -# t = time.clock() -# sigimg = arcyproc.cr_screen(filty,0.0) -# print('Old cr_screen: {0} seconds'.format(time.clock() - t)) -# -# t = time.clock() -# new_sigimg = new_cr_screen(filty,0.0) -# print('New cr_screen: {0} seconds'.format(time.clock() - t)) -# sigimg = arcyproc.cr_screen(filty,0.0) + # Old cr_screen can yield nan pixels, new one does not, meaning that + # there are differences between the returned arrays. For all + # non-nan pixels, the two algorithms are identical. + print('calling cr_screen') + t = time.clock() + _sigimg = arcyproc.cr_screen(filty,0.0) + print('Old cr_screen: {0} seconds'.format(time.clock() - t)) +# print(np.sum(np.invert(np.isfinite(_sigimg)))) + t = time.clock() sigimg = new_cr_screen(filty) + print('New cr_screen: {0} seconds'.format(time.clock() - t)) +# print(np.sum(np.invert(np.isfinite(sigimg)))) +# if np.sum(_sigimg != sigimg) != 0: +# plt.imshow(_sigimg, origin='lower', interpolation='nearest', aspect='auto') +# plt.show() +# plt.imshow(sigimg, origin='lower', interpolation='nearest', aspect='auto') +# plt.show() +# plt.imshow(_sigimg-sigimg, origin='lower', interpolation='nearest', aspect='auto') +# plt.colorbar() +# plt.show() +# r = np.ma.divide(_sigimg,sigimg)-1 +# print(np.ma.sum(r)) +# plt.imshow(np.ma.divide(_sigimg,sigimg)-1, origin='lower', interpolation='nearest', aspect='auto') +# plt.colorbar() +# plt.show() +# assert np.sum(_sigimg != sigimg) == 0, 'Difference between old and new cr_screen' # print(sigimg.shape) # print(new_sigimg.shape) @@ -1748,7 +1836,15 @@ def lacosmic(slf, fitsdict, det, sciframe, scidx, maxiter=1, grow=1.5, maskval=- sigmask[np.where(sigsmth>sigclip)] = True crmask = np.logical_and(crmask, sigmask) msgs.info("Growing cosmic ray mask by 1 pixel") - crmask = arcyutils.grow_masked(crmask.astype(np.float), grow, 1.0) + print('calling grow_masked') + t = time.clock() + _crmask = arcyutils.grow_masked(crmask.astype(np.float), grow, 1.0) + print('Old grow_masked: {0} seconds'.format(time.clock() - t)) + t = time.clock() + crmask = new_grow_masked(crmask.astype(np.float), grow, 1.0) + print('New grow_masked: {0} seconds'.format(time.clock() - t)) + assert np.sum(_crmask != crmask) == 0, 'Difference between old and new grow_masked' + return crmask @@ -1789,7 +1885,59 @@ def new_cr_screen(a, mask_value=0.0, spatial_axis=1): d = np.absolute(_a - meda[:,None]) mada = 1.4826*np.ma.median(d, axis=spatial_axis) # Return the ratio of the difference to the standard deviation - return np.ma.divide(d, mada[:,None]).filled(0.0) + return np.ma.divide(d, mada[:,None]).filled(mask_value) + + +def new_grow_masked(img, grow, growval): + + if not np.any(img == growval): + return img + + _img = img.copy() + sz_x, sz_y = img.shape + d = int(1+grow) + rsqr = grow*grow + + # Grow any masked values by the specified amount + for x in range(sz_x): + for y in range(sz_y): + if img[x,y] != growval: + continue + + mnx = 0 if x-d < 0 else x-d + mxx = x+d+1 if x+d+1 < sz_x else sz_x + mny = 0 if y-d < 0 else y-d + mxy = y+d+1 if y+d+1 < sz_y else sz_y + + for i in range(mnx,mxx): + for j in range(mny, mxy): + if (i-x)*(i-x)+(j-y)*(j-y) <= rsqr: + _img[i,j] = growval + return _img + + +def new_order_pixels(pixlocn, lord, rord): + """ + Based on physical pixel locations, determine which pixels are within the orders + """ + + sz_x, sz_y, _ = pixlocn.shape + sz_o = lord.shape[1] + + outfr = np.zeros((sz_x, sz_y), dtype=int) + + for y in range(sz_y): + for o in range(sz_o): + indx = (lord[:,o] < rord[:,o]) & (pixlocn[:,y,1] > lord[:,o])\ + & (pixlocn[:,y,1] < rord[:,o]) + indx |= ( (lord[:,o] > rord[:,o]) & (pixlocn[:,y,1] < lord[:,o]) + & (pixlocn[:,y,1] > rord[:,o]) ) + if np.any(indx): + # Only assign a single order to a given pixel + outfr[indx,y] = o+1 + break + + return outfr def gain_frame(slf, det): diff --git a/pypit/artrace.py b/pypit/artrace.py index 498722ee54..3c5ab9a030 100644 --- a/pypit/artrace.py +++ b/pypit/artrace.py @@ -14,6 +14,8 @@ import scipy.ndimage as ndimage from collections import Counter +from pypit.filter import BoxcarFilter + # Logging msgs = armsgs.get_logger() @@ -123,16 +125,15 @@ def assign_slits(binarr, edgearr, ednum=100000, lor=-1): break pks = wpk+2 # Shifted by 2 because of the peak finding algorithm above print('calling find_peak_limits') - t = time.clock() - pedges = arcytrace.find_peak_limits(smedgehist, pks) + _pedges = arcytrace.find_peak_limits(smedgehist, pks) print('Old find_peak_limits: {0} seconds'.format(time.clock() - t)) - print(pedges.shape) - print(pedges[:,0]) t = time.clock() pedges = new_find_peak_limits(smedgehist, pks) print('New find_peak_limits: {0} seconds'.format(time.clock() - t)) - exit() + assert np.sum(_pedges != pedges) == 0, \ + 'Difference between old and new find_peak_limits' + if np.all(pedges[:, 1]-pedges[:, 0] == 0): # Remaining peaks have no width break @@ -300,6 +301,7 @@ def expand_slits(slf, mstrace, det, ordcen, extord): # Calculate the pixel locations of th eorder edges pixcen = phys_to_pix(ordcen, slf._pixlocn[det - 1], 1) msgs.info("Expanding slit traces to slit edges") + exit() mordwid, pordwid = arcytrace.expand_slits(mstrace, pixcen, extord.astype(np.int)) # Fit a function for the difference between left edge and the centre trace ldiff_coeff, ldiff_fit = arutils.polyfitter2d(mordwid, mask=-1, @@ -538,7 +540,51 @@ def trace_object(slf, det, sciframe, varframe, crmask, trim=2, msgs.info("Estimating object profiles") # Smooth the S/N frame smthby, rejhilo = tracepar['smthby'], tracepar['rejhilo'] - rec_sigframe_bin = arcyutils.smooth_x(rec_sciframe/np.sqrt(rec_varframe), 1.0-rec_crmask, smthby, rejhilo, maskval) + print('calling smooth_x') + t = time.clock() + _rec_sigframe_bin = arcyutils.smooth_x(rec_sciframe/np.sqrt(rec_varframe), 1.0-rec_crmask, + smthby, rejhilo, maskval) + print('Old smooth_x: {0} seconds'.format(time.clock() - t)) + tmp = np.ma.MaskedArray(rec_sciframe/np.sqrt(rec_varframe), mask=rec_crmask.astype(bool)) + # TODO: Add rejection to BoxcarFilter? + t = time.clock() + rec_sigframe_bin = BoxcarFilter(smthby).smooth(tmp.T).T + print('BoxcarFilter: {0} seconds'.format(time.clock() - t)) +# t = time.clock() +# rec_sigframe_bin = new_smooth_x(rec_sciframe/np.sqrt(rec_varframe), 1.0-rec_crmask, smthby, +# rejhilo, maskval) +# print('New smooth_x: {0} seconds'.format(time.clock() - t)) + # TODO: BoxcarFilter and smooth_x will provide different results. + # Need to assess their importance. +# if np.sum(_rec_sigframe_bin != rec_sigframe_bin) != 0: +# +# plt.plot(tmp[:,200]) +# plt.plot(rec_sigframe_bin[:,200]) +# plt.plot(_rec_sigframe_bin[:,200]) +# plt.show() +# +# plt.imshow( np.ma.MaskedArray(_rec_sigframe_bin, mask=rec_crmask.astype(bool)), +# origin='lower', interpolation='nearest', aspect='auto') +# plt.colorbar() +# plt.show() +# plt.imshow(np.ma.MaskedArray(rec_sigframe_bin, mask=rec_crmask.astype(bool)), +# origin='lower', interpolation='nearest', aspect='auto') +# plt.colorbar() +# plt.show() +# plt.imshow(np.ma.MaskedArray(_rec_sigframe_bin-rec_sigframe_bin, +# mask=rec_crmask.astype(bool)), +# origin='lower', interpolation='nearest', aspect='auto') +# plt.colorbar() +# plt.show() +# t = np.ma.divide(_rec_sigframe_bin,rec_sigframe_bin) +# t[rec_crmask.astype(bool)] = np.ma.masked +# plt.imshow(t, +# origin='lower', interpolation='nearest', aspect='auto') +# plt.colorbar() +# plt.show() +# assert np.sum(_rec_sigframe_bin != rec_sigframe_bin) == 0, \ +# 'Difference between old and new smooth_x' + #rec_varframe_bin = arcyutils.smooth_x(rec_varframe, 1.0-rec_crmask, smthby, rejhilo, maskval) #rec_sigframe_bin = np.sqrt(rec_varframe_bin/(smthby-2.0*rejhilo)) #sigframe = rec_sciframe_bin*(1.0-rec_crmask)/rec_sigframe_bin @@ -578,13 +624,27 @@ def trace_object(slf, det, sciframe, varframe, crmask, trim=2, trcprof2 = np.mean(rec_sciframe, axis=0) objl, objr, bckl, bckr = find_obj_minima(trcprof2, triml=triml, trimr=trimr, nsmooth=nsmooth) elif settings.argflag['trace']['object']['find'] == 'standard': -# print('calling find_objects') -# t = time.clock() -# objl, objr, bckl, bckr = arcytrace.find_objects(trcprof, bgreg, mad) -# print('Old find_objects: {0} seconds'.format(time.clock() - t)) -# t = time.clock() + print('calling find_objects') + t = time.clock() + _objl, _objr, _bckl, _bckr = arcytrace.find_objects(trcprof, bgreg, mad) + print('Old find_objects: {0} seconds'.format(time.clock() - t)) + t = time.clock() objl, objr, bckl, bckr = new_find_objects(trcprof, bgreg, mad) -# print('New find_objects: {0} seconds'.format(time.clock() - t)) + print('New find_objects: {0} seconds'.format(time.clock() - t)) +# print('objl:', objl) +# print('_objl:', _objl) +# print('objr:', objr) +# print('_objr:', _objr) +# print(_objl.shape, objl.shape) +# print(_objr.shape, objr.shape) +# print(np.sum(_objl != objl)) +# print(np.sum(_objr != objr)) + assert np.sum(_objl != objl) == 0, 'Difference between old and new find_objects, objl' + assert np.sum(_objr != objr) == 0, 'Difference between old and new find_objects, objr' + assert np.sum(_bckl != bckl) == 0, 'Difference between old and new find_objects, bckl' + assert np.sum(_bckr != bckr) == 0, 'Difference between old and new find_objects, bckr' +# objl, objr, bckl, bckr = new_find_objects(trcprof, bgreg, mad) + else: msgs.error("Bad object identification algorithm!!") if msgs._debug['trace_obj']: @@ -715,6 +775,47 @@ def trace_object(slf, det, sciframe, varframe, crmask, trim=2, return tracedict +def new_find_between(edgdet, ledgem, ledgep, dirc): + + if len(edgdet.shape) != 2: + msgs.error('Edge pixels array must be 2D.') + if len(ledgem.shape) != 1 or len(ledgep.shape) !=1: + msgs.error('Input must be 1D.') + + sz_x, sz_y = edgdet.shape + + # Setup the coefficient arrays + edgbtwn = np.full(3, -1, dtype=int) + + for x in range(0,sz_x): + rng = np.sort([ledgem[x],ledgep[x]]) + if not np.any(edgdet[x,slice(*rng)] > 0): + continue + e = edgdet[x,slice(*rng)][edgdet[x,slice(*rng)] > 0] + if edgbtwn[0] == -1: + edgbtwn[0] = e[0] + indx = e != edgbtwn[0] + if edgbtwn[1] == -1 and np.any(indx): + edgbtwn[1] = e[indx][0] + + # If no right order edges were found between these two left order + # edges, find the next right order edge + if edgbtwn[0] == -1 and edgbtwn[1] == -1: + for x in range(0,sz_x): + ystrt = np.max([ledgem[x],ledgep[x]]) + if dirc == 1: + emin = np.min(edgdet[x,ystrt:][edgdet[x,ystrt:] > 0]) + if edgbtwn[2] == -1 or emin < edgbtwn[2]: + edgbtwn[2] = emin + else: + emax = np.max(edgdet[x,:ystrt+1][edgdet[x,:ystrt+1] > 0]) + if edgbtwn[2] == -1 or emax > edgbtwn[2]: + edgbtwn[2] = emax + + # Now return the array + return edgbtwn + + def new_find_objects(profile, bgreg, stddev): """ Find significantly detected objects in the profile array @@ -743,11 +844,14 @@ def new_find_objects(profile, bgreg, stddev): while np.any(gt_5_sigma & np.invert(_profile.mask)): # Find next maximum flux point imax = np.ma.argmax(_profile) - maxv = _profile[imax] # Find the valid source pixels around the peak f = np.arange(sz_x)[np.roll(not_gt_3_sigma, -imax)] - objl[obj] = imax-sz_x+f[-1] - objr[obj] = f[0]+imax + # TODO: the ifs below feel like kludges to match old + # find_objects function. In particular, should objr be treated + # as exclusive or inclusive? + objl[obj] = imax-sz_x+f[-1] if imax-sz_x+f[-1] > 0 else 1 + objr[obj] = f[0]+imax if f[0]+imax < sz_x else sz_x-1 +# print('object found: ', imax, f[-1], objl[obj], f[0], objr[obj], sz_x) # Mask source pixels and increment for next iteration has_obj[objl[obj]:objr[obj]+1] = True _profile[objl[obj]:objr[obj]+1] = np.ma.masked @@ -778,69 +882,59 @@ def new_find_peak_limits(hist, pks): Find all values between the zeros of hist hist and pks are expected to be 1d vectors - """ + if len(hist.shape) != 1 or len(pks.shape) != 1: + msgs.error('Arrays provided to find_peak_limits must be vectors.') + # Pixel indices in hist for each peak + hn = np.arange(hist.shape[0]) + indx = np.ma.MaskedArray(np.array([hn]*pks.shape[0])) + # Instantiate output + edges = np.zeros((pks.shape[0],2), dtype=int) + # Find the left edges + indx.mask = (hist != 0)[None,:] | (hn[None,:] > pks[:,None]) + edges[:,0] = np.ma.amax(indx, axis=1) + # Find the right edges + indx.mask = (hist != 0)[None,:] | (hn[None,:] < pks[:,None]) + edges[:,1] = np.ma.amin(indx, axis=1) + return edges - sz_i = pks.shape[0] - sz_h = hist.shape[0] - - edges = np.zeros((sz_i,2), dtype=int) - - indx = np.ma.MaskedArray(np.array([np.arange(sz_h)]*sz_i), - mask=(hist == 0)[None,:] | (np.arange(sz_h)[None,:] > pks[:,None])) - - plt.imshow(indx, origin='lower', interpolation='nearest', aspect='auto') - plt.show() - print(hist) +def new_find_shift(mstrace, minarr, lopos, diffarr, numsrch): - print(pks[0]) - print(indx[0,np.invert(indx.mask[0,:])]) + sz_y = mstrace.shape[1] + maxcnts = -999999.9 + shift = 0 + d = mstrace - minarr[:,None] - print(pks[1]) - print(indx[1,np.invert(indx.mask[1,:])]) - exit() - print(sz_i, sz_h) - print(pks) - print(indx.shape) - edges[:,0] = np.ma.amin(indx, axis=1) - print(edges[:,0]) - exit() + for s in range(0,numsrch): + cnts = 0.0 - hi_indx = hist_ne_0[None,:] & (np.arange(sz_h)[None,:] > pks[:,None]) + ymin = lopos + s + ymin[ymin < 0] = 0 + ymax = ymin + diffarr + ymax[ymax > sz_y] = sz_y + indx = ymax > ymin - for ii in range(0, sz_i): + if np.sum(indx) == 0: + continue - indx = (hist != 0) & (np.arange(sz_h) <= pks[ii]) + cnts = np.sum([ np.sum(t[l:h]) for t,l,h in zip(d[indx], ymin[indx], ymax[indx]) ]) \ + / np.sum(ymax[indx]-ymin[indx]) + if cnts > maxcnts: + maxcnts = cnts + shift = s - # Search below the peak - lim = pks[ii] - while True: - if lim < 0: - break - if hist[lim] == 0: - break - lim -= 1 - # Store the limit - edges[ii, 0] = lim - # Search above the peak - lim = pks[ii] - while True: - if lim > sz_h-1: - break - if hist[lim] == 0: - break - lim += 1 - # Store the limit - edges[ii, 1] = lim - return edges + return shift def new_ignore_orders(edgdet, fracpix, lmin, lmax, rmin, rmax): + """ + .. warning:: - sz_x = edgdet.shape[0] - sz_y = edgdet.shape[1] + edgdet is alted by the function. + """ + sz_x, sz_y = edgdet.shape lsize = lmax-lmin+1 larr = np.zeros((2,lsize), dtype=int) @@ -884,6 +978,266 @@ def new_ignore_orders(edgdet, fracpix, lmin, lmax, rmin, rmax): return lnc, lxc, rnc, rxc, larr, rarr +def new_limit_yval(yc, maxv): + yn = 0 if yc == 0 else (-yc if yc < 3 else -3) + yx = maxv-yc if yc > maxv-4 and yc < maxv else 4 + return yn, yx + + +def new_match_edges(edgdet, ednum): + """ + ednum is a large dummy number used for slit edge assignment. ednum + should be larger than the number of edges detected + + This function alters edgdet! + """ + # mr is the minimum number of acceptable pixels required to form the + # detection of an order edge + mr = 5 + mrxarr = np.zeros(mr, dtype=int) + mryarr = np.zeros(mr, dtype=int) + + sz_x, sz_y = edgdet.shape + + lcnt = 2*ednum + rcnt = 2*ednum + for y in range(sz_y): + for x in range(sz_x): + if edgdet[x,y] != -1 and edgdet[x,y] != 1: + continue + + anyt = 0 + left = edgdet[x,y] == -1 + + # Search upwards from x,y + xs = x + 1 + yt = y + while xs <= sz_x-1: + xr = 10 if xs + 10 < sz_x else sz_x - xs - 1 + yn, yx = new_limit_yval(yt, sz_y) + + suc = 0 + for s in range(xs, xs+xr): + suc = 0 + for t in range(yt + yn, yt + yx): + if edgdet[s, t] == -1 and left: + edgdet[s, t] = -lcnt + elif edgdet[s, t] == 1 and not left: + edgdet[s, t] = rcnt + else: + continue + + suc = 1 + if anyt < mr: + mrxarr[anyt] = s + mryarr[anyt] = t + anyt += 1 + yt = t + break + + if suc == 1: + xs = s + 1 + break + if suc == 0: # The trace is lost! + break + + # Search downwards from x,y + xs = x - 1 + yt = y + while xs >= 0: + xr = xs if xs-10 < 0 else 10 + yn, yx = new_limit_yval(yt, sz_y) + + suc = 0 + for s in range(0, xr): + suc = 0 + for t in range(yt+yn, yt+yx): + if edgdet[xs-s, t] == -1 and left: + edgdet[xs-s, t] = -lcnt + elif edgdet[xs-s, t] == 1 and not left: + edgdet[xs-s, t] = rcnt + else: + continue + + suc = 1 + if anyt < mr: + mrxarr[anyt] = xs-s + mryarr[anyt] = t + anyt += 1 + yt = t + break + + if suc == 1: + xs = xs - s - 1 + break + if suc == 0: # The trace is lost! + break + + if anyt > mr and left: + edgdet[x, y] = -lcnt + lcnt = lcnt + 1 + elif anyt > mr and not left: + edgdet[x, y] = rcnt + rcnt = rcnt + 1 + else: + edgdet[x, y] = 0 + for s in range(anyt): + if mrxarr[s] != 0 and mryarr[s] != 0: + edgdet[mrxarr[s], mryarr[s]] = 0 + return lcnt-2*ednum, rcnt-2*ednum + + +def new_mean_weight(array, weight, rejhilo, maskval): + _a = array if rejhilo == 0 else np.sort(array) + sumw = np.sum(weight[rejhilo:-rejhilo]) + sumwa = np.sum(weight[rejhilo:-rejhilo]*_a[rejhilo:-rejhilo]) + return maskval if sumw == 0.0 else sumwa/sumw + + +def new_minbetween(mstrace, loord, hiord): + # TODO: Check shapes + ymin = np.clip(loord, 0, mstrace.shape[1]) + ymax = np.clip(hiord, 0, mstrace.shape[1]) + minarr = np.zeros(mstrace.shape[0]) + indx = ymax > ymin + minarr[indx] = np.array([ np.amin(t[l:h]) + for t,l,h in zip(mstrace[indx], ymin[indx], ymax[indx]) ]) + return minarr + + +def new_phys_to_pix(array, diff): + if len(array.shape) > 2: + msgs.error('Input array must have two dimensions or less!') + if len(diff.shape) != 1: + msgs.error('Input difference array must be 1D!') + _array = np.atleast_2d(array) + doravel = len(array.shape) != 2 + pix = np.argmin(np.absolute(_array[:,:,None] - diff[None,None,:]), axis=2) + return pix.ravel() if doravel else pix + + sz_a, sz_n = _array.shape + sz_d = diff.size + + pix = np.zeros((sz_a,sz_n), dtype=int) + for n in range(0,sz_n): + for a in range(0,sz_a): + mind = 0 + mindv = _array[a,n]-diff[0] + + for d in range(1,sz_d): + test = _array[a,n]-diff[d] + if test < 0.0: test *= -1.0 + if test < mindv: + mindv = test + mind = d + if _array[a,n]-diff[d] < 0.0: break + pix[a,n] = mind + return pix.ravel() if doravel else pix + +# Weighted boxcar smooothing with rejection +def new_smooth_x(array, weight, fact, rejhilo, maskval): + hf = fact // 2 + + sz_x, sz_y = array.shape + + smtarr = np.zeros((sz_x,sz_y), dtype=float) + medarr = np.zeros((fact+1), dtype=float) + wgtarr = np.zeros((fact+1), dtype=float) + + for y in range(sz_y): + for x in range(sz_x): + for b in range(fact+1): + if (x+b-hf < 0) or (x+b-hf >= sz_x): + wgtarr[b] = 0.0 + else: + medarr[b] = array[x+b-hf,y] + wgtarr[b] = weight[x+b-hf,y] + smtarr[x,y] = new_mean_weight(medarr, wgtarr, rejhilo, maskval) + return smtarr + + +def new_tilts_image(tilts, lordloc, rordloc, pad, sz_y): + """ + Using the tilt (assumed to be fit with a first order polynomial) + generate an image of the tilts for each slit. + + Parameters + ---------- + tilts : ndarray + An (m x n) 2D array specifying the tilt (i.e. gradient) at each + pixel along the spectral direction (m) for each slit (n). + lordloc : ndarray + Location of the left slit edges + rordloc : ndarray + Location of the right slit edges + pad : int + Set the tilts within each slit, and extend a number of pixels + outside the slit edges (this number is set by pad). + sz_y : int + Number of detector pixels in the spatial direction. + + Returns + ------- + tiltsimg : ndarray + An image the same size as the science frame, containing values from 0-1. + 0/1 corresponds to the bottom/top of the detector (in the spectral direction), + and constant wavelength is represented by a single value from 0-1. + + """ + sz_x, sz_o = tilts.shape + dszx = (sz_x-1.0) + + tiltsimg = np.zeros((sz_x,sz_y), dtype=float) + for o in range(sz_o): + for x in range(sz_x): + ow = (rordloc[x,o]-lordloc[x,o])/2.0 + oc = (rordloc[x,o]+lordloc[x,o])/2.0 + ymin = int(oc-ow) - pad + ymax = int(oc+ow) + 1 + pad + # Check we are in bounds + if ymin < 0: + ymin = 0 + elif ymax < 0: + continue + if ymax > sz_y-1: + ymax = sz_y-1 + elif ymin > sz_y-1: + continue + # Set the tilt value at each pixel in this row + for y in range(ymin, ymax): + yv = (y-lordloc[x, o])/ow - 1.0 + tiltsimg[x,y] = (tilts[x,o]*yv + x)/dszx + return tiltsimg + + + sz_x, sz_o = tilts.shape + dszx = (sz_x-1.0) + + ow = (rordloc-lordloc)/2.0 + oc = (rordloc+lordloc)/2.0 + + ymin = (oc-ow).astype(int) - pad + ymax = (oc+ow).astype(int) + 1 + pad + + indx = (ymax >= 0) & (ymin < sz_y) + ymin[ymin < 0] = 0 + ymax[ymax > sz_y-1] = sz_y-1 + + tiltsimg = np.zeros((sz_x,sz_y), dtype=float) + + xv = np.arange(sz_x).astype(int) + for o in range(sz_o): + if np.sum(indx[:,o]) == 0: + continue + for x in xv[indx[:,o]]: + # Set the tilt value at each pixel in this row + for y in range(ymin[x,o], ymax[x,o]): + yv = (y-lordloc[x,o])/ow[x,o] - 1.0 + tiltsimg[x,y] = (tilts[x,o]*yv + x)/dszx + + return tiltsimg + + def trace_slits(slf, mstrace, det, pcadesc="", maskBadRows=False, min_sqm=30.): """ This routine traces the locations of the slit edges @@ -1034,7 +1388,26 @@ def trace_slits(slf, mstrace, det, pcadesc="", maskBadRows=False, min_sqm=30.): # Assign a number to each of the edges msgs.info("Matching slit edges") - lcnt, rcnt = arcytrace.match_edges(edgearr, ednum) + + _edgearr = edgearr.copy() + t = time.clock() + _lcnt, _rcnt = arcytrace.match_edges(_edgearr, ednum) + print('Old match_edges: {0} seconds'.format(time.clock() - t)) + __edgearr = edgearr.copy() + t = time.clock() + lcnt, rcnt = new_match_edges(__edgearr, ednum) + print('New match_edges: {0} seconds'.format(time.clock() - t)) +# print(lcnt, _lcnt, rcnt, _rcnt) +# print(np.sum(__edgearr != _edgearr)) +# plt.imshow(_edgearr, origin='lower', interpolation='nearest', aspect='auto') +# plt.show() +# plt.imshow(__edgearr, origin='lower', interpolation='nearest', aspect='auto') +# plt.show() + assert np.sum(_lcnt != lcnt) == 0, 'Difference between old and new match_edges, lcnt' + assert np.sum(_rcnt != rcnt) == 0, 'Difference between old and new match_edges, rcnt' + assert np.sum(__edgearr != _edgearr) == 0, 'Difference between old and new match_edges, edgearr' + edgearr = __edgearr + if lcnt >= ednum or rcnt >= ednum: msgs.error("Found more edges than allowed by ednum. Set ednum to a larger number.") if lcnt == 1: @@ -1250,14 +1623,25 @@ def trace_slits(slf, mstrace, det, pcadesc="", maskBadRows=False, min_sqm=30.): msgs.info("Ignoring any slit that spans < {0:3.2f}x{1:d} pixels on the detector".format(settings.argflag['trace']['slits']['fracignore'], int(edgearr.shape[0]))) fracpix = int(settings.argflag['trace']['slits']['fracignore']*edgearr.shape[0]) -# print('calling ignore_orders') -# t = time.clock() -# lnc, lxc, rnc, rxc, ldarr, rdarr = arcytrace.ignore_orders(edgearr, fracpix, lmin, lmax, rmin, rmax) -# print('Old ignore_orders: {0} seconds'.format(time.clock() - t)) -# t = time.clock() - lnc, lxc, rnc, rxc, ldarr, rdarr = new_ignore_orders(edgearr, fracpix, lmin, lmax, rmin, + print('calling ignore_orders') + t = time.clock() + _edgearr = edgearr.copy() + _lnc, _lxc, _rnc, _rxc, _ldarr, _rdarr = arcytrace.ignore_orders(_edgearr, fracpix, lmin, lmax, rmin, rmax) + print('Old ignore_orders: {0} seconds'.format(time.clock() - t)) + t = time.clock() + __edgearr = edgearr.copy() + lnc, lxc, rnc, rxc, ldarr, rdarr = new_ignore_orders(__edgearr, fracpix, lmin, lmax, rmin, rmax) -# print('New ignore_orders: {0} seconds'.format(time.clock() - t)) + print('New ignore_orders: {0} seconds'.format(time.clock() - t)) + assert np.sum(_lnc != lnc) == 0, 'Difference between old and new ignore_orders, lnc' + assert np.sum(_lxc != lxc) == 0, 'Difference between old and new ignore_orders, lxc' + assert np.sum(_rnc != rnc) == 0, 'Difference between old and new ignore_orders, rnc' + assert np.sum(_rxc != rxc) == 0, 'Difference between old and new ignore_orders, rxc' + assert np.sum(_ldarr != ldarr) == 0, 'Difference between old and new ignore_orders, ldarr' + assert np.sum(_rdarr != rdarr) == 0, 'Difference between old and new ignore_orders, rdarr' + assert np.sum(__edgearr != _edgearr) == 0, 'Difference between old and new ignore_orders, edgearr' + + edgearr = __edgearr lmin += lnc rmin += rnc @@ -1401,9 +1785,14 @@ def trace_slits(slf, mstrace, det, pcadesc="", maskBadRows=False, min_sqm=30.): if mnvalp > mnvalm: lvp = (arutils.func_val(lcoeff[:, lval+1-lmin], xv, settings.argflag['trace']['slits']['function'], minv=minvf, maxv=maxvf)+0.5).astype(np.int) - print('calling find_between') - exit() - edgbtwn = arcytrace.find_between(edgearr, lv, lvp, 1) + t = time.clock() + _edgbtwn = arcytrace.find_between(edgearr, lv, lvp, 1) + print('Old find_between: {0} seconds'.format(time.clock() - t)) + t = time.clock() + edgbtwn = new_find_between(edgearr, lv, lvp, 1) + print('New find_between: {0} seconds'.format(time.clock() - t)) + assert np.sum(_edgbtwn != edgbtwn) == 0, 'Difference between old and new find_between' + # edgbtwn is a 3 element array that determines what is between two adjacent left edges # edgbtwn[0] is the next right order along, from left order lval # edgbtwn[1] is only !=-1 when there's an order overlap. @@ -1417,7 +1806,14 @@ def trace_slits(slf, mstrace, det, pcadesc="", maskBadRows=False, min_sqm=30.): else: lvp = (arutils.func_val(lcoeff[:, lval-1-lmin], xv, settings.argflag['trace']['slits']['function'], minv=minvf, maxv=maxvf)+0.5).astype(np.int) - edgbtwn = arcytrace.find_between(edgearr, lvp, lv, -1) + t = time.clock() + _edgbtwn = arcytrace.find_between(edgearr, lvp, lv, -1) + print('Old find_between: {0} seconds'.format(time.clock() - t)) + t = time.clock() + edgbtwn = new_find_between(edgearr, lvp, lv, -1) + assert np.sum(_edgbtwn != edgbtwn) == 0, 'Difference between old and new find_between' + print('New find_between: {0} seconds'.format(time.clock() - t)) + if edgbtwn[0] == -1 and edgbtwn[1] == -1: rsub = edgbtwn[2]-(lval-1) # There's an order overlap elif edgbtwn[1] == -1: # No overlap @@ -1733,16 +2129,37 @@ def refine_traces(binarr, outpar, extrap_cent, extrap_diff, extord, orders, loord = hiord hiord = nxord nxord = phys_to_pix(extrap_cent[:,-i], locations, 1) - minarrL = arcytrace.minbetween(binarr, loord, hiord) # Minimum counts between loord and hiord - minarrR = arcytrace.minbetween(binarr, hiord, nxord) + + # Minimum counts between loord and hiord + print('calling minbetween') + t = time.clock() + _minarrL = arcytrace.minbetween(binarr, loord, hiord) + _minarrR = arcytrace.minbetween(binarr, hiord, nxord) + print('Old minbetween: {0} seconds'.format(time.clock() - t)) + t = time.clock() + minarrL = new_minbetween(binarr, loord, hiord) + minarrR = new_minbetween(binarr, hiord, nxord) + print('New minbetween: {0} seconds'.format(time.clock() - t)) + assert np.sum(_minarrL != minarrL) == 0, \ + 'Difference between old and new minbetween, minarrL' + assert np.sum(_minarrR != minarrR) == 0, \ + 'Difference between old and new minbetween, minarrR' + minarr = 0.5*(minarrL+minarrR) srchz = np.abs(extfit[:,-i]-extfit[:,-i-1])/3.0 lopos = phys_to_pix(extfit[:,-i]-srchz, locations, 1) # The pixel indices for the bottom of the search window numsrch = np.int(np.max(np.round(2.0*srchz-extrap_diff[:,-i]))) diffarr = np.round(extrap_diff[:,-i]).astype(np.int) + print('calling find_shift') - exit() - shift = arcytrace.find_shift(binarr, minarr, lopos, diffarr, numsrch) + t = time.clock() + _shift = arcytrace.find_shift(binarr, minarr, lopos, diffarr, numsrch) + print('Old find_shift: {0} seconds'.format(time.clock() - t)) + t = time.clock() + shift = new_find_shift(binarr, minarr, lopos, diffarr, numsrch) + print('New find_shift: {0} seconds'.format(time.clock() - t)) + assert np.sum(_shift != shift) == 0, 'Difference between old and new find_shift, shift' + relshift = np.mean(shift+extrap_diff[:,-i]/2-srchz) if shift == -1: msgs.info(" Refining order {0:d}: NO relative shift applied".format(int(orders[-i]))) @@ -1764,14 +2181,30 @@ def refine_traces(binarr, outpar, extrap_cent, extrap_diff, extord, orders, while i > 0: hiord = loord loord = phys_to_pix(extfit[:,i], locations, 1) - minarr = arcytrace.minbetween(binarr,loord, hiord) + + print('calling minbetween') + t = time.clock() + _minarr = arcytrace.minbetween(binarr,loord, hiord) + print('Old minbetween: {0} seconds'.format(time.clock() - t)) + t = time.clock() + minarr = new_minbetween(binarr,loord, hiord) + print('New minbetween: {0} seconds'.format(time.clock() - t)) + assert np.sum(_minarr != minarr) == 0, 'Difference between old and new minbetween, minarr' + srchz = np.abs(extfit[:,i]-extfit[:,i-1])/3.0 lopos = phys_to_pix(extfit[:,i-1]-srchz, locations, 1) numsrch = np.int(np.max(np.round(2.0*srchz-extrap_diff[:,i-1]))) diffarr = np.round(extrap_diff[:,i-1]).astype(np.int) + print('calling find_shift') - exit() - shift = arcytrace.find_shift(binarr, minarr, lopos, diffarr, numsrch) + t = time.clock() + _shift = arcytrace.find_shift(binarr, minarr, lopos, diffarr, numsrch) + print('Old find_shift: {0} seconds'.format(time.clock() - t)) + t = time.clock() + shift = new_find_shift(binarr, minarr, lopos, diffarr, numsrch) + print('New find_shift: {0} seconds'.format(time.clock() - t)) + assert np.sum(_shift != shift) == 0, 'Difference between old and new find_shift, shift' + relshift = np.mean(shift+extrap_diff[:,i-1]/2-srchz) if shift == -1: msgs.info(" Refining order {0:d}: NO relative shift applied".format(int(orders[i-1]))) @@ -2372,8 +2805,17 @@ def echelle_tilt(slf, msarc, det, pcadesc="PCA trace of the spectral tilts", mas tilts = np.zeros_like(slf._lordloc) # Generate tilts image - tiltsimg = arcytrace.tilts_image(tilts, slf._lordloc[det-1], slf._rordloc[det-1], + print('calling tilts_image') + t = time.clock() + _tiltsimg = arcytrace.tilts_image(tilts, slf._lordloc[det-1], slf._rordloc[det-1], settings.argflag['trace']['slits']['pad'], msarc.shape[1]) + print('Old tilts_image: {0} seconds'.format(time.clock() - t)) + t = time.clock() + tiltsimg = new_tilts_image(tilts, slf._lordloc[det-1], slf._rordloc[det-1], + settings.argflag['trace']['slits']['pad'], msarc.shape[1]) + print('New tilts_image: {0} seconds'.format(time.clock() - t)) + assert np.sum(_tiltsimg != tiltsimg) == 0, 'Difference between old and new tilts_image' + return tiltsimg, satmask, outpar @@ -2716,10 +3158,40 @@ def get_censpec(slf, frame, det, gen_satmask=False): ordwid = 0.5*np.abs(slf._lordloc[det-1]-slf._rordloc[det-1]) if gen_satmask: msgs.info("Generating a mask of arc line saturation streaks") - satmask = arcyarc.saturation_mask(frame, settings.spect[dnum]['saturation']*settings.spect[dnum]['nonlinear']) + t = time.clock() + _satmask = arcyarc.saturation_mask(frame, + settings.spect[dnum]['saturation']*settings.spect[dnum]['nonlinear']) + print('Old saturation_mask: {0} seconds'.format(time.clock() - t)) + satmask = ararc.new_saturation_mask(frame, + settings.spect[dnum]['saturation']*settings.spect[dnum]['nonlinear']) + print('New saturation_mask: {0} seconds'.format(time.clock() - t)) + # Allow for minor differences + if np.sum(_satmask != satmask) > 0.1*np.prod(satmask.shape): + plt.imshow(_satmask, origin='lower', interpolation='nearest', aspect='auto') + plt.colorbar() + plt.show() + plt.imshow(satmask, origin='lower', interpolation='nearest', aspect='auto') + plt.colorbar() + plt.show() + plt.imshow(_satmask-satmask, origin='lower', interpolation='nearest', aspect='auto') + plt.colorbar() + plt.show() + + assert np.sum(_satmask != satmask) < 0.1*np.prod(satmask.shape), \ + 'Old and new saturation_mask are too different' + print('calling order saturation') - exit() - satsnd = arcyarc.order_saturation(satmask, (ordcen+0.5).astype(np.int), (ordwid+0.5).astype(np.int)) + t = time.clock() + _satsnd = arcyarc.order_saturation(satmask, (ordcen+0.5).astype(int), + (ordwid+0.5).astype(int)) + print('Old order_saturation: {0} seconds'.format(time.clock() - t)) + t = time.clock() + satsnd = ararc.new_order_saturation(satmask, (ordcen+0.5).astype(int), + (ordwid+0.5).astype(int)) + print('New order_saturation: {0} seconds'.format(time.clock() - t)) + assert np.sum(_satsnd != satsnd) == 0, \ + 'Difference between old and new order_saturation, satsnd' + # Extract a rough spectrum of the arc in each slit msgs.info("Extracting an approximate arc spectrum at the centre of each slit") tordcen = None @@ -2840,14 +3312,18 @@ def phys_to_pix(array, pixlocn, axis): """ from pypit import arcytrace - if axis == 0: - diff = pixlocn[:,0,0] - else: - diff = pixlocn[0,:,1] - if len(np.shape(array)) == 1: - pixarr = arcytrace.phys_to_pix(np.array([array]).T, diff).flatten() - else: - pixarr = arcytrace.phys_to_pix(array, diff) + diff = pixlocn[:,0,0] if axis == 0 else pixlocn[0,:,1] + + print('calling phys_to_pix') + t = time.clock() + _pixarr = arcytrace.phys_to_pix(np.array([array]).T, diff).flatten() \ + if len(np.shape(array)) == 1 else arcytrace.phys_to_pix(array, diff) + print('Old phys_to_pix: {0} seconds'.format(time.clock() - t)) + t = time.clock() + pixarr = new_phys_to_pix(array, diff) + print('New phys_to_pix: {0} seconds'.format(time.clock() - t)) + assert np.sum(_pixarr != pixarr) == 0, 'Difference between old and new phys_to_pix, pixarr' + return pixarr diff --git a/pypit/arutils.py b/pypit/arutils.py index fe13b72557..75c7edd357 100644 --- a/pypit/arutils.py +++ b/pypit/arutils.py @@ -730,6 +730,8 @@ def gfunc(x,ampl,cent,sigm,cons,tilt): if np.any(y<0.0): return [0.0, 0.0, 0.0], True # Obtain a quick first guess at the parameters + print('calling fit_gauss') + exit() ampl, cent, sigm, good = arcyarc.fit_gauss(x, y, np.zeros(3,dtype=np.float), 0, x.size, float(pcen)) if good == 0: return [0.0, 0.0, 0.0], True @@ -755,6 +757,8 @@ def gauss_fit(x, y, pcen): return [0.0, 0.0, 0.0], True if x.size <= 3: return [0.0, 0.0, 0.0], True + print('calling fit_gauss') + exit() ampl, cent, sigm, good = arcyarc.fit_gauss(x, y, np.zeros(3,dtype=np.float), 0, x.size, float(pcen)) if good == 0: return [0.0, 0.0, 0.0], True diff --git a/pypit/filter.py b/pypit/filter.py new file mode 100644 index 0000000000..59e3e17fca --- /dev/null +++ b/pypit/filter.py @@ -0,0 +1,553 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +# -*- coding: utf-8 -*- +""" +A set of functions used to filter arrays. + +*License*: + Copyright (c) 2017, SDSS-IV/MaNGA Pipeline Group + Licensed under BSD 3-clause license - see LICENSE.rst + +*Source location*: + $MANGADAP_DIR/python/mangadap/util/filter.py + +*Imports and python version compliance*: + :: + + from __future__ import division + from __future__ import print_function + from __future__ import absolute_import + from __future__ import unicode_literals + + import sys + if sys.version > '3': + long = int + + import numpy + +*Revision history*: + | **26 Jan 2017**: Original implementation by K. Westfall (KBW) + | **06 Apr 2017**: Interpolate sigma vectors as well as the smoothed + vectors in :class:`BoxcarFilter`. + | **21 Dec 2017**: Add weighting functionality to + :class:`BoxcarFilter`. +""" + +from __future__ import division +from __future__ import print_function +from __future__ import absolute_import +from __future__ import unicode_literals + +import sys +if sys.version > '3': + long = int + +import warnings +import numpy +from scipy import interpolate, sparse + +# Debug +from matplotlib import pyplot +from matplotlib.ticker import NullFormatter + +def high_pass_filter(flux, dw=0, k=None, Dw=None): + """ + Pulled from FIREFLY's hpf() function and edited on 17 Nov 2016. + + Apply a high-pass filter to the input vector. + + """ + + n = flux.size + _dw = int(dw) + + if k is None and Dw is None: + Dw = 100 + _k = n//Dw + elif k is not None and Dw is None: + _k = int(k) + Dw = n//_k + elif k is None and Dw is not None: + _k = n//Dw + else: + raise ValueError('Cannot provide both k and Dw.') + + # PRINT A REPORT + + # Rita's typical inputs for SDSS: + # w = 10 # 10 + # windowsize = 20 # 20 + + # My MaNGA inputs: + # if w == 0 and windowsize == 0: + # print "HPF parameters not specified: choosing default window (stepsize=40)" + # w = 40 + # windowsize = 0 + + h = numpy.fft.fft(flux) + h_filtered = numpy.zeros(n, dtype=complex) + window = numpy.zeros(n) + unwindow = numpy.zeros(n) + window[0] = 1 # keep F0 for normalisation + unwindow[0] = 1 + + for i in range(_dw): + window[_k+i] = (i+1.0)/_dw + window[n-1-(_k+_dw-i)] = (_dw-i)/_dw + window[_k+_dw:n-(_k+_dw)] = 1 + + unwindow = 1 - window + unwindow[0] = 1 + + h_filtered = h * window + un_h_filtered = h * unwindow + + res = numpy.real(numpy.fft.ifft(h_filtered)) + unres = numpy.real(numpy.fft.ifft(un_h_filtered)) + res_out = (1.0+(res-numpy.median(res))/unres) * numpy.median(res) + + return res_out, window, res, unres + +#def off_diagonal_identity(size, win): +# r""" +# Construct a matrix with ones within a window along the diagonal. +# +# Args: +# size (int) : Size for the square matrix; i.e., :math:`N` for the +# :math:`N\timesN` matrix. +# win (int): Number of ones in each row along the diagonal. +# +# Raises: +# ValueError: Raised if the window is larger than 2*size-1. +# +# """ +# if win > 2*size-1: +# raise ValueError('Window too large for matrix size.') +# if win == 2*size-1: +# return numpy.ones((size,size), dtype=int) +# x = numpy.zeros((size,size), dtype=int)#numpy.identity(size).astype(int) +# for i in range(1,(win+1)//2): +# x[:-i,i:] = x[:-i,i:] + numpy.identity(size-i).astype(int) +# x += x.T +# if win % 2 != 1: +# x[:-i-1,i+1:] = x[:-i-1,i+1:] + numpy.identity(size-i-1).astype(int) +# return x + numpy.identity(size).astype(int) + +def off_diagonal_identity(size, win, return_sparse=False): + r""" + Construct a matrix with ones within a window along the diagonal. + + Args: + size (int) : Size for the square matrix; i.e., :math:`N` for the + :math:`N\timesN` matrix. + win (int): Number of ones in each row along the diagonal. + + Raises: + ValueError: Raised if the window is larger than 2*size-1. + + """ + if win > 2*size-1: + raise ValueError('Window too large for matrix size.') + if win == 2*size-1: + return numpy.ones((size,size), dtype=int) + + # Indices of diagonal + ii = numpy.arange(size).astype(int) + + # Build the upper triangle + i = numpy.empty(0, dtype=int) + j = numpy.empty(0, dtype=int) + for k in range(1,(win+1)//2): + i = numpy.append(i, ii[:size-k]) + j = numpy.append(j, ii[k:size]) + + # Copy to the lower triangle + _i = numpy.append(i,j) + j = numpy.append(j,i) + + # Add the diagonal + i = numpy.append(_i, ii) + j = numpy.append(j, ii) + + # Accommodate an even window + if win % 2 != 1: + i = numpy.append(i, ii[:size-k-1]) + j = numpy.append(j, ii[k+1:size]) + + # Construct and return the array + if return_sparse: + return sparse.coo_matrix((numpy.ones(len(i), dtype=int),(i,j)), shape=(size,size)).tocsr() + + a = numpy.zeros((size,size), dtype=int) + a[i,j] = 1 + return a + + +def build_smoothing_mask(x, pix_buffer, default=None, mask_x=None): + r""" + Construct a mask for the provided vector that masks the first and + last pix_buffer pixels in the coordinate vector. + + The mask is instantiated as fully unmasked, unless a default mask is + provided. + + To mask specified ranges in x, provide mask_x with a shape + :math:`N_{\rm mask}\times 2` where each mask is defined by the + starting and ending value of x to exclude. + """ + # Smooth the ratio of the data to the model + if len(x.shape) != 1: + raise ValueError('Input must be a vector.') + npix = x.size + mask = numpy.zeros(npix, dtype=bool) if default is None else default.copy() + if mask.size != npix: + raise ValueError('Provided default has an incorrect length.') + mask[:pix_buffer] = True + mask[-pix_buffer:] = True + if mask_x is None: + return mask + + for m in mask_x: + mask |= numpy.logical_and(x > m[0], x < m[1]) + return mask + + +class BoxcarFilter(): + def __init__(self, boxcar, lo=None, hi=None, niter=None, y=None, wgt=None, mask=None, + local_sigma=None): + + if boxcar <= 1: + raise ValueError('Boxcar must be greater than 1!') + + self.y = None + self.wgt = None + self.input_mask = None + self.output_mask = None + self.boxcar = boxcar + self.lo_rej = lo + self.hi_rej = hi + self.nrej = -1 if niter is None else niter + self.nvec = None + self.npix = None + self.local_sigma = False if local_sigma is None else local_sigma + + self.smoothed_wgt = None + self.smoothed_n = None + self.smoothed_y = None + self.sigma_y = None + self.rdx_index = None + + if y is not None: + self.smooth(y, wgt=wgt, mask=mask, boxcar=boxcar, lo=lo, hi=hi, niter=niter, + local_sigma=local_sigma) + + + def _assign_par(self, boxcar, lo, hi, niter, local_sigma): + if boxcar is not None: + self.boxcar = boxcar + if lo is not None: + self.lo_rej = lo + if hi is not None: + self.hi_rej = hi + if niter is not None: + self.nrej = niter + if local_sigma is not None: + self.local_sigma = local_sigma + + + def _check_par(self): + if self.boxcar <= 1: + raise ValueError('Boxcar must be greater than 1!') + + if self.nrej is not None and self.nrej > 0 and self.lo_rej is None and self.hi_rej is None: + raise ValueError('Must provide lo or hi if niter provided') + + + def _reduce_indices(self): + indx = numpy.array([numpy.arange(self.npix)-self.boxcar//2, + numpy.arange(self.npix)+self.boxcar//2+1]) + if self.boxcar % 2 != 1: + indx[:self.npix] += 1 + return indx.T.ravel().clip(0,self.npix-1) + + + def _apply(self): + self.smoothed_n = numpy.add.reduceat(numpy.invert(self.output_mask), self.rdx_index, axis=1, + dtype=int)[:,::2] + self.smoothed_wgt = numpy.add.reduceat(self.input_wgt, self.rdx_index, axis=1, + dtype=float)[:,::2] + self.smoothed_y = numpy.add.reduceat(self.input_wgt*self.y.filled(0.0), self.rdx_index, + axis=1, dtype=float)[:,::2] + smoothed_y2 = numpy.add.reduceat(self.input_wgt*numpy.square(self.y.filled(0.0)), + self.rdx_index, axis=1, dtype=float)[:,::2] + + self.smoothed_y = numpy.ma.divide(self.smoothed_y, self.smoothed_wgt) + self.smoothed_y[self.output_mask | (self.smoothed_n == 0)] = numpy.ma.masked + + self.sigma_y = numpy.ma.sqrt((numpy.ma.divide(smoothed_y2, self.smoothed_wgt) + - numpy.square(self.smoothed_y)) + * numpy.ma.divide(self.smoothed_n, self.smoothed_n-1)) \ + if self.local_sigma else \ + numpy.ma.MaskedArray(numpy.array([numpy.std(self.y - self.smoothed_y, + axis=1)]*self.npix).T) + + self.output_mask = numpy.ma.getmaskarray(self.smoothed_y) \ + | numpy.ma.getmaskarray(self.sigma_y) + + self.smoothed_y[self.output_mask | (self.smoothed_n == 0)] = numpy.ma.masked + self.sigma_y[self.output_mask | (self.smoothed_n == 0)] = numpy.ma.masked + + return + + w,h = pyplot.figaspect(1) + fig = pyplot.figure(figsize=(1.5*w,1.5*h)) + + minf = numpy.amin( numpy.ma.log10(numpy.append(self.y.compressed(), self.smoothed_y.compressed())) ) + maxf = numpy.amax( numpy.ma.log10(numpy.append(self.y.compressed(), self.smoothed_y.compressed())) ) + + mins = numpy.amin(numpy.ma.log10(self.sigma_y).compressed()) + maxs = numpy.amax(numpy.ma.log10(self.sigma_y).compressed()) + + minr = numpy.amin(numpy.ma.log10(numpy.ma.divide(self.y, self.smoothed_y)).compressed()) + maxr = numpy.amax(numpy.ma.log10(numpy.ma.divide(self.y, self.smoothed_y)).compressed()) + + ax = fig.add_axes([0.05, 0.50, 0.45, 0.45]) + ax.xaxis.set_major_formatter(NullFormatter()) + ax.imshow(numpy.ma.log10(self.y), origin='lower', interpolation='nearest', vmin=minf, + vmax=maxf, aspect='auto') + ax = fig.add_axes([0.50, 0.50, 0.45, 0.45]) + ax.xaxis.set_major_formatter(NullFormatter()) + ax.yaxis.set_major_formatter(NullFormatter()) + ax.imshow(numpy.ma.log10(self.smoothed_y), origin='lower', interpolation='nearest', + vmin=minf, vmax=maxf, aspect='auto') + ax = fig.add_axes([0.05, 0.05, 0.45, 0.45]) + ax.yaxis.set_major_formatter(NullFormatter()) + ax.imshow(numpy.ma.log10(numpy.ma.divide(self.y,self.smoothed_y)), origin='lower', + interpolation='nearest', aspect='auto', vmin=minr, vmax=maxr) + ax = fig.add_axes([0.50, 0.05, 0.45, 0.45]) + ax.yaxis.set_major_formatter(NullFormatter()) + ax.imshow(numpy.ma.log10(self.sigma_y), origin='lower', interpolation='nearest', + aspect='auto', vmin=mins, vmax=maxs) + + pyplot.show() + exit() + + + def _interpolate(self): + """ + Interpolate the smoothed image across the masked regions. + Leading and trailing masked regions of each row are set to the + first and last unmasked value, respectively. + + interpolate both the smoothed data and the sigma + + """ + # Boolean arrays with bad pixels (should be the same for both + # smoothed_y and sigma_y) + badpix = numpy.ma.getmaskarray(self.smoothed_y) + + # Deal with any vectors that are fully masked + goodvec = numpy.sum(numpy.invert(badpix), axis=1) > 0 + ngood = numpy.sum(goodvec) + veci = numpy.arange(self.nvec)[goodvec] + + # Find the first and last unflagged pixels in the good vectors + pixcoo = numpy.ma.MaskedArray(numpy.array([numpy.arange(self.npix)]*ngood), + mask=badpix[goodvec,:]).astype(int) + mini = numpy.ma.amin(pixcoo, axis=1) + maxi = numpy.ma.amax(pixcoo, axis=1) + + # Copy those to the first and last pixels for each row + self.smoothed_y[veci,0] = numpy.array([self.smoothed_y[i,m] for i,m in zip(veci, mini)]) + self.smoothed_y[veci,-1] = numpy.array([self.smoothed_y[i,m] for i,m in zip(veci, maxi)]) + + self.sigma_y[veci,0] = numpy.array([self.sigma_y[i,m] for i,m in zip(veci, mini)]) + self.sigma_y[veci,-1] = numpy.array([self.sigma_y[i,m] for i,m in zip(veci, maxi)]) + + # Detach badpix from self.smoothed_y + badpix = numpy.ma.getmaskarray(self.smoothed_y).copy() + + # Interpolate the smoothed array + x = numpy.ma.MaskedArray(numpy.arange(self.nvec*self.npix), mask=badpix) + interpolator = interpolate.interp1d(x.compressed(), self.smoothed_y.compressed(), + assume_sorted=True, fill_value='extrapolate') + self.smoothed_y.ravel()[badpix.ravel()] = interpolator(x.data[badpix.ravel()]) + self.smoothed_y[numpy.invert(goodvec),:] = 0.0 + + # Interpolate the sigma array + interpolator = interpolate.interp1d(x.compressed(), self.sigma_y.compressed(), + assume_sorted=True, fill_value='extrapolate') + self.sigma_y.ravel()[badpix.ravel()] = interpolator(x.data[badpix.ravel()]) + self.sigma_y[numpy.invert(goodvec),:] = 0.0 + + + def smooth(self, y, wgt=None, mask=None, boxcar=None, lo=None, hi=None, niter=None, + local_sigma=None): + """ + Smooth a vector or array of vectors by a boxcar with specified + rejection. + """ + # Assign and check the binning parameters + self._assign_par(boxcar, lo, hi, niter, local_sigma) + self._check_par() + + # Check the input vector/array + if len(y.shape) > 2: + raise ValueError('Can only deal with vectors or matrices.') + + # Set the weighting + self.input_wgt = numpy.ones(y.shape, dtype=bool) if wgt is None else wgt + if self.input_wgt.shape != y.shape: + raise ValueError('Weights shape does not match data.') + if isinstance(y, numpy.ma.MaskedArray): + self.input_wgt[numpy.ma.getmaskarray(y)] = 0.0 + + # Set the mask + self.input_mask = numpy.zeros(y.shape, dtype=bool) if mask is None else mask + if self.input_mask.shape != y.shape: + raise ValueError('Mask shape does not match data.') + if isinstance(y, numpy.ma.MaskedArray): + self.input_mask |= numpy.ma.getmaskarray(y) + + # Save the input + provided_vector = len(y.shape) == 1 + self.y = numpy.ma.atleast_2d(numpy.ma.MaskedArray(y.copy(), mask=self.input_mask)) + self.input_wgt = numpy.atleast_2d(self.input_wgt) + self.input_mask = numpy.atleast_2d(self.input_mask) + self.output_mask = self.input_mask.copy() + self.nvec, self.npix = self.y.shape + + # Create the list of reduceat indices + self.rdx_index = self._reduce_indices() + + # Get the first iteration of the smoothed vectors + self._apply() + + # No rejection iterations requested so return the result + if self.lo_rej is None and self.hi_rej is None: + self._interpolate() + return self.smoothed_y[0,:] if provided_vector else self.smoothed_y + + # Iteratively reject outliers + nmasked = numpy.sum(numpy.ma.getmaskarray(self.y)) + i=0 + while i > -1: + nbad = numpy.sum(self.output_mask) + if self.lo_rej is not None: + self.output_mask = numpy.logical_or(self.output_mask, self.y.data - self.smoothed_y + < (-self.lo_rej*self.sigma_y)) + if self.hi_rej is not None: + self.output_mask = numpy.logical_or(self.output_mask, self.y.data - self.smoothed_y + > (self.hi_rej*self.sigma_y)) + self.y[self.output_mask] = numpy.ma.masked + self._apply() + + nmasked = numpy.sum(self.output_mask) - nbad +# print('Niter: {0}; Nmasked: {1}'.format(i+1, nmasked)) + + i += 1 + if i == self.nrej or nmasked == 0: + break + + self._interpolate() + return self.smoothed_y[0,:] if provided_vector else self.smoothed_y + + + @property + def mask(self): + return self.output_mask.copy() + + + +def smooth_masked_vector(x, nsmooth): + """ + Smooth a masked vector by a box of size nsmooth. + """ + n = off_diagonal_identity(x.size, nsmooth)*numpy.invert(numpy.ma.getmaskarray(x))[None,:] + nn = numpy.sum(n,axis=1) + return numpy.ma.MaskedArray(numpy.dot(n,x.data), mask=numpy.invert(nn>0) | x.mask)/nn + + +def interpolate_masked_vector(y, quiet=True, extrap_with_median=False): + """ + Interpolate over the masked pixels in an input vector using linear + interpolation. + """ + x = numpy.arange(y.size) + indx = numpy.ma.getmaskarray(y) + if numpy.sum(numpy.invert(indx)) < 2: + if not quiet: + warnings.warn('Input vector has fewer than 2 unmasked values! Returning zero vector.') + return numpy.zeros(y.size, dtype=y.dtype.name) + interpolator = interpolate.interp1d(x[numpy.invert(indx)], y[numpy.invert(indx)], + fill_value='extrapolate') + _y = y.data.copy() + _y[indx] = interpolator(x[indx]) + + if extrap_with_median: + med = numpy.median(interpolator.y) + m_indx = (x[indx] < interpolator.x[0]) | (x[indx] > interpolator.x[-1]) + _y[indx][m_indx] = med + + return _y + + +def boxcar_smooth_vector(x, boxcar, mask=None, lo=None, hi=None, niter=None, return_mask=False): + """ + Boxcar smooth an input vector. + - Ignores masked pixels. + - Allows for iterative positive and negative outliers. + - Can return the mask separately from the smoothed vector. + """ + if niter is not None and lo is None and hi is None: + raise ValueError('Must provide lo or hi if niter provided') + + _mask = numpy.zeros(x.size, dtype=bool) if mask is None else mask + _x = x if isinstance(x, numpy.ma.MaskedArray) else numpy.ma.MaskedArray(x) + _x[_mask] = numpy.ma.masked + + sx = interpolate_masked_vector( smooth_masked_vector(_x, boxcar) ) + +# pyplot.step(numpy.arange(_x.size), _x.data, where='mid', color='0.3', lw=0.5) +# pyplot.step(numpy.arange(_x.size), _x, where='mid', color='k', lw=0.5) +# pyplot.plot(numpy.arange(_x.size), sx, color='r', lw=1) +# pyplot.show() + + if numpy.all([ x is None for x in [ lo, hi, niter ]]): + if return_mask: + return sx, numpy.ma.getmaskarray(_x).copy() + return sx + + _niter = -1 if niter is None else niter + nrej = numpy.sum(numpy.ma.getmaskarray(_x)) + i=0 + while i > -1: + sig = numpy.std((_x - sx).compressed()) + mask = numpy.ma.getmaskarray(_x).copy() + if lo is not None: + mask = numpy.logical_or(mask, _x.data - sx < -lo*sig) + if hi is not None: + mask = numpy.logical_or(mask, _x.data - sx > hi*sig) + nrej = numpy.sum(mask) - numpy.sum(_x.mask) +# print('Niter: {0}; Nrej: {1}'.format(i+1, nrej)) + _x[mask] = numpy.ma.masked + sx = interpolate_masked_vector( smooth_masked_vector(_x, boxcar) ) + +# pyplot.step(numpy.arange(_x.size), _x.data, where='mid', color='cyan', lw=0.5) +# pyplot.step(numpy.arange(_x.size), _x, where='mid', color='k', lw=0.5) +# pyplot.plot(numpy.arange(_x.size), sx, color='r', lw=1) +# pyplot.show() + + i += 1 + if i == _niter or nrej == 0: + break + + pyplot.step(numpy.arange(_x.size), _x.data, where='mid', color='cyan', lw=0.5) + pyplot.step(numpy.arange(_x.size), _x, where='mid', color='k', lw=0.5) + pyplot.plot(numpy.arange(_x.size), sx, color='r', lw=1) + pyplot.show() + + if return_mask: + return sx, numpy.ma.getmaskarray(_x).copy() + return sx + + From 835fbb9ca24d34743df09621daa7a58018192a7b Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Mon, 12 Mar 2018 08:36:41 -0700 Subject: [PATCH 03/36] minor comments --- pypit/arextract.py | 3 ++- pypit/arutils.py | 4 ---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/pypit/arextract.py b/pypit/arextract.py index c9b9a3ea8c..fdd780df7c 100644 --- a/pypit/arextract.py +++ b/pypit/arextract.py @@ -534,9 +534,10 @@ def new_func2d_fit_val(y, order, x=None, w=None): msgs.error('Input w must match y axis length or y shape.') # Prep the output model ym = np.empty(_y.shape, dtype=float) + # Weight the data __y = w * _y for i in range(ny): - # Fit with the same weight for each vector + # Weight the Vandermonde matrix for this y vector _vand = w[i,:,None] * vand c = np.linalg.lstsq(_vand, __y[i,:])[0] ym[i,:] = np.sum(c[:,None] * vand.T[:,:], axis=0) diff --git a/pypit/arutils.py b/pypit/arutils.py index 75c7edd357..fe13b72557 100644 --- a/pypit/arutils.py +++ b/pypit/arutils.py @@ -730,8 +730,6 @@ def gfunc(x,ampl,cent,sigm,cons,tilt): if np.any(y<0.0): return [0.0, 0.0, 0.0], True # Obtain a quick first guess at the parameters - print('calling fit_gauss') - exit() ampl, cent, sigm, good = arcyarc.fit_gauss(x, y, np.zeros(3,dtype=np.float), 0, x.size, float(pcen)) if good == 0: return [0.0, 0.0, 0.0], True @@ -757,8 +755,6 @@ def gauss_fit(x, y, pcen): return [0.0, 0.0, 0.0], True if x.size <= 3: return [0.0, 0.0, 0.0], True - print('calling fit_gauss') - exit() ampl, cent, sigm, good = arcyarc.fit_gauss(x, y, np.zeros(3,dtype=np.float), 0, x.size, float(pcen)) if good == 0: return [0.0, 0.0, 0.0], True From 8a2d6c55b2f32fbb762e2e45fefc5fa3ea84b133 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Thu, 22 Mar 2018 11:35:58 -0700 Subject: [PATCH 04/36] removing asserts and print statements for cython substitutions --- pypit/ararc.py | 30 ++--- pypit/arcomb.py | 80 ++++++------- pypit/arproc.py | 68 +++++------ pypit/artrace.py | 290 +++++++++++++++++++++++------------------------ 4 files changed, 234 insertions(+), 234 deletions(-) diff --git a/pypit/ararc.py b/pypit/ararc.py index e957b902c8..befa3a6a8a 100644 --- a/pypit/ararc.py +++ b/pypit/ararc.py @@ -77,23 +77,23 @@ def detect_lines(slf, det, msarc, censpec=None, MK_SATMASK=False): if MK_SATMASK: ordwid = 0.5*np.abs(slf._lordloc[det-1] - slf._rordloc[det-1]) msgs.info("Generating a mask of arc line saturation streaks") - print('calling saturation_mask') - t = time.clock() - _satmask = arcyarc.saturation_mask(msarc, slf._nonlinear[det-1]) - print('Old saturation_mask: {0} seconds'.format(time.clock() - t)) - t = time.clock() +# print('calling saturation_mask') +# t = time.clock() +# _satmask = arcyarc.saturation_mask(msarc, slf._nonlinear[det-1]) +# print('Old saturation_mask: {0} seconds'.format(time.clock() - t)) +# t = time.clock() satmask = new_saturation_mask(msarc, slf._nonlinear[det-1]) - print('New saturation_mask: {0} seconds'.format(time.clock() - t)) - assert np.sum(_satmask != satmask) == 0, 'Difference between old and new saturation_mask' - - print('calling order_saturation') - t = time.clock() - _satsnd = arcyarc.order_saturation(satmask, ordcen, (ordwid+0.5).astype(np.int)) - print('Old order_saturation: {0} seconds'.format(time.clock() - t)) - t = time.clock() +# print('New saturation_mask: {0} seconds'.format(time.clock() - t)) +# assert np.sum(_satmask != satmask) == 0, 'Difference between old and new saturation_mask' + +# print('calling order_saturation') +# t = time.clock() +# _satsnd = arcyarc.order_saturation(satmask, ordcen, (ordwid+0.5).astype(np.int)) +# print('Old order_saturation: {0} seconds'.format(time.clock() - t)) +# t = time.clock() satsnd = new_order_saturation(satmask, ordcen, (ordwid+0.5).astype(np.int)) - print('New order_saturation: {0} seconds'.format(time.clock() - t)) - assert np.sum(_satsnd != satsnd) == 0, 'Difference between old and new order_saturation' +# print('New order_saturation: {0} seconds'.format(time.clock() - t)) +# assert np.sum(_satsnd != satsnd) == 0, 'Difference between old and new order_saturation' else: satsnd = np.zeros_like(ordcen) # Detect the location of the arc lines diff --git a/pypit/arcomb.py b/pypit/arcomb.py index 009fa201f6..80d4a30b0a 100644 --- a/pypit/arcomb.py +++ b/pypit/arcomb.py @@ -92,31 +92,31 @@ def comb_frames(frames_arr, det, frametype, weights=None, maskvalue=1048577, pri allrej_arr = np.median(frames_arr, axis=2) elif reject['replace'] == 'weightmean': msgs.work("No weights are implemented yet") - print('calling masked_weightmean') - _allrej_arr = frames_arr.copy() - t = time.clock() - _allrej_arr = arcycomb.masked_weightmean(_allrej_arr, maskvalue) - print('Old masked_weightmean: {0} seconds'.format(time.clock() - t)) +# print('calling masked_weightmean') +# _allrej_arr = frames_arr.copy() +# t = time.clock() +# _allrej_arr = arcycomb.masked_weightmean(_allrej_arr, maskvalue) +# print('Old masked_weightmean: {0} seconds'.format(time.clock() - t)) __allrej_arr = frames_arr.copy() - t = time.clock() +# t = time.clock() __allrej_arr = new_masked_weightmean(__allrej_arr, maskvalue) - print('New masked_weightmean: {0} seconds'.format(time.clock() - t)) - print(__allrej_arr.shape) - assert np.sum(__allrej_arr != _allrej_arr) == 0, \ - 'Difference between old and new masked_weightmean' +# print('New masked_weightmean: {0} seconds'.format(time.clock() - t)) +# print(__allrej_arr.shape) +# assert np.sum(__allrej_arr != _allrej_arr) == 0, \ +# 'Difference between old and new masked_weightmean' allrej_arr = __allrej_arr ## allrej_arr = arcycomb.masked_weightmean(frames_arr, maskvalue) # allrej_arr = new_masked_weightmean(frames_arr, maskvalue) elif reject['replace'] == 'maxnonsat': - print('calling maxnonsat') - _allrej_arr = frames_arr.copy() - t = time.clock() - _allrej_arr = arcycomb.maxnonsat(_allrej_arr, saturation) - print('Old maxnonsat: {0} seconds'.format(time.clock() - t)) +# print('calling maxnonsat') +# _allrej_arr = frames_arr.copy() +# t = time.clock() +# _allrej_arr = arcycomb.maxnonsat(_allrej_arr, saturation) +# print('Old maxnonsat: {0} seconds'.format(time.clock() - t)) __allrej_arr = frames_arr.copy() - t = time.clock() +# t = time.clock() __allrej_arr = new_maxnonsat(__allrej_arr, saturation) - print('New maxnonsat: {0} seconds'.format(time.clock() - t)) +# print('New maxnonsat: {0} seconds'.format(time.clock() - t)) # Bug in arcycomb.maxnonsat ? # assert np.sum(__allrej_arr != _allrej_arr) == 0, 'Difference between old and new maxnonsat' allrej_arr = __allrej_arr @@ -228,31 +228,31 @@ def comb_frames(frames_arr, det, frametype, weights=None, maskvalue=1048577, pri # frames_arr = arcycomb.masked_median(frames_arr, maskvalue) frames_arr = np.ma.median(np.ma.MaskedArray(frames_arr, mask=frames_arr==maskvalue), axis=2) elif method == 'weightmean': - print('calling masked_weightmean') - _frames_arr = frames_arr.copy() - t = time.clock() - _frames_arr = arcycomb.masked_weightmean(_frames_arr, maskvalue) - print('Old masked_weightmean: {0} seconds'.format(time.clock() - t)) +# print('calling masked_weightmean') +# _frames_arr = frames_arr.copy() +# t = time.clock() +# _frames_arr = arcycomb.masked_weightmean(_frames_arr, maskvalue) +# print('Old masked_weightmean: {0} seconds'.format(time.clock() - t)) __frames_arr = frames_arr.copy() - t = time.clock() +# t = time.clock() __frames_arr = new_masked_weightmean(__frames_arr, maskvalue) - print('New masked_weightmean: {0} seconds'.format(time.clock() - t)) - print(__frames_arr.shape) - - if np.sum(np.absolute(__frames_arr-_frames_arr) > 1e-10) != 0: - print(np.sum(np.absolute(__frames_arr-_frames_arr) > 1e-10)) - plt.imshow(_frames_arr, origin='lower', interpolation='nearest', aspect='auto') - plt.show() - plt.imshow(__frames_arr, origin='lower', interpolation='nearest', aspect='auto') - plt.show() - plt.imshow(_frames_arr - __frames_arr, origin='lower', interpolation='nearest', aspect='auto') - plt.show() - plt.imshow(np.ma.divide(_frames_arr,__frames_arr) - 1, origin='lower', interpolation='nearest', aspect='auto') - plt.colorbar() - plt.show() - - assert np.sum( np.absolute(__frames_arr-_frames_arr) > 1e-10 ) == 0, \ - 'Difference between old and new masked_weightmean' +# print('New masked_weightmean: {0} seconds'.format(time.clock() - t)) +# print(__frames_arr.shape) +# +# if np.sum(np.absolute(__frames_arr-_frames_arr) > 1e-10) != 0: +# print(np.sum(np.absolute(__frames_arr-_frames_arr) > 1e-10)) +# plt.imshow(_frames_arr, origin='lower', interpolation='nearest', aspect='auto') +# plt.show() +# plt.imshow(__frames_arr, origin='lower', interpolation='nearest', aspect='auto') +# plt.show() +# plt.imshow(_frames_arr - __frames_arr, origin='lower', interpolation='nearest', aspect='auto') +# plt.show() +# plt.imshow(np.ma.divide(_frames_arr,__frames_arr) - 1, origin='lower', interpolation='nearest', aspect='auto') +# plt.colorbar() +# plt.show() +# +# assert np.sum( np.absolute(__frames_arr-_frames_arr) > 1e-10 ) == 0, \ +# 'Difference between old and new masked_weightmean' frames_arr = __frames_arr else: msgs.error("Combination type '{0:s}' is unknown".format(method)) diff --git a/pypit/arproc.py b/pypit/arproc.py index acd1cd81ab..1d1111d62a 100644 --- a/pypit/arproc.py +++ b/pypit/arproc.py @@ -196,19 +196,19 @@ def bg_subtraction(slf, det, sciframe, varframe, crpix, tracemask=None, errframe = np.sqrt(varframe) # Find which pixels are within the order edges msgs.info("Identifying pixels within each order") - print('calling order_pixels') - t = time.clock() - _ordpix = arcyutils.order_pixels(slf._pixlocn[det-1], - slf._lordloc[det-1]*0.95+slf._rordloc[det-1]*0.05, - slf._lordloc[det-1]*0.05+slf._rordloc[det-1]*0.95) - print('Old order_pixels: {0} seconds'.format(time.clock() - t)) - t = time.clock() +# print('calling order_pixels') +# t = time.clock() +# _ordpix = arcyutils.order_pixels(slf._pixlocn[det-1], +# slf._lordloc[det-1]*0.95+slf._rordloc[det-1]*0.05, +# slf._lordloc[det-1]*0.05+slf._rordloc[det-1]*0.95) +# print('Old order_pixels: {0} seconds'.format(time.clock() - t)) +# t = time.clock() ordpix = new_order_pixels(slf._pixlocn[det-1], slf._lordloc[det-1]*0.95+slf._rordloc[det-1]*0.05, slf._lordloc[det-1]*0.05+slf._rordloc[det-1]*0.95) - print('New order_pixels: {0} seconds'.format(time.clock() - t)) - assert np.sum(_ordpix != ordpix) == 0, \ - 'Difference between old and new order_pixels' +# print('New order_pixels: {0} seconds'.format(time.clock() - t)) +# assert np.sum(_ordpix != ordpix) == 0, \ +# 'Difference between old and new order_pixels' msgs.info("Applying bad pixel mask") ordpix *= (1-slf._bpix[det-1].astype(np.int)) * (1-crpix.astype(np.int)) @@ -451,15 +451,15 @@ def flatnorm(slf, det, msflat, maskval=-999999.9, overpix=6, plotdesc=""): # Find which pixels are within the order edges msgs.info("Identifying pixels within each order") - print('calling order_pixels') - t = time.clock() - _ordpix = arcyutils.order_pixels(slf._pixlocn[det-1], slf._lordloc[det-1], slf._rordloc[det-1]) - print('Old order_pixels: {0} seconds'.format(time.clock() - t)) - t = time.clock() +# print('calling order_pixels') +# t = time.clock() +# _ordpix = arcyutils.order_pixels(slf._pixlocn[det-1], slf._lordloc[det-1], slf._rordloc[det-1]) +# print('Old order_pixels: {0} seconds'.format(time.clock() - t)) +# t = time.clock() ordpix = new_order_pixels(slf._pixlocn[det-1], slf._lordloc[det-1], slf._rordloc[det-1]) - print('New order_pixels: {0} seconds'.format(time.clock() - t)) - assert np.sum(_ordpix != ordpix) == 0, \ - 'Difference between old and new order_pixels' +# print('New order_pixels: {0} seconds'.format(time.clock() - t)) +# assert np.sum(_ordpix != ordpix) == 0, \ +# 'Difference between old and new order_pixels' msgs.info("Applying bad pixel mask") ordpix *= (1-slf._bpix[det-1].astype(np.int)) @@ -1186,17 +1186,17 @@ def slit_pixels(slf, frameshape, det): for o in range(nslits): lordloc = slf._lordloc[det - 1][:, o] rordloc = slf._rordloc[det - 1][:, o] - print('calling locate_order') - t = time.clock() - _ordloc = arcytrace.locate_order(lordloc, rordloc, frameshape[0], frameshape[1], - settings.argflag['trace']['slits']['pad']) - print('Old locate_order: {0} seconds'.format(time.clock() - t)) - t = time.clock() +# print('calling locate_order') +# t = time.clock() +# _ordloc = arcytrace.locate_order(lordloc, rordloc, frameshape[0], frameshape[1], +# settings.argflag['trace']['slits']['pad']) +# print('Old locate_order: {0} seconds'.format(time.clock() - t)) +# t = time.clock() ordloc = new_locate_order(lordloc, rordloc, frameshape[0], frameshape[1], settings.argflag['trace']['slits']['pad']) - print('New locate_order: {0} seconds'.format(time.clock() - t)) - assert np.sum(_ordloc != ordloc) == 0, \ - 'Difference between old and new locate_order' +# print('New locate_order: {0} seconds'.format(time.clock() - t)) +# assert np.sum(_ordloc != ordloc) == 0, \ +# 'Difference between old and new locate_order' word = np.where(ordloc != 0) if word[0].size == 0: msgs.warn("There are no pixels in slit {0:d}".format(o + 1)) @@ -1836,14 +1836,14 @@ def lacosmic(slf, fitsdict, det, sciframe, scidx, maxiter=1, grow=1.5, maskval=- sigmask[np.where(sigsmth>sigclip)] = True crmask = np.logical_and(crmask, sigmask) msgs.info("Growing cosmic ray mask by 1 pixel") - print('calling grow_masked') - t = time.clock() - _crmask = arcyutils.grow_masked(crmask.astype(np.float), grow, 1.0) - print('Old grow_masked: {0} seconds'.format(time.clock() - t)) - t = time.clock() +# print('calling grow_masked') +# t = time.clock() +# _crmask = arcyutils.grow_masked(crmask.astype(np.float), grow, 1.0) +# print('Old grow_masked: {0} seconds'.format(time.clock() - t)) +# t = time.clock() crmask = new_grow_masked(crmask.astype(np.float), grow, 1.0) - print('New grow_masked: {0} seconds'.format(time.clock() - t)) - assert np.sum(_crmask != crmask) == 0, 'Difference between old and new grow_masked' +# print('New grow_masked: {0} seconds'.format(time.clock() - t)) +# assert np.sum(_crmask != crmask) == 0, 'Difference between old and new grow_masked' return crmask diff --git a/pypit/artrace.py b/pypit/artrace.py index 3c5ab9a030..8c1d5dac6f 100644 --- a/pypit/artrace.py +++ b/pypit/artrace.py @@ -124,15 +124,15 @@ def assign_slits(binarr, edgearr, ednum=100000, lor=-1): # After pruning, there are no more peaks break pks = wpk+2 # Shifted by 2 because of the peak finding algorithm above - print('calling find_peak_limits') - t = time.clock() - _pedges = arcytrace.find_peak_limits(smedgehist, pks) - print('Old find_peak_limits: {0} seconds'.format(time.clock() - t)) - t = time.clock() +# print('calling find_peak_limits') +# t = time.clock() +# _pedges = arcytrace.find_peak_limits(smedgehist, pks) +# print('Old find_peak_limits: {0} seconds'.format(time.clock() - t)) +# t = time.clock() pedges = new_find_peak_limits(smedgehist, pks) - print('New find_peak_limits: {0} seconds'.format(time.clock() - t)) - assert np.sum(_pedges != pedges) == 0, \ - 'Difference between old and new find_peak_limits' +# print('New find_peak_limits: {0} seconds'.format(time.clock() - t)) +# assert np.sum(_pedges != pedges) == 0, \ +# 'Difference between old and new find_peak_limits' if np.all(pedges[:, 1]-pedges[:, 0] == 0): # Remaining peaks have no width @@ -301,7 +301,7 @@ def expand_slits(slf, mstrace, det, ordcen, extord): # Calculate the pixel locations of th eorder edges pixcen = phys_to_pix(ordcen, slf._pixlocn[det - 1], 1) msgs.info("Expanding slit traces to slit edges") - exit() +# exit() mordwid, pordwid = arcytrace.expand_slits(mstrace, pixcen, extord.astype(np.int)) # Fit a function for the difference between left edge and the centre trace ldiff_coeff, ldiff_fit = arutils.polyfitter2d(mordwid, mask=-1, @@ -540,16 +540,16 @@ def trace_object(slf, det, sciframe, varframe, crmask, trim=2, msgs.info("Estimating object profiles") # Smooth the S/N frame smthby, rejhilo = tracepar['smthby'], tracepar['rejhilo'] - print('calling smooth_x') - t = time.clock() - _rec_sigframe_bin = arcyutils.smooth_x(rec_sciframe/np.sqrt(rec_varframe), 1.0-rec_crmask, - smthby, rejhilo, maskval) - print('Old smooth_x: {0} seconds'.format(time.clock() - t)) +# print('calling smooth_x') +# t = time.clock() +# _rec_sigframe_bin = arcyutils.smooth_x(rec_sciframe/np.sqrt(rec_varframe), 1.0-rec_crmask, +# smthby, rejhilo, maskval) +# print('Old smooth_x: {0} seconds'.format(time.clock() - t)) tmp = np.ma.MaskedArray(rec_sciframe/np.sqrt(rec_varframe), mask=rec_crmask.astype(bool)) - # TODO: Add rejection to BoxcarFilter? - t = time.clock() +# # TODO: Add rejection to BoxcarFilter? +# t = time.clock() rec_sigframe_bin = BoxcarFilter(smthby).smooth(tmp.T).T - print('BoxcarFilter: {0} seconds'.format(time.clock() - t)) +# print('BoxcarFilter: {0} seconds'.format(time.clock() - t)) # t = time.clock() # rec_sigframe_bin = new_smooth_x(rec_sciframe/np.sqrt(rec_varframe), 1.0-rec_crmask, smthby, # rejhilo, maskval) @@ -624,13 +624,13 @@ def trace_object(slf, det, sciframe, varframe, crmask, trim=2, trcprof2 = np.mean(rec_sciframe, axis=0) objl, objr, bckl, bckr = find_obj_minima(trcprof2, triml=triml, trimr=trimr, nsmooth=nsmooth) elif settings.argflag['trace']['object']['find'] == 'standard': - print('calling find_objects') - t = time.clock() - _objl, _objr, _bckl, _bckr = arcytrace.find_objects(trcprof, bgreg, mad) - print('Old find_objects: {0} seconds'.format(time.clock() - t)) - t = time.clock() +# print('calling find_objects') +# t = time.clock() +# _objl, _objr, _bckl, _bckr = arcytrace.find_objects(trcprof, bgreg, mad) +# print('Old find_objects: {0} seconds'.format(time.clock() - t)) +# t = time.clock() objl, objr, bckl, bckr = new_find_objects(trcprof, bgreg, mad) - print('New find_objects: {0} seconds'.format(time.clock() - t)) +# print('New find_objects: {0} seconds'.format(time.clock() - t)) # print('objl:', objl) # print('_objl:', _objl) # print('objr:', objr) @@ -639,10 +639,10 @@ def trace_object(slf, det, sciframe, varframe, crmask, trim=2, # print(_objr.shape, objr.shape) # print(np.sum(_objl != objl)) # print(np.sum(_objr != objr)) - assert np.sum(_objl != objl) == 0, 'Difference between old and new find_objects, objl' - assert np.sum(_objr != objr) == 0, 'Difference between old and new find_objects, objr' - assert np.sum(_bckl != bckl) == 0, 'Difference between old and new find_objects, bckl' - assert np.sum(_bckr != bckr) == 0, 'Difference between old and new find_objects, bckr' +# assert np.sum(_objl != objl) == 0, 'Difference between old and new find_objects, objl' +# assert np.sum(_objr != objr) == 0, 'Difference between old and new find_objects, objr' +# assert np.sum(_bckl != bckl) == 0, 'Difference between old and new find_objects, bckl' +# assert np.sum(_bckr != bckr) == 0, 'Difference between old and new find_objects, bckr' # objl, objr, bckl, bckr = new_find_objects(trcprof, bgreg, mad) else: @@ -1389,23 +1389,23 @@ def trace_slits(slf, mstrace, det, pcadesc="", maskBadRows=False, min_sqm=30.): # Assign a number to each of the edges msgs.info("Matching slit edges") - _edgearr = edgearr.copy() - t = time.clock() - _lcnt, _rcnt = arcytrace.match_edges(_edgearr, ednum) - print('Old match_edges: {0} seconds'.format(time.clock() - t)) +# _edgearr = edgearr.copy() +# t = time.clock() +# _lcnt, _rcnt = arcytrace.match_edges(_edgearr, ednum) +# print('Old match_edges: {0} seconds'.format(time.clock() - t)) __edgearr = edgearr.copy() - t = time.clock() +# t = time.clock() lcnt, rcnt = new_match_edges(__edgearr, ednum) - print('New match_edges: {0} seconds'.format(time.clock() - t)) +# print('New match_edges: {0} seconds'.format(time.clock() - t)) # print(lcnt, _lcnt, rcnt, _rcnt) # print(np.sum(__edgearr != _edgearr)) # plt.imshow(_edgearr, origin='lower', interpolation='nearest', aspect='auto') # plt.show() # plt.imshow(__edgearr, origin='lower', interpolation='nearest', aspect='auto') # plt.show() - assert np.sum(_lcnt != lcnt) == 0, 'Difference between old and new match_edges, lcnt' - assert np.sum(_rcnt != rcnt) == 0, 'Difference between old and new match_edges, rcnt' - assert np.sum(__edgearr != _edgearr) == 0, 'Difference between old and new match_edges, edgearr' +# assert np.sum(_lcnt != lcnt) == 0, 'Difference between old and new match_edges, lcnt' +# assert np.sum(_rcnt != rcnt) == 0, 'Difference between old and new match_edges, rcnt' +# assert np.sum(__edgearr != _edgearr) == 0, 'Difference between old and new match_edges, edgearr' edgearr = __edgearr if lcnt >= ednum or rcnt >= ednum: @@ -1466,8 +1466,8 @@ def trace_slits(slf, mstrace, det, pcadesc="", maskBadRows=False, min_sqm=30.): assign_slits(binarr, edgearrcp, lor=+1) if settings.argflag['trace']['slits']['maxgap'] is not None: vals = np.sort(np.unique(edgearrcp[np.where(edgearrcp != 0)])) - print('calling close_edges') - exit() +# print('calling close_edges') +# exit() hasedge = arcytrace.close_edges(edgearrcp, vals, int(settings.argflag['trace']['slits']['maxgap'])) # Find all duplicate edges edgedup = vals[np.where(hasedge == 1)] @@ -1552,14 +1552,14 @@ def trace_slits(slf, mstrace, det, pcadesc="", maskBadRows=False, min_sqm=30.): wvla = np.unique(edgearr[wdup][wdda]) wvlb = np.unique(edgearr[wdup][wddb]) # Now generate the dual edge - print('calling dual_edge') - exit() +# print('calling dual_edge') +# exit() arcytrace.dual_edge(edgearr, edgearrcp, wdup[0], wdup[1], wvla, wvlb, shadj, int(settings.argflag['trace']['slits']['maxgap']), edgedup[jj]) # Now introduce new edge locations vals = np.sort(np.unique(edgearrcp[np.where(edgearrcp != 0)])) - print('calling close_slits') - exit() +# print('calling close_slits') +# exit() edgearrcp = arcytrace.close_slits(binarr, edgearrcp, vals, int(settings.argflag['trace']['slits']['maxgap'])) # Update edgearr edgearr = edgearrcp.copy() @@ -1623,23 +1623,23 @@ def trace_slits(slf, mstrace, det, pcadesc="", maskBadRows=False, min_sqm=30.): msgs.info("Ignoring any slit that spans < {0:3.2f}x{1:d} pixels on the detector".format(settings.argflag['trace']['slits']['fracignore'], int(edgearr.shape[0]))) fracpix = int(settings.argflag['trace']['slits']['fracignore']*edgearr.shape[0]) - print('calling ignore_orders') - t = time.clock() - _edgearr = edgearr.copy() - _lnc, _lxc, _rnc, _rxc, _ldarr, _rdarr = arcytrace.ignore_orders(_edgearr, fracpix, lmin, lmax, rmin, rmax) - print('Old ignore_orders: {0} seconds'.format(time.clock() - t)) - t = time.clock() +# print('calling ignore_orders') +# t = time.clock() +# _edgearr = edgearr.copy() +# _lnc, _lxc, _rnc, _rxc, _ldarr, _rdarr = arcytrace.ignore_orders(_edgearr, fracpix, lmin, lmax, rmin, rmax) +# print('Old ignore_orders: {0} seconds'.format(time.clock() - t)) +# t = time.clock() __edgearr = edgearr.copy() lnc, lxc, rnc, rxc, ldarr, rdarr = new_ignore_orders(__edgearr, fracpix, lmin, lmax, rmin, rmax) - print('New ignore_orders: {0} seconds'.format(time.clock() - t)) - assert np.sum(_lnc != lnc) == 0, 'Difference between old and new ignore_orders, lnc' - assert np.sum(_lxc != lxc) == 0, 'Difference between old and new ignore_orders, lxc' - assert np.sum(_rnc != rnc) == 0, 'Difference between old and new ignore_orders, rnc' - assert np.sum(_rxc != rxc) == 0, 'Difference between old and new ignore_orders, rxc' - assert np.sum(_ldarr != ldarr) == 0, 'Difference between old and new ignore_orders, ldarr' - assert np.sum(_rdarr != rdarr) == 0, 'Difference between old and new ignore_orders, rdarr' - assert np.sum(__edgearr != _edgearr) == 0, 'Difference between old and new ignore_orders, edgearr' +# print('New ignore_orders: {0} seconds'.format(time.clock() - t)) +# assert np.sum(_lnc != lnc) == 0, 'Difference between old and new ignore_orders, lnc' +# assert np.sum(_lxc != lxc) == 0, 'Difference between old and new ignore_orders, lxc' +# assert np.sum(_rnc != rnc) == 0, 'Difference between old and new ignore_orders, rnc' +# assert np.sum(_rxc != rxc) == 0, 'Difference between old and new ignore_orders, rxc' +# assert np.sum(_ldarr != ldarr) == 0, 'Difference between old and new ignore_orders, ldarr' +# assert np.sum(_rdarr != rdarr) == 0, 'Difference between old and new ignore_orders, rdarr' +# assert np.sum(__edgearr != _edgearr) == 0, 'Difference between old and new ignore_orders, edgearr' edgearr = __edgearr @@ -1785,13 +1785,13 @@ def trace_slits(slf, mstrace, det, pcadesc="", maskBadRows=False, min_sqm=30.): if mnvalp > mnvalm: lvp = (arutils.func_val(lcoeff[:, lval+1-lmin], xv, settings.argflag['trace']['slits']['function'], minv=minvf, maxv=maxvf)+0.5).astype(np.int) - t = time.clock() - _edgbtwn = arcytrace.find_between(edgearr, lv, lvp, 1) - print('Old find_between: {0} seconds'.format(time.clock() - t)) - t = time.clock() +# t = time.clock() +# _edgbtwn = arcytrace.find_between(edgearr, lv, lvp, 1) +# print('Old find_between: {0} seconds'.format(time.clock() - t)) +# t = time.clock() edgbtwn = new_find_between(edgearr, lv, lvp, 1) - print('New find_between: {0} seconds'.format(time.clock() - t)) - assert np.sum(_edgbtwn != edgbtwn) == 0, 'Difference between old and new find_between' +# print('New find_between: {0} seconds'.format(time.clock() - t)) +# assert np.sum(_edgbtwn != edgbtwn) == 0, 'Difference between old and new find_between' # edgbtwn is a 3 element array that determines what is between two adjacent left edges # edgbtwn[0] is the next right order along, from left order lval @@ -1806,13 +1806,13 @@ def trace_slits(slf, mstrace, det, pcadesc="", maskBadRows=False, min_sqm=30.): else: lvp = (arutils.func_val(lcoeff[:, lval-1-lmin], xv, settings.argflag['trace']['slits']['function'], minv=minvf, maxv=maxvf)+0.5).astype(np.int) - t = time.clock() - _edgbtwn = arcytrace.find_between(edgearr, lvp, lv, -1) - print('Old find_between: {0} seconds'.format(time.clock() - t)) - t = time.clock() +# t = time.clock() +# _edgbtwn = arcytrace.find_between(edgearr, lvp, lv, -1) +# print('Old find_between: {0} seconds'.format(time.clock() - t)) +# t = time.clock() edgbtwn = new_find_between(edgearr, lvp, lv, -1) - assert np.sum(_edgbtwn != edgbtwn) == 0, 'Difference between old and new find_between' - print('New find_between: {0} seconds'.format(time.clock() - t)) +# assert np.sum(_edgbtwn != edgbtwn) == 0, 'Difference between old and new find_between' +# print('New find_between: {0} seconds'.format(time.clock() - t)) if edgbtwn[0] == -1 and edgbtwn[1] == -1: rsub = edgbtwn[2]-(lval-1) # There's an order overlap @@ -2131,19 +2131,19 @@ def refine_traces(binarr, outpar, extrap_cent, extrap_diff, extord, orders, nxord = phys_to_pix(extrap_cent[:,-i], locations, 1) # Minimum counts between loord and hiord - print('calling minbetween') - t = time.clock() - _minarrL = arcytrace.minbetween(binarr, loord, hiord) - _minarrR = arcytrace.minbetween(binarr, hiord, nxord) - print('Old minbetween: {0} seconds'.format(time.clock() - t)) - t = time.clock() +# print('calling minbetween') +# t = time.clock() +# _minarrL = arcytrace.minbetween(binarr, loord, hiord) +# _minarrR = arcytrace.minbetween(binarr, hiord, nxord) +# print('Old minbetween: {0} seconds'.format(time.clock() - t)) +# t = time.clock() minarrL = new_minbetween(binarr, loord, hiord) minarrR = new_minbetween(binarr, hiord, nxord) - print('New minbetween: {0} seconds'.format(time.clock() - t)) - assert np.sum(_minarrL != minarrL) == 0, \ - 'Difference between old and new minbetween, minarrL' - assert np.sum(_minarrR != minarrR) == 0, \ - 'Difference between old and new minbetween, minarrR' +# print('New minbetween: {0} seconds'.format(time.clock() - t)) +# assert np.sum(_minarrL != minarrL) == 0, \ +# 'Difference between old and new minbetween, minarrL' +# assert np.sum(_minarrR != minarrR) == 0, \ +# 'Difference between old and new minbetween, minarrR' minarr = 0.5*(minarrL+minarrR) srchz = np.abs(extfit[:,-i]-extfit[:,-i-1])/3.0 @@ -2151,14 +2151,14 @@ def refine_traces(binarr, outpar, extrap_cent, extrap_diff, extord, orders, numsrch = np.int(np.max(np.round(2.0*srchz-extrap_diff[:,-i]))) diffarr = np.round(extrap_diff[:,-i]).astype(np.int) - print('calling find_shift') - t = time.clock() - _shift = arcytrace.find_shift(binarr, minarr, lopos, diffarr, numsrch) - print('Old find_shift: {0} seconds'.format(time.clock() - t)) - t = time.clock() +# print('calling find_shift') +# t = time.clock() +# _shift = arcytrace.find_shift(binarr, minarr, lopos, diffarr, numsrch) +# print('Old find_shift: {0} seconds'.format(time.clock() - t)) +# t = time.clock() shift = new_find_shift(binarr, minarr, lopos, diffarr, numsrch) - print('New find_shift: {0} seconds'.format(time.clock() - t)) - assert np.sum(_shift != shift) == 0, 'Difference between old and new find_shift, shift' +# print('New find_shift: {0} seconds'.format(time.clock() - t)) +# assert np.sum(_shift != shift) == 0, 'Difference between old and new find_shift, shift' relshift = np.mean(shift+extrap_diff[:,-i]/2-srchz) if shift == -1: @@ -2182,28 +2182,28 @@ def refine_traces(binarr, outpar, extrap_cent, extrap_diff, extord, orders, hiord = loord loord = phys_to_pix(extfit[:,i], locations, 1) - print('calling minbetween') - t = time.clock() - _minarr = arcytrace.minbetween(binarr,loord, hiord) - print('Old minbetween: {0} seconds'.format(time.clock() - t)) - t = time.clock() +# print('calling minbetween') +# t = time.clock() +# _minarr = arcytrace.minbetween(binarr,loord, hiord) +# print('Old minbetween: {0} seconds'.format(time.clock() - t)) +# t = time.clock() minarr = new_minbetween(binarr,loord, hiord) - print('New minbetween: {0} seconds'.format(time.clock() - t)) - assert np.sum(_minarr != minarr) == 0, 'Difference between old and new minbetween, minarr' +# print('New minbetween: {0} seconds'.format(time.clock() - t)) +# assert np.sum(_minarr != minarr) == 0, 'Difference between old and new minbetween, minarr' srchz = np.abs(extfit[:,i]-extfit[:,i-1])/3.0 lopos = phys_to_pix(extfit[:,i-1]-srchz, locations, 1) numsrch = np.int(np.max(np.round(2.0*srchz-extrap_diff[:,i-1]))) diffarr = np.round(extrap_diff[:,i-1]).astype(np.int) - print('calling find_shift') - t = time.clock() - _shift = arcytrace.find_shift(binarr, minarr, lopos, diffarr, numsrch) - print('Old find_shift: {0} seconds'.format(time.clock() - t)) - t = time.clock() +# print('calling find_shift') +# t = time.clock() +# _shift = arcytrace.find_shift(binarr, minarr, lopos, diffarr, numsrch) +# print('Old find_shift: {0} seconds'.format(time.clock() - t)) +# t = time.clock() shift = new_find_shift(binarr, minarr, lopos, diffarr, numsrch) - print('New find_shift: {0} seconds'.format(time.clock() - t)) - assert np.sum(_shift != shift) == 0, 'Difference between old and new find_shift, shift' +# print('New find_shift: {0} seconds'.format(time.clock() - t)) +# assert np.sum(_shift != shift) == 0, 'Difference between old and new find_shift, shift' relshift = np.mean(shift+extrap_diff[:,i-1]/2-srchz) if shift == -1: @@ -2805,16 +2805,16 @@ def echelle_tilt(slf, msarc, det, pcadesc="PCA trace of the spectral tilts", mas tilts = np.zeros_like(slf._lordloc) # Generate tilts image - print('calling tilts_image') - t = time.clock() - _tiltsimg = arcytrace.tilts_image(tilts, slf._lordloc[det-1], slf._rordloc[det-1], - settings.argflag['trace']['slits']['pad'], msarc.shape[1]) - print('Old tilts_image: {0} seconds'.format(time.clock() - t)) - t = time.clock() +# print('calling tilts_image') +# t = time.clock() +# _tiltsimg = arcytrace.tilts_image(tilts, slf._lordloc[det-1], slf._rordloc[det-1], +# settings.argflag['trace']['slits']['pad'], msarc.shape[1]) +# print('Old tilts_image: {0} seconds'.format(time.clock() - t)) +# t = time.clock() tiltsimg = new_tilts_image(tilts, slf._lordloc[det-1], slf._rordloc[det-1], settings.argflag['trace']['slits']['pad'], msarc.shape[1]) - print('New tilts_image: {0} seconds'.format(time.clock() - t)) - assert np.sum(_tiltsimg != tiltsimg) == 0, 'Difference between old and new tilts_image' +# print('New tilts_image: {0} seconds'.format(time.clock() - t)) +# assert np.sum(_tiltsimg != tiltsimg) == 0, 'Difference between old and new tilts_image' return tiltsimg, satmask, outpar @@ -3158,39 +3158,39 @@ def get_censpec(slf, frame, det, gen_satmask=False): ordwid = 0.5*np.abs(slf._lordloc[det-1]-slf._rordloc[det-1]) if gen_satmask: msgs.info("Generating a mask of arc line saturation streaks") - t = time.clock() - _satmask = arcyarc.saturation_mask(frame, - settings.spect[dnum]['saturation']*settings.spect[dnum]['nonlinear']) - print('Old saturation_mask: {0} seconds'.format(time.clock() - t)) +# t = time.clock() +# _satmask = arcyarc.saturation_mask(frame, +# settings.spect[dnum]['saturation']*settings.spect[dnum]['nonlinear']) +# print('Old saturation_mask: {0} seconds'.format(time.clock() - t)) satmask = ararc.new_saturation_mask(frame, settings.spect[dnum]['saturation']*settings.spect[dnum]['nonlinear']) - print('New saturation_mask: {0} seconds'.format(time.clock() - t)) - # Allow for minor differences - if np.sum(_satmask != satmask) > 0.1*np.prod(satmask.shape): - plt.imshow(_satmask, origin='lower', interpolation='nearest', aspect='auto') - plt.colorbar() - plt.show() - plt.imshow(satmask, origin='lower', interpolation='nearest', aspect='auto') - plt.colorbar() - plt.show() - plt.imshow(_satmask-satmask, origin='lower', interpolation='nearest', aspect='auto') - plt.colorbar() - plt.show() - - assert np.sum(_satmask != satmask) < 0.1*np.prod(satmask.shape), \ - 'Old and new saturation_mask are too different' - - print('calling order saturation') - t = time.clock() - _satsnd = arcyarc.order_saturation(satmask, (ordcen+0.5).astype(int), - (ordwid+0.5).astype(int)) - print('Old order_saturation: {0} seconds'.format(time.clock() - t)) - t = time.clock() +# print('New saturation_mask: {0} seconds'.format(time.clock() - t)) +# # Allow for minor differences +# if np.sum(_satmask != satmask) > 0.1*np.prod(satmask.shape): +# plt.imshow(_satmask, origin='lower', interpolation='nearest', aspect='auto') +# plt.colorbar() +# plt.show() +# plt.imshow(satmask, origin='lower', interpolation='nearest', aspect='auto') +# plt.colorbar() +# plt.show() +# plt.imshow(_satmask-satmask, origin='lower', interpolation='nearest', aspect='auto') +# plt.colorbar() +# plt.show() +# +# assert np.sum(_satmask != satmask) < 0.1*np.prod(satmask.shape), \ +# 'Old and new saturation_mask are too different' + +# print('calling order saturation') +# t = time.clock() +# _satsnd = arcyarc.order_saturation(satmask, (ordcen+0.5).astype(int), +# (ordwid+0.5).astype(int)) +# print('Old order_saturation: {0} seconds'.format(time.clock() - t)) +# t = time.clock() satsnd = ararc.new_order_saturation(satmask, (ordcen+0.5).astype(int), (ordwid+0.5).astype(int)) - print('New order_saturation: {0} seconds'.format(time.clock() - t)) - assert np.sum(_satsnd != satsnd) == 0, \ - 'Difference between old and new order_saturation, satsnd' +# print('New order_saturation: {0} seconds'.format(time.clock() - t)) +# assert np.sum(_satsnd != satsnd) == 0, \ +# 'Difference between old and new order_saturation, satsnd' # Extract a rough spectrum of the arc in each slit msgs.info("Extracting an approximate arc spectrum at the centre of each slit") @@ -3314,15 +3314,15 @@ def phys_to_pix(array, pixlocn, axis): diff = pixlocn[:,0,0] if axis == 0 else pixlocn[0,:,1] - print('calling phys_to_pix') - t = time.clock() - _pixarr = arcytrace.phys_to_pix(np.array([array]).T, diff).flatten() \ - if len(np.shape(array)) == 1 else arcytrace.phys_to_pix(array, diff) - print('Old phys_to_pix: {0} seconds'.format(time.clock() - t)) - t = time.clock() +# print('calling phys_to_pix') +# t = time.clock() +# _pixarr = arcytrace.phys_to_pix(np.array([array]).T, diff).flatten() \ +# if len(np.shape(array)) == 1 else arcytrace.phys_to_pix(array, diff) +# print('Old phys_to_pix: {0} seconds'.format(time.clock() - t)) +# t = time.clock() pixarr = new_phys_to_pix(array, diff) - print('New phys_to_pix: {0} seconds'.format(time.clock() - t)) - assert np.sum(_pixarr != pixarr) == 0, 'Difference between old and new phys_to_pix, pixarr' +# print('New phys_to_pix: {0} seconds'.format(time.clock() - t)) +# assert np.sum(_pixarr != pixarr) == 0, 'Difference between old and new phys_to_pix, pixarr' return pixarr From 11c83a8273907fad2ddfd37c04d45b499e577924 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Thu, 22 Mar 2018 11:41:16 -0700 Subject: [PATCH 05/36] remove print statements in arextract.py --- pypit/arextract.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pypit/arextract.py b/pypit/arextract.py index fdd780df7c..4b7a392164 100644 --- a/pypit/arextract.py +++ b/pypit/arextract.py @@ -93,15 +93,15 @@ def boxcar(slf, det, specobjs, sciframe, varframe, skyframe, crmask, scitrace): mask_sci = np.ma.array(sciframe, mask=bg_mask, fill_value=0.) clip_image = sigma_clip(mask_sci, axis=1, sigma=3.) # For the mask only # Fit - print('calling func2d_fit_val') - t = time.clock() - _bgframe = arcyutils.func2d_fit_val(bgfit, sciframe, - (~clip_image.mask)*bckreg*cr_mask, bgfitord) - print('Old func2d_fit_val: {0} seconds'.format(time.clock() - t)) - t = time.clock() +# print('calling func2d_fit_val') +# t = time.clock() +# _bgframe = arcyutils.func2d_fit_val(bgfit, sciframe, +# (~clip_image.mask)*bckreg*cr_mask, bgfitord) +# print('Old func2d_fit_val: {0} seconds'.format(time.clock() - t)) +# t = time.clock() bgframe = new_func2d_fit_val(sciframe, bgfitord, x=bgfit, w=(~clip_image.mask)*bckreg*cr_mask) - print('New func2d_fit_val: {0} seconds'.format(time.clock() - t)) +# print('New func2d_fit_val: {0} seconds'.format(time.clock() - t)) # Some fits are really wonky ... in both methods # if np.sum(bgframe != _bgframe) != 0: # plt.imshow(np.ma.log10(sciframe), origin='lower', interpolation='nearest', From 2080274e37f50996c04eab517672d60edc17c0fe Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Thu, 22 Mar 2018 11:44:27 -0700 Subject: [PATCH 06/36] remove print statements in arproc.py --- pypit/arproc.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pypit/arproc.py b/pypit/arproc.py index 1d1111d62a..e6588b5bb1 100644 --- a/pypit/arproc.py +++ b/pypit/arproc.py @@ -1788,14 +1788,14 @@ def lacosmic(slf, fitsdict, det, sciframe, scidx, maxiter=1, grow=1.5, maskval=- # Old cr_screen can yield nan pixels, new one does not, meaning that # there are differences between the returned arrays. For all # non-nan pixels, the two algorithms are identical. - print('calling cr_screen') - t = time.clock() - _sigimg = arcyproc.cr_screen(filty,0.0) - print('Old cr_screen: {0} seconds'.format(time.clock() - t)) +# print('calling cr_screen') +# t = time.clock() +# _sigimg = arcyproc.cr_screen(filty,0.0) +# print('Old cr_screen: {0} seconds'.format(time.clock() - t)) # print(np.sum(np.invert(np.isfinite(_sigimg)))) - t = time.clock() +# t = time.clock() sigimg = new_cr_screen(filty) - print('New cr_screen: {0} seconds'.format(time.clock() - t)) +# print('New cr_screen: {0} seconds'.format(time.clock() - t)) # print(np.sum(np.invert(np.isfinite(sigimg)))) # if np.sum(_sigimg != sigimg) != 0: # plt.imshow(_sigimg, origin='lower', interpolation='nearest', aspect='auto') From 807c84027e4d4b6f08724e7e7a9088e39b242696 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Thu, 22 Mar 2018 14:49:58 -0700 Subject: [PATCH 07/36] cleaned up filter.py --- pypit/filter.py | 342 +----------------------------------------------- 1 file changed, 6 insertions(+), 336 deletions(-) diff --git a/pypit/filter.py b/pypit/filter.py index 59e3e17fca..d69ba3be3e 100644 --- a/pypit/filter.py +++ b/pypit/filter.py @@ -1,35 +1,9 @@ -# Licensed under a 3-clause BSD style license - see LICENSE.rst # -*- coding: utf-8 -*- """ A set of functions used to filter arrays. -*License*: - Copyright (c) 2017, SDSS-IV/MaNGA Pipeline Group - Licensed under BSD 3-clause license - see LICENSE.rst - -*Source location*: - $MANGADAP_DIR/python/mangadap/util/filter.py - -*Imports and python version compliance*: - :: - - from __future__ import division - from __future__ import print_function - from __future__ import absolute_import - from __future__ import unicode_literals - - import sys - if sys.version > '3': - long = int - - import numpy - -*Revision history*: - | **26 Jan 2017**: Original implementation by K. Westfall (KBW) - | **06 Apr 2017**: Interpolate sigma vectors as well as the smoothed - vectors in :class:`BoxcarFilter`. - | **21 Dec 2017**: Add weighting functionality to - :class:`BoxcarFilter`. +Originally pulled from SDSS-IV/MaNGA Data Analysis Pipeline, licensed +under BSD 3-clause license. """ from __future__ import division @@ -45,173 +19,8 @@ import numpy from scipy import interpolate, sparse -# Debug -from matplotlib import pyplot -from matplotlib.ticker import NullFormatter - -def high_pass_filter(flux, dw=0, k=None, Dw=None): - """ - Pulled from FIREFLY's hpf() function and edited on 17 Nov 2016. - - Apply a high-pass filter to the input vector. - - """ - - n = flux.size - _dw = int(dw) - - if k is None and Dw is None: - Dw = 100 - _k = n//Dw - elif k is not None and Dw is None: - _k = int(k) - Dw = n//_k - elif k is None and Dw is not None: - _k = n//Dw - else: - raise ValueError('Cannot provide both k and Dw.') - - # PRINT A REPORT - - # Rita's typical inputs for SDSS: - # w = 10 # 10 - # windowsize = 20 # 20 - - # My MaNGA inputs: - # if w == 0 and windowsize == 0: - # print "HPF parameters not specified: choosing default window (stepsize=40)" - # w = 40 - # windowsize = 0 - - h = numpy.fft.fft(flux) - h_filtered = numpy.zeros(n, dtype=complex) - window = numpy.zeros(n) - unwindow = numpy.zeros(n) - window[0] = 1 # keep F0 for normalisation - unwindow[0] = 1 - - for i in range(_dw): - window[_k+i] = (i+1.0)/_dw - window[n-1-(_k+_dw-i)] = (_dw-i)/_dw - window[_k+_dw:n-(_k+_dw)] = 1 - - unwindow = 1 - window - unwindow[0] = 1 - - h_filtered = h * window - un_h_filtered = h * unwindow - - res = numpy.real(numpy.fft.ifft(h_filtered)) - unres = numpy.real(numpy.fft.ifft(un_h_filtered)) - res_out = (1.0+(res-numpy.median(res))/unres) * numpy.median(res) - - return res_out, window, res, unres - -#def off_diagonal_identity(size, win): -# r""" -# Construct a matrix with ones within a window along the diagonal. -# -# Args: -# size (int) : Size for the square matrix; i.e., :math:`N` for the -# :math:`N\timesN` matrix. -# win (int): Number of ones in each row along the diagonal. -# -# Raises: -# ValueError: Raised if the window is larger than 2*size-1. -# -# """ -# if win > 2*size-1: -# raise ValueError('Window too large for matrix size.') -# if win == 2*size-1: -# return numpy.ones((size,size), dtype=int) -# x = numpy.zeros((size,size), dtype=int)#numpy.identity(size).astype(int) -# for i in range(1,(win+1)//2): -# x[:-i,i:] = x[:-i,i:] + numpy.identity(size-i).astype(int) -# x += x.T -# if win % 2 != 1: -# x[:-i-1,i+1:] = x[:-i-1,i+1:] + numpy.identity(size-i-1).astype(int) -# return x + numpy.identity(size).astype(int) - -def off_diagonal_identity(size, win, return_sparse=False): - r""" - Construct a matrix with ones within a window along the diagonal. - - Args: - size (int) : Size for the square matrix; i.e., :math:`N` for the - :math:`N\timesN` matrix. - win (int): Number of ones in each row along the diagonal. - - Raises: - ValueError: Raised if the window is larger than 2*size-1. - - """ - if win > 2*size-1: - raise ValueError('Window too large for matrix size.') - if win == 2*size-1: - return numpy.ones((size,size), dtype=int) - - # Indices of diagonal - ii = numpy.arange(size).astype(int) - - # Build the upper triangle - i = numpy.empty(0, dtype=int) - j = numpy.empty(0, dtype=int) - for k in range(1,(win+1)//2): - i = numpy.append(i, ii[:size-k]) - j = numpy.append(j, ii[k:size]) - - # Copy to the lower triangle - _i = numpy.append(i,j) - j = numpy.append(j,i) - - # Add the diagonal - i = numpy.append(_i, ii) - j = numpy.append(j, ii) - - # Accommodate an even window - if win % 2 != 1: - i = numpy.append(i, ii[:size-k-1]) - j = numpy.append(j, ii[k+1:size]) - - # Construct and return the array - if return_sparse: - return sparse.coo_matrix((numpy.ones(len(i), dtype=int),(i,j)), shape=(size,size)).tocsr() - - a = numpy.zeros((size,size), dtype=int) - a[i,j] = 1 - return a - - -def build_smoothing_mask(x, pix_buffer, default=None, mask_x=None): - r""" - Construct a mask for the provided vector that masks the first and - last pix_buffer pixels in the coordinate vector. - - The mask is instantiated as fully unmasked, unless a default mask is - provided. - - To mask specified ranges in x, provide mask_x with a shape - :math:`N_{\rm mask}\times 2` where each mask is defined by the - starting and ending value of x to exclude. - """ - # Smooth the ratio of the data to the model - if len(x.shape) != 1: - raise ValueError('Input must be a vector.') - npix = x.size - mask = numpy.zeros(npix, dtype=bool) if default is None else default.copy() - if mask.size != npix: - raise ValueError('Provided default has an incorrect length.') - mask[:pix_buffer] = True - mask[-pix_buffer:] = True - if mask_x is None: - return mask - - for m in mask_x: - mask |= numpy.logical_and(x > m[0], x < m[1]) - return mask - - -class BoxcarFilter(): +class BoxcarFilter(object): + """Boxcar filter a 2D image along columns.""" def __init__(self, boxcar, lo=None, hi=None, niter=None, y=None, wgt=None, mask=None, local_sigma=None): @@ -240,7 +49,6 @@ def __init__(self, boxcar, lo=None, hi=None, niter=None, y=None, wgt=None, mask= self.smooth(y, wgt=wgt, mask=mask, boxcar=boxcar, lo=lo, hi=hi, niter=niter, local_sigma=local_sigma) - def _assign_par(self, boxcar, lo, hi, niter, local_sigma): if boxcar is not None: self.boxcar = boxcar @@ -253,7 +61,6 @@ def _assign_par(self, boxcar, lo, hi, niter, local_sigma): if local_sigma is not None: self.local_sigma = local_sigma - def _check_par(self): if self.boxcar <= 1: raise ValueError('Boxcar must be greater than 1!') @@ -261,7 +68,6 @@ def _check_par(self): if self.nrej is not None and self.nrej > 0 and self.lo_rej is None and self.hi_rej is None: raise ValueError('Must provide lo or hi if niter provided') - def _reduce_indices(self): indx = numpy.array([numpy.arange(self.npix)-self.boxcar//2, numpy.arange(self.npix)+self.boxcar//2+1]) @@ -269,10 +75,9 @@ def _reduce_indices(self): indx[:self.npix] += 1 return indx.T.ravel().clip(0,self.npix-1) - def _apply(self): - self.smoothed_n = numpy.add.reduceat(numpy.invert(self.output_mask), self.rdx_index, axis=1, - dtype=int)[:,::2] + self.smoothed_n = numpy.add.reduceat(numpy.invert(self.output_mask), self.rdx_index, + axis=1, dtype=int)[:,::2] self.smoothed_wgt = numpy.add.reduceat(self.input_wgt, self.rdx_index, axis=1, dtype=float)[:,::2] self.smoothed_y = numpy.add.reduceat(self.input_wgt*self.y.filled(0.0), self.rdx_index, @@ -296,42 +101,6 @@ def _apply(self): self.smoothed_y[self.output_mask | (self.smoothed_n == 0)] = numpy.ma.masked self.sigma_y[self.output_mask | (self.smoothed_n == 0)] = numpy.ma.masked - return - - w,h = pyplot.figaspect(1) - fig = pyplot.figure(figsize=(1.5*w,1.5*h)) - - minf = numpy.amin( numpy.ma.log10(numpy.append(self.y.compressed(), self.smoothed_y.compressed())) ) - maxf = numpy.amax( numpy.ma.log10(numpy.append(self.y.compressed(), self.smoothed_y.compressed())) ) - - mins = numpy.amin(numpy.ma.log10(self.sigma_y).compressed()) - maxs = numpy.amax(numpy.ma.log10(self.sigma_y).compressed()) - - minr = numpy.amin(numpy.ma.log10(numpy.ma.divide(self.y, self.smoothed_y)).compressed()) - maxr = numpy.amax(numpy.ma.log10(numpy.ma.divide(self.y, self.smoothed_y)).compressed()) - - ax = fig.add_axes([0.05, 0.50, 0.45, 0.45]) - ax.xaxis.set_major_formatter(NullFormatter()) - ax.imshow(numpy.ma.log10(self.y), origin='lower', interpolation='nearest', vmin=minf, - vmax=maxf, aspect='auto') - ax = fig.add_axes([0.50, 0.50, 0.45, 0.45]) - ax.xaxis.set_major_formatter(NullFormatter()) - ax.yaxis.set_major_formatter(NullFormatter()) - ax.imshow(numpy.ma.log10(self.smoothed_y), origin='lower', interpolation='nearest', - vmin=minf, vmax=maxf, aspect='auto') - ax = fig.add_axes([0.05, 0.05, 0.45, 0.45]) - ax.yaxis.set_major_formatter(NullFormatter()) - ax.imshow(numpy.ma.log10(numpy.ma.divide(self.y,self.smoothed_y)), origin='lower', - interpolation='nearest', aspect='auto', vmin=minr, vmax=maxr) - ax = fig.add_axes([0.50, 0.05, 0.45, 0.45]) - ax.yaxis.set_major_formatter(NullFormatter()) - ax.imshow(numpy.ma.log10(self.sigma_y), origin='lower', interpolation='nearest', - aspect='auto', vmin=mins, vmax=maxs) - - pyplot.show() - exit() - - def _interpolate(self): """ Interpolate the smoothed image across the masked regions. @@ -339,7 +108,6 @@ def _interpolate(self): first and last unmasked value, respectively. interpolate both the smoothed data and the sigma - """ # Boolean arrays with bad pixels (should be the same for both # smoothed_y and sigma_y) @@ -379,7 +147,6 @@ def _interpolate(self): self.sigma_y.ravel()[badpix.ravel()] = interpolator(x.data[badpix.ravel()]) self.sigma_y[numpy.invert(goodvec),:] = 0.0 - def smooth(self, y, wgt=None, mask=None, boxcar=None, lo=None, hi=None, niter=None, local_sigma=None): """ @@ -442,7 +209,6 @@ def smooth(self, y, wgt=None, mask=None, boxcar=None, lo=None, hi=None, niter=No self._apply() nmasked = numpy.sum(self.output_mask) - nbad -# print('Niter: {0}; Nmasked: {1}'.format(i+1, nmasked)) i += 1 if i == self.nrej or nmasked == 0: @@ -451,103 +217,7 @@ def smooth(self, y, wgt=None, mask=None, boxcar=None, lo=None, hi=None, niter=No self._interpolate() return self.smoothed_y[0,:] if provided_vector else self.smoothed_y - @property def mask(self): return self.output_mask.copy() - - -def smooth_masked_vector(x, nsmooth): - """ - Smooth a masked vector by a box of size nsmooth. - """ - n = off_diagonal_identity(x.size, nsmooth)*numpy.invert(numpy.ma.getmaskarray(x))[None,:] - nn = numpy.sum(n,axis=1) - return numpy.ma.MaskedArray(numpy.dot(n,x.data), mask=numpy.invert(nn>0) | x.mask)/nn - - -def interpolate_masked_vector(y, quiet=True, extrap_with_median=False): - """ - Interpolate over the masked pixels in an input vector using linear - interpolation. - """ - x = numpy.arange(y.size) - indx = numpy.ma.getmaskarray(y) - if numpy.sum(numpy.invert(indx)) < 2: - if not quiet: - warnings.warn('Input vector has fewer than 2 unmasked values! Returning zero vector.') - return numpy.zeros(y.size, dtype=y.dtype.name) - interpolator = interpolate.interp1d(x[numpy.invert(indx)], y[numpy.invert(indx)], - fill_value='extrapolate') - _y = y.data.copy() - _y[indx] = interpolator(x[indx]) - - if extrap_with_median: - med = numpy.median(interpolator.y) - m_indx = (x[indx] < interpolator.x[0]) | (x[indx] > interpolator.x[-1]) - _y[indx][m_indx] = med - - return _y - - -def boxcar_smooth_vector(x, boxcar, mask=None, lo=None, hi=None, niter=None, return_mask=False): - """ - Boxcar smooth an input vector. - - Ignores masked pixels. - - Allows for iterative positive and negative outliers. - - Can return the mask separately from the smoothed vector. - """ - if niter is not None and lo is None and hi is None: - raise ValueError('Must provide lo or hi if niter provided') - - _mask = numpy.zeros(x.size, dtype=bool) if mask is None else mask - _x = x if isinstance(x, numpy.ma.MaskedArray) else numpy.ma.MaskedArray(x) - _x[_mask] = numpy.ma.masked - - sx = interpolate_masked_vector( smooth_masked_vector(_x, boxcar) ) - -# pyplot.step(numpy.arange(_x.size), _x.data, where='mid', color='0.3', lw=0.5) -# pyplot.step(numpy.arange(_x.size), _x, where='mid', color='k', lw=0.5) -# pyplot.plot(numpy.arange(_x.size), sx, color='r', lw=1) -# pyplot.show() - - if numpy.all([ x is None for x in [ lo, hi, niter ]]): - if return_mask: - return sx, numpy.ma.getmaskarray(_x).copy() - return sx - - _niter = -1 if niter is None else niter - nrej = numpy.sum(numpy.ma.getmaskarray(_x)) - i=0 - while i > -1: - sig = numpy.std((_x - sx).compressed()) - mask = numpy.ma.getmaskarray(_x).copy() - if lo is not None: - mask = numpy.logical_or(mask, _x.data - sx < -lo*sig) - if hi is not None: - mask = numpy.logical_or(mask, _x.data - sx > hi*sig) - nrej = numpy.sum(mask) - numpy.sum(_x.mask) -# print('Niter: {0}; Nrej: {1}'.format(i+1, nrej)) - _x[mask] = numpy.ma.masked - sx = interpolate_masked_vector( smooth_masked_vector(_x, boxcar) ) - -# pyplot.step(numpy.arange(_x.size), _x.data, where='mid', color='cyan', lw=0.5) -# pyplot.step(numpy.arange(_x.size), _x, where='mid', color='k', lw=0.5) -# pyplot.plot(numpy.arange(_x.size), sx, color='r', lw=1) -# pyplot.show() - - i += 1 - if i == _niter or nrej == 0: - break - - pyplot.step(numpy.arange(_x.size), _x.data, where='mid', color='cyan', lw=0.5) - pyplot.step(numpy.arange(_x.size), _x, where='mid', color='k', lw=0.5) - pyplot.plot(numpy.arange(_x.size), sx, color='r', lw=1) - pyplot.show() - - if return_mask: - return sx, numpy.ma.getmaskarray(_x).copy() - return sx - - From b0a121e6d5b66efa45c9d0804b394035684e37d1 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Fri, 23 Mar 2018 12:15:20 -0700 Subject: [PATCH 08/36] move msgs to __init__.py; some restructuring and doc changes --- CHANGES.rst | 3 + pypit/__init__.py | 45 +++ pypit/ararc.py | 5 +- pypit/ararclines.py | 5 +- pypit/arcoadd.py | 5 +- pypit/arcomb.py | 5 +- pypit/ardebug.py | 7 +- pypit/ardevtest.py | 5 +- pypit/arextract.py | 5 +- pypit/arflux.py | 5 +- pypit/arload.py | 5 +- pypit/arlris.py | 5 +- pypit/armasters.py | 5 +- pypit/armbase.py | 5 +- pypit/armed.py | 5 +- pypit/armlsd.py | 5 +- pypit/armsgs.py | 515 ++++++++++++++++++----------------- pypit/arparse.py | 5 +- pypit/arpca.py | 12 +- pypit/arproc.py | 5 +- pypit/arqa.py | 47 ++-- pypit/arsave.py | 5 +- pypit/arsciexp.py | 5 +- pypit/arsort.py | 5 +- pypit/arspecobj.py | 5 +- pypit/artrace.py | 5 +- pypit/arutils.py | 5 +- pypit/arwave.py | 7 +- pypit/ginga.py | 5 +- pypit/pypit.py | 166 +++++------ pypit/pyputils.py | 84 +++--- pypit/scripts/run_pypit.py | 92 ++++--- pypit/tests/test_pyputils.py | 4 +- 33 files changed, 619 insertions(+), 473 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index a6641ee203..50f309a2b5 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -32,6 +32,9 @@ * Added functionality for fully automated wavelength calibration with arclines * Switched settings files to allow IRAF style data sections to be defined * Allowed data sections to be extracted from header information +* Removed majority of cython functionality +* Moved logging to be a package object using the main __init__.py file +* Begin to adhere to PEP8 (mostly) 0.7 (2017-02-07) ---------------- diff --git a/pypit/__init__.py b/pypit/__init__.py index e69de29bb2..26b5901186 100644 --- a/pypit/__init__.py +++ b/pypit/__init__.py @@ -0,0 +1,45 @@ +""" +pypit package initialization. + +The current main purpose of this is to provide package-level globals +that can be imported by submodules. +""" + +# Set version +__version__ = '0.8.0.dev0' +# TODO: Do we need this? A release should be tied to a date, and the +# date information could be put in the CHANGES.rst +__last_updated__ = '23Mar2018' + +# Imports for singal and log handling +import sys +import signal +import warnings + +# Import and instantiate the logger +from pypit import armsgs +msgs = armsgs.Messages() + +# Import the close_qa method so that it can be called when a hard stop +# is requested by the user +# TODO: Is this necessary? As far as I can tell, close_qa() doesn't +# actually close any already opened qa files +from pypit.arqa import close_qa + +# Send all signals to messages to be dealt with (i.e. someone hits ctrl+c) +def signal_handler(signalnum, handler): + """ + Handle signals sent by the keyboard during code execution + """ + if signalnum == 2: + msgs.info('Ctrl+C was pressed. Ending processes...') + close_qa(msgs.pypit_file) + msgs.close() + sys.exit() + +signal.signal(signal.SIGINT, signal_handler) + +# Ignore all warnings given by python +warnings.resetwarnings() +warnings.simplefilter('ignore') + diff --git a/pypit/ararc.py b/pypit/ararc.py index befa3a6a8a..48d8591b2c 100644 --- a/pypit/ararc.py +++ b/pypit/ararc.py @@ -3,7 +3,8 @@ import numpy as np from pypit import arpca from pypit import arparse as settings -from pypit import armsgs +#from pypit import armsgs +from pypit import msgs from pypit import arsave from pypit import arutils from pypit import ararclines @@ -15,7 +16,7 @@ from pypit import ardebug as debugger # Logging -msgs = armsgs.get_logger() +#msgs = armsgs.get_logger() def detect_lines(slf, det, msarc, censpec=None, MK_SATMASK=False): diff --git a/pypit/ararclines.py b/pypit/ararclines.py index 16b39354cb..c198747a2d 100644 --- a/pypit/ararclines.py +++ b/pypit/ararclines.py @@ -5,13 +5,14 @@ import glob, copy import yaml -from pypit import armsgs +#from pypit import armsgs +from pypit import msgs from pypit import arparse as settings from pypit import ardebug as debugger # Logging -msgs = armsgs.get_logger() +#msgs = armsgs.get_logger() def parse_nist(slf,ion): diff --git a/pypit/arcoadd.py b/pypit/arcoadd.py index bf46f3dbf2..c83c5b1cfb 100644 --- a/pypit/arcoadd.py +++ b/pypit/arcoadd.py @@ -11,14 +11,15 @@ from astropy.stats import sigma_clipped_stats from pypit import arload -from pypit import armsgs +#from pypit import armsgs +from pypit import msgs from pypit import arqa from pypit import arutils from pypit import ardebug as debugger # Logging -msgs = armsgs.get_logger() +#msgs = armsgs.get_logger() # TODO # Shift spectra diff --git a/pypit/arcomb.py b/pypit/arcomb.py index 80d4a30b0a..d51ed07ace 100644 --- a/pypit/arcomb.py +++ b/pypit/arcomb.py @@ -2,12 +2,13 @@ import time import numpy as np -from pypit import armsgs +#from pypit import armsgs +from pypit import msgs from pypit import arparse as settings from matplotlib import pyplot as plt # Logging -msgs = armsgs.get_logger() +#msgs = armsgs.get_logger() def comb_frames(frames_arr, det, frametype, weights=None, maskvalue=1048577, printtype=None): """ Combine several frames diff --git a/pypit/ardebug.py b/pypit/ardebug.py index 8088f9dc1d..9f1e6e4a3a 100644 --- a/pypit/ardebug.py +++ b/pypit/ardebug.py @@ -1,5 +1,8 @@ from __future__ import (print_function, absolute_import, division, unicode_literals) +import matplotlib.pyplot as plt +import numpy as np + # These need to be outside of the def's from pypit.ginga import show_image from pypit.ginga import chk_arc_tilts @@ -49,8 +52,6 @@ def plot1d(*args, **kwargs): True for a scatter plot NOTE: Any extra parameters are fed as kwargs to plt.plot() """ - import matplotlib.pyplot as plt - import numpy as np # Error checking if len(args) == 0: print('x_guis.simple_splot: No arguments!') @@ -145,4 +146,4 @@ def plot1d(*args, **kwargs): plt.show() return -from pdb import * +#from pdb import * diff --git a/pypit/ardevtest.py b/pypit/ardevtest.py index 4cd63bbec7..0257e555c6 100644 --- a/pypit/ardevtest.py +++ b/pypit/ardevtest.py @@ -5,13 +5,14 @@ import scipy import glob -from pypit import armsgs +#from pypit import armsgs +from pypit import msgs from pypit import arparse as settings from pypit import ardebug as debugger # Logging -msgs = armsgs.get_logger() +#msgs = armsgs.get_logger() def set_param(argf, specname): diff --git a/pypit/arextract.py b/pypit/arextract.py index 4b7a392164..c825173adc 100644 --- a/pypit/arextract.py +++ b/pypit/arextract.py @@ -5,14 +5,15 @@ import copy import numpy as np from astropy import units as u -from pypit import armsgs +#from pypit import armsgs +from pypit import msgs from pypit import arparse as settings from pypit import artrace from pypit import arutils from pypit import arqa # Logging -msgs = armsgs.get_logger() +#msgs = armsgs.get_logger() from pypit import ardebug as debugger diff --git a/pypit/arflux.py b/pypit/arflux.py index d348d95370..4a6e38bfef 100644 --- a/pypit/arflux.py +++ b/pypit/arflux.py @@ -11,7 +11,8 @@ from astropy import units as u from astropy import coordinates as coords -from pypit import armsgs +#from pypit import armsgs +from pypit import msgs from pypit import arparse as settings try: @@ -22,7 +23,7 @@ from pypit import ardebug as debugger # Logging -msgs = armsgs.get_logger() +#msgs = armsgs.get_logger() def apply_sensfunc(slf, det, scidx, fitsdict, MAX_EXTRAP=0.05, standard=False): diff --git a/pypit/arload.py b/pypit/arload.py index ae24023e28..ec38be5d7f 100644 --- a/pypit/arload.py +++ b/pypit/arload.py @@ -6,7 +6,8 @@ from astropy.time import Time import numpy as np from pypit import arparse as settings -from pypit import armsgs +#from pypit import armsgs +from pypit import msgs from pypit import arproc from pypit import arlris #from multiprocessing import Pool as mpPool @@ -21,7 +22,7 @@ from pypit import ardebug as debugger # Logging -msgs = armsgs.get_logger() +#msgs = armsgs.get_logger() def load_headers(datlines): diff --git a/pypit/arlris.py b/pypit/arlris.py index 9598eb0a4a..08b478a0ff 100644 --- a/pypit/arlris.py +++ b/pypit/arlris.py @@ -5,7 +5,8 @@ import glob import astropy.io.fits as pyfits -from pypit import armsgs +#from pypit import armsgs +from pypit import msgs from pypit.arparse import load_sections from pypit import ardebug as debugger @@ -16,7 +17,7 @@ basestring = str # Logging -msgs = armsgs.get_logger() +#msgs = armsgs.get_logger() def read_lris(raw_file, det=None, TRIM=False): diff --git a/pypit/armasters.py b/pypit/armasters.py index af4efe9945..2f2b21938c 100644 --- a/pypit/armasters.py +++ b/pypit/armasters.py @@ -1,7 +1,8 @@ from __future__ import (print_function, absolute_import, division, unicode_literals) import numpy as np -from pypit import armsgs +#from pypit import armsgs +from pypit import msgs from pypit import arload from pypit import arparse as settings from pypit import arsave @@ -13,7 +14,7 @@ basestring = str # Logging -msgs = armsgs.get_logger() +#msgs = armsgs.get_logger() from pypit import ardebug as debugger diff --git a/pypit/armbase.py b/pypit/armbase.py index 3812319085..4c6fbe2012 100644 --- a/pypit/armbase.py +++ b/pypit/armbase.py @@ -8,13 +8,14 @@ from collections import OrderedDict from pypit import arparse as settings -from pypit import armsgs +#from pypit import armsgs +from pypit import msgs from pypit import arsort from pypit import arsciexp from pypit import arparse # Logging -msgs = armsgs.get_logger() +#msgs = armsgs.get_logger() from pypit import ardebug as debugger diff --git a/pypit/armed.py b/pypit/armed.py index cbecab8a11..2a1165802e 100644 --- a/pypit/armed.py +++ b/pypit/armed.py @@ -7,7 +7,8 @@ from pypit import arload from pypit import armasters from pypit import armbase -from pypit import armsgs +#from pypit import armsgs +from pypit import msgs from pypit import arproc from pypit import arsave from pypit import arsort @@ -17,7 +18,7 @@ from pypit import ardebug as debugger # Logging -msgs = armsgs.get_logger() +#msgs = armsgs.get_logger() def ARMED(fitsdict, reuseMaster=False, reloadMaster=True): diff --git a/pypit/armlsd.py b/pypit/armlsd.py index 847d8b5250..dc19dc5994 100644 --- a/pypit/armlsd.py +++ b/pypit/armlsd.py @@ -8,7 +8,8 @@ from pypit import arload from pypit import armasters from pypit import armbase -from pypit import armsgs +#from pypit import armsgs +from pypit import msgs from pypit import arproc from pypit import arsave from pypit import arsort @@ -20,7 +21,7 @@ from pypit import ardebug as debugger # Logging -msgs = armsgs.get_logger() +#msgs = armsgs.get_logger() def ARMLSD(fitsdict, reuseMaster=False, reloadMaster=True): diff --git a/pypit/armsgs.py b/pypit/armsgs.py index 0b51e77d7b..223da6a650 100644 --- a/pypit/armsgs.py +++ b/pypit/armsgs.py @@ -1,12 +1,29 @@ -from __future__ import absolute_import, division, print_function +""" +Module for terminal and file logging. + +.. todo:: + Why not use pythons native logging package? + +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals import sys -from os.path import dirname, basename -from textwrap import wrap as wraptext -from inspect import currentframe, getouterframes -from glob import glob +import os +import glob +import textwrap +import inspect + +# Imported for versioning +import scipy +import numpy +import astropy -pypit_logger = None +from pypit import __version__, __last_updated__ + +#pypit_logger = None class Messages: """ @@ -14,275 +31,289 @@ class Messages: For further details on colours see the following example: http://ascii-table.com/ansi-escape-sequences.php - """ - def __init__(self, log, debug, verbosity, colors=True): - """ - Initialize the Message logging class - - Parameters - ---------- - log : str or None - Name of saved log file (no log will be saved if log=="") - debug : dict - dict used for debugging. - 'LOAD', 'BIAS', 'ARC', 'TRACE' - verbosity : int (0,1,2) - Level of verbosity: - 0 = No output - 1 = Minimal output (default - suitable for the average user) - 2 = All output - colors : bool - If true, the screen output will have colors, otherwise - normal screen output will be displayed - """ - # Version - from pypit import pyputils - version, last_updated = pyputils.get_version() - # Import for version - import scipy - import numpy - import astropy + Parameters + ---------- + log : str or None + Name of saved log file (no log will be saved if log=="") + debug : dict + dict used for debugging. + 'LOAD', 'BIAS', 'ARC', 'TRACE' + verbosity : int (0,1,2) + Level of verbosity: + 0 = No output + 1 = Minimal output (default - suitable for the average user) + 2 = All output + colors : bool + If true, the screen output will have colors, otherwise + normal screen output will be displayed + """ + def __init__(self, log=None, debug=None, verbosity=None, colors=True): - # Initialize the log - if log is not None: - self._log = open(log, 'w') - else: - self._log = log # Initialize other variables + # TODO: debug could just be develop=True or False self._debug = debug - self._last_updated = last_updated - self._version = version - self._verbosity = verbosity + self._last_updated = __last_updated__ + self._version = __version__ + self._verbosity = 1 if verbosity is None else verbosity + + # TODO: Why are these two necessary? It would seem better to + # provide Messages with member functions that can operate on + # sciexp and pypit_file instead of having them kept within the + # object itself... self.sciexp = None self.pypit_file = None - # Save the version of the code including last update information to the log file - if self._log: - self._log.write("------------------------------------------------------\n\n") - self._log.write("PYPIT was last updated {0:s}\n".format(last_updated)) - self._log.write("This log was generated with version {0:s} of PYPIT\n\n".format(version)) - self._log.write("You are using scipy version={:s}\n".format(scipy.__version__)) - self._log.write("You are using numpy version={:s}\n".format(numpy.__version__)) - self._log.write("You are using astropy version={:s}\n\n".format(astropy.__version__)) - self._log.write("------------------------------------------------------\n\n") + + # Initialize the log + self._log = None + self._initialize_log_file(log=log) + # Use colors? - self._start, self._end = "", "" - self._black_CL, self._yellow_CL, self._blue_CL, self._green_CL, self._red_CL = "", "", "", "", "" - self._white_RD, self._white_GR, self._white_BK = "", "", "" - self._white_BL, self._black_YL, self._yellow_BK = "", "", "" + self._start = None + self._end = None + self._black_CL = None + self._yellow_CL = None + self._blue_CL = None + self._green_CL = None + self._red_CL = None + self._white_RD = None + self._white_GR = None + self._white_BK = None + self._white_BL = None + self._black_YL = None + self._yellow_BK = None + + self.disablecolors() if colors: self.enablecolors() + + def _cleancolors(self, msg): + cols = [self._end, self._start, + self._black_CL, self._yellow_CL, self._blue_CL, self._green_CL, self._red_CL, + self._white_RD, self._white_GR, self._white_BK, self._white_BL, + self._black_YL, self._yellow_BK] + for i in cols: + msg = msg.replace(i, '') + return msg + + def _debugmessage(self): + if self._debug is not None and self._debug['develop']: + info = inspect.getouterframes(inspect.currentframe())[2] + dbgmsg = self._start + self._blue_CL + info[1].split('/')[-1] + ' ' + str(info[2]) \ + + ' ' + info[3] + '()' + self._end + ' - ' else: - self.disablecolors() + dbgmsg = '' + return dbgmsg + + def _print(self, premsg, msg, last=True, debug=True): + """ + Print to standard error and the log file + """ + dbgmsg = self._debugmessage() if debug else '' + _msg = premsg+dbgmsg+msg + print(_msg, file=sys.stderr) + if self._log: + clean_msg = self._cleancolors(_msg) + self._log.write(clean_msg+'\n' if last else clean_msg) + + def _initialize_log_file(self, log=None): + """ + Expects self._log is already None. + """ + if log is None: + return + + # Initialize the log + self._log = open(log, 'w') + + self._log.write("------------------------------------------------------\n\n") + self._log.write("PYPIT was last updated {0:s}\n".format(__last_updated__)) + self._log.write("This log was generated with version {0:s} of PYPIT\n\n".format( + __version__)) + self._log.write("You are using scipy version={:s}\n".format(scipy.__version__)) + self._log.write("You are using numpy version={:s}\n".format(numpy.__version__)) + self._log.write("You are using astropy version={:s}\n\n".format(astropy.__version__)) + self._log.write("------------------------------------------------------\n\n") + + def reset(self, log=None, debug=None, verbosity=None, colors=True): + """ + Reinitialize the object. + + Needed so that there can be a default object for all modules, + but also a dynamically defined log file. + """ + # Initialize other variables + self._debug = debug + self._verbosity = 1 if verbosity is None else verbosity + self.reset_log_file(log) + self.disablecolors() + if colors: + self.enablecolors() + + def reset_log_file(self, log): + if self._log: + self._log.close() + self._log = None + self._initialize_log_file(log=log) # Headers and usage + # TODO: Move this to the ARMED class... def armedheader(self, prognm): """ Get the info header for ARMED """ - header = "## " - header += self._start + self._white_GR + "ARMED : " - header += "Automated Reduction and Modelling of Echelle Data v{0:s}".format(self._version) + self._end + "\n" - header += "## " - header += "Usage : " - header += "python %s [options] filelist".format(prognm) + header = '## ' + header += self._start + self._white_GR + 'ARMED : ' + header += 'Automated Reduction and Modelling of Echelle Data v{0:s}'.format( + self._version) + self._end + '\n' + header += '## ' + header += 'Usage : ' + header += 'python %s [options] filelist'.format(prognm) return header def pypitheader(self, prognm): """ Get the info header for PYPIT """ - header = "## " - header += self._start + self._white_GR + "PYPIT : " - header += "The Python Spectroscopic Data Reduction Pipeline v{0:s}".format(self._version) + self._end + "\n" - header += "## " - #header += "Usage : " - #if prognm is None: - # header += "pypit [options] filename.red" - #else: - # header += "python %s [options] filename.red".format(prognm) + header = '## ' + header += self._start + self._white_GR + 'PYPIT : ' + header += 'The Python Spectroscopic Data Reduction Pipeline v{0:s}'.format( + self._version) + self._end + '\n' + header += '## ' return header def usage(self, prognm): - stgs_arm = glob(dirname(__file__)+"/data/settings/settings.arm*") - stgs_all = glob(dirname(__file__)+"/data/settings/settings.*") + """ + Print pypit usage data. + """ + stgs_arm = glob.glob(os.path.dirname(__file__)+'/data/settings/settings.arm*') + stgs_all = glob.glob(os.path.dirname(__file__)+'/data/settings/settings.*') stgs_spc = list(set(stgs_arm) ^ set(stgs_all)) - armlist = basename(stgs_arm[0]).split(".")[-1] + + armlist = os.path.basename(stgs_arm[0]).split('.')[-1] for i in range(1, len(stgs_arm)): - armlist += ", " + basename(stgs_arm[i]).split(".")[-1] - spclist = basename(stgs_spc[0]).split(".")[-1] + armlist += ', ' + os.path.basename(stgs_arm[i]).split('.')[-1] + spclist = os.path.basename(stgs_spc[0]).split('.')[-1] for kk,istsp in enumerate(stgs_spc): if (kk == 0) or ('base' in istsp) or ('py' in istsp.split('.')[-1]): continue - spclist += ", " + istsp.split(".")[-1] - spcl = wraptext(spclist, width=60) - #print("\n#################################################################") - #print(self.pypitheader(prognm)) + spclist += ', ' + istsp.split('.')[-1] + + spcl = textwrap.wrap(spclist, width=60) descs = self.pypitheader(prognm) - #print("## -------------------------------------------------------------") - #print("## Options: (default values in brackets)") - #print("## -c or --cpus : (all) Number of cpu cores to use") - #print("## -h or --help : Print this message") - #print("## -v or --verbosity : (2) Level of verbosity (0-2)") - #print("## -m or --use_masters : Use files in MasterFrames for reduction") - #print("## -d or --develop : Turn develop debugging on") - #print("## -------------------------------------------------------------") - descs += "\n## Available pipelines include:" - #print("## Available pipelines include:") - descs += "\n## " + armlist - #print("## Available spectrographs include:") - descs += "\n## Available spectrographs include:" + + descs += '\n## Available pipelines include:' + descs += '\n## ' + armlist + + descs += '\n## Available spectrographs include:' for ispcl in spcl: - descs += "\n## " + ispcl - #print("## " + i) - #print("## -------------------------------------------------------------") - #print("## Last updated: {0:s}".format(self._last_updated)) - descs += "\n## Last updated: {0:s}".format(self._last_updated) - #print("#################################################################\n") - #sys.exit() - return descs + descs += '\n## ' + ispcl - def debugmessage(self): - if self._debug['develop']: - info = getouterframes(currentframe())[2] - dbgmsg = self._start+self._blue_CL+info[1].split("/")[-1]+" "+str(info[2])+" "+info[3]+"()"+self._end+" - " - else: - dbgmsg = "" - return dbgmsg + descs += '\n## Last updated: {0:s}'.format(self._last_updated) - def close(self): - """ - Close the log file and QA PDFs before the code exits - """ - from pypit import arqa - # QA HTML - if self.pypit_file is not None: # Likely testing - try: - arqa.gen_mf_html(self.pypit_file) - except: # Likely crashed real early - pass - else: - arqa.gen_exp_html() - # Close log - if self._log: - self._log.close() - return + return descs - def signal_handler(self, signalnum, handler): - """ - Handle signals sent by the keyboard during code execution - """ - if signalnum == 2: - self.info("Ctrl+C was pressed. Ending processes...") - self.close() - sys.exit() - return + def close(self): + ''' + Close the log file before the code exits + ''' + return self.reset_log_file(None) +# from pypit import arqa # Circular import! +# # QA HTML +# if self.pypit_file is not None: # Likely testing +# try: +# arqa.gen_mf_html(self.pypit_file) +# except: # Likely crashed real early +# pass +# else: +# arqa.gen_exp_html() +# # Close log +# if self._log: +# self._log.close() + +# def signal_handler(self, signalnum, handler): +# """ +# Handle signals sent by the keyboard during code execution +# """ +# if signalnum == 2: +# self.info('Ctrl+C was pressed. Ending processes...') +# self.close() +# sys.exit() def error(self, msg, usage=False): """ Print an error message """ - dbgmsg = self.debugmessage() - premsg = "\n"+self._start + self._white_RD + "[ERROR] ::" + self._end + " " - print(premsg+dbgmsg+msg, file=sys.stderr) - if self._log: - self._log.write(self.cleancolors(premsg+dbgmsg+msg)+"\n") - # Close PDFs and log file + premsg = '\n'+self._start + self._white_RD + '[ERROR] ::' + self._end + ' ' + self._print(premsg, msg) + + # Close log file + # TODO: This no longer "closes" the QA plots self.close() + # Print command line usage if usage: self.usage(None) + sys.exit(1) def info(self, msg): """ Print an information message """ - dbgmsg = self.debugmessage() - premsg = self._start + self._green_CL + "[INFO] ::" + self._end + " " - print(premsg+dbgmsg+msg, file=sys.stderr) - if self._log: - self._log.write(self.cleancolors(premsg+dbgmsg+msg)+"\n") - return + premsg = self._start + self._green_CL + '[INFO] ::' + self._end + ' ' + self._print(premsg, msg) def info_update(self, msg, last=False): """ Print an information message that needs to be updated """ - dbgmsg = self.debugmessage() - premsg = "\r" + self._start + self._green_CL + "[INFO] ::" + self._end + " " - if last: - print(premsg+dbgmsg+msg, file=sys.stderr) - if self._log: - self._log.write(self.cleancolors(premsg+dbgmsg+msg)+"\n") - else: - print(premsg+dbgmsg+msg, file=sys.stderr) - if self._log: - self._log.write(self.cleancolors(premsg+dbgmsg+msg)) - return + premsg = '\r' + self._start + self._green_CL + '[INFO] ::' + self._end + ' ' + self._print(premsg, msg, last=last) def test(self, msg): """ Print a test message """ if self._verbosity == 2: - dbgmsg = self.debugmessage() - premsg = self._start + self._white_BL + "[TEST] ::" + self._end + " " - print(premsg+dbgmsg+msg, file=sys.stderr) - if self._log: - self._log.write(self.cleancolors(premsg+dbgmsg+msg)+"\n") - return + premsg = self._start + self._white_BL + '[TEST] ::' + self._end + ' ' + self._print(premsg, msg) def warn(self, msg): """ Print a warning message """ - dbgmsg = self.debugmessage() - premsg = self._start + self._red_CL + "[WARNING] ::" + self._end + " " - print(premsg+dbgmsg+msg, file=sys.stderr) - if self._log: - self._log.write(self.cleancolors(premsg+dbgmsg+msg)+"\n") - return + premsg = self._start + self._red_CL + '[WARNING] ::' + self._end + ' ' + self._print(premsg, msg) def bug(self, msg): """ Print a bug message """ - dbgmsg = self.debugmessage() - premsg = self._start + self._white_BK + "[BUG] ::" + self._end + " " - print(premsg+dbgmsg+msg, file=sys.stderr) - if self._log: - self._log.write(self.cleancolors(premsg+dbgmsg+msg)) - return + premsg = self._start + self._white_BK + '[BUG] ::' + self._end + ' ' + self._print(premsg, msg) def work(self, msg): """ Print a work in progress message """ - if self._debug['develop']: - dbgmsg = self.debugmessage() - premsgp = self._start + self._black_CL + "[WORK IN ]::" + self._end + "\n" - premsgs = self._start + self._yellow_CL + "[PROGRESS]::" + self._end + " " - print(premsgp+premsgs+dbgmsg+msg, file=sys.stderr) - if self._log: - self._log.write(self.cleancolors(premsgp+premsgs+dbgmsg+msg)+"\n") - return + if self._debug is not None and self._debug['develop']: + premsgp = self._start + self._black_CL + '[WORK IN ]::' + self._end + '\n' + premsgs = self._start + self._yellow_CL + '[PROGRESS]::' + self._end + ' ' + self._print(premsgp+premsgs, msg) def prindent(self, msg): """ Print an indent """ - premsg = " " - print(premsg+msg, file=sys.stderr) - if self._log: - self._log.write(self.cleancolors(premsg+msg)+"\n") - return + premsg = ' ' + self._print(premsg, msg, debug=False) def input(self): """ Return a text string to be used to display input required from the user """ - premsg = self._start + self._blue_CL + "[INPUT] ::" + self._end + " " + premsg = self._start + self._blue_CL + '[INPUT] ::' + self._end + ' ' return premsg @staticmethod @@ -290,14 +321,14 @@ def newline(): """ Return a text string containing a newline to be used with messages """ - return "\n " + return '\n ' @staticmethod def indent(): """ Return a text string containing an indent to be used with messages """ - return " " + return ' ' # Set the colors def enablecolors(self): @@ -306,31 +337,23 @@ def enablecolors(self): """ # Start and end coloured text - self._start = "\x1B[" - self._end = "\x1B[" + "0m" + self._start = '\x1B[' + self._end = '\x1B[' + '0m' # Clear Backgrounds - self._black_CL = "1;30m" - self._yellow_CL = "1;33m" - self._blue_CL = "1;34m" - self._green_CL = "1;32m" - self._red_CL = "1;31m" + self._black_CL = '1;30m' + self._yellow_CL = '1;33m' + self._blue_CL = '1;34m' + self._green_CL = '1;32m' + self._red_CL = '1;31m' # Coloured Backgrounds - self._white_RD = "1;37;41m" - self._white_GR = "1;37;42m" - self._white_BK = "1;37;40m" - self._white_BL = "1;37;44m" - self._black_YL = "1;37;43m" - self._yellow_BK = "1;33;40m" - - def cleancolors(self, msg): - cols = [self._end, self._start, - self._black_CL, self._yellow_CL, self._blue_CL, self._green_CL, self._red_CL, - self._white_RD, self._white_GR, self._white_BK, self._white_BL, self._black_YL, self._yellow_BK] - for i in cols: - msg = msg.replace(i, "") - return msg + self._white_RD = '1;37;41m' + self._white_GR = '1;37;42m' + self._white_BK = '1;37;40m' + self._white_BL = '1;37;44m' + self._black_YL = '1;37;43m' + self._yellow_BK = '1;33;40m' def disablecolors(self): """ @@ -338,43 +361,43 @@ def disablecolors(self): """ # Start and end coloured text - self._start = "" - self._end = "" + self._start = '' + self._end = '' # Clear Backgrounds - self._black_CL = "" - self._yellow_CL = "" - self._blue_CL = "" - self._green_CL = "" - self._red_CL = "" + self._black_CL = '' + self._yellow_CL = '' + self._blue_CL = '' + self._green_CL = '' + self._red_CL = '' # Coloured Backgrounds - self._white_RD = "" - self._white_GR = "" - self._white_BK = "" - self._white_BL = "" - self._black_YL = "" - self._yellow_BK = "" - - -def get_logger(init=None): - """ Logger - Parameters - ---------- - init : tuple - For instantiation - (log, debug, verbosity) - - Returns - ------- - msgs : Messages - """ - global pypit_logger - - # Instantiate?? - if init is not None: - pypit_logger = Messages(init[0], init[1], init[2]) - - return pypit_logger + self._white_RD = '' + self._white_GR = '' + self._white_BK = '' + self._white_BL = '' + self._black_YL = '' + self._yellow_BK = '' + + +#def get_logger(init=None): +# """ Logger +# Parameters +# ---------- +# init : tuple +# For instantiation +# (log, debug, verbosity) +# +# Returns +# ------- +# msgs : Messages +# """ +# global pypit_logger +# +# # Instantiate?? +# if init is not None: +# pypit_logger = Messages(init[0], init[1], init[2]) +# +# return pypit_logger diff --git a/pypit/arparse.py b/pypit/arparse.py index 4ba53aa178..bf7cead7b3 100644 --- a/pypit/arparse.py +++ b/pypit/arparse.py @@ -12,9 +12,10 @@ # Logging from pypit import ardebug -from pypit import armsgs +#from pypit import armsgs +from pypit import msgs debug = ardebug.init() -msgs = armsgs.get_logger() +#msgs = armsgs.get_logger() # Initialize the settings variables argflag, spect = None, None diff --git a/pypit/arpca.py b/pypit/arpca.py index 9df9b9b18e..bfdb94997c 100644 --- a/pypit/arpca.py +++ b/pypit/arpca.py @@ -4,14 +4,16 @@ from matplotlib import pyplot as plt import numpy as np -from pypit import armsgs +#from pypit import armsgs +from pypit import msgs from pypit import arutils -from pypit.arqa import get_dimen, set_qa_filename +#from pypit.arqa import get_dimen, set_qa_filename +from pypit import arqa from pypit import ardebug as debugger # Logging -msgs = armsgs.get_logger() +#msgs = armsgs.get_logger() # Force the default matplotlib plotting parameters plt.rcdefaults() @@ -297,7 +299,7 @@ def pc_plot_extcenwid(tempcen, cenwid, binval, plotsdir="Plots", pcatype=" sky_spec.wvmin) & (sky_lines < sky_spec.wvmax))[0] if len(gdsky) == 0: msgs.warn("No sky lines for Flexure QA") @@ -404,7 +404,7 @@ def obj_trace_qa(slf, frame, ltrace, rtrace, objids, det, fig = plt.figure(dpi=1200) plt.rcParams['font.family'] = 'times new roman' - ticks_font = matplotlib.font_manager.FontProperties(family='times new roman', + ticks_font = font_manager.FontProperties(family='times new roman', style='normal', size=16, weight='normal', stretch='normal') ax = plt.gca() for label in ax.get_yticklabels() : @@ -1430,3 +1430,16 @@ def gen_exp_html(): msgs.info("Wrote: {:s}".format(exp_filename)) except: pass + + +def close_qa(pypit_file): + if pypit_file is None: + return + try: + gen_mf_html(pypit_file) + except: # Likely crashed real early + pass + else: + gen_exp_html() + + diff --git a/pypit/arsave.py b/pypit/arsave.py index 0f0ae9bd19..15e53dd5d5 100644 --- a/pypit/arsave.py +++ b/pypit/arsave.py @@ -12,7 +12,8 @@ import h5py -from pypit import armsgs +#from pypit import armsgs +from pypit import msgs from pypit import arutils from pypit import arparse as settings @@ -22,7 +23,7 @@ except NameError: pass # Logging -msgs = armsgs.get_logger() +#msgs = armsgs.get_logger() def save_arcids(fname, pixels): diff --git a/pypit/arsciexp.py b/pypit/arsciexp.py index c34f1e84cf..d2c17e129f 100644 --- a/pypit/arsciexp.py +++ b/pypit/arsciexp.py @@ -14,7 +14,8 @@ from pypit import arflux from pypit import arlris from pypit import armasters -from pypit import armsgs +#from pypit import armsgs +from pypit import msgs from pypit import arproc from pypit import arsort from pypit import arutils @@ -23,7 +24,7 @@ from pypit import ardebug as debugger # Logging -msgs = armsgs.get_logger() +#msgs = armsgs.get_logger() class ScienceExposure: diff --git a/pypit/arsort.py b/pypit/arsort.py index 2403b67e26..3cd9198e39 100644 --- a/pypit/arsort.py +++ b/pypit/arsort.py @@ -8,7 +8,8 @@ import numpy as np import yaml -from pypit import armsgs +#from pypit import armsgs +from pypit import msgs from pypit import arparse as settings from pypit import arutils from pypit.arflux import find_standard_file @@ -29,7 +30,7 @@ except NameError: pass # Logging -msgs = armsgs.get_logger() +#msgs = armsgs.get_logger() def sort_data(fitsdict, flag_unknown=False): diff --git a/pypit/arspecobj.py b/pypit/arspecobj.py index cab3c01730..882921f81f 100644 --- a/pypit/arspecobj.py +++ b/pypit/arspecobj.py @@ -5,11 +5,12 @@ import numpy as np import copy -from pypit import armsgs +#from pypit import armsgs +from pypit import msgs from pypit import arparse as settings # Logging -msgs = armsgs.get_logger() +#msgs = armsgs.get_logger() from pypit import ardebug as debugger diff --git a/pypit/artrace.py b/pypit/artrace.py index 8c1d5dac6f..adf44ef936 100644 --- a/pypit/artrace.py +++ b/pypit/artrace.py @@ -5,7 +5,8 @@ import copy from pypit import arqa from pypit import ararc -from pypit import armsgs +#from pypit import armsgs +from pypit import msgs from pypit import arutils from pypit import arpca from pypit import arparse as settings @@ -17,7 +18,7 @@ from pypit.filter import BoxcarFilter # Logging -msgs = armsgs.get_logger() +#msgs = armsgs.get_logger() from pypit import ardebug as debugger diff --git a/pypit/arutils.py b/pypit/arutils.py index fe13b72557..0d0d0b84ae 100644 --- a/pypit/arutils.py +++ b/pypit/arutils.py @@ -8,13 +8,14 @@ from scipy import interpolate import itertools import numpy as np -from pypit import armsgs +#from pypit import armsgs +from pypit import msgs #from pypit import arcyutils #from pypit import arcyarc import warnings # Logging -msgs = armsgs.get_logger() +#msgs = armsgs.get_logger() from pypit import ardebug as debugger diff --git a/pypit/arwave.py b/pypit/arwave.py index c0d6142bf8..e9f5f7acb7 100644 --- a/pypit/arwave.py +++ b/pypit/arwave.py @@ -11,12 +11,13 @@ from pypit import ararc from pypit import arextract -from pypit import armsgs +#from pypit import armsgs +from pypit import msgs from pypit import arparse as settings from pypit import arutils # Logging -msgs = armsgs.get_logger() +#msgs = armsgs.get_logger() from pypit import ardebug as debugger @@ -484,4 +485,4 @@ def vactoair(wave): new_wave = wavelength*u.AA new_wave.to(wave.unit) - return new_wave \ No newline at end of file + return new_wave diff --git a/pypit/ginga.py b/pypit/ginga.py index a51dc12dc7..1f03613db6 100644 --- a/pypit/ginga.py +++ b/pypit/ginga.py @@ -5,7 +5,8 @@ import os import numpy as np -from pypit import armsgs +#from pypit import armsgs +from pypit import msgs from pypit import pyputils # CANNOT LOAD DEBUGGER AS THIS MODULE IS CALLED BY ARDEBUG @@ -18,7 +19,7 @@ basestring = str # Logging -msgs = armsgs.get_logger() # THESE MAY NOT WORK.. +#msgs = armsgs.get_logger() # THESE MAY NOT WORK.. ''' if msgs is None: # For usage outside of PYPIT msgs = pyputils.get_dummy_logger() diff --git a/pypit/pypit.py b/pypit/pypit.py index fb70ac53e4..d8cd7ba247 100644 --- a/pypit/pypit.py +++ b/pypit/pypit.py @@ -1,22 +1,31 @@ -from __future__ import absolute_import, division, print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals -from signal import SIGINT, signal as sigsignal -from warnings import resetwarnings, simplefilter -from time import time -from pypit import armsgs -from pypit import archeck +import os +import time +import signal +import warnings import glob import numpy as np -# Import PYPIT routines - try: from linetools.spectra.xspectrum1d import XSpectrum1D except ImportError: pass -from pypit import ardebug as debugger +from pypit import ardebug +from pypit import msgs +from pypit import archeck +from pypit import arparse +from pypit import ardevtest +from pypit.arload import load_headers +from pypit import arqa + +from pypit import armed +from pypit import armlsd def PYPIT(redname, debug=None, progname=__file__, quick=False, ncpus=1, verbosity=1, use_masters=False, devtest=False, logname=None): @@ -58,31 +67,31 @@ def PYPIT(redname, debug=None, progname=__file__, quick=False, ncpus=1, verbosit last_updated : str --------------------------------------------------- """ - from pypit import ardebug # Init logger if debug is None: debug = ardebug.init() - msgs = armsgs.get_logger((logname, debug, verbosity)) - msgs.pypit_file = redname - # This needs to be loaded after msgs - from pypit import arparse + # Reset the global logger +# msgs = armsgs.get_logger((logname, debug, verbosity)) + msgs.reset(log=logname, debug=debug, verbosity=verbosity) + msgs.pypit_file = redname # version checking try: archeck.version_check() except archeck.VersionError as err: msgs.error(err.message) - - # First send all signals to messages to be dealt with (i.e. someone hits ctrl+c) - sigsignal(SIGINT, msgs.signal_handler) - # Ignore all warnings given by python - resetwarnings() - simplefilter("ignore") +# MOVED to __init__.py +# # First send all signals to messages to be dealt with (i.e. someone hits ctrl+c) +# signal.signal(signal.SIGINT, signal_handler) +# +# # Ignore all warnings given by python +# warnings.resetwarnings() +# warnings.simplefilter('ignore') # Record the starting time - tstart = time() + tstart = time.time() # Load the input file pyp_dict = load_input(redname, msgs) @@ -98,28 +107,29 @@ def PYPIT(redname, debug=None, progname=__file__, quick=False, ncpus=1, verbosit for i in range(len(parlines)): parspl = parlines[i].split() if len(parspl) < 3: - msgs.error("There appears to be a missing argument on the following input line" + msgs.newline() + - parlines[i]) + msgs.error('There appears to be a missing argument on the following input line' + + msgs.newline() + parlines[i]) if (parspl[0] == 'run') and (parspl[1] == 'spectrograph'): specname = parspl[2] break if specname is None: - msgs.error("Please specify the spectrograph settings to be used with the command" + msgs.newline() + - "run spectrograph ") - msgs.info("Reducing data from the {0:s} spectrograph".format(specname)) + msgs.error('Please specify the spectrograph settings to be used with the command' + + msgs.newline() + 'run spectrograph ') + msgs.info('Reducing data from the {0:s} spectrograph'.format(specname)) # Determine the type of reduction used for this spectrograph redtype = None # Get the software path prgn_spl = progname.split('/') - tfname = "/".join(prgn_spl[:-1]) + "/" + tfname = '/'.join(prgn_spl[:-1]) + '/' # Settings file fname = tfname + 'data/settings/settings.' + specname try: spl = open(fname, 'r').readlines() except IOError: - msgs.error("The following instrument settings file cannot be found:" + msgs.newline() + fname + msgs.newline() + - "Please check the settings file exists, and that the instrument name is spelt correctly.") + msgs.error('The following instrument settings file cannot be found:' + msgs.newline() + + fname + msgs.newline() + 'Please check the settings file exists, and that' + + ' the instrument name is spelt correctly.') for i in range(len(spl)): parspl = spl[i].split() if len(parspl) < 3: @@ -128,12 +138,13 @@ def PYPIT(redname, debug=None, progname=__file__, quick=False, ncpus=1, verbosit redtype = parspl[2] break if redtype is None: - msgs.bug("The {0:s} instrument settings file must contain the reduction type".format(specname)) - msgs.error("Please specify the reduction type with the command" + msgs.newline() + - "mosaic reduction ") + msgs.bug('The {0:s} instrument settings file must contain the reduction type'.format( + specname)) + msgs.error('Please specify the reduction type with the command' + msgs.newline() + + 'mosaic reduction ') # Load default reduction arguments/flags, and set any command line arguments - argf = arparse.get_argflag_class((redtype.upper(), ".".join(redname.split(".")[:-1]))) + argf = arparse.get_argflag_class((redtype.upper(), '.'.join(redname.split('.')[:-1]))) argf.init_param() # Run specific argf.set_param('run pypitdir {0:s}'.format(tfname)) @@ -148,7 +159,7 @@ def PYPIT(redname, debug=None, progname=__file__, quick=False, ncpus=1, verbosit argf.set_paramlist(lines) # Load default spectrograph settings - spect = arparse.get_spect_class((redtype.upper(), specname, ".".join(redname.split(".")[:-1]))) + spect = arparse.get_spect_class((redtype.upper(), specname, '.'.join(redname.split('.')[:-1]))) lines = spect.load_file(base=True) # Base spectrograph settings spect.set_paramlist(lines) lines = spect.load_file() # Instrument specific @@ -172,17 +183,16 @@ def PYPIT(redname, debug=None, progname=__file__, quick=False, ncpus=1, verbosit argf.set_param('output verbosity {0:d}'.format(verbosity)) if use_masters: argf.set_param('reduce masters reuse True') - msgs.work("Make appropriate changes to quick reduction") + msgs.work('Make appropriate changes to quick reduction') # Load Development suite changes if devtest: - msgs.info("Loading instrument specific argurment for Development Suite tests") - from pypit import ardevtest + msgs.info('Loading instrument specific argurment for Development Suite tests') ardevtest.set_param(argf, specname) if quick: # If a quick reduction has been requested, make sure the requested pipeline # is the quick implementation (if it exists), otherwise run the standard pipeline. - msgs.work("QUICK REDUCTION TO STILL BE DONE") + msgs.work('QUICK REDUCTION TO STILL BE DONE') # Setup from PYPIT file? if len(pyp_dict['setup']['name']) == 1: argf.set_param('setup name {:s}'.format(pyp_dict['setup']['name'][0])) @@ -193,17 +203,16 @@ def PYPIT(redname, debug=None, progname=__file__, quick=False, ncpus=1, verbosit # Now that all of the relevant settings are loaded, globalize the settings arparse.init(argf, spect) - ''' + """ # Test that a maximum of one .setup files is present from pypit import arsort setup_file, nexist = arsort.get_setup_file() if nexist == 1: - msgs.info("Found setup_file: {:s}".format(setup_file)) - msgs.info("Will use this to guide the data reduction.") - ''' + msgs.info('Found setup_file: {:s}'.format(setup_file)) + msgs.info('Will use this to guide the data reduction.') + """ # Load the important information from the fits headers - from pypit.arload import load_headers fitsdict, updates = load_headers(datlines) # If some settings were updated because of the fits headers, globalize the settings again @@ -229,49 +238,49 @@ def PYPIT(redname, debug=None, progname=__file__, quick=False, ncpus=1, verbosit # Update the amplifier/data/overscan sections for i in range(arparse.spect[ddnum]['numamplifiers']): # Flip the order of the sections - arparse.spect[ddnum]['datasec{0:02d}'.format(i + 1)] = arparse.spect[ddnum][ - 'datasec{0:02d}'.format(i + 1)][::-1] - arparse.spect[ddnum]['oscansec{0:02d}'.format(i + 1)] = arparse.spect[ddnum][ - 'oscansec{0:02d}'.format(i + 1)][::-1] + arparse.spect[ddnum]['datasec{0:02d}'.format(i + 1)] \ + = arparse.spect[ddnum]['datasec{0:02d}'.format(i + 1)][::-1] + arparse.spect[ddnum]['oscansec{0:02d}'.format(i + 1)] \ + = arparse.spect[ddnum]['oscansec{0:02d}'.format(i + 1)][::-1] + # Reduce the data! status = 0 # Send the data away to be reduced if spect.__dict__['_spect']['mosaic']['reduction'] == 'ARMLSD': - msgs.info("Data reduction will be performed using PYPIT-ARMLSD") - from pypit import armlsd + msgs.info('Data reduction will be performed using PYPIT-ARMLSD') status = armlsd.ARMLSD(fitsdict) elif spect.__dict__['_spect']['mosaic']['reduction'] == 'ARMED': - msgs.info("Data reduction will be performed using PYPIT-ARMED") - from pypit import armed + msgs.info('Data reduction will be performed using PYPIT-ARMED') status = armed.ARMED(fitsdict) + # Check for successful reduction if status == 0: - from pypit import arqa - msgs.info("Data reduction complete") + msgs.info('Data reduction complete') # QA HTML - msgs.info("Generating QA HTML") + msgs.info('Generating QA HTML') arqa.gen_mf_html(redname) arqa.gen_exp_html() elif status == 1: - msgs.info("Setup complete") + msgs.info('Setup complete') elif status == 2: - msgs.info("Calcheck complete") + msgs.info('Calcheck complete') else: - msgs.error("Data reduction failed with status ID {0:d}".format(status)) + msgs.error('Data reduction failed with status ID {0:d}'.format(status)) + # Capture the end time and print it to user - tend = time() + tend = time.time() codetime = tend-tstart if codetime < 60.0: - msgs.info("Data reduction execution time: {0:.2f}s".format(codetime)) + msgs.info('Data reduction execution time: {0:.2f}s'.format(codetime)) elif codetime/60.0 < 60.0: mns = int(codetime/60.0) scs = codetime - 60.0*mns - msgs.info("Data reduction execution time: {0:d}m {1:.2f}s".format(mns, scs)) + msgs.info('Data reduction execution time: {0:d}m {1:.2f}s'.format(mns, scs)) else: hrs = int(codetime/3600.0) mns = int(60.0*(codetime/3600.0 - hrs)) scs = codetime - 60.0*mns - 3600.0*hrs - msgs.info("Data reduction execution time: {0:d}h {1:d}m {2:.2f}s".format(hrs, mns, scs)) + msgs.info('Data reduction execution time: {0:d}h {1:d}m {2:.2f}s'.format(hrs, mns, scs)) return @@ -317,13 +326,12 @@ def load_input(redname, msgs): 'ftype' : dict dict of filename: frametype """ - import os # Read in the model file - msgs.info("Loading the input file") + msgs.info('Loading the input file') try: infile = open(redname, 'r') except IOError: - msgs.error("The filename does not exist -" + msgs.newline() + redname) + msgs.error('The filename does not exist -' + msgs.newline() + redname) lines = infile.readlines() parlines = [] datlines = [] @@ -349,7 +357,7 @@ def load_input(redname, msgs): for kk,datfile in enumerate(datlines): if skip_file in datfile: keep[kk] = False - msgs.warn("Skipping file {:s}".format(skip_file)) + msgs.warn('Skipping file {:s}'.format(skip_file)) # Save datlines = np.array(datlines)[keep].tolist() continue @@ -373,9 +381,11 @@ def load_input(redname, msgs): elif dfname[:4] == 'skip': skip_files.append(dfname.split(' ')[1]) elif dfname[0] != '/': - msgs.error("You must specify the full datapath for the file:" + msgs.newline() + dfname) + msgs.error('You must specify the full datapath for the file:' + + msgs.newline() + dfname) elif len(dfname.split()) != 1: - msgs.error("There must be no spaces when specifying the datafile:" + msgs.newline() + dfname) + msgs.error('There must be no spaces when specifying the datafile:' + + msgs.newline() + dfname) dfnames.append(dfname) listing = glob.glob(dfname) for lst in listing: datlines.append(lst) @@ -396,7 +406,8 @@ def load_input(redname, msgs): datlines.append(path+linspl[dfile_col]) ftype_dict[linspl[dfile_col]] = linspl[ftype_col] continue - elif rddata == 0 and linspl[0] == 'data' and linspl[1] == 'read': # Begin data read block + elif rddata == 0 and linspl[0] == 'data' and linspl[1] == 'read': + # Begin data read block rddata += 1 continue if rdsetup == 1: # Read setup command @@ -407,7 +418,8 @@ def load_input(redname, msgs): setups.append(lines[i][6:].strip()) setuplines.append(lines[i]) continue - elif rdsetup == 0 and linspl[0] == 'setup' and linspl[1] == 'read': # Begin setup read block + elif rdsetup == 0 and linspl[0] == 'setup' and linspl[1] == 'read': + # Begin setup read block rdsetup += 1 continue if rdspec == 1: # Read spect command @@ -416,7 +428,8 @@ def load_input(redname, msgs): continue spclines.append(lines[i]) continue - elif rdspec == 0 and linspl[0] == 'spect' and linspl[1] == 'read': # Begin spect read block + elif rdspec == 0 and linspl[0] == 'spect' and linspl[1] == 'read': + # Begin spect read block rdspec += 1 continue if lines[i].lstrip()[0] == '#': continue @@ -427,18 +440,18 @@ def load_input(redname, msgs): elif rddata == 1: msgs.error("Missing 'data end' in " + redname) if rddata == 0: - msgs.info("Using Default spectrograph parameters") + msgs.info('Using Default spectrograph parameters') elif rddata != 2: msgs.error("Missing 'spect end' in " + redname) # Check there are no duplicate inputs if len(datlines) != len(set(datlines)): - msgs.error("There are duplicate files in the list of data.") + msgs.error('There are duplicate files in the list of data.') if len(datlines) == 0: - msgs.error("There are no raw data frames" + msgs.newline() + - "Perhaps the path to the data is incorrect?") + msgs.error('There are no raw data frames' + msgs.newline() + + 'Perhaps the path to the data is incorrect?') else: - msgs.info("Found {0:d} raw data frames".format(len(datlines))) - msgs.info("Input file loaded successfully") + msgs.info('Found {0:d} raw data frames'.format(len(datlines))) + msgs.info('Input file loaded successfully') # Let's return a dict pypit_dict = dict(par=parlines, dat=datlines, spc=spclines, dfn=dfnames, setup={'name': setups, 'lines': setuplines}, @@ -446,4 +459,3 @@ def load_input(redname, msgs): return pypit_dict # parlines, datlines, spclines, dfnames, setup, setuplines, ftype_dict - diff --git a/pypit/pyputils.py b/pypit/pyputils.py index 846109cc5d..f8bcc1a5b2 100644 --- a/pypit/pyputils.py +++ b/pypit/pyputils.py @@ -1,38 +1,46 @@ """ Some PYPIT utilities for the package """ -from __future__ import (print_function, absolute_import, division, - unicode_literals) +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals import re -from os.path import realpath, join - -this_file = realpath(__file__) -this_path = this_file[:this_file.rfind('/')] - - -def get_version(): - """Get the value of ``__version__`` without having to import the module. - Copied from DESI desiutils - - Parameters - ---------- - - Returns - ------- - :class:`str` - The value of ``__version__``. - """ - ver = 'unknown' - version_file = join(this_path, '_version.py') - with open(version_file, "r") as f: - for line in f.readlines(): - mo = re.match("__version__ = '(.*)'", line) - lu = re.match("__lastupdate__ = '(.*)'", line) - if mo: - ver = mo.group(1) - if lu: - upd = lu.group(1) - return ver, upd +import os + +from pypit import armsgs +from pypit import ardebug + +# this_file = realpath(__file__) +# this_path = this_file[:this_file.rfind('/')] + +#def get_version(): +# """Get the value of ``__version__`` without having to import the module. +# Copied from DESI desiutils +# +# Parameters +# ---------- +# +# Returns +# ------- +# :class:`str` +# The value of ``__version__``. +# """ +# ver = 'unknown' +# +# this_file = os.path.realpath(__file__) +# this_path = this_file[:this_file.rfind('/')] +# +# version_file = os.path.join(this_path, '_version.py') +# with open(version_file, "r") as f: +# for line in f.readlines(): +# mo = re.match("__version__ = '(.*)'", line) +# lu = re.match("__lastupdate__ = '(.*)'", line) +# if mo: +# ver = mo.group(1) +# if lu: +# upd = lu.group(1) +# return ver, upd def get_dummy_logger(develop=False): @@ -42,13 +50,17 @@ def get_dummy_logger(develop=False): ------- """ - from pypit import ardebug - from pypit import armsgs as pyparm debug = ardebug.init() debug['develop'] = develop - - pyparm.pypit_logger = pyparm.Messages(None, debug, 0) - return pyparm.pypit_logger + return armsgs.Messages(log=None, debug=debug, verbosity=0) + +# from pypit import ardebug +# from pypit import armsgs as pyparm +# debug = ardebug.init() +# debug['develop'] = develop +# +# pyparm.pypit_logger = pyparm.Messages(None, debug, 0) +# return pyparm.pypit_logger def make_pypit_file(pyp_file, spectrograph, dfnames, parlines=None, diff --git a/pypit/scripts/run_pypit.py b/pypit/scripts/run_pypit.py index 5815583c65..0e05094767 100644 --- a/pypit/scripts/run_pypit.py +++ b/pypit/scripts/run_pypit.py @@ -3,19 +3,26 @@ # See top-level LICENSE file for Copyright information # # -*- coding: utf-8 -*- - - """ This script runs PYPIT """ -from __future__ import (print_function, absolute_import, division, - unicode_literals) - -import pdb as debugger +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + +import os +import sys +import argparse +import traceback + +from pypit import pypit +from pypit import msgs +from pypit import ardebug +#from pypit.scripts import qa_html # Globals -from pypit import ardebug -debug = ardebug.init() +#debug = ardebug.init() #debug['develop'] = True #debug['arc'] = True #debug['sky_sub'] = True @@ -26,43 +33,41 @@ #debug['flexure'] = True #debug['no_qa'] = True -from pypit.armsgs import Messages as Initmsg -initmsgs = Initmsg(None, debug, 1) +#from pypit.armsgs import Messages as Initmsg +#initmsgs = Initmsg(None, debug, 1) def parser(options=None): - import argparse - parser = argparse.ArgumentParser(description=initmsgs.usage('PYPIT'), + parser = argparse.ArgumentParser(description=msgs.usage('PYPIT'), formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument("pypit_file", type=str, help="PYPIT reduction file (must have .pypit extension)") - parser.add_argument("-v", "--verbosity", type=int, default=2, help="(2) Level of verbosity (0-2)") - parser.add_argument("-m", "--use_masters", default=False, action='store_true', help="Load previously generated MasterFrames") - parser.add_argument("-d", "--develop", default=False, action='store_true', help="Turn develop debugging on") - parser.add_argument("--devtest", default=False, action='store_true', help="Running development tests") - parser.add_argument("--debug_arc", default=False, action='store_true', help="Turn wavelength/arc debugging on") - #parser.add_argument("-q", "--quick", default=False, help="Quick reduction", action="store_true") - #parser.add_argument("-c", "--cpus", default=False, help="Number of CPUs for parallel processing", action="store_true") - #parser.print_help() + parser.add_argument('pypit_file', type=str, + help='PYPIT reduction file (must have .pypit extension)') + parser.add_argument('-v', '--verbosity', type=int, default=2, + help='(2) Level of verbosity (0-2)') + parser.add_argument('-m', '--use_masters', default=False, action='store_true', + help='Load previously generated MasterFrames') + parser.add_argument('-d', '--develop', default=False, action='store_true', + help='Turn develop debugging on') + parser.add_argument('--devtest', default=False, action='store_true', + help='Running development tests') + parser.add_argument('--debug_arc', default=False, action='store_true', + help='Turn wavelength/arc debugging on') +# parser.add_argument('-q', '--quick', default=False, help='Quick reduction', +# action='store_true') +# parser.add_argument('-c', '--cpus', default=False, action='store_true', +# help='Number of CPUs for parallel processing') +# parser.print_help() if options is None: pargs = parser.parse_args() else: pargs = parser.parse_args(options) - # return pargs def main(args): - import sys, os - from pypit import pypit - from pypit.scripts import qa_html - import traceback - - # Import PYPIT routines - - # Initiate logging for bugs and command line help # These messages will not be saved to a log file # Set the default variables @@ -71,21 +76,24 @@ def main(args): #vrb = 2 # Load options from command line + debug = ardebug.init() debug['develop'] = debug['develop'] or args.develop debug['arc'] = debug['arc'] or args.debug_arc splitnm = os.path.splitext(args.pypit_file) if splitnm[1] != '.pypit': - initmsgs.error("Bad extension for PYPIT reduction file."+initmsgs.newline()+".pypit is required") + msgs.error("Bad extension for PYPIT reduction file."+msgs.newline()+".pypit is required") logname = splitnm[0] + ".log" # Execute the reduction, and catch any bugs for printout if debug['develop']: - pypit.PYPIT(args.pypit_file, progname=pypit.__file__, quick=qck, ncpus=cpu, verbosity=args.verbosity, - use_masters=args.use_masters, devtest=args.devtest, logname=logname, debug=debug) + pypit.PYPIT(args.pypit_file, progname=pypit.__file__, quick=qck, ncpus=cpu, + verbosity=args.verbosity, use_masters=args.use_masters, devtest=args.devtest, + logname=logname, debug=debug) else: try: - pypit.PYPIT(args.pypit_file, progname=pypit.__file__, quick=qck, ncpus=cpu, verbosity=args.verbosity, - use_masters=args.use_masters, devtest=args.devtest, logname=logname, debug=debug) + pypit.PYPIT(args.pypit_file, progname=pypit.__file__, quick=qck, ncpus=cpu, + verbosity=args.verbosity, use_masters=args.use_masters, + devtest=args.devtest, logname=logname, debug=debug) except: # There is a bug in the code, print the file and line number of the error. et, ev, tb = sys.exc_info() @@ -100,11 +108,13 @@ def main(args): tb = tb.tb_next filename = filename.split('/')[-1] if str(ev) != "": - initmsgs.bug("There appears to be a bug on Line " + line_no + " of " + filename + " with error:" + - initmsgs.newline() + str(ev) + initmsgs.newline() + - "---> please contact the authors") - # Get armsgs instance to terminate - from pypit.armsgs import get_logger - get_logger().close() + msgs.bug("There appears to be a bug on Line " + line_no + " of " + filename + + " with error:" + msgs.newline() + str(ev) + msgs.newline() + + "---> please contact the authors") + msgs.close() +# # Get armsgs instance to terminate +# from pypit.armsgs import get_logger +# # TODO: Close logger and close QA +# get_logger().close() return 1 return 0 diff --git a/pypit/tests/test_pyputils.py b/pypit/tests/test_pyputils.py index b50d2f318e..4478d2399c 100644 --- a/pypit/tests/test_pyputils.py +++ b/pypit/tests/test_pyputils.py @@ -13,10 +13,12 @@ # data_dir = os.path.join(os.path.dirname(__file__), 'files') # return os.path.join(data_dir, filename) +from pypit import __version__, __last_update__ def test_version(): # Dummy self - ver,upd = pyputils.get_version() +# ver,upd = pyputils.get_version() + ver,upd = __version__, __last_update__ assert isinstance(ver,basestring) From 6c594942d38f4610dd56806d90c9bd97645bf16c Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Mon, 26 Mar 2018 09:15:17 -0700 Subject: [PATCH 09/36] resolve conflict --- pypit/filter.py | 372 ------------------------------------------------ 1 file changed, 372 deletions(-) diff --git a/pypit/filter.py b/pypit/filter.py index 9e88b1fdb5..d69ba3be3e 100644 --- a/pypit/filter.py +++ b/pypit/filter.py @@ -2,38 +2,8 @@ """ A set of functions used to filter arrays. -<<<<<<< HEAD Originally pulled from SDSS-IV/MaNGA Data Analysis Pipeline, licensed under BSD 3-clause license. -======= -*License*: - Copyright (c) 2017, SDSS-IV/MaNGA Pipeline Group - Licensed under BSD 3-clause license - see LICENSE.rst - -*Source location*: - $MANGADAP_DIR/python/mangadap/util/filter.py - -*Imports and python version compliance*: - :: - - from __future__ import division - from __future__ import print_function - from __future__ import absolute_import - from __future__ import unicode_literals - - import sys - if sys.version > '3': - long = int - - import numpy - -*Revision history*: - | **26 Jan 2017**: Original implementation by K. Westfall (KBW) - | **06 Apr 2017**: Interpolate sigma vectors as well as the smoothed - vectors in :class:`BoxcarFilter`. - | **21 Dec 2017**: Add weighting functionality to - :class:`BoxcarFilter`. ->>>>>>> master """ from __future__ import division @@ -49,178 +19,8 @@ import numpy from scipy import interpolate, sparse -<<<<<<< HEAD class BoxcarFilter(object): """Boxcar filter a 2D image along columns.""" -======= -# Debug -from matplotlib import pyplot -from matplotlib.ticker import NullFormatter - -def high_pass_filter(flux, dw=0, k=None, Dw=None): - """ - Pulled from FIREFLY's hpf() function and edited on 17 Nov 2016. - - Apply a high-pass filter to the input vector. - - """ - - n = flux.size - _dw = int(dw) - - if k is None and Dw is None: - Dw = 100 - _k = n//Dw - elif k is not None and Dw is None: - _k = int(k) - Dw = n//_k - elif k is None and Dw is not None: - _k = n//Dw - else: - raise ValueError('Cannot provide both k and Dw.') - - # PRINT A REPORT - - # Rita's typical inputs for SDSS: - # w = 10 # 10 - # windowsize = 20 # 20 - - # My MaNGA inputs: - # if w == 0 and windowsize == 0: - # print "HPF parameters not specified: choosing default window (stepsize=40)" - # w = 40 - # windowsize = 0 - - h = numpy.fft.fft(flux) - h_filtered = numpy.zeros(n, dtype=complex) - window = numpy.zeros(n) - unwindow = numpy.zeros(n) - window[0] = 1 # keep F0 for normalisation - unwindow[0] = 1 - - for i in range(_dw): - window[_k+i] = (i+1.0)/_dw - window[n-1-(_k+_dw-i)] = (_dw-i)/_dw - window[_k+_dw:n-(_k+_dw)] = 1 - - unwindow = 1 - window - unwindow[0] = 1 - - h_filtered = h * window - un_h_filtered = h * unwindow - - res = numpy.real(numpy.fft.ifft(h_filtered)) - unres = numpy.real(numpy.fft.ifft(un_h_filtered)) - res_out = (1.0+(res-numpy.median(res))/unres) * numpy.median(res) - - return res_out, window, res, unres - -#def off_diagonal_identity(size, win): -# r""" -# Construct a matrix with ones within a window along the diagonal. -# -# Args: -# size (int) : Size for the square matrix; i.e., :math:`N` for the -# :math:`N\timesN` matrix. -# win (int): Number of ones in each row along the diagonal. -# -# Raises: -# ValueError: Raised if the window is larger than 2*size-1. -# -# """ -# if win > 2*size-1: -# raise ValueError('Window too large for matrix size.') -# if win == 2*size-1: -# return numpy.ones((size,size), dtype=int) -# x = numpy.zeros((size,size), dtype=int)#numpy.identity(size).astype(int) -# for i in range(1,(win+1)//2): -# x[:-i,i:] = x[:-i,i:] + numpy.identity(size-i).astype(int) -# x += x.T -# if win % 2 != 1: -# x[:-i-1,i+1:] = x[:-i-1,i+1:] + numpy.identity(size-i-1).astype(int) -# return x + numpy.identity(size).astype(int) - -def off_diagonal_identity(size, win, return_sparse=False): - r""" - Construct a matrix with ones within a window along the diagonal. - - Args: - size (int) : Size for the square matrix; i.e., :math:`N` for the - :math:`N\timesN` matrix. - win (int): Number of ones in each row along the diagonal. - - Raises: - ValueError: Raised if the window is larger than 2*size-1. - - """ - if win > 2*size-1: - raise ValueError('Window too large for matrix size.') - if win == 2*size-1: - return numpy.ones((size,size), dtype=int) - - # Indices of diagonal - ii = numpy.arange(size).astype(int) - - # Build the upper triangle - i = numpy.empty(0, dtype=int) - j = numpy.empty(0, dtype=int) - for k in range(1,(win+1)//2): - i = numpy.append(i, ii[:size-k]) - j = numpy.append(j, ii[k:size]) - - # Copy to the lower triangle - _i = numpy.append(i,j) - j = numpy.append(j,i) - - # Add the diagonal - i = numpy.append(_i, ii) - j = numpy.append(j, ii) - - # Accommodate an even window - if win % 2 != 1: - i = numpy.append(i, ii[:size-k-1]) - j = numpy.append(j, ii[k+1:size]) - - # Construct and return the array - if return_sparse: - return sparse.coo_matrix((numpy.ones(len(i), dtype=int),(i,j)), shape=(size,size)).tocsr() - - a = numpy.zeros((size,size), dtype=int) - a[i,j] = 1 - return a - - -def build_smoothing_mask(x, pix_buffer, default=None, mask_x=None): - r""" - Construct a mask for the provided vector that masks the first and - last pix_buffer pixels in the coordinate vector. - - The mask is instantiated as fully unmasked, unless a default mask is - provided. - - To mask specified ranges in x, provide mask_x with a shape - :math:`N_{\rm mask}\times 2` where each mask is defined by the - starting and ending value of x to exclude. - """ - # Smooth the ratio of the data to the model - if len(x.shape) != 1: - raise ValueError('Input must be a vector.') - npix = x.size - mask = numpy.zeros(npix, dtype=bool) if default is None else default.copy() - if mask.size != npix: - raise ValueError('Provided default has an incorrect length.') - mask[:pix_buffer] = True - mask[-pix_buffer:] = True - if mask_x is None: - return mask - - for m in mask_x: - mask |= numpy.logical_and(x > m[0], x < m[1]) - return mask - - -class BoxcarFilter(): ->>>>>>> master def __init__(self, boxcar, lo=None, hi=None, niter=None, y=None, wgt=None, mask=None, local_sigma=None): @@ -249,10 +49,6 @@ def __init__(self, boxcar, lo=None, hi=None, niter=None, y=None, wgt=None, mask= self.smooth(y, wgt=wgt, mask=mask, boxcar=boxcar, lo=lo, hi=hi, niter=niter, local_sigma=local_sigma) -<<<<<<< HEAD -======= - ->>>>>>> master def _assign_par(self, boxcar, lo, hi, niter, local_sigma): if boxcar is not None: self.boxcar = boxcar @@ -265,10 +61,6 @@ def _assign_par(self, boxcar, lo, hi, niter, local_sigma): if local_sigma is not None: self.local_sigma = local_sigma -<<<<<<< HEAD -======= - ->>>>>>> master def _check_par(self): if self.boxcar <= 1: raise ValueError('Boxcar must be greater than 1!') @@ -276,10 +68,6 @@ def _check_par(self): if self.nrej is not None and self.nrej > 0 and self.lo_rej is None and self.hi_rej is None: raise ValueError('Must provide lo or hi if niter provided') -<<<<<<< HEAD -======= - ->>>>>>> master def _reduce_indices(self): indx = numpy.array([numpy.arange(self.npix)-self.boxcar//2, numpy.arange(self.npix)+self.boxcar//2+1]) @@ -287,16 +75,9 @@ def _reduce_indices(self): indx[:self.npix] += 1 return indx.T.ravel().clip(0,self.npix-1) -<<<<<<< HEAD def _apply(self): self.smoothed_n = numpy.add.reduceat(numpy.invert(self.output_mask), self.rdx_index, axis=1, dtype=int)[:,::2] -======= - - def _apply(self): - self.smoothed_n = numpy.add.reduceat(numpy.invert(self.output_mask), self.rdx_index, axis=1, - dtype=int)[:,::2] ->>>>>>> master self.smoothed_wgt = numpy.add.reduceat(self.input_wgt, self.rdx_index, axis=1, dtype=float)[:,::2] self.smoothed_y = numpy.add.reduceat(self.input_wgt*self.y.filled(0.0), self.rdx_index, @@ -320,45 +101,6 @@ def _apply(self): self.smoothed_y[self.output_mask | (self.smoothed_n == 0)] = numpy.ma.masked self.sigma_y[self.output_mask | (self.smoothed_n == 0)] = numpy.ma.masked -<<<<<<< HEAD -======= - return - - w,h = pyplot.figaspect(1) - fig = pyplot.figure(figsize=(1.5*w,1.5*h)) - - minf = numpy.amin( numpy.ma.log10(numpy.append(self.y.compressed(), self.smoothed_y.compressed())) ) - maxf = numpy.amax( numpy.ma.log10(numpy.append(self.y.compressed(), self.smoothed_y.compressed())) ) - - mins = numpy.amin(numpy.ma.log10(self.sigma_y).compressed()) - maxs = numpy.amax(numpy.ma.log10(self.sigma_y).compressed()) - - minr = numpy.amin(numpy.ma.log10(numpy.ma.divide(self.y, self.smoothed_y)).compressed()) - maxr = numpy.amax(numpy.ma.log10(numpy.ma.divide(self.y, self.smoothed_y)).compressed()) - - ax = fig.add_axes([0.05, 0.50, 0.45, 0.45]) - ax.xaxis.set_major_formatter(NullFormatter()) - ax.imshow(numpy.ma.log10(self.y), origin='lower', interpolation='nearest', vmin=minf, - vmax=maxf, aspect='auto') - ax = fig.add_axes([0.50, 0.50, 0.45, 0.45]) - ax.xaxis.set_major_formatter(NullFormatter()) - ax.yaxis.set_major_formatter(NullFormatter()) - ax.imshow(numpy.ma.log10(self.smoothed_y), origin='lower', interpolation='nearest', - vmin=minf, vmax=maxf, aspect='auto') - ax = fig.add_axes([0.05, 0.05, 0.45, 0.45]) - ax.yaxis.set_major_formatter(NullFormatter()) - ax.imshow(numpy.ma.log10(numpy.ma.divide(self.y,self.smoothed_y)), origin='lower', - interpolation='nearest', aspect='auto', vmin=minr, vmax=maxr) - ax = fig.add_axes([0.50, 0.05, 0.45, 0.45]) - ax.yaxis.set_major_formatter(NullFormatter()) - ax.imshow(numpy.ma.log10(self.sigma_y), origin='lower', interpolation='nearest', - aspect='auto', vmin=mins, vmax=maxs) - - pyplot.show() - exit() - - ->>>>>>> master def _interpolate(self): """ Interpolate the smoothed image across the masked regions. @@ -366,10 +108,6 @@ def _interpolate(self): first and last unmasked value, respectively. interpolate both the smoothed data and the sigma -<<<<<<< HEAD -======= - ->>>>>>> master """ # Boolean arrays with bad pixels (should be the same for both # smoothed_y and sigma_y) @@ -409,10 +147,6 @@ def _interpolate(self): self.sigma_y.ravel()[badpix.ravel()] = interpolator(x.data[badpix.ravel()]) self.sigma_y[numpy.invert(goodvec),:] = 0.0 -<<<<<<< HEAD -======= - ->>>>>>> master def smooth(self, y, wgt=None, mask=None, boxcar=None, lo=None, hi=None, niter=None, local_sigma=None): """ @@ -475,10 +209,6 @@ def smooth(self, y, wgt=None, mask=None, boxcar=None, lo=None, hi=None, niter=No self._apply() nmasked = numpy.sum(self.output_mask) - nbad -<<<<<<< HEAD -======= -# print('Niter: {0}; Nmasked: {1}'.format(i+1, nmasked)) ->>>>>>> master i += 1 if i == self.nrej or nmasked == 0: @@ -487,109 +217,7 @@ def smooth(self, y, wgt=None, mask=None, boxcar=None, lo=None, hi=None, niter=No self._interpolate() return self.smoothed_y[0,:] if provided_vector else self.smoothed_y -<<<<<<< HEAD -======= - ->>>>>>> master @property def mask(self): return self.output_mask.copy() -<<<<<<< HEAD -======= - - -def smooth_masked_vector(x, nsmooth): - """ - Smooth a masked vector by a box of size nsmooth. - """ - n = off_diagonal_identity(x.size, nsmooth)*numpy.invert(numpy.ma.getmaskarray(x))[None,:] - nn = numpy.sum(n,axis=1) - return numpy.ma.MaskedArray(numpy.dot(n,x.data), mask=numpy.invert(nn>0) | x.mask)/nn - - -def interpolate_masked_vector(y, quiet=True, extrap_with_median=False): - """ - Interpolate over the masked pixels in an input vector using linear - interpolation. - """ - x = numpy.arange(y.size) - indx = numpy.ma.getmaskarray(y) - if numpy.sum(numpy.invert(indx)) < 2: - if not quiet: - warnings.warn('Input vector has fewer than 2 unmasked values! Returning zero vector.') - return numpy.zeros(y.size, dtype=y.dtype.name) - interpolator = interpolate.interp1d(x[numpy.invert(indx)], y[numpy.invert(indx)], - fill_value='extrapolate') - _y = y.data.copy() - _y[indx] = interpolator(x[indx]) - - if extrap_with_median: - med = numpy.median(interpolator.y) - m_indx = (x[indx] < interpolator.x[0]) | (x[indx] > interpolator.x[-1]) - _y[indx][m_indx] = med - - return _y - - -def boxcar_smooth_vector(x, boxcar, mask=None, lo=None, hi=None, niter=None, return_mask=False): - """ - Boxcar smooth an input vector. - - Ignores masked pixels. - - Allows for iterative positive and negative outliers. - - Can return the mask separately from the smoothed vector. - """ - if niter is not None and lo is None and hi is None: - raise ValueError('Must provide lo or hi if niter provided') - - _mask = numpy.zeros(x.size, dtype=bool) if mask is None else mask - _x = x if isinstance(x, numpy.ma.MaskedArray) else numpy.ma.MaskedArray(x) - _x[_mask] = numpy.ma.masked - - sx = interpolate_masked_vector( smooth_masked_vector(_x, boxcar) ) - -# pyplot.step(numpy.arange(_x.size), _x.data, where='mid', color='0.3', lw=0.5) -# pyplot.step(numpy.arange(_x.size), _x, where='mid', color='k', lw=0.5) -# pyplot.plot(numpy.arange(_x.size), sx, color='r', lw=1) -# pyplot.show() - - if numpy.all([ x is None for x in [ lo, hi, niter ]]): - if return_mask: - return sx, numpy.ma.getmaskarray(_x).copy() - return sx - - _niter = -1 if niter is None else niter - nrej = numpy.sum(numpy.ma.getmaskarray(_x)) - i=0 - while i > -1: - sig = numpy.std((_x - sx).compressed()) - mask = numpy.ma.getmaskarray(_x).copy() - if lo is not None: - mask = numpy.logical_or(mask, _x.data - sx < -lo*sig) - if hi is not None: - mask = numpy.logical_or(mask, _x.data - sx > hi*sig) - nrej = numpy.sum(mask) - numpy.sum(_x.mask) -# print('Niter: {0}; Nrej: {1}'.format(i+1, nrej)) - _x[mask] = numpy.ma.masked - sx = interpolate_masked_vector( smooth_masked_vector(_x, boxcar) ) - -# pyplot.step(numpy.arange(_x.size), _x.data, where='mid', color='cyan', lw=0.5) -# pyplot.step(numpy.arange(_x.size), _x, where='mid', color='k', lw=0.5) -# pyplot.plot(numpy.arange(_x.size), sx, color='r', lw=1) -# pyplot.show() - - i += 1 - if i == _niter or nrej == 0: - break - - pyplot.step(numpy.arange(_x.size), _x.data, where='mid', color='cyan', lw=0.5) - pyplot.step(numpy.arange(_x.size), _x, where='mid', color='k', lw=0.5) - pyplot.plot(numpy.arange(_x.size), sx, color='r', lw=1) - pyplot.show() - - if return_mask: - return sx, numpy.ma.getmaskarray(_x).copy() - return sx - - ->>>>>>> master From 78d18621e104dd2740ffdfe5b857b873e301305b Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Mon, 26 Mar 2018 09:45:21 -0700 Subject: [PATCH 10/36] lingering armsgs code --- pypit/arsetup.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pypit/arsetup.py b/pypit/arsetup.py index 938fe57fb4..775a4464de 100644 --- a/pypit/arsetup.py +++ b/pypit/arsetup.py @@ -15,7 +15,8 @@ from linetools import utils as ltu -from pypit import armsgs +#from pypit import armsgs +from pypit import msgs from pypit import arparse as settings from pypit import arutils from pypit.arflux import find_standard_file @@ -32,7 +33,7 @@ except NameError: pass # Logging -msgs = armsgs.get_logger() +#msgs = armsgs.get_logger() def dummy_setup_dict(filesort, fitsdict): @@ -528,4 +529,4 @@ def build_group_dict(filesort, setupIDs, sciexp, fitsdict): if key == 'science': # Add target name group_dict[config_key]['sciobj'].append(fitsdict['target'][scidx]) - return group_dict \ No newline at end of file + return group_dict From 4e0620beabd9081a30cf16eff61d9526e5f52147 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Mon, 26 Mar 2018 10:56:28 -0700 Subject: [PATCH 11/36] minor debugging; remove ginga imports from ardebug.py --- pypit/ardebug.py | 4 ++-- pypit/tests/test_pyputils.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pypit/ardebug.py b/pypit/ardebug.py index 9f1e6e4a3a..3e871d43a0 100644 --- a/pypit/ardebug.py +++ b/pypit/ardebug.py @@ -4,8 +4,8 @@ import numpy as np # These need to be outside of the def's -from pypit.ginga import show_image -from pypit.ginga import chk_arc_tilts +#from pypit.ginga import show_image +#from pypit.ginga import chk_arc_tilts def init(): """ diff --git a/pypit/tests/test_pyputils.py b/pypit/tests/test_pyputils.py index 4478d2399c..42a075e6ec 100644 --- a/pypit/tests/test_pyputils.py +++ b/pypit/tests/test_pyputils.py @@ -13,12 +13,12 @@ # data_dir = os.path.join(os.path.dirname(__file__), 'files') # return os.path.join(data_dir, filename) -from pypit import __version__, __last_update__ +from pypit import __version__, __last_updated__ def test_version(): # Dummy self # ver,upd = pyputils.get_version() - ver,upd = __version__, __last_update__ + ver,upd = __version__, __last_updated__ assert isinstance(ver,basestring) From 959d115ce376040eb4bfd10c5fa82be75d32ffc5 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Mon, 26 Mar 2018 18:04:03 -0700 Subject: [PATCH 12/36] try with version_check defined in setup.py --- setup.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index aaa740f91b..dc1271a87c 100755 --- a/setup.py +++ b/setup.py @@ -19,9 +19,25 @@ #from desiutil.setup import DesiTest, DesiVersion, get_version # Check dependencies +class VersionError(Exception): + pass + +minimum_versions = {'scipy': '0.17.0'} + + +def version_check(): + """ + Raises an error if there is a mismatched dependency. + """ + # loop through dependencies and versions + for dep, ver in minimum_versions.items(): + if LooseVersion(globals()[dep].__version__) < LooseVersion(ver): + raise VersionError('Update ' + dep + ' to at least version ' + ver + '!') + if sys.argv[1] != 'egg_info': - from pypit import archeck - archeck.version_check() +# from pypit import archeck +# archeck.version_check() + version_check() # # Begin setup From e9e686cdbc6a4a9c8414ecf7a77e72823f02ef38 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Mon, 26 Mar 2018 18:05:38 -0700 Subject: [PATCH 13/36] setup.py bug --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index dc1271a87c..c4be620820 100755 --- a/setup.py +++ b/setup.py @@ -19,12 +19,12 @@ #from desiutil.setup import DesiTest, DesiVersion, get_version # Check dependencies +from distutils.version import LooseVersion class VersionError(Exception): pass minimum_versions = {'scipy': '0.17.0'} - def version_check(): """ Raises an error if there is a mismatched dependency. From af68663a1caf77bf18c4340617197bf9cd443f9d Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Tue, 27 Mar 2018 07:03:02 -0700 Subject: [PATCH 14/36] bug in setup.py --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c4be620820..51c6cec3d9 100755 --- a/setup.py +++ b/setup.py @@ -19,12 +19,14 @@ #from desiutil.setup import DesiTest, DesiVersion, get_version # Check dependencies -from distutils.version import LooseVersion class VersionError(Exception): pass +import scipy + minimum_versions = {'scipy': '0.17.0'} +from distutils.version import LooseVersion def version_check(): """ Raises an error if there is a mismatched dependency. From 2224459ae5f21ba3841e2c2ace5d775c2225846c Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Wed, 28 Mar 2018 10:40:53 -0700 Subject: [PATCH 15/36] rewrite setup.py --- setup.py | 270 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 138 insertions(+), 132 deletions(-) diff --git a/setup.py b/setup.py index 51c6cec3d9..cb7eded6da 100755 --- a/setup.py +++ b/setup.py @@ -1,147 +1,153 @@ -#!/usr/bin/env python -# Licensed under a 3-clause BSD style license - see LICENSE.rst -from __future__ import absolute_import, division, print_function +# !usr/bin/env python +# -*- coding: utf-8 -*- # -# Standard imports +# Licensed under a 3-clause BSD license. # +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + import sys -import glob, os -import pdb -#from distutils.extension import Extension -# -# setuptools' sdist command ignores MANIFEST.in -# -#from distutils.command.sdist import sdist as DistutilsSdist +import os +import glob + from setuptools import setup, find_packages -# -# DESI support code. -# -#from desiutil.setup import DesiTest, DesiVersion, get_version -# Check dependencies -class VersionError(Exception): - pass +# For building in Cython functions ------------------------------------- +import numpy +from Cython.Distutils import build_ext +from distutils.extension import Extension -import scipy +def gsl_dirs(): + """ Test for and return the necessary GSL directories. """ + subdir = [ 'include', 'lib' ] + dirs = [] + for i,sd in enumerate(subdir): + dirs.append(os.path.join(os.getenv('GSL_PATH'), sd)) + if not os.path.isdir(dirs[i]): + raise NotADirectoryError('No directory {0}. '.format(dirs[i]) + + 'Must define GSL_PATH environmental variable.') + return tuple(dirs) -minimum_versions = {'scipy': '0.17.0'} -from distutils.version import LooseVersion -def version_check(): - """ - Raises an error if there is a mismatched dependency. - """ - # loop through dependencies and versions - for dep, ver in minimum_versions.items(): - if LooseVersion(globals()[dep].__version__) < LooseVersion(ver): - raise VersionError('Update ' + dep + ' to at least version ' + ver + '!') +def get_extensions(): + """ Build the extension modules for GSL. """ + include_gsl_dir, lib_gsl_dir = gsl_dirs() -if sys.argv[1] != 'egg_info': -# from pypit import archeck -# archeck.version_check() - version_check() + cython_files = glob.glob('pypit/*.pyx') + ext_modules = [] + for f in cython_files: + _f = f.split('.')[0].split('/')[1] + ext_modules.append(Extension('pypit.'+_f, [f], + include_dirs=[numpy.get_include(), include_gsl_dir], + library_dirs=[lib_gsl_dir], + libraries=['gsl','gslcblas'])) + return ext_modules +# ---------------------------------------------------------------------- -# -# Begin setup -# -setup_keywords = dict() -# -# THESE SETTINGS NEED TO BE CHANGED FOR EVERY PRODUCT. -# -setup_keywords['name'] = 'pypit' -setup_keywords['description'] = 'PYPIT Spectroscopic Reduction' -setup_keywords['author'] = 'PYPIT Collaboration' -setup_keywords['author_email'] = 'pypit@ucolick.org' -setup_keywords['license'] = 'BSD' -setup_keywords['url'] = 'https://github.com/pypit/pypit' -# -# END OF SETTINGS THAT NEED TO BE CHANGED. -# -setup_keywords['version'] = '0.8.dev0' #get_version(setup_keywords['name']) -# -# Use README.rst as long_description. -# -setup_keywords['long_description'] = '' -if os.path.exists('README.md'): - with open('README.md') as readme: - setup_keywords['long_description'] = readme.read() -# -# Set other keywords for the setup function. These are automated, & should -# be left alone unless you are an expert. -# -# Treat everything in bin/ except *.rst as a script to be installed. -# -if os.path.isdir('bin'): - setup_keywords['scripts'] = [fname for fname in glob.glob(os.path.join('bin', '*')) - if not os.path.basename(fname).endswith('.rst')] -setup_keywords['provides'] = [setup_keywords['name']] -setup_keywords['requires'] = ['Python (>2.7.0)'] -# setup_keywords['install_requires'] = ['Python (>2.7.0)'] -setup_keywords['zip_safe'] = False -setup_keywords['use_2to3'] = False -setup_keywords['packages'] = find_packages() -#setup_keywords['package_dir'] = {'':''} -#setup_keywords['cmdclass'] = {'version': DesiVersion, 'test': DesiTest, 'sdist': DistutilsSdist} -#setup_keywords['test_suite']='{name}.tests.{name}_test_suite.{name}_test_suite'.format(**setup_keywords) -setup_keywords['setup_requires']=['pytest-runner'] -setup_keywords['tests_require']=['pytest'] - -# Cython -import numpy, os -from Cython.Distutils import build_ext -from Cython.Build import cythonize -from distutils.extension import Extension -include_gsl_dir = os.getenv('GSL_PATH')+'/include/' -lib_gsl_dir = os.getenv('GSL_PATH')+'/lib/' -pyx_files = glob.glob('pypit/*.pyx') -setup_keywords['ext_modules']=[] -for pyx_file in pyx_files: - pyx_split = pyx_file.split('.') - pyx_split2 = pyx_split[0].split('/') - # Generate Extension - #ext = Extension(pyx_split2[1], [pyx_file], - ext = Extension('pypit.'+pyx_split2[1], [pyx_file], - include_dirs=[numpy.get_include(), - include_gsl_dir], - library_dirs=[lib_gsl_dir], - libraries=["gsl","gslcblas"] - ) - # Append - setup_keywords['ext_modules'].append(ext) -#for pyx_file in pyx_files: -# pyx_split = pyx_file.split('/') -# ext = cythonize(pyx_split[1]) -# setup_keywords['ext_modules'].append(ext) - -setup_keywords['cmdclass']={'build_ext': build_ext} - -# Autogenerate command-line scripts. -# -# setup_keywords['entry_points'] = {'console_scripts':['desiInstall = desiutil.install.main:main']} +def get_data_files(): + """ Build the list of data files to include. """ + data_files = [] -# -# Add internal data directories. -# + # Walk through the data directory, adding all files + data_generator = os.walk('pypit/data') + for path, directories, files in data_generator: + for f in files: + data_path = '/'.join(path.split('/')[1:]) + data_files.append(os.path.join(data_path, f)) -data_files = [] - -# walk through the data directory, adding all files -data_generator = os.walk('pypit/data') -for path, directories, files in data_generator: - for f in files: - data_path = '/'.join(path.split('/')[1:]) - data_files.append(data_path + '/' + f) -# add pipeline and spectrograph settings -settings = glob.glob('pypit/settings/settings.*') -settings = ['/'.join(path.split('/')[1:]) for path in settings] -data_files.extend(settings) -setup_keywords['package_data'] = {'pypit': data_files, - '': ['*.rst', '*.txt']} -setup_keywords['include_package_data'] = True + # Add pipeline and spectrograph settings + settings = glob.glob('pypit/settings/settings.*') + settings = ['/'.join(path.split('/')[1:]) for path in settings] + data_files.extend(settings) + + return data_files + + +def get_scripts(): + """ Grab all the scripts in the bin directory. """ + scripts = [] + if os.path.isdir('bin'): + scripts = [ fname for fname in glob.glob(os.path.join('bin', '*')) + if not os.path.basename(fname).endswith('.rst') ] + return scripts + + +def get_requirements(): + """ Get the requirements from a system file. """ + name = 'requirements.txt' + + requirements_file = os.path.join(os.path.dirname(__file__), name) + install_requires = [line.strip().replace('==', '>=') for line in open(requirements_file) + if not line.strip().startswith('#') and line.strip() != ''] + return install_requires + + +NAME = 'pypit' +# do not use x.x.x-dev. things complain. instead use x.x.xdev +VERSION = '0.8.0dev' +RELEASE = 'dev' not in VERSION + + +def run_setup(data_files, scripts, packages, ext_modules, install_requires): + + # TODO: Are any/all of the *'d keyword arguments needed? I.e., what + # are the default values? + + setup(name=NAME, + provides=NAME, # * + version=VERSION, + license='BSD3', # TODO: Or is this 'BSD'? + description='PYPIT Spectroscopic Reduction', + long_description=open('README.md').read(), + author='PYPIT Collaboration', + author_email='pypit@ucolick.org', + keywords='pypit PYPIT astronomy Keck UCO Lick data reduction', + url='https://github.com/pypit/pypit', + packages=packages, + package_data={'pypit': data_files, '': ['*.rst', '*.txt']}, + include_package_data=True, + scripts=scripts, + install_requires=install_requires, + requires=[ 'Python (>2.7.0)' ], # * + zip_safe=False, # * + use_2to3=False, # * + setup_requires=[ 'pytest-runner' ], # * + tests_require=[ 'pytest' ], # * + ext_modules=ext_modules, # only needed for cython + cmdclass={'build_ext': build_ext}, # only needed for cython + classifiers=[ # TODO: Check these + 'Development Status :: 4 - Beta', + 'Intended Audience :: Science/Research', + 'License :: OSI Approved :: BSD License', + 'Natural Language :: English', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', + 'Topic :: Documentation :: Sphinx', + 'Topic :: Scientific/Engineering :: Astronomy', + 'Topic :: Software Development :: Libraries :: Python Modules', + 'Topic :: Software Development :: User Interfaces' + ], + ) + +#----------------------------------------------------------------------- +if __name__ == '__main__': + + # Compile the data files to include + data_files = get_data_files() + # Compile the scripts in the bin/ directory + scripts = get_scripts() + # Get the packages to include + packages = find_packages() + # Include the module extensions; CURRENTLY ONLY NEEDED FOR CYTHON + ext_modules = get_extensions() + # Collate the dependencies based on the system text file + install_requires = get_requirements() + # Run setup from setuptools + run_setup(data_files, scripts, packages, ext_modules, install_requires) -# -# Run setup command. -# -setup(**setup_keywords) From 1263449e680539d3819c1532188f7f4541dbc723 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Wed, 28 Mar 2018 10:46:40 -0700 Subject: [PATCH 16/36] add requirements.txt and edited CHANGES.rst --- CHANGES.rst | 1 + requirements.txt | 29 +++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 requirements.txt diff --git a/CHANGES.rst b/CHANGES.rst index 381d43f26c..f1245ad971 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -37,6 +37,7 @@ * Removed majority of cython functionality * Moved logging to be a package object using the main __init__.py file * Begin to adhere to PEP8 (mostly) +* setup.py rewritten. Modeled after https://github.com/sdss/marvin/blob/master/setup.py . Added requirements.txt with the package versions required. 0.7 (2017-02-07) ---------------- diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000..ced754ae2d --- /dev/null +++ b/requirements.txt @@ -0,0 +1,29 @@ +# Required packages for PYPIT +# +# TODO: Make linetools and arclines pip installable so that they can be +# added here, or add them as submodule dependencies in extern/ directory +# +# Users +numpy>=1.11.2 +scipy>=0.18.1 +matplotlib>=1.5.3 +astropy>=1.2.1 +Cython>=0.27.3 +PyYAML>=3.11 +future>=0.16.0 + +# PyQt5>=5.10 ? + +# linetools +# arclines + +# Develop +ginga>=2.7.0 +h5py>=2.7.1 +pytest>=3.0.7 +# pickle? + +# Documentation +sphinx>=1.6 + + From 14c547785282d8177eded81a3da08d1694765cf8 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Wed, 28 Mar 2018 11:42:38 -0700 Subject: [PATCH 17/36] remove import of unicode_literals in setup.py --- setup.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index cb7eded6da..a19bbb0877 100755 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from __future__ import unicode_literals +#from __future__ import unicode_literals import sys import os @@ -38,8 +38,9 @@ def get_extensions(): cython_files = glob.glob('pypit/*.pyx') ext_modules = [] for f in cython_files: - _f = f.split('.')[0].split('/')[1] - ext_modules.append(Extension('pypit.'+_f, [f], + name = ('pypit.'+f.split('.')[0].split('/')[1]) + sources = [f] + ext_modules.append(Extension(name, sources, include_dirs=[numpy.get_include(), include_gsl_dir], library_dirs=[lib_gsl_dir], libraries=['gsl','gslcblas'])) From c03dd7995091539863ad8efa96b2fec7d4275f7b Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Wed, 28 Mar 2018 12:39:50 -0700 Subject: [PATCH 18/36] add call to cythonize --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a19bbb0877..bd7d3b7398 100755 --- a/setup.py +++ b/setup.py @@ -16,6 +16,7 @@ # For building in Cython functions ------------------------------------- import numpy +from Cython.Build import cythonize from Cython.Distutils import build_ext from distutils.extension import Extension @@ -44,7 +45,7 @@ def get_extensions(): include_dirs=[numpy.get_include(), include_gsl_dir], library_dirs=[lib_gsl_dir], libraries=['gsl','gslcblas'])) - return ext_modules + return cythonize(ext_modules) # ---------------------------------------------------------------------- From e291b98854e3332f32bfb2ea6e98b2490bbd4750 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Wed, 28 Mar 2018 16:09:07 -0700 Subject: [PATCH 19/36] try without cythonize, but with cython as a setup requirement --- setup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index bd7d3b7398..4e7b614a27 100755 --- a/setup.py +++ b/setup.py @@ -45,7 +45,8 @@ def get_extensions(): include_dirs=[numpy.get_include(), include_gsl_dir], library_dirs=[lib_gsl_dir], libraries=['gsl','gslcblas'])) - return cythonize(ext_modules) +# return cythonize(ext_modules) + return ext_modules # ---------------------------------------------------------------------- @@ -116,7 +117,7 @@ def run_setup(data_files, scripts, packages, ext_modules, install_requires): requires=[ 'Python (>2.7.0)' ], # * zip_safe=False, # * use_2to3=False, # * - setup_requires=[ 'pytest-runner' ], # * + setup_requires=[ 'pytest-runner', 'cython>=0.27' ], # * tests_require=[ 'pytest' ], # * ext_modules=ext_modules, # only needed for cython cmdclass={'build_ext': build_ext}, # only needed for cython From bfb9b6c7cd95dcc900a72282e59988f973dec87c Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Wed, 28 Mar 2018 17:26:07 -0700 Subject: [PATCH 20/36] attempting to fix imports --- pypit/__init__.py | 17 +++++++++++------ pypit/armsgs.py | 9 +++++---- pypit/arplot.py | 3 +++ pypit/pyputils.py | 5 ++++- pypit/tests/test_ararc.py | 7 +++++-- pypit/tests/test_ararclines.py | 5 ++++- pypit/tests/test_arcoadd.py | 4 ++++ pypit/tests/test_arflexure.py | 4 ++++ pypit/tests/test_arflux.py | 4 ++++ pypit/tests/test_arload.py | 4 ++++ pypit/tests/test_armasters.py | 4 ++++ pypit/tests/test_armbase.py | 4 ++++ pypit/tests/test_armeta.py | 4 ++++ pypit/tests/test_armlsd.py | 4 ++++ pypit/tests/test_armsgs.py | 4 ++++ pypit/tests/test_arparse.py | 4 ++++ pypit/tests/test_arqa.py | 4 ++++ pypit/tests/test_arsave.py | 4 ++++ pypit/tests/test_arsort.py | 4 ++++ pypit/tests/test_arspecobj.py | 4 ++++ pypit/tests/test_arutils.py | 4 ++++ pypit/tests/test_arvcorr.py | 4 ++++ pypit/tests/test_datasec.py | 4 ++++ pypit/tests/test_objfind.py | 4 ++++ pypit/tests/test_pypit.py | 4 ++++ pypit/tests/test_pyputils.py | 4 ++++ pypit/tests/test_scripts.py | 4 ++++ pypit/tests/test_settings.py | 4 ++++ pypit/tests/test_setups.py | 4 ++++ 29 files changed, 124 insertions(+), 14 deletions(-) diff --git a/pypit/__init__.py b/pypit/__init__.py index 26b5901186..5dfb912bc7 100644 --- a/pypit/__init__.py +++ b/pypit/__init__.py @@ -5,20 +5,25 @@ that can be imported by submodules. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +# Imports for signal and log handling +import sys +import signal +import warnings + # Set version __version__ = '0.8.0.dev0' # TODO: Do we need this? A release should be tied to a date, and the # date information could be put in the CHANGES.rst __last_updated__ = '23Mar2018' -# Imports for singal and log handling -import sys -import signal -import warnings - # Import and instantiate the logger from pypit import armsgs -msgs = armsgs.Messages() +msgs = armsgs.Messages(pypit_version=__version__, pypit_updated=__last_updated__) # Import the close_qa method so that it can be called when a hard stop # is requested by the user diff --git a/pypit/armsgs.py b/pypit/armsgs.py index fc4641890c..19ec0f79fa 100644 --- a/pypit/armsgs.py +++ b/pypit/armsgs.py @@ -21,7 +21,7 @@ import numpy import astropy -from pypit import __version__, __last_updated__ +# from pypit import __version__, __last_updated__ #pypit_logger = None @@ -51,14 +51,15 @@ class Messages: If true, the screen output will have colors, otherwise normal screen output will be displayed """ - def __init__(self, log=None, debug=None, verbosity=None, colors=True): + def __init__(self, log=None, debug=None, verbosity=None, colors=True, + pypit_version=None, pypit_updated=None): # Initialize other variables # TODO: debug could just be develop=True or False self._debug = debug - self._last_updated = __last_updated__ - self._version = __version__ self._verbosity = 1 if verbosity is None else verbosity + self._last_updated = pypit_updated + self._version = pypit_version # TODO: Why are these two necessary? It would seem better to # provide Messages with member functions that can operate on diff --git a/pypit/arplot.py b/pypit/arplot.py index d765b9eaaf..a95f842110 100644 --- a/pypit/arplot.py +++ b/pypit/arplot.py @@ -1,3 +1,6 @@ + +from __future__ import (print_function, absolute_import, division, unicode_literals) + import numpy as np from matplotlib import pyplot as plt diff --git a/pypit/pyputils.py b/pypit/pyputils.py index f8bcc1a5b2..e1d9a36dab 100644 --- a/pypit/pyputils.py +++ b/pypit/pyputils.py @@ -11,6 +11,8 @@ from pypit import armsgs from pypit import ardebug +from pypit import __version__, __last_updated__ + # this_file = realpath(__file__) # this_path = this_file[:this_file.rfind('/')] @@ -52,7 +54,8 @@ def get_dummy_logger(develop=False): """ debug = ardebug.init() debug['develop'] = develop - return armsgs.Messages(log=None, debug=debug, verbosity=0) + return armsgs.Messages(log=None, debug=debug, verbosity=0, pypit_version=__version__, + pypit_updated=__last_updated__) # from pypit import ardebug # from pypit import armsgs as pyparm diff --git a/pypit/tests/test_ararc.py b/pypit/tests/test_ararc.py index 758c7e06a0..b47a7a4d61 100644 --- a/pypit/tests/test_ararc.py +++ b/pypit/tests/test_ararc.py @@ -1,12 +1,15 @@ # Module to run tests on ararclines - +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals import os import numpy as np import pytest from pypit import pyputils -msgs = pyputils.get_dummy_logger() +#msgs = pyputils.get_dummy_logger() from pypit import arparse as settings from pypit import ararc as pyarc from pypit import arutils as arut diff --git a/pypit/tests/test_ararclines.py b/pypit/tests/test_ararclines.py index 1c33148abf..1095005f37 100644 --- a/pypit/tests/test_ararclines.py +++ b/pypit/tests/test_ararclines.py @@ -1,5 +1,8 @@ # Module to run tests on ararclines - +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals import numpy as np import pytest diff --git a/pypit/tests/test_arcoadd.py b/pypit/tests/test_arcoadd.py index 3c23fc2e95..b1fd07eb26 100644 --- a/pypit/tests/test_arcoadd.py +++ b/pypit/tests/test_arcoadd.py @@ -1,4 +1,8 @@ # Module to run tests on arcoadd +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals ### TEST_UNICODE_LITERALS diff --git a/pypit/tests/test_arflexure.py b/pypit/tests/test_arflexure.py index 3186b65f39..815d79b10b 100644 --- a/pypit/tests/test_arflexure.py +++ b/pypit/tests/test_arflexure.py @@ -1,4 +1,8 @@ # Module to run tests on simple fitting routines for arrays +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals ### TEST_UNICODE_LITERALS diff --git a/pypit/tests/test_arflux.py b/pypit/tests/test_arflux.py index 2ad0b833a2..bb37589b06 100644 --- a/pypit/tests/test_arflux.py +++ b/pypit/tests/test_arflux.py @@ -1,4 +1,8 @@ # Module to run tests on simple fitting routines for arrays +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals ### TEST_UNICODE_LITERALS diff --git a/pypit/tests/test_arload.py b/pypit/tests/test_arload.py index 91de3cd5ef..4d8cbf1cce 100644 --- a/pypit/tests/test_arload.py +++ b/pypit/tests/test_arload.py @@ -1,4 +1,8 @@ # Module to run tests on arload +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals ### TEST_UNICODE_LITERALS diff --git a/pypit/tests/test_armasters.py b/pypit/tests/test_armasters.py index 483178c138..1404c56a61 100644 --- a/pypit/tests/test_armasters.py +++ b/pypit/tests/test_armasters.py @@ -1,4 +1,8 @@ # Module to run tests on armasters +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals import pytest diff --git a/pypit/tests/test_armbase.py b/pypit/tests/test_armbase.py index 1950ee3b0c..2c7922df9b 100644 --- a/pypit/tests/test_armbase.py +++ b/pypit/tests/test_armbase.py @@ -1,4 +1,8 @@ # Module to run tests on armbase module +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals # TEST_UNICODE_LITERALS diff --git a/pypit/tests/test_armeta.py b/pypit/tests/test_armeta.py index 4aed302fa3..1a531cfacc 100644 --- a/pypit/tests/test_armeta.py +++ b/pypit/tests/test_armeta.py @@ -1,4 +1,8 @@ # Module to run tests on ararclines +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals import os diff --git a/pypit/tests/test_armlsd.py b/pypit/tests/test_armlsd.py index c88ab568fb..e889490b92 100644 --- a/pypit/tests/test_armlsd.py +++ b/pypit/tests/test_armlsd.py @@ -1,4 +1,8 @@ # Module to run tests on armsld module +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals # TEST_UNICODE_LITERALS diff --git a/pypit/tests/test_armsgs.py b/pypit/tests/test_armsgs.py index 833d5e75ab..124b1d0e95 100644 --- a/pypit/tests/test_armsgs.py +++ b/pypit/tests/test_armsgs.py @@ -1,4 +1,8 @@ # Module to run tests on armsgs +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals import numpy as np diff --git a/pypit/tests/test_arparse.py b/pypit/tests/test_arparse.py index cb82d41854..7a986734ac 100644 --- a/pypit/tests/test_arparse.py +++ b/pypit/tests/test_arparse.py @@ -1,4 +1,8 @@ # Module to run tests on arparse +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals import pytest diff --git a/pypit/tests/test_arqa.py b/pypit/tests/test_arqa.py index 460062710e..edecf8d916 100644 --- a/pypit/tests/test_arqa.py +++ b/pypit/tests/test_arqa.py @@ -1,4 +1,8 @@ # Module to run tests on arqa +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals from pypit import pyputils from pypit import arqa diff --git a/pypit/tests/test_arsave.py b/pypit/tests/test_arsave.py index 71d1b6591c..0af474ba3e 100644 --- a/pypit/tests/test_arsave.py +++ b/pypit/tests/test_arsave.py @@ -1,4 +1,8 @@ # Module to run tests on arsave +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals import numpy as np diff --git a/pypit/tests/test_arsort.py b/pypit/tests/test_arsort.py index 7b1a495b2d..acb6e3700f 100644 --- a/pypit/tests/test_arsort.py +++ b/pypit/tests/test_arsort.py @@ -1,4 +1,8 @@ # Module to run tests on arsort and arsetup +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals import pytest diff --git a/pypit/tests/test_arspecobj.py b/pypit/tests/test_arspecobj.py index 1c8573d01b..4f0527895d 100644 --- a/pypit/tests/test_arspecobj.py +++ b/pypit/tests/test_arspecobj.py @@ -1,4 +1,8 @@ # Module to run tests on simple fitting routines for arrays +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals ### TEST_UNICODE_LITERALS diff --git a/pypit/tests/test_arutils.py b/pypit/tests/test_arutils.py index 699f6ee26a..9f8f128960 100644 --- a/pypit/tests/test_arutils.py +++ b/pypit/tests/test_arutils.py @@ -1,4 +1,8 @@ # Module to run tests on ararclines +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals import os diff --git a/pypit/tests/test_arvcorr.py b/pypit/tests/test_arvcorr.py index b94c6f1cdc..01d8487776 100644 --- a/pypit/tests/test_arvcorr.py +++ b/pypit/tests/test_arvcorr.py @@ -1,4 +1,8 @@ # Module to run tests on arvcorr +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals import numpy as np import pytest diff --git a/pypit/tests/test_datasec.py b/pypit/tests/test_datasec.py index f3c97f59e3..9f535c6613 100644 --- a/pypit/tests/test_datasec.py +++ b/pypit/tests/test_datasec.py @@ -1,4 +1,8 @@ # Module to run tests on ampsec definition +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals import pytest diff --git a/pypit/tests/test_objfind.py b/pypit/tests/test_objfind.py index f320d806ad..91ff203a3e 100644 --- a/pypit/tests/test_objfind.py +++ b/pypit/tests/test_objfind.py @@ -1,4 +1,8 @@ # Module to run tests on ararclines +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals import os diff --git a/pypit/tests/test_pypit.py b/pypit/tests/test_pypit.py index 24fd13ba29..25a045dc5c 100644 --- a/pypit/tests/test_pypit.py +++ b/pypit/tests/test_pypit.py @@ -1,4 +1,8 @@ # Module to run tests on arsave +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals import os import pytest diff --git a/pypit/tests/test_pyputils.py b/pypit/tests/test_pyputils.py index 42a075e6ec..16c045fc06 100644 --- a/pypit/tests/test_pyputils.py +++ b/pypit/tests/test_pyputils.py @@ -1,4 +1,8 @@ # Module to run tests on simple fitting routines for arrays +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals # TEST_UNICODE_LITERALS diff --git a/pypit/tests/test_scripts.py b/pypit/tests/test_scripts.py index 74a5f9d349..b80cfe70fe 100644 --- a/pypit/tests/test_scripts.py +++ b/pypit/tests/test_scripts.py @@ -1,4 +1,8 @@ # Module to run tests on scripts +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals import matplotlib matplotlib.use('agg') # For Travis diff --git a/pypit/tests/test_settings.py b/pypit/tests/test_settings.py index 5022b9768a..165c5b84b9 100644 --- a/pypit/tests/test_settings.py +++ b/pypit/tests/test_settings.py @@ -1,4 +1,8 @@ # Module to run tests on scripts +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals import matplotlib matplotlib.use('agg') # For Travis diff --git a/pypit/tests/test_setups.py b/pypit/tests/test_setups.py index 47fa26db20..2f715ddf99 100644 --- a/pypit/tests/test_setups.py +++ b/pypit/tests/test_setups.py @@ -1,4 +1,8 @@ # Module to run tests on scripts +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals import matplotlib matplotlib.use('agg') # For Travis From c44aa56cb21da4c7c255d19e3af5a9c718adf884 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Thu, 29 Mar 2018 07:09:57 -0700 Subject: [PATCH 21/36] bug fix --- pypit/__init__.py | 2 +- pypit/armsgs.py | 13 ++++++------- pypit/pyputils.py | 5 ++--- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/pypit/__init__.py b/pypit/__init__.py index 5dfb912bc7..801acb8e96 100644 --- a/pypit/__init__.py +++ b/pypit/__init__.py @@ -23,7 +23,7 @@ # Import and instantiate the logger from pypit import armsgs -msgs = armsgs.Messages(pypit_version=__version__, pypit_updated=__last_updated__) +msgs = armsgs.Messages() # Import the close_qa method so that it can be called when a hard stop # is requested by the user diff --git a/pypit/armsgs.py b/pypit/armsgs.py index 19ec0f79fa..9754b7fb50 100644 --- a/pypit/armsgs.py +++ b/pypit/armsgs.py @@ -21,7 +21,7 @@ import numpy import astropy -# from pypit import __version__, __last_updated__ +from pypit import __version__, __last_updated__ #pypit_logger = None @@ -51,15 +51,14 @@ class Messages: If true, the screen output will have colors, otherwise normal screen output will be displayed """ - def __init__(self, log=None, debug=None, verbosity=None, colors=True, - pypit_version=None, pypit_updated=None): + def __init__(self, log=None, debug=None, verbosity=None, colors=True): # Initialize other variables # TODO: debug could just be develop=True or False self._debug = debug self._verbosity = 1 if verbosity is None else verbosity - self._last_updated = pypit_updated - self._version = pypit_version + self._last_updated = __last_updated__ + self._version = __version__ # TODO: Why are these two necessary? It would seem better to # provide Messages with member functions that can operate on @@ -131,9 +130,9 @@ def _initialize_log_file(self, log=None): self._log = open(log, 'w') self._log.write("------------------------------------------------------\n\n") - self._log.write("PYPIT was last updated {0:s}\n".format(__last_updated__)) + self._log.write("PYPIT was last updated {0:s}\n".format(self._last_updated)) self._log.write("This log was generated with version {0:s} of PYPIT\n\n".format( - __version__)) + self._version)) self._log.write("You are using scipy version={:s}\n".format(scipy.__version__)) self._log.write("You are using numpy version={:s}\n".format(numpy.__version__)) self._log.write("You are using astropy version={:s}\n\n".format(astropy.__version__)) diff --git a/pypit/pyputils.py b/pypit/pyputils.py index e1d9a36dab..f375806a6f 100644 --- a/pypit/pyputils.py +++ b/pypit/pyputils.py @@ -11,7 +11,7 @@ from pypit import armsgs from pypit import ardebug -from pypit import __version__, __last_updated__ +#from pypit import __version__, __last_updated__ # this_file = realpath(__file__) # this_path = this_file[:this_file.rfind('/')] @@ -54,8 +54,7 @@ def get_dummy_logger(develop=False): """ debug = ardebug.init() debug['develop'] = develop - return armsgs.Messages(log=None, debug=debug, verbosity=0, pypit_version=__version__, - pypit_updated=__last_updated__) + return armsgs.Messages(log=None, debug=debug, verbosity=0) # from pypit import ardebug # from pypit import armsgs as pyparm From f91a2a4791eef77cdb975b5d47b83759aa9c69ed Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Thu, 29 Mar 2018 07:26:04 -0700 Subject: [PATCH 22/36] remove arqa import from init --- pypit/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pypit/__init__.py b/pypit/__init__.py index 801acb8e96..9ee46c710f 100644 --- a/pypit/__init__.py +++ b/pypit/__init__.py @@ -29,7 +29,9 @@ # is requested by the user # TODO: Is this necessary? As far as I can tell, close_qa() doesn't # actually close any already opened qa files -from pypit.arqa import close_qa + +# TODO: Causes circular imports +# from pypit.arqa import close_qa # Send all signals to messages to be dealt with (i.e. someone hits ctrl+c) def signal_handler(signalnum, handler): @@ -38,7 +40,7 @@ def signal_handler(signalnum, handler): """ if signalnum == 2: msgs.info('Ctrl+C was pressed. Ending processes...') - close_qa(msgs.pypit_file) +# close_qa(msgs.pypit_file) msgs.close() sys.exit() From 1d7e5c2bd447fc6f21bf4bc63d2a77c2b24ab9ee Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Fri, 30 Mar 2018 08:54:09 -0700 Subject: [PATCH 23/36] reorganize imports; move most qa plotting routines to relevant module; optimal extraction function now returns model --- pypit/ararc.py | 137 ++++- pypit/arcoadd.py | 118 +++- pypit/arextract.py | 100 +++- pypit/armed.py | 26 +- pypit/armlsd.py | 12 +- pypit/arpca.py | 253 ++++++++- pypit/arplot.py | 7 +- pypit/arproc.py | 385 +++++++++++-- pypit/arqa.py | 1089 +------------------------------------ pypit/arsciexp.py | 22 +- pypit/artrace.py | 430 +++++++++++++-- pypit/tests/test_ararc.py | 19 +- 12 files changed, 1330 insertions(+), 1268 deletions(-) diff --git a/pypit/ararc.py b/pypit/ararc.py index 48d8591b2c..19801bd304 100644 --- a/pypit/ararc.py +++ b/pypit/ararc.py @@ -1,23 +1,24 @@ from __future__ import (print_function, absolute_import, division, unicode_literals) +import os +import time +import inspect + import numpy as np +from matplotlib import pyplot as plt +from matplotlib import gridspec, font_manager + +from arclines.holy.grail import basic, semi_brute, general + from pypit import arpca from pypit import arparse as settings -#from pypit import armsgs from pypit import msgs from pypit import arsave from pypit import arutils from pypit import ararclines from pypit import arqa -from matplotlib import pyplot as plt -import os -import time - from pypit import ardebug as debugger - -# Logging -#msgs = armsgs.get_logger() - +from pypit import arcyarc def detect_lines(slf, det, msarc, censpec=None, MK_SATMASK=False): """ @@ -54,7 +55,6 @@ def detect_lines(slf, det, msarc, censpec=None, MK_SATMASK=False): The spectrum used to find detections. This spectrum has had any "continuum" emission subtracted off """ - from pypit import arcyarc # Extract a rough spectrum of the arc in each order msgs.info("Detecting lines") msgs.info("Extracting an approximate arc spectrum at the centre of the chip") @@ -549,7 +549,8 @@ def simple_calib(slf, det, get_poly=False): xrej=xrej, yrej=yrej, mask=mask, spec=yprep, nrej=aparm['nsig_rej_final'], shift=0., tcent=tcent) # QA - arqa.arc_fit_qa(slf, final_fit) +# arqa.arc_fit_qa(slf, final_fit) + arc_fit_qa(slf, final_fit) # RMS rms_ang = arutils.calc_fit_rms(xfit, yfit, fit, aparm['func'], minv=fmin, maxv=fmax) wave = arutils.func_val(fit, np.arange(slf._msarc[det-1].shape[0])/float(slf._msarc[det-1].shape[0]), @@ -575,7 +576,6 @@ def calib_with_arclines(slf, det, get_poly=False, use_method="general"): final_fit : dict Dict of fit info """ - from arclines.holy.grail import basic, semi_brute, general # Parameters (just for convenience) aparm = slf._arcparam[det-1] # Extract the arc @@ -590,7 +590,8 @@ def calib_with_arclines(slf, det, get_poly=False, use_method="general"): else: # Now preferred best_dict, final_fit = general(spec, aparm['lamps'], fit_parm=aparm, min_ampl=aparm['min_ampl']) - arqa.arc_fit_qa(slf, final_fit) +# arqa.arc_fit_qa(slf, final_fit) + arc_fit_qa(slf, final_fit) # return final_fit @@ -662,3 +663,113 @@ def new_saturation_mask(a, satlevel): return mask.astype(int) + +def arc_fit_qa(slf, fit, outfile=None, ids_only=False, title=None): + """ + QA for Arc spectrum + + Parameters + ---------- + fit : Wavelength fit + arc_spec : ndarray + Arc spectrum + outfile : str, optional + Name of output file + """ + + plt.rcdefaults() + plt.rcParams['font.family']= 'times new roman' + + # Grab the named of the method + method = inspect.stack()[0][3] + # Outfil + if outfile is None: + outfile = arqa.set_qa_filename(slf.setup, method) + # + arc_spec = fit['spec'] + + # Begin + if not ids_only: + plt.figure(figsize=(8, 4.0)) + plt.clf() + gs = gridspec.GridSpec(2, 2) + idfont = 'xx-small' + else: + plt.figure(figsize=(11, 8.5)) + plt.clf() + gs = gridspec.GridSpec(1, 1) + idfont = 'small' + + # Simple spectrum plot + ax_spec = plt.subplot(gs[:,0]) + ax_spec.plot(np.arange(len(arc_spec)), arc_spec) + ymin, ymax = 0., np.max(arc_spec) + ysep = ymax*0.03 + for kk, x in enumerate(fit['xfit']*fit['xnorm']): + yline = np.max(arc_spec[int(x)-2:int(x)+2]) + # Tick mark + ax_spec.plot([x,x], [yline+ysep*0.25, yline+ysep], 'g-') + # label + ax_spec.text(x, yline+ysep*1.3, + '{:s} {:g}'.format(fit['ions'][kk], fit['yfit'][kk]), ha='center', va='bottom', + size=idfont, rotation=90., color='green') + ax_spec.set_xlim(0., len(arc_spec)) + ax_spec.set_ylim(ymin, ymax*1.2) + ax_spec.set_xlabel('Pixel') + ax_spec.set_ylabel('Flux') + if title is not None: + ax_spec.text(0.04, 0.93, title, transform=ax_spec.transAxes, + size='x-large', ha='left')#, bbox={'facecolor':'white'}) + if ids_only: + plt.tight_layout(pad=0.2, h_pad=0.0, w_pad=0.0) + plt.savefig(outfile, dpi=800) + plt.close() + return + + # Arc Fit + ax_fit = plt.subplot(gs[0, 1]) + # Points + ax_fit.scatter(fit['xfit']*fit['xnorm'], fit['yfit'], marker='x') + if len(fit['xrej']) > 0: + ax_fit.scatter(fit['xrej']*fit['xnorm'], fit['yrej'], marker='o', + edgecolor='gray', facecolor='none') + # Solution + xval = np.arange(len(arc_spec)) + wave = arutils.func_val(fit['fitc'], xval/fit['xnorm'], 'legendre', + minv=fit['fmin'], maxv=fit['fmax']) + ax_fit.plot(xval, wave, 'r-') + xmin, xmax = 0., len(arc_spec) + ax_fit.set_xlim(xmin, xmax) + ymin,ymax = np.min(wave)*.95, np.max(wave)*1.05 + ax_fit.set_ylim(np.min(wave)*.95, np.max(wave)*1.05) + ax_fit.set_ylabel('Wavelength') + ax_fit.get_xaxis().set_ticks([]) # Suppress labeling + # Stats + wave_fit = arutils.func_val(fit['fitc'], fit['xfit'], 'legendre', + minv=fit['fmin'], maxv=fit['fmax']) + rms = np.sqrt(np.sum((fit['yfit']-wave_fit)**2)/len(fit['xfit'])) # Ang + dwv_pix = np.median(np.abs(wave-np.roll(wave,1))) + ax_fit.text(0.1*len(arc_spec), 0.90*ymin+(ymax-ymin), + r'$\Delta\lambda$={:.3f}$\AA$ (per pix)'.format(dwv_pix), size='small') + ax_fit.text(0.1*len(arc_spec), 0.80*ymin+(ymax-ymin), + 'RMS={:.3f} (pixels)'.format(rms/dwv_pix), size='small') + # Arc Residuals + ax_res = plt.subplot(gs[1,1]) + res = fit['yfit']-wave_fit + ax_res.scatter(fit['xfit']*fit['xnorm'], res/dwv_pix, marker='x') + ax_res.plot([xmin,xmax], [0.,0], 'k--') + ax_res.set_xlim(xmin, xmax) + ax_res.set_xlabel('Pixel') + ax_res.set_ylabel('Residuals (Pix)') + + # Finish + plt.tight_layout(pad=0.2, h_pad=0.0, w_pad=0.0) + plt.savefig(outfile, dpi=800) + plt.close() + + plt.rcdefaults() + + return + + + diff --git a/pypit/arcoadd.py b/pypit/arcoadd.py index 9364974b8e..b32b495f8b 100644 --- a/pypit/arcoadd.py +++ b/pypit/arcoadd.py @@ -3,24 +3,25 @@ from __future__ import absolute_import, division, print_function import numpy as np -import astropy.stats -import scipy.interpolate +from numpy.ma.core import MaskedArray +import scipy.stats from scipy.signal import medfilt -from astropy import units as u -from astropy.stats import sigma_clipped_stats +from matplotlib import pyplot as plt +from matplotlib import gridspec, font_manager +from matplotlib.backends.backend_pdf import PdfPages + +import astropy.stats +from astropy import units + +from linetools.spectra.xspectrum1d import XSpectrum1D +from linetools.spectra.utils import collate -from pypit import arload -#from pypit import armsgs from pypit import msgs -from pypit import arqa +from pypit import arload from pypit import arutils - from pypit import ardebug as debugger -# Logging -#msgs = armsgs.get_logger() - # TODO # Shift spectra # Scale by poly @@ -60,7 +61,6 @@ def new_wave_grid(waves, wave_method='iref', iref=0, A_pix=None, v_pix=None, **k # slf._argflag['reduce']['pixelsize'] = 2.5?? This won't work # if running coadding outside of PYPIT, which we'd like as an # option! - from numpy.ma.core import MaskedArray if not isinstance(waves, MaskedArray): waves = np.ma.array(waves) @@ -305,7 +305,7 @@ def median_ratio_flux(spec, smask, ispec, iref, nsig=3., niter=5, **kwargs): # Ratio med_flux = fluxes[iref,allok] / fluxes[ispec,allok] # Clip - mn_scale, med_scale, std_scale = sigma_clipped_stats(med_flux, sigma=nsig, iters=niter, **kwargs) + mn_scale, med_scale, std_scale = astropy.stats.sigma_clipped_stats(med_flux, sigma=nsig, iters=niter, **kwargs) # Return return med_scale @@ -338,11 +338,11 @@ def median_flux(spec, smask, nsig=3., niter=5, **kwargs): mfluxes = np.ma.array(fluxes, mask=smask) #goodpix = WHERE(refivar GT 0.0 AND finite(refflux) AND finite(refivar) $ # AND refmask EQ 1 AND refivar LT 1.0d8) - mean_spec, med_spec, std_spec = sigma_clipped_stats(mfluxes, sigma=nsig, iters=niter, **kwargs) + mean_spec, med_spec, std_spec = astropy.stats.sigma_clipped_stats(mfluxes, sigma=nsig, iters=niter, **kwargs) # Clip a bit #badpix = np.any([spec.flux.value < 0.5*np.abs(med_spec)]) badpix = mfluxes.filled(0.) < 0.5*np.abs(med_spec) - mean_spec, med_spec, std_spec = sigma_clipped_stats(mfluxes.filled(0.), mask=badpix, + mean_spec, med_spec, std_spec = astropy.stats.sigma_clipped_stats(mfluxes.filled(0.), mask=badpix, sigma=nsig, iters=niter, **kwargs) debugger.set_trace() # Return @@ -587,7 +587,6 @@ def one_d_coadd(spectra, smask, weights, debug=False, **kwargs): coadd : XSpectrum1D """ - from linetools.spectra.xspectrum1d import XSpectrum1D # Setup fluxes, sigs, wave = unpack_spec(spectra) variances = (sigs > 0.) * sigs**2 @@ -637,7 +636,6 @@ def load_spec(files, iextensions=None, extract='opt', flux=True): spectra : XSpectrum1D -- All spectra are collated in this one object """ - from linetools.spectra.utils import collate # Extensions if iextensions is None: msgs.warn("Extensions not provided. Assuming first extension for all") @@ -752,7 +750,8 @@ def coadd_spectra(spectra, wave_grid_method='concatenate', niter=5, new_wave = new_wave_grid(spectra.data['wave'], wave_method=wave_grid_method, **kwargs) # Rebin - rspec = spectra.rebin(new_wave*u.AA, all=True, do_sig=True, grow_bad_sig=True, masking='none') + rspec = spectra.rebin(new_wave*units.AA, all=True, do_sig=True, grow_bad_sig=True, + masking='none') # Define mask -- THIS IS THE ONLY ONE TO USE rmask = rspec.data['sig'].filled(0.) <= 0. @@ -899,7 +898,7 @@ def coadd_spectra(spectra, wave_grid_method='concatenate', niter=5, # QA if qafile is not None: msgs.info("Writing QA file: {:s}".format(qafile)) - arqa.coaddspec_qa(spectra, rspec, rmask, spec1d, qafile=qafile) + coaddspec_qa(spectra, rspec, rmask, spec1d, qafile=qafile) # Write to disk? if outfile is not None: @@ -917,3 +916,84 @@ def write_to_disk(spec1d, outfile): spec1d.write_to_fits(outfile) return + + +def coaddspec_qa(ispectra, rspec, rmask, spec1d, qafile=None, yscale=2.): + """ QA plot for 1D coadd of spectra + + Parameters + ---------- + ispectra : XSpectrum1D + Multi-spectra object + rspec : XSpectrum1D + Rebinned spectra with updated variance + spec1d : XSpectrum1D + Final coadd + yscale : float, optional + Scale median flux by this parameter for the spectral plot + + """ + + plt.rcdefaults() + plt.rcParams['font.family']= 'times new roman' + + if qafile is not None: + pp = PdfPages(qafile) + + plt.clf() + plt.figure() + gs = gridspec.GridSpec(1,2) + + # Deviate + std_dev, dev_sig = get_std_dev(rspec, rmask, spec1d) + #dev_sig = (rspec.data['flux'] - spec1d.flux) / (rspec.data['sig']**2 + spec1d.sig**2) + #std_dev = np.std(sigma_clip(dev_sig, sigma=5, iters=2)) + if dev_sig is not None: + flat_dev_sig = dev_sig.flatten() + + xmin = -10 + xmax = 10 + n_bins = 100 + + # Deviation + ax = plt.subplot(gs[0]) + if dev_sig is not None: + hist, edges = np.histogram(flat_dev_sig, range=(xmin, xmax), bins=n_bins) + area = len(flat_dev_sig)*((xmax-xmin)/float(n_bins)) + xppf = np.linspace(scipy.stats.norm.ppf(0.0001), scipy.stats.norm.ppf(0.9999), 100) + ax.plot(xppf, area*scipy.stats.norm.pdf(xppf), color='black', linewidth=2.0) + ax.bar(edges[:-1], hist, width=((xmax-xmin)/float(n_bins)), alpha=0.5) + + # Coadd on individual + # yrange + medf = np.median(spec1d.flux) + ylim = (medf/10., yscale*medf) + # Plot + ax = plt.subplot(gs[1]) + for idx in range(ispectra.nspec): + ispectra.select = idx + ax.plot(ispectra.wavelength, ispectra.flux, alpha=0.5)#, label='individual exposure') + + ax.plot(spec1d.wavelength, spec1d.flux, color='black', label='coadded spectrum') + ax.set_ylim(ylim) + debug=False + if debug: + ax.set_ylim(0., 180.) + ax.set_xlim(3840, 3860.) + plt.legend() + plt.title('Coadded + Original Spectra') + + plt.tight_layout(pad=0.2,h_pad=0.,w_pad=0.2) + if qafile is not None: + pp.savefig(bbox_inches='tight') + pp.close() + msgs.info("Wrote coadd QA: {:s}".format(qafile)) + plt.close() + + plt.rcdefaults() + + return + + + + diff --git a/pypit/arextract.py b/pypit/arextract.py index 775743eff8..5566b6d5fe 100644 --- a/pypit/arextract.py +++ b/pypit/arextract.py @@ -1,22 +1,26 @@ from __future__ import (print_function, absolute_import, division, unicode_literals) import time -from matplotlib import pyplot as plt import copy +import inspect + import numpy as np -from astropy import units as u -#from pypit import armsgs + +from matplotlib import pyplot as plt +from matplotlib import gridspec, font_manager + +from astropy import units +from astropy.stats import sigma_clip + from pypit import msgs +from pypit import arqa from pypit import arparse as settings from pypit import artrace from pypit import arutils -from pypit import arqa - -# Logging -#msgs = armsgs.get_logger() - from pypit import ardebug as debugger +from pypit import arcyutils + # MASK VALUES FROM EXTRACTION # 0 # 2**0 = Flagged as bad detector pixel @@ -53,8 +57,6 @@ def boxcar(slf, det, specobjs, sciframe, varframe, skyframe, crmask, scitrace): bgcorr : ndarray Correction to the sky background in the object window """ - from pypit import arcyutils - from astropy.stats import sigma_clip bgfitord = 1 # Polynomial order used to fit the background nslit = len(scitrace) @@ -195,7 +197,7 @@ def boxcar(slf, det, specobjs, sciframe, varframe, skyframe, crmask, scitrace): debugger.set_trace() msgs.error("Bad match to specobj in boxcar!") # Fill - specobjs[sl][o].boxcar['wave'] = wvsum.copy()*u.AA # Yes, units enter here + specobjs[sl][o].boxcar['wave'] = wvsum.copy()*units.AA # Yes, units enter here specobjs[sl][o].boxcar['counts'] = scisum.copy() specobjs[sl][o].boxcar['var'] = varsum.copy() if np.sum(specobjs[sl][o].boxcar['var']) == 0.: @@ -335,7 +337,6 @@ def obj_profiles(slf, det, specobjs, sciframe, varframe, skyframe, crmask, mx = np.max(slit_val[gdp]) xval = np.linspace(mn, mx, 1000) model = arutils.func_val(gfit, xval, fdict['func']) - import matplotlib.pyplot as plt plt.clf() ax = plt.gca() ax.scatter(slit_val[gdp], flux_val[gdp], marker='.', s=0.7, edgecolor='none', facecolor='black') @@ -359,10 +360,63 @@ def obj_profiles(slf, det, specobjs, sciframe, varframe, skyframe, crmask, # QA if doqa: #not msgs._debug['no_qa'] and doqa: msgs.info("Preparing QA for spatial object profiles") - arqa.obj_profile_qa(slf, specobjs, scitrace, det) +# arqa.obj_profile_qa(slf, specobjs, scitrace, det) + obj_profile_qa(slf, specobjs, scitrace, det) return +def obj_profile_qa(slf, specobjs, scitrace, det): + """ Generate a QA plot for the object spatial profile + Parameters + ---------- + """ + + plt.rcdefaults() + plt.rcParams['font.family']= 'times new roman' + + method = inspect.stack()[0][3] + for sl in range(len(specobjs)): + # Setup + nobj = scitrace[sl]['traces'].shape[1] + ncol = min(3, nobj) + nrow = nobj // ncol + ((nobj % ncol) > 0) + # Outfile + outfile = arqa.set_qa_filename(slf._basename, method, det=det, slit=specobjs[sl][0].slitid) + # Plot + plt.figure(figsize=(8, 5.0)) + plt.clf() + gs = gridspec.GridSpec(nrow, ncol) + + # Plot + for o in range(nobj): + fdict = scitrace[sl]['opt_profile'][o] + if 'param' not in fdict.keys(): # Not optimally extracted + continue + ax = plt.subplot(gs[o//ncol, o % ncol]) + + # Data + gdp = fdict['mask'] == 0 + ax.scatter(fdict['slit_val'][gdp], fdict['flux_val'][gdp], marker='.', + s=0.5, edgecolor='none') + + # Fit + mn = np.min(fdict['slit_val'][gdp]) + mx = np.max(fdict['slit_val'][gdp]) + xval = np.linspace(mn, mx, 1000) + fit = arutils.func_val(fdict['param'], xval, fdict['func']) + ax.plot(xval, fit, 'r') + # Axes + ax.set_xlim(mn,mx) + # Label + ax.text(0.02, 0.90, 'Obj={:s}'.format(specobjs[sl][o].idx), + transform=ax.transAxes, ha='left', size='small') + + plt.savefig(outfile, dpi=500) + plt.close() + + plt.rcdefaults() + + def optimal_extract(slf, det, specobjs, sciframe, varframe, skyframe, crmask, scitrace, pickle_file=None, profiles=None): @@ -386,7 +440,6 @@ def optimal_extract(slf, det, specobjs, sciframe, varframe, newvar : ndarray Updated variance array that includes object model """ - from pypit import arproc # Setup #rnimg = arproc.rn_frame(slf,det) #model_var = np.abs(skyframe + sciframe - np.sqrt(2)*rnimg + rnimg**2) # sqrt 2 term deals with negative flux/sky @@ -457,7 +510,7 @@ def optimal_extract(slf, det, specobjs, sciframe, varframe, opt_ivar = opt_num * arutils.calc_ivar(ivar_den) # Save - specobjs[sl][o].optimal['wave'] = opt_wave.copy()*u.AA # Yes, units enter here + specobjs[sl][o].optimal['wave'] = opt_wave.copy()*units.AA # Yes, units enter here specobjs[sl][o].optimal['counts'] = opt_flux.copy() gdiv = (opt_ivar > 0.) & (ivar_den > 0.) opt_var = np.zeros_like(opt_ivar) @@ -473,11 +526,18 @@ def optimal_extract(slf, det, specobjs, sciframe, varframe, debugger.set_trace() debugger.xplot(opt_wave, opt_flux, np.sqrt(opt_var)) ''' - # Generate new variance image - newvar = arproc.variance_frame(slf, det, sciframe, -1, - skyframe=skyframe, objframe=obj_model) - # Return - return newvar + + # KBW: Using variance_frame here produces a circular import. I've + # changed this function to return the object model, then this last + # step is done in arproc. + +# # Generate new variance image +# newvar = arproc.variance_frame(slf, det, sciframe, -1, +# skyframe=skyframe, objframe=obj_model) +# # Return +# return newvar + + return obj_model def boxcar_cen(slf, det, img): diff --git a/pypit/armed.py b/pypit/armed.py index b6b620b29a..d9eadddd0f 100644 --- a/pypit/armed.py +++ b/pypit/armed.py @@ -3,24 +3,20 @@ from __future__ import (print_function, absolute_import, division, unicode_literals) import numpy as np + +from pypit import msgs + from pypit import arparse as settings from pypit import arload from pypit import armasters from pypit import armbase -#from pypit import armsgs -from pypit import msgs from pypit import arproc from pypit import arsave from pypit import arsetup from pypit import artrace -from pypit import arqa from pypit import ardebug as debugger -# Logging -#msgs = armsgs.get_logger() - - def ARMED(fitsdict, reuseMaster=False, reloadMaster=True): """ Automatic Reduction and Modeling of Echelle Data @@ -167,8 +163,11 @@ def ARMED(fitsdict, reuseMaster=False, reloadMaster=True): # Save to disk armasters.save_masters(slf, det, mftype='trace') # Save QA for slit traces - arqa.slit_trace_qa(slf, slf._mstrace[det-1], slf._lordpix[det-1], slf._rordpix[det - 1], extord, - desc="Trace of the slit edges", normalize=False) +# arqa.slit_trace_qa(slf, slf._mstrace[det-1], slf._lordpix[det-1], slf._rordpix[det - 1], extord, +# desc="Trace of the slit edges", normalize=False) + artrace.slit_trace_qa(slf, slf._mstrace[det-1], slf._lordpix[det-1], + slf._rordpix[det - 1], extord, + desc="Trace of the slit edges", normalize=False) armbase.UpdateMasters(sciexp, sc, det, ftype="flat", chktype="trace") ############### @@ -220,10 +219,13 @@ def ARMED(fitsdict, reuseMaster=False, reloadMaster=True): slf.SetFrame(slf._msblaze, msblaze, det) # Prepare some QA for the average slit profile along the slit msgs.info("Preparing QA of each slit profile") - arqa.slit_profile(slf, mstracenrm, slit_profiles, slf._lordloc[det - 1], slf._rordloc[det - 1], - slf._slitpix[det - 1], desc="Slit profile") + arproc.slit_profile_qa(slf, mstracenrm, slit_profiles, slf._lordloc[det - 1], + slf._rordloc[det - 1], slf._slitpix[det - 1], + desc="Slit profile") msgs.info("Saving blaze function QA") - arqa.plot_orderfits(slf, msblaze, flat_ext1d, desc="Blaze function", textplt="Order") +# arqa.plot_orderfits(slf, msblaze, flat_ext1d, desc="Blaze function", textplt="Order") + artrace.plot_orderfits(slf, msblaze, flat_ext1d, desc="Blaze function", + textplt="Order") ############### # Generate/load a master wave frame diff --git a/pypit/armlsd.py b/pypit/armlsd.py index 46ca1c89eb..b559277169 100644 --- a/pypit/armlsd.py +++ b/pypit/armlsd.py @@ -14,7 +14,7 @@ from pypit import arsave from pypit import arsetup from pypit import artrace -from pypit import arqa +#from pypit import arqa from linetools import utils as ltu @@ -149,9 +149,13 @@ def ARMLSD(fitsdict, reuseMaster=False, reloadMaster=True): # Save to disk armasters.save_masters(slf, det, mftype='trace') # Save QA for slit traces - arqa.slit_trace_qa(slf, slf._mstrace[det-1], slf._lordpix[det-1], - slf._rordpix[det-1], extord, - desc="Trace of the slit edges D{:02d}".format(det), use_slitid=det) +# arqa.slit_trace_qa(slf, slf._mstrace[det-1], slf._lordpix[det-1], +# slf._rordpix[det-1], extord, +# desc="Trace of the slit edges D{:02d}".format(det), use_slitid=det) + artrace.slit_trace_qa(slf, slf._mstrace[det-1], slf._lordpix[det-1], + slf._rordpix[det-1], extord, + desc="Trace of the slit edges D{:02d}".format(det), + use_slitid=det) armbase.UpdateMasters(sciexp, sc, det, ftype="flat", chktype="trace") ############### diff --git a/pypit/arpca.py b/pypit/arpca.py index bfdb94997c..cdd7ec5f0d 100644 --- a/pypit/arpca.py +++ b/pypit/arpca.py @@ -2,24 +2,16 @@ import inspect -from matplotlib import pyplot as plt import numpy as np -#from pypit import armsgs + +from matplotlib import pyplot as plt + from pypit import msgs from pypit import arutils -#from pypit.arqa import get_dimen, set_qa_filename from pypit import arqa -from pypit import ardebug as debugger - -# Logging -#msgs = armsgs.get_logger() - -# Force the default matplotlib plotting parameters -plt.rcdefaults() - - -def basis(xfit, yfit, coeff, npc, pnpc, weights=None, skipx0=True, x0in=None, mask=None, function='polynomial'): +def basis(xfit, yfit, coeff, npc, pnpc, weights=None, skipx0=True, x0in=None, mask=None, + function='polynomial'): nrow = xfit.shape[0] ntrace = xfit.shape[1] if x0in is None: @@ -379,3 +371,238 @@ def pc_plot_extcenwid(tempcen, cenwid, binval, plotsdir="Plots", pcatype="