diff --git a/.travis.yml b/.travis.yml index a149d72a23..1a4e718b66 100644 --- a/.travis.yml +++ b/.travis.yml @@ -152,6 +152,7 @@ install: - git clone git://github.com/astropy/ci-helpers.git - source ci-helpers/travis/setup_conda_$TRAVIS_OS_NAME.sh - if [[ $SETUP_CMD != egg_info ]]; then $PIP_INSTALL git+https://github.com/linetools/linetools.git; fi + - if [[ $SETUP_CMD != egg_info ]]; then $PIP_INSTALL git+https://github.com/PYPIT/arclines.git@circular_imports; fi - conda install numpy - conda install Cython diff --git a/CHANGES.rst b/CHANGES.rst index ede5244434..8c63c386aa 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -34,6 +34,12 @@ * Allowed data sections to be extracted from header information * Significant refactor of routines related to pypit_setup * Various small improvements, primarly to handle Gemini/GMOS data [not yet fully supported in PYPIT] +* Removed majority of cython functionality +* Moved logging to be a package object using the main __init__.py file +* Begin to adhere to PEP8 (mostly) +* setup.py rewritten. Modeled after https://github.com/sdss/marvin/blob/master/setup.py . Added requirements.txt with the package versions required. +* Updates archeck +* Loads NIST arclines from arclines instead of PYPIT 0.7 (2017-02-07) ---------------- diff --git a/pypit/__init__.py b/pypit/__init__.py index e69de29bb2..9ee46c710f 100644 --- a/pypit/__init__.py +++ b/pypit/__init__.py @@ -0,0 +1,52 @@ +""" +pypit package initialization. + +The current main purpose of this is to provide package-level globals +that can be imported by submodules. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +# Imports for signal and log handling +import sys +import signal +import warnings + +# Set version +__version__ = '0.8.0.dev0' +# TODO: Do we need this? A release should be tied to a date, and the +# date information could be put in the CHANGES.rst +__last_updated__ = '23Mar2018' + +# Import and instantiate the logger +from pypit import armsgs +msgs = armsgs.Messages() + +# Import the close_qa method so that it can be called when a hard stop +# is requested by the user +# TODO: Is this necessary? As far as I can tell, close_qa() doesn't +# actually close any already opened qa files + +# TODO: Causes circular imports +# from pypit.arqa import close_qa + +# Send all signals to messages to be dealt with (i.e. someone hits ctrl+c) +def signal_handler(signalnum, handler): + """ + Handle signals sent by the keyboard during code execution + """ + if signalnum == 2: + msgs.info('Ctrl+C was pressed. Ending processes...') +# close_qa(msgs.pypit_file) + msgs.close() + sys.exit() + +signal.signal(signal.SIGINT, signal_handler) + +# Ignore all warnings given by python +warnings.resetwarnings() +warnings.simplefilter('ignore') + diff --git a/pypit/ararc.py b/pypit/ararc.py index befa3a6a8a..669f65b37c 100644 --- a/pypit/ararc.py +++ b/pypit/ararc.py @@ -1,21 +1,25 @@ from __future__ import (print_function, absolute_import, division, unicode_literals) +import os +import time +import inspect + import numpy as np +from matplotlib import pyplot as plt +from matplotlib import gridspec, font_manager + +import arclines +#from arclines.holy.grail import basic, semi_brute, general + from pypit import arpca from pypit import arparse as settings -from pypit import armsgs +from pypit import msgs from pypit import arsave from pypit import arutils from pypit import ararclines from pypit import arqa -from matplotlib import pyplot as plt -import os -import time - from pypit import ardebug as debugger - -# Logging -msgs = armsgs.get_logger() +from pypit import arcyarc def detect_lines(slf, det, msarc, censpec=None, MK_SATMASK=False): @@ -53,7 +57,6 @@ def detect_lines(slf, det, msarc, censpec=None, MK_SATMASK=False): The spectrum used to find detections. This spectrum has had any "continuum" emission subtracted off """ - from pypit import arcyarc # Extract a rough spectrum of the arc in each order msgs.info("Detecting lines") msgs.info("Extracting an approximate arc spectrum at the centre of the chip") @@ -548,7 +551,8 @@ def simple_calib(slf, det, get_poly=False): xrej=xrej, yrej=yrej, mask=mask, spec=yprep, nrej=aparm['nsig_rej_final'], shift=0., tcent=tcent) # QA - arqa.arc_fit_qa(slf, final_fit) +# arqa.arc_fit_qa(slf, final_fit) + arc_fit_qa(slf, final_fit) # RMS rms_ang = arutils.calc_fit_rms(xfit, yfit, fit, aparm['func'], minv=fmin, maxv=fmax) wave = arutils.func_val(fit, np.arange(slf._msarc[det-1].shape[0])/float(slf._msarc[det-1].shape[0]), @@ -574,7 +578,6 @@ def calib_with_arclines(slf, det, get_poly=False, use_method="general"): final_fit : dict Dict of fit info """ - from arclines.holy.grail import basic, semi_brute, general # Parameters (just for convenience) aparm = slf._arcparam[det-1] # Extract the arc @@ -582,14 +585,15 @@ def calib_with_arclines(slf, det, get_poly=False, use_method="general"): tampl, tcent, twid, w, satsnd, spec = detect_lines(slf, det, slf._msarc[det-1]) if use_method == "semi-brute": - best_dict, final_fit = semi_brute(spec, aparm['lamps'], aparm['wv_cen'], aparm['disp'], fit_parm=aparm, min_ampl=aparm['min_ampl']) + best_dict, final_fit = arclines.holy.grail.semi_brute(spec, aparm['lamps'], aparm['wv_cen'], aparm['disp'], fit_parm=aparm, min_ampl=aparm['min_ampl']) elif use_method == "basic": - stuff = basic(spec, aparm['lamps'], aparm['wv_cen'], aparm['disp']) + stuff = arclines.holy.grail.basic(spec, aparm['lamps'], aparm['wv_cen'], aparm['disp']) status, ngd_match, match_idx, scores, final_fit = stuff else: # Now preferred - best_dict, final_fit = general(spec, aparm['lamps'], fit_parm=aparm, min_ampl=aparm['min_ampl']) - arqa.arc_fit_qa(slf, final_fit) + best_dict, final_fit = arclines.holy.grail.general(spec, aparm['lamps'], fit_parm=aparm, min_ampl=aparm['min_ampl']) +# arqa.arc_fit_qa(slf, final_fit) + arc_fit_qa(slf, final_fit) # return final_fit @@ -661,3 +665,113 @@ def new_saturation_mask(a, satlevel): return mask.astype(int) + +def arc_fit_qa(slf, fit, outfile=None, ids_only=False, title=None): + """ + QA for Arc spectrum + + Parameters + ---------- + fit : Wavelength fit + arc_spec : ndarray + Arc spectrum + outfile : str, optional + Name of output file + """ + + plt.rcdefaults() + plt.rcParams['font.family']= 'times new roman' + + # Grab the named of the method + method = inspect.stack()[0][3] + # Outfil + if outfile is None: + outfile = arqa.set_qa_filename(slf.setup, method) + # + arc_spec = fit['spec'] + + # Begin + if not ids_only: + plt.figure(figsize=(8, 4.0)) + plt.clf() + gs = gridspec.GridSpec(2, 2) + idfont = 'xx-small' + else: + plt.figure(figsize=(11, 8.5)) + plt.clf() + gs = gridspec.GridSpec(1, 1) + idfont = 'small' + + # Simple spectrum plot + ax_spec = plt.subplot(gs[:,0]) + ax_spec.plot(np.arange(len(arc_spec)), arc_spec) + ymin, ymax = 0., np.max(arc_spec) + ysep = ymax*0.03 + for kk, x in enumerate(fit['xfit']*fit['xnorm']): + yline = np.max(arc_spec[int(x)-2:int(x)+2]) + # Tick mark + ax_spec.plot([x,x], [yline+ysep*0.25, yline+ysep], 'g-') + # label + ax_spec.text(x, yline+ysep*1.3, + '{:s} {:g}'.format(fit['ions'][kk], fit['yfit'][kk]), ha='center', va='bottom', + size=idfont, rotation=90., color='green') + ax_spec.set_xlim(0., len(arc_spec)) + ax_spec.set_ylim(ymin, ymax*1.2) + ax_spec.set_xlabel('Pixel') + ax_spec.set_ylabel('Flux') + if title is not None: + ax_spec.text(0.04, 0.93, title, transform=ax_spec.transAxes, + size='x-large', ha='left')#, bbox={'facecolor':'white'}) + if ids_only: + plt.tight_layout(pad=0.2, h_pad=0.0, w_pad=0.0) + plt.savefig(outfile, dpi=800) + plt.close() + return + + # Arc Fit + ax_fit = plt.subplot(gs[0, 1]) + # Points + ax_fit.scatter(fit['xfit']*fit['xnorm'], fit['yfit'], marker='x') + if len(fit['xrej']) > 0: + ax_fit.scatter(fit['xrej']*fit['xnorm'], fit['yrej'], marker='o', + edgecolor='gray', facecolor='none') + # Solution + xval = np.arange(len(arc_spec)) + wave = arutils.func_val(fit['fitc'], xval/fit['xnorm'], 'legendre', + minv=fit['fmin'], maxv=fit['fmax']) + ax_fit.plot(xval, wave, 'r-') + xmin, xmax = 0., len(arc_spec) + ax_fit.set_xlim(xmin, xmax) + ymin,ymax = np.min(wave)*.95, np.max(wave)*1.05 + ax_fit.set_ylim(np.min(wave)*.95, np.max(wave)*1.05) + ax_fit.set_ylabel('Wavelength') + ax_fit.get_xaxis().set_ticks([]) # Suppress labeling + # Stats + wave_fit = arutils.func_val(fit['fitc'], fit['xfit'], 'legendre', + minv=fit['fmin'], maxv=fit['fmax']) + rms = np.sqrt(np.sum((fit['yfit']-wave_fit)**2)/len(fit['xfit'])) # Ang + dwv_pix = np.median(np.abs(wave-np.roll(wave,1))) + ax_fit.text(0.1*len(arc_spec), 0.90*ymin+(ymax-ymin), + r'$\Delta\lambda$={:.3f}$\AA$ (per pix)'.format(dwv_pix), size='small') + ax_fit.text(0.1*len(arc_spec), 0.80*ymin+(ymax-ymin), + 'RMS={:.3f} (pixels)'.format(rms/dwv_pix), size='small') + # Arc Residuals + ax_res = plt.subplot(gs[1,1]) + res = fit['yfit']-wave_fit + ax_res.scatter(fit['xfit']*fit['xnorm'], res/dwv_pix, marker='x') + ax_res.plot([xmin,xmax], [0.,0], 'k--') + ax_res.set_xlim(xmin, xmax) + ax_res.set_xlabel('Pixel') + ax_res.set_ylabel('Residuals (Pix)') + + # Finish + plt.tight_layout(pad=0.2, h_pad=0.0, w_pad=0.0) + plt.savefig(outfile, dpi=800) + plt.close() + + plt.rcdefaults() + + return + + + diff --git a/pypit/ararclines.py b/pypit/ararclines.py index 16b39354cb..3644e138c8 100644 --- a/pypit/ararclines.py +++ b/pypit/ararclines.py @@ -4,14 +4,18 @@ from astropy.table import Table, Column, vstack import glob, copy import yaml +from pkg_resources import resource_filename -from pypit import armsgs +from arclines.io import load_line_list + +#from pypit import armsgs +from pypit import msgs from pypit import arparse as settings from pypit import ardebug as debugger # Logging -msgs = armsgs.get_logger() +#msgs = armsgs.get_logger() def parse_nist(slf,ion): @@ -23,47 +27,10 @@ def parse_nist(slf,ion): ion : str Name of ion """ - # Root (for development only) - if slf is None: - from pypit import arutils as arut - msgs.warn("Using arutils.dummy_self. Better know what you are doing.") - slf = arut.dummy_self() - root = settings.argflag['run']['pypitdir'] - # Find file - srch_file = root + '/data/arc_lines/NIST/'+ion+'_vacuum.ascii' - nist_file = glob.glob(srch_file) - if len(nist_file) == 0: - msgs.error("Cannot find NIST file {:s}".format(srch_file)) - elif len(nist_file) != 1: - msgs.error("Multiple NIST files for {:s}".format(srch_file)) - # Read - nist_tbl = Table.read(nist_file[0], format='ascii.fixed_width', comment='#') - gdrow = nist_tbl['Observed'] > 0. # Eliminate dummy lines - nist_tbl = nist_tbl[gdrow] - # Now unique values only (no duplicates) - uniq, indices = np.unique(nist_tbl['Observed'],return_index=True) - nist_tbl = nist_tbl[indices] - # Deal with Rel - agdrel = [] - for row in nist_tbl: - try: - gdrel = int(row['Rel.']) - except: - try: - gdrel = int(row['Rel.'][:-1]) - except: - gdrel = 0 - agdrel.append(gdrel) - agdrel = np.array(agdrel) - # Remove and add - nist_tbl.remove_column('Rel.') - nist_tbl.remove_column('Ritz') - nist_tbl.add_column(Column(agdrel,name='RelInt')) - nist_tbl.remove_column('Acc.') - nist_tbl.remove_column('Type') - #nist_tbl.add_column(Column([ion]*len(nist_tbl), name='Ion', dtype='S5')) - nist_tbl.add_column(Column([ion]*len(nist_tbl), name='Ion', dtype='U5')) - nist_tbl.rename_column('Observed','wave') + nist_path = resource_filename('arclines', 'data/NIST/') + line_file = nist_path+'{:s}_vacuum.ascii'.format(ion) + nist_tbl = load_line_list(line_file, NIST=True) + # Return return nist_tbl diff --git a/pypit/archeck.py b/pypit/archeck.py index 4cff454982..cd81c300dd 100644 --- a/pypit/archeck.py +++ b/pypit/archeck.py @@ -4,6 +4,23 @@ from __future__ import absolute_import, division, print_function +import pkg_resources + +requirements_file = pkg_resources.resource_filename('pypit', 'requirements.txt') +install_requires = [line.strip().replace('==', '>=') for line in open(requirements_file) + if not line.strip().startswith('#') and line.strip() != ''] +for requirement in install_requires: + pkg, version = requirement.split('>=') + try: + pv = pkg_resources.get_distribution(pkg).version + except pkg_resources.DistributionNotFound: + raise ImportError("Package: {:s} not installed!".format(pkg)) + else: + if pkg_resources.parse_version(pv) < pkg_resources.parse_version(version): + print("Version of package {:s} = {:s}".format(pkg, pv)) + raise ImportError("You need version >= {:s}".format(version)) + +''' from distutils.version import LooseVersion import warnings @@ -42,3 +59,4 @@ def version_check(): for dep, ver in minimum_versions.items(): if LooseVersion(globals()[dep].__version__) < LooseVersion(ver): raise VersionError('Update ' + dep + ' to at least version ' + ver + '!') +''' diff --git a/pypit/arcoadd.py b/pypit/arcoadd.py index 367648a4ab..b32b495f8b 100644 --- a/pypit/arcoadd.py +++ b/pypit/arcoadd.py @@ -3,23 +3,25 @@ from __future__ import absolute_import, division, print_function import numpy as np -import astropy.stats -import scipy.interpolate +from numpy.ma.core import MaskedArray +import scipy.stats from scipy.signal import medfilt -from astropy import units as u -from astropy.stats import sigma_clipped_stats +from matplotlib import pyplot as plt +from matplotlib import gridspec, font_manager +from matplotlib.backends.backend_pdf import PdfPages + +import astropy.stats +from astropy import units + +from linetools.spectra.xspectrum1d import XSpectrum1D +from linetools.spectra.utils import collate +from pypit import msgs from pypit import arload -from pypit import armsgs -from pypit import arqa from pypit import arutils - from pypit import ardebug as debugger -# Logging -msgs = armsgs.get_logger() - # TODO # Shift spectra # Scale by poly @@ -59,7 +61,6 @@ def new_wave_grid(waves, wave_method='iref', iref=0, A_pix=None, v_pix=None, **k # slf._argflag['reduce']['pixelsize'] = 2.5?? This won't work # if running coadding outside of PYPIT, which we'd like as an # option! - from numpy.ma.core import MaskedArray if not isinstance(waves, MaskedArray): waves = np.ma.array(waves) @@ -304,7 +305,7 @@ def median_ratio_flux(spec, smask, ispec, iref, nsig=3., niter=5, **kwargs): # Ratio med_flux = fluxes[iref,allok] / fluxes[ispec,allok] # Clip - mn_scale, med_scale, std_scale = sigma_clipped_stats(med_flux, sigma=nsig, iters=niter, **kwargs) + mn_scale, med_scale, std_scale = astropy.stats.sigma_clipped_stats(med_flux, sigma=nsig, iters=niter, **kwargs) # Return return med_scale @@ -337,11 +338,11 @@ def median_flux(spec, smask, nsig=3., niter=5, **kwargs): mfluxes = np.ma.array(fluxes, mask=smask) #goodpix = WHERE(refivar GT 0.0 AND finite(refflux) AND finite(refivar) $ # AND refmask EQ 1 AND refivar LT 1.0d8) - mean_spec, med_spec, std_spec = sigma_clipped_stats(mfluxes, sigma=nsig, iters=niter, **kwargs) + mean_spec, med_spec, std_spec = astropy.stats.sigma_clipped_stats(mfluxes, sigma=nsig, iters=niter, **kwargs) # Clip a bit #badpix = np.any([spec.flux.value < 0.5*np.abs(med_spec)]) badpix = mfluxes.filled(0.) < 0.5*np.abs(med_spec) - mean_spec, med_spec, std_spec = sigma_clipped_stats(mfluxes.filled(0.), mask=badpix, + mean_spec, med_spec, std_spec = astropy.stats.sigma_clipped_stats(mfluxes.filled(0.), mask=badpix, sigma=nsig, iters=niter, **kwargs) debugger.set_trace() # Return @@ -586,7 +587,6 @@ def one_d_coadd(spectra, smask, weights, debug=False, **kwargs): coadd : XSpectrum1D """ - from linetools.spectra.xspectrum1d import XSpectrum1D # Setup fluxes, sigs, wave = unpack_spec(spectra) variances = (sigs > 0.) * sigs**2 @@ -636,7 +636,6 @@ def load_spec(files, iextensions=None, extract='opt', flux=True): spectra : XSpectrum1D -- All spectra are collated in this one object """ - from linetools.spectra.utils import collate # Extensions if iextensions is None: msgs.warn("Extensions not provided. Assuming first extension for all") @@ -751,7 +750,8 @@ def coadd_spectra(spectra, wave_grid_method='concatenate', niter=5, new_wave = new_wave_grid(spectra.data['wave'], wave_method=wave_grid_method, **kwargs) # Rebin - rspec = spectra.rebin(new_wave*u.AA, all=True, do_sig=True, grow_bad_sig=True, masking='none') + rspec = spectra.rebin(new_wave*units.AA, all=True, do_sig=True, grow_bad_sig=True, + masking='none') # Define mask -- THIS IS THE ONLY ONE TO USE rmask = rspec.data['sig'].filled(0.) <= 0. @@ -898,7 +898,7 @@ def coadd_spectra(spectra, wave_grid_method='concatenate', niter=5, # QA if qafile is not None: msgs.info("Writing QA file: {:s}".format(qafile)) - arqa.coaddspec_qa(spectra, rspec, rmask, spec1d, qafile=qafile) + coaddspec_qa(spectra, rspec, rmask, spec1d, qafile=qafile) # Write to disk? if outfile is not None: @@ -916,3 +916,84 @@ def write_to_disk(spec1d, outfile): spec1d.write_to_fits(outfile) return + + +def coaddspec_qa(ispectra, rspec, rmask, spec1d, qafile=None, yscale=2.): + """ QA plot for 1D coadd of spectra + + Parameters + ---------- + ispectra : XSpectrum1D + Multi-spectra object + rspec : XSpectrum1D + Rebinned spectra with updated variance + spec1d : XSpectrum1D + Final coadd + yscale : float, optional + Scale median flux by this parameter for the spectral plot + + """ + + plt.rcdefaults() + plt.rcParams['font.family']= 'times new roman' + + if qafile is not None: + pp = PdfPages(qafile) + + plt.clf() + plt.figure() + gs = gridspec.GridSpec(1,2) + + # Deviate + std_dev, dev_sig = get_std_dev(rspec, rmask, spec1d) + #dev_sig = (rspec.data['flux'] - spec1d.flux) / (rspec.data['sig']**2 + spec1d.sig**2) + #std_dev = np.std(sigma_clip(dev_sig, sigma=5, iters=2)) + if dev_sig is not None: + flat_dev_sig = dev_sig.flatten() + + xmin = -10 + xmax = 10 + n_bins = 100 + + # Deviation + ax = plt.subplot(gs[0]) + if dev_sig is not None: + hist, edges = np.histogram(flat_dev_sig, range=(xmin, xmax), bins=n_bins) + area = len(flat_dev_sig)*((xmax-xmin)/float(n_bins)) + xppf = np.linspace(scipy.stats.norm.ppf(0.0001), scipy.stats.norm.ppf(0.9999), 100) + ax.plot(xppf, area*scipy.stats.norm.pdf(xppf), color='black', linewidth=2.0) + ax.bar(edges[:-1], hist, width=((xmax-xmin)/float(n_bins)), alpha=0.5) + + # Coadd on individual + # yrange + medf = np.median(spec1d.flux) + ylim = (medf/10., yscale*medf) + # Plot + ax = plt.subplot(gs[1]) + for idx in range(ispectra.nspec): + ispectra.select = idx + ax.plot(ispectra.wavelength, ispectra.flux, alpha=0.5)#, label='individual exposure') + + ax.plot(spec1d.wavelength, spec1d.flux, color='black', label='coadded spectrum') + ax.set_ylim(ylim) + debug=False + if debug: + ax.set_ylim(0., 180.) + ax.set_xlim(3840, 3860.) + plt.legend() + plt.title('Coadded + Original Spectra') + + plt.tight_layout(pad=0.2,h_pad=0.,w_pad=0.2) + if qafile is not None: + pp.savefig(bbox_inches='tight') + pp.close() + msgs.info("Wrote coadd QA: {:s}".format(qafile)) + plt.close() + + plt.rcdefaults() + + return + + + + diff --git a/pypit/arcomb.py b/pypit/arcomb.py index 80d4a30b0a..d51ed07ace 100644 --- a/pypit/arcomb.py +++ b/pypit/arcomb.py @@ -2,12 +2,13 @@ import time import numpy as np -from pypit import armsgs +#from pypit import armsgs +from pypit import msgs from pypit import arparse as settings from matplotlib import pyplot as plt # Logging -msgs = armsgs.get_logger() +#msgs = armsgs.get_logger() def comb_frames(frames_arr, det, frametype, weights=None, maskvalue=1048577, printtype=None): """ Combine several frames diff --git a/pypit/ardebug.py b/pypit/ardebug.py index 8088f9dc1d..49a764936f 100644 --- a/pypit/ardebug.py +++ b/pypit/ardebug.py @@ -1,8 +1,14 @@ from __future__ import (print_function, absolute_import, division, unicode_literals) +import matplotlib.pyplot as plt +import numpy as np + # These need to be outside of the def's from pypit.ginga import show_image from pypit.ginga import chk_arc_tilts +# Moved to the top and changed to only import set_trace +#from pdb import * +from pdb import set_trace def init(): """ @@ -49,8 +55,6 @@ def plot1d(*args, **kwargs): True for a scatter plot NOTE: Any extra parameters are fed as kwargs to plt.plot() """ - import matplotlib.pyplot as plt - import numpy as np # Error checking if len(args) == 0: print('x_guis.simple_splot: No arguments!') @@ -145,4 +149,3 @@ def plot1d(*args, **kwargs): plt.show() return -from pdb import * diff --git a/pypit/ardevtest.py b/pypit/ardevtest.py index 4cd63bbec7..0257e555c6 100644 --- a/pypit/ardevtest.py +++ b/pypit/ardevtest.py @@ -5,13 +5,14 @@ import scipy import glob -from pypit import armsgs +#from pypit import armsgs +from pypit import msgs from pypit import arparse as settings from pypit import ardebug as debugger # Logging -msgs = armsgs.get_logger() +#msgs = armsgs.get_logger() def set_param(argf, specname): diff --git a/pypit/arextract.py b/pypit/arextract.py index 5b408700da..5566b6d5fe 100644 --- a/pypit/arextract.py +++ b/pypit/arextract.py @@ -1,21 +1,26 @@ from __future__ import (print_function, absolute_import, division, unicode_literals) import time -from matplotlib import pyplot as plt import copy +import inspect + import numpy as np -from astropy import units as u -from pypit import armsgs + +from matplotlib import pyplot as plt +from matplotlib import gridspec, font_manager + +from astropy import units +from astropy.stats import sigma_clip + +from pypit import msgs +from pypit import arqa from pypit import arparse as settings from pypit import artrace from pypit import arutils -from pypit import arqa - -# Logging -msgs = armsgs.get_logger() - from pypit import ardebug as debugger +from pypit import arcyutils + # MASK VALUES FROM EXTRACTION # 0 # 2**0 = Flagged as bad detector pixel @@ -52,8 +57,6 @@ def boxcar(slf, det, specobjs, sciframe, varframe, skyframe, crmask, scitrace): bgcorr : ndarray Correction to the sky background in the object window """ - from pypit import arcyutils - from astropy.stats import sigma_clip bgfitord = 1 # Polynomial order used to fit the background nslit = len(scitrace) @@ -194,7 +197,7 @@ def boxcar(slf, det, specobjs, sciframe, varframe, skyframe, crmask, scitrace): debugger.set_trace() msgs.error("Bad match to specobj in boxcar!") # Fill - specobjs[sl][o].boxcar['wave'] = wvsum.copy()*u.AA # Yes, units enter here + specobjs[sl][o].boxcar['wave'] = wvsum.copy()*units.AA # Yes, units enter here specobjs[sl][o].boxcar['counts'] = scisum.copy() specobjs[sl][o].boxcar['var'] = varsum.copy() if np.sum(specobjs[sl][o].boxcar['var']) == 0.: @@ -334,7 +337,6 @@ def obj_profiles(slf, det, specobjs, sciframe, varframe, skyframe, crmask, mx = np.max(slit_val[gdp]) xval = np.linspace(mn, mx, 1000) model = arutils.func_val(gfit, xval, fdict['func']) - import matplotlib.pyplot as plt plt.clf() ax = plt.gca() ax.scatter(slit_val[gdp], flux_val[gdp], marker='.', s=0.7, edgecolor='none', facecolor='black') @@ -358,10 +360,63 @@ def obj_profiles(slf, det, specobjs, sciframe, varframe, skyframe, crmask, # QA if doqa: #not msgs._debug['no_qa'] and doqa: msgs.info("Preparing QA for spatial object profiles") - arqa.obj_profile_qa(slf, specobjs, scitrace, det) +# arqa.obj_profile_qa(slf, specobjs, scitrace, det) + obj_profile_qa(slf, specobjs, scitrace, det) return +def obj_profile_qa(slf, specobjs, scitrace, det): + """ Generate a QA plot for the object spatial profile + Parameters + ---------- + """ + + plt.rcdefaults() + plt.rcParams['font.family']= 'times new roman' + + method = inspect.stack()[0][3] + for sl in range(len(specobjs)): + # Setup + nobj = scitrace[sl]['traces'].shape[1] + ncol = min(3, nobj) + nrow = nobj // ncol + ((nobj % ncol) > 0) + # Outfile + outfile = arqa.set_qa_filename(slf._basename, method, det=det, slit=specobjs[sl][0].slitid) + # Plot + plt.figure(figsize=(8, 5.0)) + plt.clf() + gs = gridspec.GridSpec(nrow, ncol) + + # Plot + for o in range(nobj): + fdict = scitrace[sl]['opt_profile'][o] + if 'param' not in fdict.keys(): # Not optimally extracted + continue + ax = plt.subplot(gs[o//ncol, o % ncol]) + + # Data + gdp = fdict['mask'] == 0 + ax.scatter(fdict['slit_val'][gdp], fdict['flux_val'][gdp], marker='.', + s=0.5, edgecolor='none') + + # Fit + mn = np.min(fdict['slit_val'][gdp]) + mx = np.max(fdict['slit_val'][gdp]) + xval = np.linspace(mn, mx, 1000) + fit = arutils.func_val(fdict['param'], xval, fdict['func']) + ax.plot(xval, fit, 'r') + # Axes + ax.set_xlim(mn,mx) + # Label + ax.text(0.02, 0.90, 'Obj={:s}'.format(specobjs[sl][o].idx), + transform=ax.transAxes, ha='left', size='small') + + plt.savefig(outfile, dpi=500) + plt.close() + + plt.rcdefaults() + + def optimal_extract(slf, det, specobjs, sciframe, varframe, skyframe, crmask, scitrace, pickle_file=None, profiles=None): @@ -385,7 +440,6 @@ def optimal_extract(slf, det, specobjs, sciframe, varframe, newvar : ndarray Updated variance array that includes object model """ - from pypit import arproc # Setup #rnimg = arproc.rn_frame(slf,det) #model_var = np.abs(skyframe + sciframe - np.sqrt(2)*rnimg + rnimg**2) # sqrt 2 term deals with negative flux/sky @@ -456,7 +510,7 @@ def optimal_extract(slf, det, specobjs, sciframe, varframe, opt_ivar = opt_num * arutils.calc_ivar(ivar_den) # Save - specobjs[sl][o].optimal['wave'] = opt_wave.copy()*u.AA # Yes, units enter here + specobjs[sl][o].optimal['wave'] = opt_wave.copy()*units.AA # Yes, units enter here specobjs[sl][o].optimal['counts'] = opt_flux.copy() gdiv = (opt_ivar > 0.) & (ivar_den > 0.) opt_var = np.zeros_like(opt_ivar) @@ -472,11 +526,18 @@ def optimal_extract(slf, det, specobjs, sciframe, varframe, debugger.set_trace() debugger.xplot(opt_wave, opt_flux, np.sqrt(opt_var)) ''' - # Generate new variance image - newvar = arproc.variance_frame(slf, det, sciframe, -1, - skyframe=skyframe, objframe=obj_model) - # Return - return newvar + + # KBW: Using variance_frame here produces a circular import. I've + # changed this function to return the object model, then this last + # step is done in arproc. + +# # Generate new variance image +# newvar = arproc.variance_frame(slf, det, sciframe, -1, +# skyframe=skyframe, objframe=obj_model) +# # Return +# return newvar + + return obj_model def boxcar_cen(slf, det, img): diff --git a/pypit/arflux.py b/pypit/arflux.py index d348d95370..4a6e38bfef 100644 --- a/pypit/arflux.py +++ b/pypit/arflux.py @@ -11,7 +11,8 @@ from astropy import units as u from astropy import coordinates as coords -from pypit import armsgs +#from pypit import armsgs +from pypit import msgs from pypit import arparse as settings try: @@ -22,7 +23,7 @@ from pypit import ardebug as debugger # Logging -msgs = armsgs.get_logger() +#msgs = armsgs.get_logger() def apply_sensfunc(slf, det, scidx, fitsdict, MAX_EXTRAP=0.05, standard=False): diff --git a/pypit/arload.py b/pypit/arload.py index 969867b41c..704cb77c9b 100644 --- a/pypit/arload.py +++ b/pypit/arload.py @@ -6,7 +6,8 @@ from astropy.time import Time import numpy as np from pypit import arparse as settings -from pypit import armsgs +#from pypit import armsgs +from pypit import msgs from pypit import arproc from pypit import arlris #from multiprocessing import Pool as mpPool @@ -21,7 +22,7 @@ from pypit import ardebug as debugger # Logging -msgs = armsgs.get_logger() +#msgs = armsgs.get_logger() def load_headers(datlines): diff --git a/pypit/arlris.py b/pypit/arlris.py index 9598eb0a4a..08b478a0ff 100644 --- a/pypit/arlris.py +++ b/pypit/arlris.py @@ -5,7 +5,8 @@ import glob import astropy.io.fits as pyfits -from pypit import armsgs +#from pypit import armsgs +from pypit import msgs from pypit.arparse import load_sections from pypit import ardebug as debugger @@ -16,7 +17,7 @@ basestring = str # Logging -msgs = armsgs.get_logger() +#msgs = armsgs.get_logger() def read_lris(raw_file, det=None, TRIM=False): diff --git a/pypit/armasters.py b/pypit/armasters.py index af4efe9945..2f2b21938c 100644 --- a/pypit/armasters.py +++ b/pypit/armasters.py @@ -1,7 +1,8 @@ from __future__ import (print_function, absolute_import, division, unicode_literals) import numpy as np -from pypit import armsgs +#from pypit import armsgs +from pypit import msgs from pypit import arload from pypit import arparse as settings from pypit import arsave @@ -13,7 +14,7 @@ basestring = str # Logging -msgs = armsgs.get_logger() +#msgs = armsgs.get_logger() from pypit import ardebug as debugger diff --git a/pypit/armbase.py b/pypit/armbase.py index 2143a33ea1..74acb45122 100644 --- a/pypit/armbase.py +++ b/pypit/armbase.py @@ -5,14 +5,15 @@ import numpy as np from pypit import arparse as settings -from pypit import armsgs +#from pypit import armsgs +from pypit import msgs from pypit import arsetup from pypit import arsort from pypit import arsciexp from pypit import arparse # Logging -msgs = armsgs.get_logger() +#msgs = armsgs.get_logger() from pypit import ardebug as debugger diff --git a/pypit/armed.py b/pypit/armed.py index 80430a962c..d9eadddd0f 100644 --- a/pypit/armed.py +++ b/pypit/armed.py @@ -3,23 +3,20 @@ from __future__ import (print_function, absolute_import, division, unicode_literals) import numpy as np + +from pypit import msgs + from pypit import arparse as settings from pypit import arload from pypit import armasters from pypit import armbase -from pypit import armsgs from pypit import arproc from pypit import arsave from pypit import arsetup from pypit import artrace -from pypit import arqa from pypit import ardebug as debugger -# Logging -msgs = armsgs.get_logger() - - def ARMED(fitsdict, reuseMaster=False, reloadMaster=True): """ Automatic Reduction and Modeling of Echelle Data @@ -166,8 +163,11 @@ def ARMED(fitsdict, reuseMaster=False, reloadMaster=True): # Save to disk armasters.save_masters(slf, det, mftype='trace') # Save QA for slit traces - arqa.slit_trace_qa(slf, slf._mstrace[det-1], slf._lordpix[det-1], slf._rordpix[det - 1], extord, - desc="Trace of the slit edges", normalize=False) +# arqa.slit_trace_qa(slf, slf._mstrace[det-1], slf._lordpix[det-1], slf._rordpix[det - 1], extord, +# desc="Trace of the slit edges", normalize=False) + artrace.slit_trace_qa(slf, slf._mstrace[det-1], slf._lordpix[det-1], + slf._rordpix[det - 1], extord, + desc="Trace of the slit edges", normalize=False) armbase.UpdateMasters(sciexp, sc, det, ftype="flat", chktype="trace") ############### @@ -219,10 +219,13 @@ def ARMED(fitsdict, reuseMaster=False, reloadMaster=True): slf.SetFrame(slf._msblaze, msblaze, det) # Prepare some QA for the average slit profile along the slit msgs.info("Preparing QA of each slit profile") - arqa.slit_profile(slf, mstracenrm, slit_profiles, slf._lordloc[det - 1], slf._rordloc[det - 1], - slf._slitpix[det - 1], desc="Slit profile") + arproc.slit_profile_qa(slf, mstracenrm, slit_profiles, slf._lordloc[det - 1], + slf._rordloc[det - 1], slf._slitpix[det - 1], + desc="Slit profile") msgs.info("Saving blaze function QA") - arqa.plot_orderfits(slf, msblaze, flat_ext1d, desc="Blaze function", textplt="Order") +# arqa.plot_orderfits(slf, msblaze, flat_ext1d, desc="Blaze function", textplt="Order") + artrace.plot_orderfits(slf, msblaze, flat_ext1d, desc="Blaze function", + textplt="Order") ############### # Generate/load a master wave frame diff --git a/pypit/armlsd.py b/pypit/armlsd.py index 546b4672ff..b559277169 100644 --- a/pypit/armlsd.py +++ b/pypit/armlsd.py @@ -8,19 +8,20 @@ from pypit import arload from pypit import armasters from pypit import armbase -from pypit import armsgs +#from pypit import armsgs +from pypit import msgs from pypit import arproc from pypit import arsave from pypit import arsetup from pypit import artrace -from pypit import arqa +#from pypit import arqa from linetools import utils as ltu from pypit import ardebug as debugger # Logging -msgs = armsgs.get_logger() +#msgs = armsgs.get_logger() def ARMLSD(fitsdict, reuseMaster=False, reloadMaster=True): @@ -148,9 +149,13 @@ def ARMLSD(fitsdict, reuseMaster=False, reloadMaster=True): # Save to disk armasters.save_masters(slf, det, mftype='trace') # Save QA for slit traces - arqa.slit_trace_qa(slf, slf._mstrace[det-1], slf._lordpix[det-1], - slf._rordpix[det-1], extord, - desc="Trace of the slit edges D{:02d}".format(det), use_slitid=det) +# arqa.slit_trace_qa(slf, slf._mstrace[det-1], slf._lordpix[det-1], +# slf._rordpix[det-1], extord, +# desc="Trace of the slit edges D{:02d}".format(det), use_slitid=det) + artrace.slit_trace_qa(slf, slf._mstrace[det-1], slf._lordpix[det-1], + slf._rordpix[det-1], extord, + desc="Trace of the slit edges D{:02d}".format(det), + use_slitid=det) armbase.UpdateMasters(sciexp, sc, det, ftype="flat", chktype="trace") ############### diff --git a/pypit/armsgs.py b/pypit/armsgs.py index a2a2538ee4..f6b026db14 100644 --- a/pypit/armsgs.py +++ b/pypit/armsgs.py @@ -1,12 +1,29 @@ -from __future__ import absolute_import, division, print_function +""" +Module for terminal and file logging. + +.. todo:: + Why not use pythons native logging package? + +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals import sys -from os.path import dirname, basename -from textwrap import wrap as wraptext -from inspect import currentframe, getouterframes -from glob import glob +import os +import glob +import textwrap +import inspect + +# Imported for versioning +import scipy +import numpy +import astropy -pypit_logger = None +from pypit import __version__, __last_updated__ + +#pypit_logger = None class PypitError(Exception): pass @@ -17,181 +34,227 @@ class Messages: For further details on colours see the following example: http://ascii-table.com/ansi-escape-sequences.php - """ - def __init__(self, log, debug, verbosity, colors=True): - """ - Initialize the Message logging class - - Parameters - ---------- - log : str or None - Name of saved log file (no log will be saved if log=="") - debug : dict - dict used for debugging. - 'LOAD', 'BIAS', 'ARC', 'TRACE' - verbosity : int (0,1,2) - Level of verbosity: - 0 = No output - 1 = Minimal output (default - suitable for the average user) - 2 = All output - colors : bool - If true, the screen output will have colors, otherwise - normal screen output will be displayed - """ - # Version - from pypit import pyputils - version, last_updated = pyputils.get_version() - # Import for version - import scipy - import numpy - import astropy + Parameters + ---------- + log : str or None + Name of saved log file (no log will be saved if log=="") + debug : dict + dict used for debugging. + 'LOAD', 'BIAS', 'ARC', 'TRACE' + verbosity : int (0,1,2) + Level of verbosity: + 0 = No output + 1 = Minimal output (default - suitable for the average user) + 2 = All output + colors : bool + If true, the screen output will have colors, otherwise + normal screen output will be displayed + """ + def __init__(self, log=None, debug=None, verbosity=None, colors=True): - # Initialize the log - if log is not None: - self._log = open(log, 'w') - else: - self._log = log # Initialize other variables + # TODO: debug could just be develop=True or False self._debug = debug - self._last_updated = last_updated - self._version = version - self._verbosity = verbosity + self._verbosity = 1 if verbosity is None else verbosity + self._last_updated = __last_updated__ + self._version = __version__ + + # TODO: Why are these two necessary? It would seem better to + # provide Messages with member functions that can operate on + # sciexp and pypit_file instead of having them kept within the + # object itself... self.sciexp = None self.pypit_file = None - # Save the version of the code including last update information to the log file - if self._log: - self._log.write("------------------------------------------------------\n\n") - self._log.write("PYPIT was last updated {0:s}\n".format(last_updated)) - self._log.write("This log was generated with version {0:s} of PYPIT\n\n".format(version)) - self._log.write("You are using scipy version={:s}\n".format(scipy.__version__)) - self._log.write("You are using numpy version={:s}\n".format(numpy.__version__)) - self._log.write("You are using astropy version={:s}\n\n".format(astropy.__version__)) - self._log.write("------------------------------------------------------\n\n") + + # Initialize the log + self._log = None + self._initialize_log_file(log=log) + # Use colors? - self._start, self._end = "", "" - self._black_CL, self._yellow_CL, self._blue_CL, self._green_CL, self._red_CL = "", "", "", "", "" - self._white_RD, self._white_GR, self._white_BK = "", "", "" - self._white_BL, self._black_YL, self._yellow_BK = "", "", "" + self._start = None + self._end = None + self._black_CL = None + self._yellow_CL = None + self._blue_CL = None + self._green_CL = None + self._red_CL = None + self._white_RD = None + self._white_GR = None + self._white_BK = None + self._white_BL = None + self._black_YL = None + self._yellow_BK = None + + self.disablecolors() if colors: self.enablecolors() + + def _cleancolors(self, msg): + cols = [self._end, self._start, + self._black_CL, self._yellow_CL, self._blue_CL, self._green_CL, self._red_CL, + self._white_RD, self._white_GR, self._white_BK, self._white_BL, + self._black_YL, self._yellow_BK] + for i in cols: + msg = msg.replace(i, '') + return msg + + def _debugmessage(self): + if self._debug is not None and self._debug['develop']: + info = inspect.getouterframes(inspect.currentframe())[2] + dbgmsg = self._start + self._blue_CL + info[1].split('/')[-1] + ' ' + str(info[2]) \ + + ' ' + info[3] + '()' + self._end + ' - ' else: - self.disablecolors() + dbgmsg = '' + return dbgmsg + + def _print(self, premsg, msg, last=True, debug=True): + """ + Print to standard error and the log file + """ + dbgmsg = self._debugmessage() if debug else '' + _msg = premsg+dbgmsg+msg + print(_msg, file=sys.stderr) + if self._log: + clean_msg = self._cleancolors(_msg) + self._log.write(clean_msg+'\n' if last else clean_msg) + + def _initialize_log_file(self, log=None): + """ + Expects self._log is already None. + """ + if log is None: + return + + # Initialize the log + self._log = open(log, 'w') + + self._log.write("------------------------------------------------------\n\n") + self._log.write("PYPIT was last updated {0:s}\n".format(self._last_updated)) + self._log.write("This log was generated with version {0:s} of PYPIT\n\n".format( + self._version)) + self._log.write("You are using scipy version={:s}\n".format(scipy.__version__)) + self._log.write("You are using numpy version={:s}\n".format(numpy.__version__)) + self._log.write("You are using astropy version={:s}\n\n".format(astropy.__version__)) + self._log.write("------------------------------------------------------\n\n") + + def reset(self, log=None, debug=None, verbosity=None, colors=True): + """ + Reinitialize the object. + + Needed so that there can be a default object for all modules, + but also a dynamically defined log file. + """ + # Initialize other variables + self._debug = debug + self._verbosity = 1 if verbosity is None else verbosity + self.reset_log_file(log) + self.disablecolors() + if colors: + self.enablecolors() + + def reset_log_file(self, log): + if self._log: + self._log.close() + self._log = None + self._initialize_log_file(log=log) # Headers and usage + # TODO: Move this to the ARMED class... def armedheader(self, prognm): """ Get the info header for ARMED """ - header = "## " - header += self._start + self._white_GR + "ARMED : " - header += "Automated Reduction and Modelling of Echelle Data v{0:s}".format(self._version) + self._end + "\n" - header += "## " - header += "Usage : " - header += "python %s [options] filelist".format(prognm) + header = '## ' + header += self._start + self._white_GR + 'ARMED : ' + header += 'Automated Reduction and Modelling of Echelle Data v{0:s}'.format( + self._version) + self._end + '\n' + header += '## ' + header += 'Usage : ' + header += 'python %s [options] filelist'.format(prognm) return header def pypitheader(self, prognm): """ Get the info header for PYPIT """ - header = "## " - header += self._start + self._white_GR + "PYPIT : " - header += "The Python Spectroscopic Data Reduction Pipeline v{0:s}".format(self._version) + self._end + "\n" - header += "## " - #header += "Usage : " - #if prognm is None: - # header += "pypit [options] filename.red" - #else: - # header += "python %s [options] filename.red".format(prognm) + header = '## ' + header += self._start + self._white_GR + 'PYPIT : ' + header += 'The Python Spectroscopic Data Reduction Pipeline v{0:s}'.format( + self._version) + self._end + '\n' + header += '## ' return header def usage(self, prognm): - stgs_arm = glob(dirname(__file__)+"/data/settings/settings.arm*") - stgs_all = glob(dirname(__file__)+"/data/settings/settings.*") + """ + Print pypit usage data. + """ + stgs_arm = glob.glob(os.path.dirname(__file__)+'/data/settings/settings.arm*') + stgs_all = glob.glob(os.path.dirname(__file__)+'/data/settings/settings.*') stgs_spc = list(set(stgs_arm) ^ set(stgs_all)) - armlist = basename(stgs_arm[0]).split(".")[-1] + + armlist = os.path.basename(stgs_arm[0]).split('.')[-1] for i in range(1, len(stgs_arm)): - armlist += ", " + basename(stgs_arm[i]).split(".")[-1] - spclist = basename(stgs_spc[0]).split(".")[-1] + armlist += ', ' + os.path.basename(stgs_arm[i]).split('.')[-1] + spclist = os.path.basename(stgs_spc[0]).split('.')[-1] for kk,istsp in enumerate(stgs_spc): if (kk == 0) or ('base' in istsp) or ('py' in istsp.split('.')[-1]): continue - spclist += ", " + istsp.split(".")[-1] - spcl = wraptext(spclist, width=60) - #print("\n#################################################################") - #print(self.pypitheader(prognm)) + spclist += ', ' + istsp.split('.')[-1] + + spcl = textwrap.wrap(spclist, width=60) descs = self.pypitheader(prognm) - #print("## -------------------------------------------------------------") - #print("## Options: (default values in brackets)") - #print("## -c or --cpus : (all) Number of cpu cores to use") - #print("## -h or --help : Print this message") - #print("## -v or --verbosity : (2) Level of verbosity (0-2)") - #print("## -m or --use_masters : Use files in MasterFrames for reduction") - #print("## -d or --develop : Turn develop debugging on") - #print("## -------------------------------------------------------------") - descs += "\n## Available pipelines include:" - #print("## Available pipelines include:") - descs += "\n## " + armlist - #print("## Available spectrographs include:") - descs += "\n## Available spectrographs include:" + + descs += '\n## Available pipelines include:' + descs += '\n## ' + armlist + + descs += '\n## Available spectrographs include:' for ispcl in spcl: - descs += "\n## " + ispcl - #print("## " + i) - #print("## -------------------------------------------------------------") - #print("## Last updated: {0:s}".format(self._last_updated)) - descs += "\n## Last updated: {0:s}".format(self._last_updated) - #print("#################################################################\n") - #sys.exit() - return descs + descs += '\n## ' + ispcl - def debugmessage(self): - if self._debug['develop']: - info = getouterframes(currentframe())[2] - dbgmsg = self._start+self._blue_CL+info[1].split("/")[-1]+" "+str(info[2])+" "+info[3]+"()"+self._end+" - " - else: - dbgmsg = "" - return dbgmsg + descs += '\n## Last updated: {0:s}'.format(self._last_updated) + + return descs def close(self): - """ - Close the log file and QA PDFs before the code exits - """ + ''' + Close the log file before the code exits + ''' from pypit import arqa # QA HTML if self.pypit_file is not None: # Likely testing try: arqa.gen_mf_html(self.pypit_file) - except: # Likely crashed real early + except: # Likely crashed very early pass else: arqa.gen_exp_html() - # Close log - if self._log: - self._log.close() - return + return self.reset_log_file(None) - def signal_handler(self, signalnum, handler): - """ - Handle signals sent by the keyboard during code execution - """ - if signalnum == 2: - self.info("Ctrl+C was pressed. Ending processes...") - self.close() - sys.exit() - return +# # Close log +# if self._log: +# self._log.close() + +# def signal_handler(self, signalnum, handler): +# """ +# Handle signals sent by the keyboard during code execution +# """ +# if signalnum == 2: +# self.info('Ctrl+C was pressed. Ending processes...') +# self.close() +# sys.exit() def error(self, msg, usage=False): """ Print an error message """ - dbgmsg = self.debugmessage() - premsg = "\n"+self._start + self._white_RD + "[ERROR] ::" + self._end + " " - print(premsg+dbgmsg+msg, file=sys.stderr) - if self._log: - self._log.write(self.cleancolors(premsg+dbgmsg+msg)+"\n") - # Close PDFs and log file + premsg = '\n'+self._start + self._white_RD + '[ERROR] ::' + self._end + ' ' + self._print(premsg, msg) + + # Close log file + # TODO: This no longer "closes" the QA plots self.close() + # Print command line usage if usage: self.usage(None) @@ -203,91 +266,59 @@ def info(self, msg): """ Print an information message """ - dbgmsg = self.debugmessage() - premsg = self._start + self._green_CL + "[INFO] ::" + self._end + " " - print(premsg+dbgmsg+msg, file=sys.stderr) - if self._log: - self._log.write(self.cleancolors(premsg+dbgmsg+msg)+"\n") - return + premsg = self._start + self._green_CL + '[INFO] ::' + self._end + ' ' + self._print(premsg, msg) def info_update(self, msg, last=False): """ Print an information message that needs to be updated """ - dbgmsg = self.debugmessage() - premsg = "\r" + self._start + self._green_CL + "[INFO] ::" + self._end + " " - if last: - print(premsg+dbgmsg+msg, file=sys.stderr) - if self._log: - self._log.write(self.cleancolors(premsg+dbgmsg+msg)+"\n") - else: - print(premsg+dbgmsg+msg, file=sys.stderr) - if self._log: - self._log.write(self.cleancolors(premsg+dbgmsg+msg)) - return + premsg = '\r' + self._start + self._green_CL + '[INFO] ::' + self._end + ' ' + self._print(premsg, msg, last=last) def test(self, msg): """ Print a test message """ if self._verbosity == 2: - dbgmsg = self.debugmessage() - premsg = self._start + self._white_BL + "[TEST] ::" + self._end + " " - print(premsg+dbgmsg+msg, file=sys.stderr) - if self._log: - self._log.write(self.cleancolors(premsg+dbgmsg+msg)+"\n") - return + premsg = self._start + self._white_BL + '[TEST] ::' + self._end + ' ' + self._print(premsg, msg) def warn(self, msg): """ Print a warning message """ - dbgmsg = self.debugmessage() - premsg = self._start + self._red_CL + "[WARNING] ::" + self._end + " " - print(premsg+dbgmsg+msg, file=sys.stderr) - if self._log: - self._log.write(self.cleancolors(premsg+dbgmsg+msg)+"\n") - return + premsg = self._start + self._red_CL + '[WARNING] ::' + self._end + ' ' + self._print(premsg, msg) def bug(self, msg): """ Print a bug message """ - dbgmsg = self.debugmessage() - premsg = self._start + self._white_BK + "[BUG] ::" + self._end + " " - print(premsg+dbgmsg+msg, file=sys.stderr) - if self._log: - self._log.write(self.cleancolors(premsg+dbgmsg+msg)) - return + premsg = self._start + self._white_BK + '[BUG] ::' + self._end + ' ' + self._print(premsg, msg) def work(self, msg): """ Print a work in progress message """ - if self._debug['develop']: - dbgmsg = self.debugmessage() - premsgp = self._start + self._black_CL + "[WORK IN ]::" + self._end + "\n" - premsgs = self._start + self._yellow_CL + "[PROGRESS]::" + self._end + " " - print(premsgp+premsgs+dbgmsg+msg, file=sys.stderr) - if self._log: - self._log.write(self.cleancolors(premsgp+premsgs+dbgmsg+msg)+"\n") - return + if self._debug is not None and self._debug['develop']: + premsgp = self._start + self._black_CL + '[WORK IN ]::' + self._end + '\n' + premsgs = self._start + self._yellow_CL + '[PROGRESS]::' + self._end + ' ' + self._print(premsgp+premsgs, msg) def prindent(self, msg): """ Print an indent """ - premsg = " " - print(premsg+msg, file=sys.stderr) - if self._log: - self._log.write(self.cleancolors(premsg+msg)+"\n") - return + premsg = ' ' + self._print(premsg, msg, debug=False) def input(self): """ Return a text string to be used to display input required from the user """ - premsg = self._start + self._blue_CL + "[INPUT] ::" + self._end + " " + premsg = self._start + self._blue_CL + '[INPUT] ::' + self._end + ' ' return premsg @staticmethod @@ -295,14 +326,14 @@ def newline(): """ Return a text string containing a newline to be used with messages """ - return "\n " + return '\n ' @staticmethod def indent(): """ Return a text string containing an indent to be used with messages """ - return " " + return ' ' # Set the colors def enablecolors(self): @@ -311,31 +342,23 @@ def enablecolors(self): """ # Start and end coloured text - self._start = "\x1B[" - self._end = "\x1B[" + "0m" + self._start = '\x1B[' + self._end = '\x1B[' + '0m' # Clear Backgrounds - self._black_CL = "1;30m" - self._yellow_CL = "1;33m" - self._blue_CL = "1;34m" - self._green_CL = "1;32m" - self._red_CL = "1;31m" + self._black_CL = '1;30m' + self._yellow_CL = '1;33m' + self._blue_CL = '1;34m' + self._green_CL = '1;32m' + self._red_CL = '1;31m' # Coloured Backgrounds - self._white_RD = "1;37;41m" - self._white_GR = "1;37;42m" - self._white_BK = "1;37;40m" - self._white_BL = "1;37;44m" - self._black_YL = "1;37;43m" - self._yellow_BK = "1;33;40m" - - def cleancolors(self, msg): - cols = [self._end, self._start, - self._black_CL, self._yellow_CL, self._blue_CL, self._green_CL, self._red_CL, - self._white_RD, self._white_GR, self._white_BK, self._white_BL, self._black_YL, self._yellow_BK] - for i in cols: - msg = msg.replace(i, "") - return msg + self._white_RD = '1;37;41m' + self._white_GR = '1;37;42m' + self._white_BK = '1;37;40m' + self._white_BL = '1;37;44m' + self._black_YL = '1;37;43m' + self._yellow_BK = '1;33;40m' def disablecolors(self): """ @@ -343,43 +366,43 @@ def disablecolors(self): """ # Start and end coloured text - self._start = "" - self._end = "" + self._start = '' + self._end = '' # Clear Backgrounds - self._black_CL = "" - self._yellow_CL = "" - self._blue_CL = "" - self._green_CL = "" - self._red_CL = "" + self._black_CL = '' + self._yellow_CL = '' + self._blue_CL = '' + self._green_CL = '' + self._red_CL = '' # Coloured Backgrounds - self._white_RD = "" - self._white_GR = "" - self._white_BK = "" - self._white_BL = "" - self._black_YL = "" - self._yellow_BK = "" - - -def get_logger(init=None): - """ Logger - Parameters - ---------- - init : tuple - For instantiation - (log, debug, verbosity) - - Returns - ------- - msgs : Messages - """ - global pypit_logger - - # Instantiate?? - if init is not None: - pypit_logger = Messages(init[0], init[1], init[2]) - - return pypit_logger + self._white_RD = '' + self._white_GR = '' + self._white_BK = '' + self._white_BL = '' + self._black_YL = '' + self._yellow_BK = '' + + +#def get_logger(init=None): +# """ Logger +# Parameters +# ---------- +# init : tuple +# For instantiation +# (log, debug, verbosity) +# +# Returns +# ------- +# msgs : Messages +# """ +# global pypit_logger +# +# # Instantiate?? +# if init is not None: +# pypit_logger = Messages(init[0], init[1], init[2]) +# +# return pypit_logger diff --git a/pypit/arparse.py b/pypit/arparse.py index 4ba53aa178..bf7cead7b3 100644 --- a/pypit/arparse.py +++ b/pypit/arparse.py @@ -12,9 +12,10 @@ # Logging from pypit import ardebug -from pypit import armsgs +#from pypit import armsgs +from pypit import msgs debug = ardebug.init() -msgs = armsgs.get_logger() +#msgs = armsgs.get_logger() # Initialize the settings variables argflag, spect = None, None diff --git a/pypit/arpca.py b/pypit/arpca.py index 9df9b9b18e..cdd7ec5f0d 100644 --- a/pypit/arpca.py +++ b/pypit/arpca.py @@ -2,22 +2,16 @@ import inspect -from matplotlib import pyplot as plt import numpy as np -from pypit import armsgs -from pypit import arutils -from pypit.arqa import get_dimen, set_qa_filename - -from pypit import ardebug as debugger -# Logging -msgs = armsgs.get_logger() - -# Force the default matplotlib plotting parameters -plt.rcdefaults() +from matplotlib import pyplot as plt +from pypit import msgs +from pypit import arutils +from pypit import arqa -def basis(xfit, yfit, coeff, npc, pnpc, weights=None, skipx0=True, x0in=None, mask=None, function='polynomial'): +def basis(xfit, yfit, coeff, npc, pnpc, weights=None, skipx0=True, x0in=None, mask=None, + function='polynomial'): nrow = xfit.shape[0] ntrace = xfit.shape[1] if x0in is None: @@ -297,7 +291,7 @@ def pc_plot_extcenwid(tempcen, cenwid, binval, plotsdir="Plots", pcatype="