added back ressource files, use level attributes for output definition

setup.py

@@ -31,7 +31,7 @@
     packages=setuptools.find_packages(where="src"),
     python_requires=">=3.8",
     package_data={
-    	"pypromice.ressources": ["variable_description.csv", "variable_processing_info.csv"],
+    	"pypromice.ressources": ["variables.csv", "file_attributes.csv", "variable_aliases_GC-Net.csv"],
     	"pypromice.tx": ["payload_formats.csv", "payload_types.csv"],
         "pypromice.qc.percentiles": ["thresholds.csv"],
         "pypromice.postprocess": ["station_configurations.toml", "positions_seed.csv"],

src/pypromice/process/L0toL1.py

@@ -31,6 +31,7 @@ def toL1(L0, vars_df, T_0=273.15, tilt_threshold=-100):
     '''
     assert(type(L0) == xr.Dataset)
     ds = L0
+    ds.attrs['level'] = 'L1'
 
     for l in list(ds.keys()):
         if l not in ['time', 'msg_i', 'gps_lat', 'gps_lon', 'gps_alt', 'gps_time']:

src/pypromice/process/L1toL2.py

@@ -70,6 +70,7 @@ def toL2(
         Level 2 dataset
     '''
     ds = L1.copy(deep=True)                                                    # Reassign dataset
+    ds.attrs['level'] = 'L2'
     try:
         ds = adjustTime(ds)                                                    # Adjust time after a user-defined csv files
         ds = flagNAN(ds)                                                       # Flag NaNs after a user-defined csv files

src/pypromice/process/L2toL3.py

@@ -32,6 +32,7 @@ def toL3(L2, T_0=273.15, z_0=0.001, R_d=287.05, eps=0.622, es_0=6.1071,
         1013.246.
     '''
     ds = L2
+    ds.attrs['level'] = 'L3'
 
     T_100 = _getTempK(T_0)                                                     # Get steam point temperature as K 
 

src/pypromice/process/get_l2.py

@@ -4,7 +4,6 @@
 import pypromice
 from pypromice.process.aws import AWS
 from pypromice.process.write import prepare_and_write
-from pypromice.process.load import getVars, getMeta
 
 def parse_arguments_l2():
     parser = ArgumentParser(description="AWS L2 processor")
@@ -30,26 +29,14 @@ def get_l2():
         level=logging.INFO,
         stream=sys.stdout,
     )
-
-    # Define variables (either from file or pypromice package defaults)
-    if args.variables is None:
-        v = os.path.join(os.path.dirname(pypromice.__file__),'process/variables.csv')
-    else:
-        v = args.variables
-
-    # Define metadata (either from file or pypromice package defaults)
-    if args.variables is None:
-        m = os.path.join(os.path.dirname(pypromice.__file__),'process/metadata.csv')
-    else:
-        m = args.metadata
 
     # Define input path
     station_name = args.config_file.split('/')[-1].split('.')[0] 
     station_path = os.path.join(args.inpath, station_name)
     if os.path.exists(station_path):
-        aws = AWS(args.config_file, station_path, v, m)
+        aws = AWS(args.config_file, station_path, args.variables, args.metadata)
     else:
-        aws = AWS(args.config_file, args.inpath, v, m)
+        aws = AWS(args.config_file, args.inpath, args.variables, args.metadata)
 
     # Perform level 1 and 2 processing
     aws.getL1()
@@ -60,8 +47,8 @@ def get_l2():
         if not os.path.isdir(args.outpath):
             os.mkdir(args.outpath)
         if aws.L2.attrs['format'] == 'raw':
-            prepare_and_write(aws.L2, args.outpath, getVars(), getMeta(), '10min')
-        prepare_and_write(aws.L2, args.outpath, getVars(), getMeta(), '60min')
+            prepare_and_write(aws.L2, args.outpath, args.variables, args.metadata, '10min')
+        prepare_and_write(aws.L2, args.outpath, args.variables, args.metadata, '60min')
 
 
 if __name__ == "__main__":  

src/pypromice/process/get_l2tol3.py

@@ -3,7 +3,6 @@
 import xarray as xr
 from argparse import ArgumentParser
 import pypromice
-from pypromice.process.load import getVars, getMeta
 from pypromice.process.L2toL3 import toL3
 from pypromice.process.write import prepare_and_write
 
@@ -34,10 +33,6 @@ def get_l2tol3():
         level=logging.INFO,
         stream=sys.stdout,
     )
-
-    # Define variables and metadata (either from file or pypromice package defaults)
-    v = getVars(args.variables)
-    m = getMeta(args.metadata)
 
     # Define Level 2 dataset from file
     with xr.open_dataset(args.inpath) as l2:
@@ -55,9 +50,9 @@ def get_l2tol3():
 
     # Write Level 3 dataset to file if output directory given
     if args.outpath is not None:
-        prepare_and_write(l3, args.outpath, v, m, '60min')
-        prepare_and_write(l3, args.outpath, v, m, '1D')
-        prepare_and_write(l3, args.outpath, v, m, 'M')
+        prepare_and_write(l3, args.outpath, args.variables, args.metadata, '60min')
+        prepare_and_write(l3, args.outpath, args.variables, args.metadata, '1D')
+        prepare_and_write(l3, args.outpath, args.variables, args.metadata, 'M')
 
 if __name__ == "__main__":  
     get_l2tol3()

src/pypromice/process/join_l2.py

@@ -3,7 +3,6 @@
 import pandas as pd
 import xarray as xr
 from argparse import ArgumentParser
-from pypromice.process.load import getVars, getMeta
 from pypromice.process.utilities import addMeta, roundValues
 from pypromice.process.write import prepare_and_write
 from pypromice.process.L1toL2 import correctPrecip
@@ -51,10 +50,6 @@ def loadArr(infile):
 def join_l2():
     args = parse_arguments_join()
 
-    # Define variables and metadata (either from file or pypromice package defaults)
-    v = getVars(args.variables)
-    m = getMeta(args.metadata)
-
     # Check files
     if os.path.isfile(args.file1) and os.path.isfile(args.file2): 
 
@@ -99,7 +94,7 @@ def join_l2():
 
 
     # Resample to hourly, daily and monthly datasets and write to file
-    prepare_and_write(all_ds, args.outpath, v, m, resample = False)
+    prepare_and_write(all_ds, args.outpath, args.variables, args.metadata, resample = False)
 
     print(f'Files saved to {os.path.join(args.outpath, name)}...')
 

src/pypromice/process/join_l3.py

@@ -1,14 +1,10 @@
 #!/usr/bin/env python
 import logging, os, sys, unittest, toml, pkg_resources
 from argparse import ArgumentParser
-import pypromice
-from pypromice.process.aws import AWS
-from pypromice.process import load
 from pypromice.process.write import prepare_and_write
 import numpy as np
 import pandas as pd
 import xarray as xr
-from pypromice.process.load import getVars, getMeta
 
 def parse_arguments_joinl3(debug_args=None):
     parser = ArgumentParser(description="AWS L3 script for the processing L3 data from L2 and merging the L3 data with its historical site. An hourly, daily and monthly L3 data product is outputted to the defined output path")
@@ -173,6 +169,7 @@ def join_l3():
     conf = toml.load(config_file)
 
     l3m = xr.Dataset()
+    l3m.attrs['level'] = 'L3'
     for stid in conf['stations']:
         print(stid)
 
@@ -245,13 +242,11 @@ def join_l3():
     # Assign site id
     l3m.attrs['site_id'] = args.site
     l3m.attrs['stations'] = conf['stations']
-    v = getVars()
-    m = getMeta()
+
     if args.outpath is not None:
-        prepare_and_write(l3m, args.outpath, v, m, '60min')
-        prepare_and_write(l3m, args.outpath, v, m, '1D')
-        prepare_and_write(l3m, args.outpath, v, m, 'M')
-    # %% 
+        prepare_and_write(l3m, args.outpath, args.variables, args.metadata, '60min')
+        prepare_and_write(l3m, args.outpath, args.variables, args.metadata, '1D')
+        prepare_and_write(l3m, args.outpath, args.variables, args.metadata, 'M')
 
 if __name__ == "__main__":  
     join_l3()

src/pypromice/process/load.py

@@ -120,6 +120,7 @@ def getL0(infile, nodata, cols, skiprows, file_version,
 
     # Carry relevant metadata with ds
     ds = xr.Dataset.from_dataframe(df)
+    ds.attrs['level'] = 'L0'
     return ds
 
 def getVars(v_file=None):
@@ -136,11 +137,11 @@ def getVars(v_file=None):
        Variables dataframe
    '''
    if v_file is None:
-        with pkg_resources.resource_stream('pypromice', 'process/variables.csv') as stream:
+        with pkg_resources.resource_stream('pypromice', 'ressources/variables.csv') as stream:
             return pd.read_csv(stream, index_col=0, comment="#", encoding='utf-8')
    else:
         return pd.read_csv(v_file, index_col=0, comment="#")
-
+        
 
 def getMeta(m_file=None, delimiter=','):                                            #TODO change to DataFrame output to match variables.csv
     '''Load metadata table
@@ -159,7 +160,7 @@ def getMeta(m_file=None, delimiter=','):
     '''
     meta={}
     if m_file is None:
-        with pkg_resources.resource_stream('pypromice', 'process/metadata.csv') as stream:
+        with pkg_resources.resource_stream('pypromice', 'ressources/file_attributes.csv') as stream:
             lines = stream.read().decode("utf-8")
             lines = lines.split("\n")
     else:

src/pypromice/process/utilities.py

@@ -109,11 +109,9 @@ def populateMeta(ds, conf, skip):
     ds : xarray.Dataset
         L0 dataset with metadata populated as Dataset attributes
     '''
-    meta = {}
     # skip = ["columns", "skiprows"]
     for k in conf.keys():
-        if k not in skip: meta[k] = conf[k]
-    ds.attrs = meta
+        if k not in skip: ds.attrs[k] = conf[k]
     return ds
 
 

src/pypromice/process/write.py

@@ -8,9 +8,9 @@
 logger = logging.getLogger(__name__)
 
 from pypromice.process.resample import resample_dataset
-from pypromice.process import utilities
+from pypromice.process import utilities, load
 
-def prepare_and_write(dataset, outpath, vars_df, meta_dict, time='60min', resample=True):
+def prepare_and_write(dataset, outpath, vars_df=None, meta_dict=None, time='60min', resample=True):
     '''Prepare data with resampling, formating and metadata population; then
     write data to .nc and .csv hourly and daily files
 
@@ -48,7 +48,13 @@ def prepare_and_write(dataset, outpath, vars_df, meta_dict, time='60min', resamp
         d2 = utilities.reformat_lon(d2)
     else:
         logger.info('%s does not have gpd_lon'%name)
+
     # Add variable attributes and metadata
+    if vars_df is None:
+        vars_df = load.getVars()
+    if meta_dict is None:
+        meta_dict = load.getMeta()
+
     d2 = utilities.addVars(d2, vars_df)
     d2 = utilities.addMeta(d2, meta_dict)
 
@@ -90,6 +96,8 @@ def prepare_and_write(dataset, outpath, vars_df, meta_dict, time='60min', resamp
     logger.info(f'Written to {out_csv}')
     logger.info(f'Written to {out_nc}')
 
+
+
 def writeAll(outpath, station_id, l3_h, l3_d, l3_m, csv_order=None):
     '''Write L3 hourly, daily and monthly datasets to .nc and .csv
     files
@@ -169,16 +177,16 @@ def getColNames(vars_df, ds, remove_nan_fields=False):
     list
         Variable names
     '''
-    if 'data_type' in ds.attrs.keys():
-        if ds.attrs['data_type']=='TX':
-            vars_df = vars_df.loc[vars_df['data_type'].isin(['TX','all'])]
-        elif ds.attrs['data_type']=='STM' or ds.attrs['data_type']=='raw':
-            vars_df = vars_df.loc[vars_df['data_type'].isin(['raw','all'])]
-    if 'number_of_booms' in ds.attrs.keys():
+    # selecting variable list based on level
+    vars_df = vars_df.loc[vars_df[ds.attrs['level']] == 1]
+
+    # selecting variable list based on geometry
+    if ds.attrs['level'] in ['L0', 'L1', 'L2']:
         if ds.attrs['number_of_booms']==1:
             vars_df = vars_df.loc[vars_df['station_type'].isin(['one-boom','all'])]
         elif ds.attrs['number_of_booms']==2:
             vars_df = vars_df.loc[vars_df['station_type'].isin(['two-boom','all'])]
+
     var_list = list(vars_df.index)
     if remove_nan_fields:
         for v in var_list:

src/pypromice/ressources/file_attributes.csv

@@ -0,0 +1,56 @@
+attribute,entry
+acknowledgements,The Programme for Monitoring of the Greenland Ice Sheet (PROMICE)
+alt.axis,Z
+alt.coverage_content_type,coordinate
+gps_alt.positive,up
+cdm_data_type,
+comment,https://doi.org/10.22008/promice/data/aws
+contributor_name,
+contributor_role,
+conventions,ACDD-1.3; CF-1.7
+creater_email,pho@geus.dk
+creater_url,https://promice.dk
+creator_institution,Geological Survey of Denmark and Greenland (GEUS)
+creator_name,Penelope How
+creator_type,person
+featureType,timeSeries
+geospatial_bounds_crs,EPSG:4979
+geospatial_lat_extents_match,gps_lat
+geospatial_lat_resolution,
+geospatial_lat_units,degrees_north
+geospatial_lon_extents_match,gps_lon
+geospatial_lon_resolution,
+geospatial_lon_units,degrees_east
+geospatial_vertical_resolution,
+geospatial_vertical_units,EPSG:4979
+institution,Geological Survey of Denmark and Greenland (GEUS)
+instrument,See https://doi.org/10.5194/essd-13-3819-2021
+instrument_vocabulary,GCMD:GCMD Keywords
+keywords,GCMDSK:EARTH SCIENCE > CRYOSPHERE > GLACIERS/ICE SHEETS > ICE SHEETS > ICE SHEET MEASUREMENTS; GCMDSK:EARTH SCIENCE > CRYOSPHERE > GLACIERS/ICE SHEETS > GLACIER MASS BALANCE/ICE SHEET MASS BALANCE; GCMDSK:EARTH SCIENCE > CRYOSPHERE > SNOW/ICE > SNOW/ICE TEMPERATURE; GCMDSK:EARTH SCIENCE > CRYOSPHERE > SNOW/ICE; GCMDSK:EARTH SCIENCE > CRYOSPHERE > SNOW/ICE > SNOW MELT; GCMDSK:EARTH SCIENCE > CRYOSPHERE > SNOW/ICE > SNOW DEPTH; GCMDSK:EARTH SCIENCE > CRYOSPHERE > SNOW/ICE > ICE VELOCITY; GCMDSK:EARTH SCIENCE > CRYOSPHERE > SNOW/ICE > ALBEDO; GCMDSK:EARTH SCIENCE > TERRESTRIAL HYDROSPHERE > SNOW/ICE > ALBEDO; GCMDSK:EARTH SCIENCE > TERRESTRIAL HYDROSPHERE > SNOW/ICE > ICE GROWTH/MELT; GCMDSK:EARTH SCIENCE > TERRESTRIAL HYDROSPHERE > SNOW/ICE > ICE VELOCITY; GCMDSK:EARTH SCIENCE > TERRESTRIAL HYDROSPHERE > SNOW/ICE > SNOW DEPTH; GCMDSK:EARTH SCIENCE > TERRESTRIAL HYDROSPHERE > SNOW/ICE > SNOW MELT; GCMDSK:EARTH SCIENCE > TERRESTRIAL HYDROSPHERE > SNOW/ICE > SNOW/ICE TEMPERATURE; GCMDSK:EARTH SCIENCE > TERRESTRIAL HYDROSPHERE > SNOW/ICE; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC PRESSURE; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > ALBEDO; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > INCOMING SOLAR RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > LONGWAVE RADIATION > DOWNWELLING LONGWAVE RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > LONGWAVE RADIATION > UPWELLING LONGWAVE RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > LONGWAVE RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > NET RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > OUTGOING LONGWAVE RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > RADIATIVE FLUX; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > RADIATIVE FORCING; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > SHORTWAVE RADIATION > DOWNWELLING SHORTWAVE RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > SHORTWAVE RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > SUNSHINE; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC TEMPERATURE > SURFACE TEMPERATURE > AIR TEMPERATURE; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC WATER VAPOR > WATER VAPOR INDICATORS > HUMIDITY > ABSOLUTE HUMIDITY; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC WATER VAPOR > WATER VAPOR INDICATORS > HUMIDITY > RELATIVE HUMIDITY; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC WINDS > LOCAL WINDS; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC WINDS > SURFACE WINDS > U/V WIND COMPONENTS; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC WINDS > SURFACE WINDS > WIND DIRECTION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC WINDS > SURFACE WINDS > WIND SPEED; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC WINDS > SURFACE WINDS; GCMDSK:EARTH SCIENCE > ATMOSPHERE > CLOUDS; GCMDSK:EARTH SCIENCE > ATMOSPHERE > PRECIPITATION
+keywords_vocabulary,GCMDSK:GCMD Science Keywords:https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/sciencekeywords
+lat.axis,Y
+lat.coverage_content_type,coordinate
+lat.long_name,station latitude
+license,Creative Commons Attribution 4.0 International (CC-BY-4.0) https://creativecommons.org/licenses/by/4.0
+lon.axis,X
+lon.coverage_content_type,coordinate
+lon.long_name,station longitude
+lon.units,degrees_east
+metadata_link,
+naming_authority,dk.geus.promice
+platform,
+platform_vocabulary,GCMD:GCMD Keywords
+processing_level,Level 3
+product_status,beta
+product_version,4
+program,PROMICE
+project,PROMICE
+publisher_email,info@promice.dk
+publisher_institution,GEUS
+publisher_name,GEUS
+publisher_type,institution
+publisher_url,https://promice.dk
+references,"How, P.; Abermann, J.; Ahlstrøm, A.P.; Andersen, S.B.; Box, J. E.; Citterio, M.; Colgan, W.T.; Fausto. R.S.; Karlsson, N.B.; Jakobsen, J.; Langley, K.; Larsen, S.H.; Mankoff, K.D.; Pedersen, A.Ø.; Rutishauser, A.; Shield, C.L.; Solgaard, A.M.; van As, D.; Vandecrux, B.; Wright, P.J., 2022, ""PROMICE and GC-Net automated weather station data in Greenland"", https://doi.org/10.22008/FK2/IW73UU, GEUS Dataverse"
+references_bib,@article{How2022; doi = {10.22008/FK2/IW73UU}; url = {https://doi.org/10.22008/FK2/IW73UU}; year = {2022}; month=10; publisher= {GEUS Dataverse}; author = {Penelope How and Jakob Abermann and Andreas P. Ahlstr{\o}m and  Signe B. Andersen and Jason E. Box and Michele Citterio and  William Colgan and Robert S. Fausto and Nanna B. Karlsson and Jakob Jakobsen and Kirsty Langley and Signe Hillerup Larsen and Kenneth D. Mankoff and Allan {\O}. Pedersen and Anja Rutishauser and Christopher L. Shields and Anne M. Solgaard and Dirk van As and Baptiste Vandecrux}; title = {PROMICE and GC-Net automated weather station data in Greenland}; journal = {GEUS Dataverse}}
+standard_name_vocabulary,CF Standard Name Table (v77; 19 January 2021)
+summary,"The Programme for Monitoring of the Greenland Ice Sheet (PROMICE) and Greenland Climate Network (GC-Net) have been measuring climate and ice sheet properties since 2007 and 1995, respectively. The PROMICE weather station network monitors glacier mass balance in the melt zone of the Greenland Ice Sheet, providing ground truth data to calibrate mass budget models. GC-Net weather stations measure snowfall and surface properties in the accumulation zone, providing valuable knowledge on the Greenland Ice Sheet’s mass gain and climatology.Accurate measurements of the surface and near-surface atmospheric conditions in a changing climate is important for reliable present and future assessment of changes to the Greenland Ice Sheet. All measurements are handled and processed with pypromice, which is a peer-reviewed and freely available Python package with source code available at https://github.com/GEUS-Glaciology-and-Climate/pypromice. A user-contributable dynamic web-based database of known data quality issues is associated with the data products at https://github.com/GEUS-PROMICE/ PROMICE-AWS-data-issues/."