add CDS variable event

sdp/bin/spconst.py

@@ -30,6 +30,8 @@
 EVENT_CDF_COR_VARIABLE_O='cdf_cor_variable_o' # project with One variable per dataset
 EVENT_CDF_COR_VARIABLE_N='cdf_cor_variable_n' # project with N variable per dataset
 
+EVENT_CDS_VARIABLE='cds_variable'
+
 EVENT_STATUS_NEW='new'
 EVENT_STATUS_ANOMALY='anomaly'
 EVENT_STATUS_OLD='old'
@@ -57,8 +59,8 @@
 #
 DEFAULT_PRIORITY=1000
 #
-AUTHORIZED_PROJECT=['CORDEX','CMIP5','CMIP6']
-PROJECT_WITH_ONE_VARIABLE_PER_DATASET=['CORDEX','CMIP6']
+AUTHORIZED_PROJECT=['CORDEX','CMIP5','CMIP6','c3scmip5','c3scordex']
+PROJECT_WITH_ONE_VARIABLE_PER_DATASET=['CORDEX','CMIP6','c3scmip5','c3scordex']
 #
 TRIGGER_TYPE_NV2D='NV2D' # N 'variable pipeline' trigger 'dataset pipeline'
 TRIGGER_TYPE_V2V='V2V'   # 'variable pipeline' triggers 'variable pipeline'

sdt/bin/sdconst.py

@@ -32,6 +32,8 @@
 EVENT_CDF_COR_VARIABLE_O='cdf_cor_variable_o' # project with One variable per dataset
 EVENT_CDF_COR_VARIABLE_N='cdf_cor_variable_n' # project with N variable per dataset
 
+EVENT_CDS_VARIABLE='cds_variable'
+
 EVENT_STATUS_NEW='new'
 EVENT_STATUS_ANOMALY='anomaly'
 EVENT_STATUS_OLD='old'

sdt/bin/sdstat.py

@@ -23,64 +23,63 @@
 import argparse
 import json
 import humanize
-import sdapp
-import sdconst
 import sdstatutils
 import syndautils
 import sdexception
 import sdpipelineprocessing
+from sdtools import print_stderr
 
-def run(args):
 
+def run(args):
     try:
-        metadata=syndautils.file_full_search(args)
+        metadata = syndautils.file_full_search(args)
     except sdexception.EmptySelectionException, e:
         print_stderr("You must specify at least one facet to perform this action.")
         return 1
 
     if args.dry_run:
         return 0
 
+    statuses = sdstatutils.init_table()
+    po = sdpipelineprocessing.ProcessingObject(sdstatutils.get_statuses, statuses)
+    sdpipelineprocessing.run_pipeline(metadata, po)
 
-    statuses=sdstatutils.init_table()
-    po=sdpipelineprocessing.ProcessingObject(sdstatutils.get_statuses,statuses)
-    sdpipelineprocessing.run_pipeline(metadata,po)
-
-
-    total=sdstatutils.get_total(statuses)
-    print_summary(statuses,total)
+    total = sdstatutils.get_total(statuses)
+    print_summary(statuses, total)
 
     return 0
 
-def print_summary(statuses,total,mode='line'):
-    if mode=='delimited':
+
+def print_summary(statuses, total, mode='line'):
+    if mode == 'delimited':
         print "Total size;Pending size;Local size;Total files count;Pending files count;Local files count"
-        print "%s;%s;%s;%s;%s;%s"%( humanize.naturalsize(total['all']['size'],gnu=False),
-                                    humanize.naturalsize(total['pending']['size'],gnu=False),
-                                    humanize.naturalsize(statuses['done']['size'],gnu=False),
-                                    humanize.naturalsize(statuses['new']['size'],gnu=False),
-                                    total['all']['count'],
-                                    total['pending']['count'],
-                                    statuses['done']['count'],
-                                    statuses['new']['count'])
-    elif mode=='line':
-        print "Total files count: %s"%total['all']['count']
-        #print "Pending files count: %s"%total['pending']['count']
+        print "%s;%s;%s;%s;%s;%s" % (humanize.naturalsize(total['all']['size'], gnu=False),
+                                     humanize.naturalsize(total['pending']['size'], gnu=False),
+                                     humanize.naturalsize(statuses['done']['size'], gnu=False),
+                                     humanize.naturalsize(statuses['new']['size'], gnu=False),
+                                     total['all']['count'],
+                                     total['pending']['count'],
+                                     statuses['done']['count'],
+                                     statuses['new']['count'])
+    elif mode == 'line':
+        print "Total files count: %s" % total['all']['count']
+        # print "Pending files count: %s"%total['pending']['count']
         for s in statuses.keys():
-            count=statuses[s]['count']
-            if count>0:
-                print "%s files count: %s"%(s.title(),count)
-        print "Total size: %s"%humanize.naturalsize(total['all']['size'],gnu=False)
-        #print "Pending size: %s"%humanize.naturalsize(total['pending']['size'],gnu=False)
+            count = statuses[s]['count']
+            if count > 0:
+                print "%s files count: %s" % (s.title(), count)
+        print "Total size: %s" % humanize.naturalsize(total['all']['size'], gnu=False)
+        # print "Pending size: %s"%humanize.naturalsize(total['pending']['size'],gnu=False)
         for s in statuses.keys():
-            size=statuses[s]['size']
-            if size>0:
-                print "%s files size: %s"%(s.title(),humanize.naturalsize(size,gnu=False))
+            size = statuses[s]['size']
+            if size > 0:
+                print "%s files size: %s" % (s.title(), humanize.naturalsize(size, gnu=False))
+
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
     args = parser.parse_args()
 
-    files=json.load( sys.stdin ) # warning: load list in memory
+    files = json.load(sys.stdin)  # warning: load list in memory
 
     run(files)

sdt/bin/sdsubparser.py

@@ -218,7 +218,7 @@ def run(subparsers):
     subparser.add_argument('-c','--columns',type=int,default=1)
 
     subparser=create_subparser(subparsers,'pexec',help='Execute post-processing task')
-    subparser.add_argument('order_name',choices=['cdf'],help='Order name')
+    subparser.add_argument('order_name',choices=['cdf','cds'],help='Order name')
 
     subparser=create_subparser(subparsers,'queue',common_option=False,help='Display download queue status',example=sdcliex.queue())
     subparser.add_argument('project',nargs='?',default=None,help='ESGF project (e.g. CMIP5)')