convert_grib2_to_nc.py: toogle dask progress bar to avoid noise in logs

Ouranosinc · Dec 18, 2020 · a64a444 · a64a444
1 parent a6e4c2b
commit a64a444
Show file tree

Hide file tree

Showing 2 changed files with 32 additions and 2 deletions.
diff --git a/ECCC-datamart_sync/convert_grib2_to_nc.py b/ECCC-datamart_sync/convert_grib2_to_nc.py
@@ -201,7 +201,9 @@ def reformat_nc(job):
     ncfiles, outfile, var_dict = job
 
     print(outfile.name)
-    with ProgressBar():
+
+    @progressbar_toogle
+    def do_reformat_nc():
         ds_all = []
         for v in ncfiles:
 
@@ -250,6 +252,8 @@ def reformat_nc(job):
         proc.join()
         proc.close()
 
+    do_reformat_nc()
+
 
 def write_nc(inputs):
     ds, outfile = inputs
@@ -291,9 +295,14 @@ def convert(fn):
             encoding = {var: dict(zlib=True) for var in ds.data_vars}
             encoding["time"] = {"dtype": "single"}
             tmpfile = tempfile.NamedTemporaryFile(suffix='.nc', delete=False)
-            with ProgressBar():
+
+            @progressbar_toogle
+            def to_netcdf():
                 print('converting ', infile.name)
                 ds.to_netcdf(tmpfile.name, format='NETCDF4', engine="netcdf4", encoding=encoding)
+
+            to_netcdf()
+
             shutil.move(tmpfile.name, outpath.joinpath(infile.name.replace(".grib2", ".nc")).as_posix())
 
     except:
@@ -303,5 +312,25 @@ def convert(fn):
         pass
 
 
+def progressbar_toogle(func):
+    """Decorator to toogle usage of ProgressBar context manager.
+
+    ProgressBar is not used if environment variable
+    'CONVERT_GRIB2_TO_NC_PROGRESSBAR' exists and is set to 'false'.
+
+    Useful in automation because ProgressBar generate lots of noises in logs.
+    """
+
+    def wrapper():
+        if os.environ.get('CONVERT_GRIB2_TO_NC_PROGRESSBAR', default="") == 'false':
+            # Not using ProgressBar.
+            func()
+        else:
+            # Else use ProgressBar.
+            with ProgressBar():
+                func()
+    return wrapper
+
+
 if __name__ == '__main__':
     main()
diff --git a/ECCC-datamart_sync/run_convert_grib2_to_nc b/ECCC-datamart_sync/run_convert_grib2_to_nc
@@ -92,6 +92,7 @@ docker run --rm --name run_convert_grib2_to_nc \
     --env CONVERT_GRIB2_TO_NC_INPATH="$RUN_CONVERT_GRIB2_TO_NC_INPATH" \
     --env CONVERT_GRIB2_TO_NC_OUTPATH="$RUN_CONVERT_GRIB2_TO_NC_OUTPATH" \
     --env CONVERT_GRIB2_TO_NC_THREDDSPATH="$RUN_CONVERT_GRIB2_TO_NC_THREDDSPATH" \
+    --env CONVERT_GRIB2_TO_NC_PROGRESSBAR \
     --user $USER_ID:$GROUP_ID \
     $RUN_CONVERT_GRIB2_TO_NC_IMAGE \
     python $SCRIPT_TO_RUN "$@"