Feature/96 site config

doc/process/openquake_processes.md

@@ -19,5 +19,11 @@
    -  see `run_oq_convert_solutionhazard.py` below
 
 
+**Save an arbritrary file**
+
+```
+python3 runzi/automation/run_save_file_archive.py ~/Downloads/Floor_ADDTOT346ave-base.xml -t BG_Kiran_Floor_ADDTOT346ave-base
+```
+
 
 

docker/runzi-openquake/Dockerfile

@@ -1,9 +1,14 @@
 # Dockerfile
 # Extend base image
 # needs openquake/engine >= 3.13.0
-FROM openquake/engine:nightly
+#nightly
+FROM openquake/engine:latest
 
 USER root
+
+ADD nzshm-oq-engine-main/openquake/hazardlib/scalerel/cscaling.py /opt/openquake/lib/python3.8/site-packages/openquake/hazardlib/scalerel
+
+
 RUN apt-get update
 RUN apt-get install git -y
 RUN apt-get install nano
@@ -31,7 +36,7 @@ RUN git clone https://github.com/GNS-Science/nzshm-runzi.git
 WORKDIR /app/nzshm-runzi
 
 RUN git fetch
-RUN git checkout feature/89-integrate-openquake-and-toshiapi
+RUN git checkout feature/96-site-config
 RUN pip3 install -r requirements.txt
 RUN pip3 install -e .
 

docker/runzi-openquake/README.md

@@ -48,6 +48,7 @@ docker run -u root -it --rm \
 
 ```
 docker run -it --rm -u root \
+--memory=52g --memory-swap=52g --oom-kill-disable \
 --net=host --env-file environ \
 -v $HOME/.aws/credentials:/home/openquake/.aws/credentials:ro \
 -v $(pwd)/../../runzi/cli/config/saved_configs:/app/nzshm-runzi/runzi/cli/config/saved_configs \

runzi/automation/run_oq_hazard.py

@@ -54,29 +54,67 @@ def build_tasks(new_gt_id, args, task_type, model_type):
     headers={"x-api-key":API_KEY}
     toshi_api = ToshiApi(API_URL, None, None, with_schema_validation=True, headers=headers)
 
+    era_measures = ['PGA', 'SA(0.1)', 'SA(0.2)', 'SA(0.3)', 'SA(0.4)', 'SA(0.5)', 'SA(0.7)',
+        'SA(1.0)', 'SA(1.5)', 'SA(2.0)', 'SA(3.0)', 'SA(4.0)', 'SA(5.0)']
+    era_levels = [0.01, 0.02, 0.04, 0.06, 0.08, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9,
+        1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2, 2.4, 2.6, 2.8, 3.0, 3.5, 4, 4.5, 5.0]
+
     args = dict(
         config_archive_ids = [  # a Toshi File containing zipped configuration, ], #LOCAL'RmlsZToxOA=='],
-            "RmlsZToxMDE4MDQ=", #4-sites-many TEST RmlsZToxMDAzNTc=
-            "RmlsZToxMDE4MDY=", #PROD Wgn_005-10-300.ini RmlsZToxMDE4MDM= is BAD , PROD # TEST RmlsZToxMDA1MzA="
-            "RmlsZToxMDE4MDc=", #PROD Wgn_005-10-50.ini
-            "RmlsZToxMDE4MDg=", #PROD Wgn_005-4-300.ini
-            "RmlsZToxMDE4MDk=", #PROD Wgn_005-4-50.ini
+            'RmlsZToxOA=='
+            #"RmlsZToxMDE4MDQ=", #4-sites-many TEST RmlsZToxMDAzNTc=
+            # "RmlsZToxMDE4MDY=", #PROD Wgn_005-10-300.ini RmlsZToxMDE4MDM= is BAD , PROD # TEST RmlsZToxMDA1MzA="
+            # "RmlsZToxMDE4MDc=", #PROD Wgn_005-10-50.ini
+            # "RmlsZToxMDE4MDg=", #PROD Wgn_005-4-300.ini
+            # "RmlsZToxMDE4MDk=", #PROD Wgn_005-4-50.ini
             ],
-        source_combos = [
-            # {'tag':'combined','nrml_ids':{
-            #      'crustal':"SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0Ng==",
-            #      'hik':"SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0OQ==",
-            #      'bg': 'RmlsZToxMDA0ODg='}},
-            #{'tag':'crustal_only','nrml_ids':{
-            #     'crustal':"SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0Ng=="}},
-            #{'tag':'hik_only','nrml_ids':{'hik':"SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0OQ=="}},
-            # {'tag': 'bg_only', 'nrml_ids': {'bg': 'RmlsZToxMDA0ODg='}},
-            {'tag': 'combined', 'nrml_ids': {
-              'crustal': "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0Mw==", #PROD "b_and_n": "{'tag': 'N = 3.5, b=0.913', PROD
-              'hik': "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDMzMQ==",     #PROD "b_and_n": "{'b': 1.009, 'N': 25.6}"
-              'bg': "RmlsZToxMDE4MDI=" #BG_Kiran_fADDTOT346ave_Test4 unscaled BG TEST RmlsZToxMDA1MzU=
-            }}
-        ]
+        # source_combos = [
+        #     # {'tag':'combined','nrml_ids':{
+        #     #   'crustal':"SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0Ng==",
+        #     #   'hik':"SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0OQ==",
+        #     #   'bg': 'RmlsZToxMDA0ODg='}},
+        #     #{'tag':'crustal_only','nrml_ids':{
+        #     #   'crustal':"SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0Ng=="}},
+        #     #{'tag':'hik_only','nrml_ids':{'hik':"SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0OQ=="}},
+        #     {'tag': 'bg_only', 'nrml_ids': {'bg': 'RmlsZToxMDA0ODg='}},
+        #     # {'tag': 'combined', 'nrml_ids': {
+        #     #   'crustal': "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0Mw==", #PROD "b_and_n": "{'tag': 'N = 3.5, b=0.913', PROD
+        #     #   'hik': "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDMzMQ==",     #PROD "b_and_n": "{'b': 1.009, 'N': 25.6}"
+        #     #   'bg': "RmlsZToxMDE4MDI=" #BG_Kiran_fADDTOT346ave_Test4 unscaled BG TEST RmlsZToxMDA1MzU=
+        #     #   #'slab': "ABBBV"
+        #     #   #'puy' : "ABCB"
+        #     # }}
+        #     ],
+        logic_tree_permutations = [
+            {
+                "CR": {
+                    "CR_N7.8_b_1.111_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0NA==",
+                    "CR_N7.8_b_1.111_s2": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0NQ==",
+                    "CR_N3.5_b0.913_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0Ng==",
+                    #"CR_N3.5_b0.913_s2": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0Nw=="
+                    },
+                "HK": {
+                    "HK_N25.6_b0.942_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0OA==",
+                    "HK_N25.6_b1.009_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0OQ==",
+                    #"HK_N25.6_b1.009_s12": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM1MA=="
+                },
+                "BG": {
+                    "bgA": "RmlsZToxMDE4MDI="
+                },
+                "PY": {
+                    "PY_N": "RmlsZToxMDE4MDA="
+                }
+            },
+            #MORE of these ....
+        ],
+        intensity_specs = [
+            {"tag": "lite", "measures": ['PGA', 'SA(0.5)', 'SA(1.0)'], "levels": 'logscale(0.005, 4.00, 30)' },
+            {"tag": "fixed", "measures": era_measures, "levels": era_levels},
+            {"tag": "max10-300", "measures": era_measures, "levels": 'logscale(0.005, 10.00, 300)'}],
+        vs30s = [ 455, ],
+        location_codes = ['WLG', 'NZ4', 'NZ34'],
+        disagg_confs = [{'enabled': False, 'config': {}},
+            {'enabled': True, 'config': {}}]
     )
 
     args_list = []

runzi/automation/run_scale_solution.py

@@ -107,10 +107,10 @@ def build_subset_tasks(general_task_id, source_solutions, args):
     log = logging.getLogger(__name__)
 
     # If you wish to oversolution_ide something in the main config, do so here ..
-    WORKER_POOL_SIZE = 1
+    WORKER_POOL_SIZE = 2
     USE_API = True
     #If using API give this task a descriptive setting...
-    TASK_TITLE = "Hikurangi Scaled Inverions, scale = 0.587"
+    TASK_TITLE = "Hikurangi LTB004. Scaled 0.67, 1.41"
     TASK_DESCRIPTION = """
 
     """
@@ -121,20 +121,14 @@ def build_subset_tasks(general_task_id, source_solutions, args):
     toshi_api = ToshiApi(API_URL, S3_URL, None, with_schema_validation=True, headers=headers)
 
     source_solution_ids = [
-        #"SW52ZXJzaW9uU29sdXRpb246MTAwMDQ5",
-        #"SW52ZXJzaW9uU29sdXRpb246MTAwMDUw",
-        #"SW52ZXJzaW9uU29sdXRpb246MTAwMDU2"
-        #"R2VuZXJhbFRhc2s6MTAwMTA2",
-        "R2VuZXJhbFRhc2s6MTAwMDEz",
-        #"SW52ZXJzaW9uU29sdXRpb246MTAwMjQ4",
-        #"SW52ZXJzaW9uU29sdXRpb246MTAwMjUw",
-        #"SW52ZXJzaW9uU29sdXRpb246MTAwMjUy"
+        "R2VuZXJhbFRhc2s6MTAwMjQx",
     ]
-    #scales = [0.5,2.0]
-    #scales = [0.49, 1.63]
-    scales = [0.587, 1.419, 2, 3]
-    # scales = [0.587]
-    model_type = 'crustal'
+
+    # scales = [0.49, 1.63]
+    # scales = [0.61, 1.34]
+    scales = [0.67, 1.41]
+
+    model_type = 'subduction'
 
 
     file_generators = []

runzi/cli/config/saved_configs/HAZARD/source_logic_tree.json

@@ -0,0 +1,36 @@
+{
+"task_args": {
+
+    "logic_trees": [
+        {"tag": "test logic tree 1","branches": [
+            {"name":"branch A",
+                "weight":0.5,
+                "source_ids":{
+                    "c_N7.8_b_1.111_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0MA==",
+                    "h_N25.6_b0.942_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDMzMg==",
+                    "bg": "RmlsZToxMDE4MDI="
+                    }
+                },
+            {"name":"branch B",
+                "weight":0.5,
+                "source_ids":{
+                    "c_N3.5_b0.913_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0Mw==",
+                    "h_N25.6_b1.009_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDMzMQ==",
+                    "bg": "RmlsZToxMDE4MDI="
+                    }
+                }
+            ]
+        }
+    ]
+}
+}
+
+
+
+
+
+
+
+
+
+

runzi/configuration/oq_hazard.py

@@ -35,8 +35,22 @@ def build_hazard_tasks(general_task_id: str, subtask_type: SubtaskType, model_ty
     factory_task = runzi.execute.oq_hazard_task
     task_factory = factory_class(WORK_PATH, factory_task, task_config_path=WORK_PATH)
 
-    for config_archive_id in subtask_arguments["config_archive_ids"]:
-        for sources in subtask_arguments['source_combos']:
+
+    for (config_archive_id,
+        #logic_tree_permutation,
+        intensity_spec,
+        vs30,
+        location_code,
+        disagg_conf
+        )\
+        in itertools.product(
+            subtask_arguments["config_archive_ids"],
+            #subtask_arguments['logic_tree_permutations'],
+            subtask_arguments['intensity_specs'],
+            subtask_arguments['vs30s'],
+            subtask_arguments['location_codes'],
+            subtask_arguments['disagg_confs']
+            ):
 
             task_count +=1
             task_arguments = dict(
@@ -45,7 +59,11 @@ def build_hazard_tasks(general_task_id: str, subtask_type: SubtaskType, model_ty
                 config_archive_id = config_archive_id, #File archive object
                 #upstream_general_task=source_gt_id,
                 model_type = model_type.name,
-                sources = sources
+                logic_tree_permutations = subtask_arguments['logic_tree_permutations'],
+                intensity_spec = intensity_spec,
+                vs30 = vs30,
+                location_code = location_code,
+                disagg_conf = disagg_conf,
                 )
 
             print('')