diff --git a/doc/process/openquake_processes.md b/doc/process/openquake_processes.md index d436c855..e3d055e4 100644 --- a/doc/process/openquake_processes.md +++ b/doc/process/openquake_processes.md @@ -19,5 +19,11 @@ - see `run_oq_convert_solutionhazard.py` below +**Save an arbritrary file** + +``` +python3 runzi/automation/run_save_file_archive.py ~/Downloads/Floor_ADDTOT346ave-base.xml -t BG_Kiran_Floor_ADDTOT346ave-base +``` + diff --git a/docker/runzi-openquake/Dockerfile b/docker/runzi-openquake/Dockerfile index 1d8fff5e..8b4278f0 100644 --- a/docker/runzi-openquake/Dockerfile +++ b/docker/runzi-openquake/Dockerfile @@ -1,9 +1,14 @@ # Dockerfile # Extend base image # needs openquake/engine >= 3.13.0 -FROM openquake/engine:nightly +#nightly +FROM openquake/engine:latest USER root + +ADD nzshm-oq-engine-main/openquake/hazardlib/scalerel/cscaling.py /opt/openquake/lib/python3.8/site-packages/openquake/hazardlib/scalerel + + RUN apt-get update RUN apt-get install git -y RUN apt-get install nano @@ -31,7 +36,7 @@ RUN git clone https://github.com/GNS-Science/nzshm-runzi.git WORKDIR /app/nzshm-runzi RUN git fetch -RUN git checkout feature/89-integrate-openquake-and-toshiapi +RUN git checkout feature/96-site-config RUN pip3 install -r requirements.txt RUN pip3 install -e . diff --git a/docker/runzi-openquake/README.md b/docker/runzi-openquake/README.md index 032d3663..c8193a22 100644 --- a/docker/runzi-openquake/README.md +++ b/docker/runzi-openquake/README.md @@ -48,6 +48,7 @@ docker run -u root -it --rm \ ``` docker run -it --rm -u root \ +--memory=52g --memory-swap=52g --oom-kill-disable \ --net=host --env-file environ \ -v $HOME/.aws/credentials:/home/openquake/.aws/credentials:ro \ -v $(pwd)/../../runzi/cli/config/saved_configs:/app/nzshm-runzi/runzi/cli/config/saved_configs \ diff --git a/runzi/automation/run_oq_hazard.py b/runzi/automation/run_oq_hazard.py index 9fb3e6ef..68519a55 100644 --- a/runzi/automation/run_oq_hazard.py +++ b/runzi/automation/run_oq_hazard.py @@ -54,29 +54,67 @@ def build_tasks(new_gt_id, args, task_type, model_type): headers={"x-api-key":API_KEY} toshi_api = ToshiApi(API_URL, None, None, with_schema_validation=True, headers=headers) + era_measures = ['PGA', 'SA(0.1)', 'SA(0.2)', 'SA(0.3)', 'SA(0.4)', 'SA(0.5)', 'SA(0.7)', + 'SA(1.0)', 'SA(1.5)', 'SA(2.0)', 'SA(3.0)', 'SA(4.0)', 'SA(5.0)'] + era_levels = [0.01, 0.02, 0.04, 0.06, 0.08, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, + 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2, 2.4, 2.6, 2.8, 3.0, 3.5, 4, 4.5, 5.0] + args = dict( config_archive_ids = [ # a Toshi File containing zipped configuration, ], #LOCAL'RmlsZToxOA=='], - "RmlsZToxMDE4MDQ=", #4-sites-many TEST RmlsZToxMDAzNTc= - "RmlsZToxMDE4MDY=", #PROD Wgn_005-10-300.ini RmlsZToxMDE4MDM= is BAD , PROD # TEST RmlsZToxMDA1MzA=" - "RmlsZToxMDE4MDc=", #PROD Wgn_005-10-50.ini - "RmlsZToxMDE4MDg=", #PROD Wgn_005-4-300.ini - "RmlsZToxMDE4MDk=", #PROD Wgn_005-4-50.ini + 'RmlsZToxOA==' + #"RmlsZToxMDE4MDQ=", #4-sites-many TEST RmlsZToxMDAzNTc= + # "RmlsZToxMDE4MDY=", #PROD Wgn_005-10-300.ini RmlsZToxMDE4MDM= is BAD , PROD # TEST RmlsZToxMDA1MzA=" + # "RmlsZToxMDE4MDc=", #PROD Wgn_005-10-50.ini + # "RmlsZToxMDE4MDg=", #PROD Wgn_005-4-300.ini + # "RmlsZToxMDE4MDk=", #PROD Wgn_005-4-50.ini ], - source_combos = [ - # {'tag':'combined','nrml_ids':{ - # 'crustal':"SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0Ng==", - # 'hik':"SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0OQ==", - # 'bg': 'RmlsZToxMDA0ODg='}}, - #{'tag':'crustal_only','nrml_ids':{ - # 'crustal':"SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0Ng=="}}, - #{'tag':'hik_only','nrml_ids':{'hik':"SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0OQ=="}}, - # {'tag': 'bg_only', 'nrml_ids': {'bg': 'RmlsZToxMDA0ODg='}}, - {'tag': 'combined', 'nrml_ids': { - 'crustal': "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0Mw==", #PROD "b_and_n": "{'tag': 'N = 3.5, b=0.913', PROD - 'hik': "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDMzMQ==", #PROD "b_and_n": "{'b': 1.009, 'N': 25.6}" - 'bg': "RmlsZToxMDE4MDI=" #BG_Kiran_fADDTOT346ave_Test4 unscaled BG TEST RmlsZToxMDA1MzU= - }} - ] + # source_combos = [ + # # {'tag':'combined','nrml_ids':{ + # # 'crustal':"SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0Ng==", + # # 'hik':"SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0OQ==", + # # 'bg': 'RmlsZToxMDA0ODg='}}, + # #{'tag':'crustal_only','nrml_ids':{ + # # 'crustal':"SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0Ng=="}}, + # #{'tag':'hik_only','nrml_ids':{'hik':"SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0OQ=="}}, + # {'tag': 'bg_only', 'nrml_ids': {'bg': 'RmlsZToxMDA0ODg='}}, + # # {'tag': 'combined', 'nrml_ids': { + # # 'crustal': "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0Mw==", #PROD "b_and_n": "{'tag': 'N = 3.5, b=0.913', PROD + # # 'hik': "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDMzMQ==", #PROD "b_and_n": "{'b': 1.009, 'N': 25.6}" + # # 'bg': "RmlsZToxMDE4MDI=" #BG_Kiran_fADDTOT346ave_Test4 unscaled BG TEST RmlsZToxMDA1MzU= + # # #'slab': "ABBBV" + # # #'puy' : "ABCB" + # # }} + # ], + logic_tree_permutations = [ + { + "CR": { + "CR_N7.8_b_1.111_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0NA==", + "CR_N7.8_b_1.111_s2": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0NQ==", + "CR_N3.5_b0.913_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0Ng==", + #"CR_N3.5_b0.913_s2": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0Nw==" + }, + "HK": { + "HK_N25.6_b0.942_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0OA==", + "HK_N25.6_b1.009_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0OQ==", + #"HK_N25.6_b1.009_s12": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM1MA==" + }, + "BG": { + "bgA": "RmlsZToxMDE4MDI=" + }, + "PY": { + "PY_N": "RmlsZToxMDE4MDA=" + } + }, + #MORE of these .... + ], + intensity_specs = [ + {"tag": "lite", "measures": ['PGA', 'SA(0.5)', 'SA(1.0)'], "levels": 'logscale(0.005, 4.00, 30)' }, + {"tag": "fixed", "measures": era_measures, "levels": era_levels}, + {"tag": "max10-300", "measures": era_measures, "levels": 'logscale(0.005, 10.00, 300)'}], + vs30s = [ 455, ], + location_codes = ['WLG', 'NZ4', 'NZ34'], + disagg_confs = [{'enabled': False, 'config': {}}, + {'enabled': True, 'config': {}}] ) args_list = [] diff --git a/runzi/automation/run_scale_solution.py b/runzi/automation/run_scale_solution.py index bb1cf021..b5c315ac 100644 --- a/runzi/automation/run_scale_solution.py +++ b/runzi/automation/run_scale_solution.py @@ -107,10 +107,10 @@ def build_subset_tasks(general_task_id, source_solutions, args): log = logging.getLogger(__name__) # If you wish to oversolution_ide something in the main config, do so here .. - WORKER_POOL_SIZE = 1 + WORKER_POOL_SIZE = 2 USE_API = True #If using API give this task a descriptive setting... - TASK_TITLE = "Hikurangi Scaled Inverions, scale = 0.587" + TASK_TITLE = "Hikurangi LTB004. Scaled 0.67, 1.41" TASK_DESCRIPTION = """ """ @@ -121,20 +121,14 @@ def build_subset_tasks(general_task_id, source_solutions, args): toshi_api = ToshiApi(API_URL, S3_URL, None, with_schema_validation=True, headers=headers) source_solution_ids = [ - #"SW52ZXJzaW9uU29sdXRpb246MTAwMDQ5", - #"SW52ZXJzaW9uU29sdXRpb246MTAwMDUw", - #"SW52ZXJzaW9uU29sdXRpb246MTAwMDU2" - #"R2VuZXJhbFRhc2s6MTAwMTA2", - "R2VuZXJhbFRhc2s6MTAwMDEz", - #"SW52ZXJzaW9uU29sdXRpb246MTAwMjQ4", - #"SW52ZXJzaW9uU29sdXRpb246MTAwMjUw", - #"SW52ZXJzaW9uU29sdXRpb246MTAwMjUy" + "R2VuZXJhbFRhc2s6MTAwMjQx", ] - #scales = [0.5,2.0] - #scales = [0.49, 1.63] - scales = [0.587, 1.419, 2, 3] - # scales = [0.587] - model_type = 'crustal' + + # scales = [0.49, 1.63] + # scales = [0.61, 1.34] + scales = [0.67, 1.41] + + model_type = 'subduction' file_generators = [] diff --git a/runzi/cli/config/saved_configs/HAZARD/source_logic_tree.json b/runzi/cli/config/saved_configs/HAZARD/source_logic_tree.json new file mode 100644 index 00000000..aec5427b --- /dev/null +++ b/runzi/cli/config/saved_configs/HAZARD/source_logic_tree.json @@ -0,0 +1,36 @@ +{ +"task_args": { + + "logic_trees": [ + {"tag": "test logic tree 1","branches": [ + {"name":"branch A", + "weight":0.5, + "source_ids":{ + "c_N7.8_b_1.111_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0MA==", + "h_N25.6_b0.942_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDMzMg==", + "bg": "RmlsZToxMDE4MDI=" + } + }, + {"name":"branch B", + "weight":0.5, + "source_ids":{ + "c_N3.5_b0.913_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0Mw==", + "h_N25.6_b1.009_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDMzMQ==", + "bg": "RmlsZToxMDE4MDI=" + } + } + ] + } + ] +} +} + + + + + + + + + + diff --git a/runzi/configuration/oq_hazard.py b/runzi/configuration/oq_hazard.py index e221ff3d..b5c96527 100644 --- a/runzi/configuration/oq_hazard.py +++ b/runzi/configuration/oq_hazard.py @@ -35,8 +35,22 @@ def build_hazard_tasks(general_task_id: str, subtask_type: SubtaskType, model_ty factory_task = runzi.execute.oq_hazard_task task_factory = factory_class(WORK_PATH, factory_task, task_config_path=WORK_PATH) - for config_archive_id in subtask_arguments["config_archive_ids"]: - for sources in subtask_arguments['source_combos']: + + for (config_archive_id, + #logic_tree_permutation, + intensity_spec, + vs30, + location_code, + disagg_conf + )\ + in itertools.product( + subtask_arguments["config_archive_ids"], + #subtask_arguments['logic_tree_permutations'], + subtask_arguments['intensity_specs'], + subtask_arguments['vs30s'], + subtask_arguments['location_codes'], + subtask_arguments['disagg_confs'] + ): task_count +=1 task_arguments = dict( @@ -45,7 +59,11 @@ def build_hazard_tasks(general_task_id: str, subtask_type: SubtaskType, model_ty config_archive_id = config_archive_id, #File archive object #upstream_general_task=source_gt_id, model_type = model_type.name, - sources = sources + logic_tree_permutations = subtask_arguments['logic_tree_permutations'], + intensity_spec = intensity_spec, + vs30 = vs30, + location_code = location_code, + disagg_conf = disagg_conf, ) print('') diff --git a/runzi/execute/oq_hazard_task.py b/runzi/execute/oq_hazard_task.py index 24047eec..1119489e 100644 --- a/runzi/execute/oq_hazard_task.py +++ b/runzi/execute/oq_hazard_task.py @@ -4,7 +4,7 @@ import os import io -import zipfile +# import zipfile import subprocess import requests import platform @@ -19,8 +19,10 @@ from runzi.automation.scaling.toshi_api import ToshiApi, SubtaskType from nshm_toshi_client.task_relation import TaskRelation from runzi.automation.scaling.local_config import (API_KEY, API_URL, S3_URL, WORK_PATH, SPOOF_HAZARD) -from runzi.automation.scaling.file_utils import download_files, get_output_file_ids, get_output_file_id + from runzi.util import archive +from runzi.execute.util import ( OpenquakeConfig, build_sources_xml, + get_logic_tree_file_ids, get_logic_tree_branches ) logging.basicConfig(level=logging.INFO) @@ -34,34 +36,10 @@ log = logging.getLogger(__name__) -def build_sources_xml(sources_list): - template = """ - - - - - - - - 1.0 - - - - -""" - insert = " " * 16 #coz some indentation is nice - for filepath in sources_list: - insert += f"{filepath}\n" - - template = template.replace('', insert) - return template - def write_sources(xml_str, filepath): with open(filepath, 'w') as mf: mf.write(xml_str) - def explode_config_template(config_info, working_path: str): config_folder = Path(working_path, "config") @@ -155,23 +133,28 @@ def __init__(self, job_args): self._toshi_api = ToshiApi(API_URL, S3_URL, None, with_schema_validation=True, headers=headers) self._task_relation_api = TaskRelation(API_URL, None, with_schema_validation=True, headers=headers) - def unpack_sources(self, ta, source_path): - """download and extract the sources""" + # def unpack_sources(self, ta, source_path): + # """download and extract the sources""" - namelist = [] - for src_name, nrml_id in ta['sources']['nrml_ids'].items(): + # sources = dict() + # # for src_name, nrml_id in ta['sources']['nrml_ids'].items(): - log.info(f"get src : {src_name} {nrml_id}") + # for branch in ta['logic_tree']['branches']: + # for src_name, nrml_id in branch['sources']['source_ids'].items(): + # if nrml_id in sources.keys(): + # continue - gen = get_output_file_id(self._toshi_api, nrml_id) + # log.info(f"get src : {src_name} {nrml_id}") - source_nrml = download_files(self._toshi_api, gen, str(WORK_PATH), overwrite=False) - log.info(f"source_nrml: {source_nrml}") + # gen = get_output_file_id(self._toshi_api, nrml_id) - with zipfile.ZipFile(source_nrml[nrml_id]['filepath'], 'r') as zip_ref: - zip_ref.extractall(source_path) - namelist += zip_ref.namelist() - return namelist + # source_nrml = download_files(self._toshi_api, gen, str(WORK_PATH), overwrite=False) + # log.info(f"source_nrml: {source_nrml}") + + # with zipfile.ZipFile(source_nrml[nrml_id]['filepath'], 'r') as zip_ref: + # zip_ref.extractall(source_path) + # sources[nrml_id] = {'source_name': src_name, 'sources' : zip_ref.namelist()} + # return sources def run(self, task_arguments, job_arguments): @@ -230,22 +213,48 @@ def run(self, task_arguments, job_arguments): config_folder = explode_config_template(config_template_info, work_folder) # sources are the InversionSolutionNRML XML file(s) to include in the sources list + id_list = get_logic_tree_file_ids(task_args['logic_tree_permutations']) + sources_folder = Path(config_folder, 'sources') sources_list = self.unpack_sources(ta, sources_folder) print(f'sources_list: {sources_list}') # now the customised source_models.xml file must be written into the local configuration - src_xml = build_sources_xml(sources_list) + #src_xml = build_sources_xml(sources_list) + #print("ID lists:", list(id_list)) + #TODO build a map whlie downloading the files + + ltbs = [ltb for ltb in get_logic_tree_branches(task_args['logic_tree_permutations'])] + + print("LTB:", len(ltbs), ltbs[0]) + + nrml = build_sources_xml(ltbs, source_file_mapping) + + print(src_xml) write_sources(src_xml, Path(sources_folder, 'source_model.xml')) - # Do the heavy lifting in openquake, passing the config + #prepare the config for itm in config_template_info['meta']: if itm['k'] == "config_filename": config_filename = itm['v'] break config_file = Path(config_folder, config_filename) + def modify_config(config_file, task_arguments): + ta = task_arguments + config = OpenquakeConfig(open(config_file))\ + .set_sites(ta['location_code'])\ + .set_disaggregation(enable = ta['disagg_conf']['enabled'], + settings = ta['disagg_conf']['config'])\ + .set_iml(ta['intensity_spec']['measures'], + ta['intensity_spec']['levels'])\ + .set_vs30(ta['vs30']) + config.write(open(config_file, 'w')) + return + + modify_config(config_file, task_arguments) + logfile = Path(work_folder, f'openquake.log') oq_result = execute_openquake(config_file, logfile, task_id) @@ -262,6 +271,8 @@ def run(self, task_arguments, job_arguments): solution_id = self._toshi_api.openquake_hazard_solution.create_solution( config_id, csv_archive_id, hdf5_archive_id, produced_by=task_id) + # TODO bundle and save the as-run configuration file(s) + # update the OpenquakeHazardTask self._toshi_api.openquake_hazard_task.complete_task( dict(task_id =task_id, diff --git a/runzi/execute/util/__init__.py b/runzi/execute/util/__init__.py new file mode 100644 index 00000000..83a207cd --- /dev/null +++ b/runzi/execute/util/__init__.py @@ -0,0 +1,2 @@ +from .oq_hazard_config import OpenquakeConfig +from .oq_build_sources import get_logic_tree_file_ids, get_logic_tree_branches, build_sources_xml \ No newline at end of file diff --git a/runzi/execute/util/oq_build_sources.py b/runzi/execute/util/oq_build_sources.py new file mode 100644 index 00000000..bc34205e --- /dev/null +++ b/runzi/execute/util/oq_build_sources.py @@ -0,0 +1,652 @@ +#!oq_build_sources.py + +import itertools +import logging +from pathlib import Path + +import zipfile +from lxml import etree +from lxml.builder import ElementMaker # lxml only ! +from runzi.automation.scaling.toshi_api import ToshiApi +from runzi.automation.scaling.file_utils import download_files, get_output_file_ids, get_output_file_id + +log = logging.getLogger(__name__) + + +# def get_ltb(group): +# """old dict-kets style config""" +# for gk, gv in group.items(): +# yield list(gv.items()) + +def get_logic_tree_file_ids(ltb_groups): + ids = set() + for group in ltb_groups: #List + for sources in get_ltb(group): + for source in sources: + ids.add( source ) + return list(ids) + +def get_ltb(group): + """NEW object-syle config""" + for obj in group['permute']: + yield [(source['tag'], source['toshi_id']) for source in obj['members']] + +def get_logic_tree_branches(ltb_groups): + for group in ltb_groups: #List + for ltb in itertools.product(*get_ltb(group)): + yield ltb + +class SourceModelLoader(): + + def __init__(self, api_url, api_key, s3_url): + headers={"x-api-key":api_key} + self._toshi_api = ToshiApi(api_url, s3_url, None, with_schema_validation=True, headers=headers) + + def unpack_sources(self, logic_tree_branch_permutations, source_path): + """download and extract the sources""" + + sources = dict() + + # ltbs = [x for x in get_logic_tree_file_ids(logic_tree_branch_permutations)] + + # print(ltbs) + # print(len(ltbs)) + print(logic_tree_branch_permutations) + + for src_name, nrml_id in get_logic_tree_file_ids(logic_tree_branch_permutations): + if nrml_id in sources.keys(): + continue + + log.info(f"get src : {src_name} {nrml_id}") + + gen = get_output_file_id(self._toshi_api, nrml_id) + + source_nrml = download_files(self._toshi_api, gen, str(WORK_PATH), overwrite=False) + log.info(f"source_nrml: {source_nrml}") + + with zipfile.ZipFile(source_nrml[nrml_id]['filepath'], 'r') as zip_ref: + zip_ref.extractall(source_path) + sources[nrml_id] = {'source_name': src_name, 'sources' : zip_ref.namelist()} + + return sources + + +def build_sources_xml(logic_tree_branches, source_file_mapping): + + weight = 1/len(logic_tree_branches) + + E = ElementMaker(namespace="http://openquake.org/xmlns/nrml/0.5", + nsmap={"gml" : "http://www.opengis.net/gml", None:"http://openquake.org/xmlns/nrml/0.5"}) + NRML = E.nrml + LT = E.logicTree + LTBS = E.logicTreeBranchSet + LTBL = E.logicTreeBranchingLevel + LTB = E.logicTreeBranch + UM = E.uncertaintyModel + UW = E.uncertaintyWeight + + ltbs = LTBS(uncertaintyType="sourceModel", branchSetID="BS-NONCE1") + + for branch in logic_tree_branches: + files = "" + branch_name = "|".join([x[0] for x in branch]) + for source_tuple in branch: + #print(source_tuple) + name, src_id = source_tuple + files += "\t".join(source_file_mapping[src_id]['sources']) + "\t" + ltb = LTB( UM(files), UW(str(weight)), branchID=branch_name) + ltbs.append(ltb) + + nrml = NRML( LT( LTBL( ltbs, branchingLevelID="1" ), logicTreeID = "Combined")) + return etree.tostring(nrml, pretty_print=True).decode() + + + +if __name__ == "__main__": + from runzi.automation.scaling.local_config import (API_KEY, API_URL, S3_URL, WORK_PATH, SPOOF_HAZARD) + + # permutations = [ + # { + # "CR": { + # "CR_N2.3_b0.807_C4.2_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4MA==", + # "CR_N8.0_b1.115_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Nw==", + # "CR_N2.3_b0.807_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3OQ==", + # "CR_N3.7_b0.929_C4.2_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4MQ==", + # "CR_N3.7_b0.929_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4Mw==", + # "CR_N8.0_b1.115_C4.2_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4NQ==" + # }, + # "HK": { + # "HTC_b1.112_N22.6_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2NA==", + # "HTC_b0.97_N11.6_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2Ng==", + # "HTC_b1.3_N49.4_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3MQ==", + # "HTL_b1.112_N22.6_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3MA==", + # "HTL_b0.97_N11.6_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Mw==", + # "HTL_b1.3_N49.4_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Mg==" + # }, + # "BG": { + # "BG_floor_addtot346ave": "RmlsZToxMDIyMzA=" + # }, + # "PY": { + # "PY_b0.75_N3.4_C3.9_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Ng==" + # } + # }, + # { + # "CR": { + # "CR_N8.0_b1.115_C4.3_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3OA==", + # "CR_N2.3_b0.807_C4.3_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4Mg==", + # "CR_N3.7_b0.929_C4.3_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4NA==" + # }, + # "HK": { + # "HTC_b1.112_N22.6_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2Nw==", + # "HTC_b1.3_N49.4_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2NQ==", + # "HTC_b0.97_N11.6_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2OA==", + # "HTL_b1.112_N22.6_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2OQ==", + # "HTL_b1.3_N49.4_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3NQ==", + # "HTL_b0.97_N11.6_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3NA==" + # }, + # "BG": { + # "BG_floor_addtot346ave": "RmlsZToxMDIyMzA=" + # }, + # "PY": { + # "PY_b0.75_N3.4_C3.9_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Ng==" + # } + # } + # ] + + # #SLT001 + # permutations = [ + # { + # "tag": "core model", "weight": 1.0, + # "permute" : [ + # { + # "group": "HIK", + # "members" : [ + # {"tag": "b0.97_N11.6_C4.0_s1", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2Ng=="} + # ] + # } + # ] + # } + # ] + + + # #SLT002 + # permutations = [ + # { + # "tag": "core model", "weight": 0.5, + # "permute" : [ + # { "group": "HIK", + # "members" : [ + # {"tag": "b0.97_N11.6_C4.0_s1", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2Ng=="} + # ] + # } + # ] + # }, + + # { + # "tag": "C = 4.1", "weight": 0.5, + # "permute" : [ + # { "group": "HIK", + # "members" : [ + # {"tag": "b0.97_N11.6_C4.1_s1", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2OA=="} + # ] + # } + # ] + # } + + # ] + + + # #SLT003 + # permutations = [ + + # { + # "tag": "core model", "weight": 0.166, + # "permute" : [ + # { "group": "HIK", + # "members" : [ + # {"tag": "b0.97_N11.6_C4.0_s1", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2Ng=="} + # ] + # } + # ] + # }, + + # { + # "tag": "core scaled down", "weight": 0.166, + # "permute" : [ + # { "group": "HIK", + # "members" : [ + # {"tag": "b0.97_N11.6_C4.0_s0.54", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4Nw=="} + # ] + # } + # ] + # }, + + # { + # "tag": "core scaled up", "weight": 0.166, + # "permute" : [ + + # { "group": "HIK", + # "members" : [ + # {"tag": "b0.97_N11.6_C4.0_s1.43", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5OA=="} + # ] + # } + # ] + # }, + + # { + # "tag": "b = 1.112", "weight": 0.166, + # "permute" : [ + # { "group": "HIK", + # "members" : [ + # {"tag": "b1.112_N22.6_C4_s1", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2NA=="} + # ] + # } + # ] + # }, + + # { + # "tag": "b = 1.112 scaled down", "weight": 0.166, + # "permute" : [ + # { "group": "HIK", + # "members" : [ + # {"tag": "b1.112_N22.6_C4_s0.54", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4Ng=="} + # ] + # } + # ] + # }, + + # { + # "tag": "b = 1.112 scaled up", "weight": 0.17, + # "permute" : [ + + # { "group": "HIK", + # "members" : [ + # {"tag": "b1.112_N22.6_C4_s1.43", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5OQ=="} + # ] + # } + # ] + # } + # ] + + # #SLT004b + # permutations = [ + + # { + # "tag": "core model", "weight": 0.0833, + # "permute" : [ + # { "group": "HIK", + # "members" : [ + # {"tag": "b0.97_N11.6_C4.0_s1", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2Ng=="} + # ] + # } + # ] + # }, + + # { + # "tag": "core scaled down", "weight": 0.0833, + # "permute" : [ + # { "group": "HIK", + # "members" : [ + # {"tag": "b0.97_N11.6_C4.0_s0.54", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4Nw=="} + # ] + # } + # ] + # }, + + # { + # "tag": "core scaled up", "weight": 0.0833, + # "permute" : [ + + # { "group": "HIK", + # "members" : [ + # {"tag": "b0.97_N11.6_C4.0_s1.43", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5OA=="} + # ] + # } + # ] + # }, + + # { + # "tag": "b = 1.112", "weight": 0.0833, + # "permute" : [ + # { "group": "HIK", + # "members" : [ + # {"tag": "b1.112_N22.6_C4_s1", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2NA=="} + # ] + # } + # ] + # }, + + # { + # "tag": "b = 1.112 scaled down", "weight": 0.0833, + # "permute" : [ + # { "group": "HIK", + # "members" : [ + # {"tag": "b1.112_N22.6_C4_s0.54", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4Ng=="} + # ] + # } + # ] + # }, + + # { + # "tag": "b = 1.112 scaled up", "weight": 0.0833, + # "permute" : [ + + # { "group": "HIK", + # "members" : [ + # {"tag": "b1.112_N22.6_C4_s1.43", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5OQ=="} + # ] + # } + # ] + # }, + + + # { + # "tag": "core model, C = 4.1", "weight": 0.0833, + # "permute" : [ + # { "group": "HIK", + # "members" : [ + # {"tag": "b0.97_N11.6_C4.1_s1", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2OA=="} + # ] + # } + # ] + # }, + + # { + # "tag": "core scaled down, C = 4.1", "weight": 0.0833, + # "permute" : [ + # { "group": "HIK", + # "members" : [ + # {"tag": "b0.97_N11.6_C4.1_s0.54", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4OQ=="} + # ] + # } + # ] + # }, + + # { + # "tag": "core scaled up, C = 4.1", "weight": 0.0833, + # "permute" : [ + + # { "group": "HIK", + # "members" : [ + # {"tag": "b0.97_N11.6_C4.1_s1.43", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwMA=="} + # ] + # } + # ] + # }, + + # { + # "tag": "b = 1.112, C = 4.1", "weight": 0.0833, + # "permute" : [ + # { "group": "HIK", + # "members" : [ + # {"tag": "b1.112_N22.6_C4.1_s1", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2Nw=="} + # ] + # } + # ] + # }, + + # { + # "tag": "b = 1.112 scaled down, C = 4.1", "weight": 0.0833, + # "permute" : [ + # { "group": "HIK", + # "members" : [ + # {"tag": "b1.112_N22.6_C4.1_s0.54", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5MA=="} + # ] + # } + # ] + # }, + + # { + # "tag": "b = 1.112 scaled up, C = 4.1", "weight": 0.0837, + # "permute" : [ + + # { "group": "HIK", + # "members" : [ + # {"tag": "b1.112_N22.6_C4.1_s1.43", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwMw=="} + # ] + # } + # ] + # } + + # ] + + + #BIG + # permutations = [ + # { + # "CR": { + # "CR_N8.0_b1.115_C4.3_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3OA==", + # "CR_N2.3_b0.807_C4.2_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4MA==", + # "CR_N8.0_b1.115_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Nw==", + # "CR_N2.3_b0.807_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3OQ==", + # "CR_N3.7_b0.929_C4.2_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4MQ==", + # "CR_N3.7_b0.929_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4Mw==", + # "CR_N2.3_b0.807_C4.3_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4Mg==", + # "CR_N3.7_b0.929_C4.3_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4NA==", + # "CR_N8.0_b1.115_C4.2_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4NQ==" + # }, + # "HK": { + # "HTC_b1.112_N22.6_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2NA==", + # "HTC_b1.112_N22.6_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2Nw==", + # "HTC_b1.3_N49.4_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2NQ==", + # "HTC_b0.97_N11.6_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2Ng==", + # "HTC_b0.97_N11.6_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2OA==", + # "HTC_b1.3_N49.4_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3MQ==", + # "HTL_b1.112_N22.6_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2OQ==", + # "HTL_b1.112_N22.6_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3MA==", + # "HTL_b0.97_N11.6_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Mw==", + # "HTL_b1.3_N49.4_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3NQ==", + # "HTL_b1.3_N49.4_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Mg==", + # "HTL_b0.97_N11.6_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3NA==" + # }, + # "BG": { + # "floor_addtot346ave": "RmlsZToxMDIyMzA=" + # }, + # "PY": { + # "P_b0.75_N3.4_C3.9_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Ng==" + # } + # }, + # { + # "CR": { + # "CR_N8.0_b1.115_C4.3_s0.51": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxMg==", + # "CR_N8.0_b1.115_C4.1_s0.51": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxNQ==", + # "CR_N2.3_b0.807_C4.2_s0.51": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxNg==", + # "CR_N2.3_b0.807_C4.1_s0.51": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxMw==", + # "CR_N3.7_b0.929_C4.1_s0.51": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxNA==", + # "CR_N3.7_b0.929_C4.2_s0.51": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxNw==", + # "CR_N2.3_b0.807_C4.3_s0.51": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxOQ==", + # "CR_N8.0_b1.115_C4.2_s0.51": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyMA==", + # "CR_N3.7_b0.929_C4.3_s0.51": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyMw==" + # }, + # "HK": { + # "HTC_b1.112_N22.6_C4_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4Ng==", + # "HTC_b1.3_N49.4_C4.1_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4OA==", + # "HTC_b1.112_N22.6_C4.1_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5MA==", + # "HTC_b0.97_N11.6_C4_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4Nw==", + # "HTC_b0.97_N11.6_C4.1_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4OQ==", + # "HTL_b1.112_N22.6_C4.1_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5MQ==", + # "HTC_b1.3_N49.4_C4_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5Mg==", + # "HTL_b1.112_N22.6_C4_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5NQ==", + # "HTL_b0.97_N11.6_C4_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5Ng==", + # "HTL_b1.3_N49.4_C4_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5Mw==", + # "HTL_b1.3_N49.4_C4.1_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5NA==", + # "HTL_b0.97_N11.6_C4.1_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5Nw==" + # }, + # "BG": { + # "floor_addtot346ave": "RmlsZToxMDIyMzA=" + # }, + # "PY": { + # "P_b0.75_N3.4_C3.9_s0.61": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxMA==" + # } + # }, + # { + # "CR": { + # "CR_N2.3_b0.807_C4.2_s1.62": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxOA==", + # "CR_N8.0_b1.115_C4.3_s1.62": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyMQ==", + # "CR_N8.0_b1.115_C4.1_s1.62": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyMg==", + # "CR_N3.7_b0.929_C4.2_s1.62": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyNA==", + # "CR_N2.3_b0.807_C4.1_s1.62": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyNg==", + # "CR_N3.7_b0.929_C4.1_s1.62": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyOA==", + # "CR_N3.7_b0.929_C4.3_s1.62": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyNQ==", + # "CR_N2.3_b0.807_C4.3_s1.62": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyNw==", + # "CR_N8.0_b1.115_C4.2_s1.62": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyOQ==" + # }, + # "HK": { + # "HTC_b1.112_N22.6_C4_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5OQ==", + # "HTC_b1.3_N49.4_C4.1_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwMQ==", + # "HTC_b1.112_N22.6_C4.1_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwMw==", + # "HTC_b0.97_N11.6_C4_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5OA==", + # "HTC_b0.97_N11.6_C4.1_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwMA==", + # "HTL_b1.112_N22.6_C4.1_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwMg==", + # "HTC_b1.3_N49.4_C4_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwNA==", + # "HTL_b1.112_N22.6_C4_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwNw==", + # "HTL_b0.97_N11.6_C4_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwOA==", + # "HTL_b1.3_N49.4_C4_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwNQ==", + # "HTL_b1.3_N49.4_C4.1_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwNg==", + # "HTL_b0.97_N11.6_C4.1_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwOQ==" + # }, + # "BG": { + # "floor_addtot346ave": "RmlsZToxMDIyMzA=" + # }, + # "PY": { + # "P_b0.75_N3.4_C3.9_s1.34": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxMQ==" + # } + # } + # ] + + + #27_TAG_CONFIG + permutations = [ + { + "tag": "all rate combinations", "weight": 1.0, + "permute" : [ + { "group": "HIK", + "members" : [ + {"tag": "HTC_b0.957_N16.5_C4.1_s1", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjk0OQ=="}, + {"tag": "HTC_b1.078_N22.8_C4.1_s1", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjk1MA=="}, + {"tag": "HTL_b0.957_N16.5_C4.1_s1", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjk1MQ=="}, + {"tag": "HTL_b1.078_N22.8_C4.1_s1", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjk1Mg=="}, + {"tag": "HTC_b0.957_N16.5_C4.1_s0.54", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjk1Mw=="}, + {"tag": "HTC_b0.957_N16.5_C4.1_s1.43", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjk1NA=="}, + {"tag": "HTC_b1.078_N22.8_C4.1_s0.54", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjk1NQ=="}, + {"tag": "HTC_b1.078_N22.8_C4.1_s1.43", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjk1Ng=="}, + {"tag": "HTL_b0.957_N16.5_C4.1_s0.54", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjk1Nw=="}, + {"tag": "HTL_b0.957_N16.5_C4.1_s1.43", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjk1OA=="}, + {"tag": "HTL_b1.078_N22.8_C4.1_s0.54", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjk1OQ=="}, + {"tag": "HTL_b1.078_N22.8_C4.1_s1.43", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjk2MA=="} + ] + }, + { "group": "PUY", + "members" : [ + {"tag": "P_b0.75_N3.4_C3.9_s1", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Ng=="} + ] + }, + { "group": "CRU", + "members" : [ + {"tag": "CR_N8.0_b1.115_C4.3_s1", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3OA=="}, + {"tag": "CR_N2.3_b0.807_C4.2_s1", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4MA=="}, + {"tag": "CR_N3.7_b0.929_C4.2_s1", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4MQ=="}, + {"tag": "CR_N8.0_b1.115_C4.3_s0.51", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxMg=="}, + {"tag": "CR_N2.3_b0.807_C4.2_s0.51", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxNg=="}, + {"tag": "CR_N3.7_b0.929_C4.2_s0.51", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxNw=="}, + {"tag": "CR_N2.3_b0.807_C4.2_s1.62", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxOA=="}, + {"tag": "CR_N8.0_b1.115_C4.3_s1.62", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyMQ=="}, + {"tag": "CR_N3.7_b0.929_C4.2_s1.62", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyNA=="} + ] + }, + { "group": "BG", + "members" : [ + {"tag": "floor_addtot346ave", "toshi_id": "RmlsZToxMDIyMzA="} + ] + } + ] + } + ] + + + logging.basicConfig(level=logging.INFO) + sources_folder = Path(WORK_PATH, 'sources') + source_file_mapping = SourceModelLoader(API_URL, API_KEY, S3_URL).unpack_sources(permutations, sources_folder) + # print(source_file_mapping) + + ltbs = [ltb for ltb in get_logic_tree_branches(permutations)] + + print("LTB:", len(ltbs), ltbs[0]) + + nrml = build_sources_xml(ltbs, source_file_mapping) + + with open("source_model.xml", 'w') as f: + f.write(nrml) + + print('Done!') + + + # task_args = { + # "logic_tree_permutations" : [ + # { + # "CR": { + # "CR_N7.8_b_1.111_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0NA==", + # "CR_N7.8_b_1.111_s2": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0NQ==", + # "CR_N3.5_b0.913_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0Ng==", + # #"CR_N3.5_b0.913_s2": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0Nw==" + # }, + # "HK": { + # "HK_N25.6_b0.942_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0OA==", + # "HK_N25.6_b1.009_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0OQ==", + # #"HK_N25.6_b1.009_s12": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM1MA==" + # }, + # "BG": { + # "bgA": "RmlsZToxMDE4MDI=" + # }, + # "PY": { + # "PY_N": "RmlsZToxMDE4MDA=" + # } + # }, + # #MORE of these .... + # ] + # } + + #this is produced by + # source_file_mapping = { + # "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0NA==" : + # { + # 'source_name': 'CR_N7.8_b_1.111_s1', + # 'sources' : ['path/to/fileA', 'path/to/FileA1'] + # }, + # "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0NQ==" : + # { + # 'source_name': 'CR_N7.8_b_1.111_s1', + # 'sources' : ['path/to/fileB', 'path/to/FileB1'] + # }, + # "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0Ng==" : + # { + # 'source_name': 'CR_N7.8_b_1.111_s1', + # 'sources' : ['path/to/fileC', 'path/to/FileC1'] + # }, + # "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0OA==" : + # { + # 'source_name': 'CR_N7.8_b_1.111_s1', + # 'sources' : ['path/to/fileD', 'path/to/FileD1'] + # }, + # "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0OQ==" : + # { + # 'source_name': 'CR_N7.8_b_1.111_s2', + # 'sources' : ['path/to/fileE', 'path/to/FileE1'] + # }, + # "RmlsZToxMDE4MDI=" : + # { + # 'source_name': 'BG', + # 'sources' : ['path/to/fileBG'] + # }, + # "RmlsZToxMDE4MDA=" : + # { + # 'source_name': 'PY', + # 'sources' : ['path/to/filePY'] + # } + + # } + + # id_list = get_logic_tree_file_ids(task_args['logic_tree_permutations']) + # #print("ID lists:", list(id_list)) + # #TODO build a map whlie downloading the files + diff --git a/runzi/execute/util/oq_hazard_config.py b/runzi/execute/util/oq_hazard_config.py new file mode 100644 index 00000000..4e9ff0b5 --- /dev/null +++ b/runzi/execute/util/oq_hazard_config.py @@ -0,0 +1,145 @@ +#!python3 oq_hazard_config.py +import logging +import configparser +import io + +from pathlib import Path +log = logging.getLogger(__name__) + +SITES = dict( + WLG = {"sites": "174.7762 -41.2865"}, + NZ4 = {"sites_csv": "nz_towns_4.csv"}, + NZ34 = {"sites_csv": "nz_towns_34.csv"}, + GRD1 = {"sites_csv": "NZ_whole_country_10k.csv"}) + +# DEFAULT_DISAGG = dict( +# poes_disagg = 0.002, +# mag_bin_width = 0.25, +# distance_bin_width = 1.0, +# coordinate_bin_width = 5.0, +# num_epsilon_bins = 1) + + +#Sanjay new values +DEFAULT_DISAGG = dict( + max_sites_disagg = 1, + poes_disagg = "0.002105 0.000404", + mag_bin_width = 0.25, + distance_bin_width = 10, + coordinate_bin_width = 1, + num_epsilon_bins = 4, + disagg_outputs = "Mag_Dist Mag_Dist_Eps TRT" + ) + + +class OpenquakeConfig(): + + def __init__(self, config): + self.config = configparser.ConfigParser() + self.config.read_file(config) + + def set_sites(self, site_key: str): + """ + + """ + assert site_key in SITES.keys() + #destroy any existing site configs + self.config['site_params'].pop('sites', None) + self.config['site_params'].pop('sites_csv', None) + self.config.pop('geometry', None) + key, value = list(SITES[site_key].items())[0] + self.config.add_section('geometry') + self.config['geometry'][key] = value + return self + + def set_disaggregation(self, enable: bool, values: dict = None): + self.config['general']['calculation_mode'] = 'disaggregation' if enable else 'classical' + if enable: + self.config.pop('disagg', None) # destroy any existing disagg settings + self.config.add_section('disagg') + settings = DEFAULT_DISAGG.copy() + if values: + settings.update(values) + for k, v in settings.items(): + self.config['disagg'][k] = str(v) + return self + + def set_iml(self, measures: list, levels: object): + + self.config['calculation'].pop('intensity_measure_types_and_levels', None) + + new_iml = '{' + for m in measures: + new_iml += f'"{m}": {str(levels)}, ' + new_iml += '}' + + self.config['calculation']['intensity_measure_types_and_levels '] = new_iml + return self + + + + def set_vs30(self, vs30): + + try: + from openquake.commands.prepare_site_model import calculate_z1pt0, calculate_z2pt5_ngaw2 + except: + print("openquake librarys are not available, skipping set_vs30 ") + return self + + sect = self.config['site_params'] + #Clean up old settings + for setting in ['reference_vs30_type', 'reference_vs30_value', + 'reference_depth_to_1pt0km_per_sec', 'reference_depth_to_2pt5km_per_sec']: + sect.pop(setting, None) + + sect['reference_vs30_type'] = 'measured' + sect['reference_vs30_value'] = str(vs30) + sect['reference_depth_to_1pt0km_per_sec'] = str(round(calculate_z1pt0(vs30), 0)) + sect['reference_depth_to_2pt5km_per_sec'] = str(round(calculate_z2pt5_ngaw2(vs30), 1)) + return self + + def write(self, tofile): + self.config.write(tofile) + + +if __name__ == "__main__": + + sample_conf = """ + [general] + calculation_mode = disaggregation + + [logic_tree] + + number_of_logic_tree_samples = 0 + + [site_params] + sites = 174.7762 -41.2865 + foo=bar + + + [calculation] + intensity_measure_types_and_levels = {"SA(0.5)": logscale(0.005, 4.00, 30)} + + """ + sample = io.StringIO(sample_conf) #fake file for demo + + nc = OpenquakeConfig(sample)\ + .set_sites('NZ4')\ + .set_disaggregation(False, {"num_rlz_disagg": 0}) + + measures = ['PGA', 'SA(0.5)'] + levels0 = [0.01, 0.02, 0.04, 0.06, 0.08, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2, 2.4, 2.6, 2.8, 3.0, 3.5, 4, 4.5, 5.0] + levels1 = 'logscale(0.005, 4.00, 30)' + _4_sites_levels = [0.01, 0.02, 0.04, 0.06, 0.08, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2, 2.4, 2.6, 2.8, 3.0, 3.5, 4, 4.5, 5.0] + _4_sites_measures = ['PGA',"SA(0.1)","SA(0.2)","SA(0.3)","SA(0.4)","SA(0.5)","SA(0.7)","SA(1.0)","SA(1.5)","SA(2.0)","SA(3.0)","SA(4.0)","SA(5.0)"] + + nc.set_iml(_4_sites_measures, _4_sites_levels) + nc.set_vs30(250) + + out = io.StringIO() #aother fake file + nc.write(out) + + out.seek(0) + for l in out: + print(l) +