From ae51380cb5811a4e5ceee8f9d977c3d97882561c Mon Sep 17 00:00:00 2001 From: Chris Chamberlain Date: Tue, 29 Mar 2022 09:03:42 +1300 Subject: [PATCH 01/13] update ucerf3 converter; docs etc --- docker/runzi-openquake/examples/test_nzshm.py | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 docker/runzi-openquake/examples/test_nzshm.py diff --git a/docker/runzi-openquake/examples/test_nzshm.py b/docker/runzi-openquake/examples/test_nzshm.py new file mode 100644 index 00000000..88a1b0ea --- /dev/null +++ b/docker/runzi-openquake/examples/test_nzshm.py @@ -0,0 +1,60 @@ +import gzip +import unittest +import numpy +from openquake.baselib import parallel, general, config +from openquake.baselib.python3compat import decode +from openquake.hazardlib import InvalidFile, contexts +from openquake.hazardlib.source.rupture import get_ruptures +from openquake.hazardlib.sourcewriter import write_source_model +from openquake.commonlib import readinput +from openquake.calculators.tests import ( + CalculatorTestCase, NOT_DARWIN, strip_calc_id) + +# hacked from https://github.com/gem/oq-engine/blob/7e8dc6fdd0d50da41cfed76ec901838e5c929b32/openquake/calculators/tests/classical_test.py + +class ClassicalTestCase(CalculatorTestCase): + + def assert_curves_ok(self, expected, test_dir, delta=None, **kw): + kind = kw.pop('kind', '') + self.run_calc(test_dir, 'job.ini', **kw) + ds = self.calc.datastore + got = (export(('hcurves/' + kind, 'csv'), ds) + + export(('hmaps/' + kind, 'csv'), ds) + + export(('uhs/' + kind, 'csv'), ds)) + self.assertEqual(len(expected), len(got), str(got)) + for fname, actual in zip(expected, got): + self.assertEqualFiles('expected/%s' % fname, actual, + delta=delta) + return got + + + def test_case_65(self): + # reading/writing a multiFaultSource + + #oq = readinput.get_oqparam('job.ini', pkg=case_65) + + inifile = "/WORKING/examples/16_SRWG_TEST/oq_inputs/many-sites_3-periods_vs30-475.ini" + oq = readinput.get_oqparam(inifile) + csm = readinput.get_composite_source_model(oq) + tmpname = general.gettemp() + out = write_source_model(tmpname, csm.src_groups) + self.assertEqual(out[0], tmpname) + self.assertEqual(out[1], tmpname[:-4] + '_sections.xml') + + # running the calculation + self.run_calc(case_65.__file__, 'job.ini') + + [f] = export(('hcurves/mean', 'csv'), self.calc.datastore) + self.assertEqualFiles('expected/hcurve-mean.csv', f, delta=1E-5) + + # make sure we are not breaking event_based + self.run_calc(case_65.__file__, 'job_eb.ini') + [f] = export(('ruptures', 'csv'), self.calc.datastore) + self.assertEqualFiles('expected/ruptures.csv', f, delta=1E-5) + + rups = extract(self.calc.datastore, 'ruptures') + csv = general.gettemp(rups.array) + self.assertEqualFiles('expected/full_ruptures.csv', csv, delta=1E-5) + + files = export(('gmf_data', 'csv'), self.calc.datastore) + self.assertEqualFiles('expected/gmf_data.csv', files[0], delta=1E-4) \ No newline at end of file From 8f0844a4c405f1f67caff8729dd434a1b33894c6 Mon Sep 17 00:00:00 2001 From: Chris Chamberlain Date: Mon, 4 Apr 2022 11:25:44 +1200 Subject: [PATCH 02/13] remove redundant test;update gitignore; --- docker/runzi-openquake/examples/test_nzshm.py | 60 ------------------- 1 file changed, 60 deletions(-) delete mode 100644 docker/runzi-openquake/examples/test_nzshm.py diff --git a/docker/runzi-openquake/examples/test_nzshm.py b/docker/runzi-openquake/examples/test_nzshm.py deleted file mode 100644 index 88a1b0ea..00000000 --- a/docker/runzi-openquake/examples/test_nzshm.py +++ /dev/null @@ -1,60 +0,0 @@ -import gzip -import unittest -import numpy -from openquake.baselib import parallel, general, config -from openquake.baselib.python3compat import decode -from openquake.hazardlib import InvalidFile, contexts -from openquake.hazardlib.source.rupture import get_ruptures -from openquake.hazardlib.sourcewriter import write_source_model -from openquake.commonlib import readinput -from openquake.calculators.tests import ( - CalculatorTestCase, NOT_DARWIN, strip_calc_id) - -# hacked from https://github.com/gem/oq-engine/blob/7e8dc6fdd0d50da41cfed76ec901838e5c929b32/openquake/calculators/tests/classical_test.py - -class ClassicalTestCase(CalculatorTestCase): - - def assert_curves_ok(self, expected, test_dir, delta=None, **kw): - kind = kw.pop('kind', '') - self.run_calc(test_dir, 'job.ini', **kw) - ds = self.calc.datastore - got = (export(('hcurves/' + kind, 'csv'), ds) + - export(('hmaps/' + kind, 'csv'), ds) + - export(('uhs/' + kind, 'csv'), ds)) - self.assertEqual(len(expected), len(got), str(got)) - for fname, actual in zip(expected, got): - self.assertEqualFiles('expected/%s' % fname, actual, - delta=delta) - return got - - - def test_case_65(self): - # reading/writing a multiFaultSource - - #oq = readinput.get_oqparam('job.ini', pkg=case_65) - - inifile = "/WORKING/examples/16_SRWG_TEST/oq_inputs/many-sites_3-periods_vs30-475.ini" - oq = readinput.get_oqparam(inifile) - csm = readinput.get_composite_source_model(oq) - tmpname = general.gettemp() - out = write_source_model(tmpname, csm.src_groups) - self.assertEqual(out[0], tmpname) - self.assertEqual(out[1], tmpname[:-4] + '_sections.xml') - - # running the calculation - self.run_calc(case_65.__file__, 'job.ini') - - [f] = export(('hcurves/mean', 'csv'), self.calc.datastore) - self.assertEqualFiles('expected/hcurve-mean.csv', f, delta=1E-5) - - # make sure we are not breaking event_based - self.run_calc(case_65.__file__, 'job_eb.ini') - [f] = export(('ruptures', 'csv'), self.calc.datastore) - self.assertEqualFiles('expected/ruptures.csv', f, delta=1E-5) - - rups = extract(self.calc.datastore, 'ruptures') - csv = general.gettemp(rups.array) - self.assertEqualFiles('expected/full_ruptures.csv', csv, delta=1E-5) - - files = export(('gmf_data', 'csv'), self.calc.datastore) - self.assertEqualFiles('expected/gmf_data.csv', files[0], delta=1E-4) \ No newline at end of file From e05c0c82e3a303f0185f546c38feb1a0ef86c420 Mon Sep 17 00:00:00 2001 From: Chris Chamberlain Date: Tue, 5 Apr 2022 16:36:00 +1200 Subject: [PATCH 03/13] add toshi_api support for hazard --- .../scaling/toshi_api/openquake_hazard.py | 110 ++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 runzi/automation/scaling/toshi_api/openquake_hazard.py diff --git a/runzi/automation/scaling/toshi_api/openquake_hazard.py b/runzi/automation/scaling/toshi_api/openquake_hazard.py new file mode 100644 index 00000000..61465b38 --- /dev/null +++ b/runzi/automation/scaling/toshi_api/openquake_hazard.py @@ -0,0 +1,110 @@ + +from datetime import datetime as dt +from dateutil.tz import tzutc +from hashlib import md5 +from pathlib import PurePath + +import base64 +import json +import requests + +from nshm_toshi_client.toshi_client_base import ToshiClientBase, kvl_to_graphql + +import logging +log = logging.getLogger(__name__) + +class OpenquakeHazardConfig(object): + + def __init__(self, api): + self.api = api + assert isinstance(api, ToshiClientBase) + + def upload_hazard_config(self, task_id, source_models, filepath): + filepath = PurePath(filepath) + file_id, post_url = self._create_hazard_configuration(filepath, task_id, source_models) + self.upload_content(post_url, filepath) + + # #link file to task in role + # self.api.task_file.create_task_file(task_id, file_id, 'WRITE') + return file_id + + def upload_content(self, post_url, filepath): + log.debug(f'upload_content() POST URL: {post_url}; PATH: {filepath}') + filedata = open(filepath, 'rb') + files = {'file': filedata} + log.debug(f'upload_content() _s3_url: {self.api._s3_url}') + + response = requests.post( + url=self.api._s3_url, + data=post_url, + files=files) + log.debug(f'response {response}') + response.raise_for_status() + + def _create_hazard_configuration(self, filepath, source_models, produced_by): + qry = ''' + mutation ($created: DateTime!, $source_models: [ID!], $digest: String!, + $file_name: String!, $file_size: Int!, $produced_by: ID!) { + create_openquake_hazard_config(input: { + created: $created + source_models: $sources #must be NRML sources Openquake + md5_digest: $digest + file_name: $file_name + file_size: $file_size + produced_by_id: $produced_by + ##MFD_TABLE## + + ##META## + + ##METRICS## + + } + ) { + config { id, created, source_models { id } } + } + } + ''' + + filedata = open(filepath, 'rb') + digest = base64.b64encode(md5(filedata.read()).digest()).decode() + + filedata.seek(0) #important! + size = len(filedata.read()) + filedata.close() + + created = dt.utcnow(tzutc()).isoformat() + variables = dict(digest=digest, source_models=source_models, file_name=filepath.parts[-1], file_size=size, + produced_by=produced_by, created=created) + + #result = self.api.client.execute(qry, variable_values = variables) + #print(result) + executed = self.api.run_query(qry, variables) + #print("executed", executed) + post_url = json.loads(executed['create_inversion_solution']['inversion_solution']['post_url']) + + return (executed['create_inversion_solution']['inversion_solution']['id'], post_url) + + + def get_hazard_config(self, config_id): + + qry = ''' + query get_sol_tables ($config_id: ID!) { + node(id:$config_id) { + __typename + ... on OpenquakeHazardConfig{ + created + source_models { + id + source_solution { + id + file_name + } + } + } + } + } + } + ''' + + executed = self.api.run_query(qry, dict(config_id=config_id)) + return executed['node'] \ No newline at end of file From 8f2d771c9aba5b88d78ac536e0b488da7dbfe1c8 Mon Sep 17 00:00:00 2001 From: Chris Chamberlain Date: Tue, 12 Apr 2022 15:14:11 +1200 Subject: [PATCH 04/13] AWS tweaks --- runzi/configuration/oq_hazard.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/runzi/configuration/oq_hazard.py b/runzi/configuration/oq_hazard.py index e221ff3d..ed5a231f 100644 --- a/runzi/configuration/oq_hazard.py +++ b/runzi/configuration/oq_hazard.py @@ -24,6 +24,8 @@ HAZARD_MAX_TIME = 36*60 #minutes +HAZARD_MAX_TIME = 36*60 #minutes + def build_hazard_tasks(general_task_id: str, subtask_type: SubtaskType, model_type: ModelType, subtask_arguments): task_count = 0 From c3e7615a6bed856c5792c897e07806e296573005 Mon Sep 17 00:00:00 2001 From: Chris Chamberlain Date: Thu, 14 Apr 2022 12:58:01 +1200 Subject: [PATCH 05/13] added support for dynamic openquake configs --- runzi/execute/util/oq_hazard_config.py | 129 +++++++++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 runzi/execute/util/oq_hazard_config.py diff --git a/runzi/execute/util/oq_hazard_config.py b/runzi/execute/util/oq_hazard_config.py new file mode 100644 index 00000000..014f146e --- /dev/null +++ b/runzi/execute/util/oq_hazard_config.py @@ -0,0 +1,129 @@ +#!python3 oq_hazard_config.py +import logging +import configparser +import io + +from pathlib import Path +log = logging.getLogger(__name__) + +sample_conf = """ +[general] +calculation_mode = disaggregation + +[logic_tree] + +number_of_logic_tree_samples = 0 + +[site_params] +sites = 174.7762 -41.2865 +foo=bar + + +[calculation] +intensity_measure_types_and_levels = {"SA(0.5)": logscale(0.005, 4.00, 30)} + +""" + +SITES = dict( + WLG = {"sites": "174.7762 -41.2865"}, + NZ4 = {"sites_csv": "nz_towns_4.csv"}, + NZ34 = {"sites_csv": "nz_towns_34.csv"}, + GRD1 = {"sites_csv": "NZ_whole_country_10k.csv"}) + +DEFAULT_DISAGG = dict( + poes_disagg = 0.002, + mag_bin_width = 0.25, + distance_bin_width = 1.0, + coordinate_bin_width = 5.0, + num_epsilon_bins = 1) + +class OpenquakeConfig(): + + def __init__(self, config): + self.config = configparser.ConfigParser() + self.config.read_file(config) + + def set_sites(self, site_key: str): + """ + + """ + assert site_key in SITES.keys() + #destroy any existing site configs + self.config['site_params'].pop('sites', None) + self.config['site_params'].pop('sites_csv', None) + self.config.pop('geometry', None) + key, value = list(SITES[site_key].items())[0] + self.config.add_section('geometry') + self.config['geometry'][key] = value + return self + + def set_disaggregation(self, enable: bool, values: dict = None): + self.config['general']['calculation_mode'] = 'disaggregation' if enable else 'classical' + if enable: + self.config.pop('disagg', None) # destroy any existing disagg settings + self.config.add_section('disagg') + settings = DEFAULT_DISAGG.copy() + if values: + settings.update(values) + for k, v in settings.items(): + self.config['disagg'][k] = str(v) + return self + + def set_iml(self, measures: list, levels: object): + + self.config['calculation'].pop('intensity_measure_types_and_levels', None) + + new_iml = '{' + for m in measures: + new_iml += f'"{m}": {str(levels)}, ' + new_iml += '}' + + self.config['calculation']['intensity_measure_types_and_levels '] = new_iml + return self + + + + def set_vs30(self, vs30): + + try: + from openquake.commands.prepare_site_model import calculate_z1pt0, calculate_z2pt5_ngaw2 + except: + print("openquake librarys are not available, skipping set_vs30 ") + return self + + sect = self.config.get_section('site_params') + #Clean up old settings + for setting in ['reference_vs30_type', 'reference_vs30_value', + 'reference_depth_to_1pt0km_per_sec', 'reference_depth_to_2pt5km_per_sec']: + sect.pop(setting, None) + + sect['reference_vs30_type'] = 'measured' + sect['reference_vs30_value'] = str(vs30) + sect['reference_depth_to_1pt0km_per_sec'] = str(calculate_z1pt0(vs30)) + sect['reference_depth_to_2pt5km_per_sec'] = str(calculate_z2pt5_ngaw2(vs30)) + return self + + def write(self, tofile): + self.config.write(tofile) + + +if __name__ == "__main__": + + samp = io.StringIO(sample_conf) + nc = OpenquakeConfig(samp)\ + .set_sites('NZ4')\ + .set_disaggregation(True, {"num_rlz_disagg": 0}) + + measures = ['PGA', 'SA(0.5)'] + levels = [0.01, 0.02, 0.04, 0.06, 0.08, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2, 2.4, 2.6, 2.8, 3.0, 3.5, 4, 4.5, 5.0] + levels1 = 'logscale(0.005, 4.00, 30)' + nc.set_iml(measures, levels1) + nc.set_vs30(355) + + out = io.StringIO() + nc.write(out) + + out.seek(0) + for l in out: + print(l) + From 57f8c1a62dd242126c2e33ad30891919ac80da18 Mon Sep 17 00:00:00 2001 From: Chris Chamberlain Date: Thu, 14 Apr 2022 13:01:08 +1200 Subject: [PATCH 06/13] tidy up --- runzi/execute/util/oq_hazard_config.py | 45 +++++++++++++------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/runzi/execute/util/oq_hazard_config.py b/runzi/execute/util/oq_hazard_config.py index 014f146e..7277a840 100644 --- a/runzi/execute/util/oq_hazard_config.py +++ b/runzi/execute/util/oq_hazard_config.py @@ -6,24 +6,6 @@ from pathlib import Path log = logging.getLogger(__name__) -sample_conf = """ -[general] -calculation_mode = disaggregation - -[logic_tree] - -number_of_logic_tree_samples = 0 - -[site_params] -sites = 174.7762 -41.2865 -foo=bar - - -[calculation] -intensity_measure_types_and_levels = {"SA(0.5)": logscale(0.005, 4.00, 30)} - -""" - SITES = dict( WLG = {"sites": "174.7762 -41.2865"}, NZ4 = {"sites_csv": "nz_towns_4.csv"}, @@ -109,18 +91,37 @@ def write(self, tofile): if __name__ == "__main__": - samp = io.StringIO(sample_conf) - nc = OpenquakeConfig(samp)\ + sample_conf = """ + [general] + calculation_mode = disaggregation + + [logic_tree] + + number_of_logic_tree_samples = 0 + + [site_params] + sites = 174.7762 -41.2865 + foo=bar + + + [calculation] + intensity_measure_types_and_levels = {"SA(0.5)": logscale(0.005, 4.00, 30)} + + """ + sample = io.StringIO(sample_conf) #fake file for demo + + nc = OpenquakeConfig(sample)\ .set_sites('NZ4')\ .set_disaggregation(True, {"num_rlz_disagg": 0}) measures = ['PGA', 'SA(0.5)'] - levels = [0.01, 0.02, 0.04, 0.06, 0.08, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2, 2.4, 2.6, 2.8, 3.0, 3.5, 4, 4.5, 5.0] + levels0 = [0.01, 0.02, 0.04, 0.06, 0.08, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2, 2.4, 2.6, 2.8, 3.0, 3.5, 4, 4.5, 5.0] levels1 = 'logscale(0.005, 4.00, 30)' + nc.set_iml(measures, levels1) nc.set_vs30(355) - out = io.StringIO() + out = io.StringIO() #aother fake file nc.write(out) out.seek(0) From 336bd96acc30a160527e9ba9121e6b104be8dbcb Mon Sep 17 00:00:00 2001 From: Chris Chamberlain Date: Thu, 14 Apr 2022 15:29:54 +1200 Subject: [PATCH 07/13] set up vs30 fn --- docker/runzi-openquake/Dockerfile | 2 +- runzi/execute/util/oq_hazard_config.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/runzi-openquake/Dockerfile b/docker/runzi-openquake/Dockerfile index 1d8fff5e..4d2a52b7 100644 --- a/docker/runzi-openquake/Dockerfile +++ b/docker/runzi-openquake/Dockerfile @@ -31,7 +31,7 @@ RUN git clone https://github.com/GNS-Science/nzshm-runzi.git WORKDIR /app/nzshm-runzi RUN git fetch -RUN git checkout feature/89-integrate-openquake-and-toshiapi +RUN git checkout feature/96-site-config RUN pip3 install -r requirements.txt RUN pip3 install -e . diff --git a/runzi/execute/util/oq_hazard_config.py b/runzi/execute/util/oq_hazard_config.py index 7277a840..ef4d8d64 100644 --- a/runzi/execute/util/oq_hazard_config.py +++ b/runzi/execute/util/oq_hazard_config.py @@ -73,7 +73,7 @@ def set_vs30(self, vs30): print("openquake librarys are not available, skipping set_vs30 ") return self - sect = self.config.get_section('site_params') + sect = self.config['site_params'] #Clean up old settings for setting in ['reference_vs30_type', 'reference_vs30_value', 'reference_depth_to_1pt0km_per_sec', 'reference_depth_to_2pt5km_per_sec']: @@ -81,8 +81,8 @@ def set_vs30(self, vs30): sect['reference_vs30_type'] = 'measured' sect['reference_vs30_value'] = str(vs30) - sect['reference_depth_to_1pt0km_per_sec'] = str(calculate_z1pt0(vs30)) - sect['reference_depth_to_2pt5km_per_sec'] = str(calculate_z2pt5_ngaw2(vs30)) + sect['reference_depth_to_1pt0km_per_sec'] = str(round(calculate_z1pt0(vs30), 0)) + sect['reference_depth_to_2pt5km_per_sec'] = str(round(calculate_z2pt5_ngaw2(vs30), 1)) return self def write(self, tofile): From 63c79e146f06656291b1c9cf8a4977d0e77c7379 Mon Sep 17 00:00:00 2001 From: Chris Chamberlain Date: Thu, 14 Apr 2022 16:12:41 +1200 Subject: [PATCH 08/13] fix rebase repeats --- .../scaling/toshi_api/openquake_hazard.py | 110 ------------------ runzi/configuration/oq_hazard.py | 2 - 2 files changed, 112 deletions(-) delete mode 100644 runzi/automation/scaling/toshi_api/openquake_hazard.py diff --git a/runzi/automation/scaling/toshi_api/openquake_hazard.py b/runzi/automation/scaling/toshi_api/openquake_hazard.py deleted file mode 100644 index 61465b38..00000000 --- a/runzi/automation/scaling/toshi_api/openquake_hazard.py +++ /dev/null @@ -1,110 +0,0 @@ - -from datetime import datetime as dt -from dateutil.tz import tzutc -from hashlib import md5 -from pathlib import PurePath - -import base64 -import json -import requests - -from nshm_toshi_client.toshi_client_base import ToshiClientBase, kvl_to_graphql - -import logging -log = logging.getLogger(__name__) - -class OpenquakeHazardConfig(object): - - def __init__(self, api): - self.api = api - assert isinstance(api, ToshiClientBase) - - def upload_hazard_config(self, task_id, source_models, filepath): - filepath = PurePath(filepath) - file_id, post_url = self._create_hazard_configuration(filepath, task_id, source_models) - self.upload_content(post_url, filepath) - - # #link file to task in role - # self.api.task_file.create_task_file(task_id, file_id, 'WRITE') - return file_id - - def upload_content(self, post_url, filepath): - log.debug(f'upload_content() POST URL: {post_url}; PATH: {filepath}') - filedata = open(filepath, 'rb') - files = {'file': filedata} - log.debug(f'upload_content() _s3_url: {self.api._s3_url}') - - response = requests.post( - url=self.api._s3_url, - data=post_url, - files=files) - log.debug(f'response {response}') - response.raise_for_status() - - def _create_hazard_configuration(self, filepath, source_models, produced_by): - qry = ''' - mutation ($created: DateTime!, $source_models: [ID!], $digest: String!, - $file_name: String!, $file_size: Int!, $produced_by: ID!) { - create_openquake_hazard_config(input: { - created: $created - source_models: $sources #must be NRML sources Openquake - md5_digest: $digest - file_name: $file_name - file_size: $file_size - produced_by_id: $produced_by - ##MFD_TABLE## - - ##META## - - ##METRICS## - - } - ) { - config { id, created, source_models { id } } - } - } - ''' - - filedata = open(filepath, 'rb') - digest = base64.b64encode(md5(filedata.read()).digest()).decode() - - filedata.seek(0) #important! - size = len(filedata.read()) - filedata.close() - - created = dt.utcnow(tzutc()).isoformat() - variables = dict(digest=digest, source_models=source_models, file_name=filepath.parts[-1], file_size=size, - produced_by=produced_by, created=created) - - #result = self.api.client.execute(qry, variable_values = variables) - #print(result) - executed = self.api.run_query(qry, variables) - #print("executed", executed) - post_url = json.loads(executed['create_inversion_solution']['inversion_solution']['post_url']) - - return (executed['create_inversion_solution']['inversion_solution']['id'], post_url) - - - def get_hazard_config(self, config_id): - - qry = ''' - query get_sol_tables ($config_id: ID!) { - node(id:$config_id) { - __typename - ... on OpenquakeHazardConfig{ - created - source_models { - id - source_solution { - id - file_name - } - } - } - } - } - } - ''' - - executed = self.api.run_query(qry, dict(config_id=config_id)) - return executed['node'] \ No newline at end of file diff --git a/runzi/configuration/oq_hazard.py b/runzi/configuration/oq_hazard.py index ed5a231f..e221ff3d 100644 --- a/runzi/configuration/oq_hazard.py +++ b/runzi/configuration/oq_hazard.py @@ -24,8 +24,6 @@ HAZARD_MAX_TIME = 36*60 #minutes -HAZARD_MAX_TIME = 36*60 #minutes - def build_hazard_tasks(general_task_id: str, subtask_type: SubtaskType, model_type: ModelType, subtask_arguments): task_count = 0 From 0ddc20aeacaf09e40a5592455cf57543dd788fa4 Mon Sep 17 00:00:00 2001 From: chrisdicaprio Date: Tue, 19 Apr 2022 12:10:28 +1200 Subject: [PATCH 09/13] example source logic tree config --- runzi/automation/run_scale_solution.py | 24 +++++-------- .../HAZARD/source_logic_tree.json | 36 +++++++++++++++++++ 2 files changed, 45 insertions(+), 15 deletions(-) create mode 100644 runzi/cli/config/saved_configs/HAZARD/source_logic_tree.json diff --git a/runzi/automation/run_scale_solution.py b/runzi/automation/run_scale_solution.py index bb1cf021..b5c315ac 100644 --- a/runzi/automation/run_scale_solution.py +++ b/runzi/automation/run_scale_solution.py @@ -107,10 +107,10 @@ def build_subset_tasks(general_task_id, source_solutions, args): log = logging.getLogger(__name__) # If you wish to oversolution_ide something in the main config, do so here .. - WORKER_POOL_SIZE = 1 + WORKER_POOL_SIZE = 2 USE_API = True #If using API give this task a descriptive setting... - TASK_TITLE = "Hikurangi Scaled Inverions, scale = 0.587" + TASK_TITLE = "Hikurangi LTB004. Scaled 0.67, 1.41" TASK_DESCRIPTION = """ """ @@ -121,20 +121,14 @@ def build_subset_tasks(general_task_id, source_solutions, args): toshi_api = ToshiApi(API_URL, S3_URL, None, with_schema_validation=True, headers=headers) source_solution_ids = [ - #"SW52ZXJzaW9uU29sdXRpb246MTAwMDQ5", - #"SW52ZXJzaW9uU29sdXRpb246MTAwMDUw", - #"SW52ZXJzaW9uU29sdXRpb246MTAwMDU2" - #"R2VuZXJhbFRhc2s6MTAwMTA2", - "R2VuZXJhbFRhc2s6MTAwMDEz", - #"SW52ZXJzaW9uU29sdXRpb246MTAwMjQ4", - #"SW52ZXJzaW9uU29sdXRpb246MTAwMjUw", - #"SW52ZXJzaW9uU29sdXRpb246MTAwMjUy" + "R2VuZXJhbFRhc2s6MTAwMjQx", ] - #scales = [0.5,2.0] - #scales = [0.49, 1.63] - scales = [0.587, 1.419, 2, 3] - # scales = [0.587] - model_type = 'crustal' + + # scales = [0.49, 1.63] + # scales = [0.61, 1.34] + scales = [0.67, 1.41] + + model_type = 'subduction' file_generators = [] diff --git a/runzi/cli/config/saved_configs/HAZARD/source_logic_tree.json b/runzi/cli/config/saved_configs/HAZARD/source_logic_tree.json new file mode 100644 index 00000000..aec5427b --- /dev/null +++ b/runzi/cli/config/saved_configs/HAZARD/source_logic_tree.json @@ -0,0 +1,36 @@ +{ +"task_args": { + + "logic_trees": [ + {"tag": "test logic tree 1","branches": [ + {"name":"branch A", + "weight":0.5, + "source_ids":{ + "c_N7.8_b_1.111_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0MA==", + "h_N25.6_b0.942_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDMzMg==", + "bg": "RmlsZToxMDE4MDI=" + } + }, + {"name":"branch B", + "weight":0.5, + "source_ids":{ + "c_N3.5_b0.913_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0Mw==", + "h_N25.6_b1.009_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDMzMQ==", + "bg": "RmlsZToxMDE4MDI=" + } + } + ] + } + ] +} +} + + + + + + + + + + From dc2ad75405c0450642d2fde3939f59c4072d80f9 Mon Sep 17 00:00:00 2001 From: Chris Chamberlain Date: Wed, 20 Apr 2022 13:58:40 +1200 Subject: [PATCH 10/13] WIP on new configs --- docker/runzi-openquake/Dockerfile | 3 +- runzi/automation/run_oq_hazard.py | 78 ++++++--- runzi/configuration/oq_hazard.py | 24 ++- runzi/execute/oq_hazard_task.py | 91 +++++----- runzi/execute/util/__init__.py | 2 + runzi/execute/util/oq_build_sources.py | 227 +++++++++++++++++++++++++ runzi/execute/util/oq_hazard_config.py | 21 ++- 7 files changed, 378 insertions(+), 68 deletions(-) create mode 100644 runzi/execute/util/__init__.py create mode 100644 runzi/execute/util/oq_build_sources.py diff --git a/docker/runzi-openquake/Dockerfile b/docker/runzi-openquake/Dockerfile index 4d2a52b7..a11cbaab 100644 --- a/docker/runzi-openquake/Dockerfile +++ b/docker/runzi-openquake/Dockerfile @@ -1,7 +1,8 @@ # Dockerfile # Extend base image # needs openquake/engine >= 3.13.0 -FROM openquake/engine:nightly +#nightly +FROM openquake/engine:latest USER root RUN apt-get update diff --git a/runzi/automation/run_oq_hazard.py b/runzi/automation/run_oq_hazard.py index 9fb3e6ef..68519a55 100644 --- a/runzi/automation/run_oq_hazard.py +++ b/runzi/automation/run_oq_hazard.py @@ -54,29 +54,67 @@ def build_tasks(new_gt_id, args, task_type, model_type): headers={"x-api-key":API_KEY} toshi_api = ToshiApi(API_URL, None, None, with_schema_validation=True, headers=headers) + era_measures = ['PGA', 'SA(0.1)', 'SA(0.2)', 'SA(0.3)', 'SA(0.4)', 'SA(0.5)', 'SA(0.7)', + 'SA(1.0)', 'SA(1.5)', 'SA(2.0)', 'SA(3.0)', 'SA(4.0)', 'SA(5.0)'] + era_levels = [0.01, 0.02, 0.04, 0.06, 0.08, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, + 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2, 2.4, 2.6, 2.8, 3.0, 3.5, 4, 4.5, 5.0] + args = dict( config_archive_ids = [ # a Toshi File containing zipped configuration, ], #LOCAL'RmlsZToxOA=='], - "RmlsZToxMDE4MDQ=", #4-sites-many TEST RmlsZToxMDAzNTc= - "RmlsZToxMDE4MDY=", #PROD Wgn_005-10-300.ini RmlsZToxMDE4MDM= is BAD , PROD # TEST RmlsZToxMDA1MzA=" - "RmlsZToxMDE4MDc=", #PROD Wgn_005-10-50.ini - "RmlsZToxMDE4MDg=", #PROD Wgn_005-4-300.ini - "RmlsZToxMDE4MDk=", #PROD Wgn_005-4-50.ini + 'RmlsZToxOA==' + #"RmlsZToxMDE4MDQ=", #4-sites-many TEST RmlsZToxMDAzNTc= + # "RmlsZToxMDE4MDY=", #PROD Wgn_005-10-300.ini RmlsZToxMDE4MDM= is BAD , PROD # TEST RmlsZToxMDA1MzA=" + # "RmlsZToxMDE4MDc=", #PROD Wgn_005-10-50.ini + # "RmlsZToxMDE4MDg=", #PROD Wgn_005-4-300.ini + # "RmlsZToxMDE4MDk=", #PROD Wgn_005-4-50.ini ], - source_combos = [ - # {'tag':'combined','nrml_ids':{ - # 'crustal':"SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0Ng==", - # 'hik':"SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0OQ==", - # 'bg': 'RmlsZToxMDA0ODg='}}, - #{'tag':'crustal_only','nrml_ids':{ - # 'crustal':"SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0Ng=="}}, - #{'tag':'hik_only','nrml_ids':{'hik':"SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0OQ=="}}, - # {'tag': 'bg_only', 'nrml_ids': {'bg': 'RmlsZToxMDA0ODg='}}, - {'tag': 'combined', 'nrml_ids': { - 'crustal': "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0Mw==", #PROD "b_and_n": "{'tag': 'N = 3.5, b=0.913', PROD - 'hik': "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDMzMQ==", #PROD "b_and_n": "{'b': 1.009, 'N': 25.6}" - 'bg': "RmlsZToxMDE4MDI=" #BG_Kiran_fADDTOT346ave_Test4 unscaled BG TEST RmlsZToxMDA1MzU= - }} - ] + # source_combos = [ + # # {'tag':'combined','nrml_ids':{ + # # 'crustal':"SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0Ng==", + # # 'hik':"SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0OQ==", + # # 'bg': 'RmlsZToxMDA0ODg='}}, + # #{'tag':'crustal_only','nrml_ids':{ + # # 'crustal':"SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0Ng=="}}, + # #{'tag':'hik_only','nrml_ids':{'hik':"SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0OQ=="}}, + # {'tag': 'bg_only', 'nrml_ids': {'bg': 'RmlsZToxMDA0ODg='}}, + # # {'tag': 'combined', 'nrml_ids': { + # # 'crustal': "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0Mw==", #PROD "b_and_n": "{'tag': 'N = 3.5, b=0.913', PROD + # # 'hik': "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDMzMQ==", #PROD "b_and_n": "{'b': 1.009, 'N': 25.6}" + # # 'bg': "RmlsZToxMDE4MDI=" #BG_Kiran_fADDTOT346ave_Test4 unscaled BG TEST RmlsZToxMDA1MzU= + # # #'slab': "ABBBV" + # # #'puy' : "ABCB" + # # }} + # ], + logic_tree_permutations = [ + { + "CR": { + "CR_N7.8_b_1.111_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0NA==", + "CR_N7.8_b_1.111_s2": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0NQ==", + "CR_N3.5_b0.913_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0Ng==", + #"CR_N3.5_b0.913_s2": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0Nw==" + }, + "HK": { + "HK_N25.6_b0.942_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0OA==", + "HK_N25.6_b1.009_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0OQ==", + #"HK_N25.6_b1.009_s12": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM1MA==" + }, + "BG": { + "bgA": "RmlsZToxMDE4MDI=" + }, + "PY": { + "PY_N": "RmlsZToxMDE4MDA=" + } + }, + #MORE of these .... + ], + intensity_specs = [ + {"tag": "lite", "measures": ['PGA', 'SA(0.5)', 'SA(1.0)'], "levels": 'logscale(0.005, 4.00, 30)' }, + {"tag": "fixed", "measures": era_measures, "levels": era_levels}, + {"tag": "max10-300", "measures": era_measures, "levels": 'logscale(0.005, 10.00, 300)'}], + vs30s = [ 455, ], + location_codes = ['WLG', 'NZ4', 'NZ34'], + disagg_confs = [{'enabled': False, 'config': {}}, + {'enabled': True, 'config': {}}] ) args_list = [] diff --git a/runzi/configuration/oq_hazard.py b/runzi/configuration/oq_hazard.py index e221ff3d..b5c96527 100644 --- a/runzi/configuration/oq_hazard.py +++ b/runzi/configuration/oq_hazard.py @@ -35,8 +35,22 @@ def build_hazard_tasks(general_task_id: str, subtask_type: SubtaskType, model_ty factory_task = runzi.execute.oq_hazard_task task_factory = factory_class(WORK_PATH, factory_task, task_config_path=WORK_PATH) - for config_archive_id in subtask_arguments["config_archive_ids"]: - for sources in subtask_arguments['source_combos']: + + for (config_archive_id, + #logic_tree_permutation, + intensity_spec, + vs30, + location_code, + disagg_conf + )\ + in itertools.product( + subtask_arguments["config_archive_ids"], + #subtask_arguments['logic_tree_permutations'], + subtask_arguments['intensity_specs'], + subtask_arguments['vs30s'], + subtask_arguments['location_codes'], + subtask_arguments['disagg_confs'] + ): task_count +=1 task_arguments = dict( @@ -45,7 +59,11 @@ def build_hazard_tasks(general_task_id: str, subtask_type: SubtaskType, model_ty config_archive_id = config_archive_id, #File archive object #upstream_general_task=source_gt_id, model_type = model_type.name, - sources = sources + logic_tree_permutations = subtask_arguments['logic_tree_permutations'], + intensity_spec = intensity_spec, + vs30 = vs30, + location_code = location_code, + disagg_conf = disagg_conf, ) print('') diff --git a/runzi/execute/oq_hazard_task.py b/runzi/execute/oq_hazard_task.py index 24047eec..1119489e 100644 --- a/runzi/execute/oq_hazard_task.py +++ b/runzi/execute/oq_hazard_task.py @@ -4,7 +4,7 @@ import os import io -import zipfile +# import zipfile import subprocess import requests import platform @@ -19,8 +19,10 @@ from runzi.automation.scaling.toshi_api import ToshiApi, SubtaskType from nshm_toshi_client.task_relation import TaskRelation from runzi.automation.scaling.local_config import (API_KEY, API_URL, S3_URL, WORK_PATH, SPOOF_HAZARD) -from runzi.automation.scaling.file_utils import download_files, get_output_file_ids, get_output_file_id + from runzi.util import archive +from runzi.execute.util import ( OpenquakeConfig, build_sources_xml, + get_logic_tree_file_ids, get_logic_tree_branches ) logging.basicConfig(level=logging.INFO) @@ -34,34 +36,10 @@ log = logging.getLogger(__name__) -def build_sources_xml(sources_list): - template = """ - - - - - - - - 1.0 - - - - -""" - insert = " " * 16 #coz some indentation is nice - for filepath in sources_list: - insert += f"{filepath}\n" - - template = template.replace('', insert) - return template - def write_sources(xml_str, filepath): with open(filepath, 'w') as mf: mf.write(xml_str) - def explode_config_template(config_info, working_path: str): config_folder = Path(working_path, "config") @@ -155,23 +133,28 @@ def __init__(self, job_args): self._toshi_api = ToshiApi(API_URL, S3_URL, None, with_schema_validation=True, headers=headers) self._task_relation_api = TaskRelation(API_URL, None, with_schema_validation=True, headers=headers) - def unpack_sources(self, ta, source_path): - """download and extract the sources""" + # def unpack_sources(self, ta, source_path): + # """download and extract the sources""" - namelist = [] - for src_name, nrml_id in ta['sources']['nrml_ids'].items(): + # sources = dict() + # # for src_name, nrml_id in ta['sources']['nrml_ids'].items(): - log.info(f"get src : {src_name} {nrml_id}") + # for branch in ta['logic_tree']['branches']: + # for src_name, nrml_id in branch['sources']['source_ids'].items(): + # if nrml_id in sources.keys(): + # continue - gen = get_output_file_id(self._toshi_api, nrml_id) + # log.info(f"get src : {src_name} {nrml_id}") - source_nrml = download_files(self._toshi_api, gen, str(WORK_PATH), overwrite=False) - log.info(f"source_nrml: {source_nrml}") + # gen = get_output_file_id(self._toshi_api, nrml_id) - with zipfile.ZipFile(source_nrml[nrml_id]['filepath'], 'r') as zip_ref: - zip_ref.extractall(source_path) - namelist += zip_ref.namelist() - return namelist + # source_nrml = download_files(self._toshi_api, gen, str(WORK_PATH), overwrite=False) + # log.info(f"source_nrml: {source_nrml}") + + # with zipfile.ZipFile(source_nrml[nrml_id]['filepath'], 'r') as zip_ref: + # zip_ref.extractall(source_path) + # sources[nrml_id] = {'source_name': src_name, 'sources' : zip_ref.namelist()} + # return sources def run(self, task_arguments, job_arguments): @@ -230,22 +213,48 @@ def run(self, task_arguments, job_arguments): config_folder = explode_config_template(config_template_info, work_folder) # sources are the InversionSolutionNRML XML file(s) to include in the sources list + id_list = get_logic_tree_file_ids(task_args['logic_tree_permutations']) + sources_folder = Path(config_folder, 'sources') sources_list = self.unpack_sources(ta, sources_folder) print(f'sources_list: {sources_list}') # now the customised source_models.xml file must be written into the local configuration - src_xml = build_sources_xml(sources_list) + #src_xml = build_sources_xml(sources_list) + #print("ID lists:", list(id_list)) + #TODO build a map whlie downloading the files + + ltbs = [ltb for ltb in get_logic_tree_branches(task_args['logic_tree_permutations'])] + + print("LTB:", len(ltbs), ltbs[0]) + + nrml = build_sources_xml(ltbs, source_file_mapping) + + print(src_xml) write_sources(src_xml, Path(sources_folder, 'source_model.xml')) - # Do the heavy lifting in openquake, passing the config + #prepare the config for itm in config_template_info['meta']: if itm['k'] == "config_filename": config_filename = itm['v'] break config_file = Path(config_folder, config_filename) + def modify_config(config_file, task_arguments): + ta = task_arguments + config = OpenquakeConfig(open(config_file))\ + .set_sites(ta['location_code'])\ + .set_disaggregation(enable = ta['disagg_conf']['enabled'], + settings = ta['disagg_conf']['config'])\ + .set_iml(ta['intensity_spec']['measures'], + ta['intensity_spec']['levels'])\ + .set_vs30(ta['vs30']) + config.write(open(config_file, 'w')) + return + + modify_config(config_file, task_arguments) + logfile = Path(work_folder, f'openquake.log') oq_result = execute_openquake(config_file, logfile, task_id) @@ -262,6 +271,8 @@ def run(self, task_arguments, job_arguments): solution_id = self._toshi_api.openquake_hazard_solution.create_solution( config_id, csv_archive_id, hdf5_archive_id, produced_by=task_id) + # TODO bundle and save the as-run configuration file(s) + # update the OpenquakeHazardTask self._toshi_api.openquake_hazard_task.complete_task( dict(task_id =task_id, diff --git a/runzi/execute/util/__init__.py b/runzi/execute/util/__init__.py new file mode 100644 index 00000000..83a207cd --- /dev/null +++ b/runzi/execute/util/__init__.py @@ -0,0 +1,2 @@ +from .oq_hazard_config import OpenquakeConfig +from .oq_build_sources import get_logic_tree_file_ids, get_logic_tree_branches, build_sources_xml \ No newline at end of file diff --git a/runzi/execute/util/oq_build_sources.py b/runzi/execute/util/oq_build_sources.py new file mode 100644 index 00000000..3db28558 --- /dev/null +++ b/runzi/execute/util/oq_build_sources.py @@ -0,0 +1,227 @@ +#!oq_build_sources.py + +import itertools +import logging +from pathlib import Path + +import zipfile +from lxml import etree +from lxml.builder import ElementMaker # lxml only ! +from runzi.automation.scaling.toshi_api import ToshiApi +from runzi.automation.scaling.file_utils import download_files, get_output_file_ids, get_output_file_id + +log = logging.getLogger(__name__) + +def get_ltb(group): + for gk, gv in group.items(): + yield list(gv.items()) + +def get_logic_tree_file_ids(ltb_groups): + ids = set() + for group in ltb_groups: #List + for sources in get_ltb(group): + for source in sources: + ids.add( source ) + return list(ids) + +def get_logic_tree_branches(ltb_groups): + for group in ltb_groups: #List + for ltb in itertools.product(*get_ltb(group)): + yield ltb + +class SourceModelLoader(): + + def __init__(self, api_url, api_key, s3_url): + headers={"x-api-key":api_key} + self._toshi_api = ToshiApi(api_url, s3_url, None, with_schema_validation=True, headers=headers) + + def unpack_sources(self, logic_tree_branch_permutations, source_path): + """download and extract the sources""" + + sources = dict() + + # ltbs = [x for x in get_logic_tree_file_ids(logic_tree_branch_permutations)] + + # print(ltbs) + # print(len(ltbs)) + + for src_name, nrml_id in get_logic_tree_file_ids(logic_tree_branch_permutations): + if nrml_id in sources.keys(): + continue + + log.info(f"get src : {src_name} {nrml_id}") + + gen = get_output_file_id(self._toshi_api, nrml_id) + + source_nrml = download_files(self._toshi_api, gen, str(WORK_PATH), overwrite=False) + log.info(f"source_nrml: {source_nrml}") + + with zipfile.ZipFile(source_nrml[nrml_id]['filepath'], 'r') as zip_ref: + zip_ref.extractall(source_path) + sources[nrml_id] = {'source_name': src_name, 'sources' : zip_ref.namelist()} + return sources + + +def build_sources_xml(logic_tree_branches, source_file_mapping): + + weight = 1/len(logic_tree_branches) + + E = ElementMaker(namespace="http://openquake.org/xmlns/nrml/0.5", + nsmap={"gml" : "http://www.opengis.net/gml", None:"http://openquake.org/xmlns/nrml/0.5"}) + NRML = E.nrml + LT = E.logicTree + LTBS = E.logicTreeBranchSet + LTBL = E.logicTreeBranchingLevel + LTB = E.logicTreeBranch + UM = E.uncertaintyModel + UW = E.uncertaintyWeight + + ltbs = LTBS(uncertaintyType="sourceModel", branchSetID="BS-NONCE1") + + for branch in logic_tree_branches: + files = "" + branch_name = ", ".join([x[0] for x in branch]) + for source_tuple in branch: + #print(source_tuple) + name, src_id = source_tuple + files += "\t".join(source_file_mapping[src_id]['sources']) + "\t" + ltb = LTB( UM(files), UW(str(weight)), branchID=branch_name) + ltbs.append(ltb) + + nrml = NRML( LT( LTBL( ltbs, branchingLevelID="1" ), logicTreeID = "Combined")) + return etree.tostring(nrml, pretty_print=True).decode() + + + +if __name__ == "__main__": + from runzi.automation.scaling.local_config import (API_KEY, API_URL, S3_URL, WORK_PATH, SPOOF_HAZARD) + + permutations = [ + { + "CR": { + "CR_N2.3_b0.807_C4.2_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4MA==", + "CR_N8.0_b1.115_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Nw==", + "CR_N2.3_b0.807_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3OQ==", + "CR_N3.7_b0.929_C4.2_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4MQ==", + "CR_N3.7_b0.929_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4Mw==", + "CR_N8.0_b1.115_C4.2_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4NQ==" + }, + "HK": { + "HTC_b1.112_N22.6_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2NA==", + "HTC_b0.97_N11.6_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2Ng==", + "HTC_b1.3_N49.4_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3MQ==", + "HTL_b1.112_N22.6_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3MA==", + "HTL_b0.97_N11.6_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Mw==", + "HTL_b1.3_N49.4_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Mg==" + }, + # "BG": { + # "floor_addtot346ave": "FILL IN THE BLANK" + # }, + "PY": { + "P_b0.75, N3.4_C3.9_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Ng==" + } + }, + { + "CR": { + "CR_N8.0_b1.115_C4.3_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3OA==", + "CR_N2.3_b0.807_C4.3_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4Mg==", + "CR_N3.7_b0.929_C4.3_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4NA==" + }, + "HK": { + "HTC_b1.112_N22.6_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2Nw==", + "HTC_b1.3_N49.4_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2NQ==", + "HTC_b0.97_N11.6_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2OA==", + "HTL_b1.112_N22.6_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2OQ==", + "HTL_b1.3_N49.4_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3NQ==", + "HTL_b0.97_N11.6_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3NA==" + }, + # "BG": { + # #"floor_addtot346ave": "FILL IN THE BLANK" + # }, + "PY": { + "P_b0.75_N3.4_C3.9_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Ng==" + } + } + ] + logging.basicConfig(level=logging.INFO) + sources_folder = Path(WORK_PATH, 'sources') + source_file_mapping = SourceModelLoader(API_URL, API_KEY, S3_URL).unpack_sources(permutations, sources_folder) + # print(source_file_mapping) + + ltbs = [ltb for ltb in get_logic_tree_branches(permutations)] + + print("LTB:", len(ltbs), ltbs[0]) + + nrml = build_sources_xml(ltbs, source_file_mapping) + print(nrml) + + + # task_args = { + # "logic_tree_permutations" : [ + # { + # "CR": { + # "CR_N7.8_b_1.111_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0NA==", + # "CR_N7.8_b_1.111_s2": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0NQ==", + # "CR_N3.5_b0.913_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0Ng==", + # #"CR_N3.5_b0.913_s2": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0Nw==" + # }, + # "HK": { + # "HK_N25.6_b0.942_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0OA==", + # "HK_N25.6_b1.009_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0OQ==", + # #"HK_N25.6_b1.009_s12": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM1MA==" + # }, + # "BG": { + # "bgA": "RmlsZToxMDE4MDI=" + # }, + # "PY": { + # "PY_N": "RmlsZToxMDE4MDA=" + # } + # }, + # #MORE of these .... + # ] + # } + + #this is produced by + # source_file_mapping = { + # "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0NA==" : + # { + # 'source_name': 'CR_N7.8_b_1.111_s1', + # 'sources' : ['path/to/fileA', 'path/to/FileA1'] + # }, + # "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0NQ==" : + # { + # 'source_name': 'CR_N7.8_b_1.111_s1', + # 'sources' : ['path/to/fileB', 'path/to/FileB1'] + # }, + # "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0Ng==" : + # { + # 'source_name': 'CR_N7.8_b_1.111_s1', + # 'sources' : ['path/to/fileC', 'path/to/FileC1'] + # }, + # "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0OA==" : + # { + # 'source_name': 'CR_N7.8_b_1.111_s1', + # 'sources' : ['path/to/fileD', 'path/to/FileD1'] + # }, + # "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMDM0OQ==" : + # { + # 'source_name': 'CR_N7.8_b_1.111_s2', + # 'sources' : ['path/to/fileE', 'path/to/FileE1'] + # }, + # "RmlsZToxMDE4MDI=" : + # { + # 'source_name': 'BG', + # 'sources' : ['path/to/fileBG'] + # }, + # "RmlsZToxMDE4MDA=" : + # { + # 'source_name': 'PY', + # 'sources' : ['path/to/filePY'] + # } + + # } + + # id_list = get_logic_tree_file_ids(task_args['logic_tree_permutations']) + # #print("ID lists:", list(id_list)) + # #TODO build a map whlie downloading the files + diff --git a/runzi/execute/util/oq_hazard_config.py b/runzi/execute/util/oq_hazard_config.py index ef4d8d64..cb64e9f8 100644 --- a/runzi/execute/util/oq_hazard_config.py +++ b/runzi/execute/util/oq_hazard_config.py @@ -12,12 +12,25 @@ NZ34 = {"sites_csv": "nz_towns_34.csv"}, GRD1 = {"sites_csv": "NZ_whole_country_10k.csv"}) +# DEFAULT_DISAGG = dict( +# poes_disagg = 0.002, +# mag_bin_width = 0.25, +# distance_bin_width = 1.0, +# coordinate_bin_width = 5.0, +# num_epsilon_bins = 1) + + +#Sanjay new values DEFAULT_DISAGG = dict( - poes_disagg = 0.002, + max_sites_disagg = 1, + poes_disagg = "0.002105 0.000404", mag_bin_width = 0.25, - distance_bin_width = 1.0, - coordinate_bin_width = 5.0, - num_epsilon_bins = 1) + distance_bin_width = 10, + coordinate_bin_width = 1, + num_epsilon_bins = 4, + disagg_outputs = "Mag_Dist Mag_Dist_Eps TRT" + ) + class OpenquakeConfig(): From e8d324c3f7ffd56bc4d03be9fca5315cd1b47e77 Mon Sep 17 00:00:00 2001 From: Chris Chamberlain Date: Thu, 21 Apr 2022 15:18:32 +1200 Subject: [PATCH 11/13] configs as per SRWG #2 --- doc/process/openquake_processes.md | 6 ++++++ docker/runzi-openquake/Dockerfile | 4 ++++ docker/runzi-openquake/README.md | 1 + runzi/execute/util/oq_build_sources.py | 20 ++++++++++---------- runzi/execute/util/oq_hazard_config.py | 8 +++++--- 5 files changed, 26 insertions(+), 13 deletions(-) diff --git a/doc/process/openquake_processes.md b/doc/process/openquake_processes.md index d436c855..e3d055e4 100644 --- a/doc/process/openquake_processes.md +++ b/doc/process/openquake_processes.md @@ -19,5 +19,11 @@ - see `run_oq_convert_solutionhazard.py` below +**Save an arbritrary file** + +``` +python3 runzi/automation/run_save_file_archive.py ~/Downloads/Floor_ADDTOT346ave-base.xml -t BG_Kiran_Floor_ADDTOT346ave-base +``` + diff --git a/docker/runzi-openquake/Dockerfile b/docker/runzi-openquake/Dockerfile index a11cbaab..8b4278f0 100644 --- a/docker/runzi-openquake/Dockerfile +++ b/docker/runzi-openquake/Dockerfile @@ -5,6 +5,10 @@ FROM openquake/engine:latest USER root + +ADD nzshm-oq-engine-main/openquake/hazardlib/scalerel/cscaling.py /opt/openquake/lib/python3.8/site-packages/openquake/hazardlib/scalerel + + RUN apt-get update RUN apt-get install git -y RUN apt-get install nano diff --git a/docker/runzi-openquake/README.md b/docker/runzi-openquake/README.md index 032d3663..c8193a22 100644 --- a/docker/runzi-openquake/README.md +++ b/docker/runzi-openquake/README.md @@ -48,6 +48,7 @@ docker run -u root -it --rm \ ``` docker run -it --rm -u root \ +--memory=52g --memory-swap=52g --oom-kill-disable \ --net=host --env-file environ \ -v $HOME/.aws/credentials:/home/openquake/.aws/credentials:ro \ -v $(pwd)/../../runzi/cli/config/saved_configs:/app/nzshm-runzi/runzi/cli/config/saved_configs \ diff --git a/runzi/execute/util/oq_build_sources.py b/runzi/execute/util/oq_build_sources.py index 3db28558..733a11ca 100644 --- a/runzi/execute/util/oq_build_sources.py +++ b/runzi/execute/util/oq_build_sources.py @@ -80,7 +80,7 @@ def build_sources_xml(logic_tree_branches, source_file_mapping): for branch in logic_tree_branches: files = "" - branch_name = ", ".join([x[0] for x in branch]) + branch_name = "|".join([x[0] for x in branch]) for source_tuple in branch: #print(source_tuple) name, src_id = source_tuple @@ -99,7 +99,7 @@ def build_sources_xml(logic_tree_branches, source_file_mapping): permutations = [ { "CR": { - "CR_N2.3_b0.807_C4.2_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4MA==", + "CR_N2.3_b0.807_C4.2_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4MA==" "CR_N8.0_b1.115_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Nw==", "CR_N2.3_b0.807_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3OQ==", "CR_N3.7_b0.929_C4.2_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4MQ==", @@ -114,11 +114,11 @@ def build_sources_xml(logic_tree_branches, source_file_mapping): "HTL_b0.97_N11.6_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Mw==", "HTL_b1.3_N49.4_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Mg==" }, - # "BG": { - # "floor_addtot346ave": "FILL IN THE BLANK" - # }, + "BG": { + "BG_floor_addtot346ave": "RmlsZToxMDIyMzA=" + }, "PY": { - "P_b0.75, N3.4_C3.9_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Ng==" + "PY_b0.75_N3.4_C3.9_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Ng==" } }, { @@ -135,11 +135,11 @@ def build_sources_xml(logic_tree_branches, source_file_mapping): "HTL_b1.3_N49.4_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3NQ==", "HTL_b0.97_N11.6_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3NA==" }, - # "BG": { - # #"floor_addtot346ave": "FILL IN THE BLANK" - # }, + "BG": { + "BG_floor_addtot346ave": "RmlsZToxMDIyMzA=" + }, "PY": { - "P_b0.75_N3.4_C3.9_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Ng==" + "PY_b0.75_N3.4_C3.9_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Ng==" } } ] diff --git a/runzi/execute/util/oq_hazard_config.py b/runzi/execute/util/oq_hazard_config.py index cb64e9f8..4e9ff0b5 100644 --- a/runzi/execute/util/oq_hazard_config.py +++ b/runzi/execute/util/oq_hazard_config.py @@ -125,14 +125,16 @@ def write(self, tofile): nc = OpenquakeConfig(sample)\ .set_sites('NZ4')\ - .set_disaggregation(True, {"num_rlz_disagg": 0}) + .set_disaggregation(False, {"num_rlz_disagg": 0}) measures = ['PGA', 'SA(0.5)'] levels0 = [0.01, 0.02, 0.04, 0.06, 0.08, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2, 2.4, 2.6, 2.8, 3.0, 3.5, 4, 4.5, 5.0] levels1 = 'logscale(0.005, 4.00, 30)' + _4_sites_levels = [0.01, 0.02, 0.04, 0.06, 0.08, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2, 2.4, 2.6, 2.8, 3.0, 3.5, 4, 4.5, 5.0] + _4_sites_measures = ['PGA',"SA(0.1)","SA(0.2)","SA(0.3)","SA(0.4)","SA(0.5)","SA(0.7)","SA(1.0)","SA(1.5)","SA(2.0)","SA(3.0)","SA(4.0)","SA(5.0)"] - nc.set_iml(measures, levels1) - nc.set_vs30(355) + nc.set_iml(_4_sites_measures, _4_sites_levels) + nc.set_vs30(250) out = io.StringIO() #aother fake file nc.write(out) From 6021862056dded9ee5881824a6230141438b0cad Mon Sep 17 00:00:00 2001 From: Chris Chamberlain Date: Tue, 26 Apr 2022 13:38:47 +1200 Subject: [PATCH 12/13] WIP on large config --- runzi/execute/util/oq_build_sources.py | 143 ++++++++++++++++++++++--- 1 file changed, 126 insertions(+), 17 deletions(-) diff --git a/runzi/execute/util/oq_build_sources.py b/runzi/execute/util/oq_build_sources.py index 733a11ca..f9f55196 100644 --- a/runzi/execute/util/oq_build_sources.py +++ b/runzi/execute/util/oq_build_sources.py @@ -96,53 +96,158 @@ def build_sources_xml(logic_tree_branches, source_file_mapping): if __name__ == "__main__": from runzi.automation.scaling.local_config import (API_KEY, API_URL, S3_URL, WORK_PATH, SPOOF_HAZARD) + # permutations = [ + # { + # "CR": { + # "CR_N2.3_b0.807_C4.2_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4MA==" + # "CR_N8.0_b1.115_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Nw==", + # "CR_N2.3_b0.807_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3OQ==", + # "CR_N3.7_b0.929_C4.2_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4MQ==", + # "CR_N3.7_b0.929_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4Mw==", + # "CR_N8.0_b1.115_C4.2_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4NQ==" + # }, + # "HK": { + # "HTC_b1.112_N22.6_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2NA==", + # "HTC_b0.97_N11.6_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2Ng==", + # "HTC_b1.3_N49.4_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3MQ==", + # "HTL_b1.112_N22.6_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3MA==", + # "HTL_b0.97_N11.6_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Mw==", + # "HTL_b1.3_N49.4_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Mg==" + # }, + # "BG": { + # "BG_floor_addtot346ave": "RmlsZToxMDIyMzA=" + # }, + # "PY": { + # "PY_b0.75_N3.4_C3.9_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Ng==" + # } + # }, + # { + # "CR": { + # "CR_N8.0_b1.115_C4.3_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3OA==", + # "CR_N2.3_b0.807_C4.3_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4Mg==", + # "CR_N3.7_b0.929_C4.3_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4NA==" + # }, + # "HK": { + # "HTC_b1.112_N22.6_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2Nw==", + # "HTC_b1.3_N49.4_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2NQ==", + # "HTC_b0.97_N11.6_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2OA==", + # "HTL_b1.112_N22.6_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2OQ==", + # "HTL_b1.3_N49.4_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3NQ==", + # "HTL_b0.97_N11.6_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3NA==" + # }, + # "BG": { + # "BG_floor_addtot346ave": "RmlsZToxMDIyMzA=" + # }, + # "PY": { + # "PY_b0.75_N3.4_C3.9_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Ng==" + # } + # } + # ] + + + permutations = [ { "CR": { - "CR_N2.3_b0.807_C4.2_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4MA==" + "CR_N8.0_b1.115_C4.3_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3OA==", + "CR_N2.3_b0.807_C4.2_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4MA==", "CR_N8.0_b1.115_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Nw==", "CR_N2.3_b0.807_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3OQ==", "CR_N3.7_b0.929_C4.2_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4MQ==", "CR_N3.7_b0.929_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4Mw==", + "CR_N2.3_b0.807_C4.3_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4Mg==", + "CR_N3.7_b0.929_C4.3_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4NA==", "CR_N8.0_b1.115_C4.2_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4NQ==" }, "HK": { "HTC_b1.112_N22.6_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2NA==", + "HTC_b1.112_N22.6_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2Nw==", + "HTC_b1.3_N49.4_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2NQ==", "HTC_b0.97_N11.6_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2Ng==", + "HTC_b0.97_N11.6_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2OA==", "HTC_b1.3_N49.4_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3MQ==", + "HTL_b1.112_N22.6_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2OQ==", "HTL_b1.112_N22.6_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3MA==", "HTL_b0.97_N11.6_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Mw==", - "HTL_b1.3_N49.4_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Mg==" + "HTL_b1.3_N49.4_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3NQ==", + "HTL_b1.3_N49.4_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Mg==", + "HTL_b0.97_N11.6_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3NA==" }, "BG": { - "BG_floor_addtot346ave": "RmlsZToxMDIyMzA=" + "floor_addtot346ave": "RmlsZToxMDIyMzA=" }, "PY": { - "PY_b0.75_N3.4_C3.9_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Ng==" + "P_b0.75_N3.4_C3.9_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Ng==" } }, { "CR": { - "CR_N8.0_b1.115_C4.3_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3OA==", - "CR_N2.3_b0.807_C4.3_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4Mg==", - "CR_N3.7_b0.929_C4.3_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4NA==" + "CR_N8.0_b1.115_C4.3_s0.51": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxMg==", + "CR_N8.0_b1.115_C4.1_s0.51": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxNQ==", + "CR_N2.3_b0.807_C4.2_s0.51": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxNg==", + "CR_N2.3_b0.807_C4.1_s0.51": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxMw==", + "CR_N3.7_b0.929_C4.1_s0.51": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxNA==", + "CR_N3.7_b0.929_C4.2_s0.51": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxNw==", + "CR_N2.3_b0.807_C4.3_s0.51": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxOQ==", + "CR_N8.0_b1.115_C4.2_s0.51": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyMA==", + "CR_N3.7_b0.929_C4.3_s0.51": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyMw==" }, "HK": { - "HTC_b1.112_N22.6_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2Nw==", - "HTC_b1.3_N49.4_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2NQ==", - "HTC_b0.97_N11.6_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2OA==", - "HTL_b1.112_N22.6_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2OQ==", - "HTL_b1.3_N49.4_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3NQ==", - "HTL_b0.97_N11.6_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3NA==" + "HTC_b1.112_N22.6_C4_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4Ng==", + "HTC_b1.3_N49.4_C4.1_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4OA==", + "HTC_b1.112_N22.6_C4.1_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5MA==", + "HTC_b0.97_N11.6_C4_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4Nw==", + "HTC_b0.97_N11.6_C4.1_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4OQ==", + "HTL_b1.112_N22.6_C4.1_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5MQ==", + "HTC_b1.3_N49.4_C4_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5Mg==", + "HTL_b1.112_N22.6_C4_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5NQ==", + "HTL_b0.97_N11.6_C4_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5Ng==", + "HTL_b1.3_N49.4_C4_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5Mw==", + "HTL_b1.3_N49.4_C4.1_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5NA==", + "HTL_b0.97_N11.6_C4.1_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5Nw==" }, "BG": { - "BG_floor_addtot346ave": "RmlsZToxMDIyMzA=" + "floor_addtot346ave": "RmlsZToxMDIyMzA=" }, "PY": { - "PY_b0.75_N3.4_C3.9_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Ng==" + "P_b0.75_N3.4_C3.9_s0.61": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxMA==" + } + }, + { + "CR": { + "CR_N2.3_b0.807_C4.2_s1.62": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxOA==", + "CR_N8.0_b1.115_C4.3_s1.62": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyMQ==", + "CR_N8.0_b1.115_C4.1_s1.62": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyMg==", + "CR_N3.7_b0.929_C4.2_s1.62": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyNA==", + "CR_N2.3_b0.807_C4.1_s1.62": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyNg==", + "CR_N3.7_b0.929_C4.1_s1.62": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyOA==", + "CR_N3.7_b0.929_C4.3_s1.62": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyNQ==", + "CR_N2.3_b0.807_C4.3_s1.62": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyNw==", + "CR_N8.0_b1.115_C4.2_s1.62": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyOQ==" + }, + "HK": { + "HTC_b1.112_N22.6_C4_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5OQ==", + "HTC_b1.3_N49.4_C4.1_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwMQ==", + "HTC_b1.112_N22.6_C4.1_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwMw==", + "HTC_b0.97_N11.6_C4_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5OA==", + "HTC_b0.97_N11.6_C4.1_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwMA==", + "HTL_b1.112_N22.6_C4.1_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwMg==", + "HTC_b1.3_N49.4_C4_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwNA==", + "HTL_b1.112_N22.6_C4_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwNw==", + "HTL_b0.97_N11.6_C4_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwOA==", + "HTL_b1.3_N49.4_C4_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwNQ==", + "HTL_b1.3_N49.4_C4.1_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwNg==", + "HTL_b0.97_N11.6_C4.1_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwOQ==" + }, + "BG": { + "floor_addtot346ave": "RmlsZToxMDIyMzA=" + }, + "PY": { + "P_b0.75_N3.4_C3.9_s1.34": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxMQ==" } } ] + logging.basicConfig(level=logging.INFO) sources_folder = Path(WORK_PATH, 'sources') source_file_mapping = SourceModelLoader(API_URL, API_KEY, S3_URL).unpack_sources(permutations, sources_folder) @@ -152,8 +257,12 @@ def build_sources_xml(logic_tree_branches, source_file_mapping): print("LTB:", len(ltbs), ltbs[0]) - nrml = build_sources_xml(ltbs, source_file_mapping) - print(nrml) + nrml = build_sources_xml(ltbs[:100], source_file_mapping) + + with open("source_model.xml", 'w') as f: + f.write(nrml) + + print('Done!') # task_args = { From a068c7ae050646f30705b3e3afbf107390bb5820 Mon Sep 17 00:00:00 2001 From: Chris Chamberlain Date: Wed, 4 May 2022 11:01:50 +1200 Subject: [PATCH 13/13] recent config updates for oq hazard --- runzi/execute/util/oq_build_sources.py | 514 ++++++++++++++++++++----- 1 file changed, 415 insertions(+), 99 deletions(-) diff --git a/runzi/execute/util/oq_build_sources.py b/runzi/execute/util/oq_build_sources.py index f9f55196..bc34205e 100644 --- a/runzi/execute/util/oq_build_sources.py +++ b/runzi/execute/util/oq_build_sources.py @@ -12,9 +12,11 @@ log = logging.getLogger(__name__) -def get_ltb(group): - for gk, gv in group.items(): - yield list(gv.items()) + +# def get_ltb(group): +# """old dict-kets style config""" +# for gk, gv in group.items(): +# yield list(gv.items()) def get_logic_tree_file_ids(ltb_groups): ids = set() @@ -24,6 +26,11 @@ def get_logic_tree_file_ids(ltb_groups): ids.add( source ) return list(ids) +def get_ltb(group): + """NEW object-syle config""" + for obj in group['permute']: + yield [(source['tag'], source['toshi_id']) for source in obj['members']] + def get_logic_tree_branches(ltb_groups): for group in ltb_groups: #List for ltb in itertools.product(*get_ltb(group)): @@ -44,6 +51,7 @@ def unpack_sources(self, logic_tree_branch_permutations, source_path): # print(ltbs) # print(len(ltbs)) + print(logic_tree_branch_permutations) for src_name, nrml_id in get_logic_tree_file_ids(logic_tree_branch_permutations): if nrml_id in sources.keys(): @@ -59,6 +67,7 @@ def unpack_sources(self, logic_tree_branch_permutations, source_path): with zipfile.ZipFile(source_nrml[nrml_id]['filepath'], 'r') as zip_ref: zip_ref.extractall(source_path) sources[nrml_id] = {'source_name': src_name, 'sources' : zip_ref.namelist()} + return sources @@ -99,7 +108,7 @@ def build_sources_xml(logic_tree_branches, source_file_mapping): # permutations = [ # { # "CR": { - # "CR_N2.3_b0.807_C4.2_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4MA==" + # "CR_N2.3_b0.807_C4.2_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4MA==", # "CR_N8.0_b1.115_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Nw==", # "CR_N2.3_b0.807_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3OQ==", # "CR_N3.7_b0.929_C4.2_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4MQ==", @@ -144,110 +153,417 @@ def build_sources_xml(logic_tree_branches, source_file_mapping): # } # ] + # #SLT001 + # permutations = [ + # { + # "tag": "core model", "weight": 1.0, + # "permute" : [ + # { + # "group": "HIK", + # "members" : [ + # {"tag": "b0.97_N11.6_C4.0_s1", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2Ng=="} + # ] + # } + # ] + # } + # ] + + + # #SLT002 + # permutations = [ + # { + # "tag": "core model", "weight": 0.5, + # "permute" : [ + # { "group": "HIK", + # "members" : [ + # {"tag": "b0.97_N11.6_C4.0_s1", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2Ng=="} + # ] + # } + # ] + # }, + + # { + # "tag": "C = 4.1", "weight": 0.5, + # "permute" : [ + # { "group": "HIK", + # "members" : [ + # {"tag": "b0.97_N11.6_C4.1_s1", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2OA=="} + # ] + # } + # ] + # } + # ] + + + # #SLT003 + # permutations = [ + + # { + # "tag": "core model", "weight": 0.166, + # "permute" : [ + # { "group": "HIK", + # "members" : [ + # {"tag": "b0.97_N11.6_C4.0_s1", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2Ng=="} + # ] + # } + # ] + # }, + + # { + # "tag": "core scaled down", "weight": 0.166, + # "permute" : [ + # { "group": "HIK", + # "members" : [ + # {"tag": "b0.97_N11.6_C4.0_s0.54", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4Nw=="} + # ] + # } + # ] + # }, + + # { + # "tag": "core scaled up", "weight": 0.166, + # "permute" : [ + + # { "group": "HIK", + # "members" : [ + # {"tag": "b0.97_N11.6_C4.0_s1.43", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5OA=="} + # ] + # } + # ] + # }, + + # { + # "tag": "b = 1.112", "weight": 0.166, + # "permute" : [ + # { "group": "HIK", + # "members" : [ + # {"tag": "b1.112_N22.6_C4_s1", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2NA=="} + # ] + # } + # ] + # }, + + # { + # "tag": "b = 1.112 scaled down", "weight": 0.166, + # "permute" : [ + # { "group": "HIK", + # "members" : [ + # {"tag": "b1.112_N22.6_C4_s0.54", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4Ng=="} + # ] + # } + # ] + # }, + # { + # "tag": "b = 1.112 scaled up", "weight": 0.17, + # "permute" : [ + + # { "group": "HIK", + # "members" : [ + # {"tag": "b1.112_N22.6_C4_s1.43", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5OQ=="} + # ] + # } + # ] + # } + # ] + + # #SLT004b + # permutations = [ + + # { + # "tag": "core model", "weight": 0.0833, + # "permute" : [ + # { "group": "HIK", + # "members" : [ + # {"tag": "b0.97_N11.6_C4.0_s1", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2Ng=="} + # ] + # } + # ] + # }, + + # { + # "tag": "core scaled down", "weight": 0.0833, + # "permute" : [ + # { "group": "HIK", + # "members" : [ + # {"tag": "b0.97_N11.6_C4.0_s0.54", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4Nw=="} + # ] + # } + # ] + # }, + + # { + # "tag": "core scaled up", "weight": 0.0833, + # "permute" : [ + + # { "group": "HIK", + # "members" : [ + # {"tag": "b0.97_N11.6_C4.0_s1.43", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5OA=="} + # ] + # } + # ] + # }, + + # { + # "tag": "b = 1.112", "weight": 0.0833, + # "permute" : [ + # { "group": "HIK", + # "members" : [ + # {"tag": "b1.112_N22.6_C4_s1", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2NA=="} + # ] + # } + # ] + # }, + + # { + # "tag": "b = 1.112 scaled down", "weight": 0.0833, + # "permute" : [ + # { "group": "HIK", + # "members" : [ + # {"tag": "b1.112_N22.6_C4_s0.54", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4Ng=="} + # ] + # } + # ] + # }, + + # { + # "tag": "b = 1.112 scaled up", "weight": 0.0833, + # "permute" : [ + + # { "group": "HIK", + # "members" : [ + # {"tag": "b1.112_N22.6_C4_s1.43", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5OQ=="} + # ] + # } + # ] + # }, + + + # { + # "tag": "core model, C = 4.1", "weight": 0.0833, + # "permute" : [ + # { "group": "HIK", + # "members" : [ + # {"tag": "b0.97_N11.6_C4.1_s1", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2OA=="} + # ] + # } + # ] + # }, + + # { + # "tag": "core scaled down, C = 4.1", "weight": 0.0833, + # "permute" : [ + # { "group": "HIK", + # "members" : [ + # {"tag": "b0.97_N11.6_C4.1_s0.54", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4OQ=="} + # ] + # } + # ] + # }, + + # { + # "tag": "core scaled up, C = 4.1", "weight": 0.0833, + # "permute" : [ + + # { "group": "HIK", + # "members" : [ + # {"tag": "b0.97_N11.6_C4.1_s1.43", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwMA=="} + # ] + # } + # ] + # }, + + # { + # "tag": "b = 1.112, C = 4.1", "weight": 0.0833, + # "permute" : [ + # { "group": "HIK", + # "members" : [ + # {"tag": "b1.112_N22.6_C4.1_s1", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2Nw=="} + # ] + # } + # ] + # }, + + # { + # "tag": "b = 1.112 scaled down, C = 4.1", "weight": 0.0833, + # "permute" : [ + # { "group": "HIK", + # "members" : [ + # {"tag": "b1.112_N22.6_C4.1_s0.54", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5MA=="} + # ] + # } + # ] + # }, + + # { + # "tag": "b = 1.112 scaled up, C = 4.1", "weight": 0.0837, + # "permute" : [ + + # { "group": "HIK", + # "members" : [ + # {"tag": "b1.112_N22.6_C4.1_s1.43", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwMw=="} + # ] + # } + # ] + # } + + # ] + + + #BIG + # permutations = [ + # { + # "CR": { + # "CR_N8.0_b1.115_C4.3_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3OA==", + # "CR_N2.3_b0.807_C4.2_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4MA==", + # "CR_N8.0_b1.115_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Nw==", + # "CR_N2.3_b0.807_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3OQ==", + # "CR_N3.7_b0.929_C4.2_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4MQ==", + # "CR_N3.7_b0.929_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4Mw==", + # "CR_N2.3_b0.807_C4.3_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4Mg==", + # "CR_N3.7_b0.929_C4.3_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4NA==", + # "CR_N8.0_b1.115_C4.2_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4NQ==" + # }, + # "HK": { + # "HTC_b1.112_N22.6_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2NA==", + # "HTC_b1.112_N22.6_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2Nw==", + # "HTC_b1.3_N49.4_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2NQ==", + # "HTC_b0.97_N11.6_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2Ng==", + # "HTC_b0.97_N11.6_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2OA==", + # "HTC_b1.3_N49.4_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3MQ==", + # "HTL_b1.112_N22.6_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2OQ==", + # "HTL_b1.112_N22.6_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3MA==", + # "HTL_b0.97_N11.6_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Mw==", + # "HTL_b1.3_N49.4_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3NQ==", + # "HTL_b1.3_N49.4_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Mg==", + # "HTL_b0.97_N11.6_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3NA==" + # }, + # "BG": { + # "floor_addtot346ave": "RmlsZToxMDIyMzA=" + # }, + # "PY": { + # "P_b0.75_N3.4_C3.9_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Ng==" + # } + # }, + # { + # "CR": { + # "CR_N8.0_b1.115_C4.3_s0.51": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxMg==", + # "CR_N8.0_b1.115_C4.1_s0.51": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxNQ==", + # "CR_N2.3_b0.807_C4.2_s0.51": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxNg==", + # "CR_N2.3_b0.807_C4.1_s0.51": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxMw==", + # "CR_N3.7_b0.929_C4.1_s0.51": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxNA==", + # "CR_N3.7_b0.929_C4.2_s0.51": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxNw==", + # "CR_N2.3_b0.807_C4.3_s0.51": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxOQ==", + # "CR_N8.0_b1.115_C4.2_s0.51": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyMA==", + # "CR_N3.7_b0.929_C4.3_s0.51": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyMw==" + # }, + # "HK": { + # "HTC_b1.112_N22.6_C4_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4Ng==", + # "HTC_b1.3_N49.4_C4.1_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4OA==", + # "HTC_b1.112_N22.6_C4.1_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5MA==", + # "HTC_b0.97_N11.6_C4_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4Nw==", + # "HTC_b0.97_N11.6_C4.1_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4OQ==", + # "HTL_b1.112_N22.6_C4.1_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5MQ==", + # "HTC_b1.3_N49.4_C4_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5Mg==", + # "HTL_b1.112_N22.6_C4_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5NQ==", + # "HTL_b0.97_N11.6_C4_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5Ng==", + # "HTL_b1.3_N49.4_C4_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5Mw==", + # "HTL_b1.3_N49.4_C4.1_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5NA==", + # "HTL_b0.97_N11.6_C4.1_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5Nw==" + # }, + # "BG": { + # "floor_addtot346ave": "RmlsZToxMDIyMzA=" + # }, + # "PY": { + # "P_b0.75_N3.4_C3.9_s0.61": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxMA==" + # } + # }, + # { + # "CR": { + # "CR_N2.3_b0.807_C4.2_s1.62": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxOA==", + # "CR_N8.0_b1.115_C4.3_s1.62": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyMQ==", + # "CR_N8.0_b1.115_C4.1_s1.62": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyMg==", + # "CR_N3.7_b0.929_C4.2_s1.62": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyNA==", + # "CR_N2.3_b0.807_C4.1_s1.62": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyNg==", + # "CR_N3.7_b0.929_C4.1_s1.62": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyOA==", + # "CR_N3.7_b0.929_C4.3_s1.62": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyNQ==", + # "CR_N2.3_b0.807_C4.3_s1.62": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyNw==", + # "CR_N8.0_b1.115_C4.2_s1.62": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyOQ==" + # }, + # "HK": { + # "HTC_b1.112_N22.6_C4_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5OQ==", + # "HTC_b1.3_N49.4_C4.1_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwMQ==", + # "HTC_b1.112_N22.6_C4.1_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwMw==", + # "HTC_b0.97_N11.6_C4_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5OA==", + # "HTC_b0.97_N11.6_C4.1_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwMA==", + # "HTL_b1.112_N22.6_C4.1_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwMg==", + # "HTC_b1.3_N49.4_C4_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwNA==", + # "HTL_b1.112_N22.6_C4_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwNw==", + # "HTL_b0.97_N11.6_C4_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwOA==", + # "HTL_b1.3_N49.4_C4_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwNQ==", + # "HTL_b1.3_N49.4_C4.1_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwNg==", + # "HTL_b0.97_N11.6_C4.1_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwOQ==" + # }, + # "BG": { + # "floor_addtot346ave": "RmlsZToxMDIyMzA=" + # }, + # "PY": { + # "P_b0.75_N3.4_C3.9_s1.34": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxMQ==" + # } + # } + # ] + + + #27_TAG_CONFIG permutations = [ { - "CR": { - "CR_N8.0_b1.115_C4.3_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3OA==", - "CR_N2.3_b0.807_C4.2_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4MA==", - "CR_N8.0_b1.115_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Nw==", - "CR_N2.3_b0.807_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3OQ==", - "CR_N3.7_b0.929_C4.2_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4MQ==", - "CR_N3.7_b0.929_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4Mw==", - "CR_N2.3_b0.807_C4.3_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4Mg==", - "CR_N3.7_b0.929_C4.3_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4NA==", - "CR_N8.0_b1.115_C4.2_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4NQ==" + "tag": "all rate combinations", "weight": 1.0, + "permute" : [ + { "group": "HIK", + "members" : [ + {"tag": "HTC_b0.957_N16.5_C4.1_s1", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjk0OQ=="}, + {"tag": "HTC_b1.078_N22.8_C4.1_s1", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjk1MA=="}, + {"tag": "HTL_b0.957_N16.5_C4.1_s1", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjk1MQ=="}, + {"tag": "HTL_b1.078_N22.8_C4.1_s1", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjk1Mg=="}, + {"tag": "HTC_b0.957_N16.5_C4.1_s0.54", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjk1Mw=="}, + {"tag": "HTC_b0.957_N16.5_C4.1_s1.43", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjk1NA=="}, + {"tag": "HTC_b1.078_N22.8_C4.1_s0.54", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjk1NQ=="}, + {"tag": "HTC_b1.078_N22.8_C4.1_s1.43", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjk1Ng=="}, + {"tag": "HTL_b0.957_N16.5_C4.1_s0.54", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjk1Nw=="}, + {"tag": "HTL_b0.957_N16.5_C4.1_s1.43", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjk1OA=="}, + {"tag": "HTL_b1.078_N22.8_C4.1_s0.54", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjk1OQ=="}, + {"tag": "HTL_b1.078_N22.8_C4.1_s1.43", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjk2MA=="} + ] }, - "HK": { - "HTC_b1.112_N22.6_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2NA==", - "HTC_b1.112_N22.6_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2Nw==", - "HTC_b1.3_N49.4_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2NQ==", - "HTC_b0.97_N11.6_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2Ng==", - "HTC_b0.97_N11.6_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2OA==", - "HTC_b1.3_N49.4_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3MQ==", - "HTL_b1.112_N22.6_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE2OQ==", - "HTL_b1.112_N22.6_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3MA==", - "HTL_b0.97_N11.6_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Mw==", - "HTL_b1.3_N49.4_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3NQ==", - "HTL_b1.3_N49.4_C4_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Mg==", - "HTL_b0.97_N11.6_C4.1_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3NA==" - }, - "BG": { - "floor_addtot346ave": "RmlsZToxMDIyMzA=" - }, - "PY": { - "P_b0.75_N3.4_C3.9_s1": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Ng==" - } - }, - { - "CR": { - "CR_N8.0_b1.115_C4.3_s0.51": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxMg==", - "CR_N8.0_b1.115_C4.1_s0.51": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxNQ==", - "CR_N2.3_b0.807_C4.2_s0.51": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxNg==", - "CR_N2.3_b0.807_C4.1_s0.51": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxMw==", - "CR_N3.7_b0.929_C4.1_s0.51": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxNA==", - "CR_N3.7_b0.929_C4.2_s0.51": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxNw==", - "CR_N2.3_b0.807_C4.3_s0.51": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxOQ==", - "CR_N8.0_b1.115_C4.2_s0.51": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyMA==", - "CR_N3.7_b0.929_C4.3_s0.51": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyMw==" + { "group": "PUY", + "members" : [ + {"tag": "P_b0.75_N3.4_C3.9_s1", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3Ng=="} + ] }, - "HK": { - "HTC_b1.112_N22.6_C4_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4Ng==", - "HTC_b1.3_N49.4_C4.1_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4OA==", - "HTC_b1.112_N22.6_C4.1_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5MA==", - "HTC_b0.97_N11.6_C4_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4Nw==", - "HTC_b0.97_N11.6_C4.1_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4OQ==", - "HTL_b1.112_N22.6_C4.1_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5MQ==", - "HTC_b1.3_N49.4_C4_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5Mg==", - "HTL_b1.112_N22.6_C4_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5NQ==", - "HTL_b0.97_N11.6_C4_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5Ng==", - "HTL_b1.3_N49.4_C4_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5Mw==", - "HTL_b1.3_N49.4_C4.1_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5NA==", - "HTL_b0.97_N11.6_C4.1_s0.54": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5Nw==" - }, - "BG": { - "floor_addtot346ave": "RmlsZToxMDIyMzA=" - }, - "PY": { - "P_b0.75_N3.4_C3.9_s0.61": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxMA==" - } - }, - { - "CR": { - "CR_N2.3_b0.807_C4.2_s1.62": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxOA==", - "CR_N8.0_b1.115_C4.3_s1.62": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyMQ==", - "CR_N8.0_b1.115_C4.1_s1.62": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyMg==", - "CR_N3.7_b0.929_C4.2_s1.62": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyNA==", - "CR_N2.3_b0.807_C4.1_s1.62": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyNg==", - "CR_N3.7_b0.929_C4.1_s1.62": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyOA==", - "CR_N3.7_b0.929_C4.3_s1.62": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyNQ==", - "CR_N2.3_b0.807_C4.3_s1.62": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyNw==", - "CR_N8.0_b1.115_C4.2_s1.62": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyOQ==" + { "group": "CRU", + "members" : [ + {"tag": "CR_N8.0_b1.115_C4.3_s1", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE3OA=="}, + {"tag": "CR_N2.3_b0.807_C4.2_s1", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4MA=="}, + {"tag": "CR_N3.7_b0.929_C4.2_s1", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE4MQ=="}, + {"tag": "CR_N8.0_b1.115_C4.3_s0.51", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxMg=="}, + {"tag": "CR_N2.3_b0.807_C4.2_s0.51", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxNg=="}, + {"tag": "CR_N3.7_b0.929_C4.2_s0.51", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxNw=="}, + {"tag": "CR_N2.3_b0.807_C4.2_s1.62", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxOA=="}, + {"tag": "CR_N8.0_b1.115_C4.3_s1.62", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyMQ=="}, + {"tag": "CR_N3.7_b0.929_C4.2_s1.62", "toshi_id": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIyNA=="} + ] }, - "HK": { - "HTC_b1.112_N22.6_C4_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5OQ==", - "HTC_b1.3_N49.4_C4.1_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwMQ==", - "HTC_b1.112_N22.6_C4.1_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwMw==", - "HTC_b0.97_N11.6_C4_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjE5OA==", - "HTC_b0.97_N11.6_C4.1_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwMA==", - "HTL_b1.112_N22.6_C4.1_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwMg==", - "HTC_b1.3_N49.4_C4_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwNA==", - "HTL_b1.112_N22.6_C4_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwNw==", - "HTL_b0.97_N11.6_C4_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwOA==", - "HTL_b1.3_N49.4_C4_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwNQ==", - "HTL_b1.3_N49.4_C4.1_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwNg==", - "HTL_b0.97_N11.6_C4.1_s1.43": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIwOQ==" - }, - "BG": { - "floor_addtot346ave": "RmlsZToxMDIyMzA=" - }, - "PY": { - "P_b0.75_N3.4_C3.9_s1.34": "SW52ZXJzaW9uU29sdXRpb25Ocm1sOjEwMjIxMQ==" - } + { "group": "BG", + "members" : [ + {"tag": "floor_addtot346ave", "toshi_id": "RmlsZToxMDIyMzA="} + ] + } + ] } ] + logging.basicConfig(level=logging.INFO) sources_folder = Path(WORK_PATH, 'sources') source_file_mapping = SourceModelLoader(API_URL, API_KEY, S3_URL).unpack_sources(permutations, sources_folder) @@ -257,7 +573,7 @@ def build_sources_xml(logic_tree_branches, source_file_mapping): print("LTB:", len(ltbs), ltbs[0]) - nrml = build_sources_xml(ltbs[:100], source_file_mapping) + nrml = build_sources_xml(ltbs, source_file_mapping) with open("source_model.xml", 'w') as f: f.write(nrml)