From 9a5f78caff7114f42e8f69ecf94534f7457881c8 Mon Sep 17 00:00:00 2001 From: "Trine Alsos (CCN GGP GGP8)" Date: Wed, 11 Sep 2024 13:39:39 +0200 Subject: [PATCH 1/6] replace dates with include file in testdata --- tests/testdata_gravity/grav_subs_maps.yml | 5 ++--- tests/testdata_gravity/grav_subs_points.yml | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/testdata_gravity/grav_subs_maps.yml b/tests/testdata_gravity/grav_subs_maps.yml index 88378bb77..228a8309d 100644 --- a/tests/testdata_gravity/grav_subs_maps.yml +++ b/tests/testdata_gravity/grav_subs_maps.yml @@ -1,12 +1,11 @@ # Configuration file for use with grav_subs_maps.py input: - diffdates: - - [2020-07-01, 2018-01-01] + diffdates: !include_from global_variables.yml::global.dates.GRAVITY_DIFFDATES seabed_map: seabed.gri calculations: poisson_ratio: 0.45 coarsening: 8 phases: ["gas", "oil","water", "total"] - + diff --git a/tests/testdata_gravity/grav_subs_points.yml b/tests/testdata_gravity/grav_subs_points.yml index 57b8ac104..ae5709dd6 100644 --- a/tests/testdata_gravity/grav_subs_points.yml +++ b/tests/testdata_gravity/grav_subs_points.yml @@ -1,8 +1,7 @@ # Configuration file for use with grav_subs_points.py input: - diffdates: - - [2020-07-01, 2018-01-01] + diffdates: !include_from global_variables.yml::global.dates.GRAVITY_DIFFDATES stations: grav: @@ -13,4 +12,4 @@ stations: calculations: poisson_ratio: 0.45 phases: ["gas", "oil","water", "total"] - + From 3ac99c56e1f7613eb37949407026fc70bdde2500 Mon Sep 17 00:00:00 2001 From: "Trine Alsos (CCN GGP GGP8)" Date: Wed, 11 Sep 2024 13:44:04 +0200 Subject: [PATCH 2/6] update tests, remove root option, add include dates --- tests/test_grav_subs_maps.py | 26 +------------------------- tests/test_grav_subs_points.py | 21 --------------------- 2 files changed, 1 insertion(+), 46 deletions(-) diff --git a/tests/test_grav_subs_maps.py b/tests/test_grav_subs_maps.py index 96fc29ae7..4450e7c84 100644 --- a/tests/test_grav_subs_maps.py +++ b/tests/test_grav_subs_maps.py @@ -4,7 +4,6 @@ from pathlib import Path import pytest -import yaml from pydantic import ValidationError from subscript.grav_subs_maps import grav_subs_maps @@ -13,28 +12,6 @@ TESTDATA = Path(__file__).absolute().parent / "testdata_gravity" -def test_prepend_root_path(): - """Test that we need to prepend with root-path""" - cfg_file = TESTDATA / "grav_subs_maps.yml" - - cfg = yaml.safe_load(cfg_file.read_text(encoding="utf8")) - - with pytest.raises(ValidationError): - GravMapsConfig(**cfg) - - cfg_with_rootpath = grav_subs_maps.prepend_root_path_to_relative_files( - cfg, TESTDATA - ) - GravMapsConfig(**cfg_with_rootpath) - - # When root-path is prepended (with an absolute part) it should not - # matter if we reapply: - cfg_with_double_rootpath = grav_subs_maps.prepend_root_path_to_relative_files( - cfg_with_rootpath, TESTDATA - ) - GravMapsConfig(**cfg_with_double_rootpath) - - @pytest.fixture(name="res_data") def fixture_res_data(tmp_path): """Prepare a data directory with Eclipse binary output""" @@ -198,8 +175,7 @@ def test_ert_integration(res_data): "RUNPATH ", "", "FORWARD_MODEL GRAV_SUBS_MAPS(=.UNRST, \ - =grav_subs_maps.yml, =./, \ - =./)", + =grav_subs_maps.yml, =./)", ] ), encoding="utf8", diff --git a/tests/test_grav_subs_points.py b/tests/test_grav_subs_points.py index 25e57daa7..bc8e85234 100644 --- a/tests/test_grav_subs_points.py +++ b/tests/test_grav_subs_points.py @@ -4,7 +4,6 @@ from pathlib import Path import pytest -import yaml from pydantic import ValidationError from subscript.grav_subs_points import grav_subs_points @@ -13,25 +12,6 @@ TESTDATA = Path(__file__).absolute().parent / "testdata_gravity" -def test_prepend_root_path(): - """Test that we need to prepend with root-path""" - cfg_file = TESTDATA / "grav_subs_points.yml" - - cfg = yaml.safe_load(cfg_file.read_text(encoding="utf8")) - - cfg_with_rootpath = grav_subs_points.prepend_root_path_to_relative_files( - cfg, TESTDATA - ) - GravPointsConfig(**cfg_with_rootpath) - - # When root-path is prepended (with an absolute part) it should not - # matter if we reapply: - cfg_with_double_rootpath = grav_subs_points.prepend_root_path_to_relative_files( - cfg_with_rootpath, TESTDATA - ) - GravPointsConfig(**cfg_with_double_rootpath) - - @pytest.fixture(name="res_data") def fixture_res_data(tmp_path): """Prepare a data directory with Eclipse binary output""" @@ -192,7 +172,6 @@ def test_unrst_error(dictupdates, expected_error): grav_subs_points.main_gravpoints( unrst_file=test_resfile, config=cfg, - root_path="./", output_folder="./", pref_gendata="", ext_gendata="_1.txt", From 35a34cd8f4c1a638b79bfc9c4db95182c9d3c345 Mon Sep 17 00:00:00 2001 From: "Trine Alsos (CCN GGP GGP8)" Date: Wed, 11 Sep 2024 13:53:06 +0200 Subject: [PATCH 3/6] remove root option, add include dates --- pyproject.toml | 1 + src/subscript/config_jobs/GRAV_SUBS_MAPS | 7 +-- src/subscript/config_jobs/GRAV_SUBS_POINTS | 7 ++- .../grav_subs_maps/grav_subs_maps.py | 47 ++-------------- .../grav_subs_points/grav_subs_points.py | 55 ++----------------- 5 files changed, 18 insertions(+), 99 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f7f685e0c..308b0554c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,7 @@ dependencies = [ "ert", "ert>=10.2.0b13", "fmu-tools", + "grid3d_maps", "matplotlib", "numpy<2", "opm>=2023.04", diff --git a/src/subscript/config_jobs/GRAV_SUBS_MAPS b/src/subscript/config_jobs/GRAV_SUBS_MAPS index 30197f8f3..f9c396616 100644 --- a/src/subscript/config_jobs/GRAV_SUBS_MAPS +++ b/src/subscript/config_jobs/GRAV_SUBS_MAPS @@ -1,10 +1,9 @@ EXECUTABLE grav_subs_maps DEFAULT "./" -DEFAULT "./" -ARGLIST "--configfile" "--root-path" "--outputdir" +ARGLIST "--configfile" "--outputdir" MIN_ARG 2 -MAX_ARG 4 -ARG_TYPE 0 STRING +MAX_ARG 3 +ARG_TYPE 0 STRING \ No newline at end of file diff --git a/src/subscript/config_jobs/GRAV_SUBS_POINTS b/src/subscript/config_jobs/GRAV_SUBS_POINTS index 81a7ca677..4c40b13fb 100644 --- a/src/subscript/config_jobs/GRAV_SUBS_POINTS +++ b/src/subscript/config_jobs/GRAV_SUBS_POINTS @@ -1,12 +1,13 @@ EXECUTABLE grav_subs_points DEFAULT "./" -DEFAULT "./" + DEFAULT "" DEFAULT "_1.txt" -ARGLIST "--configfile" "--root-path" "--outputdir" "--prefix_gendata" "--extension_gendata" +ARGLIST "--configfile" "--outputdir" "--prefix_gendata" "--extension_gendata" MIN_ARG 2 -MAX_ARG 6 +MAX_ARG 5 + ARG_TYPE 0 STRING \ No newline at end of file diff --git a/src/subscript/grav_subs_maps/grav_subs_maps.py b/src/subscript/grav_subs_maps/grav_subs_maps.py index f2eefe351..0d83869e7 100644 --- a/src/subscript/grav_subs_maps/grav_subs_maps.py +++ b/src/subscript/grav_subs_maps/grav_subs_maps.py @@ -8,6 +8,7 @@ import xtgeo import yaml +from grid3d_maps.avghc._loader import FMUYamlSafeLoader from pydantic import BaseModel, Field, FilePath, field_validator from resdata.gravimetry import ResdataGrav, ResdataSubsidence from resdata.grid import Grid @@ -60,15 +61,12 @@ FORWARD_MODEL GRAV_SUBS_MAPS(=.UNRST, =grav_subs_maps.yml) FORWARD_MODEL GRAV_SUBS_MAPS(=.UNRST, =grav_subs_maps.yml, =share/results/maps) - FORWARD_MODEL GRAV_SUBS_MAPS(=.UNRST, =grav_subs_maps.yml, =/../../gravity/input/, =share/results/maps) where ``ECLBASE`` is already defined in your ERT config, pointing to the flowsimulator basename relative to ``RUNPATH``, grav_subs_maps.yml is a YAML file defining the inputs and modelling parameters and ``OUTPUT_DIR`` is the path to the output folder. If not specified OUTPUT_DIR will be defaulted to "./". -``ROOT_PATH`` is the root path assumed for any relative paths in the yaml config file. -This is optional and defaulted to "./". The directory to export maps to must exist. """ # noqa @@ -121,13 +119,6 @@ def get_parser() -> argparse.ArgumentParser: help="Name of YAML config file", required=True, ) - parser.add_argument( - "-r", - "--root-path", - type=str, - default="./", - help=("Root path assumed for relative paths" " in config file."), - ) parser.add_argument( "-o", "--outputdir", @@ -153,46 +144,21 @@ def main() -> None: # parse the config file if not Path(args.configfile).exists(): sys.exit("No such file:" + args.configfile) - config = yaml.safe_load(Path(args.configfile).read_text(encoding="utf8")) + + with open(Path(args.configfile), "r", encoding="utf8") as stream: + config = yaml.load(stream, Loader=FMUYamlSafeLoader) if not Path(args.outputdir).exists(): sys.exit("Output folder does not exist:" + args.outputdir) if not Path(args.UNRSTfile).exists(): sys.exit("UNRST file does not exist:" + args.UNRSTfile) - main_gravmaps(args.UNRSTfile, config, Path(args.root_path), Path(args.outputdir)) - - -def prepend_root_path_to_relative_files( - cfg: Dict[str, Any], root_path: Path -) -> Dict[str, Any]: - """Prepend root_path to relative files found paths in a configuration - dictionary. - - Note: This function is before prior to validation of the configuration! - - Will look for filename in the key "input["seabed_map"]" - - Args: - cfg: grav_subs_maps configuration dictionary - root_path: A relative or absolute path to be prepended - - Returns: - Modified configuration for grav_subs_maps - """ - if ( - "input" in cfg - and "seabed_map" in cfg["input"] - and not os.path.isabs(cfg["input"]["seabed_map"]) - ): - cfg["input"]["seabed_map"] = str(root_path / Path(cfg["input"]["seabed_map"])) - return cfg + main_gravmaps(args.UNRSTfile, config, Path(args.outputdir)) def main_gravmaps( unrst_file: str, config: Dict[str, Any], - root_path: Optional[Path], output_folder: Path, ) -> None: """ @@ -203,9 +169,6 @@ def main_gravmaps( config: Configuration for modelling """ - if root_path is not None: - config = prepend_root_path_to_relative_files(config, root_path) - cfg = GravMapsConfig.model_validate(config).model_dump() # Read inputs and calculation parameters diff --git a/src/subscript/grav_subs_points/grav_subs_points.py b/src/subscript/grav_subs_points/grav_subs_points.py index ed5e2db26..ceca1ee11 100644 --- a/src/subscript/grav_subs_points/grav_subs_points.py +++ b/src/subscript/grav_subs_points/grav_subs_points.py @@ -8,6 +8,7 @@ import pandas as pd import yaml +from grid3d_maps.avghc._loader import FMUYamlSafeLoader from pydantic import BaseModel, Field, FilePath, field_validator from resdata.gravimetry import ResdataGrav, ResdataSubsidence from resdata.grid import Grid @@ -68,7 +69,7 @@ FORWARD_MODEL GRAV_SUBS_POINTS(.UNRST, =grav_subs_points.yml) FORWARD_MODEL GRAV_SUBS_POINTS(.UNRST, =/../input/config/grav_subs_points.yml, =share/results/points) - FORWARD_MODEL GRAV_SUBS_POINTS(.UNRST, =grav_subs_points.yml, =/../../gravity/input, =share/results/points) + FORWARD_MODEL GRAV_SUBS_POINTS(.UNRST, =grav_subs_points.yml, ="_10.txt") FORWARD_MODEL GRAV_SUBS_POINTS(.UNRST, =grav_subs_points.yml, ="fieldA_") @@ -80,8 +81,6 @@ the output files of type GEN_DATA. The prefix can be used to separate datasets for different structures/fields within the dataset and is defaulted to an empty string, i.e. no prefix. The extension should include the report step number, and is defaulted to "_1.txt" -``ROOT_PATH`` is the root path assumed for any relative paths in the yaml config file. -This is optional and defaulted to "./". The directory to export point files to must exist. """ # noqa @@ -138,13 +137,6 @@ def get_parser() -> argparse.ArgumentParser: help="Name of YAML config file", required=True, ) - parser.add_argument( - "-r", - "--root-path", - type=str, - default="./", - help=("Root path assumed for relative paths" " in config file."), - ) parser.add_argument( "-o", "--outputdir", @@ -185,7 +177,9 @@ def main() -> None: # parse the config file if not Path(args.configfile).exists(): sys.exit("No such file:" + args.configfile) - config = yaml.safe_load(Path(args.configfile).read_text(encoding="utf8")) + + with open(Path(args.configfile), "r", encoding="utf8") as stream: + config = yaml.load(stream, Loader=FMUYamlSafeLoader) if not Path(args.outputdir).exists(): sys.exit("Output folder does not exist:" + args.outputdir) @@ -195,47 +189,12 @@ def main() -> None: main_gravpoints( args.UNRSTfile, config, - Path(args.root_path), Path(args.outputdir), args.prefix_gendata, args.extension_gendata, ) -def prepend_root_path_to_relative_files( - cfg: Dict[str, Any], root_path: Path -) -> Dict[str, Any]: - """Prepend root_path to relative files found paths in a configuration - dictionary. - - Note: This function is before prior to validation of the configuration! - - Will look for filename in the keys "grav" and "subs" - - Args: - cfg: grav_subs_points configuration dictionary - root_path: A relative or absolute path to be prepended - - Returns: - Modified configuration for grav_subs_points - """ - - stations = cfg.get("stations") - - if stations is None: - return cfg - - for key in ["grav", "subs"]: - if key in stations and isinstance(stations[key], dict): - for item in stations[key]: - if os.path.isabs(stations[key][item]): - continue - - stations[key][item] = str(root_path / Path(stations[key][item])) - - return cfg - - def export_grav_points_xyz(act_stations, phase, diff_date, out_folder) -> None: """Write points in xyz format, roxar.FileFormat.RMS_POINTS""" logger.info(f"Exporting simulated gravity values to {out_folder} as xyz points") @@ -312,7 +271,6 @@ def export_subs_points_ert( def main_gravpoints( unrst_file: str, config: Dict[str, Any], - root_path: Optional[Path], output_folder: Optional[Path], pref_gendata: Optional[str], ext_gendata: Optional[str], @@ -325,9 +283,6 @@ def main_gravpoints( config: Configuration for modelling """ - if root_path is not None: - config = prepend_root_path_to_relative_files(config, root_path) - cfg = GravPointsConfig.model_validate(config).model_dump() # Read inputs and calculation parameters From 8d9ebd01a9a132bcc31f893c076a0d82165a3675 Mon Sep 17 00:00:00 2001 From: "Trine Alsos (CCN GGP GGP8)" Date: Wed, 11 Sep 2024 13:59:04 +0200 Subject: [PATCH 4/6] add global vars file with dates --- tests/testdata_gravity/global_variables.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 tests/testdata_gravity/global_variables.yml diff --git a/tests/testdata_gravity/global_variables.yml b/tests/testdata_gravity/global_variables.yml new file mode 100644 index 000000000..3df9465fd --- /dev/null +++ b/tests/testdata_gravity/global_variables.yml @@ -0,0 +1,6 @@ +# example configuration file with dates +global: + dates: + GRAVITY_DIFFDATES: + - - 2020-07-01 + - 2018-01-01 From 998a70a3f0a653722bfdd552a6e8485df646637a Mon Sep 17 00:00:00 2001 From: "Trine Alsos (CCN GGP GGP8)" Date: Wed, 11 Sep 2024 14:14:48 +0200 Subject: [PATCH 5/6] update docs --- docs/scripts/grav_subs_maps.rst | 36 +++++++++++++++++++++++++++++++ docs/scripts/grav_subs_points.rst | 30 ++++++++++++++++++++++++-- 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/docs/scripts/grav_subs_maps.rst b/docs/scripts/grav_subs_maps.rst index 2efb62452..98b7a2444 100644 --- a/docs/scripts/grav_subs_maps.rst +++ b/docs/scripts/grav_subs_maps.rst @@ -7,4 +7,40 @@ GRAV_SUBS_MAPS :func: get_parser :prog: grav_subs_maps +Include dates +------------- + +Instead of specifying the modelling dates directly in the yaml config file +it is possible to include them from another yaml file: + +.. code-block:: text + + input: + diffdates: !include_from global_variables.yml::global.dates.GRAVITY_DIFFDATES + +This is an advantage if the dates in the global config is used also for other jobs. In this case the included file looks like this: + +.. code-block:: yaml + + # example global config file with dates + global: + dates: + GRAVITY_DIFFDATES: + - - 2020-07-01 + - 2018-01-01 +The output files +---------------- + +The output from this job are maps in irap binary format. For each difference data there will be one subsidence map and one or more gravity change maps (depending on which phases are specified to model). + +The naming of the output files have been standardised to: + +| all--subsidence--yyyymmdd_yyyymmdd.gri + +| all--delta_gravity_gas--yyyymmdd_yyyymmdd.gri +| all--delta_gravity_oil--yyyymmdd_yyyymmdd.gri +| all--delta_gravity_water--yyyymmdd_yyyymmdd.gri +| all--delta_gravity_total--yyyymmdd_yyyymmdd.gri + +The "all" prefix is indicating that the contributions from all zones is summed. diff --git a/docs/scripts/grav_subs_points.rst b/docs/scripts/grav_subs_points.rst index 009a2bdc6..3f682c3e0 100644 --- a/docs/scripts/grav_subs_points.rst +++ b/docs/scripts/grav_subs_points.rst @@ -6,6 +6,30 @@ GRAV_SUBS_POINTS :func: get_parser :prog: grav_subs_points + +Include dates +------------- + +Instead of specifying the modelling dates directly in the yaml config file +it is possible to include them from another yaml file: + +.. code-block:: text + + input: + diffdates: !include_from global_variables.yml::global.dates.GRAVITY_DIFFDATES + +This is an advantage if the dates in the global config is used also for other jobs. In this case the included file looks like this: + +.. code-block:: yaml + + # example global config file with dates + global: + dates: + GRAVITY_DIFFDATES: + - - 2020-07-01 + - 2018-01-01 + + Station coordinates ------------------- @@ -39,11 +63,13 @@ The output files There will be two types of output file from grav_subs_points. -- one column text files of modelled gravity change/subsidence that can be used with GEN_DATA observations +- one column text files of modelled gravity change/subsidence GEN_DATA files that can be used with ert GEN_OBS observations - x,y,z text file that can be used for visualisation. -The ordering of points in these two files will be the same as given in the station coordinates file used as input. +The ordering of points in these two files will be the same as given in the station coordinates file used as input. It can therefore be a good idea to order the benchmark stations in the station coordinates file in the order you would like to see them in e.g. line plots, for instance sorted by area. + +In some cases where the reservoir model covers several structures or fields it can be beneficial to split the modelling for different structures into several files. E.g. for testing in assisted history matching what effect it has if only observations for one of the structures is used as observations compared to both. To facilitate this usage the option to add a prefix to the GEN_DATA file to separate them, using the prefix_gendata option. There is also a possibility to use a different report step and extension than the default "_1.txt" by using the extension_gendata option. From 345a617e5f61534516428cbdbf3140d5411589ff Mon Sep 17 00:00:00 2001 From: "Trine Alsos (CCN GGP GGP8)" Date: Thu, 12 Sep 2024 08:21:06 +0200 Subject: [PATCH 6/6] fix typos --- docs/scripts/grav_subs_maps.rst | 4 ++-- docs/scripts/grav_subs_points.rst | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/scripts/grav_subs_maps.rst b/docs/scripts/grav_subs_maps.rst index 98b7a2444..d88975219 100644 --- a/docs/scripts/grav_subs_maps.rst +++ b/docs/scripts/grav_subs_maps.rst @@ -18,7 +18,7 @@ it is possible to include them from another yaml file: input: diffdates: !include_from global_variables.yml::global.dates.GRAVITY_DIFFDATES -This is an advantage if the dates in the global config is used also for other jobs. In this case the included file looks like this: +This is an advantage if the dates in the global config is used also for other jobs. In this example the included file looks like this: .. code-block:: yaml @@ -32,7 +32,7 @@ This is an advantage if the dates in the global config is used also for other jo The output files ---------------- -The output from this job are maps in irap binary format. For each difference data there will be one subsidence map and one or more gravity change maps (depending on which phases are specified to model). +The output from this job are maps in irap binary format. For each difference date there will be one subsidence map and one or more gravity change maps (depending on which phases are specified to model). The naming of the output files have been standardised to: diff --git a/docs/scripts/grav_subs_points.rst b/docs/scripts/grav_subs_points.rst index 3f682c3e0..00b16de3a 100644 --- a/docs/scripts/grav_subs_points.rst +++ b/docs/scripts/grav_subs_points.rst @@ -18,7 +18,7 @@ it is possible to include them from another yaml file: input: diffdates: !include_from global_variables.yml::global.dates.GRAVITY_DIFFDATES -This is an advantage if the dates in the global config is used also for other jobs. In this case the included file looks like this: +This is an advantage if the dates in the global config is used also for other jobs. In this example the included file looks like this: .. code-block:: yaml @@ -63,7 +63,7 @@ The output files There will be two types of output file from grav_subs_points. -- one column text files of modelled gravity change/subsidence GEN_DATA files that can be used with ert GEN_OBS observations +- one column text files of modelled gravity change/subsidence GEN_DATA files that can be used with ert GENERAL_OBSERVATION - x,y,z text file that can be used for visualisation.