From ec43c656b5e32db715d7ebddb5a7c3fe341c8f55 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Fri, 31 May 2024 15:08:34 +0200 Subject: [PATCH 01/18] new group_to_dict function --- python/grass/script/imagery.py | 110 +++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 python/grass/script/imagery.py diff --git a/python/grass/script/imagery.py b/python/grass/script/imagery.py new file mode 100644 index 00000000000..234ca94e2d9 --- /dev/null +++ b/python/grass/script/imagery.py @@ -0,0 +1,110 @@ +""" +Imagery related functions to be used in Python scripts. + +Usage: + +:: + + import grass.script as gs + + gs.imagery.group_to_dict(imagery_group) + ... + +(C) 2024 by Stefan Blumentrath and the GRASS Development Team +This program is free software under the GNU General Public +License (>=v2). Read the file COPYING that comes with GRASS +for details. + +.. sectionauthor:: Stefan Blumentrath +""" + +from grass.exceptions import CalledModuleError + +from .core import read_command, warning +from .raster import raster_info + + +def group_to_dict( + imagery_group_name, + dict_key="semantic_label", + full_info=True, + subgroup=None, + env=None, +): + """Create a dictionary to represent an imagery group, with it`s raster maps + and their associated semantic_labels. The dict_key option allows to choose + if the semantic_labels should be the keys in the dictionary (the default) + or the map IDs (full map names). The function can also operate on the level + of subgroups. + For raster maps in the imagery (sub-)group that do not have a semantic label + a warning is given. + If the imagery group is not found, an error message is printed and an empty + dictionary is returned. In case of non-existing subgroups no error message + is printed and an empty dictionary is returned. + + Example: + + >>> run_command("g.copy", raster="lsat7_2000_10,lsat7_2000_10") + 0 + >>> run_command("r.support", raster="lsat7_2000_10", semantic_label="L8_1") + 0 + >>> run_command("g.copy", raster="lsat7_2000_20,lsat7_2000_20") + 0 + >>> run_command("r.support", raster="lsat7_2000_20", semantic_label="L8_2") + 0 + >>> run_command("g.copy", raster="lsat7_2000_30,lsat7_2000_30") + 0 + >>> run_command("r.support", raster="lsat7_2000_30", semantic_label="L8_3") + 0 + >>> run_command("i.group", group="L8_group", + >>> input="lsat7_2000_10,lsat7_2000_20,lsat7_2000_30") + 0 + >>> group_to_dict("L8_group", full_info=False) # doctest: +ELLIPSIS + {"L8_1": "lsat7_2000_10", ... "L8_3": "lsat7_2000_30"} + >>> run_command("g.remove", flags="f", type="group", name="L8_group") + 0 + >>> run_command("g.remove", flags="f", type="raster", + >>> name="lsat7_2000_10,lsat7_2000_20,lsat7_2000_30") + 0 + + :param str table: imagery_group_name: Name of the imagery group to process (or None) + :param str dict_key: What to use as key for dictionary "semantic_labels" or map ids + :param env: environment + + :return: dictionary with maps and their semantic labels (or row indices in the + imagery group) + :rtype: dict + """ + group_dict = {} + try: + maps_in_group = ( + read_command( + "i.group", + group=imagery_group_name, + subgroup=subgroup, + flags="g", + quiet=True, + env=env, + ) + .strip() + .split() + ) + except CalledModuleError: + return group_dict + + for idx, raster_map in enumerate(maps_in_group): + raster_map_info = raster_info(raster_map, env=env) + semantic_label = raster_map_info["semantic_label"] + if not raster_map_info["semantic_label"]: + warning( + _( + "Raster map {rm} in group {igroup} does not have a semantic label." + "Using the numeric row index in the (sub-) group" + ).format(rm=raster_map, igroup=imagery_group_name) + ) + semantic_label = idx + 1 + if dict_key == "semantic_label": + group_dict[semantic_label] = raster_map_info if full_info else raster_map + else: + group_dict[raster_map] = raster_map_info if full_info else semantic_label + return group_dict From 149c84a44079a71b3a2fb84035197f5b70c4dee0 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Fri, 31 May 2024 15:10:13 +0200 Subject: [PATCH 02/18] new group_to_dict function --- python/grass/script/testsuite/test_imagery.py | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 python/grass/script/testsuite/test_imagery.py diff --git a/python/grass/script/testsuite/test_imagery.py b/python/grass/script/testsuite/test_imagery.py new file mode 100644 index 00000000000..c3bbf607225 --- /dev/null +++ b/python/grass/script/testsuite/test_imagery.py @@ -0,0 +1,65 @@ +from grass.gunittest.case import TestCase +from grass.gunittest.main import test + +import grass.script as gs + + +class TestImageryGroupToDict(TestCase): + """Tests function `group_to_dict` that returns raster maps + from an imagery group and their metadata.""" + + @classmethod + def setUpClass(cls): + cls.bands = [1, 2, 3] + cls.raster_maps = ",".join([f"lsat7_2000_{band}" for band in cls.bands]) + cls.group = "L8_group" + cls.subgroup = "L8_group_subgroup" + for band in cls.bands: + cls.runModule("g.copy", raster=f"lsat7_2000_{band}0,lsat7_2000_{band}0") + cls.runModule( + "r.support", raster=f"lsat7_2000_{band}", semantic_label=f"L8_{band}" + ) + cls.runModule("i.group", group="L8_group", input=cls.raster_maps) + + @classmethod + def tearDownClass(cls): + cls.runModule("g.remove", type="raster", name=cls.raster_maps, flags="f") + cls.runModule("g.remove", type="group", name=cls.group, flags="f") + + def test_basic_group_label_keys(self): + ref_dict = {f"L8_{band}": f"lsat7_2000_{band}" for band in self.bands} + group_info = gs.imagery.group_to_dict(self.group, full_info=False) + self.assertIsInstance(dict, group_info) + self.assertDictEqual(ref_dict, group_info) + + def test_basic_group_map_keys(self): + ref_dict = {f"lsat7_2000_{band}": f"L8_{band}" for band in self.bands} + group_info = gs.imagery.group_to_dict( + self.group, dict_key=None, full_info=False + ) + self.assertIsInstance(dict, group_info) + self.assertDictEqual(ref_dict, group_info) + + def test_full_info_group_label_keys(self): + group_info = gs.imagery.group_to_dict(self.group, full_info=False) + self.assertIsInstance(dict, group_info) + self.assertListEqual( + [f"L8_{band}" for band in self.bands], list(group_info.keys()) + ) + for band in self.bands: + # Take some metadata keys from raster_info + for metadata_key in [ + "north", + "nsres", + "cols", + "datatype", + "map", + "date", + "semantic_label", + "comments", + ]: + self.assertIn(metadata_key, group_info[f"L8_{band}"]) + + +if __name__ == "__main__": + test() From a7764b179f98d2bea149dfccac8b439dcceb3f75 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Fri, 31 May 2024 23:01:19 +0200 Subject: [PATCH 03/18] add imagery module --- python/grass/script/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/grass/script/Makefile b/python/grass/script/Makefile index 2ca98db5041..888417f8ca6 100644 --- a/python/grass/script/Makefile +++ b/python/grass/script/Makefile @@ -5,7 +5,7 @@ include $(MODULE_TOPDIR)/include/Make/Python.make DSTDIR = $(ETC)/python/grass/script -MODULES = core db raster raster3d vector array setup task utils +MODULES = core db imagery raster raster3d vector array setup task utils PYFILES := $(patsubst %,$(DSTDIR)/%.py,$(MODULES) __init__) PYCFILES := $(patsubst %,$(DSTDIR)/%.pyc,$(MODULES) __init__) From 3635ea8bdddbeca7b83d83d5d8632be6d700055d Mon Sep 17 00:00:00 2001 From: ninsbl Date: Fri, 31 May 2024 23:01:40 +0200 Subject: [PATCH 04/18] add imagery module --- python/grass/script/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/grass/script/__init__.py b/python/grass/script/__init__.py index 8589e8a1521..1d083de4dc3 100644 --- a/python/grass/script/__init__.py +++ b/python/grass/script/__init__.py @@ -3,6 +3,7 @@ from .core import * from .db import * +from .imagery import * from .raster import * from .raster3d import * from .vector import * From f78b813118a4a9dafb0b1208b70c4e84c743d7ec Mon Sep 17 00:00:00 2001 From: ninsbl Date: Fri, 31 May 2024 23:02:13 +0200 Subject: [PATCH 05/18] add imagery module --- python/grass/script/testsuite/test_imagery.py | 166 +++++++++++------- 1 file changed, 101 insertions(+), 65 deletions(-) diff --git a/python/grass/script/testsuite/test_imagery.py b/python/grass/script/testsuite/test_imagery.py index c3bbf607225..12a9dcb84c7 100644 --- a/python/grass/script/testsuite/test_imagery.py +++ b/python/grass/script/testsuite/test_imagery.py @@ -1,65 +1,101 @@ -from grass.gunittest.case import TestCase -from grass.gunittest.main import test - -import grass.script as gs - - -class TestImageryGroupToDict(TestCase): - """Tests function `group_to_dict` that returns raster maps - from an imagery group and their metadata.""" - - @classmethod - def setUpClass(cls): - cls.bands = [1, 2, 3] - cls.raster_maps = ",".join([f"lsat7_2000_{band}" for band in cls.bands]) - cls.group = "L8_group" - cls.subgroup = "L8_group_subgroup" - for band in cls.bands: - cls.runModule("g.copy", raster=f"lsat7_2000_{band}0,lsat7_2000_{band}0") - cls.runModule( - "r.support", raster=f"lsat7_2000_{band}", semantic_label=f"L8_{band}" - ) - cls.runModule("i.group", group="L8_group", input=cls.raster_maps) - - @classmethod - def tearDownClass(cls): - cls.runModule("g.remove", type="raster", name=cls.raster_maps, flags="f") - cls.runModule("g.remove", type="group", name=cls.group, flags="f") - - def test_basic_group_label_keys(self): - ref_dict = {f"L8_{band}": f"lsat7_2000_{band}" for band in self.bands} - group_info = gs.imagery.group_to_dict(self.group, full_info=False) - self.assertIsInstance(dict, group_info) - self.assertDictEqual(ref_dict, group_info) - - def test_basic_group_map_keys(self): - ref_dict = {f"lsat7_2000_{band}": f"L8_{band}" for band in self.bands} - group_info = gs.imagery.group_to_dict( - self.group, dict_key=None, full_info=False - ) - self.assertIsInstance(dict, group_info) - self.assertDictEqual(ref_dict, group_info) - - def test_full_info_group_label_keys(self): - group_info = gs.imagery.group_to_dict(self.group, full_info=False) - self.assertIsInstance(dict, group_info) - self.assertListEqual( - [f"L8_{band}" for band in self.bands], list(group_info.keys()) - ) - for band in self.bands: - # Take some metadata keys from raster_info - for metadata_key in [ - "north", - "nsres", - "cols", - "datatype", - "map", - "date", - "semantic_label", - "comments", - ]: - self.assertIn(metadata_key, group_info[f"L8_{band}"]) - - -if __name__ == "__main__": - test() +from grass.gunittest.case import TestCase +from grass.gunittest.main import test + +import grass.script as gs + + +class TestImageryGroupToDict(TestCase): + """Tests function `group_to_dict` that returns raster maps + from an imagery group and their metadata.""" + + @classmethod + def setUpClass(cls): + cls.bands = [1, 2, 3] + cls.raster_maps = [f"lsat7_2002_{band}0" for band in cls.bands] + cls.group = "L8_group" + cls.subgroup = "L8_group_subgroup" + # Create input maps with label and group + for band in cls.bands: + cls.runModule( + "g.copy", raster=[f"lsat7_2002_{band}0", f"lsat7_2002_{band}0"] + ) + cls.runModule( + "r.support", map=f"lsat7_2002_{band}0", semantic_label=f"L8_{band}" + ) + cls.runModule("i.group", group=cls.group, input=cls.raster_maps) + + @classmethod + def tearDownClass(cls): + cls.runModule("g.remove", type="raster", name=cls.raster_maps, flags="f") + cls.runModule("g.remove", type="group", name=cls.group, flags="f") + + def test_basic_group_label_keys(self): + ref_dict = {f"L8_{band}": f"lsat7_2002_{band}0" for band in self.bands} + group_info = gs.imagery.group_to_dict(self.group, full_info=False) + # Check that a dict is returned + self.assertIsInstance(group_info, dict) + self.assertListEqual(list(ref_dict.keys()), list(group_info.keys())) + self.assertListEqual( + list(ref_dict.values()), [val.split("@")[0] for val in group_info.values()] + ) + + def test_basic_group_map_keys(self): + ref_dict = {f"lsat7_2002_{band}0": f"L8_{band}" for band in self.bands} + group_info = gs.imagery.group_to_dict( + self.group, dict_key=None, full_info=False + ) + # Check that a dict is returned + self.assertIsInstance(group_info, dict) + self.assertListEqual( + list(ref_dict.keys()), [key.split("@")[0] for key in group_info.keys()] + ) + self.assertListEqual(list(ref_dict.values()), list(group_info.values())) + + def test_full_info_group_label_keys(self): + group_info = gs.imagery.group_to_dict(self.group, full_info=True) + # Check that a dict is returned + self.assertIsInstance(group_info, dict) + self.assertListEqual( + [f"L8_{band}" for band in self.bands], list(group_info.keys()) + ) + for band in self.bands: + # Take some metadata keys from raster_info + for metadata_key in [ + "north", + "nsres", + "cols", + "datatype", + "map", + "date", + "semantic_label", + "comments", + ]: + self.assertIn(metadata_key, group_info[f"L8_{band}"]) + + def test_full_info_group_label_keys_subgroup(self): + self.runModule( + "i.group", group=self.group, subgroup=self.subgroup, input=self.raster_maps + ) + group_info = gs.imagery.group_to_dict(self.group, full_info=True) + # Check that a dict is returned + self.assertIsInstance(group_info, dict) + self.assertListEqual( + [f"L8_{band}" for band in self.bands], list(group_info.keys()) + ) + for band in self.bands: + # Take some metadata keys from raster_info + for metadata_key in [ + "north", + "nsres", + "cols", + "datatype", + "map", + "date", + "semantic_label", + "comments", + ]: + self.assertIn(metadata_key, group_info[f"L8_{band}"]) + + +if __name__ == "__main__": + test() From cc86c0b712dd6663ba7b808632f3e9eeef30792b Mon Sep 17 00:00:00 2001 From: ninsbl Date: Mon, 3 Jun 2024 11:37:11 +0200 Subject: [PATCH 06/18] address code review --- python/grass/script/imagery.py | 87 +++++++++++++++++++--------------- 1 file changed, 49 insertions(+), 38 deletions(-) diff --git a/python/grass/script/imagery.py b/python/grass/script/imagery.py index 234ca94e2d9..2585f33bdf5 100644 --- a/python/grass/script/imagery.py +++ b/python/grass/script/imagery.py @@ -26,49 +26,45 @@ def group_to_dict( imagery_group_name, - dict_key="semantic_label", - full_info=True, subgroup=None, + dict_key="semantic_labels", + fill_semantic_label=True, + full_info=True, env=None, ): - """Create a dictionary to represent an imagery group, with it`s raster maps - and their associated semantic_labels. The dict_key option allows to choose - if the semantic_labels should be the keys in the dictionary (the default) - or the map IDs (full map names). The function can also operate on the level - of subgroups. - For raster maps in the imagery (sub-)group that do not have a semantic label - a warning is given. - If the imagery group is not found, an error message is printed and an empty - dictionary is returned. In case of non-existing subgroups no error message - is printed and an empty dictionary is returned. + """Create a dictionary to represent an imagery group with metadata. + + Defined by the dict_key option, the dictionary uses either the names + of the raster maps ("map_names"), their row indices in the group + ("indices") or their associated semantic_labels ("semantic_labels") as keys. + The default is to use semantic_labels as keys. Note that map metadata + of the maps in the group have to be read to get the semantic label, + in addition to the group file. The same metadata is read when the + "full_info" should be returned. + + The function can also operate on the level of subgroups. In case a + non-existing (or empty sub-group) is requested a warning is printed + and an empty dictionary is returned (following the behavior of i.group). Example: >>> run_command("g.copy", raster="lsat7_2000_10,lsat7_2000_10") - 0 >>> run_command("r.support", raster="lsat7_2000_10", semantic_label="L8_1") - 0 >>> run_command("g.copy", raster="lsat7_2000_20,lsat7_2000_20") - 0 >>> run_command("r.support", raster="lsat7_2000_20", semantic_label="L8_2") - 0 >>> run_command("g.copy", raster="lsat7_2000_30,lsat7_2000_30") - 0 >>> run_command("r.support", raster="lsat7_2000_30", semantic_label="L8_3") - 0 >>> run_command("i.group", group="L8_group", >>> input="lsat7_2000_10,lsat7_2000_20,lsat7_2000_30") - 0 >>> group_to_dict("L8_group", full_info=False) # doctest: +ELLIPSIS {"L8_1": "lsat7_2000_10", ... "L8_3": "lsat7_2000_30"} >>> run_command("g.remove", flags="f", type="group", name="L8_group") - 0 >>> run_command("g.remove", flags="f", type="raster", >>> name="lsat7_2000_10,lsat7_2000_20,lsat7_2000_30") - 0 :param str table: imagery_group_name: Name of the imagery group to process (or None) - :param str dict_key: What to use as key for dictionary "semantic_labels" or map ids + :param str dict_key: What to use as key for dictionary. Can bei either + "semantic_labels" (default), "map_names" or "indices" :param env: environment :return: dictionary with maps and their semantic labels (or row indices in the @@ -89,22 +85,37 @@ def group_to_dict( .strip() .split() ) - except CalledModuleError: - return group_dict + except CalledModuleError as cme: + raise cme - for idx, raster_map in enumerate(maps_in_group): - raster_map_info = raster_info(raster_map, env=env) - semantic_label = raster_map_info["semantic_label"] - if not raster_map_info["semantic_label"]: - warning( - _( - "Raster map {rm} in group {igroup} does not have a semantic label." - "Using the numeric row index in the (sub-) group" - ).format(rm=raster_map, igroup=imagery_group_name) + if subgroup and not maps_in_group: + warning( + _("Empty result returned for subgroup <{sg}> in group <{g}>").format( + sg=subgroup, g=imagery_group_name ) - semantic_label = idx + 1 - if dict_key == "semantic_label": - group_dict[semantic_label] = raster_map_info if full_info else raster_map - else: - group_dict[raster_map] = raster_map_info if full_info else semantic_label + ) + + for idx, raster_map in enumerate(maps_in_group): + raster_map_info = None + if full_info or dict_key == "semantic_labels": + raster_map_info = raster_info(raster_map, env=env) + + if dict_key == "indices": + key = str(idx + 1) + val = raster_map_info or raster_map + elif dict_key == "map_names": + key = raster_map + val = raster_map_info or str(idx + 1) + elif dict_key == "semantic_labels": + key = raster_map_info["semantic_label"] + if not key or key == '"none"': + warning( + _( + "Raster map {m} in group <{g}> does not have a semantic label." + ).format(m=raster_map, g=imagery_group_name) + ) + if fill_semantic_label: + key = str(idx + 1) + val = raster_map_info if full_info else raster_map + group_dict[key] = val return group_dict From f5aab583a1175da20bc07440ba4d1fc7b1c5d663 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Mon, 3 Jun 2024 12:34:33 +0200 Subject: [PATCH 07/18] address code review --- python/grass/script/testsuite/test_imagery.py | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/python/grass/script/testsuite/test_imagery.py b/python/grass/script/testsuite/test_imagery.py index 12a9dcb84c7..b508bd0ffa2 100644 --- a/python/grass/script/testsuite/test_imagery.py +++ b/python/grass/script/testsuite/test_imagery.py @@ -1,3 +1,4 @@ +from grass.exceptions import CalledModuleError from grass.gunittest.case import TestCase from grass.gunittest.main import test @@ -39,10 +40,35 @@ def test_basic_group_label_keys(self): list(ref_dict.values()), [val.split("@")[0] for val in group_info.values()] ) + def test_invalid_input(self): + # Non existing group + self.assertRaises( + CalledModuleError, gs.imagery.group_to_dict, "non_existing_group" + ) + # invalid dict_key + self.assertRaises( + CalledModuleError, + gs.imagery.group_to_dict, + self.group, + **{"dict_key": "invalid_dict_key"}, + ) + def test_basic_group_map_keys(self): ref_dict = {f"lsat7_2002_{band}0": f"L8_{band}" for band in self.bands} group_info = gs.imagery.group_to_dict( - self.group, dict_key=None, full_info=False + self.group, dict_key="map_names", full_info=False + ) + # Check that a dict is returned + self.assertIsInstance(group_info, dict) + self.assertListEqual( + list(ref_dict.keys()), [key.split("@")[0] for key in group_info.keys()] + ) + self.assertListEqual(list(ref_dict.values()), list(group_info.values())) + + def test_basic_group_index_keys(self): + ref_dict = {band: f"lsat7_2002_{band}0" for band in self.bands} + group_info = gs.imagery.group_to_dict( + self.group, dict_key="indices", full_info=False ) # Check that a dict is returned self.assertIsInstance(group_info, dict) From e2b7880798fa3e367ed020be3525f9cd6878da1b Mon Sep 17 00:00:00 2001 From: ninsbl Date: Mon, 3 Jun 2024 14:14:00 +0200 Subject: [PATCH 08/18] address test faliures --- python/grass/script/testsuite/test_imagery.py | 41 +++++++++++++------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/python/grass/script/testsuite/test_imagery.py b/python/grass/script/testsuite/test_imagery.py index b508bd0ffa2..56d49dd2a16 100644 --- a/python/grass/script/testsuite/test_imagery.py +++ b/python/grass/script/testsuite/test_imagery.py @@ -45,18 +45,32 @@ def test_invalid_input(self): self.assertRaises( CalledModuleError, gs.imagery.group_to_dict, "non_existing_group" ) - # invalid dict_key + # Invalid dict_keys self.assertRaises( - CalledModuleError, + ValueError, gs.imagery.group_to_dict, self.group, - **{"dict_key": "invalid_dict_key"}, + **{"dict_keys": "invalid_dict_key"}, ) + # Invalid dict_values + self.assertRaises( + ValueError, + gs.imagery.group_to_dict, + self.group, + **{"dict_values": "invalid_dict_value"}, + ) + group_info = gs.imagery.group_to_dict( + self.group, subgroup="non_existing_subgroup" + ) + + # Check that an empty dict is returned + self.assertIsInstance(group_info, dict) + self.assertDictEqual(group_info, {}) def test_basic_group_map_keys(self): ref_dict = {f"lsat7_2002_{band}0": f"L8_{band}" for band in self.bands} group_info = gs.imagery.group_to_dict( - self.group, dict_key="map_names", full_info=False + self.group, dict_keys="map_names", dict_values="semantic_labels" ) # Check that a dict is returned self.assertIsInstance(group_info, dict) @@ -66,10 +80,8 @@ def test_basic_group_map_keys(self): self.assertListEqual(list(ref_dict.values()), list(group_info.values())) def test_basic_group_index_keys(self): - ref_dict = {band: f"lsat7_2002_{band}0" for band in self.bands} - group_info = gs.imagery.group_to_dict( - self.group, dict_key="indices", full_info=False - ) + ref_dict = {str(band): f"lsat7_2002_{band}0" for band in self.bands} + group_info = gs.imagery.group_to_dict(self.group, dict_keys="indices") # Check that a dict is returned self.assertIsInstance(group_info, dict) self.assertListEqual( @@ -78,7 +90,7 @@ def test_basic_group_index_keys(self): self.assertListEqual(list(ref_dict.values()), list(group_info.values())) def test_full_info_group_label_keys(self): - group_info = gs.imagery.group_to_dict(self.group, full_info=True) + group_info = gs.imagery.group_to_dict(self.group, dict_values="metadata") # Check that a dict is returned self.assertIsInstance(group_info, dict) self.assertListEqual( @@ -102,11 +114,16 @@ def test_full_info_group_label_keys_subgroup(self): self.runModule( "i.group", group=self.group, subgroup=self.subgroup, input=self.raster_maps ) - group_info = gs.imagery.group_to_dict(self.group, full_info=True) + group_info = gs.imagery.group_to_dict( + self.group, + subgroup=self.subgroup, + dict_keys="map_names", + dict_values="metadata", + ) # Check that a dict is returned self.assertIsInstance(group_info, dict) self.assertListEqual( - [f"L8_{band}" for band in self.bands], list(group_info.keys()) + [f"lsat7_2002_{band}0" for band in self.bands], list(group_info.keys()) ) for band in self.bands: # Take some metadata keys from raster_info @@ -120,7 +137,7 @@ def test_full_info_group_label_keys_subgroup(self): "semantic_label", "comments", ]: - self.assertIn(metadata_key, group_info[f"L8_{band}"]) + self.assertIn(metadata_key, group_info[f"lsat7_2002_{band}0"]) if __name__ == "__main__": From 88c2e25df98d0c73e0532d0a88b365b7f80e5990 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Mon, 3 Jun 2024 14:14:19 +0200 Subject: [PATCH 09/18] improve logic --- python/grass/script/imagery.py | 72 ++++++++++++++++++++++++---------- 1 file changed, 52 insertions(+), 20 deletions(-) diff --git a/python/grass/script/imagery.py b/python/grass/script/imagery.py index 2585f33bdf5..0bce871fea0 100644 --- a/python/grass/script/imagery.py +++ b/python/grass/script/imagery.py @@ -27,26 +27,26 @@ def group_to_dict( imagery_group_name, subgroup=None, - dict_key="semantic_labels", + dict_keys="semantic_labels", + dict_values="map_names", fill_semantic_label=True, - full_info=True, env=None, ): """Create a dictionary to represent an imagery group with metadata. - Defined by the dict_key option, the dictionary uses either the names + Defined by the dict_keys option, the dictionary uses either the names of the raster maps ("map_names"), their row indices in the group ("indices") or their associated semantic_labels ("semantic_labels") as keys. - The default is to use semantic_labels as keys. Note that map metadata + The default is to use semantic_labels. Note that map metadata of the maps in the group have to be read to get the semantic label, in addition to the group file. The same metadata is read when the - "full_info" should be returned. + full_info should be returned. The function can also operate on the level of subgroups. In case a non-existing (or empty sub-group) is requested a warning is printed and an empty dictionary is returned (following the behavior of i.group). - Example: + Example:: >>> run_command("g.copy", raster="lsat7_2000_10,lsat7_2000_10") >>> run_command("r.support", raster="lsat7_2000_10", semantic_label="L8_1") @@ -56,19 +56,27 @@ def group_to_dict( >>> run_command("r.support", raster="lsat7_2000_30", semantic_label="L8_3") >>> run_command("i.group", group="L8_group", >>> input="lsat7_2000_10,lsat7_2000_20,lsat7_2000_30") - >>> group_to_dict("L8_group", full_info=False) # doctest: +ELLIPSIS + >>> group_to_dict("L8_group") # doctest: +ELLIPSIS {"L8_1": "lsat7_2000_10", ... "L8_3": "lsat7_2000_30"} >>> run_command("g.remove", flags="f", type="group", name="L8_group") >>> run_command("g.remove", flags="f", type="raster", >>> name="lsat7_2000_10,lsat7_2000_20,lsat7_2000_30") - :param str table: imagery_group_name: Name of the imagery group to process (or None) - :param str dict_key: What to use as key for dictionary. Can bei either + :param str imagery_group_name: Name of the imagery group to process (or None) + :param str subgroup: Name of the imagery sub-group to process (or None) + :param str dict_keys: What to use as key for dictionary. Can bei either "semantic_labels" (default), "map_names" or "indices" - :param env: environment - - :return: dictionary with maps and their semantic labels (or row indices in the - imagery group) + :param str dict_values: What to use as values for dictionary. Can bei either + "map_names" (default), "semanic_labels", "indices" or + "metadata" (to return dictionaries with full map metadata) + :param bool fill_semantic_label: If maps in a group do not have a semantic + label, their index in the group is used + instead (default). Otherwise None / "none" + is used. + :param dict env: Environment to use when parsing the imagery group + + :return: dictionary representing an imagery group with it's maps and their + semantic labels, row indices in the group, or metadata :rtype: dict """ group_dict = {} @@ -88,6 +96,12 @@ def group_to_dict( except CalledModuleError as cme: raise cme + if dict_keys not in ["indices", "map_names", "semantic_labels"]: + raise ValueError(f"Invalid dictionary keys <{dict_keys}> requested") + + if dict_values not in ["indices", "map_names", "semantic_labels", "metadata"]: + raise ValueError(f"Invalid dictionary values <{dict_values}> requested") + if subgroup and not maps_in_group: warning( _("Empty result returned for subgroup <{sg}> in group <{g}>").format( @@ -97,16 +111,19 @@ def group_to_dict( for idx, raster_map in enumerate(maps_in_group): raster_map_info = None - if full_info or dict_key == "semantic_labels": + # Get raster metadata if needed + if ( + dict_values in ["semantic_labels", "metadata"] + or dict_keys == "semantic_labels" + ): raster_map_info = raster_info(raster_map, env=env) - if dict_key == "indices": + # Get key for dictionary + if dict_keys == "indices": key = str(idx + 1) - val = raster_map_info or raster_map - elif dict_key == "map_names": + elif dict_keys == "map_names": key = raster_map - val = raster_map_info or str(idx + 1) - elif dict_key == "semantic_labels": + elif dict_keys == "semantic_labels": key = raster_map_info["semantic_label"] if not key or key == '"none"': warning( @@ -116,6 +133,21 @@ def group_to_dict( ) if fill_semantic_label: key = str(idx + 1) - val = raster_map_info if full_info else raster_map + + if dict_values == "indices": + val = str(idx + 1) + elif dict_values == "map_names": + val = raster_map + elif dict_values == "semantic_labels": + val = raster_map_info or raster_map + elif dict_values == "metadata": + val = raster_map_info + if key in group_dict: + warning( + _( + "Key {k} from raster map {m} already present in group dictionary." + "Overwriting existing entry..." + ).format(k=key, r=raster_map) + ) group_dict[key] = val return group_dict From 2525777474aa55ce24fe6790801332d0935c6573 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Mon, 3 Jun 2024 14:45:41 +0200 Subject: [PATCH 10/18] update docstring --- python/grass/script/imagery.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/grass/script/imagery.py b/python/grass/script/imagery.py index 0bce871fea0..99acd71f203 100644 --- a/python/grass/script/imagery.py +++ b/python/grass/script/imagery.py @@ -40,7 +40,8 @@ def group_to_dict( The default is to use semantic_labels. Note that map metadata of the maps in the group have to be read to get the semantic label, in addition to the group file. The same metadata is read when the - full_info should be returned. + "metadata" is requested as dict_values. Other supported dict_values + are "map_names" (default), "semantic_labels", or "indices". The function can also operate on the level of subgroups. In case a non-existing (or empty sub-group) is requested a warning is printed From 85d36a0c83efe0eea2d386b8dd68c970fd326f45 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Wed, 5 Jun 2024 00:51:23 +0200 Subject: [PATCH 11/18] address review --- python/grass/script/imagery.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/python/grass/script/imagery.py b/python/grass/script/imagery.py index 99acd71f203..da934d8db82 100644 --- a/python/grass/script/imagery.py +++ b/python/grass/script/imagery.py @@ -20,7 +20,7 @@ from grass.exceptions import CalledModuleError -from .core import read_command, warning +from .core import read_command, warning, fatal from .raster import raster_info @@ -34,8 +34,8 @@ def group_to_dict( ): """Create a dictionary to represent an imagery group with metadata. - Defined by the dict_keys option, the dictionary uses either the names - of the raster maps ("map_names"), their row indices in the group + Depending on the dict_keys option, the returned dictionary uses either + the names of the raster maps ("map_names"), their row indices in the group ("indices") or their associated semantic_labels ("semantic_labels") as keys. The default is to use semantic_labels. Note that map metadata of the maps in the group have to be read to get the semantic label, @@ -94,8 +94,8 @@ def group_to_dict( .strip() .split() ) - except CalledModuleError as cme: - raise cme + except CalledModuleError as error: + raise error if dict_keys not in ["indices", "map_names", "semantic_labels"]: raise ValueError(f"Invalid dictionary keys <{dict_keys}> requested") @@ -127,13 +127,14 @@ def group_to_dict( elif dict_keys == "semantic_labels": key = raster_map_info["semantic_label"] if not key or key == '"none"': - warning( - _( - "Raster map {m} in group <{g}> does not have a semantic label." - ).format(m=raster_map, g=imagery_group_name) - ) if fill_semantic_label: key = str(idx + 1) + else: + fatal( + _( + "Semantic label missing for raster map {m} in group <{g}>." + ).format(m=raster_map, g=imagery_group_name) + ) if dict_values == "indices": val = str(idx + 1) From 9e7923c9fc0213d5ca3259f5192643bdfaff469e Mon Sep 17 00:00:00 2001 From: ninsbl Date: Wed, 5 Jun 2024 23:40:25 +0200 Subject: [PATCH 12/18] fix tests --- python/grass/script/testsuite/test_imagery.py | 63 ++++++++++++------- 1 file changed, 41 insertions(+), 22 deletions(-) diff --git a/python/grass/script/testsuite/test_imagery.py b/python/grass/script/testsuite/test_imagery.py index 56d49dd2a16..3d057d705f0 100644 --- a/python/grass/script/testsuite/test_imagery.py +++ b/python/grass/script/testsuite/test_imagery.py @@ -30,9 +30,10 @@ def tearDownClass(cls): cls.runModule("g.remove", type="raster", name=cls.raster_maps, flags="f") cls.runModule("g.remove", type="group", name=cls.group, flags="f") - def test_basic_group_label_keys(self): + def test_basic_group_dict_defaults(self): + """Test with semantic labels as keys and map names as values (defaults)""" ref_dict = {f"L8_{band}": f"lsat7_2002_{band}0" for band in self.bands} - group_info = gs.imagery.group_to_dict(self.group, full_info=False) + group_info = gs.imagery.group_to_dict(self.group) # Check that a dict is returned self.assertIsInstance(group_info, dict) self.assertListEqual(list(ref_dict.keys()), list(group_info.keys())) @@ -40,34 +41,42 @@ def test_basic_group_label_keys(self): list(ref_dict.values()), [val.split("@")[0] for val in group_info.values()] ) - def test_invalid_input(self): + def test_non_existing_group(self): + """Test that function fails if group does not exist""" # Non existing group self.assertRaises( CalledModuleError, gs.imagery.group_to_dict, "non_existing_group" ) - # Invalid dict_keys + + def test_invalid_dict_key(self): + """Test that function fails if invalid keys are requested""" self.assertRaises( ValueError, gs.imagery.group_to_dict, self.group, **{"dict_keys": "invalid_dict_key"}, ) - # Invalid dict_values + + def test_invalid_dict_value(self): + """Test that function fails if invalid values are requested""" self.assertRaises( ValueError, gs.imagery.group_to_dict, self.group, **{"dict_values": "invalid_dict_value"}, ) + + def test_missing_subgroup(self): + """Test that empty dict is returned if subgroup does not exist""" group_info = gs.imagery.group_to_dict( self.group, subgroup="non_existing_subgroup" ) # Check that an empty dict is returned - self.assertIsInstance(group_info, dict) self.assertDictEqual(group_info, {}) def test_basic_group_map_keys(self): + """Test with map_names as keys and semantic_labels as values""" ref_dict = {f"lsat7_2002_{band}0": f"L8_{band}" for band in self.bands} group_info = gs.imagery.group_to_dict( self.group, dict_keys="map_names", dict_values="semantic_labels" @@ -80,21 +89,25 @@ def test_basic_group_map_keys(self): self.assertListEqual(list(ref_dict.values()), list(group_info.values())) def test_basic_group_index_keys(self): + """Test with indices as keys and mapnames as values""" ref_dict = {str(band): f"lsat7_2002_{band}0" for band in self.bands} group_info = gs.imagery.group_to_dict(self.group, dict_keys="indices") # Check that a dict is returned self.assertIsInstance(group_info, dict) + self.assertListEqual(list(ref_dict.keys()), list(group_info.keys())) self.assertListEqual( - list(ref_dict.keys()), [key.split("@")[0] for key in group_info.keys()] + list(ref_dict.values()), + list([val.split("@")[0] for val in group_info.values()]), ) - self.assertListEqual(list(ref_dict.values()), list(group_info.values())) def test_full_info_group_label_keys(self): + """Test with semantic labels as keys and full map metadata as values""" group_info = gs.imagery.group_to_dict(self.group, dict_values="metadata") # Check that a dict is returned self.assertIsInstance(group_info, dict) self.assertListEqual( - [f"L8_{band}" for band in self.bands], list(group_info.keys()) + [f"L8_{band}" for band in self.bands], + [key.split("@")[0] for key in group_info.keys()], ) for band in self.bands: # Take some metadata keys from raster_info @@ -111,6 +124,17 @@ def test_full_info_group_label_keys(self): self.assertIn(metadata_key, group_info[f"L8_{band}"]) def test_full_info_group_label_keys_subgroup(self): + """Test with map names as keys and full map metadata as values""" + metadata_keys = { + "north", + "nsres", + "cols", + "datatype", + "map", + "date", + "semantic_label", + "comments", + } self.runModule( "i.group", group=self.group, subgroup=self.subgroup, input=self.raster_maps ) @@ -123,21 +147,16 @@ def test_full_info_group_label_keys_subgroup(self): # Check that a dict is returned self.assertIsInstance(group_info, dict) self.assertListEqual( - [f"lsat7_2002_{band}0" for band in self.bands], list(group_info.keys()) + [f"lsat7_2002_{band}0" for band in self.bands], + [key.split("@")[0] for key in group_info.keys()], ) - for band in self.bands: + for key, val in group_info.items(): + # Check keys + self.assertTrue(key.startswith("lsat7_2002_")) + # Check values + self.assertIsInstance(val, dict) # Take some metadata keys from raster_info - for metadata_key in [ - "north", - "nsres", - "cols", - "datatype", - "map", - "date", - "semantic_label", - "comments", - ]: - self.assertIn(metadata_key, group_info[f"lsat7_2002_{band}0"]) + self.assertTrue(metadata_keys.issubset(set(val.keys()))) if __name__ == "__main__": From 569dff684dec048def2b3bcc9f99c21950017f94 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Wed, 5 Jun 2024 23:40:47 +0200 Subject: [PATCH 13/18] fix semantic_labels values --- python/grass/script/imagery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/grass/script/imagery.py b/python/grass/script/imagery.py index da934d8db82..65b63786e62 100644 --- a/python/grass/script/imagery.py +++ b/python/grass/script/imagery.py @@ -141,7 +141,7 @@ def group_to_dict( elif dict_values == "map_names": val = raster_map elif dict_values == "semantic_labels": - val = raster_map_info or raster_map + val = raster_map_info["semantic_label"] elif dict_values == "metadata": val = raster_map_info if key in group_dict: From 89c431d4bc8bf8b8568af942b8fddb6dfe810cc0 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Thu, 18 Jul 2024 00:08:10 +0200 Subject: [PATCH 14/18] address review --- python/grass/script/imagery.py | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/python/grass/script/imagery.py b/python/grass/script/imagery.py index 65b63786e62..92bbeded56b 100644 --- a/python/grass/script/imagery.py +++ b/python/grass/script/imagery.py @@ -18,8 +18,6 @@ .. sectionauthor:: Stefan Blumentrath """ -from grass.exceptions import CalledModuleError - from .core import read_command, warning, fatal from .raster import raster_info @@ -81,26 +79,23 @@ def group_to_dict( :rtype: dict """ group_dict = {} - try: - maps_in_group = ( - read_command( - "i.group", - group=imagery_group_name, - subgroup=subgroup, - flags="g", - quiet=True, - env=env, - ) - .strip() - .split() + maps_in_group = ( + read_command( + "i.group", + group=imagery_group_name, + subgroup=subgroup, + flags="g", + quiet=True, + env=env, ) - except CalledModuleError as error: - raise error + .strip() + .split() + ) - if dict_keys not in ["indices", "map_names", "semantic_labels"]: + if dict_keys not in {"indices", "map_names", "semantic_labels"}: raise ValueError(f"Invalid dictionary keys <{dict_keys}> requested") - if dict_values not in ["indices", "map_names", "semantic_labels", "metadata"]: + if dict_values not in {"indices", "map_names", "semantic_labels", "metadata"}: raise ValueError(f"Invalid dictionary values <{dict_values}> requested") if subgroup and not maps_in_group: @@ -114,7 +109,7 @@ def group_to_dict( raster_map_info = None # Get raster metadata if needed if ( - dict_values in ["semantic_labels", "metadata"] + dict_values in {"semantic_labels", "metadata"} or dict_keys == "semantic_labels" ): raster_map_info = raster_info(raster_map, env=env) From 1da985ad1e2d0b8a54cffe8f08781a86812b7459 Mon Sep 17 00:00:00 2001 From: Stefan Blumentrath Date: Thu, 18 Jul 2024 00:25:20 +0200 Subject: [PATCH 15/18] Apply suggestions from code review Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- python/grass/script/testsuite/test_imagery.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/grass/script/testsuite/test_imagery.py b/python/grass/script/testsuite/test_imagery.py index 3d057d705f0..543bda774b6 100644 --- a/python/grass/script/testsuite/test_imagery.py +++ b/python/grass/script/testsuite/test_imagery.py @@ -54,7 +54,7 @@ def test_invalid_dict_key(self): ValueError, gs.imagery.group_to_dict, self.group, - **{"dict_keys": "invalid_dict_key"}, + dict_keys="invalid_dict_key", ) def test_invalid_dict_value(self): @@ -63,7 +63,7 @@ def test_invalid_dict_value(self): ValueError, gs.imagery.group_to_dict, self.group, - **{"dict_values": "invalid_dict_value"}, + dict_values="invalid_dict_value", ) def test_missing_subgroup(self): @@ -97,7 +97,7 @@ def test_basic_group_index_keys(self): self.assertListEqual(list(ref_dict.keys()), list(group_info.keys())) self.assertListEqual( list(ref_dict.values()), - list([val.split("@")[0] for val in group_info.values()]), + [val.split("@")[0] for val in group_info.values()], ) def test_full_info_group_label_keys(self): From ebf46dd5a427767023c49a44d0a662f53cb15ee4 Mon Sep 17 00:00:00 2001 From: Stefan Blumentrath Date: Fri, 30 Aug 2024 21:05:39 +0200 Subject: [PATCH 16/18] Apply suggestions from code review Co-authored-by: Vaclav Petras --- python/grass/script/imagery.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/grass/script/imagery.py b/python/grass/script/imagery.py index 92bbeded56b..2c2c3868423 100644 --- a/python/grass/script/imagery.py +++ b/python/grass/script/imagery.py @@ -63,9 +63,9 @@ def group_to_dict( :param str imagery_group_name: Name of the imagery group to process (or None) :param str subgroup: Name of the imagery sub-group to process (or None) - :param str dict_keys: What to use as key for dictionary. Can bei either + :param str dict_keys: What to use as key for dictionary. It can be either "semantic_labels" (default), "map_names" or "indices" - :param str dict_values: What to use as values for dictionary. Can bei either + :param str dict_values: What to use as values for dictionary. It can be either "map_names" (default), "semanic_labels", "indices" or "metadata" (to return dictionaries with full map metadata) :param bool fill_semantic_label: If maps in a group do not have a semantic From 0a6a69711be8d982a92572492ad37a82d9a3cad7 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Sun, 8 Sep 2024 21:51:47 +0200 Subject: [PATCH 17/18] Revert "add imagery module" This reverts commit 3635ea8bdddbeca7b83d83d5d8632be6d700055d. --- python/grass/script/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/grass/script/__init__.py b/python/grass/script/__init__.py index 1d083de4dc3..8589e8a1521 100644 --- a/python/grass/script/__init__.py +++ b/python/grass/script/__init__.py @@ -3,7 +3,6 @@ from .core import * from .db import * -from .imagery import * from .raster import * from .raster3d import * from .vector import * From ac4d17a1941adff29ec80cf5966d600c3a8323fb Mon Sep 17 00:00:00 2001 From: Stefan Blumentrath Date: Tue, 10 Dec 2024 09:49:46 +0100 Subject: [PATCH 18/18] Add imagery to __init__.py --- python/grass/script/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/grass/script/__init__.py b/python/grass/script/__init__.py index 57260d69c5d..032a0eb67f3 100644 --- a/python/grass/script/__init__.py +++ b/python/grass/script/__init__.py @@ -68,6 +68,7 @@ db_table_exist, db_table_in_vector, ) +from .imagery import group_to_dict from .raster import mapcalc, mapcalc_start, raster_history, raster_info, raster_what from .raster3d import mapcalc3d, raster3d_info from .utils import ( @@ -146,6 +147,7 @@ "get_raise_on_error", "get_real_command", "gisenv", + "group_to_dict", "handle_errors", "info", "legal_name",