From 1687e7fdb471e5213eeb949cd38f63b7bbd7abcf Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Mon, 16 Jan 2023 14:55:40 -0500 Subject: [PATCH 01/52] Added unit test for JSON (Issue #494) --- podpac/core/test/test_node.py | 66 +++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/podpac/core/test/test_node.py b/podpac/core/test/test_node.py index 03853927..5a1102e8 100644 --- a/podpac/core/test/test_node.py +++ b/podpac/core/test/test_node.py @@ -1227,6 +1227,72 @@ def test_from_definition_version_warning(self): with pytest.warns(UserWarning, match="node definition version mismatch"): node = Node.from_json(s) + + def test_from_proper_json(self): + not_ordered_json = """ + { + "Arithmetic": { + "node": "core.algorithm.generic.Arithmetic", + "attrs": { + "eqn": "a+b", + "params": { + + } + }, + "inputs": { + "a": "SinCoords", + "b": "Arange" + } + }, + "SinCoords": { + "node": "core.algorithm.utility.SinCoords", + "style": { + "colormap": "jet", + "clim": [ + -1.0, + 1.0 + ] + } + }, + "Arange": { + "node": "core.algorithm.utility.Arange" + }, + "podpac_version": "3.2.0" + } + """ + ordered_json = """ + { + "SinCoords": { + "node": "core.algorithm.utility.SinCoords", + "style": { + "colormap": "jet", + "clim": [ + -1.0, + 1.0 + ] + } + }, + "Arange": { + "node": "core.algorithm.utility.Arange" + }, + "Arithmetic": { + "node": "core.algorithm.generic.Arithmetic", + "attrs": { + "eqn": "a+b", + "params": { + + } + }, + "inputs": { + "a": "SinCoords", + "b": "Arange" + } + }, + "podpac_version": "3.2.0" + } + """ + + assert(Node.from_json(not_ordered_json).definition == Node.from_json(ordered_json.definition)) class TestNoCacheMixin(object): From f8a79631cf3d9d5797c37d341b0d23557e804ec8 Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Mon, 16 Jan 2023 15:29:29 -0500 Subject: [PATCH 02/52] Incomplete json check --- podpac/core/test/test_node.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/podpac/core/test/test_node.py b/podpac/core/test/test_node.py index 5a1102e8..1e8c1bfa 100644 --- a/podpac/core/test/test_node.py +++ b/podpac/core/test/test_node.py @@ -1291,9 +1291,35 @@ def test_from_proper_json(self): "podpac_version": "3.2.0" } """ - + # Check that the order doesn't matter. Because .from_json returns the output node, also checks correct output_node is returned assert(Node.from_json(not_ordered_json).definition == Node.from_json(ordered_json.definition)) + # Check that incomplete json will throw ValueError: + incomplete_json = """ + { + "Arange": { + "node": "core.algorithm.utility.Arange" + }, + "Arithmetic": { + "node": "core.algorithm.generic.Arithmetic", + "attrs": { + "eqn": "a+b", + "params": { + + } + }, + "inputs": { + "a": "SinCoords", + "b": "Arange" + } + }, + "podpac_version": "3.2.0" + } + """ + with pytest.raises(ValueError): + Node.from_json(incomplete_json) + + class TestNoCacheMixin(object): class NoCacheNode(NoCacheMixin, Node): From a180ff6eab516af5baf8ab57bc265f78728c5a00 Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Mon, 16 Jan 2023 15:30:35 -0500 Subject: [PATCH 03/52] Incomplete json check --- podpac/core/test/test_node.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/podpac/core/test/test_node.py b/podpac/core/test/test_node.py index 5a1102e8..1e8c1bfa 100644 --- a/podpac/core/test/test_node.py +++ b/podpac/core/test/test_node.py @@ -1291,9 +1291,35 @@ def test_from_proper_json(self): "podpac_version": "3.2.0" } """ - + # Check that the order doesn't matter. Because .from_json returns the output node, also checks correct output_node is returned assert(Node.from_json(not_ordered_json).definition == Node.from_json(ordered_json.definition)) + # Check that incomplete json will throw ValueError: + incomplete_json = """ + { + "Arange": { + "node": "core.algorithm.utility.Arange" + }, + "Arithmetic": { + "node": "core.algorithm.generic.Arithmetic", + "attrs": { + "eqn": "a+b", + "params": { + + } + }, + "inputs": { + "a": "SinCoords", + "b": "Arange" + } + }, + "podpac_version": "3.2.0" + } + """ + with pytest.raises(ValueError): + Node.from_json(incomplete_json) + + class TestNoCacheMixin(object): class NoCacheNode(NoCacheMixin, Node): From bcda1c677541ed9d4d874bd73954f1633ac8e47a Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Mon, 16 Jan 2023 15:33:28 -0500 Subject: [PATCH 04/52] Revert "Incomplete json check" This reverts commit a180ff6eab516af5baf8ab57bc265f78728c5a00. --- podpac/core/test/test_node.py | 28 +--------------------------- 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/podpac/core/test/test_node.py b/podpac/core/test/test_node.py index 1e8c1bfa..5a1102e8 100644 --- a/podpac/core/test/test_node.py +++ b/podpac/core/test/test_node.py @@ -1291,34 +1291,8 @@ def test_from_proper_json(self): "podpac_version": "3.2.0" } """ - # Check that the order doesn't matter. Because .from_json returns the output node, also checks correct output_node is returned - assert(Node.from_json(not_ordered_json).definition == Node.from_json(ordered_json.definition)) - - # Check that incomplete json will throw ValueError: - incomplete_json = """ - { - "Arange": { - "node": "core.algorithm.utility.Arange" - }, - "Arithmetic": { - "node": "core.algorithm.generic.Arithmetic", - "attrs": { - "eqn": "a+b", - "params": { - - } - }, - "inputs": { - "a": "SinCoords", - "b": "Arange" - } - }, - "podpac_version": "3.2.0" - } - """ - with pytest.raises(ValueError): - Node.from_json(incomplete_json) + assert(Node.from_json(not_ordered_json).definition == Node.from_json(ordered_json.definition)) class TestNoCacheMixin(object): From d1db01bc2aeb5f0b1baa65aab23948de26d98ed3 Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Mon, 16 Jan 2023 16:26:10 -0500 Subject: [PATCH 05/52] adjust inputs, fix test_node typo --- podpac/core/node.py | 83 +++++++++++++++++++++++++++++++---- podpac/core/test/test_node.py | 28 +++++++++++- 2 files changed, 101 insertions(+), 10 deletions(-) diff --git a/podpac/core/node.py b/podpac/core/node.py index d04a85dd..b6337ca0 100644 --- a/podpac/core/node.py +++ b/podpac/core/node.py @@ -875,10 +875,10 @@ def from_definition(cls, definition): kwargs[k] = v for k, v in d.get("inputs", {}).items(): - kwargs[k] = _lookup_input(nodes, name, v) + kwargs[k], nodes = _lookup_input(nodes, name, v, definition) for k, v in d.get("lookup_attrs", {}).items(): - kwargs[k] = _lookup_attr(nodes, name, v) + kwargs[k] = _lookup_attr(nodes, name, v, definition) if "style" in d: style_class = getattr(node_class, "style", Style) @@ -1208,13 +1208,13 @@ def get_ui_spec(cls, help_as_html=False): return spec -def _lookup_input(nodes, name, value): +def _lookup_input(nodes, name, value, definition): # containers if isinstance(value, list): - return [_lookup_input(nodes, name, elem) for elem in value] + return [_lookup_input(nodes, name, elem, definition) for elem in value] if isinstance(value, dict): - return {k: _lookup_input(nodes, name, v) for k, v in value.items()} + return {k: _lookup_input(nodes, name, v, definition) for k, v in value.items()} # node reference if not isinstance(value, six.string_types): @@ -1222,28 +1222,93 @@ def _lookup_input(nodes, name, value): "Invalid definition for node '%s': invalid reference '%s' of type '%s' in inputs" % (name, value, type(value)) ) + # node not yet discovered yet + if not value in nodes: + # Look for it in the definition items: + for found_name, d in definition.items(): + if value != found_name: + continue + + # get node class + module_root = d.get("plugin", "podpac") + node_string = "%s.%s" % (module_root, d["node"]) + module_name, node_name = node_string.rsplit(".", 1) + try: + module = importlib.import_module(module_name) + except ImportError: + raise ValueError("Invalid definition for node '%s': no module found '%s'" % (name, module_name)) + try: + node_class = getattr(module, node_name) + except AttributeError: + raise ValueError( + "Invalid definition for node '%s': class '%s' not found in module '%s'" + % (found_name, node_name, module_name) + ) + + kwargs = {} + for k, v in d.get("attrs", {}).items(): + kwargs[k] = v + + for k, v in d.get("inputs", {}).items(): + kwargs[k], nodes = _lookup_input(nodes, found_name, v, definition) + + for k, v in d.get("lookup_attrs", {}).items(): + kwargs[k] = _lookup_attr(nodes, found_name, v, definition) + + if "style" in d: + style_class = getattr(node_class, "style", Style) + if isinstance(style_class, tl.TraitType): + # Now we actually have to look through the class to see + # if there is a custom initializer for style + for attr in dir(node_class): + atr = getattr(node_class, attr) + if not isinstance(atr, tl.traitlets.DefaultHandler) or atr.trait_name != "style": + continue + try: + style_class = atr(node_class) + except Exception as e: + # print ("couldn't make style from class", e) + try: + style_class = atr(node_class()) + except: + # print ("couldn't make style from class instance", e) + style_class = style_class.klass + try: + kwargs["style"] = style_class.from_definition(d["style"]) + except Exception as e: + kwargs["style"] = Style.from_definition(d["style"]) + # print ("couldn't make style from inferred style class", e) + + for k in d: + if k not in ["node", "inputs", "attrs", "lookup_attrs", "plugin", "style"]: + raise ValueError("Invalid definition for node '%s': unexpected property '%s'" % (name, k)) + + nodes[found_name] = node_class(**kwargs) + nodes = nodes + # found! + break if not value in nodes: raise ValueError( "Invalid definition for node '%s': reference to nonexistent node '%s' in inputs" % (name, value) ) - + print(value) node = nodes[value] # copy in debug mode if settings["DEBUG"]: node = deepcopy(node) - return node + return node, nodes -def _lookup_attr(nodes, name, value): +def _lookup_attr(nodes, name, value, definition): # containers if isinstance(value, list): return [_lookup_attr(nodes, name, elem) for elem in value] if isinstance(value, dict): - return {_k: _lookup_attr(nodes, name, v) for k, v in value.items()} + return {k: _lookup_attr(nodes, name, v) for k, v in value.items()} if not isinstance(value, six.string_types): raise ValueError( diff --git a/podpac/core/test/test_node.py b/podpac/core/test/test_node.py index 5a1102e8..c3da5074 100644 --- a/podpac/core/test/test_node.py +++ b/podpac/core/test/test_node.py @@ -1291,8 +1291,34 @@ def test_from_proper_json(self): "podpac_version": "3.2.0" } """ + # Check that the order doesn't matter. Because .from_json returns the output node, also checks correct output_node is returned + assert(Node.from_json(not_ordered_json).definition == Node.from_json(ordered_json).definition) + + # Check that incomplete json will throw ValueError: + incomplete_json = """ + { + "Arange": { + "node": "core.algorithm.utility.Arange" + }, + "Arithmetic": { + "node": "core.algorithm.generic.Arithmetic", + "attrs": { + "eqn": "a+b", + "params": { + + } + }, + "inputs": { + "a": "SinCoords", + "b": "Arange" + } + }, + "podpac_version": "3.2.0" + } + """ + with pytest.raises(ValueError): + Node.from_json(incomplete_json) - assert(Node.from_json(not_ordered_json).definition == Node.from_json(ordered_json.definition)) class TestNoCacheMixin(object): From e45a4a2990db31e1ca86648977021aacab2519cc Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Mon, 16 Jan 2023 16:46:36 -0500 Subject: [PATCH 06/52] added another case --- ex_json.JSON | 29 ++++++++++++++++++++++++++ podpac/core/test/test_node.py | 38 ++++++++++++++++++++++++++++++++++- 2 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 ex_json.JSON diff --git a/ex_json.JSON b/ex_json.JSON new file mode 100644 index 00000000..2547ee97 --- /dev/null +++ b/ex_json.JSON @@ -0,0 +1,29 @@ +{ + "Arithmetic": { + "node": "core.algorithm.generic.Arithmetic", + "attrs": { + "eqn": "a+b", + "params": { + + } + }, + "inputs": { + "a": "SinCoords", + "b": "Arange" + } + }, + "SinCoords": { + "node": "core.algorithm.utility.SinCoords", + "style": { + "colormap": "jet", + "clim": [ + -1.0, + 1.0 + ] + } + }, + "Arange": { + "node": "core.algorithm.utility.Arange" + }, + "podpac_version": "3.2.0" +} diff --git a/podpac/core/test/test_node.py b/podpac/core/test/test_node.py index c3da5074..2adb0200 100644 --- a/podpac/core/test/test_node.py +++ b/podpac/core/test/test_node.py @@ -1260,6 +1260,37 @@ def test_from_proper_json(self): "podpac_version": "3.2.0" } """ + not_ordered_json_2 = """ + { + "SinCoords": { + "node": "core.algorithm.utility.SinCoords", + "style": { + "colormap": "jet", + "clim": [ + -1.0, + 1.0 + ] + } + }, + "Arithmetic": { + "node": "core.algorithm.generic.Arithmetic", + "attrs": { + "eqn": "a+b", + "params": { + + } + }, + "inputs": { + "a": "SinCoords", + "b": "Arange" + } + }, + "Arange": { + "node": "core.algorithm.utility.Arange" + }, + "podpac_version": "3.2.0" + } + """ ordered_json = """ { "SinCoords": { @@ -1292,7 +1323,10 @@ def test_from_proper_json(self): } """ # Check that the order doesn't matter. Because .from_json returns the output node, also checks correct output_node is returned - assert(Node.from_json(not_ordered_json).definition == Node.from_json(ordered_json).definition) + not_ordered_pipe = Node.from_json(not_ordered_json) + not_ordered_pipe_2 = Node.from_json(not_ordered_json_2) + ordered_pipe = Node.from_json(ordered_json) + assert(not_ordered_pipe.definition == ordered_pipe.definition == not_ordered_pipe_2.definition) # Check that incomplete json will throw ValueError: incomplete_json = """ @@ -1319,6 +1353,8 @@ def test_from_proper_json(self): with pytest.raises(ValueError): Node.from_json(incomplete_json) + # Check for load issues with nodes with attributes + class TestNoCacheMixin(object): From 39bfbd77ff610f80ca80edf2f3b0adf23b055406 Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Mon, 16 Jan 2023 18:29:42 -0500 Subject: [PATCH 07/52] Cleanup --- podpac/core/node.py | 7 +++---- podpac/core/test/test_node.py | 4 ---- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/podpac/core/node.py b/podpac/core/node.py index b6337ca0..93d01b17 100644 --- a/podpac/core/node.py +++ b/podpac/core/node.py @@ -878,7 +878,7 @@ def from_definition(cls, definition): kwargs[k], nodes = _lookup_input(nodes, name, v, definition) for k, v in d.get("lookup_attrs", {}).items(): - kwargs[k] = _lookup_attr(nodes, name, v, definition) + kwargs[k] = _lookup_attr(nodes, name, v) if "style" in d: style_class = getattr(node_class, "style", Style) @@ -1253,7 +1253,7 @@ def _lookup_input(nodes, name, value, definition): kwargs[k], nodes = _lookup_input(nodes, found_name, v, definition) for k, v in d.get("lookup_attrs", {}).items(): - kwargs[k] = _lookup_attr(nodes, found_name, v, definition) + kwargs[k] = _lookup_attr(nodes, found_name, v) if "style" in d: style_class = getattr(node_class, "style", Style) @@ -1284,7 +1284,6 @@ def _lookup_input(nodes, name, value, definition): raise ValueError("Invalid definition for node '%s': unexpected property '%s'" % (name, k)) nodes[found_name] = node_class(**kwargs) - nodes = nodes # found! break @@ -1302,7 +1301,7 @@ def _lookup_input(nodes, name, value, definition): return node, nodes -def _lookup_attr(nodes, name, value, definition): +def _lookup_attr(nodes, name, value): # containers if isinstance(value, list): return [_lookup_attr(nodes, name, elem) for elem in value] diff --git a/podpac/core/test/test_node.py b/podpac/core/test/test_node.py index 2adb0200..ce567a4e 100644 --- a/podpac/core/test/test_node.py +++ b/podpac/core/test/test_node.py @@ -1353,10 +1353,6 @@ def test_from_proper_json(self): with pytest.raises(ValueError): Node.from_json(incomplete_json) - # Check for load issues with nodes with attributes - - - class TestNoCacheMixin(object): class NoCacheNode(NoCacheMixin, Node): pass From b01b9af0084b9c4ad15a4342fcfe4eeea60b205f Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Mon, 16 Jan 2023 19:16:53 -0500 Subject: [PATCH 08/52] Delete ex_json, fix err. print --- ex_json.JSON | 29 ----------------------------- podpac/core/node.py | 1 - 2 files changed, 30 deletions(-) delete mode 100644 ex_json.JSON diff --git a/ex_json.JSON b/ex_json.JSON deleted file mode 100644 index 2547ee97..00000000 --- a/ex_json.JSON +++ /dev/null @@ -1,29 +0,0 @@ -{ - "Arithmetic": { - "node": "core.algorithm.generic.Arithmetic", - "attrs": { - "eqn": "a+b", - "params": { - - } - }, - "inputs": { - "a": "SinCoords", - "b": "Arange" - } - }, - "SinCoords": { - "node": "core.algorithm.utility.SinCoords", - "style": { - "colormap": "jet", - "clim": [ - -1.0, - 1.0 - ] - } - }, - "Arange": { - "node": "core.algorithm.utility.Arange" - }, - "podpac_version": "3.2.0" -} diff --git a/podpac/core/node.py b/podpac/core/node.py index 93d01b17..4420bfd1 100644 --- a/podpac/core/node.py +++ b/podpac/core/node.py @@ -1291,7 +1291,6 @@ def _lookup_input(nodes, name, value, definition): raise ValueError( "Invalid definition for node '%s': reference to nonexistent node '%s' in inputs" % (name, value) ) - print(value) node = nodes[value] # copy in debug mode From 3fb5048ef08d00fd69e95d05343ebe02db73fc98 Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Mon, 16 Jan 2023 19:35:32 -0500 Subject: [PATCH 09/52] Encapsulate kwargs loop --- podpac/core/node.py | 176 ++++++++++++++++---------------------------- 1 file changed, 62 insertions(+), 114 deletions(-) diff --git a/podpac/core/node.py b/podpac/core/node.py index 4420bfd1..9e03f3d5 100644 --- a/podpac/core/node.py +++ b/podpac/core/node.py @@ -853,63 +853,8 @@ def from_definition(cls, definition): if "node" not in d: raise ValueError("Invalid definition for node '%s': 'node' property required" % name) - # get node class - module_root = d.get("plugin", "podpac") - node_string = "%s.%s" % (module_root, d["node"]) - module_name, node_name = node_string.rsplit(".", 1) - try: - module = importlib.import_module(module_name) - except ImportError: - raise ValueError("Invalid definition for node '%s': no module found '%s'" % (name, module_name)) - try: - node_class = getattr(module, node_name) - except AttributeError: - raise ValueError( - "Invalid definition for node '%s': class '%s' not found in module '%s'" - % (name, node_name, module_name) - ) - - # parse and configure kwargs - kwargs = {} - for k, v in d.get("attrs", {}).items(): - kwargs[k] = v - - for k, v in d.get("inputs", {}).items(): - kwargs[k], nodes = _lookup_input(nodes, name, v, definition) - - for k, v in d.get("lookup_attrs", {}).items(): - kwargs[k] = _lookup_attr(nodes, name, v) - - if "style" in d: - style_class = getattr(node_class, "style", Style) - if isinstance(style_class, tl.TraitType): - # Now we actually have to look through the class to see - # if there is a custom initializer for style - for attr in dir(node_class): - atr = getattr(node_class, attr) - if not isinstance(atr, tl.traitlets.DefaultHandler) or atr.trait_name != "style": - continue - try: - style_class = atr(node_class) - except Exception as e: - # print ("couldn't make style from class", e) - try: - style_class = atr(node_class()) - except: - # print ("couldn't make style from class instance", e) - style_class = style_class.klass - try: - kwargs["style"] = style_class.from_definition(d["style"]) - except Exception as e: - kwargs["style"] = Style.from_definition(d["style"]) - # print ("couldn't make style from inferred style class", e) - - for k in d: - if k not in ["node", "inputs", "attrs", "lookup_attrs", "plugin", "style"]: - raise ValueError("Invalid definition for node '%s': unexpected property '%s'" % (name, k)) - - nodes[name] = node_class(**kwargs) - + _process_kwargs(name, d, definition, nodes) + return list(nodes.values())[-1] @classmethod @@ -1228,63 +1173,9 @@ def _lookup_input(nodes, name, value, definition): for found_name, d in definition.items(): if value != found_name: continue + # Load the node into nodes + _process_kwargs(found_name, d, definition, nodes) - # get node class - module_root = d.get("plugin", "podpac") - node_string = "%s.%s" % (module_root, d["node"]) - module_name, node_name = node_string.rsplit(".", 1) - try: - module = importlib.import_module(module_name) - except ImportError: - raise ValueError("Invalid definition for node '%s': no module found '%s'" % (name, module_name)) - try: - node_class = getattr(module, node_name) - except AttributeError: - raise ValueError( - "Invalid definition for node '%s': class '%s' not found in module '%s'" - % (found_name, node_name, module_name) - ) - - kwargs = {} - for k, v in d.get("attrs", {}).items(): - kwargs[k] = v - - for k, v in d.get("inputs", {}).items(): - kwargs[k], nodes = _lookup_input(nodes, found_name, v, definition) - - for k, v in d.get("lookup_attrs", {}).items(): - kwargs[k] = _lookup_attr(nodes, found_name, v) - - if "style" in d: - style_class = getattr(node_class, "style", Style) - if isinstance(style_class, tl.TraitType): - # Now we actually have to look through the class to see - # if there is a custom initializer for style - for attr in dir(node_class): - atr = getattr(node_class, attr) - if not isinstance(atr, tl.traitlets.DefaultHandler) or atr.trait_name != "style": - continue - try: - style_class = atr(node_class) - except Exception as e: - # print ("couldn't make style from class", e) - try: - style_class = atr(node_class()) - except: - # print ("couldn't make style from class instance", e) - style_class = style_class.klass - try: - kwargs["style"] = style_class.from_definition(d["style"]) - except Exception as e: - kwargs["style"] = Style.from_definition(d["style"]) - # print ("couldn't make style from inferred style class", e) - - for k in d: - if k not in ["node", "inputs", "attrs", "lookup_attrs", "plugin", "style"]: - raise ValueError("Invalid definition for node '%s': unexpected property '%s'" % (name, k)) - - nodes[found_name] = node_class(**kwargs) - # found! break if not value in nodes: @@ -1299,7 +1190,6 @@ def _lookup_input(nodes, name, value, definition): return node, nodes - def _lookup_attr(nodes, name, value): # containers if isinstance(value, list): @@ -1338,6 +1228,64 @@ def _lookup_attr(nodes, name, value): return attr +def _process_kwargs(name, d, definition, nodes): + # get node class + module_root = d.get("plugin", "podpac") + node_string = "%s.%s" % (module_root, d["node"]) + module_name, node_name = node_string.rsplit(".", 1) + try: + module = importlib.import_module(module_name) + except ImportError: + raise ValueError("Invalid definition for node '%s': no module found '%s'" % (name, module_name)) + try: + node_class = getattr(module, node_name) + except AttributeError: + raise ValueError( + "Invalid definition for node '%s': class '%s' not found in module '%s'" + % (name, node_name, module_name) + ) + + kwargs = {} + for k, v in d.get("attrs", {}).items(): + kwargs[k] = v + + for k, v in d.get("inputs", {}).items(): + kwargs[k], nodes = _lookup_input(nodes, name, v, definition) + + for k, v in d.get("lookup_attrs", {}).items(): + kwargs[k] = _lookup_attr(nodes, name, v) + + if "style" in d: + style_class = getattr(node_class, "style", Style) + if isinstance(style_class, tl.TraitType): + # Now we actually have to look through the class to see + # if there is a custom initializer for style + for attr in dir(node_class): + atr = getattr(node_class, attr) + if not isinstance(atr, tl.traitlets.DefaultHandler) or atr.trait_name != "style": + continue + try: + style_class = atr(node_class) + except Exception as e: + # print ("couldn't make style from class", e) + try: + style_class = atr(node_class()) + except: + # print ("couldn't make style from class instance", e) + style_class = style_class.klass + try: + kwargs["style"] = style_class.from_definition(d["style"]) + except Exception as e: + kwargs["style"] = Style.from_definition(d["style"]) + # print ("couldn't make style from inferred style class", e) + + for k in d: + if k not in ["node", "inputs", "attrs", "lookup_attrs", "plugin", "style"]: + raise ValueError("Invalid definition for node '%s': unexpected property '%s'" % (name, k)) + + nodes[name] = node_class(**kwargs) + + # --------------------------------------------------------# # Mixins # --------------------------------------------------------# From 00f053004d786edffe9a3aef432c08051181389a Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Mon, 16 Jan 2023 20:15:43 -0500 Subject: [PATCH 10/52] Added podpac_order key for from_definition --- podpac/core/node.py | 12 ++++++++++-- podpac/core/test/test_node.py | 3 ++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/podpac/core/node.py b/podpac/core/node.py index 9e03f3d5..89c62796 100644 --- a/podpac/core/node.py +++ b/podpac/core/node.py @@ -846,7 +846,11 @@ def from_definition(cls, definition): # parse node definitions in order nodes = OrderedDict() + output_node = None for name, d in definition.items(): + if name == "podpac_output_node": + output_node = d + continue if name == "podpac_version": continue @@ -854,8 +858,12 @@ def from_definition(cls, definition): raise ValueError("Invalid definition for node '%s': 'node' property required" % name) _process_kwargs(name, d, definition, nodes) - - return list(nodes.values())[-1] + + # look for podpac_output_node attribute + if output_node is None: + return list(nodes.values())[-1] + + return nodes[output_node] @classmethod def from_json(cls, s): diff --git a/podpac/core/test/test_node.py b/podpac/core/test/test_node.py index ce567a4e..ad8bf423 100644 --- a/podpac/core/test/test_node.py +++ b/podpac/core/test/test_node.py @@ -1257,7 +1257,8 @@ def test_from_proper_json(self): "Arange": { "node": "core.algorithm.utility.Arange" }, - "podpac_version": "3.2.0" + "podpac_version": "3.2.0", + "podpac_output_node": "Arithmetic" } """ not_ordered_json_2 = """ From 9d1b5e9d3b069d29ace4b8c27e85ac31e707076c Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Mon, 16 Jan 2023 20:54:12 -0500 Subject: [PATCH 11/52] Unittest for output_node, check for invalid value --- podpac/core/node.py | 5 ++ podpac/core/test/test_node.py | 105 +++++++++++++++++++++++++++++++++- 2 files changed, 109 insertions(+), 1 deletion(-) diff --git a/podpac/core/node.py b/podpac/core/node.py index 89c62796..18360b31 100644 --- a/podpac/core/node.py +++ b/podpac/core/node.py @@ -863,6 +863,11 @@ def from_definition(cls, definition): if output_node is None: return list(nodes.values())[-1] + + if output_node not in nodes: + raise ValueError( + "Invalid definition for value 'podpac_output_node': reference to nonexistent node '%s' in lookup_attrs" % (output_node) + ) return nodes[output_node] @classmethod diff --git a/podpac/core/test/test_node.py b/podpac/core/test/test_node.py index ad8bf423..cc972254 100644 --- a/podpac/core/test/test_node.py +++ b/podpac/core/test/test_node.py @@ -1258,7 +1258,6 @@ def test_from_proper_json(self): "node": "core.algorithm.utility.Arange" }, "podpac_version": "3.2.0", - "podpac_output_node": "Arithmetic" } """ not_ordered_json_2 = """ @@ -1354,6 +1353,110 @@ def test_from_proper_json(self): with pytest.raises(ValueError): Node.from_json(incomplete_json) + def test_output_node(self): + included_json = """ + { + "Arithmetic": { + "node": "core.algorithm.generic.Arithmetic", + "attrs": { + "eqn": "a+b", + "params": { + + } + }, + "inputs": { + "a": "SinCoords", + "b": "Arange" + } + }, + "SinCoords": { + "node": "core.algorithm.utility.SinCoords", + "style": { + "colormap": "jet", + "clim": [ + -1.0, + 1.0 + ] + } + }, + "Arange": { + "node": "core.algorithm.utility.Arange" + }, + "podpac_version": "3.2.0", + "podpac_output_node": "Arithmetic" + } + """ + ordered_json = """ + { + "SinCoords": { + "node": "core.algorithm.utility.SinCoords", + "style": { + "colormap": "jet", + "clim": [ + -1.0, + 1.0 + ] + } + }, + "Arange": { + "node": "core.algorithm.utility.Arange" + }, + "Arithmetic": { + "node": "core.algorithm.generic.Arithmetic", + "attrs": { + "eqn": "a+b", + "params": { + + } + }, + "inputs": { + "a": "SinCoords", + "b": "Arange" + } + }, + "podpac_version": "3.2.0" + } + """ + included_pipe = Node.from_json(included_json) + ordered_pipe = Node.from_json(ordered_json) + assert(included_pipe.definition == ordered_pipe.definition) + + wrong_name_json = """ + { + "SinCoords": { + "node": "core.algorithm.utility.SinCoords", + "style": { + "colormap": "jet", + "clim": [ + -1.0, + 1.0 + ] + } + }, + "Arange": { + "node": "core.algorithm.utility.Arange" + }, + "Arithmetic": { + "node": "core.algorithm.generic.Arithmetic", + "attrs": { + "eqn": "a+b", + "params": { + + } + }, + "inputs": { + "a": "SinCoords", + "b": "Arange" + } + }, + "podpac_version": "3.2.0", + "podpac_output_node": "Sum" + } + """ + with pytest.raises(ValueError): + Node.from_json(wrong_name_json) + + class TestNoCacheMixin(object): class NoCacheNode(NoCacheMixin, Node): pass From 35565d2771bd8c892b9079400071a780a01eb7fe Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Mon, 16 Jan 2023 21:23:04 -0500 Subject: [PATCH 12/52] typo --- podpac/core/test/test_node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/podpac/core/test/test_node.py b/podpac/core/test/test_node.py index cc972254..188c7e4c 100644 --- a/podpac/core/test/test_node.py +++ b/podpac/core/test/test_node.py @@ -1257,7 +1257,7 @@ def test_from_proper_json(self): "Arange": { "node": "core.algorithm.utility.Arange" }, - "podpac_version": "3.2.0", + "podpac_version": "3.2.0" } """ not_ordered_json_2 = """ From 33ff5aaa4f206bc365ba561ab43a7d4885ecdf0e Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Tue, 17 Jan 2023 16:33:05 -0500 Subject: [PATCH 13/52] Fix Tests --- podpac/core/node.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/podpac/core/node.py b/podpac/core/node.py index 18360b31..18304b1c 100644 --- a/podpac/core/node.py +++ b/podpac/core/node.py @@ -1201,7 +1201,7 @@ def _lookup_input(nodes, name, value, definition): if settings["DEBUG"]: node = deepcopy(node) - return node, nodes + return node def _lookup_attr(nodes, name, value): # containers @@ -1263,7 +1263,7 @@ def _process_kwargs(name, d, definition, nodes): kwargs[k] = v for k, v in d.get("inputs", {}).items(): - kwargs[k], nodes = _lookup_input(nodes, name, v, definition) + kwargs[k]= _lookup_input(nodes, name, v, definition) for k, v in d.get("lookup_attrs", {}).items(): kwargs[k] = _lookup_attr(nodes, name, v) From 1f40f7b45030d0e00fb707678b96d53caa1cb575 Mon Sep 17 00:00:00 2001 From: Matt Ueckermann Date: Thu, 19 Jan 2023 09:05:22 -0500 Subject: [PATCH 14/52] TEST: Ensuring hash is not affected by order --- podpac/core/test/test_node.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/podpac/core/test/test_node.py b/podpac/core/test/test_node.py index 188c7e4c..055c5972 100644 --- a/podpac/core/test/test_node.py +++ b/podpac/core/test/test_node.py @@ -1227,7 +1227,7 @@ def test_from_definition_version_warning(self): with pytest.warns(UserWarning, match="node definition version mismatch"): node = Node.from_json(s) - + def test_from_proper_json(self): not_ordered_json = """ { @@ -1326,7 +1326,8 @@ def test_from_proper_json(self): not_ordered_pipe = Node.from_json(not_ordered_json) not_ordered_pipe_2 = Node.from_json(not_ordered_json_2) ordered_pipe = Node.from_json(ordered_json) - assert(not_ordered_pipe.definition == ordered_pipe.definition == not_ordered_pipe_2.definition) + assert not_ordered_pipe.definition == ordered_pipe.definition == not_ordered_pipe_2.definition + assert not_ordered_pipe.hash == ordered_pipe.hash # Check that incomplete json will throw ValueError: incomplete_json = """ @@ -1419,7 +1420,8 @@ def test_output_node(self): """ included_pipe = Node.from_json(included_json) ordered_pipe = Node.from_json(ordered_json) - assert(included_pipe.definition == ordered_pipe.definition) + assert included_pipe.definition == ordered_pipe.definition + assert included_pipe.hash == ordered_pipe.hash wrong_name_json = """ { From 6f59b42fe1c1a0273435c7d532d51178c5fca29e Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Thu, 19 Jan 2023 10:30:23 -0500 Subject: [PATCH 15/52] Added function comments --- podpac/core/node.py | 45 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/podpac/core/node.py b/podpac/core/node.py index 18304b1c..8858944b 100644 --- a/podpac/core/node.py +++ b/podpac/core/node.py @@ -1165,8 +1165,30 @@ def get_ui_spec(cls, help_as_html=False): spec.update(getattr(cls, "_ui_spec", {})) return spec - def _lookup_input(nodes, name, value, definition): + '''check if inputs of a node are stored in nodes, if not add them + + Parameters + ----------- + nodes: OrderedDict + Keys: Node names (strings) + Values: Node objects + + name: string + the Node whose inputs are being checked + + value: string, list, dictionary: + the Node (or collection of Nodes) which is being looked + + definition: pipeline definition + + Returns + -------- + node: the node searched for + + Note: this function calles _process_kwargs, which alters nodes by loading a Node if it is not yet in nodes. + + ''' # containers if isinstance(value, list): return [_lookup_input(nodes, name, elem, definition) for elem in value] @@ -1242,6 +1264,27 @@ def _lookup_attr(nodes, name, value): def _process_kwargs(name, d, definition, nodes): + '''create a node and add it to nodes + + Parameters + ----------- + nodes: OrderedDict + Keys: Node names (strings) + Values: Node objects + + name: string + the Node which will be created + + d: the definition of the node to be created + + definition: pipeline definition + + Returns + -------- + Nothing, but loads the node with name "name" and definition "d" into nodes + + + ''' # get node class module_root = d.get("plugin", "podpac") node_string = "%s.%s" % (module_root, d["node"]) From 2e54b21e1449fdfd5ed6d5ed52c478a65ad6e5e2 Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Fri, 20 Jan 2023 10:45:40 -0500 Subject: [PATCH 16/52] Refactor podpac.algorithm --- doc/source/api.rst | 44 +++++++++++++------------- podpac/{algorithm.py => algorithms.py} | 0 2 files changed, 22 insertions(+), 22 deletions(-) rename podpac/{algorithm.py => algorithms.py} (100%) diff --git a/doc/source/api.rst b/doc/source/api.rst index 4bdcc7c1..acad0da7 100644 --- a/doc/source/api.rst +++ b/doc/source/api.rst @@ -119,7 +119,7 @@ Split/Apply/Combine nodes with algorithms :toctree: api/ :template: class.rst - podpac.algorithm.Algorithm + podpac.algorithms.Algorithm .. rubric:: General Purpose @@ -127,10 +127,10 @@ Split/Apply/Combine nodes with algorithms :toctree: api/ :template: class.rst - podpac.algorithm.Arithmetic - podpac.algorithm.SinCoords - podpac.algorithm.Arange - podpac.algorithm.CoordData + podpac.algorithms.Arithmetic + podpac.algorithms.SinCoords + podpac.algorithms.Arange + podpac.algorithms.CoordData .. rubric:: Statistical Methods @@ -138,18 +138,18 @@ Split/Apply/Combine nodes with algorithms :toctree: api/ :template: class.rst - podpac.algorithm.Min - podpac.algorithm.Max - podpac.algorithm.Sum - podpac.algorithm.Count - podpac.algorithm.Mean - podpac.algorithm.Median - podpac.algorithm.Variance - podpac.algorithm.StandardDeviation - podpac.algorithm.Skew - podpac.algorithm.Kurtosis - podpac.algorithm.DayOfYear - podpac.algorithm.GroupReduce + podpac.algorithms.Min + podpac.algorithms.Max + podpac.algorithms.Sum + podpac.algorithms.Count + podpac.algorithms.Mean + podpac.algorithms.Median + podpac.algorithms.Variance + podpac.algorithms.StandardDeviation + podpac.algorithms.Skew + podpac.algorithms.Kurtosis + podpac.algorithms.DayOfYear + podpac.algorithms.GroupReduce .. rubric:: Coordinates Modification @@ -157,8 +157,8 @@ Split/Apply/Combine nodes with algorithms :toctree: api/ :template: class.rst - podpac.algorithm.ExpandCoordinates - podpac.algorithm.SelectCoordinates + podpac.algorithms.ExpandCoordinates + podpac.algorithms.SelectCoordinates .. rubric:: Signal Processing @@ -166,9 +166,9 @@ Split/Apply/Combine nodes with algorithms :toctree: api/ :template: class.rst - podpac.algorithm.Convolution - podpac.algorithm.SpatialConvolution - podpac.algorithm.TimeConvolution + podpac.algorithms.Convolution + podpac.algorithms.SpatialConvolution + podpac.algorithms.TimeConvolution Compositor Nodes ---------------- diff --git a/podpac/algorithm.py b/podpac/algorithms.py similarity index 100% rename from podpac/algorithm.py rename to podpac/algorithms.py From c6fd877e034b9640225457eac75abff04f3aa39c Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Fri, 20 Jan 2023 10:46:17 -0500 Subject: [PATCH 17/52] Revert "Refactor podpac.algorithm" This reverts commit 2e54b21e1449fdfd5ed6d5ed52c478a65ad6e5e2. --- doc/source/api.rst | 44 +++++++++++++------------- podpac/{algorithms.py => algorithm.py} | 0 2 files changed, 22 insertions(+), 22 deletions(-) rename podpac/{algorithms.py => algorithm.py} (100%) diff --git a/doc/source/api.rst b/doc/source/api.rst index acad0da7..4bdcc7c1 100644 --- a/doc/source/api.rst +++ b/doc/source/api.rst @@ -119,7 +119,7 @@ Split/Apply/Combine nodes with algorithms :toctree: api/ :template: class.rst - podpac.algorithms.Algorithm + podpac.algorithm.Algorithm .. rubric:: General Purpose @@ -127,10 +127,10 @@ Split/Apply/Combine nodes with algorithms :toctree: api/ :template: class.rst - podpac.algorithms.Arithmetic - podpac.algorithms.SinCoords - podpac.algorithms.Arange - podpac.algorithms.CoordData + podpac.algorithm.Arithmetic + podpac.algorithm.SinCoords + podpac.algorithm.Arange + podpac.algorithm.CoordData .. rubric:: Statistical Methods @@ -138,18 +138,18 @@ Split/Apply/Combine nodes with algorithms :toctree: api/ :template: class.rst - podpac.algorithms.Min - podpac.algorithms.Max - podpac.algorithms.Sum - podpac.algorithms.Count - podpac.algorithms.Mean - podpac.algorithms.Median - podpac.algorithms.Variance - podpac.algorithms.StandardDeviation - podpac.algorithms.Skew - podpac.algorithms.Kurtosis - podpac.algorithms.DayOfYear - podpac.algorithms.GroupReduce + podpac.algorithm.Min + podpac.algorithm.Max + podpac.algorithm.Sum + podpac.algorithm.Count + podpac.algorithm.Mean + podpac.algorithm.Median + podpac.algorithm.Variance + podpac.algorithm.StandardDeviation + podpac.algorithm.Skew + podpac.algorithm.Kurtosis + podpac.algorithm.DayOfYear + podpac.algorithm.GroupReduce .. rubric:: Coordinates Modification @@ -157,8 +157,8 @@ Split/Apply/Combine nodes with algorithms :toctree: api/ :template: class.rst - podpac.algorithms.ExpandCoordinates - podpac.algorithms.SelectCoordinates + podpac.algorithm.ExpandCoordinates + podpac.algorithm.SelectCoordinates .. rubric:: Signal Processing @@ -166,9 +166,9 @@ Split/Apply/Combine nodes with algorithms :toctree: api/ :template: class.rst - podpac.algorithms.Convolution - podpac.algorithms.SpatialConvolution - podpac.algorithms.TimeConvolution + podpac.algorithm.Convolution + podpac.algorithm.SpatialConvolution + podpac.algorithm.TimeConvolution Compositor Nodes ---------------- diff --git a/podpac/algorithms.py b/podpac/algorithm.py similarity index 100% rename from podpac/algorithms.py rename to podpac/algorithm.py From dd570e1683626b66a542b022977056b605d2c244 Mon Sep 17 00:00:00 2001 From: Matt Ueckermann Date: Fri, 20 Jan 2023 11:33:14 -0500 Subject: [PATCH 18/52] FIX: Ensuring that coord_type == 'slice' produces a slice even when interpolator_queue is empty. --- podpac/core/interpolation/interpolation_manager.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/podpac/core/interpolation/interpolation_manager.py b/podpac/core/interpolation/interpolation_manager.py index daaa3f23..9ed2e92d 100644 --- a/podpac/core/interpolation/interpolation_manager.py +++ b/podpac/core/interpolation/interpolation_manager.py @@ -536,7 +536,14 @@ def select_coordinates(self, source_coordinates, eval_coordinates, index_type="n if isinstance(selected_coords_idx[i], tuple): selected_coords_idx2.extend(selected_coords_idx[i]) else: - selected_coords_idx2.append(selected_coords_idx[i]) + if isinstance(selected_coords_idx[i], np.ndarray): + # This happens when the interpolator_queue is empty, so we have to turn the + # initialized coordinates into slices instead of numpy arrays + selected_coords_idx2.append( + slice(selected_coords_idx[i].min(), selected_coords_idx[i].max() + 1) + ) + else: + selected_coords_idx2.append(selected_coords_idx[i]) selected_coords_idx2 = tuple(selected_coords_idx2) else: From 1c4e720d45999631a4542e2376abc7ae8450412f Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Tue, 24 Jan 2023 15:43:38 -0500 Subject: [PATCH 19/52] MAINT: Pydap 3.3 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d58fb150..931439ce 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,7 @@ "beautifulsoup4>=4.6", "h5py>=2.9", "lxml>=4.2", - "pydap>=3.2", + "pydap>=3.3", "rasterio>=1.0", "zarr>=2.3", "owslib", From 0666aef1d1a08c8e1e4cacbfc9e7cb45b0e31141 Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Wed, 25 Jan 2023 07:24:02 -0500 Subject: [PATCH 20/52] MAINT: Propose API Scheme --- doc/source/api-scheme.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 doc/source/api-scheme.md diff --git a/doc/source/api-scheme.md b/doc/source/api-scheme.md new file mode 100644 index 00000000..53144fe0 --- /dev/null +++ b/doc/source/api-scheme.md @@ -0,0 +1,29 @@ +# API Scheme +A naming scheme for our API. The goal is to make the grammar of the entire API consistent. + +## Rules + +### Node Rules +All node names must be: +1. Nouns +2. Singular + +### Coordinates Rules +All Coordinates names must be: +1. Nouns + +### Core Directory Rules +Where a "Core Directory" is located inside podpac/core/* +All directory names must be: +1. Nouns +2. Pluarl + +### Package Rules: + +Where a "Package" is loaded by podpac.* + +All package names must be: +1. Nouns +2. Pluarl +*AND* +3. All packages must match their corresponding `podpac.core.*` directory name. \ No newline at end of file From 76798782d06cf34ad5505f00f7ddd910631ffa4a Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Wed, 25 Jan 2023 07:24:18 -0500 Subject: [PATCH 21/52] MAINT: Propose API Scheme --- doc/source/api-scheme.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 doc/source/api-scheme.md diff --git a/doc/source/api-scheme.md b/doc/source/api-scheme.md new file mode 100644 index 00000000..53144fe0 --- /dev/null +++ b/doc/source/api-scheme.md @@ -0,0 +1,29 @@ +# API Scheme +A naming scheme for our API. The goal is to make the grammar of the entire API consistent. + +## Rules + +### Node Rules +All node names must be: +1. Nouns +2. Singular + +### Coordinates Rules +All Coordinates names must be: +1. Nouns + +### Core Directory Rules +Where a "Core Directory" is located inside podpac/core/* +All directory names must be: +1. Nouns +2. Pluarl + +### Package Rules: + +Where a "Package" is loaded by podpac.* + +All package names must be: +1. Nouns +2. Pluarl +*AND* +3. All packages must match their corresponding `podpac.core.*` directory name. \ No newline at end of file From d638183b26804cb283411a71905c66d9a1e4bef7 Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Wed, 25 Jan 2023 07:24:41 -0500 Subject: [PATCH 22/52] Revert "MAINT: Propose API Scheme" This reverts commit 76798782d06cf34ad5505f00f7ddd910631ffa4a. --- doc/source/api-scheme.md | 29 ----------------------------- 1 file changed, 29 deletions(-) delete mode 100644 doc/source/api-scheme.md diff --git a/doc/source/api-scheme.md b/doc/source/api-scheme.md deleted file mode 100644 index 53144fe0..00000000 --- a/doc/source/api-scheme.md +++ /dev/null @@ -1,29 +0,0 @@ -# API Scheme -A naming scheme for our API. The goal is to make the grammar of the entire API consistent. - -## Rules - -### Node Rules -All node names must be: -1. Nouns -2. Singular - -### Coordinates Rules -All Coordinates names must be: -1. Nouns - -### Core Directory Rules -Where a "Core Directory" is located inside podpac/core/* -All directory names must be: -1. Nouns -2. Pluarl - -### Package Rules: - -Where a "Package" is loaded by podpac.* - -All package names must be: -1. Nouns -2. Pluarl -*AND* -3. All packages must match their corresponding `podpac.core.*` directory name. \ No newline at end of file From 7102024fa563aa3593e939ef9d13fef84e4a866d Mon Sep 17 00:00:00 2001 From: Matt Ueckermann Date: Wed, 25 Jan 2023 09:58:02 -0500 Subject: [PATCH 23/52] FIX: Fix error from newest version of traitlets. --- podpac/core/node.py | 47 +++++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/podpac/core/node.py b/podpac/core/node.py index 8858944b..85275e33 100644 --- a/podpac/core/node.py +++ b/podpac/core/node.py @@ -206,8 +206,8 @@ def __init__(self, **kwargs): self.set_trait(name, tkwargs.pop(name)) trait.read_only = True - # Call traitlets constructor - super(Node, self).__init__(**tkwargs) + # Call traitlets constructor + super(Node, self).__init__(**tkwargs) self._traits_initialized_guard = True @@ -862,12 +862,12 @@ def from_definition(cls, definition): # look for podpac_output_node attribute if output_node is None: return list(nodes.values())[-1] - if output_node not in nodes: - raise ValueError( - "Invalid definition for value 'podpac_output_node': reference to nonexistent node '%s' in lookup_attrs" % (output_node) - ) + raise ValueError( + "Invalid definition for value 'podpac_output_node': reference to nonexistent node '%s' in lookup_attrs" + % (output_node) + ) return nodes[output_node] @classmethod @@ -1165,21 +1165,22 @@ def get_ui_spec(cls, help_as_html=False): spec.update(getattr(cls, "_ui_spec", {})) return spec + def _lookup_input(nodes, name, value, definition): - '''check if inputs of a node are stored in nodes, if not add them + """check if inputs of a node are stored in nodes, if not add them Parameters ----------- nodes: OrderedDict Keys: Node names (strings) Values: Node objects - + name: string the Node whose inputs are being checked - + value: string, list, dictionary: - the Node (or collection of Nodes) which is being looked - + the Node (or collection of Nodes) which is being looked + definition: pipeline definition Returns @@ -1188,7 +1189,7 @@ def _lookup_input(nodes, name, value, definition): Note: this function calles _process_kwargs, which alters nodes by loading a Node if it is not yet in nodes. - ''' + """ # containers if isinstance(value, list): return [_lookup_input(nodes, name, elem, definition) for elem in value] @@ -1208,7 +1209,7 @@ def _lookup_input(nodes, name, value, definition): for found_name, d in definition.items(): if value != found_name: continue - # Load the node into nodes + # Load the node into nodes _process_kwargs(found_name, d, definition, nodes) break @@ -1225,6 +1226,7 @@ def _lookup_input(nodes, name, value, definition): return node + def _lookup_attr(nodes, name, value): # containers if isinstance(value, list): @@ -1264,19 +1266,19 @@ def _lookup_attr(nodes, name, value): def _process_kwargs(name, d, definition, nodes): - '''create a node and add it to nodes + """create a node and add it to nodes Parameters ----------- nodes: OrderedDict Keys: Node names (strings) Values: Node objects - + name: string the Node which will be created - + d: the definition of the node to be created - + definition: pipeline definition Returns @@ -1284,8 +1286,8 @@ def _process_kwargs(name, d, definition, nodes): Nothing, but loads the node with name "name" and definition "d" into nodes - ''' - # get node class + """ + # get node class module_root = d.get("plugin", "podpac") node_string = "%s.%s" % (module_root, d["node"]) module_name, node_name = node_string.rsplit(".", 1) @@ -1297,8 +1299,7 @@ def _process_kwargs(name, d, definition, nodes): node_class = getattr(module, node_name) except AttributeError: raise ValueError( - "Invalid definition for node '%s': class '%s' not found in module '%s'" - % (name, node_name, module_name) + "Invalid definition for node '%s': class '%s' not found in module '%s'" % (name, node_name, module_name) ) kwargs = {} @@ -1306,7 +1307,7 @@ def _process_kwargs(name, d, definition, nodes): kwargs[k] = v for k, v in d.get("inputs", {}).items(): - kwargs[k]= _lookup_input(nodes, name, v, definition) + kwargs[k] = _lookup_input(nodes, name, v, definition) for k, v in d.get("lookup_attrs", {}).items(): kwargs[k] = _lookup_attr(nodes, name, v) @@ -1340,7 +1341,7 @@ def _process_kwargs(name, d, definition, nodes): raise ValueError("Invalid definition for node '%s': unexpected property '%s'" % (name, k)) nodes[name] = node_class(**kwargs) - + # --------------------------------------------------------# # Mixins From 6d2b65c5a23c5ab4d987be7475c4726d11e1f882 Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Thu, 26 Jan 2023 14:36:31 -0500 Subject: [PATCH 24/52] WIP: Coordinate Resolution --- podpac/core/coordinates/coordinates.py | 56 +++++++++++++++++-- .../core/coordinates/test/test_coordinates.py | 25 +++++++++ 2 files changed, 75 insertions(+), 6 deletions(-) diff --git a/podpac/core/coordinates/coordinates.py b/podpac/core/coordinates/coordinates.py index 75653fd9..1bcf98d5 100644 --- a/podpac/core/coordinates/coordinates.py +++ b/podpac/core/coordinates/coordinates.py @@ -19,6 +19,7 @@ from six import string_types import pyproj import logging +from scipy import spatial import podpac from podpac.core.settings import settings @@ -1501,12 +1502,55 @@ def issubset(self, other): return all(c.issubset(other) for c in self.values()) - def is_stacked(self, dim): - if dim not in self.udims: - raise ValueError("Dimension {} is not in self.dims={}".format(dim, self.dims)) - elif dim not in self.dims: - return True - return False + def is_stacked(self, dim): #re-wrote to be able to iterate through c.dims + match (dim in self.dims) + (dim in self.udims): + case 0: # both false + raise ValueError("Dimension {} is not in self.dims={}".format(dim, self.dims)) + case 1: # one true, one false + return True + case 2: # both true + return False + + def horizontal_resolution(self, units='m'): + + resolution_dict = OrderedDict() + + # utility functions + def _get_shortest_distance(tree, query_point): + dd, ii = tree.query(query_point, k=2) + return dd[1] + + # BROKEN: doesn't work for stacked corods + def _get_unit_name(self, dim): + def _get_axis_index(self, dim): + for i in range(len(self.CRS.axis_info)): + if self.CRS.axis_info[i].abbrev==dim: + return i + return ValueError("Dimension {} not found in CRS.axis_info.".format(dim)) + return self.CRS.axis_info[_get_axis_index(self, dim)].unit_name + + # For each dimension + for dim in self.dims: + # if the dimension is stacked: + if self.is_stacked(dim): + # make KDTree + t = spatial.KDTree(self[dim].coordinates) + total_distance = 0 + for point in self[dim].coordinates: + total_distance += _get_shortest_distance(t, point) + resolution = total_distance / self[dim].size + # store resolution and units + resolution_dict[dim] = (resolution, _get_unit_name(self, dim.split('_')[0])) + else: + resolution = (self.bounds[dim][1] - self.bounds[dim][0]) / self.get(dim).size + resolution_dict[dim] = (resolution, _get_unit_name(self, dim)) + + # TODO: Convert resolution to desired units + + + return resolution_dict + + # ------------------------------------------------------------------------------------------------------------------ # Operators/Magic Methods diff --git a/podpac/core/coordinates/test/test_coordinates.py b/podpac/core/coordinates/test/test_coordinates.py index 97db6e36..2d410436 100644 --- a/podpac/core/coordinates/test/test_coordinates.py +++ b/podpac/core/coordinates/test/test_coordinates.py @@ -2202,3 +2202,28 @@ def test_affine_to_uniform(self): # unstacked uniform -> unstacked uniform assert c3.simplify() == c3 + + def test_resolution(self): + + lat = [0, 1, 2] + lon = [10, 40, 30] + time = ['2018-01-01', '2018-01-02'] + alt = [10, 3, -10] + + + # Test Unstacked Coords + c = Coordinates([lat, lon], dims=['lat','lon']) + print(c.horizontal_resolution()) + + # Test Stacked Coords + c = Coordinates([[lat, lon]], dims=['lat_lon']) + print(c.horizontal_resolution()) + + ''' + # Test Stacked/Unstacked Coords + c = Coordinates([[lat, lon], alt], dims=['lat_lon', 'alt']) + print(c.horizontal_resolution()) + c = Coordinates([[lat, lon], time], dims=['lat_lon', 'time']) + print(c.horizontal_resolution()) + ''' + From c9af5ade74944e4cbbeb1f5428b250eb8f688579 Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Mon, 30 Jan 2023 11:05:26 -0500 Subject: [PATCH 25/52] WIP: Horizontal Res using geodesic --- podpac/core/coordinates/coordinates.py | 87 +++++++++++-------- .../core/coordinates/test/test_coordinates.py | 9 +- 2 files changed, 60 insertions(+), 36 deletions(-) diff --git a/podpac/core/coordinates/coordinates.py b/podpac/core/coordinates/coordinates.py index 1bcf98d5..14214cb0 100644 --- a/podpac/core/coordinates/coordinates.py +++ b/podpac/core/coordinates/coordinates.py @@ -20,6 +20,8 @@ import pyproj import logging from scipy import spatial +import math +from geopy.distance import geodesic import podpac from podpac.core.settings import settings @@ -1511,45 +1513,62 @@ def is_stacked(self, dim): #re-wrote to be able to iterate through c.dims case 2: # both true return False - def horizontal_resolution(self, units='m'): - + ''' + Calculate horizontal resolution of coordinate system. + Assumes horizontal coords are in degrees. + ''' + def horizontal_resolution(self): + # dictionary: resolution_dict = OrderedDict() - # utility functions - def _get_shortest_distance(tree, query_point): - dd, ii = tree.query(query_point, k=2) - return dd[1] - - # BROKEN: doesn't work for stacked corods - def _get_unit_name(self, dim): - def _get_axis_index(self, dim): - for i in range(len(self.CRS.axis_info)): - if self.CRS.axis_info[i].abbrev==dim: - return i - return ValueError("Dimension {} not found in CRS.axis_info.".format(dim)) - return self.CRS.axis_info[_get_axis_index(self, dim)].unit_name - - # For each dimension - for dim in self.dims: - # if the dimension is stacked: - if self.is_stacked(dim): - # make KDTree - t = spatial.KDTree(self[dim].coordinates) - total_distance = 0 - for point in self[dim].coordinates: - total_distance += _get_shortest_distance(t, point) - resolution = total_distance / self[dim].size - # store resolution and units - resolution_dict[dim] = (resolution, _get_unit_name(self, dim.split('_')[0])) - else: - resolution = (self.bounds[dim][1] - self.bounds[dim][0]) / self.get(dim).size - resolution_dict[dim] = (resolution, _get_unit_name(self, dim)) - - # TODO: Convert resolution to desired units + '''Check if a dim::str is horizontal''' + def check_horizontal(dim): + for term in dim.split('_'): + if term == 'lat' or term == 'lon': + return True + return False - return resolution_dict + '''Return resolution for stacked coordinates''' + def stacked_resolution(dim): + sum = 0 + # Brute Force implementatiom + for point1 in self[dim].coordinates: + min_distance = float('inf') + for point2 in self[dim].coordinates: + # check if not current point + if point1.all() == point2.all(): + continue + # calculate distance in meters + distance = geodesic(point1, point2).m + # only add min distance + if distance < min_distance: + min_distance = distance + # add min_distance + if min_distance != float('inf'): + sum += min_distance + # return sum / total_points + return (sum / self[dim].size) * podpac.units('metre') + + '''Return resolution for unstacked coordiantes''' + def unstacked_resolution(dim): + top_bound = (self.bounds[dim][1], 0) + bottom_bound = (self.bounds[dim][0], 0) + return ((geodesic(top_bound, bottom_bound).m)/ self[dim].size) * podpac.units('metre') + for dim in self.dims: + # Is the dim lat/lon? + if not check_horizontal(dim): + continue + # stacked coordinate resolutions + if self.is_stacked(dim): + print("{} is stacked!".format(dim)) + resolution_dict[dim] = stacked_resolution(dim) + else: # unstacked resolution + resolution_dict[dim] = unstacked_resolution(dim) + print("{} is NOT stacked!".format(dim)) + + return resolution_dict # ------------------------------------------------------------------------------------------------------------------ diff --git a/podpac/core/coordinates/test/test_coordinates.py b/podpac/core/coordinates/test/test_coordinates.py index 2d410436..fdda9478 100644 --- a/podpac/core/coordinates/test/test_coordinates.py +++ b/podpac/core/coordinates/test/test_coordinates.py @@ -2203,6 +2203,8 @@ def test_affine_to_uniform(self): # unstacked uniform -> unstacked uniform assert c3.simplify() == c3 + +class TestResolutions(object): def test_resolution(self): lat = [0, 1, 2] @@ -2219,11 +2221,14 @@ def test_resolution(self): c = Coordinates([[lat, lon]], dims=['lat_lon']) print(c.horizontal_resolution()) - ''' + # Test Stacked/Unstacked Coords c = Coordinates([[lat, lon], alt], dims=['lat_lon', 'alt']) print(c.horizontal_resolution()) c = Coordinates([[lat, lon], time], dims=['lat_lon', 'time']) print(c.horizontal_resolution()) - ''' + c = Coordinates([time,[lat, lon]], dims=['time', 'lat_lon']) + print(c.horizontal_resolution()) + + From 7d453d1901a5a9dbd9956684b6e72cd309af676e Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Mon, 30 Jan 2023 13:43:27 -0500 Subject: [PATCH 26/52] WIP: Custom Ellipsoids --- podpac/core/coordinates/coordinates.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/podpac/core/coordinates/coordinates.py b/podpac/core/coordinates/coordinates.py index 14214cb0..6e1954bf 100644 --- a/podpac/core/coordinates/coordinates.py +++ b/podpac/core/coordinates/coordinates.py @@ -1520,14 +1520,16 @@ def is_stacked(self, dim): #re-wrote to be able to iterate through c.dims def horizontal_resolution(self): # dictionary: resolution_dict = OrderedDict() + ellipsoid_tuple = (self.CRS.ellipsoid.semi_major_metre/1000, self.CRS.ellipsoid.semi_minor_metre/1000, 1 / self.CRS.ellipsoid.inverse_flattening) '''Check if a dim::str is horizontal''' def check_horizontal(dim): for term in dim.split('_'): - if term == 'lat' or term == 'lon': - return True - return False + if term != 'lat' or term != 'lon': + return False + return True + '''Return resolution for stacked coordinates''' def stacked_resolution(dim): @@ -1540,7 +1542,7 @@ def stacked_resolution(dim): if point1.all() == point2.all(): continue # calculate distance in meters - distance = geodesic(point1, point2).m + distance = geodesic(point1, point2, ellipsoid=ellipsoid_tuple).m # only add min distance if distance < min_distance: min_distance = distance @@ -1554,7 +1556,7 @@ def stacked_resolution(dim): def unstacked_resolution(dim): top_bound = (self.bounds[dim][1], 0) bottom_bound = (self.bounds[dim][0], 0) - return ((geodesic(top_bound, bottom_bound).m)/ self[dim].size) * podpac.units('metre') + return ((geodesic(top_bound, bottom_bound, ellipsoid=ell_name).m)/ self[dim].size) * podpac.units('metre') for dim in self.dims: # Is the dim lat/lon? From 6cc3eff3566456f1660372ab7ac0c2ac17b8fdf8 Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Wed, 1 Feb 2023 10:15:20 -0500 Subject: [PATCH 27/52] WIP: Write Unit Test --- .../core/coordinates/test/test_coordinates.py | 48 ++++++++++--------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/podpac/core/coordinates/test/test_coordinates.py b/podpac/core/coordinates/test/test_coordinates.py index fdda9478..7cf2a502 100644 --- a/podpac/core/coordinates/test/test_coordinates.py +++ b/podpac/core/coordinates/test/test_coordinates.py @@ -8,6 +8,7 @@ from numpy.testing import assert_equal, assert_array_equal import xarray as xr import pyproj +from collections import OrderedDict import podpac from podpac.core.coordinates.coordinates1d import Coordinates1d @@ -2206,29 +2207,32 @@ def test_affine_to_uniform(self): class TestResolutions(object): def test_resolution(self): + - lat = [0, 1, 2] - lon = [10, 40, 30] - time = ['2018-01-01', '2018-01-02'] - alt = [10, 3, -10] + '''Unstacked Coordinates''' + # Intl Ellipsis + c = podpac.Coordinates([podpac.clinspace(0, 90, 11, 'lat'), podpac.clinspace(0, 90, 20, 'lon')], crs='+proj=latlon +zone=9 +north +ellps=intl +no_defs') + assert (c.horizontal_resolution() == OrderedDict([('lat', 909298.936271768*podpac.units('m')), ('lon', 500114.41494947235*podpac.units('m'))])) + # WGS84 Ellipsis + c = podpac.Coordinates([podpac.clinspace(0, 90, 11, 'lat'), podpac.clinspace(0, 90, 20, 'lon')], crs='+proj=latlon +zone=9 +north +ellps=WGS84 +no_defs') + assert (c.horizontal_resolution() == OrderedDict([('lat', 909269.611755702*podpac.units('m')), ('lon', 500098.2864656361*podpac.units('m'))])) + + '''Stacked Coordinates''' + lat = [0, 1, 2, 3, 4, 5, 6] + lon = [10, 20, 30, 50, 60, 80, 100] + c = Coordinates([[lat, lon]], dims=['lat_lon'], crs='+proj=latlon +zone=9 +north +ellps=WGS84 +no_defs') + assert (c.horizontal_resolution() == OrderedDict([('lat_lon', 4619360.82436145*podpac.units('m'))])) - # Test Unstacked Coords - c = Coordinates([lat, lon], dims=['lat','lon']) - print(c.horizontal_resolution()) - - # Test Stacked Coords - c = Coordinates([[lat, lon]], dims=['lat_lon']) - print(c.horizontal_resolution()) - - - # Test Stacked/Unstacked Coords - c = Coordinates([[lat, lon], alt], dims=['lat_lon', 'alt']) - print(c.horizontal_resolution()) - c = Coordinates([[lat, lon], time], dims=['lat_lon', 'time']) - print(c.horizontal_resolution()) - c = Coordinates([time,[lat, lon]], dims=['time', 'lat_lon']) - print(c.horizontal_resolution()) + '''Stacked and Unstacked Coordinates''' + time = ['2018-01-01', '2018-01-02'] + c_time = Coordinates([[lat, lon], time], dims=[['lat_lon'], 'time'], crs='+proj=latlon +zone=9 +north +ellps=WGS84 +no_defs') + assert (c.horizontal_resolution() == c_time.horizontal_resolution()) - - + '''Kilometer Units''' + assert(c.horizontal_resolution("km") == OrderedDict([('lat_lon', 4619.36082436145*podpac.units("km"))])) + + '''Invalid Arguments''' + # Invalid Units + with pytest.raises(ValueError): + c.horizontal_resolution(units="degree") From c0e6d062501cc48de04ed7510f74ddf8aeb1b993 Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Wed, 1 Feb 2023 10:15:44 -0500 Subject: [PATCH 28/52] WIP: allow for m and km, require degree unit --- podpac/core/coordinates/coordinates.py | 55 +++++++++++++++++++------- 1 file changed, 40 insertions(+), 15 deletions(-) diff --git a/podpac/core/coordinates/coordinates.py b/podpac/core/coordinates/coordinates.py index 6e1954bf..0e34032a 100644 --- a/podpac/core/coordinates/coordinates.py +++ b/podpac/core/coordinates/coordinates.py @@ -1515,22 +1515,29 @@ def is_stacked(self, dim): #re-wrote to be able to iterate through c.dims ''' Calculate horizontal resolution of coordinate system. - Assumes horizontal coords are in degrees. - ''' - def horizontal_resolution(self): - # dictionary: - resolution_dict = OrderedDict() - ellipsoid_tuple = (self.CRS.ellipsoid.semi_major_metre/1000, self.CRS.ellipsoid.semi_minor_metre/1000, 1 / self.CRS.ellipsoid.inverse_flattening) + Assumes horizontal coords are in degrees.''' + def horizontal_resolution(self, units = 'm'): + + '''---------------------------------------------------------------------''' + ''' Utility Functions ''' + + '''Return distance of 2 points in desired unit measurement''' + def calculate_distance(point1, point2): + if units in ['m', "metre", "meter"]: + return geodesic(point1, point2, ellipsoid=ellipsoid_tuple).m + elif units in ['km', "kilometre", "kilometer"]: + return geodesic(point1, point2, ellipsoid=ellipsoid_tuple).km + else: + return geodesic(point1, point2, ellipsoid=ellipsoid_tuple).m '''Check if a dim::str is horizontal''' def check_horizontal(dim): for term in dim.split('_'): - if term != 'lat' or term != 'lon': - return False - return True + if term == 'lat' or term == 'lon': + return True + return False - '''Return resolution for stacked coordinates''' def stacked_resolution(dim): sum = 0 @@ -1542,7 +1549,7 @@ def stacked_resolution(dim): if point1.all() == point2.all(): continue # calculate distance in meters - distance = geodesic(point1, point2, ellipsoid=ellipsoid_tuple).m + distance = calculate_distance(point1, point2) # only add min distance if distance < min_distance: min_distance = distance @@ -1550,25 +1557,43 @@ def stacked_resolution(dim): if min_distance != float('inf'): sum += min_distance # return sum / total_points - return (sum / self[dim].size) * podpac.units('metre') + return (sum / self[dim].size) * podpac.units(units) '''Return resolution for unstacked coordiantes''' def unstacked_resolution(dim): top_bound = (self.bounds[dim][1], 0) bottom_bound = (self.bounds[dim][0], 0) - return ((geodesic(top_bound, bottom_bound, ellipsoid=ell_name).m)/ self[dim].size) * podpac.units('metre') + return (calculate_distance(top_bound, bottom_bound)/ self[dim].size) * podpac.units(units) + + '''---------------------------------------------------------------------''' + ''' Function Code ''' + # dictionary: + resolution_dict = OrderedDict() + ellipsoid_tuple = (self.CRS.ellipsoid.semi_major_metre/1000, self.CRS.ellipsoid.semi_minor_metre/1000, 1 / self.CRS.ellipsoid.inverse_flattening) + + units = units.replace('s', '') + if units not in ['m', "metre", "meter",'km', "kilometre", "kilometer"]: + return ValueError("Valid options for desired units: m and km") + + # Check the units. + for axis in self.CRS.axis_info: + if check_horizontal(axis.abbrev): + if (axis.unit_name != "degree"): + return ValueError("Units of horizontal axes must be in degrees") + # main execution loop for dim in self.dims: # Is the dim lat/lon? if not check_horizontal(dim): continue # stacked coordinate resolutions if self.is_stacked(dim): - print("{} is stacked!".format(dim)) + # print("{} is stacked!".format(dim)) resolution_dict[dim] = stacked_resolution(dim) else: # unstacked resolution + # print("{} is NOT stacked!".format(dim)) resolution_dict[dim] = unstacked_resolution(dim) - print("{} is NOT stacked!".format(dim)) + return resolution_dict From 0d26fd00af80dacd2816831a8b325a59b996a908 Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Fri, 3 Feb 2023 13:20:52 -0500 Subject: [PATCH 29/52] FIX: use pint for units --- podpac/core/coordinates/coordinates.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/podpac/core/coordinates/coordinates.py b/podpac/core/coordinates/coordinates.py index 0e34032a..a84a89e4 100644 --- a/podpac/core/coordinates/coordinates.py +++ b/podpac/core/coordinates/coordinates.py @@ -1523,13 +1523,9 @@ def horizontal_resolution(self, units = 'm'): '''Return distance of 2 points in desired unit measurement''' def calculate_distance(point1, point2): - if units in ['m', "metre", "meter"]: - return geodesic(point1, point2, ellipsoid=ellipsoid_tuple).m - elif units in ['km', "kilometre", "kilometer"]: - return geodesic(point1, point2, ellipsoid=ellipsoid_tuple).km - else: - return geodesic(point1, point2, ellipsoid=ellipsoid_tuple).m - + + return (geodesic(point1, point2, ellipsoid=ellipsoid_tuple).m * podpac.units("metre)").to(podpac.units(units))) + '''Check if a dim::str is horizontal''' def check_horizontal(dim): @@ -1571,9 +1567,7 @@ def unstacked_resolution(dim): resolution_dict = OrderedDict() ellipsoid_tuple = (self.CRS.ellipsoid.semi_major_metre/1000, self.CRS.ellipsoid.semi_minor_metre/1000, 1 / self.CRS.ellipsoid.inverse_flattening) - units = units.replace('s', '') - if units not in ['m', "metre", "meter",'km', "kilometre", "kilometer"]: - return ValueError("Valid options for desired units: m and km") + # TODO: UTM Projection # Check the units. for axis in self.CRS.axis_info: From edec942be29e259b4ebf87a49aa5d8c15b19da3f Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Mon, 6 Feb 2023 12:23:59 -0500 Subject: [PATCH 30/52] MAINT: KDTree Stacked --- podpac/core/coordinates/coordinates.py | 45 ++++++++++++++++++-------- 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/podpac/core/coordinates/coordinates.py b/podpac/core/coordinates/coordinates.py index a84a89e4..e2f9e91c 100644 --- a/podpac/core/coordinates/coordinates.py +++ b/podpac/core/coordinates/coordinates.py @@ -20,7 +20,6 @@ import pyproj import logging from scipy import spatial -import math from geopy.distance import geodesic import podpac @@ -1516,15 +1515,14 @@ def is_stacked(self, dim): #re-wrote to be able to iterate through c.dims ''' Calculate horizontal resolution of coordinate system. Assumes horizontal coords are in degrees.''' - def horizontal_resolution(self, units = 'm'): + def horizontal_resolution(self, units = "metre"): '''---------------------------------------------------------------------''' ''' Utility Functions ''' '''Return distance of 2 points in desired unit measurement''' - def calculate_distance(point1, point2): - - return (geodesic(point1, point2, ellipsoid=ellipsoid_tuple).m * podpac.units("metre)").to(podpac.units(units))) + def calculate_distance(point1, point2, is_utm = False): + return ((geodesic(point1, point2, ellipsoid=ellipsoid_tuple).m) * podpac.units("metre")).to(podpac.units(units)) '''Check if a dim::str is horizontal''' @@ -1539,30 +1537,50 @@ def stacked_resolution(dim): sum = 0 # Brute Force implementatiom for point1 in self[dim].coordinates: - min_distance = float('inf') + nearest_neighbor =0 + min_distance = float('inf') * podpac.units(units) for point2 in self[dim].coordinates: # check if not current point - if point1.all() == point2.all(): + if np.array_equiv(point1,point2): + print(point1) + print(point2) continue # calculate distance in meters - distance = calculate_distance(point1, point2) + distance = calculate_distance(point1, point2) # only add min distance if distance < min_distance: + nearest_neighbor = point2 min_distance = distance # add min_distance - if min_distance != float('inf'): + if min_distance != float('inf')*podpac.units(units): + print(min_distance) sum += min_distance + print(point1) # return sum / total_points - return (sum / self[dim].size) * podpac.units(units) + return (sum / self[dim].size) + + '''Uses a KDTree to return approximate stacked resolution with some errors. + Errors arise from spatial.KDTree's use of euclidean distance for a metric.''' + def kdtree_stacked_resolution(dim): + tree = spatial.KDTree(self[dim].coordinates + [90.,180.], boxsize=[0., 360.0000000000001]) + sum_distance = 0 + for point in tree.data: + dd, ii = tree.query(point, k=2) # get nearest neighbor + sum_distance += calculate_distance(point - [90., 180.], tree.data[ii[1]] - [90., 180.]) # calculate distance + return sum_distance/len(tree.data) + + '''Return resolution for unstacked coordiantes''' def unstacked_resolution(dim): top_bound = (self.bounds[dim][1], 0) bottom_bound = (self.bounds[dim][0], 0) - return (calculate_distance(top_bound, bottom_bound)/ self[dim].size) * podpac.units(units) + return (calculate_distance(top_bound, bottom_bound)/ self[dim].size) + '''---------------------------------------------------------------------''' - ''' Function Code ''' + + # dictionary: resolution_dict = OrderedDict() ellipsoid_tuple = (self.CRS.ellipsoid.semi_major_metre/1000, self.CRS.ellipsoid.semi_minor_metre/1000, 1 / self.CRS.ellipsoid.inverse_flattening) @@ -1582,10 +1600,9 @@ def unstacked_resolution(dim): continue # stacked coordinate resolutions if self.is_stacked(dim): - # print("{} is stacked!".format(dim)) + # stacked_resolution(dim) resolution_dict[dim] = stacked_resolution(dim) else: # unstacked resolution - # print("{} is NOT stacked!".format(dim)) resolution_dict[dim] = unstacked_resolution(dim) From dda63bc44f333c075cc5be68f394266371e5f0bc Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Mon, 6 Feb 2023 12:24:25 -0500 Subject: [PATCH 31/52] FIX: make test realisitc --- .../core/coordinates/test/test_coordinates.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/podpac/core/coordinates/test/test_coordinates.py b/podpac/core/coordinates/test/test_coordinates.py index 7cf2a502..4f34f8c7 100644 --- a/podpac/core/coordinates/test/test_coordinates.py +++ b/podpac/core/coordinates/test/test_coordinates.py @@ -9,6 +9,7 @@ import xarray as xr import pyproj from collections import OrderedDict +import pint import podpac from podpac.core.coordinates.coordinates1d import Coordinates1d @@ -2218,11 +2219,17 @@ def test_resolution(self): c = podpac.Coordinates([podpac.clinspace(0, 90, 11, 'lat'), podpac.clinspace(0, 90, 20, 'lon')], crs='+proj=latlon +zone=9 +north +ellps=WGS84 +no_defs') assert (c.horizontal_resolution() == OrderedDict([('lat', 909269.611755702*podpac.units('m')), ('lon', 500098.2864656361*podpac.units('m'))])) + '''Stacked Coordinates''' - lat = [0, 1, 2, 3, 4, 5, 6] - lon = [10, 20, 30, 50, 60, 80, 100] + lat = podpac.clinspace(0, 5, 5) + lon = podpac.clinspace(-180, 180, 5) + c = Coordinates([[lat, lon]], dims=['lat_lon'], crs='+proj=latlon +zone=9 +north +ellps=WGS84 +no_defs') + assert (c.horizontal_resolution() == OrderedDict([('lat_lon', 6014010.226930527*podpac.units('m'))])) + + lat = podpac.clinspace(-90, 90, 20) + lon = podpac.clinspace(-180, 180, 20) c = Coordinates([[lat, lon]], dims=['lat_lon'], crs='+proj=latlon +zone=9 +north +ellps=WGS84 +no_defs') - assert (c.horizontal_resolution() == OrderedDict([('lat_lon', 4619360.82436145*podpac.units('m'))])) + assert (c.horizontal_resolution() == OrderedDict([('lat_lon', 1661624.9900800656*podpac.units('m'))])) '''Stacked and Unstacked Coordinates''' time = ['2018-01-01', '2018-01-02'] @@ -2230,9 +2237,9 @@ def test_resolution(self): assert (c.horizontal_resolution() == c_time.horizontal_resolution()) '''Kilometer Units''' - assert(c.horizontal_resolution("km") == OrderedDict([('lat_lon', 4619.36082436145*podpac.units("km"))])) + assert(c.horizontal_resolution("km") == OrderedDict([('lat_lon', 1661.6249900800653*podpac.units("km"))])) '''Invalid Arguments''' # Invalid Units - with pytest.raises(ValueError): + with pytest.raises(pint.errors.DimensionalityError): c.horizontal_resolution(units="degree") From 9851b1cb5ce4b32bc00e96e500ffc8c78efd7376 Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Mon, 6 Feb 2023 13:45:59 -0500 Subject: [PATCH 32/52] MAINT: impl. return types --- podpac/core/coordinates/coordinates.py | 86 +++++++++++++++++++++++--- 1 file changed, 77 insertions(+), 9 deletions(-) diff --git a/podpac/core/coordinates/coordinates.py b/podpac/core/coordinates/coordinates.py index e2f9e91c..c9e0f3e6 100644 --- a/podpac/core/coordinates/coordinates.py +++ b/podpac/core/coordinates/coordinates.py @@ -21,6 +21,7 @@ import logging from scipy import spatial from geopy.distance import geodesic +import math import podpac from podpac.core.settings import settings @@ -1514,8 +1515,15 @@ def is_stacked(self, dim): #re-wrote to be able to iterate through c.dims ''' Calculate horizontal resolution of coordinate system. - Assumes horizontal coords are in degrees.''' - def horizontal_resolution(self, units = "metre"): + Assumes horizontal coords are in degrees. + + Support different types! + - "nominal" <-- this is wrong but cheap to calculate. Give a 'nominal' resolution over the entire domain + - "summary" <-- This is still approximate, gives the mean and standard deviation to each point + - "full" <-- min distance between every point (dx, dy, basically) + - "full_exact" <-- uses brute-force for the stacked coordinates + ''' + def horizontal_resolution(self, units = "metre", type="nominal"): '''---------------------------------------------------------------------''' ''' Utility Functions ''' @@ -1532,8 +1540,10 @@ def check_horizontal(dim): return True return False + ''' STACKED COORDINATE FUNCTIONS ''' + '''Return resolution for stacked coordinates''' - def stacked_resolution(dim): + def brute_stacked_resolution(dim): sum = 0 # Brute Force implementatiom for point1 in self[dim].coordinates: @@ -1561,7 +1571,7 @@ def stacked_resolution(dim): '''Uses a KDTree to return approximate stacked resolution with some errors. Errors arise from spatial.KDTree's use of euclidean distance for a metric.''' - def kdtree_stacked_resolution(dim): + def nominal_stacked_resolution(dim): tree = spatial.KDTree(self[dim].coordinates + [90.,180.], boxsize=[0., 360.0000000000001]) sum_distance = 0 for point in tree.data: @@ -1569,14 +1579,57 @@ def kdtree_stacked_resolution(dim): sum_distance += calculate_distance(point - [90., 180.], tree.data[ii[1]] - [90., 180.]) # calculate distance return sum_distance/len(tree.data) + '''Return the approximate mean resolution and std.deviation using a KDTree''' + def summary_stacked_resolution(dim): + tree = spatial.KDTree(self[dim].coordinates + [90.,180.], boxsize=[0., 360.0000000000001]) + sum_distance = 0 + for point in tree.data: + dd, ii = tree.query(point, k=2) # get nearest neighbor + sum_distance += calculate_distance(point - [90., 180.], tree.data[ii[1]] - [90., 180.]) # calculate distance + avg_distance = sum_distance/len(tree.data) + # calculate standard deviation + std_dev = 0 + for point in tree.data: + dd, ii = tree.query(point, k=2) # get nearest neighbor + std_dev += (calculate_distance(point - [90., 180.], tree.data[ii[1]] - [90., 180.]) - avg_distance)**2 + std_dev /= len(tree.data) + std_dev = math.sqrt(std_dev.magnitude) + return (avg_distance, std_dev) + + '''Returns the exact distance between every point using brute force''' + def full_stacked_resolution(dim): + distance_matrix = np.zeros((len(self[dim].coordinates),len(self[dim].coordinates))) + for i in range(len(self[dim].coordinates)): + for j in range(len(self[dim].coordinates)): + distance_matrix[i][j] = calculate_distance(self[dim].coordinates[i],self[dim].coordinates[j]).magnitude + return distance_matrix *podpac.units(units) + + + ''' UNSTACKED COORDINATE FUNCTIONS ''' - '''Return resolution for unstacked coordiantes''' - def unstacked_resolution(dim): + '''Return resolution for unstacked coordiantes using the bounds''' + def nominal_unstacked_resolution(dim): top_bound = (self.bounds[dim][1], 0) bottom_bound = (self.bounds[dim][0], 0) return (calculate_distance(top_bound, bottom_bound)/ self[dim].size) + '''Return summary resolution for the dimension.''' + def summary_unstacked_resolution(dim): + diff = np.zeros(len(self[dim].coordinates)-1) + for i in range(len(diff)): + top_bound = (self[dim].coordinates[i+1], 0) + bottom_bound = (self[dim].coordinates[i], 0) + diff[i] = calculate_distance(top_bound, bottom_bound).magnitude + return (np.average(diff) * podpac.units(units), np.std(diff) * podpac.units(units)) + + def full_unstacked_resolution(dim): + diff = np.zeros(len(self[dim].coordinates)-1) + for i in range(len(diff)): + top_bound = (self[dim].coordinates[i+1], 0) + bottom_bound = (self[dim].coordinates[i], 0) + diff[i] = calculate_distance(top_bound, bottom_bound).magnitude + return diff *podpac.units(units) '''---------------------------------------------------------------------''' @@ -1601,11 +1654,26 @@ def unstacked_resolution(dim): # stacked coordinate resolutions if self.is_stacked(dim): # stacked_resolution(dim) - resolution_dict[dim] = stacked_resolution(dim) + match type: + case "nominal": + resolution_dict[dim] = nominal_stacked_resolution(dim) + case "summary": + resolution_dict[dim] = summary_stacked_resolution(dim) + case "full": + resolution_dict[dim] = full_stacked_resolution(dim) + case _: + return ValueError("Invalid value for type: {}".format(type)) else: # unstacked resolution - resolution_dict[dim] = unstacked_resolution(dim) + match type: + case "nominal": + resolution_dict[dim] = nominal_unstacked_resolution(dim) + case "summary": + resolution_dict[dim] = summary_unstacked_resolution(dim) + case "full": + resolution_dict[dim] = full_unstacked_resolution(dim) + case _: + return ValueError("Invalid value for type: {}".format(type)) - return resolution_dict From f4dc8b38d44793742e2ce0bbcb6e95fa787bdfe1 Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Mon, 6 Feb 2023 13:46:48 -0500 Subject: [PATCH 33/52] FIX: update header --- podpac/core/coordinates/coordinates.py | 1 - 1 file changed, 1 deletion(-) diff --git a/podpac/core/coordinates/coordinates.py b/podpac/core/coordinates/coordinates.py index c9e0f3e6..553c0487 100644 --- a/podpac/core/coordinates/coordinates.py +++ b/podpac/core/coordinates/coordinates.py @@ -1521,7 +1521,6 @@ def is_stacked(self, dim): #re-wrote to be able to iterate through c.dims - "nominal" <-- this is wrong but cheap to calculate. Give a 'nominal' resolution over the entire domain - "summary" <-- This is still approximate, gives the mean and standard deviation to each point - "full" <-- min distance between every point (dx, dy, basically) - - "full_exact" <-- uses brute-force for the stacked coordinates ''' def horizontal_resolution(self, units = "metre", type="nominal"): From e05a662688b5d600aeb29784d43b5acb8c466cbe Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Mon, 6 Feb 2023 15:48:09 -0500 Subject: [PATCH 34/52] MAINT: Support Cartesian --- podpac/core/coordinates/coordinates.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/podpac/core/coordinates/coordinates.py b/podpac/core/coordinates/coordinates.py index 553c0487..32733cf7 100644 --- a/podpac/core/coordinates/coordinates.py +++ b/podpac/core/coordinates/coordinates.py @@ -22,6 +22,7 @@ from scipy import spatial from geopy.distance import geodesic import math +import utm import podpac from podpac.core.settings import settings @@ -1529,7 +1530,10 @@ def horizontal_resolution(self, units = "metre", type="nominal"): '''Return distance of 2 points in desired unit measurement''' def calculate_distance(point1, point2, is_utm = False): - return ((geodesic(point1, point2, ellipsoid=ellipsoid_tuple).m) * podpac.units("metre")).to(podpac.units(units)) + if self.CRS.coordinate_system.name == "cartesian": + return math.dist(point1, point2) * podpac.units(units) + else: + return ((geodesic(point1, point2, ellipsoid=ellipsoid_tuple).m) * podpac.units("metre")).to(podpac.units(units)) '''Check if a dim::str is horizontal''' @@ -1633,11 +1637,12 @@ def full_unstacked_resolution(dim): '''---------------------------------------------------------------------''' + # dictionary: resolution_dict = OrderedDict() - ellipsoid_tuple = (self.CRS.ellipsoid.semi_major_metre/1000, self.CRS.ellipsoid.semi_minor_metre/1000, 1 / self.CRS.ellipsoid.inverse_flattening) - # TODO: UTM Projection + # ellipsoid Tuple + ellipsoid_tuple = (self.CRS.ellipsoid.semi_major_metre/1000, self.CRS.ellipsoid.semi_minor_metre/1000, 1 / self.CRS.ellipsoid.inverse_flattening) # Check the units. for axis in self.CRS.axis_info: From 8fb7fbb915b7bedfb5661dfdbba7e860c497fa28 Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Mon, 6 Feb 2023 16:21:36 -0500 Subject: [PATCH 35/52] MAINT: Remove unused code --- podpac/core/coordinates/coordinates.py | 29 ----------- .../core/coordinates/test/test_coordinates.py | 50 +++++++++---------- 2 files changed, 24 insertions(+), 55 deletions(-) diff --git a/podpac/core/coordinates/coordinates.py b/podpac/core/coordinates/coordinates.py index 32733cf7..8a1c0132 100644 --- a/podpac/core/coordinates/coordinates.py +++ b/podpac/core/coordinates/coordinates.py @@ -1544,33 +1544,6 @@ def check_horizontal(dim): return False ''' STACKED COORDINATE FUNCTIONS ''' - - '''Return resolution for stacked coordinates''' - def brute_stacked_resolution(dim): - sum = 0 - # Brute Force implementatiom - for point1 in self[dim].coordinates: - nearest_neighbor =0 - min_distance = float('inf') * podpac.units(units) - for point2 in self[dim].coordinates: - # check if not current point - if np.array_equiv(point1,point2): - print(point1) - print(point2) - continue - # calculate distance in meters - distance = calculate_distance(point1, point2) - # only add min distance - if distance < min_distance: - nearest_neighbor = point2 - min_distance = distance - # add min_distance - if min_distance != float('inf')*podpac.units(units): - print(min_distance) - sum += min_distance - print(point1) - # return sum / total_points - return (sum / self[dim].size) '''Uses a KDTree to return approximate stacked resolution with some errors. Errors arise from spatial.KDTree's use of euclidean distance for a metric.''' @@ -1636,8 +1609,6 @@ def full_unstacked_resolution(dim): '''---------------------------------------------------------------------''' - - # dictionary: resolution_dict = OrderedDict() diff --git a/podpac/core/coordinates/test/test_coordinates.py b/podpac/core/coordinates/test/test_coordinates.py index 4f34f8c7..f840358f 100644 --- a/podpac/core/coordinates/test/test_coordinates.py +++ b/podpac/core/coordinates/test/test_coordinates.py @@ -2208,38 +2208,36 @@ def test_affine_to_uniform(self): class TestResolutions(object): def test_resolution(self): - - '''Unstacked Coordinates''' - # Intl Ellipsis - c = podpac.Coordinates([podpac.clinspace(0, 90, 11, 'lat'), podpac.clinspace(0, 90, 20, 'lon')], crs='+proj=latlon +zone=9 +north +ellps=intl +no_defs') - assert (c.horizontal_resolution() == OrderedDict([('lat', 909298.936271768*podpac.units('m')), ('lon', 500114.41494947235*podpac.units('m'))])) - # WGS84 Ellipsis - c = podpac.Coordinates([podpac.clinspace(0, 90, 11, 'lat'), podpac.clinspace(0, 90, 20, 'lon')], crs='+proj=latlon +zone=9 +north +ellps=WGS84 +no_defs') - assert (c.horizontal_resolution() == OrderedDict([('lat', 909269.611755702*podpac.units('m')), ('lon', 500098.2864656361*podpac.units('m'))])) - + # Nominal + + # Summary + + # Exact + + # Cartesian '''Stacked Coordinates''' - lat = podpac.clinspace(0, 5, 5) - lon = podpac.clinspace(-180, 180, 5) - c = Coordinates([[lat, lon]], dims=['lat_lon'], crs='+proj=latlon +zone=9 +north +ellps=WGS84 +no_defs') - assert (c.horizontal_resolution() == OrderedDict([('lat_lon', 6014010.226930527*podpac.units('m'))])) + # Nominal + + # Summary + + # Exact + + # Cartesian - lat = podpac.clinspace(-90, 90, 20) - lon = podpac.clinspace(-180, 180, 20) - c = Coordinates([[lat, lon]], dims=['lat_lon'], crs='+proj=latlon +zone=9 +north +ellps=WGS84 +no_defs') - assert (c.horizontal_resolution() == OrderedDict([('lat_lon', 1661624.9900800656*podpac.units('m'))])) '''Stacked and Unstacked Coordinates''' - time = ['2018-01-01', '2018-01-02'] - c_time = Coordinates([[lat, lon], time], dims=[['lat_lon'], 'time'], crs='+proj=latlon +zone=9 +north +ellps=WGS84 +no_defs') - assert (c.horizontal_resolution() == c_time.horizontal_resolution()) + # Nominal + + # Summary + + # Exact + + # Cartesian - '''Kilometer Units''' - assert(c.horizontal_resolution("km") == OrderedDict([('lat_lon', 1661.6249900800653*podpac.units("km"))])) - '''Invalid Arguments''' - # Invalid Units - with pytest.raises(pint.errors.DimensionalityError): - c.horizontal_resolution(units="degree") + + + From 0232e45a79d33b0e0d8a546402505854d4ba61d7 Mon Sep 17 00:00:00 2001 From: Matt Ueckermann Date: Mon, 6 Feb 2023 16:43:55 -0500 Subject: [PATCH 36/52] FIX: Adding new depenencies. --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index 931439ce..178fcf68 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,8 @@ "lazy-import>=0.2.2", "psutil", "affine", + "utm", + "geopy", ] if sys.version_info.major == 2: From c3f38aa7f1d4cc30094664d50fabd7d7c9aab297 Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Tue, 7 Feb 2023 09:50:47 -0500 Subject: [PATCH 37/52] MAINT: tests --- .../core/coordinates/test/test_coordinates.py | 66 +++++++++++++++---- 1 file changed, 53 insertions(+), 13 deletions(-) diff --git a/podpac/core/coordinates/test/test_coordinates.py b/podpac/core/coordinates/test/test_coordinates.py index f840358f..752eea84 100644 --- a/podpac/core/coordinates/test/test_coordinates.py +++ b/podpac/core/coordinates/test/test_coordinates.py @@ -2210,34 +2210,74 @@ class TestResolutions(object): def test_resolution(self): '''Unstacked Coordinates''' + c = podpac.Coordinates([podpac.clinspace(0, 90, 11, 'lat'), podpac.clinspace(0, 90, 20, 'lon')]) + # Nominal + assert (c.horizontal_resolution(type="nominal") == OrderedDict([('lat', 909269.611755702* podpac.units("metre")), + ('lon', 500098.2864656361* podpac.units("metre"))])) # Summary + assert (c.horizontal_resolution(type="summary") == OrderedDict([('lat', + (1000196.5729312723* podpac.units("metre"), + 3548.2348372254723* podpac.units("metre"))), + ('lon', + (526419.2489111959* podpac.units("metre"), + 1873.0586112168473* podpac.units("metre")))])) + + # Full + answer = OrderedDict([('lat', + ([ 995250.27739564, 995731.65748247, 996648.35979764, + 997912.37572808, 999401.70757391, 1000971.65042421, + 1002468.54050245, 1003744.78599167, 1004673.71796683, + 1005162.65644982]) * podpac.units("m")), + ('lon', + ([523784.85224537, 523856.27075792, 523997.2071627 , 524203.90698729, + 524470.85470098, 524790.91101581, 525155.49233879, 525554.78910432, + 525978.01872173, 526413.70781726, 526849.99736995, 527274.96329269, + 527676.94406298, 528044.86624478, 528368.55824467, 528639.04248812, + 528848.79644537, 528991.97361553, 529064.57669647]) * podpac.units("m"))]) + assert answer['lat'].all() == c.horizontal_resolution(type="full")['lat'].all() and answer['lon'].all() == c.horizontal_resolution(type="full")['lon'].all() - # Exact - - # Cartesian + '''Stacked Coordinates''' + lat = podpac.clinspace(-90, 90, 5) + lon = podpac.clinspace(-180, 180, 5) + + c = Coordinates([[lat, lon]], dims=['lat_lon']) # Nominal + assert c.horizontal_resolution(type="nominal") == OrderedDict([('lat_lon', 7014367.35521632 * podpac.units("m"))]) # Summary - - # Exact + assert c.horizontal_resolution(type="summary") == OrderedDict([('lat_lon', + (7014367.35521632 * podpac.units("m"), 2446239.274648156))]) + + # Full + answer = OrderedDict([('lat_lon', + ([[ 0. , 5017021.35133498, 10001965.72931272, + 14986910.10729047, 20003931.45862544], + [ 5017021.35133498, 0. , 10010386.36103833, + 20003931.45862544, 14986910.10729047], + [10001965.72931272, 10010386.36103833, 0. , + 10010386.36103833, 10001965.72931272], + [14986910.10729047, 20003931.45862544, 10010386.36103833, + 0. , 5017021.35133498], + [20003931.45862544, 14986910.10729047, 10001965.72931272, + 5017021.35133498, 0. ]]) * podpac.units("m"))]) + + assert c.horizontal_resolution(type="full")['lat_lon'].all() == answer['lat_lon'].all() # Cartesian + c = podpac.Coordinates([[podpac.clinspace(0, 90, 20, 'lat'), podpac.clinspace(0, 90, 20, 'lon')]], dims = ['lat_lon'], crs='+proj=utm +zone=9 +north +ellps=WGS84 +datum=WGS84 +units=m +no_defs') + assert c.horizontal_resolution() == OrderedDict([('lat_lon', 6.6989063480830735 * podpac.units("m"))]) '''Stacked and Unstacked Coordinates''' + time = ['2018-01-01', '2018-01-02'] + c = Coordinates([[lat, lon], time], dims=['lat_lon', 'time']) # Nominal - - # Summary - - # Exact - - # Cartesian - - + assert c.horizontal_resolution() == OrderedDict([('lat_lon', 7014367.35521632 * podpac.units("m"))]) + From 044ede9690e1e52f5b13c081320b36cf3f85d2fe Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Tue, 7 Feb 2023 10:09:02 -0500 Subject: [PATCH 38/52] FIX: edit dependencies --- podpac/core/coordinates/coordinates.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/podpac/core/coordinates/coordinates.py b/podpac/core/coordinates/coordinates.py index 8a1c0132..1176cee2 100644 --- a/podpac/core/coordinates/coordinates.py +++ b/podpac/core/coordinates/coordinates.py @@ -22,7 +22,7 @@ from scipy import spatial from geopy.distance import geodesic import math -import utm + import podpac from podpac.core.settings import settings diff --git a/setup.py b/setup.py index 178fcf68..6065b757 100644 --- a/setup.py +++ b/setup.py @@ -26,8 +26,8 @@ "lazy-import>=0.2.2", "psutil", "affine", - "utm", "geopy", + "math" ] if sys.version_info.major == 2: From afac6d7b204b2673d089c73fb0f45164d43cb129 Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Tue, 7 Feb 2023 10:32:41 -0500 Subject: [PATCH 39/52] FIX: numpy function headers --- podpac/core/coordinates/coordinates.py | 158 ++++++++++++++++++++----- 1 file changed, 129 insertions(+), 29 deletions(-) diff --git a/podpac/core/coordinates/coordinates.py b/podpac/core/coordinates/coordinates.py index 1176cee2..434759b9 100644 --- a/podpac/core/coordinates/coordinates.py +++ b/podpac/core/coordinates/coordinates.py @@ -1514,40 +1514,91 @@ def is_stacked(self, dim): #re-wrote to be able to iterate through c.dims case 2: # both true return False - ''' - Calculate horizontal resolution of coordinate system. - Assumes horizontal coords are in degrees. - Support different types! - - "nominal" <-- this is wrong but cheap to calculate. Give a 'nominal' resolution over the entire domain - - "summary" <-- This is still approximate, gives the mean and standard deviation to each point - - "full" <-- min distance between every point (dx, dy, basically) - ''' def horizontal_resolution(self, units = "metre", type="nominal"): + ''' + Calculate horizontal resolution of coordinate system. - '''---------------------------------------------------------------------''' - ''' Utility Functions ''' + Parameters + ---------- + units : str + The desired unit the returned resolution should be in. Supports any unit supported by podpac.units + type : str + The kind of horizontal resolution that should be returned. Supported values are: + - "nominal" <-- this is wrong but cheap to calculate. Give a 'nominal' resolution over the entire domain + - "summary" <-- This is still approximate, gives the mean and standard deviation to each point + - "full" <-- min distance between every point (dx, dy, basically) - '''Return distance of 2 points in desired unit measurement''' - def calculate_distance(point1, point2, is_utm = False): + Returns + ------- + OrderedDict + A dictionary with: + keys : str + dimension names + values + resolution (format determined by 'type' parameter) + + Raises + ------ + ValueError + If the 'type' is not one of the supported resolution types + + ''' + + + def calculate_distance(point1, point2): + '''Return distance of 2 points in desired unit measurement + + Parameters + ---------- + point1 : tuple + point2 : tuple + + Returns + ------- + float + The distance between point1 and point2, according to the current coordinate system's distance metric, using the desired units + ''' if self.CRS.coordinate_system.name == "cartesian": return math.dist(point1, point2) * podpac.units(units) else: return ((geodesic(point1, point2, ellipsoid=ellipsoid_tuple).m) * podpac.units("metre")).to(podpac.units(units)) - '''Check if a dim::str is horizontal''' + def check_horizontal(dim): + '''Check if a dimension is horizontal + + Parameters + ---------- + dim : str + the name of the dimension + + Returns + ------- + bool + Whether or not the current dimension is horizontal or not + ''' for term in dim.split('_'): if term == 'lat' or term == 'lon': return True return False - ''' STACKED COORDINATE FUNCTIONS ''' - '''Uses a KDTree to return approximate stacked resolution with some errors. - Errors arise from spatial.KDTree's use of euclidean distance for a metric.''' + def nominal_stacked_resolution(dim): + '''Use a KDTree to return approximate stacked resolution with some loss of accuracy. + + Parameters + ---------- + dim : str + The name of the dimension to return the resolution of. Should be stacked. + + Returns + ------- + The average min distance of every point + + ''' tree = spatial.KDTree(self[dim].coordinates + [90.,180.], boxsize=[0., 360.0000000000001]) sum_distance = 0 for point in tree.data: @@ -1555,8 +1606,20 @@ def nominal_stacked_resolution(dim): sum_distance += calculate_distance(point - [90., 180.], tree.data[ii[1]] - [90., 180.]) # calculate distance return sum_distance/len(tree.data) - '''Return the approximate mean resolution and std.deviation using a KDTree''' + def summary_stacked_resolution(dim): + '''Return the approximate mean resolution and std.deviation using a KDTree + + Parameters + ---------- + dim : str + The name of the dimension to return the resolution of. Should be stacked. + + Returns + ------- + tuple + Average min distance of every point and standard deviation of those min distances + ''' tree = spatial.KDTree(self[dim].coordinates + [90.,180.], boxsize=[0., 360.0000000000001]) sum_distance = 0 for point in tree.data: @@ -1572,8 +1635,19 @@ def summary_stacked_resolution(dim): std_dev = math.sqrt(std_dev.magnitude) return (avg_distance, std_dev) - '''Returns the exact distance between every point using brute force''' + def full_stacked_resolution(dim): + '''Returns the exact distance between every point using brute force + + Parameters + ---------- + dim : str + The dimension to return the resolution of. Should be stacked. + + Returns + ------- + distance matrix of size (NxN), where N is the number of points in the dimension + ''' distance_matrix = np.zeros((len(self[dim].coordinates),len(self[dim].coordinates))) for i in range(len(self[dim].coordinates)): for j in range(len(self[dim].coordinates)): @@ -1581,17 +1655,38 @@ def full_stacked_resolution(dim): return distance_matrix *podpac.units(units) - - ''' UNSTACKED COORDINATE FUNCTIONS ''' - - '''Return resolution for unstacked coordiantes using the bounds''' + def nominal_unstacked_resolution(dim): + '''Return resolution for unstacked coordiantes using the bounds + + Parameters + ---------- + dim : str + The dimenion to return the resolution of. Should be unstacked. + + Returns + -------- + The average distance between each grid square for this dimension + ''' top_bound = (self.bounds[dim][1], 0) bottom_bound = (self.bounds[dim][0], 0) return (calculate_distance(top_bound, bottom_bound)/ self[dim].size) - '''Return summary resolution for the dimension.''' + def summary_unstacked_resolution(dim): + '''Return summary resolution for the dimension. + + Parameters + ---------- + dim : str + The dimension to return the resolution of. Should be unstacked. + + Returns + ------- + tuple + the average distance between grid squares + the standard deviation of those distances + ''' diff = np.zeros(len(self[dim].coordinates)-1) for i in range(len(diff)): top_bound = (self[dim].coordinates[i+1], 0) @@ -1600,6 +1695,17 @@ def summary_unstacked_resolution(dim): return (np.average(diff) * podpac.units(units), np.std(diff) * podpac.units(units)) def full_unstacked_resolution(dim): + ''' Calculate full resolution of unstacked dimension + + Parameters + ---------- + dim : str + The dimension to return the resolution of. Should be unstacked. + + Returns + ------- + An array of every distance between each grid point + ''' diff = np.zeros(len(self[dim].coordinates)-1) for i in range(len(diff)): top_bound = (self[dim].coordinates[i+1], 0) @@ -1615,12 +1721,6 @@ def full_unstacked_resolution(dim): # ellipsoid Tuple ellipsoid_tuple = (self.CRS.ellipsoid.semi_major_metre/1000, self.CRS.ellipsoid.semi_minor_metre/1000, 1 / self.CRS.ellipsoid.inverse_flattening) - # Check the units. - for axis in self.CRS.axis_info: - if check_horizontal(axis.abbrev): - if (axis.unit_name != "degree"): - return ValueError("Units of horizontal axes must be in degrees") - # main execution loop for dim in self.dims: # Is the dim lat/lon? From 6b8bd4c83be59aa0bfd24a95ea603f6141dee7ec Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Tue, 7 Feb 2023 10:35:15 -0500 Subject: [PATCH 40/52] MAINT: black formatting --- podpac/core/coordinates/coordinates.py | 156 +++++++++++++------------ 1 file changed, 79 insertions(+), 77 deletions(-) diff --git a/podpac/core/coordinates/coordinates.py b/podpac/core/coordinates/coordinates.py index 434759b9..4de90dee 100644 --- a/podpac/core/coordinates/coordinates.py +++ b/podpac/core/coordinates/coordinates.py @@ -1505,20 +1505,19 @@ def issubset(self, other): return all(c.issubset(other) for c in self.values()) - def is_stacked(self, dim): #re-wrote to be able to iterate through c.dims + def is_stacked(self, dim): # re-wrote to be able to iterate through c.dims match (dim in self.dims) + (dim in self.udims): - case 0: # both false + case 0: # both false raise ValueError("Dimension {} is not in self.dims={}".format(dim, self.dims)) - case 1: # one true, one false + case 1: # one true, one false return True - case 2: # both true + case 2: # both true return False - - def horizontal_resolution(self, units = "metre", type="nominal"): - ''' + def horizontal_resolution(self, units="metre", type="nominal"): + """ Calculate horizontal resolution of coordinate system. - + Parameters ---------- units : str @@ -1535,20 +1534,19 @@ def horizontal_resolution(self, units = "metre", type="nominal"): A dictionary with: keys : str dimension names - values + values resolution (format determined by 'type' parameter) - + Raises ------ ValueError If the 'type' is not one of the supported resolution types - ''' + """ - def calculate_distance(point1, point2): - '''Return distance of 2 points in desired unit measurement - + """Return distance of 2 points in desired unit measurement + Parameters ---------- point1 : tuple @@ -1558,37 +1556,35 @@ def calculate_distance(point1, point2): ------- float The distance between point1 and point2, according to the current coordinate system's distance metric, using the desired units - ''' + """ if self.CRS.coordinate_system.name == "cartesian": return math.dist(point1, point2) * podpac.units(units) else: - return ((geodesic(point1, point2, ellipsoid=ellipsoid_tuple).m) * podpac.units("metre")).to(podpac.units(units)) - + return ((geodesic(point1, point2, ellipsoid=ellipsoid_tuple).m) * podpac.units("metre")).to( + podpac.units(units) + ) - def check_horizontal(dim): - '''Check if a dimension is horizontal - + """Check if a dimension is horizontal + Parameters ---------- dim : str the name of the dimension - + Returns ------- bool Whether or not the current dimension is horizontal or not - ''' - for term in dim.split('_'): - if term == 'lat' or term == 'lon': + """ + for term in dim.split("_"): + if term == "lat" or term == "lon": return True return False - - def nominal_stacked_resolution(dim): - '''Use a KDTree to return approximate stacked resolution with some loss of accuracy. - + """Use a KDTree to return approximate stacked resolution with some loss of accuracy. + Parameters ---------- dim : str @@ -1597,19 +1593,20 @@ def nominal_stacked_resolution(dim): Returns ------- The average min distance of every point - - ''' - tree = spatial.KDTree(self[dim].coordinates + [90.,180.], boxsize=[0., 360.0000000000001]) + + """ + tree = spatial.KDTree(self[dim].coordinates + [90.0, 180.0], boxsize=[0.0, 360.0000000000001]) sum_distance = 0 for point in tree.data: - dd, ii = tree.query(point, k=2) # get nearest neighbor - sum_distance += calculate_distance(point - [90., 180.], tree.data[ii[1]] - [90., 180.]) # calculate distance - return sum_distance/len(tree.data) + dd, ii = tree.query(point, k=2) # get nearest neighbor + sum_distance += calculate_distance( + point - [90.0, 180.0], tree.data[ii[1]] - [90.0, 180.0] + ) # calculate distance + return sum_distance / len(tree.data) - def summary_stacked_resolution(dim): - '''Return the approximate mean resolution and std.deviation using a KDTree - + """Return the approximate mean resolution and std.deviation using a KDTree + Parameters ---------- dim : str @@ -1619,26 +1616,29 @@ def summary_stacked_resolution(dim): ------- tuple Average min distance of every point and standard deviation of those min distances - ''' - tree = spatial.KDTree(self[dim].coordinates + [90.,180.], boxsize=[0., 360.0000000000001]) + """ + tree = spatial.KDTree(self[dim].coordinates + [90.0, 180.0], boxsize=[0.0, 360.0000000000001]) sum_distance = 0 for point in tree.data: - dd, ii = tree.query(point, k=2) # get nearest neighbor - sum_distance += calculate_distance(point - [90., 180.], tree.data[ii[1]] - [90., 180.]) # calculate distance - avg_distance = sum_distance/len(tree.data) + dd, ii = tree.query(point, k=2) # get nearest neighbor + sum_distance += calculate_distance( + point - [90.0, 180.0], tree.data[ii[1]] - [90.0, 180.0] + ) # calculate distance + avg_distance = sum_distance / len(tree.data) # calculate standard deviation std_dev = 0 for point in tree.data: - dd, ii = tree.query(point, k=2) # get nearest neighbor - std_dev += (calculate_distance(point - [90., 180.], tree.data[ii[1]] - [90., 180.]) - avg_distance)**2 + dd, ii = tree.query(point, k=2) # get nearest neighbor + std_dev += ( + calculate_distance(point - [90.0, 180.0], tree.data[ii[1]] - [90.0, 180.0]) - avg_distance + ) ** 2 std_dev /= len(tree.data) std_dev = math.sqrt(std_dev.magnitude) return (avg_distance, std_dev) - def full_stacked_resolution(dim): - '''Returns the exact distance between every point using brute force - + """Returns the exact distance between every point using brute force + Parameters ---------- dim : str @@ -1647,35 +1647,34 @@ def full_stacked_resolution(dim): Returns ------- distance matrix of size (NxN), where N is the number of points in the dimension - ''' - distance_matrix = np.zeros((len(self[dim].coordinates),len(self[dim].coordinates))) + """ + distance_matrix = np.zeros((len(self[dim].coordinates), len(self[dim].coordinates))) for i in range(len(self[dim].coordinates)): for j in range(len(self[dim].coordinates)): - distance_matrix[i][j] = calculate_distance(self[dim].coordinates[i],self[dim].coordinates[j]).magnitude - return distance_matrix *podpac.units(units) - + distance_matrix[i][j] = calculate_distance( + self[dim].coordinates[i], self[dim].coordinates[j] + ).magnitude + return distance_matrix * podpac.units(units) - def nominal_unstacked_resolution(dim): - '''Return resolution for unstacked coordiantes using the bounds - + """Return resolution for unstacked coordiantes using the bounds + Parameters ---------- dim : str The dimenion to return the resolution of. Should be unstacked. - + Returns -------- The average distance between each grid square for this dimension - ''' + """ top_bound = (self.bounds[dim][1], 0) bottom_bound = (self.bounds[dim][0], 0) - return (calculate_distance(top_bound, bottom_bound)/ self[dim].size) + return calculate_distance(top_bound, bottom_bound) / self[dim].size - def summary_unstacked_resolution(dim): - '''Return summary resolution for the dimension. - + """Return summary resolution for the dimension. + Parameters ---------- dim : str @@ -1686,16 +1685,16 @@ def summary_unstacked_resolution(dim): tuple the average distance between grid squares the standard deviation of those distances - ''' - diff = np.zeros(len(self[dim].coordinates)-1) + """ + diff = np.zeros(len(self[dim].coordinates) - 1) for i in range(len(diff)): - top_bound = (self[dim].coordinates[i+1], 0) + top_bound = (self[dim].coordinates[i + 1], 0) bottom_bound = (self[dim].coordinates[i], 0) diff[i] = calculate_distance(top_bound, bottom_bound).magnitude return (np.average(diff) * podpac.units(units), np.std(diff) * podpac.units(units)) def full_unstacked_resolution(dim): - ''' Calculate full resolution of unstacked dimension + """Calculate full resolution of unstacked dimension Parameters ---------- @@ -1705,21 +1704,25 @@ def full_unstacked_resolution(dim): Returns ------- An array of every distance between each grid point - ''' - diff = np.zeros(len(self[dim].coordinates)-1) + """ + diff = np.zeros(len(self[dim].coordinates) - 1) for i in range(len(diff)): - top_bound = (self[dim].coordinates[i+1], 0) + top_bound = (self[dim].coordinates[i + 1], 0) bottom_bound = (self[dim].coordinates[i], 0) diff[i] = calculate_distance(top_bound, bottom_bound).magnitude - return diff *podpac.units(units) + return diff * podpac.units(units) - '''---------------------------------------------------------------------''' + """---------------------------------------------------------------------""" # dictionary: resolution_dict = OrderedDict() # ellipsoid Tuple - ellipsoid_tuple = (self.CRS.ellipsoid.semi_major_metre/1000, self.CRS.ellipsoid.semi_minor_metre/1000, 1 / self.CRS.ellipsoid.inverse_flattening) + ellipsoid_tuple = ( + self.CRS.ellipsoid.semi_major_metre / 1000, + self.CRS.ellipsoid.semi_minor_metre / 1000, + 1 / self.CRS.ellipsoid.inverse_flattening, + ) # main execution loop for dim in self.dims: @@ -1730,7 +1733,7 @@ def full_unstacked_resolution(dim): if self.is_stacked(dim): # stacked_resolution(dim) match type: - case "nominal": + case "nominal": resolution_dict[dim] = nominal_stacked_resolution(dim) case "summary": resolution_dict[dim] = summary_stacked_resolution(dim) @@ -1738,19 +1741,18 @@ def full_unstacked_resolution(dim): resolution_dict[dim] = full_stacked_resolution(dim) case _: return ValueError("Invalid value for type: {}".format(type)) - else: # unstacked resolution + else: # unstacked resolution match type: - case "nominal": + case "nominal": resolution_dict[dim] = nominal_unstacked_resolution(dim) case "summary": - resolution_dict[dim] = summary_unstacked_resolution(dim) + resolution_dict[dim] = summary_unstacked_resolution(dim) case "full": resolution_dict[dim] = full_unstacked_resolution(dim) case _: return ValueError("Invalid value for type: {}".format(type)) - - return resolution_dict + return resolution_dict # ------------------------------------------------------------------------------------------------------------------ # Operators/Magic Methods From 4aadde97eec3b9f7d6d4bbdf6eafffc8a007a27b Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Tue, 7 Feb 2023 13:05:59 -0500 Subject: [PATCH 41/52] FIX: match to if-else --- podpac/core/coordinates/coordinates.py | 40 ++++++++++++-------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/podpac/core/coordinates/coordinates.py b/podpac/core/coordinates/coordinates.py index 4de90dee..63bd4460 100644 --- a/podpac/core/coordinates/coordinates.py +++ b/podpac/core/coordinates/coordinates.py @@ -1506,13 +1506,13 @@ def issubset(self, other): return all(c.issubset(other) for c in self.values()) def is_stacked(self, dim): # re-wrote to be able to iterate through c.dims - match (dim in self.dims) + (dim in self.udims): - case 0: # both false - raise ValueError("Dimension {} is not in self.dims={}".format(dim, self.dims)) - case 1: # one true, one false - return True - case 2: # both true - return False + value = (dim in self.dims) + (dim in self.udims) + if value == 0: + raise ValueError("Dimension {} is not in self.dims={}".format(dim, self.dims)) + elif value == 1: # one true, one false + return True + elif value == 2: # both true + return False def horizontal_resolution(self, units="metre", type="nominal"): """ @@ -1732,24 +1732,22 @@ def full_unstacked_resolution(dim): # stacked coordinate resolutions if self.is_stacked(dim): # stacked_resolution(dim) - match type: - case "nominal": - resolution_dict[dim] = nominal_stacked_resolution(dim) - case "summary": - resolution_dict[dim] = summary_stacked_resolution(dim) - case "full": - resolution_dict[dim] = full_stacked_resolution(dim) - case _: - return ValueError("Invalid value for type: {}".format(type)) + if type == "nominal": + resolution_dict[dim] = nominal_stacked_resolution(dim) + elif type == "summary": + resolution_dict[dim] = summary_stacked_resolution(dim) + elif type == "full": + resolution_dict[dim] = full_stacked_resolution(dim) + else: + return ValueError("Invalid value for type: {}".format(type)) else: # unstacked resolution - match type: - case "nominal": + if type == "nominal": resolution_dict[dim] = nominal_unstacked_resolution(dim) - case "summary": + elif type == "summary": resolution_dict[dim] = summary_unstacked_resolution(dim) - case "full": + elif type == "full": resolution_dict[dim] = full_unstacked_resolution(dim) - case _: + else: return ValueError("Invalid value for type: {}".format(type)) return resolution_dict From 28640746e31864bb4ebb3feb94c3fe09430d97b0 Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Tue, 7 Feb 2023 13:25:56 -0500 Subject: [PATCH 42/52] FIX: O(n) std.dev and mean --- podpac/core/coordinates/coordinates.py | 34 ++++++++++++-------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/podpac/core/coordinates/coordinates.py b/podpac/core/coordinates/coordinates.py index 63bd4460..c09915e6 100644 --- a/podpac/core/coordinates/coordinates.py +++ b/podpac/core/coordinates/coordinates.py @@ -1511,7 +1511,7 @@ def is_stacked(self, dim): # re-wrote to be able to iterate through c.dims raise ValueError("Dimension {} is not in self.dims={}".format(dim, self.dims)) elif value == 1: # one true, one false return True - elif value == 2: # both true + elif value == 2: # both true return False def horizontal_resolution(self, units="metre", type="nominal"): @@ -1619,21 +1619,17 @@ def summary_stacked_resolution(dim): """ tree = spatial.KDTree(self[dim].coordinates + [90.0, 180.0], boxsize=[0.0, 360.0000000000001]) sum_distance = 0 + sum_distance_sq = 0 for point in tree.data: dd, ii = tree.query(point, k=2) # get nearest neighbor - sum_distance += calculate_distance( + distance = calculate_distance( point - [90.0, 180.0], tree.data[ii[1]] - [90.0, 180.0] ) # calculate distance + sum_distance += distance + sum_distance_sq += distance**2 avg_distance = sum_distance / len(tree.data) - # calculate standard deviation - std_dev = 0 - for point in tree.data: - dd, ii = tree.query(point, k=2) # get nearest neighbor - std_dev += ( - calculate_distance(point - [90.0, 180.0], tree.data[ii[1]] - [90.0, 180.0]) - avg_distance - ) ** 2 - std_dev /= len(tree.data) - std_dev = math.sqrt(std_dev.magnitude) + variance = (sum_distance_sq / len(tree.data)) - (avg_distance**2) + std_dev = math.sqrt(variance.magnitude) return (avg_distance, std_dev) def full_stacked_resolution(dim): @@ -1741,14 +1737,14 @@ def full_unstacked_resolution(dim): else: return ValueError("Invalid value for type: {}".format(type)) else: # unstacked resolution - if type == "nominal": - resolution_dict[dim] = nominal_unstacked_resolution(dim) - elif type == "summary": - resolution_dict[dim] = summary_unstacked_resolution(dim) - elif type == "full": - resolution_dict[dim] = full_unstacked_resolution(dim) - else: - return ValueError("Invalid value for type: {}".format(type)) + if type == "nominal": + resolution_dict[dim] = nominal_unstacked_resolution(dim) + elif type == "summary": + resolution_dict[dim] = summary_unstacked_resolution(dim) + elif type == "full": + resolution_dict[dim] = full_unstacked_resolution(dim) + else: + return ValueError("Invalid value for type: {}".format(type)) return resolution_dict From b7b54da935d6d5e273a150bb7f009c1e88cdfbe1 Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Wed, 8 Feb 2023 12:00:17 -0500 Subject: [PATCH 43/52] MAINT: vectorization pt.1 --- podpac/core/coordinates/coordinates.py | 54 +++++++++---------- .../core/coordinates/test/test_coordinates.py | 6 +-- 2 files changed, 27 insertions(+), 33 deletions(-) diff --git a/podpac/core/coordinates/coordinates.py b/podpac/core/coordinates/coordinates.py index c09915e6..7e22b3b5 100644 --- a/podpac/core/coordinates/coordinates.py +++ b/podpac/core/coordinates/coordinates.py @@ -1558,11 +1558,22 @@ def calculate_distance(point1, point2): The distance between point1 and point2, according to the current coordinate system's distance metric, using the desired units """ if self.CRS.coordinate_system.name == "cartesian": - return math.dist(point1, point2) * podpac.units(units) + return np.linalg.norm(point1 - point2, axis=-1) * podpac.units(units) else: - return ((geodesic(point1, point2, ellipsoid=ellipsoid_tuple).m) * podpac.units("metre")).to( - podpac.units(units) - ) + if not isinstance(point1, tuple) and point1.size > 2 : + distances = np.empty(len(point1)) + for i in range(len(point1)): + distances[i] = ((geodesic(point1[i], point2[i], ellipsoid=ellipsoid_tuple).m) + ) + return distances* podpac.units("metre").to(podpac.units(units)) + if not isinstance(point2, tuple) and point2.size > 2: + distances = np.empty(len(point2)) + for i in range(len(point2)): + distances[i] = geodesic(point1, point2[i], ellipsoid=ellipsoid_tuple).m + return distances * podpac.units("metre").to(podpac.units(units)) + else: + return (geodesic(point1, point2, ellipsoid=ellipsoid_tuple).m) * podpac.units("metre").to(podpac.units(units)) + def check_horizontal(dim): """Check if a dimension is horizontal @@ -1595,14 +1606,11 @@ def nominal_stacked_resolution(dim): The average min distance of every point """ - tree = spatial.KDTree(self[dim].coordinates + [90.0, 180.0], boxsize=[0.0, 360.0000000000001]) - sum_distance = 0 - for point in tree.data: - dd, ii = tree.query(point, k=2) # get nearest neighbor - sum_distance += calculate_distance( - point - [90.0, 180.0], tree.data[ii[1]] - [90.0, 180.0] - ) # calculate distance - return sum_distance / len(tree.data) + tree = spatial.KDTree(self[dim].coordinates + [0, 180.0], boxsize=[0.0, 360.0000000000001]) + return np.average(calculate_distance(tree.data - [0, 180.0], tree.data[tree.query(tree.data, k=2)[1][:,1]] - [0, 180.0])) + + + def summary_stacked_resolution(dim): """Return the approximate mean resolution and std.deviation using a KDTree @@ -1617,20 +1625,9 @@ def summary_stacked_resolution(dim): tuple Average min distance of every point and standard deviation of those min distances """ - tree = spatial.KDTree(self[dim].coordinates + [90.0, 180.0], boxsize=[0.0, 360.0000000000001]) - sum_distance = 0 - sum_distance_sq = 0 - for point in tree.data: - dd, ii = tree.query(point, k=2) # get nearest neighbor - distance = calculate_distance( - point - [90.0, 180.0], tree.data[ii[1]] - [90.0, 180.0] - ) # calculate distance - sum_distance += distance - sum_distance_sq += distance**2 - avg_distance = sum_distance / len(tree.data) - variance = (sum_distance_sq / len(tree.data)) - (avg_distance**2) - std_dev = math.sqrt(variance.magnitude) - return (avg_distance, std_dev) + tree = spatial.KDTree(self[dim].coordinates + [0, 180.0], boxsize=[0.0, 360.0000000000001]) + distances = (calculate_distance(tree.data - [0, 180.0], tree.data[tree.query(tree.data, k=2)[1][:,1]] - [0, 180.0])) + return (np.average(distances), np.std(distances)) def full_stacked_resolution(dim): """Returns the exact distance between every point using brute force @@ -1646,10 +1643,7 @@ def full_stacked_resolution(dim): """ distance_matrix = np.zeros((len(self[dim].coordinates), len(self[dim].coordinates))) for i in range(len(self[dim].coordinates)): - for j in range(len(self[dim].coordinates)): - distance_matrix[i][j] = calculate_distance( - self[dim].coordinates[i], self[dim].coordinates[j] - ).magnitude + distance_matrix[i,:] = calculate_distance(self[dim].coordinates[i], self[dim].coordinates[:]).magnitude return distance_matrix * podpac.units(units) def nominal_unstacked_resolution(dim): diff --git a/podpac/core/coordinates/test/test_coordinates.py b/podpac/core/coordinates/test/test_coordinates.py index 752eea84..8875b479 100644 --- a/podpac/core/coordinates/test/test_coordinates.py +++ b/podpac/core/coordinates/test/test_coordinates.py @@ -2246,11 +2246,11 @@ def test_resolution(self): c = Coordinates([[lat, lon]], dims=['lat_lon']) # Nominal - assert c.horizontal_resolution(type="nominal") == OrderedDict([('lat_lon', 7014367.35521632 * podpac.units("m"))]) + assert (c.horizontal_resolution(type="nominal") == OrderedDict([('lat_lon', 7014367.35521632 * podpac.units("m"))])) # Summary assert c.horizontal_resolution(type="summary") == OrderedDict([('lat_lon', - (7014367.35521632 * podpac.units("m"), 2446239.274648156))]) + (7014367.35521632 * podpac.units("m"), 2446239.274648156 * podpac.units("m")))]) # Full answer = OrderedDict([('lat_lon', @@ -2270,7 +2270,7 @@ def test_resolution(self): # Cartesian c = podpac.Coordinates([[podpac.clinspace(0, 90, 20, 'lat'), podpac.clinspace(0, 90, 20, 'lon')]], dims = ['lat_lon'], crs='+proj=utm +zone=9 +north +ellps=WGS84 +datum=WGS84 +units=m +no_defs') - assert c.horizontal_resolution() == OrderedDict([('lat_lon', 6.6989063480830735 * podpac.units("m"))]) + assert c.horizontal_resolution() == OrderedDict([('lat_lon', 6.698906348083078 * podpac.units("m"))]) '''Stacked and Unstacked Coordinates''' time = ['2018-01-01', '2018-01-02'] From fe4df5e573d0b2d8c66c7c5e9002db996244b870 Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Wed, 8 Feb 2023 14:57:37 -0500 Subject: [PATCH 44/52] FIX: math not needed --- podpac/core/coordinates/coordinates.py | 2 +- setup.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/podpac/core/coordinates/coordinates.py b/podpac/core/coordinates/coordinates.py index 7e22b3b5..c4cdde41 100644 --- a/podpac/core/coordinates/coordinates.py +++ b/podpac/core/coordinates/coordinates.py @@ -21,7 +21,7 @@ import logging from scipy import spatial from geopy.distance import geodesic -import math + import podpac diff --git a/setup.py b/setup.py index 6065b757..875c920b 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,6 @@ "psutil", "affine", "geopy", - "math" ] if sys.version_info.major == 2: From 52359cef084a019c6a6a8100476e653a06ab203e Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Fri, 10 Feb 2023 11:02:07 -0500 Subject: [PATCH 45/52] FIX: correct distances for unstacked --- podpac/core/coordinates/coordinates.py | 112 +++++++++-- .../core/coordinates/test/test_coordinates.py | 174 ++++++++++++++++-- 2 files changed, 255 insertions(+), 31 deletions(-) diff --git a/podpac/core/coordinates/coordinates.py b/podpac/core/coordinates/coordinates.py index c4cdde41..b3ac7056 100644 --- a/podpac/core/coordinates/coordinates.py +++ b/podpac/core/coordinates/coordinates.py @@ -1525,8 +1525,8 @@ def horizontal_resolution(self, units="metre", type="nominal"): type : str The kind of horizontal resolution that should be returned. Supported values are: - "nominal" <-- this is wrong but cheap to calculate. Give a 'nominal' resolution over the entire domain - - "summary" <-- This is still approximate, gives the mean and standard deviation to each point - - "full" <-- min distance between every point (dx, dy, basically) + - "summary" <-- Gives the exact mean and standard deviation for unstacked coordinates, some error for stacked coordinates + - "full" <-- Gives exact grid differences if unstacked coordiantes or distance matrix if stacked coordinnates Returns ------- @@ -1658,9 +1658,15 @@ def nominal_unstacked_resolution(dim): -------- The average distance between each grid square for this dimension """ - top_bound = (self.bounds[dim][1], 0) - bottom_bound = (self.bounds[dim][0], 0) - return calculate_distance(top_bound, bottom_bound) / self[dim].size + if dim == 'lat': + return (calculate_distance((self[dim].bounds[0],0), (self[dim].bounds[1],0)).magnitude / self[dim].size) * podpac.units(units) + elif dim == 'lon': + # Get median lat: + # need this because of difference in circumference + median_lat = ((self['lat'].bounds[1] - self['lat'].bounds[0]) / 2) + self['lat'].bounds[0] + return (calculate_distance((median_lat, self[dim].bounds[0]), (median_lat, self[dim].bounds[1])).magnitude / self[dim].size) * podpac.units(units) + else: + return ValueError("Unknown dim: {}".format(dim)) def summary_unstacked_resolution(dim): """Return summary resolution for the dimension. @@ -1676,12 +1682,40 @@ def summary_unstacked_resolution(dim): the average distance between grid squares the standard deviation of those distances """ - diff = np.zeros(len(self[dim].coordinates) - 1) - for i in range(len(diff)): - top_bound = (self[dim].coordinates[i + 1], 0) - bottom_bound = (self[dim].coordinates[i], 0) - diff[i] = calculate_distance(top_bound, bottom_bound).magnitude - return (np.average(diff) * podpac.units(units), np.std(diff) * podpac.units(units)) + if dim == "lat" or dim == "lon": + full_res = full_unstacked_resolution(dim).magnitude + return (np.average(full_res)* podpac.units(units), np.std(full_res) * podpac.units(units)) + else: + return ValueError("Unknown dim: {}".format(dim)) + + # if dim == 'lat': + # # Top bounds + # top_bounds = self[dim].coordinates[1:] + # top_bounds = np.stack([top_bounds, self['lon'].coordinates[1:]], axis=1) + + # # Bottom bounds + # bottom_bounds = self[dim].coordinates[:-1] + # bottom_bounds = np.stack([bottom_bounds, self['lon'].coordinates[1:]], axis=1) + + # # differences: + # diff = calculate_distance(top_bounds, bottom_bounds).magnitude + + # # Return standard deviation and average + # return (np.average(diff)*podpac.units(units), np.std(diff)*podpac.units(units)) + # elif dim == 'lon': + # # top bounds + # top_bounds = np.stack([self['lat'].coordinates[1:], self[dim].coordinates[1:]], axis=1) # use exact lat values + + # # Bottom bounds + # bottom_bounds = np.stack([self['lat'].coordinates[:-1], self[dim].coordinates[:-1]], axis=1) # use exact lat values + + # # differences + # diff = calculate_distance(top_bounds, bottom_bounds).magnitude + + # # Return standard deviation and average + # return (np.average(diff)*podpac.units(units), np.std(diff)*podpac.units(units)) + # else: + # return ValueError("Unknown dim: {}".format(dim)) def full_unstacked_resolution(dim): """Calculate full resolution of unstacked dimension @@ -1693,14 +1727,56 @@ def full_unstacked_resolution(dim): Returns ------- - An array of every distance between each grid point + A matrix of distances """ - diff = np.zeros(len(self[dim].coordinates) - 1) - for i in range(len(diff)): - top_bound = (self[dim].coordinates[i + 1], 0) - bottom_bound = (self[dim].coordinates[i], 0) - diff[i] = calculate_distance(top_bound, bottom_bound).magnitude - return diff * podpac.units(units) + if dim == "lat": + M = self['lat'].coordinates.size + N = self['lon'].coordinates.size + diff= np.zeros((N,M-1)) + for i in range(N): + lat_values= self['lat'].coordinates + lon_value = self['lon'].coordinates[i] + top_bounds = np.stack([lat_values[1:], np.full((lat_values[1:]).shape[0], lon_value)], axis=1) # use exact lat values + bottom_bounds = np.stack([lat_values[:-1], np.full((lat_values[:-1]).shape[0], lon_value)], axis=1) # use exact lat values + diff[i] = calculate_distance(top_bounds, bottom_bounds).magnitude + return diff * podpac.units(units) + elif dim == "lon": + M = self['lat'].coordinates.size + N = self['lon'].coordinates.size + diff= np.zeros((M,N-1)) + for i in range(M): + lat_value= self['lat'].coordinates[i] + lon_values = self['lon'].coordinates + top_bounds = np.stack([np.full((lon_values[1:]).shape[0], lat_value), lon_values[1:]], axis=1) # use exact lat values + bottom_bounds = np.stack([np.full((lon_values[:-1]).shape[0], lat_value), lon_values[:-1]], axis=1) # use exact lat values + diff[i] = calculate_distance(top_bounds, bottom_bounds).magnitude + return diff * podpac.units(units) + else: + return ValueError("Unknown dim: {}".format(dim)) + + # if dim == 'lat': + # # Top bounds + # top_bounds = self[dim].coordinates[1:] + # top_bounds = np.stack([top_bounds, np.zeros(top_bounds.shape[0])], axis=1) + + # # Bottom bounds + # bottom_bounds = self[dim].coordinates[:-1] + # bottom_bounds = np.stack([bottom_bounds, np.zeros(bottom_bounds.shape[0])], axis=1) + + # # Return differences + # return calculate_distance(top_bounds, bottom_bounds) + # elif dim == 'lon': + # # top bounds + # top_bounds = np.stack([self['lat'].coordinates[1:], self[dim].coordinates[1:]], axis=1) # use exact lat values + + # # Bottom bounds + # bottom_bounds = np.stack([self['lat'].coordinates[:-1], self[dim].coordinates[:-1]], axis=1) # use exact lat values + + # # Return differences + # return calculate_distance(top_bounds, bottom_bounds) + # else: + # return ValueError("Unknown dim: {}".format(dim)) + """---------------------------------------------------------------------""" diff --git a/podpac/core/coordinates/test/test_coordinates.py b/podpac/core/coordinates/test/test_coordinates.py index 8875b479..a6c9f0c7 100644 --- a/podpac/core/coordinates/test/test_coordinates.py +++ b/podpac/core/coordinates/test/test_coordinates.py @@ -2214,28 +2214,176 @@ def test_resolution(self): # Nominal assert (c.horizontal_resolution(type="nominal") == OrderedDict([('lat', 909269.611755702* podpac.units("metre")), - ('lon', 500098.2864656361* podpac.units("metre"))])) + ('lon', 334511.6466271357* podpac.units("metre"))])) # Summary assert (c.horizontal_resolution(type="summary") == OrderedDict([('lat', - (1000196.5729312723* podpac.units("metre"), + (1000196.5729312722* podpac.units("metre"), 3548.2348372254723* podpac.units("metre"))), ('lon', - (526419.2489111959* podpac.units("metre"), - 1873.0586112168473* podpac.units("metre")))])) + (328825.13300389395* podpac.units("metre"), + 176330.22544905098* podpac.units("metre")))])) # Full answer = OrderedDict([('lat', - ([ 995250.27739564, 995731.65748247, 996648.35979764, - 997912.37572808, 999401.70757391, 1000971.65042421, - 1002468.54050245, 1003744.78599167, 1004673.71796683, - 1005162.65644982]) * podpac.units("m")), + ([995250.27739564, 995731.65748247, 996648.35979764, + 997912.37572808, 999401.70757391, 1000971.65042421, + 1002468.54050245, 1003744.78599167, 1004673.71796683, + 1005162.65644982], + [ 995250.27739564, 995731.65748247, 996648.35979764, + 997912.37572808, 999401.70757391, 1000971.65042421, + 1002468.54050245, 1003744.78599167, 1004673.71796683, + 1005162.65644982], + [ 995250.27739564, 995731.65748247, 996648.35979764, + 997912.37572808, 999401.70757391, 1000971.65042421, + 1002468.54050245, 1003744.78599167, 1004673.71796683, + 1005162.65644982], + [ 995250.27739564, 995731.65748247, 996648.35979764, + 997912.37572808, 999401.70757391, 1000971.65042421, + 1002468.54050245, 1003744.78599167, 1004673.71796683, + 1005162.65644982], + [ 995250.27739564, 995731.65748247, 996648.35979764, + 997912.37572808, 999401.70757391, 1000971.65042421, + 1002468.54050245, 1003744.78599167, 1004673.71796683, + 1005162.65644982], + [ 995250.27739564, 995731.65748247, 996648.35979764, + 997912.37572808, 999401.70757391, 1000971.65042421, + 1002468.54050245, 1003744.78599167, 1004673.71796683, + 1005162.65644982], + [ 995250.27739564, 995731.65748247, 996648.35979764, + 997912.37572808, 999401.70757391, 1000971.65042421, + 1002468.54050245, 1003744.78599167, 1004673.71796683, + 1005162.65644982], + [ 995250.27739564, 995731.65748247, 996648.35979764, + 997912.37572808, 999401.70757391, 1000971.65042421, + 1002468.54050245, 1003744.78599167, 1004673.71796683, + 1005162.65644982], + [ 995250.27739564, 995731.65748247, 996648.35979764, + 997912.37572808, 999401.70757391, 1000971.65042421, + 1002468.54050245, 1003744.78599167, 1004673.71796683, + 1005162.65644982], + [ 995250.27739564, 995731.65748247, 996648.35979764, + 997912.37572808, 999401.70757391, 1000971.65042421, + 1002468.54050245, 1003744.78599167, 1004673.71796683, + 1005162.65644982], + [ 995250.27739564, 995731.65748247, 996648.35979764, + 997912.37572808, 999401.70757391, 1000971.65042421, + 1002468.54050245, 1003744.78599167, 1004673.71796683, + 1005162.65644982], + [ 995250.27739564, 995731.65748247, 996648.35979764, + 997912.37572808, 999401.70757391, 1000971.65042421, + 1002468.54050245, 1003744.78599167, 1004673.71796683, + 1005162.65644982], + [ 995250.27739564, 995731.65748247, 996648.35979764, + 997912.37572808, 999401.70757391, 1000971.65042421, + 1002468.54050245, 1003744.78599167, 1004673.71796683, + 1005162.65644982], + [ 995250.27739564, 995731.65748247, 996648.35979764, + 997912.37572808, 999401.70757391, 1000971.65042421, + 1002468.54050245, 1003744.78599167, 1004673.71796683, + 1005162.65644982], + [ 995250.27739564, 995731.65748247, 996648.35979764, + 997912.37572808, 999401.70757391, 1000971.65042421, + 1002468.54050245, 1003744.78599167, 1004673.71796683, + 1005162.65644982], + [ 995250.27739564, 995731.65748247, 996648.35979764, + 997912.37572808, 999401.70757391, 1000971.65042421, + 1002468.54050245, 1003744.78599167, 1004673.71796683, + 1005162.65644982], + [ 995250.27739564, 995731.65748247, 996648.35979764, + 997912.37572808, 999401.70757391, 1000971.65042421, + 1002468.54050245, 1003744.78599167, 1004673.71796683, + 1005162.65644982], + [ 995250.27739564, 995731.65748247, 996648.35979764, + 997912.37572808, 999401.70757391, 1000971.65042421, + 1002468.54050245, 1003744.78599167, 1004673.71796683, + 1005162.65644982], + [ 995250.27739564, 995731.65748247, 996648.35979764, + 997912.37572808, 999401.70757391, 1000971.65042421, + 1002468.54050245, 1003744.78599167, 1004673.71796683, + 1005162.65644982], + [ 995250.27739564, 995731.65748247, 996648.35979764, + 997912.37572808, 999401.70757391, 1000971.65042421, + 1002468.54050245, 1003744.78599167, 1004673.71796683, + 1005162.65644982]) * podpac.units("m")), ('lon', - ([523784.85224537, 523856.27075792, 523997.2071627 , 524203.90698729, - 524470.85470098, 524790.91101581, 525155.49233879, 525554.78910432, - 525978.01872173, 526413.70781726, 526849.99736995, 527274.96329269, - 527676.94406298, 528044.86624478, 528368.55824467, 528639.04248812, - 528848.79644537, 528991.97361553, 529064.57669647]) * podpac.units("m"))]) + ([527302.85112603, 527302.85112603, 527302.85112603, + 527302.85112603, 527302.85112603, 527302.85112603, + 527302.85112603, 527302.85112603, 527302.85112603, + 527302.85112603, 527302.85112603, 527302.85112603, + 527302.85112603, 527302.85112603, 527302.85112603, + 527302.85112603, 527302.85112603, 527302.85112603, + 527302.85112603], + [520849.91130839, 520849.91130839, 520849.91130839, + 520849.91130839, 520849.91130839, 520849.91130839, + 520849.91130839, 520849.91130839, 520849.91130839, + 520849.91130839, 520849.91130839, 520849.91130839, + 520849.91130839, 520849.91130839, 520849.91130839, + 520849.91130838, 520849.91130838, 520849.91130839, + 520849.91130839], + [501641.53060499, 501641.53060499, 501641.53060499, + 501641.53060499, 501641.53060499, 501641.53060499, + 501641.53060499, 501641.53060499, 501641.53060499, + 501641.53060499, 501641.53060499, 501641.53060499, + 501641.53060499, 501641.53060499, 501641.53060499, + 501641.53060499, 501641.53060499, 501641.53060499, + 501641.53060499], + [470127.1319624 , 470127.1319624 , 470127.1319624 , + 470127.1319624 , 470127.1319624 , 470127.1319624 , + 470127.1319624 , 470127.1319624 , 470127.1319624 , + 470127.1319624 , 470127.1319624 , 470127.1319624 , + 470127.1319624 , 470127.1319624 , 470127.1319624 , + 470127.1319624 , 470127.1319624 , 470127.1319624 , + 470127.1319624 ], + [427049.11294657, 427049.11294657, 427049.11294657, + 427049.11294657, 427049.11294657, 427049.11294657, + 427049.11294657, 427049.11294657, 427049.11294657, + 427049.11294657, 427049.11294657, 427049.11294657, + 427049.11294657, 427049.11294657, 427049.11294657, + 427049.11294657, 427049.11294657, 427049.11294657, + 427049.11294657], + [373431.8103952 , 373431.8103952 , 373431.8103952 , + 373431.8103952 , 373431.8103952 , 373431.8103952 , + 373431.8103952 , 373431.8103952 , 373431.8103952 , + 373431.8103952 , 373431.8103952 , 373431.8103952 , + 373431.8103952 , 373431.8103952 , 373431.8103952 , + 373431.8103952 , 373431.8103952 , 373431.8103952 , + 373431.8103952 ], + [310564.17702203, 310564.17702203, 310564.17702203, + 310564.17702203, 310564.17702203, 310564.17702203, + 310564.17702203, 310564.17702203, 310564.17702203, + 310564.17702203, 310564.17702203, 310564.17702203, + 310564.17702203, 310564.17702203, 310564.17702203, + 310564.17702203, 310564.17702203, 310564.17702203, + 310564.17702203], + [239974.89400498, 239974.89400498, 239974.89400498, + 239974.89400498, 239974.89400498, 239974.89400498, + 239974.89400498, 239974.89400498, 239974.89400498, + 239974.89400498, 239974.89400498, 239974.89400498, + 239974.89400498, 239974.89400498, 239974.89400498, + 239974.89400498, 239974.89400498, 239974.89400498, + 239974.89400498], + [163399.02083826, 163399.02083826, 163399.02083826, + 163399.02083826, 163399.02083826, 163399.02083826, + 163399.02083826, 163399.02083826, 163399.02083826, + 163399.02083826, 163399.02083826, 163399.02083826, + 163399.02083826, 163399.02083826, 163399.02083826, + 163399.02083826, 163399.02083826, 163399.02083826, + 163399.02083826], + [ 82736.02283397, 82736.02283397, 82736.02283397, + 82736.02283397, 82736.02283397, 82736.02283397, + 82736.02283397, 82736.02283397, 82736.02283397, + 82736.02283397, 82736.02283397, 82736.02283397, + 82736.02283397, 82736.02283397, 82736.02283397, + 82736.02283397, 82736.02283397, 82736.02283397, + 82736.02283397], + [ 0. , 0. , 0. , + 0. , 0. , 0. , + 0. , 0. , 0. , + 0. , 0. , 0. , + 0. , 0. , 0. , + 0. , 0. , 0. , + 0. ]) * podpac.units("m"))]) assert answer['lat'].all() == c.horizontal_resolution(type="full")['lat'].all() and answer['lon'].all() == c.horizontal_resolution(type="full")['lon'].all() From 809c61c4df4edcfd2d040d0e08d03033231d1eed Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Fri, 10 Feb 2023 15:51:27 -0500 Subject: [PATCH 46/52] FIX: 'type' param and check dims --- podpac/core/coordinates/coordinates.py | 75 ++++--------------- .../core/coordinates/test/test_coordinates.py | 12 +-- 2 files changed, 22 insertions(+), 65 deletions(-) diff --git a/podpac/core/coordinates/coordinates.py b/podpac/core/coordinates/coordinates.py index b3ac7056..b89220a3 100644 --- a/podpac/core/coordinates/coordinates.py +++ b/podpac/core/coordinates/coordinates.py @@ -1514,7 +1514,7 @@ def is_stacked(self, dim): # re-wrote to be able to iterate through c.dims elif value == 2: # both true return False - def horizontal_resolution(self, units="metre", type="nominal"): + def horizontal_resolution(self, units="metre", restype="nominal"): """ Calculate horizontal resolution of coordinate system. @@ -1688,35 +1688,6 @@ def summary_unstacked_resolution(dim): else: return ValueError("Unknown dim: {}".format(dim)) - # if dim == 'lat': - # # Top bounds - # top_bounds = self[dim].coordinates[1:] - # top_bounds = np.stack([top_bounds, self['lon'].coordinates[1:]], axis=1) - - # # Bottom bounds - # bottom_bounds = self[dim].coordinates[:-1] - # bottom_bounds = np.stack([bottom_bounds, self['lon'].coordinates[1:]], axis=1) - - # # differences: - # diff = calculate_distance(top_bounds, bottom_bounds).magnitude - - # # Return standard deviation and average - # return (np.average(diff)*podpac.units(units), np.std(diff)*podpac.units(units)) - # elif dim == 'lon': - # # top bounds - # top_bounds = np.stack([self['lat'].coordinates[1:], self[dim].coordinates[1:]], axis=1) # use exact lat values - - # # Bottom bounds - # bottom_bounds = np.stack([self['lat'].coordinates[:-1], self[dim].coordinates[:-1]], axis=1) # use exact lat values - - # # differences - # diff = calculate_distance(top_bounds, bottom_bounds).magnitude - - # # Return standard deviation and average - # return (np.average(diff)*podpac.units(units), np.std(diff)*podpac.units(units)) - # else: - # return ValueError("Unknown dim: {}".format(dim)) - def full_unstacked_resolution(dim): """Calculate full resolution of unstacked dimension @@ -1754,29 +1725,6 @@ def full_unstacked_resolution(dim): else: return ValueError("Unknown dim: {}".format(dim)) - # if dim == 'lat': - # # Top bounds - # top_bounds = self[dim].coordinates[1:] - # top_bounds = np.stack([top_bounds, np.zeros(top_bounds.shape[0])], axis=1) - - # # Bottom bounds - # bottom_bounds = self[dim].coordinates[:-1] - # bottom_bounds = np.stack([bottom_bounds, np.zeros(bottom_bounds.shape[0])], axis=1) - - # # Return differences - # return calculate_distance(top_bounds, bottom_bounds) - # elif dim == 'lon': - # # top bounds - # top_bounds = np.stack([self['lat'].coordinates[1:], self[dim].coordinates[1:]], axis=1) # use exact lat values - - # # Bottom bounds - # bottom_bounds = np.stack([self['lat'].coordinates[:-1], self[dim].coordinates[:-1]], axis=1) # use exact lat values - - # # Return differences - # return calculate_distance(top_bounds, bottom_bounds) - # else: - # return ValueError("Unknown dim: {}".format(dim)) - """---------------------------------------------------------------------""" @@ -1790,28 +1738,37 @@ def full_unstacked_resolution(dim): 1 / self.CRS.ellipsoid.inverse_flattening, ) + # validate dims: + # main execution loop for dim in self.dims: # Is the dim lat/lon? if not check_horizontal(dim): continue + # Put this check inside the loop to avoid checking if stacked coords + if dim == "lat": + if "lon" not in self.dims: + return ValueError("Need both lat and lon for resolution.") + if dim == "lon": + if "lat" not in self.dims: + return ValueError("Need both lat and lon for resolution.") # stacked coordinate resolutions if self.is_stacked(dim): # stacked_resolution(dim) - if type == "nominal": + if restype == "nominal": resolution_dict[dim] = nominal_stacked_resolution(dim) - elif type == "summary": + elif restype == "summary": resolution_dict[dim] = summary_stacked_resolution(dim) - elif type == "full": + elif restype == "full": resolution_dict[dim] = full_stacked_resolution(dim) else: return ValueError("Invalid value for type: {}".format(type)) else: # unstacked resolution - if type == "nominal": + if restype == "nominal": resolution_dict[dim] = nominal_unstacked_resolution(dim) - elif type == "summary": + elif restype == "summary": resolution_dict[dim] = summary_unstacked_resolution(dim) - elif type == "full": + elif restype == "full": resolution_dict[dim] = full_unstacked_resolution(dim) else: return ValueError("Invalid value for type: {}".format(type)) diff --git a/podpac/core/coordinates/test/test_coordinates.py b/podpac/core/coordinates/test/test_coordinates.py index a6c9f0c7..17f42ea2 100644 --- a/podpac/core/coordinates/test/test_coordinates.py +++ b/podpac/core/coordinates/test/test_coordinates.py @@ -2213,11 +2213,11 @@ def test_resolution(self): c = podpac.Coordinates([podpac.clinspace(0, 90, 11, 'lat'), podpac.clinspace(0, 90, 20, 'lon')]) # Nominal - assert (c.horizontal_resolution(type="nominal") == OrderedDict([('lat', 909269.611755702* podpac.units("metre")), + assert (c.horizontal_resolution(restype="nominal") == OrderedDict([('lat', 909269.611755702* podpac.units("metre")), ('lon', 334511.6466271357* podpac.units("metre"))])) # Summary - assert (c.horizontal_resolution(type="summary") == OrderedDict([('lat', + assert (c.horizontal_resolution(restype="summary") == OrderedDict([('lat', (1000196.5729312722* podpac.units("metre"), 3548.2348372254723* podpac.units("metre"))), ('lon', @@ -2384,7 +2384,7 @@ def test_resolution(self): 0. , 0. , 0. , 0. , 0. , 0. , 0. ]) * podpac.units("m"))]) - assert answer['lat'].all() == c.horizontal_resolution(type="full")['lat'].all() and answer['lon'].all() == c.horizontal_resolution(type="full")['lon'].all() + assert answer['lat'].all() == c.horizontal_resolution(restype="full")['lat'].all() and answer['lon'].all() == c.horizontal_resolution(restype="full")['lon'].all() @@ -2394,10 +2394,10 @@ def test_resolution(self): c = Coordinates([[lat, lon]], dims=['lat_lon']) # Nominal - assert (c.horizontal_resolution(type="nominal") == OrderedDict([('lat_lon', 7014367.35521632 * podpac.units("m"))])) + assert (c.horizontal_resolution(restype="nominal") == OrderedDict([('lat_lon', 7014367.35521632 * podpac.units("m"))])) # Summary - assert c.horizontal_resolution(type="summary") == OrderedDict([('lat_lon', + assert c.horizontal_resolution(restype="summary") == OrderedDict([('lat_lon', (7014367.35521632 * podpac.units("m"), 2446239.274648156 * podpac.units("m")))]) # Full @@ -2413,7 +2413,7 @@ def test_resolution(self): [20003931.45862544, 14986910.10729047, 10001965.72931272, 5017021.35133498, 0. ]]) * podpac.units("m"))]) - assert c.horizontal_resolution(type="full")['lat_lon'].all() == answer['lat_lon'].all() + assert c.horizontal_resolution(restype="full")['lat_lon'].all() == answer['lat_lon'].all() # Cartesian c = podpac.Coordinates([[podpac.clinspace(0, 90, 20, 'lat'), podpac.clinspace(0, 90, 20, 'lon')]], dims = ['lat_lon'], crs='+proj=utm +zone=9 +north +ellps=WGS84 +datum=WGS84 +units=m +no_defs') From 5b853814780a6d7e9acc90f2ab539d2f20d9a58d Mon Sep 17 00:00:00 2001 From: Clay Foye Date: Tue, 14 Feb 2023 11:41:58 -0500 Subject: [PATCH 47/52] FIX: Refactor resolution functions, new tests --- podpac/core/coordinates/base_coordinates.py | 10 + podpac/core/coordinates/coordinates.py | 251 ++---------------- podpac/core/coordinates/coordinates1d.py | 121 +++++++++ .../core/coordinates/stacked_coordinates.py | 97 +++++++ .../core/coordinates/test/test_coordinates.py | 243 ++--------------- .../coordinates/test/test_coordinates1d.py | 59 ++++ .../test/test_stacked_coordinates.py | 40 +++ .../core/coordinates/uniform_coordinates1d.py | 2 + podpac/core/coordinates/utils.py | 35 +++ 9 files changed, 418 insertions(+), 440 deletions(-) diff --git a/podpac/core/coordinates/base_coordinates.py b/podpac/core/coordinates/base_coordinates.py index 47976403..98844a8b 100644 --- a/podpac/core/coordinates/base_coordinates.py +++ b/podpac/core/coordinates/base_coordinates.py @@ -68,6 +68,11 @@ def definition(self): def full_definition(self): """Coordinates definition, containing all properties. For internal use.""" raise NotImplementedError + + @property + def is_stacked(self): + """stacked or unstacked property""" + raise NotImplementedError @classmethod def from_definition(cls, d): @@ -106,6 +111,11 @@ def issubset(self, other): """Report if these coordinates are a subset of other coordinates.""" raise NotImplementedError + def horizontal_resolution(self, latitude, restype="nominal", units="meter"): + """Get horizontal resolution of coordiantes.""" + raise NotImplementedError + + def __getitem__(self, index): raise NotImplementedError diff --git a/podpac/core/coordinates/coordinates.py b/podpac/core/coordinates/coordinates.py index b89220a3..31e4b427 100644 --- a/podpac/core/coordinates/coordinates.py +++ b/podpac/core/coordinates/coordinates.py @@ -23,7 +23,6 @@ from geopy.distance import geodesic - import podpac from podpac.core.settings import settings from podpac.core.utils import OrderedDictTrait, _get_query_params_from_url, _get_param, cached_property @@ -1516,7 +1515,9 @@ def is_stacked(self, dim): # re-wrote to be able to iterate through c.dims def horizontal_resolution(self, units="metre", restype="nominal"): """ - Calculate horizontal resolution of coordinate system. + Return horizontal resolution of coordinate system. + This function handles mainly edge case sanitation. + It calls StackedCoordiantes and Coordinates1d 'horizontal_resolution' methods to get the actual values. Parameters ---------- @@ -1540,240 +1541,40 @@ def horizontal_resolution(self, units="metre", restype="nominal"): Raises ------ ValueError - If the 'type' is not one of the supported resolution types + If the 'restype' is not one of the supported resolution types """ - def calculate_distance(point1, point2): - """Return distance of 2 points in desired unit measurement - - Parameters - ---------- - point1 : tuple - point2 : tuple - - Returns - ------- - float - The distance between point1 and point2, according to the current coordinate system's distance metric, using the desired units - """ - if self.CRS.coordinate_system.name == "cartesian": - return np.linalg.norm(point1 - point2, axis=-1) * podpac.units(units) - else: - if not isinstance(point1, tuple) and point1.size > 2 : - distances = np.empty(len(point1)) - for i in range(len(point1)): - distances[i] = ((geodesic(point1[i], point2[i], ellipsoid=ellipsoid_tuple).m) - ) - return distances* podpac.units("metre").to(podpac.units(units)) - if not isinstance(point2, tuple) and point2.size > 2: - distances = np.empty(len(point2)) - for i in range(len(point2)): - distances[i] = geodesic(point1, point2[i], ellipsoid=ellipsoid_tuple).m - return distances * podpac.units("metre").to(podpac.units(units)) - else: - return (geodesic(point1, point2, ellipsoid=ellipsoid_tuple).m) * podpac.units("metre").to(podpac.units(units)) - - - def check_horizontal(dim): - """Check if a dimension is horizontal - - Parameters - ---------- - dim : str - the name of the dimension - - Returns - ------- - bool - Whether or not the current dimension is horizontal or not - """ - for term in dim.split("_"): - if term == "lat" or term == "lon": - return True - return False - - def nominal_stacked_resolution(dim): - """Use a KDTree to return approximate stacked resolution with some loss of accuracy. - - Parameters - ---------- - dim : str - The name of the dimension to return the resolution of. Should be stacked. - - Returns - ------- - The average min distance of every point - - """ - tree = spatial.KDTree(self[dim].coordinates + [0, 180.0], boxsize=[0.0, 360.0000000000001]) - return np.average(calculate_distance(tree.data - [0, 180.0], tree.data[tree.query(tree.data, k=2)[1][:,1]] - [0, 180.0])) - - - - - def summary_stacked_resolution(dim): - """Return the approximate mean resolution and std.deviation using a KDTree - - Parameters - ---------- - dim : str - The name of the dimension to return the resolution of. Should be stacked. - - Returns - ------- - tuple - Average min distance of every point and standard deviation of those min distances - """ - tree = spatial.KDTree(self[dim].coordinates + [0, 180.0], boxsize=[0.0, 360.0000000000001]) - distances = (calculate_distance(tree.data - [0, 180.0], tree.data[tree.query(tree.data, k=2)[1][:,1]] - [0, 180.0])) - return (np.average(distances), np.std(distances)) - - def full_stacked_resolution(dim): - """Returns the exact distance between every point using brute force - - Parameters - ---------- - dim : str - The dimension to return the resolution of. Should be stacked. - - Returns - ------- - distance matrix of size (NxN), where N is the number of points in the dimension - """ - distance_matrix = np.zeros((len(self[dim].coordinates), len(self[dim].coordinates))) - for i in range(len(self[dim].coordinates)): - distance_matrix[i,:] = calculate_distance(self[dim].coordinates[i], self[dim].coordinates[:]).magnitude - return distance_matrix * podpac.units(units) - - def nominal_unstacked_resolution(dim): - """Return resolution for unstacked coordiantes using the bounds - - Parameters - ---------- - dim : str - The dimenion to return the resolution of. Should be unstacked. - - Returns - -------- - The average distance between each grid square for this dimension - """ - if dim == 'lat': - return (calculate_distance((self[dim].bounds[0],0), (self[dim].bounds[1],0)).magnitude / self[dim].size) * podpac.units(units) - elif dim == 'lon': - # Get median lat: - # need this because of difference in circumference - median_lat = ((self['lat'].bounds[1] - self['lat'].bounds[0]) / 2) + self['lat'].bounds[0] - return (calculate_distance((median_lat, self[dim].bounds[0]), (median_lat, self[dim].bounds[1])).magnitude / self[dim].size) * podpac.units(units) - else: - return ValueError("Unknown dim: {}".format(dim)) - - def summary_unstacked_resolution(dim): - """Return summary resolution for the dimension. - - Parameters - ---------- - dim : str - The dimension to return the resolution of. Should be unstacked. - - Returns - ------- - tuple - the average distance between grid squares - the standard deviation of those distances - """ - if dim == "lat" or dim == "lon": - full_res = full_unstacked_resolution(dim).magnitude - return (np.average(full_res)* podpac.units(units), np.std(full_res) * podpac.units(units)) - else: - return ValueError("Unknown dim: {}".format(dim)) - - def full_unstacked_resolution(dim): - """Calculate full resolution of unstacked dimension - - Parameters - ---------- - dim : str - The dimension to return the resolution of. Should be unstacked. - - Returns - ------- - A matrix of distances - """ - if dim == "lat": - M = self['lat'].coordinates.size - N = self['lon'].coordinates.size - diff= np.zeros((N,M-1)) - for i in range(N): - lat_values= self['lat'].coordinates - lon_value = self['lon'].coordinates[i] - top_bounds = np.stack([lat_values[1:], np.full((lat_values[1:]).shape[0], lon_value)], axis=1) # use exact lat values - bottom_bounds = np.stack([lat_values[:-1], np.full((lat_values[:-1]).shape[0], lon_value)], axis=1) # use exact lat values - diff[i] = calculate_distance(top_bounds, bottom_bounds).magnitude - return diff * podpac.units(units) - elif dim == "lon": - M = self['lat'].coordinates.size - N = self['lon'].coordinates.size - diff= np.zeros((M,N-1)) - for i in range(M): - lat_value= self['lat'].coordinates[i] - lon_values = self['lon'].coordinates - top_bounds = np.stack([np.full((lon_values[1:]).shape[0], lat_value), lon_values[1:]], axis=1) # use exact lat values - bottom_bounds = np.stack([np.full((lon_values[:-1]).shape[0], lat_value), lon_values[:-1]], axis=1) # use exact lat values - diff[i] = calculate_distance(top_bounds, bottom_bounds).magnitude - return diff * podpac.units(units) - else: - return ValueError("Unknown dim: {}".format(dim)) - - - """---------------------------------------------------------------------""" - - # dictionary: - resolution_dict = OrderedDict() - - # ellipsoid Tuple + # ellipsoid tuple to pass to geodesic ellipsoid_tuple = ( self.CRS.ellipsoid.semi_major_metre / 1000, self.CRS.ellipsoid.semi_minor_metre / 1000, 1 / self.CRS.ellipsoid.inverse_flattening, ) - # validate dims: - # main execution loop - for dim in self.dims: - # Is the dim lat/lon? - if not check_horizontal(dim): - continue - # Put this check inside the loop to avoid checking if stacked coords - if dim == "lat": - if "lon" not in self.dims: - return ValueError("Need both lat and lon for resolution.") - if dim == "lon": - if "lat" not in self.dims: - return ValueError("Need both lat and lon for resolution.") - # stacked coordinate resolutions - if self.is_stacked(dim): - # stacked_resolution(dim) - if restype == "nominal": - resolution_dict[dim] = nominal_stacked_resolution(dim) - elif restype == "summary": - resolution_dict[dim] = summary_stacked_resolution(dim) - elif restype == "full": - resolution_dict[dim] = full_stacked_resolution(dim) - else: - return ValueError("Invalid value for type: {}".format(type)) - else: # unstacked resolution - if restype == "nominal": - resolution_dict[dim] = nominal_unstacked_resolution(dim) - elif restype == "summary": - resolution_dict[dim] = summary_unstacked_resolution(dim) - elif restype == "full": - resolution_dict[dim] = full_unstacked_resolution(dim) - else: - return ValueError("Invalid value for type: {}".format(type)) + resolutions = OrderedDict() # To return + for name, dim in self.items(): + if dim.is_stacked: + for stack_name in dim.dims: # make sure ['lat_lon'] is stacked before returning horizontal_resolution + if stack_name != "lat" and stack_name != "lon": # check for valid stacking + raise ValueError( + "Must be lat and lon stacked, cannot return resolution for stacked {}".format(stack_name) + ) + resolutions[name] = dim.horizontal_resolution( + ellipsoid_tuple, self.CRS.coordinate_system.name, restype, units + ) + else: + if ( + name == "lat" or name == "lon" + ): # need to do this inside of loop in case of stacked [[alt,time]] but unstacked [lat, lon] + if "lat" not in self.dims: # require latitude + raise ValueError("Latitude required for horizontal resolution.") + resolutions[name] = dim.horizontal_resolution( + self["lat"], ellipsoid_tuple, self.CRS.coordinate_system.name, restype, units + ) - return resolution_dict + return resolutions # ------------------------------------------------------------------------------------------------------------------ # Operators/Magic Methods diff --git a/podpac/core/coordinates/coordinates1d.py b/podpac/core/coordinates/coordinates1d.py index e855255e..6a18d916 100644 --- a/podpac/core/coordinates/coordinates1d.py +++ b/podpac/core/coordinates/coordinates1d.py @@ -10,10 +10,12 @@ import numpy as np import traitlets as tl +import podpac from podpac.core.utils import ArrayTrait, TupleTrait from podpac.core.coordinates.utils import make_coord_value, make_coord_delta, make_coord_delta_array from podpac.core.coordinates.utils import add_coord, divide_delta, lower_precision_time_bounds from podpac.core.coordinates.utils import Dimension +from podpac.core.coordinates.utils import calculate_distance from podpac.core.coordinates.base_coordinates import BaseCoordinates @@ -187,6 +189,10 @@ def _get_definition(self, full=True): def _full_properties(self): return {"name": self.name} + @property + def is_stacked(self): + return False + # ------------------------------------------------------------------------------------------------------------------ # Methods # ------------------------------------------------------------------------------------------------------------------ @@ -423,3 +429,118 @@ def issubset(self, other): other_coordinates = other_coordinates.astype(my_coordinates.dtype) return set(my_coordinates).issubset(other_coordinates) + + def horizontal_resolution(self, latitude, ellipsoid_tuple, coordinate_name, restype="nominal", units="meter"): + """Return the horizontal resolution of a Uniform 1D Coordinate + + Parameters + ---------- + latitude: Coordinates1D + The latitude coordiantes of the current coordinate system, required for both lat and lon resolution. + ellipsoid_tuple: tuple + a tuple containing ellipsoid information from the the original coordinates to pass to geopy + coordinate_name: str + "cartesian" or "ellipsoidal", to tell calculate_distance what kind of calculation to do + restype: str + The kind of horizontal resolution that should be returned. Supported values are: + - "nominal" <-- returns average resolution based upon bounds and number of grid squares + - "summary" <-- Gives the exact mean and standard deviation of grid square resolutions + - "full" <-- Gives exact grid differences. + + Returns + ------- + float * (podpac.unit) + if restype == "nominal", return average distance in desired units + tuple * (podpac.unit) + if restype == "summary", return average distance and standard deviation in desired units + np.ndarray * (podpac.unit) + if restype == "full", give full resolution. 1D array for latitude, MxN matrix for longitude. + """ + + def nominal_unstacked_resolution(): + """Return resolution for unstacked coordiantes using the bounds + + Returns + -------- + The average distance between each grid square for this dimension + """ + if self.name == "lat": + return ( + calculate_distance( + (self.bounds[0], 0), (self.bounds[1], 0), ellipsoid_tuple, coordinate_name, units + ).magnitude + / self.size + ) * podpac.units(units) + elif self.name == "lon": + median_lat = ((latitude.bounds[1] - latitude.bounds[0]) / 2) + latitude.bounds[0] + return ( + calculate_distance( + (median_lat, self.bounds[0]), + (median_lat, self.bounds[1]), + ellipsoid_tuple, + coordinate_name, + units, + ).magnitude + / self.size + ) * podpac.units(units) + else: + return ValueError("Unknown dim: {}".format(self.name)) + + def summary_unstacked_resolution(): + """Return summary resolution for the dimension. + + Returns + ------- + tuple + the average distance between grid squares + the standard deviation of those distances + """ + if self.name == "lat" or self.name == "lon": + full_res = full_unstacked_resolution().magnitude + return (np.average(full_res) * podpac.units(units), np.std(full_res) * podpac.units(units)) + else: + return ValueError("Unknown dim: {}".format(self.name)) + + def full_unstacked_resolution(): + """Calculate full resolution of unstacked dimension + + Returns + ------- + A matrix of distances + """ + if self.name == "lat": + top_bounds = np.stack( + [latitude.coordinates[1:], np.full((latitude.coordinates[1:]).shape[0], 0)], axis=1 + ) # use exact lat values + bottom_bounds = np.stack( + [latitude.coordinates[:-1], np.full((latitude.coordinates[:-1]).shape[0], 0)], axis=1 + ) # use exact lat values + return calculate_distance(top_bounds, bottom_bounds, ellipsoid_tuple, coordinate_name, units) + elif self.name == "lon": + M = latitude.coordinates.size + N = self.size + diff = np.zeros((M, N - 1)) + for i in range(M): + lat_value = latitude.coordinates[i] + lon_values = self.coordinates + top_bounds = np.stack( + [np.full((lon_values[1:]).shape[0], lat_value), lon_values[1:]], axis=1 + ) # use exact lat values + bottom_bounds = np.stack( + [np.full((lon_values[:-1]).shape[0], lat_value), lon_values[:-1]], axis=1 + ) # use exact lat values + diff[i] = calculate_distance( + top_bounds, bottom_bounds, ellipsoid_tuple, coordinate_name, units + ).magnitude + return diff * podpac.units(units) + else: + raise ValueError("Unknown dim: {}".format(self.name)) + + if restype == "nominal": + return nominal_unstacked_resolution() + elif restype == "summary": + return summary_unstacked_resolution() + elif restype == "full": + return full_unstacked_resolution() + else: + raise ValueError("Invalid value for type: {}".format(restype)) diff --git a/podpac/core/coordinates/stacked_coordinates.py b/podpac/core/coordinates/stacked_coordinates.py index 9ffd45b4..d891a9e0 100644 --- a/podpac/core/coordinates/stacked_coordinates.py +++ b/podpac/core/coordinates/stacked_coordinates.py @@ -9,12 +9,15 @@ import traitlets as tl from six import string_types import lazy_import +from scipy import spatial +import podpac from podpac.core.coordinates.base_coordinates import BaseCoordinates from podpac.core.coordinates.coordinates1d import Coordinates1d from podpac.core.coordinates.array_coordinates1d import ArrayCoordinates1d from podpac.core.coordinates.uniform_coordinates1d import UniformCoordinates1d from podpac.core.coordinates.utils import make_coord_value +from podpac.core.coordinates.utils import calculate_distance class StackedCoordinates(BaseCoordinates): @@ -358,6 +361,10 @@ def full_definition(self): return [c.full_definition for c in self._coords] + @property + def is_stacked(self): + return True + # ----------------------------------------------------------------------------------------------------------------- # Methods # ----------------------------------------------------------------------------------------------------------------- @@ -700,3 +707,93 @@ def is_affine(self): return False return True + + def horizontal_resolution(self, ellipsoid_tuple, coordinate_name, restype="nominal", units="meter"): + """Return the horizontal resolution of a Uniform 1D Coordinate + + Parameters + ---------- + ellipsoid_tuple: tuple + a tuple containing ellipsoid information from the the original coordinates to pass to geopy + coordinate_name: str + "cartesian" or "ellipsoidal", to tell calculate_distance what kind of calculation to do + restype: str + The kind of horizontal resolution that should be returned. Supported values are: + - "nominal" <-- Gives average nearest distance of all points, with some error + - "summary" <-- Gives the mean and standard deviation of nearest distance betweem points, with some error + - "full" <-- Gives exact distance matrix + units: str + desired unit to return + + Returns + ------- + float * (podpac.unit) + If restype == "nominal", return the average nearest distance with some error + tuple * (podpac.unit) + If restype == "summary", return average and std.dev of nearest distances, with some error + np.ndarray * (podpac.unit) + if restype == "full", return exact distance matrix + ValueError + if unknown restype + + """ + + def nominal_stacked_resolution(): + """Use a KDTree to return approximate stacked resolution with some loss of accuracy. + + Returns + ------- + The average min distance of every point + + """ + tree = spatial.KDTree(self.coordinates + [0, 180.0], boxsize=[0.0, 360.0000000000001]) + return np.average( + calculate_distance( + tree.data - [0, 180.0], + tree.data[tree.query(tree.data, k=2)[1][:, 1]] - [0, 180.0], + ellipsoid_tuple, + coordinate_name, + units, + ) + ) + + def summary_stacked_resolution(): + """Return the approximate mean resolution and std.deviation using a KDTree + + Returns + ------- + tuple + Average min distance of every point and standard deviation of those min distances + """ + tree = spatial.KDTree(self.coordinates + [0, 180.0], boxsize=[0.0, 360.0000000000001]) + distances = calculate_distance( + tree.data - [0, 180.0], + tree.data[tree.query(tree.data, k=2)[1][:, 1]] - [0, 180.0], + ellipsoid_tuple, + coordinate_name, + units, + ) + return (np.average(distances), np.std(distances)) + + def full_stacked_resolution(): + """Returns the exact distance between every point using brute force + + Returns + ------- + distance matrix of size (NxN), where N is the number of points in the dimension + """ + distance_matrix = np.zeros((len(self.coordinates), len(self.coordinates))) + for i in range(len(self.coordinates)): + distance_matrix[i, :] = calculate_distance( + self.coordinates[i], self.coordinates[:], ellipsoid_tuple, coordinate_name, units + ).magnitude + return distance_matrix * podpac.units(units) + + if restype == "nominal": + return nominal_stacked_resolution() + elif restype == "summary": + return summary_stacked_resolution() + elif restype == "full": + return full_stacked_resolution() + else: + raise ValueError("Invalid value for type: {}".format(restype)) diff --git a/podpac/core/coordinates/test/test_coordinates.py b/podpac/core/coordinates/test/test_coordinates.py index 17f42ea2..51b14055 100644 --- a/podpac/core/coordinates/test/test_coordinates.py +++ b/podpac/core/coordinates/test/test_coordinates.py @@ -2207,225 +2207,38 @@ def test_affine_to_uniform(self): class TestResolutions(object): - def test_resolution(self): - - '''Unstacked Coordinates''' - c = podpac.Coordinates([podpac.clinspace(0, 90, 11, 'lat'), podpac.clinspace(0, 90, 20, 'lon')]) - - # Nominal - assert (c.horizontal_resolution(restype="nominal") == OrderedDict([('lat', 909269.611755702* podpac.units("metre")), - ('lon', 334511.6466271357* podpac.units("metre"))])) - - # Summary - assert (c.horizontal_resolution(restype="summary") == OrderedDict([('lat', - (1000196.5729312722* podpac.units("metre"), - 3548.2348372254723* podpac.units("metre"))), - ('lon', - (328825.13300389395* podpac.units("metre"), - 176330.22544905098* podpac.units("metre")))])) - - # Full - answer = OrderedDict([('lat', - ([995250.27739564, 995731.65748247, 996648.35979764, - 997912.37572808, 999401.70757391, 1000971.65042421, - 1002468.54050245, 1003744.78599167, 1004673.71796683, - 1005162.65644982], - [ 995250.27739564, 995731.65748247, 996648.35979764, - 997912.37572808, 999401.70757391, 1000971.65042421, - 1002468.54050245, 1003744.78599167, 1004673.71796683, - 1005162.65644982], - [ 995250.27739564, 995731.65748247, 996648.35979764, - 997912.37572808, 999401.70757391, 1000971.65042421, - 1002468.54050245, 1003744.78599167, 1004673.71796683, - 1005162.65644982], - [ 995250.27739564, 995731.65748247, 996648.35979764, - 997912.37572808, 999401.70757391, 1000971.65042421, - 1002468.54050245, 1003744.78599167, 1004673.71796683, - 1005162.65644982], - [ 995250.27739564, 995731.65748247, 996648.35979764, - 997912.37572808, 999401.70757391, 1000971.65042421, - 1002468.54050245, 1003744.78599167, 1004673.71796683, - 1005162.65644982], - [ 995250.27739564, 995731.65748247, 996648.35979764, - 997912.37572808, 999401.70757391, 1000971.65042421, - 1002468.54050245, 1003744.78599167, 1004673.71796683, - 1005162.65644982], - [ 995250.27739564, 995731.65748247, 996648.35979764, - 997912.37572808, 999401.70757391, 1000971.65042421, - 1002468.54050245, 1003744.78599167, 1004673.71796683, - 1005162.65644982], - [ 995250.27739564, 995731.65748247, 996648.35979764, - 997912.37572808, 999401.70757391, 1000971.65042421, - 1002468.54050245, 1003744.78599167, 1004673.71796683, - 1005162.65644982], - [ 995250.27739564, 995731.65748247, 996648.35979764, - 997912.37572808, 999401.70757391, 1000971.65042421, - 1002468.54050245, 1003744.78599167, 1004673.71796683, - 1005162.65644982], - [ 995250.27739564, 995731.65748247, 996648.35979764, - 997912.37572808, 999401.70757391, 1000971.65042421, - 1002468.54050245, 1003744.78599167, 1004673.71796683, - 1005162.65644982], - [ 995250.27739564, 995731.65748247, 996648.35979764, - 997912.37572808, 999401.70757391, 1000971.65042421, - 1002468.54050245, 1003744.78599167, 1004673.71796683, - 1005162.65644982], - [ 995250.27739564, 995731.65748247, 996648.35979764, - 997912.37572808, 999401.70757391, 1000971.65042421, - 1002468.54050245, 1003744.78599167, 1004673.71796683, - 1005162.65644982], - [ 995250.27739564, 995731.65748247, 996648.35979764, - 997912.37572808, 999401.70757391, 1000971.65042421, - 1002468.54050245, 1003744.78599167, 1004673.71796683, - 1005162.65644982], - [ 995250.27739564, 995731.65748247, 996648.35979764, - 997912.37572808, 999401.70757391, 1000971.65042421, - 1002468.54050245, 1003744.78599167, 1004673.71796683, - 1005162.65644982], - [ 995250.27739564, 995731.65748247, 996648.35979764, - 997912.37572808, 999401.70757391, 1000971.65042421, - 1002468.54050245, 1003744.78599167, 1004673.71796683, - 1005162.65644982], - [ 995250.27739564, 995731.65748247, 996648.35979764, - 997912.37572808, 999401.70757391, 1000971.65042421, - 1002468.54050245, 1003744.78599167, 1004673.71796683, - 1005162.65644982], - [ 995250.27739564, 995731.65748247, 996648.35979764, - 997912.37572808, 999401.70757391, 1000971.65042421, - 1002468.54050245, 1003744.78599167, 1004673.71796683, - 1005162.65644982], - [ 995250.27739564, 995731.65748247, 996648.35979764, - 997912.37572808, 999401.70757391, 1000971.65042421, - 1002468.54050245, 1003744.78599167, 1004673.71796683, - 1005162.65644982], - [ 995250.27739564, 995731.65748247, 996648.35979764, - 997912.37572808, 999401.70757391, 1000971.65042421, - 1002468.54050245, 1003744.78599167, 1004673.71796683, - 1005162.65644982], - [ 995250.27739564, 995731.65748247, 996648.35979764, - 997912.37572808, 999401.70757391, 1000971.65042421, - 1002468.54050245, 1003744.78599167, 1004673.71796683, - 1005162.65644982]) * podpac.units("m")), - ('lon', - ([527302.85112603, 527302.85112603, 527302.85112603, - 527302.85112603, 527302.85112603, 527302.85112603, - 527302.85112603, 527302.85112603, 527302.85112603, - 527302.85112603, 527302.85112603, 527302.85112603, - 527302.85112603, 527302.85112603, 527302.85112603, - 527302.85112603, 527302.85112603, 527302.85112603, - 527302.85112603], - [520849.91130839, 520849.91130839, 520849.91130839, - 520849.91130839, 520849.91130839, 520849.91130839, - 520849.91130839, 520849.91130839, 520849.91130839, - 520849.91130839, 520849.91130839, 520849.91130839, - 520849.91130839, 520849.91130839, 520849.91130839, - 520849.91130838, 520849.91130838, 520849.91130839, - 520849.91130839], - [501641.53060499, 501641.53060499, 501641.53060499, - 501641.53060499, 501641.53060499, 501641.53060499, - 501641.53060499, 501641.53060499, 501641.53060499, - 501641.53060499, 501641.53060499, 501641.53060499, - 501641.53060499, 501641.53060499, 501641.53060499, - 501641.53060499, 501641.53060499, 501641.53060499, - 501641.53060499], - [470127.1319624 , 470127.1319624 , 470127.1319624 , - 470127.1319624 , 470127.1319624 , 470127.1319624 , - 470127.1319624 , 470127.1319624 , 470127.1319624 , - 470127.1319624 , 470127.1319624 , 470127.1319624 , - 470127.1319624 , 470127.1319624 , 470127.1319624 , - 470127.1319624 , 470127.1319624 , 470127.1319624 , - 470127.1319624 ], - [427049.11294657, 427049.11294657, 427049.11294657, - 427049.11294657, 427049.11294657, 427049.11294657, - 427049.11294657, 427049.11294657, 427049.11294657, - 427049.11294657, 427049.11294657, 427049.11294657, - 427049.11294657, 427049.11294657, 427049.11294657, - 427049.11294657, 427049.11294657, 427049.11294657, - 427049.11294657], - [373431.8103952 , 373431.8103952 , 373431.8103952 , - 373431.8103952 , 373431.8103952 , 373431.8103952 , - 373431.8103952 , 373431.8103952 , 373431.8103952 , - 373431.8103952 , 373431.8103952 , 373431.8103952 , - 373431.8103952 , 373431.8103952 , 373431.8103952 , - 373431.8103952 , 373431.8103952 , 373431.8103952 , - 373431.8103952 ], - [310564.17702203, 310564.17702203, 310564.17702203, - 310564.17702203, 310564.17702203, 310564.17702203, - 310564.17702203, 310564.17702203, 310564.17702203, - 310564.17702203, 310564.17702203, 310564.17702203, - 310564.17702203, 310564.17702203, 310564.17702203, - 310564.17702203, 310564.17702203, 310564.17702203, - 310564.17702203], - [239974.89400498, 239974.89400498, 239974.89400498, - 239974.89400498, 239974.89400498, 239974.89400498, - 239974.89400498, 239974.89400498, 239974.89400498, - 239974.89400498, 239974.89400498, 239974.89400498, - 239974.89400498, 239974.89400498, 239974.89400498, - 239974.89400498, 239974.89400498, 239974.89400498, - 239974.89400498], - [163399.02083826, 163399.02083826, 163399.02083826, - 163399.02083826, 163399.02083826, 163399.02083826, - 163399.02083826, 163399.02083826, 163399.02083826, - 163399.02083826, 163399.02083826, 163399.02083826, - 163399.02083826, 163399.02083826, 163399.02083826, - 163399.02083826, 163399.02083826, 163399.02083826, - 163399.02083826], - [ 82736.02283397, 82736.02283397, 82736.02283397, - 82736.02283397, 82736.02283397, 82736.02283397, - 82736.02283397, 82736.02283397, 82736.02283397, - 82736.02283397, 82736.02283397, 82736.02283397, - 82736.02283397, 82736.02283397, 82736.02283397, - 82736.02283397, 82736.02283397, 82736.02283397, - 82736.02283397], - [ 0. , 0. , 0. , - 0. , 0. , 0. , - 0. , 0. , 0. , - 0. , 0. , 0. , - 0. , 0. , 0. , - 0. , 0. , 0. , - 0. ]) * podpac.units("m"))]) - assert answer['lat'].all() == c.horizontal_resolution(restype="full")['lat'].all() and answer['lon'].all() == c.horizontal_resolution(restype="full")['lon'].all() - - - - '''Stacked Coordinates''' - lat = podpac.clinspace(-90, 90, 5) - lon = podpac.clinspace(-180, 180, 5) - - c = Coordinates([[lat, lon]], dims=['lat_lon']) - # Nominal - assert (c.horizontal_resolution(restype="nominal") == OrderedDict([('lat_lon', 7014367.35521632 * podpac.units("m"))])) - - # Summary - assert c.horizontal_resolution(restype="summary") == OrderedDict([('lat_lon', - (7014367.35521632 * podpac.units("m"), 2446239.274648156 * podpac.units("m")))]) + def test_horizontal_resolution(self): + """Test edge cases of resolutions, and that Resolutions are returned correctly in an OrderedDict. + StackedCoordinates and Coordiantes1d handle the resolution calculations, so correctness of the resolutions are tested there. + """ - # Full - answer = OrderedDict([('lat_lon', - ([[ 0. , 5017021.35133498, 10001965.72931272, - 14986910.10729047, 20003931.45862544], - [ 5017021.35133498, 0. , 10010386.36103833, - 20003931.45862544, 14986910.10729047], - [10001965.72931272, 10010386.36103833, 0. , - 10010386.36103833, 10001965.72931272], - [14986910.10729047, 20003931.45862544, 10010386.36103833, - 0. , 5017021.35133498], - [20003931.45862544, 14986910.10729047, 10001965.72931272, - 5017021.35133498, 0. ]]) * podpac.units("m"))]) + # Dimensions + lat = podpac.clinspace(-80, 80, 5) + lon = podpac.clinspace(-180, 180, 5) + time = ["2018-01-01", "2018-01-02", "2018-01-03", "2018-01-04", "2018-01-05"] - assert c.horizontal_resolution(restype="full")['lat_lon'].all() == answer['lat_lon'].all() + # Invalid stacked dims check: + c = Coordinates([[lat, time]], dims=["lat_time"]) + with pytest.raises(ValueError): + c.horizontal_resolution() - # Cartesian - c = podpac.Coordinates([[podpac.clinspace(0, 90, 20, 'lat'), podpac.clinspace(0, 90, 20, 'lon')]], dims = ['lat_lon'], crs='+proj=utm +zone=9 +north +ellps=WGS84 +datum=WGS84 +units=m +no_defs') + # Stacked and Unstacked valid Check: + c = Coordinates([[lat, time], lon], dims=["lat_time", "lon"]) + with pytest.raises(ValueError): + c.horizontal_resolution() - assert c.horizontal_resolution() == OrderedDict([('lat_lon', 6.698906348083078 * podpac.units("m"))]) + # Require latitude + c = Coordinates([lon], dims=["lon"]) + with pytest.raises(ValueError): + c.horizontal_resolution() - '''Stacked and Unstacked Coordinates''' - time = ['2018-01-01', '2018-01-02'] - c = Coordinates([[lat, lon], time], dims=['lat_lon', 'time']) - # Nominal - assert c.horizontal_resolution() == OrderedDict([('lat_lon', 7014367.35521632 * podpac.units("m"))]) - + # Valid dims check: + c = Coordinates([[lat, lon]], dims=["lat_lon"]) + c.horizontal_resolution() + # Corect name for restype: + with pytest.raises(ValueError): + c.horizontal_resolution(restype="whateverIwant") - + # Unstacked + c = Coordinates([[lat, lon]], dims=["lat_lon"]) diff --git a/podpac/core/coordinates/test/test_coordinates1d.py b/podpac/core/coordinates/test/test_coordinates1d.py index 35d1422f..020e6509 100644 --- a/podpac/core/coordinates/test/test_coordinates1d.py +++ b/podpac/core/coordinates/test/test_coordinates1d.py @@ -1,6 +1,9 @@ import pytest +import numpy as np +import podpac from podpac.core.coordinates.coordinates1d import Coordinates1d +from podpac import clinspace class TestCoordinates1d(object): @@ -82,3 +85,59 @@ def test_common_api(self): c.issubset(c) except NotImplementedError: pass + + def test_horizontal_resolution(self): + """Test horizontal resolution implentation for Coordinates1d. Edge cases are handled in Coordinates.py""" + # Latitude + lat = clinspace(-80, 80, 5) + lat.name = "lat" # normally assigned when creating Coords object + assert type(lat) == podpac.core.coordinates.uniform_coordinates1d.UniformCoordinates1d + + # Longitude + lon = podpac.clinspace(-180, 180, 5) + lon.name = "lon" + assert type(lon) == podpac.core.coordinates.uniform_coordinates1d.UniformCoordinates1d + + # Sample Ellipsoid Tuple + ell_tuple = (6378.137, 6356.752314245179, 0.0033528106647474805) + + # Sample Coordinate name: + coord_name = "ellipsoidal" + + # Resolution: nominal + assert lat.horizontal_resolution(lat, ell_tuple, coord_name) == 3554055.948774749 * podpac.units("meter") + assert lon.horizontal_resolution(lat, ell_tuple, coord_name) == 0.0 * podpac.units("meter") + + # Resolution: summary + assert lat.horizontal_resolution(lat, ell_tuple, coord_name, restype="summary") == ( + 4442569.935968436 * podpac.units("meter"), + 13040.905617921147 * podpac.units("meter"), + ) + assert lon.horizontal_resolution(lat, ell_tuple, coord_name, restype="summary") == ( + 5558704.3695234 * podpac.units("meter"), + 3399219.0171971265 * podpac.units("meter"), + ) + + # Resolution: full + lat_answer = [4455610.84158636, 4429529.03035052, 4429529.03035052, 4455610.84158636] + + lon_answer = [ + [1575399.99090356, 1575399.99090356, 1575399.99090356, 1575399.99090356], + [7311983.84720763, 7311983.84720763, 7311983.84720763, 7311983.84720763], + [10018754.17139462, 10018754.17139462, 10018754.17139462, 10018754.17139462], + [7311983.84720763, 7311983.84720763, 7311983.84720763, 7311983.84720763], + [1575399.99090356, 1575399.99090356, 1575399.99090356, 1575399.99090356], + ] + + np.testing.assert_array_almost_equal( + lat.horizontal_resolution(lat, ell_tuple, coord_name, restype="full").magnitude, lat_answer + ) + np.testing.assert_array_almost_equal( + lon.horizontal_resolution(lat, ell_tuple, coord_name, restype="full").magnitude, lon_answer + ) + + # Different Units + np.testing.assert_almost_equal( + lat.horizontal_resolution(lat, ell_tuple, coord_name).to(podpac.units("feet")).magnitude, + lat.horizontal_resolution(lat, ell_tuple, coord_name, units="feet").magnitude, + ) diff --git a/podpac/core/coordinates/test/test_stacked_coordinates.py b/podpac/core/coordinates/test/test_stacked_coordinates.py index d9460a05..f9f303c2 100644 --- a/podpac/core/coordinates/test/test_stacked_coordinates.py +++ b/podpac/core/coordinates/test/test_stacked_coordinates.py @@ -981,3 +981,43 @@ def test_reshape(self): assert c.reshape((4, 3)) == StackedCoordinates([lat.reshape((4, 3)), lon.reshape((4, 3))]) assert c.flatten().reshape((3, 4)) == c + + def test_horizontal_resolution(self): + """Test Horizontal Resolution of Stacked Coordinates. Edge cases are handled in Coordinates.py""" + lat = podpac.clinspace(-80, 80, 5) + lat.name = "lat" # normally assigned when creating Coords object + lon = podpac.clinspace(-180, 180, 5) + lon.name = "lon" + c = StackedCoordinates([lat, lon]) + + # Sample Ellipsoid Tuple + ell_tuple = (6378.137, 6356.752314245179, 0.0033528106647474805) + + # Sample Coordinate name: + coord_name = "ellipsoidal" + + # Nominal resolution: + np.testing.assert_almost_equal( + c.horizontal_resolution(ell_tuple, coord_name, restype="nominal").magnitude, 7397047.845631437 + ) + + # Summary resolution + np.testing.assert_almost_equal( + c.horizontal_resolution(ell_tuple, coord_name, restype="summary")[0].magnitude, 7397047.845631437 + ) + np.testing.assert_almost_equal( + c.horizontal_resolution(ell_tuple, coord_name, restype="summary")[1].magnitude, 2134971.4571846593 + ) + + # Full resolution + distance_matrix = [ + [0.0, 5653850.95046188, 11118791.58668857, 14351078.11393555, 17770279.74387375], + [5653850.95046188, 0.0, 10011843.18838578, 20003931.45862544, 14351078.11393555], + [11118791.58668857, 10011843.18838578, 0.0, 10011843.18838578, 11118791.58668857], + [14351078.11393555, 20003931.45862544, 10011843.18838578, 0.0, 5653850.95046188], + [17770279.74387375, 14351078.11393555, 11118791.58668857, 5653850.95046188, 0.0], + ] + + np.testing.assert_array_almost_equal( + c.horizontal_resolution(ell_tuple, coord_name, restype="full"), distance_matrix + ) diff --git a/podpac/core/coordinates/uniform_coordinates1d.py b/podpac/core/coordinates/uniform_coordinates1d.py index f972c2d1..bbb13dfe 100644 --- a/podpac/core/coordinates/uniform_coordinates1d.py +++ b/podpac/core/coordinates/uniform_coordinates1d.py @@ -541,3 +541,5 @@ def _select(self, bounds, return_index, outer): return self[I], I else: return self[I] + + \ No newline at end of file diff --git a/podpac/core/coordinates/utils.py b/podpac/core/coordinates/utils.py index ef4f0b8d..a65ec35f 100644 --- a/podpac/core/coordinates/utils.py +++ b/podpac/core/coordinates/utils.py @@ -19,6 +19,9 @@ import traitlets as tl from six import string_types import pyproj +from geopy.distance import geodesic + +import podpac def get_timedelta(s): @@ -602,3 +605,35 @@ def has_alt_units(crs): with warnings.catch_warnings(): warnings.simplefilter("ignore") return crs.is_vertical or "vunits" in crs.to_dict() or any(axis.direction == "up" for axis in crs.axis_info) + + +def calculate_distance(point1, point2, ellipsoid_tuple, coordinate_name, units="meter"): + """Return distance of 2 points in desired unit measurement + + Parameters + ---------- + point1 : tuple + point2 : tuple + + Returns + ------- + float + The distance between point1 and point2, according to the current coordinate system's distance metric, using the desired units + """ + if coordinate_name == "cartesian": + return np.linalg.norm(point1 - point2, axis=-1, units="meter") * podpac.units(units) + else: + if not isinstance(point1, tuple) and point1.size > 2: + distances = np.empty(len(point1)) + for i in range(len(point1)): + distances[i] = geodesic(point1[i], point2[i], ellipsoid=ellipsoid_tuple).m + return distances * podpac.units("metre").to(podpac.units(units)) + if not isinstance(point2, tuple) and point2.size > 2: + distances = np.empty(len(point2)) + for i in range(len(point2)): + distances[i] = geodesic(point1, point2[i], ellipsoid=ellipsoid_tuple).m + return distances * podpac.units("metre").to(podpac.units(units)) + else: + return (geodesic(point1, point2, ellipsoid=ellipsoid_tuple).m) * podpac.units("metre").to( + podpac.units(units) + ) From 4dcb57e07a5425b51c9ab3eeec5416c14d7b08ee Mon Sep 17 00:00:00 2001 From: Matt Ueckermann Date: Mon, 6 Mar 2023 15:02:59 -0500 Subject: [PATCH 48/52] FIX: A few minor fixed to horizontal_resolution * Made the interface to the horizontal_resolution method the same * Fixed fence-post error (size-1) * Some documentation updates, spelling, rewording, etc. * Fixed ordering bug on stacked coordinates * Added a few more tests --- podpac/core/coordinates/base_coordinates.py | 5 +- podpac/core/coordinates/coordinates.py | 53 +++++++++++-------- podpac/core/coordinates/coordinates1d.py | 4 +- .../core/coordinates/stacked_coordinates.py | 9 ++-- .../core/coordinates/test/test_coordinates.py | 26 ++++++--- .../coordinates/test/test_coordinates1d.py | 2 +- .../test/test_stacked_coordinates.py | 34 ++++++++++-- 7 files changed, 89 insertions(+), 44 deletions(-) diff --git a/podpac/core/coordinates/base_coordinates.py b/podpac/core/coordinates/base_coordinates.py index 98844a8b..2b14e4b9 100644 --- a/podpac/core/coordinates/base_coordinates.py +++ b/podpac/core/coordinates/base_coordinates.py @@ -68,7 +68,7 @@ def definition(self): def full_definition(self): """Coordinates definition, containing all properties. For internal use.""" raise NotImplementedError - + @property def is_stacked(self): """stacked or unstacked property""" @@ -111,11 +111,10 @@ def issubset(self, other): """Report if these coordinates are a subset of other coordinates.""" raise NotImplementedError - def horizontal_resolution(self, latitude, restype="nominal", units="meter"): + def horizontal_resolution(self, latitude, ellipsoid_tuple, coordinate_name, restype="nominal", units="meter"): """Get horizontal resolution of coordiantes.""" raise NotImplementedError - def __getitem__(self, index): raise NotImplementedError diff --git a/podpac/core/coordinates/coordinates.py b/podpac/core/coordinates/coordinates.py index 31e4b427..5f76db8e 100644 --- a/podpac/core/coordinates/coordinates.py +++ b/podpac/core/coordinates/coordinates.py @@ -1513,21 +1513,19 @@ def is_stacked(self, dim): # re-wrote to be able to iterate through c.dims elif value == 2: # both true return False - def horizontal_resolution(self, units="metre", restype="nominal"): + def horizontal_resolution(self, units="meter", restype="nominal"): """ - Return horizontal resolution of coordinate system. - This function handles mainly edge case sanitation. - It calls StackedCoordiantes and Coordinates1d 'horizontal_resolution' methods to get the actual values. + Returns horizontal resolution of coordinate system. Parameters ---------- units : str - The desired unit the returned resolution should be in. Supports any unit supported by podpac.units - type : str + The desired unit the returned resolution should be in. Supports any unit supported by podpac.units (i.e. pint). Default is 'meter'. + restype : str The kind of horizontal resolution that should be returned. Supported values are: - - "nominal" <-- this is wrong but cheap to calculate. Give a 'nominal' resolution over the entire domain - - "summary" <-- Gives the exact mean and standard deviation for unstacked coordinates, some error for stacked coordinates - - "full" <-- Gives exact grid differences if unstacked coordiantes or distance matrix if stacked coordinnates + - "nominal" <-- Returns a number. Gives a 'nominal' resolution over the entire domain. This is wrong but fast. + - "summary" <-- Returns a tuple (mean, standard deviation). Gives the exact mean and standard deviation for unstacked coordinates, some error for stacked coordinates + - "full" <-- Returns a 1 or 2-D array. Gives exact grid differences if unstacked coordinates or distance matrix if stacked coordinates Returns ------- @@ -1543,7 +1541,13 @@ def horizontal_resolution(self, units="metre", restype="nominal"): ValueError If the 'restype' is not one of the supported resolution types + """ + # This function handles mainly edge case sanitation. + # It calls StackedCoordinates and Coordinates1d 'horizontal_resolution' methods to get the actual values. + + if "lat" not in self.udims: # require latitude + raise ValueError("Latitude required for horizontal resolution.") # ellipsoid tuple to pass to geodesic ellipsoid_tuple = ( @@ -1556,23 +1560,26 @@ def horizontal_resolution(self, units="metre", restype="nominal"): resolutions = OrderedDict() # To return for name, dim in self.items(): if dim.is_stacked: - for stack_name in dim.dims: # make sure ['lat_lon'] is stacked before returning horizontal_resolution - if stack_name != "lat" and stack_name != "lon": # check for valid stacking - raise ValueError( - "Must be lat and lon stacked, cannot return resolution for stacked {}".format(stack_name) - ) - resolutions[name] = dim.horizontal_resolution( - ellipsoid_tuple, self.CRS.coordinate_system.name, restype, units - ) - else: - if ( - name == "lat" or name == "lon" - ): # need to do this inside of loop in case of stacked [[alt,time]] but unstacked [lat, lon] - if "lat" not in self.dims: # require latitude - raise ValueError("Latitude required for horizontal resolution.") + if "lat" in dim.dims and "lon" in dim.dims: resolutions[name] = dim.horizontal_resolution( + None, ellipsoid_tuple, self.CRS.coordinate_system.name, restype, units + ) + elif "lat" in dim.dims: + # Calling self['lat'] forces UniformCoordinates1d, even if stacked + resolutions["lat"] = self["lat"].horizontal_resolution( + self["lat"], ellipsoid_tuple, self.CRS.coordinate_system.name, restype, units + ) + elif "lon" in dim.dims: + # Calling self['lon'] forces UniformCoordinates1d, even if stacked + resolutions["lon"] = self["lon"].dim.horizontal_resolution( self["lat"], ellipsoid_tuple, self.CRS.coordinate_system.name, restype, units ) + elif ( + name == "lat" or name == "lon" + ): # need to do this inside of loop in case of stacked [[alt,time]] but unstacked [lat, lon] + resolutions[name] = dim.horizontal_resolution( + self["lat"], ellipsoid_tuple, self.CRS.coordinate_system.name, restype, units + ) return resolutions diff --git a/podpac/core/coordinates/coordinates1d.py b/podpac/core/coordinates/coordinates1d.py index 6a18d916..38972497 100644 --- a/podpac/core/coordinates/coordinates1d.py +++ b/podpac/core/coordinates/coordinates1d.py @@ -469,7 +469,7 @@ def nominal_unstacked_resolution(): calculate_distance( (self.bounds[0], 0), (self.bounds[1], 0), ellipsoid_tuple, coordinate_name, units ).magnitude - / self.size + / (self.size - 1) ) * podpac.units(units) elif self.name == "lon": median_lat = ((latitude.bounds[1] - latitude.bounds[0]) / 2) + latitude.bounds[0] @@ -481,7 +481,7 @@ def nominal_unstacked_resolution(): coordinate_name, units, ).magnitude - / self.size + / (self.size - 1) ) * podpac.units(units) else: return ValueError("Unknown dim: {}".format(self.name)) diff --git a/podpac/core/coordinates/stacked_coordinates.py b/podpac/core/coordinates/stacked_coordinates.py index d891a9e0..3d7a1861 100644 --- a/podpac/core/coordinates/stacked_coordinates.py +++ b/podpac/core/coordinates/stacked_coordinates.py @@ -708,7 +708,7 @@ def is_affine(self): return True - def horizontal_resolution(self, ellipsoid_tuple, coordinate_name, restype="nominal", units="meter"): + def horizontal_resolution(self, latitude, ellipsoid_tuple, coordinate_name, restype="nominal", units="meter"): """Return the horizontal resolution of a Uniform 1D Coordinate Parameters @@ -737,6 +737,7 @@ def horizontal_resolution(self, ellipsoid_tuple, coordinate_name, restype="nomin if unknown restype """ + order = tuple([self.dims.index(d) for d in ["lat", "lon"]]) def nominal_stacked_resolution(): """Use a KDTree to return approximate stacked resolution with some loss of accuracy. @@ -746,7 +747,7 @@ def nominal_stacked_resolution(): The average min distance of every point """ - tree = spatial.KDTree(self.coordinates + [0, 180.0], boxsize=[0.0, 360.0000000000001]) + tree = spatial.KDTree(self.coordinates[:, order] + [0, 180.0], boxsize=[0.0, 360.0000000000001]) return np.average( calculate_distance( tree.data - [0, 180.0], @@ -765,7 +766,7 @@ def summary_stacked_resolution(): tuple Average min distance of every point and standard deviation of those min distances """ - tree = spatial.KDTree(self.coordinates + [0, 180.0], boxsize=[0.0, 360.0000000000001]) + tree = spatial.KDTree(self.coordinates[:, order] + [0, 180.0], boxsize=[0.0, 360.0000000000001]) distances = calculate_distance( tree.data - [0, 180.0], tree.data[tree.query(tree.data, k=2)[1][:, 1]] - [0, 180.0], @@ -785,7 +786,7 @@ def full_stacked_resolution(): distance_matrix = np.zeros((len(self.coordinates), len(self.coordinates))) for i in range(len(self.coordinates)): distance_matrix[i, :] = calculate_distance( - self.coordinates[i], self.coordinates[:], ellipsoid_tuple, coordinate_name, units + self.coordinates[i, order], self.coordinates[:, order], ellipsoid_tuple, coordinate_name, units ).magnitude return distance_matrix * podpac.units(units) diff --git a/podpac/core/coordinates/test/test_coordinates.py b/podpac/core/coordinates/test/test_coordinates.py index 51b14055..11124122 100644 --- a/podpac/core/coordinates/test/test_coordinates.py +++ b/podpac/core/coordinates/test/test_coordinates.py @@ -2218,12 +2218,7 @@ def test_horizontal_resolution(self): time = ["2018-01-01", "2018-01-02", "2018-01-03", "2018-01-04", "2018-01-05"] # Invalid stacked dims check: - c = Coordinates([[lat, time]], dims=["lat_time"]) - with pytest.raises(ValueError): - c.horizontal_resolution() - - # Stacked and Unstacked valid Check: - c = Coordinates([[lat, time], lon], dims=["lat_time", "lon"]) + c = Coordinates([[lon, time]], dims=["lon_time"]) with pytest.raises(ValueError): c.horizontal_resolution() @@ -2236,9 +2231,26 @@ def test_horizontal_resolution(self): c = Coordinates([[lat, lon]], dims=["lat_lon"]) c.horizontal_resolution() + c = Coordinates([[lon, lat]], dims=["lon_lat"]) + c.horizontal_resolution() + # Corect name for restype: with pytest.raises(ValueError): c.horizontal_resolution(restype="whateverIwant") # Unstacked - c = Coordinates([[lat, lon]], dims=["lat_lon"]) + c = Coordinates([lat, lon], dims=["lat", "lon"]) + c.horizontal_resolution() + + c = Coordinates([lat, lon], dims=["lat", "lon"]) + c.horizontal_resolution(restype="summary") + + # Mixed stacked, unstacked, but still valid + # Stacked and Unstacked valid Check: + c = Coordinates([[lat, time], lon], dims=["lat_time", "lon"]) + c2 = Coordinates([lat, lon], dims=["lat", "lon"]) + np.testing.assert_array_equal(c.horizontal_resolution(), c2.horizontal_resolution()) + # Lat only + c = Coordinates([[lat, time]], dims=["lat_time"]) + c2 = Coordinates([lat], dims=["lat"]) + np.testing.assert_array_equal(c.horizontal_resolution(), c2.horizontal_resolution()) diff --git a/podpac/core/coordinates/test/test_coordinates1d.py b/podpac/core/coordinates/test/test_coordinates1d.py index 020e6509..b7f7d3d5 100644 --- a/podpac/core/coordinates/test/test_coordinates1d.py +++ b/podpac/core/coordinates/test/test_coordinates1d.py @@ -105,7 +105,7 @@ def test_horizontal_resolution(self): coord_name = "ellipsoidal" # Resolution: nominal - assert lat.horizontal_resolution(lat, ell_tuple, coord_name) == 3554055.948774749 * podpac.units("meter") + assert lat.horizontal_resolution(lat, ell_tuple, coord_name) == 4442569.935968436 * podpac.units("meter") assert lon.horizontal_resolution(lat, ell_tuple, coord_name) == 0.0 * podpac.units("meter") # Resolution: summary diff --git a/podpac/core/coordinates/test/test_stacked_coordinates.py b/podpac/core/coordinates/test/test_stacked_coordinates.py index f9f303c2..c2b78440 100644 --- a/podpac/core/coordinates/test/test_stacked_coordinates.py +++ b/podpac/core/coordinates/test/test_stacked_coordinates.py @@ -998,15 +998,15 @@ def test_horizontal_resolution(self): # Nominal resolution: np.testing.assert_almost_equal( - c.horizontal_resolution(ell_tuple, coord_name, restype="nominal").magnitude, 7397047.845631437 + c.horizontal_resolution(None, ell_tuple, coord_name, restype="nominal").magnitude, 7397047.845631437 ) # Summary resolution np.testing.assert_almost_equal( - c.horizontal_resolution(ell_tuple, coord_name, restype="summary")[0].magnitude, 7397047.845631437 + c.horizontal_resolution(None, ell_tuple, coord_name, restype="summary")[0].magnitude, 7397047.845631437 ) np.testing.assert_almost_equal( - c.horizontal_resolution(ell_tuple, coord_name, restype="summary")[1].magnitude, 2134971.4571846593 + c.horizontal_resolution(None, ell_tuple, coord_name, restype="summary")[1].magnitude, 2134971.4571846593 ) # Full resolution @@ -1019,5 +1019,31 @@ def test_horizontal_resolution(self): ] np.testing.assert_array_almost_equal( - c.horizontal_resolution(ell_tuple, coord_name, restype="full"), distance_matrix + c.horizontal_resolution(None, ell_tuple, coord_name, restype="full"), distance_matrix + ) + + # Test different order of lat/lon still works + c2 = StackedCoordinates([lon, lat]) + np.testing.assert_equal( + c.horizontal_resolution(None, ell_tuple, "lat").magnitude, + c2.horizontal_resolution(None, ell_tuple, "lat").magnitude, + ) + + # Test multiple stacked coordinates + alt = podpac.clinspace(0, 1, 5, "alt") + + c2 = StackedCoordinates([lon, alt, lat]) + np.testing.assert_equal( + c.horizontal_resolution(None, ell_tuple, "lat").magnitude, + c2.horizontal_resolution(None, ell_tuple, "lat").magnitude, + ) + c2 = StackedCoordinates([alt, lon, lat]) + np.testing.assert_equal( + c.horizontal_resolution(None, ell_tuple, "lat").magnitude, + c2.horizontal_resolution(None, ell_tuple, "lat").magnitude, + ) + c2 = StackedCoordinates([lat, alt, lon]) + np.testing.assert_equal( + c.horizontal_resolution(None, ell_tuple, "lat").magnitude, + c2.horizontal_resolution(None, ell_tuple, "lat").magnitude, ) From 3b8294497404a0217619d523871b27a56111cd43 Mon Sep 17 00:00:00 2001 From: Matt Ueckermann Date: Thu, 13 Jul 2023 10:08:23 -0400 Subject: [PATCH 49/52] BUGFIX: The Coordinates.from_url function should shrink the bounding-box. * The bbox specifies the cell edges, but PODPAC uses cell centers for coordinates * The original unittest was incorrect, also now fixed. --- podpac/core/coordinates/coordinates.py | 10 ++- .../core/coordinates/test/test_coordinates.py | 65 +++++++++++-------- 2 files changed, 46 insertions(+), 29 deletions(-) diff --git a/podpac/core/coordinates/coordinates.py b/podpac/core/coordinates/coordinates.py index 5f76db8e..6a22dd46 100644 --- a/podpac/core/coordinates/coordinates.py +++ b/podpac/core/coordinates/coordinates.py @@ -481,9 +481,15 @@ def from_url(cls, url): r = 1 # Extract bounding box information and translate to PODPAC coordinates + size = np.array([_get_param(params, "HEIGHT"), _get_param(params, "WIDTH")], int) start = bbox[:2][::r] - stop = bbox[2::][::r] - size = np.array([_get_param(params, "WIDTH"), _get_param(params, "HEIGHT")], int)[::r] + stop = bbox[2:][::r] + + # The bbox gives the edges of the pixels, but our coordinates use the + # box centers -- so we have to shrink the start/stop portions + dx = (stop - start) / (size) # This should take care of signs + start = start + dx / 2 + stop = stop - dx / 2 coords["coords"] = [ {"name": "lat", "start": stop[0], "stop": start[0], "size": size[0]}, diff --git a/podpac/core/coordinates/test/test_coordinates.py b/podpac/core/coordinates/test/test_coordinates.py index 11124122..0f9968a2 100644 --- a/podpac/core/coordinates/test/test_coordinates.py +++ b/podpac/core/coordinates/test/test_coordinates.py @@ -380,57 +380,63 @@ def test_from_url(self): url = ( r"http://testwms/?map=map&&service=WMS&request=GetMap&layers=layer&styles=&format=image%2Fpng" - r"&transparent=true&version={version}&transparency=true&width=256&height=256&srs=EPSG%3A{epsg}" + r"&transparent=true&version={version}&transparency=true&width=128&height=256&srs=EPSG%3A{epsg}" r"&bbox={},{},{},{}&time={}" ) # version 1.1.1 c = Coordinates.from_url( url.format( - min(crds2.bounds["lon"]), - min(crds2.bounds["lat"]), - max(crds2.bounds["lon"]), - max(crds2.bounds["lat"]), + min(crds2.bounds["lon"]) - np.abs(crds2["lon"].step / 127 / 2), + min(crds2.bounds["lat"]) - np.abs(crds2["lat"].step / 255 / 2), + max(crds2.bounds["lon"]) + np.abs(crds2["lon"].step / 127 / 2), + max(crds2.bounds["lat"]) + np.abs(crds2["lat"].step / 255 / 2), crds2.bounds["time"][0], version="1.1.1", epsg="3857", ) ) assert c.bounds == crds2.bounds + assert crds2.bounds["lon"][0] - c["lon"].step / 2 == c.geotransform[0] + assert crds2.bounds["lat"][1] - c["lat"].step / 2 == c.geotransform[3] c = Coordinates.from_url( url.format( - min(crds.bounds["lon"]), - min(crds.bounds["lat"]), - max(crds.bounds["lon"]), - max(crds.bounds["lat"]), + min(crds.bounds["lon"]) - np.abs(crds["lon"].step / 127 / 2), + min(crds.bounds["lat"]) - np.abs(crds["lat"].step / 255 / 2), + max(crds.bounds["lon"]) + np.abs(crds["lon"].step / 127 / 2), + max(crds.bounds["lat"]) + np.abs(crds["lat"].step / 255 / 2), crds.bounds["time"][0], version="1.1.1", epsg="4326", ) ) assert c.bounds == crds.bounds + assert crds.bounds["lon"][0] - c["lon"].step / 2 == c.geotransform[0] + assert crds.bounds["lat"][1] - c["lat"].step / 2 == c.geotransform[3] # version 1.3 c = Coordinates.from_url( url.format( - min(crds2.bounds["lon"]), - min(crds2.bounds["lat"]), - max(crds2.bounds["lon"]), - max(crds2.bounds["lat"]), + min(crds2.bounds["lon"]) - np.abs(crds2["lon"].step / 127 / 2), + min(crds2.bounds["lat"]) - np.abs(crds2["lat"].step / 255 / 2), + max(crds2.bounds["lon"]) + np.abs(crds2["lon"].step / 127 / 2), + max(crds2.bounds["lat"]) + np.abs(crds2["lat"].step / 255 / 2), crds2.bounds["time"][0], version="1.3", epsg="3857", ) ) assert c.bounds == crds2.bounds + assert crds2.bounds["lon"][0] - c["lon"].step / 2 == c.geotransform[0] + assert crds2.bounds["lat"][1] - c["lat"].step / 2 == c.geotransform[3] c = Coordinates.from_url( url.format( - min(crds.bounds["lat"]), - min(crds.bounds["lon"]), - max(crds.bounds["lat"]), - max(crds.bounds["lon"]), + min(crds.bounds["lat"]) - np.abs(crds["lat"].step / 255 / 2), + min(crds.bounds["lon"]) - np.abs(crds["lon"].step / 127 / 2), + max(crds.bounds["lat"]) + np.abs(crds["lat"].step / 255 / 2), + max(crds.bounds["lon"]) + np.abs(crds["lon"].step / 127 / 2), crds.bounds["time"][0], version="1.3", epsg="4326", @@ -438,6 +444,8 @@ def test_from_url(self): ) assert c.bounds == crds.bounds + assert crds.bounds["lon"][0] - c["lon"].step / 2 == c.geotransform[0] + assert crds.bounds["lat"][1] - c["lat"].step / 2 == c.geotransform[3] # WCS version crds = Coordinates([[41, 40], [-71, -70], "2018-05-19"], dims=["lat", "lon", "time"]) @@ -445,23 +453,24 @@ def test_from_url(self): url = ( r"http://testwms/?map=map&&service=WCS&request=GetMap&layers=layer&styles=&format=image%2Fpng" - r"&transparent=true&version={version}&transparency=true&width=256&height=256&srs=EPSG%3A{epsg}" + r"&transparent=true&version={version}&transparency=true&width=128&height=256&srs=EPSG%3A{epsg}" r"&bbox={},{},{},{}&time={}" ) c = Coordinates.from_url( url.format( - min(crds2.bounds["lon"]), - min(crds2.bounds["lat"]), - max(crds2.bounds["lon"]), - max(crds2.bounds["lat"]), + min(crds2.bounds["lon"]) - np.abs(crds2["lon"].step / 127 / 2), + min(crds2.bounds["lat"]) - np.abs(crds2["lat"].step / 255 / 2), + max(crds2.bounds["lon"]) + np.abs(crds2["lon"].step / 127 / 2), + max(crds2.bounds["lat"]) + np.abs(crds2["lat"].step / 255 / 2), crds2.bounds["time"][0], version="1.1", epsg="3857", ) ) assert c.bounds == crds2.bounds - + assert crds2.bounds["lon"][0] - c["lon"].step / 2 == c.geotransform[0] + assert crds2.bounds["lat"][1] - c["lat"].step / 2 == c.geotransform[3] # Based on all the documentation I've read, this should be correct, but # based on the server's I've checked, this does not seem correct # c = Coordinates.from_url( @@ -478,10 +487,10 @@ def test_from_url(self): c = Coordinates.from_url( url.format( - min(crds.bounds["lon"]), - min(crds.bounds["lat"]), - max(crds.bounds["lon"]), - max(crds.bounds["lat"]), + min(crds.bounds["lon"]) - np.abs(crds["lon"].step / 127 / 2), + min(crds.bounds["lat"]) - np.abs(crds["lat"].step / 255 / 2), + max(crds.bounds["lon"]) + np.abs(crds["lon"].step / 127 / 2), + max(crds.bounds["lat"]) + np.abs(crds["lat"].step / 255 / 2), crds.bounds["time"][0], version="1.1", epsg="4326", @@ -489,6 +498,8 @@ def test_from_url(self): ) assert c.bounds == crds.bounds + assert crds.bounds["lon"][0] - c["lon"].step / 2 == c.geotransform[0] + assert crds.bounds["lat"][1] - c["lat"].step / 2 == c.geotransform[3] def test_from_xarray(self): lat = [0, 1, 2] From 62845de04e2bbe9229b44b5274c25e1986a339ac Mon Sep 17 00:00:00 2001 From: Matt Ueckermann Date: Sun, 6 Aug 2023 03:38:29 -0400 Subject: [PATCH 50/52] DOC: Just adding a note --- podpac/core/coordinates/coordinates.py | 1 + 1 file changed, 1 insertion(+) diff --git a/podpac/core/coordinates/coordinates.py b/podpac/core/coordinates/coordinates.py index 6a22dd46..e21a9096 100644 --- a/podpac/core/coordinates/coordinates.py +++ b/podpac/core/coordinates/coordinates.py @@ -481,6 +481,7 @@ def from_url(cls, url): r = 1 # Extract bounding box information and translate to PODPAC coordinates + # Not, size does not get re-ordered, Height == Lat and width = lon -- ALWAYS size = np.array([_get_param(params, "HEIGHT"), _get_param(params, "WIDTH")], int) start = bbox[:2][::r] stop = bbox[2:][::r] From 2042327e4283e693fee01969359d84b3c5edd504 Mon Sep 17 00:00:00 2001 From: Matt Ueckermann Date: Thu, 24 Aug 2023 15:05:41 +0000 Subject: [PATCH 51/52] FIX: Ensuring geopy is optional as a dependency --- podpac/core/coordinates/coordinates.py | 2 -- podpac/core/coordinates/utils.py | 4 +++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/podpac/core/coordinates/coordinates.py b/podpac/core/coordinates/coordinates.py index e21a9096..b5885099 100644 --- a/podpac/core/coordinates/coordinates.py +++ b/podpac/core/coordinates/coordinates.py @@ -20,8 +20,6 @@ import pyproj import logging from scipy import spatial -from geopy.distance import geodesic - import podpac from podpac.core.settings import settings diff --git a/podpac/core/coordinates/utils.py b/podpac/core/coordinates/utils.py index a65ec35f..b0639262 100644 --- a/podpac/core/coordinates/utils.py +++ b/podpac/core/coordinates/utils.py @@ -19,7 +19,9 @@ import traitlets as tl from six import string_types import pyproj -from geopy.distance import geodesic + +from lazy_import import lazy_function +geodesic = lazy_function("geopy.distance.geodesic") import podpac From 3663837cb4998003abfbc331b7223ec847f8c839 Mon Sep 17 00:00:00 2001 From: Matt Ueckermann Date: Fri, 25 Aug 2023 13:23:24 -0400 Subject: [PATCH 52/52] FIX: Coordinates now return values when select bounds are between coordinates. * When selecting coordinates based on bounds, if the bounds were between two coordinates we used to return an empty sub-selected coordinates. (e.g. bounds = [5.5, 5.6], coords = [4, 5, 6]) * Now we return the coordinates that have data for those bounds. That is, we interpret the coordinate to represent constant data in the interval between coordinates * |------0--b-b--|------1-------| --> returns 0 * |------0-----b-|-b----1-------| --> returns 0 and 1 --- .../core/coordinates/array_coordinates1d.py | 41 ++++++++++++++++++- .../test/test_array_coordinates1d.py | 41 +++++++++++++++++++ .../core/coordinates/test/test_coordinates.py | 2 +- .../test/test_uniform_coordinates1d.py | 40 +++++++++++++----- .../core/coordinates/uniform_coordinates1d.py | 10 +++-- 5 files changed, 119 insertions(+), 15 deletions(-) diff --git a/podpac/core/coordinates/array_coordinates1d.py b/podpac/core/coordinates/array_coordinates1d.py index 2d4c2f4a..2bc03897 100644 --- a/podpac/core/coordinates/array_coordinates1d.py +++ b/podpac/core/coordinates/array_coordinates1d.py @@ -377,7 +377,46 @@ def _select(self, bounds, return_index, outer): gt = self.coordinates >= bounds[0] lt = self.coordinates <= bounds[1] b = gt & lt - + b2 = gt | lt + if b2.sum() == b2.size and b.sum() == 0 and self.is_monotonic: + # bounds between data points + indlt = np.argwhere(lt).squeeze() + indgt = np.argwhere(gt).squeeze() + if self._is_descending: + if indlt.size > 0: + indlt = indlt[0] + else: + indlt = b.size - 1 + if indgt.size > 0: + indgt = indgt[-1] + else: + indgt = 0 + else: + if indlt.size > 0: + indlt = indlt[-1] + else: + indlt = 0 + if indgt.size > 0: + indgt = indgt[0] + else: + indgt = b.size - 1 + + ind0 = min(indlt, indgt) + ind1 = max(indlt, indgt) + 1 + b[ind0:ind1] = True + if b.sum() > 1: + # These two coordinates are candidates, we need + # to make sure that the bounds cross the edge between + # the two points (selects both) or not (only selects) + crds = self.coordinates[b] + step = np.diff(self.coordinates[b])[0] + edge = crds[0] + step / 2 + bounds_lt = bounds <= edge + bounds_gt = bounds > edge + keep_point = [np.any(bounds_lt), np.any(bounds_gt)] + if self._is_descending: + keep_point = keep_point[::-1] + b[ind0:ind1] = keep_point elif self.is_monotonic: gt = np.where(self.coordinates >= bounds[0])[0] lt = np.where(self.coordinates <= bounds[1])[0] diff --git a/podpac/core/coordinates/test/test_array_coordinates1d.py b/podpac/core/coordinates/test/test_array_coordinates1d.py index 211e62fa..804393bf 100644 --- a/podpac/core/coordinates/test/test_array_coordinates1d.py +++ b/podpac/core/coordinates/test/test_array_coordinates1d.py @@ -836,6 +836,47 @@ def test_select(self): assert_equal(s.coordinates, []) assert_equal(c.coordinates[I], []) + def test_select_one_between_coords(self): + # Ascending + c = ArrayCoordinates1d([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) + c1, inds1 = c.select([5.6, 6.1], return_index=True, outer=False) + assert np.argwhere(inds1).squeeze() == 6 + + c2, inds2 = c.select([5.6, 5.6], return_index=True, outer=False) + assert np.argwhere(inds2).squeeze() == 6 + + c3, inds3 = c.select([5.4, 5.4], return_index=True, outer=False) + assert np.argwhere(inds3).squeeze() == 5 + + c3b, inds3b = c.select([5.4, 5.6], return_index=True, outer=False) + assert np.all(np.argwhere(inds3b).squeeze() == [5, 6]) + + c4, inds4 = c.select([9.1, 9.1], return_index=True, outer=False) + assert inds4 == [] + + c5, inds5 = c.select([-0.1, -0.1], return_index=True, outer=False) + assert inds5 == [] + + # Decending + c = ArrayCoordinates1d([0, 1, 2, 3, 4, 5, 6, 7, 8, 9][::-1]) + c1, inds1 = c.select([5.6, 6.1], return_index=True, outer=False) + assert np.argwhere(inds1).squeeze() == 3 + + c2, inds2 = c.select([5.6, 5.6], return_index=True, outer=False) + assert np.argwhere(inds2).squeeze() == 3 + + c3, inds3 = c.select([5.4, 5.4], return_index=True, outer=False) + assert np.argwhere(inds3).squeeze() == 4 + + c3b, inds3b = c.select([5.4, 5.6], return_index=True, outer=False) + assert np.all(np.argwhere(inds3b).squeeze() == [3, 4]) + + c4, inds4 = c.select([9.1, 9.1], return_index=True, outer=False) + assert inds4 == [] + + c5, inds5 = c.select([-0.1, -0.1], return_index=True, outer=False) + assert inds5 == [] + def test_select_outer_ascending(self): c = ArrayCoordinates1d([10.0, 20.0, 40.0, 50.0, 60.0, 90.0]) diff --git a/podpac/core/coordinates/test/test_coordinates.py b/podpac/core/coordinates/test/test_coordinates.py index 0f9968a2..b4dd1d7a 100644 --- a/podpac/core/coordinates/test/test_coordinates.py +++ b/podpac/core/coordinates/test/test_coordinates.py @@ -1458,7 +1458,7 @@ def test_intersect(self): assert ct2.size == 3 ct2 = ct.intersect(cti, outer=False) - assert ct2.size == 0 # Is this behavior desired? + assert ct2.size == 1 # def test_intersect_dims(self): lat = ArrayCoordinates1d([0, 1, 2, 3, 4, 5], name="lat") diff --git a/podpac/core/coordinates/test/test_uniform_coordinates1d.py b/podpac/core/coordinates/test/test_uniform_coordinates1d.py index 51f1924a..a25b40a0 100644 --- a/podpac/core/coordinates/test/test_uniform_coordinates1d.py +++ b/podpac/core/coordinates/test/test_uniform_coordinates1d.py @@ -944,13 +944,13 @@ def test_select_ascending(self): # between coordinates s = c.select([52, 55]) - assert isinstance(s, ArrayCoordinates1d) - assert_equal(s.coordinates, []) + assert isinstance(s, UniformCoordinates1d) + assert_equal(s.coordinates, c[3:5].coordinates) s, I = c.select([52, 55], return_index=True) - assert isinstance(s, ArrayCoordinates1d) - assert_equal(s.coordinates, []) - assert_equal(c.coordinates[I], []) + assert isinstance(s, UniformCoordinates1d) + assert_equal(s.coordinates, c[3:5].coordinates) + assert_equal(c.coordinates[I], c[3:5].coordinates) # backwards bounds s = c.select([70, 30]) @@ -1015,13 +1015,13 @@ def test_select_descending(self): # between coordinates s = c.select([52, 55]) - assert isinstance(s, ArrayCoordinates1d) - assert_equal(s.coordinates, []) + assert isinstance(s, UniformCoordinates1d) + assert_equal(s.coordinates, [60, 50]) s, I = c.select([52, 55], return_index=True) - assert isinstance(s, ArrayCoordinates1d) - assert_equal(s.coordinates, []) - assert_equal(c.coordinates[I], []) + assert isinstance(s, UniformCoordinates1d) + assert_equal(s.coordinates, [60, 50]) + assert_equal(c.coordinates[I], [60, 50]) # backwards bounds s = c.select([70, 30]) @@ -1116,6 +1116,26 @@ def test_select_time_variable_precision(self): assert s1.size == 0 assert s2.size == 1 + def test_select_one_floating_point_error(self): + c = UniformCoordinates1d(0, 9, 1) # FIX THE PROBLEM HERE!! + c1, inds1 = c.select([5.6, 6.1], return_index=True, outer=False) + assert inds1 == slice(6, 7) + + c2, inds2 = c.select([5.6, 5.6], return_index=True, outer=False) + assert inds2 == slice(6, 7) + + c3, inds3 = c.select([5.4, 5.4], return_index=True, outer=False) + assert inds3 == slice(5, 6) + + c3, inds3b = c.select([5.4, 5.6], return_index=True, outer=False) + assert inds3b == slice(5, 7) + + c4, inds4 = c.select([9.1, 9.1], return_index=True, outer=False) + assert inds4 == [] + + c5, inds5 = c.select([-0.1, -0.1], return_index=True, outer=False) + assert inds5 == [] + class TestUniformCoordinatesMethods(object): def test_unique(self): diff --git a/podpac/core/coordinates/uniform_coordinates1d.py b/podpac/core/coordinates/uniform_coordinates1d.py index bbb13dfe..3f7ae039 100644 --- a/podpac/core/coordinates/uniform_coordinates1d.py +++ b/podpac/core/coordinates/uniform_coordinates1d.py @@ -530,8 +530,14 @@ def _select(self, bounds, return_index, outer): imin = np.clip(imin, 0, self.size) # empty case - if imin >= imax: + if imin > imax: return self._select_empty(return_index) + if imax == imin: + # could have been selected between two existing coordinates + imin = int(np.round(fmin)) + imax = int(np.round(fmax)) + 1 + if imin >= (self.size - 1) | imin < 0: + return self._select_empty(return_index) if self.is_descending: imax, imin = self.size - imin, self.size - imax @@ -541,5 +547,3 @@ def _select(self, bounds, return_index, outer): return self[I], I else: return self[I] - - \ No newline at end of file