From 2a82d57ec557ee32a7c863ec42f8eaf93f584527 Mon Sep 17 00:00:00 2001 From: Markus Nagel Date: Thu, 20 Aug 2020 17:01:52 +0200 Subject: [PATCH 01/38] #23 initial push. Validate an xarray against an dictionary which sets dtypes and values of the dimensions. - prototype! --- weldx/utility.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/weldx/utility.py b/weldx/utility.py index 9bdaf1d06..6bf44feba 100644 --- a/weldx/utility.py +++ b/weldx/utility.py @@ -560,6 +560,36 @@ def xr_interp_like( return result +def xr_check_format(dax: xr.DataArray, ref: dict): + """Validate the format of the DataArray against a dictionary. + + Does nothing if everything is valid? + + """ + + # use this error? + from asdf import ValidationError + + # check labels in ref + # there should only be "time" and "c"? + for key in ref: + if key is "time": + # everything with key = time + if ref[key].get("optional"): + # if time is optional - no comparison + pass + else: + # if time not optional - compare + for n, dim in enumerate(dax.coords.dims): + if not getattr(dax, dim).dtype == np.dtype(ref[key]["dtype"][n]): + ValidationError(f"Datatype mismatch in dimension {dim}.") + else: + # in c are the dimensions of the xarray + for dim in ref[key]["values"]: + if not hasattr(dax, dim): + ValidationError(f"Dimension {dim} is missing in the xarray.") + + def xr_3d_vector(data, times=None) -> xr.DataArray: """Create an xarray 3d vector with correctly named dimensions and coordinates. From d603bb221083cc3f38800886671e758249ad45d5 Mon Sep 17 00:00:00 2001 From: Markus Nagel Date: Tue, 8 Sep 2020 12:58:15 +0200 Subject: [PATCH 02/38] #23 Added 3 functions xr_valid_key, xr_check_coords, xr_check_dtype. --- weldx/utility.py | 106 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 84 insertions(+), 22 deletions(-) diff --git a/weldx/utility.py b/weldx/utility.py index 6bf44feba..de4416536 100644 --- a/weldx/utility.py +++ b/weldx/utility.py @@ -560,34 +560,96 @@ def xr_interp_like( return result -def xr_check_format(dax: xr.DataArray, ref: dict): - """Validate the format of the DataArray against a dictionary. +# Notes: +# 2 functions: xr_check_coords, xr_check_dtype, (xr_check_unit - not +# possible with xr yet) +# both functions check if the keys in ref are valid in the xarray +# keywords with functionality: +# dtype: 1 of the dtype in the list should be valid +# values: check if the values are the same as in the xarray +# optional: default false, if true it has to have some attributes (dtype, values) - Does nothing if everything is valid? - """ +def xr_valid_key(dax: xr.DataArray, ref: dict): + """Validate if the keys in ref are present in the DataArray. + + TODO: Warnings or raise an error? Would suggest a raise here. + Since the other functions will get stuck otherwise + """ # use this error? - from asdf import ValidationError + # from asdf import ValidationError - # check labels in ref - # there should only be "time" and "c"? for key in ref: - if key is "time": - # everything with key = time - if ref[key].get("optional"): - # if time is optional - no comparison - pass - else: - # if time not optional - compare - for n, dim in enumerate(dax.coords.dims): - if not getattr(dax, dim).dtype == np.dtype(ref[key]["dtype"][n]): - ValidationError(f"Datatype mismatch in dimension {dim}.") - else: - # in c are the dimensions of the xarray - for dim in ref[key]["values"]: - if not hasattr(dax, dim): - ValidationError(f"Dimension {dim} is missing in the xarray.") + if not hasattr(dax, ref): + # Attributes not found in dax + print("TODO: add error message / raise Error.") + + +def xr_check_coords(dax: xr.DataArray, ref: dict): + """Validate the coordinates of the DataArray against a dictionary. + + TODO: Warnings or raise an error? + + """ + + # check if the keys in ref are also in dax + xr_valid_key(dax, ref) + + for key in ref: + # check if the optional key is set to true + if hasattr(ref[key], "optional"): + if ( + getattr(ref[key]["optional"]) is True + and not hasattr(ref[key], "dtype") + and not hasattr(ref[key], "values") + ): + # Optional is True so there has to be "dtype" or "values" + print( + "TODO: add error message / raise Error. " + "The dtype or values have to be in ref." + ) + + # only if the key "values" is given do the validation + if hasattr(ref[key], "values"): + # TODO: compare against a set? what if some variables are doubled? + # Suggestion: Values should be in an array which should be identical to the + # array to compare against! + # Solution for the suggestion: + if (getattr(dax, key).values == ref[key]["values"]).all(): + # TODO: Error or Warning? + print("Value mismatch") + + +def xr_check_dtype(dax: xr.DataArray, ref: dict): + """Validate the dtype of the DataArray against a dictionary. + + TODO: Warnings or raise an error? + + """ + + # check if the keys in ref are also in dax + xr_valid_key(dax, ref) + + for key in ref: + # check if the optional key is set to true + if hasattr(ref[key], "optional"): + if ( + getattr(ref[key]["optional"]) is True + and not hasattr(ref[key], "dtype") + and not hasattr(ref[key], "values") + ): + # Optional is True so there has to be "dtype" or "values" + print( + "TODO: add error message / raise Error. " + "Either dtype or values have to be in ref." + ) + + # only if the key "dtype" is given do the validation + if hasattr(ref[key], "dtype"): + if getattr(dax, key).dtype not in [np.dtype(x) for x in ref[key]["dtype"]]: + # TODO: Error or Warning? + print("Data type mismatch of ref and dax.") def xr_3d_vector(data, times=None) -> xr.DataArray: From cc1bc097e48333cdf53aaefe8807689ce1dc4117 Mon Sep 17 00:00:00 2001 From: Markus Nagel Date: Tue, 8 Sep 2020 15:07:52 +0200 Subject: [PATCH 03/38] added a testfile to show functionality --- 23_test.py | 51 ++++++++++++++++++++++++++++++++++++++++++++++++ weldx/utility.py | 24 +++++++++++------------ 2 files changed, 63 insertions(+), 12 deletions(-) create mode 100644 23_test.py diff --git a/23_test.py b/23_test.py new file mode 100644 index 000000000..42cefdf33 --- /dev/null +++ b/23_test.py @@ -0,0 +1,51 @@ +import numpy as np +import pandas as pd +import xarray as xr + +from weldx.utility import xr_check_coords, xr_check_dtype + +data = np.array([[0, 1], [2, 3]]) + +time_labels = ["2020-05-01", "2020-05-03"] +d1 = np.array([-1, 1]) +d2 = pd.DatetimeIndex(time_labels) +coords = {"d1": d1, "d2": d2, "time_labels": (["d2"], time_labels)} + +dax = xr.DataArray(data=data, dims=["d1", "d2"], coords=coords) + +dax.attrs = {"answer": 42} + +ref = { + "d1": {"values": np.array([-1, 1])}, + "d2": {"optional": True, "dtype": ["datetime64[ns]", "timedelta64[ns]"]}, +} + +ref2 = { + "d1": {"values": np.array([-1, 2])}, + "d2": {"optional": True, "dtype": ["int32"]}, +} + +ref_additional = { + "d1": {"values": np.array([-1, 1])}, + "d2": {"optional": True, "dtype": ["datetime64[ns]", "timedelta64[ns]"]}, + "d3": {"values": np.array([-1, 1])}, +} + +# testing +print("dtype check:") +xr_check_dtype(dax, ref) +print() +print("coords check:") +xr_check_coords(dax, ref) +print() +print("dtype check - wrong dtype:") +xr_check_dtype(dax, ref2) +print() +print("coords check - wrong coords:") +xr_check_coords(dax, ref2) +print() +print("dtype check - additional entry:") +xr_check_dtype(dax, ref_additional) +print() +print("coords check - additional entry:") +xr_check_coords(dax, ref_additional) diff --git a/weldx/utility.py b/weldx/utility.py index de4416536..dcbd974f3 100644 --- a/weldx/utility.py +++ b/weldx/utility.py @@ -581,7 +581,7 @@ def xr_valid_key(dax: xr.DataArray, ref: dict): # from asdf import ValidationError for key in ref: - if not hasattr(dax, ref): + if not hasattr(dax, key): # Attributes not found in dax print("TODO: add error message / raise Error.") @@ -598,11 +598,11 @@ def xr_check_coords(dax: xr.DataArray, ref: dict): for key in ref: # check if the optional key is set to true - if hasattr(ref[key], "optional"): + if "optional" in ref[key]: if ( - getattr(ref[key]["optional"]) is True - and not hasattr(ref[key], "dtype") - and not hasattr(ref[key], "values") + ref[key]["optional"] is True + and "dtype" not in ref[key] + and "values" not in ref[key] ): # Optional is True so there has to be "dtype" or "values" print( @@ -611,12 +611,12 @@ def xr_check_coords(dax: xr.DataArray, ref: dict): ) # only if the key "values" is given do the validation - if hasattr(ref[key], "values"): + if "values" in ref[key]: # TODO: compare against a set? what if some variables are doubled? # Suggestion: Values should be in an array which should be identical to the # array to compare against! # Solution for the suggestion: - if (getattr(dax, key).values == ref[key]["values"]).all(): + if not (getattr(dax, key).values == ref[key]["values"]).all(): # TODO: Error or Warning? print("Value mismatch") @@ -633,11 +633,11 @@ def xr_check_dtype(dax: xr.DataArray, ref: dict): for key in ref: # check if the optional key is set to true - if hasattr(ref[key], "optional"): + if "optional" in ref[key]: if ( - getattr(ref[key]["optional"]) is True - and not hasattr(ref[key], "dtype") - and not hasattr(ref[key], "values") + ref[key]["optional"] is True + and "dtype" not in ref[key] + and "values" not in ref[key] ): # Optional is True so there has to be "dtype" or "values" print( @@ -646,7 +646,7 @@ def xr_check_dtype(dax: xr.DataArray, ref: dict): ) # only if the key "dtype" is given do the validation - if hasattr(ref[key], "dtype"): + if "dtype" in ref[key]: if getattr(dax, key).dtype not in [np.dtype(x) for x in ref[key]["dtype"]]: # TODO: Error or Warning? print("Data type mismatch of ref and dax.") From 2e1c676ea631498aee77e0512c2524383afe46c1 Mon Sep 17 00:00:00 2001 From: Markus Nagel Date: Tue, 8 Sep 2020 15:17:13 +0200 Subject: [PATCH 04/38] #23 xr_valid_key now raises an error. --- 23_test.py | 10 ++++++++-- weldx/utility.py | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/23_test.py b/23_test.py index 42cefdf33..5057f9e6a 100644 --- a/23_test.py +++ b/23_test.py @@ -45,7 +45,13 @@ xr_check_coords(dax, ref2) print() print("dtype check - additional entry:") -xr_check_dtype(dax, ref_additional) +try: + xr_check_dtype(dax, ref_additional) +except AttributeError as err: + print(err) print() print("coords check - additional entry:") -xr_check_coords(dax, ref_additional) +try: + xr_check_coords(dax, ref_additional) +except AttributeError as err: + print(err) diff --git a/weldx/utility.py b/weldx/utility.py index dcbd974f3..6bd9a86a4 100644 --- a/weldx/utility.py +++ b/weldx/utility.py @@ -583,7 +583,7 @@ def xr_valid_key(dax: xr.DataArray, ref: dict): for key in ref: if not hasattr(dax, key): # Attributes not found in dax - print("TODO: add error message / raise Error.") + raise AttributeError(f"Data array 'dax' has no attribute '{key}'.") def xr_check_coords(dax: xr.DataArray, ref: dict): From 0b2ff67f1053171a0686868b567a5879c19da7fc Mon Sep 17 00:00:00 2001 From: Cagtay Fabry <43667554+CagtayFabry@users.noreply.github.com> Date: Tue, 8 Sep 2020 16:18:17 +0200 Subject: [PATCH 05/38] add xr_check_coords testcase --- tests/test_utility.py | 53 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/tests/test_utility.py b/tests/test_utility.py index 035b8cf7f..a9a25bdcf 100644 --- a/tests/test_utility.py +++ b/tests/test_utility.py @@ -376,3 +376,56 @@ def test_xf_fill_all(): with pytest.raises(ValueError): ut.xr_fill_all(da3, order="wrong") + + +_dax_check = xr.DataArray( + data=np.ones((2, 2, 2, 4, 3)), + dims=["d1", "d2", "d3", "d4", "d5"], + coords={ + "d1": np.array([-1, 1], dtype=float), + "d2": np.array([-1, 1], dtype=int), + "d3": pd.DatetimeIndex(["2020-05-01", "2020-05-03"]), + "d4": pd.TimedeltaIndex([0, 1, 2, 3], "s"), + "d5": ["x", "y", "z"], + }, +) + +_dax_ref = dict( + d1={"values": np.array([-1, 1]), "dtype": "float"}, + d2={"values": np.array([-1, 1]), "dtype": int}, + d3={ + "values": pd.DatetimeIndex(["2020-05-01", "2020-05-03"]), + "dtype": ["datetime64[ns]", "timedelta64[ns]"], + }, + d4={ + "values": pd.TimedeltaIndex([0, 1, 2, 3], "s"), + "dtype": ["datetime64[ns]", "timedelta64[ns]"], + }, + d5={"values": ["x", "y", "z"], "dtype": " Date: Tue, 8 Sep 2020 17:34:05 +0200 Subject: [PATCH 06/38] #23 pytests are now working. Changed one pytest to AttributeError. added exceptions in xr_check_coords. --- tests/test_utility.py | 6 ++-- weldx/utility.py | 82 +++++++++++++------------------------------ 2 files changed, 29 insertions(+), 59 deletions(-) diff --git a/tests/test_utility.py b/tests/test_utility.py index a9a25bdcf..22a67b8c7 100644 --- a/tests/test_utility.py +++ b/tests/test_utility.py @@ -412,10 +412,11 @@ def test_xf_fill_all(): (_dax_check, {"d1": {"dtype": ["float64", int]}}), (_dax_check, {"d2": {"dtype": ["float64", int]}}), (_dax_check, {"no_dim": {"optional": True, "dtype": float}}), - # (_dax_check, {"d5": {"dtype": str}}), # das wäre schöner als " xr.DataArray: From 7097a0604e74c4d5cbfb4b43809dfac0c759db02 Mon Sep 17 00:00:00 2001 From: Cagtay Fabry <43667554+CagtayFabry@users.noreply.github.com> Date: Fri, 11 Sep 2020 09:56:15 +0200 Subject: [PATCH 07/38] small test changes --- tests/test_utility.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_utility.py b/tests/test_utility.py index 22a67b8c7..6be9d6bde 100644 --- a/tests/test_utility.py +++ b/tests/test_utility.py @@ -412,7 +412,7 @@ def test_xf_fill_all(): (_dax_check, {"d1": {"dtype": ["float64", int]}}), (_dax_check, {"d2": {"dtype": ["float64", int]}}), (_dax_check, {"no_dim": {"optional": True, "dtype": float}}), - # (_dax_check, {"d5": {"dtype": str}}), # das wäre schöner als " Date: Mon, 28 Sep 2020 19:31:37 +0200 Subject: [PATCH 08/38] Added docstrings. Code cleanup. --- tests/test_utility.py | 10 ++++-- weldx/utility.py | 71 +++++++++++++++++++++++++++++++++++++------ 2 files changed, 68 insertions(+), 13 deletions(-) diff --git a/tests/test_utility.py b/tests/test_utility.py index 6be9d6bde..a169a62ef 100644 --- a/tests/test_utility.py +++ b/tests/test_utility.py @@ -412,11 +412,13 @@ def test_xf_fill_all(): (_dax_check, {"d1": {"dtype": ["float64", int]}}), (_dax_check, {"d2": {"dtype": ["float64", int]}}), (_dax_check, {"no_dim": {"optional": True, "dtype": float}}), - (_dax_check, {"d5": {"dtype": str}}), # das wäre schöner als " xr.DataArray: """Create an xarray 3d vector with correctly named dimensions and coordinates. From 8eaf067cddff2f0497fd46e6245d34dc657c3e45 Mon Sep 17 00:00:00 2001 From: Markus Nagel Date: Mon, 28 Sep 2020 19:38:15 +0200 Subject: [PATCH 09/38] Manually linebreaks due to wrong PyCharm settings. --- tests/test_utility.py | 3 ++- weldx/utility.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_utility.py b/tests/test_utility.py index a169a62ef..c73496bdd 100644 --- a/tests/test_utility.py +++ b/tests/test_utility.py @@ -431,7 +431,8 @@ def test_xr_check_coords(dax, ref_dict): ], ) def test_xr_check_coords_exception(dax, ref_dict, exception_type): - """Test of the validator function weldx.utility.xr_check_coords especially checking for exceptions. + """Test of the validator function weldx.utility.xr_check_coords especially + checking for exceptions. Validate the coordinates of an DataArray against different dictionarys.""" with pytest.raises(exception_type): diff --git a/weldx/utility.py b/weldx/utility.py index 34d4bf46c..2ddd03a23 100644 --- a/weldx/utility.py +++ b/weldx/utility.py @@ -563,7 +563,8 @@ def xr_interp_like( def xr_valid_key(dax: xr.DataArray, ref: dict): """Validate if the keys in ref are present in the DataArray. - This is a helper function for xr_check_coords. Throws an exception when the dimension is requested in dax and not optional. + This is a helper function for xr_check_coords. Throws an exception when the dimension is requested in dax and not + optional. Parameters ---------- From 36191a3d0bee9a978c85073db4042183890d9801 Mon Sep 17 00:00:00 2001 From: Markus Nagel Date: Mon, 28 Sep 2020 19:47:48 +0200 Subject: [PATCH 10/38] Changed PyCharm to softwrap at 89 characters. --- weldx/utility.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/weldx/utility.py b/weldx/utility.py index 2ddd03a23..899d3810e 100644 --- a/weldx/utility.py +++ b/weldx/utility.py @@ -563,8 +563,8 @@ def xr_interp_like( def xr_valid_key(dax: xr.DataArray, ref: dict): """Validate if the keys in ref are present in the DataArray. - This is a helper function for xr_check_coords. Throws an exception when the dimension is requested in dax and not - optional. + This is a helper function for xr_check_coords. Throws an exception when the + dimension is requested in dax and not optional. Parameters ---------- From 3bee7ebb11c23e9f7974da0b9276d983cf4c5c80 Mon Sep 17 00:00:00 2001 From: Markus Nagel Date: Mon, 28 Sep 2020 20:00:08 +0200 Subject: [PATCH 11/38] added a functionality: it is now allowed to test all numpy strings against 'str'. --- weldx/utility.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/weldx/utility.py b/weldx/utility.py index 899d3810e..1320c9e3f 100644 --- a/weldx/utility.py +++ b/weldx/utility.py @@ -665,9 +665,13 @@ def xr_check_coords(dax: xr.DataArray, ref: dict): f"Mismatch in the dtype of the DataArray and ref['{key}']" ) elif getattr(dax, key).dtype != np.dtype(ref[key]["dtype"]): - raise Exception( - f"Mismatch in the dtype of the DataArray and ref['{key}']" - ) + if not ( + np.issubdtype(getattr(dax, key).dtype, np.dtype(ref[key]["dtype"])) + and np.dtype(ref[key]["dtype"]) == str + ): + raise Exception( + f"Mismatch in the dtype of the DataArray and ref['{key}']" + ) return True From f328506e142f346dcdd88cf52c54a2e431285326 Mon Sep 17 00:00:00 2001 From: Markus Nagel Date: Mon, 28 Sep 2020 20:10:49 +0200 Subject: [PATCH 12/38] Added tests to check reachability of all lines of the code. Now all lines are tested. --- tests/test_utility.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_utility.py b/tests/test_utility.py index c73496bdd..ff69f3422 100644 --- a/tests/test_utility.py +++ b/tests/test_utility.py @@ -428,6 +428,8 @@ def test_xr_check_coords(dax, ref_dict): (_dax_check, {"d1": {"dtype": int}}, Exception), (_dax_check, {"d1": {"dtype": int, "optional": True}}, Exception), (_dax_check, {"no_dim": {"dtype": float}}, AttributeError), + (_dax_check, {"d5": {"values": ["x", "noty", "z"], "dtype": "str"}}, Exception), + (_dax_check, {"d1": {"dtype": [int, str, bool]}}, Exception), ], ) def test_xr_check_coords_exception(dax, ref_dict, exception_type): From 0b8dc046e8994a8053fcc5bcea41896ee565db9c Mon Sep 17 00:00:00 2001 From: Cagtay Fabry <43667554+CagtayFabry@users.noreply.github.com> Date: Tue, 29 Sep 2020 14:22:24 +0200 Subject: [PATCH 13/38] remove 23_test.py --- 23_test.py | 57 ------------------------------------------------------ 1 file changed, 57 deletions(-) delete mode 100644 23_test.py diff --git a/23_test.py b/23_test.py deleted file mode 100644 index 5057f9e6a..000000000 --- a/23_test.py +++ /dev/null @@ -1,57 +0,0 @@ -import numpy as np -import pandas as pd -import xarray as xr - -from weldx.utility import xr_check_coords, xr_check_dtype - -data = np.array([[0, 1], [2, 3]]) - -time_labels = ["2020-05-01", "2020-05-03"] -d1 = np.array([-1, 1]) -d2 = pd.DatetimeIndex(time_labels) -coords = {"d1": d1, "d2": d2, "time_labels": (["d2"], time_labels)} - -dax = xr.DataArray(data=data, dims=["d1", "d2"], coords=coords) - -dax.attrs = {"answer": 42} - -ref = { - "d1": {"values": np.array([-1, 1])}, - "d2": {"optional": True, "dtype": ["datetime64[ns]", "timedelta64[ns]"]}, -} - -ref2 = { - "d1": {"values": np.array([-1, 2])}, - "d2": {"optional": True, "dtype": ["int32"]}, -} - -ref_additional = { - "d1": {"values": np.array([-1, 1])}, - "d2": {"optional": True, "dtype": ["datetime64[ns]", "timedelta64[ns]"]}, - "d3": {"values": np.array([-1, 1])}, -} - -# testing -print("dtype check:") -xr_check_dtype(dax, ref) -print() -print("coords check:") -xr_check_coords(dax, ref) -print() -print("dtype check - wrong dtype:") -xr_check_dtype(dax, ref2) -print() -print("coords check - wrong coords:") -xr_check_coords(dax, ref2) -print() -print("dtype check - additional entry:") -try: - xr_check_dtype(dax, ref_additional) -except AttributeError as err: - print(err) -print() -print("coords check - additional entry:") -try: - xr_check_coords(dax, ref_additional) -except AttributeError as err: - print(err) From 35cd3983cc3713d383792e5836a2998ce6ea160e Mon Sep 17 00:00:00 2001 From: Cagtay Fabry <43667554+CagtayFabry@users.noreply.github.com> Date: Wed, 30 Sep 2020 09:49:18 +0200 Subject: [PATCH 14/38] add single entry list testcase for str --- tests/test_utility.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_utility.py b/tests/test_utility.py index dd9a1f772..de9f8615f 100644 --- a/tests/test_utility.py +++ b/tests/test_utility.py @@ -430,6 +430,7 @@ def test_xf_fill_all(): (_dax_check, {"d2": {"dtype": ["float64", int]}}), (_dax_check, {"no_dim": {"optional": True, "dtype": float}}), (_dax_check, {"d5": {"dtype": str}}), + (_dax_check, {"d5": {"dtype": [str]}}), ], ) def test_xr_check_coords(dax, ref_dict): From 6124f5d3ba3f0ff9f1c2f93f62b32e684ac19cb3 Mon Sep 17 00:00:00 2001 From: Markus Nagel Date: Wed, 30 Sep 2020 13:07:58 +0200 Subject: [PATCH 15/38] Added missing lines for str as subtype in a list. --- weldx/utility.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/weldx/utility.py b/weldx/utility.py index 35dc69bce..9f87861fe 100644 --- a/weldx/utility.py +++ b/weldx/utility.py @@ -662,9 +662,13 @@ def xr_check_coords(dax: xr.DataArray, ref: dict): if getattr(dax, key).dtype not in [ np.dtype(x) for x in ref[key]["dtype"] ]: - raise Exception( - f"Mismatch in the dtype of the DataArray and ref['{key}']" - ) + if not ( + str in [np.dtype(x) for x in ref[key]["dtype"]] + and np.issubdtype(getattr(dax, key).dtype, str) + ): + raise Exception( + f"Mismatch in the dtype of the DataArray and ref['{key}']" + ) elif getattr(dax, key).dtype != np.dtype(ref[key]["dtype"]): if not ( np.issubdtype(getattr(dax, key).dtype, np.dtype(ref[key]["dtype"])) From 6dbc693f9bf89e17943360987e4680dbe4e8efc9 Mon Sep 17 00:00:00 2001 From: Cagtay Fabry <43667554+CagtayFabry@users.noreply.github.com> Date: Wed, 30 Sep 2020 14:42:32 +0200 Subject: [PATCH 16/38] test numpydoc example syntax --- weldx/utility.py | 48 +++++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/weldx/utility.py b/weldx/utility.py index 9f87861fe..a22b82d9e 100644 --- a/weldx/utility.py +++ b/weldx/utility.py @@ -602,30 +602,32 @@ def xr_check_coords(dax: xr.DataArray, ref: dict): 'dtype': Any allowed data type from XArray is allowed here. - Example: + Examples: --------- - import numpy as np - import pandas as pd - import xarray as xr - - dax = xr.DataArray( - data=np.ones((3,2,3)), - dims=["d1", "d2", "d3"], - coords={ - "d1": np.array([-1, 0, 2], dtype=int), - "d2": pd.DatetimeIndex(["2020-05-01", "2020-05-03"]), - "d3": ["x", "y", "z"], - } - ) - ref = dict( - d1={"optional": True, "values": np.array([-1, 0, 2], dtype=int)}, - d2={ - "values": pd.DatetimeIndex(["2020-05-01", "2020-05-03"]), - "dtype": ["datetime64[ns]", "timedelta64[ns]"], - }, - d3={"values": ["x", "y", "z"], "dtype": ">> import pandas as pd + ... import xarray as xr + ... import weldx as wx + ... + ... dax = xr.DataArray( + ... data=np.ones((3, 2, 3)), + ... dims=["d1", "d2", "d3"], + ... coords={ + ... "d1": np.array([-1, 0, 2], dtype=int), + ... "d2": pd.DatetimeIndex(["2020-05-01", "2020-05-03"]), + ... "d3": ["x", "y", "z"], + ... } + ... ) + ... ref = dict( + ... d1={"optional": True, "values": np.array([-1, 0, 2], dtype=int)}, + ... d2={ + ... "values": pd.DatetimeIndex(["2020-05-01", "2020-05-03"]), + ... "dtype": ["datetime64[ns]", "timedelta64[ns]"], + ... }, + ... d3={"values": ["x", "y", "z"], "dtype": " Date: Wed, 30 Sep 2020 15:01:22 +0200 Subject: [PATCH 17/38] fix example syntax in xr_check_coords --- weldx/utility.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/weldx/utility.py b/weldx/utility.py index e3d76f555..ec28160e4 100644 --- a/weldx/utility.py +++ b/weldx/utility.py @@ -603,13 +603,12 @@ def xr_check_coords(dax: xr.DataArray, ref: dict): 'dtype': Any allowed data type from XArray is allowed here. - Examples: - --------- + Examples + -------- >>> import pandas as pd - ... import xarray as xr - ... import weldx as wx - ... - ... dax = xr.DataArray( + >>> import xarray as xr + >>> import weldx as wx + >>> dax = xr.DataArray( ... data=np.ones((3, 2, 3)), ... dims=["d1", "d2", "d3"], ... coords={ @@ -618,7 +617,7 @@ def xr_check_coords(dax: xr.DataArray, ref: dict): ... "d3": ["x", "y", "z"], ... } ... ) - ... ref = dict( + >>> ref = dict( ... d1={"optional": True, "values": np.array([-1, 0, 2], dtype=int)}, ... d2={ ... "values": pd.DatetimeIndex(["2020-05-01", "2020-05-03"]), @@ -626,7 +625,7 @@ def xr_check_coords(dax: xr.DataArray, ref: dict): ... }, ... d3={"values": ["x", "y", "z"], "dtype": ">> wx.utility.xr_check_coords(dax, ref) True From 83c9ec535930821b5972d0fa9d539893e5b82679 Mon Sep 17 00:00:00 2001 From: Cagtay Fabry <43667554+CagtayFabry@users.noreply.github.com> Date: Wed, 30 Sep 2020 15:43:27 +0200 Subject: [PATCH 18/38] update docstring syntax --- weldx/utility.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/weldx/utility.py b/weldx/utility.py index ec28160e4..1ea9d792f 100644 --- a/weldx/utility.py +++ b/weldx/utility.py @@ -596,12 +596,15 @@ def xr_check_coords(dax: xr.DataArray, ref: dict): The reference dictionary should have the dimensions as keys and those contain dictionaries with the following keywords (all optional): - 'optional': - boolean, default False - if True, the dimension has to be in the DataArray dax - 'values': - Any allowed value from XArray is allowed here. (numpy.ndarray) - 'dtype': - Any allowed data type from XArray is allowed here. + + - ``values``: Specify exact coordinate values to match. + + - ``dtype``: Ensure coordinate dtype matches at least one of the given dtypes. + + - ``optional``: boolean, default ``False`` - + if ``True``, the dimension has to be in the DataArray dax + + Examples -------- @@ -628,8 +631,6 @@ def xr_check_coords(dax: xr.DataArray, ref: dict): >>> wx.utility.xr_check_coords(dax, ref) True - - Parameters ---------- dax: From 227270b1c21241738020f2a95d19bc61bf46a985 Mon Sep 17 00:00:00 2001 From: Cagtay Fabry <43667554+CagtayFabry@users.noreply.github.com> Date: Wed, 30 Sep 2020 15:52:21 +0200 Subject: [PATCH 19/38] update docstring syntax --- weldx/utility.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/weldx/utility.py b/weldx/utility.py index 1ea9d792f..213afe959 100644 --- a/weldx/utility.py +++ b/weldx/utility.py @@ -597,14 +597,14 @@ def xr_check_coords(dax: xr.DataArray, ref: dict): The reference dictionary should have the dimensions as keys and those contain dictionaries with the following keywords (all optional): - - ``values``: Specify exact coordinate values to match. - - - ``dtype``: Ensure coordinate dtype matches at least one of the given dtypes. - - - ``optional``: boolean, default ``False`` - - if ``True``, the dimension has to be in the DataArray dax + ``values`` + Specify exact coordinate values to match. + ``dtype`` : str or type + Ensure coordinate dtype matches at least one of the given dtypes. + ``optional`` : boolean + default ``False`` - if ``True``, the dimension has to be in the DataArray dax Examples -------- From 2df5dcaa38b1acb26803766b3de25aeb66a836ad Mon Sep 17 00:00:00 2001 From: Cagtay Fabry <43667554+CagtayFabry@users.noreply.github.com> Date: Wed, 30 Sep 2020 16:04:41 +0200 Subject: [PATCH 20/38] move example code to end of docstring and test xarray format in LocalCoordinateSystem --- weldx/transformations.py | 17 +++++++++++++++-- weldx/utility.py | 26 +++++++++++++------------- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/weldx/transformations.py b/weldx/transformations.py index 68a0dd07f..821b10e03 100644 --- a/weldx/transformations.py +++ b/weldx/transformations.py @@ -629,8 +629,15 @@ def _build_orientation( """ if isinstance(orientation, xr.DataArray): + ut.xr_check_coords( + orientation, + dict( + c={"values": ["x", "y", "z"]}, + v={"values": [0, 1, 2]}, + time={"dtype": "timedelta64[ns]", "optional": True}, + ), + ) return orientation - # TODO: Test if xarray has correct format time_orientation = None if isinstance(orientation, Rot): @@ -660,8 +667,14 @@ def _build_coordinates(coordinates, time: pd.DatetimeIndex = None): """ if isinstance(coordinates, xr.DataArray): + ut.xr_check_coords( + coordinates, + dict( + c={"values": ["x", "y", "z"]}, + time={"dtype": "timedelta64[ns]", "optional": True}, + ), + ) return coordinates - # TODO: Test if xarray has correct format time_coordinates = None if not isinstance(coordinates, (np.ndarray, pint.Quantity)): diff --git a/weldx/utility.py b/weldx/utility.py index 213afe959..76a0e9c10 100644 --- a/weldx/utility.py +++ b/weldx/utility.py @@ -591,7 +591,7 @@ def xr_valid_key(dax: xr.DataArray, ref: dict): raise AttributeError(f"Data array has no attribute '{key}'.") -def xr_check_coords(dax: xr.DataArray, ref: dict): +def xr_check_coords(dax: xr.DataArray, ref: dict) -> bool: """Validate the coordinates of the DataArray against a reference dictionary. The reference dictionary should have the dimensions as keys and those contain @@ -606,6 +606,18 @@ def xr_check_coords(dax: xr.DataArray, ref: dict): ``optional`` : boolean default ``False`` - if ``True``, the dimension has to be in the DataArray dax + Parameters + ---------- + dax: + xarray object which should be validated + ref: + reference dictionary + + Returns + ------- + bool + True, if the test was a success, else throws an exception + Examples -------- >>> import pandas as pd @@ -630,18 +642,6 @@ def xr_check_coords(dax: xr.DataArray, ref: dict): ... ) >>> wx.utility.xr_check_coords(dax, ref) True - - Parameters - ---------- - dax: - xarray object which should be validated - ref: - reference dictionary - - Returns - ------- - bool - True, if the test was a success, else throws an exception """ # check if the keys in ref are also in dax From 9dd59bfa2a0c00fc600cd73b78974f51dbaba360 Mon Sep 17 00:00:00 2001 From: Cagtay Fabry <43667554+CagtayFabry@users.noreply.github.com> Date: Wed, 30 Sep 2020 17:19:31 +0200 Subject: [PATCH 21/38] move LCS xarray coord checks to construction check stage --- weldx/transformations.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/weldx/transformations.py b/weldx/transformations.py index 821b10e03..f140a3c9b 100644 --- a/weldx/transformations.py +++ b/weldx/transformations.py @@ -408,6 +408,23 @@ def __init__( coordinates = self._build_coordinates(coordinates, time) if construction_checks: + ut.xr_check_coords( + coordinates, + dict( + c={"values": ["x", "y", "z"]}, + time={"dtype": "timedelta64[ns]", "optional": True}, + ), + ) + + ut.xr_check_coords( + orientation, + dict( + c={"values": ["x", "y", "z"]}, + v={"values": [0, 1, 2]}, + time={"dtype": "timedelta64[ns]", "optional": True}, + ), + ) + orientation = xr.apply_ufunc( normalize, orientation, @@ -629,14 +646,6 @@ def _build_orientation( """ if isinstance(orientation, xr.DataArray): - ut.xr_check_coords( - orientation, - dict( - c={"values": ["x", "y", "z"]}, - v={"values": [0, 1, 2]}, - time={"dtype": "timedelta64[ns]", "optional": True}, - ), - ) return orientation time_orientation = None @@ -667,13 +676,6 @@ def _build_coordinates(coordinates, time: pd.DatetimeIndex = None): """ if isinstance(coordinates, xr.DataArray): - ut.xr_check_coords( - coordinates, - dict( - c={"values": ["x", "y", "z"]}, - time={"dtype": "timedelta64[ns]", "optional": True}, - ), - ) return coordinates time_coordinates = None From d7b57543cbf3d97e658051e13da9d293d3fcba31 Mon Sep 17 00:00:00 2001 From: Cagtay Fabry <43667554+CagtayFabry@users.noreply.github.com> Date: Wed, 30 Sep 2020 17:35:44 +0200 Subject: [PATCH 22/38] fix codacy issue --- weldx/utility.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weldx/utility.py b/weldx/utility.py index 76a0e9c10..538ffa31f 100644 --- a/weldx/utility.py +++ b/weldx/utility.py @@ -661,7 +661,7 @@ def xr_check_coords(dax: xr.DataArray, ref: dict) -> bool: # only if the key "dtype" is given do the validation if "dtype" in ref[key]: - if type(ref[key]["dtype"]) is list: + if isinstance(ref[key]["dtype"], list): if getattr(dax, key).dtype not in [ np.dtype(x) for x in ref[key]["dtype"] ]: From 5f6db5a1684dc10907c9b8c22c57dc7310746c83 Mon Sep 17 00:00:00 2001 From: Cagtay Fabry <43667554+CagtayFabry@users.noreply.github.com> Date: Thu, 1 Oct 2020 20:00:24 +0200 Subject: [PATCH 23/38] fix pydocstring issues --- tests/test_utility.py | 9 +++++---- weldx/utility.py | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/test_utility.py b/tests/test_utility.py index de9f8615f..03d6424e7 100644 --- a/tests/test_utility.py +++ b/tests/test_utility.py @@ -436,7 +436,8 @@ def test_xf_fill_all(): def test_xr_check_coords(dax, ref_dict): """Test of the validator function weldx.utility.xr_check_coords. - Validate the coordinates of an DataArray against different dictionarys.""" + Validate the coordinates of an DataArray against different dictionarys. + """ assert ut.xr_check_coords(dax, ref_dict) @@ -451,9 +452,9 @@ def test_xr_check_coords(dax, ref_dict): ], ) def test_xr_check_coords_exception(dax, ref_dict, exception_type): - """Test of the validator function weldx.utility.xr_check_coords especially - checking for exceptions. + """Test weldx.utility.xr_check_coords exceptions. - Validate the coordinates of an DataArray against different dictionarys.""" + Validate the coordinates of an DataArray against different dictionarys. + """ with pytest.raises(exception_type): ut.xr_check_coords(dax, ref_dict) diff --git a/weldx/utility.py b/weldx/utility.py index 538ffa31f..5735db902 100644 --- a/weldx/utility.py +++ b/weldx/utility.py @@ -579,8 +579,8 @@ def xr_valid_key(dax: xr.DataArray, ref: dict): ------- None Does not have a return value. Throws an exception if - """ + """ for key in ref: # if optional is set to true, then dax does not need to have this key if "optional" in ref[key]: @@ -642,8 +642,8 @@ def xr_check_coords(dax: xr.DataArray, ref: dict) -> bool: ... ) >>> wx.utility.xr_check_coords(dax, ref) True - """ + """ # check if the keys in ref are also in dax xr_valid_key(dax, ref) From 01539b69047bd152099459ae318e5753f9a5c811 Mon Sep 17 00:00:00 2001 From: Markus Nagel Date: Wed, 7 Oct 2020 12:15:54 +0200 Subject: [PATCH 24/38] added support for general 'timedelta64' and 'datetime64' dtype. --- tests/test_utility.py | 3 +++ weldx/utility.py | 22 +++++++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/test_utility.py b/tests/test_utility.py index 03d6424e7..214b7e8aa 100644 --- a/tests/test_utility.py +++ b/tests/test_utility.py @@ -431,6 +431,8 @@ def test_xf_fill_all(): (_dax_check, {"no_dim": {"optional": True, "dtype": float}}), (_dax_check, {"d5": {"dtype": str}}), (_dax_check, {"d5": {"dtype": [str]}}), + (_dax_check, {"d4": {"dtype": "timedelta64"}}), + (_dax_check, {"d3": {"dtype": ["datetime64", "timedelta64"]}}), ], ) def test_xr_check_coords(dax, ref_dict): @@ -449,6 +451,7 @@ def test_xr_check_coords(dax, ref_dict): (_dax_check, {"no_dim": {"dtype": float}}, AttributeError), (_dax_check, {"d5": {"values": ["x", "noty", "z"], "dtype": "str"}}, Exception), (_dax_check, {"d1": {"dtype": [int, str, bool]}}, Exception), + (_dax_check, {"d4": {"dtype": "datetime64"}}, Exception), ], ) def test_xr_check_coords_exception(dax, ref_dict, exception_type): diff --git a/weldx/utility.py b/weldx/utility.py index 5735db902..febf23ddd 100644 --- a/weldx/utility.py +++ b/weldx/utility.py @@ -662,17 +662,37 @@ def xr_check_coords(dax: xr.DataArray, ref: dict) -> bool: # only if the key "dtype" is given do the validation if "dtype" in ref[key]: if isinstance(ref[key]["dtype"], list): + if [ + x + for x in [x for x in ref[key]["dtype"] if isinstance(x, str)] + if "timedelta64" in x or "datetime64" in x + ]: + if np.issubdtype( + getattr(dax, key).dtype, np.dtype("timedelta64") + ) or np.issubdtype(getattr(dax, key).dtype, np.dtype("datetime64")): + continue + if getattr(dax, key).dtype not in [ np.dtype(x) for x in ref[key]["dtype"] ]: if not ( str in [np.dtype(x) for x in ref[key]["dtype"]] - and np.issubdtype(getattr(dax, key).dtype, str) + and np.issubdtype(getattr(dax, key).dtype, np.str_) ): raise Exception( f"Mismatch in the dtype of the DataArray and ref['{key}']" ) elif getattr(dax, key).dtype != np.dtype(ref[key]["dtype"]): + if isinstance(ref[key]["dtype"], str): + if ( + "timedelta64" in ref[key]["dtype"] + or "datetime64" in ref[key]["dtype"] + ): + if np.issubdtype( + getattr(dax, key).dtype, np.dtype(ref[key]["dtype"]) + ): + continue + if not ( np.issubdtype(getattr(dax, key).dtype, np.dtype(ref[key]["dtype"])) and np.dtype(ref[key]["dtype"]) == str From 3b169e81d74713d188ec7e87b14049f4fb13f0d6 Mon Sep 17 00:00:00 2001 From: Markus Nagel Date: Wed, 7 Oct 2020 13:49:55 +0200 Subject: [PATCH 25/38] change docstring in test functions. --- tests/test_utility.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/test_utility.py b/tests/test_utility.py index 214b7e8aa..f60543a18 100644 --- a/tests/test_utility.py +++ b/tests/test_utility.py @@ -436,10 +436,7 @@ def test_xf_fill_all(): ], ) def test_xr_check_coords(dax, ref_dict): - """Test of the validator function weldx.utility.xr_check_coords. - - Validate the coordinates of an DataArray against different dictionarys. - """ + """Test weldx.utility.xr_check_coords function.""" assert ut.xr_check_coords(dax, ref_dict) @@ -455,9 +452,6 @@ def test_xr_check_coords(dax, ref_dict): ], ) def test_xr_check_coords_exception(dax, ref_dict, exception_type): - """Test weldx.utility.xr_check_coords exceptions. - - Validate the coordinates of an DataArray against different dictionarys. - """ + """Test weldx.utility.xr_check_coords function.""" with pytest.raises(exception_type): ut.xr_check_coords(dax, ref_dict) From 203a6e954fcfab3db472855296725b332ffbe977 Mon Sep 17 00:00:00 2001 From: Markus Nagel Date: Wed, 7 Oct 2020 15:29:40 +0200 Subject: [PATCH 26/38] now we only llok at the coordinates when procesing the validation. --- weldx/utility.py | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/weldx/utility.py b/weldx/utility.py index febf23ddd..0a9a2d9a4 100644 --- a/weldx/utility.py +++ b/weldx/utility.py @@ -562,7 +562,7 @@ def xr_interp_like( return result -def xr_valid_key(dax: xr.DataArray, ref: dict): +def _xr_valid_key(coords, ref: dict): """Validate if the keys in ref are present in the DataArray. This is a helper function for xr_check_coords. Throws an exception when the @@ -570,8 +570,8 @@ def xr_valid_key(dax: xr.DataArray, ref: dict): Parameters ---------- - dax: - xarray object + coords: + xarray coords ref: reference dictionary @@ -586,11 +586,18 @@ def xr_valid_key(dax: xr.DataArray, ref: dict): if "optional" in ref[key]: if ref[key]["optional"]: continue - if not hasattr(dax, key): - # Attributes not found in dax + if key not in coords: + # Attributes not found in coords raise AttributeError(f"Data array has no attribute '{key}'.") +def _check_dtype(var_dtype, ref_dtype) -> bool: + """Checks if dtype mtaches a reference dtype. + + Returns True or False + """ + + def xr_check_coords(dax: xr.DataArray, ref: dict) -> bool: """Validate the coordinates of the DataArray against a reference dictionary. @@ -644,19 +651,25 @@ def xr_check_coords(dax: xr.DataArray, ref: dict) -> bool: True """ + # only process the coords of the xarray + if isinstance(dax, (xr.DataArray, xr.Dataset)): + coords = dax.coords + else: + coords = dax + # check if the keys in ref are also in dax - xr_valid_key(dax, ref) + _xr_valid_key(coords, ref) for key in ref: # check if the optional key is set to true if "optional" in ref[key]: - if ref[key]["optional"] and not hasattr(dax, key): + if ref[key]["optional"] and key not in coords: # skip this key - it is not in dax continue # only if the key "values" is given do the validation if "values" in ref[key]: - if not (getattr(dax, key).values == ref[key]["values"]).all(): + if not (coords[key].values == ref[key]["values"]).all(): raise Exception(f"Value mismatch in DataArray and ref['{key}']") # only if the key "dtype" is given do the validation @@ -668,21 +681,19 @@ def xr_check_coords(dax: xr.DataArray, ref: dict) -> bool: if "timedelta64" in x or "datetime64" in x ]: if np.issubdtype( - getattr(dax, key).dtype, np.dtype("timedelta64") - ) or np.issubdtype(getattr(dax, key).dtype, np.dtype("datetime64")): + coords[key].dtype, np.dtype("timedelta64") + ) or np.issubdtype(coords[key].dtype, np.dtype("datetime64")): continue - if getattr(dax, key).dtype not in [ - np.dtype(x) for x in ref[key]["dtype"] - ]: + if coords[key].dtype not in [np.dtype(x) for x in ref[key]["dtype"]]: if not ( str in [np.dtype(x) for x in ref[key]["dtype"]] - and np.issubdtype(getattr(dax, key).dtype, np.str_) + and np.issubdtype(coords[key].dtype, np.str_) ): raise Exception( f"Mismatch in the dtype of the DataArray and ref['{key}']" ) - elif getattr(dax, key).dtype != np.dtype(ref[key]["dtype"]): + elif coords[key].dtype != np.dtype(ref[key]["dtype"]): if isinstance(ref[key]["dtype"], str): if ( "timedelta64" in ref[key]["dtype"] From 8e5558861c8e856f523bf899864cb13dc98eaa18 Mon Sep 17 00:00:00 2001 From: Markus Nagel Date: Wed, 7 Oct 2020 15:56:36 +0200 Subject: [PATCH 27/38] added helper function '_check_dtype' to simplify the code. --- weldx/utility.py | 56 ++++++++++++++++++++---------------------------- 1 file changed, 23 insertions(+), 33 deletions(-) diff --git a/weldx/utility.py b/weldx/utility.py index 0a9a2d9a4..7e3997459 100644 --- a/weldx/utility.py +++ b/weldx/utility.py @@ -594,8 +594,23 @@ def _xr_valid_key(coords, ref: dict): def _check_dtype(var_dtype, ref_dtype) -> bool: """Checks if dtype mtaches a reference dtype. - Returns True or False + Returns True or raises exception. """ + if var_dtype != np.dtype(ref_dtype): + if isinstance(ref_dtype, str): + if ( + "timedelta64" in ref_dtype + or "datetime64" in ref_dtype + and np.issubdtype(var_dtype, np.dtype(ref_dtype)) + ): + return True + + if not ( + np.issubdtype(var_dtype, np.dtype(ref_dtype)) and np.dtype(ref_dtype) == str + ): + return False + + return True def xr_check_coords(dax: xr.DataArray, ref: dict) -> bool: @@ -675,42 +690,17 @@ def xr_check_coords(dax: xr.DataArray, ref: dict) -> bool: # only if the key "dtype" is given do the validation if "dtype" in ref[key]: if isinstance(ref[key]["dtype"], list): - if [ - x - for x in [x for x in ref[key]["dtype"] if isinstance(x, str)] - if "timedelta64" in x or "datetime64" in x - ]: - if np.issubdtype( - coords[key].dtype, np.dtype("timedelta64") - ) or np.issubdtype(coords[key].dtype, np.dtype("datetime64")): - continue - - if coords[key].dtype not in [np.dtype(x) for x in ref[key]["dtype"]]: - if not ( - str in [np.dtype(x) for x in ref[key]["dtype"]] - and np.issubdtype(coords[key].dtype, np.str_) - ): - raise Exception( - f"Mismatch in the dtype of the DataArray and ref['{key}']" - ) - elif coords[key].dtype != np.dtype(ref[key]["dtype"]): - if isinstance(ref[key]["dtype"], str): - if ( - "timedelta64" in ref[key]["dtype"] - or "datetime64" in ref[key]["dtype"] - ): - if np.issubdtype( - getattr(dax, key).dtype, np.dtype(ref[key]["dtype"]) - ): - continue - - if not ( - np.issubdtype(getattr(dax, key).dtype, np.dtype(ref[key]["dtype"])) - and np.dtype(ref[key]["dtype"]) == str + if not any( + _check_dtype(coords[key].dtype, var_dtype) + for var_dtype in ref[key]["dtype"] ): raise Exception( f"Mismatch in the dtype of the DataArray and ref['{key}']" ) + elif not _check_dtype(coords[key].dtype, ref[key]["dtype"]): + raise Exception( + f"Mismatch in the dtype of the DataArray and ref['{key}']" + ) return True From 9cbe51a4d4330078dc4b5f677f4fce7da3d7683e Mon Sep 17 00:00:00 2001 From: Markus Nagel Date: Wed, 7 Oct 2020 16:03:51 +0200 Subject: [PATCH 28/38] changed the Error types to the corresponding Error types. TypeError and ValueError. --- tests/test_utility.py | 15 ++++++++++----- weldx/utility.py | 8 ++++---- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/tests/test_utility.py b/tests/test_utility.py index f60543a18..6373460f5 100644 --- a/tests/test_utility.py +++ b/tests/test_utility.py @@ -426,6 +426,7 @@ def test_xf_fill_all(): "dax, ref_dict", [ (_dax_check, _dax_ref), + (_dax_check.coords, _dax_ref), (_dax_check, {"d1": {"dtype": ["float64", int]}}), (_dax_check, {"d2": {"dtype": ["float64", int]}}), (_dax_check, {"no_dim": {"optional": True, "dtype": float}}), @@ -443,12 +444,16 @@ def test_xr_check_coords(dax, ref_dict): @pytest.mark.parametrize( "dax, ref_dict, exception_type", [ - (_dax_check, {"d1": {"dtype": int}}, Exception), - (_dax_check, {"d1": {"dtype": int, "optional": True}}, Exception), + (_dax_check, {"d1": {"dtype": int}}, TypeError), + (_dax_check, {"d1": {"dtype": int, "optional": True}}, TypeError), (_dax_check, {"no_dim": {"dtype": float}}, AttributeError), - (_dax_check, {"d5": {"values": ["x", "noty", "z"], "dtype": "str"}}, Exception), - (_dax_check, {"d1": {"dtype": [int, str, bool]}}, Exception), - (_dax_check, {"d4": {"dtype": "datetime64"}}, Exception), + ( + _dax_check, + {"d5": {"values": ["x", "noty", "z"], "dtype": "str"}}, + ValueError, + ), + (_dax_check, {"d1": {"dtype": [int, str, bool]}}, TypeError), + (_dax_check, {"d4": {"dtype": "datetime64"}}, TypeError), ], ) def test_xr_check_coords_exception(dax, ref_dict, exception_type): diff --git a/weldx/utility.py b/weldx/utility.py index 7e3997459..bddf46bb0 100644 --- a/weldx/utility.py +++ b/weldx/utility.py @@ -594,7 +594,7 @@ def _xr_valid_key(coords, ref: dict): def _check_dtype(var_dtype, ref_dtype) -> bool: """Checks if dtype mtaches a reference dtype. - Returns True or raises exception. + Returns True or False. """ if var_dtype != np.dtype(ref_dtype): if isinstance(ref_dtype, str): @@ -685,7 +685,7 @@ def xr_check_coords(dax: xr.DataArray, ref: dict) -> bool: # only if the key "values" is given do the validation if "values" in ref[key]: if not (coords[key].values == ref[key]["values"]).all(): - raise Exception(f"Value mismatch in DataArray and ref['{key}']") + raise ValueError(f"Value mismatch in DataArray and ref['{key}']") # only if the key "dtype" is given do the validation if "dtype" in ref[key]: @@ -694,11 +694,11 @@ def xr_check_coords(dax: xr.DataArray, ref: dict) -> bool: _check_dtype(coords[key].dtype, var_dtype) for var_dtype in ref[key]["dtype"] ): - raise Exception( + raise TypeError( f"Mismatch in the dtype of the DataArray and ref['{key}']" ) elif not _check_dtype(coords[key].dtype, ref[key]["dtype"]): - raise Exception( + raise TypeError( f"Mismatch in the dtype of the DataArray and ref['{key}']" ) From 6770c33030fbab953f7f2acaf1400a40c8ca01cc Mon Sep 17 00:00:00 2001 From: MNagel Date: Wed, 7 Oct 2020 16:24:53 +0200 Subject: [PATCH 29/38] Update weldx/utility.py suggestion accepted! Co-authored-by: Cagtay Fabry <43667554+CagtayFabry@users.noreply.github.com> --- weldx/utility.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/weldx/utility.py b/weldx/utility.py index bddf46bb0..3f0412110 100644 --- a/weldx/utility.py +++ b/weldx/utility.py @@ -689,15 +689,12 @@ def xr_check_coords(dax: xr.DataArray, ref: dict) -> bool: # only if the key "dtype" is given do the validation if "dtype" in ref[key]: - if isinstance(ref[key]["dtype"], list): - if not any( - _check_dtype(coords[key].dtype, var_dtype) - for var_dtype in ref[key]["dtype"] - ): - raise TypeError( - f"Mismatch in the dtype of the DataArray and ref['{key}']" - ) - elif not _check_dtype(coords[key].dtype, ref[key]["dtype"]): + dtype_list = ref[key]["dtype"] + if not isinstance(dtype_list, list): + dtype_list = [dtype_list] + if not any( + _check_dtype(coords[key].dtype, var_dtype) for var_dtype in dtype_list + ): raise TypeError( f"Mismatch in the dtype of the DataArray and ref['{key}']" ) From c10d4fb29bb4bc360d414b0624f7aad367489b1d Mon Sep 17 00:00:00 2001 From: Markus Nagel Date: Wed, 7 Oct 2020 16:27:22 +0200 Subject: [PATCH 30/38] typo change --- weldx/utility.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weldx/utility.py b/weldx/utility.py index 3f0412110..c4910265e 100644 --- a/weldx/utility.py +++ b/weldx/utility.py @@ -592,7 +592,7 @@ def _xr_valid_key(coords, ref: dict): def _check_dtype(var_dtype, ref_dtype) -> bool: - """Checks if dtype mtaches a reference dtype. + """Checks if dtype matches a reference dtype. Returns True or False. """ From 94e20bcbb39d9a623463b5bda2b59ba11b8680ec Mon Sep 17 00:00:00 2001 From: Cagtay Fabry <43667554+CagtayFabry@users.noreply.github.com> Date: Wed, 7 Oct 2020 16:40:58 +0200 Subject: [PATCH 31/38] pydocstyle cleanup --- weldx/utility.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/weldx/utility.py b/weldx/utility.py index c4910265e..f4ff8d3ea 100644 --- a/weldx/utility.py +++ b/weldx/utility.py @@ -592,9 +592,20 @@ def _xr_valid_key(coords, ref: dict): def _check_dtype(var_dtype, ref_dtype) -> bool: - """Checks if dtype matches a reference dtype. + """Check if dtype matches a reference dtype (or is subdtype). + + Parameters + ---------- + var_dtype: + A numpy-dtype to test against. + ref_dtype + Python type or string description + + Returns + ------- + bool + True if dtypes matches. - Returns True or False. """ if var_dtype != np.dtype(ref_dtype): if isinstance(ref_dtype, str): From 61cc841c2f04865abd05081f945d07ee12ca6ac9 Mon Sep 17 00:00:00 2001 From: Cagtay Fabry <43667554+CagtayFabry@users.noreply.github.com> Date: Thu, 8 Oct 2020 08:39:47 +0200 Subject: [PATCH 32/38] update CHANGELOG.md --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 425757a80..8485728e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,10 +28,11 @@ - add basic schema layout and `GmawProcess` class for arc welding process implementation [#104] - add example notebook and documentation for arc welding process [#104] - fix propagating the `name` attribute when reading an ndarray `TimeSeries` object back from ASDF files [#104] -- fix `pint` regression in `TimeSeries` when mixing integer and float values +- fix `pint` regression in `TimeSeries` when mixing integer and float values [#121] - add `pint` compatibility to some `geometry` classes (**experimental**) - when passing quantities to constructors (and some functions), values get converted to default unit `mm` and passed on as magnitude - old behavior is preserved +- add `weldx.utility.xr_check_coords` function to check coordinates of xarray object against dtype and value restrictions [#125] ## 0.2.0 (30.07.2020) From 4a8e8b4829bae9959f5422d1f6820a1a4375a05e Mon Sep 17 00:00:00 2001 From: Cagtay Fabry <43667554+CagtayFabry@users.noreply.github.com> Date: Thu, 8 Oct 2020 10:16:02 +0200 Subject: [PATCH 33/38] removed _xr_valid_key and changed exception type --- tests/test_utility.py | 2 +- weldx/utility.py | 36 ++++-------------------------------- 2 files changed, 5 insertions(+), 33 deletions(-) diff --git a/tests/test_utility.py b/tests/test_utility.py index 6373460f5..684f1e71c 100644 --- a/tests/test_utility.py +++ b/tests/test_utility.py @@ -446,7 +446,7 @@ def test_xr_check_coords(dax, ref_dict): [ (_dax_check, {"d1": {"dtype": int}}, TypeError), (_dax_check, {"d1": {"dtype": int, "optional": True}}, TypeError), - (_dax_check, {"no_dim": {"dtype": float}}, AttributeError), + (_dax_check, {"no_dim": {"dtype": float}}, KeyError), ( _dax_check, {"d5": {"values": ["x", "noty", "z"], "dtype": "str"}}, diff --git a/weldx/utility.py b/weldx/utility.py index f4ff8d3ea..88bd00415 100644 --- a/weldx/utility.py +++ b/weldx/utility.py @@ -562,35 +562,6 @@ def xr_interp_like( return result -def _xr_valid_key(coords, ref: dict): - """Validate if the keys in ref are present in the DataArray. - - This is a helper function for xr_check_coords. Throws an exception when the - dimension is requested in dax and not optional. - - Parameters - ---------- - coords: - xarray coords - ref: - reference dictionary - - Returns - ------- - None - Does not have a return value. Throws an exception if - - """ - for key in ref: - # if optional is set to true, then dax does not need to have this key - if "optional" in ref[key]: - if ref[key]["optional"]: - continue - if key not in coords: - # Attributes not found in coords - raise AttributeError(f"Data array has no attribute '{key}'.") - - def _check_dtype(var_dtype, ref_dtype) -> bool: """Check if dtype matches a reference dtype (or is subdtype). @@ -683,9 +654,6 @@ def xr_check_coords(dax: xr.DataArray, ref: dict) -> bool: else: coords = dax - # check if the keys in ref are also in dax - _xr_valid_key(coords, ref) - for key in ref: # check if the optional key is set to true if "optional" in ref[key]: @@ -693,6 +661,10 @@ def xr_check_coords(dax: xr.DataArray, ref: dict) -> bool: # skip this key - it is not in dax continue + if key not in coords: + # Attributes not found in coords + raise KeyError(f"Could not find required coordinate '{key}'.") + # only if the key "values" is given do the validation if "values" in ref[key]: if not (coords[key].values == ref[key]["values"]).all(): From 5ee49cbd7795c48c3941b7408c5e306791882dff Mon Sep 17 00:00:00 2001 From: Cagtay Fabry <43667554+CagtayFabry@users.noreply.github.com> Date: Thu, 8 Oct 2020 10:20:33 +0200 Subject: [PATCH 34/38] change loop signature --- weldx/utility.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/weldx/utility.py b/weldx/utility.py index 88bd00415..9afc74dc2 100644 --- a/weldx/utility.py +++ b/weldx/utility.py @@ -654,10 +654,10 @@ def xr_check_coords(dax: xr.DataArray, ref: dict) -> bool: else: coords = dax - for key in ref: + for key, check in ref.items(): # check if the optional key is set to true - if "optional" in ref[key]: - if ref[key]["optional"] and key not in coords: + if "optional" in check: + if check["optional"] and key not in coords: # skip this key - it is not in dax continue @@ -666,13 +666,13 @@ def xr_check_coords(dax: xr.DataArray, ref: dict) -> bool: raise KeyError(f"Could not find required coordinate '{key}'.") # only if the key "values" is given do the validation - if "values" in ref[key]: - if not (coords[key].values == ref[key]["values"]).all(): + if "values" in check: + if not (coords[key].values == check["values"]).all(): raise ValueError(f"Value mismatch in DataArray and ref['{key}']") # only if the key "dtype" is given do the validation - if "dtype" in ref[key]: - dtype_list = ref[key]["dtype"] + if "dtype" in check: + dtype_list = check["dtype"] if not isinstance(dtype_list, list): dtype_list = [dtype_list] if not any( From 45b09bcd8d57080449f4f18ecc06deb4d6507807 Mon Sep 17 00:00:00 2001 From: Cagtay Fabry <43667554+CagtayFabry@users.noreply.github.com> Date: Thu, 8 Oct 2020 12:35:53 +0200 Subject: [PATCH 35/38] allow different timedelta dimensions --- weldx/transformations.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/weldx/transformations.py b/weldx/transformations.py index a5955cc0a..e47f928f4 100644 --- a/weldx/transformations.py +++ b/weldx/transformations.py @@ -418,7 +418,7 @@ def __init__( coordinates, dict( c={"values": ["x", "y", "z"]}, - time={"dtype": "timedelta64[ns]", "optional": True}, + time={"dtype": "timedelta64", "optional": True}, ), ) @@ -427,7 +427,7 @@ def __init__( dict( c={"values": ["x", "y", "z"]}, v={"values": [0, 1, 2]}, - time={"dtype": "timedelta64[ns]", "optional": True}, + time={"dtype": "timedelta64", "optional": True}, ), ) From cc1e037861f5268e43bf0c9dfd9d14a45b8abb4e Mon Sep 17 00:00:00 2001 From: Cagtay Fabry <43667554+CagtayFabry@users.noreply.github.com> Date: Thu, 8 Oct 2020 12:49:41 +0200 Subject: [PATCH 36/38] docstring changes Co-authored-by: vhirtham --- weldx/utility.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/weldx/utility.py b/weldx/utility.py index 9afc74dc2..21813bbe6 100644 --- a/weldx/utility.py +++ b/weldx/utility.py @@ -612,15 +612,15 @@ def xr_check_coords(dax: xr.DataArray, ref: dict) -> bool: Parameters ---------- - dax: + dax : xr.DataArray xarray object which should be validated - ref: + ref : dict reference dictionary Returns ------- bool - True, if the test was a success, else throws an exception + True, if the test was a success, else an exception is raised Examples -------- From 680f8c03222a310e19a43fe9210a8f62b4e2d746 Mon Sep 17 00:00:00 2001 From: Cagtay Fabry <43667554+CagtayFabry@users.noreply.github.com> Date: Thu, 8 Oct 2020 12:59:48 +0200 Subject: [PATCH 37/38] add wrong input type exception --- tests/test_utility.py | 1 + weldx/utility.py | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/tests/test_utility.py b/tests/test_utility.py index 684f1e71c..c4a67705d 100644 --- a/tests/test_utility.py +++ b/tests/test_utility.py @@ -454,6 +454,7 @@ def test_xr_check_coords(dax, ref_dict): ), (_dax_check, {"d1": {"dtype": [int, str, bool]}}, TypeError), (_dax_check, {"d4": {"dtype": "datetime64"}}, TypeError), + ({"d4": np.arange(4)}, {"d4": {"dtype": "int"}}, ValueError), ], ) def test_xr_check_coords_exception(dax, ref_dict, exception_type): diff --git a/weldx/utility.py b/weldx/utility.py index 9afc74dc2..31a9f2ca9 100644 --- a/weldx/utility.py +++ b/weldx/utility.py @@ -562,14 +562,14 @@ def xr_interp_like( return result -def _check_dtype(var_dtype, ref_dtype) -> bool: +def _check_dtype(var_dtype, ref_dtype: dict) -> bool: """Check if dtype matches a reference dtype (or is subdtype). Parameters ---------- - var_dtype: + var_dtype : numpy dtype A numpy-dtype to test against. - ref_dtype + ref_dtype : dict Python type or string description Returns @@ -651,8 +651,16 @@ def xr_check_coords(dax: xr.DataArray, ref: dict) -> bool: # only process the coords of the xarray if isinstance(dax, (xr.DataArray, xr.Dataset)): coords = dax.coords - else: + elif isinstance( + dax, + ( + xr.core.coordinates.DataArrayCoordinates, + xr.core.coordinates.DatasetCoordinates, + ), + ): coords = dax + else: + raise ValueError("Input variable is not an xarray object") for key, check in ref.items(): # check if the optional key is set to true From 781b4fe4ad900bd0734e389731ad5e4c39d1b27e Mon Sep 17 00:00:00 2001 From: Cagtay Fabry <43667554+CagtayFabry@users.noreply.github.com> Date: Fri, 9 Oct 2020 10:12:48 +0200 Subject: [PATCH 38/38] try fix doc & RTD builds --- doc/conf.py | 6 +++--- weldx/utility.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index c77af5e2e..768bc381a 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -179,9 +179,9 @@ "pandas": ("https://pandas.pydata.org/pandas-docs/stable", None), "xarray": ("http://xarray.pydata.org/en/stable", None), "scipy": ("https://docs.scipy.org/doc/scipy/reference", None), - "matplotlib": ("https://matplotlib.org", None), - "dask": ("https://docs.dask.org/en/latest", None), - "numba": ("https://numba.pydata.org/numba-doc/latest", None), + # "matplotlib": ("https://matplotlib.org", None), + # "dask": ("https://docs.dask.org/en/latest", None), + # "numba": ("https://numba.pydata.org/numba-doc/latest", None), "pint": ("https://pint.readthedocs.io/en/stable", None), } diff --git a/weldx/utility.py b/weldx/utility.py index 6c6707da3..363515dd7 100644 --- a/weldx/utility.py +++ b/weldx/utility.py @@ -612,7 +612,7 @@ def xr_check_coords(dax: xr.DataArray, ref: dict) -> bool: Parameters ---------- - dax : xr.DataArray + dax : xarray.DataArray xarray object which should be validated ref : dict reference dictionary