From cc5f39ae919f4a5202f9bab5017d8c23cc6b4020 Mon Sep 17 00:00:00 2001 From: git-abhishek <36498066+git-abhishek@users.noreply.github.com> Date: Fri, 8 Jun 2018 17:40:38 -0400 Subject: [PATCH 01/15] Removed dead code, added support to set center at min values, added test case --- yt/data_objects/data_containers.py | 30 +++---------------- yt/data_objects/tests/test_data_containers.py | 0 yt/data_objects/tests/test_spheres.py | 20 ++++++++++--- 3 files changed, 20 insertions(+), 30 deletions(-) create mode 100644 yt/data_objects/tests/test_data_containers.py diff --git a/yt/data_objects/data_containers.py b/yt/data_objects/data_containers.py index a318db840ba..967a2c1bcc6 100644 --- a/yt/data_objects/data_containers.py +++ b/yt/data_objects/data_containers.py @@ -80,19 +80,6 @@ def force_array(item, shape): else: return np.zeros(shape, dtype='bool') -def restore_field_information_state(func): - """ - A decorator that takes a function with the API of (self, grid, field) - and ensures that after the function is called, the field_parameters will - be returned to normal. - """ - def save_state(self, grid, field=None, *args, **kwargs): - old_params = grid.field_parameters - grid.field_parameters = self.field_parameters - tr = func(self, grid, field, *args, **kwargs) - grid.field_parameters = old_params - return tr - return save_state def sanitize_weight_field(ds, field, weight): field_object = ds._get_field_info(field) @@ -215,6 +202,10 @@ def _set_center(self, center): self.center = self.ds.find_max(("gas", "density"))[1] elif center.startswith("max_"): self.center = self.ds.find_max(center[4:])[1] + elif center.lower() == "min": + self.center = self.ds.find_min(("gas", "density"))[1] + elif center.startswith("min_"): + self.center = self.ds.find_min(center[4:])[1] else: self.center = self.ds.arr(center, 'code_length', dtype='float64') self.set_field_parameter('center', self.center) @@ -1990,20 +1981,7 @@ def extract_connected_sets(self, field, num_levels, min_val, max_val, {'contour_slices_%s' % contour_key: cids}) return cons, contours - def paint_grids(self, field, value, default_value=None): - """ - This function paints every cell in our dataset with a given *value*. - If default_value is given, the other values for the given in every grid - are discarded and replaced with *default_value*. Otherwise, the field is - mandated to 'know how to exist' in the grid. - Note that this only paints the cells *in the dataset*, so cells in grids - with child cells are left untouched. - """ - for grid in self._grids: - if default_value is not None: - grid[field] = np.ones(grid.ActiveDimensions)*default_value - grid[field][self._get_point_indices(grid)] = value def volume(self): """ diff --git a/yt/data_objects/tests/test_data_containers.py b/yt/data_objects/tests/test_data_containers.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/yt/data_objects/tests/test_spheres.py b/yt/data_objects/tests/test_spheres.py index b6d2b3318b3..3ca85007495 100644 --- a/yt/data_objects/tests/test_spheres.py +++ b/yt/data_objects/tests/test_spheres.py @@ -1,10 +1,9 @@ import numpy as np +from numpy.testing import assert_array_equal from yt.data_objects.profiles import create_profile -from yt.testing import \ - fake_random_ds, \ - assert_equal, \ - periodicity_cases +from yt.testing import fake_random_ds, assert_equal, periodicity_cases + def setup(): from yt.config import ytcfg @@ -65,3 +64,16 @@ def test_domain_sphere(): for f in _fields_to_compare: sp[f].sort() assert_equal(sp[f], ref_sp[f]) + +def test_sphere_center(): + ds = fake_random_ds(16, nprocs=8, + fields=("density", "temperature", "velocity_x")) + + # Test if we obtain same center in different ways + sp1 = ds.sphere("max", (0.25, 'unitary')) + sp2 = ds.sphere("max_density", (0.25, 'unitary')) + assert_array_equal(sp1.center, sp2.center) + + sp1 = ds.sphere("min", (0.25, 'unitary')) + sp2 = ds.sphere("min_density", (0.25, 'unitary')) + assert_array_equal(sp1.center, sp2.center) From 3fca6bcec57cac6e885aa7cbe5a743c73230eedc Mon Sep 17 00:00:00 2001 From: git-abhishek <36498066+git-abhishek@users.noreply.github.com> Date: Mon, 11 Jun 2018 17:15:30 -0400 Subject: [PATCH 02/15] update fiel i/o using context mgr; fixed to_frb --- yt/data_objects/data_containers.py | 36 +++++++++++++++--------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/yt/data_objects/data_containers.py b/yt/data_objects/data_containers.py index 967a2c1bcc6..ee43b73a83d 100644 --- a/yt/data_objects/data_containers.py +++ b/yt/data_objects/data_containers.py @@ -80,7 +80,6 @@ def force_array(item, shape): else: return np.zeros(shape, dtype='bool') - def sanitize_weight_field(ds, field, weight): field_object = ds._get_field_info(field) if weight is None: @@ -420,16 +419,18 @@ def _parameter_iterate(self, seq): def write_out(self, filename, fields=None, format="%0.16e"): if fields is None: fields=sorted(self.field_data.keys()) if self._key_fields is None: raise ValueError - field_order = self._key_fields[:] - for field in field_order: self[field] - field_order += [field for field in fields if field not in field_order] - fid = open(filename,"w") - fid.write("\t".join(["#"] + field_order + ["\n"])) - field_data = np.array([self.field_data[field] for field in field_order]) - for line in range(field_data.shape[1]): - field_data[:,line].tofile(fid, sep="\t", format=format) - fid.write("\n") - fid.close() + field_order = self._determine_fields(self._key_fields) + for field in field_order: + self[field] + field_order += [self._determine_fields(field)[0] for field in fields + if field not in field_order] + field_header = [str(f) for f in field_order] + with open(filename,"w") as fid: + fid.write("\t".join(["#"] + field_header + ["\n"])) + field_data = np.array([self.field_data[field] for field in field_order]) + for line in range(field_data.shape[1]): + field_data[:,line].tofile(fid, sep="\t", format=format) + fid.write("\n") def save_object(self, name, filename = None): """ @@ -438,11 +439,10 @@ def save_object(self, name, filename = None): :meth:`yt.data_objects.api.GridIndex.save_object`. """ if filename is not None: - ds = shelve.open(filename, protocol=-1) - if name in ds: - mylog.info("Overwriting %s in %s", name, filename) - ds[name] = self - ds.close() + with shelve.open(filename, protocol=-1) as ds: + if name in ds: + mylog.info("Overwriting %s in %s", name, filename) + ds[name] = self else: self.index.save_object(self, name) @@ -1664,8 +1664,8 @@ def to_frb(self, width, resolution, center=None, height=None, "Currently we only support images centered at R=0. " + "We plan to generalize this in the near future") from yt.visualization.fixed_resolution import CylindricalFixedResolutionBuffer - if iterable(width): - radius = max(width) + if isinstance(width, tuple): + radius = width[0] else: radius = width if iterable(resolution): resolution = max(resolution) From ef197194a9e28b7155d1b5617c32d727ddb36261 Mon Sep 17 00:00:00 2001 From: git-abhishek <36498066+git-abhishek@users.noreply.github.com> Date: Mon, 11 Jun 2018 17:18:19 -0400 Subject: [PATCH 03/15] Fixed YTSmoothedCoveringGrid __init__ --- yt/data_objects/construction_data_containers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/yt/data_objects/construction_data_containers.py b/yt/data_objects/construction_data_containers.py index 0d5b21d4f07..135fb6942be 100644 --- a/yt/data_objects/construction_data_containers.py +++ b/yt/data_objects/construction_data_containers.py @@ -922,7 +922,8 @@ def __init__(self, *args, **kwargs): self._final_start_index = self.global_startindex def _setup_data_source(self, level_state = None): - if level_state is None: return + if level_state is None: + return super(YTSmoothedCoveringGrid, self)._setup_data_source() # We need a buffer region to allow for zones that contribute to the # interpolation but are not directly inside our bounds level_state.data_source = self.ds.region( From bf72557c370b6d575f4d389f6ffa893fd5bc6300 Mon Sep 17 00:00:00 2001 From: git-abhishek <36498066+git-abhishek@users.noreply.github.com> Date: Mon, 11 Jun 2018 17:29:58 -0400 Subject: [PATCH 04/15] Added test cases --- yt/data_objects/tests/test_clone.py | 7 ++ yt/data_objects/tests/test_connected_sets.py | 21 ++-- yt/data_objects/tests/test_data_containers.py | 114 ++++++++++++++++++ 3 files changed, 130 insertions(+), 12 deletions(-) diff --git a/yt/data_objects/tests/test_clone.py b/yt/data_objects/tests/test_clone.py index e44bac092a0..8b3087b2b91 100644 --- a/yt/data_objects/tests/test_clone.py +++ b/yt/data_objects/tests/test_clone.py @@ -22,3 +22,10 @@ def test_clone_sphere(): sp1["density"] assert_array_equal(sp0["density"], sp1["density"]) + +def test_clone_cut_region(): + ds = fake_random_ds(64, nprocs=4, fields=("density", "temperature")) + dd = ds.all_data() + reg1 = dd.cut_region(["obj['temperature'] > 0.5", "obj['density'] < 0.75"]) + reg2 = reg1.clone() + assert_array_equal(reg1['density'], reg2['density']) diff --git a/yt/data_objects/tests/test_connected_sets.py b/yt/data_objects/tests/test_connected_sets.py index 284cf0f4957..af5aa034b38 100644 --- a/yt/data_objects/tests/test_connected_sets.py +++ b/yt/data_objects/tests/test_connected_sets.py @@ -1,14 +1,11 @@ -from yt.utilities.answer_testing.level_sets_tests import \ - ExtractConnectedSetsTest -from yt.utilities.answer_testing.framework import \ - requires_ds, \ - data_dir_load +from yt.testing import fake_random_ds +from yt.utilities.answer_testing.level_sets_tests import ExtractConnectedSetsTest -g30 = "IsolatedGalaxy/galaxy0030/galaxy0030" -@requires_ds(g30, big_data=True) def test_connected_sets(): - ds = data_dir_load(g30) - data_source = ds.disk([0.5, 0.5, 0.5], [0., 0., 1.], - (8, 'kpc'), (1, 'kpc')) - yield ExtractConnectedSetsTest(g30, data_source, ("gas", "density"), - 5, 1e-24, 8e-24) + ds = fake_random_ds(16, nprocs=8, particles=16**3) + data_source = ds.disk([0.5, 0.5, 0.5], [0., 0., 1.], (8, 'kpc'), (1, 'kpc')) + field = ("gas", "density") + min_val, max_val = data_source[field].min()/2, data_source[field].max()/2 + data_source.extract_connected_sets(field, 5, min_val, max_val, + log_space=True, cumulative=True) + yield ExtractConnectedSetsTest(ds, data_source, field, 5, min_val, max_val) diff --git a/yt/data_objects/tests/test_data_containers.py b/yt/data_objects/tests/test_data_containers.py index e69de29bb2d..bbcd7434734 100644 --- a/yt/data_objects/tests/test_data_containers.py +++ b/yt/data_objects/tests/test_data_containers.py @@ -0,0 +1,114 @@ +import shelve + +import numpy as np +from nose.tools import assert_raises +from numpy.testing import assert_array_equal + +from yt.data_objects.data_containers import YTDataContainer +from yt.testing import assert_equal, fake_random_ds, fake_amr_ds, fake_particle_ds +from yt.utilities.exceptions import YTFieldNotFound + + +def test_yt_data_container(): + # Test if ds could be None + with assert_raises(RuntimeError) as err: + YTDataContainer(None, None) + desired = 'Error: ds must be set either through class type or parameter' \ + ' to the constructor' + assert_equal(str(err.exception), desired) + + # Test if field_data key exists + ds = fake_random_ds(5) + proj = ds.proj("density", 0, data_source=ds.all_data()) + assert_equal(proj.has_key('px'), True) + assert_equal(proj.has_key('pz'), False) + + # Delete the key and check if exits + proj.__delitem__('px') + assert_equal(proj.has_key('px'), False) + proj.__delitem__('density') + assert_equal(proj.has_key('density'), False) + + # Delete a non-existent field + with assert_raises(YTFieldNotFound) as ex: + proj.__delitem__('p_mass') + desired = "Could not find field '('stream', 'p_mass')' in UniformGridData." + assert_equal(str(ex.exception), desired) + + # Test the convert method + assert_equal(proj.convert('HydroMethod'), -1) + +def test_write_out(): + filename = "sphere.txt" + ds = fake_particle_ds() + sp = ds.sphere(ds.domain_center, 0.25) + sp.write_out(filename) + + with open(filename, "r") as file: + file_row_1 = file.readline() + file_row_2 = file.readline() + file_row_2 = np.array(file_row_2.split('\t'), dtype=np.float64) + _keys = [str(k) for k in sp.field_data.keys()] + _keys = "\t".join(["#"] + _keys + ["\n"]) + _data = [sp.field_data[k][0] for k in sp.field_data.keys()] + + assert_equal(_keys, file_row_1) + assert_array_equal(_data, file_row_2) + +def test_save_object(): + ds = fake_particle_ds() + sp = ds.sphere(ds.domain_center, 0.25) + sp.save_object("my_sphere_1", filename="test_save_obj") + with shelve.open("test_save_obj", protocol=-1) as obj: + loaded_sphere = obj["my_sphere_1"][1] + assert_array_equal(loaded_sphere.center, sp.center) + assert_equal(loaded_sphere.radius, sp.radius) + + sp.save_object("my_sphere_2") + +def test_to_dataframe(): + fields = ["density", "velocity_z"] + ds = fake_random_ds(6) + dd = ds.all_data() + df1 = dd.to_dataframe(fields) + assert_array_equal(dd[fields[0]], df1[fields[0]]) + assert_array_equal(dd[fields[1]], df1[fields[1]]) + +def test_std(): + ds = fake_random_ds(3) + ds.all_data().std('density', weight="velocity_z") + +def test_to_frb(): + ds = fake_amr_ds(fields=["density", "cell_mass"], geometry="cylindrical", + particles=16**3) + proj = ds.proj("density", weight_field="cell_mass", axis=1, + data_source=ds.all_data()) + proj.to_frb((1.0, 'unitary'), 64) + +def test_extract_isocontours(): + # Test isocontour properties for AMRGridData + ds = fake_amr_ds(fields=["density", "cell_mass"], particles=16**3) + dd = ds.all_data() + rho = dd.quantities["WeightedAverageQuantity"]("density", + weight="cell_mass") + dd.extract_isocontours("density", rho, "triangles.obj", True) + dd.calculate_isocontour_flux("density", rho, "x", "y", "z", + "dx") + + # Test error in case of ParticleData + ds = fake_particle_ds() + dd = ds.all_data() + rho = dd.quantities["WeightedAverageQuantity"]("particle_velocity_x", + weight="particle_mass") + with assert_raises(NotImplementedError): + dd.extract_isocontours("density", rho, sample_values='x') + +def test_extract_connected_sets(): + + ds = fake_random_ds(16, nprocs=8, particles=16 ** 3) + data_source = ds.disk([0.5, 0.5, 0.5], [0., 0., 1.], (8, 'kpc'), (1, 'kpc')) + field = ("gas", "density") + min_val, max_val = data_source[field].min() / 2, data_source[field].max() / 2 + + data_source.extract_connected_sets(field, 3, min_val, max_val, + log_space=True, cumulative=True) From 680fb9213d8e87c2531a7038191189ae112102d3 Mon Sep 17 00:00:00 2001 From: git-abhishek <36498066+git-abhishek@users.noreply.github.com> Date: Mon, 11 Jun 2018 17:51:06 -0400 Subject: [PATCH 05/15] fixed flake8 warnings --- yt/data_objects/tests/test_data_containers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/yt/data_objects/tests/test_data_containers.py b/yt/data_objects/tests/test_data_containers.py index bbcd7434734..382afe66e4c 100644 --- a/yt/data_objects/tests/test_data_containers.py +++ b/yt/data_objects/tests/test_data_containers.py @@ -20,14 +20,14 @@ def test_yt_data_container(): # Test if field_data key exists ds = fake_random_ds(5) proj = ds.proj("density", 0, data_source=ds.all_data()) - assert_equal(proj.has_key('px'), True) - assert_equal(proj.has_key('pz'), False) + assert_equal('px' in proj.keys(), True) + assert_equal('pz' in proj.keys(), False) # Delete the key and check if exits proj.__delitem__('px') - assert_equal(proj.has_key('px'), False) + assert_equal('px' in proj.keys(), False) proj.__delitem__('density') - assert_equal(proj.has_key('density'), False) + assert_equal('density' in proj.keys(), False) # Delete a non-existent field with assert_raises(YTFieldNotFound) as ex: From 218e33c8ebc1a1f1e07c5352ea625d71f79901c6 Mon Sep 17 00:00:00 2001 From: git-abhishek <36498066+git-abhishek@users.noreply.github.com> Date: Mon, 11 Jun 2018 18:57:34 -0400 Subject: [PATCH 06/15] Revert the use of context manager due to python 2 --- yt/data_objects/data_containers.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/yt/data_objects/data_containers.py b/yt/data_objects/data_containers.py index ee43b73a83d..72c4ccdaf79 100644 --- a/yt/data_objects/data_containers.py +++ b/yt/data_objects/data_containers.py @@ -419,18 +419,17 @@ def _parameter_iterate(self, seq): def write_out(self, filename, fields=None, format="%0.16e"): if fields is None: fields=sorted(self.field_data.keys()) if self._key_fields is None: raise ValueError - field_order = self._determine_fields(self._key_fields) - for field in field_order: - self[field] - field_order += [self._determine_fields(field)[0] for field in fields - if field not in field_order] + field_order = self._key_fields[:] + for field in field_order: self[field] + field_order += [field for field in fields if field not in field_order] + fid = open(filename,"w") field_header = [str(f) for f in field_order] - with open(filename,"w") as fid: - fid.write("\t".join(["#"] + field_header + ["\n"])) - field_data = np.array([self.field_data[field] for field in field_order]) - for line in range(field_data.shape[1]): - field_data[:,line].tofile(fid, sep="\t", format=format) - fid.write("\n") + fid.write("\t".join(["#"] + field_header + ["\n"])) + field_data = np.array([self.field_data[field] for field in field_order]) + for line in range(field_data.shape[1]): + field_data[:,line].tofile(fid, sep="\t", format=format) + fid.write("\n") + fid.close() def save_object(self, name, filename = None): """ @@ -439,10 +438,11 @@ def save_object(self, name, filename = None): :meth:`yt.data_objects.api.GridIndex.save_object`. """ if filename is not None: - with shelve.open(filename, protocol=-1) as ds: - if name in ds: - mylog.info("Overwriting %s in %s", name, filename) - ds[name] = self + ds = shelve.open(filename, protocol=-1) + if name in ds: + mylog.info("Overwriting %s in %s", name, filename) + ds[name] = self + ds.close() else: self.index.save_object(self, name) From e2b4d70ea34731eff6b1cf30760d283c199fe838 Mon Sep 17 00:00:00 2001 From: git-abhishek <36498066+git-abhishek@users.noreply.github.com> Date: Mon, 11 Jun 2018 19:43:22 -0400 Subject: [PATCH 07/15] Fixed Pandas test, sort writing to file --- yt/data_objects/data_containers.py | 2 +- yt/data_objects/tests/test_data_containers.py | 21 ++++++++++++------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/yt/data_objects/data_containers.py b/yt/data_objects/data_containers.py index 72c4ccdaf79..d0c8d5d295a 100644 --- a/yt/data_objects/data_containers.py +++ b/yt/data_objects/data_containers.py @@ -419,7 +419,7 @@ def _parameter_iterate(self, seq): def write_out(self, filename, fields=None, format="%0.16e"): if fields is None: fields=sorted(self.field_data.keys()) if self._key_fields is None: raise ValueError - field_order = self._key_fields[:] + field_order = sorted(self._determine_fields(self._key_fields)) for field in field_order: self[field] field_order += [field for field in fields if field not in field_order] fid = open(filename,"w") diff --git a/yt/data_objects/tests/test_data_containers.py b/yt/data_objects/tests/test_data_containers.py index 382afe66e4c..36eb467c527 100644 --- a/yt/data_objects/tests/test_data_containers.py +++ b/yt/data_objects/tests/test_data_containers.py @@ -48,9 +48,10 @@ def test_write_out(): file_row_1 = file.readline() file_row_2 = file.readline() file_row_2 = np.array(file_row_2.split('\t'), dtype=np.float64) - _keys = [str(k) for k in sp.field_data.keys()] + sorted_keys = sorted(sp.field_data.keys()) + _keys = [str(k) for k in sorted_keys] _keys = "\t".join(["#"] + _keys + ["\n"]) - _data = [sp.field_data[k][0] for k in sp.field_data.keys()] + _data = [sp.field_data[k][0] for k in sorted_keys] assert_equal(_keys, file_row_1) assert_array_equal(_data, file_row_2) @@ -67,12 +68,16 @@ def test_save_object(): sp.save_object("my_sphere_2") def test_to_dataframe(): - fields = ["density", "velocity_z"] - ds = fake_random_ds(6) - dd = ds.all_data() - df1 = dd.to_dataframe(fields) - assert_array_equal(dd[fields[0]], df1[fields[0]]) - assert_array_equal(dd[fields[1]], df1[fields[1]]) + try: + import pandas as pd + fields = ["density", "velocity_z"] + ds = fake_random_ds(6) + dd = ds.all_data() + df1 = dd.to_dataframe(fields) + assert_array_equal(dd[fields[0]], df1[fields[0]]) + assert_array_equal(dd[fields[1]], df1[fields[1]]) + except ImportError: + pass def test_std(): ds = fake_random_ds(3) From 1550eab59930afbd44fdf20c1fe729dfd9127dc1 Mon Sep 17 00:00:00 2001 From: git-abhishek <36498066+git-abhishek@users.noreply.github.com> Date: Mon, 11 Jun 2018 20:18:48 -0400 Subject: [PATCH 08/15] fixed pandas import flake8 error --- yt/data_objects/tests/test_data_containers.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/yt/data_objects/tests/test_data_containers.py b/yt/data_objects/tests/test_data_containers.py index 36eb467c527..ceece1fd19c 100644 --- a/yt/data_objects/tests/test_data_containers.py +++ b/yt/data_objects/tests/test_data_containers.py @@ -69,13 +69,15 @@ def test_save_object(): def test_to_dataframe(): try: - import pandas as pd - fields = ["density", "velocity_z"] - ds = fake_random_ds(6) - dd = ds.all_data() - df1 = dd.to_dataframe(fields) - assert_array_equal(dd[fields[0]], df1[fields[0]]) - assert_array_equal(dd[fields[1]], df1[fields[1]]) + import importlib + pandas_loader = importlib.util.find_spec("pandas") + if pandas_loader is not None: + fields = ["density", "velocity_z"] + ds = fake_random_ds(6) + dd = ds.all_data() + df1 = dd.to_dataframe(fields) + assert_array_equal(dd[fields[0]], df1[fields[0]]) + assert_array_equal(dd[fields[1]], df1[fields[1]]) except ImportError: pass From 11f62ca633a12e1c96eaf99865c4932a92d6eb56 Mon Sep 17 00:00:00 2001 From: git-abhishek <36498066+git-abhishek@users.noreply.github.com> Date: Mon, 11 Jun 2018 21:35:02 -0400 Subject: [PATCH 09/15] closing file explicitly due to py2 with context mgr --- yt/data_objects/tests/test_data_containers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/yt/data_objects/tests/test_data_containers.py b/yt/data_objects/tests/test_data_containers.py index ceece1fd19c..d0b99cba9f1 100644 --- a/yt/data_objects/tests/test_data_containers.py +++ b/yt/data_objects/tests/test_data_containers.py @@ -64,6 +64,7 @@ def test_save_object(): loaded_sphere = obj["my_sphere_1"][1] assert_array_equal(loaded_sphere.center, sp.center) assert_equal(loaded_sphere.radius, sp.radius) + obj.close() sp.save_object("my_sphere_2") From 77c27d52ff15696f09df9eea071d83456427a637 Mon Sep 17 00:00:00 2001 From: git-abhishek <36498066+git-abhishek@users.noreply.github.com> Date: Mon, 11 Jun 2018 23:47:23 -0400 Subject: [PATCH 10/15] Removed context mgr usage from test --- yt/data_objects/tests/test_data_containers.py | 4 ++-- yt/utilities/input_validator.py | 0 yt/utilities/tests/test_input_validator.py | 0 3 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 yt/utilities/input_validator.py create mode 100644 yt/utilities/tests/test_input_validator.py diff --git a/yt/data_objects/tests/test_data_containers.py b/yt/data_objects/tests/test_data_containers.py index d0b99cba9f1..a7d0c825d9b 100644 --- a/yt/data_objects/tests/test_data_containers.py +++ b/yt/data_objects/tests/test_data_containers.py @@ -60,8 +60,8 @@ def test_save_object(): ds = fake_particle_ds() sp = ds.sphere(ds.domain_center, 0.25) sp.save_object("my_sphere_1", filename="test_save_obj") - with shelve.open("test_save_obj", protocol=-1) as obj: - loaded_sphere = obj["my_sphere_1"][1] + obj = shelve.open("test_save_obj", protocol=-1) + loaded_sphere = obj["my_sphere_1"][1] assert_array_equal(loaded_sphere.center, sp.center) assert_equal(loaded_sphere.radius, sp.radius) obj.close() diff --git a/yt/utilities/input_validator.py b/yt/utilities/input_validator.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/yt/utilities/tests/test_input_validator.py b/yt/utilities/tests/test_input_validator.py new file mode 100644 index 00000000000..e69de29bb2d From 9abeca028086b6f60c5cb66cdf64fef156aeb60e Mon Sep 17 00:00:00 2001 From: git-abhishek <36498066+git-abhishek@users.noreply.github.com> Date: Tue, 12 Jun 2018 02:53:57 -0400 Subject: [PATCH 11/15] fixed incompatibility b/w py 2 and 3 --- yt/data_objects/tests/test_data_containers.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/yt/data_objects/tests/test_data_containers.py b/yt/data_objects/tests/test_data_containers.py index a7d0c825d9b..2a7d0bcd894 100644 --- a/yt/data_objects/tests/test_data_containers.py +++ b/yt/data_objects/tests/test_data_containers.py @@ -70,15 +70,14 @@ def test_save_object(): def test_to_dataframe(): try: - import importlib - pandas_loader = importlib.util.find_spec("pandas") - if pandas_loader is not None: - fields = ["density", "velocity_z"] - ds = fake_random_ds(6) - dd = ds.all_data() - df1 = dd.to_dataframe(fields) - assert_array_equal(dd[fields[0]], df1[fields[0]]) - assert_array_equal(dd[fields[1]], df1[fields[1]]) + import pandas as pd + pd # to ignore Flake8 error + fields = ["density", "velocity_z"] + ds = fake_random_ds(6) + dd = ds.all_data() + df1 = dd.to_dataframe(fields) + assert_array_equal(dd[fields[0]], df1[fields[0]]) + assert_array_equal(dd[fields[1]], df1[fields[1]]) except ImportError: pass From 70682e52ae2a4895bb39497150074ef53bfa2bf5 Mon Sep 17 00:00:00 2001 From: git-abhishek <36498066+git-abhishek@users.noreply.github.com> Date: Thu, 14 Jun 2018 00:58:37 -0400 Subject: [PATCH 12/15] Updated as per first code review comments --- .travis.yml | 3 +- appveyor.yml | 2 +- .../construction_data_containers.py | 3 +- yt/data_objects/data_containers.py | 95 +++++++++++------ yt/data_objects/tests/test_data_containers.py | 100 +++++++++--------- yt/utilities/input_validator.py | 0 yt/utilities/tests/test_input_validator.py | 0 7 files changed, 118 insertions(+), 85 deletions(-) delete mode 100644 yt/utilities/input_validator.py delete mode 100644 yt/utilities/tests/test_input_validator.py diff --git a/.travis.yml b/.travis.yml index 488619283b0..caec2064af5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,6 +23,7 @@ env: FLAKE8=flake8 CODECOV=codecov COVERAGE=coverage + PANDAS=pandas before_install: - | @@ -50,7 +51,7 @@ install: pip install --upgrade wheel pip install --upgrade setuptools # Install dependencies - pip install mock $NUMPY $SCIPY $H5PY $CYTHON $MATPLOTLIB $SYMPY $FASTCACHE $IPYTHON $FLAKE8 $CODECOV $COVERAGE nose nose-timer + pip install mock $NUMPY $SCIPY $H5PY $CYTHON $MATPLOTLIB $SYMPY $FASTCACHE $IPYTHON $FLAKE8 $CODECOV $COVERAGE $PANDAS nose nose-timer # install yt pip install -e . diff --git a/appveyor.yml b/appveyor.yml index 6b888d503ce..5671182a393 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -29,7 +29,7 @@ install: - "python --version" # Install specified version of numpy and dependencies - - "conda install --yes -c conda-forge numpy scipy nose setuptools ipython Cython sympy fastcache h5py matplotlib flake8 mock" + - "conda install --yes -c conda-forge numpy scipy nose setuptools ipython Cython sympy fastcache h5py matplotlib flake8 mock pandas" - "pip install -e ." # Not a .NET project diff --git a/yt/data_objects/construction_data_containers.py b/yt/data_objects/construction_data_containers.py index 135fb6942be..41a02da3149 100644 --- a/yt/data_objects/construction_data_containers.py +++ b/yt/data_objects/construction_data_containers.py @@ -923,7 +923,8 @@ def __init__(self, *args, **kwargs): def _setup_data_source(self, level_state = None): if level_state is None: - return super(YTSmoothedCoveringGrid, self)._setup_data_source() + super(YTSmoothedCoveringGrid, self)._setup_data_source() + return # We need a buffer region to allow for zones that contribute to the # interpolation but are not directly inside our bounds level_state.data_source = self.ds.region( diff --git a/yt/data_objects/data_containers.py b/yt/data_objects/data_containers.py index d0c8d5d295a..3764fa846d8 100644 --- a/yt/data_objects/data_containers.py +++ b/yt/data_objects/data_containers.py @@ -32,7 +32,7 @@ mylog, \ ensure_list, \ fix_axis, \ - iterable + iterable, validate_width_tuple from yt.units.unit_object import UnitParseError from yt.units.yt_array import \ YTArray, \ @@ -50,7 +50,7 @@ YTDataSelectorNotImplemented, \ YTDimensionalityError, \ YTBooleanObjectError, \ - YTBooleanObjectsWrongDataset + YTBooleanObjectsWrongDataset, YTException from yt.utilities.lib.marching_cubes import \ march_cubes_grid, march_cubes_grid_flux from yt.utilities.parallel_tools.parallel_analysis_interface import \ @@ -71,15 +71,6 @@ data_object_registry = {} -def force_array(item, shape): - try: - return item.copy() - except AttributeError: - if item: - return np.ones(shape, dtype='bool') - else: - return np.zeros(shape, dtype='bool') - def sanitize_weight_field(ds, field, weight): field_object = ds._get_field_info(field) if weight is None: @@ -232,13 +223,6 @@ def has_field_parameter(self, name): """ return name in self.field_parameters - def convert(self, datatype): - """ - This will attempt to convert a given unit to cgs from code units. - It either returns the multiplicative factor or throws a KeyError. - """ - return self.ds[datatype] - def clear_data(self): """ Clears out all data from the YTDataContainer instance, freeing memory. @@ -417,21 +401,73 @@ def _parameter_iterate(self, seq): _key_fields = None def write_out(self, filename, fields=None, format="%0.16e"): - if fields is None: fields=sorted(self.field_data.keys()) - if self._key_fields is None: raise ValueError - field_order = sorted(self._determine_fields(self._key_fields)) + """Write out the YTDataContainer object in a text file. + + This function will take a data object and produce a tab delimited text + file containing the fields presently existing and the fields given in + the ``fields`` list. + + Parameters + ---------- + filename : String + The name of the file to write to. + + fields : List of string, Default = None + If this is supplied, these fields will be added to the list of + fields to be saved to disk. If not supplied, whatever fields + presently exist will be used. + + format : String, Default = "%0.16e" + Format of numbers to be written in the file. + + Raises + ------ + ValueError + Raised when there is no existing field. + + YTException + Raised when field_type of supplied fields is inconsistent with the + field_type of existing fields. + + Examples + -------- + >>> ds = fake_particle_ds() + >>> sp = ds.sphere(ds.domain_center, 0.25) + >>> sp.write_out("sphere_1.txt") + >>> sp.write_out("sphere_2.txt", fields=["cell_volume"]) + """ + if fields is None: + fields = sorted(self.field_data.keys()) + + if self._key_fields is None: + raise ValueError + + field_order = self._key_fields + diff_fields = [field for field in fields if field not in field_order] + field_order += diff_fields + field_order = sorted(self._determine_fields(field_order)) + field_types = {u for u, v in field_order} + + if len(field_types) != 1: + diff_fields = self._determine_fields(diff_fields) + req_ftype = self._determine_fields(self._key_fields[0])[0][0] + f_type = {f for f in diff_fields if f[0] != req_ftype } + msg = ("Field type %s of the supplied field %s is inconsistent" + " with field type '%s'." % + ([f[0] for f in f_type], [f[1] for f in f_type], req_ftype)) + raise YTException(msg) + for field in field_order: self[field] - field_order += [field for field in fields if field not in field_order] - fid = open(filename,"w") + fid = open(filename, "w") field_header = [str(f) for f in field_order] fid.write("\t".join(["#"] + field_header + ["\n"])) field_data = np.array([self.field_data[field] for field in field_order]) for line in range(field_data.shape[1]): - field_data[:,line].tofile(fid, sep="\t", format=format) + field_data[:, line].tofile(fid, sep="\t", format=format) fid.write("\n") fid.close() - def save_object(self, name, filename = None): + def save_object(self, name, filename=None): """ Save an object. If *filename* is supplied, it will be stored in a :mod:`shelve` file of that name. Otherwise, it will be stored via @@ -446,7 +482,7 @@ def save_object(self, name, filename = None): else: self.index.save_object(self, name) - def to_dataframe(self, fields = None): + def to_dataframe(self, fields=None): r"""Export a data object to a pandas DataFrame. This function will take a data object and construct from it and @@ -1664,12 +1700,9 @@ def to_frb(self, width, resolution, center=None, height=None, "Currently we only support images centered at R=0. " + "We plan to generalize this in the near future") from yt.visualization.fixed_resolution import CylindricalFixedResolutionBuffer - if isinstance(width, tuple): - radius = width[0] - else: - radius = width + validate_width_tuple(width) if iterable(resolution): resolution = max(resolution) - frb = CylindricalFixedResolutionBuffer(self, radius, resolution) + frb = CylindricalFixedResolutionBuffer(self, width, resolution) return frb if center is None: diff --git a/yt/data_objects/tests/test_data_containers.py b/yt/data_objects/tests/test_data_containers.py index 2a7d0bcd894..2a887d8f634 100644 --- a/yt/data_objects/tests/test_data_containers.py +++ b/yt/data_objects/tests/test_data_containers.py @@ -5,16 +5,16 @@ from numpy.testing import assert_array_equal from yt.data_objects.data_containers import YTDataContainer -from yt.testing import assert_equal, fake_random_ds, fake_amr_ds, fake_particle_ds -from yt.utilities.exceptions import YTFieldNotFound - +from yt.testing import assert_equal, fake_random_ds, fake_amr_ds,\ + fake_particle_ds, requires_module +from yt.utilities.exceptions import YTFieldNotFound, YTException def test_yt_data_container(): # Test if ds could be None with assert_raises(RuntimeError) as err: YTDataContainer(None, None) - desired = 'Error: ds must be set either through class type or parameter' \ - ' to the constructor' + desired = ('Error: ds must be set either through class' + ' type or parameter to the constructor') assert_equal(str(err.exception), desired) # Test if field_data key exists @@ -24,37 +24,41 @@ def test_yt_data_container(): assert_equal('pz' in proj.keys(), False) # Delete the key and check if exits - proj.__delitem__('px') + del proj['px'] assert_equal('px' in proj.keys(), False) - proj.__delitem__('density') + del proj['density'] assert_equal('density' in proj.keys(), False) # Delete a non-existent field with assert_raises(YTFieldNotFound) as ex: - proj.__delitem__('p_mass') + del proj['p_mass'] desired = "Could not find field '('stream', 'p_mass')' in UniformGridData." assert_equal(str(ex.exception), desired) - # Test the convert method - assert_equal(proj.convert('HydroMethod'), -1) - def test_write_out(): filename = "sphere.txt" ds = fake_particle_ds() sp = ds.sphere(ds.domain_center, 0.25) - sp.write_out(filename) + sp.write_out(filename, fields=["cell_volume"]) with open(filename, "r") as file: file_row_1 = file.readline() file_row_2 = file.readline() file_row_2 = np.array(file_row_2.split('\t'), dtype=np.float64) sorted_keys = sorted(sp.field_data.keys()) - _keys = [str(k) for k in sorted_keys] - _keys = "\t".join(["#"] + _keys + ["\n"]) - _data = [sp.field_data[k][0] for k in sorted_keys] + keys = [str(k) for k in sorted_keys] + keys = "\t".join(["#"] + keys + ["\n"]) + data = [sp.field_data[k][0] for k in sorted_keys] + + assert_equal(keys, file_row_1) + assert_array_equal(data, file_row_2) - assert_equal(_keys, file_row_1) - assert_array_equal(_data, file_row_2) + # Test for exception + with assert_raises(YTException) as ex: + sp.write_out(filename, fields=["particle_position_x"]) + desired = ("Field type ['all'] of the supplied field ['particle_position_x']" + " is in consistent with field type 'gas'.") + assert_equal(str(ex.exception)[:50], desired[:50]) def test_save_object(): ds = fake_particle_ds() @@ -62,60 +66,54 @@ def test_save_object(): sp.save_object("my_sphere_1", filename="test_save_obj") obj = shelve.open("test_save_obj", protocol=-1) loaded_sphere = obj["my_sphere_1"][1] + obj.close() assert_array_equal(loaded_sphere.center, sp.center) assert_equal(loaded_sphere.radius, sp.radius) - obj.close() + for k in loaded_sphere._key_fields: + assert_array_equal(loaded_sphere[k], sp[k]) - sp.save_object("my_sphere_2") + # Object is saved but retrieval is not working + # sp.save_object("my_sphere_2") + # loaded_sphere = ds.index.load_object("my_sphere_2") + # for k in loaded_sphere._key_fields: + # assert_array_equal(loaded_sphere[k], sp[k]) +@requires_module("pandas") def test_to_dataframe(): - try: - import pandas as pd - pd # to ignore Flake8 error - fields = ["density", "velocity_z"] - ds = fake_random_ds(6) - dd = ds.all_data() - df1 = dd.to_dataframe(fields) - assert_array_equal(dd[fields[0]], df1[fields[0]]) - assert_array_equal(dd[fields[1]], df1[fields[1]]) - except ImportError: - pass + fields = ["density", "velocity_z"] + ds = fake_random_ds(6) + dd = ds.all_data() + df1 = dd.to_dataframe(fields) + assert_array_equal(dd[fields[0]], df1[fields[0]]) + assert_array_equal(dd[fields[1]], df1[fields[1]]) def test_std(): ds = fake_random_ds(3) ds.all_data().std('density', weight="velocity_z") def test_to_frb(): - ds = fake_amr_ds(fields=["density", "cell_mass"], geometry="cylindrical", - particles=16**3) - proj = ds.proj("density", weight_field="cell_mass", axis=1, - data_source=ds.all_data()) - proj.to_frb((1.0, 'unitary'), 64) + # Test cylindrical geometry + fields = ["density", "cell_mass"] + ds = fake_amr_ds(fields=fields, geometry="cylindrical", particles=16**3) + dd = ds.all_data() + proj = ds.proj("density", weight_field="cell_mass", axis=1, data_source=dd) + frb = proj.to_frb((1.0, 'unitary'), 64) + assert_equal(frb.radius, (1.0, 'unitary')) + assert_equal(frb.buff_size, 64) def test_extract_isocontours(): # Test isocontour properties for AMRGridData ds = fake_amr_ds(fields=["density", "cell_mass"], particles=16**3) dd = ds.all_data() - rho = dd.quantities["WeightedAverageQuantity"]("density", - weight="cell_mass") + q = dd.quantities["WeightedAverageQuantity"] + rho = q("density", weight="cell_mass") dd.extract_isocontours("density", rho, "triangles.obj", True) - dd.calculate_isocontour_flux("density", rho, "x", "y", "z", - "dx") + dd.calculate_isocontour_flux("density", rho, "x", "y", "z", "dx") # Test error in case of ParticleData ds = fake_particle_ds() dd = ds.all_data() - rho = dd.quantities["WeightedAverageQuantity"]("particle_velocity_x", - weight="particle_mass") + q = dd.quantities["WeightedAverageQuantity"] + rho = q("particle_velocity_x", weight="particle_mass") with assert_raises(NotImplementedError): dd.extract_isocontours("density", rho, sample_values='x') - -def test_extract_connected_sets(): - - ds = fake_random_ds(16, nprocs=8, particles=16 ** 3) - data_source = ds.disk([0.5, 0.5, 0.5], [0., 0., 1.], (8, 'kpc'), (1, 'kpc')) - field = ("gas", "density") - min_val, max_val = data_source[field].min() / 2, data_source[field].max() / 2 - - data_source.extract_connected_sets(field, 3, min_val, max_val, - log_space=True, cumulative=True) diff --git a/yt/utilities/input_validator.py b/yt/utilities/input_validator.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/yt/utilities/tests/test_input_validator.py b/yt/utilities/tests/test_input_validator.py deleted file mode 100644 index e69de29bb2d..00000000000 From cb3bf7185978cf1fd38e9745e838234f820e4d0e Mon Sep 17 00:00:00 2001 From: git-abhishek <36498066+git-abhishek@users.noreply.github.com> Date: Thu, 14 Jun 2018 02:23:37 -0400 Subject: [PATCH 13/15] Resolved pandas dependency issue --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index caec2064af5..55b69685fd3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,7 @@ env: FLAKE8=flake8 CODECOV=codecov COVERAGE=coverage - PANDAS=pandas + PANDAS=pandas<0.21 before_install: - | @@ -59,7 +59,7 @@ jobs: include: - stage: lint python: 3.4 - env: NUMPY=numpy==1.10.4 CYTHON=cython==0.24 MATPLOTLIB= SYMPY= H5PY= SCIPY= FASTCACHE= IPYTHON= + env: NUMPY=numpy==1.10.4 CYTHON=cython==0.24 MATPLOTLIB= SYMPY= H5PY= SCIPY= FASTCACHE= IPYTHON= PANDAS= script: flake8 yt/ - stage: tests From 4f8fb5dac6480d47d56c920bbbb6dc1a624eb7d9 Mon Sep 17 00:00:00 2001 From: git-abhishek <36498066+git-abhishek@users.noreply.github.com> Date: Thu, 14 Jun 2018 14:51:13 -0400 Subject: [PATCH 14/15] Updated Lint stage to run in py3.6; pinned pandas for py3.4 --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 55b69685fd3..97cd65dc20b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,7 @@ env: FLAKE8=flake8 CODECOV=codecov COVERAGE=coverage - PANDAS=pandas<0.21 + PANDAS=pandas before_install: - | @@ -58,7 +58,7 @@ install: jobs: include: - stage: lint - python: 3.4 + python: 3.6 env: NUMPY=numpy==1.10.4 CYTHON=cython==0.24 MATPLOTLIB= SYMPY= H5PY= SCIPY= FASTCACHE= IPYTHON= PANDAS= script: flake8 yt/ @@ -74,7 +74,7 @@ jobs: - stage: tests python: 3.4 - env: FLAKE8= + env: FLAKE8= PANDAS=pandas==0.20 script: coverage run $(which nosetests) -c nose.cfg yt - stage: tests From d12ca6e1033c61c2c2c4fa7a443f5e74d4b5b30e Mon Sep 17 00:00:00 2001 From: git-abhishek <36498066+git-abhishek@users.noreply.github.com> Date: Mon, 18 Jun 2018 16:26:58 -0400 Subject: [PATCH 15/15] using with statement for file write ops --- yt/data_objects/data_containers.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/yt/data_objects/data_containers.py b/yt/data_objects/data_containers.py index 3764fa846d8..f503fc16602 100644 --- a/yt/data_objects/data_containers.py +++ b/yt/data_objects/data_containers.py @@ -458,14 +458,13 @@ def write_out(self, filename, fields=None, format="%0.16e"): raise YTException(msg) for field in field_order: self[field] - fid = open(filename, "w") - field_header = [str(f) for f in field_order] - fid.write("\t".join(["#"] + field_header + ["\n"])) - field_data = np.array([self.field_data[field] for field in field_order]) - for line in range(field_data.shape[1]): - field_data[:, line].tofile(fid, sep="\t", format=format) - fid.write("\n") - fid.close() + with open(filename, "w") as fid: + field_header = [str(f) for f in field_order] + fid.write("\t".join(["#"] + field_header + ["\n"])) + field_data = np.array([self.field_data[field] for field in field_order]) + for line in range(field_data.shape[1]): + field_data[:, line].tofile(fid, sep="\t", format=format) + fid.write("\n") def save_object(self, name, filename=None): """