From f0243bdb9f93fb1ca2b160db39f674426c339e9e Mon Sep 17 00:00:00 2001 From: Joep Vanlier Date: Thu, 4 Jul 2024 21:51:28 +0200 Subject: [PATCH 1/5] tracker: allow widget backend --- changelog.md | 1 + lumicks/pylake/nb_widgets/kymotracker_widgets.py | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/changelog.md b/changelog.md index df3da3f3f..0f39c913d 100644 --- a/changelog.md +++ b/changelog.md @@ -3,6 +3,7 @@ ## v1.5.1 | 2024-06-03 * Fixed bug that prevented loading an `h5` file where only a subset of the photon channels are available. This bug was introduced in Pylake `1.4.0`. +* Fixed bug that prevented opening the kymotracking widget when using it with the `widget` backend on `matplotlib >= 3.9.0`. ## v1.5.0 | 2024-05-28 diff --git a/lumicks/pylake/nb_widgets/kymotracker_widgets.py b/lumicks/pylake/nb_widgets/kymotracker_widgets.py index d0f6c4f8d..682aacebd 100644 --- a/lumicks/pylake/nb_widgets/kymotracker_widgets.py +++ b/lumicks/pylake/nb_widgets/kymotracker_widgets.py @@ -418,13 +418,17 @@ def _create_widgets(self): import matplotlib.pyplot as plt from IPython.display import display - if not max([backend in plt.get_backend() for backend in ("nbAgg", "ipympl")]): + if not max( + # Note: Some, but not all versions of matplotlib lower the backend names. Hence, we + # always lower them to be on the safe side. + [backend in plt.get_backend().lower() for backend in ("nbagg", "ipympl", "widget")] + ): raise RuntimeError( ( "Please enable an interactive matplotlib backend for this plot to work. In " "jupyter notebook or lab you can do this by invoking either " - "%matplotlib widget. Please note that you may have to restart the notebook " - "kernel for this to work." + "%matplotlib widget or %matplotlib ipympl. Please note that you may have to " + "restart the notebook kernel for this to work." ) ) From 0dd6e6122c165f017337b2bc260de06aaa36c056 Mon Sep 17 00:00:00 2001 From: Joep Vanlier Date: Tue, 2 Jul 2024 15:19:02 +0200 Subject: [PATCH 2/5] kymotrack: load photon counts from file --- changelog.md | 8 +- lumicks/pylake/kymotracker/kymotrack.py | 71 ++++++++++++++- lumicks/pylake/kymotracker/tests/conftest.py | 19 ++++ lumicks/pylake/kymotracker/tests/test_io.py | 96 +++++++++++++------- 4 files changed, 155 insertions(+), 39 deletions(-) diff --git a/changelog.md b/changelog.md index 0f39c913d..c25f1a4de 100644 --- a/changelog.md +++ b/changelog.md @@ -1,9 +1,15 @@ # Changelog +## v1.5.2 | t.b.d. + +#### Bug fixes + +* Fixed bug that prevented opening the kymotracking widget when using it with the `widget` backend on `matplotlib >= 3.9.0`. +* Fixed bug where photon counts were not being loaded from a csv file generated with the kymotracker. + ## v1.5.1 | 2024-06-03 * Fixed bug that prevented loading an `h5` file where only a subset of the photon channels are available. This bug was introduced in Pylake `1.4.0`. -* Fixed bug that prevented opening the kymotracking widget when using it with the `widget` backend on `matplotlib >= 3.9.0`. ## v1.5.0 | 2024-05-28 diff --git a/lumicks/pylake/kymotracker/kymotrack.py b/lumicks/pylake/kymotracker/kymotrack.py index 5522c72f9..9fe78f154 100644 --- a/lumicks/pylake/kymotracker/kymotrack.py +++ b/lumicks/pylake/kymotracker/kymotrack.py @@ -163,6 +163,40 @@ def store_column(column_title, format_string, new_data): np.savetxt(filename, data, fmt=fmt, header=header, delimiter=delimiter) +def _check_summing_mismatch(track, sampling_width): + """Checks calling sample_from_image on a loaded track reproduces the same result as is in the + file.""" + wrong_kymo_warning = ( + "Photon counts do not match the photon counts found in the file. It is " + "possible that the loaded kymo or channel doesn't match the one used to " + "create this file." + ) + try: + if not np.allclose( + track.sample_from_image((sampling_width - 1) // 2, correct_origin=True), + track.photon_counts, + ): + if np.allclose( + track.sample_from_image((sampling_width - 1) // 2, correct_origin=False), + track.photon_counts, + ): + return RuntimeWarning( + "Photon counts do not match the photon counts found in the file. Prior to " + "Pylake v1.1.0, the method `sample_from_image` had a bug that assumed the " + "origin of a pixel to be at the edge rather than the center of the pixel. " + "Consequently, the sampled window could be off by one pixel. This file was " + "likely created using the incorrect origin. " + "Note that Pylake loaded the counts found in the file as is, so if the " + "used summing window was very small, there may be a bias in the counts." + "To recreate these counts without bias invoke:" + f"`track.sample_from_image({(sampling_width - 1) // 2}, correct_origin=True)`" + ) + else: + return RuntimeWarning(wrong_kymo_warning) + except IndexError: + return RuntimeWarning(wrong_kymo_warning) + + def import_kymotrackgroup_from_csv(filename, kymo, channel, delimiter=";"): """Import a KymoTrackGroup from a csv file. @@ -210,9 +244,13 @@ def import_kymotrackgroup_from_csv(filename, kymo, channel, delimiter=";"): stacklevel=2, ) - def create_track(time, coord, min_length=None): + def create_track(time, coord, min_length=None, counts=None): if min_length is not None: min_length = float(np.unique(min_length).squeeze()) + + if counts is not None: + coord = CentroidLocalizationModel(coord * kymo.pixelsize_um, counts) + return KymoTrack(time.astype(int), coord, kymo, channel, min_length) if csv_version == 3: @@ -232,11 +270,34 @@ def create_track(time, coord, min_length=None): else: min_duration_field = "minimum observable duration (seconds)" - if min_duration_field in data: - mandatory_fields.append(min_duration_field) + count_field = [key for key in data.keys() if "counts" in key] + sampling_width = None + if count_field: + count_field = count_field[0] + if match := re.findall(r"over (\d*) pixels", count_field): + sampling_width = int(match[0]) + + tracks = [] + resampling_mismatch = None + for track_idx in range(len(data[mandatory_fields[0]])): + tracks.append( + create_track( + time=data[mandatory_fields[0]][track_idx], + coord=data[mandatory_fields[1]][track_idx], + min_length=data[min_duration_field][track_idx] + if min_duration_field in data + else None, + counts=data[count_field][track_idx] if count_field else None, + ) + ) + + if sampling_width is not None: + resampling_mismatch = _check_summing_mismatch(tracks[-1], sampling_width) + + if resampling_mismatch: + warnings.warn(resampling_mismatch, stacklevel=2) - data = [data[f] for f in mandatory_fields] - return KymoTrackGroup([create_track(*track_data) for track_data in zip(*data)]) + return KymoTrackGroup(tracks) class KymoTrack: diff --git a/lumicks/pylake/kymotracker/tests/conftest.py b/lumicks/pylake/kymotracker/tests/conftest.py index f5d1d953b..ee77c672d 100644 --- a/lumicks/pylake/kymotracker/tests/conftest.py +++ b/lumicks/pylake/kymotracker/tests/conftest.py @@ -33,6 +33,25 @@ def kymo_integration_test_data(): ) +@pytest.fixture +def kymo_integration_tracks(kymo_integration_test_data): + track_coordinates = [ + (np.arange(10, 20), np.full(10, 11)), + (np.arange(15, 25), np.full(10, 21.51)), + ] + + tracks = KymoTrackGroup( + [ + KymoTrack( + np.array(time_idx), np.array(position_idx), kymo_integration_test_data, "red", 0.1 + ) + for time_idx, position_idx in track_coordinates + ] + ) + + return tracks + + @pytest.fixture def kymo_pixel_calibrations(): image = raw_test_data() diff --git a/lumicks/pylake/kymotracker/tests/test_io.py b/lumicks/pylake/kymotracker/tests/test_io.py index 429337ce9..657abf5a8 100644 --- a/lumicks/pylake/kymotracker/tests/test_io.py +++ b/lumicks/pylake/kymotracker/tests/test_io.py @@ -1,7 +1,6 @@ import io import re import inspect -import contextlib from copy import copy from pathlib import Path @@ -96,8 +95,9 @@ def test_kymotrackgroup_io(tmpdir_factory, dt, dx, delimiter, sampling_width, sa assert len([key for key in data.keys() if "counts" in key]) == 0 else: count_field = [key for key in data.keys() if "counts" in key][0] - for track1, cnt in zip(tracks, data[count_field]): + for track1, imported_track, cnt in zip(tracks, imported_tracks, data[count_field]): np.testing.assert_allclose([sampling_outcome] * len(track1.coordinate_idx), cnt) + np.testing.assert_allclose(imported_track.photon_counts, cnt) def test_export_sources(tmpdir_factory): @@ -128,7 +128,7 @@ def test_export_sources(tmpdir_factory): [[";", 0, True], [",", 0, True], [";", 1, True], [";", None, True]], ) def test_roundtrip_without_file( - delimiter, sampling_width, correct_origin, kymo_integration_test_data + delimiter, sampling_width, correct_origin, kymo_integration_test_data, kymo_integration_tracks ): # Validate that this also works when providing a string handle (this is the API LV uses). @@ -137,43 +137,67 @@ def get_args(func): # This helps us ensure that if we get additional arguments to this function, we don't forget to # add them to the parametrization here. - assert set(get_args(KymoTrackGroup.save)[2:]) == set(get_args(test_roundtrip_without_file)[:-1]) - - track_coordinates = [ - ((1, 2, 3), (2, 3, 4)), - ((2, 3, 4), (3, 4, 5)), - ((3, 4, 5), (4, 5, 6)), - ((4, 5, 6), (5, 6, 7)), - ] - - tracks = KymoTrackGroup( - [ - KymoTrack( - np.array(time_idx), np.array(position_idx), kymo_integration_test_data, "red", 0.1 - ) - for time_idx, position_idx in track_coordinates - ] - ) + assert set(get_args(KymoTrackGroup.save)[2:]) == set(get_args(test_roundtrip_without_file)[:-2]) with io.StringIO() as s: - tracks.save( + kymo_integration_tracks.save( s, delimiter=delimiter, sampling_width=sampling_width, correct_origin=correct_origin ) string_representation = s.getvalue() with io.StringIO(string_representation) as s: read_tracks = import_kymotrackgroup_from_csv( - s, kymo_integration_test_data, "green", delimiter=delimiter + s, kymo_integration_test_data, "red", delimiter=delimiter + ) + + compare_kymotrack_group(kymo_integration_tracks, read_tracks) + + +def test_photon_count_validation(kymo_integration_test_data, kymo_integration_tracks): + with io.StringIO() as s: + kymo_integration_tracks.save(s, sampling_width=0, correct_origin=False) + biased_tracks = s.getvalue() + + with io.StringIO() as s: + kymo_integration_tracks.save(s, sampling_width=0, correct_origin=True) + good_tracks = s.getvalue() + + with pytest.warns( + RuntimeWarning, + match="origin of a pixel to be at the edge rather than the center of the pixel", + ): + _ = import_kymotrackgroup_from_csv( + io.StringIO(biased_tracks), kymo_integration_test_data, "red" + ) + + # We can also fail by having the wrong kymo + with pytest.warns( + RuntimeWarning, + match="loaded kymo or channel doesn't match the one used to create this file", + ): + _ = import_kymotrackgroup_from_csv( + io.StringIO(good_tracks), kymo_integration_test_data, "green" + ) + + # Or by having the wrong one where it actually completely fails to sample. This tests whether + # the exception inside import_kymotrackgroup_from_csv is correctly caught and handled + with pytest.warns( + RuntimeWarning, + match="loaded kymo or channel doesn't match the one used to create this file", + ): + _ = import_kymotrackgroup_from_csv( + io.StringIO(good_tracks), kymo_integration_test_data[:"1s"], "red" ) - compare_kymotrack_group(tracks, read_tracks) + # Control for good tracks + import_kymotrackgroup_from_csv(io.StringIO(good_tracks), kymo_integration_test_data, "red") @pytest.mark.parametrize( "version, read_with_version", [[0, False], [1, False], [2, True], [3, True], [4, True]], ) -def test_csv_version(version, read_with_version): +def test_csv_version(version, read_with_version, recwarn): # Test that header is parsed properly on CSV import # Version 2 has 2 header lines, <2 only has 1 header line @@ -198,14 +222,15 @@ def test_csv_version(version, read_with_version): ) testfile = Path(__file__).parent / f"./data/tracks_v{version}.csv" - with pytest.warns( - RuntimeWarning, - match=( - "This CSV file is from a version in which the minimum observable track duration " - "was incorrectly rounded to an integer." - ), - ) if version == 3 else contextlib.nullcontext(): - imported_tracks = import_kymotrackgroup_from_csv(testfile, kymo, "red", delimiter=";") + imported_tracks = import_kymotrackgroup_from_csv(testfile, kymo, "red", delimiter=";") + + match version: + case 3: + assert "minimum observable track duration" in str(recwarn[0].message) + case 4: + assert "loaded kymo or channel doesn't match the one used to create this file" in str( + recwarn[0].message + ) data, pylake_version, csv_version = _read_txt(testfile, ";") @@ -229,7 +254,12 @@ def test_bad_csv(filename, blank_kymo): def test_min_obs_csv_regression(tmpdir_factory, blank_kymo): """This tests a regression where saving a freshly imported older file does not function""" testfile = Path(__file__).parent / f"./data/tracks_v0.csv" - imported_tracks = import_kymotrackgroup_from_csv(testfile, blank_kymo, "red", delimiter=";") + with pytest.warns( + RuntimeWarning, + match="loaded kymo or channel doesn't match the one used to create this file", + ): + imported_tracks = import_kymotrackgroup_from_csv(testfile, blank_kymo, "red", delimiter=";") + out_file = f"{tmpdir_factory.mktemp('pylake')}/no_min_lengths.csv" err_msg = "Loaded tracks have no minimum length metadata defined" From 84963de0298bb4676e1f1f02af8298429daa87ac Mon Sep 17 00:00:00 2001 From: Joep Vanlier Date: Mon, 22 Jul 2024 14:42:43 +0200 Subject: [PATCH 3/5] scan: allow loading truncated scans --- changelog.md | 4 ++ lumicks/pylake/channel.py | 8 ++++ lumicks/pylake/detail/confocal.py | 37 ++++++++++++++++--- .../tests/test_imaging_confocal/test_kymo.py | 37 +++++++++++++++++++ 4 files changed, 81 insertions(+), 5 deletions(-) diff --git a/changelog.md b/changelog.md index c25f1a4de..24430ed93 100644 --- a/changelog.md +++ b/changelog.md @@ -2,6 +2,10 @@ ## v1.5.2 | t.b.d. +#### Improvements + +* Added a fallback which allows loading scans or kymographs that have truncated photon count channels. + #### Bug fixes * Fixed bug that prevented opening the kymotracking widget when using it with the `widget` backend on `matplotlib >= 3.9.0`. diff --git a/lumicks/pylake/channel.py b/lumicks/pylake/channel.py index 39437b5cf..c1e4e2c71 100644 --- a/lumicks/pylake/channel.py +++ b/lumicks/pylake/channel.py @@ -849,6 +849,14 @@ def data(self) -> npt.ArrayLike: def timestamps(self) -> npt.ArrayLike: return np.empty(0) + @property + def start(self): + return None + + @property + def stop(self): + return None + empty_slice = Slice(Empty()) diff --git a/lumicks/pylake/detail/confocal.py b/lumicks/pylake/detail/confocal.py index 0c2916085..e290cf7bb 100644 --- a/lumicks/pylake/detail/confocal.py +++ b/lumicks/pylake/detail/confocal.py @@ -40,11 +40,37 @@ def timestamp_mean(a, axis=None): return minimum + _int_mean(a - minimum, a.size if axis is None else a.shape[axis], axis) -def _default_image_factory(self: "ConfocalImage", color): +def _get_confocal_data(self: "ConfocalImage", color): channel = getattr(self, f"{color}_photon_count") + infowave = self.infowave + + # Early out for an empty channel + if not channel: + return channel, infowave + + if infowave.stop and channel.stop and len(infowave.data) != len(channel.data): + if channel.stop < infowave.stop: + warnings.warn( + RuntimeWarning( + f"Warning: {self.__class__.__name__} is truncated. Photon count data ends " + f"{(infowave.stop - channel.stop) / 1e9:.2g} seconds before the end of the " + "info wave (which encodes how the data should be read)." + ) + ) + + return ( + channel[: min(infowave.stop, channel.stop)], + infowave[: min(infowave.stop, channel.stop)], + ) + else: + return channel, infowave + + +def _default_image_factory(self: "ConfocalImage", color): + channel, infowave = _get_confocal_data(self, color) raw_image = reconstruct_image_sum( - channel.data.astype(float) if channel else np.zeros(self.infowave.data.size), - self.infowave.data, + channel.data.astype(float) if channel else np.zeros(infowave.data.size), + infowave.data, self._reconstruction_shape, ) return self._to_spatial(raw_image) @@ -53,14 +79,15 @@ def _default_image_factory(self: "ConfocalImage", color): def _default_timestamp_factory(self: "ConfocalImage", reduce=timestamp_mean): # Uses the timestamps from the first non-zero-sized photon channel for color in ("red", "green", "blue"): - channel_data = getattr(self, f"{color}_photon_count").timestamps + channel_data, infowave = _get_confocal_data(self, color) + channel_data = channel_data.timestamps if len(channel_data) != 0: break else: raise RuntimeError("Can't get pixel timestamps if there are no pixels") raw_image = reconstruct_image( - channel_data, self.infowave.data, self._reconstruction_shape, reduce=reduce + channel_data, infowave.data, self._reconstruction_shape, reduce=reduce ) return self._to_spatial(raw_image) diff --git a/lumicks/pylake/tests/test_imaging_confocal/test_kymo.py b/lumicks/pylake/tests/test_imaging_confocal/test_kymo.py index 5bb44c5ad..29245a853 100644 --- a/lumicks/pylake/tests/test_imaging_confocal/test_kymo.py +++ b/lumicks/pylake/tests/test_imaging_confocal/test_kymo.py @@ -1,3 +1,5 @@ +import copy + import numpy as np import pytest @@ -129,6 +131,41 @@ def test_partial_pixel_kymo(): np.testing.assert_equal(kymo.get_image("red")[-2, -1], 0) +@pytest.mark.parametrize("samples_from_end, pixels_from_end", [(1, 1), (47, 1), (48, 2)]) +def test_unequal_size(samples_from_end, pixels_from_end): + """This function checks whether the handling for kymographs with differences between the length + of the infowave and photon counts get reconstructed correctly.""" + dt = int(1e9 / 78125) + np.random.seed(15451345) + kymo = generate_kymo( + "Mock", + (np.random.rand(5, 5) * 100).astype(int), + pixel_size_nm=100, + start=1536582124217030400, + dt=dt, + samples_per_pixel=47, + line_padding=0, + ) + + # we need to make a copy of the kymo to make sure the original doesn't get cached + kymo_copy = copy.copy(kymo) + ref_img = copy.copy(kymo_copy.get_image("red")) + ref_ts = copy.copy(kymo_copy.timestamps) + + # Make the photon counts shorter than the info-wave (reproduce truncated output). + ch = kymo.file.red_photon_count + kymo.file.red_photon_count._src = ch[: ch.timestamps[-samples_from_end]]._src + + with pytest.warns(RuntimeWarning, match="Kymo is truncated"): + kymo_truncated = kymo.get_image("red") + timestamps_truncated = kymo.timestamps + + ref_img[-pixels_from_end:, -1] = 0 + ref_ts[-pixels_from_end:, -1] = 0 + np.testing.assert_equal(kymo_truncated, ref_img) + np.testing.assert_equal(timestamps_truncated, ref_ts) + + @pytest.mark.parametrize( "data, ref_line_time, pixels_per_line", [ From 6570ede3ba2cee6100df0de0973c8e9d96df303e Mon Sep 17 00:00:00 2001 From: Joep Vanlier Date: Mon, 22 Jul 2024 13:26:09 +0200 Subject: [PATCH 4/5] install: don't add lower bound to instructions --- docs/install.rst | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/docs/install.rst b/docs/install.rst index 228a210e1..aa1d5f078 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -29,7 +29,7 @@ The easiest way to install Python and SciPy is with `Anaconda`_, a free scientif .. code-block:: python - conda create -n pylake conda>=23.7.2 + conda create -n pylake #. The environment can then be activated:: @@ -362,15 +362,8 @@ The full error message is:: Exception: HTTPSConnectionPool(host='conda.anaconda.org', port=443): Max retries exceeded with url: /conda-forge/win-64/current_repodata.json (Caused by SSLError("Can't connect to HTTPS URL because the SSL module is not available.")) -This issue has been solved upstream by conda. Make sure you install a new enough version:: - - conda create -n pylake conda>=23.7.2 - -And then follow the rest of the installation instructions. -If you already have an environment named pylake, you can remove this environment, before creating it again. Another option is to create an environment with a different name, eg:: - - conda create -n pylake2 conda>=23.7.2 - conda activate pylake2 +This issue has been solved upstream by conda. +The best option is to reinstall `conda` and then follow the rest of the installation instructions. **I tried the installation instructions, but I cannot import Pylake inside a Jupyter notebook** From 90ffcd731b6dd92ac0d49f29a8f2de287d1d3301 Mon Sep 17 00:00:00 2001 From: Joep Vanlier Date: Mon, 22 Jul 2024 10:29:52 +0200 Subject: [PATCH 5/5] release: release Pylake v1.5.2 --- changelog.md | 2 +- lumicks/pylake/__about__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/changelog.md b/changelog.md index 24430ed93..ed0d34893 100644 --- a/changelog.md +++ b/changelog.md @@ -1,6 +1,6 @@ # Changelog -## v1.5.2 | t.b.d. +## v1.5.2 | 2024-07-24 #### Improvements diff --git a/lumicks/pylake/__about__.py b/lumicks/pylake/__about__.py index 43edd4d32..1b590b424 100644 --- a/lumicks/pylake/__about__.py +++ b/lumicks/pylake/__about__.py @@ -1,5 +1,5 @@ __title__ = "lumicks.pylake" -__version__ = "1.5.1" +__version__ = "1.5.2" __summary__ = "Bluelake data analysis tools" __url__ = "https://github.com/lumicks/pylake"