From 6c4fd6cd8dd4b22e84ead6ac2c1594b3eb460bc2 Mon Sep 17 00:00:00 2001 From: Joep Vanlier Date: Thu, 1 Aug 2024 16:35:26 +0200 Subject: [PATCH] lowlevel: add low level api to create objects Adds tested functions to directly create confocal objects and slices --- lumicks/pylake/benchmark.py | 14 +- lumicks/pylake/detail/confocal.py | 30 ++- lumicks/pylake/detail/imaging_mixins.py | 2 +- lumicks/pylake/kymo.py | 2 +- lumicks/pylake/lowlevel.py | 61 ++++++ lumicks/pylake/scan.py | 2 +- lumicks/pylake/tests/data/mock_confocal.py | 196 +++++++++--------- .../tests/test_channels/test_channels.py | 23 +- .../tests/test_imaging_confocal/conftest.py | 7 +- .../tests/test_imaging_confocal/test_kymo.py | 6 +- 10 files changed, 221 insertions(+), 122 deletions(-) create mode 100644 lumicks/pylake/lowlevel.py diff --git a/lumicks/pylake/benchmark.py b/lumicks/pylake/benchmark.py index 94d137e4d..8b3937339 100644 --- a/lumicks/pylake/benchmark.py +++ b/lumicks/pylake/benchmark.py @@ -2,6 +2,7 @@ import timeit import tempfile import contextlib +from copy import copy import numpy as np @@ -41,14 +42,15 @@ def _generate_kymo_for_tracking(duration, line_count, samples_per_pixel=1): def _generate_blank_kymo_data(samples=1000000): """This is a different function since generating a junk data kymo is significantly faster than generating one with sensible image content.""" - from lumicks.pylake.tests.data.mock_confocal import MockConfocalFile + from lumicks.pylake.tests.data.mock_confocal import mock_confocal_from_arrays counts = np.ones(samples) infowave = np.ones(samples) infowave[::2] = 0 infowave[::10] = 2 - return MockConfocalFile.from_streams( + return mock_confocal_from_arrays( + "kymo", start=0, dt=int(1e9), axes=[0], @@ -86,8 +88,8 @@ class _KymoImage(_Benchmark): loops = 60 def context(self): - confocal_file, metadata, stop = _generate_blank_kymo_data() - yield lambda: lk.kymo.Kymo("big_kymo", confocal_file, 0, stop, metadata).get_image("red") + kymo = _generate_blank_kymo_data() + yield lambda: copy(kymo).get_image("red") # Copy to not use the cache! class _KymoTimestamps(_Benchmark): @@ -95,8 +97,8 @@ class _KymoTimestamps(_Benchmark): loops = 45 def context(self): - confocal_file, metadata, stop = _generate_blank_kymo_data() - yield lambda: lk.kymo.Kymo("big_kymo", confocal_file, 0, stop, metadata).timestamps + kymo = _generate_blank_kymo_data() + yield lambda: copy(kymo).timestamps # Copy to not use the cache! class _Tracking(_Benchmark): diff --git a/lumicks/pylake/detail/confocal.py b/lumicks/pylake/detail/confocal.py index e290cf7bb..ed645aa36 100644 --- a/lumicks/pylake/detail/confocal.py +++ b/lumicks/pylake/detail/confocal.py @@ -6,6 +6,8 @@ import numpy as np from numpy import typing as npt +from lumicks.pylake.channel import Slice, empty_slice + from .image import reconstruct_image, reconstruct_image_sum from .mixin import PhotonCounts, ExcitationLaserPower from .plotting import parse_color_channel @@ -157,7 +159,9 @@ def scan_order(self): @classmethod def from_json(cls, json_string): - json_dict = json.loads(json_string)["value0"] + json_dict = json.loads(json_string) + if "value0" in json_dict: + json_dict = json_dict["value0"] axes = [ ScanAxis(ax["axis"], ax["num of pixels"], ax["pixel size (nm)"] / 1000) @@ -167,6 +171,28 @@ def from_json(cls, json_string): return cls(axes, json_dict["scan volume"]["center point (um)"], json_dict["scan count"]) +class ConfocalFileProxy: + """Class with the minimal requirements to reconstruct confocal scans from photon streams""" + + def __init__( + self, + infowave: Slice, + red_channel: Slice = empty_slice, + green_channel: Slice = empty_slice, + blue_channel: Slice = empty_slice, + ): + self.infowave = infowave + self.red_photon_count = red_channel + self.green_photon_count = green_channel + self.blue_photon_count = blue_channel + + def __getitem__(self, key): + if key == "Info wave": + return {"Info wave": self.infowave} + else: + raise KeyError(f"Key {key} not found.") + + class BaseScan(PhotonCounts, ExcitationLaserPower): """Base class for confocal scans @@ -174,7 +200,7 @@ class BaseScan(PhotonCounts, ExcitationLaserPower): ---------- name : str confocal scan name - file : lumicks.pylake.File + file : lumicks.pylake.File | lumicks.pylake.ConfocalFile Parent file. Contains the channel data. start : int Start point in the relevant info wave. diff --git a/lumicks/pylake/detail/imaging_mixins.py b/lumicks/pylake/detail/imaging_mixins.py index d110b01f9..63c54f238 100644 --- a/lumicks/pylake/detail/imaging_mixins.py +++ b/lumicks/pylake/detail/imaging_mixins.py @@ -9,7 +9,7 @@ from .timeindex import to_timestamp from ..adjustments import no_adjustment -_FIRST_TIMESTAMP = 1388534400 +_FIRST_TIMESTAMP = 1388534400000000000 class TiffExport: diff --git a/lumicks/pylake/kymo.py b/lumicks/pylake/kymo.py index fca9b2f5a..5480f71d0 100644 --- a/lumicks/pylake/kymo.py +++ b/lumicks/pylake/kymo.py @@ -71,7 +71,7 @@ class Kymo(ConfocalImage): ---------- name : str Kymograph name - file : lumicks.pylake.File + file : lumicks.pylake.File | lumicks.pylake.detail.confocal.ConfocalFile Parent file. Contains the channel data. start : int Start point in the relevant info wave. diff --git a/lumicks/pylake/lowlevel.py b/lumicks/pylake/lowlevel.py new file mode 100644 index 000000000..3a87aeba4 --- /dev/null +++ b/lumicks/pylake/lowlevel.py @@ -0,0 +1,61 @@ +import numpy as np + +from lumicks.pylake.kymo import Kymo +from lumicks.pylake.scan import Scan +from lumicks.pylake.channel import Slice, Continuous, empty_slice +from lumicks.pylake.point_scan import PointScan +from lumicks.pylake.detail.confocal import ScanMetaData, ConfocalFileProxy +from lumicks.pylake.detail.imaging_mixins import _FIRST_TIMESTAMP + + +def create_confocal_object( + name, + infowave, + json_metadata, + red_channel=empty_slice, + green_channel=empty_slice, + blue_channel=empty_slice, +) -> Kymo | Scan | PointScan: + """Create a confocal object from slices and json metadata + + Parameters + ---------- + name : str + Name of this object + infowave : lumicks.pylake.Slice + Info wave that encodes how the photon counts should be assembled into an image. + red_channel, green_channel, blue_channel : lumicks.pylake.Slice + Photon counts + json_metadata : str + json metadata generated by Bluelake. + """ + metadata = ScanMetaData.from_json(json_metadata) + file = ConfocalFileProxy(infowave, red_channel, green_channel, blue_channel) + confocal_cls = {0: PointScan, 1: Kymo, 2: Scan} + return confocal_cls[metadata.num_axes](name, file, infowave.start, infowave.stop, metadata) + + +def make_continuous_slice(data, start, dt, y_label="y", name="") -> Slice: + """Make a continuous slice of data + + Converts a raw `array_like` to a pylake `Slice`. + + Parameters + ---------- + data : array_like + Source of data + start : int + Start timestamp in nanoseconds since epoch + dt : int + Timestep in nanoseconds + y_label : str + Label to show on the y-axis. + name : str + Name of the slice (used on plot titles). + """ + if start < _FIRST_TIMESTAMP: + raise ValueError( + f"Starting timestamp must be larger than {_FIRST_TIMESTAMP}. You provided: {start}." + ) + + return Slice(Continuous(np.asarray(data), start, dt), labels={"title": name, "y": y_label}) diff --git a/lumicks/pylake/scan.py b/lumicks/pylake/scan.py index 6d14ae163..169d15bf3 100644 --- a/lumicks/pylake/scan.py +++ b/lumicks/pylake/scan.py @@ -18,7 +18,7 @@ class Scan(ConfocalImage, VideoExport, FrameIndex): ---------- name : str Scan name - file : lumicks.pylake.File + file : lumicks.pylake.File | lumicks.pylake.detail.confocal.ConfocalFile Parent file. Contains the channel data. start : int Start point in the relevant info wave. diff --git a/lumicks/pylake/tests/data/mock_confocal.py b/lumicks/pylake/tests/data/mock_confocal.py index 783cf0e1b..78a1915d8 100644 --- a/lumicks/pylake/tests/data/mock_confocal.py +++ b/lumicks/pylake/tests/data/mock_confocal.py @@ -3,15 +3,21 @@ from dataclasses import dataclass import numpy as np +from numpy.typing import ArrayLike from lumicks.pylake.kymo import Kymo from lumicks.pylake.scan import Scan from lumicks.pylake.channel import Slice, Continuous, empty_slice +from lumicks.pylake.lowlevel import create_confocal_object +from lumicks.pylake.point_scan import PointScan from lumicks.pylake.detail.image import InfowaveCode -from lumicks.pylake.detail.confocal import ScanMetaData, ConfocalImage +from lumicks.pylake.detail.confocal import ConfocalImage from .mock_json import mock_json +_default_start = int(20e9) +_default_dt = int(62.5e6) + def generate_scan_json(axes): """Generate a mock JSON for a Scan or Kymo. @@ -120,41 +126,74 @@ def generate_photon_count_line(line): ) -class MockConfocalFile: - def __init__(self, infowave, red_channel=None, green_channel=None, blue_channel=None): - self.infowave = infowave - self.red_photon_count = red_channel if red_channel is not None else empty_slice - self.green_photon_count = green_channel if green_channel is not None else empty_slice - self.blue_photon_count = blue_channel if blue_channel is not None else empty_slice +def mock_confocal_from_image( + name: str, + image: np.ndarray, + pixel_sizes_nm: list, + axes: list | None = None, + start: int = _default_start, + dt: int = _default_dt, + samples_per_pixel: int = 5, + line_padding: int = 3, + multi_color: bool = False, +) -> Kymo | Scan | PointScan: + """Generates a Kymo, Scan or PointScan depending on the number of axes""" + axes = [0, 1] if axes is None else axes + + if len(axes) == 2 and axes[0] < axes[1]: + # The fast axis of the physical scannning process has the lower physical axis number + # (e.g. x) and comes before the slow axis with the greater physical axis number (e.g. y + # or z). The convention of indexing numpy arrays is to have the slow indexed y axis + # before the fast indexed x axis. Therefore, flip the axes to ensure correct indexing + # order when creating the infowave: + image = image.swapaxes(-1 - multi_color, -2 - multi_color) + infowave, photon_counts = generate_image_data( + image, samples_per_pixel, line_padding, multi_color=multi_color + ) + json_string = generate_scan_json( + [ + { + "axis": axis, + "num of pixels": num_pixels, + "pixel size (nm)": pixel_size, + } + for pixel_size, axis, num_pixels in zip( + pixel_sizes_nm, axes, image.shape[-2 - multi_color : image.ndim - multi_color] + ) + ] + ) + + return create_confocal_object( + name, + Slice(Continuous(infowave, start=start, dt=dt)), + json_string, + *(Slice(Continuous(channel, start=start, dt=dt)) for channel in photon_counts), + ) + - def __getitem__(self, key): - if key == "Info wave": - return {"Info wave": self.infowave} +def mock_confocal_from_arrays( + name: str, + start: int, + dt: int, + axes: list, + num_pixels: list, + pixel_sizes_nm: list, + infowave: ArrayLike, + red_photon_counts: ArrayLike | None = None, + blue_photon_counts: ArrayLike | None = None, + green_photon_counts: ArrayLike | None = None, +) -> Kymo | Scan | PointScan: + """Generates a Kymo, Scan or PointScan depending on the number of axes""" + + def make_slice(data): + if data is None: + return empty_slice + else: + return Slice(Continuous(np.asarray(data), start, dt)) - @staticmethod - def from_image( - image, - pixel_sizes_nm, - axes=None, - start=int(20e9), - dt=int(62.5e6), - samples_per_pixel=5, - line_padding=3, - multi_color=False, - ): - """Generate a mock file that can be read by Kymo or Scan""" - axes = [0, 1] if axes is None else axes - - if len(axes) == 2 and axes[0] < axes[1]: - # The fast axis of the physical scannning process has the lower physical axis number - # (e.g. x) and comes before the slow axis with the greater physical axis number (e.g. y - # or z). The convention of indexing numpy arrays is to have the slow indexed y axis - # before the fast indexed x axis. Therefore, flip the axes to ensure correct indexing - # order when creating the infowave: - image = image.swapaxes(-1 - multi_color, -2 - multi_color) - infowave, photon_counts = generate_image_data( - image, samples_per_pixel, line_padding, multi_color=multi_color - ) + if axes == [] and num_pixels == [] and pixel_sizes_nm == []: + json_string = generate_scan_json([]) + else: json_string = generate_scan_json( [ { @@ -162,78 +201,32 @@ def from_image( "num of pixels": num_pixels, "pixel size (nm)": pixel_size, } - for pixel_size, axis, num_pixels in zip( - pixel_sizes_nm, axes, image.shape[-2 - multi_color : image.ndim - multi_color] - ) + for (axis, pixel_size, num_pixels) in zip(axes, pixel_sizes_nm, num_pixels) ] ) - return ( - MockConfocalFile( - Slice(Continuous(infowave, start=start, dt=dt)), - *(Slice(Continuous(channel, start=start, dt=dt)) for channel in photon_counts), - ), - ScanMetaData.from_json(json_string), - start + len(infowave) * dt, - ) - - @staticmethod - def from_streams( - start, - dt, - axes, - num_pixels, - pixel_sizes_nm, - infowave, - red_photon_counts=None, - blue_photon_counts=None, - green_photon_counts=None, - ): - def make_slice(data): - if data is None: - return empty_slice - else: - return Slice(Continuous(data, start, dt)) - - if axes == [] and num_pixels == [] and pixel_sizes_nm == []: - json_string = generate_scan_json([]) - else: - json_string = generate_scan_json( - [ - { - "axis": axis, - "num of pixels": num_pixels, - "pixel size (nm)": pixel_size, - } - for (axis, pixel_size, num_pixels) in zip(axes, pixel_sizes_nm, num_pixels) - ] - ) - - return ( - MockConfocalFile( - infowave=make_slice(infowave), - red_channel=make_slice(red_photon_counts), - blue_channel=make_slice(blue_photon_counts), - green_channel=make_slice(green_photon_counts), - ), - ScanMetaData.from_json(json_string), - start + len(infowave) * dt, - ) + return create_confocal_object( + name, + infowave=make_slice(infowave), + red_channel=make_slice(red_photon_counts), + blue_channel=make_slice(blue_photon_counts), + green_channel=make_slice(green_photon_counts), + json_metadata=json_string, + ) def generate_kymo( name, image, pixel_size_nm=10.0, - start=int(20e9), - dt=int(62.5e6), + start=_default_start, + dt=_default_dt, samples_per_pixel=5, line_padding=3, ): """Generate a kymo based on provided image data""" return _generate_confocal( name, - Kymo, image, multi_color=image.ndim > 2, pixel_sizes_nm=[pixel_size_nm], @@ -255,8 +248,8 @@ def generate_scan( image, pixel_sizes_nm, axes=None, - start=int(20e9), - dt=int(62.5e6), + start=_default_start, + dt=_default_dt, samples_per_pixel=5, line_padding=3, multi_color=False, @@ -264,7 +257,6 @@ def generate_scan( """Generate a scan based on provided image data""" return _generate_confocal( name, - Scan, image, multi_color=multi_color, pixel_sizes_nm=pixel_sizes_nm, @@ -278,7 +270,6 @@ def generate_scan( def _generate_confocal( name, - confocal_class, image, multi_color, pixel_sizes_nm, @@ -292,7 +283,8 @@ def _generate_confocal( start = np.int64(start) dt = np.int64(dt) - confocal_file, metadata, stop = MockConfocalFile.from_image( + return mock_confocal_from_image( + name, image, pixel_sizes_nm=pixel_sizes_nm, axes=axes, @@ -303,15 +295,13 @@ def _generate_confocal( multi_color=multi_color, ) - return confocal_class(name, confocal_file, start, stop, metadata) - def generate_kymo_with_ref( name, image, pixel_size_nm=10.0, - start=int(20e9), - dt=int(62.5e6), + start=_default_start, + dt=_default_dt, samples_per_pixel=5, line_padding=3, ): @@ -337,8 +327,8 @@ def generate_scan_with_ref( image, pixel_sizes_nm, axes=None, - start=int(20e9), - dt=int(62.5e6), + start=_default_start, + dt=_default_dt, samples_per_pixel=5, line_padding=3, multi_color=False, @@ -451,7 +441,7 @@ def generate_timestamps( scan=False, x_axis_fast=True, ): - """Calculate reference timestamps of a kymo or scan created with `MockConfocal.from_image()` + """Calculate reference timestamps of a kymo or scan created with `mock_confocal_from_image()` Parameters ---------- diff --git a/lumicks/pylake/tests/test_channels/test_channels.py b/lumicks/pylake/tests/test_channels/test_channels.py index 9e121b4c0..b08ba43d1 100644 --- a/lumicks/pylake/tests/test_channels/test_channels.py +++ b/lumicks/pylake/tests/test_channels/test_channels.py @@ -8,6 +8,7 @@ import matplotlib as mpl from lumicks.pylake import channel +from lumicks.pylake.lowlevel import make_continuous_slice from lumicks.pylake.calibration import ForceCalibrationList @@ -66,7 +67,7 @@ def test_calibration_timeseries_channels(): # This slices off everything nested_slice = nested_slice[120:] assert len(nested_slice.calibration) == 0 - assert type(nested_slice.calibration) is list + assert isinstance(nested_slice.calibration, list) def test_calibration_continuous_channels(): @@ -987,3 +988,23 @@ def start(self): with pytest.raises(TypeError, match=invalid_range): s[BadType(0)] + + +def test_low_level_construction(): + start = 1388534400000000000 + with pytest.raises(ValueError, match="Starting timestamp must be larger than"): + make_continuous_slice(np.array([1, 2, 3]), start - 1, int(1e9 / 78125)) + + data = np.arange(200) + slc = make_continuous_slice(data, start, int(1e9 / 78125)) + assert slc.start == start + assert slc._timesteps == int(1e9 / 78125) + assert slc.sample_rate == 78125 + assert slc.labels["y"] == "y" + assert slc.labels["title"] == "" + np.testing.assert_equal(slc.data, data) + assert isinstance(slc._src, channel.Continuous) + + slc = make_continuous_slice(data, start, int(1e9 / 78125), name="hi", y_label="there") + assert slc.labels["title"] == "hi" + assert slc.labels["y"] == "there" diff --git a/lumicks/pylake/tests/test_imaging_confocal/conftest.py b/lumicks/pylake/tests/test_imaging_confocal/conftest.py index da64aefc3..51dffbaaf 100644 --- a/lumicks/pylake/tests/test_imaging_confocal/conftest.py +++ b/lumicks/pylake/tests/test_imaging_confocal/conftest.py @@ -6,15 +6,14 @@ from lumicks.pylake.kymo import _kymo_from_array from lumicks.pylake.channel import Slice, Continuous -from lumicks.pylake.point_scan import PointScan from lumicks.pylake.detail.imaging_mixins import _FIRST_TIMESTAMP from ..data.mock_file import MockDataFile_v2 from ..data.mock_confocal import ( - MockConfocalFile, generate_scan_json, generate_kymo_with_ref, generate_scan_with_ref, + mock_confocal_from_arrays, ) start = np.int64(20e9) @@ -233,7 +232,8 @@ def test_point_scan(): n_samples = 90 data = {c: np.random.poisson(15, n_samples) for c in ("red", "green", "blue")} - mock_file, metadata, stop = MockConfocalFile.from_streams( + point_scan = mock_confocal_from_arrays( + "PointScan1", start, dt, [], @@ -244,7 +244,6 @@ def test_point_scan(): green_photon_counts=data["green"], blue_photon_counts=data["blue"], ) - point_scan = PointScan("PointScan1", mock_file, start, stop, metadata) reference = { "data": data, diff --git a/lumicks/pylake/tests/test_imaging_confocal/test_kymo.py b/lumicks/pylake/tests/test_imaging_confocal/test_kymo.py index 29245a853..700e9662e 100644 --- a/lumicks/pylake/tests/test_imaging_confocal/test_kymo.py +++ b/lumicks/pylake/tests/test_imaging_confocal/test_kymo.py @@ -5,9 +5,9 @@ from lumicks.pylake.kymo import Kymo, _default_line_time_factory from lumicks.pylake.channel import Slice, Continuous -from lumicks.pylake.detail.confocal import ScanMetaData +from lumicks.pylake.detail.confocal import ScanMetaData, ConfocalFileProxy -from ..data.mock_confocal import MockConfocalFile, generate_kymo, generate_scan_json +from ..data.mock_confocal import generate_kymo, generate_scan_json def test_kymo_properties(test_kymo): @@ -211,7 +211,7 @@ def kymo_with_wave(infowave_data): infowave = Slice(Continuous(np.array(infowave_data), 0, int(1e9))) json_string = generate_scan_json([{"axis": 1, "num of pixels": 2, "pixel size (nm)": 1}]) metadata = ScanMetaData.from_json(json_string) - confocal_file = MockConfocalFile(infowave, None, None, None) + confocal_file = ConfocalFileProxy(infowave, None, None, None) return Kymo("test", confocal_file, 0, int(6e9), metadata) kymo = kymo_with_wave([1, 1, 1, 1])