Sonify plugin updates jtedits

.github/workflows/ci_cron_weekly.yml

@@ -54,3 +54,24 @@ jobs:
         python -m pip install tox
     - name: Test with tox
       run: tox -e py311-test-devdeps-romandeps
+
+  ci_cron_tests_stable_strauss:
+    name: Python 3.12 with stable versions of dependencies and Strauss
+    runs-on: ubuntu-latest
+    if: (github.repository == 'spacetelescope/jdaviz' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'Extra CI')))
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
+      with:
+        fetch-depth: 0
+    - name: Set up python
+      uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b  # v5.3.0
+      with:
+        python-version: '3.12'
+    - name: Install base dependencies
+      run: |
+        sudo apt-get install libportaudio2
+        python -m pip install --upgrade pip
+        python -m pip install tox
+    - name: Test with tox
+      run: tox -e py312-test-straussdeps

CHANGES.rst

@@ -11,6 +11,8 @@ New Features
 Cubeviz
 ^^^^^^^
 
+- Add Sonify Data plugin which uses the Strauss package to turn a data cube into sound. [#3269]
+
 Imviz
 ^^^^^
 

docs/cubeviz/plugins.rst

@@ -338,6 +338,19 @@ have valid flux units. For 3D data, the current :ref:`slice` is used.
     :ref:`Imviz Aperture Photometry <aper-phot-simple>`
         Imviz documentation describing the concept of aperture photometry in Jdaviz.
 
+.. _cubeviz-sonify-data:
+
+Sonify Data
+===========
+
+This plugin uses the `Strauss <https://strauss.readthedocs.io/en/latest/>`_ package
+to turn data cubes into audio grids (by pressing the
+:guilabel:`Sonify Data` button) that can be played while the spectrum-at-spaxel tool is active
+and the mouse is hovering over the flux viewer. A range of the cube can be sonified by creating
+and selecting a spectral subset from the :guilabel:`Spectral range` dropdown and then pressing
+the :guilabel:`Sonify Data` button. The output device for sound can be changed by using the
+:guilabel:`Sound device` dropdown.
+
 .. _cubeviz-export-plot:
 
 Export

jdaviz/configs/cubeviz/cubeviz.yaml

@@ -33,6 +33,7 @@ tray:
   - specviz-line-analysis
   - cubeviz-moment-maps
   - imviz-aper-phot-simple
+  - cubeviz-sonify-data
   - export
   - about
 viewer_area:

jdaviz/configs/cubeviz/plugins/__init__.py

@@ -4,4 +4,5 @@
 from .moment_maps.moment_maps import *  # noqa
 from .slice.slice import * # noqa
 from .spectral_extraction.spectral_extraction import * # noqa
+from .sonify_data.sonify_data import * # noqa
 from .tools import *  # noqa

jdaviz/configs/cubeviz/plugins/cube_listener.py

@@ -0,0 +1,149 @@
+import numpy as np
+from contextlib import contextmanager
+import sys
+import os
+import time
+try:
+    from strauss.sonification import Sonification
+    from strauss.sources import Events
+    from strauss.score import Score
+    from strauss.generator import Spectralizer
+except ImportError:
+    pass
+
+#  smallest fraction of the max audio amplitude that can be represented by a 16-bit signed integer
+MINVOL = 1/(2**15 - 1)
+
+
+@contextmanager
+def suppress_stderr():
+    with open(os.devnull, "w") as devnull:
+        old_stderr = sys.stderr
+        sys.stderr = devnull
+        try:
+            yield
+        finally:
+            sys.stderr = old_stderr
+
+
+def sonify_spectrum(spec, duration, overlap=0.05, system='mono', srate=44100, fmin=40, fmax=1300,
+                    eln=False):
+    notes = [["A2"]]
+    score = Score(notes, duration)
+    # set up spectralizer generator
+    generator = Spectralizer(samprate=srate)
+
+    # Lets pick the mapping frequency range for the spectrum...
+    generator.modify_preset({'min_freq': fmin, 'max_freq': fmax,
+                             'fit_spec_multiples': False,
+                             'interpolation_type': 'preserve_power',
+                             'equal_loudness_normalisation': eln})
+
+    data = {'spectrum': [spec], 'pitch': [1]}
+
+    # again, use maximal range for the mapped parameters
+    lims = {'spectrum': (0, '100')}
+
+    # set up source
+    sources = Events(data.keys())
+    sources.fromdict(data)
+    sources.apply_mapping_functions(map_lims=lims)
+
+    # render and play sonification!
+    soni = Sonification(score, sources, generator, system, samprate=srate)
+    soni.render()
+    soni._make_seamless(overlap)
+
+    return soni.loop_channels['0'].values
+
+
+class CubeListenerData:
+    def __init__(self, cube, wlens, samplerate=44100, duration=1, overlap=0.05, buffsize=1024,
+                 bdepth=16, wl_bounds=None, wl_unit=None, audfrqmin=50, audfrqmax=1500,
+                 eln=False, vol=None):
+        self.siglen = int(samplerate*(duration-overlap))
+        self.cube = cube
+        self.dur = duration
+        self.bdepth = bdepth
+        self.srate = samplerate
+        self.maxval = pow(2, bdepth-1) - 1
+        self.fadedx = 0
+
+        if vol is None:
+            self.atten_level = 1
+        else:
+            self.atten_level = int(np.clip((vol/100)**2, MINVOL, 1))
+
+        self.wl_bounds = wl_bounds
+        self.wl_unit = wl_unit
+        self.wlens = wlens
+
+        # control fades
+        fade = np.linspace(0, 1, buffsize+1)
+        self.ifade = fade[:-1]
+        self.ofade = fade[::-1][:-1]
+
+        # mapping frequency limits in Hz
+        self.audfrqmin = audfrqmin
+        self.audfrqmax = audfrqmax
+
+        # do we normalise for equal loudness?
+        self.eln = eln
+
+        self.idx1 = 0
+        self.idx2 = 0
+        self.cbuff = False
+        self.cursig = np.zeros(self.siglen, dtype='int16')
+        self.newsig = np.zeros(self.siglen, dtype='int16')
+
+        if self.cursig.nbytes * pow(1024, -3) > 2:
+            raise Exception("Cube projected to be > 2Gb!")
+
+        self.sigcube = np.zeros((*self.cube.shape[:2], self.siglen), dtype='int16')
+
+    def set_wl_bounds(self, w1, w2):
+        """
+        set the wavelength bounds for indexing spectra
+        """
+        wsrt = np.sort([w1, w2])
+        self.wl_bounds = tuple(wsrt)
+
+    def sonify_cube(self):
+        """
+        Iterate through the cube, convert each spectrum to a signal, and store
+        in class attributes
+        """
+        lo2hi = self.wlens.argsort()[::-1]
+
+        t0 = time.time()
+        for i in range(self.cube.shape[0]):
+            for j in range(self.cube.shape[1]):
+                with suppress_stderr():
+                    if self.cube[i, j, lo2hi].any():
+                        sig = sonify_spectrum(self.cube[i, j, lo2hi], self.dur,
+                                              srate=self.srate,
+                                              fmin=self.audfrqmin,
+                                              fmax=self.audfrqmax,
+                                              eln=self.eln)
+                        sig = (sig*self.maxval).astype('int16')
+                        self.sigcube[i, j, :] = sig
+                    else:
+                        continue
+        self.cursig[:] = self.sigcube[self.idx1, self.idx2, :]
+        self.newsig[:] = self.cursig[:]
+        t1 = time.time()
+        print(f"Took {t1-t0}s to process {self.cube.shape[0]*self.cube.shape[1]} spaxels")
+
+    def player_callback(self, outdata, frames, time, status):
+        cur = self.cursig
+        new = self.newsig
+        sdx = int(time.outputBufferDacTime*self.srate)
+        dxs = np.arange(sdx, sdx+frames).astype(int) % self.sigcube.shape[-1]
+        if self.cbuff:
+            outdata[:, 0] = (cur[dxs] * self.ofade).astype('int16')
+            outdata[:, 0] += (new[dxs] * self.ifade).astype('int16')
+            self.cursig[:] = self.newsig[:]
+            self.cbuff = False
+        else:
+            outdata[:, 0] = self.cursig[dxs]
+        outdata[:, 0] //= self.atten_level

jdaviz/configs/cubeviz/plugins/sonify_data/sonify_data.py

@@ -0,0 +1,146 @@
+from traitlets import Bool, List, Unicode, observe
+import astropy.units as u
+
+from jdaviz.core.custom_traitlets import IntHandleEmpty, FloatHandleEmpty
+from jdaviz.core.registries import tray_registry
+from jdaviz.core.template_mixin import (PluginTemplateMixin, DatasetSelectMixin,
+                                        SpectralSubsetSelectMixin, with_spinner)
+from jdaviz.core.user_api import PluginUserApi
+
+
+__all__ = ['SonifyData']
+
+try:
+    import strauss  # noqa
+    import sounddevice as sd
+except ImportError:
+    class Empty:
+        pass
+    sd = Empty()
+    sd.default = Empty()
+    sd.default.device = [-1, -1]
+    _has_strauss = False
+else:
+    _has_strauss = True
+
+
+@tray_registry('cubeviz-sonify-data', label="Sonify Data",
+               viewer_requirements=['spectrum', 'image'])
+class SonifyData(PluginTemplateMixin, DatasetSelectMixin, SpectralSubsetSelectMixin):
+    """
+    See the :ref:`Sonify Data Plugin Documentation <cubeviz-sonify-data>` for more details.
+
+    Only the following attributes and methods are available through the
+    :ref:`public plugin API <plugin-apis>`:
+
+    * :meth:`~jdaviz.core.template_mixin.PluginTemplateMixin.show`
+    * :meth:`~jdaviz.core.template_mixin.PluginTemplateMixin.open_in_tray`
+    * :meth:`~jdaviz.core.template_mixin.PluginTemplateMixin.close_in_tray`
+    """
+    template_file = __file__, "sonify_data.vue"
+
+    sample_rate = 44100 #IntHandleEmpty(44100).tag(sync=True)
+    buffer_size = 2048 #IntHandleEmpty(2048).tag(sync=True)
+    assidx = FloatHandleEmpty(2.5).tag(sync=True)
+    ssvidx = FloatHandleEmpty(0.65).tag(sync=True)
+    eln = Bool(True).tag(sync=True)
+    audfrqmin = FloatHandleEmpty(50).tag(sync=True)
+    audfrqmax = FloatHandleEmpty(1000).tag(sync=True)
+    use_pccut = Bool(True).tag(sync=True)
+    pccut = IntHandleEmpty(20).tag(sync=True)
+    volume = IntHandleEmpty(100).tag(sync=True)
+    stream_active = Bool(True).tag(sync=True)
+    has_strauss = Bool(_has_strauss).tag(sync=True)
+
+    # TODO: can we referesh the list, so sounddevices are up-to-date when dropdown clicked?
+    sound_devices_items = List().tag(sync=True)
+    sound_devices_selected = Unicode('').tag(sync=True)
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._plugin_description = 'Sonify a data cube'
+        self.docs_description = 'Sonify a data cube using the Strauss package.'
+        if not self.has_strauss or sd.default.device[1] < 0:
+            self.disabled_msg = ('To use Sonify Data, install strauss and restart Jdaviz. You '
+                                 'can do this by running pip install strauss in the command'
+                                 ' line and then launching Jdaviz. Currently, this plugin only'
+                                 ' works on devices with valid sound output.')
+
+        else:
+            devices, indexes = self.build_device_lists()
+            self.sound_device_indexes = dict(zip(devices, indexes))
+            self.sound_devices_items = devices
+            self.sound_devices_selected = dict(zip(indexes, devices))[sd.default.device[1]]
+
+        # TODO: Remove hardcoded range and flux viewer
+        self.spec_viewer = self.app.get_viewer('spectrum-viewer')
+        self.flux_viewer = self.app.get_viewer('flux-viewer')
+
+    @property
+    def user_api(self):
+        expose = []
+        return PluginUserApi(self, expose)
+
+    @with_spinner()
+    def vue_sonify_cube(self, *args):
+        if self.disabled_msg:
+            raise ValueError('Unable to sonify cube')
+
+        # Get index of selected device
+        selected_device_index = self.sound_device_indexes[self.sound_devices_selected]
+
+        # Apply spectral subset bounds
+        if self.spectral_subset_selected is not self.spectral_subset.default_text:
+            display_unit = self.spec_viewer.state.x_display_unit
+            min_wavelength = self.spectral_subset.selected_obj.lower.to_value(u.Unit(display_unit))
+            max_wavelength = self.spectral_subset.selected_obj.upper.to_value(u.Unit(display_unit))
+            self.flux_viewer.update_listener_wls((min_wavelength, max_wavelength), display_unit)
+
+        # Ensure the current spectral region bounds are up-to-date at render time
+        self.update_wavelength_range(None)
+        # generate the sonified cube
+        self.flux_viewer.get_sonified_cube(self.sample_rate, self.buffer_size,
+                                           selected_device_index, self.assidx, self.ssvidx,
+                                           self.pccut, self.audfrqmin,
+                                           self.audfrqmax, self.eln, self.use_pccut)
+
+        # Automatically select spectrum-at-spaxel tool
+        spec_at_spaxel_tool = self.flux_viewer.toolbar.tools['jdaviz:spectrumperspaxel']
+        self.flux_viewer.toolbar.active_tool = spec_at_spaxel_tool
+
+    def vue_start_stop_stream(self, *args):
+        self.stream_active = not self.stream_active
+        self.flux_viewer.stream_active = not self.flux_viewer.stream_active
+
+    @observe('spectral_subset_selected')
+    def update_wavelength_range(self, event):
+        if not hasattr(self, 'spec_viewer'):
+            return
+        display_unit = self.spec_viewer.state.x_display_unit
+        # is this spectral selection or the entire spectrum?
+        if hasattr(self.spectral_subset.selected_obj, "subregions"):
+            wlbounds = self.spectral_subset.selected_obj.subregions
+        else:
+            wlbounds = None
+        self.flux_viewer.update_listener_wls(wlbounds, display_unit)
+
+    @observe('volume')
+    def update_volume_level(self, event):
+        self.flux_viewer.update_volume_level(event['new'])
+
+    @observe('sound_devices_selected')
+    def update_sound_device(self, event):
+        if event['new'] != event['old']:
+            didx = dict(zip(*self.build_device_lists()))[event['new']]
+            self.flux_viewer.update_sound_device(didx)
+
+    def build_device_lists(self):
+        # dedicated function to build the current *output*
+        # device and index lists
+        devices = []
+        device_indexes = []
+        for index, device in enumerate(sd.query_devices()):
+            if device['max_output_channels'] > 0 and device['name'] not in devices:
+                devices.append(device['name'])
+                device_indexes.append(index)
+        return devices, device_indexes