Add music visualiser

frontend/package.json

@@ -30,6 +30,7 @@
     "@jellyfin/sdk": "0.8.2",
     "@vueuse/components": "10.2.1",
     "@vueuse/core": "10.2.1",
+    "audiomotion-analyzer": "4.0.0",
     "axios": "1.4.0",
     "blurhash": "2.0.5",
     "comlink": "4.4.1",

frontend/src/components/Buttons/LikeButton.vue

@@ -1,7 +1,7 @@
 <template>
   <v-btn
     :icon="isFavorite ? IMdiHeart : IMdiHeartOutline"
-    size="small"
+    :size="size"
     :color="isFavorite ? 'primary' : undefined"
     :loading="loading"
     @click.stop.prevent="isFavorite = !isFavorite" />
@@ -15,7 +15,12 @@ import IMdiHeart from 'virtual:icons/mdi/heart';
 import IMdiHeartOutline from 'virtual:icons/mdi/heart-outline';
 import { useRemote } from '@/composables';
 
-const props = defineProps<{ item: BaseItemDto }>();
+const props = withDefaults(
+  defineProps<{ item: BaseItemDto; size?: string }>(),
+  {
+    size: 'small'
+  }
+);
 const remote = useRemote();
 const loading = ref(false);
 

frontend/src/components/Layout/AudioControls.vue

@@ -64,9 +64,7 @@
             </div>
           </v-col>
           <v-col cols="3" class="d-none d-md-flex align-center justify-end">
-            <like-button
-              :item="playbackManager.currentItem"
-              class="active-button" />
+            <like-button :item="playbackManager.currentItem" />
             <queue-button />
             <div class="hidden-lg-and-down">
               <volume-slider />

frontend/src/components/Layout/VolumeSlider.vue

@@ -1,10 +1,6 @@
 <template>
   <div class="volume-slider d-flex align-center justify-center">
-    <v-btn
-      class="active-button"
-      icon
-      size="small"
-      @click="playbackManager.toggleMute">
+    <v-btn icon size="small" @click="playbackManager.toggleMute">
       <v-icon :icon="icon" />
     </v-btn>
     <v-slider

frontend/src/components/Playback/MusicVisualizer.vue

@@ -0,0 +1,39 @@
+<template>
+  <div ref="visualizerElement" />
+</template>
+
+<script setup lang="ts">
+/**
+ * TODO: When the WebAudio node is connected to audiomotion-analyzer, the volume
+ * of the media increases abruptly. Investigate why and fix.
+ */
+import { shallowRef, onMounted, onBeforeUnmount } from 'vue';
+import AudioMotionAnalyzer from 'audiomotion-analyzer';
+import { mediaWebAudio } from '@/store';
+
+let visualizerInstance: AudioMotionAnalyzer;
+const visualizerElement = shallowRef<HTMLDivElement>();
+
+onMounted(() => {
+  visualizerInstance = new AudioMotionAnalyzer(visualizerElement.value, {
+    source: mediaWebAudio.sourceNode,
+    mode: 2,
+    gradient: 'prism',
+    reflexRatio: 0.025,
+    overlay: true,
+    showBgColor: false,
+    fftSize: 16_384,
+    frequencyScale: 'bark',
+    showScaleX: false,
+    smoothing: 0.9
+  });
+});
+
+onBeforeUnmount(() => {
+  if (visualizerInstance) {
+    visualizerInstance.disconnectInput();
+    visualizerInstance.disconnectOutput();
+    visualizerInstance = undefined;
+  }
+});
+</script>

frontend/src/components/Playback/PlayerElement.vue

@@ -32,7 +32,8 @@ import Hls, { ErrorData } from 'hls.js';
 import {
   playbackManagerStore,
   playerElementStore,
-  mediaElementRef
+  mediaElementRef,
+  mediaWebAudio
 } from '@/store';
 import { getImageInfo } from '@/utils/images';
 import { useSnackbar } from '@/composables';
@@ -60,6 +61,18 @@ function detachHls(): void {
   }
 }
 
+/**
+ * Suspends WebAudio when no playback is in place
+ */
+async function detachWebAudio(): Promise<void> {
+  if (mediaWebAudio.sourceNode) {
+    mediaWebAudio.sourceNode.disconnect();
+    mediaWebAudio.sourceNode = undefined;
+  }
+
+  await mediaWebAudio.context.suspend();
+}
+
 const mediaElementType = computed<'audio' | 'video' | undefined>(() => {
   if (playbackManager.currentlyPlayingMediaType === 'Audio') {
     return 'audio';
@@ -153,10 +166,19 @@ watch(
 watch(mediaElementRef, async () => {
   await nextTick();
   detachHls();
+  await detachWebAudio();
+
+  if (mediaElementRef.value) {
+    if (mediaElementType.value === 'video' && hls) {
+      hls.attachMedia(mediaElementRef.value);
+      hls.on(Hls.Events.ERROR, onHlsEror);
+    }
 
-  if (mediaElementRef.value && mediaElementType.value === 'video' && hls) {
-    hls.attachMedia(mediaElementRef.value);
-    hls.on(Hls.Events.ERROR, onHlsEror);
+    await mediaWebAudio.context.resume();
+    mediaWebAudio.sourceNode = mediaWebAudio.context.createMediaElementSource(
+      mediaElementRef.value
+    );
+    mediaWebAudio.sourceNode.connect(mediaWebAudio.context.destination);
   }
 });
 

frontend/src/pages/playback/music/index.vue

@@ -1,5 +1,5 @@
 <template>
-  <v-main>
+  <v-main v-if="playbackManager.queue">
     <v-app-bar color="transparent">
       <app-bar-button-layout @click="$router.back()">
         <template #icon>
@@ -8,43 +8,73 @@
           </v-icon>
         </template>
       </app-bar-button-layout>
+      <v-spacer />
+      <app-bar-button-layout @click="isVisualizing = !isVisualizing">
+        <template #icon>
+          <v-icon>
+            <i-dashicons-album v-if="isVisualizing" />
+            <i-mdi-chart-bar v-else />
+          </v-icon>
+        </template>
+      </app-bar-button-layout>
     </v-app-bar>
     <v-col class="px-0">
-      <swiper
-        v-if="playbackManager.queue"
-        class="d-flex justify-center align-center user-select-none"
-        :modules="modules"
-        :slides-per-view="4"
-        centered-slides
-        :autoplay="false"
-        effect="coverflow"
-        :coverflow-effect="coverflowEffect"
-        keyboard
-        a11y
-        virtual
-        @slide-change="onSlideChange"
-        @swiper="setControlledSwiper">
-        <swiper-slide
-          v-for="(item, index) in playbackManager.queue"
-          :key="`${item.Id}-${index}`"
-          :virtual-index="`${item.Id}-${index}`"
-          class="d-flex justify-center">
-          <div class="album-cover">
-            <blurhash-image :item="item" />
-          </div>
-        </swiper-slide>
-      </swiper>
+      <v-fade-transition mode="out-in">
+        <swiper
+          v-if="!isVisualizing"
+          class="d-flex justify-center align-center user-select-none"
+          :modules="modules"
+          :slides-per-view="4"
+          centered-slides
+          :autoplay="false"
+          effect="coverflow"
+          :coverflow-effect="coverflowEffect"
+          keyboard
+          a11y
+          virtual
+          @slide-change="onSlideChange"
+          @swiper="setControlledSwiper">
+          <swiper-slide
+            v-for="(item, index) in playbackManager.queue"
+            :key="`${item.Id}-${index}`"
+            :virtual-index="`${item.Id}-${index}`"
+            class="d-flex justify-center">
+            <div class="album-cover presentation-height">
+              <blurhash-image :item="item" />
+            </div>
+          </swiper-slide>
+        </swiper>
+        <music-visualizer
+          v-else
+<<<<<<< HEAD
+          class="d-flex justify-center align-center user-select-none presentation-height" />
+=======
+          class="d-flex justify-center align-center user-select-none"
+          style="height: 65vh" />
+>>>>>>> 1b7be01f (fix: music visualizer transitions and music hangs)
+      </v-fade-transition>
       <v-row class="justify-center align-center mt-3">
         <v-col cols="6">
           <v-row class="justify-center align-center">
-            <h1 class="text-h4">
-              {{ playbackManager.currentItem?.Name }}
-            </h1>
-          </v-row>
-          <v-row class="justify-center align-center">
-            <span class="text-subtitle">
-              {{ artistString }}
-            </span>
+            <v-col>
+              <v-row>
+                <h1 class="text-h4">
+                  {{ playbackManager.currentItem?.Name }}
+                </h1>
+              </v-row>
+              <v-row>
+                <span class="text-subtitle">
+                  {{ artistString }}
+                </span>
+              </v-row>
+            </v-col>
+            <!-- TODO: Fix alignment with the end time of TimeSlider -->
+            <v-col class="d-flex justify-end">
+              <like-button
+                v-if="playbackManager.currentItem"
+                :item="playbackManager?.currentItem"
+                size="x-large" />
+            </v-col>
           </v-row>
           <v-row class="justify-center align-center mt-3">
             <time-slider />
@@ -73,7 +103,7 @@ meta:
 </route>
 
 <script setup lang="ts">
-import { computed, ref, watch } from 'vue';
+import { computed, ref, watch, nextTick } from 'vue';
 import { ImageType } from '@jellyfin/sdk/lib/generated-client';
 import { A11y, Keyboard, Virtual, EffectCoverflow } from 'swiper';
 import type SwiperType from 'swiper';
@@ -92,13 +122,16 @@ const modules = [A11y, Keyboard, Virtual, EffectCoverflow];
 const route = useRoute();
 
 const playbackManager = playbackManagerStore();
+
 const coverflowEffect = {
   depth: 500,
   slideShadows: false,
   rotate: 0,
   stretch: -400
 };
 
+const isVisualizing = ref(false);
+
 const backdropHash = computed(() => {
   return playbackManager.currentItem
     ? getBlurhash(playbackManager.currentItem, ImageType.Primary)
@@ -144,6 +177,13 @@ watch(
   { immediate: true }
 );
 
+watch(isVisualizing, async () => {
+  if (!isVisualizing.value) {
+    await nextTick();
+    swiperInstance.value?.update();
+  }
+});
+
 /**
  * Handle slide changes
  */
@@ -157,8 +197,11 @@ function onSlideChange(): void {
 <style lang="scss" scoped>
 .album-cover {
   position: relative;
-  height: 65vh;
   min-width: 65vh;
   width: 65vh;
 }
+
+.presentation-height {
+  height: 65vh;
+}
 </style>

frontend/src/store/globals.ts

@@ -22,3 +22,10 @@ export const mediaElementRef = shallowRef<HTMLMediaElement>();
  * Reactive media controls of the local media player
  */
 export const mediaControls = useMediaControls(mediaElementRef);
+/**
+ * WebAudio instance of the local media player
+ */
+export const mediaWebAudio = {
+  context: new AudioContext(),
+  sourceNode: undefined as undefined | MediaElementAudioSourceNode
+};

frontend/types/global/components.d.ts

@@ -28,6 +28,7 @@ declare module 'vue' {
     DraggableQueue: typeof import('./../../src/components/Playback/DraggableQueue.vue')['default']
     FilterButton: typeof import('./../../src/components/Buttons/FilterButton.vue')['default']
     HomeSection: typeof import('./../../src/components/Layout/HomeSection.vue')['default']
+    IDashiconsAlbum: typeof import('~icons/dashicons/album')['default']
     IdentifyDialog: typeof import('./../../src/components/Item/Identify/IdentifyDialog.vue')['default']
     IdentifyResults: typeof import('./../../src/components/Item/Identify/IdentifyResults.vue')['default']
     ImageEditor: typeof import('./../../src/components/Item/Metadata/ImageEditor.vue')['default']
@@ -41,6 +42,7 @@ declare module 'vue' {
     IMdiBrightnessAuto: typeof import('~icons/mdi/brightness-auto')['default']
     IMdiCalendarRange: typeof import('~icons/mdi/calendar-range')['default']
     IMdiCast: typeof import('~icons/mdi/cast')['default']
+    IMdiChartBar: typeof import('~icons/mdi/chart-bar')['default']
     IMdiCheck: typeof import('~icons/mdi/check')['default']
     IMdiChevronRight: typeof import('~icons/mdi/chevron-right')['default']
     IMdiChevronUp: typeof import('~icons/mdi/chevron-up')['default']
@@ -100,6 +102,7 @@ declare module 'vue' {
     MediaStreamSelector: typeof import('./../../src/components/Item/MediaStreamSelector.vue')['default']
     MetadataEditor: typeof import('./../../src/components/Item/Metadata/MetadataEditor.vue')['default']
     MetadataEditorDialog: typeof import('./../../src/components/Item/Metadata/MetadataEditorDialog.vue')['default']
+    MusicVisualizer: typeof import('./../../src/components/Playback/MusicVisualizer.vue')['default']
     NavigationDrawer: typeof import('./../../src/components/Layout/Navigation/NavigationDrawer.vue')['default']
     NextTrackButton: typeof import('./../../src/components/Buttons/Playback/NextTrackButton.vue')['default']
     PeopleList: typeof import('./../../src/components/Item/PeopleList.vue')['default']