Add support for mp4, webm, and ogg

common/fs.ts

@@ -43,6 +43,10 @@ export async function isDirEmpty(dir: string) {
   return dirContents.length === 0 || (dirContents.length === 1 && dirContents[0] === '.DS_Store');
 }
 
+export function isFileExtensionVideo(fileExtension: string) {
+  return (fileExtension === 'webm' || fileExtension === 'mp4' || fileExtension === 'ogg');
+}
+
 function hashString(s: string) {
   let hash = 0;
   let chr = 0;

resources/style/content.scss

@@ -59,7 +59,7 @@
     pointer-events: none;
   }
 
-  > img {
+  > :is(img, video) {
     width: 100%;
     height: 100%;
     cursor: pointer;
@@ -72,7 +72,7 @@
   &[data-dnd-target='true'] {
     background-color: var(--accent-color-yellow);
 
-    img {
+    :is(img, video) {
       clip-path: inset(0.5rem round 0.125rem);
     }
   }
@@ -81,16 +81,14 @@
 [aria-selected='true'] > .thumbnail {
   // If selected, show a blue border
   background-color: var(--accent-color);
-  > img,
-  > .image-error {
+  > :is(img, video) {
     clip-path: inset(0.25rem round 0.125rem); // old design
   }
 
   // If selected AND drop target: big yellow border
   &[data-dnd-target='true'] {
     background-color: var(--accent-color-yellow);
-    > img,
-    > .image-error {
+    > :is(img, video) {
       clip-path: inset(0.5rem round 0.125rem);
     }
   }
@@ -278,7 +276,7 @@
     width: var(--thumbnail-size);
   }
 
-  .col-name > img {
+  .col-name > :is(img, video) {
     object-fit: cover;
     padding: 2px;
   }
@@ -330,11 +328,11 @@
   position: relative;
 }
 
-.thumbnail-square img {
+.thumbnail-square img, video {
   background-position: center;
   object-fit: cover;
 }
-.thumbnail-letterbox img {
+.thumbnail-letterbox img, video {
   object-fit: contain;
 }
 
@@ -422,7 +420,7 @@
     right: 0;
   }
 
-  img {
+  img, video {
     // Let the compositor know that the image will transform for better performance
     will-change: transform;
 

src/api/file.ts

@@ -53,5 +53,8 @@ export const IMG_EXTENSIONS = [
   // 'avif',
   // 'heic', // not supported by Sharp out of the box https://github.com/lovell/sharp/issues/2871
   // TODO: 'blend', raw, etc.?
+  'mp4',
+  'webm',
+  'ogg',
 ] as const;
-export type IMG_EXTENSIONS_TYPE = typeof IMG_EXTENSIONS[number];
+export type IMG_EXTENSIONS_TYPE = (typeof IMG_EXTENSIONS)[number];

src/frontend/containers/ContentView/GalleryItem.tsx

@@ -4,19 +4,23 @@ import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'
 import fse from 'fs-extra';
 import { ClientFile } from 'src/entities/File';
 import { ellipsize, humanFileSize } from 'common/fmt';
-import { encodeFilePath } from 'common/fs';
+import { encodeFilePath, isFileExtensionVideo } from 'common/fs';
 import { IconButton, IconSet, Tag } from 'widgets';
 import { ITransform } from './Masonry/layout-helpers';
 import { ClientTag } from 'src/entities/Tag';
 import { useStore } from 'src/frontend/contexts/StoreContext';
 import { usePromise } from 'src/frontend/hooks/usePromise';
 import { CommandDispatcher, MousePointerEvent } from './Commands';
+import { GalleryVideoPlaybackMode } from 'src/frontend/stores/UiStore';
 
 interface ItemProps {
   file: ClientFile;
   mounted: boolean;
   // Will use the original image instead of the thumbnail
   forceNoThumbnail?: boolean;
+  hovered?: boolean;
+  galleryVideoPlaybackMode?: GalleryVideoPlaybackMode;
+  isSlideMode?: boolean;
 }
 
 interface MasonryItemProps extends ItemProps {
@@ -32,9 +36,17 @@ export const MasonryCell = observer(
     transform: [width, height, top, left],
   }: MasonryItemProps) => {
     const { uiStore, fileStore } = useStore();
+    const [isHovered, setIsHovered] = useState(false);
     const style = { height, width, transform: `translate(${left}px,${top}px)` };
     const eventManager = useMemo(() => new CommandDispatcher(file), [file]);
 
+    const handleMouseEnter = useCallback((e: React.MouseEvent): void => {
+      setIsHovered(true);
+    }, []);
+    const handleMouseLeave = useCallback((e: React.MouseEvent): void => {
+      setIsHovered(false);
+    }, []);
+
     return (
       <div data-masonrycell aria-selected={uiStore.fileSelection.has(file)} style={style}>
         <div
@@ -48,8 +60,25 @@ export const MasonryCell = observer(
           onDragLeave={eventManager.dragLeave}
           onDrop={eventManager.drop}
           onDragEnd={eventManager.dragEnd}
+          onMouseEnter={
+            uiStore.galleryVideoPlaybackMode === 'hover' && isFileExtensionVideo(file.extension)
+              ? handleMouseEnter
+              : (e: React.MouseEvent): void => {}
+          }
+          onMouseLeave={
+            uiStore.galleryVideoPlaybackMode === 'hover' && isFileExtensionVideo(file.extension)
+              ? handleMouseLeave
+              : (e: React.MouseEvent): void => {}
+          }
         >
-          <Thumbnail mounted={mounted} file={file} forceNoThumbnail={forceNoThumbnail} />
+          <Thumbnail
+            mounted={mounted}
+            file={file}
+            forceNoThumbnail={forceNoThumbnail}
+            hovered={isHovered}
+            galleryVideoPlaybackMode={uiStore.galleryVideoPlaybackMode}
+            isSlideMode={uiStore.isSlideMode}
+          />
         </div>
         {file.isBroken === true && !fileStore.showsMissingContent && (
           <IconButton
@@ -87,89 +116,154 @@ export const MasonryCell = observer(
 
 // TODO: When a filename contains https://x/y/z.abc?323 etc., it can't be found
 // e.g. %2F should be %252F on filesystems. Something to do with decodeURI, but seems like only on the filename - not the whole path
-export const Thumbnail = observer(({ file, mounted, forceNoThumbnail }: ItemProps) => {
-  const { uiStore, imageLoader } = useStore();
-  const { thumbnailPath, isBroken } = file;
-
-  // This will check whether a thumbnail exists, generate it if needed
-  const imageSource = usePromise(
+export const Thumbnail = observer(
+  ({
     file,
-    isBroken,
     mounted,
-    thumbnailPath,
-    uiStore.isList || !forceNoThumbnail,
-    async (file, isBroken, mounted, thumbnailPath, useThumbnail) => {
-      // If it is broken, only show thumbnail if it exists.
-      if (!mounted || isBroken === true) {
-        if (await fse.pathExists(thumbnailPath)) {
-          return thumbnailPath;
-        } else {
-          throw new Error('No thumbnail available.');
+    forceNoThumbnail,
+    hovered,
+    galleryVideoPlaybackMode,
+    isSlideMode,
+  }: ItemProps) => {
+    const { uiStore, imageLoader } = useStore();
+    const { thumbnailPath, isBroken } = file;
+
+    // This will check whether a thumbnail exists, generate it if needed
+    const imageSource = usePromise(
+      file,
+      isBroken,
+      mounted,
+      thumbnailPath,
+      uiStore.isList || !forceNoThumbnail,
+      async (file, isBroken, mounted, thumbnailPath, useThumbnail) => {
+        // If it is broken, only show thumbnail if it exists.
+        if (!mounted || isBroken === true) {
+          if (await fse.pathExists(thumbnailPath)) {
+            return thumbnailPath;
+          } else {
+            throw new Error('No thumbnail available.');
+          }
         }
-      }
 
-      if (useThumbnail) {
-        const freshlyGenerated = await imageLoader.ensureThumbnail(file);
-        // The thumbnailPath of an image is always set, but may not exist yet.
-        // When the thumbnail is finished generating, the path will be changed to `${thumbnailPath}?v=1`.
-        if (freshlyGenerated) {
-          await when(() => file.thumbnailPath.endsWith('?v=1'), { timeout: 10000 });
-          if (!getThumbnail(file).endsWith('?v=1')) {
-            throw new Error('Thumbnail generation timeout.');
+        if (useThumbnail) {
+          const freshlyGenerated = await imageLoader.ensureThumbnail(file);
+          // The thumbnailPath of an image is always set, but may not exist yet.
+          // When the thumbnail is finished generating, the path will be changed to `${thumbnailPath}?v=1`.
+          if (freshlyGenerated) {
+            await when(() => file.thumbnailPath.endsWith('?v=1'), { timeout: 10000 });
+            if (!getThumbnail(file).endsWith('?v=1')) {
+              throw new Error('Thumbnail generation timeout.');
+            }
+          }
+          return getThumbnail(file);
+        } else {
+          const src = await imageLoader.getImageSrc(file);
+          if (src !== undefined) {
+            return src;
+          } else {
+            throw new Error('No thumbnail available.');
           }
         }
-        return getThumbnail(file);
+      },
+    );
+
+    // Even though all thumbnail errors should be caught in the above usePromise,
+    // there is a chance that the image cannot be loaded, and we don't want to show broken image icons
+    const fileId = file.id;
+    const fileIdRef = useRef(fileId);
+    const [loadError, setLoadError] = useState(false);
+    const handleImageError = useCallback(() => {
+      if (fileIdRef.current === fileId) {
+        setLoadError(true);
+      }
+    }, [fileId]);
+    useEffect(() => {
+      fileIdRef.current = fileId;
+      setLoadError(false);
+    }, [fileId]);
+
+    // Plays and pauses video
+    const thumbnailRef = useRef<HTMLVideoElement>(null);
+    useEffect(() => {
+      if (thumbnailRef.current === null || !isFileExtensionVideo(file.extension)) {
+        return;
+      }
+      if (hovered) {
+        thumbnailRef.current.play();
       } else {
-        const src = await imageLoader.getImageSrc(file);
-        if (src !== undefined) {
-          return src;
-        } else {
-          throw new Error('No thumbnail available.');
+        thumbnailRef.current.pause();
+        thumbnailRef.current.currentTime = 0;
+      }
+    }, [thumbnailRef, hovered]);
+    useEffect(() => {
+      if (thumbnailRef.current === null || !isFileExtensionVideo(file.extension)) {
+        return;
+      }
+      if (galleryVideoPlaybackMode === 'auto') {
+        thumbnailRef.current.play();
+      } else {
+        thumbnailRef.current.pause();
+        thumbnailRef.current.currentTime = 0;
+      }
+    }, [thumbnailRef, galleryVideoPlaybackMode]);
+
+    // Pause video when slide mode, don't want to decode when video isn't visible
+    useEffect(() => {
+      if (thumbnailRef.current === null || !isFileExtensionVideo(file.extension)) {
+        return;
+      }
+      if (isSlideMode) {
+        thumbnailRef.current.pause();
+      } else {
+        if (galleryVideoPlaybackMode === 'auto') {
+          thumbnailRef.current.play();
         }
       }
-    },
-  );
+    }, [thumbnailRef, isSlideMode]);
 
-  // Even though all thumbnail errors should be caught in the above usePromise,
-  // there is a chance that the image cannot be loaded, and we don't want to show broken image icons
-  const fileId = file.id;
-  const fileIdRef = useRef(fileId);
-  const [loadError, setLoadError] = useState(false);
-  const handleImageError = useCallback(() => {
-    if (fileIdRef.current === fileId) {
-      setLoadError(true);
-    }
-  }, [fileId]);
-  useEffect(() => {
-    fileIdRef.current = fileId;
-    setLoadError(false);
-  }, [fileId]);
-
-  if (!mounted) {
-    return <span className="image-placeholder" />;
-  } else if (loadError) {
-    return <span className="image-loading" />;
-  } else if (imageSource.tag === 'ready') {
-    if ('ok' in imageSource.value) {
-      const is_lowres = file.width < 320 || file.height < 320;
-      return (
-        <img
-          src={encodeFilePath(imageSource.value.ok)}
-          alt=""
-          data-file-id={file.id}
-          onError={handleImageError}
-          style={
-            is_lowres && uiStore.upscaleMode == 'pixelated' ? { imageRendering: 'pixelated' } : {}
+    if (!mounted) {
+      return <span className="image-placeholder" />;
+    } else if (loadError) {
+      return <span className="image-loading" />;
+    } else if (imageSource.tag === 'ready') {
+      if ('ok' in imageSource.value) {
+        const is_lowres = file.width < 320 || file.height < 320;
+        // TODO: add thumbnails to video for performance in gallery view
+        if (isFileExtensionVideo(file.extension)) {
+          const videoProps = {
+            src: encodeFilePath(imageSource.value.ok),
+            'data-file-id': file.id,
+            onError: handleImageError,
+            muted: true,
+            autoPlay: false,
+            loop: true,
+          };
+          if (galleryVideoPlaybackMode === 'auto' || hovered) {
+            videoProps.autoPlay = true;
           }
-        />
-      );
+
+          return <video ref={thumbnailRef} {...videoProps} />;
+        }
+
+        return (
+          <img
+            src={encodeFilePath(imageSource.value.ok)}
+            alt=""
+            data-file-id={file.id}
+            onError={handleImageError}
+            style={
+              is_lowres && uiStore.upscaleMode == 'pixelated' ? { imageRendering: 'pixelated' } : {}
+            }
+          />
+        );
+      } else {
+        return <span className="image-error" />;
+      }
     } else {
-      return <span className="image-error" />;
+      return <span className="image-loading" />;
     }
-  } else {
-    return <span className="image-loading" />;
-  }
-});
+  },
+);
 
 const getThumbnail = action((file: ClientFile) => file.thumbnailPath);
 

src/frontend/containers/ContentView/SlideMode/ZoomPan.tsx

@@ -31,9 +31,7 @@ const ANIMATION_SPEED = 0.1;
 export type SlideTransform = Transform;
 
 export interface ZoomPanProps {
-  children: (
-    props: React.ImgHTMLAttributes<HTMLImageElement>,
-  ) => React.ReactElement<HTMLImageElement>;
+  children: (props: React.ImgHTMLAttributes<HTMLElement>) => React.ReactElement<HTMLElement>;
   initialScale: number | 'auto';
   minScale: number;
   maxScale: number;