add detect video

src/i18n/zhCN.ts

@@ -8,6 +8,7 @@ export default {
     image: '图片',
     video: '视频',
     webcam: '摄像',
-    'Open Image': '打开图片'
+    'Open Image': '打开图片',
+    'Open Video': '打开视频'
   }
 }

src/utils/tf.ts

@@ -47,7 +47,7 @@ function preprocess(source: HTMLVideoElement | HTMLImageElement, modelWidth: num
 
     // padding image to square => [n, m] to [n, n], n > m
     const [h, w] = img.shape.slice(0, 2) // get source width and height
-    console.log('image', h, w)
+    // console.log('image', h, w)
 
     const maxSize = Math.max(w, h) // get max size
     const imgPadded = img.pad([
@@ -85,12 +85,12 @@ export async function detect(
 ) {
   tf.engine().startScope() // start scoping tf engine
   const [modelWidth, modelHeight] = inputShape.slice(1, 3) // get model width and height
-  console.log('shape', modelWidth, modelHeight)
+  // console.log('shape', modelWidth, modelHeight)
 
   const [input, xRatio, yRatio] = preprocess(source, modelWidth, modelHeight) // preprocess image
-  console.log('ratio', xRatio, yRatio)
+  // console.log('ratio', xRatio, yRatio)
 
-  const res = toRaw(model).execute(input) as Tensor<Rank> // inference model
+  const res = toRaw(model).execute(input) as Tensor<Rank> // Must use toRaw() inference model.
   const transRes = res.transpose([0, 2, 1]) // transpose result [b, det, n] => [b, n, det]
 
   const boxes = tf.tidy(() => {
@@ -129,3 +129,36 @@ export async function detect(
 
   tf.engine().endScope() // end of scoping
 }
+
+/**
+ * Function to detect video from every source.
+ * @param {HTMLVideoElement} source video source
+ * @param {tf.GraphModel} model loaded YOLOv8 tensorflow.js model
+ * @param {HTMLCanvasElement} canvasRef canvas reference
+ */
+export function detectVideo(model: GraphModel<string | io.IOHandler>, inputShape: number[], source: HTMLVideoElement, canvasRef: HTMLCanvasElement) {
+  /**
+   * Function to detect every frame from video
+   */
+  let animationId = -1
+  const detectFrame = async () => {
+    if (source.videoWidth === 0 && source.srcObject === null) {
+      console.warn('source.srcObject === null')
+      const ctx = canvasRef.getContext('2d')
+      ctx && ctx.clearRect(0, 0, ctx.canvas.width, ctx.canvas.height) // clean canvas
+      return // handle if source is closed
+    }
+
+    detect(model, inputShape, source, canvasRef, () => {
+      animationId = requestAnimationFrame(detectFrame) // get another frame
+    })
+  }
+
+  detectFrame() // initialize to detect every frame
+  return animationId
+}
+
+export function unDetectVideo(id: number) {
+  cancelAnimationFrame(id)
+  console.warn('unDetectVideo', id)
+}

src/views/image.vue

@@ -2,7 +2,7 @@
   <span class="absolute w-full h-full flex justify-center items-center">
     <van-uploader :after-read="afterRead" reupload max-count="1">
       <canvas v-if="imageUrl" ref="canvasRef" class="absolute top-0 w-full h-full" :width="yolo().inputShape[1]" :height="yolo().inputShape[2]" />
-      <img v-if="imageUrl" ref="imageRef" :src="imageUrl" @load="onImageLoadDetect" class="w-full" />
+      <img v-if="imageUrl" ref="imageRef" :src="imageUrl" @load="onImageLoadDetect" class="w-full h-full" />
       <van-button v-else icon="plus" type="primary">{{ i18n.t('Open Image') }}</van-button>
     </van-uploader>
   </span>

src/views/video.vue

@@ -1,7 +1,36 @@
 <template>
-  <span>video</span>
+  <span class="absolute w-full h-full flex justify-center items-center">
+    <van-uploader :after-read="afterRead" reupload max-count="1" accept="video/*">
+      <canvas v-if="videoUrl" ref="canvasRef" class="absolute top-0 w-full h-full" :width="yolo().inputShape[1]" :height="yolo().inputShape[2]" />
+      <video v-if="videoUrl" ref="videoRef" class="w-full h-full" :src="videoUrl" controls autoplay @play="onVideoPlayDetect" />
+      <van-button v-else icon="plus" type="primary">{{ i18n.t('Open Video') }}</van-button>
+    </van-uploader>
+  </span>
 </template>
 
-<script setup></script>
+<script lang="ts" setup>
+import type { GraphModel, io } from '@tensorflow/tfjs'
+import { UploaderFileListItem } from 'vant'
+import { detectVideo, unDetectVideo } from '@/utils/tf'
+import i18n from '@/vue-i18n'
+import { yolo } from '@/vue-pinia'
+
+const videoUrl = ref()
+const videoRef = ref()
+const canvasRef = ref()
+let videoID = ref()
+
+// uploader
+function afterRead(file: UploaderFileListItem | UploaderFileListItem[]) {
+  videoUrl.value = (file as UploaderFileListItem).objectUrl
+}
+function onVideoPlayDetect() {
+  videoID.value = detectVideo(yolo().model as GraphModel<string | io.IOHandler>, yolo().inputShape, videoRef.value, canvasRef.value)
+}
+
+onBeforeUnmount(() => {
+  unDetectVideo(videoID.value)
+})
+</script>
 
 <style lang="less" scoped></style>