CR fixes

package.json

@@ -88,7 +88,6 @@
     "patch-package": "^6.5.0",
     "pg": "8.8.0",
     "swagger-ui-express": "^4.6.2",
-    "tinyld": "^1.3.4",
     "type-graphql": "^1.2.0-rc.1",
     "typeorm": "^0.3.11",
     "ua-parser-js": "^1.0.34",
@@ -106,8 +105,8 @@
     "@subsquid/substrate-typegen": "^2.1.0",
     "@subsquid/typeorm-codegen": "0.3.1",
     "@types/async-lock": "^1.1.3",
-    "@types/chai": "^4.3.11",
     "@types/big-json": "^3.2.4",
+    "@types/chai": "^4.3.11",
     "@types/cookie-parser": "^1.4.3",
     "@types/express-rate-limit": "^6.0.0",
     "@types/mocha": "^10.0.1",

src/mappings/content/video.ts

@@ -16,10 +16,7 @@ import {
   VideoPosted,
   VideoViewEvent,
 } from '../../model'
-import { VIDEO_ORION_LANGUAGE_CURSOR_NAME } from '../../utils/customMigrations/setOrionLanguageProvider'
 import { EventHandlerContext } from '../../utils/events'
-import { predictVideoLanguage } from '../../utils/language'
-import { OrionVideoLanguageManager } from '../../utils/OrionVideoLanguageManager'
 import {
   deserializeMetadata,
   genericEventFields,
@@ -125,12 +122,7 @@ export async function processVideoCreatedEvent({
     }
   }
 
-  video.orionLanguage = VIDEO_ORION_LANGUAGE_CURSOR_NAME
-    ? null
-    : predictVideoLanguage({
-        title: video.title ?? '',
-        description: video.description ?? '',
-      })
+  video.orionLanguage = null
 
   channel.totalVideosCreated += 1
 
@@ -197,14 +189,7 @@ export async function processVideoUpdatedEvent({
     )
   }
 
-  if (VIDEO_ORION_LANGUAGE_CURSOR_NAME) {
-    orionVideoLanguageManager.scheduleVideoForDetection(video.id)
-  } else {
-    video.orionLanguage = predictVideoLanguage({
-      title: video.title ?? '',
-      description: video.description ?? '',
-    })
-  }
+  orionVideoLanguageManager.scheduleVideoForDetection(video.id)
 
   if (autoIssueNft) {
     await processNft(overlay, block, indexInBlock, extrinsicHash, video, contentActor, autoIssueNft)

src/utils/OrionVideoLanguageManager.ts

@@ -52,7 +52,9 @@ export class OrionVideoLanguageManager {
   private async updateLoop(intervalMs: number): Promise<void> {
     const em = await globalEm
     while (true) {
-      await this.updateScheduledVideoLanguage(em)
+      await this.updateScheduledVideoLanguage(em).catch((e) => {
+        console.log(`Updating scheduled videos Orion language with provider failed`, e)
+      })
       await this.updateOrionVideoLanguage().catch((e) => {
         console.log(`Updating Orion language with provider failed`, e)
       })

src/utils/customMigrations/setOrionLanguageProvider.ts

@@ -1,7 +1,7 @@
 import { EntityManager } from 'typeorm'
 import { OrionOffchainCursor } from '../../model'
 import { globalEm } from '../globalEm'
-import { predictLanguageForArray } from '../language'
+import { predictLanguageWithProvider } from '../language'
 
 const batchSize = 5_000 // Adjust the batch size based on your database and network performance
 
@@ -16,7 +16,7 @@ export const VIDEO_ORION_LANGUAGE_CURSOR_NAME = 'video_orion_language'
 export async function updateVideoLanguages(em: EntityManager, videos: VideoUpdateType[]) {
   const mappedVideos = videos.map((video) => `${video.title} ${video.description}`)
 
-  const predictionForVideos = await predictLanguageForArray(mappedVideos)
+  const predictionForVideos = await predictLanguageWithProvider(mappedVideos)
 
   const videosWithDetections = videos.map((video, index) => ({
     ...video,
@@ -40,32 +40,36 @@ export async function updateVideoLanguages(em: EntityManager, videos: VideoUpdat
 
 export async function detectVideoLanguageWithProvider() {
   const em: EntityManager = await globalEm
-  const cursorEntity: { value: string }[] = await em.query(
+  let cursorEntity: { value: number }[] = await em.query(
     `SELECT value FROM orion_offchain_cursor WHERE cursor_name='${VIDEO_ORION_LANGUAGE_CURSOR_NAME}'`
   )
-  const cursor = +(cursorEntity[0]?.value ?? 0)
+  while (true) {
+    const cursor = +(cursorEntity[0]?.value ?? 0)
 
-  const videos: VideoUpdateType[] = await em.query(`
+    const videos: VideoUpdateType[] = await em.query(`
     SELECT id, title, description
     FROM admin.video
     ORDER BY id::INTEGER ASC
     OFFSET ${cursor}
     LIMIT ${batchSize}
-  `)
+     `)
 
-  if (!videos.length) {
-    console.log('No more videos!')
-    return
-  }
-  await updateVideoLanguages(em, videos)
-  const newCursor = new OrionOffchainCursor({
-    cursorName: VIDEO_ORION_LANGUAGE_CURSOR_NAME,
-    value: cursor + Math.min(batchSize, videos.length),
-  })
-  await em.save(newCursor)
-  console.log(
-    `Updated languages for videos in range ${cursor}-${cursor + Math.min(batchSize, videos.length)}`
-  )
+    if (!videos.length) {
+      console.log('No more videos!')
+      break
+    }
 
-  await detectVideoLanguageWithProvider()
+    await updateVideoLanguages(em, videos)
+    const newCursor = new OrionOffchainCursor({
+      cursorName: VIDEO_ORION_LANGUAGE_CURSOR_NAME,
+      value: cursor + Math.min(batchSize, videos.length),
+    })
+    await em.save(newCursor)
+    cursorEntity = [newCursor]
+    console.log(
+      `Updated languages for videos in range ${cursor}-${
+        cursor + Math.min(batchSize, videos.length)
+      }`
+    )
+  }
 }

src/utils/language.ts

@@ -1,4 +1,3 @@
-import { detectAll } from 'tinyld'
 import DetectLanguage from 'detectlanguage'
 
 const languageDetectionApiKey = process.env.DETECTLANGUAGE_API_KEY
@@ -15,32 +14,8 @@ function cleanString(input: string): string {
   return cleanedString
 }
 
-function predictLanguage(text: string): { lang: string; accuracy: number } | undefined {
-  const cleanedText = cleanString(text)
-
-  // Get the most accurate language prediction
-  return detectAll(cleanedText)?.[0]
-}
-
-export async function predictLanguageForArray(texts: string[]) {
-  const result = await languageDetectionInstace.detect(texts)
+export async function predictLanguageWithProvider(texts: string[]) {
+  const cleanedTexts = texts.map(cleanString)
+  const result = await languageDetectionInstace.detect(cleanedTexts)
   return result.map((row) => row[0].language)
 }
-
-export function predictVideoLanguage({ title, description }: any): string | undefined {
-  let detectedLang: string | undefined
-
-  const titleLang = predictLanguage(title ?? '')
-
-  detectedLang = titleLang?.lang
-
-  if ((titleLang?.accuracy || 0) < 0.5) {
-    const titleAndDescriptionLang = predictLanguage(`${title} ${description}`)
-    if ((titleAndDescriptionLang?.accuracy || 0) > (titleLang?.accuracy || 0)) {
-      // then
-      detectedLang = titleAndDescriptionLang?.lang
-    }
-  }
-
-  return detectedLang
-}