Better orion language detection (#337)

* New model to track orion language processing

* New function to detect language

* New custom migration

* Add cursor tracker and video orion language to offchain export

* Running `custom-migration` command

* Add manger to trigger video language updates

* Adjust orion video language manager to support video update

* Revert "Running `custom-migration` command"

This reverts commit 71c4997.

* Second run of `create-migrations` command

* CR fixes

* move 'orion_offchain_cursor' table to admin schema

* bump package version and add change log

* fix: bug in case detected language is undefined

---------

Co-authored-by: Zeeshan Akram <97m.zeeshan@gmail.com>

.env

@@ -62,6 +62,9 @@ TRUST_PROXY=uniquelocal
 SENDGRID_API_KEY=
 SENDGRID_FROM_EMAIL=gateway@example.com
 
+# Detectlanguage 
+DETECTLANGUAGE_API_KEY=
+
 # Debug settings
 SQD_DEBUG=api:*
 

CHANGELOG.md

@@ -1,3 +1,8 @@
+# 4.0.4
+
+## Bug Fixes:
+- Fixed: improve the accuracy of `Video.orionLanguage` field by reworking the `predictVideoLanguage` function in `src/utils/language.ts`
+
 # 4.0.3
 
 ## Misc

db/migrations/1720623003671-Data.js

@@ -0,0 +1,11 @@
+module.exports = class Data1720623003671 {
+    name = 'Data1720623003671'
+
+    async up(db) {
+        await db.query(`CREATE TABLE "admin"."orion_offchain_cursor" ("cursor_name" character varying NOT NULL, "value" bigint NOT NULL, CONSTRAINT "PK_7083797352af5a21224b6c8ccbc" PRIMARY KEY ("cursor_name"))`)
+    }
+
+    async down(db) {
+        await db.query(`DROP TABLE "admin"."orion_offchain_cursor"`)
+    }
+}

db/migrations/1709641962433-Views.js → db/migrations/1720623003800-Views.js

@@ -1,8 +1,8 @@
 
 const { getViewDefinitions } = require('../viewDefinitions')
 
-module.exports = class Views1709641962433 {
-  name = 'Views1709641962433'
+module.exports = class Views1720623003800 {
+  name = 'Views1720623003800'
 
   async up(db) {
     const viewDefinitions = getViewDefinitions(db);

package.json

@@ -1,6 +1,6 @@
 {
   "name": "orion",
-  "version": "4.0.3",
+  "version": "4.0.4",
   "engines": {
     "node": ">=16"
   },
@@ -72,6 +72,7 @@
     "cookie-parser": "^1.4.6",
     "csv-stringify": "^6.3.0",
     "dayjs": "^1.11.7",
+    "detectlanguage": "^2.1.0",
     "dotenv": "^16.0.3",
     "dotenv-expand": "^10.0.0",
     "express-openapi-validator": "^5.0.3",
@@ -87,7 +88,6 @@
     "patch-package": "^6.5.0",
     "pg": "8.8.0",
     "swagger-ui-express": "^4.6.2",
-    "tinyld": "^1.3.4",
     "type-graphql": "^1.2.0-rc.1",
     "typeorm": "^0.3.11",
     "ua-parser-js": "^1.0.34",
@@ -105,8 +105,8 @@
     "@subsquid/substrate-typegen": "^2.1.0",
     "@subsquid/typeorm-codegen": "0.3.1",
     "@types/async-lock": "^1.1.3",
-    "@types/chai": "^4.3.11",
     "@types/big-json": "^3.2.4",
+    "@types/chai": "^4.3.11",
     "@types/cookie-parser": "^1.4.3",
     "@types/express-rate-limit": "^6.0.0",
     "@types/mocha": "^10.0.1",

src/mappings/content/video.ts

@@ -17,10 +17,10 @@ import {
   VideoViewEvent,
 } from '../../model'
 import { EventHandlerContext } from '../../utils/events'
-import { predictVideoLanguage } from '../../utils/language'
 import {
   deserializeMetadata,
   genericEventFields,
+  orionVideoLanguageManager,
   u8aToBytes,
   videoRelevanceManager,
 } from '../utils'
@@ -122,10 +122,7 @@ export async function processVideoCreatedEvent({
     }
   }
 
-  video.orionLanguage = predictVideoLanguage({
-    title: video.title ?? '',
-    description: video.description ?? '',
-  })
+  video.orionLanguage = null
 
   channel.totalVideosCreated += 1
 
@@ -192,10 +189,7 @@ export async function processVideoUpdatedEvent({
     )
   }
 
-  video.orionLanguage = predictVideoLanguage({
-    title: video.title ?? '',
-    description: video.description ?? '',
-  })
+  orionVideoLanguageManager.scheduleVideoForDetection(video.id)
 
   if (autoIssueNft) {
     await processNft(overlay, block, indexInBlock, extrinsicHash, video, contentActor, autoIssueNft)

src/mappings/utils.ts

@@ -10,10 +10,16 @@ import { Event, MetaprotocolTransactionResultFailed, NftActivity, NftHistoryEntr
 import { CommentCountersManager } from '../utils/CommentsCountersManager'
 import { VideoRelevanceManager } from '../utils/VideoRelevanceManager'
 import { EntityManagerOverlay } from '../utils/overlay'
+import { OrionVideoLanguageManager } from '../utils/OrionVideoLanguageManager'
 
+export const orionVideoLanguageManager = new OrionVideoLanguageManager()
 export const commentCountersManager = new CommentCountersManager()
 export const videoRelevanceManager = new VideoRelevanceManager()
 // eslint-disable-next-line no-void
+void orionVideoLanguageManager.init(
+  1000 * 60 * 5 // 5 mins
+)
+// eslint-disable-next-line no-void
 void videoRelevanceManager.init({
   fullUpdateLoopTime: 1000 * 60 * 60 * 12, // 12 hrs
   scheduledUpdateLoopTime: 1000 * 60 * 10, // 10 mins

src/model/OrionOffchainCursor.ts

@@ -0,0 +1,20 @@
+import { Column, Entity, PrimaryColumn } from 'typeorm'
+
+@Entity({ schema: 'admin' })
+export class OrionOffchainCursor {
+  constructor(props?: Partial<OrionOffchainCursor>) {
+    Object.assign(this, props)
+  }
+
+  /**
+   * Name of the offchain cursor
+   */
+  @PrimaryColumn()
+  cursorName!: string
+
+  /**
+   * Value of the cursor
+   */
+  @Column('int8', { nullable: false })
+  value!: number
+}

src/model/index.ts

@@ -1,2 +1,3 @@
 export * from './generated'
 export { NextEntityId } from './NextEntityId'
+export { OrionOffchainCursor } from './OrionOffchainCursor'

src/utils/OrionVideoLanguageManager.ts

@@ -0,0 +1,64 @@
+import { EntityManager } from 'typeorm'
+import {
+  detectVideoLanguageWithProvider,
+  updateVideoLanguages,
+  VIDEO_ORION_LANGUAGE_CURSOR_NAME,
+} from './customMigrations/setOrionLanguageProvider'
+import { globalEm } from './globalEm'
+
+export class OrionVideoLanguageManager {
+  private videoToDetect: Set<string> = new Set()
+
+  async init(intervalMs: number): Promise<void> {
+    if (!VIDEO_ORION_LANGUAGE_CURSOR_NAME) {
+      return
+    }
+
+    this.updateLoop(intervalMs)
+      .then(() => {
+        /* Do nothing */
+      })
+      .catch((err) => {
+        console.error(err)
+        process.exit(-1)
+      })
+  }
+
+  scheduleVideoForDetection(id: string | null | undefined) {
+    if (id) {
+      this.videoToDetect.add(id)
+    }
+  }
+
+  async updateScheduledVideoLanguage(em: EntityManager) {
+    if (!this.videoToDetect.size) {
+      return
+    }
+
+    const videos = await em.query(`
+    SELECT id, title, description
+    FROM admin.video
+    WHERE id in (${[...this.videoToDetect.values()].map((id) => `'${id}'`).join(',')})
+    `)
+
+    await updateVideoLanguages(em, videos)
+    this.videoToDetect.clear()
+  }
+
+  async updateOrionVideoLanguage() {
+    return detectVideoLanguageWithProvider()
+  }
+
+  private async updateLoop(intervalMs: number): Promise<void> {
+    const em = await globalEm
+    while (true) {
+      await this.updateScheduledVideoLanguage(em).catch((e) => {
+        console.log(`Updating scheduled videos Orion language with provider failed`, e)
+      })
+      await this.updateOrionVideoLanguage().catch((e) => {
+        console.log(`Updating Orion language with provider failed`, e)
+      })
+      await new Promise((resolve) => setTimeout(resolve, intervalMs))
+    }
+  }
+}