Merge remote-tracking branch 'upstream/master' into deffer-typeorm-un…

…ique-constraints

.env

@@ -62,6 +62,9 @@ TRUST_PROXY=uniquelocal
 SENDGRID_API_KEY=
 SENDGRID_FROM_EMAIL=gateway@example.com
 
+# Detectlanguage 
+DETECTLANGUAGE_API_KEY=
+
 # Debug settings
 SQD_DEBUG=api:*
 

.github/workflows/checks.yml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        node-version: [16.x]
+        node-version: [18.x]
       fail-fast: true
     steps:
       - uses: actions/checkout@v2
@@ -26,7 +26,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        node-version: [16.x]
+        node-version: [18.x]
       fail-fast: true
     steps:
       - uses: actions/checkout@v2
@@ -42,10 +42,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Use Node.js 16.x
+      - name: Use Node.js 18.x
         uses: actions/setup-node@v1
         with:
-          node-version: 16.x
+          node-version: 18.x
       - name: Prepare workspace
         run: make prepare
       - name: create joystream_default network

.github/workflows/tests.yml

@@ -7,10 +7,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Use Node.js 16.x
+      - name: Use Node.js 18.x
         uses: actions/setup-node@v1
         with:
-          node-version: 16.x
+          node-version: 18.x
       - name: Create joystream_default network
         run: docker network create joystream_default
       - name: Prepare workspace
@@ -31,7 +31,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        node-version: [16.x]
+        node-version: [18.x]
       fail-fast: true
     steps:
       - uses: actions/checkout@v2
@@ -50,7 +50,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        node-version: [16.x]
+        node-version: [18.x]
       fail-fast: true
     steps:
       - uses: actions/checkout@v2
@@ -71,7 +71,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        node-version: [16.x]
+        node-version: [18.x]
       fail-fast: true
     steps:
       - uses: actions/checkout@v2

CHANGELOG.md

@@ -1,3 +1,15 @@
+# 4.0.4
+
+## Bug Fixes:
+- Fixed: improve the accuracy of `Video.orionLanguage` field by reworking the `predictVideoLanguage` function in `src/utils/language.ts`
+
+# 4.0.3
+
+## Misc
+
+- Update Email notification subjects and tests [#333](https://github.com/Joystream/orion/pull/333)
+- Update Node.js version to 18.x in all github actions.
+
 # 4.0.2
 
 ## Bug Fixes:

db/migrations/1720623003671-Data.js

@@ -0,0 +1,11 @@
+module.exports = class Data1720623003671 {
+    name = 'Data1720623003671'
+
+    async up(db) {
+        await db.query(`CREATE TABLE "admin"."orion_offchain_cursor" ("cursor_name" character varying NOT NULL, "value" bigint NOT NULL, CONSTRAINT "PK_7083797352af5a21224b6c8ccbc" PRIMARY KEY ("cursor_name"))`)
+    }
+
+    async down(db) {
+        await db.query(`DROP TABLE "admin"."orion_offchain_cursor"`)
+    }
+}

db/migrations/1709641962433-Views.js → db/migrations/1720623003800-Views.js

@@ -1,8 +1,8 @@
 
 const { getViewDefinitions } = require('../viewDefinitions')
 
-module.exports = class Views1709641962433 {
-  name = 'Views1709641962433'
+module.exports = class Views1720623003800 {
+  name = 'Views1720623003800'
 
   async up(db) {
     const viewDefinitions = getViewDefinitions(db);

package.json

@@ -1,6 +1,6 @@
 {
   "name": "orion",
-  "version": "4.0.2",
+  "version": "4.0.4",
   "engines": {
     "node": ">=16"
   },
@@ -72,6 +72,7 @@
     "cookie-parser": "^1.4.6",
     "csv-stringify": "^6.3.0",
     "dayjs": "^1.11.7",
+    "detectlanguage": "^2.1.0",
     "dotenv": "^16.0.3",
     "dotenv-expand": "^10.0.0",
     "express-openapi-validator": "^5.0.3",
@@ -87,7 +88,6 @@
     "patch-package": "^6.5.0",
     "pg": "8.8.0",
     "swagger-ui-express": "^4.6.2",
-    "tinyld": "^1.3.4",
     "type-graphql": "^1.2.0-rc.1",
     "typeorm": "^0.3.11",
     "ua-parser-js": "^1.0.34",
@@ -105,8 +105,8 @@
     "@subsquid/substrate-typegen": "^2.1.0",
     "@subsquid/typeorm-codegen": "0.3.1",
     "@types/async-lock": "^1.1.3",
-    "@types/chai": "^4.3.11",
     "@types/big-json": "^3.2.4",
+    "@types/chai": "^4.3.11",
     "@types/cookie-parser": "^1.4.3",
     "@types/express-rate-limit": "^6.0.0",
     "@types/mocha": "^10.0.1",

src/mail-scheduler/index.ts

@@ -5,6 +5,7 @@ import { ConfigVariable, config } from '../utils/config'
 import { uniqueId } from '../utils/crypto'
 import { globalEm } from '../utils/globalEm'
 import { createMailContent, executeMailDelivery } from './utils'
+import { updateJoystreamPrice } from '../utils/joystreamPrice'
 
 export async function getMaxAttempts(em: EntityManager): Promise<number> {
   const maxAttempts = await config.get(ConfigVariable.EmailNotificationDeliveryMaxAttempts, em)
@@ -31,6 +32,7 @@ export async function mailsToDeliver(em: EntityManager): Promise<NotificationEma
 
 export async function deliverEmails() {
   const em = await globalEm
+  await updateJoystreamPrice()
   const newEmailDeliveries = await mailsToDeliver(em)
   const maxAttempts = await getMaxAttempts(em)
   const appName = await config.get(ConfigVariable.AppName, em)

src/mappings/content/video.ts

@@ -17,10 +17,10 @@ import {
   VideoViewEvent,
 } from '../../model'
 import { EventHandlerContext } from '../../utils/events'
-import { predictVideoLanguage } from '../../utils/language'
 import {
   deserializeMetadata,
   genericEventFields,
+  orionVideoLanguageManager,
   u8aToBytes,
   videoRelevanceManager,
 } from '../utils'
@@ -122,10 +122,7 @@ export async function processVideoCreatedEvent({
     }
   }
 
-  video.orionLanguage = predictVideoLanguage({
-    title: video.title ?? '',
-    description: video.description ?? '',
-  })
+  video.orionLanguage = null
 
   channel.totalVideosCreated += 1
 
@@ -192,10 +189,7 @@ export async function processVideoUpdatedEvent({
     )
   }
 
-  video.orionLanguage = predictVideoLanguage({
-    title: video.title ?? '',
-    description: video.description ?? '',
-  })
+  orionVideoLanguageManager.scheduleVideoForDetection(video.id)
 
   if (autoIssueNft) {
     await processNft(overlay, block, indexInBlock, extrinsicHash, video, contentActor, autoIssueNft)

src/mappings/utils.ts

@@ -10,10 +10,16 @@ import { Event, MetaprotocolTransactionResultFailed, NftActivity, NftHistoryEntr
 import { CommentCountersManager } from '../utils/CommentsCountersManager'
 import { VideoRelevanceManager } from '../utils/VideoRelevanceManager'
 import { EntityManagerOverlay } from '../utils/overlay'
+import { OrionVideoLanguageManager } from '../utils/OrionVideoLanguageManager'
 
+export const orionVideoLanguageManager = new OrionVideoLanguageManager()
 export const commentCountersManager = new CommentCountersManager()
 export const videoRelevanceManager = new VideoRelevanceManager()
 // eslint-disable-next-line no-void
+void orionVideoLanguageManager.init(
+  1000 * 60 * 5 // 5 mins
+)
+// eslint-disable-next-line no-void
 void videoRelevanceManager.init({
   fullUpdateLoopTime: 1000 * 60 * 60 * 12, // 12 hrs
   scheduledUpdateLoopTime: 1000 * 60 * 10, // 10 mins

src/model/OrionOffchainCursor.ts

@@ -0,0 +1,20 @@
+import { Column, Entity, PrimaryColumn } from 'typeorm'
+
+@Entity({ schema: 'admin' })
+export class OrionOffchainCursor {
+  constructor(props?: Partial<OrionOffchainCursor>) {
+    Object.assign(this, props)
+  }
+
+  /**
+   * Name of the offchain cursor
+   */
+  @PrimaryColumn()
+  cursorName!: string
+
+  /**
+   * Value of the cursor
+   */
+  @Column('int8', { nullable: false })
+  value!: number
+}

src/model/index.ts

@@ -1,2 +1,3 @@
 export * from './generated'
 export { NextEntityId } from './NextEntityId'
+export { OrionOffchainCursor } from './OrionOffchainCursor'

src/server-extension/resolvers/StateResolver/index.ts

@@ -1,4 +1,4 @@
-import { Resolver, Root, Subscription } from 'type-graphql'
+import { Resolver, Root, Subscription, Query, ObjectType, Field } from 'type-graphql'
 import type { EntityManager } from 'typeorm'
 import { ProcessorState } from './types'
 import _, { isObject } from 'lodash'
@@ -73,4 +73,63 @@ export class StateResolver {
   processorState(@Root() state: ProcessorState): ProcessorState {
     return state
   }
+
+  @Query(() => EarningStatsOutput)
+  async totalJoystreamEarnings(): Promise<EarningStatsOutput> {
+    const em = await this.tx()
+    const result = (
+      await em.query<
+        {
+          total_rewards_volume: string
+          total_crt_volume: string
+          total_nft_volume: string
+        }[]
+      >(
+        `
+      SELECT
+          SUM(
+          COALESCE(event.data->>'amount', '0')::bigint
+        ) AS "total_rewards_volume",
+        SUM(
+          COALESCE(amm_buy.price_paid, '0')::bigint
+        ) AS "total_crt_volume",
+        SUM(
+          COALESCE(event.data->>'price', '0')::bigint + COALESCE(winning_bid.amount, 0)
+        ) AS "total_nft_volume"
+      FROM
+          "event"
+      LEFT JOIN amm_transaction AS amm_buy ON "data"->>'ammMintTransaction' = amm_buy.id
+      LEFT JOIN bid AS winning_bid ON "data"->>'winningBid' = winning_bid.id
+      WHERE
+          "event"."data"->>'isTypeOf' IN (
+              'ChannelPaymentMadeEventData',
+              'CreatorTokenMarketMintEventData',
+              'NftBoughtEventData',
+              'EnglishAuctionSettledEventData',
+              'BidMadeCompletingAuctionEventData',
+              'OpenAuctionBidAcceptedEventData'
+          )
+
+      `
+      )
+    )[0]
+
+    return {
+      crtSaleVolume: result.total_crt_volume ?? 0,
+      nftSaleVolume: result.total_nft_volume ?? 0,
+      totalRewardsPaid: result.total_rewards_volume ?? 0,
+    }
+  }
+}
+
+@ObjectType()
+export class EarningStatsOutput {
+  @Field({ nullable: false })
+  crtSaleVolume: string
+
+  @Field({ nullable: false })
+  totalRewardsPaid: string
+
+  @Field({ nullable: false })
+  nftSaleVolume: string
 }