Simple public homefeed query and mutation (Joystream#304)

* update graphql schema

* add partial index on 'video.include_in_home_feed' field

* update video view definition to only include public videos

* regenerate migrations

* add dumbPublicFeedVideos custom query

* add setPublicFeedVideos mutation

* fix lint issue

* add arg to skip video IDs

* revert: update video view definition to only include public videos

* add feat. to unset public feed videos

* address requested change

* bump package version and update CHANGELOG

.prettierignore

@@ -5,7 +5,8 @@ lib
 # Autogenerated stuff
 src/types
 src/model/generated
-db/migrations/*.js
+db/migrations/*-Data.js
+db/migrations/*-Views.js
 schema.graphql
 /scripts/orion-v1-migration/data
-/db/export
+/db/export

CHANGELOG.md

@@ -1,3 +1,19 @@
+# 3.6.0
+
+## Schema changes
+- Added `includeInHomeFeed` field to `Video` entity indicating if the video should be included in the home feed/page.
+
+## Mutations
+### Additions
+- `setOrUnsetPublicFeedVideos`: mutation to set or unset the `includeInHomeFeed` field of a video by the Operator. 
+
+### Queries
+#### Additions
+- `dumbPublicFeedVideos`: resolver to retrieve random `N` videos from list of all homepage videos.
+
+## DB Migrations
+- Added partial index on `Video` entity to include only videos that are included in the home feed (in `db/migrations/2200000000000-Indexes.js`)
+
 # 3.5.0
 
 ## Schema changes

db/migrations/1000000000000-Admin.js

@@ -5,7 +5,9 @@ module.exports = class Admin1000000000000 {
     // Create a new "admin" schema through which the "hidden" entities can be accessed
     await db.query(`CREATE SCHEMA "admin"`)
     // Create admin user with "admin" schema in default "search_path"
-    await db.query(`CREATE USER "${process.env.DB_ADMIN_USER}" WITH PASSWORD '${process.env.DB_ADMIN_PASS}'`)
+    await db.query(
+      `CREATE USER "${process.env.DB_ADMIN_USER}" WITH PASSWORD '${process.env.DB_ADMIN_PASS}'`
+    )
     await db.query(`GRANT pg_read_all_data TO "${process.env.DB_ADMIN_USER}"`)
     await db.query(`GRANT pg_write_all_data TO "${process.env.DB_ADMIN_USER}"`)
     await db.query(`ALTER USER "${process.env.DB_ADMIN_USER}" SET search_path TO admin,public`)

db/migrations/1708791753999-Data.js

@@ -0,0 +1,11 @@
+module.exports = class Data1708791753999 {
+    name = 'Data1708791753999'
+
+    async up(db) {
+        await db.query(`ALTER TABLE "admin"."video" ADD "include_in_home_feed" boolean`)
+    }
+
+    async down(db) {
+        await db.query(`ALTER TABLE "admin"."video" DROP COLUMN "include_in_home_feed"`)
+    }
+}

db/migrations/1708500753231-Views.js → db/migrations/1708791754135-Views.js

@@ -1,8 +1,7 @@
-
 const { getViewDefinitions } = require('../viewDefinitions')
 
-module.exports = class Views1708500753231 {
-  name = 'Views1708500753231'
+module.exports = class Views1708791754135 {
+  name = 'Views1708791754135'
 
   async up(db) {
     const viewDefinitions = getViewDefinitions(db);

db/migrations/2200000000000-Indexes.js

@@ -2,19 +2,29 @@ module.exports = class Indexes2200000000000 {
   name = 'Indexes2200000000000'
 
   async up(db) {
-    await db.query(`CREATE INDEX "events_video" ON "admin"."event" USING BTREE (("data"->>'video'));`)
-    await db.query(`CREATE INDEX "events_comment" ON "admin"."event" USING BTREE (("data"->>'comment'));`)
+    await db.query(
+      `CREATE INDEX "events_video" ON "admin"."event" USING BTREE (("data"->>'video'));`
+    )
+    await db.query(
+      `CREATE INDEX "events_comment" ON "admin"."event" USING BTREE (("data"->>'comment'));`
+    )
     await db.query(
       `CREATE INDEX "events_nft_owner_member" ON "admin"."event" USING BTREE (("data"->'nftOwner'->>'member'));`
     )
     await db.query(
       `CREATE INDEX "events_nft_owner_channel" ON "admin"."event" USING BTREE (("data"->'nftOwner'->>'channel'));`
     )
-    await db.query(`CREATE INDEX "events_auction" ON "admin"."event" USING BTREE (("data"->>'auction'));`)
-    await db.query(`CREATE INDEX "events_type" ON "admin"."event" USING BTREE (("data"->>'isTypeOf'));`)
+    await db.query(
+      `CREATE INDEX "events_auction" ON "admin"."event" USING BTREE (("data"->>'auction'));`
+    )
+    await db.query(
+      `CREATE INDEX "events_type" ON "admin"."event" USING BTREE (("data"->>'isTypeOf'));`
+    )
     await db.query(`CREATE INDEX "events_nft" ON "admin"."event" USING BTREE (("data"->>'nft'));`)
     await db.query(`CREATE INDEX "events_bid" ON "admin"."event" USING BTREE (("data"->>'bid'));`)
-    await db.query(`CREATE INDEX "events_member" ON "admin"."event" USING BTREE (("data"->>'member'));`)
+    await db.query(
+      `CREATE INDEX "events_member" ON "admin"."event" USING BTREE (("data"->>'member'));`
+    )
     await db.query(
       `CREATE INDEX "events_winning_bid" ON "admin"."event" USING BTREE (("data"->>'winningBid'));`
     )
@@ -24,10 +34,21 @@ module.exports = class Indexes2200000000000 {
     await db.query(
       `CREATE INDEX "events_previous_nft_owner_channel" ON "admin"."event" USING BTREE (("data"->'previousNftOwner'->>'channel'));`
     )
-    await db.query(`CREATE INDEX "events_buyer" ON "admin"."event" USING BTREE (("data"->>'buyer'));`)
-    await db.query(`CREATE INDEX "auction_type" ON "admin"."auction" USING BTREE (("auction_type"->>'isTypeOf'));`)
-    await db.query(`CREATE INDEX "member_metadata_avatar" ON "member_metadata" USING BTREE (("avatar"->>'avatarObject'));`)
-    await db.query(`CREATE INDEX "owned_nft_auction" ON "admin"."owned_nft" USING BTREE (("transactional_status"->>'auction'));`)
+    await db.query(
+      `CREATE INDEX "events_buyer" ON "admin"."event" USING BTREE (("data"->>'buyer'));`
+    )
+    await db.query(
+      `CREATE INDEX "auction_type" ON "admin"."auction" USING BTREE (("auction_type"->>'isTypeOf'));`
+    )
+    await db.query(
+      `CREATE INDEX "member_metadata_avatar" ON "member_metadata" USING BTREE (("avatar"->>'avatarObject'));`
+    )
+    await db.query(
+      `CREATE INDEX "owned_nft_auction" ON "admin"."owned_nft" USING BTREE (("transactional_status"->>'auction'));`
+    )
+    await db.query(
+      `CREATE INDEX video_include_in_home_feed_idx ON admin.video (include_in_home_feed) WHERE include_in_home_feed = true;`
+    )
   }
 
   async down(db) {
@@ -44,5 +65,6 @@ module.exports = class Indexes2200000000000 {
     await db.query(`DROP INDEX "events_previous_nft_owner_member"`)
     await db.query(`DROP INDEX "events_previous_nft_owner_channel"`)
     await db.query(`DROP INDEX "events_buyer"`)
+    await db.query(`DROP INDEX "video_include_in_home_feed_idx"`)
   }
 }

db/migrations/2300000000000-Operator.js

@@ -1,11 +1,11 @@
-const { randomAsHex } = require("@polkadot/util-crypto")
-const { existsSync } = require("fs")
+const { randomAsHex } = require('@polkadot/util-crypto')
+const { existsSync } = require('fs')
 const path = require('path')
 const { OffchainState } = require('../../lib/utils/offchainState')
 
 module.exports = class Operator2300000000000 {
   name = 'Operator2300000000000'
-  
+
   async up(db) {
     // Support only one operator account at the moment to avoid confusion
     const exportFilePath = path.join(__dirname, '../export/export.json')
@@ -25,7 +25,7 @@ module.exports = class Operator2300000000000 {
     // Create pg_stat_statements extension for analyzing query stats
     await db.query(`CREATE EXTENSION pg_stat_statements;`)
   }
-  
+
   async down(db) {
     await db.query(`DELETE FROM "admin"."user" WHERE "is_root" = true;`)
     await db.query(`DROP EXTENSION pg_stat_statements;`)

package.json

@@ -1,6 +1,6 @@
 {
   "name": "orion",
-  "version": "3.5.0",
+  "version": "3.6.0",
   "engines": {
     "node": ">=16"
   },

schema/auth.graphql

@@ -11,6 +11,7 @@ enum OperatorPermission {
   SET_FEATURED_NFTS
   EXCLUDE_CONTENT
   RESTORE_CONTENT
+  SET_PUBLIC_FEED_VIDEOS
 }
 
 type User @entity @schema(name: "admin") {

schema/videos.graphql

@@ -129,6 +129,9 @@ type Video @entity @schema(name: "admin") {
 
   "Whether the video is a short format, vertical video (e.g. Youtube Shorts, TikTok, Instagram Reels)"
   isShort: Boolean
+
+  "Optional boolean flag to indicate if the video should be included in the home feed/page."
+  includeInHomeFeed: Boolean
 }
 
 type VideoFeaturedInCategory

src/server-extension/resolvers/VideosResolver/index.ts

@@ -1,53 +1,58 @@
-import 'reflect-metadata'
-import { Arg, Args, Ctx, Info, Mutation, Query, Resolver, UseMiddleware } from 'type-graphql'
-import { EntityManager, MoreThan } from 'typeorm'
-import {
-  AddVideoViewResult,
-  ExcludeVideoInfo,
-  MostViewedVideosConnectionArgs,
-  ReportVideoArgs,
-  VideoReportInfo,
-} from './types'
-import { VideosConnection } from '../baseTypes'
-import {
-  VideoViewEvent,
-  Video,
-  Report,
-  Exclusion,
-  Account,
-  VideoExcluded,
-  ChannelRecipient,
-} from '../../../model'
-import { ensureArray } from '@subsquid/openreader/lib/util/util'
-import { UserInputError } from 'apollo-server-core'
-import { parseOrderBy } from '@subsquid/openreader/lib/opencrud/orderBy'
-import { parseWhere } from '@subsquid/openreader/lib/opencrud/where'
 import {
-  decodeRelayConnectionCursor,
   RelayConnectionRequest,
+  decodeRelayConnectionCursor,
 } from '@subsquid/openreader/lib/ir/connection'
 import { AnyFields } from '@subsquid/openreader/lib/ir/fields'
+import { getConnectionSize } from '@subsquid/openreader/lib/limit.size'
+import { parseOrderBy } from '@subsquid/openreader/lib/opencrud/orderBy'
+import { parseAnyTree, parseSqlArguments } from '@subsquid/openreader/lib/opencrud/tree'
+import { parseWhere } from '@subsquid/openreader/lib/opencrud/where'
+import { ConnectionQuery, CountQuery, ListQuery } from '@subsquid/openreader/lib/sql/query'
 import {
   getResolveTree,
   getTreeRequest,
   hasTreeRequest,
   simplifyResolveTree,
 } from '@subsquid/openreader/lib/util/resolve-tree'
-import { model } from '../model'
+import { ensureArray } from '@subsquid/openreader/lib/util/util'
+import { UserInputError } from 'apollo-server-core'
 import { GraphQLResolveInfo } from 'graphql'
-import { parseAnyTree } from '@subsquid/openreader/lib/opencrud/tree'
-import { getConnectionSize } from '@subsquid/openreader/lib/limit.size'
-import { ConnectionQuery, CountQuery } from '@subsquid/openreader/lib//sql/query'
-import { extendClause, overrideClause, withHiddenEntities } from '../../../utils/sql'
-import { config, ConfigVariable } from '../../../utils/config'
-import { Context } from '../../check'
 import { isObject } from 'lodash'
-import { has } from '../../../utils/misc'
+import 'reflect-metadata'
+import { Arg, Args, Ctx, Info, Mutation, Query, Resolver, UseMiddleware } from 'type-graphql'
+import { EntityManager, In, MoreThan } from 'typeorm'
+import { parseVideoTitle } from '../../../mappings/content/utils'
 import { videoRelevanceManager } from '../../../mappings/utils'
+import {
+  Account,
+  ChannelRecipient,
+  Exclusion,
+  OperatorPermission,
+  Report,
+  Video,
+  VideoExcluded,
+  VideoViewEvent,
+} from '../../../model'
+import { ConfigVariable, config } from '../../../utils/config'
 import { uniqueId } from '../../../utils/crypto'
+import { has } from '../../../utils/misc'
 import { addNotification } from '../../../utils/notification'
-import { parseVideoTitle } from '../../../mappings/content/utils'
-import { UserOnly, OperatorOnly } from '../middleware'
+import { extendClause, overrideClause, withHiddenEntities } from '../../../utils/sql'
+import { Context } from '../../check'
+import { Video as VideoReturnType, VideosConnection } from '../baseTypes'
+import { OperatorOnly, UserOnly } from '../middleware'
+import { model } from '../model'
+import {
+  AddVideoViewResult,
+  DumbPublicFeedArgs,
+  ExcludeVideoInfo,
+  MostViewedVideosConnectionArgs,
+  PublicFeedOperationType,
+  ReportVideoArgs,
+  SetOrUnsetPublicFeedArgs,
+  SetOrUnsetPublicFeedResult,
+  VideoReportInfo,
+} from './types'
 
 @Resolver()
 export class VideosResolver {
@@ -194,6 +199,63 @@ export class VideosResolver {
     return result as VideosConnection
   }
 
+  @Query(() => [VideoReturnType])
+  async dumbPublicFeedVideos(
+    @Args() args: DumbPublicFeedArgs,
+    @Info() info: GraphQLResolveInfo,
+    @Ctx() ctx: Context
+  ): Promise<VideoReturnType[]> {
+    const tree = getResolveTree(info)
+
+    const sqlArgs = parseSqlArguments(model, 'Video', {
+      limit: args.limit,
+      where: args.where,
+    })
+
+    const videoFields = parseAnyTree(model, 'Video', info.schema, tree)
+
+    const listQuery = new ListQuery(model, ctx.openreader.dialect, 'Video', videoFields, sqlArgs)
+
+    let listQuerySql = listQuery.sql
+
+    listQuerySql = extendClause(
+      listQuerySql,
+      'WHERE',
+      `"video"."include_in_home_feed" = true AND "video"."id" NOT IN (${args.skipVideoIds
+        .map((id) => `'${id}'`)
+        .join(', ')})`,
+      'AND'
+    )
+
+    listQuerySql = extendClause(listQuerySql, 'ORDER BY', 'RANDOM()', '')
+    ;(listQuery as { sql: string }).sql = listQuerySql
+
+    const result = await ctx.openreader.executeQuery(listQuery)
+
+    return result as VideoReturnType[]
+  }
+
+  @Mutation(() => SetOrUnsetPublicFeedResult)
+  @UseMiddleware(OperatorOnly(OperatorPermission.SET_PUBLIC_FEED_VIDEOS))
+  async setOrUnsetPublicFeedVideos(
+    @Args() { videoIds, operation }: SetOrUnsetPublicFeedArgs
+  ): Promise<SetOrUnsetPublicFeedResult> {
+    const em = await this.em()
+
+    return withHiddenEntities(em, async () => {
+      const result = await em
+        .createQueryBuilder()
+        .update<Video>(Video)
+        .set({ includeInHomeFeed: operation === PublicFeedOperationType.SET })
+        .where({ id: In(videoIds) })
+        .execute()
+
+      return {
+        numberOfEntitiesAffected: result.affected || 0,
+      }
+    })
+  }
+
   @UseMiddleware(UserOnly)
   @Mutation(() => AddVideoViewResult)
   async addVideoView(

src/server-extension/resolvers/VideosResolver/types.ts

@@ -1,7 +1,7 @@
-import { ArgsType, Field, ObjectType, Int } from 'type-graphql'
+import { MaxLength } from 'class-validator'
+import { ArgsType, Field, Int, ObjectType, registerEnumType } from 'type-graphql'
 import { Video, VideoOrderByInput, VideoWhereInput } from '../baseTypes'
 import { EntityReportInfo } from '../commonTypes'
-import { MaxLength } from 'class-validator'
 
 @ObjectType()
 export class VideosSearchResult {
@@ -94,3 +94,47 @@ export class ExcludeVideoInfo extends EntityReportInfo {
   @Field(() => String, { nullable: false })
   videoId!: string
 }
+
+@ArgsType()
+export class DumbPublicFeedArgs {
+  @Field(() => VideoWhereInput, { nullable: true })
+  where?: Record<string, unknown>
+
+  @Field(() => [String], {
+    nullable: true,
+    description:
+      'The list of video ids to skip/exclude from the public feed videos. Maybe because they are already shown to the user.',
+  })
+  skipVideoIds!: string[]
+
+  @Field(() => Int, {
+    nullable: true,
+    defaultValue: 100,
+    description: 'The number of videos to return',
+  })
+  limit?: number
+}
+
+export enum PublicFeedOperationType {
+  SET = 'set',
+  UNSET = 'unset',
+}
+registerEnumType(PublicFeedOperationType, { name: 'PublicFeedOperationType' })
+
+@ArgsType()
+export class SetOrUnsetPublicFeedArgs {
+  @Field(() => [String], { nullable: false })
+  videoIds!: string[]
+
+  @Field(() => PublicFeedOperationType, {
+    nullable: false,
+    description: 'Type of operation to perform',
+  })
+  operation: PublicFeedOperationType
+}
+
+@ObjectType()
+export class SetOrUnsetPublicFeedResult {
+  @Field(() => Int)
+  numberOfEntitiesAffected!: number
+}