From 4ee063c9e208f24a811cdc5a0322c067768898dd Mon Sep 17 00:00:00 2001 From: Ignazio Bovo Date: Fri, 17 Nov 2023 10:05:36 +0100 Subject: [PATCH] Notification branch fixes for issued arised during Q&A (#225) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update squid.yaml fix: squid.yml version numbering * Release PR: Orion v3.0.1 (#185) * fix: processor's overlay bug (#170) * atlas homepage query speed optimisation (#177) * fix: gitignored src/model/generated * add index on video.video_relevance field * fix typo in developer guide docs * regenerate db/migrations/*-Data.js file * Fix/unblock graphql (#183) * Make session optional and apply changes for auth handles * Create new middleware for users and use it on some resolvers * Remove remaining auth guards * Small docs change * Add middleware for channel report * Prettier * docs: :memo: update changelog (#184) --------- Co-authored-by: Zeeshan Akram <37098720+zeeshanakram3@users.noreply.github.com> Co-authored-by: WRadoslaw <92513933+WRadoslaw@users.noreply.github.com> * fix: :ambulance: fix accountId global counter not being migrated (#188) * docs: :memo: add back up guide (#196) * mark 'VideoDeletedByModerator' & 'ChannelDeletedByModerator' events deprecated (#193) * fix: :bug: encoding issues with member handle (#214) * fix: :bug: encoding issues with member handle * fix: :bug: pr issues * fix: :bug: refactor and update changelog * docs: update version number * fix: :ambulance: account counter (#215) * Orion v3.0.3 (#224) * Remove n+1 problem for StorageDataObject (#209) * Remove n+1 problem for StorageDataObject * Inform user about incorrect query * fix: :sparkles: add global counter migratino to account for 303 (#217) * Orion 303 final touches (#223) * feat: :sparkles: add CORS origins for atlas local testing * chore: :memo: adds 303 release notes * Fix typos (#176) * fix typos * fix typos * fix typos * fix typos * fix typos * fix typo * fix typos * fix typos * fix typos * fix typos * fix typos * fix typos --------- Co-authored-by: WRadoslaw <92513933+WRadoslaw@users.noreply.github.com> Co-authored-by: omahs <73983677+omahs@users.noreply.github.com> * fix(notifications): uninitialized access fields * fix(notifications): :bug: add channelId to new auction * Orion v3.0.4 - Release (#230) * 🧮 Aggregate channel payments (#222) * Fix channels `cumulativeRewardClaimed_DESC` order * Aggregate channel payments * Make `cumulativeRewardClaimed` and `cumulativeRewardPaid` non null * Generate db migrations * Rename the `cumulativeReward` field * Re-generate db migrations * Bump Orion's version * feat: :zap: add migration to version 3.0.4 for account id counter (#228) --------- Co-authored-by: Theophile Sandoz * Orion v3.1.0 (#238) * Add granular permissions support for Gateway operator users (#231) * Add granular permissions support for Gateway operator users * fix lint issues * revert docker-compose port change * mark 'grantPermissions' & 'revokePermissions' input fields are non-nullable & return new permissions instead of boolean * Set Channel Weight (`setChannelsWeights`) mutation (#232) * Add granular permissions support for Gateway operator users * fix lint issues * add mutation to set channel weight/bias for homepage video relevance * revert docker-compose port change * mark 'grantPermissions' & 'revokePermissions' input fields are non-nullable & return new permissions instead of boolean * bump package version * update global migration counter map * bumped package version & updated CHANGELOG --------- Co-authored-by: Ignazio Bovo * Postgres performance improvements (#235) * add index in video.createdAt field * add pg_stat_extenstion extenstion for queries stats * docs: :sparkles: changelog and fix data-js (#237) --------- Co-authored-by: Zeeshan Akram <37098720+zeeshanakram3@users.noreply.github.com> * Hotfix/3.1.0 (#240) * Orion v3.1.0 (#238) * Add granular permissions support for Gateway operator users (#231) * Add granular permissions support for Gateway operator users * fix lint issues * revert docker-compose port change * mark 'grantPermissions' & 'revokePermissions' input fields are non-nullable & return new permissions instead of boolean * Set Channel Weight (`setChannelsWeights`) mutation (#232) * Add granular permissions support for Gateway operator users * fix lint issues * add mutation to set channel weight/bias for homepage video relevance * revert docker-compose port change * mark 'grantPermissions' & 'revokePermissions' input fields are non-nullable & return new permissions instead of boolean * bump package version * update global migration counter map * bumped package version & updated CHANGELOG --------- Co-authored-by: Ignazio Bovo * Postgres performance improvements (#235) * add index in video.createdAt field * add pg_stat_extenstion extenstion for queries stats * docs: :sparkles: changelog and fix data-js (#237) --------- Co-authored-by: Zeeshan Akram <37098720+zeeshanakram3@users.noreply.github.com> * fix: :ambulance: hotfix default value for channel weight no matter what --------- Co-authored-by: Zeeshan Akram <37098720+zeeshanakram3@users.noreply.github.com> * fix: :white_check_mark: add test for comment reply and metadata * fix: :sparkles: solve conflicts * test: :white_check_mark: add test for channel verification + refactoring * chore: :rotating_light: linter * fix: :green_heart: add 10 seconds sleep time for CI checks * fix: :bug: channel excluded title * docs: :memo: typo --------- Co-authored-by: Zeeshan Akram <37098720+zeeshanakram3@users.noreply.github.com> Co-authored-by: WRadoslaw <92513933+WRadoslaw@users.noreply.github.com> Co-authored-by: omahs <73983677+omahs@users.noreply.github.com> Co-authored-by: Theophile Sandoz --- .env | 5 +- CHANGELOG.md | 13 +- ...37206830-Data.js => 1700052773258-Data.js} | 10 +- db/migrations/2200000000000-Operator.js | 4 + db/postgres.conf | 4 +- docker-compose.yml | 3 +- package-lock.json | 4 +- package.json | 2 +- schema/auth.graphql | 18 ++ schema/channels.graphql | 5 +- schema/notifications.graphql | 4 +- src/mail-scheduler/index.ts | 3 - src/mappings/content/commentsAndReactions.ts | 4 +- src/mappings/content/nft.ts | 4 +- src/mappings/content/utils.ts | 7 +- src/server-extension/check.ts | 4 +- .../resolvers/AdminResolver/index.ts | 177 ++++++++++++---- .../resolvers/AdminResolver/types.ts | 68 +++++- .../resolvers/ChannelsResolver/index.ts | 111 +++++----- .../resolvers/VideosResolver/index.ts | 10 +- src/server-extension/resolvers/middleware.ts | 34 ++- src/tests/integration/notifications.test.ts | 199 ++++++++++++++---- src/tests/integration/testUtils.ts | 5 +- src/tests/migrations/migration.test.ts | 2 +- src/tests/migrations/run.sh | 2 +- src/utils/VideoRelevanceManager.ts | 21 +- src/utils/config.ts | 4 +- src/utils/notification/notificationsData.ts | 4 +- src/utils/offchainState.ts | 13 +- 29 files changed, 553 insertions(+), 191 deletions(-) rename db/migrations/{1699537206830-Data.js => 1700052773258-Data.js} (99%) diff --git a/.env b/.env index d3501078c..2de6afd2d 100644 --- a/.env +++ b/.env @@ -38,9 +38,10 @@ VIDEO_RELEVANCE_VIEWS_TICK=50 # views weight, # comments weight, # rections weights, -# [joystream creation weight, YT creation weight] +# [joystream creation weight, YT creation weight], +# Default channel weight/bias # ] -RELEVANCE_WEIGHTS="[1, 0.03, 0.3, 0.5, [7,3]]" +RELEVANCE_WEIGHTS="[1, 0.03, 0.3, 0.5, [7,3], 1]" MAX_CACHED_ENTITIES=1000 APP_PRIVATE_KEY=this-is-not-so-secret-change-it SESSION_EXPIRY_AFTER_INACTIVITY_MINUTES=60 diff --git a/CHANGELOG.md b/CHANGELOG.md index fb60c0f49..33c00c5b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,17 @@ -# 3.1.0 +# 3.2.0 This release adds notifications to the orion infrastructure... +# 3.1.0 + +### Entities +- Adds `User.permission` to the `User` entity, this however doesn't require migration logic. +- Adds `Channel.channelWeights` in order to boost channel relevance. This value can be set via the `setChannelWeights` mutation +### Resolvers +- Adds supports for new permissions model for gateway operator users. Now the root user can assign/revoke operator permission/s to users using `grantPermissions` & `revokePermissions` mutations +- Adds new `setChannelWeights` operator mutation to set weight/bias for any channel/s which will be used to calculate the Atlas homepage video relevance scores +### Performance +- Adds `Video.createdAt` as index in order to speed up Atlas home page queries + # 3.0.4 ### Misc diff --git a/db/migrations/1699537206830-Data.js b/db/migrations/1700052773258-Data.js similarity index 99% rename from db/migrations/1699537206830-Data.js rename to db/migrations/1700052773258-Data.js index 0a34df4a5..6d7461788 100644 --- a/db/migrations/1699537206830-Data.js +++ b/db/migrations/1700052773258-Data.js @@ -1,5 +1,5 @@ -module.exports = class Data1699537206830 { - name = 'Data1699537206830' +module.exports = class Data1700052773258 { + name = 'Data1700052773258' async up(db) { await db.query(`CREATE TABLE "channel_follow" ("id" character varying NOT NULL, "user_id" character varying, "channel_id" text NOT NULL, "timestamp" TIMESTAMP WITH TIME ZONE NOT NULL, CONSTRAINT "PK_9410df2b9a316af3f0d216f9487" PRIMARY KEY ("id"))`) @@ -15,7 +15,7 @@ module.exports = class Data1699537206830 { await db.query(`CREATE TABLE "nft_featuring_request" ("id" character varying NOT NULL, "user_id" character varying, "nft_id" text NOT NULL, "timestamp" TIMESTAMP WITH TIME ZONE NOT NULL, "rationale" text NOT NULL, CONSTRAINT "PK_d0b1ccb74336b30b9575387d328" PRIMARY KEY ("id"))`) await db.query(`CREATE INDEX "IDX_519be2a41216c278c35f254dcb" ON "nft_featuring_request" ("user_id") `) await db.query(`CREATE INDEX "IDX_76d87e26cce72ac2e7ffa28dfb" ON "nft_featuring_request" ("nft_id") `) - await db.query(`CREATE TABLE "user" ("id" character varying NOT NULL, "is_root" boolean NOT NULL, CONSTRAINT "PK_cace4a159ff9f2512dd42373760" PRIMARY KEY ("id"))`) + await db.query(`CREATE TABLE "user" ("id" character varying NOT NULL, "is_root" boolean NOT NULL, "permissions" character varying(34) array, CONSTRAINT "PK_cace4a159ff9f2512dd42373760" PRIMARY KEY ("id"))`) await db.query(`CREATE TABLE "storage_bucket" ("id" character varying NOT NULL, "operator_status" jsonb NOT NULL, "accepting_new_bags" boolean NOT NULL, "data_objects_size_limit" numeric NOT NULL, "data_object_count_limit" numeric NOT NULL, "data_objects_count" numeric NOT NULL, "data_objects_size" numeric NOT NULL, CONSTRAINT "PK_97cd0c3fe7f51e34216822e5f91" PRIMARY KEY ("id"))`) await db.query(`CREATE TABLE "storage_bucket_bag" ("id" character varying NOT NULL, "storage_bucket_id" character varying, "bag_id" character varying, CONSTRAINT "StorageBucketBag_storageBucket_bag" UNIQUE ("storage_bucket_id", "bag_id") DEFERRABLE INITIALLY DEFERRED, CONSTRAINT "PK_9d54c04557134225652d566cc82" PRIMARY KEY ("id"))`) await db.query(`CREATE INDEX "IDX_aaf00b2c7d0cba49f97da14fbb" ON "storage_bucket_bag" ("bag_id") `) @@ -37,7 +37,8 @@ module.exports = class Data1699537206830 { await db.query(`CREATE TABLE "app" ("id" character varying NOT NULL, "name" text NOT NULL, "owner_member_id" character varying, "website_url" text, "use_uri" text, "small_icon" text, "medium_icon" text, "big_icon" text, "one_liner" text, "description" text, "terms_of_service" text, "platforms" text array, "category" text, "auth_key" text, CONSTRAINT "App_name" UNIQUE ("name") DEFERRABLE INITIALLY DEFERRED, CONSTRAINT "PK_9478629fc093d229df09e560aea" PRIMARY KEY ("id"))`) await db.query(`CREATE INDEX "IDX_f36adbb7b096ceeb6f3e80ad14" ON "app" ("name") `) await db.query(`CREATE INDEX "IDX_c9cc395bbc485f70a15be64553" ON "app" ("owner_member_id") `) - await db.query(`CREATE TABLE "channel" ("id" character varying NOT NULL, "created_at" TIMESTAMP WITH TIME ZONE NOT NULL, "owner_member_id" character varying, "title" text, "description" text, "cover_photo_id" character varying, "avatar_photo_id" character varying, "is_public" boolean, "is_censored" boolean NOT NULL, "is_excluded" boolean NOT NULL, "language" text, "created_in_block" integer NOT NULL, "reward_account" text NOT NULL, "channel_state_bloat_bond" numeric NOT NULL, "follows_num" integer NOT NULL, "video_views_num" integer NOT NULL, "entry_app_id" character varying, "total_videos_created" integer NOT NULL, "cumulative_reward_claimed" numeric NOT NULL, "cumulative_reward" numeric NOT NULL, "ypp_status" jsonb NOT NULL, CONSTRAINT "PK_590f33ee6ee7d76437acf362e39" PRIMARY KEY ("id"))`) + await db.query(`CREATE TABLE "channel" ("id" character varying NOT NULL, "created_at" TIMESTAMP WITH TIME ZONE NOT NULL, "owner_member_id" character varying, "title" text, "description" text, "cover_photo_id" character varying, "avatar_photo_id" character varying, "is_public" boolean, "is_censored" boolean NOT NULL, "is_excluded" boolean NOT NULL, "language" text, "created_in_block" integer NOT NULL, "reward_account" text NOT NULL, "channel_state_bloat_bond" numeric NOT NULL, "follows_num" integer NOT NULL, "video_views_num" integer NOT NULL, "entry_app_id" character varying, "total_videos_created" integer NOT NULL, "cumulative_reward_claimed" numeric NOT NULL, "cumulative_reward" numeric NOT NULL, "channel_weight" numeric, "ypp_status" jsonb NOT NULL, CONSTRAINT "PK_590f33ee6ee7d76437acf362e39" PRIMARY KEY ("id"))`) + await db.query(`CREATE INDEX "IDX_a4752a0a0899dedc4d18077dd0" ON "channel" ("created_at") `) await db.query(`CREATE INDEX "IDX_25c85bc448b5e236a4c1a5f789" ON "channel" ("owner_member_id") `) await db.query(`CREATE INDEX "IDX_a77e12f3d8c6ced020e179a5e9" ON "channel" ("cover_photo_id") `) await db.query(`CREATE INDEX "IDX_6997e94413b3f2f25a84e4a96f" ON "channel" ("avatar_photo_id") `) @@ -259,6 +260,7 @@ module.exports = class Data1699537206830 { await db.query(`DROP INDEX "public"."IDX_f36adbb7b096ceeb6f3e80ad14"`) await db.query(`DROP INDEX "public"."IDX_c9cc395bbc485f70a15be64553"`) await db.query(`DROP TABLE "channel"`) + await db.query(`DROP INDEX "public"."IDX_a4752a0a0899dedc4d18077dd0"`) await db.query(`DROP INDEX "public"."IDX_25c85bc448b5e236a4c1a5f789"`) await db.query(`DROP INDEX "public"."IDX_a77e12f3d8c6ced020e179a5e9"`) await db.query(`DROP INDEX "public"."IDX_6997e94413b3f2f25a84e4a96f"`) diff --git a/db/migrations/2200000000000-Operator.js b/db/migrations/2200000000000-Operator.js index 3cd270948..c3fe72716 100644 --- a/db/migrations/2200000000000-Operator.js +++ b/db/migrations/2200000000000-Operator.js @@ -19,9 +19,13 @@ module.exports = class Operator2300000000000 { INSERT INTO "admin"."user" ("id", "is_root") VALUES ('${process.env.OPERATOR_SECRET || randomAsHex(32)}', true); `) + + // Create pg_stat_statements extension for analyzing query stats + await db.query(`CREATE EXTENSION pg_stat_statements`) } async down(db) { await db.query(`DELETE FROM "admin"."user" WHERE "is_root" = true;`) + await db.query(`DROP EXTENSION pg_stat_statements`) } } diff --git a/db/postgres.conf b/db/postgres.conf index 9514b42e0..fd121d761 100644 --- a/db/postgres.conf +++ b/db/postgres.conf @@ -3,4 +3,6 @@ log_statement = all autovacuum_analyze_scale_factor = 0.01 shared_buffers=2GB jit=off -random_page_cost=1.0 \ No newline at end of file +random_page_cost=1.0 +shared_preload_libraries = 'pg_stat_statements' +pg_stat_statements.track = all diff --git a/docker-compose.yml b/docker-compose.yml index 8af3bbb97..6b6b59ddb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,8 +10,7 @@ services: POSTGRES_DB: squid POSTGRES_PASSWORD: squid ports: - - '127.0.0.1:${DB_PORT}:${DB_PORT}' - - '[::1]:${DB_PORT}:${DB_PORT}' + - '${DB_PORT}:${DB_PORT}' command: ['postgres', '-c', 'config_file=/etc/postgresql/postgresql.conf', '-p', '${DB_PORT}'] shm_size: 1g volumes: diff --git a/package-lock.json b/package-lock.json index cb2a8653d..073f860cc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "orion", - "version": "3.1.0", + "version": "3.2.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "orion", - "version": "3.1.0", + "version": "3.2.0", "hasInstallScript": true, "dependencies": { "@joystream/js": "^1.4.0", diff --git a/package.json b/package.json index a03b92bb0..8ef4a08f6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "orion", - "version": "3.1.0", + "version": "3.2.0", "engines": { "node": ">=16" }, diff --git a/schema/auth.graphql b/schema/auth.graphql index cd11f911a..3b48c4035 100644 --- a/schema/auth.graphql +++ b/schema/auth.graphql @@ -1,3 +1,18 @@ +enum OperatorPermission { + GRANT_OPERATOR_PERMISSIONS + REVOKE_OPERATOR_PERMISSIONS + SET_VIDEO_WEIGHTS + SET_CHANNEL_WEIGHTS + SET_KILL_SWITCH + SET_VIDEO_VIEW_PER_USER_TIME_LIMIT + SET_VIDEO_HERO + SET_CATEGORY_FEATURED_VIDEOS + SET_SUPPORTED_CATEGORIES + SET_FEATURED_NFTS + EXCLUDE_CONTENT + RESTORE_CONTENT +} + type User @entity { "Unique identifier (32-byte string, securely random)" id: ID! @@ -5,6 +20,9 @@ type User @entity { "Whether the user has root (gateway operator) privileges" isRoot: Boolean! + "List of all the gateway operator permissions that this user has" + permissions: [OperatorPermission!] + "The account associated with the user (if any)" account: Account @derivedFrom(field: "user") diff --git a/schema/channels.graphql b/schema/channels.graphql index ec886430d..2fd4a206c 100644 --- a/schema/channels.graphql +++ b/schema/channels.graphql @@ -3,7 +3,7 @@ type Channel @entity { id: ID! "Timestamp of the block the channel was created at" - createdAt: DateTime! + createdAt: DateTime! @index "Current member-owner of the channel (if owned by a member)" ownerMember: Membership @@ -65,6 +65,9 @@ type Channel @entity { "Cumulative rewards paid to this channel" cumulativeReward: BigInt! + "Weight/Bias of the channel affecting video relevance in the Homepage" + channelWeight: Float + "Channel Ypp Status: either unverified , verified or suspended" yppStatus: ChannelYppStatus! } diff --git a/schema/notifications.graphql b/schema/notifications.graphql index 6e4ef8cce..5b861882f 100644 --- a/schema/notifications.graphql +++ b/schema/notifications.graphql @@ -125,11 +125,13 @@ type NftFeaturedOnMarketPlace @variant { } type ChannelVerified @variant { + "no data needed as recipient is channel" phantom: Int } type ChannelExcluded @variant { - phantom: Int + "title for the channel used for notification text" + channelTitle: String! } type VideoExcluded @variant { diff --git a/src/mail-scheduler/index.ts b/src/mail-scheduler/index.ts index 5a71cc051..68f22f417 100644 --- a/src/mail-scheduler/index.ts +++ b/src/mail-scheduler/index.ts @@ -1,6 +1,3 @@ -import { config as dontenvConfig } from 'dotenv' -import path from 'path' - import { ConfigVariable, config } from '../utils/config' import { EmailDeliveryAttempt, NotificationEmailDelivery } from '../model' import { EntityManager } from 'typeorm' diff --git a/src/mappings/content/commentsAndReactions.ts b/src/mappings/content/commentsAndReactions.ts index 638b0a24e..2175c5c40 100644 --- a/src/mappings/content/commentsAndReactions.ts +++ b/src/mappings/content/commentsAndReactions.ts @@ -513,11 +513,13 @@ export async function processCreateCommentMessage( videoId: video.id, videoTitle: parseVideoTitle(video), memberHandle: await memberHandleById(overlay, memberId), + memberId, } + const memberRecipientId = parentComment.authorId || undefined await addNotification( overlay, authorAccount, - new MemberRecipient({ membership: comment.authorId }), + new MemberRecipient({ membership: memberRecipientId }), new CommentReply(notificationData), event ) diff --git a/src/mappings/content/nft.ts b/src/mappings/content/nft.ts index 0f9f746be..894bea94c 100644 --- a/src/mappings/content/nft.ts +++ b/src/mappings/content/nft.ts @@ -50,7 +50,6 @@ import { NftOfferedEventData, Account, MemberRecipient, - AuctionTypeOpen, } from '../../model' import { addNftActivity, addNftHistoryEntry, genericEventFields } from '../utils' import { SubstrateBlock, assertNotNull } from '@subsquid/substrate-processor' @@ -91,6 +90,7 @@ export async function processOpenAuctionStartedEvent({ if (video.channelId) { const channelTitle = await getChannelTitleById(overlay, video.channelId) const notificationData = new NewAuction({ + channelId: video.channelId, channelTitle, videoId: video.id, videoTitle: parseVideoTitle(video), @@ -138,6 +138,7 @@ export async function processEnglishAuctionStartedEvent({ if (video.channelId) { const channelTitle = await getChannelTitleById(overlay, video.channelId) const notificationData = new NewAuction({ + channelId: video.channelId, channelTitle, videoId: video.id, videoTitle: parseVideoTitle(video), @@ -547,6 +548,7 @@ export async function processNftSellOrderMadeEvent({ if (video?.channelId) { const channelTitle = await getChannelTitleById(overlay, video.channelId) const notificationData = new NewNftOnSale({ + channelId: video.channelId, channelTitle, videoId: video.id, videoTitle: parseVideoTitle(video), diff --git a/src/mappings/content/utils.ts b/src/mappings/content/utils.ts index e81e70886..2303f7e31 100644 --- a/src/mappings/content/utils.ts +++ b/src/mappings/content/utils.ts @@ -866,11 +866,11 @@ export function computeRoyalty(royaltyPct: number, price: bigint): bigint { } export function parseChannelTitle(channel: Flat): string { - return channel.title || '??' + return channel.title || FALLBACK_CHANNEL_TITLE } export function parseVideoTitle(video: Flat