From 3ea30ede663339b163be24858b97646f2074a944 Mon Sep 17 00:00:00 2001 From: Zeeshan Akram <37098720+zeeshanakram3@users.noreply.github.com> Date: Thu, 14 Mar 2024 03:50:16 +0500 Subject: [PATCH] Nara off-chain migration script fixes (#315) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add is short field to video entity (#301) * add isShort field to video entity * regenerate db migrations * remove @joystream/metadata-protobuf patch from assets/patches * fix lint issue * Disable both in Appp and eail notifications for video posted events (#299) * bump package version and update CHANGELOG (#302) * bump package version and update CHANGELOG * change release version * Simple public homefeed query and mutation (#304) * update graphql schema * add partial index on 'video.include_in_home_feed' field * update video view definition to only include public videos * regenerate migrations * add dumbPublicFeedVideos custom query * add setPublicFeedVideos mutation * fix lint issue * add arg to skip video IDs * revert: update video view definition to only include public videos * add feat. to unset public feed videos * address requested change * bump package version and update CHANGELOG * Update `nara` from `master` (#300) * Adds mappings for `ChannelAssetsDeletedByModerator` & `VideoAssetsDeletedByModerator` events (#199) * mark 'VideoDeletedByModerator' & 'ChannelDeletedByModerator' events deprecated * Implements mappings for 'Content.VideoAssetsDeletedByModerator and 'Content.ChannelAssetsDeletedByModerator' runtime events * remove unused import * Nara/crt update (#244) * feat: build orion * feat: start generating schema * fix: extra entities * fixup! * fix: continue implementing design specs * fix: review and fix foreign key relationships * fix: formatting * fix: generation errors * fix: add comment * fix: relations * fix: final review * fixup! * fix: add ending blocks * fix: generate type & set typegen to ipv4 * fix: add support for event backward compatibility * feat: start adding mappings * fix: continue with mappnigs * feat: init sale * feat: patronage decreased to & fixed build * feat: claim patronage event * feat: tokens bought on amm * feat: tokens sold on amm * fix: add relation between sales and vesting schedules * feat: add Tokens sold on sale vente * feat: update upcoming sale * feat: revenue share issued * feat: member joined whitelist * feat: amm deactivated * feat: burned token * feat: transfer policy changed to permissionless * feat: sale finalized * feat: finish mappings * fix: review * fix: remove cascade deletions * fix: renaming & formatting * fixup! * fixup! * fix: patched protobuf packages with token proto * feat: update metadata and add event handler scheleton * feat: token metadata * feat: sale metadata * fix: review comments * fix: formatting * fix: revenue * Revert "fix: revenue" This reverts commit 0821abe1e5e19be37a8d9a25d621dd148f03be9c. * fix: token status after sale * fix: fixmes * fix: formatting * fix: funds accounting during sale * fix: amount accounting * fix: linter * fix: review * fix: review 2 * fix: review * fix: linter * feat: migration for new db scheam * fix: update event versions * fix: patch types with crt_release types * fix: patch types * fix: generate all events versions since mainnet * fix: temp fix after event version generation * fix: event versioning * fix: add migration * fix: mignations * fix: solve channel not being added * fix: add id to TokenChannel * fix: non-nullable deleted field set * fix: format * feat: creator token init sale re enabling * feat: re enable sale init code * fix: update types * fix: amm id * fix: id computation for revenue share * fix: amm id computation for token * fix: issuer transfer accounting * fix: amm tx id * fix: destination accounting * feat: minor fix on holder transfer processing * fix: re-enable metadata * fix: metadata parsing * fix: post reword cleanup * fix: format * fix: silence ci checks * fix: event version * fix: address PR changes I edited all the entity that have a composite index like TokenAccount so that they have a synthetic ID and an optionally unique @index * fix: add hidden entities conditions * fix: add extra fields to token in order to keep track of ongoing status * fix: build errors * fix: adapt mapping to new token fields * fix: format * feat: add trailer video entity this is required so we can simply make trailer video hidden if video is hidden * fix: linter * chore: prettier * fix: from PR review * fix: vesting schedule schema & mappings I have replaced the vesting schedule back to the original schema with: - VestingSchedule: holding vesting schedule information such being amount agnostic - VestedAccount: contains information regarded to a vested account, the goal is to mimic the runtime logic * fix: burning from vesting * patch: metadata-protobuf package * patch: metadata-protobuf package * fix: generate migrations * fix: purchase token on sale * Update schema/token.graphql Co-authored-by: Leszek Wiesner * Update schema/token.graphql Co-authored-by: Leszek Wiesner * fix: address PR * fix: hidden entities * fix: migration ok * feat: add extra check for migrations * fix: docker network * fix: format * fix: remove unrequired constraint * fix: :bug: post rebase fixes * feat: :art: add metadata processing for issue token * feat(crt-v1): :sparkles: chain metadata for v 2003 * fix(crt-v1): :ambulance: comment out view element for orion playgroud * fix(crt-v1): :art: add playground config variable to .env * feat: :white_check_mark: add tests * fix(crt-v1): :package: packages and patches * fix(crt-v1): :white_check_mark: update entity id used and other minor fixes * fix(crt-v1): :white_check_mark: update entity id used and other minor fixes * test(crt-v1): :bug: misc fixes to have tests working * test(crt-v1): :bug: misc fixes to have tests working * fix(crt-v1): :bug: metadata and trailer video * feat(crt-v1): :art: update types * fix(crt-v1): :sparkles: Add correct Ratio denomination (Permill) * update with master * fix: :bug: metadata not being set * fix: :bug: parameters order * test: :test_tube: fixing integration tests * test(crt-v1): :test_tube: fix integration tests * feat(crt-v1): :sparkles: last price for token and recovered field for rev share part * feat: :sparkles: add resolver for dividend amount * feat(crt-v1): :sparkles: start adding channel fields for trackingtotal revenue * feat(crt-v1): :sparkles: add utils for royalty computation * feat(crt-v1): :sparkles: cumulative revenue on channel * feat(crt-v1): :sparkles: add resolver for transferrable amount * fix(crt-v1): :sparkles: add `acquiredAt` to pinpoint latest vesting schedule for account * Token metadata processing update * Prettier * chore(crt-v1): :zap: dbgen * fix(crt-v1): :test_tube: fix integration tests * fix(crt-v1): :bug: missing fields in token sale vesting source * test(crt-v1): :test_tube: test for transferrable balance amount * fix(crt-v1): :bug: transferrable amount * test: :test_tube: update tests after resolver fix * fix: :bug: error on vesting schedules array * fix: :art: CI fixes * docs: update gitignore * fix: :rotating_light: prettier * build: :pushpin: chai depnedencies --------- Co-authored-by: Leszek Wiesner Co-authored-by: WRadoslaw * Clear benefits even if not passed (#282) * 🤑 Fix revenue share dividend estimation (#297) * Fix on revenue share dividend estimation * Fix type on result * 🛕 Historical revenue share participants (#286) * New field for revenue share * Set potential revenue share particitants at the time of start * fix: .gitignore not working * fix lint issues * re-generate db migrations * commit register.html.mst file * fix: notifications integration test --------- Co-authored-by: Ignazio Bovo Co-authored-by: Leszek Wiesner Co-authored-by: WRadoslaw Co-authored-by: WRadoslaw <92513933+WRadoslaw@users.noreply.github.com> * Revert "Update `nara` from `master` (#300)" (#306) This reverts commit 887427c75548417dedc741f9b258f7d49e1a0b4e. * generate auth api docs and types * add is short derived field to video entity (#310) * add is shirt derived field to video entity * add indices on is short fields * fix: video language detection fix (#309) * fix: video language detection fix * address requested changes * fix: predictVideoLanguage function * fix: include max 1 video per channel in homepage videos (#313) * fix: include max 1 video per channel in homepage videos * update setOrionLanguage Migration script * format updateVideoRelevanceValue SQL query * fix: use UTC midnight epoch instead of current epoch to calculate video relevance score * bump package version and update CHANGELOG * fix: lint bug * remove NextEntityIdManager migration script * [offchainState] add v4.0.0 (CRT release) migrations * [offchainState] remove ORDER BY clause from UPDATE statements * add migration for NextEntityId * bump package version and update CHANGELOG --------- Co-authored-by: Ignazio Bovo Co-authored-by: Leszek Wiesner Co-authored-by: WRadoslaw Co-authored-by: WRadoslaw <92513933+WRadoslaw@users.noreply.github.com> --- CHANGELOG.md | 4 ++ package-lock.json | 4 +- package.json | 2 +- src/mappings/utils.ts | 16 ++++---- src/processor.ts | 3 +- src/utils/NextEntityIdManager.ts | 26 ------------ src/utils/notification/helpers.ts | 2 +- src/utils/offchainState.ts | 68 +++++++++++++++++++++++++++++-- src/utils/overlay.ts | 5 --- 9 files changed, 81 insertions(+), 49 deletions(-) delete mode 100644 src/utils/NextEntityIdManager.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index b4b994c84..7f7787229 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# 4.0.0 + +This is Creator Tokens (CRT) release. It introduces the CRT mappings, custom resolvers and mutations. + # 3.7.0 ## Schema changes diff --git a/package-lock.json b/package-lock.json index dc52e4dc7..6c36b0368 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "orion", - "version": "3.7.0", + "version": "4.0.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "orion", - "version": "3.7.0", + "version": "4.0.0", "hasInstallScript": true, "workspaces": [ "network-tests" diff --git a/package.json b/package.json index f7849c5ee..7a5583828 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "orion", - "version": "3.7.0", + "version": "4.0.0", "engines": { "node": ">=16" }, diff --git a/src/mappings/utils.ts b/src/mappings/utils.ts index 900c750fd..49d1f8877 100644 --- a/src/mappings/utils.ts +++ b/src/mappings/utils.ts @@ -1,20 +1,18 @@ -import { metaToObject } from '@joystream/metadata-protobuf/utils' import { AnyMetadataClass, DecodedMetadataObject } from '@joystream/metadata-protobuf/types' -import { Logger } from '../logger' -import { SubstrateBlock } from '@subsquid/substrate-processor' -import { Event, MetaprotocolTransactionResultFailed, NftActivity, NftHistoryEntry } from '../model' -import { encodeAddress } from '@polkadot/util-crypto' -import { EntityManagerOverlay } from '../utils/overlay' -import { Bytes } from '@polkadot/types/primitive' +import { metaToObject } from '@joystream/metadata-protobuf/utils' import { createType } from '@joystream/types' +import { Bytes } from '@polkadot/types/primitive' import { u8aToHex } from '@polkadot/util' +import { encodeAddress } from '@polkadot/util-crypto' +import { SubstrateBlock } from '@subsquid/substrate-processor' +import { Logger } from '../logger' +import { Event, MetaprotocolTransactionResultFailed, NftActivity, NftHistoryEntry } from '../model' import { CommentCountersManager } from '../utils/CommentsCountersManager' import { VideoRelevanceManager } from '../utils/VideoRelevanceManager' -import { NextEntityIdManager } from '../utils/NextEntityIdManager' +import { EntityManagerOverlay } from '../utils/overlay' export const commentCountersManager = new CommentCountersManager() export const videoRelevanceManager = new VideoRelevanceManager() -export const migrateCounters = new NextEntityIdManager() // eslint-disable-next-line no-void void videoRelevanceManager.init({ fullUpdateLoopTime: 1000 * 60 * 60 * 12, // 12 hrs diff --git a/src/processor.ts b/src/processor.ts index da542ec1f..2bc28e9c2 100644 --- a/src/processor.ts +++ b/src/processor.ts @@ -113,7 +113,7 @@ import { processUpcomingTokenSaleUpdatedEvent, processUserParticipatedInSplitEvent, } from './mappings/token' -import { commentCountersManager, migrateCounters, videoRelevanceManager } from './mappings/utils' +import { commentCountersManager, videoRelevanceManager } from './mappings/utils' import { Event } from './types/support' import { EventHandler, EventInstance, EventNames, eventConstructors } from './utils/events' import { assertAssignable } from './utils/misc' @@ -415,7 +415,6 @@ processor.run(new TypeormDatabase({ isolationLevel: 'READ COMMITTED' }), async ( await commentCountersManager.updateVideoCommentsCounters(em, true) await commentCountersManager.updateParentRepliesCounters(em, true) await videoRelevanceManager.updateVideoRelevanceValue(em, true) - await migrateCounters.migrateCounters(overlay) ctx.log.info(`Offchain state successfully imported!`) } } diff --git a/src/utils/NextEntityIdManager.ts b/src/utils/NextEntityIdManager.ts deleted file mode 100644 index d4f226736..000000000 --- a/src/utils/NextEntityIdManager.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { createLogger } from '@subsquid/logger' -import { NextEntityId } from '../model' -import { EntityManagerOverlay } from './overlay' - -export class NextEntityIdManager { - private _migrationDone = false - private logger = createLogger('NextEntityIdManager') - private entities = ['Account'] - - public async migrateCounters(overlay: EntityManagerOverlay): Promise { - if (this._migrationDone) { - return - } - // TODO (^3.2.0): use better migration logic for migrating Ids OFFCHAIN_NOTIFICATION_ID_TAG + nextId & RUNTIME_NOTIFICATION_ID_TAG + nextId - const em = overlay.getEm() - for (const entityName of this.entities) { - // build query that gets the entityName with the highest id - overlay.invalidateRepository(entityName) - const rowNumber = await em.query(`SELECT COUNT(*) FROM ${entityName}`) - const latestId = parseInt(rowNumber[0].count) - this.logger.info(`🔄 Migrating next entity if for: ${entityName}`) - await em.save(new NextEntityId({ entityName, nextId: latestId + 1 })) - this._migrationDone = true - } - } -} diff --git a/src/utils/notification/helpers.ts b/src/utils/notification/helpers.ts index 393837455..3b705f539 100644 --- a/src/utils/notification/helpers.ts +++ b/src/utils/notification/helpers.ts @@ -19,7 +19,7 @@ import { EntityManagerOverlay } from '../overlay' export const RUNTIME_NOTIFICATION_ID_TAG = 'RuntimeNotification' export const OFFCHAIN_NOTIFICATION_ID_TAG = 'OffchainNotification' -function notificationPrefAllTrue(): NotificationPreference { +export function notificationPrefAllTrue(): NotificationPreference { return new NotificationPreference({ inAppEnabled: true, emailEnabled: true }) } diff --git a/src/utils/offchainState.ts b/src/utils/offchainState.ts index be15c8bbd..3e1cd514f 100644 --- a/src/utils/offchainState.ts +++ b/src/utils/offchainState.ts @@ -5,8 +5,15 @@ import fs from 'fs' import path from 'path' import { EntityManager } from 'typeorm' import * as model from '../model' +import { + AccountNotificationPreferences, + fromJsonDeliveryStatus, + fromJsonNotificationType, + fromJsonReadOrUnread, + fromJsonRecipientType, +} from '../model' import { uniqueId } from './crypto' -import { defaultNotificationPreferences } from './notification/helpers' +import { defaultNotificationPreferences, notificationPrefAllTrue } from './notification/helpers' const DEFAULT_EXPORT_PATH = path.resolve(__dirname, '../../db/export/export.json') @@ -112,11 +119,51 @@ function migrateExportDataToV320(data: ExportedData): ExportedData { return data } +export function setCrtNotificationPreferences( + notificationPreferencesObj: any +): AccountNotificationPreferences { + notificationPreferencesObj.crtIssued = notificationPrefAllTrue() + notificationPreferencesObj.crtMarketStarted = notificationPrefAllTrue() + notificationPreferencesObj.crtMarketMint = notificationPrefAllTrue() + notificationPreferencesObj.crtMarketBurn = notificationPrefAllTrue() + notificationPreferencesObj.crtSaleStarted = notificationPrefAllTrue() + notificationPreferencesObj.crtSaleMint = notificationPrefAllTrue() + notificationPreferencesObj.crtRevenueShareStarted = notificationPrefAllTrue() + notificationPreferencesObj.crtRevenueSharePlanned = notificationPrefAllTrue() + notificationPreferencesObj.crtRevenueShareEnded = notificationPrefAllTrue() + const notificationPreferences = new AccountNotificationPreferences( + undefined, + notificationPreferencesObj + ) + return notificationPreferences +} + +function migrateExportDataToV400(data: ExportedData): ExportedData { + data.Account?.values.forEach((account) => { + // account will find himself with all CRT notification pref. enabled by default + account.notificationPreferences = setCrtNotificationPreferences( + account.notificationPreferences as AccountNotificationPreferences + ) + }) + + data.Notification?.values.forEach((notification) => { + notification.notificationType = fromJsonNotificationType(notification.notificationType) + notification.status = fromJsonReadOrUnread(notification.status) + notification.recipient = fromJsonRecipientType(notification.recipient) + }) + + data.EmailDeliveryAttempt?.values.forEach((emailDeliveryAttempt) => { + emailDeliveryAttempt.status = fromJsonDeliveryStatus(emailDeliveryAttempt.status) + }) + return data +} + export class OffchainState { private logger = createLogger('offchainState') private _isImported = false private migrations: Migrations = { + '4.0.0': migrateExportDataToV400, '3.2.0': migrateExportDataToV320, '3.0.0': migrateExportDataToV300, } @@ -252,7 +299,6 @@ export class OffchainState { }) .join(', ')} ) AS "data" - ORDER BY "id" WHERE "${meta.tableName}"."id" = "data"."id"`, fieldNames.map((fieldName) => batch.map((v) => v[fieldName])) ) @@ -269,7 +315,23 @@ export class OffchainState { values.length } entities left)...` ) - await em.getRepository(entityName).insert(batch) + + // UPSERT operation specifically for NextEntityId + if (entityName === 'NextEntityId') { + for (const entity of batch) { + await em.query( + ` + INSERT INTO "next_entity_id" ("entity_name", "next_id") + VALUES ($1, $2) + ON CONFLICT (entity_name) + DO UPDATE SET next_id = EXCLUDED.next_id; + `, + [entity.entityName, entity.nextId] + ) + } + } else { + await em.getRepository(entityName).insert(batch) + } } } this.logger.info( diff --git a/src/utils/overlay.ts b/src/utils/overlay.ts index f8e6e4ff8..82c83644b 100644 --- a/src/utils/overlay.ts +++ b/src/utils/overlay.ts @@ -361,11 +361,6 @@ export class EntityManagerOverlay { return this.em } - // reason: during migration the overlay would write to the database the old nextId - public invalidateRepository(entityName: string) { - this.repositories.delete(entityName) - } - // Create an entity repository overlay or load already cached one public getRepository(entityClass: Constructor): RepositoryOverlay { const loadedRepository = this.repositories.get(entityClass.name)