Nara off-chain migration script fixes

CHANGELOG.md

@@ -1,3 +1,7 @@
+# 4.0.0
+
+This is Creator Tokens (CRT) release. It introduces the CRT mappings, custom resolvers and mutations.
+
 # 3.7.0
 
 ## Schema changes

package.json

@@ -1,6 +1,6 @@
 {
   "name": "orion",
-  "version": "3.7.0",
+  "version": "4.0.0",
   "engines": {
     "node": ">=16"
   },

src/mappings/utils.ts

@@ -1,20 +1,18 @@
-import { metaToObject } from '@joystream/metadata-protobuf/utils'
 import { AnyMetadataClass, DecodedMetadataObject } from '@joystream/metadata-protobuf/types'
-import { Logger } from '../logger'
-import { SubstrateBlock } from '@subsquid/substrate-processor'
-import { Event, MetaprotocolTransactionResultFailed, NftActivity, NftHistoryEntry } from '../model'
-import { encodeAddress } from '@polkadot/util-crypto'
-import { EntityManagerOverlay } from '../utils/overlay'
-import { Bytes } from '@polkadot/types/primitive'
+import { metaToObject } from '@joystream/metadata-protobuf/utils'
 import { createType } from '@joystream/types'
+import { Bytes } from '@polkadot/types/primitive'
 import { u8aToHex } from '@polkadot/util'
+import { encodeAddress } from '@polkadot/util-crypto'
+import { SubstrateBlock } from '@subsquid/substrate-processor'
+import { Logger } from '../logger'
+import { Event, MetaprotocolTransactionResultFailed, NftActivity, NftHistoryEntry } from '../model'
 import { CommentCountersManager } from '../utils/CommentsCountersManager'
 import { VideoRelevanceManager } from '../utils/VideoRelevanceManager'
-import { NextEntityIdManager } from '../utils/NextEntityIdManager'
+import { EntityManagerOverlay } from '../utils/overlay'
 
 export const commentCountersManager = new CommentCountersManager()
 export const videoRelevanceManager = new VideoRelevanceManager()
-export const migrateCounters = new NextEntityIdManager()
 // eslint-disable-next-line no-void
 void videoRelevanceManager.init({
   fullUpdateLoopTime: 1000 * 60 * 60 * 12, // 12 hrs

src/processor.ts

@@ -113,7 +113,7 @@ import {
   processUpcomingTokenSaleUpdatedEvent,
   processUserParticipatedInSplitEvent,
 } from './mappings/token'
-import { commentCountersManager, migrateCounters, videoRelevanceManager } from './mappings/utils'
+import { commentCountersManager, videoRelevanceManager } from './mappings/utils'
 import { Event } from './types/support'
 import { EventHandler, EventInstance, EventNames, eventConstructors } from './utils/events'
 import { assertAssignable } from './utils/misc'
@@ -415,7 +415,6 @@ processor.run(new TypeormDatabase({ isolationLevel: 'READ COMMITTED' }), async (
       await commentCountersManager.updateVideoCommentsCounters(em, true)
       await commentCountersManager.updateParentRepliesCounters(em, true)
       await videoRelevanceManager.updateVideoRelevanceValue(em, true)
-      await migrateCounters.migrateCounters(overlay)
       ctx.log.info(`Offchain state successfully imported!`)
     }
   }

src/utils/notification/helpers.ts

@@ -19,7 +19,7 @@ import { EntityManagerOverlay } from '../overlay'
 export const RUNTIME_NOTIFICATION_ID_TAG = 'RuntimeNotification'
 export const OFFCHAIN_NOTIFICATION_ID_TAG = 'OffchainNotification'
 
-function notificationPrefAllTrue(): NotificationPreference {
+export function notificationPrefAllTrue(): NotificationPreference {
   return new NotificationPreference({ inAppEnabled: true, emailEnabled: true })
 }
 

src/utils/offchainState.ts

@@ -5,8 +5,15 @@ import fs from 'fs'
 import path from 'path'
 import { EntityManager } from 'typeorm'
 import * as model from '../model'
+import {
+  AccountNotificationPreferences,
+  fromJsonDeliveryStatus,
+  fromJsonNotificationType,
+  fromJsonReadOrUnread,
+  fromJsonRecipientType,
+} from '../model'
 import { uniqueId } from './crypto'
-import { defaultNotificationPreferences } from './notification/helpers'
+import { defaultNotificationPreferences, notificationPrefAllTrue } from './notification/helpers'
 
 const DEFAULT_EXPORT_PATH = path.resolve(__dirname, '../../db/export/export.json')
 
@@ -112,11 +119,51 @@ function migrateExportDataToV320(data: ExportedData): ExportedData {
   return data
 }
 
+export function setCrtNotificationPreferences(
+  notificationPreferencesObj: any
+): AccountNotificationPreferences {
+  notificationPreferencesObj.crtIssued = notificationPrefAllTrue()
+  notificationPreferencesObj.crtMarketStarted = notificationPrefAllTrue()
+  notificationPreferencesObj.crtMarketMint = notificationPrefAllTrue()
+  notificationPreferencesObj.crtMarketBurn = notificationPrefAllTrue()
+  notificationPreferencesObj.crtSaleStarted = notificationPrefAllTrue()
+  notificationPreferencesObj.crtSaleMint = notificationPrefAllTrue()
+  notificationPreferencesObj.crtRevenueShareStarted = notificationPrefAllTrue()
+  notificationPreferencesObj.crtRevenueSharePlanned = notificationPrefAllTrue()
+  notificationPreferencesObj.crtRevenueShareEnded = notificationPrefAllTrue()
+  const notificationPreferences = new AccountNotificationPreferences(
+    undefined,
+    notificationPreferencesObj
+  )
+  return notificationPreferences
+}
+
+function migrateExportDataToV400(data: ExportedData): ExportedData {
+  data.Account?.values.forEach((account) => {
+    // account will find himself with all CRT notification pref. enabled by default
+    account.notificationPreferences = setCrtNotificationPreferences(
+      account.notificationPreferences as AccountNotificationPreferences
+    )
+  })
+
+  data.Notification?.values.forEach((notification) => {
+    notification.notificationType = fromJsonNotificationType(notification.notificationType)
+    notification.status = fromJsonReadOrUnread(notification.status)
+    notification.recipient = fromJsonRecipientType(notification.recipient)
+  })
+
+  data.EmailDeliveryAttempt?.values.forEach((emailDeliveryAttempt) => {
+    emailDeliveryAttempt.status = fromJsonDeliveryStatus(emailDeliveryAttempt.status)
+  })
+  return data
+}
+
 export class OffchainState {
   private logger = createLogger('offchainState')
   private _isImported = false
 
   private migrations: Migrations = {
+    '4.0.0': migrateExportDataToV400,
     '3.2.0': migrateExportDataToV320,
     '3.0.0': migrateExportDataToV300,
   }
@@ -252,7 +299,6 @@ export class OffchainState {
                 })
                 .join(', ')}
             ) AS "data"
-            ORDER BY "id"
             WHERE "${meta.tableName}"."id" = "data"."id"`,
             fieldNames.map((fieldName) => batch.map((v) => v[fieldName]))
           )
@@ -269,7 +315,23 @@ export class OffchainState {
               values.length
             } entities left)...`
           )
-          await em.getRepository(entityName).insert(batch)
+
+          // UPSERT operation specifically for NextEntityId
+          if (entityName === 'NextEntityId') {
+            for (const entity of batch) {
+              await em.query(
+                `
+                INSERT INTO "next_entity_id" ("entity_name", "next_id")
+                VALUES ($1, $2)
+                ON CONFLICT (entity_name)
+                DO UPDATE SET next_id = EXCLUDED.next_id;
+              `,
+                [entity.entityName, entity.nextId]
+              )
+            }
+          } else {
+            await em.getRepository(entityName).insert(batch)
+          }
         }
       }
       this.logger.info(

src/utils/overlay.ts

@@ -361,11 +361,6 @@ export class EntityManagerOverlay {
     return this.em
   }
 
-  // reason: during migration the overlay would write to the database the old nextId
-  public invalidateRepository(entityName: string) {
-    this.repositories.delete(entityName)
-  }
-
   // Create an entity repository overlay or load already cached one
   public getRepository<E extends AnyEntity>(entityClass: Constructor<E>): RepositoryOverlay<E> {
     const loadedRepository = this.repositories.get(entityClass.name)