ci: 💚 add ci checks for migration

.github/workflows/tests.yml

@@ -62,3 +62,22 @@ jobs:
         run: make prepare
       - name: Run tests
         run: npm run tests:notifications
+  post-update-migrations:
+    name: Post update migrations tests
+    needs: [migrations]
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        node-version: [16.x]
+      fail-fast: true
+    steps:
+      - uses: actions/checkout@v2
+      - name: Use Node.js ${{matrix.node-version}}
+        uses: actions/setup-node@v1
+        with:
+          node-version: ${{matrix.node-version}}
+      - name: Prepare workspace
+        run: make prepare
+      - name: Run tests
+        run: npm run tests:migrations

package.json

@@ -21,6 +21,7 @@
     "tests:compareState": "npx ts-node ./src/tests/compareState.ts",
     "tests:benchmark": "npx ts-node ./src/tests/benchmarks/index.ts",
     "tests:auth-api": "./src/auth-server/tests/run.sh",
+    "tests:migrations": "./src/tests/migrations/run.sh",
     "tests:notifications": "./src/tests/integration/run.sh",
     "offchain-state:export": "node ./lib/scripts/export.js",
     "get-public-key": "node ./lib/scripts/getPublicKey.js"

src/tests/migrations/migration.test.ts

@@ -42,7 +42,7 @@ describe('Migration from 3.0.1 to 3.1.0', () => {
     })
   })
   describe('Channels are migrated properly', () => {
-    let aliceChannelId = '1'
+    const aliceChannelId = '1'
     let aliceChannel: Channel
     before(async () => {
       aliceChannel = await em.getRepository(Channel).findOneByOrFail({ id: aliceChannelId })

src/tests/migrations/run.sh

@@ -15,19 +15,20 @@ cd ..
 cp ./src/tests/migrations/export.json ./db/export/export.json
 
 # load the archive db
-until docker logs orion_archive_db | grep "database system is ready to accept connections"; do
+until docker logs orion_archive_db | grep "database system is ready to accept connections" >/dev/null; do
     sleep 1
 done
 docker cp ./src/tests/migrations/archive_dump orion_archive_db:/tmp/archive_dump
-docker exec orion_archive_db psql -f /tmp/archive_dump -U postgres postgres -p 12345
+docker exec orion_archive_db psql -f /tmp/archive_dump -U postgres -p 12345 >/dev/null
 
 # start the orion stack
 docker-compose up -d orion_db
-until docker logs orion_db | grep "database system is ready to accept connections"; do
+until docker logs orion_db | grep "database system is ready to accept connections" >/dev/null; do
     sleep 1
 done
 make migrate >/dev/null
 docker-compose up -d orion_processor
+# TODO: improve this
 # until docker logs orion_processor | grep "Saving database updates..." >/dev/null; do
 #     sleep 1
 # done

src/utils/NextEntityIdManager.ts

@@ -11,7 +11,7 @@ export class NextEntityIdManager {
     if (this._migrationDone) {
       return
     }
-    //TODO (^3.2.0): use better migration logic for migrating Ids OFFCHAIN_NOTIFICATION_ID_TAG + nextId & RUNTIME_NOTIFICATION_ID_TAG + nextId
+    // TODO (^3.2.0): use better migration logic for migrating Ids OFFCHAIN_NOTIFICATION_ID_TAG + nextId & RUNTIME_NOTIFICATION_ID_TAG + nextId
     const em = overlay.getEm()
     for (const entityName of this.entities) {
       // build query that gets the entityName with the highest id

src/utils/offchainState.ts

@@ -5,7 +5,6 @@ import { createLogger } from '@subsquid/logger'
 import assert from 'assert'
 import { uniqueId } from './crypto'
 import { defaultNotificationPreferences } from './notification/helpers'
-import { NextEntityId, YppUnverified } from '../model'
 
 const DEFAULT_EXPORT_PATH = path.resolve(__dirname, '../../db/export/export.json')
 
@@ -96,12 +95,6 @@ export class OffchainState {
   private logger = createLogger('offchainState')
   private _isImported = false
 
-  private globalCountersMigration = {
-    // destination version : [global counters names]
-    '3.0.1': ['Account'],
-    '3.1.0': ['Account'],
-  }
-
   private migrations: Migrations = {
     '3.1.0': migrateExportDataToV310,
     '3.0.0': migrateExportDataToV300,
@@ -275,23 +268,4 @@ export class OffchainState {
     this.logger.info(`Last export block number established: ${blockNumber}`)
     return blockNumber
   }
-
-  private async migrateCounters(exportedVersion: string, em: EntityManager): Promise<void> {
-    //TODO (^3.2.0): use better migration logic for migrating Ids OFFCHAIN_NOTIFICATION_ID_TAG + nextId & RUNTIME_NOTIFICATION_ID_TAG + nextId
-    const migrationData = Object.entries(this.globalCountersMigration).sort(
-      ([a], [b]) => this.versionToNumber(a) - this.versionToNumber(b)
-    ) // sort in increasing order
-
-    for (const [version, counters] of migrationData) {
-      if (this.versionToNumber(exportedVersion) < this.versionToNumber(version)) {
-        this.logger.info(`Migrating global counters to version ${version}`)
-        for (const entityName of counters) {
-          // build query that gets the entityName with the highest id
-          const rowNumber = await em.query(`SELECT COUNT(*) FROM ${entityName}`)
-          const latestId = parseInt(rowNumber[0].count)
-          await em.save(new NextEntityId({ entityName, nextId: latestId + 1 }))
-        }
-      }
-    }
-  }
 }

src/utils/overlay.ts

@@ -6,7 +6,7 @@
 import { Store } from '@subsquid/typeorm-store'
 import AsyncLock from 'async-lock'
 import _, { isObject } from 'lodash'
-import { EntityManager, FindOptionsWhere, In, Not, Repository, SimpleConsoleLogger } from 'typeorm'
+import { EntityManager, FindOptionsWhere, In, Not, Repository } from 'typeorm'
 import { Logger } from '../logger'
 import { NextEntityId } from '../model/NextEntityId'
 import { criticalError, idStringFromNumber } from './misc'