Orion db migrations and state export fixes

.gitignore

@@ -15,3 +15,7 @@ src/auth-server/emails/templates/preview
 /db/persisted
 /scripts/orion-v1-migration/data
 /db/export
+/db/migrations_dir*
+
+# LOGS
+chron_mail_scheduler.log

CHANGELOG.md

@@ -1,3 +1,26 @@
+# 3.4.0
+
+## Schema changes
+- Added `@schema(name: "admin")` directive to hide entities (from public GRAPHQL API) in Graphql schema definitions.
+
+## Misc
+- Patch `@subsquid/typeorm-config` & `@subsquid/typeorm-migration` packages to change `squid-typeorm-migration apply` command to apply a single migrations file too using `--filename` option instead of applying the whole `db/migrations` directory.
+- Patch `@subsquid/openreader` and `@subsquid/typeorm-codegen` dependencies to include the db schema `name` too in the generated typeorm/postgres migrations, and an optional `schema` directive to specify the schema of any GRAPHQL entity.
+
+## DB Migrations
+- Update `generate-migrations` makefile command. Now the existing `*-Data.js` will not be overwritten, instead a new `*-Data.js` migration file (containing only changes compared to the previous DB state) will be added whenever there are GRAPHQL schema changes. The `*-Views.js` migration file will also be updated whenever the GRAPHQL schema changes.
+- Create `generateViewsMigration.js` script to create new `*-Views.js` migration file.
+- Separate the view definitions(in `db/viewDefinitions.js`) from views migration file(`*-Views.js`).
+- Add `*-Admin.js` migration file to create an `admin` schema & user, previously the `admin` schema and user was being created in the `*-Views.js` migration.
+- Regenerate the postgres db migrations.
+
+## Documentation
+- Updated documentation for [upgrading-orion.md](docs/operator-guide/tutorials/upgrading-orion.md)
+- Updated documentation for [entity-visibility.md#managing-entity-visibility](docs/developer-guide/tutorials/entity-visibility.md)
+
+### Bug Fixes:
+- read/write `export.json` file, containing the offchain state, using `big-json` package, instead using Javascript native `JSON.stringify` function which does not work on large JSON objects
+
 # 3.3.0
 
 ## Schema

Makefile

@@ -24,16 +24,25 @@ dbgen:
 	@npx squid-typeorm-migration generate
 
 generate-migrations: 
-	@rm db/migrations/*-Data.js || true
 	@docker run -d --name temp_migrations_db \
 		-e POSTGRES_DB=squid \
 		-e POSTGRES_HOST_AUTH_METHOD=trust \
 		-v temp_migrations_db_volume:/var/lib/postgresql/data \
 		-v ./db/postgres.conf:/etc/postgresql/postgresql.conf \
 		-p 5555:5555 postgres:14 postgres -p 5555 || true
-	@export DB_PORT=5555 && sleep 5 && npx squid-typeorm-migration generate
+	@sleep 5
+	@export DB_PORT=5555 && npx squid-typeorm-migration apply --filename *-Admin.js
+	@ls db/migrations > db/migrations_dir_before.txt
+	@export DB_PORT=5555 && npx squid-typeorm-migration apply --filename *-Data.js
+	@export DB_PORT=5555 && npx squid-typeorm-migration generate || true
+	@ls db/migrations > db/migrations_dir_after.txt
+	@if ! diff db/migrations_dir_before.txt db/migrations_dir_after.txt > /dev/null ; then \
+		rm db/migrations/*-Views.js; \
+		node db/generateViewsMigration.js; \
+	fi;
 	@docker rm temp_migrations_db -vf || true
 	@docker volume rm temp_migrations_db_volume || true
+	@rm db/migrations_dir_before.txt db/migrations_dir_after.txt || true
 
 codegen:
 	@npm run generate:schema || true

assets/patches/@subsquid+openreader+3.1.7.patch

@@ -20,6 +20,54 @@ index ccb64b5..faa6199 100644
  }
  //# sourceMappingURL=context.d.ts.map
 \ No newline at end of file
+diff --git a/node_modules/@subsquid/openreader/lib/model.schema.js b/node_modules/@subsquid/openreader/lib/model.schema.js
+index 25be2da..74f14da 100644
+--- a/node_modules/@subsquid/openreader/lib/model.schema.js
++++ b/node_modules/@subsquid/openreader/lib/model.schema.js
+@@ -11,6 +11,7 @@ const model_tools_1 = require("./model.tools");
+ const scalars_1 = require("./scalars");
+ const baseSchema = (0, graphql_1.buildASTSchema)((0, graphql_1.parse)(`
+     directive @entity on OBJECT
++    directive @schema(name: String!) on OBJECT
+     directive @query on INTERFACE
+     directive @derivedFrom(field: String!) on FIELD_DEFINITION
+     directive @unique on FIELD_DEFINITION
+@@ -57,11 +58,12 @@ function addEntityOrJsonObjectOrInterface(model, type) {
+     let properties = {};
+     let interfaces = [];
+     let indexes = type instanceof graphql_1.GraphQLObjectType ? checkEntityIndexes(type) : [];
++    let schema = type instanceof graphql_1.GraphQLObjectType ? checkEntitySchema(type) : undefined;
+     let cardinality = checkEntityCardinality(type);
+     let description = type.description || undefined;
+     switch (kind) {
+         case 'entity':
+-            model[type.name] = { kind, properties, description, interfaces, indexes, ...cardinality };
++            model[type.name] = {kind, properties, description, interfaces, indexes, schema, ...cardinality}
+             break;
+         case 'object':
+             model[type.name] = { kind, properties, description, interfaces };
+@@ -393,6 +395,21 @@ function checkEntityIndexes(type) {
+     });
+     return indexes;
+ }
++function checkEntitySchema(type) {
++    let schema;
++    type.astNode?.directives?.forEach(d => {
++        if (d.name.value != 'schema')
++            return;
++        if (!isEntityType(type))
++            throw new SchemaError(`@schema was applied to ${type.name}, but only entities can have base schema`);
++        let fieldsArg = d.arguments?.find(arg => arg.name.value == 'name');
++        if (fieldsArg == null)
++            throw new SchemaError(`@schema was applied to ${type.name}, but no name were specified`);
++        (0, assert_1.default)(fieldsArg.value.kind == 'StringValue');
++        schema = fieldsArg.value.value;
++    });
++    return schema;
++}
+ function checkDerivedFrom(type, f) {
+     let directives = f.astNode?.directives?.filter(d => d.name.value == 'derivedFrom') || [];
+     if (directives.length == 0)
 diff --git a/node_modules/@subsquid/openreader/lib/server.d.ts b/node_modules/@subsquid/openreader/lib/server.d.ts
 index da31a85..3691776 100644
 --- a/node_modules/@subsquid/openreader/lib/server.d.ts

assets/patches/@subsquid+typeorm-codegen+0.3.1.patch

@@ -1,8 +1,8 @@
 diff --git a/node_modules/@subsquid/typeorm-codegen/lib/codegen.js b/node_modules/@subsquid/typeorm-codegen/lib/codegen.js
-index e40f9bc..648e5e8 100644
+index e40f9bc..1055524 100644
 --- a/node_modules/@subsquid/typeorm-codegen/lib/codegen.js
 +++ b/node_modules/@subsquid/typeorm-codegen/lib/codegen.js
-@@ -62,9 +62,17 @@ function generateOrmModels(model, dir) {
+@@ -62,11 +62,19 @@ function generateOrmModels(model, dir) {
          out.line();
          printComment(entity, out);
          entity.indexes?.forEach(index => {
@@ -19,8 +19,11 @@ index e40f9bc..648e5e8 100644
 -            out.line(`@Index_([${index.fields.map(f => `"${f.name}"`).join(', ')}], {unique: ${!!index.unique}})`);
 +            out.line(`@Index_([${index.fields.map(f => `"${f.name}"`).join(', ')}])`);
          });
-         out.line('@Entity_()');
+-        out.line('@Entity_()');
++        out.line(`@Entity_(${entity.schema ? `{schema: "${entity.schema}"}` : ''})`);
          out.block(`export class ${name}`, () => {
+             out.block(`constructor(props?: Partial<${name}>)`, () => {
+                 out.line('Object.assign(this, props)');
 @@ -109,8 +117,8 @@ function generateOrmModels(model, dir) {
                      case 'fk':
                          if (getFieldIndex(entity, key)?.unique) {

assets/patches/@subsquid+typeorm-config+2.0.2.patch

@@ -0,0 +1,33 @@
+diff --git a/node_modules/@subsquid/typeorm-config/lib/config.js b/node_modules/@subsquid/typeorm-config/lib/config.js
+index 046611f..28fd4e2 100644
+--- a/node_modules/@subsquid/typeorm-config/lib/config.js
++++ b/node_modules/@subsquid/typeorm-config/lib/config.js
+@@ -28,22 +28,22 @@ const path = __importStar(require("path"));
+ const process = __importStar(require("process"));
+ const connectionOptions_1 = require("./connectionOptions");
+ const namingStrategy_1 = require("./namingStrategy");
+-exports.MIGRATIONS_DIR = 'db/migrations';
++exports.MIGRATIONS_DIR = "db/migrations";
+ function createOrmConfig(options) {
+     let dir = path.resolve(options?.projectDir || process.cwd());
+-    let model = resolveModel(path.join(dir, 'lib/model'));
++    let model = resolveModel(path.join(dir, "lib/model"));
+     let migrationsDir = path.join(dir, exports.MIGRATIONS_DIR);
+     return {
+-        type: 'postgres',
++        type: "postgres",
+         namingStrategy: new namingStrategy_1.SnakeNamingStrategy(),
+         entities: [model],
+-        migrations: [migrationsDir + '/*.js'],
+-        ...(0, connectionOptions_1.createConnectionOptions)()
++        migrations: [migrationsDir + `/${options?.file || "*.js"}`],
++        ...(0, connectionOptions_1.createConnectionOptions)(),
+     };
+ }
+ exports.createOrmConfig = createOrmConfig;
+ function resolveModel(model) {
+-    model = path.resolve(model || 'lib/model');
++    model = path.resolve(model || "lib/model");
+     try {
+         return require.resolve(model);
+     }

assets/patches/@subsquid+typeorm-migration+0.1.4.patch

@@ -0,0 +1,30 @@
+diff --git a/node_modules/@subsquid/typeorm-migration/lib/apply.js b/node_modules/@subsquid/typeorm-migration/lib/apply.js
+index a8dd9c8..999ed2a 100644
+--- a/node_modules/@subsquid/typeorm-migration/lib/apply.js
++++ b/node_modules/@subsquid/typeorm-migration/lib/apply.js
+@@ -29,10 +29,14 @@ const commander_1 = require("commander");
+ const dotenv = __importStar(require("dotenv"));
+ const typeorm_1 = require("typeorm");
+ (0, util_internal_1.runProgram)(async () => {
+-    commander_1.program.description('Apply pending migrations').parse();
++    commander_1.program
++        .description("Apply pending migrations")
++        .option("-f, --filename <type>", "Specify a filename")
++        .parse();
++    const options = commander_1.program.opts();
+     dotenv.config();
+     let connection = new typeorm_1.DataSource({
+-        ...(0, typeorm_config_1.createOrmConfig)(),
++        ...(0, typeorm_config_1.createOrmConfig)({ file: options.filename }),
+         subscribers: [],
+         synchronize: false,
+         migrationsRun: false,
+@@ -41,7 +45,7 @@ const typeorm_1 = require("typeorm");
+     });
+     await connection.initialize();
+     try {
+-        await connection.runMigrations({ transaction: 'all' });
++        await connection.runMigrations({ transaction: "all" });
+     }
+     finally {
+         await connection.destroy().catch(() => null);

db/generateViewsMigration.js

@@ -0,0 +1,49 @@
+const fs = require('fs')
+const path = require('path')
+
+const migrationsDir = path.join(__dirname, 'migrations')
+
+// Function to generate the new views migration file
+const generateViewsMigration = (versionNumber) => {
+  const className = `Views${versionNumber}`
+  const fileName = `${versionNumber}-Views.js`
+  const fileContent = `
+const { getViewDefinitions } = require('../viewDefinitions')
+
+module.exports = class ${className} {
+  name = '${className}'
+
+  async up(db) {
+    const viewDefinitions = getViewDefinitions(db);
+    for (const [tableName, viewConditions] of Object.entries(viewDefinitions)) {
+      if (Array.isArray(viewConditions)) {
+        await db.query(\`
+          CREATE OR REPLACE VIEW "\${tableName}" AS
+            SELECT *
+            FROM "admin"."\${tableName}" AS "this"
+            WHERE \${viewConditions.map(cond => \`(\${cond})\`).join(' AND ')}
+        \`);
+      } else {
+        await db.query(\`
+          CREATE OR REPLACE VIEW "\${tableName}" AS (\${viewConditions})
+        \`);
+      }
+    }
+  }  
+
+  async down(db) {
+    const viewDefinitions = this.getViewDefinitions(db)
+    for (const viewName of Object.keys(viewDefinitions)) {
+      await db.query(\`DROP VIEW "\${viewName}"\`)
+    }
+  }
+}
+`
+
+  const filePath = path.join(migrationsDir, fileName)
+  fs.writeFileSync(filePath, fileContent)
+  console.log(`Generated new views migration: ${filePath}`)
+}
+
+// Get the latest data migration number and generate the views migration
+generateViewsMigration(Date.now())

db/migrations/1000000000000-Admin.js

@@ -0,0 +1,17 @@
+module.exports = class Admin1000000000000 {
+  name = 'Admin1000000000000'
+
+  async up(db) {
+    // Create a new "admin" schema through which the "hidden" entities can be accessed
+    await db.query(`CREATE SCHEMA "admin"`)
+    // Create admin user with "admin" schema in default "search_path"
+    await db.query(`CREATE USER "${process.env.DB_ADMIN_USER}" WITH PASSWORD '${process.env.DB_ADMIN_PASS}'`)
+    await db.query(`GRANT pg_read_all_data TO "${process.env.DB_ADMIN_USER}"`)
+    await db.query(`GRANT pg_write_all_data TO "${process.env.DB_ADMIN_USER}"`)
+    await db.query(`ALTER USER "${process.env.DB_ADMIN_USER}" SET search_path TO admin,public`)
+  }
+
+  async down(db) {
+    await db.query(`DROP SCHEMA "admin"`)
+  }
+}