Merge pull request #4873 from mnaamani/nara-update-from-master

Nara update from master
Joystream · Sep 19, 2023 · 5dfd153 · 5dfd153
2 parents 19de3f9 + 21d7177
commit 5dfd153
Show file tree

Hide file tree

Showing 28 changed files with 663 additions and 299 deletions.
diff --git a/.env b/.env
@@ -92,35 +92,71 @@ BALANCE_CREDIT=300000000000
 BALANCE_LOCKED=300000000000
 
 # ====================================== ORION ======================================
-# Orion db config
-# Overrides `DB_*` environment variables for orion services in docker-compose.yml
-# We use a single postgres service for both:
-# - Subsquid archive's database (squid-archive, squid-archive-gateway) and
-# - Orion's database (orion_processor, orion_graphql-server).
-# ORION_DB_NAME is the name of the Orion's database, while SQUID_ARCHIVE_DB_NAME
-# is the name of the Subsquid archive database.
-SQUID_ARCHIVE_DB_NAME=squid_archive
-ORION_DB_NAME=orion
-ORION_DB_USER=postgres
-ORION_DB_PASS=postgres
+# LOCAL DEV ENVIRONMENT
+
+ORION_ENV=development
+DEV_DISABLE_SAME_SITE=true
+
+# Db config
+ORION_DB_NAME=squid
+ORION_DB_PASS=squid
+DB_ADMIN_USER=admin
+DB_ADMIN_PASS=admin
 ORION_DB_PORT=5433
 
-# Orion / Subsquid archive service ports
-ORION_API_PORT=4350
+# archive configuration
+ARCHIVE_DB_PORT=12345
+ARCHIVE_GATEWAY_PORT=8888
+WS_SOURCE=ws://joystream-node:9944
+ARCHIVE_EXPLORER_PORT=4444
+
+# Processor service prometheus port
 PROCESSOR_PROMETHEUS_PORT=3337
-SQUID_ARCHIVE_GATEWAY_PORT=8000
+# Graphql server port
+GRAPHQL_API_PORT=4350
+# Auth api port
+AUTH_API_PORT=4074
 
-# Other Orion configs
+APP_NAME=Gleev
 SUPPORT_NO_CATEGORY_VIDEOS=true
 SUPPORT_NEW_CATEGORIES=true
 KILL_SWITCH_ON=false
-VIDEO_VIEW_PER_IP_TIME_LIMIT=30
-VIDEO_RELEVANCE_VIEWS_TICK=50
-RELEVANCE_WEIGHTS="[1, 0.03, 0.3, 0.5]"
+# 10 seconds
+VIDEO_VIEW_PER_USER_TIME_LIMIT=10
+# Operator API secret
 OPERATOR_SECRET=this-is-not-so-secret-change-it
+# every 50 views video relevance score will be recalculated
+VIDEO_RELEVANCE_VIEWS_TICK=50
+# [
+#   newness (negative number of days since created) weight,
+#   views weight,
+#   comments weight,
+#   rections weights,
+#   [joystream creation weight, YT creation weight]
+# ]
+RELEVANCE_WEIGHTS="[1, 0.03, 0.3, 0.5, [7,3]]"
 MAX_CACHED_ENTITIES=1000
 APP_PRIVATE_KEY=this-is-not-so-secret-change-it
-TRUSTED_REVERSE_PROXIES=1
+SESSION_EXPIRY_AFTER_INACTIVITY_MINUTES=60
+SESSION_MAX_DURATION_HOURS=720
+EMAIL_CONFIRMATION_ROUTE=http://localhost:4074/api/v1/confirm-email?token={token}
+EMAIL_CONFIRMATION_TOKEN_EXPIRY_TIME_HOURS=24
+EMAIL_CONFIRMATION_TOKEN_RATE_LIMIT=5
+ACCOUNT_OWNERSHIP_PROOF_EXPIRY_TIME_SECONDS=300 # 5 minutes
+COOKIE_SECRET=this-is-not-so-secret-change-it
+
+TRUST_PROXY=uniquelocal
+
+# Sendgrid API
+ORION_SENDGRID_API_KEY=
+ORION_SENDGRID_FROM_EMAIL=gateway@example.com
+
+# Debug settings
+SQD_DEBUG=api:*
+OPENAPI_PLAYGROUND=true
+
+ARCHIVE_GATEWAY_URL=${CUSTOM_ARCHIVE_GATEWAY_URL:-http://squid-archive-gateway:8000/graphql}
+
 # =====================================================================================
 
 ## Telemetry

diff --git a/distributor-node/src/services/networking/query-node/schema.graphql b/distributor-node/src/services/networking/query-node/schema.graphql
@@ -2024,6 +2024,8 @@ type MemberMetadata implements BaseGraphQLObject {
   about: String
   member: Membership
   externalResources: [MembershipExternalResource!]
+  isVerifiedValidator: Boolean
+  validatorAccount: String
   membercreatedeventmetadata: [MemberCreatedEvent!]
   memberinvitedeventmetadata: [MemberInvitedEvent!]
   memberprofileupdatedeventnewMetadata: [MemberProfileUpdatedEvent!]
@@ -17711,6 +17713,13 @@ input MemberMetadataWhereInput {
   about_startsWith: String
   about_endsWith: String
   about_in: [String!]
+  isVerifiedValidator_eq: Boolean
+  isVerifiedValidator_in: [Boolean!]
+  validatorAccount_eq: String
+  validatorAccount_contains: String
+  validatorAccount_startsWith: String
+  validatorAccount_endsWith: String
+  validatorAccount_in: [String!]
   member: MembershipWhereInput
   externalResources_none: MembershipExternalResourceWhereInput
   externalResources_some: MembershipExternalResourceWhereInput
@@ -17743,12 +17752,16 @@ input MemberMetadataCreateInput {
   name: String
   avatar: JSONObject!
   about: String
+  isVerifiedValidator: Boolean
+  validatorAccount: String
 }
 
 input MemberMetadataUpdateInput {
   name: String
   avatar: JSONObject
   about: String
+  isVerifiedValidator: Boolean
+  validatorAccount: String
 }
 
 input MembershipGiftedEventWhereInput {
@@ -30451,6 +30464,10 @@ enum MemberMetadataOrderByInput {
   name_DESC
   about_ASC
   about_DESC
+  isVerifiedValidator_ASC
+  isVerifiedValidator_DESC
+  validatorAccount_ASC
+  validatorAccount_DESC
 }
 
 enum MemberProfileUpdatedEventOrderByInput {

diff --git a/docker-compose-no-bind-volumes.yml b/docker-compose-no-bind-volumes.yml
@@ -164,23 +164,6 @@ services:
       POSTGRES_DB: ${INDEXER_DB_NAME}
       PGPORT: ${DB_PORT}
 
-  orion-db:
-    image: postgres:14
-    container_name: orion-db
-    restart: unless-stopped
-    ports:
-      - '127.0.0.1:${ORION_DB_PORT}:${ORION_DB_PORT}'
-    volumes:
-      - orion-db-data:/var/lib/postgresql/data
-    env_file:
-      # relative to working directory where docker-compose was run from
-      - .env
-    environment:
-      POSTGRES_USER: ${ORION_DB_USER}
-      POSTGRES_PASSWORD: ${ORION_DB_PASS}
-      POSTGRES_DB: ${ORION_DB_NAME}
-      PGPORT: ${ORION_DB_PORT}
-
   graphql-server:
     image: joystream/query-node
     build:
@@ -307,92 +290,162 @@ services:
     ports:
       - "3002:3002"
 
-  squid-archive:
-    container_name: squid-archive
-    image: subsquid/substrate-ingest:firesquid
-    depends_on:
-      - orion-db
+  orion-db:
+    image: postgres:14
+    container_name: orion-db
     restart: unless-stopped
     env_file:
       - .env
     environment:
-      - DB_NAME=${SUBSQUID_ARCHIVE_DB_NAME}
-      - DB_USER=${ORION_DB_USER}
-      - DB_PASS=${ORION_DB_PASS}
-      - DB_HOST=orion-db
-      - DB_PORT=${ORION_DB_PORT}
-    command: [
-        '-e',
-        '${JOYSTREAM_NODE_WS}',
-        '-c',
-        '20',
-        '--out',
-        'postgres://postgres:postgres@orion-db:${ORION_DB_PORT}/${SUBSQUID_ARCHIVE_DB_NAME}',
-      ]
+      POSTGRES_PASSWORD: ${ORION_DB_PASS}
+      POSTGRES_DB: ${ORION_DB_NAME}
+    ports:
+      - '127.0.0.1:${ORION_DB_PORT}:${ORION_DB_PORT}'
+    command: ['postgres', '-c', 'config_file=/etc/postgresql/postgresql.conf', '-p', '${ORION_DB_PORT}']
+    # command: ['postgres', '-p', '${ORION_DB_PORT}']
+    shm_size: 1g
+    volumes:
+      - orion-db-data:/var/lib/postgresql/data
+      - ./orion-postgres.conf:/etc/postgresql/postgresql.conf
 
-  squid-archive-gateway:
-    container_name: squid-archive-gateway
-    image: subsquid/substrate-gateway:firesquid
+  # Orion processor
+  orion-processor:
+    image: joystream/orion:latest
+    container_name: orion-processor
+    restart: unless-stopped
     depends_on:
       - orion-db
-    restart: unless-stopped
     env_file:
       - .env
     environment:
-      - DB_NAME=${SUBSQUID_ARCHIVE_DB_NAME}
-      - DB_USER=${ORION_DB_USER}
+      - DB_NAME=${ORION_DB_NAME}
       - DB_PASS=${ORION_DB_PASS}
-      - DB_HOST=orion-db
       - DB_PORT=${ORION_DB_PORT}
-      - DATABASE_MAX_CONNECTIONS=5
-      - RUST_LOG=actix_web=info,actix_server=info
-    command: [
-        '--database-url',
-        'postgres://postgres:postgres@orion-db:${ORION_DB_PORT}/${SUBSQUID_ARCHIVE_DB_NAME}',
-      ]
+      - DB_HOST=orion-db
+      - GQL_PORT=${GRAPHQL_API_PORT}
+    volumes:
+      - ./db/export:/squid/db/export
+    command: >
+      sh -c "npx squid-typeorm-migration apply && \
+             node lib/processor.js"
     ports:
-      - '127.0.0.1:${SQUID_ARCHIVE_GATEWAY_PORT}:8000'
+      - '127.0.0.1:${PROCESSOR_PROMETHEUS_PORT}:${PROCESSOR_PROMETHEUS_PORT}'
 
-  orion-processor:
-    container_name: orion-processor
+  # Orion GraphQL API
+  orion-graphql-api:
+    container_name: orion-graphql-api
     image: joystream/orion:latest
     restart: unless-stopped
-    env_file:
-      - .env
     depends_on:
       - orion-db
+    env_file:
+      - .env
     environment:
       - DB_NAME=${ORION_DB_NAME}
-      - DB_USER=${ORION_DB_USER}
       - DB_PASS=${ORION_DB_PASS}
-      - DB_HOST=orion-db
       - DB_PORT=${ORION_DB_PORT}
-      - GQL_PORT=${ORION_API_PORT}
-      - ARCHIVE_GATEWAY_URL=http://squid-archive-gateway:8000/graphql
-    command: >
-      sh -c "npx squid-typeorm-migration apply && \
-             node lib/processor.js"
+      - DB_HOST=orion-db
+      - GQL_PORT=${GRAPHQL_API_PORT}
+    command: ['npm', 'run', 'query-node-start']
     ports:
-      - '127.0.0.1:${PROCESSOR_PROMETHEUS_PORT}:${PROCESSOR_PROMETHEUS_PORT}'
+      - '127.0.0.1:${GRAPHQL_API_PORT}:${GRAPHQL_API_PORT}'
 
-  orion-api:
-    container_name: orion-api
+  # Orion Authentication API
+  orion-auth-api:
+    container_name: orion-auth-api
     image: joystream/orion:latest
     restart: unless-stopped
-    env_file:
-      - .env
     depends_on:
       - orion-db
+    env_file:
+      - .env
     environment:
       - DB_NAME=${ORION_DB_NAME}
-      - DB_USER=${ORION_DB_USER}
       - DB_PASS=${ORION_DB_PASS}
-      - DB_HOST=orion-db
       - DB_PORT=${ORION_DB_PORT}
-      - GQL_PORT=${ORION_API_PORT}
-    command: ["npm", "run", "query-node-start"]
+      - DB_HOST=orion-db
+      - SENDGRID_API_KEY=${ORION_SENDGRID_API_KEY}
+      - SENDGRID_FROM_EMAIL=${ORION_SENDGRID_FROM_EMAIL}
+    command: ['npm', 'run', 'auth-server-start']
+    ports:
+      - '127.0.0.1:${AUTH_API_PORT}:${AUTH_API_PORT}'
+
+  # Orion archive node
+  squid-archive-db:
+    container_name: orion_archive_db
+    hostname: orion_archive_db
+    image: postgres:14 # CockroachDB cluster might be a better fit for production deployment
+    restart: unless-stopped
+    volumes:
+      - orion_archive_db_data:/var/lib/postgresql/data
+    env_file:
+      - .env
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: squid-archive
+    ports:
+      - '127.0.0.1:${ARCHIVE_DB_PORT}:${ARCHIVE_DB_PORT}'
+      - '[::1]:${ARCHIVE_DB_PORT}:${ARCHIVE_DB_PORT}'
+    command: ['postgres', '-c', 'log_statement=all', '-p', '${ARCHIVE_DB_PORT}']
+
+  squid-archive-ingest:
+    container_name: orion_archive_ingest
+    depends_on:
+      - squid-archive-db
+    restart: unless-stopped
+    env_file:
+      - .env
+    image: subsquid/substrate-ingest:firesquid
+    command: [
+        '-e',
+        '${WS_SOURCE}',
+        '-c',
+        '20', # allow up to 20 pending requests for the above endpoint (default is 5)
+        #  "--start-block", "1000000", # uncomment to specify a non-zero start block
+        '--out',
+        'postgres://postgres:postgres@orion_archive_db:${ARCHIVE_DB_PORT}/squid-archive',
+      ]
+
+  squid-archive-gateway:
+    container_name: orion_archive_gateway
+    hostname: orion_archive_gateway
+    depends_on:
+      - squid-archive-db
+    restart: unless-stopped
+    image: subsquid/substrate-gateway:firesquid
+    env_file:
+      - .env
+    environment:
+      DATABASE_MAX_CONNECTIONS: 5
+      RUST_LOG: 'actix_web=info,actix_server=info'
+    command: [
+        '--database-url',
+        'postgres://postgres:postgres@orion_archive_db:${ARCHIVE_DB_PORT}/squid-archive',
+      ]
     ports:
-      - '${ORION_API_PORT}:${ORION_API_PORT}'
+      - '127.0.0.1:${ARCHIVE_GATEWAY_PORT}:8000'
+      - '[::1]:${ARCHIVE_GATEWAY_PORT}:8000'
+
+  # Explorer service is optional.
+  # It provides rich GraphQL API for querying archived data.
+  # Many developers find it very useful for exploration and debugging.
+  squid-archive-explorer:
+    container_name: orion_archive_explorer
+    hostname: orion_archive_explorer
+    restart: unless-stopped
+    image: subsquid/substrate-explorer:firesquid
+    env_file:
+      - .env
+    environment:
+      DB_TYPE: postgres # set to `cockroach` for Cockroach DB
+      DB_HOST: orion_archive_db
+      DB_NAME: 'squid-archive'
+      DB_USER: 'postgres'
+      DB_PASS: 'postgres'
+      DB_PORT: ${ARCHIVE_DB_PORT}
+    ports:
+      - '${ARCHIVE_EXPLORER_PORT}:3000'
 
 volumes:
   chain-data:
@@ -425,3 +478,5 @@ volumes:
     driver: local
   orion-db-data:
     driver: local
+  orion_archive_db_data:
+    driver: local