Merge remote-tracking branch 'upstream/master' into nara

.env

@@ -39,9 +39,10 @@ VIDEO_RELEVANCE_VIEWS_TICK=50
 #   views weight,
 #   comments weight,
 #   rections weights,
-#   [joystream creation weight, YT creation weight]
+#   [joystream creation weight, YT creation weight],
+#   Default channel weight/bias
 # ]
-RELEVANCE_WEIGHTS="[1, 0.03, 0.3, 0.5, [7,3]]"
+RELEVANCE_WEIGHTS="[1, 0.03, 0.3, 0.5, [7,3], 1]"
 MAX_CACHED_ENTITIES=1000
 APP_PRIVATE_KEY=this-is-not-so-secret-change-it
 SESSION_EXPIRY_AFTER_INACTIVITY_MINUTES=60
@@ -52,6 +53,9 @@ EMAIL_CONFIRMATION_TOKEN_RATE_LIMIT=5
 ACCOUNT_OWNERSHIP_PROOF_EXPIRY_TIME_SECONDS=300 # 5 minutes
 COOKIE_SECRET=this-is-not-so-secret-change-it
 
+# Notification related variables
+APP_ROOT_DOMAIN=gleev.xyz
+
 TRUST_PROXY=uniquelocal
 
 # Sendgrid API
@@ -63,3 +67,19 @@ SQD_DEBUG=api:*
 
 # playground
 OPENAPI_PLAYGROUND=true
+
+# max number of attempts to deliver email notification
+EMAIL_NOTIFICATION_DELIVERY_MAX_ATTEMPTS=5
+
+# Notification emails storage links
+APP_ASSET_STORAGE=https://raw.githubusercontent.com/Joystream/atlas-notification-assets/main/logos/gleev
+APP_NAME_ALT=Gleev.xyz
+NOTIFICATION_ASSET_ROOT=https://raw.githubusercontent.com/Joystream/atlas-notification-assets/main/icons
+
+# =====================================================================================
+# Telemetry
+# =====================================================================================
+
+# yes/no
+TELEMETRY_ENABLED=no
+TELEMETRY_ENDPOINT=http://apm-server:8200

.eslintrc.js

@@ -16,6 +16,7 @@ module.exports = {
     'standard',
     'eslint:recommended',
     'plugin:@typescript-eslint/recommended',
+    'plugin:chai-friendly/recommended',
     'plugin:prettier/recommended',
     'prettier',
   ],

.github/workflows/docker.yml

@@ -1,22 +1,58 @@
-name: Build and publish to Docker Hub
+name: Build & Publish Orion Docker image
 
 on:
-  push:
-    branches:
-      - 'master'
+  workflow_dispatch:
 
 jobs:
-  docker:
+  build_and_publish:
+    name: Build and Publish
     runs-on: ubuntu-latest
     steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '18.x'
+
+      - name: Extract Package Version
+        id: extract_version
+        shell: bash
+        run: |
+          echo "orion_version=$(cat package.json | jq -r '.version')" >> $GITHUB_OUTPUT
+
+      - name: Make some space
+        shell: bash
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf "/usr/local/share/boost"
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        if: github.event_name == 'workflow_dispatch'
+        uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Build and push
-        id: docker_build
-        uses: docker/build-push-action@v2
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      - name: Build orion
+        uses: docker/build-push-action@v3
         with:
-          push: true
+          context: .
+          file: Dockerfile
+          push: false
+          load: true
           tags: joystream/orion:latest
+
+      - name: Push Orion Image
+        run: |
+          docker image tag joystream/orion:latest joystream/orion:${{ steps.extract_version.outputs.orion_version }}
+          docker push joystream/orion:${{ steps.extract_version.outputs.orion_version }}
+
+      - name: Push Orion latest tag
+        if: github.ref == 'refs/heads/master'
+        run: |
+          docker push joystream/orion:latest

.github/workflows/tests.yml

@@ -2,8 +2,31 @@ name: Tests
 on: [push, pull_request]
 
 jobs:
+  migrations:
+    name: Run migrations
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Use Node.js 16.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 16.x
+      - name: Create joystream_default network
+        run: docker network create joystream_default
+      - name: Prepare workspace
+        run: make prepare
+      - name: Start db
+        run: docker-compose up -d orion_db
+      - name: Wait for db
+        run: |
+          while ! docker-compose exec -T orion_db pg_isready -U postgres -p 23798; do
+            sleep 1
+          done
+      - name: Run migrations
+        run: make migrate
   auth-api:
     name: Auth API tests
+    needs: [migrations]
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -20,3 +43,43 @@ jobs:
         run: make prepare
       - name: Run tests
         run: npm run tests:auth-api
+  notifications:
+    name: orion_db<->service tests for Q&A failing notifications
+    needs: [migrations]
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        node-version: [16.x]
+      fail-fast: true
+    steps:
+      - uses: actions/checkout@v2
+      - name: Use Node.js ${{matrix.node-version}}
+        uses: actions/setup-node@v1
+        with:
+          node-version: ${{matrix.node-version}}
+      - name: Prepare workspace
+        run: make prepare
+      - name: Run tests
+        run: npm run tests:notifications
+      - name: Run utils tests
+        run: npm run tests:notifications-utils
+  mail-scheduler:
+    name: Mail scheduler tests
+    needs: [migrations]
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        node-version: [16.x]
+      fail-fast: true
+    steps:
+      - uses: actions/checkout@v2
+      - name: Use Node.js ${{matrix.node-version}}
+        uses: actions/setup-node@v1
+        with:
+          node-version: ${{matrix.node-version}}
+      - name: Prepare workspace
+        run: make prepare
+      - name: Run tests
+        run: npm run tests:mail-scheduler

.gitignore

@@ -5,17 +5,23 @@
 /**Versions.jsonl
 
 # IDE files
+/.vscode
 /.idea
 src/model/generated
-src/model/generated/index.ts
-src/model/generated/.DS_STORE
+src/types/
+src/auth-server/emails/templates/*.mst
+src/auth-server/emails/templates/preview
 /schema.graphql
 /db/persisted
 /scripts/orion-v1-migration/data
-/db/export.json*
+/db/export
+/db/migrations_dir*
 .DS_STORE
 
 # network tests
 network-tests/graphql/generated
 network-tests/node_modules
 network-tests/lib
+
+# LOGS
+chron_mail_scheduler.log

CHANGELOG.md

@@ -1,7 +1,124 @@
+# 3.4.0
+
+## Schema changes
+- Added `@schema(name: "admin")` directive to hide entities (from public GRAPHQL API) in Graphql schema definitions.
+
+## Misc
+- Patch `@subsquid/typeorm-config` & `@subsquid/typeorm-migration` packages to change `squid-typeorm-migration apply` command to apply a single migrations file too using `--filename` option instead of applying the whole `db/migrations` directory.
+- Patch `@subsquid/openreader` and `@subsquid/typeorm-codegen` dependencies to include the db schema `name` too in the generated typeorm/postgres migrations, and an optional `schema` directive to specify the schema of any GRAPHQL entity.
+
+## DB Migrations
+- Update `generate-migrations` makefile command. Now the existing `*-Data.js` will not be overwritten, instead a new `*-Data.js` migration file (containing only changes compared to the previous DB state) will be added whenever there are GRAPHQL schema changes. The `*-Views.js` migration file will also be updated whenever the GRAPHQL schema changes.
+- Create `generateViewsMigration.js` script to create new `*-Views.js` migration file.
+- Separate the view definitions(in `db/viewDefinitions.js`) from views migration file(`*-Views.js`).
+- Add `*-Admin.js` migration file to create an `admin` schema & user, previously the `admin` schema and user was being created in the `*-Views.js` migration.
+- Regenerate the postgres db migrations.
+
+## Documentation
+- Updated documentation for [upgrading-orion.md](docs/operator-guide/tutorials/upgrading-orion.md)
+- Updated documentation for [entity-visibility.md#managing-entity-visibility](docs/developer-guide/tutorials/entity-visibility.md)
+
+### Bug Fixes:
+- read/write `export.json` file, containing the offchain state, using `big-json` package, instead using Javascript native `JSON.stringify` function which does not work on large JSON objects
+
+# 3.3.0
+
+## Schema
+- `orionLanguage` property has been added.
+
+## Mappings 
+- Language detection is used to populate new property on video update and creation.
+
+# 3.2.1
+
+## Misc
+- Added index on timestamp fields on `Video`, `Event` and `OwnedNft` entities
+- update `docker.yml` github workflow to build and publish docker images.
+- added `generate-migrations` makefile command to generate migrations
+- adds OpenTelemetry tracing integration with `graphql-server` and `auth-server`
+- update nodejs version to `node:18`
+
+### Bug Fixes:
+- Extend session based on activity after initial expiry is over.
+- Fix/notification email title.
+- Improves off-chain import/export script
+
+# 3.2.0
+This release adds notifications to the orion infrastructure...
+## Schema changes
+- Introduced `NotificationEmailDelivery` entity to handle email notifications. It includes fields for tracking the notification, delivery attempts, and a discard flag.
+- Added `EmailDeliveryAttempt` entity to track each delivery attempt. It includes a status and timestamp.
+- Created `DeliveryStatus` union type that can be either `EmailSuccess` or `EmailFailure` (which includes an error status).
+- Introduced `Read` and `Unread` types to track if a notification has been read. Both are part of the `ReadOrUnread` union.
+- Started defining a `Notification` entity. It includes fields for the account, notification type, event, status, in-app visibility, creation timestamp, and recipient.
+- Defined RecipientType as a union of `MemberRecipient` and `ChannelRecipient`.
+- Introduced various notification types as part of the NotificationType union. Each type has specific fields relevant to the notification.
+- Added `ChannelSuspended, NftFeaturedOnMarketPlace, ChannelVerified, ChannelExcluded, VideoExcluded, NewChannelFollower, CommentPostedToVideo, VideoLiked, VideoDisliked, NftPurchased, CreatorReceivesAuctionBid, NftOffered, DirectChannelPaymentByMember, EnglishAuctionSettled, ChannelFundsWithdrawn, ChannelCreated, CommentReply, ReactionToComment, VideoPosted, NewAuction, NewNftOnSale, HigherBidPlaced, AuctionWon, AuctionLost, BidMadeCompletingAuction, NftRoyaltyPaid` types. Each type includes fields relevant to the specific notification.
+- Introduced `AccountNotificationPreferences` type to handle user notification preferences.
+- Added fields for each notification type, each of which is of `NotificationPreference` type.
+- Channel notifications include: `channelExcludedFromApp, videoExcludedFromApp, nftFeaturedOnMarketPlace, newChannelFollower, videoCommentCreated, videoLiked, videoDisliked, yppChannelVerified, yppSignupSuccessful, yppChannelSuspended, nftBought, creatorTimedAuctionExpired, bidMadeOnNft, royaltyReceived, channelPaymentReceived, channelReceivedFundsFromWg, newPayoutUpdatedByCouncil, channelFundsWithdrawn`.
+Member notifications include: `channelCreated, replyToComment, reactionToComment, videoPosted, newNftOnAuction, newNftOnSale, timedAuctionExpired, higherBidThanYoursMade, auctionWon, auctionLost, openAuctionBidCanBeWithdrawn, fundsFromCouncilReceived, fundsToExternalWalletSent, fundsFromWgReceived`.
+- Introduced `ChannelYppStatus` as a union type in the GraphQL schema. This type represents the YouTube Partner Program (YPP) status of a channel.
+The ChannelYppStatus can be one of three types: `YppUnverified, YppVerified, YppSuspended`.
+## Resolvers 
+- Ypp status mutation resolvers have been introduced, intended for the Ypp verification ops team:
+    - Added `verifyChannel` mutation is protected by the `OperatorOnly` middleware, meaning it can only be accessed by operators returning a list of `VerifyChannelResults` (each contaning the `id` of the new suspension, the `channelId` of the suspended channel, and the `createdAt` timestamp of when the suspension was created)
+    - Added `excludeChannel` mutation (protected by the `OperatorOnly` middleware) to the GraphQL schema. This mutation is used to exclude a channel. It takes channelId and rationale as arguments and returns an `ExcludeChannelResult` (contaning the `id` of the new suspension, the `channelId` of the suspended channel, and the `createdAt` timestamp of when the suspension was created)
+    - Added `suspendChannels` mutation (protected by the `OperatorOnly` middleware) to the GraphQL schema. This mutation is used to suspend. It takes `channelIds` as an argument and returns a list of `SuspendChannelResult` (each contaning the `id` of the new suspension, the `channelId` of the suspended channel, and the `createdAt` timestamp of when the suspension was created)
+- Notification-related resolvers (accessible only through the `AccountOnly` middleware) are intened for the use with the front end app:
+    - Added `markNotificationsAsRead` mutation. This mutation marks specified notifications as read. It takes an array of `notificationIds` and returns an object with an array of IDs of notifications that were successfully marked as read.
+    - Added `setAccountNotificationPreferences` mutation. This mutation updates the notification preferences for the current account. It takes an object of `newPreferences` and returns the updated notification preferences.
+    - Both mutations are protected by the `AccountOnly` middleware, meaning they can only be accessed by authenticated accounts.
+    - Added a helper function `maybeUpdateNotificationPreference` to update individual notification preferences if a new value is provided. This function is used in the `setAccountNotificationPreferences` mutation to update each preference.
+- Notification email attepmts and assets
+    - Added `setMaxAttemptsOnMailDelivery` mutation. This mutation sets the maximum number of attempts to deliver an email notification. It takes `newMaxAttempts` as an argument and returns the new maximum attempts. The mutation is protected by the `OperatorOnly` middleware.
+    - Introduced `setNewNotificationCenterPath` mutation. This mutation sets the new notification center path. It takes `newMaxAttempts` as an argument and returns the new maximum attempts. The mutation is protected by the `OperatorOnly` middleware.
+    - Added `setNewAppRootDomain` mutation. This mutation sets the new application root domain. It takes `newRootDomain` as an argument and returns an object indicating whether the new root domain was applied. The mutation is protected by the `OperatorOnly` middleware.
+## Mail Scheduler module
+- mail template generation using mjml (see `./src/auth-server/emails/templates/mjml/notification.html.mst.mjml`)
+- Introduced functions to handle email notifications.
+- Added `getMaxAttempts` function to fetch the maximum number of email delivery attempts from the configuration.
+- Added `mailsToDeliver` function to fetch all email notifications that have not been discarded.
+- Added `deliverEmails` function to process each email notification. It creates the email content, executes the delivery, and records the attempt. If the delivery is successful or the maximum number of attempts has been reached, the notification is discarded.
+- The `deliverEmails` function is called in the `main` function, which logs the result of the email delivery process.
+- The email delivery is meant (for the moment) to be scheduled via chron job (as explained on operator documentation)
+## Misc
+- several unit test have been introduced along with CI checks
+- Improved Home page scoring schedule logic
+- Refactored migration logic in order to migrate global account counter, which will be used also to migrate Notifications counters in future releases
+## Environment
+- Added `EMAIL_NOTIFICATION_DELIVERY_MAX_ATTEMPTS` environment variable to configure the maximum number of attempts to deliver an email notification, before the scheduler stops attending them
+- Introduced `APP_ASSET_STORAGE` environment variable to specify the URL where the application's assets are stored.
+- Added `APP_NAME_ALT` environment variable to set an alternative name for the application.
+- Introduced `NOTIFICATION_ASSET_ROOT` environment variable to specify the URL where the notification icons are stored.
+## Documentation
+- added documentation for setting up the email scheduler having a [Sendgrid](https://sendgrid.com) account
+
 # 3.1.0
-### Additions
-- Implements mappings for `Content.VideoAssetsDeletedByModerator` and `Content.ChannelAssetsDeletedByModerator` runtime events
 
+### Entities
+- Adds `User.permission` to the `User` entity, this however doesn't require migration logic.
+- Adds `Channel.channelWeights` in order to boost channel relevance. This value can be set via the `setChannelWeights` mutation
+### Resolvers
+- Adds supports for new permissions model for gateway operator users. Now the root user can assign/revoke operator permission/s to users using `grantPermissions` & `revokePermissions` mutations
+- Adds new `setChannelWeights` operator mutation to set weight/bias for any channel/s which will be used to calculate the Atlas homepage video relevance scores
+### Performance
+- Adds `Video.createdAt` as index in order to speed up Atlas home page queries 
+
+# 3.0.4
+
+### Misc
+- sum channels rewards into a new `Channel.cumulativeReward` field
+- start `Channel.cumulativeRewardClaimed` at zero instead of null
+
+# 3.0.3
+### Optimisation:
+- Solves n+1 query issue on the `AssetResolver` `resolvedUrls` field resolver, by reusing `storageBagId` field if available. Resulting in faster query times and inpacting home page loading speed for front end application significantly. This fix involves only the `orion_graphql-server` microservice
+### Misc
+- adds `http://localhost:3000` and `http://127.0.0.1:3000` as authorized CORS origins for local front end testing 
+# 3.0.2
+### Bug Fixes:
+- Store membership handles both as utf-8 string and raw bytes - [#4950](https://github.com/Joystream/joystream/pull/4950)
 # 3.0.1
 ### Misc
 - add migration for the `Account` id field

Dockerfile

@@ -1,4 +1,4 @@
-FROM node:16-alpine AS node
+FROM node:18-alpine AS node
 
 FROM node AS node-with-gyp
 RUN apk add g++ make python3
@@ -10,10 +10,13 @@ ADD package-lock.json .
 ADD assets assets
 RUN npm ci
 ADD tsconfig.json .
+ADD typegen.json .
+ADD joystream.jsonl .
 ADD src src
 ADD schema schema
 ADD scripts scripts
 RUN npx squid-typeorm-codegen
+RUN npx squid-substrate-typegen typegen.json
 RUN npm run build
 
 FROM node-with-gyp AS deps
@@ -45,4 +48,4 @@ CMD ["npm", "run", "processor-start"]
 
 
 FROM squid AS query-node
-CMD ["npm", "run", "query-node-start"]
+CMD ["npm", "run", "graphql-server-start"]