Merge pull request #1176 from IntersectMBO/develop

Test Optimization, PDF Integration, Bug Fixes.
IntersectMBO · Jun 7, 2024 · 9b2593c · 9b2593c
2 parents aaced8b + dde0653
commit 9b2593c
Show file tree

Hide file tree

Showing 140 changed files with 39,075 additions and 18,478 deletions.
diff --git a/.github/workflows/build-and-deploy-beta.yml b/.github/workflows/build-and-deploy-beta.yml
@@ -27,6 +27,7 @@ jobs:
       GRAFANA_SLACK_OAUTH_TOKEN: ${{ secrets.GRAFANA_SLACK_OAUTH_TOKEN }}
       GRAFANA_SLACK_RECIPIENT: ${{ secrets.GRAFANA_SLACK_RECIPIENT }}
       GTM_ID: ${{ secrets.GTM_ID }}
+      NPMRC_TOKEN: ${{ secrets.NPMRC_TOKEN }}
       IP_ADDRESS_BYPASSING_BASIC_AUTH1: ${{ secrets.IP_ADDRESS_BYPASSING_BASIC_AUTH1 }}
       IP_ADDRESS_BYPASSING_BASIC_AUTH2: ${{ secrets.IP_ADDRESS_BYPASSING_BASIC_AUTH2 }}
       NEXT_PUBLIC_API_URL: "https://participation.sanchogov.tools"

diff --git a/.github/workflows/build-and-deploy-dev.yml b/.github/workflows/build-and-deploy-dev.yml
@@ -3,6 +3,15 @@ run-name: Deploy by @${{ github.actor }}
 
 on:
   workflow_dispatch:
+    inputs:
+      isProposalDiscussionForumEnabled:
+        description: "Enable proposal discussion forum"
+        required: true
+        type: choice
+        default: "disabled"
+        options:
+          - "enabled"
+          - "disabled"
 
 env:
   ENVIRONMENT: "dev"
@@ -26,6 +35,7 @@ jobs:
       GRAFANA_SLACK_OAUTH_TOKEN: ${{ secrets.GRAFANA_SLACK_OAUTH_TOKEN }}
       GRAFANA_SLACK_RECIPIENT: ${{ secrets.GRAFANA_SLACK_RECIPIENT }}
       GTM_ID: ${{ secrets.GTM_ID }}
+      NPMRC_TOKEN: ${{ secrets.NPMRC_TOKEN }}
       IP_ADDRESS_BYPASSING_BASIC_AUTH1: ${{ secrets.IP_ADDRESS_BYPASSING_BASIC_AUTH1 }}
       IP_ADDRESS_BYPASSING_BASIC_AUTH2: ${{ secrets.IP_ADDRESS_BYPASSING_BASIC_AUTH2 }}
       NEXT_PUBLIC_API_URL: "https://participation.sanchogov.tools"
@@ -37,6 +47,7 @@ jobs:
       SENTRY_IGNORE_API_RESOLUTION_ERROR: "1"
       TRAEFIK_LE_EMAIL: "admin+govtool@binarapps.com"
       USERSNAP_SPACE_API_KEY: ${{ secrets.USERSNAP_SPACE_API_KEY }}
+      IS_PROPOSAL_DISCUSSION_FORUM_ENABLED: ${{ inputs.isProposalDiscussionForumEnabled == 'enabled' }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v4

diff --git a/.github/workflows/build-and-deploy-staging.yml b/.github/workflows/build-and-deploy-staging.yml
@@ -28,6 +28,7 @@ jobs:
       GRAFANA_SLACK_OAUTH_TOKEN: ${{ secrets.GRAFANA_SLACK_OAUTH_TOKEN }}
       GRAFANA_SLACK_RECIPIENT: ${{ secrets.GRAFANA_SLACK_RECIPIENT }}
       GTM_ID: ${{ secrets.GTM_ID }}
+      NPMRC_TOKEN: ${{ secrets.NPMRC_TOKEN }}
       IP_ADDRESS_BYPASSING_BASIC_AUTH1: ${{ secrets.IP_ADDRESS_BYPASSING_BASIC_AUTH1 }}
       IP_ADDRESS_BYPASSING_BASIC_AUTH2: ${{ secrets.IP_ADDRESS_BYPASSING_BASIC_AUTH2 }}
       NEXT_PUBLIC_API_URL: "https://participation.sanchogov.tools"

diff --git a/.github/workflows/build-and-deploy-test-stack.yml b/.github/workflows/build-and-deploy-test-stack.yml
@@ -20,6 +20,7 @@ jobs:
       GRAFANA_SLACK_OAUTH_TOKEN: ${{ secrets.GRAFANA_SLACK_OAUTH_TOKEN }}
       SENTRY_DSN_BACKEND: ${{ secrets.SENTRY_DSN_BACKEND }}
       GTM_ID: ${{ secrets.GTM_ID }}
+      NPMRC_TOKEN: ${{ secrets.NPMRC_TOKEN }}
       SENTRY_DSN: ${{ secrets.SENTRY_DSN_FRONTEND }}
       PIPELINE_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
       USERSNAP_SPACE_API_KEY: ${{ secrets.USERSNAP_SPACE_API_KEY }}
@@ -46,4 +47,4 @@ jobs:
           options: |
             --verbose
         env:
-          GOVTOOL_TAG: ${{ github.sha }}
+          GOVTOOL_TAG: ${{ github.sha }}
diff --git a/.github/workflows/build-and-deploy-test.yml b/.github/workflows/build-and-deploy-test.yml
@@ -28,6 +28,7 @@ jobs:
       GRAFANA_SLACK_OAUTH_TOKEN: ${{ secrets.GRAFANA_SLACK_OAUTH_TOKEN }}
       GRAFANA_SLACK_RECIPIENT: ${{ secrets.GRAFANA_SLACK_RECIPIENT }}
       GTM_ID: ${{ secrets.GTM_ID }}
+      NPMRC_TOKEN: ${{ secrets.NPMRC_TOKEN }}
       IP_ADDRESS_BYPASSING_BASIC_AUTH1: ${{ secrets.IP_ADDRESS_BYPASSING_BASIC_AUTH1 }}
       IP_ADDRESS_BYPASSING_BASIC_AUTH2: ${{ secrets.IP_ADDRESS_BYPASSING_BASIC_AUTH2 }}
       NEXT_PUBLIC_API_URL: "https://participation.sanchogov.tools"

diff --git a/.github/workflows/code_check_frontend.yml b/.github/workflows/code_check_frontend.yml
@@ -4,7 +4,7 @@ on:
   push:
     paths:
       - govtool/frontend/**
-      - .github/workflows/test_frontend.yml
+      - .github/workflows/code_check_frontend.yml
 
 defaults:
   run:
@@ -26,13 +26,17 @@ jobs:
       - name: Set up Node.js
         uses: actions/setup-node@v4
         with:
-          node-version-file: "govtool/frontend/.nvmrc"
+          registry-url: "https://registry.npmjs.org/"
+          node-version-file: "./govtool/frontend/.nvmrc"
+          scope: "@intersect.mbo"
+      - run: npm ci
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPMRC_TOKEN }}
 
       - name: 🧪 Test
         env:
-          NODE_OPTIONS: "--max_old_space_size=4096"
+          NODE_OPTIONS: "--max_old_space_size=8192"
         run: |
-          npm install
           npm run test
 
   lint:
@@ -50,11 +54,15 @@ jobs:
       - name: Set up Node.js
         uses: actions/setup-node@v4
         with:
-          node-version-file: "govtool/frontend/.nvmrc"
+          registry-url: "https://registry.npmjs.org/"
+          node-version-file: "./govtool/frontend/.nvmrc"
+          scope: "@intersect.mbo"
+      - run: npm ci
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPMRC_TOKEN }}
 
       - name: 👕 Lint
         run: |
-          npm install
           npm run lint
 
   type_check:
@@ -72,9 +80,13 @@ jobs:
       - name: Set up Node.js
         uses: actions/setup-node@v4
         with:
-          node-version-file: "govtool/frontend/.nvmrc"
+          registry-url: "https://registry.npmjs.org/"
+          node-version-file: "./govtool/frontend/.nvmrc"
+          scope: "@intersect.mbo"
+      - run: npm ci
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPMRC_TOKEN }}
 
       - name: 🔍 Type Check
         run: |
-          npm install
           npm run tsc
diff --git a/.github/workflows/deploy-dev-from-tag.yml b/.github/workflows/deploy-dev-from-tag.yml
@@ -28,6 +28,7 @@ jobs:
       SENTRY_DSN_BACKEND: ${{ secrets.SENTRY_DSN_BACKEND }}
       TRAEFIK_LE_EMAIL: "admin+govtool@binarapps.com"
       GTM_ID: ${{ secrets.GTM_ID }}
+      NPMRC_TOKEN: ${{ secrets.NPMRC_TOKEN }}
       SENTRY_DSN: ${{ secrets.SENTRY_DSN_FRONTEND }}
       PIPELINE_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
       IP_ADDRESS_BYPASSING_BASIC_AUTH1: ${{ secrets.IP_ADDRESS_BYPASSING_BASIC_AUTH1 }}

diff --git a/.github/workflows/resync-cardano-node-and-db-sync.yml b/.github/workflows/resync-cardano-node-and-db-sync.yml
@@ -36,6 +36,7 @@ jobs:
       SENTRY_DSN_BACKEND: ${{ secrets.SENTRY_DSN_BACKEND }}
       TRAEFIK_LE_EMAIL: "admin+govtool@binarapps.com"
       GTM_ID: ${{ secrets.GTM_ID }}
+      NPMRC_TOKEN: ${{ secrets.NPMRC_TOKEN }}
       SENTRY_DSN: ${{ secrets.SENTRY_DSN_FRONTEND }}
       PIPELINE_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
       IP_ADDRESS_BYPASSING_BASIC_AUTH1: ${{ secrets.IP_ADDRESS_BYPASSING_BASIC_AUTH1 }}

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -24,6 +24,8 @@ changes.
 
 ### Added
 
+- added `bio` `dRepName` `email` `references` `metadataValid` and `metadataStatus` fields to `drep/list`
+- added `metadatavalidationmaxconcurrentrequests` field to the backend config
 - added `metadata/validate` endpoint [Issue 876](https://github.com/IntersectMBO/govtool/issues/876)
 - added pagination to `drep/list` [Issue 756](https://github.com/IntersectMBO/govtool/issues/756)
 - added search query param to the `drep/getVotes` [Issue 640](https://github.com/IntersectMBO/govtool/issues/640)
@@ -60,6 +62,7 @@ changes.
 - Add type check & lint to github actions [Issue 512](https://github.com/IntersectMBO/govtool/issues/512)
 - Add eslint & prettier to frontend package [Issue 166](https://github.com/IntersectMBO/govtool/issues/166)
 - Add DRep list pagination [Issue 740](https://github.com/IntersectMBO/govtool/issues/740)
+- Add PDF pillar [Issue 1090](https://github.com/IntersectMBO/govtool/issues/1090)
 
 ### Fixed
 
@@ -92,6 +95,7 @@ changes.
 
 ### Changed
 
+- `proposal.about` changed to `proposal.abstract`
 - `drep/info` now returns 4 different tx hashes instead of one latest tx hash [Issue 688](https://github.com/IntersectMBO/govtool/issues/688)
 - `proposal/list` allows user to search by tx hash [Issue 603](https://github.com/IntersectMBO/govtool/issues/603)
 - `proposal/list` returns additional data such ass `expiryEpochNo`, `createdEpochNo`, `title`, `about`, `motivation`,

diff --git a/govtool/backend/app/Main.hs b/govtool/backend/app/Main.hs
@@ -8,6 +8,7 @@
 
 module Main where
 
+import           Control.Concurrent.QSem                (newQSem)
 import           Control.Exception                      (Exception,
                                                          SomeException,
                                                          fromException, throw)
@@ -115,7 +116,8 @@ startApp vvaConfig = do
     dRepVotingPowerCache <- newCache
     dRepListCache <- newCache
     networkMetricsCache <- newCache
-    metadataValidationCache <- newCache
+    proposalMetadataValidationCache <- newCache
+    dRepMetadataValidationCache <- newCache
     return $ CacheEnv
       { proposalListCache
       , getProposalCache
@@ -127,12 +129,13 @@ startApp vvaConfig = do
       , dRepVotingPowerCache
       , dRepListCache
       , networkMetricsCache
-      , metadataValidationCache
+      , proposalMetadataValidationCache
+      , dRepMetadataValidationCache
       }
   connectionPool <- createPool (connectPostgreSQL (encodeUtf8 (dbSyncConnectionString $ getter vvaConfig))) close 1 1 60
   vvaTlsManager <- newManager tlsManagerSettings
-
-  let appEnv = AppEnv {vvaConfig=vvaConfig, vvaCache=cacheEnv, vvaConnectionPool=connectionPool, vvaTlsManager}
+  qsem <- newQSem (metadataValidationMaxConcurrentRequests vvaConfig)
+  let appEnv = AppEnv {vvaConfig=vvaConfig, vvaCache=cacheEnv, vvaConnectionPool=connectionPool, vvaTlsManager, vvaMetadataQSem=qsem}
   server' <- mkVVAServer appEnv
   runSettings settings server'
 

diff --git a/govtool/backend/example-config.json b/govtool/backend/example-config.json
@@ -11,5 +11,6 @@
     "cachedurationseconds": 20,
     "sentrydsn": "https://username:password@senty.host/id",
     "metadatavalidationhost": "localhost",
-    "metadatavalidationport": 3001
+    "metadatavalidationport": 3001,
+    "metadatavalidationmaxconcurrentrequests": 10
 }