Merge pull request #2038 from w3f/will-worker

Add Worker
w3f · Oct 17, 2022 · 9ecc2de · ironoa · Oct 17, 2022 · 9ecc2de
2 parents 3087e85 + c3aa51a
commit 9ecc2de
Show file tree

Hide file tree

Showing 32 changed files with 505 additions and 7,109 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -62,7 +62,18 @@ jobs:
       - setup_remote_docker
       - run:
           command: |
-            /scripts/build-image.sh web3f/otv-gateway . --build-arg PACKAGE=telemetry
+            /scripts/build-image.sh web3f/otv-telemetry . --build-arg PACKAGE=telemetry
+
+  buildWorkerImage:
+    docker:
+      - image: web3f/ci-commons:v3.1.6
+    resource_class: xlarge
+    steps:
+      - checkout
+      - setup_remote_docker
+      - run:
+          command: |
+            /scripts/build-image.sh web3f/otv-worker . --build-arg PACKAGE=worker
 
   integrationTests:
     docker:
@@ -105,6 +116,16 @@ jobs:
           command: |
             /scripts/publish-image.sh web3f/otv-telemetry
 
+  publishWorkerImage:
+    docker:
+      - image: web3f/ci-commons:v3.1.6
+    steps:
+      - checkout
+      - setup_remote_docker
+      - run:
+          command: |
+            /scripts/publish-image.sh web3f/otv-worker
+
   publishChart:
     docker:
       - image: web3f/ci-commons:v3.1.6
@@ -160,6 +181,11 @@ workflows:
 #          context: dockerhub-bot
 #          filters:
 #            tags:
+#              only: /.*/
+#      - buildWorkerImage:
+#          context: dockerhub-bot
+#          filters:
+#            tags:
 #              only: /.*/
       - integrationTests:
           filters:
@@ -195,6 +221,15 @@ workflows:
 #            tags:
 #              only: /v[0-9]+(\.[0-9]+)*/
 #          requires:
+#            - integrationTests
+#      - publishWorkerImage:
+#          context: dockerhub-bot
+#          filters:
+#            branches:
+#              ignore: /.*/
+#            tags:
+#              only: /v[0-9]+(\.[0-9]+)*/
+#          requires:
 #            - integrationTests
       - publishChart:
           context: github-bot

diff --git a/charts/otv-backend/Chart.yaml b/charts/otv-backend/Chart.yaml
@@ -1,4 +1,4 @@
 description: 1K Validators Backend
 name: otv-backend
-version: v2.6.86
+version: v2.6.87
 apiVersion: v2
diff --git a/charts/otv-backend/values.yaml b/charts/otv-backend/values.yaml
@@ -17,7 +17,7 @@ resources:
 
 image:
   repo: web3f/otv-backend
-  tag: 2.6.86
+  tag: 2.6.87
 
 certificate:
   enabled: true

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -132,12 +132,22 @@ services:
         dockerfile: ./Dockerfile-dev
       volumes:
         - .:/app
-#      ports:
-#        - "3300:3300"
       networks:
         testing_net:
           ipv4_address: 172.28.1.9
 
+    1kv-worker:
+      build:
+        context: .
+        args:
+          PACKAGE: worker
+        dockerfile: ./Dockerfile-dev
+      volumes:
+        - .:/app
+      networks:
+        testing_net:
+          ipv4_address: 172.28.1.16
+
 
     telemetry-backend-core:
         image: parity/substrate-telemetry-backend:latest
@@ -210,6 +220,22 @@ services:
         testing_net:
           ipv4_address: 172.28.1.14
 
+    redis:
+      image: redis:alpine
+      expose:
+        - "6379"
+      volumes:
+        - ./redis:/data
+      restart: always
+      command: ["redis-server", "--bind", "redis", "--port", "6379"]
+      logging:
+        driver: "json-file"
+        options:
+          max-size: "50m"
+      networks:
+        default:
+        testing_net:
+          ipv4_address: 172.28.1.13
 
 
 

diff --git a/helmfile.d/10-otv-backend.yaml b/helmfile.d/10-otv-backend.yaml
@@ -12,7 +12,7 @@ releases:
     namespace: kusama
     {{ if eq .Environment.Name "production" }}
     chart: w3f/otv-backend
-    version: v2.6.86
+    version: v2.6.87
     {{ else }}
     chart: ../charts/otv-backend
     {{ end }}
@@ -30,7 +30,7 @@ releases:
     namespace: polkadot
     {{ if eq .Environment.Name "production" }}
     chart: w3f/otv-backend
-    version: v2.6.86
+    version: v2.6.87
     {{ else }}
     chart: ../charts/otv-backend
     {{ end }} 

diff --git a/helmfile.d/config/kusama/otv-backend-ci.yaml.gotmpl b/helmfile.d/config/kusama/otv-backend-ci.yaml.gotmpl
@@ -17,7 +17,8 @@ config: |
       "networkPrefix": 2,
       "test": false,
       "retroactive": false,
-      "historicalNominations": false
+      "historicalNominations": false,
+      "apiEndpoints": []
     },
     "constraints": {
       "skipConnectionTime": false,

diff --git a/helmfile.d/config/kusama/otv-backend-prod.yaml.gotmpl b/helmfile.d/config/kusama/otv-backend-prod.yaml.gotmpl
@@ -17,7 +17,8 @@ config: |
       "networkPrefix": 2,
       "test": false,
       "retroactive": false,
-      "historicalNominations": false
+      "historicalNominations": false,
+      "apiEndpoints": []
     },
     "constraints": {
       "skipConnectionTime": false,
@@ -2883,9 +2884,11 @@ config: |
       "nominating": true
     },
     "server": {
-      "port": 3300
+      "port": 3300,
+      "enable": true
     },
     "telemetry": {
+      "enable": true,
       "chains": [
           "0xb0a8d493285c2df73290dfb7e61f870f17b41801197a149ca93654499ea3dafe"
       ],

diff --git a/helmfile.d/config/polkadot/otv-backend-ci.yaml.gotmpl b/helmfile.d/config/polkadot/otv-backend-ci.yaml.gotmpl
@@ -17,7 +17,8 @@ config: |
       "networkPrefix": 0,
       "test": false,
       "retroactive": false,
-      "historicalNominations": false
+      "historicalNominations": false,
+      "apiEndpoints": []
     },
     "constraints": {
       "skipConnectionTime": false,

diff --git a/helmfile.d/config/polkadot/otv-backend-prod.yaml.gotmpl b/helmfile.d/config/polkadot/otv-backend-prod.yaml.gotmpl
@@ -17,7 +17,8 @@ config: |
       "networkPrefix": 0,
       "test": false,
       "retroactive": false,
-      "historicalNominations": false
+      "historicalNominations": false,
+      "apiEndpoints": []
     },
     "constraints": {
       "skipConnectionTime": false,
@@ -1266,9 +1267,11 @@ config: |
       "nominating": true
     },
     "server": {
-      "port": 3300
+      "port": 3300,
+      "enable": true
     },
     "telemetry": {
+      "enable": true
       "chains": [
           "0x91b171bb158e2d3848fa23a9f1c25182fb8e20313b2c1eb49219da7a70ce90c3"
       ],

diff --git a/package.json b/package.json
@@ -6,7 +6,8 @@
     "packages/common",
     "packages/gateway",
     "packages/telemetry",
-    "packages/core"
+    "packages/core",
+    "packages/worker"
   ],
   "scripts": {
     "create-config:all": "yarn workspaces foreach run create-config",
@@ -21,7 +22,9 @@
     "start:dev:core": "yarn workspace @1kv/core run start:dev",
     "start:js:core": "NODE_OPTIONS='--max-old-space-size=6096' node packages/core/build/index.js start",
     "start:dev:telemetry": "yarn workspace @1kv/telemetry run start:dev",
-    "start:js:telemetry": "NODE_OPTIONS='--max-old-space-size=6096' node packages/telemetry/build/index.js start"
+    "start:js:telemetry": "NODE_OPTIONS='--max-old-space-size=6096' node packages/telemetry/build/index.js start",
+    "start:dev:worker": "yarn workspace @1kv/worker run start:dev",
+    "start:js:worker": "NODE_OPTIONS='--max-old-space-size=6096' node packages/worker/build/index.js start"
   },
   "devDependencies": {
     "@types/node": "^18.8.4",

diff --git a/packages/common/package.json b/packages/common/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@1kv/common",
-  "version": "2.6.82",
+  "version": "2.6.87",
   "description": "Services for running the Thousand Validator Program.",
   "main": "build/index.js",
   "types": "build/index.d.ts",

diff --git a/packages/common/src/config.ts b/packages/common/src/config.ts
@@ -65,6 +65,7 @@ export type ConfigSchema = {
     test: boolean;
     retroactive: boolean;
     historicalNominations: boolean;
+    apiEndpoints: string[];
   };
   matrix: {
     accessToken: string;

diff --git a/packages/core/package.json b/packages/core/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@1kv/core",
-  "version": "2.6.83",
+  "version": "2.6.87",
   "description": "Services for running the Thousand Validator Program.",
   "main": "index.js",
   "scripts": {

diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts
@@ -13,7 +13,7 @@ import { startClearAccumulatedOfflineTimeJob, startMonitorJob } from "./cron";
 
 const isCI = process.env.CI;
 
-const version = "v2.6.83";
+const version = "v2.6.87";
 
 const catchAndQuit = async (fn: any) => {
   try {

diff --git a/packages/gateway/package.json b/packages/gateway/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@1kv/gateway",
-  "version": "2.6.82",
+  "version": "2.6.87",
   "description": "Services for running the Thousand Validator Program.",
   "main": "build/index.js",
   "types": "build/index.d.ts",
@@ -25,7 +25,7 @@
     }
   },
   "dependencies": {
-    "@1kv/common": "2.6.82",
+    "@1kv/common": "workspace:^",
     "typescript": "^4.8.4"
   },
   "devDependencies": {

diff --git a/packages/gateway/src/run.ts b/packages/gateway/src/run.ts
@@ -2,7 +2,7 @@ import Server from "./server";
 import { Command } from "commander";
 import { Db, Config, logger } from "@1kv/common";
 
-const version = "v3.0.0";
+const version = "v2.6.87";
 
 const catchAndQuit = async (fn: any) => {
   try {

diff --git a/packages/telemetry/package.json b/packages/telemetry/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@1kv/telemetry",
-  "version": "2.6.82",
+  "version": "2.6.87",
   "description": "Services for running the Thousand Validator Program.",
   "main": "build/index.js",
   "types": "build/index.d.ts",
@@ -25,7 +25,7 @@
     }
   },
   "dependencies": {
-    "@1kv/common": "2.6.82",
+    "@1kv/common": "workspace:^",
     "typescript": "^4.8.4"
   },
   "devDependencies": {

diff --git a/packages/telemetry/src/run.ts b/packages/telemetry/src/run.ts
@@ -2,7 +2,7 @@ import { Config, Db, logger } from "@1kv/common";
 import TelemetryClient from "./telemetry";
 import { Command } from "commander";
 
-const version = "v3.0.0";
+const version = "v2.6.87";
 
 const catchAndQuit = async (fn: any) => {
   try {

diff --git a/packages/worker/.dockerignore b/packages/worker/.dockerignore
@@ -0,0 +1,10 @@
+alice
+bob
+node_modules
+scripts
+test
+zero
+*.db
+charts
+helmfile.d
+build
diff --git a/packages/worker/.eslintrc.js b/packages/worker/.eslintrc.js
@@ -0,0 +1,16 @@
+module.exports = {
+	parser: "@typescript-eslint/parser", // Specifies the ESLint parser
+	extends: [
+		"plugin:@typescript-eslint/recommended", // Uses the recommended rules from the @typescript-eslint/eslint-plugin
+		"plugin:security/recommended",
+		"prettier", // Uses eslint-config-prettier to disable ESLint rules from @typescript-eslint/eslint-plugin that would conflict with prettier
+		"plugin:prettier/recommended" // Enables eslint-plugin-prettier and eslint-config-prettier. This will display prettier errors as ESLint errors. Make sure this is always the last configuration in the extends array.
+	],
+	parserOptions: {
+		ecmaVersion: 2018, // Allows for the parsing of modern ECMAScript features
+		sourceType: "module" // Allows for the use of imports
+	},
+	rules: {
+		"@typescript-eslint/ban-ts-comment": "off"
+	}
+};
diff --git a/packages/worker/Dockerfile b/packages/worker/Dockerfile
@@ -0,0 +1,7 @@
+FROM node:17-slim
+ARG MATRIX_TOKEN
+WORKDIR /code
+COPY . .
+RUN ["npm", "i"]
+RUN ["npm", "run", "build"]
+CMD ["npm", "run", "js:start"]
diff --git a/packages/worker/Dockerfile-dev b/packages/worker/Dockerfile-dev
@@ -0,0 +1,8 @@
+FROM node:18 AS builder
+COPY . /app
+WORKDIR /app
+RUN yarn set version 3.2.2 && \
+    yarn install && \
+    yarn workspace @1kv/common build && \
+    yarn workspace @1kv/gateway build
+CMD yarn start:dev:gateway