Merge pull request #2694 from w3f/will-v3.1.3

v3.1.3

apps/1kv-backend/templates/kusama-otv-backend.yaml

@@ -17,7 +17,7 @@ spec:
   source:
     repoURL: https://w3f.github.io/helm-charts/
     chart: otv-backend
-    targetRevision: v3.1.2
+    targetRevision: v3.1.3
     plugin:
       env:
         - name: HELM_VALUES

apps/1kv-backend/templates/polkadot-otv-backend.yaml

@@ -17,7 +17,7 @@ spec:
   source:
     repoURL: https://w3f.github.io/helm-charts/
     chart: otv-backend
-    targetRevision: v3.1.2
+    targetRevision: v3.1.3
     plugin:
       env:
         - name: HELM_VALUES

charts/otv-backend/Chart.yaml

@@ -1,5 +1,5 @@
 description: 1K Validators Backend
 name: otv-backend
-version: v3.1.2
-appVersion: v3.1.2
+version: v3.1.3
+appVersion: v3.1.3
 apiVersion: v2

package.json

@@ -79,8 +79,8 @@
     "typedoc-plugin-markdown": "^3.17.1"
   },
   "dependencies": {
-    "@polkadot/api": "^10.11.2",
-    "@polkadot/rpc-provider": "^10.11.2",
+    "@polkadot/api": "^10.12.1",
+    "@polkadot/rpc-provider": "^10.12.1",
     "@types/ws": "^8.5.10",
     "axios": "^1.6.7",
     "chalk": "4.1.2",

packages/common/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@1kv/common",
-  "version": "3.1.2",
+  "version": "3.1.3",
   "description": "Services for running the Thousand Validator Program.",
   "main": "build/index.js",
   "types": "build/index.d.ts",

packages/common/src/chaindata/queries/Era.ts

@@ -203,16 +203,17 @@ export const findEraBlockHash = async (
       if (!blockHash) {
         return ["", "Block hash is null"];
       }
-      const testEra =
-        await chaindata?.api?.query.staking.activeEra.at(blockHash);
-      if (testEra && testEra.isNone) {
-        logger.info(`Test era is none`);
+      const apiAt = await chaindata?.api?.at(blockHash);
+      if (!apiAt) {
+        return ["", "API at block hash is null"];
+      }
+
+      const testEra = await apiAt.query.staking.currentEra();
+      if (testEra && testEra.isEmpty) {
+        logger.info(`Test era is empty: ${JSON.stringify(testEra)}`);
         return ["", "Test era is none"];
       }
-      const testIndex =
-        testEra && testEra?.unwrap && testEra?.unwrap().index?.toNumber()
-          ? testEra?.unwrap().index.toNumber()
-          : 0;
+      const testIndex = testEra.unwrap().toNumber();
       if (era == testIndex) {
         return [blockHash.toString(), null];
       }

packages/common/src/constants.ts

@@ -1,6 +1,8 @@
 /// One week in milliseconds.
 import WS from "ws";
 
+export const TWO_DAYS_IN_MS = 2 * 24 * 60 * 60 * 1000;
+
 export const FIVE_MINUTES = 5 * 60 * 1000;
 
 export const WEEK = 7 * 24 * 60 * 60 * 1000;
@@ -45,6 +47,9 @@ export const CHAINDATA_SLEEP = 300;
 
 export const API_PROVIDER_TIMEOUT = 10000;
 
+// The number of eras a nominator should wait until making a next nomination
+export const NOMINATOR_SHOULD_NOMINATE_ERAS_THRESHOLD = 1;
+
 /// List of Kusama endpoints we can switch between.
 export const KusamaEndpoints = [
   "wss://kusama-rpc-tn.dwellir.com",
@@ -100,8 +105,8 @@ export const TIME_DELAY_BLOCKS = 10850;
 // The number of blocks after a time delay proxy call was announced that we want to cancel the tx. Should be 36 hours
 export const CANCEL_THRESHOLD = 21700;
 
-// Monitor Cron job for checking if clients have upgraded. This runs ever 15 minutes by default
-export const MONITOR_CRON = "0 */15 * * * *";
+// Monitor Cron job for checking if clients have upgraded. This runs ever 3 minutes by default
+export const MONITOR_CRON = "0 */3 * * * *";
 
 // Clear Offline Time Cron Job. This runs once every sunday  by default
 // export const CLEAR_OFFLINE_CRON = "0 0 0 * * 0";

packages/common/src/constraints/ScoreCandidates.ts

@@ -210,7 +210,9 @@ export const scoreCandidate = async (
     const nominatorStakeScore =
       scaledNominatorStake * constraints.WEIGHT_CONFIG.NOMINATIONS_WEIGHT;
 
-    const isAlternativeClient = candidate?.implementation != "Parity Polkadot";
+    const isAlternativeClient = candidate?.implementation
+      ? candidate?.implementation != "Parity Polkadot"
+      : false;
     const clientScore = isAlternativeClient
       ? constraints.WEIGHT_CONFIG.CLIENT_WEIGHT
       : 0;

packages/common/src/constraints/ValidityChecks.ts

@@ -89,7 +89,7 @@ export const checkLatestClientVersion = async (
 ): Promise<boolean> => {
   try {
     const skipClientUpgrade = config.constraints?.skipClientUpgrade || false;
-    if (skipClientUpgrade!) {
+    if (!skipClientUpgrade) {
       if (candidate?.implementation == "Kagome Node") {
         await setLatestClientReleaseValidity(candidate.stash, true);
         return true;
@@ -120,18 +120,19 @@ export const checkLatestClientVersion = async (
           return true;
         }
       } else {
+        await setLatestClientReleaseValidity(candidate.stash, false);
         return false;
       }
     } else {
       await setLatestClientReleaseValidity(candidate.stash, true);
       return true;
     }
-    return true;
   } catch (e) {
     logger.error(
       `Error checking latest client version: ${e}`,
       constraintsLabel,
     );
+    await setLatestClientReleaseValidity(candidate.stash, false);
     return false;
   }
 };

packages/common/src/db/models.ts

@@ -463,6 +463,12 @@ export const CandidateSchema = new Schema({
 
 export const CandidateModel = mongoose.model("Candidate", CandidateSchema);
 
+export interface Era {
+  lastNominatedEraIndex: string;
+  nextNomination: number;
+  when: number;
+}
+
 export const EraSchema = new Schema({
   // The last era a nomination took place
   lastNominatedEraIndex: { type: String, default: "0" },
@@ -726,6 +732,7 @@ export interface ValidatorScore {
   nominatorStake: number;
   // The randomness factor used to buffer the total
   randomness: number;
+  client: number;
 }
 
 export const ValidatorScoreSchema = new Schema({
@@ -763,6 +770,7 @@ export const ValidatorScoreSchema = new Schema({
   country: Number,
   provider: Number,
   nominatorStake: Number,
+  client: Number,
   // The randomness factor used to buffer the total
   randomness: Number,
 });

packages/common/src/db/queries/Candidate.ts

@@ -334,7 +334,7 @@ export const getIdentityName = async (
       .lean<Identity>()
       .select({ name: 1 });
 
-    return identity?.name;
+    return identity?.name || null;
   }
 };
 
@@ -516,6 +516,7 @@ export const updateCandidateOnlineTelemetryDetails = async (
             telemetryId: { $literal: telemetryNodeDetails.telemetryId },
             onlineSince: { $literal: Date.now() },
             offlineSince: { $literal: 0 },
+            version: telemetryNodeDetails.version,
             implementation: {
               $literal: telemetryNodeDetails.nodeImplementation,
             },

packages/common/src/db/queries/Era.ts

@@ -1,33 +1,39 @@
-import { EraModel } from "../models";
+import { Era, EraModel } from "../models";
+import logger from "../../logger";
 
 export const setLastNominatedEraIndex = async (
   index: number,
 ): Promise<boolean> => {
-  const data = await EraModel.findOne({}).lean();
-  if (!data) {
-    const eraIndex = new EraModel({
-      lastNominatedEraIndex: index.toString(),
-      when: Date.now(),
-    });
-    await eraIndex.save();
-    return true;
-  }
-
-  await EraModel.findOneAndUpdate(
-    { lastNominatedEraIndex: /.*/ },
-    {
-      $set: {
+  try {
+    const data = await EraModel.findOne({}).lean();
+    if (!data) {
+      const eraIndex = new EraModel({
         lastNominatedEraIndex: index.toString(),
         when: Date.now(),
-        nextNomination: Date.now() + 86400000,
+      });
+      await eraIndex.save();
+      return true;
+    }
+
+    await EraModel.findOneAndUpdate(
+      { lastNominatedEraIndex: /.*/ },
+      {
+        $set: {
+          lastNominatedEraIndex: index.toString(),
+          when: Date.now(),
+          nextNomination: Date.now() + 86400000,
+        },
       },
-    },
-  ).exec();
-  return true;
+    ).exec();
+    return true;
+  } catch (e) {
+    logger.error(
+      `Error setting last nominated era index: ${JSON.stringify(e)}`,
+    );
+    return false;
+  }
 };
 
-export const getLastNominatedEraIndex = async (): Promise<any> => {
-  return EraModel.findOne({ lastNominatedEraIndex: /[0-9]+/ })
-    .lean()
-    .exec();
+export const getLastNominatedEraIndex = async (): Promise<Era | null> => {
+  return EraModel.findOne({ lastNominatedEraIndex: /[0-9]+/ }).lean<Era>();
 };

packages/common/src/db/queries/Location.ts

@@ -9,6 +9,7 @@ import { logger } from "../../index";
 import { getLatestSession } from "./Session";
 import { HardwareSpec } from "../../types";
 import { dbLabel } from "../index";
+import { TWO_DAYS_IN_MS } from "../../constants";
 
 export const getAllLocations = async (): Promise<Location[]> => {
   return LocationModel.find({}).lean<Location[]>();
@@ -131,6 +132,22 @@ export const cleanBlankLocations = async (): Promise<any> => {
   }).exec();
 };
 
+// Remove all location data older than two days
+export const cleanOldLocations = async (): Promise<boolean> => {
+  const twoDaysAgo = Date.now() - TWO_DAYS_IN_MS;
+
+  try {
+    await LocationModel.deleteMany({ updated: { $lt: twoDaysAgo } }).exec();
+    return true;
+  } catch (error) {
+    logger.info(
+      `Error cleaning old locations: ${JSON.stringify(error)}`,
+      dbLabel,
+    );
+    return false;
+  }
+};
+
 // Sets a location from heartbeats
 
 export const iitExists = async (): Promise<any> => {

packages/common/src/db/queries/NominatorStake.ts

@@ -1,4 +1,7 @@
 import { NominatorStake, NominatorStakeModel } from "../models";
+import { TWO_DAYS_IN_MS } from "../../constants";
+import { logger } from "../../index";
+import { dbLabel } from "../index";
 
 export const setNominatorStake = async (
   validator: string,
@@ -84,3 +87,20 @@ export const getNominatorStake = async (
     .sort("-era")
     .limit(limit ? limit : 100);
 };
+
+export const cleanOldNominatorStakes = async (): Promise<boolean> => {
+  const twoDaysAgo = Date.now() - TWO_DAYS_IN_MS;
+
+  try {
+    await NominatorStakeModel.deleteMany({
+      updated: { $lt: twoDaysAgo },
+    }).exec();
+    return true;
+  } catch (error) {
+    logger.info(
+      `Error cleaning old nominator stakes: ${JSON.stringify(error)}`,
+      dbLabel,
+    );
+    return false;
+  }
+};

packages/common/src/db/queries/ValidatorScore.ts

@@ -1,4 +1,5 @@
 import { ValidatorScore, ValidatorScoreModel } from "../models";
+import { TWO_DAYS_IN_MS } from "../../constants";
 
 export const setValidatorScore = async (
   address: string,
@@ -25,6 +26,7 @@ export const setValidatorScore = async (
       nominatorStake,
       randomness,
       updated,
+      client,
     } = score;
 
     const data = await ValidatorScoreModel.findOne({
@@ -54,6 +56,7 @@ export const setValidatorScore = async (
         provider,
         nominatorStake,
         randomness,
+        client,
       });
       await score.save();
       return true;
@@ -82,6 +85,7 @@ export const setValidatorScore = async (
         country,
         provider,
         nominatorStake,
+        client,
         randomness,
       },
     ).exec();
@@ -107,18 +111,15 @@ export const getValidatorScore = async (
 
 export const getLatestValidatorScore = async (
   address: string,
-): Promise<ValidatorScore> => {
+): Promise<ValidatorScore | null> => {
   return ValidatorScoreModel.findOne({ address: address }, { _id: 0, __v: 0 })
     .sort({ session: -1 })
     .limit(1)
     .lean<ValidatorScore>();
 };
 
 export const deleteOldValidatorScores = async (): Promise<any> => {
-  const FIVE_MINUTES = 300000;
-  const ONE_WEEK = 604800016.56;
-  const ONE_MONTH = 2629800000;
-  const timeWindow = Date.now() - ONE_WEEK;
+  const timeWindow = Date.now() - TWO_DAYS_IN_MS;
   const scoreToDelete = await ValidatorScoreModel.find({
     updated: { $lt: timeWindow },
   }).exec();