From a75f950c67c80e968092dd4fdbd4a1ac1411825b Mon Sep 17 00:00:00 2001 From: Uladzislau Hubar Date: Tue, 16 Jan 2024 11:45:06 +0100 Subject: [PATCH 01/96] Added Services for Hashing and Proximity/Score Calculations, implemented Script for KAs Distribution Simulations --- .gitignore | 3 + package-lock.json | 2728 ++++++++++++++++- package.json | 3 + src/constants/constants.js | 4 + src/service/hashing-service.js | 31 + src/service/proximity-scoring-service.js | 148 + .../mocks/blockchain-module-manager-mock.js | 47 + .../plots/.gitkeep | 0 .../simulation.js | 416 +++ 9 files changed, 3368 insertions(+), 12 deletions(-) create mode 100644 src/service/hashing-service.js create mode 100644 src/service/proximity-scoring-service.js create mode 100644 tools/knowledge-assets-distribution-simulation/mocks/blockchain-module-manager-mock.js create mode 100644 tools/knowledge-assets-distribution-simulation/plots/.gitkeep create mode 100644 tools/knowledge-assets-distribution-simulation/simulation.js diff --git a/.gitignore b/.gitignore index be72117326..3484fcd91e 100644 --- a/.gitignore +++ b/.gitignore @@ -115,3 +115,6 @@ data* # VS code files .vscode/launch.json + +# KAs Distribution Simulation Script Plots +tools/knowledge-assets-distribution-simulation/plots/*jpg diff --git a/package-lock.json b/package-lock.json index 71744602bc..2c474e188b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -66,6 +66,8 @@ "devDependencies": { "@cucumber/cucumber": "^8.5.2", "chai": "^4.3.6", + "d3": "^7.8.5", + "d3-node": "^3.0.0", "dkg.js": "^6.0.2", "eslint": "^8.23.0", "eslint-config-airbnb": "^19.0.4", @@ -75,6 +77,7 @@ "mocha": "^10.0.0", "nyc": "^15.1.0", "prettier": "^2.7.1", + "sharp": "^0.32.6", "sinon": "^14.0.0", "slugify": "^1.6.5" }, @@ -5396,6 +5399,15 @@ "node": ">=10.17" } }, + "node_modules/@tootallnate/once": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz", + "integrity": "sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==", + "dev": true, + "engines": { + "node": ">= 6" + } + }, "node_modules/@tsconfig/node10": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.9.tgz", @@ -5650,6 +5662,13 @@ "resolved": "https://registry.npmjs.org/@vascosantos/moving-average/-/moving-average-1.1.0.tgz", "integrity": "sha512-MVEJ4vWAPNbrGLjz7ITnHYg+YXZ6ijAqtH5/cHwSoCpbvuJ98aLXwFfPKAUfZpJMQR5uXB58UJajbY130IRF/w==" }, + "node_modules/abab": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.6.tgz", + "integrity": "sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA==", + "deprecated": "Use your platform's native atob() and btoa() methods instead", + "dev": true + }, "node_modules/abort-controller": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", @@ -5715,6 +5734,37 @@ "node": ">=0.4.0" } }, + "node_modules/acorn-globals": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/acorn-globals/-/acorn-globals-6.0.0.tgz", + "integrity": "sha512-ZQl7LOWaF5ePqqcX4hLuv/bLXYQNfNWw2c0/yX/TsPRKamzHcTGQnlCjHT3TsmkOUVEPS3crCxiPfdzE/Trlhg==", + "dev": true, + "dependencies": { + "acorn": "^7.1.1", + "acorn-walk": "^7.1.1" + } + }, + "node_modules/acorn-globals/node_modules/acorn": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz", + "integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==", + "dev": true, + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-globals/node_modules/acorn-walk": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-7.2.0.tgz", + "integrity": "sha512-OPdCF6GsMIP+Az+aWfAAOEt2/+iVDKE7oy6lJ098aoe59oAmK76qV6Gw60SbZ8jHuG2wH058GF4pLFbYamYrVA==", + "dev": true, + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/acorn-jsx": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", @@ -6228,6 +6278,12 @@ "deep-equal": "^2.0.5" } }, + "node_modules/b4a": { + "version": "1.6.4", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.4.tgz", + "integrity": "sha512-fpWrvyVHEKyeEvbKZTVOeZF3VSKKWtJxFIxX/jaVPf+cLbGUSitjb49pHLqPV2BUNNZ0LcoeEGfE/YCpyDYHIw==", + "dev": true + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -6450,6 +6506,12 @@ "run-parallel-limit": "^1.1.0" } }, + "node_modules/browser-process-hrtime": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz", + "integrity": "sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow==", + "dev": true + }, "node_modules/browser-stdout": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz", @@ -7402,6 +7464,30 @@ "resolved": "https://registry.npmjs.org/crypto-js/-/crypto-js-3.3.0.tgz", "integrity": "sha512-DIT51nX0dCfKltpRiXV+/TVZq+Qq2NgF4644+K7Ttnla7zEzqc+kjJyiB96BHNyUTBxyjzRcZYpUdZa+QAqi6Q==" }, + "node_modules/cssom": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.4.4.tgz", + "integrity": "sha512-p3pvU7r1MyyqbTk+WbNJIgJjG2VmTIaB10rI93LzVPrmDJKkzKYMtxxyAvQXR/NS6otuzveI7+7BBq3SjBS2mw==", + "dev": true + }, + "node_modules/cssstyle": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-2.3.0.tgz", + "integrity": "sha512-AZL67abkUzIuvcHqk7c09cezpGNcxUxU4Ioi/05xHk4DQeTkWmGYftIE6ctU6AEt+Gn4n1lDStOtj7FKycP71A==", + "dev": true, + "dependencies": { + "cssom": "~0.3.6" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cssstyle/node_modules/cssom": { + "version": "0.3.8", + "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.3.8.tgz", + "integrity": "sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg==", + "dev": true + }, "node_modules/d": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/d/-/d-1.0.1.tgz", @@ -7411,6 +7497,751 @@ "type": "^1.0.1" } }, + "node_modules/d3": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/d3/-/d3-7.8.5.tgz", + "integrity": "sha512-JgoahDG51ncUfJu6wX/1vWQEqOflgXyl4MaHqlcSruTez7yhaRKR9i8VjjcQGeS2en/jnFivXuaIMnseMMt0XA==", + "dev": true, + "dependencies": { + "d3-array": "3", + "d3-axis": "3", + "d3-brush": "3", + "d3-chord": "3", + "d3-color": "3", + "d3-contour": "4", + "d3-delaunay": "6", + "d3-dispatch": "3", + "d3-drag": "3", + "d3-dsv": "3", + "d3-ease": "3", + "d3-fetch": "3", + "d3-force": "3", + "d3-format": "3", + "d3-geo": "3", + "d3-hierarchy": "3", + "d3-interpolate": "3", + "d3-path": "3", + "d3-polygon": "3", + "d3-quadtree": "3", + "d3-random": "3", + "d3-scale": "4", + "d3-scale-chromatic": "3", + "d3-selection": "3", + "d3-shape": "3", + "d3-time": "3", + "d3-time-format": "4", + "d3-timer": "3", + "d3-transition": "3", + "d3-zoom": "3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-array": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-3.2.4.tgz", + "integrity": "sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==", + "dev": true, + "dependencies": { + "internmap": "1 - 2" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-axis": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-axis/-/d3-axis-3.0.0.tgz", + "integrity": "sha512-IH5tgjV4jE/GhHkRV0HiVYPDtvfjHQlQfJHs0usq7M30XcSBvOotpmH1IgkcXsO/5gEQZD43B//fc7SRT5S+xw==", + "dev": true, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-brush": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-brush/-/d3-brush-3.0.0.tgz", + "integrity": "sha512-ALnjWlVYkXsVIGlOsuWH1+3udkYFI48Ljihfnh8FZPF2QS9o+PzGLBslO0PjzVoHLZ2KCVgAM8NVkXPJB2aNnQ==", + "dev": true, + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-drag": "2 - 3", + "d3-interpolate": "1 - 3", + "d3-selection": "3", + "d3-transition": "3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-chord": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-chord/-/d3-chord-3.0.1.tgz", + "integrity": "sha512-VE5S6TNa+j8msksl7HwjxMHDM2yNK3XCkusIlpX5kwauBfXuyLAtNg9jCp/iHH61tgI4sb6R/EIMWCqEIdjT/g==", + "dev": true, + "dependencies": { + "d3-path": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-collection": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/d3-collection/-/d3-collection-1.0.7.tgz", + "integrity": "sha512-ii0/r5f4sjKNTfh84Di+DpztYwqKhEyUlKoPrzUFfeSkWxjW49xU2QzO9qrPrNkpdI0XJkfzvmTu8V2Zylln6A==", + "dev": true + }, + "node_modules/d3-color": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-3.1.0.tgz", + "integrity": "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==", + "dev": true, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-contour": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/d3-contour/-/d3-contour-4.0.2.tgz", + "integrity": "sha512-4EzFTRIikzs47RGmdxbeUvLWtGedDUNkTcmzoeyg4sP/dvCexO47AaQL7VKy/gul85TOxw+IBgA8US2xwbToNA==", + "dev": true, + "dependencies": { + "d3-array": "^3.2.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-delaunay": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/d3-delaunay/-/d3-delaunay-6.0.4.tgz", + "integrity": "sha512-mdjtIZ1XLAM8bm/hx3WwjfHt6Sggek7qH043O8KEjDXN40xi3vx/6pYSVTwLjEgiXQTbvaouWKynLBiUZ6SK6A==", + "dev": true, + "dependencies": { + "delaunator": "5" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-dispatch": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-dispatch/-/d3-dispatch-3.0.1.tgz", + "integrity": "sha512-rzUyPU/S7rwUflMyLc1ETDeBj0NRuHKKAcvukozwhshr6g6c5d8zh4c2gQjY2bZ0dXeGLWc1PF174P2tVvKhfg==", + "dev": true, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-drag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-drag/-/d3-drag-3.0.0.tgz", + "integrity": "sha512-pWbUJLdETVA8lQNJecMxoXfH6x+mO2UQo8rSmZ+QqxcbyA3hfeprFgIT//HW2nlHChWeIIMwS2Fq+gEARkhTkg==", + "dev": true, + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-selection": "3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-dsv": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-dsv/-/d3-dsv-3.0.1.tgz", + "integrity": "sha512-UG6OvdI5afDIFP9w4G0mNq50dSOsXHJaRE8arAS5o9ApWnIElp8GZw1Dun8vP8OyHOZ/QJUKUJwxiiCCnUwm+Q==", + "dev": true, + "dependencies": { + "commander": "7", + "iconv-lite": "0.6", + "rw": "1" + }, + "bin": { + "csv2json": "bin/dsv2json.js", + "csv2tsv": "bin/dsv2dsv.js", + "dsv2dsv": "bin/dsv2dsv.js", + "dsv2json": "bin/dsv2json.js", + "json2csv": "bin/json2dsv.js", + "json2dsv": "bin/json2dsv.js", + "json2tsv": "bin/json2dsv.js", + "tsv2csv": "bin/dsv2dsv.js", + "tsv2json": "bin/dsv2json.js" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-dsv/node_modules/commander": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", + "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==", + "dev": true, + "engines": { + "node": ">= 10" + } + }, + "node_modules/d3-dsv/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "dev": true, + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/d3-ease": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-ease/-/d3-ease-3.0.1.tgz", + "integrity": "sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==", + "dev": true, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-fetch": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-fetch/-/d3-fetch-3.0.1.tgz", + "integrity": "sha512-kpkQIM20n3oLVBKGg6oHrUchHM3xODkTzjMoj7aWQFq5QEM+R6E4WkzT5+tojDY7yjez8KgCBRoj4aEr99Fdqw==", + "dev": true, + "dependencies": { + "d3-dsv": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-force": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-force/-/d3-force-3.0.0.tgz", + "integrity": "sha512-zxV/SsA+U4yte8051P4ECydjD/S+qeYtnaIyAs9tgHCqfguma/aAQDjo85A9Z6EKhBirHRJHXIgJUlffT4wdLg==", + "dev": true, + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-quadtree": "1 - 3", + "d3-timer": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-format": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-format/-/d3-format-3.1.0.tgz", + "integrity": "sha512-YyUI6AEuY/Wpt8KWLgZHsIU86atmikuoOmCfommt0LYHiQSPjvX2AcFc38PX0CBpr2RCyZhjex+NS/LPOv6YqA==", + "dev": true, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-geo": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-geo/-/d3-geo-3.1.0.tgz", + "integrity": "sha512-JEo5HxXDdDYXCaWdwLRt79y7giK8SbhZJbFWXqbRTolCHFI5jRqteLzCsq51NKbUoX0PjBVSohxrx+NoOUujYA==", + "dev": true, + "dependencies": { + "d3-array": "2.5.0 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-hierarchy": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/d3-hierarchy/-/d3-hierarchy-3.1.2.tgz", + "integrity": "sha512-FX/9frcub54beBdugHjDCdikxThEqjnR93Qt7PvQTOHxyiNCAlvMrHhclk3cD5VeAaq9fxmfRp+CnWw9rEMBuA==", + "dev": true, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-interpolate": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-3.0.1.tgz", + "integrity": "sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==", + "dev": true, + "dependencies": { + "d3-color": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-node": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-node/-/d3-node-3.0.0.tgz", + "integrity": "sha512-JuuBToljSQBo+KHSuPVOEdKawsxuLDlrB8TZY8AkPkpvyKaGgX+g0ksgjRg+Mgat9fB77qFQrk09j72cD0yk7w==", + "dev": true, + "dependencies": { + "d3": "^5.16.0", + "jsdom": "^16.7.0" + } + }, + "node_modules/d3-node/node_modules/commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", + "dev": true + }, + "node_modules/d3-node/node_modules/d3": { + "version": "5.16.0", + "resolved": "https://registry.npmjs.org/d3/-/d3-5.16.0.tgz", + "integrity": "sha512-4PL5hHaHwX4m7Zr1UapXW23apo6pexCgdetdJ5kTmADpG/7T9Gkxw0M0tf/pjoB63ezCCm0u5UaFYy2aMt0Mcw==", + "dev": true, + "dependencies": { + "d3-array": "1", + "d3-axis": "1", + "d3-brush": "1", + "d3-chord": "1", + "d3-collection": "1", + "d3-color": "1", + "d3-contour": "1", + "d3-dispatch": "1", + "d3-drag": "1", + "d3-dsv": "1", + "d3-ease": "1", + "d3-fetch": "1", + "d3-force": "1", + "d3-format": "1", + "d3-geo": "1", + "d3-hierarchy": "1", + "d3-interpolate": "1", + "d3-path": "1", + "d3-polygon": "1", + "d3-quadtree": "1", + "d3-random": "1", + "d3-scale": "2", + "d3-scale-chromatic": "1", + "d3-selection": "1", + "d3-shape": "1", + "d3-time": "1", + "d3-time-format": "2", + "d3-timer": "1", + "d3-transition": "1", + "d3-voronoi": "1", + "d3-zoom": "1" + } + }, + "node_modules/d3-node/node_modules/d3-array": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-1.2.4.tgz", + "integrity": "sha512-KHW6M86R+FUPYGb3R5XiYjXPq7VzwxZ22buHhAEVG5ztoEcZZMLov530mmccaqA1GghZArjQV46fuc8kUqhhHw==", + "dev": true + }, + "node_modules/d3-node/node_modules/d3-axis": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/d3-axis/-/d3-axis-1.0.12.tgz", + "integrity": "sha512-ejINPfPSNdGFKEOAtnBtdkpr24c4d4jsei6Lg98mxf424ivoDP2956/5HDpIAtmHo85lqT4pruy+zEgvRUBqaQ==", + "dev": true + }, + "node_modules/d3-node/node_modules/d3-brush": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/d3-brush/-/d3-brush-1.1.6.tgz", + "integrity": "sha512-7RW+w7HfMCPyZLifTz/UnJmI5kdkXtpCbombUSs8xniAyo0vIbrDzDwUJB6eJOgl9u5DQOt2TQlYumxzD1SvYA==", + "dev": true, + "dependencies": { + "d3-dispatch": "1", + "d3-drag": "1", + "d3-interpolate": "1", + "d3-selection": "1", + "d3-transition": "1" + } + }, + "node_modules/d3-node/node_modules/d3-chord": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/d3-chord/-/d3-chord-1.0.6.tgz", + "integrity": "sha512-JXA2Dro1Fxw9rJe33Uv+Ckr5IrAa74TlfDEhE/jfLOaXegMQFQTAgAw9WnZL8+HxVBRXaRGCkrNU7pJeylRIuA==", + "dev": true, + "dependencies": { + "d3-array": "1", + "d3-path": "1" + } + }, + "node_modules/d3-node/node_modules/d3-color": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-1.4.1.tgz", + "integrity": "sha512-p2sTHSLCJI2QKunbGb7ocOh7DgTAn8IrLx21QRc/BSnodXM4sv6aLQlnfpvehFMLZEfBc6g9pH9SWQccFYfJ9Q==", + "dev": true + }, + "node_modules/d3-node/node_modules/d3-contour": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/d3-contour/-/d3-contour-1.3.2.tgz", + "integrity": "sha512-hoPp4K/rJCu0ladiH6zmJUEz6+u3lgR+GSm/QdM2BBvDraU39Vr7YdDCicJcxP1z8i9B/2dJLgDC1NcvlF8WCg==", + "dev": true, + "dependencies": { + "d3-array": "^1.1.1" + } + }, + "node_modules/d3-node/node_modules/d3-dispatch": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/d3-dispatch/-/d3-dispatch-1.0.6.tgz", + "integrity": "sha512-fVjoElzjhCEy+Hbn8KygnmMS7Or0a9sI2UzGwoB7cCtvI1XpVN9GpoYlnb3xt2YV66oXYb1fLJ8GMvP4hdU1RA==", + "dev": true + }, + "node_modules/d3-node/node_modules/d3-drag": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/d3-drag/-/d3-drag-1.2.5.tgz", + "integrity": "sha512-rD1ohlkKQwMZYkQlYVCrSFxsWPzI97+W+PaEIBNTMxRuxz9RF0Hi5nJWHGVJ3Om9d2fRTe1yOBINJyy/ahV95w==", + "dev": true, + "dependencies": { + "d3-dispatch": "1", + "d3-selection": "1" + } + }, + "node_modules/d3-node/node_modules/d3-dsv": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/d3-dsv/-/d3-dsv-1.2.0.tgz", + "integrity": "sha512-9yVlqvZcSOMhCYzniHE7EVUws7Fa1zgw+/EAV2BxJoG3ME19V6BQFBwI855XQDsxyOuG7NibqRMTtiF/Qup46g==", + "dev": true, + "dependencies": { + "commander": "2", + "iconv-lite": "0.4", + "rw": "1" + }, + "bin": { + "csv2json": "bin/dsv2json", + "csv2tsv": "bin/dsv2dsv", + "dsv2dsv": "bin/dsv2dsv", + "dsv2json": "bin/dsv2json", + "json2csv": "bin/json2dsv", + "json2dsv": "bin/json2dsv", + "json2tsv": "bin/json2dsv", + "tsv2csv": "bin/dsv2dsv", + "tsv2json": "bin/dsv2json" + } + }, + "node_modules/d3-node/node_modules/d3-ease": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/d3-ease/-/d3-ease-1.0.7.tgz", + "integrity": "sha512-lx14ZPYkhNx0s/2HX5sLFUI3mbasHjSSpwO/KaaNACweVwxUruKyWVcb293wMv1RqTPZyZ8kSZ2NogUZNcLOFQ==", + "dev": true + }, + "node_modules/d3-node/node_modules/d3-fetch": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/d3-fetch/-/d3-fetch-1.2.0.tgz", + "integrity": "sha512-yC78NBVcd2zFAyR/HnUiBS7Lf6inSCoWcSxFfw8FYL7ydiqe80SazNwoffcqOfs95XaLo7yebsmQqDKSsXUtvA==", + "dev": true, + "dependencies": { + "d3-dsv": "1" + } + }, + "node_modules/d3-node/node_modules/d3-force": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/d3-force/-/d3-force-1.2.1.tgz", + "integrity": "sha512-HHvehyaiUlVo5CxBJ0yF/xny4xoaxFxDnBXNvNcfW9adORGZfyNF1dj6DGLKyk4Yh3brP/1h3rnDzdIAwL08zg==", + "dev": true, + "dependencies": { + "d3-collection": "1", + "d3-dispatch": "1", + "d3-quadtree": "1", + "d3-timer": "1" + } + }, + "node_modules/d3-node/node_modules/d3-format": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/d3-format/-/d3-format-1.4.5.tgz", + "integrity": "sha512-J0piedu6Z8iB6TbIGfZgDzfXxUFN3qQRMofy2oPdXzQibYGqPB/9iMcxr/TGalU+2RsyDO+U4f33id8tbnSRMQ==", + "dev": true + }, + "node_modules/d3-node/node_modules/d3-geo": { + "version": "1.12.1", + "resolved": "https://registry.npmjs.org/d3-geo/-/d3-geo-1.12.1.tgz", + "integrity": "sha512-XG4d1c/UJSEX9NfU02KwBL6BYPj8YKHxgBEw5om2ZnTRSbIcego6dhHwcxuSR3clxh0EpE38os1DVPOmnYtTPg==", + "dev": true, + "dependencies": { + "d3-array": "1" + } + }, + "node_modules/d3-node/node_modules/d3-hierarchy": { + "version": "1.1.9", + "resolved": "https://registry.npmjs.org/d3-hierarchy/-/d3-hierarchy-1.1.9.tgz", + "integrity": "sha512-j8tPxlqh1srJHAtxfvOUwKNYJkQuBFdM1+JAUfq6xqH5eAqf93L7oG1NVqDa4CpFZNvnNKtCYEUC8KY9yEn9lQ==", + "dev": true + }, + "node_modules/d3-node/node_modules/d3-interpolate": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-1.4.0.tgz", + "integrity": "sha512-V9znK0zc3jOPV4VD2zZn0sDhZU3WAE2bmlxdIwwQPPzPjvyLkd8B3JUVdS1IDUFDkWZ72c9qnv1GK2ZagTZ8EA==", + "dev": true, + "dependencies": { + "d3-color": "1" + } + }, + "node_modules/d3-node/node_modules/d3-path": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/d3-path/-/d3-path-1.0.9.tgz", + "integrity": "sha512-VLaYcn81dtHVTjEHd8B+pbe9yHWpXKZUC87PzoFmsFrJqgFwDe/qxfp5MlfsfM1V5E/iVt0MmEbWQ7FVIXh/bg==", + "dev": true + }, + "node_modules/d3-node/node_modules/d3-polygon": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/d3-polygon/-/d3-polygon-1.0.6.tgz", + "integrity": "sha512-k+RF7WvI08PC8reEoXa/w2nSg5AUMTi+peBD9cmFc+0ixHfbs4QmxxkarVal1IkVkgxVuk9JSHhJURHiyHKAuQ==", + "dev": true + }, + "node_modules/d3-node/node_modules/d3-quadtree": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/d3-quadtree/-/d3-quadtree-1.0.7.tgz", + "integrity": "sha512-RKPAeXnkC59IDGD0Wu5mANy0Q2V28L+fNe65pOCXVdVuTJS3WPKaJlFHer32Rbh9gIo9qMuJXio8ra4+YmIymA==", + "dev": true + }, + "node_modules/d3-node/node_modules/d3-random": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/d3-random/-/d3-random-1.1.2.tgz", + "integrity": "sha512-6AK5BNpIFqP+cx/sreKzNjWbwZQCSUatxq+pPRmFIQaWuoD+NrbVWw7YWpHiXpCQ/NanKdtGDuB+VQcZDaEmYQ==", + "dev": true + }, + "node_modules/d3-node/node_modules/d3-scale": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/d3-scale/-/d3-scale-2.2.2.tgz", + "integrity": "sha512-LbeEvGgIb8UMcAa0EATLNX0lelKWGYDQiPdHj+gLblGVhGLyNbaCn3EvrJf0A3Y/uOOU5aD6MTh5ZFCdEwGiCw==", + "dev": true, + "dependencies": { + "d3-array": "^1.2.0", + "d3-collection": "1", + "d3-format": "1", + "d3-interpolate": "1", + "d3-time": "1", + "d3-time-format": "2" + } + }, + "node_modules/d3-node/node_modules/d3-scale-chromatic": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/d3-scale-chromatic/-/d3-scale-chromatic-1.5.0.tgz", + "integrity": "sha512-ACcL46DYImpRFMBcpk9HhtIyC7bTBR4fNOPxwVSl0LfulDAwyiHyPOTqcDG1+t5d4P9W7t/2NAuWu59aKko/cg==", + "dev": true, + "dependencies": { + "d3-color": "1", + "d3-interpolate": "1" + } + }, + "node_modules/d3-node/node_modules/d3-selection": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-1.4.2.tgz", + "integrity": "sha512-SJ0BqYihzOjDnnlfyeHT0e30k0K1+5sR3d5fNueCNeuhZTnGw4M4o8mqJchSwgKMXCNFo+e2VTChiSJ0vYtXkg==", + "dev": true + }, + "node_modules/d3-node/node_modules/d3-shape": { + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-1.3.7.tgz", + "integrity": "sha512-EUkvKjqPFUAZyOlhY5gzCxCeI0Aep04LwIRpsZ/mLFelJiUfnK56jo5JMDSE7yyP2kLSb6LtF+S5chMk7uqPqw==", + "dev": true, + "dependencies": { + "d3-path": "1" + } + }, + "node_modules/d3-node/node_modules/d3-time": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/d3-time/-/d3-time-1.1.0.tgz", + "integrity": "sha512-Xh0isrZ5rPYYdqhAVk8VLnMEidhz5aP7htAADH6MfzgmmicPkTo8LhkLxci61/lCB7n7UmE3bN0leRt+qvkLxA==", + "dev": true + }, + "node_modules/d3-node/node_modules/d3-time-format": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/d3-time-format/-/d3-time-format-2.3.0.tgz", + "integrity": "sha512-guv6b2H37s2Uq/GefleCDtbe0XZAuy7Wa49VGkPVPMfLL9qObgBST3lEHJBMUp8S7NdLQAGIvr2KXk8Hc98iKQ==", + "dev": true, + "dependencies": { + "d3-time": "1" + } + }, + "node_modules/d3-node/node_modules/d3-timer": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/d3-timer/-/d3-timer-1.0.10.tgz", + "integrity": "sha512-B1JDm0XDaQC+uvo4DT79H0XmBskgS3l6Ve+1SBCfxgmtIb1AVrPIoqd+nPSv+loMX8szQ0sVUhGngL7D5QPiXw==", + "dev": true + }, + "node_modules/d3-node/node_modules/d3-transition": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/d3-transition/-/d3-transition-1.3.2.tgz", + "integrity": "sha512-sc0gRU4PFqZ47lPVHloMn9tlPcv8jxgOQg+0zjhfZXMQuvppjG6YuwdMBE0TuqCZjeJkLecku/l9R0JPcRhaDA==", + "dev": true, + "dependencies": { + "d3-color": "1", + "d3-dispatch": "1", + "d3-ease": "1", + "d3-interpolate": "1", + "d3-selection": "^1.1.0", + "d3-timer": "1" + } + }, + "node_modules/d3-node/node_modules/d3-zoom": { + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/d3-zoom/-/d3-zoom-1.8.3.tgz", + "integrity": "sha512-VoLXTK4wvy1a0JpH2Il+F2CiOhVu7VRXWF5M/LroMIh3/zBAC3WAt7QoIvPibOavVo20hN6/37vwAsdBejLyKQ==", + "dev": true, + "dependencies": { + "d3-dispatch": "1", + "d3-drag": "1", + "d3-interpolate": "1", + "d3-selection": "1", + "d3-transition": "1" + } + }, + "node_modules/d3-path": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-path/-/d3-path-3.1.0.tgz", + "integrity": "sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ==", + "dev": true, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-polygon": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-polygon/-/d3-polygon-3.0.1.tgz", + "integrity": "sha512-3vbA7vXYwfe1SYhED++fPUQlWSYTTGmFmQiany/gdbiWgU/iEyQzyymwL9SkJjFFuCS4902BSzewVGsHHmHtXg==", + "dev": true, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-quadtree": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-quadtree/-/d3-quadtree-3.0.1.tgz", + "integrity": "sha512-04xDrxQTDTCFwP5H6hRhsRcb9xxv2RzkcsygFzmkSIOJy3PeRJP7sNk3VRIbKXcog561P9oU0/rVH6vDROAgUw==", + "dev": true, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-random": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-random/-/d3-random-3.0.1.tgz", + "integrity": "sha512-FXMe9GfxTxqd5D6jFsQ+DJ8BJS4E/fT5mqqdjovykEB2oFbTMDVdg1MGFxfQW+FBOGoB++k8swBrgwSHT1cUXQ==", + "dev": true, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-scale": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/d3-scale/-/d3-scale-4.0.2.tgz", + "integrity": "sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ==", + "dev": true, + "dependencies": { + "d3-array": "2.10.0 - 3", + "d3-format": "1 - 3", + "d3-interpolate": "1.2.0 - 3", + "d3-time": "2.1.1 - 3", + "d3-time-format": "2 - 4" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-scale-chromatic": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-scale-chromatic/-/d3-scale-chromatic-3.0.0.tgz", + "integrity": "sha512-Lx9thtxAKrO2Pq6OO2Ua474opeziKr279P/TKZsMAhYyNDD3EnCffdbgeSYN5O7m2ByQsxtuP2CSDczNUIZ22g==", + "dev": true, + "dependencies": { + "d3-color": "1 - 3", + "d3-interpolate": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-selection": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz", + "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", + "dev": true, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-shape": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-3.2.0.tgz", + "integrity": "sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA==", + "dev": true, + "dependencies": { + "d3-path": "^3.1.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-time": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-time/-/d3-time-3.1.0.tgz", + "integrity": "sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q==", + "dev": true, + "dependencies": { + "d3-array": "2 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-time-format": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/d3-time-format/-/d3-time-format-4.1.0.tgz", + "integrity": "sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg==", + "dev": true, + "dependencies": { + "d3-time": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-timer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-timer/-/d3-timer-3.0.1.tgz", + "integrity": "sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==", + "dev": true, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-transition": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-transition/-/d3-transition-3.0.1.tgz", + "integrity": "sha512-ApKvfjsSR6tg06xrL434C0WydLr7JewBB3V+/39RMHsaXTOG0zmt/OAXeng5M5LBm0ojmxJrpomQVZ1aPvBL4w==", + "dev": true, + "dependencies": { + "d3-color": "1 - 3", + "d3-dispatch": "1 - 3", + "d3-ease": "1 - 3", + "d3-interpolate": "1 - 3", + "d3-timer": "1 - 3" + }, + "engines": { + "node": ">=12" + }, + "peerDependencies": { + "d3-selection": "2 - 3" + } + }, + "node_modules/d3-voronoi": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/d3-voronoi/-/d3-voronoi-1.1.4.tgz", + "integrity": "sha512-dArJ32hchFsrQ8uMiTBLq256MpnZjeuBtdHpaDlYuQyjU0CVzCJl/BVW+SkszaAeH95D/8gxqAhgx0ouAWAfRg==", + "dev": true + }, + "node_modules/d3-zoom": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-zoom/-/d3-zoom-3.0.0.tgz", + "integrity": "sha512-b8AmV3kfQaqWAuacbPuNbL6vahnOJflOhexLzMMNLga62+/nh0JzvJ0aO/5a5MVgUFGS7Hu1P9P03o3fJkDCyw==", + "dev": true, + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-drag": "2 - 3", + "d3-interpolate": "1 - 3", + "d3-selection": "2 - 3", + "d3-transition": "2 - 3" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/damerau-levenshtein": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/damerau-levenshtein/-/damerau-levenshtein-1.0.8.tgz", @@ -7438,6 +8269,55 @@ "node": ">= 12" } }, + "node_modules/data-urls": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-2.0.0.tgz", + "integrity": "sha512-X5eWTSXO/BJmpdIKCRuKUgSCgAN0OwliVK3yPKbwIWU1Tdw5BRajxlzMidvh+gwko9AfQ9zIj52pzF91Q3YAvQ==", + "dev": true, + "dependencies": { + "abab": "^2.0.3", + "whatwg-mimetype": "^2.3.0", + "whatwg-url": "^8.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/data-urls/node_modules/tr46": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-2.1.0.tgz", + "integrity": "sha512-15Ih7phfcdP5YxqiB+iDtLoaTz4Nd35+IiAv0kQ5FNKHzXgdWqPoTIqEDDJmXceQt4JZk6lVPT8lnDlPpGDppw==", + "dev": true, + "dependencies": { + "punycode": "^2.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/data-urls/node_modules/webidl-conversions": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-6.1.0.tgz", + "integrity": "sha512-qBIvFLGiBpLjfwmYAaHPXsn+ho5xZnGvyGvsarywGNc8VyQJUMHJ8OBKGGrPER0okBeMDaan4mNBlgBROxuI8w==", + "dev": true, + "engines": { + "node": ">=10.4" + } + }, + "node_modules/data-urls/node_modules/whatwg-url": { + "version": "8.7.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-8.7.0.tgz", + "integrity": "sha512-gAojqb/m9Q8a5IV96E3fHJM70AzCkgt4uXYX2O7EmuyOnLrViCQlsEBmF9UQIu3/aeAIp2U17rtbpZWNntQqdg==", + "dev": true, + "dependencies": { + "lodash": "^4.7.0", + "tr46": "^2.1.0", + "webidl-conversions": "^6.1.0" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/dateformat": { "version": "4.6.3", "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-4.6.3.tgz", @@ -7622,6 +8502,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/delaunator": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/delaunator/-/delaunator-5.0.0.tgz", + "integrity": "sha512-AyLvtyJdbv/U1GkiS6gUUzclRoAY4Gs75qkMygJJhU75LW4DNuSF2RMzpxs9jw9Oz1BobHjTdkG3zdP55VxAqw==", + "dev": true, + "dependencies": { + "robust-predicates": "^3.0.0" + } + }, "node_modules/delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -7655,6 +8544,15 @@ "npm": "1.2.8000 || >= 1.4.16" } }, + "node_modules/detect-libc": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.2.tgz", + "integrity": "sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw==", + "dev": true, + "engines": { + "node": ">=8" + } + }, "node_modules/diff": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", @@ -8343,6 +9241,28 @@ } ] }, + "node_modules/domexception": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/domexception/-/domexception-2.0.1.tgz", + "integrity": "sha512-yxJ2mFy/sibVQlu5qHjOkf9J3K6zgmCxgJ94u2EdvDOV09H+32LtRswEcUsmUWN72pVLOEnTSRaIVVzVQgS0dg==", + "deprecated": "Use your platform's native DOMException instead", + "dev": true, + "dependencies": { + "webidl-conversions": "^5.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/domexception/node_modules/webidl-conversions": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-5.0.0.tgz", + "integrity": "sha512-VlZwKPCkYKxQgeSbH5EyngOmRp7Ww7I9rQLERETtf5ofd9pGeswWiOtogpEO850jziPRarreGxn5QIiTqpb2wA==", + "dev": true, + "engines": { + "node": ">=8" + } + }, "node_modules/domhandler": { "version": "5.0.3", "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", @@ -8800,6 +9720,27 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/escodegen": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", + "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==", + "dev": true, + "dependencies": { + "esprima": "^4.0.1", + "estraverse": "^5.2.0", + "esutils": "^2.0.2" + }, + "bin": { + "escodegen": "bin/escodegen.js", + "esgenerate": "bin/esgenerate.js" + }, + "engines": { + "node": ">=6.0" + }, + "optionalDependencies": { + "source-map": "~0.6.1" + } + }, "node_modules/eslint": { "version": "8.36.0", "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.36.0.tgz", @@ -9537,6 +10478,15 @@ "url": "https://github.com/sindresorhus/execa?sponsor=1" } }, + "node_modules/expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", + "dev": true, + "engines": { + "node": ">=6" + } + }, "node_modules/express": { "version": "4.18.2", "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz", @@ -9652,9 +10602,9 @@ "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" }, "node_modules/fast-fifo": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.1.0.tgz", - "integrity": "sha512-Kl29QoNbNvn4nhDsLYjyIAaIqaJB6rBx5p3sL9VjaefJ+eMFBWVZiaoguaoZfzEKr5RhAti0UgM8703akGPJ6g==" + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==" }, "node_modules/fast-glob": { "version": "3.2.12", @@ -10058,6 +11008,12 @@ } ] }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "dev": true + }, "node_modules/fs-extra": { "version": "10.1.0", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz", @@ -10315,6 +11271,12 @@ "assert-plus": "^1.0.0" } }, + "node_modules/github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", + "dev": true + }, "node_modules/glob-parent": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", @@ -11203,6 +12165,18 @@ "minimalistic-crypto-utils": "^1.0.1" } }, + "node_modules/html-encoding-sniffer": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-2.0.1.tgz", + "integrity": "sha512-D5JbOMBIR/TVZkubHT+OyT2705QvogUW4IBn6nHd756OwieSF9aDYFj4dv6HHEVGYbHaLETa3WggZYWWMyy3ZQ==", + "dev": true, + "dependencies": { + "whatwg-encoding": "^1.0.5" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/html-escaper": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", @@ -11262,6 +12236,20 @@ "node": ">=6.0.0" } }, + "node_modules/http-proxy-agent": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz", + "integrity": "sha512-k0zdNgqWTGA6aeIRVpvfVob4fL52dTfaehylg0Y4UvSySvOq/Y+BOyPrgpUrA7HylqvU8vIZGsRuXmspskV0Tg==", + "dev": true, + "dependencies": { + "@tootallnate/once": "1", + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/http-signature": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", @@ -11494,6 +12482,15 @@ "node": ">= 0.4" } }, + "node_modules/internmap": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/internmap/-/internmap-2.0.3.tgz", + "integrity": "sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==", + "dev": true, + "engines": { + "node": ">=12" + } + }, "node_modules/io-ts": { "version": "1.10.4", "resolved": "https://registry.npmjs.org/io-ts/-/io-ts-1.10.4.tgz", @@ -11858,6 +12855,12 @@ "node": ">=8" } }, + "node_modules/is-potential-custom-element-name": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz", + "integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==", + "dev": true + }, "node_modules/is-property": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz", @@ -12395,6 +13398,144 @@ "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-1.1.0.tgz", "integrity": "sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==" }, + "node_modules/jsdom": { + "version": "16.7.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-16.7.0.tgz", + "integrity": "sha512-u9Smc2G1USStM+s/x1ru5Sxrl6mPYCbByG1U/hUmqaVsm4tbNyS7CicOSRyuGQYZhTu0h84qkZZQ/I+dzizSVw==", + "dev": true, + "dependencies": { + "abab": "^2.0.5", + "acorn": "^8.2.4", + "acorn-globals": "^6.0.0", + "cssom": "^0.4.4", + "cssstyle": "^2.3.0", + "data-urls": "^2.0.0", + "decimal.js": "^10.2.1", + "domexception": "^2.0.1", + "escodegen": "^2.0.0", + "form-data": "^3.0.0", + "html-encoding-sniffer": "^2.0.1", + "http-proxy-agent": "^4.0.1", + "https-proxy-agent": "^5.0.0", + "is-potential-custom-element-name": "^1.0.1", + "nwsapi": "^2.2.0", + "parse5": "6.0.1", + "saxes": "^5.0.1", + "symbol-tree": "^3.2.4", + "tough-cookie": "^4.0.0", + "w3c-hr-time": "^1.0.2", + "w3c-xmlserializer": "^2.0.0", + "webidl-conversions": "^6.1.0", + "whatwg-encoding": "^1.0.5", + "whatwg-mimetype": "^2.3.0", + "whatwg-url": "^8.5.0", + "ws": "^7.4.6", + "xml-name-validator": "^3.0.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "canvas": "^2.5.0" + }, + "peerDependenciesMeta": { + "canvas": { + "optional": true + } + } + }, + "node_modules/jsdom/node_modules/saxes": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/saxes/-/saxes-5.0.1.tgz", + "integrity": "sha512-5LBh1Tls8c9xgGjw3QrMwETmTMVk0oFgvrFSvWx62llR2hcEInrKNZ2GZCCuuy2lvWrdl5jhbpeqc5hRYKFOcw==", + "dev": true, + "dependencies": { + "xmlchars": "^2.2.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/jsdom/node_modules/tough-cookie": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.3.tgz", + "integrity": "sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==", + "dev": true, + "dependencies": { + "psl": "^1.1.33", + "punycode": "^2.1.1", + "universalify": "^0.2.0", + "url-parse": "^1.5.3" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/jsdom/node_modules/tr46": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-2.1.0.tgz", + "integrity": "sha512-15Ih7phfcdP5YxqiB+iDtLoaTz4Nd35+IiAv0kQ5FNKHzXgdWqPoTIqEDDJmXceQt4JZk6lVPT8lnDlPpGDppw==", + "dev": true, + "dependencies": { + "punycode": "^2.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/jsdom/node_modules/universalify": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz", + "integrity": "sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==", + "dev": true, + "engines": { + "node": ">= 4.0.0" + } + }, + "node_modules/jsdom/node_modules/webidl-conversions": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-6.1.0.tgz", + "integrity": "sha512-qBIvFLGiBpLjfwmYAaHPXsn+ho5xZnGvyGvsarywGNc8VyQJUMHJ8OBKGGrPER0okBeMDaan4mNBlgBROxuI8w==", + "dev": true, + "engines": { + "node": ">=10.4" + } + }, + "node_modules/jsdom/node_modules/whatwg-url": { + "version": "8.7.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-8.7.0.tgz", + "integrity": "sha512-gAojqb/m9Q8a5IV96E3fHJM70AzCkgt4uXYX2O7EmuyOnLrViCQlsEBmF9UQIu3/aeAIp2U17rtbpZWNntQqdg==", + "dev": true, + "dependencies": { + "lodash": "^4.7.0", + "tr46": "^2.1.0", + "webidl-conversions": "^6.1.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/jsdom/node_modules/ws": { + "version": "7.5.9", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz", + "integrity": "sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==", + "dev": true, + "engines": { + "node": ">=8.3.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": "^5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, "node_modules/jsesc": { "version": "2.5.2", "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz", @@ -13850,6 +14991,12 @@ "mkdirp": "bin/cmd.js" } }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", + "dev": true + }, "node_modules/mkdirp-promise": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/mkdirp-promise/-/mkdirp-promise-5.0.1.tgz", @@ -14318,6 +15465,12 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, + "node_modules/napi-build-utils": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", + "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==", + "dev": true + }, "node_modules/napi-macros": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/napi-macros/-/napi-macros-2.2.2.tgz", @@ -14442,6 +15595,18 @@ "node": ">= 10.13" } }, + "node_modules/node-abi": { + "version": "3.54.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.54.0.tgz", + "integrity": "sha512-p7eGEiQil0YUV3ItH4/tBb781L5impVmmx2E9FRKF7d18XXzp4PGT2tdYMFY6wQqgxD0IwNZOiSJ0/K0fSi/OA==", + "dev": true, + "dependencies": { + "semver": "^7.3.5" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/node-addon-api": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-2.0.2.tgz", @@ -14567,6 +15732,12 @@ "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.6.tgz", "integrity": "sha512-XWwnNNFCuuSQ0m3r3C4LE3EiORltHd9M05pq6FOlVeiophzRbMo50Sbz1ehl8K3Z+jw9+vmgnXefY1hz8X+2wA==" }, + "node_modules/nwsapi": { + "version": "2.2.7", + "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.7.tgz", + "integrity": "sha512-ub5E4+FBPKwAZx0UwIQOjYWGHTEq5sPqHQNRN8Z9e4A7u3Tj1weLJsL59yH9vmvqEtBHaOmT6cYQKIZOxp35FQ==", + "dev": true + }, "node_modules/nyc": { "version": "15.1.0", "resolved": "https://registry.npmjs.org/nyc/-/nyc-15.1.0.tgz", @@ -15235,6 +16406,12 @@ "integrity": "sha512-ft3iAoLOB/MlwbNXgzy43SWGP6sQki2jQvAyBg/zDFAgr9bfNWZIUj42Kw2eJIl8kEi4PbgE6U1Zau/HwI75HA==", "dev": true }, + "node_modules/parse5": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-6.0.1.tgz", + "integrity": "sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw==", + "dev": true + }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -15535,6 +16712,134 @@ "node": ">=12.0.0" } }, + "node_modules/prebuild-install": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz", + "integrity": "sha512-jAXscXWMcCK8GgCoHOfIr0ODh5ai8mj63L2nWrjuAgXE6tDyYGnx4/8o/rCgU+B4JSyZBKbeZqzhtwtC3ovxjw==", + "dev": true, + "dependencies": { + "detect-libc": "^2.0.0", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp-classic": "^0.5.3", + "napi-build-utils": "^1.0.1", + "node-abi": "^3.3.0", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^4.0.0", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0" + }, + "bin": { + "prebuild-install": "bin.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/prebuild-install/node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "dev": true, + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/prebuild-install/node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/prebuild-install/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/prebuild-install/node_modules/simple-get": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", + "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "decompress-response": "^6.0.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, + "node_modules/prebuild-install/node_modules/tar-fs": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", + "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==", + "dev": true, + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/prebuild-install/node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "dev": true, + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -15740,6 +17045,12 @@ "node": ">=0.10.0" } }, + "node_modules/querystringify": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", + "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==", + "dev": true + }, "node_modules/queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", @@ -15759,6 +17070,12 @@ } ] }, + "node_modules/queue-tick": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/queue-tick/-/queue-tick-1.0.1.tgz", + "integrity": "sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==", + "dev": true + }, "node_modules/quick-format-unescaped": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz", @@ -16259,6 +17576,12 @@ "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==", "dev": true }, + "node_modules/requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==", + "dev": true + }, "node_modules/resolve": { "version": "1.22.1", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.1.tgz", @@ -16432,6 +17755,12 @@ "rlp": "bin/rlp" } }, + "node_modules/robust-predicates": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/robust-predicates/-/robust-predicates-3.0.2.tgz", + "integrity": "sha512-IXgzBWvWQwE6PrDI05OvmXUIruQTcoMDzRsOd5CDvHCVLcLHMTSYvOK5Cm46kWqlV3yAbuSpBZdJ5oP5OUoStg==", + "dev": true + }, "node_modules/rolling-rate-limiter": { "version": "0.2.13", "resolved": "https://registry.npmjs.org/rolling-rate-limiter/-/rolling-rate-limiter-0.2.13.tgz", @@ -16493,6 +17822,12 @@ "resolved": "https://registry.npmjs.org/rustbn.js/-/rustbn.js-0.2.0.tgz", "integrity": "sha512-4VlvkRUuCJvr2J6Y0ImW7NvTCriMi7ErOAqWk1y69vAdoNIzCF3yPmgeNzx+RQTLEDFq5sHfscn1MwHxP9hNfA==" }, + "node_modules/rw": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/rw/-/rw-1.3.3.tgz", + "integrity": "sha512-PdhdWy89SiZogBLaw42zdeqtRJ//zFd2PgQavcICDUgJT5oW10QCRKbJ6bg4r0/UY2M6BWd5tkxuGFRvCkgfHQ==", + "dev": true + }, "node_modules/rxjs": { "version": "7.8.0", "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.0.tgz", @@ -16629,9 +17964,9 @@ "integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==" }, "node_modules/semver": { - "version": "7.5.2", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.2.tgz", - "integrity": "sha512-SoftuTROv/cRjCze/scjGyiDtcUyxw1rgYQSZY7XTmtR5hX+dm76iDbTH8TkLPHCQmlbQVSSbNZCPM2hb0knnQ==", + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", + "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", "dependencies": { "lru-cache": "^6.0.0" }, @@ -16858,6 +18193,73 @@ "rdf-string-ttl": "^1.3.2" } }, + "node_modules/sharp": { + "version": "0.32.6", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.32.6.tgz", + "integrity": "sha512-KyLTWwgcR9Oe4d9HwCwNM2l7+J0dUQwn/yf7S0EnTtb0eVS4RxO0eUSvxPtzT4F3SY+C4K6fqdv/DO27sJ/v/w==", + "dev": true, + "hasInstallScript": true, + "dependencies": { + "color": "^4.2.3", + "detect-libc": "^2.0.2", + "node-addon-api": "^6.1.0", + "prebuild-install": "^7.1.1", + "semver": "^7.5.4", + "simple-get": "^4.0.1", + "tar-fs": "^3.0.4", + "tunnel-agent": "^0.6.0" + }, + "engines": { + "node": ">=14.15.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/sharp/node_modules/color": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz", + "integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==", + "dev": true, + "dependencies": { + "color-convert": "^2.0.1", + "color-string": "^1.9.0" + }, + "engines": { + "node": ">=12.5.0" + } + }, + "node_modules/sharp/node_modules/node-addon-api": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-6.1.0.tgz", + "integrity": "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==", + "dev": true + }, + "node_modules/sharp/node_modules/simple-get": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", + "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "decompress-response": "^6.0.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -17445,6 +18847,16 @@ "node": ">=10.0.0" } }, + "node_modules/streamx": { + "version": "2.15.6", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.15.6.tgz", + "integrity": "sha512-q+vQL4AAz+FdfT137VF69Cc/APqUbxy+MDOImRrMvchJpigHj9GksgDU2LYbO9rx7RX6osWgxJB2WxhYv4SZAw==", + "dev": true, + "dependencies": { + "fast-fifo": "^1.1.0", + "queue-tick": "^1.0.1" + } + }, "node_modules/strict-uri-encode": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz", @@ -17793,6 +19205,12 @@ "node": ">= 4.0.0" } }, + "node_modules/symbol-tree": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz", + "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==", + "dev": true + }, "node_modules/tar": { "version": "4.4.19", "resolved": "https://registry.npmjs.org/tar/-/tar-4.4.19.tgz", @@ -17811,6 +19229,28 @@ "node": ">=4.5" } }, + "node_modules/tar-fs": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.4.tgz", + "integrity": "sha512-5AFQU8b9qLfZCX9zp2duONhPmZv0hGYiBPJsyUdqMjzq/mqVpy/rEUSeHk1+YitmxugaptgBh5oDGU3VsAJq4w==", + "dev": true, + "dependencies": { + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^3.1.5" + } + }, + "node_modules/tar-stream": { + "version": "3.1.6", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.6.tgz", + "integrity": "sha512-B/UyjYwPpMBv+PaFSWAmtYjwdrlEaZQEhMIBFNC5oEG8lpiW8XjcSdmEaClj28ArfKScKHs2nshz3k2le6crsg==", + "dev": true, + "dependencies": { + "b4a": "^1.6.4", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + }, "node_modules/test-exclude": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", @@ -18439,6 +19879,16 @@ "resolved": "https://registry.npmjs.org/uritemplate/-/uritemplate-0.3.4.tgz", "integrity": "sha512-enADBvHfhjrwxFMTVWeIIYz51SZ91uC6o2MR/NQTVljJB6HTZ8eQL3Q7JBj3RxNISA14MOwJaU3vpf5R6dyxHA==" }, + "node_modules/url-parse": { + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", + "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", + "dev": true, + "dependencies": { + "querystringify": "^2.1.1", + "requires-port": "^1.0.0" + } + }, "node_modules/url-set-query": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/url-set-query/-/url-set-query-1.0.0.tgz", @@ -18565,6 +20015,28 @@ "node": ">=0.6.0" } }, + "node_modules/w3c-hr-time": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/w3c-hr-time/-/w3c-hr-time-1.0.2.tgz", + "integrity": "sha512-z8P5DvDNjKDoFIHK7q8r8lackT6l+jo/Ye3HOle7l9nICP9lf1Ci25fy9vHd0JOWewkIFzXIEig3TdKT7JQ5fQ==", + "deprecated": "Use your platform's native performance.now() and performance.timeOrigin.", + "dev": true, + "dependencies": { + "browser-process-hrtime": "^1.0.0" + } + }, + "node_modules/w3c-xmlserializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-2.0.0.tgz", + "integrity": "sha512-4tzD0mF8iSiMiNs30BiLO3EpfGLZUT2MSX/G+o7ZywDzliWQ3OPtTZ0PTC3B3ca1UAf4cJMHB+2Bf56EriJuRA==", + "dev": true, + "dependencies": { + "xml-name-validator": "^3.0.0" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/web-streams-polyfill": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz", @@ -19010,6 +20482,21 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, + "node_modules/whatwg-encoding": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-1.0.5.tgz", + "integrity": "sha512-b5lim54JOPN9HtzvK9HFXvBma/rnfFeqsic0hSpjtDbVxR3dJKLc+KB4V6GgiGOvl7CY/KNh8rxSo9DKQrnUEw==", + "dev": true, + "dependencies": { + "iconv-lite": "0.4.24" + } + }, + "node_modules/whatwg-mimetype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz", + "integrity": "sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g==", + "dev": true + }, "node_modules/whatwg-url": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", @@ -19268,6 +20755,12 @@ "xhr-request": "^1.1.0" } }, + "node_modules/xml-name-validator": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz", + "integrity": "sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==", + "dev": true + }, "node_modules/xml2js": { "version": "0.5.0", "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.5.0.tgz", @@ -23922,6 +25415,12 @@ "integrity": "sha512-TL1adzq1HdxUf9WYduLcQ/DNGYiz71U31QRgbnr0Ef1cPyOUOsBojxHVWpFeOSUucB6Lrs0LxFRA14ntgtkc9w==", "dev": true }, + "@tootallnate/once": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz", + "integrity": "sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==", + "dev": true + }, "@tsconfig/node10": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.9.tgz", @@ -24177,6 +25676,12 @@ "resolved": "https://registry.npmjs.org/@vascosantos/moving-average/-/moving-average-1.1.0.tgz", "integrity": "sha512-MVEJ4vWAPNbrGLjz7ITnHYg+YXZ6ijAqtH5/cHwSoCpbvuJ98aLXwFfPKAUfZpJMQR5uXB58UJajbY130IRF/w==" }, + "abab": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.6.tgz", + "integrity": "sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA==", + "dev": true + }, "abort-controller": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", @@ -24227,6 +25732,30 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.2.tgz", "integrity": "sha512-xjIYgE8HBrkpd/sJqOGNspf8uHG+NOHGOw6a/Urj8taM2EXfdNAH2oFcPeIFfsv3+kz/mJrS5VuMqbNLjCa2vw==" }, + "acorn-globals": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/acorn-globals/-/acorn-globals-6.0.0.tgz", + "integrity": "sha512-ZQl7LOWaF5ePqqcX4hLuv/bLXYQNfNWw2c0/yX/TsPRKamzHcTGQnlCjHT3TsmkOUVEPS3crCxiPfdzE/Trlhg==", + "dev": true, + "requires": { + "acorn": "^7.1.1", + "acorn-walk": "^7.1.1" + }, + "dependencies": { + "acorn": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz", + "integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==", + "dev": true + }, + "acorn-walk": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-7.2.0.tgz", + "integrity": "sha512-OPdCF6GsMIP+Az+aWfAAOEt2/+iVDKE7oy6lJ098aoe59oAmK76qV6Gw60SbZ8jHuG2wH058GF4pLFbYamYrVA==", + "dev": true + } + } + }, "acorn-jsx": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", @@ -24643,6 +26172,12 @@ "deep-equal": "^2.0.5" } }, + "b4a": { + "version": "1.6.4", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.4.tgz", + "integrity": "sha512-fpWrvyVHEKyeEvbKZTVOeZF3VSKKWtJxFIxX/jaVPf+cLbGUSitjb49pHLqPV2BUNNZ0LcoeEGfE/YCpyDYHIw==", + "dev": true + }, "balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -24829,6 +26364,12 @@ "run-parallel-limit": "^1.1.0" } }, + "browser-process-hrtime": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz", + "integrity": "sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow==", + "dev": true + }, "browser-stdout": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz", @@ -25554,6 +27095,29 @@ "resolved": "https://registry.npmjs.org/crypto-js/-/crypto-js-3.3.0.tgz", "integrity": "sha512-DIT51nX0dCfKltpRiXV+/TVZq+Qq2NgF4644+K7Ttnla7zEzqc+kjJyiB96BHNyUTBxyjzRcZYpUdZa+QAqi6Q==" }, + "cssom": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.4.4.tgz", + "integrity": "sha512-p3pvU7r1MyyqbTk+WbNJIgJjG2VmTIaB10rI93LzVPrmDJKkzKYMtxxyAvQXR/NS6otuzveI7+7BBq3SjBS2mw==", + "dev": true + }, + "cssstyle": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-2.3.0.tgz", + "integrity": "sha512-AZL67abkUzIuvcHqk7c09cezpGNcxUxU4Ioi/05xHk4DQeTkWmGYftIE6ctU6AEt+Gn4n1lDStOtj7FKycP71A==", + "dev": true, + "requires": { + "cssom": "~0.3.6" + }, + "dependencies": { + "cssom": { + "version": "0.3.8", + "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.3.8.tgz", + "integrity": "sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg==", + "dev": true + } + } + }, "d": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/d/-/d-1.0.1.tgz", @@ -25563,6 +27127,631 @@ "type": "^1.0.1" } }, + "d3": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/d3/-/d3-7.8.5.tgz", + "integrity": "sha512-JgoahDG51ncUfJu6wX/1vWQEqOflgXyl4MaHqlcSruTez7yhaRKR9i8VjjcQGeS2en/jnFivXuaIMnseMMt0XA==", + "dev": true, + "requires": { + "d3-array": "3", + "d3-axis": "3", + "d3-brush": "3", + "d3-chord": "3", + "d3-color": "3", + "d3-contour": "4", + "d3-delaunay": "6", + "d3-dispatch": "3", + "d3-drag": "3", + "d3-dsv": "3", + "d3-ease": "3", + "d3-fetch": "3", + "d3-force": "3", + "d3-format": "3", + "d3-geo": "3", + "d3-hierarchy": "3", + "d3-interpolate": "3", + "d3-path": "3", + "d3-polygon": "3", + "d3-quadtree": "3", + "d3-random": "3", + "d3-scale": "4", + "d3-scale-chromatic": "3", + "d3-selection": "3", + "d3-shape": "3", + "d3-time": "3", + "d3-time-format": "4", + "d3-timer": "3", + "d3-transition": "3", + "d3-zoom": "3" + } + }, + "d3-array": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-3.2.4.tgz", + "integrity": "sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==", + "dev": true, + "requires": { + "internmap": "1 - 2" + } + }, + "d3-axis": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-axis/-/d3-axis-3.0.0.tgz", + "integrity": "sha512-IH5tgjV4jE/GhHkRV0HiVYPDtvfjHQlQfJHs0usq7M30XcSBvOotpmH1IgkcXsO/5gEQZD43B//fc7SRT5S+xw==", + "dev": true + }, + "d3-brush": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-brush/-/d3-brush-3.0.0.tgz", + "integrity": "sha512-ALnjWlVYkXsVIGlOsuWH1+3udkYFI48Ljihfnh8FZPF2QS9o+PzGLBslO0PjzVoHLZ2KCVgAM8NVkXPJB2aNnQ==", + "dev": true, + "requires": { + "d3-dispatch": "1 - 3", + "d3-drag": "2 - 3", + "d3-interpolate": "1 - 3", + "d3-selection": "3", + "d3-transition": "3" + } + }, + "d3-chord": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-chord/-/d3-chord-3.0.1.tgz", + "integrity": "sha512-VE5S6TNa+j8msksl7HwjxMHDM2yNK3XCkusIlpX5kwauBfXuyLAtNg9jCp/iHH61tgI4sb6R/EIMWCqEIdjT/g==", + "dev": true, + "requires": { + "d3-path": "1 - 3" + } + }, + "d3-collection": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/d3-collection/-/d3-collection-1.0.7.tgz", + "integrity": "sha512-ii0/r5f4sjKNTfh84Di+DpztYwqKhEyUlKoPrzUFfeSkWxjW49xU2QzO9qrPrNkpdI0XJkfzvmTu8V2Zylln6A==", + "dev": true + }, + "d3-color": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-3.1.0.tgz", + "integrity": "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==", + "dev": true + }, + "d3-contour": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/d3-contour/-/d3-contour-4.0.2.tgz", + "integrity": "sha512-4EzFTRIikzs47RGmdxbeUvLWtGedDUNkTcmzoeyg4sP/dvCexO47AaQL7VKy/gul85TOxw+IBgA8US2xwbToNA==", + "dev": true, + "requires": { + "d3-array": "^3.2.0" + } + }, + "d3-delaunay": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/d3-delaunay/-/d3-delaunay-6.0.4.tgz", + "integrity": "sha512-mdjtIZ1XLAM8bm/hx3WwjfHt6Sggek7qH043O8KEjDXN40xi3vx/6pYSVTwLjEgiXQTbvaouWKynLBiUZ6SK6A==", + "dev": true, + "requires": { + "delaunator": "5" + } + }, + "d3-dispatch": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-dispatch/-/d3-dispatch-3.0.1.tgz", + "integrity": "sha512-rzUyPU/S7rwUflMyLc1ETDeBj0NRuHKKAcvukozwhshr6g6c5d8zh4c2gQjY2bZ0dXeGLWc1PF174P2tVvKhfg==", + "dev": true + }, + "d3-drag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-drag/-/d3-drag-3.0.0.tgz", + "integrity": "sha512-pWbUJLdETVA8lQNJecMxoXfH6x+mO2UQo8rSmZ+QqxcbyA3hfeprFgIT//HW2nlHChWeIIMwS2Fq+gEARkhTkg==", + "dev": true, + "requires": { + "d3-dispatch": "1 - 3", + "d3-selection": "3" + } + }, + "d3-dsv": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-dsv/-/d3-dsv-3.0.1.tgz", + "integrity": "sha512-UG6OvdI5afDIFP9w4G0mNq50dSOsXHJaRE8arAS5o9ApWnIElp8GZw1Dun8vP8OyHOZ/QJUKUJwxiiCCnUwm+Q==", + "dev": true, + "requires": { + "commander": "7", + "iconv-lite": "0.6", + "rw": "1" + }, + "dependencies": { + "commander": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", + "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==", + "dev": true + }, + "iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "dev": true, + "requires": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + } + } + } + }, + "d3-ease": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-ease/-/d3-ease-3.0.1.tgz", + "integrity": "sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==", + "dev": true + }, + "d3-fetch": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-fetch/-/d3-fetch-3.0.1.tgz", + "integrity": "sha512-kpkQIM20n3oLVBKGg6oHrUchHM3xODkTzjMoj7aWQFq5QEM+R6E4WkzT5+tojDY7yjez8KgCBRoj4aEr99Fdqw==", + "dev": true, + "requires": { + "d3-dsv": "1 - 3" + } + }, + "d3-force": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-force/-/d3-force-3.0.0.tgz", + "integrity": "sha512-zxV/SsA+U4yte8051P4ECydjD/S+qeYtnaIyAs9tgHCqfguma/aAQDjo85A9Z6EKhBirHRJHXIgJUlffT4wdLg==", + "dev": true, + "requires": { + "d3-dispatch": "1 - 3", + "d3-quadtree": "1 - 3", + "d3-timer": "1 - 3" + } + }, + "d3-format": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-format/-/d3-format-3.1.0.tgz", + "integrity": "sha512-YyUI6AEuY/Wpt8KWLgZHsIU86atmikuoOmCfommt0LYHiQSPjvX2AcFc38PX0CBpr2RCyZhjex+NS/LPOv6YqA==", + "dev": true + }, + "d3-geo": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-geo/-/d3-geo-3.1.0.tgz", + "integrity": "sha512-JEo5HxXDdDYXCaWdwLRt79y7giK8SbhZJbFWXqbRTolCHFI5jRqteLzCsq51NKbUoX0PjBVSohxrx+NoOUujYA==", + "dev": true, + "requires": { + "d3-array": "2.5.0 - 3" + } + }, + "d3-hierarchy": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/d3-hierarchy/-/d3-hierarchy-3.1.2.tgz", + "integrity": "sha512-FX/9frcub54beBdugHjDCdikxThEqjnR93Qt7PvQTOHxyiNCAlvMrHhclk3cD5VeAaq9fxmfRp+CnWw9rEMBuA==", + "dev": true + }, + "d3-interpolate": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-3.0.1.tgz", + "integrity": "sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==", + "dev": true, + "requires": { + "d3-color": "1 - 3" + } + }, + "d3-node": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-node/-/d3-node-3.0.0.tgz", + "integrity": "sha512-JuuBToljSQBo+KHSuPVOEdKawsxuLDlrB8TZY8AkPkpvyKaGgX+g0ksgjRg+Mgat9fB77qFQrk09j72cD0yk7w==", + "dev": true, + "requires": { + "d3": "^5.16.0", + "jsdom": "^16.7.0" + }, + "dependencies": { + "commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", + "dev": true + }, + "d3": { + "version": "5.16.0", + "resolved": "https://registry.npmjs.org/d3/-/d3-5.16.0.tgz", + "integrity": "sha512-4PL5hHaHwX4m7Zr1UapXW23apo6pexCgdetdJ5kTmADpG/7T9Gkxw0M0tf/pjoB63ezCCm0u5UaFYy2aMt0Mcw==", + "dev": true, + "requires": { + "d3-array": "1", + "d3-axis": "1", + "d3-brush": "1", + "d3-chord": "1", + "d3-collection": "1", + "d3-color": "1", + "d3-contour": "1", + "d3-dispatch": "1", + "d3-drag": "1", + "d3-dsv": "1", + "d3-ease": "1", + "d3-fetch": "1", + "d3-force": "1", + "d3-format": "1", + "d3-geo": "1", + "d3-hierarchy": "1", + "d3-interpolate": "1", + "d3-path": "1", + "d3-polygon": "1", + "d3-quadtree": "1", + "d3-random": "1", + "d3-scale": "2", + "d3-scale-chromatic": "1", + "d3-selection": "1", + "d3-shape": "1", + "d3-time": "1", + "d3-time-format": "2", + "d3-timer": "1", + "d3-transition": "1", + "d3-voronoi": "1", + "d3-zoom": "1" + } + }, + "d3-array": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-1.2.4.tgz", + "integrity": "sha512-KHW6M86R+FUPYGb3R5XiYjXPq7VzwxZ22buHhAEVG5ztoEcZZMLov530mmccaqA1GghZArjQV46fuc8kUqhhHw==", + "dev": true + }, + "d3-axis": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/d3-axis/-/d3-axis-1.0.12.tgz", + "integrity": "sha512-ejINPfPSNdGFKEOAtnBtdkpr24c4d4jsei6Lg98mxf424ivoDP2956/5HDpIAtmHo85lqT4pruy+zEgvRUBqaQ==", + "dev": true + }, + "d3-brush": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/d3-brush/-/d3-brush-1.1.6.tgz", + "integrity": "sha512-7RW+w7HfMCPyZLifTz/UnJmI5kdkXtpCbombUSs8xniAyo0vIbrDzDwUJB6eJOgl9u5DQOt2TQlYumxzD1SvYA==", + "dev": true, + "requires": { + "d3-dispatch": "1", + "d3-drag": "1", + "d3-interpolate": "1", + "d3-selection": "1", + "d3-transition": "1" + } + }, + "d3-chord": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/d3-chord/-/d3-chord-1.0.6.tgz", + "integrity": "sha512-JXA2Dro1Fxw9rJe33Uv+Ckr5IrAa74TlfDEhE/jfLOaXegMQFQTAgAw9WnZL8+HxVBRXaRGCkrNU7pJeylRIuA==", + "dev": true, + "requires": { + "d3-array": "1", + "d3-path": "1" + } + }, + "d3-color": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-1.4.1.tgz", + "integrity": "sha512-p2sTHSLCJI2QKunbGb7ocOh7DgTAn8IrLx21QRc/BSnodXM4sv6aLQlnfpvehFMLZEfBc6g9pH9SWQccFYfJ9Q==", + "dev": true + }, + "d3-contour": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/d3-contour/-/d3-contour-1.3.2.tgz", + "integrity": "sha512-hoPp4K/rJCu0ladiH6zmJUEz6+u3lgR+GSm/QdM2BBvDraU39Vr7YdDCicJcxP1z8i9B/2dJLgDC1NcvlF8WCg==", + "dev": true, + "requires": { + "d3-array": "^1.1.1" + } + }, + "d3-dispatch": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/d3-dispatch/-/d3-dispatch-1.0.6.tgz", + "integrity": "sha512-fVjoElzjhCEy+Hbn8KygnmMS7Or0a9sI2UzGwoB7cCtvI1XpVN9GpoYlnb3xt2YV66oXYb1fLJ8GMvP4hdU1RA==", + "dev": true + }, + "d3-drag": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/d3-drag/-/d3-drag-1.2.5.tgz", + "integrity": "sha512-rD1ohlkKQwMZYkQlYVCrSFxsWPzI97+W+PaEIBNTMxRuxz9RF0Hi5nJWHGVJ3Om9d2fRTe1yOBINJyy/ahV95w==", + "dev": true, + "requires": { + "d3-dispatch": "1", + "d3-selection": "1" + } + }, + "d3-dsv": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/d3-dsv/-/d3-dsv-1.2.0.tgz", + "integrity": "sha512-9yVlqvZcSOMhCYzniHE7EVUws7Fa1zgw+/EAV2BxJoG3ME19V6BQFBwI855XQDsxyOuG7NibqRMTtiF/Qup46g==", + "dev": true, + "requires": { + "commander": "2", + "iconv-lite": "0.4", + "rw": "1" + } + }, + "d3-ease": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/d3-ease/-/d3-ease-1.0.7.tgz", + "integrity": "sha512-lx14ZPYkhNx0s/2HX5sLFUI3mbasHjSSpwO/KaaNACweVwxUruKyWVcb293wMv1RqTPZyZ8kSZ2NogUZNcLOFQ==", + "dev": true + }, + "d3-fetch": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/d3-fetch/-/d3-fetch-1.2.0.tgz", + "integrity": "sha512-yC78NBVcd2zFAyR/HnUiBS7Lf6inSCoWcSxFfw8FYL7ydiqe80SazNwoffcqOfs95XaLo7yebsmQqDKSsXUtvA==", + "dev": true, + "requires": { + "d3-dsv": "1" + } + }, + "d3-force": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/d3-force/-/d3-force-1.2.1.tgz", + "integrity": "sha512-HHvehyaiUlVo5CxBJ0yF/xny4xoaxFxDnBXNvNcfW9adORGZfyNF1dj6DGLKyk4Yh3brP/1h3rnDzdIAwL08zg==", + "dev": true, + "requires": { + "d3-collection": "1", + "d3-dispatch": "1", + "d3-quadtree": "1", + "d3-timer": "1" + } + }, + "d3-format": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/d3-format/-/d3-format-1.4.5.tgz", + "integrity": "sha512-J0piedu6Z8iB6TbIGfZgDzfXxUFN3qQRMofy2oPdXzQibYGqPB/9iMcxr/TGalU+2RsyDO+U4f33id8tbnSRMQ==", + "dev": true + }, + "d3-geo": { + "version": "1.12.1", + "resolved": "https://registry.npmjs.org/d3-geo/-/d3-geo-1.12.1.tgz", + "integrity": "sha512-XG4d1c/UJSEX9NfU02KwBL6BYPj8YKHxgBEw5om2ZnTRSbIcego6dhHwcxuSR3clxh0EpE38os1DVPOmnYtTPg==", + "dev": true, + "requires": { + "d3-array": "1" + } + }, + "d3-hierarchy": { + "version": "1.1.9", + "resolved": "https://registry.npmjs.org/d3-hierarchy/-/d3-hierarchy-1.1.9.tgz", + "integrity": "sha512-j8tPxlqh1srJHAtxfvOUwKNYJkQuBFdM1+JAUfq6xqH5eAqf93L7oG1NVqDa4CpFZNvnNKtCYEUC8KY9yEn9lQ==", + "dev": true + }, + "d3-interpolate": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-1.4.0.tgz", + "integrity": "sha512-V9znK0zc3jOPV4VD2zZn0sDhZU3WAE2bmlxdIwwQPPzPjvyLkd8B3JUVdS1IDUFDkWZ72c9qnv1GK2ZagTZ8EA==", + "dev": true, + "requires": { + "d3-color": "1" + } + }, + "d3-path": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/d3-path/-/d3-path-1.0.9.tgz", + "integrity": "sha512-VLaYcn81dtHVTjEHd8B+pbe9yHWpXKZUC87PzoFmsFrJqgFwDe/qxfp5MlfsfM1V5E/iVt0MmEbWQ7FVIXh/bg==", + "dev": true + }, + "d3-polygon": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/d3-polygon/-/d3-polygon-1.0.6.tgz", + "integrity": "sha512-k+RF7WvI08PC8reEoXa/w2nSg5AUMTi+peBD9cmFc+0ixHfbs4QmxxkarVal1IkVkgxVuk9JSHhJURHiyHKAuQ==", + "dev": true + }, + "d3-quadtree": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/d3-quadtree/-/d3-quadtree-1.0.7.tgz", + "integrity": "sha512-RKPAeXnkC59IDGD0Wu5mANy0Q2V28L+fNe65pOCXVdVuTJS3WPKaJlFHer32Rbh9gIo9qMuJXio8ra4+YmIymA==", + "dev": true + }, + "d3-random": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/d3-random/-/d3-random-1.1.2.tgz", + "integrity": "sha512-6AK5BNpIFqP+cx/sreKzNjWbwZQCSUatxq+pPRmFIQaWuoD+NrbVWw7YWpHiXpCQ/NanKdtGDuB+VQcZDaEmYQ==", + "dev": true + }, + "d3-scale": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/d3-scale/-/d3-scale-2.2.2.tgz", + "integrity": "sha512-LbeEvGgIb8UMcAa0EATLNX0lelKWGYDQiPdHj+gLblGVhGLyNbaCn3EvrJf0A3Y/uOOU5aD6MTh5ZFCdEwGiCw==", + "dev": true, + "requires": { + "d3-array": "^1.2.0", + "d3-collection": "1", + "d3-format": "1", + "d3-interpolate": "1", + "d3-time": "1", + "d3-time-format": "2" + } + }, + "d3-scale-chromatic": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/d3-scale-chromatic/-/d3-scale-chromatic-1.5.0.tgz", + "integrity": "sha512-ACcL46DYImpRFMBcpk9HhtIyC7bTBR4fNOPxwVSl0LfulDAwyiHyPOTqcDG1+t5d4P9W7t/2NAuWu59aKko/cg==", + "dev": true, + "requires": { + "d3-color": "1", + "d3-interpolate": "1" + } + }, + "d3-selection": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-1.4.2.tgz", + "integrity": "sha512-SJ0BqYihzOjDnnlfyeHT0e30k0K1+5sR3d5fNueCNeuhZTnGw4M4o8mqJchSwgKMXCNFo+e2VTChiSJ0vYtXkg==", + "dev": true + }, + "d3-shape": { + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-1.3.7.tgz", + "integrity": "sha512-EUkvKjqPFUAZyOlhY5gzCxCeI0Aep04LwIRpsZ/mLFelJiUfnK56jo5JMDSE7yyP2kLSb6LtF+S5chMk7uqPqw==", + "dev": true, + "requires": { + "d3-path": "1" + } + }, + "d3-time": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/d3-time/-/d3-time-1.1.0.tgz", + "integrity": "sha512-Xh0isrZ5rPYYdqhAVk8VLnMEidhz5aP7htAADH6MfzgmmicPkTo8LhkLxci61/lCB7n7UmE3bN0leRt+qvkLxA==", + "dev": true + }, + "d3-time-format": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/d3-time-format/-/d3-time-format-2.3.0.tgz", + "integrity": "sha512-guv6b2H37s2Uq/GefleCDtbe0XZAuy7Wa49VGkPVPMfLL9qObgBST3lEHJBMUp8S7NdLQAGIvr2KXk8Hc98iKQ==", + "dev": true, + "requires": { + "d3-time": "1" + } + }, + "d3-timer": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/d3-timer/-/d3-timer-1.0.10.tgz", + "integrity": "sha512-B1JDm0XDaQC+uvo4DT79H0XmBskgS3l6Ve+1SBCfxgmtIb1AVrPIoqd+nPSv+loMX8szQ0sVUhGngL7D5QPiXw==", + "dev": true + }, + "d3-transition": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/d3-transition/-/d3-transition-1.3.2.tgz", + "integrity": "sha512-sc0gRU4PFqZ47lPVHloMn9tlPcv8jxgOQg+0zjhfZXMQuvppjG6YuwdMBE0TuqCZjeJkLecku/l9R0JPcRhaDA==", + "dev": true, + "requires": { + "d3-color": "1", + "d3-dispatch": "1", + "d3-ease": "1", + "d3-interpolate": "1", + "d3-selection": "^1.1.0", + "d3-timer": "1" + } + }, + "d3-zoom": { + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/d3-zoom/-/d3-zoom-1.8.3.tgz", + "integrity": "sha512-VoLXTK4wvy1a0JpH2Il+F2CiOhVu7VRXWF5M/LroMIh3/zBAC3WAt7QoIvPibOavVo20hN6/37vwAsdBejLyKQ==", + "dev": true, + "requires": { + "d3-dispatch": "1", + "d3-drag": "1", + "d3-interpolate": "1", + "d3-selection": "1", + "d3-transition": "1" + } + } + } + }, + "d3-path": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-path/-/d3-path-3.1.0.tgz", + "integrity": "sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ==", + "dev": true + }, + "d3-polygon": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-polygon/-/d3-polygon-3.0.1.tgz", + "integrity": "sha512-3vbA7vXYwfe1SYhED++fPUQlWSYTTGmFmQiany/gdbiWgU/iEyQzyymwL9SkJjFFuCS4902BSzewVGsHHmHtXg==", + "dev": true + }, + "d3-quadtree": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-quadtree/-/d3-quadtree-3.0.1.tgz", + "integrity": "sha512-04xDrxQTDTCFwP5H6hRhsRcb9xxv2RzkcsygFzmkSIOJy3PeRJP7sNk3VRIbKXcog561P9oU0/rVH6vDROAgUw==", + "dev": true + }, + "d3-random": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-random/-/d3-random-3.0.1.tgz", + "integrity": "sha512-FXMe9GfxTxqd5D6jFsQ+DJ8BJS4E/fT5mqqdjovykEB2oFbTMDVdg1MGFxfQW+FBOGoB++k8swBrgwSHT1cUXQ==", + "dev": true + }, + "d3-scale": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/d3-scale/-/d3-scale-4.0.2.tgz", + "integrity": "sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ==", + "dev": true, + "requires": { + "d3-array": "2.10.0 - 3", + "d3-format": "1 - 3", + "d3-interpolate": "1.2.0 - 3", + "d3-time": "2.1.1 - 3", + "d3-time-format": "2 - 4" + } + }, + "d3-scale-chromatic": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-scale-chromatic/-/d3-scale-chromatic-3.0.0.tgz", + "integrity": "sha512-Lx9thtxAKrO2Pq6OO2Ua474opeziKr279P/TKZsMAhYyNDD3EnCffdbgeSYN5O7m2ByQsxtuP2CSDczNUIZ22g==", + "dev": true, + "requires": { + "d3-color": "1 - 3", + "d3-interpolate": "1 - 3" + } + }, + "d3-selection": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz", + "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", + "dev": true + }, + "d3-shape": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-3.2.0.tgz", + "integrity": "sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA==", + "dev": true, + "requires": { + "d3-path": "^3.1.0" + } + }, + "d3-time": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-time/-/d3-time-3.1.0.tgz", + "integrity": "sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q==", + "dev": true, + "requires": { + "d3-array": "2 - 3" + } + }, + "d3-time-format": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/d3-time-format/-/d3-time-format-4.1.0.tgz", + "integrity": "sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg==", + "dev": true, + "requires": { + "d3-time": "1 - 3" + } + }, + "d3-timer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-timer/-/d3-timer-3.0.1.tgz", + "integrity": "sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==", + "dev": true + }, + "d3-transition": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-transition/-/d3-transition-3.0.1.tgz", + "integrity": "sha512-ApKvfjsSR6tg06xrL434C0WydLr7JewBB3V+/39RMHsaXTOG0zmt/OAXeng5M5LBm0ojmxJrpomQVZ1aPvBL4w==", + "dev": true, + "requires": { + "d3-color": "1 - 3", + "d3-dispatch": "1 - 3", + "d3-ease": "1 - 3", + "d3-interpolate": "1 - 3", + "d3-timer": "1 - 3" + } + }, + "d3-voronoi": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/d3-voronoi/-/d3-voronoi-1.1.4.tgz", + "integrity": "sha512-dArJ32hchFsrQ8uMiTBLq256MpnZjeuBtdHpaDlYuQyjU0CVzCJl/BVW+SkszaAeH95D/8gxqAhgx0ouAWAfRg==", + "dev": true + }, + "d3-zoom": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-zoom/-/d3-zoom-3.0.0.tgz", + "integrity": "sha512-b8AmV3kfQaqWAuacbPuNbL6vahnOJflOhexLzMMNLga62+/nh0JzvJ0aO/5a5MVgUFGS7Hu1P9P03o3fJkDCyw==", + "dev": true, + "requires": { + "d3-dispatch": "1 - 3", + "d3-drag": "2 - 3", + "d3-interpolate": "1 - 3", + "d3-selection": "2 - 3", + "d3-transition": "2 - 3" + } + }, "damerau-levenshtein": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/damerau-levenshtein/-/damerau-levenshtein-1.0.8.tgz", @@ -25584,6 +27773,45 @@ "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==" }, + "data-urls": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-2.0.0.tgz", + "integrity": "sha512-X5eWTSXO/BJmpdIKCRuKUgSCgAN0OwliVK3yPKbwIWU1Tdw5BRajxlzMidvh+gwko9AfQ9zIj52pzF91Q3YAvQ==", + "dev": true, + "requires": { + "abab": "^2.0.3", + "whatwg-mimetype": "^2.3.0", + "whatwg-url": "^8.0.0" + }, + "dependencies": { + "tr46": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-2.1.0.tgz", + "integrity": "sha512-15Ih7phfcdP5YxqiB+iDtLoaTz4Nd35+IiAv0kQ5FNKHzXgdWqPoTIqEDDJmXceQt4JZk6lVPT8lnDlPpGDppw==", + "dev": true, + "requires": { + "punycode": "^2.1.1" + } + }, + "webidl-conversions": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-6.1.0.tgz", + "integrity": "sha512-qBIvFLGiBpLjfwmYAaHPXsn+ho5xZnGvyGvsarywGNc8VyQJUMHJ8OBKGGrPER0okBeMDaan4mNBlgBROxuI8w==", + "dev": true + }, + "whatwg-url": { + "version": "8.7.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-8.7.0.tgz", + "integrity": "sha512-gAojqb/m9Q8a5IV96E3fHJM70AzCkgt4uXYX2O7EmuyOnLrViCQlsEBmF9UQIu3/aeAIp2U17rtbpZWNntQqdg==", + "dev": true, + "requires": { + "lodash": "^4.7.0", + "tr46": "^2.1.0", + "webidl-conversions": "^6.1.0" + } + } + } + }, "dateformat": { "version": "4.6.3", "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-4.6.3.tgz", @@ -25716,6 +27944,15 @@ "object-keys": "^1.1.1" } }, + "delaunator": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/delaunator/-/delaunator-5.0.0.tgz", + "integrity": "sha512-AyLvtyJdbv/U1GkiS6gUUzclRoAY4Gs75qkMygJJhU75LW4DNuSF2RMzpxs9jw9Oz1BobHjTdkG3zdP55VxAqw==", + "dev": true, + "requires": { + "robust-predicates": "^3.0.0" + } + }, "delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -25736,6 +27973,12 @@ "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==" }, + "detect-libc": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.2.tgz", + "integrity": "sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw==", + "dev": true + }, "diff": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", @@ -26253,6 +28496,23 @@ "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==" }, + "domexception": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/domexception/-/domexception-2.0.1.tgz", + "integrity": "sha512-yxJ2mFy/sibVQlu5qHjOkf9J3K6zgmCxgJ94u2EdvDOV09H+32LtRswEcUsmUWN72pVLOEnTSRaIVVzVQgS0dg==", + "dev": true, + "requires": { + "webidl-conversions": "^5.0.0" + }, + "dependencies": { + "webidl-conversions": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-5.0.0.tgz", + "integrity": "sha512-VlZwKPCkYKxQgeSbH5EyngOmRp7Ww7I9rQLERETtf5ofd9pGeswWiOtogpEO850jziPRarreGxn5QIiTqpb2wA==", + "dev": true + } + } + }, "domhandler": { "version": "5.0.3", "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", @@ -26645,6 +28905,18 @@ "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==" }, + "escodegen": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", + "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==", + "dev": true, + "requires": { + "esprima": "^4.0.1", + "estraverse": "^5.2.0", + "esutils": "^2.0.2", + "source-map": "~0.6.1" + } + }, "eslint": { "version": "8.36.0", "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.36.0.tgz", @@ -27241,6 +29513,12 @@ "strip-final-newline": "^2.0.0" } }, + "expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", + "dev": true + }, "express": { "version": "4.18.2", "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz", @@ -27346,9 +29624,9 @@ "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" }, "fast-fifo": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.1.0.tgz", - "integrity": "sha512-Kl29QoNbNvn4nhDsLYjyIAaIqaJB6rBx5p3sL9VjaefJ+eMFBWVZiaoguaoZfzEKr5RhAti0UgM8703akGPJ6g==" + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==" }, "fast-glob": { "version": "3.2.12", @@ -27644,6 +29922,12 @@ "integrity": "sha512-cHEpEQHUg0f8XdtZCc2ZAhrHzKzT0MrFUTcvx+hfxYu7rGMDc5SKoXFh+n4YigxsHXRzc6OrCshdR1bWH6HHyg==", "dev": true }, + "fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "dev": true + }, "fs-extra": { "version": "10.1.0", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz", @@ -27838,6 +30122,12 @@ "assert-plus": "^1.0.0" } }, + "github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", + "dev": true + }, "glob-parent": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", @@ -28532,6 +30822,15 @@ "minimalistic-crypto-utils": "^1.0.1" } }, + "html-encoding-sniffer": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-2.0.1.tgz", + "integrity": "sha512-D5JbOMBIR/TVZkubHT+OyT2705QvogUW4IBn6nHd756OwieSF9aDYFj4dv6HHEVGYbHaLETa3WggZYWWMyy3ZQ==", + "dev": true, + "requires": { + "whatwg-encoding": "^1.0.5" + } + }, "html-escaper": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", @@ -28578,6 +30877,17 @@ "resolved": "https://registry.npmjs.org/http-link-header/-/http-link-header-1.1.0.tgz", "integrity": "sha512-pj6N1yxOz/ANO8HHsWGg/OoIL1kmRYvQnXQ7PIRpgp+15AnEsRH8fmIJE6D1OdWG2Bov+BJHVla1fFXxg1JbbA==" }, + "http-proxy-agent": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz", + "integrity": "sha512-k0zdNgqWTGA6aeIRVpvfVob4fL52dTfaehylg0Y4UvSySvOq/Y+BOyPrgpUrA7HylqvU8vIZGsRuXmspskV0Tg==", + "dev": true, + "requires": { + "@tootallnate/once": "1", + "agent-base": "6", + "debug": "4" + } + }, "http-signature": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", @@ -28740,6 +31050,12 @@ "side-channel": "^1.0.4" } }, + "internmap": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/internmap/-/internmap-2.0.3.tgz", + "integrity": "sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==", + "dev": true + }, "io-ts": { "version": "1.10.4", "resolved": "https://registry.npmjs.org/io-ts/-/io-ts-1.10.4.tgz", @@ -28983,6 +31299,12 @@ "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-2.1.0.tgz", "integrity": "sha512-YWnfyRwxL/+SsrWYfOpUtz5b3YD+nyfkHvjbcanzk8zgyO4ASD67uVMRt8k5bM4lLMDnXfriRhOpemw+NfT1eA==" }, + "is-potential-custom-element-name": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz", + "integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==", + "dev": true + }, "is-property": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz", @@ -29424,6 +31746,103 @@ "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-1.1.0.tgz", "integrity": "sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==" }, + "jsdom": { + "version": "16.7.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-16.7.0.tgz", + "integrity": "sha512-u9Smc2G1USStM+s/x1ru5Sxrl6mPYCbByG1U/hUmqaVsm4tbNyS7CicOSRyuGQYZhTu0h84qkZZQ/I+dzizSVw==", + "dev": true, + "requires": { + "abab": "^2.0.5", + "acorn": "^8.2.4", + "acorn-globals": "^6.0.0", + "cssom": "^0.4.4", + "cssstyle": "^2.3.0", + "data-urls": "^2.0.0", + "decimal.js": "^10.2.1", + "domexception": "^2.0.1", + "escodegen": "^2.0.0", + "form-data": "^3.0.0", + "html-encoding-sniffer": "^2.0.1", + "http-proxy-agent": "^4.0.1", + "https-proxy-agent": "^5.0.0", + "is-potential-custom-element-name": "^1.0.1", + "nwsapi": "^2.2.0", + "parse5": "6.0.1", + "saxes": "^5.0.1", + "symbol-tree": "^3.2.4", + "tough-cookie": "^4.0.0", + "w3c-hr-time": "^1.0.2", + "w3c-xmlserializer": "^2.0.0", + "webidl-conversions": "^6.1.0", + "whatwg-encoding": "^1.0.5", + "whatwg-mimetype": "^2.3.0", + "whatwg-url": "^8.5.0", + "ws": "^7.4.6", + "xml-name-validator": "^3.0.0" + }, + "dependencies": { + "saxes": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/saxes/-/saxes-5.0.1.tgz", + "integrity": "sha512-5LBh1Tls8c9xgGjw3QrMwETmTMVk0oFgvrFSvWx62llR2hcEInrKNZ2GZCCuuy2lvWrdl5jhbpeqc5hRYKFOcw==", + "dev": true, + "requires": { + "xmlchars": "^2.2.0" + } + }, + "tough-cookie": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.3.tgz", + "integrity": "sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==", + "dev": true, + "requires": { + "psl": "^1.1.33", + "punycode": "^2.1.1", + "universalify": "^0.2.0", + "url-parse": "^1.5.3" + } + }, + "tr46": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-2.1.0.tgz", + "integrity": "sha512-15Ih7phfcdP5YxqiB+iDtLoaTz4Nd35+IiAv0kQ5FNKHzXgdWqPoTIqEDDJmXceQt4JZk6lVPT8lnDlPpGDppw==", + "dev": true, + "requires": { + "punycode": "^2.1.1" + } + }, + "universalify": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz", + "integrity": "sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==", + "dev": true + }, + "webidl-conversions": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-6.1.0.tgz", + "integrity": "sha512-qBIvFLGiBpLjfwmYAaHPXsn+ho5xZnGvyGvsarywGNc8VyQJUMHJ8OBKGGrPER0okBeMDaan4mNBlgBROxuI8w==", + "dev": true + }, + "whatwg-url": { + "version": "8.7.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-8.7.0.tgz", + "integrity": "sha512-gAojqb/m9Q8a5IV96E3fHJM70AzCkgt4uXYX2O7EmuyOnLrViCQlsEBmF9UQIu3/aeAIp2U17rtbpZWNntQqdg==", + "dev": true, + "requires": { + "lodash": "^4.7.0", + "tr46": "^2.1.0", + "webidl-conversions": "^6.1.0" + } + }, + "ws": { + "version": "7.5.9", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz", + "integrity": "sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==", + "dev": true, + "requires": {} + } + } + }, "jsesc": { "version": "2.5.2", "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz", @@ -30582,6 +33001,12 @@ "minimist": "^1.2.6" } }, + "mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", + "dev": true + }, "mkdirp-promise": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/mkdirp-promise/-/mkdirp-promise-5.0.1.tgz", @@ -30956,6 +33381,12 @@ "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.4.tgz", "integrity": "sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==" }, + "napi-build-utils": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", + "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==", + "dev": true + }, "napi-macros": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/napi-macros/-/napi-macros-2.2.2.tgz", @@ -31067,6 +33498,15 @@ "propagate": "^2.0.0" } }, + "node-abi": { + "version": "3.54.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.54.0.tgz", + "integrity": "sha512-p7eGEiQil0YUV3ItH4/tBb781L5impVmmx2E9FRKF7d18XXzp4PGT2tdYMFY6wQqgxD0IwNZOiSJ0/K0fSi/OA==", + "dev": true, + "requires": { + "semver": "^7.3.5" + } + }, "node-addon-api": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-2.0.2.tgz", @@ -31147,6 +33587,12 @@ } } }, + "nwsapi": { + "version": "2.2.7", + "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.7.tgz", + "integrity": "sha512-ub5E4+FBPKwAZx0UwIQOjYWGHTEq5sPqHQNRN8Z9e4A7u3Tj1weLJsL59yH9vmvqEtBHaOmT6cYQKIZOxp35FQ==", + "dev": true + }, "nyc": { "version": "15.1.0", "resolved": "https://registry.npmjs.org/nyc/-/nyc-15.1.0.tgz", @@ -31641,6 +34087,12 @@ "integrity": "sha512-ft3iAoLOB/MlwbNXgzy43SWGP6sQki2jQvAyBg/zDFAgr9bfNWZIUj42Kw2eJIl8kEi4PbgE6U1Zau/HwI75HA==", "dev": true }, + "parse5": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-6.0.1.tgz", + "integrity": "sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw==", + "dev": true + }, "parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -31873,6 +34325,96 @@ "resolved": "https://registry.npmjs.org/pony-cause/-/pony-cause-2.1.8.tgz", "integrity": "sha512-c4E5sVlX+npoRiKb72SkMOYMZ/c+jehaKAytybykhh49tEQYkjHxKSZ3YHUyH1v3oEL+wl3aXtkyYhJp+cNXDg==" }, + "prebuild-install": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz", + "integrity": "sha512-jAXscXWMcCK8GgCoHOfIr0ODh5ai8mj63L2nWrjuAgXE6tDyYGnx4/8o/rCgU+B4JSyZBKbeZqzhtwtC3ovxjw==", + "dev": true, + "requires": { + "detect-libc": "^2.0.0", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp-classic": "^0.5.3", + "napi-build-utils": "^1.0.1", + "node-abi": "^3.3.0", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^4.0.0", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0" + }, + "dependencies": { + "bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "dev": true, + "requires": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "dev": true, + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + }, + "simple-get": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", + "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", + "dev": true, + "requires": { + "decompress-response": "^6.0.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, + "tar-fs": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", + "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==", + "dev": true, + "requires": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "dev": true, + "requires": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + } + } + } + }, "prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -32033,11 +34575,23 @@ "strict-uri-encode": "^1.0.0" } }, + "querystringify": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", + "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==", + "dev": true + }, "queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==" }, + "queue-tick": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/queue-tick/-/queue-tick-1.0.1.tgz", + "integrity": "sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==", + "dev": true + }, "quick-format-unescaped": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz", @@ -32469,6 +35023,12 @@ "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==", "dev": true }, + "requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==", + "dev": true + }, "resolve": { "version": "1.22.1", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.1.tgz", @@ -32602,6 +35162,12 @@ "bn.js": "^5.2.0" } }, + "robust-predicates": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/robust-predicates/-/robust-predicates-3.0.2.tgz", + "integrity": "sha512-IXgzBWvWQwE6PrDI05OvmXUIruQTcoMDzRsOd5CDvHCVLcLHMTSYvOK5Cm46kWqlV3yAbuSpBZdJ5oP5OUoStg==", + "dev": true + }, "rolling-rate-limiter": { "version": "0.2.13", "resolved": "https://registry.npmjs.org/rolling-rate-limiter/-/rolling-rate-limiter-0.2.13.tgz", @@ -32632,6 +35198,12 @@ "resolved": "https://registry.npmjs.org/rustbn.js/-/rustbn.js-0.2.0.tgz", "integrity": "sha512-4VlvkRUuCJvr2J6Y0ImW7NvTCriMi7ErOAqWk1y69vAdoNIzCF3yPmgeNzx+RQTLEDFq5sHfscn1MwHxP9hNfA==" }, + "rw": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/rw/-/rw-1.3.3.tgz", + "integrity": "sha512-PdhdWy89SiZogBLaw42zdeqtRJ//zFd2PgQavcICDUgJT5oW10QCRKbJ6bg4r0/UY2M6BWd5tkxuGFRvCkgfHQ==", + "dev": true + }, "rxjs": { "version": "7.8.0", "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.0.tgz", @@ -32743,9 +35315,9 @@ "integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==" }, "semver": { - "version": "7.5.2", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.2.tgz", - "integrity": "sha512-SoftuTROv/cRjCze/scjGyiDtcUyxw1rgYQSZY7XTmtR5hX+dm76iDbTH8TkLPHCQmlbQVSSbNZCPM2hb0knnQ==", + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", + "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", "requires": { "lru-cache": "^6.0.0" }, @@ -32916,6 +35488,51 @@ "rdf-string-ttl": "^1.3.2" } }, + "sharp": { + "version": "0.32.6", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.32.6.tgz", + "integrity": "sha512-KyLTWwgcR9Oe4d9HwCwNM2l7+J0dUQwn/yf7S0EnTtb0eVS4RxO0eUSvxPtzT4F3SY+C4K6fqdv/DO27sJ/v/w==", + "dev": true, + "requires": { + "color": "^4.2.3", + "detect-libc": "^2.0.2", + "node-addon-api": "^6.1.0", + "prebuild-install": "^7.1.1", + "semver": "^7.5.4", + "simple-get": "^4.0.1", + "tar-fs": "^3.0.4", + "tunnel-agent": "^0.6.0" + }, + "dependencies": { + "color": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz", + "integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==", + "dev": true, + "requires": { + "color-convert": "^2.0.1", + "color-string": "^1.9.0" + } + }, + "node-addon-api": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-6.1.0.tgz", + "integrity": "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==", + "dev": true + }, + "simple-get": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", + "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", + "dev": true, + "requires": { + "decompress-response": "^6.0.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + } + } + }, "shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -33391,6 +36008,16 @@ "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==" }, + "streamx": { + "version": "2.15.6", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.15.6.tgz", + "integrity": "sha512-q+vQL4AAz+FdfT137VF69Cc/APqUbxy+MDOImRrMvchJpigHj9GksgDU2LYbO9rx7RX6osWgxJB2WxhYv4SZAw==", + "dev": true, + "requires": { + "fast-fifo": "^1.1.0", + "queue-tick": "^1.0.1" + } + }, "strict-uri-encode": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz", @@ -33649,6 +36276,12 @@ } } }, + "symbol-tree": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz", + "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==", + "dev": true + }, "tar": { "version": "4.4.19", "resolved": "https://registry.npmjs.org/tar/-/tar-4.4.19.tgz", @@ -33664,6 +36297,28 @@ "yallist": "^3.1.1" } }, + "tar-fs": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.4.tgz", + "integrity": "sha512-5AFQU8b9qLfZCX9zp2duONhPmZv0hGYiBPJsyUdqMjzq/mqVpy/rEUSeHk1+YitmxugaptgBh5oDGU3VsAJq4w==", + "dev": true, + "requires": { + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^3.1.5" + } + }, + "tar-stream": { + "version": "3.1.6", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.6.tgz", + "integrity": "sha512-B/UyjYwPpMBv+PaFSWAmtYjwdrlEaZQEhMIBFNC5oEG8lpiW8XjcSdmEaClj28ArfKScKHs2nshz3k2le6crsg==", + "dev": true, + "requires": { + "b4a": "^1.6.4", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + }, "test-exclude": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", @@ -34156,6 +36811,16 @@ "resolved": "https://registry.npmjs.org/uritemplate/-/uritemplate-0.3.4.tgz", "integrity": "sha512-enADBvHfhjrwxFMTVWeIIYz51SZ91uC6o2MR/NQTVljJB6HTZ8eQL3Q7JBj3RxNISA14MOwJaU3vpf5R6dyxHA==" }, + "url-parse": { + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", + "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", + "dev": true, + "requires": { + "querystringify": "^2.1.1", + "requires-port": "^1.0.0" + } + }, "url-set-query": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/url-set-query/-/url-set-query-1.0.0.tgz", @@ -34259,6 +36924,24 @@ "extsprintf": "^1.2.0" } }, + "w3c-hr-time": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/w3c-hr-time/-/w3c-hr-time-1.0.2.tgz", + "integrity": "sha512-z8P5DvDNjKDoFIHK7q8r8lackT6l+jo/Ye3HOle7l9nICP9lf1Ci25fy9vHd0JOWewkIFzXIEig3TdKT7JQ5fQ==", + "dev": true, + "requires": { + "browser-process-hrtime": "^1.0.0" + } + }, + "w3c-xmlserializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-2.0.0.tgz", + "integrity": "sha512-4tzD0mF8iSiMiNs30BiLO3EpfGLZUT2MSX/G+o7ZywDzliWQ3OPtTZ0PTC3B3ca1UAf4cJMHB+2Bf56EriJuRA==", + "dev": true, + "requires": { + "xml-name-validator": "^3.0.0" + } + }, "web-streams-polyfill": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz", @@ -34645,6 +37328,21 @@ } } }, + "whatwg-encoding": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-1.0.5.tgz", + "integrity": "sha512-b5lim54JOPN9HtzvK9HFXvBma/rnfFeqsic0hSpjtDbVxR3dJKLc+KB4V6GgiGOvl7CY/KNh8rxSo9DKQrnUEw==", + "dev": true, + "requires": { + "iconv-lite": "0.4.24" + } + }, + "whatwg-mimetype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz", + "integrity": "sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g==", + "dev": true + }, "whatwg-url": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", @@ -34850,6 +37548,12 @@ "xhr-request": "^1.1.0" } }, + "xml-name-validator": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz", + "integrity": "sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==", + "dev": true + }, "xml2js": { "version": "0.5.0", "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.5.0.tgz", diff --git a/package.json b/package.json index 36d67b1463..b6b88d8776 100644 --- a/package.json +++ b/package.json @@ -48,6 +48,8 @@ "devDependencies": { "@cucumber/cucumber": "^8.5.2", "chai": "^4.3.6", + "d3": "^7.8.5", + "d3-node": "^3.0.0", "dkg.js": "^6.0.2", "eslint": "^8.23.0", "eslint-config-airbnb": "^19.0.4", @@ -57,6 +59,7 @@ "mocha": "^10.0.0", "nyc": "^15.1.0", "prettier": "^2.7.1", + "sharp": "^0.32.6", "sinon": "^14.0.0", "slugify": "^1.6.5" }, diff --git a/src/constants/constants.js b/src/constants/constants.js index 052f17df4c..3c66be43d7 100644 --- a/src/constants/constants.js +++ b/src/constants/constants.js @@ -10,8 +10,12 @@ export const RPC_PROVIDER_STALL_TIMEOUT = 60 * 1000; export const UINT256_MAX_BN = BigNumber.from(2).pow(256).sub(1); +export const UINT128_MAX_BN = BigNumber.from(2).pow(128).sub(1); + export const UINT32_MAX_BN = BigNumber.from(2).pow(32).sub(1); +export const HASH_RING_SIZE = BigNumber.from(2).pow(256); + export const STAKE_UINT256_MULTIPLIER_BN = UINT256_MAX_BN.div(500000000); export const UINT256_UINT32_DIVISOR_BN = UINT256_MAX_BN.div(UINT32_MAX_BN); diff --git a/src/service/hashing-service.js b/src/service/hashing-service.js new file mode 100644 index 0000000000..0a0da3fab5 --- /dev/null +++ b/src/service/hashing-service.js @@ -0,0 +1,31 @@ +import { ethers } from 'ethers'; + +class HashingService { + constructor(ctx) { + this.config = ctx.config; + this.logger = ctx.logger; + + this.hashFunctions = { + 1: 'sha256', + }; + } + + async callHashFunction(hashFunctionId, data) { + const hashFunctionName = this.getHashFunctionName(hashFunctionId); + return this[hashFunctionName](data); + } + + getHashFunctionName(hashFunctionId) { + return this.hashFunctions[hashFunctionId]; + } + + async sha256(data) { + if (!ethers.utils.isBytesLike(data)) { + const bytesLikeData = ethers.utils.toUtf8Bytes(data); + return ethers.utils.sha256(bytesLikeData); + } + return ethers.utils.sha256(data); + } +} + +export default HashingService; diff --git a/src/service/proximity-scoring-service.js b/src/service/proximity-scoring-service.js new file mode 100644 index 0000000000..b47d923b35 --- /dev/null +++ b/src/service/proximity-scoring-service.js @@ -0,0 +1,148 @@ +import { xor as uint8ArrayXor } from 'uint8arrays/xor'; +import { HASH_RING_SIZE, UINT128_MAX_BN } from '../constants/constants.js'; + +class ProximityScoringService { + constructor(ctx) { + this.config = ctx.config; + this.logger = ctx.logger; + + this.blockchainModuleManager = ctx.blockchainModuleManager; + + this.proximityScoreFunctionsPairs = { + 1: [this.calculateBinaryXOR.bind(this), this.Log2PLDSF.bind(this)], + 2: [this.calculateProximityOnHashRing.bind(this), this.LinearLogisticSum.bind(this)], + }; + } + + async callProximityFunction( + blockchain, + proximityFunctionId, + peerHash, + keyHash, + ...additionalArgs + ) { + return this.proximityScoreFunctionsPairs[proximityFunctionId][0]( + blockchain, + peerHash, + keyHash, + ...additionalArgs, + ); + } + + async callScoreFunction(blockchain, scoreFunctionId, distance, stake, ...additionalArgs) { + return this.proximityScoreFunctionsPairs[scoreFunctionId][1]( + blockchain, + distance, + stake, + ...additionalArgs, + ); + } + + async calculateBinaryXOR(blockchain, peerHash, keyHash) { + const distance = uint8ArrayXor( + this.blockchainModuleManager.convertBytesToUint8Array(blockchain, peerHash), + this.blockchainModuleManager.convertBytesToUint8Array(blockchain, keyHash), + ); + + const distanceHex = await this.blockchainModuleManager.convertUint8ArrayToHex( + blockchain, + distance, + ); + + return this.blockchainModuleManager.toBigNumber(blockchain, distanceHex); + } + + async calculateProximityOnHashRing(blockchain, peerHash, keyHash) { + const peerPositionOnHashRing = await this.blockchainModuleManager.toBigNumber( + blockchain, + peerHash, + ); + const keyPositionOnHashRing = await this.blockchainModuleManager.toBigNumber( + blockchain, + keyHash, + ); + + let directDistance = peerPositionOnHashRing.sub(keyPositionOnHashRing); + + if (directDistance.lt(0)) { + directDistance = directDistance.add(HASH_RING_SIZE); + } + + const wraparoundDistance = HASH_RING_SIZE.sub(directDistance); + + return directDistance.lt(wraparoundDistance) ? directDistance : wraparoundDistance; + } + + async Log2PLDSF(blockchain, distance, stake) { + const log2PLDSFParams = await this.blockchainModuleManager.getLog2PLDSFParams(blockchain); + + const { + distanceMappingCoefficient, + stakeMappingCoefficient, + multiplier, + logArgumentConstant, + a, + stakeExponent, + b, + c, + distanceExponent, + d, + } = log2PLDSFParams; + + const mappedStake = this.blockchainModuleManager + .toBigNumber(blockchain, this.blockchainModuleManager.convertToWei(blockchain, stake)) + .div(stakeMappingCoefficient); + + const mappedDistance = distance.div(distanceMappingCoefficient); + + const dividend = mappedStake.pow(stakeExponent).mul(a).add(b); + const divisor = mappedDistance.pow(distanceExponent).mul(c).add(d); + + return Math.floor( + Number(multiplier) * + Math.log2(Number(logArgumentConstant) + dividend.toNumber() / divisor.toNumber()), + ); + } + + // Using Maclaurin Series to approximate e^x + _approximateExp(x, precision = 20) { + let xPow = x; + let factorial = 1; + let result = 1; + + for (let i = 1; i <= precision; i += 1) { + factorial *= i; + result += xPow / factorial; + xPow *= x; + } + + return result; + } + + async LinearLogisticSum(blockchain, distance, stake, maxNeighborhoodDistance) { + const linearLogisticSumParams = + await this.blockchainModuleManager.getLinearLogisticSumParams(blockchain); + const { distanceScaleFactor, exponentMultiplier, x0, w1, w2 } = linearLogisticSumParams; + + let dividend = distance; + let divisor = maxNeighborhoodDistance; + if (dividend.gt(UINT128_MAX_BN) || divisor.gt(UINT128_MAX_BN)) { + dividend = dividend.div(distanceScaleFactor); + divisor = divisor.div(distanceScaleFactor); + } + + const divResult = dividend.mul(distanceScaleFactor).div(divisor); + const mappedDistance = + parseFloat(divResult.toString()) / parseFloat(distanceScaleFactor.toString()); + + const exponentPart = exponentMultiplier * (stake - x0); + const mappedStake = 2 / (1 + this._approximateExp(exponentPart)) - 1; + + const proximityScore = w1 * (1 - mappedDistance); + const stakeScore = w2 * mappedStake; + + return proximityScore + stakeScore; + } +} + +export default ProximityScoringService; diff --git a/tools/knowledge-assets-distribution-simulation/mocks/blockchain-module-manager-mock.js b/tools/knowledge-assets-distribution-simulation/mocks/blockchain-module-manager-mock.js new file mode 100644 index 0000000000..c4a260ab52 --- /dev/null +++ b/tools/knowledge-assets-distribution-simulation/mocks/blockchain-module-manager-mock.js @@ -0,0 +1,47 @@ +import { ethers } from 'ethers'; + +class BlockchainModuleManagerMock { + convertBytesToUint8Array(blockchain, bytesLikeData) { + return ethers.utils.arrayify(bytesLikeData); + } + + convertUint8ArrayToHex(blockchain, uint8Array) { + return ethers.utils.hexlify(uint8Array); + } + + convertToWei(blockchain, value, fromUnit = 'ether') { + return ethers.utils.parseUnits(value.toString(), fromUnit).toString(); + } + + toBigNumber(blockchain, value) { + return ethers.BigNumber.from(value); + } + + getLog2PLDSFParams(blockchain) { + return { + distanceMappingCoefficient: + '115792089237316195423570985008687907853269984665640564039457584007913129639', + stakeMappingCoefficient: `${Math.floor(5000000 / 200000)}000000000000000000`, + multiplier: 10000, + logArgumentConstant: 1, + a: 1, + stakeExponent: 1, + b: 0, + c: 1, + distanceExponent: 2, + d: 1, + }; + } + + getLinearLogisticSumParams(blockchain) { + return { + distanceScaleFactor: '1000000000000000000', + exponentMultiplier: -0.000001, + x0: 50000, + w1: 1, + w2: 1, + }; + } +} + +export default BlockchainModuleManagerMock; diff --git a/tools/knowledge-assets-distribution-simulation/plots/.gitkeep b/tools/knowledge-assets-distribution-simulation/plots/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tools/knowledge-assets-distribution-simulation/simulation.js b/tools/knowledge-assets-distribution-simulation/simulation.js new file mode 100644 index 0000000000..8d3f51345d --- /dev/null +++ b/tools/knowledge-assets-distribution-simulation/simulation.js @@ -0,0 +1,416 @@ +/* eslint-disable import/no-extraneous-dependencies */ +/* eslint-disable no-await-in-loop */ +import 'dotenv/config'; +import crypto from 'crypto'; +import DeepExtend from 'deep-extend'; +import rc from 'rc'; +import fs from 'fs-extra'; +import D3Node from 'd3-node'; +import * as d3 from 'd3'; +import sharp from 'sharp'; +import { readFile } from 'fs/promises'; +import { create as createLibP2PKey, createFromPrivKey } from 'peer-id'; +import BlockchainModuleManagerMock from './mocks/blockchain-module-manager-mock.js'; +import HashingService from '../../src/service/hashing-service.js'; +import ProximityScoringService from '../../src/service/proximity-scoring-service.js'; +import Logger from '../../src/logger/logger.js'; +import configjson from '../../config/config.json'; +import pjson from '../../package.json'; + +function getConfig() { + let config; + let userConfig; + + if (process.env.USER_CONFIG_PATH) { + const configurationFilename = process.env.USER_CONFIG_PATH; + const pathSplit = configurationFilename.split('/'); + userConfig = JSON.parse(fs.readFileSync(configurationFilename)); + userConfig.configFilename = pathSplit[pathSplit.length - 1]; + } + + const defaultConfig = JSON.parse(JSON.stringify(configjson[process.env.NODE_ENV])); + + if (userConfig) { + config = DeepExtend(defaultConfig, userConfig); + } else { + config = rc(pjson.name, defaultConfig); + } + + if (!config.configFilename) { + config.configFilename = '.origintrail_noderc'; + } + return config; +} + +function getLogger() { + return new Logger('debug'); +} + +const config = getConfig(); +const logger = getLogger(); +const blockchain = 'hardhat1:31337'; +const blockchainModuleManagerMock = new BlockchainModuleManagerMock(); +const hashingService = new HashingService({ config, logger }); +const proximityScoringService = new ProximityScoringService({ + config, + logger, + blockchainModuleManager: blockchainModuleManagerMock, +}); + +function generateRandomNumber(min, max) { + return Math.floor(Math.random() * (max - min + 1)) + min; +} + +function generateRandomHashes(numberOfHashes) { + const hashes = []; + + for (let i = 0; i < numberOfHashes; i += 1) { + const randomString = crypto.randomBytes(20).toString('hex'); + const hash = `0x${crypto.createHash('sha256').update(randomString).digest('hex')}`; + hashes.push(hash); + } + + return numberOfHashes === 1 ? hashes[0] : hashes; +} + +async function generateRandomNodes(numberOfNodes, hashFunctionId = 1) { + const nodes = []; + + for (let i = 0; i < numberOfNodes; i += 1) { + const libp2pPrivKey = (await createLibP2PKey({ bits: 1024, keyType: 'RSA' })).toJSON() + .privKey; + const nodeId = (await createFromPrivKey(libp2pPrivKey)).toB58String(); + const sha256 = await hashingService.callHashFunction(hashFunctionId, nodeId); + const stake = String(generateRandomNumber(50000, 5000000)); + + nodes.push({ nodeId, sha256, stake }); + } + + return nodes; +} + +async function readJsonFromFile(filePath) { + try { + const data = await readFile(filePath, 'utf8'); + return JSON.parse(data); + } catch (error) { + logger.error(`Error reading file: ${error.stack}`); + throw error; + } +} + +function convertSvgToJpg(svgString, outputImageName) { + sharp(Buffer.from(svgString)) + .jpeg({ quality: 100 }) + .toBuffer() + .then((buffer) => { + fs.writeFile( + `tools/knowledge-assets-distribution-simulation/plots/${outputImageName}.jpg`, + buffer, + (err) => { + if (err) { + logger.error(`Error saving the JPG file: ${err.stack}`); + } else { + logger.info( + `Plot saved as "tools/knowledge-assets-distribution-simulation/plots/${outputImageName}.jpg"`, + ); + } + }, + ); + }) + .catch((error) => { + logger.error('Error converting SVG to JPG:', error); + }); +} + +function generateStakeDistributionPlot(data, outputImageName) { + logger.info('Generating Node-Stake Distribution Plot.'); + + const d3n = new D3Node(); + const margin = { top: 60, right: 30, bottom: 60, left: 90 }; + const width = 2000 - margin.left - margin.right; + const height = 1000 - margin.top - margin.bottom; + + data.sort((a, b) => d3.descending(a.stake, b.stake)); + + const svg = d3n + .createSVG(width + margin.left + margin.right, height + margin.top + margin.bottom) + .append('g') + .attr('transform', `translate(${margin.left},${margin.top})`); + + svg.append('text') + .attr('x', width / 2) + .attr('y', 0 - margin.top / 2) + .attr('text-anchor', 'middle') + .style('font-size', '20px') + .style('text-decoration', 'underline') + .text('Node-Stake Distribution Plot'); + + const x = d3 + .scaleBand() + .range([0, width]) + .domain(data.map((_, i) => i + 1)) + .padding(0.2); + + svg.append('text') + .attr('transform', `translate(${width / 2}, ${height + margin.top - 20})`) + .style('text-anchor', 'middle') + .text('Node Index'); + + const y = d3 + .scaleLinear() + .domain([0, Math.round(d3.max(data.map((node) => node.stake)) * 1.1)]) + .range([height, 0]); + + svg.append('text') + .attr('transform', 'rotate(-90)') + .attr('y', 0 - margin.left) + .attr('x', 0 - height / 2) + .attr('dy', '1em') + .style('text-anchor', 'middle') + .text('Stake'); + + svg.selectAll('rect') + .data(data) + .enter() + .append('rect') + .attr('x', (_, i) => x(i + 1)) + .attr('width', x.bandwidth()) + .attr('y', (d) => y(d.stake)) + .attr('height', (d) => height - y(d.stake)) + .attr('fill', 'steelblue'); + + svg.append('g') + .attr('transform', `translate(0, ${height})`) + .call(d3.axisBottom(x)) + .selectAll('text') + .style('text-anchor', 'end') + .attr('dx', '-.8em') + .attr('dy', '.15em') + .attr('transform', 'rotate(-65)'); + + svg.append('g').call(d3.axisLeft(y)); + + convertSvgToJpg(d3n.svgString(), outputImageName); +} + +function generateScatterPlot(data, metric, outputImageName) { + const d3n = new D3Node(); + const margin = { top: 60, right: 30, bottom: 60, left: 60 }; + const width = 2000 - margin.left - margin.right; + const height = 1000 - margin.top - margin.bottom; + + const svg = d3n + .createSVG(width + margin.left + margin.right, height + margin.top + margin.bottom) + .append('g') + .attr('transform', `translate(${margin.left},${margin.top})`); + + svg.append('text') + .attr('x', width / 2) + .attr('y', 0 - margin.top / 2) + .attr('text-anchor', 'middle') + .style('font-size', '20px') + .style('text-decoration', 'underline') + .text(`Stake / KAs ${metric[0].toUpperCase() + metric.slice(1)} Relation Plot`); + + const minStake = d3.min(data.map((node) => node.stake)); + const maxStake = d3.max(data.map((node) => node.stake)); + + const x = d3 + .scaleLinear() + .domain([Math.round(minStake * 0.9), Math.round(maxStake * 1.1)]) + .range([0, width]); + + svg.append('text') + .attr('transform', `translate(${width / 2}, ${height + margin.top - 20})`) + .style('text-anchor', 'middle') + .text('Stake'); + + const minKAsNum = d3.min(data.map((node) => node[metric])); + const maxKAsNum = d3.max(data.map((node) => node[metric])); + + const y = d3 + .scaleLinear() + .domain([Math.round(minKAsNum * 0.9), Math.round(maxKAsNum * 1.1)]) + .range([height, 0]); + + svg.append('text') + .attr('transform', 'rotate(-90)') + .attr('y', 0 - margin.left) + .attr('x', 0 - height / 2) + .attr('dy', '1em') + .style('text-anchor', 'middle') + .text(`Knowledge Assets ${metric}`); + + svg.append('g').attr('transform', `translate(0, ${height})`).call(d3.axisBottom(x)); + + svg.append('g').call(d3.axisLeft(y).ticks(30)); + + svg.append('g') + .selectAll('dot') + .data(data) + .enter() + .append('circle') + .attr('cx', (d) => x(d.stake)) + .attr('cy', (d) => y(d[metric])) + .attr('r', 5) + .style('fill', '#69b3a2'); + + svg.append('line') + .style('stroke', 'red') + .style('stroke-width', 2) + .style('stroke-dasharray', '3, 3') + .attr('x1', 0) + .attr('y1', y(minKAsNum)) + .attr('x2', width) + .attr('y2', y(minKAsNum)); + + svg.append('line') + .style('stroke', 'red') + .style('stroke-width', 2) + .style('stroke-dasharray', '3, 3') + .attr('x1', 0) + .attr('y1', y(maxKAsNum)) + .attr('x2', width) + .attr('y2', y(maxKAsNum)); + + convertSvgToJpg(d3n.svgString(), outputImageName); +} + +async function runSimulation( + mode, + filePath, + numberOfNodes, + numberOfKAs, + proximityScoreFunctionsPairId, + r0 = 3, + r2 = 20, +) { + let nodes = []; + + if (mode === 'load') { + nodes = await readJsonFromFile(filePath); + } else if (mode === 'generate') { + nodes = await generateRandomNodes(numberOfNodes); + } else { + logger.error(`Invalid mode: ${mode}. Use "load" or "generate".`); + return; + } + + logger.info( + `Running simulation in '${mode}' mode with ${nodes.length} nodes and ${numberOfKAs} KAs.`, + ); + + generateStakeDistributionPlot( + nodes.map((node) => ({ ...node, stake: Number(node.stake) })), + `${mode}-${nodes.length}-${numberOfKAs}-${proximityScoreFunctionsPairId}-nodes-stake-distribution`, + ); + + const knowledgeAssets = await generateRandomHashes(numberOfKAs); + const distances = []; + const replicas = {}; + + for (const node of nodes) { + replicas[node.nodeId] = { + stake: Number(node.stake), + replicated: 0, + won: 0, + }; + } + + for (const key of knowledgeAssets) { + const nodesWithDistances = await Promise.all( + nodes.map(async (node) => { + const distance = await proximityScoringService.callProximityFunction( + blockchain, + proximityScoreFunctionsPairId, + node.sha256, + key, + ); + + distances.push({ nodeId: node.nodeId, assertionId: key, distance }); + + return { ...node, distance }; + }), + ); + + const nodesSortedByDistance = nodesWithDistances + .sort((a, b) => { + if (a.distance.lt(b.distance)) { + return -1; + } + if (a.distance.gt(b.distance)) { + return 1; + } + return 0; + }) + .slice(0, r2); + + const maxDistance = nodesSortedByDistance[nodesSortedByDistance.length - 1].distance; + + for (const node of nodesSortedByDistance) { + replicas[node.nodeId].replicated += 1; + } + + const nodesWithScores = await Promise.all( + nodesSortedByDistance.map(async (node) => { + const score = await proximityScoringService.callScoreFunction( + blockchain, + proximityScoreFunctionsPairId, + node.distance, + node.stake, + maxDistance, + ); + + return { ...node, score }; + }), + ); + + const nodesSortedByScore = nodesWithScores.sort((a, b) => b.score - a.score); + + for (const [index, node] of nodesSortedByScore.entries()) { + if (index < r0) { + replicas[node.nodeId].won += 1; + } else { + break; + } + } + } + + generateScatterPlot( + Object.values(replicas), + 'replicated', + `${mode}-${nodes.length}-${numberOfKAs}-${proximityScoreFunctionsPairId}-stake-replications-relation`, + ); + generateScatterPlot( + Object.values(replicas), + 'won', + `${mode}-${nodes.length}-${numberOfKAs}-${proximityScoreFunctionsPairId}-stake-wins-relation`, + ); +} + +const args = process.argv.slice(2); +const mode = args[0]; +const filePath = mode === 'load' ? args[1] : undefined; +const numberOfNodes = mode === 'generate' ? parseInt(args[1], 10) : undefined; +const numberOfKAs = parseInt(args[2], 10); +const proximityScoreFunctionsPairId = parseInt(args[3], 10); + +if ( + (mode === 'load' && !filePath) || + (mode === 'generate' && numberOfNodes === undefined) || + numberOfKAs === undefined || + proximityScoreFunctionsPairId === undefined +) { + logger.error('Incorrect arguments. Please provide the correct format.'); + logger.error( + 'To load nodes list from the JSON file: node simulate.js load ', + ); + logger.error( + 'To generate random nodes: node simulate.js generate ', + ); +} else { + runSimulation(mode, filePath, numberOfNodes, numberOfKAs, proximityScoreFunctionsPairId).catch( + (error) => logger.error(`Simulation error: ${error.stack}`), + ); +} From 72fdcc39504a4dd94e1c7f880e0c66ec9f904be9 Mon Sep 17 00:00:00 2001 From: Uladzislau Hubar Date: Tue, 16 Jan 2024 11:55:24 +0100 Subject: [PATCH 02/96] Fixed JSON imports and Plots generation in simulation tool --- .../simulation.js | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tools/knowledge-assets-distribution-simulation/simulation.js b/tools/knowledge-assets-distribution-simulation/simulation.js index 8d3f51345d..ab98319de7 100644 --- a/tools/knowledge-assets-distribution-simulation/simulation.js +++ b/tools/knowledge-assets-distribution-simulation/simulation.js @@ -9,13 +9,16 @@ import D3Node from 'd3-node'; import * as d3 from 'd3'; import sharp from 'sharp'; import { readFile } from 'fs/promises'; +import { createRequire } from 'module'; import { create as createLibP2PKey, createFromPrivKey } from 'peer-id'; import BlockchainModuleManagerMock from './mocks/blockchain-module-manager-mock.js'; import HashingService from '../../src/service/hashing-service.js'; import ProximityScoringService from '../../src/service/proximity-scoring-service.js'; import Logger from '../../src/logger/logger.js'; -import configjson from '../../config/config.json'; -import pjson from '../../package.json'; + +const require = createRequire(import.meta.url); +const configjson = require('../../config/config.json'); +const pjson = require('../../package.json'); function getConfig() { let config; @@ -73,7 +76,12 @@ function generateRandomHashes(numberOfHashes) { return numberOfHashes === 1 ? hashes[0] : hashes; } -async function generateRandomNodes(numberOfNodes, hashFunctionId = 1) { +async function generateRandomNodes( + numberOfNodes, + stakeMin = 50000, + stakeMax = 1000000, + hashFunctionId = 1, +) { const nodes = []; for (let i = 0; i < numberOfNodes; i += 1) { @@ -81,7 +89,7 @@ async function generateRandomNodes(numberOfNodes, hashFunctionId = 1) { .privKey; const nodeId = (await createFromPrivKey(libp2pPrivKey)).toB58String(); const sha256 = await hashingService.callHashFunction(hashFunctionId, nodeId); - const stake = String(generateRandomNumber(50000, 5000000)); + const stake = String(generateRandomNumber(stakeMin, stakeMax)); nodes.push({ nodeId, sha256, stake }); } @@ -101,6 +109,7 @@ async function readJsonFromFile(filePath) { function convertSvgToJpg(svgString, outputImageName) { sharp(Buffer.from(svgString)) + .flatten({ background: '#FFFFFF' }) .jpeg({ quality: 100 }) .toBuffer() .then((buffer) => { @@ -242,7 +251,7 @@ function generateScatterPlot(data, metric, outputImageName) { .style('text-anchor', 'middle') .text(`Knowledge Assets ${metric}`); - svg.append('g').attr('transform', `translate(0, ${height})`).call(d3.axisBottom(x)); + svg.append('g').attr('transform', `translate(0, ${height})`).call(d3.axisBottom(x).ticks(15)); svg.append('g').call(d3.axisLeft(y).ticks(30)); From cb4db124b293dd66207014cee2d94c7122c8751e Mon Sep 17 00:00:00 2001 From: Uladzislau Hubar Date: Tue, 16 Jan 2024 12:03:28 +0100 Subject: [PATCH 03/96] Moved Maclaurin Series degree for e^x approximation to be a parameter in the smart contract --- src/service/proximity-scoring-service.js | 9 +++++---- .../mocks/blockchain-module-manager-mock.js | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/service/proximity-scoring-service.js b/src/service/proximity-scoring-service.js index b47d923b35..1cfd4fd439 100644 --- a/src/service/proximity-scoring-service.js +++ b/src/service/proximity-scoring-service.js @@ -105,12 +105,12 @@ class ProximityScoringService { } // Using Maclaurin Series to approximate e^x - _approximateExp(x, precision = 20) { + _approximateExp(x, degree) { let xPow = x; let factorial = 1; let result = 1; - for (let i = 1; i <= precision; i += 1) { + for (let i = 1; i <= degree; i += 1) { factorial *= i; result += xPow / factorial; xPow *= x; @@ -122,7 +122,8 @@ class ProximityScoringService { async LinearLogisticSum(blockchain, distance, stake, maxNeighborhoodDistance) { const linearLogisticSumParams = await this.blockchainModuleManager.getLinearLogisticSumParams(blockchain); - const { distanceScaleFactor, exponentMultiplier, x0, w1, w2 } = linearLogisticSumParams; + const { distanceScaleFactor, exponentMultiplier, maclaurinSeriesDegree, x0, w1, w2 } = + linearLogisticSumParams; let dividend = distance; let divisor = maxNeighborhoodDistance; @@ -136,7 +137,7 @@ class ProximityScoringService { parseFloat(divResult.toString()) / parseFloat(distanceScaleFactor.toString()); const exponentPart = exponentMultiplier * (stake - x0); - const mappedStake = 2 / (1 + this._approximateExp(exponentPart)) - 1; + const mappedStake = 2 / (1 + this._approximateExp(exponentPart, maclaurinSeriesDegree)) - 1; const proximityScore = w1 * (1 - mappedDistance); const stakeScore = w2 * mappedStake; diff --git a/tools/knowledge-assets-distribution-simulation/mocks/blockchain-module-manager-mock.js b/tools/knowledge-assets-distribution-simulation/mocks/blockchain-module-manager-mock.js index c4a260ab52..a4488d398b 100644 --- a/tools/knowledge-assets-distribution-simulation/mocks/blockchain-module-manager-mock.js +++ b/tools/knowledge-assets-distribution-simulation/mocks/blockchain-module-manager-mock.js @@ -37,6 +37,7 @@ class BlockchainModuleManagerMock { return { distanceScaleFactor: '1000000000000000000', exponentMultiplier: -0.000001, + maclaurinSeriesDegree: 20, x0: 50000, w1: 1, w2: 1, From e437ebece9319f964e365763aecf7f234538d601 Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Tue, 16 Jan 2024 14:01:22 +0100 Subject: [PATCH 04/96] Use hashinService insteast validationModuleManager for new functions --- src/migration/pull-sharding-table-migration.js | 9 +++------ src/service/blockchain-event-listener-service.js | 4 ++-- src/service/service-agreement-service.js | 11 ++++------- src/service/sharding-table-service.js | 8 ++++---- 4 files changed, 13 insertions(+), 19 deletions(-) diff --git a/src/migration/pull-sharding-table-migration.js b/src/migration/pull-sharding-table-migration.js index 6a02e219fa..f382fdcae5 100644 --- a/src/migration/pull-sharding-table-migration.js +++ b/src/migration/pull-sharding-table-migration.js @@ -7,12 +7,12 @@ class PullBlockchainShardingTableMigration extends BaseMigration { config, repositoryModuleManager, blockchainModuleManager, - validationModuleManager, + hashingService, ) { super(migrationName, logger, config); this.repositoryModuleManager = repositoryModuleManager; this.blockchainModuleManager = blockchainModuleManager; - this.validationModuleManager = validationModuleManager; + this.hashingService = hashingService; } async executeMigration() { @@ -77,10 +77,7 @@ class PullBlockchainShardingTableMigration extends BaseMigration { peer.stake, 'ether', ), - sha256: await this.validationModuleManager.callHashFunction( - 1, - nodeId, - ), + sha256: await this.hashingService.callHashFunction(1, nodeId), }; }), ), diff --git a/src/service/blockchain-event-listener-service.js b/src/service/blockchain-event-listener-service.js index 39aab4c22b..6bcee0206c 100644 --- a/src/service/blockchain-event-listener-service.js +++ b/src/service/blockchain-event-listener-service.js @@ -18,10 +18,10 @@ class BlockchainEventListenerService { this.logger = ctx.logger; this.blockchainModuleManager = ctx.blockchainModuleManager; this.repositoryModuleManager = ctx.repositoryModuleManager; - this.validationModuleManager = ctx.validationModuleManager; this.tripleStoreService = ctx.tripleStoreService; this.pendingStorageService = ctx.pendingStorageService; this.ualService = ctx.ualService; + this.hashingService = ctx.hashingService; } async initialize() { @@ -303,7 +303,7 @@ class BlockchainEventListenerService { eventData.nodeId, ); - const nodeIdSha256 = await this.validationModuleManager.callHashFunction( + const nodeIdSha256 = await this.hashingService.callHashFunction( // TODO: How to add more hashes? CONTENT_ASSET_HASH_FUNCTION_ID, nodeId, diff --git a/src/service/service-agreement-service.js b/src/service/service-agreement-service.js index d1f0f17ca1..53e59be713 100644 --- a/src/service/service-agreement-service.js +++ b/src/service/service-agreement-service.js @@ -2,15 +2,15 @@ class ServiceAgreementService { constructor(ctx) { this.logger = ctx.logger; - this.validationModuleManager = ctx.validationModuleManager; this.blockchainModuleManager = ctx.blockchainModuleManager; this.repositoryModuleManager = ctx.repositoryModuleManager; this.shardingTableService = ctx.shardingTableService; this.networkModuleManager = ctx.networkModuleManager; + this.hashingService = ctx.hashingService; } async generateId(blockchain, assetTypeContract, tokenId, keyword, hashFunctionId) { - return this.validationModuleManager.callHashFunction( + return this.hashingService.callHashFunction( hashFunctionId, this.blockchainModuleManager.encodePacked( blockchain, @@ -26,12 +26,9 @@ class ServiceAgreementService { async calculateScore(peerId, blockchainId, keyword, hashFunctionId) { const peerRecord = await this.repositoryModuleManager.getPeerRecord(peerId, blockchainId); - const keyHash = await this.validationModuleManager.callHashFunction( - hashFunctionId, - keyword, - ); + const keyHash = await this.hashingService.callHashFunction(hashFunctionId, keyword); - const hashFunctionName = this.validationModuleManager.getHashFunctionName(hashFunctionId); + const hashFunctionName = this.hashingService.getHashFunctionName(hashFunctionId); const distanceUint8Array = this.shardingTableService.calculateDistance( blockchainId, diff --git a/src/service/sharding-table-service.js b/src/service/sharding-table-service.js index 87688c0a42..c8d07eb3d8 100644 --- a/src/service/sharding-table-service.js +++ b/src/service/sharding-table-service.js @@ -14,7 +14,7 @@ class ShardingTableService { this.blockchainModuleManager = ctx.blockchainModuleManager; this.repositoryModuleManager = ctx.repositoryModuleManager; this.networkModuleManager = ctx.networkModuleManager; - this.validationModuleManager = ctx.validationModuleManager; + this.hashingService = ctx.hashingService; this.memoryCachedPeerIds = {}; } @@ -107,7 +107,7 @@ class ShardingTableService { peer.stake, 'ether', ), - sha256: await this.validationModuleManager.callHashFunction(1, nodeId), + sha256: await this.hashingService.callHashFunction(1, nodeId), }; }), ), @@ -119,13 +119,13 @@ class ShardingTableService { blockchainId, filterLastSeen, ); - const keyHash = await this.validationModuleManager.callHashFunction(hashFunctionId, key); + const keyHash = await this.hashingService.callHashFunction(hashFunctionId, key); return this.sortPeers(blockchainId, keyHash, peers, r2, hashFunctionId); } async sortPeers(blockchainId, keyHash, peers, count, hashFunctionId) { - const hashFunctionName = this.validationModuleManager.getHashFunctionName(hashFunctionId); + const hashFunctionName = this.hashingService.getHashFunctionName(hashFunctionId); return peers .map((peer) => ({ From 2c9604483e92cba07ac95ac520c5a973268ba201 Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Tue, 16 Jan 2024 15:35:37 +0100 Subject: [PATCH 05/96] Update sharding-table-service to use proximityScoringService --- src/service/sharding-table-service.js | 59 +++++++++++++++++++-------- 1 file changed, 43 insertions(+), 16 deletions(-) diff --git a/src/service/sharding-table-service.js b/src/service/sharding-table-service.js index c8d07eb3d8..fb4d36c938 100644 --- a/src/service/sharding-table-service.js +++ b/src/service/sharding-table-service.js @@ -1,6 +1,3 @@ -import { xor as uint8ArrayXor } from 'uint8arrays/xor'; -import { compare as uint8ArrayCompare } from 'uint8arrays/compare'; - import { BYTES_IN_KILOBYTE, CONTRACTS, @@ -15,6 +12,7 @@ class ShardingTableService { this.repositoryModuleManager = ctx.repositoryModuleManager; this.networkModuleManager = ctx.networkModuleManager; this.hashingService = ctx.hashingService; + this.proximityScoringService = ctx.proximityScoringService; this.memoryCachedPeerIds = {}; } @@ -114,36 +112,63 @@ class ShardingTableService { ); } - async findNeighbourhood(blockchainId, key, r2, hashFunctionId, filterLastSeen) { + async findNeighbourhood( + blockchainId, + key, + r2, + hashFunctionId, + proximityScoreFunctionsPairId, + filterLastSeen, + ) { const peers = await this.repositoryModuleManager.getAllPeerRecords( blockchainId, filterLastSeen, ); const keyHash = await this.hashingService.callHashFunction(hashFunctionId, key); - return this.sortPeers(blockchainId, keyHash, peers, r2, hashFunctionId); + return this.sortPeers( + blockchainId, + keyHash, + peers, + r2, + hashFunctionId, + proximityScoreFunctionsPairId, + ); } - async sortPeers(blockchainId, keyHash, peers, count, hashFunctionId) { + async sortPeers( + blockchainId, + keyHash, + peers, + count, + hashFunctionId, + proximityScoreFunctionsPairId, + ) { const hashFunctionName = this.hashingService.getHashFunctionName(hashFunctionId); return peers - .map((peer) => ({ + .map(async (peer) => ({ peer, - distance: this.calculateDistance(blockchainId, keyHash, peer[hashFunctionName]), + distance: await this.proximityScoringService.callProximityFunction( + blockchainId, + proximityScoreFunctionsPairId, + peer[hashFunctionName], + keyHash, + ), })) - .sort((a, b) => uint8ArrayCompare(a.distance, b.distance)) + .sort((a, b) => { + if (a.lt(b)) { + return -1; + } + if (a.gt(b)) { + return 1; + } + return 0; + }) .slice(0, count) .map((pd) => pd.peer); } - calculateDistance(blockchain, peerHash, keyHash) { - return uint8ArrayXor( - this.blockchainModuleManager.convertBytesToUint8Array(blockchain, peerHash), - this.blockchainModuleManager.convertBytesToUint8Array(blockchain, keyHash), - ); - } - async getBidSuggestion( blockchainId, epochsNumber, @@ -151,6 +176,7 @@ class ShardingTableService { contentAssetStorageAddress, firstAssertionId, hashFunctionId, + proximityScoreFunctionsPairId, ) { const kbSize = assertionSize < BYTES_IN_KILOBYTE ? BYTES_IN_KILOBYTE : assertionSize; const peerRecords = await this.findNeighbourhood( @@ -162,6 +188,7 @@ class ShardingTableService { ), await this.blockchainModuleManager.getR2(blockchainId), hashFunctionId, + proximityScoreFunctionsPairId, true, ); const r1 = await this.blockchainModuleManager.getR1(blockchainId); From 98c2001e3df97bbdf3c3c4f3bca4668c46374446 Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Tue, 16 Jan 2024 16:09:28 +0100 Subject: [PATCH 06/96] Update findNeighbourhood function calls to use proximityScoreFunctionsPairId --- src/commands/protocols/common/epoch-check-command.js | 3 ++- src/commands/protocols/common/find-nodes-command.js | 9 ++++++++- .../protocols/common/handle-protocol-message-command.js | 9 ++++++++- .../v1.0.0/v1-0-0-handle-update-request-command.js | 3 ++- 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/commands/protocols/common/epoch-check-command.js b/src/commands/protocols/common/epoch-check-command.js index d8da892f21..564a24ef8b 100644 --- a/src/commands/protocols/common/epoch-check-command.js +++ b/src/commands/protocols/common/epoch-check-command.js @@ -239,12 +239,13 @@ class EpochCheckCommand extends Command { ]); } - async calculateRank(blockchain, keyword, hashFunctionId, r2) { + async calculateRank(blockchain, keyword, hashFunctionId, proximityScoreFunctionsPairId, r2) { const neighbourhood = await this.shardingTableService.findNeighbourhood( blockchain, keyword, r2, hashFunctionId, + proximityScoreFunctionsPairId, true, ); diff --git a/src/commands/protocols/common/find-nodes-command.js b/src/commands/protocols/common/find-nodes-command.js index bf9e204301..13b137c5f8 100644 --- a/src/commands/protocols/common/find-nodes-command.js +++ b/src/commands/protocols/common/find-nodes-command.js @@ -66,7 +66,13 @@ class FindNodesCommand extends Command { ); } - async findNodes(blockchainId, keyword, operationId, hashFunctionId) { + async findNodes( + blockchainId, + keyword, + operationId, + hashFunctionId, + proximityScoreFunctionsPairId, + ) { await this.operationIdService.updateOperationIdStatus( operationId, blockchainId, @@ -78,6 +84,7 @@ class FindNodesCommand extends Command { keyword, r2, hashFunctionId, + proximityScoreFunctionsPairId, true, ); diff --git a/src/commands/protocols/common/handle-protocol-message-command.js b/src/commands/protocols/common/handle-protocol-message-command.js index 92b7966697..fb348de771 100644 --- a/src/commands/protocols/common/handle-protocol-message-command.js +++ b/src/commands/protocols/common/handle-protocol-message-command.js @@ -47,12 +47,19 @@ class HandleProtocolMessageCommand extends Command { throw Error('prepareMessage not implemented'); } - async validateNeighborhood(blockchain, keyword, hashFunctionId, ual) { + async validateNeighborhood( + blockchain, + keyword, + hashFunctionId, + proximityScoreFunctionsPairId, + ual, + ) { const closestNodes = await this.shardingTableService.findNeighbourhood( blockchain, keyword, await this.blockchainModuleManager.getR2(blockchain), hashFunctionId, + proximityScoreFunctionsPairId, true, ); const peerId = this.networkModuleManager.getPeerId().toB58String(); diff --git a/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js b/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js index c02ae88649..fb9220c475 100644 --- a/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js +++ b/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js @@ -148,12 +148,13 @@ class HandleUpdateRequestCommand extends HandleProtocolMessageCommand { return delay; } - async calculateRank(blockchain, keyword, hashFunctionId, r2) { + async calculateRank(blockchain, keyword, hashFunctionId, proximityScoreFunctionsPairId, r2) { const neighbourhood = await this.shardingTableService.findNeighbourhood( blockchain, keyword, r2, hashFunctionId, + proximityScoreFunctionsPairId, true, ); From 52567ef506a4ba602ad249f3477cb2cccf5860c6 Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Tue, 16 Jan 2024 16:19:04 +0100 Subject: [PATCH 07/96] Update epoch-check-command --- src/commands/protocols/common/epoch-check-command.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/commands/protocols/common/epoch-check-command.js b/src/commands/protocols/common/epoch-check-command.js index 564a24ef8b..b4873d9524 100644 --- a/src/commands/protocols/common/epoch-check-command.js +++ b/src/commands/protocols/common/epoch-check-command.js @@ -129,6 +129,7 @@ class EpochCheckCommand extends Command { blockchain, serviceAgreement.keyword, serviceAgreement.hashFunctionId, + serviceAgreement.scoreFunctionId, r2, ); @@ -261,6 +262,7 @@ class EpochCheckCommand extends Command { blockchain, keyword, hashFunctionId, + proximityScoreFunctionsPairId, ), peerId: node.peerId, })), From bf7ed5a3f324e1439e1d2f2663dc87a014030ef1 Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Tue, 16 Jan 2024 16:40:02 +0100 Subject: [PATCH 08/96] Update calculateScore function in ServiceAgreementService to use ProximityScoringService --- src/service/service-agreement-service.js | 54 ++++++------------------ 1 file changed, 14 insertions(+), 40 deletions(-) diff --git a/src/service/service-agreement-service.js b/src/service/service-agreement-service.js index 53e59be713..dc68c63e29 100644 --- a/src/service/service-agreement-service.js +++ b/src/service/service-agreement-service.js @@ -7,6 +7,7 @@ class ServiceAgreementService { this.shardingTableService = ctx.shardingTableService; this.networkModuleManager = ctx.networkModuleManager; this.hashingService = ctx.hashingService; + this.proximityScoringService = ctx.proximityScoringService; } async generateId(blockchain, assetTypeContract, tokenId, keyword, hashFunctionId) { @@ -24,57 +25,30 @@ class ServiceAgreementService { return Math.floor(Math.random() * (max - min + 1) + min); } - async calculateScore(peerId, blockchainId, keyword, hashFunctionId) { + async calculateScore( + peerId, + blockchainId, + keyword, + hashFunctionId, + proximityScoreFunctionsPairId, + ) { const peerRecord = await this.repositoryModuleManager.getPeerRecord(peerId, blockchainId); const keyHash = await this.hashingService.callHashFunction(hashFunctionId, keyword); const hashFunctionName = this.hashingService.getHashFunctionName(hashFunctionId); - const distanceUint8Array = this.shardingTableService.calculateDistance( + const distance = await this.proximityScoringService.callProximityFunction( blockchainId, + proximityScoreFunctionsPairId, peerRecord[hashFunctionName], keyHash, ); - // todo: store this in a more appropriate way - if (!this.log2PLDSFParams) { - this.log2PLDSFParams = await this.blockchainModuleManager.getLog2PLDSFParams( - blockchainId, - ); - } - - const { - distanceMappingCoefficient, - stakeMappingCoefficient, - multiplier, - logArgumentConstant, - a, - stakeExponent, - b, - c, - distanceExponent, - d, - } = this.log2PLDSFParams; - - const distanceUint256BN = this.blockchainModuleManager.toBigNumber( + return this.proximityScoringService.callScoreFunction( blockchainId, - distanceUint8Array, - ); - - const mappedStake = this.blockchainModuleManager - .toBigNumber( - blockchainId, - this.blockchainModuleManager.convertToWei(blockchainId, peerRecord.stake), - ) - .div(stakeMappingCoefficient); - const mappedDistance = distanceUint256BN.div(distanceMappingCoefficient); - - const dividend = mappedStake.pow(stakeExponent).mul(a).add(b); - const divisor = mappedDistance.pow(distanceExponent).mul(c).add(d); - - return Math.floor( - Number(multiplier) * - Math.log2(Number(logArgumentConstant) + dividend.toNumber() / divisor.toNumber()), + proximityScoreFunctionsPairId, + distance, + peerRecord.stake, ); } } From 62665ec854eed3ce5d7b2d9d7d0c514adb3842b0 Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Tue, 16 Jan 2024 16:43:34 +0100 Subject: [PATCH 09/96] Update commands to use proximityScoreFunctionsPairId --- src/commands/protocols/common/find-nodes-command.js | 9 ++++++++- .../common/handle-protocol-message-command.js | 11 ++++++++++- .../v1.0.0/v1-0-0-handle-store-init-command.js | 13 +++++++++++-- .../v1.0.0/v1-0-0-handle-update-init-command.js | 13 +++++++++++-- .../v1.0.0/v1-0-0-handle-update-request-command.js | 10 +++++++++- 5 files changed, 49 insertions(+), 7 deletions(-) diff --git a/src/commands/protocols/common/find-nodes-command.js b/src/commands/protocols/common/find-nodes-command.js index 13b137c5f8..a3f6e19e0c 100644 --- a/src/commands/protocols/common/find-nodes-command.js +++ b/src/commands/protocols/common/find-nodes-command.js @@ -22,6 +22,7 @@ class FindNodesCommand extends Command { networkProtocols, hashFunctionId, minAckResponses, + proximityScoreFunctionsPairId, } = command.data; this.errorType = errorType; @@ -29,7 +30,13 @@ class FindNodesCommand extends Command { // TODO: protocol selection const closestNodes = []; - const foundNodes = await this.findNodes(blockchain, keyword, operationId, hashFunctionId); + const foundNodes = await this.findNodes( + blockchain, + keyword, + operationId, + hashFunctionId, + proximityScoreFunctionsPairId, + ); for (const node of foundNodes) { if (node.id !== this.networkModuleManager.getPeerId().toB58String()) { closestNodes.push({ id: node.id, protocol: networkProtocols[0] }); diff --git a/src/commands/protocols/common/handle-protocol-message-command.js b/src/commands/protocols/common/handle-protocol-message-command.js index fb348de771..1da17dcbcc 100644 --- a/src/commands/protocols/common/handle-protocol-message-command.js +++ b/src/commands/protocols/common/handle-protocol-message-command.js @@ -186,11 +186,20 @@ class HandleProtocolMessageCommand extends Command { tokenId, keyword, hashFunctionId, + proximityScoreFunctionsPairId, ) { const ual = this.ualService.deriveUAL(blockchain, contract, tokenId); this.logger.trace(`Validating neighborhood for ual: ${ual}`); - if (!(await this.validateNeighborhood(blockchain, keyword, hashFunctionId, ual))) { + if ( + !(await this.validateNeighborhood( + blockchain, + keyword, + hashFunctionId, + proximityScoreFunctionsPairId, + ual, + )) + ) { return { messageType: NETWORK_MESSAGE_TYPES.RESPONSES.NACK, messageData: { errorMessage: 'Invalid neighbourhood' }, diff --git a/src/commands/protocols/publish/receiver/v1.0.0/v1-0-0-handle-store-init-command.js b/src/commands/protocols/publish/receiver/v1.0.0/v1-0-0-handle-store-init-command.js index d9c529ffe8..77e3bd36c4 100644 --- a/src/commands/protocols/publish/receiver/v1.0.0/v1-0-0-handle-store-init-command.js +++ b/src/commands/protocols/publish/receiver/v1.0.0/v1-0-0-handle-store-init-command.js @@ -11,8 +11,16 @@ class HandleStoreInitCommand extends HandleProtocolMessageCommand { } async prepareMessage(commandData) { - const { operationId, assertionId, blockchain, contract, tokenId, keyword, hashFunctionId } = - commandData; + const { + operationId, + assertionId, + blockchain, + contract, + tokenId, + keyword, + hashFunctionId, + proximityScoreFunctionsPairId, + } = commandData; await this.operationIdService.updateOperationIdStatus( operationId, @@ -28,6 +36,7 @@ class HandleStoreInitCommand extends HandleProtocolMessageCommand { tokenId, keyword, hashFunctionId, + proximityScoreFunctionsPairId, ); this.operationIdService.updateOperationIdStatus( diff --git a/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-init-command.js b/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-init-command.js index ca0c1720ab..d0c814cecb 100644 --- a/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-init-command.js +++ b/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-init-command.js @@ -14,8 +14,16 @@ class HandleUpdateInitCommand extends HandleProtocolMessageCommand { } async prepareMessage(commandData) { - const { operationId, assertionId, blockchain, contract, tokenId, keyword, hashFunctionId } = - commandData; + const { + operationId, + assertionId, + blockchain, + contract, + tokenId, + keyword, + hashFunctionId, + proximityScoreFunctionsPairId, + } = commandData; await this.operationIdService.updateOperationIdStatus( operationId, blockchain, @@ -39,6 +47,7 @@ class HandleUpdateInitCommand extends HandleProtocolMessageCommand { tokenId, keyword, hashFunctionId, + proximityScoreFunctionsPairId, ); await this.operationIdService.updateOperationIdStatus( diff --git a/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js b/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js index fb9220c475..f468472bf4 100644 --- a/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js +++ b/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js @@ -34,6 +34,7 @@ class HandleUpdateRequestCommand extends HandleProtocolMessageCommand { agreementId, keyword, hashFunctionId, + proximityScoreFunctionsPairId, agreementData, } = commandData; @@ -66,7 +67,13 @@ class HandleUpdateRequestCommand extends HandleProtocolMessageCommand { const r2 = await this.blockchainModuleManager.getR2(blockchain); const scheduleCommandsPromises = []; - const rank = await this.calculateRank(blockchain, keyword, hashFunctionId, r2); + const rank = await this.calculateRank( + blockchain, + keyword, + hashFunctionId, + proximityScoreFunctionsPairId, + r2, + ); if (rank != null) { this.logger.trace(`Calculated rank: ${rank + 1} for agreement id: ${agreementId}`); const finalizationCommitsNumber = @@ -170,6 +177,7 @@ class HandleUpdateRequestCommand extends HandleProtocolMessageCommand { blockchain, keyword, hashFunctionId, + proximityScoreFunctionsPairId, ), peerId: node.peerId, })), From f9ba6d8ddd8c4488f4d3df4d8fee8848f0e26ac9 Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Wed, 17 Jan 2024 11:50:05 +0100 Subject: [PATCH 10/96] Update bid sugestion controller --- .../http-api/v0/bid-suggestion-http-api-controller-v0.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/controllers/http-api/v0/bid-suggestion-http-api-controller-v0.js b/src/controllers/http-api/v0/bid-suggestion-http-api-controller-v0.js index 4d711228fa..df171bee7c 100644 --- a/src/controllers/http-api/v0/bid-suggestion-http-api-controller-v0.js +++ b/src/controllers/http-api/v0/bid-suggestion-http-api-controller-v0.js @@ -6,6 +6,7 @@ class BidSuggestionController extends BaseController { this.repositoryModuleManager = ctx.repositoryModuleManager; this.blockchainModuleManager = ctx.blockchainModuleManager; this.shardingTableService = ctx.shardingTableService; + this.serviceAgreementService = ctx.serviceAgreementService; } async handleRequest(req, res) { @@ -49,7 +50,14 @@ class BidSuggestionController extends BaseController { firstAssertionId, hashFunctionId, } = req.query; + let { proximityScoreFunctionsPairId } = req.query; try { + // ADD-DOCS + if (!proximityScoreFunctionsPairId) { + if (blockchain.startsWith('otp')) proximityScoreFunctionsPairId = 1; + else if (blockchain.startsWith('gnosis')) proximityScoreFunctionsPairId = 2; + } + const bidSuggestion = await this.shardingTableService.getBidSuggestion( blockchain, epochsNumber, @@ -57,6 +65,7 @@ class BidSuggestionController extends BaseController { contentAssetStorageAddress, firstAssertionId, hashFunctionId, + proximityScoreFunctionsPairId, ); this.returnResponse(res, 200, { bidSuggestion }); From 164ccd6fd5460f33b6fc5d63fcffa7d44fe59c06 Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Wed, 17 Jan 2024 12:18:13 +0100 Subject: [PATCH 11/96] Update operation commands --- .../common/network-protocol-command.js | 15 ++++++++++++++- .../protocols/common/protocol-init-command.js | 19 +++++++++++++++++-- .../protocol-schedule-messages-command.js | 18 ++++++++++++++++-- .../sender/v1.0.0/v1-0-0-get-init-command.js | 4 +++- .../v1.0.0/v1-0-0-get-request-command.js | 10 +++++++++- .../v0/publish-http-api-controller-v0.js | 3 +++ 6 files changed, 62 insertions(+), 7 deletions(-) diff --git a/src/commands/protocols/common/network-protocol-command.js b/src/commands/protocols/common/network-protocol-command.js index fb10f1d3b3..ac7d5cb534 100644 --- a/src/commands/protocols/common/network-protocol-command.js +++ b/src/commands/protocols/common/network-protocol-command.js @@ -11,12 +11,24 @@ class NetworkProtocolCommand extends Command { * @param command */ async execute(command) { - const { blockchain } = command.data; + const { blockchain, contract, tokenId } = command.data; const keywords = await this.getKeywords(command); const batchSize = await this.getBatchSize(blockchain); const minAckResponses = await this.getMinAckResponses(blockchain); + const serviceAgreementId = this.serviceAgreementService.generateId( + blockchain, + contract, + tokenId, + keywords, + ); + const agreementData = this.blockchainModuleManager.getAgreementData( + blockchain, + serviceAgreementId, + ); + const proximityScoreFunctionsPairId = agreementData.scoreFunctionId; + const commandSequence = [ 'findNodesCommand', `${this.operationService.getOperationName()}ScheduleMessagesCommand`, @@ -34,6 +46,7 @@ class NetworkProtocolCommand extends Command { minAckResponses, errorType: this.errorType, networkProtocols: this.operationService.getNetworkProtocols(), + proximityScoreFunctionsPairId, }, transactional: false, }), diff --git a/src/commands/protocols/common/protocol-init-command.js b/src/commands/protocols/common/protocol-init-command.js index 6a76e0318b..df324248fa 100644 --- a/src/commands/protocols/common/protocol-init-command.js +++ b/src/commands/protocols/common/protocol-init-command.js @@ -3,12 +3,27 @@ import { NETWORK_MESSAGE_TYPES } from '../../../constants/constants.js'; class ProtocolInitCommand extends ProtocolMessageCommand { async prepareMessage(command) { - const { assertionId, contract, tokenId, keyword, hashFunctionId } = command.data; + const { + assertionId, + contract, + tokenId, + keyword, + hashFunctionId, + proximityScoreFunctionsPairId, + } = command.data; // TODO: Backwards compatibility, send blockchain without chainId const blockchain = command.data.blockchain.split(':')[0]; - return { assertionId, blockchain, contract, tokenId, keyword, hashFunctionId }; + return { + assertionId, + blockchain, + contract, + tokenId, + keyword, + hashFunctionId, + proximityScoreFunctionsPairId, + }; } async execute(command) { diff --git a/src/commands/protocols/common/protocol-schedule-messages-command.js b/src/commands/protocols/common/protocol-schedule-messages-command.js index c29b66559d..6d2022d71b 100644 --- a/src/commands/protocols/common/protocol-schedule-messages-command.js +++ b/src/commands/protocols/common/protocol-schedule-messages-command.js @@ -69,8 +69,22 @@ class ProtocolScheduleMessagesCommand extends Command { } getNextCommandData(command) { - const { assertionId, blockchain, contract, tokenId, hashFunctionId } = command.data; - return { assertionId, blockchain, contract, tokenId, hashFunctionId }; + const { + assertionId, + blockchain, + contract, + tokenId, + hashFunctionId, + proximityScoreFunctionsPairId, + } = command.data; + return { + assertionId, + blockchain, + contract, + tokenId, + hashFunctionId, + proximityScoreFunctionsPairId, + }; } /** diff --git a/src/commands/protocols/get/sender/v1.0.0/v1-0-0-get-init-command.js b/src/commands/protocols/get/sender/v1.0.0/v1-0-0-get-init-command.js index 2c366cd952..f0ccabe408 100644 --- a/src/commands/protocols/get/sender/v1.0.0/v1-0-0-get-init-command.js +++ b/src/commands/protocols/get/sender/v1.0.0/v1-0-0-get-init-command.js @@ -10,7 +10,8 @@ class GetInitCommand extends ProtocolInitCommand { } async prepareMessage(command) { - const { contract, tokenId, keyword, assertionId, state } = command.data; + const { contract, tokenId, keyword, assertionId, state, proximityScoreFunctionsPairId } = + command.data; // TODO: Backwards compatibility, send blockchain without chainId const blockchain = command.data.blockchain.split(':')[0]; @@ -22,6 +23,7 @@ class GetInitCommand extends ProtocolInitCommand { keyword, assertionId, state, + proximityScoreFunctionsPairId, }; } diff --git a/src/commands/protocols/get/sender/v1.0.0/v1-0-0-get-request-command.js b/src/commands/protocols/get/sender/v1.0.0/v1-0-0-get-request-command.js index 6780a99871..82ec902859 100644 --- a/src/commands/protocols/get/sender/v1.0.0/v1-0-0-get-request-command.js +++ b/src/commands/protocols/get/sender/v1.0.0/v1-0-0-get-request-command.js @@ -31,7 +31,14 @@ class GetRequestCommand extends ProtocolRequestCommand { } async prepareMessage(command) { - const { contract, tokenId, assertionId, state, hashFunctionId } = command.data; + const { + contract, + tokenId, + assertionId, + state, + hashFunctionId, + proximityScoreFunctionsPairId, + } = command.data; // TODO: Backwards compatibility, send blockchain without chainId const blockchain = command.data.blockchain.split(':')[0]; @@ -43,6 +50,7 @@ class GetRequestCommand extends ProtocolRequestCommand { assertionId, state, hashFunctionId, + proximityScoreFunctionsPairId, }; } diff --git a/src/controllers/http-api/v0/publish-http-api-controller-v0.js b/src/controllers/http-api/v0/publish-http-api-controller-v0.js index dd132f8eac..f3de5c7c4b 100644 --- a/src/controllers/http-api/v0/publish-http-api-controller-v0.js +++ b/src/controllers/http-api/v0/publish-http-api-controller-v0.js @@ -14,6 +14,9 @@ class PublishController extends BaseController { this.commandExecutor = ctx.commandExecutor; this.operationIdService = ctx.operationIdService; this.repositoryModuleManager = ctx.repositoryModuleManager; + this.ualService = ctx.ualService; + this.serviceAgreementService = ctx.serviceAgreementService; + this.blockchainModuleManager = ctx.blockchainModuleManager; } async handleRequest(req, res) { From 902838f661d549a9c553fc999299633bbf1c3789 Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Wed, 17 Jan 2024 12:29:46 +0100 Subject: [PATCH 12/96] Update rpc-controllers to handle reciving proximityScoreFunctionsPairId --- src/controllers/rpc/publish-rpc-controller.js | 1 + src/controllers/rpc/update-rpc-controller.js | 1 + 2 files changed, 2 insertions(+) diff --git a/src/controllers/rpc/publish-rpc-controller.js b/src/controllers/rpc/publish-rpc-controller.js index 84010abb3c..c435b58dc8 100644 --- a/src/controllers/rpc/publish-rpc-controller.js +++ b/src/controllers/rpc/publish-rpc-controller.js @@ -54,6 +54,7 @@ class PublishController extends BaseController { tokenId: dataSource.tokenId, keyword: dataSource.keyword, hashFunctionId: message.data.hashFunctionId ?? CONTENT_ASSET_HASH_FUNCTION_ID, + proximityScoreFunctionsPairId: dataSource.proximityScoreFunctionsPairId, }; await this.commandExecutor.add(command); diff --git a/src/controllers/rpc/update-rpc-controller.js b/src/controllers/rpc/update-rpc-controller.js index f9fd8764d5..53d6c44a53 100644 --- a/src/controllers/rpc/update-rpc-controller.js +++ b/src/controllers/rpc/update-rpc-controller.js @@ -54,6 +54,7 @@ class UpdateController extends BaseController { tokenId: dataSource.tokenId, keyword: dataSource.keyword, hashFunctionId: dataSource.hashFunctionId ?? CONTENT_ASSET_HASH_FUNCTION_ID, + proximityScoreFunctionsPairId: dataSource.proximityScoreFunctionsPairId, }; await this.commandExecutor.add(command); From 2cb4d7b6925ea73ddae9e841a1878262eb072f0b Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Wed, 17 Jan 2024 15:18:40 +0100 Subject: [PATCH 13/96] Use getAgreementScoreFunctionId to get proximityScoreFunctionsPairId from blockchain --- .../protocols/common/network-protocol-command.js | 11 ++++++----- src/modules/blockchain/blockchain-module-manager.js | 6 ++++++ src/modules/blockchain/implementation/web3-service.js | 8 ++++++++ 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/commands/protocols/common/network-protocol-command.js b/src/commands/protocols/common/network-protocol-command.js index ac7d5cb534..2ee65129cb 100644 --- a/src/commands/protocols/common/network-protocol-command.js +++ b/src/commands/protocols/common/network-protocol-command.js @@ -4,6 +4,7 @@ class NetworkProtocolCommand extends Command { constructor(ctx) { super(ctx); this.commandExecutor = ctx.commandExecutor; + this.blockchainModuleManager = ctx.this.blockchainModuleManager; } /** @@ -23,11 +24,11 @@ class NetworkProtocolCommand extends Command { tokenId, keywords, ); - const agreementData = this.blockchainModuleManager.getAgreementData( - blockchain, - serviceAgreementId, - ); - const proximityScoreFunctionsPairId = agreementData.scoreFunctionId; + const proximityScoreFunctionsPairId = + this.blockchainModuleManager.getAgreementScoreFunctionId( + blockchain, + serviceAgreementId, + ); const commandSequence = [ 'findNodesCommand', diff --git a/src/modules/blockchain/blockchain-module-manager.js b/src/modules/blockchain/blockchain-module-manager.js index dfce41cee5..7809e9a10c 100644 --- a/src/modules/blockchain/blockchain-module-manager.js +++ b/src/modules/blockchain/blockchain-module-manager.js @@ -391,6 +391,12 @@ class BlockchainModuleManager extends BaseModuleManager { async hasPendingUpdate(blockchain, tokenId) { return this.callImplementationFunction(blockchain, 'hasPendingUpdate', [tokenId]); } + + async getAgreementScoreFunctionId(blockchain, agreementId) { + return this.callImplementationFunction(blockchain, 'getAgreementScoreFunctionId', [ + agreementId, + ]); + } } export default BlockchainModuleManager; diff --git a/src/modules/blockchain/implementation/web3-service.js b/src/modules/blockchain/implementation/web3-service.js index 3d00134bfd..5ca3afe5bd 100644 --- a/src/modules/blockchain/implementation/web3-service.js +++ b/src/modules/blockchain/implementation/web3-service.js @@ -1203,6 +1203,14 @@ class Web3Service { tokenId, ]); } + + async getAgreementScoreFunctionId(agreementId) { + return this.callContractFunction( + this.ServiceAgreementStorageProxyContract, + 'getAgreementScoreFunctionId', + [agreementId], + ); + } } export default Web3Service; From 0d4f0ef5209bcb87d87aaa0cd078d538a568d714 Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Wed, 17 Jan 2024 15:42:56 +0100 Subject: [PATCH 14/96] Update bid-suggerstion-schema to include proximityScoreFunctionsPairId --- .../http-api/v0/request-schema/bid-suggestion-schema-v0.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/controllers/http-api/v0/request-schema/bid-suggestion-schema-v0.js b/src/controllers/http-api/v0/request-schema/bid-suggestion-schema-v0.js index f9259aa3ef..715b37ce60 100644 --- a/src/controllers/http-api/v0/request-schema/bid-suggestion-schema-v0.js +++ b/src/controllers/http-api/v0/request-schema/bid-suggestion-schema-v0.js @@ -35,5 +35,10 @@ export default (argumentsObject) => ({ minimum: 1, maximum: 1, }, + proximityScoreFunctionsPairId: { + type: 'number', + minimum: 1, + maximum: 2, + }, }, }); From 6d5747f97405630d0a3229784511b95bf8d6738c Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Thu, 18 Jan 2024 11:39:17 +0100 Subject: [PATCH 15/96] Add test to check publish, get, update handling of hashFunctionId & scoreFunctionId --- test/bdd/steps/api/get.mjs | 41 +++++++++++++++++++++++++++ test/bdd/steps/api/publish.mjs | 51 +++++++++++++++++++++++++++++++++ test/bdd/steps/api/update.mjs | 52 ++++++++++++++++++++++++++++++++++ 3 files changed, 144 insertions(+) diff --git a/test/bdd/steps/api/get.mjs b/test/bdd/steps/api/get.mjs index 7b49dfd543..0331785678 100644 --- a/test/bdd/steps/api/get.mjs +++ b/test/bdd/steps/api/get.mjs @@ -105,3 +105,44 @@ When('I wait for latest Get to finalize', { timeout: 80000 }, async function get await setTimeout(4000); } }); + +When( + /^I call Get directly on the node (\d+) with ([^"]*) on blockchain ([^"]*) with hashFunctionId (\d+)/, + { timeout: 30000 }, + async function getFromNode(node, requestName, blockchain, hashFunctionId) { + this.logger.log(`I call get directly on the node ${node} on blockchain ${blockchain}`); + + expect( + !!this.state.localBlockchains[blockchain], + `Blockchain with name ${blockchain} not found`, + ).to.be.equal(true); + + expect( + !!requests[requestName], + `Request body with name: ${requestName} not found!`, + ).to.be.equal(true); + + expect( + !Number.isInteger(hashFunctionId), + `hashFunctionId value: ${hashFunctionId} is not an integer!`, + ).to.be.equal(true); + + const requestBody = JSON.parse(JSON.stringify(requests[requestName])); + requestBody.id = requestBody.id.replace('blockchain', blockchain); + requestBody.hashFunctionId = hashFunctionId; + + try { + const result = await httpApiHelper.get( + this.state.nodes[node - 1].nodeRpcUrl, + requestBody, + ); + const { operationId } = result.data; + this.state.latestGetData = { + nodeId: node - 1, + operationId, + }; + } catch (error) { + this.state.latestError = error; + } + }, +); diff --git a/test/bdd/steps/api/publish.mjs b/test/bdd/steps/api/publish.mjs index 18b6bf3bb3..8e46ab9a48 100644 --- a/test/bdd/steps/api/publish.mjs +++ b/test/bdd/steps/api/publish.mjs @@ -124,3 +124,54 @@ When( ); }, ); + +When( + /^I call Publish on the node (\d+) with ([^"]*) on blockchain ([^"]*) with hashFunctionId (\d+) and scoreFunctionId (\d+)/, + { timeout: 120000 }, + async function publish(node, assertionName, blockchain, hashFunctionId, scoreFunctionId) { + this.logger.log(`I call publish route on the node ${node} on blockchain ${blockchain}`); + + expect( + !!this.state.localBlockchains[blockchain], + `Blockchain with name ${blockchain} not found`, + ).to.be.equal(true); + + expect( + !!assertions[assertionName], + `Assertion with name: ${assertionName} not found!`, + ).to.be.equal(true); + + expect( + !Number.isInteger(hashFunctionId), + `hashFunctionId value: ${hashFunctionId} is not an integer!`, + ).to.be.equal(true); + + expect( + !Number.isInteger(scoreFunctionId), + `scoreFunctionId value: ${scoreFunctionId} not an integer!`, + ).to.be.equal(true); + + const assertion = assertions[assertionName]; + const options = { + blockchain: this.state.nodes[node - 1].clientBlockchainOptions[blockchain], + hashFunctionId, + scoreFunctionId, + }; + const result = await this.state.nodes[node - 1].client + .publish(assertion, options) + .catch((error) => { + assert.fail(`Error while trying to publish assertion. ${error}`); + }); + const { operationId } = result.operation; + this.state.latestPublishData = { + nodeId: node - 1, + UAL: result.UAL, + assertionId: result.assertionId, + operationId, + assertion: assertions[assertionName], + status: result.operation.status, + errorType: result.operation.errorType, + result, + }; + }, +); diff --git a/test/bdd/steps/api/update.mjs b/test/bdd/steps/api/update.mjs index 4740083d75..993ef7d192 100644 --- a/test/bdd/steps/api/update.mjs +++ b/test/bdd/steps/api/update.mjs @@ -102,3 +102,55 @@ When('I wait for latest Update to finalize', { timeout: 80000 }, async function await setTimeout(4000); } }); + +When( + /^I call Update on the node (\d+) for the latest published UAL with ([^"]*) on blockchain ([^"]*) with hashFunctionId (\d+) and scoreFunctionId (\d+)/, + { timeout: 120000 }, + async function update(node, assertionName, blockchain, hashFunctionId, scoreFunctionId) { + this.logger.log(`I call update route on the node ${node} on blockchain ${blockchain}`); + + expect( + !!this.state.localBlockchains[blockchain], + `Blockchain with name ${blockchain} not found`, + ).to.be.equal(true); + + expect( + !!assertions[assertionName], + `Assertion with name: ${assertionName} not found!`, + ).to.be.equal(true); + + expect( + !Number.isInteger(hashFunctionId), + `hashFunctionId value: ${hashFunctionId} is not an integer!`, + ).to.be.equal(true); + + expect( + !Number.isInteger(scoreFunctionId), + `hashFunctionId value: ${scoreFunctionId} is not an integer!`, + ).to.be.equal(true); + + const assertion = assertions[assertionName]; + const { UAL } = this.state.latestPublishData; + const options = { + blockchain: this.state.nodes[node - 1].clientBlockchainOptions[blockchain], + hashFunctionId, + scoreFunctionId, + }; + const result = await this.state.nodes[node - 1].client + .update(UAL, assertion, options) + .catch((error) => { + assert.fail(`Error while trying to update assertion. ${error}`); + }); + const { operationId } = result.operation; + this.state.latestUpdateData = { + nodeId: node - 1, + UAL, + assertionId: result.assertionId, + operationId, + assertion: assertions[assertionName], + status: result.operation.status, + errorType: result.operation.errorType, + result, + }; + }, +); From f02a2a5f70025491b8f6067cdebb145c99e6ccbf Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Thu, 18 Jan 2024 14:29:08 +0100 Subject: [PATCH 16/96] Add bid suggestion tests --- test/bdd/steps/api/bid-suggestion.mjs | 67 +++++++++++++++++++++++++++ test/bdd/steps/api/update.mjs | 2 +- test/utilities/dkg-client-helper.mjs | 18 +++++++ 3 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 test/bdd/steps/api/bid-suggestion.mjs diff --git a/test/bdd/steps/api/bid-suggestion.mjs b/test/bdd/steps/api/bid-suggestion.mjs new file mode 100644 index 0000000000..738ca4c5ff --- /dev/null +++ b/test/bdd/steps/api/bid-suggestion.mjs @@ -0,0 +1,67 @@ +import { When } from '@cucumber/cucumber'; +import { expect, assert } from 'chai'; +import { readFile } from 'fs/promises'; + +const assertions = JSON.parse(await readFile('test/bdd/steps/api/datasets/assertions.json')); + +When( + /^I call Get Bid Suggestion on the node (\d+) with ([^"]*) on blockchain ([^"]*) with hashFunctionId (\d+) and scoreFunctionId (\d+)/, + { timeout: 120000 }, + async function getBidSuggestion(node, assertionName, blockchain) { + this.logger.log( + `I call get bid suggestion route on the node ${node} on blockchain ${blockchain}`, + ); + + expect( + !!this.state.localBlockchains[blockchain], + `Blockchain with name ${blockchain} not found`, + ).to.be.equal(true); + + expect( + !!assertions[assertionName], + `Assertion with name: ${assertionName} not found!`, + ).to.be.equal(true); + + expect( + !Number.isInteger(hashFunctionId), + `hashFunctionId value: ${hashFunctionId} is not an integer!`, + ).to.be.equal(true); + + expect( + !Number.isInteger(scoreFunctionId), + `scoreFunctionId value: ${scoreFunctionId} is not an integer!`, + ).to.be.equal(true); + + const assertion = assertions[assertionName]; + + const publicAssertionId = await this.state.nodes[node - 1].client + .getPublicAssertionId(assertion) + .catch((error) => { + assert.fail(`Error while trying to get public assertion id. ${error}`); + }); + + const sizeInBytes = await this.state.nodes[node - 1].client + .getSizeInBytes(assertion) + .catch((error) => { + assert.fail(`Error while trying to get size in bytes. ${error}`); + }); + + const options = { + blockchain: this.state.nodes[node - 1].clientBlockchainOptions[blockchain], + hashFunctionId, + scoreFunctionId, + }; + const result = await this.state.nodes[node - 1].client + .getBidSuggestion(publicAssertionId, sizeInBytes, options) + .catch((error) => { + assert.fail(`Error while trying to get bid suggestion. ${error}`); + }); + this.state.latestBidSuggestionResult = { + nodeId: node - 1, + publicAssertionId, + sizeInBytes, + assertion: assertions[assertionName], + result, + }; + }, +); diff --git a/test/bdd/steps/api/update.mjs b/test/bdd/steps/api/update.mjs index 993ef7d192..d1eca6a678 100644 --- a/test/bdd/steps/api/update.mjs +++ b/test/bdd/steps/api/update.mjs @@ -126,7 +126,7 @@ When( expect( !Number.isInteger(scoreFunctionId), - `hashFunctionId value: ${scoreFunctionId} is not an integer!`, + `scoreFunctionId value: ${scoreFunctionId} is not an integer!`, ).to.be.equal(true); const assertion = assertions[assertionName]; diff --git a/test/utilities/dkg-client-helper.mjs b/test/utilities/dkg-client-helper.mjs index 6b41114a0a..2e216a3dd6 100644 --- a/test/utilities/dkg-client-helper.mjs +++ b/test/utilities/dkg-client-helper.mjs @@ -46,6 +46,24 @@ class DkgClientHelper { async query(query) { return this.client.query(query); } + + async getBidSuggestion(publicAssertionId, sizeInBytes, userOptions = {}) { + const defaultOptions = { + epochsNum: 2, + }; + + const options = { ...defaultOptions, ...userOptions }; + + return this.client.network.getBidSuggestion(publicAssertionId, sizeInBytes, options); + } + + async getPublicAssertionId(content) { + return this.client.assertion.getPublicAssertionId(content); + } + + async getSizeInBytes(content) { + return this.client.assertion.getSizeInBytes(content); + } } export default DkgClientHelper; From 85b1d4a42fe49b990b76d8ff2aaab1e38121e678 Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Thu, 18 Jan 2024 14:30:17 +0100 Subject: [PATCH 17/96] Fix updated code --- .../bid-suggestion-http-api-controller-v0.js | 6 ++-- .../blockchain/blockchain-module-manager.js | 4 +++ .../blockchain/implementation/web3-service.js | 6 +++- src/service/sharding-table-service.js | 33 ++++++++++--------- 4 files changed, 30 insertions(+), 19 deletions(-) diff --git a/src/controllers/http-api/v0/bid-suggestion-http-api-controller-v0.js b/src/controllers/http-api/v0/bid-suggestion-http-api-controller-v0.js index df171bee7c..9f4be4b180 100644 --- a/src/controllers/http-api/v0/bid-suggestion-http-api-controller-v0.js +++ b/src/controllers/http-api/v0/bid-suggestion-http-api-controller-v0.js @@ -54,8 +54,10 @@ class BidSuggestionController extends BaseController { try { // ADD-DOCS if (!proximityScoreFunctionsPairId) { - if (blockchain.startsWith('otp')) proximityScoreFunctionsPairId = 1; - else if (blockchain.startsWith('gnosis')) proximityScoreFunctionsPairId = 2; + if (blockchain.startsWith('otp') || blockchain.startsWith('hardhat1')) + proximityScoreFunctionsPairId = 1; + else if (blockchain.startsWith('gnosis') || blockchain.startsWith('hardhat2')) + proximityScoreFunctionsPairId = 2; } const bidSuggestion = await this.shardingTableService.getBidSuggestion( diff --git a/src/modules/blockchain/blockchain-module-manager.js b/src/modules/blockchain/blockchain-module-manager.js index 7809e9a10c..c9e2c53ede 100644 --- a/src/modules/blockchain/blockchain-module-manager.js +++ b/src/modules/blockchain/blockchain-module-manager.js @@ -397,6 +397,10 @@ class BlockchainModuleManager extends BaseModuleManager { agreementId, ]); } + + convertUint8ArrayToHex(blockchain, uint8Array) { + return this.callImplementationFunction(blockchain, 'convertUint8ArrayToHex', [uint8Array]); + } } export default BlockchainModuleManager; diff --git a/src/modules/blockchain/implementation/web3-service.js b/src/modules/blockchain/implementation/web3-service.js index 5ca3afe5bd..33e7ca10ea 100644 --- a/src/modules/blockchain/implementation/web3-service.js +++ b/src/modules/blockchain/implementation/web3-service.js @@ -1079,8 +1079,12 @@ class Web3Service { return ethers.utils.solidityPack(types, values); } + convertUint8ArrayToHex(uint8Array) { + return ethers.utils.hexlify(uint8Array); + } + convertAsciiToHex(string) { - return ethers.utils.hexlify(ethers.utils.toUtf8Bytes(string)); + return this.convertUint8ArrayToHex(ethers.utils.toUtf8Bytes(string)); } convertHexToAscii(hexString) { diff --git a/src/service/sharding-table-service.js b/src/service/sharding-table-service.js index fb4d36c938..d39f624446 100644 --- a/src/service/sharding-table-service.js +++ b/src/service/sharding-table-service.js @@ -126,7 +126,7 @@ class ShardingTableService { ); const keyHash = await this.hashingService.callHashFunction(hashFunctionId, key); - return this.sortPeers( + const soretedPeers = this.sortPeers( blockchainId, keyHash, peers, @@ -134,6 +134,7 @@ class ShardingTableService { hashFunctionId, proximityScoreFunctionsPairId, ); + return soretedPeers; } async sortPeers( @@ -145,9 +146,8 @@ class ShardingTableService { proximityScoreFunctionsPairId, ) { const hashFunctionName = this.hashingService.getHashFunctionName(hashFunctionId); - - return peers - .map(async (peer) => ({ + const peersWithDistance = await Promise.all( + peers.map(async (peer) => ({ peer, distance: await this.proximityScoringService.callProximityFunction( blockchainId, @@ -155,18 +155,19 @@ class ShardingTableService { peer[hashFunctionName], keyHash, ), - })) - .sort((a, b) => { - if (a.lt(b)) { - return -1; - } - if (a.gt(b)) { - return 1; - } - return 0; - }) - .slice(0, count) - .map((pd) => pd.peer); + })), + ); + peersWithDistance.sort((a, b) => { + if (a.distance < b.distance) { + return -1; + } + if (a.distance > b.distance) { + return 1; + } + return 0; + }); + const result = peersWithDistance.slice(0, count).map((pd) => pd.peer); + return result; } async getBidSuggestion( From 98781de72b28d959985c4736f8db8d930e01dcae Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Thu, 18 Jan 2024 15:58:24 +0100 Subject: [PATCH 18/96] Add scenario --- test/bdd/features/bid-suggestion.feature | 31 +++++++++++ test/bdd/steps/api/bid-suggestion.mjs | 66 +++++++++++++++++++++++- 2 files changed, 95 insertions(+), 2 deletions(-) create mode 100644 test/bdd/features/bid-suggestion.feature diff --git a/test/bdd/features/bid-suggestion.feature b/test/bdd/features/bid-suggestion.feature new file mode 100644 index 0000000000..b4a19c3ead --- /dev/null +++ b/test/bdd/features/bid-suggestion.feature @@ -0,0 +1,31 @@ +Feature: Release related tests + Background: Setup local blockchain, bootstraps and nodes + Given the blockchains are set up + And 1 bootstrap is running + + @release + Scenario: Publishing a valid assertion on both blockchains + Given I set R0 to be 1 on blockchain hardhat1:31337 + And I set R1 to be 2 on blockchain hardhat1:31337 + And I set R0 to be 1 on blockchain hardhat2:31337 + And I set R1 to be 2 on blockchain hardhat2:31337 + And I setup 4 nodes + And I wait for 5 seconds + + I call Get Bid Suggestion on the node 2 with validPublish_1ForValidUpdate_1 on blockchain hardhat1:31337 with hashFunctionId 1 and scoreFunctionId 1 + And I call Info route on the node 2 + + I call Get Bid Suggestion on the node 2 with validPublish_1ForValidUpdate_1 on blockchain hardhat2:31337 with hashFunctionId 1 and scoreFunctionId 1 + And I call Info route on the node 2 + + I call Get Bid Suggestion on the node 2 with validPublish_1ForValidUpdate_1 on blockchain hardhat1:31337 with hashFunctionId 1 and scoreFunctionId 2 + And I call Info route on the node 2 + + I call Get Bid Suggestion on the node 2 with validPublish_1ForValidUpdate_1 on blockchain hardhat2:31337 with hashFunctionId 1 and scoreFunctionId 2 + And I call Info route on the node 2 + + I call Get Bid Suggestion on the node 2 with validPublish_1ForValidUpdate_1 on blockchain hardhat1:31337 + And I call Info route on the node 2 + + I call Get Bid Suggestion on the node 2 with validPublish_1ForValidUpdate_1 on blockchain hardhat2:31337 + And I call Info route on the node 2 \ No newline at end of file diff --git a/test/bdd/steps/api/bid-suggestion.mjs b/test/bdd/steps/api/bid-suggestion.mjs index 738ca4c5ff..c6f93ae630 100644 --- a/test/bdd/steps/api/bid-suggestion.mjs +++ b/test/bdd/steps/api/bid-suggestion.mjs @@ -6,8 +6,14 @@ const assertions = JSON.parse(await readFile('test/bdd/steps/api/datasets/assert When( /^I call Get Bid Suggestion on the node (\d+) with ([^"]*) on blockchain ([^"]*) with hashFunctionId (\d+) and scoreFunctionId (\d+)/, - { timeout: 120000 }, - async function getBidSuggestion(node, assertionName, blockchain) { + { timeout: 300000 }, + async function getBidSuggestion( + node, + assertionName, + blockchain, + hashFunctionId, + scoreFunctionId, + ) { this.logger.log( `I call get bid suggestion route on the node ${node} on blockchain ${blockchain}`, ); @@ -51,9 +57,64 @@ When( hashFunctionId, scoreFunctionId, }; + let getBidSuggestionError; + const result = await this.state.nodes[node - 1].client + .getBidSuggestion(publicAssertionId, sizeInBytes, options) + .catch((error) => { + getBidSuggestionError = error; + assert.fail(`Error while trying to get bid suggestion. ${error}`); + }); + this.state.latestBidSuggestionResult = { + nodeId: node - 1, + publicAssertionId, + sizeInBytes, + assertion: assertions[assertionName], + result, + getBidSuggestionError, + }; + }, +); + +When( + /^I call Get Bid Suggestion on the node (\d+) with ([^"]*) on blockchain ([^"]*)/, + { timeout: 300000 }, + async function getBidSuggestion(node, assertionName, blockchain) { + this.logger.log( + `I call get bid suggestion route on the node ${node} on blockchain ${blockchain}`, + ); + + expect( + !!this.state.localBlockchains[blockchain], + `Blockchain with name ${blockchain} not found`, + ).to.be.equal(true); + + expect( + !!assertions[assertionName], + `Assertion with name: ${assertionName} not found!`, + ).to.be.equal(true); + + const assertion = assertions[assertionName]; + + const publicAssertionId = await this.state.nodes[node - 1].client + .getPublicAssertionId(assertion) + .catch((error) => { + assert.fail(`Error while trying to get public assertion id. ${error}`); + }); + + const sizeInBytes = await this.state.nodes[node - 1].client + .getSizeInBytes(assertion) + .catch((error) => { + assert.fail(`Error while trying to get size in bytes. ${error}`); + }); + + const options = { + blockchain: this.state.nodes[node - 1].clientBlockchainOptions[blockchain], + }; + let getBidSuggestionError; const result = await this.state.nodes[node - 1].client .getBidSuggestion(publicAssertionId, sizeInBytes, options) .catch((error) => { + getBidSuggestionError = error; assert.fail(`Error while trying to get bid suggestion. ${error}`); }); this.state.latestBidSuggestionResult = { @@ -62,6 +123,7 @@ When( sizeInBytes, assertion: assertions[assertionName], result, + getBidSuggestionError, }; }, ); From 3bb9abd1afc9458ab820c72896e06eabfcbc3393 Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Fri, 19 Jan 2024 12:14:35 +0100 Subject: [PATCH 19/96] Fixes --- .../protocols/common/network-protocol-command.js | 12 +++++++----- src/controllers/rpc/update-rpc-controller.js | 6 ++++-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/commands/protocols/common/network-protocol-command.js b/src/commands/protocols/common/network-protocol-command.js index 2ee65129cb..8bbf2affbc 100644 --- a/src/commands/protocols/common/network-protocol-command.js +++ b/src/commands/protocols/common/network-protocol-command.js @@ -4,7 +4,8 @@ class NetworkProtocolCommand extends Command { constructor(ctx) { super(ctx); this.commandExecutor = ctx.commandExecutor; - this.blockchainModuleManager = ctx.this.blockchainModuleManager; + this.blockchainModuleManager = ctx.blockchainModuleManager; + this.serviceAgreementService = ctx.serviceAgreementService; } /** @@ -12,20 +13,21 @@ class NetworkProtocolCommand extends Command { * @param command */ async execute(command) { - const { blockchain, contract, tokenId } = command.data; + const { blockchain, contract, tokenId, hashFunctionId } = command.data; const keywords = await this.getKeywords(command); const batchSize = await this.getBatchSize(blockchain); const minAckResponses = await this.getMinAckResponses(blockchain); - const serviceAgreementId = this.serviceAgreementService.generateId( + const serviceAgreementId = await this.serviceAgreementService.generateId( blockchain, contract, tokenId, - keywords, + keywords[0], + hashFunctionId, ); const proximityScoreFunctionsPairId = - this.blockchainModuleManager.getAgreementScoreFunctionId( + await this.blockchainModuleManager.getAgreementScoreFunctionId( blockchain, serviceAgreementId, ); diff --git a/src/controllers/rpc/update-rpc-controller.js b/src/controllers/rpc/update-rpc-controller.js index 53d6c44a53..355723ef42 100644 --- a/src/controllers/rpc/update-rpc-controller.js +++ b/src/controllers/rpc/update-rpc-controller.js @@ -24,7 +24,8 @@ class UpdateController extends BaseController { command.name = handleInitCommand; command.period = 5000; command.retries = 3; - + command.data.proximityScoreFunctionsPairId = + dataSource.proximityScoreFunctionsPairId; break; case NETWORK_MESSAGE_TYPES.REQUESTS.PROTOCOL_REQUEST: // eslint-disable-next-line no-case-declarations @@ -37,6 +38,8 @@ class UpdateController extends BaseController { command.data.keyword = message.data.keyword; command.data.agreementId = dataSource.agreementId; command.data.agreementData = dataSource.agreementData; + command.data.proximityScoreFunctionsPairId = + dataSource.agreementData.scoreFunctionId; break; default: throw Error('unknown message type'); @@ -54,7 +57,6 @@ class UpdateController extends BaseController { tokenId: dataSource.tokenId, keyword: dataSource.keyword, hashFunctionId: dataSource.hashFunctionId ?? CONTENT_ASSET_HASH_FUNCTION_ID, - proximityScoreFunctionsPairId: dataSource.proximityScoreFunctionsPairId, }; await this.commandExecutor.add(command); From 7c9b6ea1c7a57b51eaad92c095b953dc73636bca Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Fri, 19 Jan 2024 16:07:54 +0100 Subject: [PATCH 20/96] Update test scenario --- test/bdd/features/bid-suggestion.feature | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/test/bdd/features/bid-suggestion.feature b/test/bdd/features/bid-suggestion.feature index b4a19c3ead..be5be1e309 100644 --- a/test/bdd/features/bid-suggestion.feature +++ b/test/bdd/features/bid-suggestion.feature @@ -4,28 +4,16 @@ Feature: Release related tests And 1 bootstrap is running @release - Scenario: Publishing a valid assertion on both blockchains + Scenario: Get big suggestion with a valid assertion and valid hashFunctionId and scoreFunctionId on both blockchains Given I set R0 to be 1 on blockchain hardhat1:31337 And I set R1 to be 2 on blockchain hardhat1:31337 And I set R0 to be 1 on blockchain hardhat2:31337 And I set R1 to be 2 on blockchain hardhat2:31337 And I setup 4 nodes And I wait for 5 seconds - - I call Get Bid Suggestion on the node 2 with validPublish_1ForValidUpdate_1 on blockchain hardhat1:31337 with hashFunctionId 1 and scoreFunctionId 1 - And I call Info route on the node 2 - - I call Get Bid Suggestion on the node 2 with validPublish_1ForValidUpdate_1 on blockchain hardhat2:31337 with hashFunctionId 1 and scoreFunctionId 1 - And I call Info route on the node 2 - - I call Get Bid Suggestion on the node 2 with validPublish_1ForValidUpdate_1 on blockchain hardhat1:31337 with hashFunctionId 1 and scoreFunctionId 2 - And I call Info route on the node 2 - - I call Get Bid Suggestion on the node 2 with validPublish_1ForValidUpdate_1 on blockchain hardhat2:31337 with hashFunctionId 1 and scoreFunctionId 2 - And I call Info route on the node 2 - I call Get Bid Suggestion on the node 2 with validPublish_1ForValidUpdate_1 on blockchain hardhat1:31337 - And I call Info route on the node 2 + When I call Get Bid Suggestion on the node 2 with validPublish_1ForValidUpdate_1 on blockchain hardhat1:31337 + Then I call Info route on the node 2 - I call Get Bid Suggestion on the node 2 with validPublish_1ForValidUpdate_1 on blockchain hardhat2:31337 - And I call Info route on the node 2 \ No newline at end of file + When I call Get Bid Suggestion on the node 2 with validPublish_1ForValidUpdate_1 on blockchain hardhat2:31337 + Then I call Info route on the node 2 \ No newline at end of file From 641c2f5958c967b742c563eedfc46a30df24400c Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Mon, 22 Jan 2024 09:21:32 +0100 Subject: [PATCH 21/96] Fix scenario typo --- test/bdd/features/bid-suggestion.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/bdd/features/bid-suggestion.feature b/test/bdd/features/bid-suggestion.feature index be5be1e309..7e3b1ed245 100644 --- a/test/bdd/features/bid-suggestion.feature +++ b/test/bdd/features/bid-suggestion.feature @@ -4,7 +4,7 @@ Feature: Release related tests And 1 bootstrap is running @release - Scenario: Get big suggestion with a valid assertion and valid hashFunctionId and scoreFunctionId on both blockchains + Scenario: Get bid suggestion with a valid assertion and valid hashFunctionId and scoreFunctionId on both blockchains Given I set R0 to be 1 on blockchain hardhat1:31337 And I set R1 to be 2 on blockchain hardhat1:31337 And I set R0 to be 1 on blockchain hardhat2:31337 From 4133c358bf97dfe81e0c2a1de9763b55355876ba Mon Sep 17 00:00:00 2001 From: Uladzislau Hubar Date: Mon, 22 Jan 2024 09:54:36 +0100 Subject: [PATCH 22/96] Added box plots for distances and scores, added additional distance/score functions for testing purposes --- .gitignore | 2 +- src/service/proximity-scoring-service.js | 156 ++++++++++++- .../mocks/blockchain-module-manager-mock.js | 16 ++ .../simulation.js | 206 +++++++++++++++--- 4 files changed, 343 insertions(+), 37 deletions(-) diff --git a/.gitignore b/.gitignore index 3484fcd91e..78f95e223b 100644 --- a/.gitignore +++ b/.gitignore @@ -117,4 +117,4 @@ data* .vscode/launch.json # KAs Distribution Simulation Script Plots -tools/knowledge-assets-distribution-simulation/plots/*jpg +tools/knowledge-assets-distribution-simulation/plots/**/*jpg diff --git a/src/service/proximity-scoring-service.js b/src/service/proximity-scoring-service.js index 1cfd4fd439..37a74c82fc 100644 --- a/src/service/proximity-scoring-service.js +++ b/src/service/proximity-scoring-service.js @@ -11,6 +11,13 @@ class ProximityScoringService { this.proximityScoreFunctionsPairs = { 1: [this.calculateBinaryXOR.bind(this), this.Log2PLDSF.bind(this)], 2: [this.calculateProximityOnHashRing.bind(this), this.LinearLogisticSum.bind(this)], + 3: [this.calculateProximityOnHashRing.bind(this), this.LinearSum.bind(this)], + 4: [this.calculateProximityOnHashRing.bind(this), this.LinearDivision.bind(this)], + 5: [ + this.calculateProximityOnHashRing.bind(this), + this.LinearEMANormalization.bind(this), + ], + 6: [this.calculateRelativeDistance.bind(this), this.RelativeNormalization.bind(this)], }; } @@ -62,17 +69,55 @@ class ProximityScoringService { keyHash, ); - let directDistance = peerPositionOnHashRing.sub(keyPositionOnHashRing); - - if (directDistance.lt(0)) { - directDistance = directDistance.add(HASH_RING_SIZE); - } - + const directDistance = peerPositionOnHashRing.gt(keyPositionOnHashRing) + ? peerPositionOnHashRing.sub(keyPositionOnHashRing) + : keyPositionOnHashRing.sub(peerPositionOnHashRing); const wraparoundDistance = HASH_RING_SIZE.sub(directDistance); return directDistance.lt(wraparoundDistance) ? directDistance : wraparoundDistance; } + async calculateRelativeDistance(blockchain, peerHash, keyHash, nodes) { + const peerHashBN = await this.blockchainModuleManager.toBigNumber(blockchain, peerHash); + const keyHashBN = await this.blockchainModuleManager.toBigNumber(blockchain, keyHash); + + const positions = await Promise.all( + nodes.map(async (node) => + this.blockchainModuleManager.toBigNumber(blockchain, node.sha256), + ), + ); + + const closestNode = positions.reduce((prev, curr) => { + const diffCurr = curr.sub(keyHashBN).abs(); + const diffPrev = prev.sub(keyHashBN).abs(); + + return diffCurr.lt(diffPrev) ? curr : prev; + }); + + const sortedPositions = positions.sort((a, b) => a.sub(b)); + + const closestNodeIndex = sortedPositions.findIndex((pos) => pos.eq(closestNode)); + const peerIndex = sortedPositions.findIndex((pos) => pos.eq(peerHashBN)); + + return this.blockchainModuleManager.toBigNumber( + blockchain, + Math.abs(peerIndex - closestNodeIndex), + ); + } + + async RelativeNormalization(blockchain, distance, stake) { + const w1 = 3; + const w2 = 1; + + const mappedDistance = distance / 10; + const proximityScore = 1 - mappedDistance; + const mappedStake = (stake - 50000) / (1000000 - 50000); + + const score = w1 * proximityScore + w2 * mappedStake; + + return { mappedDistance, mappedStake, score }; + } + async Log2PLDSF(blockchain, distance, stake) { const log2PLDSFParams = await this.blockchainModuleManager.getLog2PLDSFParams(blockchain); @@ -98,10 +143,16 @@ class ProximityScoringService { const dividend = mappedStake.pow(stakeExponent).mul(a).add(b); const divisor = mappedDistance.pow(distanceExponent).mul(c).add(d); - return Math.floor( - Number(multiplier) * - Math.log2(Number(logArgumentConstant) + dividend.toNumber() / divisor.toNumber()), - ); + return { + mappedDistance, + mappedStake, + score: Math.floor( + Number(multiplier) * + Math.log2( + Number(logArgumentConstant) + dividend.toNumber() / divisor.toNumber(), + ), + ), + }; } // Using Maclaurin Series to approximate e^x @@ -142,7 +193,90 @@ class ProximityScoringService { const proximityScore = w1 * (1 - mappedDistance); const stakeScore = w2 * mappedStake; - return proximityScore + stakeScore; + return { mappedDistance, mappedStake, score: proximityScore + stakeScore }; + } + + async LinearSum(blockchain, distance, stake) { + const linearSumParams = await this.blockchainModuleManager.getLinearSumParams(blockchain); + const { distanceScaleFactor, w1, w2 } = linearSumParams; + + let dividend = distance; + let divisor = HASH_RING_SIZE.div(2); + if (dividend.gt(UINT128_MAX_BN) || divisor.gt(UINT128_MAX_BN)) { + dividend = dividend.div(distanceScaleFactor); + divisor = divisor.div(distanceScaleFactor); + } + + const divResult = dividend.mul(distanceScaleFactor).div(divisor); + const mappedDistance = + parseFloat(divResult.toString()) / parseFloat(distanceScaleFactor.toString()); + + const maxStake = await this.blockchainModuleManager.getMaximumStake(blockchain); + const mappedStake = + stake / Number(await this.blockchainModuleManager.convertFromWei(maxStake)); + + const proximityScore = w1 * (1 - mappedDistance); + const stakeScore = w2 * mappedStake; + + return { mappedDistance, mappedStake, score: proximityScore + stakeScore }; + } + + async LinearDivision(blockchain, distance, stake) { + const linearDivisionParams = await this.blockchainModuleManager.getLinearDivisionParams( + blockchain, + ); + const { distanceScaleFactor, w1, w2 } = linearDivisionParams; + + let dividend = distance; + let divisor = HASH_RING_SIZE.div(2); + if (dividend.gt(UINT128_MAX_BN) || divisor.gt(UINT128_MAX_BN)) { + dividend = dividend.div(distanceScaleFactor); + divisor = divisor.div(distanceScaleFactor); + } + + const divResult = dividend.mul(distanceScaleFactor).div(divisor); + const mappedDistance = + parseFloat(divResult.toString()) / parseFloat(distanceScaleFactor.toString()); + + const maxStake = await this.blockchainModuleManager.getMaximumStake(blockchain); + const mappedStake = + Math.log(stake + 1) / + Math.log(1 + Number(await this.blockchainModuleManager.convertFromWei(maxStake))); + + const proximityScore = w1 * (1 - mappedDistance); + const stakeScore = w2 * mappedStake; + + return { mappedDistance, mappedStake, score: stakeScore / proximityScore }; + } + + async LinearEMANormalization(blockchain, distance, stake, nodeDistanceEMA) { + // const linearEMANormalizationParams = await this.blockchainModuleManager.getLinearEMANormalizationParams(blockchain); + // const { w1, w2 } = linearEMANormalizationParams; + + const w1 = 2; + const w2 = 1; + + const distanceScaleFactor = '1000000000000000000'; + + let dividend = distance; + let divisor = nodeDistanceEMA; + if (dividend.gt(UINT128_MAX_BN) || divisor.gt(UINT128_MAX_BN)) { + dividend = dividend.div(distanceScaleFactor); + divisor = divisor.div(distanceScaleFactor); + } + + const divResult = dividend.mul(distanceScaleFactor).div(divisor); + const mappedDistance = + parseFloat(divResult.toString()) / parseFloat(distanceScaleFactor.toString()); + + const maxStake = await this.blockchainModuleManager.getMaximumStake(blockchain); + const mappedStake = + stake / Number(await this.blockchainModuleManager.convertFromWei(maxStake)); + + const proximityScore = w1 * (1 - mappedDistance); + const stakeScore = w2 * mappedStake; + + return { mappedDistance, mappedStake, score: proximityScore + stakeScore }; } } diff --git a/tools/knowledge-assets-distribution-simulation/mocks/blockchain-module-manager-mock.js b/tools/knowledge-assets-distribution-simulation/mocks/blockchain-module-manager-mock.js index a4488d398b..3a285927d7 100644 --- a/tools/knowledge-assets-distribution-simulation/mocks/blockchain-module-manager-mock.js +++ b/tools/knowledge-assets-distribution-simulation/mocks/blockchain-module-manager-mock.js @@ -9,6 +9,10 @@ class BlockchainModuleManagerMock { return ethers.utils.hexlify(uint8Array); } + convertFromWei(value, toUnit = 'ether') { + return ethers.utils.formatUnits(value, toUnit); + } + convertToWei(blockchain, value, fromUnit = 'ether') { return ethers.utils.parseUnits(value.toString(), fromUnit).toString(); } @@ -17,6 +21,10 @@ class BlockchainModuleManagerMock { return ethers.BigNumber.from(value); } + getMaximumStake(blockchain) { + return '1000000000000000000000000'; + } + getLog2PLDSFParams(blockchain) { return { distanceMappingCoefficient: @@ -43,6 +51,14 @@ class BlockchainModuleManagerMock { w2: 1, }; } + + getLinearSumParams(blockchain) { + return { distanceScaleFactor: '1000000000000000000', w1: 1, w2: 0.25 }; + } + + getLinearDivisionParams(blockchain) { + return { distanceScaleFactor: '1000000000000000000', w1: 1, w2: 0.1 }; + } } export default BlockchainModuleManagerMock; diff --git a/tools/knowledge-assets-distribution-simulation/simulation.js b/tools/knowledge-assets-distribution-simulation/simulation.js index ab98319de7..bb14a79855 100644 --- a/tools/knowledge-assets-distribution-simulation/simulation.js +++ b/tools/knowledge-assets-distribution-simulation/simulation.js @@ -286,6 +286,119 @@ function generateScatterPlot(data, metric, outputImageName) { convertSvgToJpg(d3n.svgString(), outputImageName); } +function generateBoxPlot(data, metric, outputImageName) { + const d3n = new D3Node(); + const margin = { top: 60, right: 30, bottom: 60, left: 60 }; + const width = 2000 - margin.left - margin.right; + const height = 1000 - margin.top - margin.bottom; + + const svg = d3n + .createSVG(width + margin.left + margin.right, height + margin.top + margin.bottom) + .append('g') + .attr('transform', `translate(${margin.left},${margin.top})`); + + svg.append('text') + .attr('x', width / 2) + .attr('y', 0 - margin.top / 2) + .attr('text-anchor', 'middle') + .style('font-size', '20px') + .style('text-decoration', 'underline') + .text(`${metric[0].toUpperCase() + metric.slice(1)} Distribution Plot`); + + const groupedData = d3.group(data, (d) => d.nodeId); + const valuesPerNode = Array.from(groupedData.values(), (d) => d.map((item) => item[metric])); + + const x = d3 + .scaleBand() + .domain(valuesPerNode.map((_, i) => i)) + .range([0, width]); + + svg.append('text') + .attr('transform', `translate(${width / 2}, ${height + margin.top - 20})`) + .style('text-anchor', 'middle') + .text('Node Index'); + + const y = d3 + .scaleLinear() + .domain([0, d3.max(data, (d) => d[metric])]) + .nice() + .range([height, 0]); + + svg.append('text') + .attr('transform', 'rotate(-90)') + .attr('y', 0 - margin.left) + .attr('x', 0 - height / 2) + .attr('dy', '1em') + .style('text-anchor', 'middle') + .text(metric[0].toUpperCase() + metric.slice(1)); + + svg.append('g') + .attr('transform', `translate(0, ${height})`) + .call(d3.axisBottom(x)) + .selectAll('text') + .style('text-anchor', 'end') + .attr('dx', '-.8em') + .attr('dy', '.15em') + .attr('transform', 'rotate(-65)'); + + svg.append('g').call(d3.axisLeft(y).ticks(30)); + + const boxWidth = x.bandwidth() - 3; + + valuesPerNode.forEach((d, i) => { + const box = svg + .append('g') + .attr('transform', `translate(${x(i) + (x.bandwidth() - boxWidth) / 2},0)`); + + const q1 = d3.quantile(d, 0.25); + const median = d3.quantile(d, 0.5); + const q3 = d3.quantile(d, 0.75); + const iqr = q3 - q1; + const lowerLimit = q1 - 1.5 * iqr; + const upperLimit = q3 + 1.5 * iqr; + + const lowerWhisker = d3.max([d3.min(data, (n) => n[metric]), lowerLimit]); + const upperWhisker = d3.min([d3.max(data, (n) => n[metric]), upperLimit]); + + box.append('rect') + .attr('y', y(q3)) + .attr('height', y(q1) - y(q3)) + .attr('width', boxWidth) + .style('fill', '#69b3a2'); + + box.append('line') + .attr('y1', y(median)) + .attr('y2', y(median)) + .attr('x1', 0) + .attr('x2', boxWidth) + .style('stroke', 'black') + .style('width', 80); + + box.append('line') + .attr('y1', y(lowerWhisker)) + .attr('y2', y(upperWhisker)) + .attr('x1', boxWidth / 2) + .attr('x2', boxWidth / 2) + .style('stroke', 'black'); + + box.append('line') + .attr('y1', y(lowerWhisker)) + .attr('y2', y(lowerWhisker)) + .attr('x1', boxWidth * 0.25) + .attr('x2', boxWidth * 0.75) + .style('stroke', 'black'); + + box.append('line') + .attr('y1', y(upperWhisker)) + .attr('y2', y(upperWhisker)) + .attr('x1', boxWidth * 0.25) + .attr('x2', boxWidth * 0.75) + .style('stroke', 'black'); + }); + + convertSvgToJpg(d3n.svgString(), outputImageName); +} + async function runSimulation( mode, filePath, @@ -316,9 +429,12 @@ async function runSimulation( ); const knowledgeAssets = await generateRandomHashes(numberOfKAs); - const distances = []; + const metrics = []; const replicas = {}; + const N = 100; + const EMAs = {}; + for (const node of nodes) { replicas[node.nodeId] = { stake: Number(node.stake), @@ -327,32 +443,48 @@ async function runSimulation( }; } + const positions = await Promise.all( + nodes.map(async (node) => blockchainModuleManagerMock.toBigNumber(blockchain, node.sha256)), + ); + const sortedPositions = positions.sort((a, b) => a.sub(b)); + for (const key of knowledgeAssets) { + const closestNode = sortedPositions.reduce((prev, curr) => { + const diffCurr = curr.sub(key).abs(); + const diffPrev = prev.sub(key).abs(); + + return diffCurr.lt(diffPrev) ? curr : prev; + }); + + const closestNodeIndex = sortedPositions.findIndex((pos) => pos.eq(closestNode)); + const nodesWithDistances = await Promise.all( nodes.map(async (node) => { - const distance = await proximityScoringService.callProximityFunction( - blockchain, - proximityScoreFunctionsPairId, - node.sha256, - key, - ); - - distances.push({ nodeId: node.nodeId, assertionId: key, distance }); + const peerIndex = sortedPositions.findIndex((pos) => pos.eq(node.sha256)); + + let distance; + if (proximityScoreFunctionsPairId === 6) { + const directDistance = Math.abs(closestNodeIndex - peerIndex); + const wraparoundDistance = positions.length - directDistance; + distance = await blockchainModuleManagerMock.toBigNumber( + blockchain, + Math.min(directDistance, wraparoundDistance), + ); + } else { + distance = await proximityScoringService.callProximityFunction( + blockchain, + proximityScoreFunctionsPairId, + node.sha256, + key, + ); + } return { ...node, distance }; }), ); const nodesSortedByDistance = nodesWithDistances - .sort((a, b) => { - if (a.distance.lt(b.distance)) { - return -1; - } - if (a.distance.gt(b.distance)) { - return 1; - } - return 0; - }) + .sort((a, b) => a.distance.sub(b.distance)) .slice(0, r2); const maxDistance = nodesSortedByDistance[nodesSortedByDistance.length - 1].distance; @@ -363,13 +495,26 @@ async function runSimulation( const nodesWithScores = await Promise.all( nodesSortedByDistance.map(async (node) => { - const score = await proximityScoringService.callScoreFunction( - blockchain, - proximityScoreFunctionsPairId, - node.distance, - node.stake, - maxDistance, - ); + const { mappedDistance, mappedStake, score } = + await proximityScoringService.callScoreFunction( + blockchain, + proximityScoreFunctionsPairId, + node.distance, + node.stake, + maxDistance, + EMAs[node.nodeId], + ); + + metrics.push({ nodeId: node.nodeId, mappedDistance, mappedStake, score }); + + if (!(node.nodeId in EMAs)) { + EMAs[node.nodeId] = node.distance; + } else { + EMAs[node.nodeId] = node.distance + .mul(2) + .div(N + 1) + .add(EMAs[node.nodeId].mul(N - 1).div(N + 1)); + } return { ...node, score }; }), @@ -396,6 +541,17 @@ async function runSimulation( 'won', `${mode}-${nodes.length}-${numberOfKAs}-${proximityScoreFunctionsPairId}-stake-wins-relation`, ); + + generateBoxPlot( + metrics, + 'mappedDistance', + `${mode}-${nodes.length}-${numberOfKAs}-${proximityScoreFunctionsPairId}-mapped-distances-distribution`, + ); + generateBoxPlot( + metrics, + 'score', + `${mode}-${nodes.length}-${numberOfKAs}-${proximityScoreFunctionsPairId}-scores-distribution`, + ); } const args = process.argv.slice(2); From 2351e7a54bd553b76c689bb521889bf7ef061d35 Mon Sep 17 00:00:00 2001 From: Uladzislau Hubar Date: Mon, 22 Jan 2024 10:29:06 +0100 Subject: [PATCH 23/96] Removed abandoned proximity/distance functions, adapted simulation script, added proximity/score function that is defined as final proposal in the spec --- .../blockchain/blockchain-module-manager.js | 8 + .../blockchain/implementation/web3-service.js | 22 +++ src/service/proximity-scoring-service.js | 161 +----------------- .../mocks/blockchain-module-manager-mock.js | 19 +-- .../simulation.js | 83 ++++----- 5 files changed, 74 insertions(+), 219 deletions(-) diff --git a/src/modules/blockchain/blockchain-module-manager.js b/src/modules/blockchain/blockchain-module-manager.js index dfce41cee5..92916c78e2 100644 --- a/src/modules/blockchain/blockchain-module-manager.js +++ b/src/modules/blockchain/blockchain-module-manager.js @@ -326,6 +326,14 @@ class BlockchainModuleManager extends BaseModuleManager { ]); } + async getMinimumStake(blockchain) { + return this.callImplementationFunction(blockchain, 'getMinimumStake'); + } + + async getMaximumStake(blockchain) { + return this.callImplementationFunction(blockchain, 'getMaximumStake'); + } + async getR2(blockchain) { return this.callImplementationFunction(blockchain, 'getR2'); } diff --git a/src/modules/blockchain/implementation/web3-service.js b/src/modules/blockchain/implementation/web3-service.js index 3d00134bfd..65a4f1c347 100644 --- a/src/modules/blockchain/implementation/web3-service.js +++ b/src/modules/blockchain/implementation/web3-service.js @@ -913,6 +913,28 @@ class Web3Service { })); } + async getMinimumStake() { + const minimumStake = await this.callContractFunction( + this.ParametersStorageContract, + 'minimumStake', + [], + CONTRACTS.PARAMETERS_STORAGE_CONTRACT, + ); + + return Number(ethers.utils.formatEther(minimumStake)); + } + + async getMaximumStake() { + const maximumStake = await this.callContractFunction( + this.ParametersStorageContract, + 'maximumStake', + [], + CONTRACTS.PARAMETERS_STORAGE_CONTRACT, + ); + + return Number(ethers.utils.formatEther(maximumStake)); + } + async getR2() { const r2 = await this.callContractFunction( this.ParametersStorageContract, diff --git a/src/service/proximity-scoring-service.js b/src/service/proximity-scoring-service.js index 37a74c82fc..4975c0dfb4 100644 --- a/src/service/proximity-scoring-service.js +++ b/src/service/proximity-scoring-service.js @@ -10,14 +10,7 @@ class ProximityScoringService { this.proximityScoreFunctionsPairs = { 1: [this.calculateBinaryXOR.bind(this), this.Log2PLDSF.bind(this)], - 2: [this.calculateProximityOnHashRing.bind(this), this.LinearLogisticSum.bind(this)], - 3: [this.calculateProximityOnHashRing.bind(this), this.LinearSum.bind(this)], - 4: [this.calculateProximityOnHashRing.bind(this), this.LinearDivision.bind(this)], - 5: [ - this.calculateProximityOnHashRing.bind(this), - this.LinearEMANormalization.bind(this), - ], - 6: [this.calculateRelativeDistance.bind(this), this.RelativeNormalization.bind(this)], + 2: [this.calculateProximityOnHashRing.bind(this), this.LinearSum.bind(this)], }; } @@ -77,47 +70,6 @@ class ProximityScoringService { return directDistance.lt(wraparoundDistance) ? directDistance : wraparoundDistance; } - async calculateRelativeDistance(blockchain, peerHash, keyHash, nodes) { - const peerHashBN = await this.blockchainModuleManager.toBigNumber(blockchain, peerHash); - const keyHashBN = await this.blockchainModuleManager.toBigNumber(blockchain, keyHash); - - const positions = await Promise.all( - nodes.map(async (node) => - this.blockchainModuleManager.toBigNumber(blockchain, node.sha256), - ), - ); - - const closestNode = positions.reduce((prev, curr) => { - const diffCurr = curr.sub(keyHashBN).abs(); - const diffPrev = prev.sub(keyHashBN).abs(); - - return diffCurr.lt(diffPrev) ? curr : prev; - }); - - const sortedPositions = positions.sort((a, b) => a.sub(b)); - - const closestNodeIndex = sortedPositions.findIndex((pos) => pos.eq(closestNode)); - const peerIndex = sortedPositions.findIndex((pos) => pos.eq(peerHashBN)); - - return this.blockchainModuleManager.toBigNumber( - blockchain, - Math.abs(peerIndex - closestNodeIndex), - ); - } - - async RelativeNormalization(blockchain, distance, stake) { - const w1 = 3; - const w2 = 1; - - const mappedDistance = distance / 10; - const proximityScore = 1 - mappedDistance; - const mappedStake = (stake - 50000) / (1000000 - 50000); - - const score = w1 * proximityScore + w2 * mappedStake; - - return { mappedDistance, mappedStake, score }; - } - async Log2PLDSF(blockchain, distance, stake) { const log2PLDSFParams = await this.blockchainModuleManager.getLog2PLDSFParams(blockchain); @@ -155,128 +107,29 @@ class ProximityScoringService { }; } - // Using Maclaurin Series to approximate e^x - _approximateExp(x, degree) { - let xPow = x; - let factorial = 1; - let result = 1; - - for (let i = 1; i <= degree; i += 1) { - factorial *= i; - result += xPow / factorial; - xPow *= x; - } - - return result; - } - - async LinearLogisticSum(blockchain, distance, stake, maxNeighborhoodDistance) { - const linearLogisticSumParams = - await this.blockchainModuleManager.getLinearLogisticSumParams(blockchain); - const { distanceScaleFactor, exponentMultiplier, maclaurinSeriesDegree, x0, w1, w2 } = - linearLogisticSumParams; - - let dividend = distance; - let divisor = maxNeighborhoodDistance; - if (dividend.gt(UINT128_MAX_BN) || divisor.gt(UINT128_MAX_BN)) { - dividend = dividend.div(distanceScaleFactor); - divisor = divisor.div(distanceScaleFactor); - } - - const divResult = dividend.mul(distanceScaleFactor).div(divisor); - const mappedDistance = - parseFloat(divResult.toString()) / parseFloat(distanceScaleFactor.toString()); - - const exponentPart = exponentMultiplier * (stake - x0); - const mappedStake = 2 / (1 + this._approximateExp(exponentPart, maclaurinSeriesDegree)) - 1; - - const proximityScore = w1 * (1 - mappedDistance); - const stakeScore = w2 * mappedStake; - - return { mappedDistance, mappedStake, score: proximityScore + stakeScore }; - } - - async LinearSum(blockchain, distance, stake) { + async LinearSum(blockchain, distance, stake, maxNeighborhoodDistance) { const linearSumParams = await this.blockchainModuleManager.getLinearSumParams(blockchain); const { distanceScaleFactor, w1, w2 } = linearSumParams; + const minimumStake = await this.blockchainModuleManager.getMinimumStake(blockchain); + const maximumStake = await this.blockchainModuleManager.getMaximumStake(blockchain); let dividend = distance; - let divisor = HASH_RING_SIZE.div(2); + let divisor = maxNeighborhoodDistance; if (dividend.gt(UINT128_MAX_BN) || divisor.gt(UINT128_MAX_BN)) { dividend = dividend.div(distanceScaleFactor); divisor = divisor.div(distanceScaleFactor); } const divResult = dividend.mul(distanceScaleFactor).div(divisor); - const mappedDistance = - parseFloat(divResult.toString()) / parseFloat(distanceScaleFactor.toString()); - - const maxStake = await this.blockchainModuleManager.getMaximumStake(blockchain); - const mappedStake = - stake / Number(await this.blockchainModuleManager.convertFromWei(maxStake)); - - const proximityScore = w1 * (1 - mappedDistance); - const stakeScore = w2 * mappedStake; - - return { mappedDistance, mappedStake, score: proximityScore + stakeScore }; - } - - async LinearDivision(blockchain, distance, stake) { - const linearDivisionParams = await this.blockchainModuleManager.getLinearDivisionParams( - blockchain, - ); - const { distanceScaleFactor, w1, w2 } = linearDivisionParams; - - let dividend = distance; - let divisor = HASH_RING_SIZE.div(2); - if (dividend.gt(UINT128_MAX_BN) || divisor.gt(UINT128_MAX_BN)) { - dividend = dividend.div(distanceScaleFactor); - divisor = divisor.div(distanceScaleFactor); - } - const divResult = dividend.mul(distanceScaleFactor).div(divisor); const mappedDistance = parseFloat(divResult.toString()) / parseFloat(distanceScaleFactor.toString()); - - const maxStake = await this.blockchainModuleManager.getMaximumStake(blockchain); - const mappedStake = - Math.log(stake + 1) / - Math.log(1 + Number(await this.blockchainModuleManager.convertFromWei(maxStake))); - - const proximityScore = w1 * (1 - mappedDistance); - const stakeScore = w2 * mappedStake; - - return { mappedDistance, mappedStake, score: stakeScore / proximityScore }; - } - - async LinearEMANormalization(blockchain, distance, stake, nodeDistanceEMA) { - // const linearEMANormalizationParams = await this.blockchainModuleManager.getLinearEMANormalizationParams(blockchain); - // const { w1, w2 } = linearEMANormalizationParams; - - const w1 = 2; - const w2 = 1; - - const distanceScaleFactor = '1000000000000000000'; - - let dividend = distance; - let divisor = nodeDistanceEMA; - if (dividend.gt(UINT128_MAX_BN) || divisor.gt(UINT128_MAX_BN)) { - dividend = dividend.div(distanceScaleFactor); - divisor = divisor.div(distanceScaleFactor); - } - - const divResult = dividend.mul(distanceScaleFactor).div(divisor); - const mappedDistance = - parseFloat(divResult.toString()) / parseFloat(distanceScaleFactor.toString()); - - const maxStake = await this.blockchainModuleManager.getMaximumStake(blockchain); - const mappedStake = - stake / Number(await this.blockchainModuleManager.convertFromWei(maxStake)); + const mappedStake = (stake - minimumStake) / (maximumStake - minimumStake); const proximityScore = w1 * (1 - mappedDistance); const stakeScore = w2 * mappedStake; - return { mappedDistance, mappedStake, score: proximityScore + stakeScore }; + return proximityScore + stakeScore; } } diff --git a/tools/knowledge-assets-distribution-simulation/mocks/blockchain-module-manager-mock.js b/tools/knowledge-assets-distribution-simulation/mocks/blockchain-module-manager-mock.js index 3a285927d7..5651dddee7 100644 --- a/tools/knowledge-assets-distribution-simulation/mocks/blockchain-module-manager-mock.js +++ b/tools/knowledge-assets-distribution-simulation/mocks/blockchain-module-manager-mock.js @@ -21,8 +21,12 @@ class BlockchainModuleManagerMock { return ethers.BigNumber.from(value); } + getMinimumStake(blockchain) { + return 50000; + } + getMaximumStake(blockchain) { - return '1000000000000000000000000'; + return 1000000; } getLog2PLDSFParams(blockchain) { @@ -41,24 +45,13 @@ class BlockchainModuleManagerMock { }; } - getLinearLogisticSumParams(blockchain) { + getLinearSumParams(blockchain) { return { distanceScaleFactor: '1000000000000000000', - exponentMultiplier: -0.000001, - maclaurinSeriesDegree: 20, - x0: 50000, w1: 1, w2: 1, }; } - - getLinearSumParams(blockchain) { - return { distanceScaleFactor: '1000000000000000000', w1: 1, w2: 0.25 }; - } - - getLinearDivisionParams(blockchain) { - return { distanceScaleFactor: '1000000000000000000', w1: 1, w2: 0.1 }; - } } export default BlockchainModuleManagerMock; diff --git a/tools/knowledge-assets-distribution-simulation/simulation.js b/tools/knowledge-assets-distribution-simulation/simulation.js index bb14a79855..3f79743a5a 100644 --- a/tools/knowledge-assets-distribution-simulation/simulation.js +++ b/tools/knowledge-assets-distribution-simulation/simulation.js @@ -11,6 +11,7 @@ import sharp from 'sharp'; import { readFile } from 'fs/promises'; import { createRequire } from 'module'; import { create as createLibP2PKey, createFromPrivKey } from 'peer-id'; +import { UINT128_MAX_BN } from '../../src/constants/constants.js'; import BlockchainModuleManagerMock from './mocks/blockchain-module-manager-mock.js'; import HashingService from '../../src/service/hashing-service.js'; import ProximityScoringService from '../../src/service/proximity-scoring-service.js'; @@ -432,9 +433,6 @@ async function runSimulation( const metrics = []; const replicas = {}; - const N = 100; - const EMAs = {}; - for (const node of nodes) { replicas[node.nodeId] = { stake: Number(node.stake), @@ -443,41 +441,20 @@ async function runSimulation( }; } - const positions = await Promise.all( - nodes.map(async (node) => blockchainModuleManagerMock.toBigNumber(blockchain, node.sha256)), - ); - const sortedPositions = positions.sort((a, b) => a.sub(b)); + const linearSumParams = await blockchainModuleManagerMock.getLinearSumParams(blockchain); + const { distanceScaleFactor } = linearSumParams; + const minimumStake = await blockchainModuleManagerMock.getMinimumStake(blockchain); + const maximumStake = await blockchainModuleManagerMock.getMaximumStake(blockchain); for (const key of knowledgeAssets) { - const closestNode = sortedPositions.reduce((prev, curr) => { - const diffCurr = curr.sub(key).abs(); - const diffPrev = prev.sub(key).abs(); - - return diffCurr.lt(diffPrev) ? curr : prev; - }); - - const closestNodeIndex = sortedPositions.findIndex((pos) => pos.eq(closestNode)); - const nodesWithDistances = await Promise.all( nodes.map(async (node) => { - const peerIndex = sortedPositions.findIndex((pos) => pos.eq(node.sha256)); - - let distance; - if (proximityScoreFunctionsPairId === 6) { - const directDistance = Math.abs(closestNodeIndex - peerIndex); - const wraparoundDistance = positions.length - directDistance; - distance = await blockchainModuleManagerMock.toBigNumber( - blockchain, - Math.min(directDistance, wraparoundDistance), - ); - } else { - distance = await proximityScoringService.callProximityFunction( - blockchain, - proximityScoreFunctionsPairId, - node.sha256, - key, - ); - } + const distance = await proximityScoringService.callProximityFunction( + blockchain, + proximityScoreFunctionsPairId, + node.sha256, + key, + ); return { ...node, distance }; }), @@ -495,26 +472,28 @@ async function runSimulation( const nodesWithScores = await Promise.all( nodesSortedByDistance.map(async (node) => { - const { mappedDistance, mappedStake, score } = - await proximityScoringService.callScoreFunction( - blockchain, - proximityScoreFunctionsPairId, - node.distance, - node.stake, - maxDistance, - EMAs[node.nodeId], - ); + const score = await proximityScoringService.callScoreFunction( + blockchain, + proximityScoreFunctionsPairId, + node.distance, + node.stake, + maxDistance, + ); + + let dividend = node.distance; + let divisor = maxDistance; + if (dividend.gt(UINT128_MAX_BN) || divisor.gt(UINT128_MAX_BN)) { + dividend = dividend.div(distanceScaleFactor); + divisor = divisor.div(distanceScaleFactor); + } - metrics.push({ nodeId: node.nodeId, mappedDistance, mappedStake, score }); + const divResult = dividend.mul(distanceScaleFactor).div(divisor); - if (!(node.nodeId in EMAs)) { - EMAs[node.nodeId] = node.distance; - } else { - EMAs[node.nodeId] = node.distance - .mul(2) - .div(N + 1) - .add(EMAs[node.nodeId].mul(N - 1).div(N + 1)); - } + const mappedDistance = + parseFloat(divResult.toString()) / parseFloat(distanceScaleFactor.toString()); + const mappedStake = (node.stake - minimumStake) / (maximumStake - minimumStake); + + metrics.push({ nodeId: node.nodeId, mappedDistance, mappedStake, score }); return { ...node, score }; }), From e3d15f648ebd960229db2ee8833a4391330f27a6 Mon Sep 17 00:00:00 2001 From: Uladzislau Hubar Date: Mon, 22 Jan 2024 11:48:13 +0100 Subject: [PATCH 24/96] Added clockwise distance calculation on the Hash Ring --- src/service/proximity-scoring-service.js | 29 +++++++++++++++++-- .../simulation.js | 18 ++++++++++-- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/src/service/proximity-scoring-service.js b/src/service/proximity-scoring-service.js index 4975c0dfb4..d9ebe47ded 100644 --- a/src/service/proximity-scoring-service.js +++ b/src/service/proximity-scoring-service.js @@ -10,7 +10,11 @@ class ProximityScoringService { this.proximityScoreFunctionsPairs = { 1: [this.calculateBinaryXOR.bind(this), this.Log2PLDSF.bind(this)], - 2: [this.calculateProximityOnHashRing.bind(this), this.LinearSum.bind(this)], + 2: [this.calculateClockwiseProximityOnHashRing.bind(this), this.LinearSum.bind(this)], + 3: [ + this.calculateBidirectionalProximityOnHashRing.bind(this), + this.LinearSum.bind(this), + ], }; } @@ -52,7 +56,28 @@ class ProximityScoringService { return this.blockchainModuleManager.toBigNumber(blockchain, distanceHex); } - async calculateProximityOnHashRing(blockchain, peerHash, keyHash) { + async calculateClockwiseProximityOnHashRing(blockchain, peerHash, keyHash) { + const peerPositionOnHashRing = await this.blockchainModuleManager.toBigNumber( + blockchain, + peerHash, + ); + const keyPositionOnHashRing = await this.blockchainModuleManager.toBigNumber( + blockchain, + keyHash, + ); + + let clockwiseDistance; + if (peerPositionOnHashRing.gt(keyPositionOnHashRing)) { + const distanceToEnd = HASH_RING_SIZE.sub(peerPositionOnHashRing); + clockwiseDistance = distanceToEnd.add(keyPositionOnHashRing); + } else { + clockwiseDistance = keyPositionOnHashRing.sub(peerPositionOnHashRing); + } + + return clockwiseDistance; + } + + async calculateBidirectionalProximityOnHashRing(blockchain, peerHash, keyHash) { const peerPositionOnHashRing = await this.blockchainModuleManager.toBigNumber( blockchain, peerHash, diff --git a/tools/knowledge-assets-distribution-simulation/simulation.js b/tools/knowledge-assets-distribution-simulation/simulation.js index 3f79743a5a..60b3cd9593 100644 --- a/tools/knowledge-assets-distribution-simulation/simulation.js +++ b/tools/knowledge-assets-distribution-simulation/simulation.js @@ -11,7 +11,7 @@ import sharp from 'sharp'; import { readFile } from 'fs/promises'; import { createRequire } from 'module'; import { create as createLibP2PKey, createFromPrivKey } from 'peer-id'; -import { UINT128_MAX_BN } from '../../src/constants/constants.js'; +import { HASH_RING_SIZE, UINT128_MAX_BN } from '../../src/constants/constants.js'; import BlockchainModuleManagerMock from './mocks/blockchain-module-manager-mock.js'; import HashingService from '../../src/service/hashing-service.js'; import ProximityScoringService from '../../src/service/proximity-scoring-service.js'; @@ -441,6 +441,15 @@ async function runSimulation( }; } + const nodesNumber = nodes.length; + let IDEAL_MAX_DISTANCE_IN_NEIGHBORHOOD; + + if (proximityScoreFunctionsPairId === 2) { + IDEAL_MAX_DISTANCE_IN_NEIGHBORHOOD = HASH_RING_SIZE.div(nodesNumber).mul(20); + } else if (proximityScoreFunctionsPairId === 3) { + IDEAL_MAX_DISTANCE_IN_NEIGHBORHOOD = HASH_RING_SIZE.div(nodesNumber).mul(10); + } + const linearSumParams = await blockchainModuleManagerMock.getLinearSumParams(blockchain); const { distanceScaleFactor } = linearSumParams; const minimumStake = await blockchainModuleManagerMock.getMinimumStake(blockchain); @@ -464,7 +473,12 @@ async function runSimulation( .sort((a, b) => a.distance.sub(b.distance)) .slice(0, r2); - const maxDistance = nodesSortedByDistance[nodesSortedByDistance.length - 1].distance; + const maxDistanceInNeighborhood = + nodesSortedByDistance[nodesSortedByDistance.length - 1].distance; + const maxDistance = + maxDistanceInNeighborhood > IDEAL_MAX_DISTANCE_IN_NEIGHBORHOOD + ? IDEAL_MAX_DISTANCE_IN_NEIGHBORHOOD + : maxDistanceInNeighborhood; for (const node of nodesSortedByDistance) { replicas[node.nodeId].replicated += 1; From c9d8c7dd8e3acc3d145f41f3053153e248404402 Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Mon, 22 Jan 2024 20:23:49 +0100 Subject: [PATCH 25/96] Update dkg.js version --- package-lock.json | 34 +++++++++++++++++----------------- package.json | 2 +- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2c474e188b..9f02771c78 100644 --- a/package-lock.json +++ b/package-lock.json @@ -68,7 +68,7 @@ "chai": "^4.3.6", "d3": "^7.8.5", "d3-node": "^3.0.0", - "dkg.js": "^6.0.2", + "dkg.js": "^6.1.2", "eslint": "^8.23.0", "eslint-config-airbnb": "^19.0.4", "eslint-config-prettier": "^8.5.0", @@ -6111,9 +6111,9 @@ } }, "node_modules/assertion-tools": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/assertion-tools/-/assertion-tools-2.0.2.tgz", - "integrity": "sha512-wntV3+hNWfd0vAs1vaBsSBEaa7W7Qs3Wbu9Sh0WeZEr+RZ55ImjFICh1Bfi1ArGTXYgjd/zQa5YSSOFvs18UGQ==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/assertion-tools/-/assertion-tools-2.1.0.tgz", + "integrity": "sha512-HCI/K2G9x/PlTpj0XuB2K3gR09C5VSXfqOfhzeqONC1x3WHRlUOIZmSSodaA5uLamUE45x8pnHjJxSyqxOI7tA==", "dependencies": { "ethers": "^5.7.2", "jsonld": "^8.1.0", @@ -9081,14 +9081,14 @@ "optional": true }, "node_modules/dkg.js": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/dkg.js/-/dkg.js-6.0.2.tgz", - "integrity": "sha512-F1+XGbOTQh//n5fOLpPJCZLNMlTl9Q//A5dAAdBxhA/cXjDWnMJOX2w98Fd2b2/Q4ZsWGuLnwdILqO8GmhG9bw==", + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/dkg.js/-/dkg.js-6.1.2.tgz", + "integrity": "sha512-flqu4wEDEVcMQnnO8WkuAUzeiEn/6wyTIDkRe72j8mwtClVHvA0+QGWrvuv2zfJJuDHx6e3wjrzhkuMdxjez9g==", "dev": true, "dependencies": { - "assertion-tools": "^2.0.2", + "assertion-tools": "^2.1.0", "axios": "^0.27.2", - "dkg-evm-module": "^4.0.4", + "dkg-evm-module": "^4.1.0", "ethers": "^6.1.0", "jsonld": "^8.1.0", "web3": "^1.7.3" @@ -26028,9 +26028,9 @@ } }, "assertion-tools": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/assertion-tools/-/assertion-tools-2.0.2.tgz", - "integrity": "sha512-wntV3+hNWfd0vAs1vaBsSBEaa7W7Qs3Wbu9Sh0WeZEr+RZ55ImjFICh1Bfi1ArGTXYgjd/zQa5YSSOFvs18UGQ==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/assertion-tools/-/assertion-tools-2.1.0.tgz", + "integrity": "sha512-HCI/K2G9x/PlTpj0XuB2K3gR09C5VSXfqOfhzeqONC1x3WHRlUOIZmSSodaA5uLamUE45x8pnHjJxSyqxOI7tA==", "requires": { "ethers": "^5.7.2", "jsonld": "^8.1.0", @@ -28382,14 +28382,14 @@ } }, "dkg.js": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/dkg.js/-/dkg.js-6.0.2.tgz", - "integrity": "sha512-F1+XGbOTQh//n5fOLpPJCZLNMlTl9Q//A5dAAdBxhA/cXjDWnMJOX2w98Fd2b2/Q4ZsWGuLnwdILqO8GmhG9bw==", + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/dkg.js/-/dkg.js-6.1.2.tgz", + "integrity": "sha512-flqu4wEDEVcMQnnO8WkuAUzeiEn/6wyTIDkRe72j8mwtClVHvA0+QGWrvuv2zfJJuDHx6e3wjrzhkuMdxjez9g==", "dev": true, "requires": { - "assertion-tools": "^2.0.2", + "assertion-tools": "^2.1.0", "axios": "^0.27.2", - "dkg-evm-module": "^4.0.4", + "dkg-evm-module": "^4.1.0", "ethers": "^6.1.0", "jsonld": "^8.1.0", "web3": "^1.7.3" diff --git a/package.json b/package.json index b6b88d8776..83890b211d 100644 --- a/package.json +++ b/package.json @@ -50,7 +50,7 @@ "chai": "^4.3.6", "d3": "^7.8.5", "d3-node": "^3.0.0", - "dkg.js": "^6.0.2", + "dkg.js": "^6.1.2", "eslint": "^8.23.0", "eslint-config-airbnb": "^19.0.4", "eslint-config-prettier": "^8.5.0", From b8b4e0b9cafffd5a423e0a5505b5835c35663857 Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Mon, 22 Jan 2024 20:28:47 +0100 Subject: [PATCH 26/96] Fix getBidSuggestion tests --- test/bdd/features/bid-suggestion.feature | 20 +++++++++++++++++++- test/bdd/steps/api/bid-suggestion.mjs | 24 ++++++++++-------------- 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/test/bdd/features/bid-suggestion.feature b/test/bdd/features/bid-suggestion.feature index 7e3b1ed245..544ec47240 100644 --- a/test/bdd/features/bid-suggestion.feature +++ b/test/bdd/features/bid-suggestion.feature @@ -16,4 +16,22 @@ Feature: Release related tests Then I call Info route on the node 2 When I call Get Bid Suggestion on the node 2 with validPublish_1ForValidUpdate_1 on blockchain hardhat2:31337 - Then I call Info route on the node 2 \ No newline at end of file + Then I call Info route on the node 2 + + When I call Get Bid Suggestion on node 2 using parameters validPublish_1ForValidUpdate_1, hashFunctionId 1, scoreFunctionId 1, within blockchain hardhat1:31337 + Then I call Info route on the node 2 + + When I call Get Bid Suggestion on node 2 using parameters validPublish_1ForValidUpdate_1, hashFunctionId 1, scoreFunctionId 1, within blockchain hardhat2:31337 + Then I call Info route on the node 2 + + When I call Get Bid Suggestion on node 2 using parameters validPublish_1ForValidUpdate_1, hashFunctionId 1, scoreFunctionId 2, within blockchain hardhat1:31337 + Then I call Info route on the node 2 + + When I call Get Bid Suggestion on node 2 using parameters validPublish_1ForValidUpdate_1, hashFunctionId 1, scoreFunctionId 2, within blockchain hardhat2:31337 + Then I call Info route on the node 2 + + When I call Get Bid Suggestion on node 2 using parameters validPublish_1ForValidUpdate_1, hashFunctionId 1, scoreFunctionId 4, within blockchain hardhat1:31337 + Then I call Info route on the node 2 + + When I call Get Bid Suggestion on node 2 using parameters validPublish_1ForValidUpdate_1, hashFunctionId 1, scoreFunctionId 4, within blockchain hardhat2:31337 + Then I call Info route on the node 2 diff --git a/test/bdd/steps/api/bid-suggestion.mjs b/test/bdd/steps/api/bid-suggestion.mjs index c6f93ae630..72d27845e5 100644 --- a/test/bdd/steps/api/bid-suggestion.mjs +++ b/test/bdd/steps/api/bid-suggestion.mjs @@ -5,17 +5,17 @@ import { readFile } from 'fs/promises'; const assertions = JSON.parse(await readFile('test/bdd/steps/api/datasets/assertions.json')); When( - /^I call Get Bid Suggestion on the node (\d+) with ([^"]*) on blockchain ([^"]*) with hashFunctionId (\d+) and scoreFunctionId (\d+)/, + /^I call Get Bid Suggestion on node (\d+) using parameters ([^"]*), hashFunctionId (\d+), scoreFunctionId (\d+), within blockchain ([^"]*)/, { timeout: 300000 }, - async function getBidSuggestion( + async function getBidSuggestionWithHashAndScore( node, assertionName, - blockchain, hashFunctionId, scoreFunctionId, + blockchain, ) { this.logger.log( - `I call get bid suggestion route on the node ${node} on blockchain ${blockchain}`, + `I call get bid suggestion route on the node ${node} on blockchain ${blockchain} with hashFunctionId ${hashFunctionId} and scoreFunctionId ${scoreFunctionId}`, ); expect( @@ -29,17 +29,16 @@ When( ).to.be.equal(true); expect( - !Number.isInteger(hashFunctionId), + Number.isInteger(hashFunctionId), `hashFunctionId value: ${hashFunctionId} is not an integer!`, ).to.be.equal(true); expect( - !Number.isInteger(scoreFunctionId), + Number.isInteger(scoreFunctionId), `scoreFunctionId value: ${scoreFunctionId} is not an integer!`, ).to.be.equal(true); const assertion = assertions[assertionName]; - const publicAssertionId = await this.state.nodes[node - 1].client .getPublicAssertionId(assertion) .catch((error) => { @@ -53,9 +52,9 @@ When( }); const options = { - blockchain: this.state.nodes[node - 1].clientBlockchainOptions[blockchain], - hashFunctionId, - scoreFunctionId, + ...this.state.nodes[node - 1].clientBlockchainOptions[blockchain], + hashFunctionId: hashFunctionId, + scoreFunctionId: scoreFunctionId, }; let getBidSuggestionError; const result = await this.state.nodes[node - 1].client @@ -94,7 +93,6 @@ When( ).to.be.equal(true); const assertion = assertions[assertionName]; - const publicAssertionId = await this.state.nodes[node - 1].client .getPublicAssertionId(assertion) .catch((error) => { @@ -107,9 +105,7 @@ When( assert.fail(`Error while trying to get size in bytes. ${error}`); }); - const options = { - blockchain: this.state.nodes[node - 1].clientBlockchainOptions[blockchain], - }; + const options = this.state.nodes[node - 1].clientBlockchainOptions[blockchain]; let getBidSuggestionError; const result = await this.state.nodes[node - 1].client .getBidSuggestion(publicAssertionId, sizeInBytes, options) From 05498576bf532b3d7671166812a410544cfb04e0 Mon Sep 17 00:00:00 2001 From: Uladzislau Hubar Date: Wed, 24 Jan 2024 10:30:16 +0100 Subject: [PATCH 27/96] Removed clockwise distance calculation in proximity scoring service --- src/service/proximity-scoring-service.js | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/src/service/proximity-scoring-service.js b/src/service/proximity-scoring-service.js index d9ebe47ded..6cc958ff93 100644 --- a/src/service/proximity-scoring-service.js +++ b/src/service/proximity-scoring-service.js @@ -10,8 +10,7 @@ class ProximityScoringService { this.proximityScoreFunctionsPairs = { 1: [this.calculateBinaryXOR.bind(this), this.Log2PLDSF.bind(this)], - 2: [this.calculateClockwiseProximityOnHashRing.bind(this), this.LinearSum.bind(this)], - 3: [ + 2: [ this.calculateBidirectionalProximityOnHashRing.bind(this), this.LinearSum.bind(this), ], @@ -56,27 +55,6 @@ class ProximityScoringService { return this.blockchainModuleManager.toBigNumber(blockchain, distanceHex); } - async calculateClockwiseProximityOnHashRing(blockchain, peerHash, keyHash) { - const peerPositionOnHashRing = await this.blockchainModuleManager.toBigNumber( - blockchain, - peerHash, - ); - const keyPositionOnHashRing = await this.blockchainModuleManager.toBigNumber( - blockchain, - keyHash, - ); - - let clockwiseDistance; - if (peerPositionOnHashRing.gt(keyPositionOnHashRing)) { - const distanceToEnd = HASH_RING_SIZE.sub(peerPositionOnHashRing); - clockwiseDistance = distanceToEnd.add(keyPositionOnHashRing); - } else { - clockwiseDistance = keyPositionOnHashRing.sub(peerPositionOnHashRing); - } - - return clockwiseDistance; - } - async calculateBidirectionalProximityOnHashRing(blockchain, peerHash, keyHash) { const peerPositionOnHashRing = await this.blockchainModuleManager.toBigNumber( blockchain, From 01592207c6c0011da0555ead96a84b97e1ab6c83 Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Wed, 24 Jan 2024 10:32:44 +0100 Subject: [PATCH 28/96] Use big number comparison --- src/service/sharding-table-service.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/service/sharding-table-service.js b/src/service/sharding-table-service.js index d39f624446..ffd83fdc97 100644 --- a/src/service/sharding-table-service.js +++ b/src/service/sharding-table-service.js @@ -158,10 +158,10 @@ class ShardingTableService { })), ); peersWithDistance.sort((a, b) => { - if (a.distance < b.distance) { + if (a.distance.lt(b.distance)) { return -1; } - if (a.distance > b.distance) { + if (a.distance.gt(b.distance)) { return 1; } return 0; From b89590d86e90dabe1b76888cee59aafd61a1ad35 Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Wed, 24 Jan 2024 16:40:40 +0100 Subject: [PATCH 29/96] Find asset neighbourhood edge --- .../protocols/common/epoch-check-command.js | 114 ++++++++++++++++-- .../protocols/common/submit-commit-command.js | 20 ++- .../blockchain/implementation/web3-service.js | 16 ++- src/service/sharding-table-service.js | 4 +- 4 files changed, 136 insertions(+), 18 deletions(-) diff --git a/src/commands/protocols/common/epoch-check-command.js b/src/commands/protocols/common/epoch-check-command.js index b4873d9524..3d3ac5169a 100644 --- a/src/commands/protocols/common/epoch-check-command.js +++ b/src/commands/protocols/common/epoch-check-command.js @@ -7,6 +7,7 @@ import { OPERATION_ID_STATUS, ERROR_TYPE, NODE_ENVIRONMENTS, + // HASH_RING_SIZE, } from '../../../constants/constants.js'; import MigrationExecutor from '../../../migration/migration-executor.js'; @@ -124,6 +125,15 @@ class EpochCheckCommand extends Command { for (const serviceAgreement of eligibleAgreementForSubmitCommit) { if (scheduleSubmitCommitCommands.length >= maxTransactions) break; + const neighbourhood = await this.shardingTableService.findNeighbourhood( + blockchain, + serviceAgreement.keyword, + r2, + serviceAgreement.hashFunctionId, + serviceAgreement.scoreFunctionId, + true, + ); + try { const rank = await this.calculateRank( blockchain, @@ -131,6 +141,7 @@ class EpochCheckCommand extends Command { serviceAgreement.hashFunctionId, serviceAgreement.scoreFunctionId, r2, + neighbourhood, ); updateServiceAgreementsLastCommitEpoch.push( @@ -165,9 +176,14 @@ class EpochCheckCommand extends Command { serviceAgreement.agreementId }. Scheduling submitCommitCommand.`, ); - + const neighbourhoodEdges = this.getNeighboorhoodEdges( + neighbourhood, + blockchain, + serviceAgreement.hashFunctionId, + serviceAgreement.scoreFunctionId, + ); scheduleSubmitCommitCommands.push( - this.scheduleSubmitCommitCommand(serviceAgreement), + this.scheduleSubmitCommitCommand(serviceAgreement, neighbourhoodEdges), ); } catch (error) { this.logger.warn( @@ -240,16 +256,14 @@ class EpochCheckCommand extends Command { ]); } - async calculateRank(blockchain, keyword, hashFunctionId, proximityScoreFunctionsPairId, r2) { - const neighbourhood = await this.shardingTableService.findNeighbourhood( - blockchain, - keyword, - r2, - hashFunctionId, - proximityScoreFunctionsPairId, - true, - ); - + async calculateRank( + blockchain, + keyword, + hashFunctionId, + proximityScoreFunctionsPairId, + r2, + neighbourhood, + ) { const peerId = this.networkModuleManager.getPeerId().toB58String(); if (!neighbourhood.some((node) => node.peerId === peerId)) { return; @@ -291,7 +305,7 @@ class EpochCheckCommand extends Command { return false; } - async scheduleSubmitCommitCommand(agreement) { + async scheduleSubmitCommitCommand(agreement, neighbourhoodEdges, closestNode) { const commandData = { operationId: this.operationIdService.generateId(), blockchain: agreement.blockchainId, @@ -302,6 +316,9 @@ class EpochCheckCommand extends Command { epoch: agreement.currentEpoch, agreementId: agreement.agreementId, stateIndex: agreement.stateIndex, + closestNode, + leftNeighborhoodEdge: neighbourhoodEdges.leftNeighborhoodEdge, + rightNeighborhoodEdge: neighbourhoodEdges.rightNeighborhoodEdge, }; await this.commandExecutor.add({ @@ -368,6 +385,77 @@ class EpochCheckCommand extends Command { return devEnvironment ? 30_000 : 120_000; } + async getNeighboorhoodEdges( + neighbourhood, + blockchainId, + hashFunctionId, + proximityScoreFunctionsPairId, + ) { + const closestNode = neighbourhood[0]; + + const hashFunctionName = this.hashingService.getHashFunctionName(hashFunctionId); + const nodesWithDistanceFromClosestNode = await Promise.all( + neighbourhood.map(async (node) => ({ + node, + distance: await this.proximityScoringService.callProximityFunction( + blockchainId, + proximityScoreFunctionsPairId, + node[hashFunctionName], + closestNode[hashFunctionName], + ), + })), + ); + + const maxDistanceFromClosestNode = Math.max( + ...nodesWithDistanceFromClosestNode.map((node) => node.distance), + ); + const firstEdge = nodesWithDistanceFromClosestNode.find( + (node) => node.distance === maxDistanceFromClosestNode, + )?.node; + + // const nodesWithDistanceFromFirstEdge = + await Promise.all( + neighbourhood.map(async (node) => ({ + node, + distance: await this.proximityScoringService.callProximityFunction( + blockchainId, + proximityScoreFunctionsPairId, + node[hashFunctionName], + firstEdge[hashFunctionName], + ), + })), + ); + + // const maxDistanceFromFirstEdge = Math.max( + // ...nodesWithDistanceFromFirstEdge.map((node) => node.distance), + // ); + // const secondEdge = nodesWithDistanceFromFirstEdge.find( + // (node) => node.distance === maxDistanceFromFirstEdge, + // )?.node; + + // const firstEgdePositionOnRing = await this.blockchainModuleManager.toBigNumber( + // blockchainId, + // firstEdge[hashFunctionId], + // ); + // const secondEdgePositionOnRing = await this.blockchainModuleManager.toBigNumber( + // blockchainId, + // secondEdge[hashFunctionId], + // ); + // const closestNodePositionOnRing = await this.blockchainModuleManager.toBigNumber( + // blockchainId, + // closestNode[hashFunctionId], + // ); + + // const firstEdgeClockWiseDistanceFromClossestNode = + // firstEgdePositionOnRing <= closestNodePositionOnRing + // ? closestNodePositionOnRing - firstEgdePositionOnRing + // : HASH_RING_SIZE - firstEgdePositionOnRing + closestNodePositionOnRing + 1; + // const secondEdgeClockWiseDistanceFromClossestNode = + // secondEdgePositionOnRing <= closestNodePositionOnRing + // ? closestNodePositionOnRing - secondEdgePositionOnRing + // : HASH_RING_SIZE - secondEdgePositionOnRing + closestNodePositionOnRing + 1; + } + /** * Recover system from failure * @param command diff --git a/src/commands/protocols/common/submit-commit-command.js b/src/commands/protocols/common/submit-commit-command.js index a0484d1c6c..4776202b32 100644 --- a/src/commands/protocols/common/submit-commit-command.js +++ b/src/commands/protocols/common/submit-commit-command.js @@ -28,6 +28,9 @@ class SubmitCommitCommand extends Command { agreementId, stateIndex, gasPrice, + closestNode, + leftNeighborhoodEdge, + rightNeighborhoodEdge, } = command.data; this.logger.trace( @@ -35,6 +38,8 @@ class SubmitCommitCommand extends Command { `Blockchain: ${blockchain}, Contract: ${contract}, Token ID: ${tokenId}, ` + `Keyword: ${keyword}, Hash function ID: ${hashFunctionId}, Epoch: ${epoch}, ` + `State Index: ${stateIndex}, Operation ID: ${operationId}, ` + + `Clossest Node: ${closestNode}, Left neighborhood edge: ${leftNeighborhoodEdge}, ` + + `Right neighborhood edge: ${rightNeighborhoodEdge}` + `Retry number: ${COMMAND_RETRIES.SUBMIT_COMMIT - command.retries + 1}`, ); @@ -60,7 +65,9 @@ class SubmitCommitCommand extends Command { `Knowledge Asset was updated, not sending Commit for the Service Agreement with the ID: ${agreementId}, ` + `Blockchain: ${blockchain}, Contract: ${contract}, Token ID: ${tokenId}, ` + `Keyword: ${keyword}, Hash function ID: ${hashFunctionId}, Epoch: ${epoch}, ` + - `State Index: ${stateIndex}, Operation ID: ${operationId}`, + `State Index: ${stateIndex}, Operation ID: ${operationId}` + + `Clossest Node: ${closestNode}, Left neighborhood edge: ${leftNeighborhoodEdge}, ` + + `Right neighborhood edge: ${rightNeighborhoodEdge}`, ); return Command.empty(); @@ -78,7 +85,9 @@ class SubmitCommitCommand extends Command { `Commit has already been submitted for the Service Agreement with the ID: ${agreementId}, ` + `Blockchain: ${blockchain}, Contract: ${contract}, Token ID: ${tokenId}, ` + `Keyword: ${keyword}, Hash function ID: ${hashFunctionId}, Epoch: ${epoch}, ` + - `State Index: ${stateIndex}, Operation ID: ${operationId}`, + `State Index: ${stateIndex}, Operation ID: ${operationId}` + + `Clossest Node: ${closestNode}, Left neighborhood edge: ${leftNeighborhoodEdge}, ` + + `Right neighborhood edge: ${rightNeighborhoodEdge}`, ); return Command.empty(); @@ -94,6 +103,9 @@ class SubmitCommitCommand extends Command { keyword, hashFunctionId, epoch, + closestNode, + leftNeighborhoodEdge, + rightNeighborhoodEdge, stateIndex, (result) => { if (result?.error) { @@ -119,6 +131,8 @@ class SubmitCommitCommand extends Command { `with the ID: ${agreementId}, Blockchain: ${blockchain}, Contract: ${contract}, ` + `Token ID: ${tokenId}, Keyword: ${keyword}, Hash function ID: ${hashFunctionId}, ` + `Epoch: ${epoch}, State Index: ${stateIndex}, Operation ID: ${operationId}, ` + + `Clossest Node: ${closestNode}, Left neighborhood edge: ${leftNeighborhoodEdge}, ` + + `Right neighborhood edge: ${rightNeighborhoodEdge}` + `Retry number: ${COMMAND_RETRIES.SUBMIT_COMMIT - command.retries + 1}.`, ); @@ -160,6 +174,8 @@ class SubmitCommitCommand extends Command { `Blockchain: ${blockchain}, Contract: ${contract}, Token ID: ${tokenId}, ` + `Keyword: ${keyword}, Hash function ID: ${hashFunctionId}, Epoch: ${epoch}, ` + `State Index: ${stateIndex}, Operation ID: ${operationId}, ` + + `Clossest Node: ${closestNode}, Left neighborhood edge: ${leftNeighborhoodEdge}, ` + + `Right neighborhood edge: ${rightNeighborhoodEdge}` + `Retry number: ${COMMAND_RETRIES.SUBMIT_COMMIT - command.retries + 1}`, ); diff --git a/src/modules/blockchain/implementation/web3-service.js b/src/modules/blockchain/implementation/web3-service.js index 363677ed1c..c831244e0d 100644 --- a/src/modules/blockchain/implementation/web3-service.js +++ b/src/modules/blockchain/implementation/web3-service.js @@ -981,6 +981,9 @@ class Web3Service { keyword, hashFunctionId, epoch, + closestNode, + leftNeighborhoodEdge, + rightNeighborhoodEdge, latestStateIndex, callback, gasPrice, @@ -988,7 +991,18 @@ class Web3Service { return this.queueTransaction( this.selectCommitManagerContract(latestStateIndex), 'submitCommit', - [[assetContractAddress, tokenId, keyword, hashFunctionId, epoch]], + [ + [ + assetContractAddress, + tokenId, + keyword, + hashFunctionId, + epoch, + closestNode, + leftNeighborhoodEdge, + rightNeighborhoodEdge, + ], + ], callback, gasPrice, ); diff --git a/src/service/sharding-table-service.js b/src/service/sharding-table-service.js index d39f624446..ffd83fdc97 100644 --- a/src/service/sharding-table-service.js +++ b/src/service/sharding-table-service.js @@ -158,10 +158,10 @@ class ShardingTableService { })), ); peersWithDistance.sort((a, b) => { - if (a.distance < b.distance) { + if (a.distance.lt(b.distance)) { return -1; } - if (a.distance > b.distance) { + if (a.distance.gt(b.distance)) { return 1; } return 0; From c14ecf8ff6a690c70f8168b66f057063187cdd44 Mon Sep 17 00:00:00 2001 From: Djordje Kovacevic Date: Thu, 25 Jan 2024 10:12:48 +0100 Subject: [PATCH 30/96] Added new migration --- ot-node.js | 6 +++++ src/migration/migration-executor.js | 24 +++++++++++++++++++ ...-agreements-for-chiado-devnet-migration.js | 18 ++++++++++++++ .../service-agreement-repository.js | 8 +++++++ .../repository/repository-module-manager.js | 6 +++++ 5 files changed, 62 insertions(+) create mode 100644 src/migration/remove-service-agreements-for-chiado-devnet-migration.js diff --git a/ot-node.js b/ot-node.js index 6bdcf4fa9b..a8916ce8b7 100644 --- a/ot-node.js +++ b/ot-node.js @@ -49,6 +49,12 @@ class OTNode { await this.initializeModules(); + await MigrationExecutor.executeRemoveServiceAgreementsForChiadoDevnetMigration( + this.container, + this.logger, + this.config, + ); + await this.createProfiles(); await this.initializeCommandExecutor(); diff --git a/src/migration/migration-executor.js b/src/migration/migration-executor.js index 847a9308cf..e24dc97869 100644 --- a/src/migration/migration-executor.js +++ b/src/migration/migration-executor.js @@ -15,6 +15,7 @@ import ServiceAgreementsInvalidDataMigration from './service-agreements-invalid- import UalExtensionUserConfigurationMigration from './ual-extension-user-configuration-migration.js'; import UalExtensionTripleStoreMigration from './ual-extension-triple-store-migration.js'; import MarkStakingEventsAsProcessedMigration from './mark-staking-events-as-processed-migration.js'; +import RemoveServiceAgreementsForChiadoDevnetMigration from './remove-service-agreements-for-chiado-devnet-migration.js'; class MigrationExecutor { static async executePullShardingTableMigration(container, logger, config) { @@ -364,6 +365,29 @@ class MigrationExecutor { } } + static async executeRemoveServiceAgreementsForChiadoDevnetMigration(container, logger, config) { + if (process.env.NODE_ENV === NODE_ENVIRONMENTS.DEVNET) { + const repositoryModuleManager = container.resolve('repositoryModuleManager'); + + const migration = new RemoveServiceAgreementsForChiadoDevnetMigration( + 'removeServiceAgreementsForChiadoDevnetMigration', + logger, + config, + repositoryModuleManager, + ); + if (!(await migration.migrationAlreadyExecuted())) { + try { + await migration.migrate(); + } catch (error) { + logger.error( + `Unable to execute remove service agreements for Chiado Devnet migration. Error: ${error.message}`, + ); + this.exitNode(1); + } + } + } + } + static exitNode(code = 0) { process.exit(code); } diff --git a/src/migration/remove-service-agreements-for-chiado-devnet-migration.js b/src/migration/remove-service-agreements-for-chiado-devnet-migration.js new file mode 100644 index 0000000000..7c8dab87c5 --- /dev/null +++ b/src/migration/remove-service-agreements-for-chiado-devnet-migration.js @@ -0,0 +1,18 @@ +import BaseMigration from './base-migration.js'; + +const GNOSIS_DEVNET_CHAIN_ID = 'gnosis:10200'; + +class RemoveServiceAgreementsForChiadoDevnetMigration extends BaseMigration { + constructor(migrationName, logger, config, repositoryModuleManager) { + super(migrationName, logger, config); + this.repositoryModuleManager = repositoryModuleManager; + } + + async executeMigration() { + await this.repositoryModuleManager.removeServiceAgreementsForBlockchain( + GNOSIS_DEVNET_CHAIN_ID, + ); + } +} + +export default RemoveServiceAgreementsForChiadoDevnetMigration; diff --git a/src/modules/repository/implementation/sequelize/repositories/service-agreement-repository.js b/src/modules/repository/implementation/sequelize/repositories/service-agreement-repository.js index deec201839..1e8124f3cf 100644 --- a/src/modules/repository/implementation/sequelize/repositories/service-agreement-repository.js +++ b/src/modules/repository/implementation/sequelize/repositories/service-agreement-repository.js @@ -21,6 +21,14 @@ class ServiceAgreementRepository { }); } + async removeServiceAgreementsForBlockchain(blockchainId) { + await this.model.destroy({ + where: { + blockchainId, + }, + }); + } + async updateServiceAgreementRecord( blockchainId, assetStorageContractAddress, diff --git a/src/modules/repository/repository-module-manager.js b/src/modules/repository/repository-module-manager.js index 0a58b7570a..11e610d824 100644 --- a/src/modules/repository/repository-module-manager.js +++ b/src/modules/repository/repository-module-manager.js @@ -416,6 +416,12 @@ class RepositoryModuleManager extends BaseModuleManager { return this.getRepository('service_agreement').removeServiceAgreements(agreementIds); } + async removeServiceAgreementsForBlockchain(blockchainId) { + return this.getRepository('service_agreement').removeServiceAgreementsForBlockchain( + blockchainId, + ); + } + async updateServiceAgreementEpochsNumber(agreementId, epochsNumber) { return this.getRepository('service_agreement').updateServiceAgreementEpochsNumber( agreementId, From 3fdbeb5ff5e8333aff34649e9382206a80d2d3b0 Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Thu, 25 Jan 2024 10:22:22 +0100 Subject: [PATCH 31/96] Implement findEdges method --- .../protocols/common/epoch-check-command.js | 107 +++++++----------- 1 file changed, 43 insertions(+), 64 deletions(-) diff --git a/src/commands/protocols/common/epoch-check-command.js b/src/commands/protocols/common/epoch-check-command.js index 3d3ac5169a..dbc199a06c 100644 --- a/src/commands/protocols/common/epoch-check-command.js +++ b/src/commands/protocols/common/epoch-check-command.js @@ -7,7 +7,6 @@ import { OPERATION_ID_STATUS, ERROR_TYPE, NODE_ENVIRONMENTS, - // HASH_RING_SIZE, } from '../../../constants/constants.js'; import MigrationExecutor from '../../../migration/migration-executor.js'; @@ -21,6 +20,8 @@ class EpochCheckCommand extends Command { this.blockchainModuleManager = ctx.blockchainModuleManager; this.serviceAgreementService = ctx.serviceAgreementService; this.fileService = ctx.fileService; + this.proximityScoringService = ctx.proximityScoringService; + this.hashingService = ctx.hashingService; this.errorType = ERROR_TYPE.COMMIT_PROOF.EPOCH_CHECK_ERROR; } @@ -176,11 +177,12 @@ class EpochCheckCommand extends Command { serviceAgreement.agreementId }. Scheduling submitCommitCommand.`, ); - const neighbourhoodEdges = this.getNeighboorhoodEdges( + const neighbourhoodEdges = this.getNeighboorhoodEdgeNodes( neighbourhood, blockchain, serviceAgreement.hashFunctionId, serviceAgreement.scoreFunctionId, + serviceAgreement.keyword, ); scheduleSubmitCommitCommands.push( this.scheduleSubmitCommitCommand(serviceAgreement, neighbourhoodEdges), @@ -316,9 +318,9 @@ class EpochCheckCommand extends Command { epoch: agreement.currentEpoch, agreementId: agreement.agreementId, stateIndex: agreement.stateIndex, - closestNode, - leftNeighborhoodEdge: neighbourhoodEdges.leftNeighborhoodEdge, - rightNeighborhoodEdge: neighbourhoodEdges.rightNeighborhoodEdge, + closestNode: closestNode.peerId, + leftNeighborhoodEdge: neighbourhoodEdges.leftNeighborhoodEdge.peerId, + rightNeighborhoodEdge: neighbourhoodEdges.rightNeighborhoodEdge.peerId, }; await this.commandExecutor.add({ @@ -385,75 +387,52 @@ class EpochCheckCommand extends Command { return devEnvironment ? 30_000 : 120_000; } - async getNeighboorhoodEdges( + async getNeighboorhoodEdgeNodes( neighbourhood, blockchainId, hashFunctionId, proximityScoreFunctionsPairId, + assetHash, ) { - const closestNode = neighbourhood[0]; - const hashFunctionName = this.hashingService.getHashFunctionName(hashFunctionId); - const nodesWithDistanceFromClosestNode = await Promise.all( - neighbourhood.map(async (node) => ({ - node, - distance: await this.proximityScoringService.callProximityFunction( - blockchainId, - proximityScoreFunctionsPairId, - node[hashFunctionName], - closestNode[hashFunctionName], - ), - })), + const assetPositionOnHashRing = await this.blockchainModuleManager.toBigNumber( + blockchainId, + assetHash, ); + const hashRing = [assetPositionOnHashRing]; - const maxDistanceFromClosestNode = Math.max( - ...nodesWithDistanceFromClosestNode.map((node) => node.distance), - ); - const firstEdge = nodesWithDistanceFromClosestNode.find( - (node) => node.distance === maxDistanceFromClosestNode, - )?.node; - - // const nodesWithDistanceFromFirstEdge = - await Promise.all( - neighbourhood.map(async (node) => ({ - node, - distance: await this.proximityScoringService.callProximityFunction( - blockchainId, - proximityScoreFunctionsPairId, - node[hashFunctionName], - firstEdge[hashFunctionName], - ), - })), + const maxDistance = this.proximityScoringService.callProximityFunction( + blockchainId, + proximityScoreFunctionsPairId, + neighbourhood[neighbourhood.length - 1], + assetHash, ); + for (const neighbour of neighbourhood) { + const neighbourPositionOnHashRing = await this.blockchainModuleManager.toBigNumber( + blockchainId, + neighbour[hashFunctionName], + ); + if (assetPositionOnHashRing.lte(neighbourPositionOnHashRing)) { + if (neighbourPositionOnHashRing.sub(assetPositionOnHashRing).lt(maxDistance)) { + hashRing.push(neighbourPositionOnHashRing); + } else { + hashRing.unshift(neighbourPositionOnHashRing); + } + } else if (assetPositionOnHashRing.gt(neighbourPositionOnHashRing)) { + if (assetPositionOnHashRing.sub(neighbourPositionOnHashRing).lt(maxDistance)) { + hashRing.unshift(neighbourPositionOnHashRing); + } else { + hashRing.push(neighbourPositionOnHashRing); + } + } + } - // const maxDistanceFromFirstEdge = Math.max( - // ...nodesWithDistanceFromFirstEdge.map((node) => node.distance), - // ); - // const secondEdge = nodesWithDistanceFromFirstEdge.find( - // (node) => node.distance === maxDistanceFromFirstEdge, - // )?.node; - - // const firstEgdePositionOnRing = await this.blockchainModuleManager.toBigNumber( - // blockchainId, - // firstEdge[hashFunctionId], - // ); - // const secondEdgePositionOnRing = await this.blockchainModuleManager.toBigNumber( - // blockchainId, - // secondEdge[hashFunctionId], - // ); - // const closestNodePositionOnRing = await this.blockchainModuleManager.toBigNumber( - // blockchainId, - // closestNode[hashFunctionId], - // ); - - // const firstEdgeClockWiseDistanceFromClossestNode = - // firstEgdePositionOnRing <= closestNodePositionOnRing - // ? closestNodePositionOnRing - firstEgdePositionOnRing - // : HASH_RING_SIZE - firstEgdePositionOnRing + closestNodePositionOnRing + 1; - // const secondEdgeClockWiseDistanceFromClossestNode = - // secondEdgePositionOnRing <= closestNodePositionOnRing - // ? closestNodePositionOnRing - secondEdgePositionOnRing - // : HASH_RING_SIZE - secondEdgePositionOnRing + closestNodePositionOnRing + 1; + return { + leftEdge: neighbourhood.find((node) => node[hashFunctionName] === hashRing[0]), + rightEdge: neighbourhood.find( + (node) => node[hashFunctionName] === hashRing[hashRing.length() - 1], + ), + }; } /** From 5d53b5f58fa2e369f0ba68a9780549509f700cf1 Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Thu, 25 Jan 2024 11:05:11 +0100 Subject: [PATCH 32/96] Add index to neighborhood peers --- src/commands/protocols/common/epoch-check-command.js | 6 +++--- src/service/sharding-table-service.js | 5 ++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/commands/protocols/common/epoch-check-command.js b/src/commands/protocols/common/epoch-check-command.js index dbc199a06c..87fa492f74 100644 --- a/src/commands/protocols/common/epoch-check-command.js +++ b/src/commands/protocols/common/epoch-check-command.js @@ -318,9 +318,9 @@ class EpochCheckCommand extends Command { epoch: agreement.currentEpoch, agreementId: agreement.agreementId, stateIndex: agreement.stateIndex, - closestNode: closestNode.peerId, - leftNeighborhoodEdge: neighbourhoodEdges.leftNeighborhoodEdge.peerId, - rightNeighborhoodEdge: neighbourhoodEdges.rightNeighborhoodEdge.peerId, + closestNode: closestNode.index, + leftNeighborhoodEdge: neighbourhoodEdges.leftNeighborhoodEdge.index, + rightNeighborhoodEdge: neighbourhoodEdges.rightNeighborhoodEdge.index, }; await this.commandExecutor.add({ diff --git a/src/service/sharding-table-service.js b/src/service/sharding-table-service.js index ffd83fdc97..cc2495bcbf 100644 --- a/src/service/sharding-table-service.js +++ b/src/service/sharding-table-service.js @@ -120,10 +120,13 @@ class ShardingTableService { proximityScoreFunctionsPairId, filterLastSeen, ) { - const peers = await this.repositoryModuleManager.getAllPeerRecords( + let peers = await this.repositoryModuleManager.getAllPeerRecords( blockchainId, filterLastSeen, ); + + peers = peers.map((peer, index) => ({ ...peer, index })); + const keyHash = await this.hashingService.callHashFunction(hashFunctionId, key); const soretedPeers = this.sortPeers( From 6e5634902665ff490642235e57e85237cb4e79ad Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Thu, 25 Jan 2024 11:31:54 +0100 Subject: [PATCH 33/96] Fix index addition --- src/service/sharding-table-service.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/service/sharding-table-service.js b/src/service/sharding-table-service.js index cc2495bcbf..cb5b347804 100644 --- a/src/service/sharding-table-service.js +++ b/src/service/sharding-table-service.js @@ -125,7 +125,7 @@ class ShardingTableService { filterLastSeen, ); - peers = peers.map((peer, index) => ({ ...peer, index })); + peers = peers.map((peer, index) => ({ ...peer.dataValues, index })); const keyHash = await this.hashingService.callHashFunction(hashFunctionId, key); From ccfbb58813577b607e51c7d5b31b22d972de09da Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Mon, 29 Jan 2024 11:27:02 +0100 Subject: [PATCH 34/96] Store sha256 as blob in sql --- .../pull-sharding-table-migration.js | 10 +++++- .../blockchain/implementation/web3-service.js | 6 ++-- .../20240126120000-shard-add-sha256blobl.js | 31 +++++++++++++++++++ .../implementation/sequelize/models/shard.js | 3 ++ .../repositories/shard-repository.js | 6 +++- .../blockchain-event-listener-service.js | 9 ++++-- src/service/proximity-scoring-service.js | 4 +-- src/service/sharding-table-service.js | 8 +++-- 8 files changed, 64 insertions(+), 13 deletions(-) create mode 100644 src/modules/repository/implementation/sequelize/migrations/20240126120000-shard-add-sha256blobl.js diff --git a/src/migration/pull-sharding-table-migration.js b/src/migration/pull-sharding-table-migration.js index f382fdcae5..9b0673c33d 100644 --- a/src/migration/pull-sharding-table-migration.js +++ b/src/migration/pull-sharding-table-migration.js @@ -64,6 +64,13 @@ class PullBlockchainShardingTableMigration extends BaseMigration { peer.nodeId, ); + const sha256 = await this.hashingService.callHashFunction(1, nodeId); + + const cleanHexString = sha256.startsWith('0x') + ? sha256.slice(2) + : sha256; + const sha256Blob = Buffer.from(cleanHexString, 'hex'); + return { peerId: nodeId, blockchainId, @@ -77,7 +84,8 @@ class PullBlockchainShardingTableMigration extends BaseMigration { peer.stake, 'ether', ), - sha256: await this.hashingService.callHashFunction(1, nodeId), + sha256, + sha256Blob, }; }), ), diff --git a/src/modules/blockchain/implementation/web3-service.js b/src/modules/blockchain/implementation/web3-service.js index c831244e0d..7d5808b660 100644 --- a/src/modules/blockchain/implementation/web3-service.js +++ b/src/modules/blockchain/implementation/web3-service.js @@ -41,12 +41,12 @@ const ABIs = { ProfileStorage: require('dkg-evm-module/abi/ProfileStorage.json'), ScoringProxy: require('dkg-evm-module/abi/ScoringProxy.json'), ServiceAgreementV1: require('dkg-evm-module/abi/ServiceAgreementV1.json'), - CommitManagerV1: require('dkg-evm-module/abi/CommitManagerV1.json'), + CommitManagerV1: require('dkg-evm-module/abi/CommitManagerV2.json'), CommitManagerV1U1: require('dkg-evm-module/abi/CommitManagerV1U1.json'), ProofManagerV1: require('dkg-evm-module/abi/ProofManagerV1.json'), ProofManagerV1U1: require('dkg-evm-module/abi/ProofManagerV1U1.json'), - ShardingTable: require('dkg-evm-module/abi/ShardingTable.json'), - ShardingTableStorage: require('dkg-evm-module/abi/ShardingTableStorage.json'), + ShardingTable: require('dkg-evm-module/abi/ShardingTableV2.json'), + ShardingTableStorage: require('dkg-evm-module/abi/ShardingTableStorageV2.json'), ServiceAgreementStorageProxy: require('dkg-evm-module/abi/ServiceAgreementStorageProxy.json'), UnfinalizedStateStorage: require('dkg-evm-module/abi/UnfinalizedStateStorage.json'), }; diff --git a/src/modules/repository/implementation/sequelize/migrations/20240126120000-shard-add-sha256blobl.js b/src/modules/repository/implementation/sequelize/migrations/20240126120000-shard-add-sha256blobl.js new file mode 100644 index 0000000000..7db8aceed3 --- /dev/null +++ b/src/modules/repository/implementation/sequelize/migrations/20240126120000-shard-add-sha256blobl.js @@ -0,0 +1,31 @@ +export async function up({ context: { queryInterface, Sequelize } }) { + await queryInterface.addColumn('shard', 'sha256_blob', { + type: Sequelize.BLOB, + }); + + const shards = await queryInterface.sequelize.query( + 'SELECT peer_id, blockchain_id, sha256 FROM shard', + { type: queryInterface.sequelize.QueryTypes.SELECT }, + ); + + for (const shard of shards) { + const sha256Blob = Buffer.from(shard.sha256, 'hex'); + + // eslint-disable-next-line no-await-in-loop + await queryInterface.sequelize.query( + 'UPDATE shard SET sha256_blob = :sha256Blob WHERE peer_id = :peerId AND blockchain_id = :blockchainId', + { + replacements: { + sha256Blob, + peerId: shard.peerId, + blockchainId: shard.blockchainId, + }, + type: queryInterface.sequelize.QueryTypes.UPDATE, + }, + ); + } +} + +export async function down({ context: { queryInterface } }) { + await queryInterface.removeColumn('shard', 'sha256_blob'); +} diff --git a/src/modules/repository/implementation/sequelize/models/shard.js b/src/modules/repository/implementation/sequelize/models/shard.js index cd14aa539f..7e5b884e29 100644 --- a/src/modules/repository/implementation/sequelize/models/shard.js +++ b/src/modules/repository/implementation/sequelize/models/shard.js @@ -26,6 +26,9 @@ export default (sequelize, DataTypes) => { type: DataTypes.STRING, allowNull: false, }, + sha256Blob: { + type: DataTypes.BLOB, + }, }, { underscored: true }, ); diff --git a/src/modules/repository/implementation/sequelize/repositories/shard-repository.js b/src/modules/repository/implementation/sequelize/repositories/shard-repository.js index 08dc04df82..661a220de1 100644 --- a/src/modules/repository/implementation/sequelize/repositories/shard-repository.js +++ b/src/modules/repository/implementation/sequelize/repositories/shard-repository.js @@ -19,7 +19,7 @@ class ShardRepository { }); } - async createPeerRecord(peerId, blockchainId, ask, stake, lastSeen, sha256) { + async createPeerRecord(peerId, blockchainId, ask, stake, lastSeen, sha256, sha256Blob) { return this.model.create( { peerId, @@ -28,6 +28,7 @@ class ShardRepository { stake, lastSeen, sha256, + sha256Blob, }, { ignoreDuplicates: true, @@ -40,6 +41,7 @@ class ShardRepository { where: { blockchainId, }, + order: [['sha256Blob', 'ASC']], }; if (filterLastSeen) { @@ -144,6 +146,8 @@ class ShardRepository { where: blockchainId ? { blockchainId } : {}, }); } + + async; } export default ShardRepository; diff --git a/src/service/blockchain-event-listener-service.js b/src/service/blockchain-event-listener-service.js index 6bcee0206c..22d5f0a32a 100644 --- a/src/service/blockchain-event-listener-service.js +++ b/src/service/blockchain-event-listener-service.js @@ -303,12 +303,14 @@ class BlockchainEventListenerService { eventData.nodeId, ); - const nodeIdSha256 = await this.hashingService.callHashFunction( - // TODO: How to add more hashes? + const sha256 = await this.hashingService.callHashFunction( CONTENT_ASSET_HASH_FUNCTION_ID, nodeId, ); + const cleanHexString = sha256.startsWith('0x') ? sha256.slice(2) : sha256; + const sha256Blob = Buffer.from(cleanHexString, 'hex'); + this.logger.trace(`Adding peer id: ${nodeId} to sharding table.`); return { peerId: nodeId, @@ -322,7 +324,8 @@ class BlockchainEventListenerService { eventData.stake, ), lastSeen: new Date(0), - sha256: nodeIdSha256, + sha256, + sha256Blob, }; }), ); diff --git a/src/service/proximity-scoring-service.js b/src/service/proximity-scoring-service.js index 6cc958ff93..8b02fbad77 100644 --- a/src/service/proximity-scoring-service.js +++ b/src/service/proximity-scoring-service.js @@ -12,7 +12,7 @@ class ProximityScoringService { 1: [this.calculateBinaryXOR.bind(this), this.Log2PLDSF.bind(this)], 2: [ this.calculateBidirectionalProximityOnHashRing.bind(this), - this.LinearSum.bind(this), + this.linearSum.bind(this), ], }; } @@ -110,7 +110,7 @@ class ProximityScoringService { }; } - async LinearSum(blockchain, distance, stake, maxNeighborhoodDistance) { + async linearSum(blockchain, distance, stake, maxNeighborhoodDistance) { const linearSumParams = await this.blockchainModuleManager.getLinearSumParams(blockchain); const { distanceScaleFactor, w1, w2 } = linearSumParams; const minimumStake = await this.blockchainModuleManager.getMinimumStake(blockchain); diff --git a/src/service/sharding-table-service.js b/src/service/sharding-table-service.js index cb5b347804..12451d918a 100644 --- a/src/service/sharding-table-service.js +++ b/src/service/sharding-table-service.js @@ -91,7 +91,10 @@ class ShardingTableService { blockchainId, peer.nodeId, ); + const sha256 = await this.hashingService.callHashFunction(1, nodeId); + const cleanHexString = sha256.startsWith('0x') ? sha256.slice(2) : sha256; + const sha256Blob = Buffer.from(cleanHexString, 'hex'); return { peerId: nodeId, blockchainId, @@ -105,7 +108,8 @@ class ShardingTableService { peer.stake, 'ether', ), - sha256: await this.hashingService.callHashFunction(1, nodeId), + sha256, + sha256Blob, }; }), ), @@ -124,9 +128,7 @@ class ShardingTableService { blockchainId, filterLastSeen, ); - peers = peers.map((peer, index) => ({ ...peer.dataValues, index })); - const keyHash = await this.hashingService.callHashFunction(hashFunctionId, key); const soretedPeers = this.sortPeers( From 8b0525056e81beed57096d1bb9cd97ed42f21192 Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Mon, 29 Jan 2024 12:25:48 +0100 Subject: [PATCH 35/96] Add liearSum params get --- .../protocols/common/epoch-check-command.js | 19 ++++++++++++------- .../blockchain/blockchain-module-manager.js | 4 ++++ .../blockchain/implementation/web3-service.js | 15 +++++++++++++++ 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/src/commands/protocols/common/epoch-check-command.js b/src/commands/protocols/common/epoch-check-command.js index 87fa492f74..6d55688c10 100644 --- a/src/commands/protocols/common/epoch-check-command.js +++ b/src/commands/protocols/common/epoch-check-command.js @@ -135,6 +135,14 @@ class EpochCheckCommand extends Command { true, ); + const neighbourhoodEdges = this.getNeighboorhoodEdgeNodes( + neighbourhood, + blockchain, + serviceAgreement.hashFunctionId, + serviceAgreement.scoreFunctionId, + serviceAgreement.keyword, + ); + try { const rank = await this.calculateRank( blockchain, @@ -143,6 +151,7 @@ class EpochCheckCommand extends Command { serviceAgreement.scoreFunctionId, r2, neighbourhood, + neighbourhoodEdges, ); updateServiceAgreementsLastCommitEpoch.push( @@ -177,13 +186,7 @@ class EpochCheckCommand extends Command { serviceAgreement.agreementId }. Scheduling submitCommitCommand.`, ); - const neighbourhoodEdges = this.getNeighboorhoodEdgeNodes( - neighbourhood, - blockchain, - serviceAgreement.hashFunctionId, - serviceAgreement.scoreFunctionId, - serviceAgreement.keyword, - ); + scheduleSubmitCommitCommands.push( this.scheduleSubmitCommitCommand(serviceAgreement, neighbourhoodEdges), ); @@ -271,6 +274,8 @@ class EpochCheckCommand extends Command { return; } + // const maxNeighborhoodDistance = this.proximityScoringService.g + const scores = await Promise.all( neighbourhood.map(async (node) => ({ score: await this.serviceAgreementService.calculateScore( diff --git a/src/modules/blockchain/blockchain-module-manager.js b/src/modules/blockchain/blockchain-module-manager.js index 447811c5ea..0252e1940f 100644 --- a/src/modules/blockchain/blockchain-module-manager.js +++ b/src/modules/blockchain/blockchain-module-manager.js @@ -409,6 +409,10 @@ class BlockchainModuleManager extends BaseModuleManager { convertUint8ArrayToHex(blockchain, uint8Array) { return this.callImplementationFunction(blockchain, 'convertUint8ArrayToHex', [uint8Array]); } + + getLinearSumParams(blockchain) { + return this.callImplementationFunction(blockchain, 'getLinearSumParams'); + } } export default BlockchainModuleManager; diff --git a/src/modules/blockchain/implementation/web3-service.js b/src/modules/blockchain/implementation/web3-service.js index 7d5808b660..39b77ec081 100644 --- a/src/modules/blockchain/implementation/web3-service.js +++ b/src/modules/blockchain/implementation/web3-service.js @@ -49,10 +49,12 @@ const ABIs = { ShardingTableStorage: require('dkg-evm-module/abi/ShardingTableStorageV2.json'), ServiceAgreementStorageProxy: require('dkg-evm-module/abi/ServiceAgreementStorageProxy.json'), UnfinalizedStateStorage: require('dkg-evm-module/abi/UnfinalizedStateStorage.json'), + LinearSum: require('dkg-evm-module/abi/LinearSum.json'), }; const SCORING_FUNCTIONS = { 1: 'Log2PLDSF', + 2: 'LinearSum', }; class Web3Service { @@ -1251,6 +1253,19 @@ class Web3Service { [agreementId], ); } + + async getLinearSumParams() { + const linearSumParams = await this.callContractFunction( + this.scoringFunctionsContracts[2], + 'getParameters', + [], + ); + return { + distanceScaleFactor: linearSumParams[0], + w1: linearSumParams[1], + w2: linearSumParams[2], + }; + } } export default Web3Service; From 55af92e3492f925e75855c292f8acc409b31127b Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Mon, 29 Jan 2024 13:17:11 +0100 Subject: [PATCH 36/96] Get rank fix --- .../protocols/common/epoch-check-command.js | 31 ++++++++++++++----- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/src/commands/protocols/common/epoch-check-command.js b/src/commands/protocols/common/epoch-check-command.js index 6d55688c10..336032acb1 100644 --- a/src/commands/protocols/common/epoch-check-command.js +++ b/src/commands/protocols/common/epoch-check-command.js @@ -135,7 +135,7 @@ class EpochCheckCommand extends Command { true, ); - const neighbourhoodEdges = this.getNeighboorhoodEdgeNodes( + const neighbourhoodEdges = await this.getNeighboorhoodEdgeNodes( neighbourhood, blockchain, serviceAgreement.hashFunctionId, @@ -268,13 +268,21 @@ class EpochCheckCommand extends Command { proximityScoreFunctionsPairId, r2, neighbourhood, + neighbourhoodEdges, ) { const peerId = this.networkModuleManager.getPeerId().toB58String(); if (!neighbourhood.some((node) => node.peerId === peerId)) { return; } - // const maxNeighborhoodDistance = this.proximityScoringService.g + const hashFunctionName = this.hashingService.getHashFunctionName(hashFunctionId); + + const maxNeighborhoodDistance = await this.proximityScoringService.callProximityFunction( + blockchain, + proximityScoreFunctionsPairId, + neighbourhoodEdges.leftEdge[hashFunctionName], + neighbourhoodEdges.rightEdge[hashFunctionName], + ); const scores = await Promise.all( neighbourhood.map(async (node) => ({ @@ -284,6 +292,7 @@ class EpochCheckCommand extends Command { keyword, hashFunctionId, proximityScoreFunctionsPairId, + maxNeighborhoodDistance, ), peerId: node.peerId, })), @@ -406,10 +415,10 @@ class EpochCheckCommand extends Command { ); const hashRing = [assetPositionOnHashRing]; - const maxDistance = this.proximityScoringService.callProximityFunction( + const maxDistance = await this.proximityScoringService.callProximityFunction( blockchainId, proximityScoreFunctionsPairId, - neighbourhood[neighbourhood.length - 1], + neighbourhood[neighbourhood.length - 1][hashFunctionName], assetHash, ); for (const neighbour of neighbourhood) { @@ -433,10 +442,16 @@ class EpochCheckCommand extends Command { } return { - leftEdge: neighbourhood.find((node) => node[hashFunctionName] === hashRing[0]), - rightEdge: neighbourhood.find( - (node) => node[hashFunctionName] === hashRing[hashRing.length() - 1], - ), + leftEdge: neighbourhood.find(async (node) => { + await this.blockchainModuleManager + .toBigNumber(blockchainId, node[hashFunctionName]) + .eq(hashRing[0]); + }), + rightEdge: neighbourhood.find(async (node) => { + await this.blockchainModuleManager + .toBigNumber(blockchainId, node[hashFunctionName]) + .eq(hashRing[hashRing.length - 1]); + }), }; } From efb3d65016b365d086bbc206ff431ab72c4e2503 Mon Sep 17 00:00:00 2001 From: Djordje Kovacevic Date: Mon, 29 Jan 2024 14:06:02 +0100 Subject: [PATCH 37/96] Added proximity score functions pair id where we need to choose commit manager contract --- .../protocols/common/epoch-check-command.js | 13 +++- .../protocols/common/submit-commit-command.js | 18 +++-- .../protocols/common/submit-proofs-command.js | 11 +++- .../receiver/submit-update-commit-command.js | 8 +++ .../v1-0-0-handle-update-request-command.js | 1 + .../blockchain/blockchain-module-manager.js | 43 +++++++++++- .../blockchain/implementation/web3-service.js | 65 +++++++++++-------- 7 files changed, 122 insertions(+), 37 deletions(-) diff --git a/src/commands/protocols/common/epoch-check-command.js b/src/commands/protocols/common/epoch-check-command.js index 336032acb1..09e0a21bd2 100644 --- a/src/commands/protocols/common/epoch-check-command.js +++ b/src/commands/protocols/common/epoch-check-command.js @@ -228,6 +228,7 @@ class EpochCheckCommand extends Command { serviceAgreement.currentEpoch, serviceAgreement.stateIndex, r0, + serviceAgreement.scoreFunctionId, ); if (eligibleForReward) { this.logger.trace( @@ -303,13 +304,21 @@ class EpochCheckCommand extends Command { return scores.findIndex((node) => node.peerId === peerId); } - async isEligibleForRewards(blockchain, agreementId, epoch, stateIndex, r0) { + async isEligibleForRewards( + blockchain, + agreementId, + epoch, + stateIndex, + r0, + proximityScoreFunctionsPairId, + ) { const identityId = await this.blockchainModuleManager.getIdentityId(blockchain); const commits = await this.blockchainModuleManager.getTopCommitSubmissions( blockchain, agreementId, epoch, stateIndex, + proximityScoreFunctionsPairId, ); for (const commit of commits.slice(0, r0)) { @@ -329,6 +338,7 @@ class EpochCheckCommand extends Command { tokenId: agreement.tokenId, keyword: agreement.keyword, hashFunctionId: agreement.hashFunctionId, + proximityScoreFunctionsPairId: agreement.scoreFunctionId, epoch: agreement.currentEpoch, agreementId: agreement.agreementId, stateIndex: agreement.stateIndex, @@ -354,6 +364,7 @@ class EpochCheckCommand extends Command { tokenId: agreement.tokenId, keyword: agreement.keyword, hashFunctionId: agreement.hashFunctionId, + proximityScoreFunctionsPairId: agreement.scoreFunctionId, epoch: agreement.currentEpoch, agreementId: agreement.agreementId, assertionId: agreement.assertionId, diff --git a/src/commands/protocols/common/submit-commit-command.js b/src/commands/protocols/common/submit-commit-command.js index 4776202b32..e4e631534a 100644 --- a/src/commands/protocols/common/submit-commit-command.js +++ b/src/commands/protocols/common/submit-commit-command.js @@ -31,6 +31,7 @@ class SubmitCommitCommand extends Command { closestNode, leftNeighborhoodEdge, rightNeighborhoodEdge, + proximityScoreFunctionsPairId, } = command.data; this.logger.trace( @@ -79,6 +80,7 @@ class SubmitCommitCommand extends Command { agreementId, epoch, stateIndex, + proximityScoreFunctionsPairId, ); if (alreadySubmitted) { this.logger.trace( @@ -103,9 +105,6 @@ class SubmitCommitCommand extends Command { keyword, hashFunctionId, epoch, - closestNode, - leftNeighborhoodEdge, - rightNeighborhoodEdge, stateIndex, (result) => { if (result?.error) { @@ -119,6 +118,10 @@ class SubmitCommitCommand extends Command { resolve(true); }, txGasPrice, + proximityScoreFunctionsPairId, + closestNode, + leftNeighborhoodEdge, + rightNeighborhoodEdge, ); }); @@ -182,12 +185,19 @@ class SubmitCommitCommand extends Command { return Command.empty(); } - async commitAlreadySubmitted(blockchain, agreementId, epoch, stateIndex) { + async commitAlreadySubmitted( + blockchain, + agreementId, + epoch, + stateIndex, + proximityScoreFunctionsPairId, + ) { const commits = await this.blockchainModuleManager.getTopCommitSubmissions( blockchain, agreementId, epoch, stateIndex, + proximityScoreFunctionsPairId, ); const identityId = await this.blockchainModuleManager.getIdentityId(blockchain); diff --git a/src/commands/protocols/common/submit-proofs-command.js b/src/commands/protocols/common/submit-proofs-command.js index f3c7e6d9ca..a077e3b031 100644 --- a/src/commands/protocols/common/submit-proofs-command.js +++ b/src/commands/protocols/common/submit-proofs-command.js @@ -34,6 +34,7 @@ class SubmitProofsCommand extends Command { assertionId, stateIndex, gasPrice, + proximityScoreFunctionsPairId, } = command.data; this.logger.trace( @@ -122,6 +123,7 @@ class SubmitProofsCommand extends Command { agreementId, epoch, stateIndex, + proximityScoreFunctionsPairId, ); if (alreadySubmitted) { this.logger.trace( @@ -218,12 +220,19 @@ class SubmitProofsCommand extends Command { return Command.empty(); } - async proofAlreadySubmitted(blockchain, agreementId, epoch, stateIndex) { + async proofAlreadySubmitted( + blockchain, + agreementId, + epoch, + stateIndex, + proximityScoreFunctionsPairId, + ) { const commits = await this.blockchainModuleManager.getTopCommitSubmissions( blockchain, agreementId, epoch, stateIndex, + proximityScoreFunctionsPairId, ); const identityId = await this.blockchainModuleManager.getIdentityId(blockchain); diff --git a/src/commands/protocols/update/receiver/submit-update-commit-command.js b/src/commands/protocols/update/receiver/submit-update-commit-command.js index d8c9b27297..af1fde3fa6 100644 --- a/src/commands/protocols/update/receiver/submit-update-commit-command.js +++ b/src/commands/protocols/update/receiver/submit-update-commit-command.js @@ -27,6 +27,10 @@ class SubmitUpdateCommitCommand extends Command { agreementId, operationId, gasPrice, + proximityScoreFunctionsPairId, + closestNode, + leftNeighborhoodEdge, + rightNeighborhoodEdge, } = command.data; this.logger.trace( @@ -86,6 +90,10 @@ class SubmitUpdateCommitCommand extends Command { resolve(); }, txGasPrice, + proximityScoreFunctionsPairId, + closestNode, + leftNeighborhoodEdge, + rightNeighborhoodEdge, ); }); diff --git a/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js b/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js index f468472bf4..4f111c0b51 100644 --- a/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js +++ b/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js @@ -98,6 +98,7 @@ class HandleUpdateRequestCommand extends HandleProtocolMessageCommand { r0, r2, updateCommitWindowDuration, + proximityScoreFunctionsPairId, }, transactional: false, }), diff --git a/src/modules/blockchain/blockchain-module-manager.js b/src/modules/blockchain/blockchain-module-manager.js index 0252e1940f..696d937d1c 100644 --- a/src/modules/blockchain/blockchain-module-manager.js +++ b/src/modules/blockchain/blockchain-module-manager.js @@ -194,27 +194,48 @@ class BlockchainModuleManager extends BaseModuleManager { return this.callImplementationFunction(blockchain, 'convertFromWei', [value, toUnit]); } - async isCommitWindowOpen(blockchain, agreementId, epoch, stateIndex) { + async isCommitWindowOpen( + blockchain, + agreementId, + epoch, + stateIndex, + proximityScoreFunctionsPairId, + ) { return this.callImplementationFunction(blockchain, 'isCommitWindowOpen', [ agreementId, epoch, stateIndex, + proximityScoreFunctionsPairId, ]); } - async isUpdateCommitWindowOpen(blockchain, agreementId, epoch, latestStateIndex) { + async isUpdateCommitWindowOpen( + blockchain, + agreementId, + epoch, + latestStateIndex, + proximityScoreFunctionsPairId, + ) { return this.callImplementationFunction(blockchain, 'isUpdateCommitWindowOpen', [ agreementId, epoch, latestStateIndex, + proximityScoreFunctionsPairId, ]); } - async getTopCommitSubmissions(blockchain, agreementId, epoch, latestStateIndex) { + async getTopCommitSubmissions( + blockchain, + agreementId, + epoch, + latestStateIndex, + proximityScoreFunctionsPairId, + ) { return this.callImplementationFunction(blockchain, 'getTopCommitSubmissions', [ agreementId, epoch, latestStateIndex, + proximityScoreFunctionsPairId, ]); } @@ -248,6 +269,10 @@ class BlockchainModuleManager extends BaseModuleManager { latestStateIndex, callback, gasPrice, + proximityScoreFunctionsPairId, + closestNode, + leftNeighborhoodEdge, + rightNeighborhoodEdge, ) { return this.callImplementationFunction(blockchain, 'submitCommit', [ assetContractAddress, @@ -258,6 +283,10 @@ class BlockchainModuleManager extends BaseModuleManager { latestStateIndex, callback, gasPrice, + proximityScoreFunctionsPairId, + closestNode, + leftNeighborhoodEdge, + rightNeighborhoodEdge, ]); } @@ -270,6 +299,10 @@ class BlockchainModuleManager extends BaseModuleManager { epoch, callback, gasPrice, + proximityScoreFunctionsPairId, + closestNode, + leftNeighborhoodEdge, + rightNeighborhoodEdge, ) { return this.callImplementationFunction(blockchain, 'submitUpdateCommit', [ assetContractAddress, @@ -279,6 +312,10 @@ class BlockchainModuleManager extends BaseModuleManager { epoch, callback, gasPrice, + proximityScoreFunctionsPairId, + closestNode, + leftNeighborhoodEdge, + rightNeighborhoodEdge, ]); } diff --git a/src/modules/blockchain/implementation/web3-service.js b/src/modules/blockchain/implementation/web3-service.js index 39b77ec081..3b45d89935 100644 --- a/src/modules/blockchain/implementation/web3-service.js +++ b/src/modules/blockchain/implementation/web3-service.js @@ -41,8 +41,10 @@ const ABIs = { ProfileStorage: require('dkg-evm-module/abi/ProfileStorage.json'), ScoringProxy: require('dkg-evm-module/abi/ScoringProxy.json'), ServiceAgreementV1: require('dkg-evm-module/abi/ServiceAgreementV1.json'), - CommitManagerV1: require('dkg-evm-module/abi/CommitManagerV2.json'), + CommitManagerV1: require('dkg-evm-module/abi/CommitManagerV1.json'), CommitManagerV1U1: require('dkg-evm-module/abi/CommitManagerV1U1.json'), + CommitManagerV2: require('dkg-evm-module/abi/CommitManagerV2.json'), + CommitManagerV2U1: require('dkg-evm-module/abi/CommitManagerV2U1.json'), ProofManagerV1: require('dkg-evm-module/abi/ProofManagerV1.json'), ProofManagerV1U1: require('dkg-evm-module/abi/ProofManagerV1U1.json'), ShardingTable: require('dkg-evm-module/abi/ShardingTableV2.json'), @@ -873,34 +875,39 @@ class Web3Service { return Number(assertionChunksNumber); } - selectCommitManagerContract(latestStateIndex) { + selectCommitManagerContract(latestStateIndex, proximityScoreFunctionsPairId = 1) { return latestStateIndex === 0 - ? this.CommitManagerV1Contract - : this.CommitManagerV1U1Contract; + ? this[`CommitManagerV${proximityScoreFunctionsPairId}Contract`] + : this[`CommitManagerV${proximityScoreFunctionsPairId}U1Contract`]; } - async isCommitWindowOpen(agreementId, epoch, latestStateIndex) { + async isCommitWindowOpen(agreementId, epoch, latestStateIndex, proximityScoreFunctionsPairId) { return this.callContractFunction( - this.selectCommitManagerContract(latestStateIndex), + this.selectCommitManagerContract(latestStateIndex, proximityScoreFunctionsPairId), 'isCommitWindowOpen', [agreementId, epoch], ); } - async isUpdateCommitWindowOpen(agreementId, epoch, stateIndex) { + async isUpdateCommitWindowOpen(agreementId, epoch, stateIndex, proximityScoreFunctionsPairId) { return this.callContractFunction( - this.CommitManagerV1U1Contract, + this.selectCommitManagerContract(1, proximityScoreFunctionsPairId), 'isUpdateCommitWindowOpen', [agreementId, epoch, stateIndex], ); } - async getTopCommitSubmissions(agreementId, epoch, latestStateIndex) { + async getTopCommitSubmissions( + agreementId, + epoch, + latestStateIndex, + proximityScoreFunctionsPairId, + ) { const args = latestStateIndex === 0 ? [agreementId, epoch] : [agreementId, epoch, latestStateIndex]; const commits = await this.callContractFunction( - this.selectCommitManagerContract(latestStateIndex), + this.selectCommitManagerContract(latestStateIndex, proximityScoreFunctionsPairId), 'getTopCommitSubmissions', args, ); @@ -983,28 +990,22 @@ class Web3Service { keyword, hashFunctionId, epoch, - closestNode, - leftNeighborhoodEdge, - rightNeighborhoodEdge, latestStateIndex, callback, gasPrice, + proximityScoreFunctionsPairId = 1, + closestNode = null, + leftNeighborhoodEdge = null, + rightNeighborhoodEdge = null, ) { + const submitCommitArgs = [assetContractAddress, tokenId, keyword, hashFunctionId, epoch]; + if (closestNode && leftNeighborhoodEdge && rightNeighborhoodEdge) { + submitCommitArgs.push(closestNode, leftNeighborhoodEdge, rightNeighborhoodEdge); + } return this.queueTransaction( - this.selectCommitManagerContract(latestStateIndex), + this.selectCommitManagerContract(latestStateIndex, proximityScoreFunctionsPairId), 'submitCommit', - [ - [ - assetContractAddress, - tokenId, - keyword, - hashFunctionId, - epoch, - closestNode, - leftNeighborhoodEdge, - rightNeighborhoodEdge, - ], - ], + [submitCommitArgs], callback, gasPrice, ); @@ -1018,11 +1019,19 @@ class Web3Service { epoch, callback, gasPrice, + proximityScoreFunctionsPairId = 1, + closestNode = null, + leftNeighborhoodEdge = null, + rightNeighborhoodEdge = null, ) { + const submitCommitArgs = [assetContractAddress, tokenId, keyword, hashFunctionId, epoch]; + if (closestNode && leftNeighborhoodEdge && rightNeighborhoodEdge) { + submitCommitArgs.push(closestNode, leftNeighborhoodEdge, rightNeighborhoodEdge); + } return this.queueTransaction( - this.CommitManagerV1U1Contract, + this.selectCommitManagerContract(1, proximityScoreFunctionsPairId), 'submitUpdateCommit', - [[assetContractAddress, tokenId, keyword, hashFunctionId, epoch]], + [submitCommitArgs], callback, gasPrice, ); From 91bfb2c7dd65d23a18afdd1edf20a77500d9cc3c Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Mon, 29 Jan 2024 14:32:38 +0100 Subject: [PATCH 38/96] Update caluclateRank in update command --- .../protocols/common/epoch-check-command.js | 34 +++---- .../v1-0-0-handle-update-request-command.js | 97 +++++++++++++++++-- src/service/service-agreement-service.js | 2 + 3 files changed, 104 insertions(+), 29 deletions(-) diff --git a/src/commands/protocols/common/epoch-check-command.js b/src/commands/protocols/common/epoch-check-command.js index 09e0a21bd2..795f2b9237 100644 --- a/src/commands/protocols/common/epoch-check-command.js +++ b/src/commands/protocols/common/epoch-check-command.js @@ -186,9 +186,13 @@ class EpochCheckCommand extends Command { serviceAgreement.agreementId }. Scheduling submitCommitCommand.`, ); - + const closestNode = neighbourhood[0]; scheduleSubmitCommitCommands.push( - this.scheduleSubmitCommitCommand(serviceAgreement, neighbourhoodEdges), + this.scheduleSubmitCommitCommand( + serviceAgreement, + neighbourhoodEdges, + closestNode, + ), ); } catch (error) { this.logger.warn( @@ -343,8 +347,8 @@ class EpochCheckCommand extends Command { agreementId: agreement.agreementId, stateIndex: agreement.stateIndex, closestNode: closestNode.index, - leftNeighborhoodEdge: neighbourhoodEdges.leftNeighborhoodEdge.index, - rightNeighborhoodEdge: neighbourhoodEdges.rightNeighborhoodEdge.index, + leftNeighborhoodEdge: neighbourhoodEdges.leftEdge.index, + rightNeighborhoodEdge: neighbourhoodEdges.rightEdge.index, }; await this.commandExecutor.add({ @@ -424,7 +428,7 @@ class EpochCheckCommand extends Command { blockchainId, assetHash, ); - const hashRing = [assetPositionOnHashRing]; + const hashRing = []; const maxDistance = await this.proximityScoringService.callProximityFunction( blockchainId, @@ -439,30 +443,22 @@ class EpochCheckCommand extends Command { ); if (assetPositionOnHashRing.lte(neighbourPositionOnHashRing)) { if (neighbourPositionOnHashRing.sub(assetPositionOnHashRing).lt(maxDistance)) { - hashRing.push(neighbourPositionOnHashRing); + hashRing.push(neighbour); } else { - hashRing.unshift(neighbourPositionOnHashRing); + hashRing.unshift(neighbour); } } else if (assetPositionOnHashRing.gt(neighbourPositionOnHashRing)) { if (assetPositionOnHashRing.sub(neighbourPositionOnHashRing).lt(maxDistance)) { - hashRing.unshift(neighbourPositionOnHashRing); + hashRing.unshift(neighbour); } else { - hashRing.push(neighbourPositionOnHashRing); + hashRing.push(neighbour); } } } return { - leftEdge: neighbourhood.find(async (node) => { - await this.blockchainModuleManager - .toBigNumber(blockchainId, node[hashFunctionName]) - .eq(hashRing[0]); - }), - rightEdge: neighbourhood.find(async (node) => { - await this.blockchainModuleManager - .toBigNumber(blockchainId, node[hashFunctionName]) - .eq(hashRing[hashRing.length - 1]); - }), + leftEdge: hashRing[0], + rightEdge: hashRing[hashRing.length - 1], }; } diff --git a/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js b/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js index 4f111c0b51..7c7ac26432 100644 --- a/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js +++ b/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js @@ -21,6 +21,8 @@ class HandleUpdateRequestCommand extends HandleProtocolMessageCommand { this.tripleStoreService = ctx.tripleStoreService; this.ualService = ctx.ualService; this.pendingStorageService = ctx.pendingStorageService; + this.shardingTableService = ctx.shardingTableService; + this.hashingService = ctx.hashingService; this.errorType = ERROR_TYPE.UPDATE.UPDATE_LOCAL_STORE_REMOTE_ERROR; } @@ -67,12 +69,31 @@ class HandleUpdateRequestCommand extends HandleProtocolMessageCommand { const r2 = await this.blockchainModuleManager.getR2(blockchain); const scheduleCommandsPromises = []; + const neighbourhood = await this.shardingTableService.findNeighbourhood( + blockchain, + keyword, + r2, + hashFunctionId, + proximityScoreFunctionsPairId, + true, + ); + + const neighbourhoodEdges = await this.getNeighboorhoodEdgeNodes( + neighbourhood, + blockchain, + hashFunctionId, + proximityScoreFunctionsPairId, + keyword, + ); + const rank = await this.calculateRank( blockchain, keyword, hashFunctionId, proximityScoreFunctionsPairId, r2, + neighbourhood, + neighbourhoodEdges, ); if (rank != null) { this.logger.trace(`Calculated rank: ${rank + 1} for agreement id: ${agreementId}`); @@ -156,21 +177,29 @@ class HandleUpdateRequestCommand extends HandleProtocolMessageCommand { return delay; } - async calculateRank(blockchain, keyword, hashFunctionId, proximityScoreFunctionsPairId, r2) { - const neighbourhood = await this.shardingTableService.findNeighbourhood( - blockchain, - keyword, - r2, - hashFunctionId, - proximityScoreFunctionsPairId, - true, - ); - + async calculateRank( + blockchain, + keyword, + hashFunctionId, + proximityScoreFunctionsPairId, + r2, + neighbourhood, + neighbourhoodEdges, + ) { const peerId = this.networkModuleManager.getPeerId().toB58String(); if (!neighbourhood.some((node) => node.peerId === peerId)) { return; } + const hashFunctionName = this.hashingService.getHashFunctionName(hashFunctionId); + + const maxNeighborhoodDistance = await this.proximityScoringService.callProximityFunction( + blockchain, + proximityScoreFunctionsPairId, + neighbourhoodEdges.leftEdge[hashFunctionName], + neighbourhoodEdges.rightEdge[hashFunctionName], + ); + const scores = await Promise.all( neighbourhood.map(async (node) => ({ score: await this.serviceAgreementService.calculateScore( @@ -179,6 +208,7 @@ class HandleUpdateRequestCommand extends HandleProtocolMessageCommand { keyword, hashFunctionId, proximityScoreFunctionsPairId, + maxNeighborhoodDistance, ), peerId: node.peerId, })), @@ -189,6 +219,53 @@ class HandleUpdateRequestCommand extends HandleProtocolMessageCommand { return scores.findIndex((node) => node.peerId === peerId); } + async getNeighboorhoodEdgeNodes( + neighbourhood, + blockchainId, + hashFunctionId, + proximityScoreFunctionsPairId, + assetHash, + ) { + const hashFunctionName = this.hashingService.getHashFunctionName(hashFunctionId); + const assetPositionOnHashRing = await this.blockchainModuleManager.toBigNumber( + blockchainId, + assetHash, + ); + const hashRing = []; + + const maxDistance = await this.proximityScoringService.callProximityFunction( + blockchainId, + proximityScoreFunctionsPairId, + neighbourhood[neighbourhood.length - 1][hashFunctionName], + assetHash, + ); + for (const neighbour of neighbourhood) { + // eslint-disable-next-line no-await-in-loop + const neighbourPositionOnHashRing = await this.blockchainModuleManager.toBigNumber( + blockchainId, + neighbour[hashFunctionName], + ); + if (assetPositionOnHashRing.lte(neighbourPositionOnHashRing)) { + if (neighbourPositionOnHashRing.sub(assetPositionOnHashRing).lt(maxDistance)) { + hashRing.push(neighbour); + } else { + hashRing.unshift(neighbour); + } + } else if (assetPositionOnHashRing.gt(neighbourPositionOnHashRing)) { + if (assetPositionOnHashRing.sub(neighbourPositionOnHashRing).lt(maxDistance)) { + hashRing.unshift(neighbour); + } else { + hashRing.push(neighbour); + } + } + } + + return { + leftEdge: hashRing[0], + rightEdge: hashRing[hashRing.length - 1], + }; + } + /** * Builds default HandleUpdateRequestCommand * @param map diff --git a/src/service/service-agreement-service.js b/src/service/service-agreement-service.js index dc68c63e29..da856cf71d 100644 --- a/src/service/service-agreement-service.js +++ b/src/service/service-agreement-service.js @@ -31,6 +31,7 @@ class ServiceAgreementService { keyword, hashFunctionId, proximityScoreFunctionsPairId, + maxNeighborhoodDistance, ) { const peerRecord = await this.repositoryModuleManager.getPeerRecord(peerId, blockchainId); const keyHash = await this.hashingService.callHashFunction(hashFunctionId, keyword); @@ -49,6 +50,7 @@ class ServiceAgreementService { proximityScoreFunctionsPairId, distance, peerRecord.stake, + maxNeighborhoodDistance, ); } } From 036afc22356dd9073f10f54bfc65379fcbf5ff08 Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Mon, 29 Jan 2024 14:50:44 +0100 Subject: [PATCH 39/96] submitUpdateCommitCommand pass correct data --- .../receiver/v1.0.0/v1-0-0-handle-update-request-command.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js b/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js index 7c7ac26432..e2da230bbc 100644 --- a/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js +++ b/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js @@ -23,6 +23,7 @@ class HandleUpdateRequestCommand extends HandleProtocolMessageCommand { this.pendingStorageService = ctx.pendingStorageService; this.shardingTableService = ctx.shardingTableService; this.hashingService = ctx.hashingService; + this.proximityScoringService = ctx.proximityScoringService; this.errorType = ERROR_TYPE.UPDATE.UPDATE_LOCAL_STORE_REMOTE_ERROR; } @@ -78,6 +79,8 @@ class HandleUpdateRequestCommand extends HandleProtocolMessageCommand { true, ); + const closestNode = neighbourhood[0]; + const neighbourhoodEdges = await this.getNeighboorhoodEdgeNodes( neighbourhood, blockchain, @@ -120,6 +123,9 @@ class HandleUpdateRequestCommand extends HandleProtocolMessageCommand { r2, updateCommitWindowDuration, proximityScoreFunctionsPairId, + closestNode: closestNode.index, + leftNeighborhoodEdge: neighbourhoodEdges.leftEdge.index, + rightNeighborhoodEdge: neighbourhoodEdges.rightEdge.index, }, transactional: false, }), From 8406d3862b778724df3e1329cacedd7cf1815a6c Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Mon, 29 Jan 2024 15:19:52 +0100 Subject: [PATCH 40/96] Update submitUpdateCommitCommand logs --- .../receiver/submit-update-commit-command.js | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/commands/protocols/update/receiver/submit-update-commit-command.js b/src/commands/protocols/update/receiver/submit-update-commit-command.js index af1fde3fa6..58c2687b81 100644 --- a/src/commands/protocols/update/receiver/submit-update-commit-command.js +++ b/src/commands/protocols/update/receiver/submit-update-commit-command.js @@ -37,7 +37,9 @@ class SubmitUpdateCommitCommand extends Command { `Started ${command.name} for the Service Agreement with the ID: ${agreementId}, ` + `Blockchain: ${blockchain}, Contract: ${contract}, Token ID: ${tokenId}, ` + `Keyword: ${keyword}, Hash function ID: ${hashFunctionId}, Operation ID: ${operationId}, ` + - `Retry number: ${COMMAND_RETRIES.SUBMIT_UPDATE_COMMIT - command.retries + 1}`, + `Clossest Node: ${closestNode}, Left neighborhood edge: ${leftNeighborhoodEdge}, ` + + `Right neighborhood edge: ${rightNeighborhoodEdge}, `, + `Retry number: ${COMMAND_RETRIES.SUBMIT_UPDATE_COMMIT - command.retries + 1}`, ); const epoch = await this.calculateCurrentEpoch( @@ -66,7 +68,9 @@ class SubmitUpdateCommitCommand extends Command { `Not submitting update commit as state has been already finalized for the Service Agreement ` + `with the ID: ${agreementId}, Blockchain: ${blockchain}, Contract: ${contract}, ` + `Token ID: ${tokenId}, Keyword: ${keyword}, Hash function ID: ${hashFunctionId}, ` + - `Epoch: ${epoch}, Operation ID: ${operationId}`, + `Clossest Node: ${closestNode}, Left neighborhood edge: ${leftNeighborhoodEdge}, ` + + `Right neighborhood edge: ${rightNeighborhoodEdge}, `, + +`Epoch: ${epoch}, Operation ID: ${operationId}`, ); return Command.empty(); @@ -104,9 +108,11 @@ class SubmitUpdateCommitCommand extends Command { `Failed to execute ${command.name}, Error Message: ${error.message} for the Service Agreement ` + `with the ID: ${agreementId}, Blockchain: ${blockchain}, Contract: ${contract}, ` + `Token ID: ${tokenId}, Keyword: ${keyword}, Hash function ID: ${hashFunctionId}, ` + - `Epoch: ${epoch}, Operation ID: ${operationId}, Retry number: ${ - COMMAND_RETRIES.SUBMIT_UPDATE_COMMIT - command.retries + 1 - }.`, + `Clossest Node: ${closestNode}, Left neighborhood edge: ${leftNeighborhoodEdge}, ` + + `Right neighborhood edge: ${rightNeighborhoodEdge}, `, + +`Epoch: ${epoch}, Operation ID: ${operationId}, Retry number: ${ + COMMAND_RETRIES.SUBMIT_UPDATE_COMMIT - command.retries + 1 + }.`, ); let newGasPrice; @@ -133,9 +139,11 @@ class SubmitUpdateCommitCommand extends Command { `Successfully executed ${command.name} for the Service Agreement with the ID: ${agreementId}, ` + `Blockchain: ${blockchain}, Contract: ${contract}, Token ID: ${tokenId}, ` + `Keyword: ${keyword}, Hash function ID: ${hashFunctionId}, Epoch: ${epoch}, ` + - `Operation ID: ${operationId}, Retry number: ${ - COMMAND_RETRIES.SUBMIT_UPDATE_COMMIT - command.retries + 1 - }`, + `Clossest Node: ${closestNode}, Left neighborhood edge: ${leftNeighborhoodEdge}, ` + + `Right neighborhood edge: ${rightNeighborhoodEdge}, `, + +`Operation ID: ${operationId}, Retry number: ${ + COMMAND_RETRIES.SUBMIT_UPDATE_COMMIT - command.retries + 1 + }`, ); this.operationIdService.emitChangeEvent( From 40280c39eef8ff99310bff86bdc0ec2593608192 Mon Sep 17 00:00:00 2001 From: Djordje Kovacevic Date: Mon, 29 Jan 2024 15:32:25 +0100 Subject: [PATCH 41/96] Reverted changes --- .../protocols/common/epoch-check-command.js | 13 +---- .../protocols/common/submit-commit-command.js | 18 ++----- .../protocols/common/submit-proofs-command.js | 11 +---- .../receiver/submit-update-commit-command.js | 2 - .../blockchain/blockchain-module-manager.js | 37 +++------------ .../blockchain/implementation/web3-service.js | 47 ++++++++----------- 6 files changed, 31 insertions(+), 97 deletions(-) diff --git a/src/commands/protocols/common/epoch-check-command.js b/src/commands/protocols/common/epoch-check-command.js index 795f2b9237..7923af6145 100644 --- a/src/commands/protocols/common/epoch-check-command.js +++ b/src/commands/protocols/common/epoch-check-command.js @@ -232,7 +232,6 @@ class EpochCheckCommand extends Command { serviceAgreement.currentEpoch, serviceAgreement.stateIndex, r0, - serviceAgreement.scoreFunctionId, ); if (eligibleForReward) { this.logger.trace( @@ -308,21 +307,13 @@ class EpochCheckCommand extends Command { return scores.findIndex((node) => node.peerId === peerId); } - async isEligibleForRewards( - blockchain, - agreementId, - epoch, - stateIndex, - r0, - proximityScoreFunctionsPairId, - ) { + async isEligibleForRewards(blockchain, agreementId, epoch, stateIndex, r0) { const identityId = await this.blockchainModuleManager.getIdentityId(blockchain); const commits = await this.blockchainModuleManager.getTopCommitSubmissions( blockchain, agreementId, epoch, stateIndex, - proximityScoreFunctionsPairId, ); for (const commit of commits.slice(0, r0)) { @@ -342,7 +333,6 @@ class EpochCheckCommand extends Command { tokenId: agreement.tokenId, keyword: agreement.keyword, hashFunctionId: agreement.hashFunctionId, - proximityScoreFunctionsPairId: agreement.scoreFunctionId, epoch: agreement.currentEpoch, agreementId: agreement.agreementId, stateIndex: agreement.stateIndex, @@ -368,7 +358,6 @@ class EpochCheckCommand extends Command { tokenId: agreement.tokenId, keyword: agreement.keyword, hashFunctionId: agreement.hashFunctionId, - proximityScoreFunctionsPairId: agreement.scoreFunctionId, epoch: agreement.currentEpoch, agreementId: agreement.agreementId, assertionId: agreement.assertionId, diff --git a/src/commands/protocols/common/submit-commit-command.js b/src/commands/protocols/common/submit-commit-command.js index e4e631534a..4776202b32 100644 --- a/src/commands/protocols/common/submit-commit-command.js +++ b/src/commands/protocols/common/submit-commit-command.js @@ -31,7 +31,6 @@ class SubmitCommitCommand extends Command { closestNode, leftNeighborhoodEdge, rightNeighborhoodEdge, - proximityScoreFunctionsPairId, } = command.data; this.logger.trace( @@ -80,7 +79,6 @@ class SubmitCommitCommand extends Command { agreementId, epoch, stateIndex, - proximityScoreFunctionsPairId, ); if (alreadySubmitted) { this.logger.trace( @@ -105,6 +103,9 @@ class SubmitCommitCommand extends Command { keyword, hashFunctionId, epoch, + closestNode, + leftNeighborhoodEdge, + rightNeighborhoodEdge, stateIndex, (result) => { if (result?.error) { @@ -118,10 +119,6 @@ class SubmitCommitCommand extends Command { resolve(true); }, txGasPrice, - proximityScoreFunctionsPairId, - closestNode, - leftNeighborhoodEdge, - rightNeighborhoodEdge, ); }); @@ -185,19 +182,12 @@ class SubmitCommitCommand extends Command { return Command.empty(); } - async commitAlreadySubmitted( - blockchain, - agreementId, - epoch, - stateIndex, - proximityScoreFunctionsPairId, - ) { + async commitAlreadySubmitted(blockchain, agreementId, epoch, stateIndex) { const commits = await this.blockchainModuleManager.getTopCommitSubmissions( blockchain, agreementId, epoch, stateIndex, - proximityScoreFunctionsPairId, ); const identityId = await this.blockchainModuleManager.getIdentityId(blockchain); diff --git a/src/commands/protocols/common/submit-proofs-command.js b/src/commands/protocols/common/submit-proofs-command.js index a077e3b031..f3c7e6d9ca 100644 --- a/src/commands/protocols/common/submit-proofs-command.js +++ b/src/commands/protocols/common/submit-proofs-command.js @@ -34,7 +34,6 @@ class SubmitProofsCommand extends Command { assertionId, stateIndex, gasPrice, - proximityScoreFunctionsPairId, } = command.data; this.logger.trace( @@ -123,7 +122,6 @@ class SubmitProofsCommand extends Command { agreementId, epoch, stateIndex, - proximityScoreFunctionsPairId, ); if (alreadySubmitted) { this.logger.trace( @@ -220,19 +218,12 @@ class SubmitProofsCommand extends Command { return Command.empty(); } - async proofAlreadySubmitted( - blockchain, - agreementId, - epoch, - stateIndex, - proximityScoreFunctionsPairId, - ) { + async proofAlreadySubmitted(blockchain, agreementId, epoch, stateIndex) { const commits = await this.blockchainModuleManager.getTopCommitSubmissions( blockchain, agreementId, epoch, stateIndex, - proximityScoreFunctionsPairId, ); const identityId = await this.blockchainModuleManager.getIdentityId(blockchain); diff --git a/src/commands/protocols/update/receiver/submit-update-commit-command.js b/src/commands/protocols/update/receiver/submit-update-commit-command.js index 58c2687b81..0213c31bce 100644 --- a/src/commands/protocols/update/receiver/submit-update-commit-command.js +++ b/src/commands/protocols/update/receiver/submit-update-commit-command.js @@ -27,7 +27,6 @@ class SubmitUpdateCommitCommand extends Command { agreementId, operationId, gasPrice, - proximityScoreFunctionsPairId, closestNode, leftNeighborhoodEdge, rightNeighborhoodEdge, @@ -94,7 +93,6 @@ class SubmitUpdateCommitCommand extends Command { resolve(); }, txGasPrice, - proximityScoreFunctionsPairId, closestNode, leftNeighborhoodEdge, rightNeighborhoodEdge, diff --git a/src/modules/blockchain/blockchain-module-manager.js b/src/modules/blockchain/blockchain-module-manager.js index 696d937d1c..5f7d3f7691 100644 --- a/src/modules/blockchain/blockchain-module-manager.js +++ b/src/modules/blockchain/blockchain-module-manager.js @@ -194,48 +194,27 @@ class BlockchainModuleManager extends BaseModuleManager { return this.callImplementationFunction(blockchain, 'convertFromWei', [value, toUnit]); } - async isCommitWindowOpen( - blockchain, - agreementId, - epoch, - stateIndex, - proximityScoreFunctionsPairId, - ) { + async isCommitWindowOpen(blockchain, agreementId, epoch, stateIndex) { return this.callImplementationFunction(blockchain, 'isCommitWindowOpen', [ agreementId, epoch, stateIndex, - proximityScoreFunctionsPairId, ]); } - async isUpdateCommitWindowOpen( - blockchain, - agreementId, - epoch, - latestStateIndex, - proximityScoreFunctionsPairId, - ) { + async isUpdateCommitWindowOpen(blockchain, agreementId, epoch, latestStateIndex) { return this.callImplementationFunction(blockchain, 'isUpdateCommitWindowOpen', [ agreementId, epoch, latestStateIndex, - proximityScoreFunctionsPairId, ]); } - async getTopCommitSubmissions( - blockchain, - agreementId, - epoch, - latestStateIndex, - proximityScoreFunctionsPairId, - ) { + async getTopCommitSubmissions(blockchain, agreementId, epoch, latestStateIndex) { return this.callImplementationFunction(blockchain, 'getTopCommitSubmissions', [ agreementId, epoch, latestStateIndex, - proximityScoreFunctionsPairId, ]); } @@ -265,14 +244,13 @@ class BlockchainModuleManager extends BaseModuleManager { tokenId, keyword, hashFunctionId, + closestNode, + leftNeighborhoodEdge, + rightNeighborhoodEdge, epoch, latestStateIndex, callback, gasPrice, - proximityScoreFunctionsPairId, - closestNode, - leftNeighborhoodEdge, - rightNeighborhoodEdge, ) { return this.callImplementationFunction(blockchain, 'submitCommit', [ assetContractAddress, @@ -283,7 +261,6 @@ class BlockchainModuleManager extends BaseModuleManager { latestStateIndex, callback, gasPrice, - proximityScoreFunctionsPairId, closestNode, leftNeighborhoodEdge, rightNeighborhoodEdge, @@ -299,7 +276,6 @@ class BlockchainModuleManager extends BaseModuleManager { epoch, callback, gasPrice, - proximityScoreFunctionsPairId, closestNode, leftNeighborhoodEdge, rightNeighborhoodEdge, @@ -312,7 +288,6 @@ class BlockchainModuleManager extends BaseModuleManager { epoch, callback, gasPrice, - proximityScoreFunctionsPairId, closestNode, leftNeighborhoodEdge, rightNeighborhoodEdge, diff --git a/src/modules/blockchain/implementation/web3-service.js b/src/modules/blockchain/implementation/web3-service.js index 3b45d89935..1c326bbd48 100644 --- a/src/modules/blockchain/implementation/web3-service.js +++ b/src/modules/blockchain/implementation/web3-service.js @@ -41,10 +41,8 @@ const ABIs = { ProfileStorage: require('dkg-evm-module/abi/ProfileStorage.json'), ScoringProxy: require('dkg-evm-module/abi/ScoringProxy.json'), ServiceAgreementV1: require('dkg-evm-module/abi/ServiceAgreementV1.json'), - CommitManagerV1: require('dkg-evm-module/abi/CommitManagerV1.json'), - CommitManagerV1U1: require('dkg-evm-module/abi/CommitManagerV1U1.json'), - CommitManagerV2: require('dkg-evm-module/abi/CommitManagerV2.json'), - CommitManagerV2U1: require('dkg-evm-module/abi/CommitManagerV2U1.json'), + CommitManagerV1: require('dkg-evm-module/abi/CommitManagerV2.json'), + CommitManagerV1U1: require('dkg-evm-module/abi/CommitManagerV2U1.json'), ProofManagerV1: require('dkg-evm-module/abi/ProofManagerV1.json'), ProofManagerV1U1: require('dkg-evm-module/abi/ProofManagerV1U1.json'), ShardingTable: require('dkg-evm-module/abi/ShardingTableV2.json'), @@ -875,39 +873,34 @@ class Web3Service { return Number(assertionChunksNumber); } - selectCommitManagerContract(latestStateIndex, proximityScoreFunctionsPairId = 1) { + selectCommitManagerContract(latestStateIndex) { return latestStateIndex === 0 - ? this[`CommitManagerV${proximityScoreFunctionsPairId}Contract`] - : this[`CommitManagerV${proximityScoreFunctionsPairId}U1Contract`]; + ? this.CommitManagerV1Contract + : this.CommitManagerV1U1Contract; } - async isCommitWindowOpen(agreementId, epoch, latestStateIndex, proximityScoreFunctionsPairId) { + async isCommitWindowOpen(agreementId, epoch, latestStateIndex) { return this.callContractFunction( - this.selectCommitManagerContract(latestStateIndex, proximityScoreFunctionsPairId), + this.selectCommitManagerContract(latestStateIndex), 'isCommitWindowOpen', [agreementId, epoch], ); } - async isUpdateCommitWindowOpen(agreementId, epoch, stateIndex, proximityScoreFunctionsPairId) { + async isUpdateCommitWindowOpen(agreementId, epoch, stateIndex) { return this.callContractFunction( - this.selectCommitManagerContract(1, proximityScoreFunctionsPairId), + this.CommitManagerV1U1Contract, 'isUpdateCommitWindowOpen', [agreementId, epoch, stateIndex], ); } - async getTopCommitSubmissions( - agreementId, - epoch, - latestStateIndex, - proximityScoreFunctionsPairId, - ) { + async getTopCommitSubmissions(agreementId, epoch, latestStateIndex) { const args = latestStateIndex === 0 ? [agreementId, epoch] : [agreementId, epoch, latestStateIndex]; const commits = await this.callContractFunction( - this.selectCommitManagerContract(latestStateIndex, proximityScoreFunctionsPairId), + this.selectCommitManagerContract(latestStateIndex), 'getTopCommitSubmissions', args, ); @@ -990,20 +983,19 @@ class Web3Service { keyword, hashFunctionId, epoch, + closestNode, + leftNeighborhoodEdge, + rightNeighborhoodEdge, latestStateIndex, callback, gasPrice, - proximityScoreFunctionsPairId = 1, - closestNode = null, - leftNeighborhoodEdge = null, - rightNeighborhoodEdge = null, ) { const submitCommitArgs = [assetContractAddress, tokenId, keyword, hashFunctionId, epoch]; if (closestNode && leftNeighborhoodEdge && rightNeighborhoodEdge) { submitCommitArgs.push(closestNode, leftNeighborhoodEdge, rightNeighborhoodEdge); } return this.queueTransaction( - this.selectCommitManagerContract(latestStateIndex, proximityScoreFunctionsPairId), + this.selectCommitManagerContract(latestStateIndex), 'submitCommit', [submitCommitArgs], callback, @@ -1016,20 +1008,19 @@ class Web3Service { tokenId, keyword, hashFunctionId, + closestNode, + leftNeighborhoodEdge, + rightNeighborhoodEdge, epoch, callback, gasPrice, - proximityScoreFunctionsPairId = 1, - closestNode = null, - leftNeighborhoodEdge = null, - rightNeighborhoodEdge = null, ) { const submitCommitArgs = [assetContractAddress, tokenId, keyword, hashFunctionId, epoch]; if (closestNode && leftNeighborhoodEdge && rightNeighborhoodEdge) { submitCommitArgs.push(closestNode, leftNeighborhoodEdge, rightNeighborhoodEdge); } return this.queueTransaction( - this.selectCommitManagerContract(1, proximityScoreFunctionsPairId), + this.CommitManagerV1U1Contract, 'submitUpdateCommit', [submitCommitArgs], callback, From fdbab89cae548d268213e82cfe1b2065d99961a3 Mon Sep 17 00:00:00 2001 From: Djordje Kovacevic Date: Mon, 29 Jan 2024 15:53:18 +0100 Subject: [PATCH 42/96] moved calculate rank and find neighbourhood edges to service classes --- .../protocols/common/epoch-check-command.js | 111 +++-------------- .../v1-0-0-handle-update-request-command.js | 116 +++--------------- src/service/service-agreement-service.js | 53 +++++++- src/service/sharding-table-service.js | 47 +++++++ 4 files changed, 131 insertions(+), 196 deletions(-) diff --git a/src/commands/protocols/common/epoch-check-command.js b/src/commands/protocols/common/epoch-check-command.js index 7923af6145..df5a31bb72 100644 --- a/src/commands/protocols/common/epoch-check-command.js +++ b/src/commands/protocols/common/epoch-check-command.js @@ -135,16 +135,23 @@ class EpochCheckCommand extends Command { true, ); - const neighbourhoodEdges = await this.getNeighboorhoodEdgeNodes( - neighbourhood, - blockchain, - serviceAgreement.hashFunctionId, - serviceAgreement.scoreFunctionId, - serviceAgreement.keyword, - ); + let neighbourhoodEdges = null; + if (serviceAgreement.scoreFunctionId === 2) { + neighbourhoodEdges = await this.shardingTableService.getNeighboorhoodEdgeNodes( + neighbourhood, + blockchain, + serviceAgreement.hashFunctionId, + serviceAgreement.scoreFunctionId, + serviceAgreement.keyword, + ); + } + + if (!neighbourhoodEdges && serviceAgreement.scoreFunctionId === 2) { + throw Error('Unable to find neighbourhood edges for asset'); + } try { - const rank = await this.calculateRank( + const rank = await this.serviceAgreementService.calculateRank( blockchain, serviceAgreement.keyword, serviceAgreement.hashFunctionId, @@ -265,48 +272,6 @@ class EpochCheckCommand extends Command { ]); } - async calculateRank( - blockchain, - keyword, - hashFunctionId, - proximityScoreFunctionsPairId, - r2, - neighbourhood, - neighbourhoodEdges, - ) { - const peerId = this.networkModuleManager.getPeerId().toB58String(); - if (!neighbourhood.some((node) => node.peerId === peerId)) { - return; - } - - const hashFunctionName = this.hashingService.getHashFunctionName(hashFunctionId); - - const maxNeighborhoodDistance = await this.proximityScoringService.callProximityFunction( - blockchain, - proximityScoreFunctionsPairId, - neighbourhoodEdges.leftEdge[hashFunctionName], - neighbourhoodEdges.rightEdge[hashFunctionName], - ); - - const scores = await Promise.all( - neighbourhood.map(async (node) => ({ - score: await this.serviceAgreementService.calculateScore( - node.peerId, - blockchain, - keyword, - hashFunctionId, - proximityScoreFunctionsPairId, - maxNeighborhoodDistance, - ), - peerId: node.peerId, - })), - ); - - scores.sort((a, b) => b.score - a.score); - - return scores.findIndex((node) => node.peerId === peerId); - } - async isEligibleForRewards(blockchain, agreementId, epoch, stateIndex, r0) { const identityId = await this.blockchainModuleManager.getIdentityId(blockchain); const commits = await this.blockchainModuleManager.getTopCommitSubmissions( @@ -405,52 +370,6 @@ class EpochCheckCommand extends Command { return devEnvironment ? 30_000 : 120_000; } - async getNeighboorhoodEdgeNodes( - neighbourhood, - blockchainId, - hashFunctionId, - proximityScoreFunctionsPairId, - assetHash, - ) { - const hashFunctionName = this.hashingService.getHashFunctionName(hashFunctionId); - const assetPositionOnHashRing = await this.blockchainModuleManager.toBigNumber( - blockchainId, - assetHash, - ); - const hashRing = []; - - const maxDistance = await this.proximityScoringService.callProximityFunction( - blockchainId, - proximityScoreFunctionsPairId, - neighbourhood[neighbourhood.length - 1][hashFunctionName], - assetHash, - ); - for (const neighbour of neighbourhood) { - const neighbourPositionOnHashRing = await this.blockchainModuleManager.toBigNumber( - blockchainId, - neighbour[hashFunctionName], - ); - if (assetPositionOnHashRing.lte(neighbourPositionOnHashRing)) { - if (neighbourPositionOnHashRing.sub(assetPositionOnHashRing).lt(maxDistance)) { - hashRing.push(neighbour); - } else { - hashRing.unshift(neighbour); - } - } else if (assetPositionOnHashRing.gt(neighbourPositionOnHashRing)) { - if (assetPositionOnHashRing.sub(neighbourPositionOnHashRing).lt(maxDistance)) { - hashRing.unshift(neighbour); - } else { - hashRing.push(neighbour); - } - } - } - - return { - leftEdge: hashRing[0], - rightEdge: hashRing[hashRing.length - 1], - }; - } - /** * Recover system from failure * @param command diff --git a/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js b/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js index e2da230bbc..d30c6b384d 100644 --- a/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js +++ b/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js @@ -81,13 +81,20 @@ class HandleUpdateRequestCommand extends HandleProtocolMessageCommand { const closestNode = neighbourhood[0]; - const neighbourhoodEdges = await this.getNeighboorhoodEdgeNodes( - neighbourhood, - blockchain, - hashFunctionId, - proximityScoreFunctionsPairId, - keyword, - ); + let neighbourhoodEdges = null; + if (proximityScoreFunctionsPairId === 2) { + neighbourhoodEdges = await this.shardingTableService.getNeighboorhoodEdgeNodes( + neighbourhood, + blockchain, + hashFunctionId, + proximityScoreFunctionsPairId, + keyword, + ); + } + + if (!neighbourhoodEdges && proximityScoreFunctionsPairId === 2) { + throw Error('Unable to find neighbourhood edges for asset'); + } const rank = await this.calculateRank( blockchain, @@ -123,9 +130,9 @@ class HandleUpdateRequestCommand extends HandleProtocolMessageCommand { r2, updateCommitWindowDuration, proximityScoreFunctionsPairId, - closestNode: closestNode.index, - leftNeighborhoodEdge: neighbourhoodEdges.leftEdge.index, - rightNeighborhoodEdge: neighbourhoodEdges.rightEdge.index, + closestNode: closestNode?.index, + leftNeighborhoodEdge: neighbourhoodEdges?.leftEdge?.index, + rightNeighborhoodEdge: neighbourhoodEdges?.rightEdge?.index, }, transactional: false, }), @@ -183,95 +190,6 @@ class HandleUpdateRequestCommand extends HandleProtocolMessageCommand { return delay; } - async calculateRank( - blockchain, - keyword, - hashFunctionId, - proximityScoreFunctionsPairId, - r2, - neighbourhood, - neighbourhoodEdges, - ) { - const peerId = this.networkModuleManager.getPeerId().toB58String(); - if (!neighbourhood.some((node) => node.peerId === peerId)) { - return; - } - - const hashFunctionName = this.hashingService.getHashFunctionName(hashFunctionId); - - const maxNeighborhoodDistance = await this.proximityScoringService.callProximityFunction( - blockchain, - proximityScoreFunctionsPairId, - neighbourhoodEdges.leftEdge[hashFunctionName], - neighbourhoodEdges.rightEdge[hashFunctionName], - ); - - const scores = await Promise.all( - neighbourhood.map(async (node) => ({ - score: await this.serviceAgreementService.calculateScore( - node.peerId, - blockchain, - keyword, - hashFunctionId, - proximityScoreFunctionsPairId, - maxNeighborhoodDistance, - ), - peerId: node.peerId, - })), - ); - - scores.sort((a, b) => b.score - a.score); - - return scores.findIndex((node) => node.peerId === peerId); - } - - async getNeighboorhoodEdgeNodes( - neighbourhood, - blockchainId, - hashFunctionId, - proximityScoreFunctionsPairId, - assetHash, - ) { - const hashFunctionName = this.hashingService.getHashFunctionName(hashFunctionId); - const assetPositionOnHashRing = await this.blockchainModuleManager.toBigNumber( - blockchainId, - assetHash, - ); - const hashRing = []; - - const maxDistance = await this.proximityScoringService.callProximityFunction( - blockchainId, - proximityScoreFunctionsPairId, - neighbourhood[neighbourhood.length - 1][hashFunctionName], - assetHash, - ); - for (const neighbour of neighbourhood) { - // eslint-disable-next-line no-await-in-loop - const neighbourPositionOnHashRing = await this.blockchainModuleManager.toBigNumber( - blockchainId, - neighbour[hashFunctionName], - ); - if (assetPositionOnHashRing.lte(neighbourPositionOnHashRing)) { - if (neighbourPositionOnHashRing.sub(assetPositionOnHashRing).lt(maxDistance)) { - hashRing.push(neighbour); - } else { - hashRing.unshift(neighbour); - } - } else if (assetPositionOnHashRing.gt(neighbourPositionOnHashRing)) { - if (assetPositionOnHashRing.sub(neighbourPositionOnHashRing).lt(maxDistance)) { - hashRing.unshift(neighbour); - } else { - hashRing.push(neighbour); - } - } - } - - return { - leftEdge: hashRing[0], - rightEdge: hashRing[hashRing.length - 1], - }; - } - /** * Builds default HandleUpdateRequestCommand * @param map diff --git a/src/service/service-agreement-service.js b/src/service/service-agreement-service.js index da856cf71d..965993e4c2 100644 --- a/src/service/service-agreement-service.js +++ b/src/service/service-agreement-service.js @@ -25,13 +25,55 @@ class ServiceAgreementService { return Math.floor(Math.random() * (max - min + 1) + min); } + async calculateRank( + blockchain, + keyword, + hashFunctionId, + proximityScoreFunctionsPairId, + r2, + neighbourhood, + neighbourhoodEdges, + ) { + const peerId = this.networkModuleManager.getPeerId().toB58String(); + if (!neighbourhood.some((node) => node.peerId === peerId)) { + return; + } + + const hashFunctionName = this.hashingService.getHashFunctionName(hashFunctionId); + + const maxNeighborhoodDistance = await this.proximityScoringService.callProximityFunction( + blockchain, + proximityScoreFunctionsPairId, + neighbourhoodEdges.leftEdge[hashFunctionName], + neighbourhoodEdges.rightEdge[hashFunctionName], + ); + + const scores = await Promise.all( + neighbourhood.map(async (node) => ({ + score: await this.calculateScore( + node.peerId, + blockchain, + keyword, + hashFunctionId, + proximityScoreFunctionsPairId, + maxNeighborhoodDistance, + ), + peerId: node.peerId, + })), + ); + + scores.sort((a, b) => b.score - a.score); + + return scores.findIndex((node) => node.peerId === peerId); + } + async calculateScore( peerId, blockchainId, keyword, hashFunctionId, proximityScoreFunctionsPairId, - maxNeighborhoodDistance, + neighbourhoodEdges, ) { const peerRecord = await this.repositoryModuleManager.getPeerRecord(peerId, blockchainId); const keyHash = await this.hashingService.callHashFunction(hashFunctionId, keyword); @@ -44,6 +86,15 @@ class ServiceAgreementService { peerRecord[hashFunctionName], keyHash, ); + let maxNeighborhoodDistance; + if (neighbourhoodEdges) { + maxNeighborhoodDistance = await this.proximityScoringService.callProximityFunction( + blockchainId, + proximityScoreFunctionsPairId, + neighbourhoodEdges.leftEdge[hashFunctionName], + neighbourhoodEdges.rightEdge[hashFunctionName], + ); + } return this.proximityScoringService.callScoreFunction( blockchainId, diff --git a/src/service/sharding-table-service.js b/src/service/sharding-table-service.js index 12451d918a..3a7aa15de9 100644 --- a/src/service/sharding-table-service.js +++ b/src/service/sharding-table-service.js @@ -311,6 +311,53 @@ class ShardingTableService { protocols: peerInfo?.protocols ?? [], }; } + + async getNeighboorhoodEdgeNodes( + neighbourhood, + blockchainId, + hashFunctionId, + proximityScoreFunctionsPairId, + assetHash, + ) { + const hashFunctionName = this.hashingService.getHashFunctionName(hashFunctionId); + const assetPositionOnHashRing = await this.blockchainModuleManager.toBigNumber( + blockchainId, + assetHash, + ); + const hashRing = []; + + const maxDistance = await this.proximityScoringService.callProximityFunction( + blockchainId, + proximityScoreFunctionsPairId, + neighbourhood[neighbourhood.length - 1][hashFunctionName], + assetHash, + ); + for (const neighbour of neighbourhood) { + // eslint-disable-next-line no-await-in-loop + const neighbourPositionOnHashRing = await this.blockchainModuleManager.toBigNumber( + blockchainId, + neighbour[hashFunctionName], + ); + if (assetPositionOnHashRing.lte(neighbourPositionOnHashRing)) { + if (neighbourPositionOnHashRing.sub(assetPositionOnHashRing).lt(maxDistance)) { + hashRing.push(neighbour); + } else { + hashRing.unshift(neighbour); + } + } else if (assetPositionOnHashRing.gt(neighbourPositionOnHashRing)) { + if (assetPositionOnHashRing.sub(neighbourPositionOnHashRing).lt(maxDistance)) { + hashRing.unshift(neighbour); + } else { + hashRing.push(neighbour); + } + } + } + + return { + leftEdge: hashRing[0], + rightEdge: hashRing[hashRing.length - 1], + }; + } } export default ShardingTableService; From c7730f3063af432c59c3954653dec84114232d34 Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Mon, 29 Jan 2024 15:59:41 +0100 Subject: [PATCH 43/96] Remove sha256Blob from select allPears --- .../implementation/sequelize/repositories/shard-repository.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/modules/repository/implementation/sequelize/repositories/shard-repository.js b/src/modules/repository/implementation/sequelize/repositories/shard-repository.js index 661a220de1..067711f03f 100644 --- a/src/modules/repository/implementation/sequelize/repositories/shard-repository.js +++ b/src/modules/repository/implementation/sequelize/repositories/shard-repository.js @@ -41,6 +41,7 @@ class ShardRepository { where: { blockchainId, }, + attributes: ['peerId', 'blockchainId', 'ask', 'stake', 'lastSeen', 'sha256'], order: [['sha256Blob', 'ASC']], }; From c296795dd2133fe4fc1cdcd8994e1ef503a612a3 Mon Sep 17 00:00:00 2001 From: Djordje Kovacevic Date: Mon, 29 Jan 2024 16:13:02 +0100 Subject: [PATCH 44/96] Resolved bug with neighbourhood edges in epoch check command --- src/service/service-agreement-service.js | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/src/service/service-agreement-service.js b/src/service/service-agreement-service.js index 965993e4c2..1dc580158c 100644 --- a/src/service/service-agreement-service.js +++ b/src/service/service-agreement-service.js @@ -39,15 +39,6 @@ class ServiceAgreementService { return; } - const hashFunctionName = this.hashingService.getHashFunctionName(hashFunctionId); - - const maxNeighborhoodDistance = await this.proximityScoringService.callProximityFunction( - blockchain, - proximityScoreFunctionsPairId, - neighbourhoodEdges.leftEdge[hashFunctionName], - neighbourhoodEdges.rightEdge[hashFunctionName], - ); - const scores = await Promise.all( neighbourhood.map(async (node) => ({ score: await this.calculateScore( @@ -56,7 +47,7 @@ class ServiceAgreementService { keyword, hashFunctionId, proximityScoreFunctionsPairId, - maxNeighborhoodDistance, + neighbourhoodEdges, ), peerId: node.peerId, })), From bef165d5b3fc6c3ca9284fbbc3df5d1047b9d361 Mon Sep 17 00:00:00 2001 From: Djordje Kovacevic Date: Mon, 29 Jan 2024 16:22:07 +0100 Subject: [PATCH 45/96] Resolved bug with sending submit commit --- .../protocols/common/submit-commit-command.js | 2 +- src/constants/constants.js | 2 ++ .../blockchain/blockchain-module-manager.js | 18 +++++++++--------- .../blockchain/implementation/web3-service.js | 2 +- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/commands/protocols/common/submit-commit-command.js b/src/commands/protocols/common/submit-commit-command.js index 4776202b32..c8a56730be 100644 --- a/src/commands/protocols/common/submit-commit-command.js +++ b/src/commands/protocols/common/submit-commit-command.js @@ -102,10 +102,10 @@ class SubmitCommitCommand extends Command { tokenId, keyword, hashFunctionId, - epoch, closestNode, leftNeighborhoodEdge, rightNeighborhoodEdge, + epoch, stateIndex, (result) => { if (result?.error) { diff --git a/src/constants/constants.js b/src/constants/constants.js index 3c66be43d7..8acb8159c5 100644 --- a/src/constants/constants.js +++ b/src/constants/constants.js @@ -615,6 +615,8 @@ export const CACHED_FUNCTIONS = { commitWindowDurationPerc: CACHE_DATA_TYPES.NUMBER, proofWindowDurationPerc: CACHE_DATA_TYPES.NUMBER, epochLength: CACHE_DATA_TYPES.NUMBER, + maximumStake: CACHE_DATA_TYPES.NUMBER, + minimumStake: CACHE_DATA_TYPES.NUMBER, }, IdentityStorageContract: { getIdentityId: CACHE_DATA_TYPES.NUMBER, diff --git a/src/modules/blockchain/blockchain-module-manager.js b/src/modules/blockchain/blockchain-module-manager.js index 5f7d3f7691..aea8031a66 100644 --- a/src/modules/blockchain/blockchain-module-manager.js +++ b/src/modules/blockchain/blockchain-module-manager.js @@ -257,13 +257,13 @@ class BlockchainModuleManager extends BaseModuleManager { tokenId, keyword, hashFunctionId, + closestNode, + leftNeighborhoodEdge, + rightNeighborhoodEdge, epoch, latestStateIndex, callback, gasPrice, - closestNode, - leftNeighborhoodEdge, - rightNeighborhoodEdge, ]); } @@ -273,24 +273,24 @@ class BlockchainModuleManager extends BaseModuleManager { tokenId, keyword, hashFunctionId, - epoch, - callback, - gasPrice, closestNode, leftNeighborhoodEdge, rightNeighborhoodEdge, + epoch, + callback, + gasPrice, ) { return this.callImplementationFunction(blockchain, 'submitUpdateCommit', [ assetContractAddress, tokenId, keyword, hashFunctionId, - epoch, - callback, - gasPrice, closestNode, leftNeighborhoodEdge, rightNeighborhoodEdge, + epoch, + callback, + gasPrice, ]); } diff --git a/src/modules/blockchain/implementation/web3-service.js b/src/modules/blockchain/implementation/web3-service.js index 1c326bbd48..a1c307c4f4 100644 --- a/src/modules/blockchain/implementation/web3-service.js +++ b/src/modules/blockchain/implementation/web3-service.js @@ -982,10 +982,10 @@ class Web3Service { tokenId, keyword, hashFunctionId, - epoch, closestNode, leftNeighborhoodEdge, rightNeighborhoodEdge, + epoch, latestStateIndex, callback, gasPrice, From 8b2d1b10ce28bb8aa2ac3c7c1f1d00e23ab99a91 Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Mon, 29 Jan 2024 16:23:33 +0100 Subject: [PATCH 46/96] Call serviceAgreementService.calculateRank in handle update request --- .../receiver/v1.0.0/v1-0-0-handle-update-request-command.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js b/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js index d30c6b384d..00ec748806 100644 --- a/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js +++ b/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js @@ -96,7 +96,7 @@ class HandleUpdateRequestCommand extends HandleProtocolMessageCommand { throw Error('Unable to find neighbourhood edges for asset'); } - const rank = await this.calculateRank( + const rank = await this.serviceAgreementService.calculateRank( blockchain, keyword, hashFunctionId, From e02a1f865c590c6734db4b9fbc797530f9c98efe Mon Sep 17 00:00:00 2001 From: Djordje Kovacevic Date: Mon, 29 Jan 2024 16:35:35 +0100 Subject: [PATCH 47/96] Updated caching of params --- src/constants/constants.js | 2 -- src/service/proximity-scoring-service.js | 8 ++++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/constants/constants.js b/src/constants/constants.js index 8acb8159c5..3c66be43d7 100644 --- a/src/constants/constants.js +++ b/src/constants/constants.js @@ -615,8 +615,6 @@ export const CACHED_FUNCTIONS = { commitWindowDurationPerc: CACHE_DATA_TYPES.NUMBER, proofWindowDurationPerc: CACHE_DATA_TYPES.NUMBER, epochLength: CACHE_DATA_TYPES.NUMBER, - maximumStake: CACHE_DATA_TYPES.NUMBER, - minimumStake: CACHE_DATA_TYPES.NUMBER, }, IdentityStorageContract: { getIdentityId: CACHE_DATA_TYPES.NUMBER, diff --git a/src/service/proximity-scoring-service.js b/src/service/proximity-scoring-service.js index 8b02fbad77..0ea1bba144 100644 --- a/src/service/proximity-scoring-service.js +++ b/src/service/proximity-scoring-service.js @@ -74,7 +74,11 @@ class ProximityScoringService { } async Log2PLDSF(blockchain, distance, stake) { - const log2PLDSFParams = await this.blockchainModuleManager.getLog2PLDSFParams(blockchain); + if (!this.log2PLDSFParams) { + this.log2PLDSFParams = await this.blockchainModuleManager.getLog2PLDSFParams( + blockchain, + ); + } const { distanceMappingCoefficient, @@ -87,7 +91,7 @@ class ProximityScoringService { c, distanceExponent, d, - } = log2PLDSFParams; + } = this.log2PLDSFParams; const mappedStake = this.blockchainModuleManager .toBigNumber(blockchain, this.blockchainModuleManager.convertToWei(blockchain, stake)) From 954830286bf51226d326386ec845caebc069206d Mon Sep 17 00:00:00 2001 From: Djordje Kovacevic Date: Mon, 29 Jan 2024 17:35:23 +0100 Subject: [PATCH 48/96] Updated contract call caching, updated sharding table fetching --- src/constants/constants.js | 6 ++++++ src/modules/blockchain/implementation/web3-service.js | 3 ++- src/service/proximity-scoring-service.js | 8 ++------ src/service/sharding-table-service.js | 2 +- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/constants/constants.js b/src/constants/constants.js index 3c66be43d7..3ce7b61eb6 100644 --- a/src/constants/constants.js +++ b/src/constants/constants.js @@ -560,6 +560,7 @@ export const CONTRACTS = { SERVICE_AGREEMENT_V1_CONTRACT: 'ServiceAgreementV1Contract', PARAMETERS_STORAGE_CONTRACT: 'ParametersStorageContract', IDENTITY_STORAGE_CONTRACT: 'IdentityStorageContract', + Log2PLDSF: 'Log2PLDSF', }; export const CONTRACT_EVENTS = { @@ -615,8 +616,13 @@ export const CACHED_FUNCTIONS = { commitWindowDurationPerc: CACHE_DATA_TYPES.NUMBER, proofWindowDurationPerc: CACHE_DATA_TYPES.NUMBER, epochLength: CACHE_DATA_TYPES.NUMBER, + minimumStake: null, + maximumStake: null, }, IdentityStorageContract: { getIdentityId: CACHE_DATA_TYPES.NUMBER, }, + Log2PLDSF: { + getParameters: null, + }, }; diff --git a/src/modules/blockchain/implementation/web3-service.js b/src/modules/blockchain/implementation/web3-service.js index a1c307c4f4..3f79314619 100644 --- a/src/modules/blockchain/implementation/web3-service.js +++ b/src/modules/blockchain/implementation/web3-service.js @@ -1077,7 +1077,7 @@ class Web3Service { } async getShardingTableHead() { - return this.callContractFunction(this.ShardingTableStorageContract, 'head', []); + return this.callContractFunction(this.ShardingTableStorageContract, 'getNodeByIndex', [0]); } async getShardingTableLength() { @@ -1207,6 +1207,7 @@ class Web3Service { this.scoringFunctionsContracts[1], 'getParameters', [], + CONTRACTS.Log2PLDSF, ); const params = {}; diff --git a/src/service/proximity-scoring-service.js b/src/service/proximity-scoring-service.js index 0ea1bba144..8b02fbad77 100644 --- a/src/service/proximity-scoring-service.js +++ b/src/service/proximity-scoring-service.js @@ -74,11 +74,7 @@ class ProximityScoringService { } async Log2PLDSF(blockchain, distance, stake) { - if (!this.log2PLDSFParams) { - this.log2PLDSFParams = await this.blockchainModuleManager.getLog2PLDSFParams( - blockchain, - ); - } + const log2PLDSFParams = await this.blockchainModuleManager.getLog2PLDSFParams(blockchain); const { distanceMappingCoefficient, @@ -91,7 +87,7 @@ class ProximityScoringService { c, distanceExponent, d, - } = this.log2PLDSFParams; + } = log2PLDSFParams; const mappedStake = this.blockchainModuleManager .toBigNumber(blockchain, this.blockchainModuleManager.convertToWei(blockchain, stake)) diff --git a/src/service/sharding-table-service.js b/src/service/sharding-table-service.js index 3a7aa15de9..f9fd8a7d7c 100644 --- a/src/service/sharding-table-service.js +++ b/src/service/sharding-table-service.js @@ -72,7 +72,7 @@ class ShardingTableService { // eslint-disable-next-line no-await-in-loop const nodes = await this.blockchainModuleManager.getShardingTablePage( blockchainId, - startingIdentityId, + Number(startingIdentityId.identityId), pageSize, ); shardingTable.push(...nodes.slice(sliceIndex).filter((node) => node.nodeId !== '0x')); From 6b2243a10353b48af7f5fc27d0864479ca6a86bc Mon Sep 17 00:00:00 2001 From: Djordje Kovacevic Date: Mon, 29 Jan 2024 17:48:57 +0100 Subject: [PATCH 49/96] Updated constants --- src/constants/constants.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/constants/constants.js b/src/constants/constants.js index 3ce7b61eb6..5a568c3dbb 100644 --- a/src/constants/constants.js +++ b/src/constants/constants.js @@ -597,6 +597,7 @@ export const TRANSACTION_CONFIRMATIONS = 1; export const CACHE_DATA_TYPES = { NUMBER: 'number', + ANY: 'any', }; /** @@ -616,13 +617,13 @@ export const CACHED_FUNCTIONS = { commitWindowDurationPerc: CACHE_DATA_TYPES.NUMBER, proofWindowDurationPerc: CACHE_DATA_TYPES.NUMBER, epochLength: CACHE_DATA_TYPES.NUMBER, - minimumStake: null, - maximumStake: null, + minimumStake: CACHE_DATA_TYPES.ANY, + maximumStake: CACHE_DATA_TYPES.ANY, }, IdentityStorageContract: { getIdentityId: CACHE_DATA_TYPES.NUMBER, }, Log2PLDSF: { - getParameters: null, + getParameters: CACHE_DATA_TYPES.ANY, }, }; From 79bfedfc26c0aad70a953d061a9c51bbf1b42827 Mon Sep 17 00:00:00 2001 From: NZT48 Date: Tue, 30 Jan 2024 00:40:34 +0100 Subject: [PATCH 50/96] Fix pulling of sharding table; --- src/migration/pull-sharding-table-migration.js | 1 + src/service/sharding-table-service.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/migration/pull-sharding-table-migration.js b/src/migration/pull-sharding-table-migration.js index 9b0673c33d..968cc8b9d9 100644 --- a/src/migration/pull-sharding-table-migration.js +++ b/src/migration/pull-sharding-table-migration.js @@ -49,6 +49,7 @@ class PullBlockchainShardingTableMigration extends BaseMigration { ...nodes.slice(sliceIndex).filter((node) => node.nodeId !== '0x'), ); sliceIndex = 1; + // TODO: Should we fix it here also startingIdentityId = nodes[nodes.length - 1].identityId; } diff --git a/src/service/sharding-table-service.js b/src/service/sharding-table-service.js index f9fd8a7d7c..fe0b243014 100644 --- a/src/service/sharding-table-service.js +++ b/src/service/sharding-table-service.js @@ -77,7 +77,7 @@ class ShardingTableService { ); shardingTable.push(...nodes.slice(sliceIndex).filter((node) => node.nodeId !== '0x')); sliceIndex = 1; - startingIdentityId = nodes[nodes.length - 1].identityId; + startingIdentityId = nodes[nodes.length - 1]; } this.logger.debug( From 438ce4dcc6a7575ec9c142daa0b5e00272e7423d Mon Sep 17 00:00:00 2001 From: NZT48 Date: Tue, 30 Jan 2024 01:01:49 +0100 Subject: [PATCH 51/96] Add commit manager v2 contracts --- src/modules/blockchain/implementation/web3-service.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/modules/blockchain/implementation/web3-service.js b/src/modules/blockchain/implementation/web3-service.js index 3f79314619..e8a2fa36cc 100644 --- a/src/modules/blockchain/implementation/web3-service.js +++ b/src/modules/blockchain/implementation/web3-service.js @@ -43,6 +43,8 @@ const ABIs = { ServiceAgreementV1: require('dkg-evm-module/abi/ServiceAgreementV1.json'), CommitManagerV1: require('dkg-evm-module/abi/CommitManagerV2.json'), CommitManagerV1U1: require('dkg-evm-module/abi/CommitManagerV2U1.json'), + CommitManagerV2: require('dkg-evm-module/abi/CommitManagerV2.json'), + CommitManagerV2U1: require('dkg-evm-module/abi/CommitManagerV2U1.json'), ProofManagerV1: require('dkg-evm-module/abi/ProofManagerV1.json'), ProofManagerV1U1: require('dkg-evm-module/abi/ProofManagerV1U1.json'), ShardingTable: require('dkg-evm-module/abi/ShardingTableV2.json'), @@ -875,8 +877,8 @@ class Web3Service { selectCommitManagerContract(latestStateIndex) { return latestStateIndex === 0 - ? this.CommitManagerV1Contract - : this.CommitManagerV1U1Contract; + ? this.CommitManagerV2Contract + : this.CommitManagerV2U1Contract; } async isCommitWindowOpen(agreementId, epoch, latestStateIndex) { @@ -889,7 +891,7 @@ class Web3Service { async isUpdateCommitWindowOpen(agreementId, epoch, stateIndex) { return this.callContractFunction( - this.CommitManagerV1U1Contract, + this.CommitManagerV2U1Contract, 'isUpdateCommitWindowOpen', [agreementId, epoch, stateIndex], ); @@ -1020,7 +1022,7 @@ class Web3Service { submitCommitArgs.push(closestNode, leftNeighborhoodEdge, rightNeighborhoodEdge); } return this.queueTransaction( - this.CommitManagerV1U1Contract, + this.CommitManagerV2U1Contract, 'submitUpdateCommit', [submitCommitArgs], callback, From be68bc4fc86481e4d7e510dd94926b3b33d73372 Mon Sep 17 00:00:00 2001 From: NZT48 Date: Tue, 30 Jan 2024 01:03:42 +0100 Subject: [PATCH 52/96] Add ToDo comments --- src/constants/constants.js | 1 + src/service/blockchain-event-listener-service.js | 1 + 2 files changed, 2 insertions(+) diff --git a/src/constants/constants.js b/src/constants/constants.js index 5a568c3dbb..211066d0a5 100644 --- a/src/constants/constants.js +++ b/src/constants/constants.js @@ -556,6 +556,7 @@ export const CONTRACTS = { STAKING_CONTRACT: 'StakingContract', PROFILE_CONTRACT: 'ProfileContract', HUB_CONTRACT: 'HubContract', + // TODO: Update with new commit Managers COMMIT_MANAGER_V1_U1_CONTRACT: 'CommitManagerV1U1Contract', SERVICE_AGREEMENT_V1_CONTRACT: 'ServiceAgreementV1Contract', PARAMETERS_STORAGE_CONTRACT: 'ParametersStorageContract', diff --git a/src/service/blockchain-event-listener-service.js b/src/service/blockchain-event-listener-service.js index 22d5f0a32a..b389092527 100644 --- a/src/service/blockchain-event-listener-service.js +++ b/src/service/blockchain-event-listener-service.js @@ -67,6 +67,7 @@ class BlockchainEventListenerService { currentBlock, CONTRACT_EVENTS.PROFILE, ), + // TODO: Update with new commit managers this.getContractEvents( blockchainId, CONTRACTS.COMMIT_MANAGER_V1_U1_CONTRACT, From 93b5c8a49daeffc73b4da5c969fea6e8c3d1f9fe Mon Sep 17 00:00:00 2001 From: Nikola Todorovic Date: Tue, 30 Jan 2024 09:03:34 +0100 Subject: [PATCH 53/96] Update src/commands/protocols/common/submit-commit-command.js --- src/commands/protocols/common/submit-commit-command.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/protocols/common/submit-commit-command.js b/src/commands/protocols/common/submit-commit-command.js index c8a56730be..db76d0a11d 100644 --- a/src/commands/protocols/common/submit-commit-command.js +++ b/src/commands/protocols/common/submit-commit-command.js @@ -132,7 +132,7 @@ class SubmitCommitCommand extends Command { `Token ID: ${tokenId}, Keyword: ${keyword}, Hash function ID: ${hashFunctionId}, ` + `Epoch: ${epoch}, State Index: ${stateIndex}, Operation ID: ${operationId}, ` + `Clossest Node: ${closestNode}, Left neighborhood edge: ${leftNeighborhoodEdge}, ` + - `Right neighborhood edge: ${rightNeighborhoodEdge}` + + `Right neighborhood edge: ${rightNeighborhoodEdge}, ` + `Retry number: ${COMMAND_RETRIES.SUBMIT_COMMIT - command.retries + 1}.`, ); From 1d54d1017919073df58ca0a776b689039628b765 Mon Sep 17 00:00:00 2001 From: Djordje Kovacevic Date: Tue, 30 Jan 2024 10:20:40 +0100 Subject: [PATCH 54/96] Updated pull sharding table --- src/modules/blockchain/implementation/web3-service.js | 4 +--- src/service/sharding-table-service.js | 4 ++-- test/bdd/steps/lib/local-blockchain.mjs | 3 +++ 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/modules/blockchain/implementation/web3-service.js b/src/modules/blockchain/implementation/web3-service.js index e8a2fa36cc..722ec93db4 100644 --- a/src/modules/blockchain/implementation/web3-service.js +++ b/src/modules/blockchain/implementation/web3-service.js @@ -43,8 +43,6 @@ const ABIs = { ServiceAgreementV1: require('dkg-evm-module/abi/ServiceAgreementV1.json'), CommitManagerV1: require('dkg-evm-module/abi/CommitManagerV2.json'), CommitManagerV1U1: require('dkg-evm-module/abi/CommitManagerV2U1.json'), - CommitManagerV2: require('dkg-evm-module/abi/CommitManagerV2.json'), - CommitManagerV2U1: require('dkg-evm-module/abi/CommitManagerV2U1.json'), ProofManagerV1: require('dkg-evm-module/abi/ProofManagerV1.json'), ProofManagerV1U1: require('dkg-evm-module/abi/ProofManagerV1U1.json'), ShardingTable: require('dkg-evm-module/abi/ShardingTableV2.json'), @@ -1079,7 +1077,7 @@ class Web3Service { } async getShardingTableHead() { - return this.callContractFunction(this.ShardingTableStorageContract, 'getNodeByIndex', [0]); + return this.callContractFunction(this.ShardingTableStorageContract, 'head', []); } async getShardingTableLength() { diff --git a/src/service/sharding-table-service.js b/src/service/sharding-table-service.js index fe0b243014..3a7aa15de9 100644 --- a/src/service/sharding-table-service.js +++ b/src/service/sharding-table-service.js @@ -72,12 +72,12 @@ class ShardingTableService { // eslint-disable-next-line no-await-in-loop const nodes = await this.blockchainModuleManager.getShardingTablePage( blockchainId, - Number(startingIdentityId.identityId), + startingIdentityId, pageSize, ); shardingTable.push(...nodes.slice(sliceIndex).filter((node) => node.nodeId !== '0x')); sliceIndex = 1; - startingIdentityId = nodes[nodes.length - 1]; + startingIdentityId = nodes[nodes.length - 1].identityId; } this.logger.debug( diff --git a/test/bdd/steps/lib/local-blockchain.mjs b/test/bdd/steps/lib/local-blockchain.mjs index 19410f9504..b686cec378 100644 --- a/test/bdd/steps/lib/local-blockchain.mjs +++ b/test/bdd/steps/lib/local-blockchain.mjs @@ -22,6 +22,9 @@ const testParametersStorageParams = { proofWindowDurationPerc: 33, // 2 minutes updateCommitWindowDuration: 60, // 1 minute finalizationCommitsNumber: 3, + r0: 3, + r1: 5, + r2: 6, }; /** * LocalBlockchain represent small wrapper around the Ganache. From cb746886f5d74ea714e514d08efa07d85b361cc5 Mon Sep 17 00:00:00 2001 From: Djordje Kovacevic Date: Tue, 30 Jan 2024 10:29:28 +0100 Subject: [PATCH 55/96] Updated contracts in web3service --- src/modules/blockchain/implementation/web3-service.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/modules/blockchain/implementation/web3-service.js b/src/modules/blockchain/implementation/web3-service.js index 722ec93db4..ed360997be 100644 --- a/src/modules/blockchain/implementation/web3-service.js +++ b/src/modules/blockchain/implementation/web3-service.js @@ -875,8 +875,8 @@ class Web3Service { selectCommitManagerContract(latestStateIndex) { return latestStateIndex === 0 - ? this.CommitManagerV2Contract - : this.CommitManagerV2U1Contract; + ? this.CommitManagerV1Contract + : this.CommitManagerV1U1Contract; } async isCommitWindowOpen(agreementId, epoch, latestStateIndex) { @@ -889,7 +889,7 @@ class Web3Service { async isUpdateCommitWindowOpen(agreementId, epoch, stateIndex) { return this.callContractFunction( - this.CommitManagerV2U1Contract, + this.CommitManagerV1U1Contract, 'isUpdateCommitWindowOpen', [agreementId, epoch, stateIndex], ); @@ -1020,7 +1020,7 @@ class Web3Service { submitCommitArgs.push(closestNode, leftNeighborhoodEdge, rightNeighborhoodEdge); } return this.queueTransaction( - this.CommitManagerV2U1Contract, + this.CommitManagerV1U1Contract, 'submitUpdateCommit', [submitCommitArgs], callback, From 6f30c243fa75c11e0f0d9b47d12c3846f768d8be Mon Sep 17 00:00:00 2001 From: Djordje Kovacevic Date: Tue, 30 Jan 2024 11:59:10 +0100 Subject: [PATCH 56/96] Updated key calculation --- src/service/sharding-table-service.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/service/sharding-table-service.js b/src/service/sharding-table-service.js index 3a7aa15de9..72c5394f98 100644 --- a/src/service/sharding-table-service.js +++ b/src/service/sharding-table-service.js @@ -317,12 +317,14 @@ class ShardingTableService { blockchainId, hashFunctionId, proximityScoreFunctionsPairId, - assetHash, + key, ) { + const keyHash = await this.hashingService.callHashFunction(hashFunctionId, key); + const hashFunctionName = this.hashingService.getHashFunctionName(hashFunctionId); const assetPositionOnHashRing = await this.blockchainModuleManager.toBigNumber( blockchainId, - assetHash, + keyHash, ); const hashRing = []; @@ -330,7 +332,7 @@ class ShardingTableService { blockchainId, proximityScoreFunctionsPairId, neighbourhood[neighbourhood.length - 1][hashFunctionName], - assetHash, + keyHash, ); for (const neighbour of neighbourhood) { // eslint-disable-next-line no-await-in-loop From 2e9deea221f58978a3aae310a0d1ae553696908f Mon Sep 17 00:00:00 2001 From: Djordje Kovacevic Date: Tue, 30 Jan 2024 12:14:57 +0100 Subject: [PATCH 57/96] Updated submit commit method signature --- src/modules/blockchain/implementation/web3-service.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/modules/blockchain/implementation/web3-service.js b/src/modules/blockchain/implementation/web3-service.js index ed360997be..52e5db9544 100644 --- a/src/modules/blockchain/implementation/web3-service.js +++ b/src/modules/blockchain/implementation/web3-service.js @@ -991,12 +991,15 @@ class Web3Service { gasPrice, ) { const submitCommitArgs = [assetContractAddress, tokenId, keyword, hashFunctionId, epoch]; + let functionName = 'submitCommit((address, uint256, bytes, uint8, uint16))'; if (closestNode && leftNeighborhoodEdge && rightNeighborhoodEdge) { submitCommitArgs.push(closestNode, leftNeighborhoodEdge, rightNeighborhoodEdge); + functionName = + 'submitCommit((address, uint256, bytes, uint8, uint16, uint72, uint72, uint72))'; } return this.queueTransaction( this.selectCommitManagerContract(latestStateIndex), - 'submitCommit', + functionName, [submitCommitArgs], callback, gasPrice, From b9734399a54b8d900c01da929f9524ecadc327ae Mon Sep 17 00:00:00 2001 From: Djordje Kovacevic Date: Tue, 30 Jan 2024 12:36:58 +0100 Subject: [PATCH 58/96] Updated submit commit function call --- src/modules/blockchain/implementation/web3-service.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/modules/blockchain/implementation/web3-service.js b/src/modules/blockchain/implementation/web3-service.js index 52e5db9544..74c4c493d6 100644 --- a/src/modules/blockchain/implementation/web3-service.js +++ b/src/modules/blockchain/implementation/web3-service.js @@ -991,11 +991,11 @@ class Web3Service { gasPrice, ) { const submitCommitArgs = [assetContractAddress, tokenId, keyword, hashFunctionId, epoch]; - let functionName = 'submitCommit((address, uint256, bytes, uint8, uint16))'; + let functionName = 'submitCommit((address,uint256,bytes,uint8,uint16))'; if (closestNode && leftNeighborhoodEdge && rightNeighborhoodEdge) { submitCommitArgs.push(closestNode, leftNeighborhoodEdge, rightNeighborhoodEdge); functionName = - 'submitCommit((address, uint256, bytes, uint8, uint16, uint72, uint72, uint72))'; + 'submitCommit((address,uint256,bytes,uint8,uint16,uint72,uint72,uint72))'; } return this.queueTransaction( this.selectCommitManagerContract(latestStateIndex), From beb7ba32f997ce5cb067c67dbd8a571712d2e4a5 Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Tue, 30 Jan 2024 12:40:14 +0100 Subject: [PATCH 59/96] Fix _executeContractFunction gasLimit error handling --- src/modules/blockchain/implementation/web3-service.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/modules/blockchain/implementation/web3-service.js b/src/modules/blockchain/implementation/web3-service.js index 74c4c493d6..8116847566 100644 --- a/src/modules/blockchain/implementation/web3-service.js +++ b/src/modules/blockchain/implementation/web3-service.js @@ -482,6 +482,12 @@ class Web3Service { } catch (error) { const decodedErrorData = this._decodeErrorData(error, contractInstance.interface); + if (error.transaction === undefined) { + throw new Error( + `Gas estimation for ${functionName} has failed, reason: ${decodedErrorData}`, + ); + } + const functionFragment = contractInstance.interface.getFunction( error.transaction.data.slice(0, 10), ); From 0dc872399a50286e20ebf48f219ea99efb88e74c Mon Sep 17 00:00:00 2001 From: Djordje Kovacevic Date: Tue, 30 Jan 2024 12:52:54 +0100 Subject: [PATCH 60/96] Added logs for sharding table --- src/service/sharding-table-service.js | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/service/sharding-table-service.js b/src/service/sharding-table-service.js index 72c5394f98..e296b51876 100644 --- a/src/service/sharding-table-service.js +++ b/src/service/sharding-table-service.js @@ -334,12 +334,22 @@ class ShardingTableService { neighbourhood[neighbourhood.length - 1][hashFunctionName], keyHash, ); + this.logger.info(`Searching for edges:`); + this.logger.info(`****************`); + for (const neighbour of neighbourhood) { // eslint-disable-next-line no-await-in-loop const neighbourPositionOnHashRing = await this.blockchainModuleManager.toBigNumber( blockchainId, neighbour[hashFunctionName], ); + this.logger.info( + `peerId: ${neighbour.peerId} sha256 : ${ + neighbour.sha256 + }, asset position: ${assetPositionOnHashRing}, maxDistance: ${maxDistance.toString()} ring position: ${neighbourPositionOnHashRing.toString()} index: ${ + neighbour.index + }`, + ); if (assetPositionOnHashRing.lte(neighbourPositionOnHashRing)) { if (neighbourPositionOnHashRing.sub(assetPositionOnHashRing).lt(maxDistance)) { hashRing.push(neighbour); @@ -354,6 +364,11 @@ class ShardingTableService { } } } + this.logger.info('Hash ring:'); + hashRing.forEach((node) => { + this.logger.info(JSON.stringify(node, null, 2)); + }); + this.logger.info(`****************`); return { leftEdge: hashRing[0], From bf2b3e794b23e75afaf3843cb9e2d9d6048fe9af Mon Sep 17 00:00:00 2001 From: Djordje Kovacevic Date: Tue, 30 Jan 2024 13:06:16 +0100 Subject: [PATCH 61/96] Updated max distance calculation for hash ring position --- src/service/sharding-table-service.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/service/sharding-table-service.js b/src/service/sharding-table-service.js index e296b51876..847e67f484 100644 --- a/src/service/sharding-table-service.js +++ b/src/service/sharding-table-service.js @@ -351,13 +351,13 @@ class ShardingTableService { }`, ); if (assetPositionOnHashRing.lte(neighbourPositionOnHashRing)) { - if (neighbourPositionOnHashRing.sub(assetPositionOnHashRing).lt(maxDistance)) { + if (neighbourPositionOnHashRing.sub(assetPositionOnHashRing).lte(maxDistance)) { hashRing.push(neighbour); } else { hashRing.unshift(neighbour); } } else if (assetPositionOnHashRing.gt(neighbourPositionOnHashRing)) { - if (assetPositionOnHashRing.sub(neighbourPositionOnHashRing).lt(maxDistance)) { + if (assetPositionOnHashRing.sub(neighbourPositionOnHashRing).lte(maxDistance)) { hashRing.unshift(neighbour); } else { hashRing.push(neighbour); From 27649a366e69d954bba03bd349a25b4b15624ee0 Mon Sep 17 00:00:00 2001 From: Djordje Kovacevic Date: Tue, 30 Jan 2024 13:19:07 +0100 Subject: [PATCH 62/96] Fixed submit commit contract call --- src/modules/blockchain/implementation/web3-service.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/modules/blockchain/implementation/web3-service.js b/src/modules/blockchain/implementation/web3-service.js index 8116847566..a0ab460295 100644 --- a/src/modules/blockchain/implementation/web3-service.js +++ b/src/modules/blockchain/implementation/web3-service.js @@ -998,7 +998,11 @@ class Web3Service { ) { const submitCommitArgs = [assetContractAddress, tokenId, keyword, hashFunctionId, epoch]; let functionName = 'submitCommit((address,uint256,bytes,uint8,uint16))'; - if (closestNode && leftNeighborhoodEdge && rightNeighborhoodEdge) { + if ( + closestNode !== undefined && + leftNeighborhoodEdge !== undefined && + rightNeighborhoodEdge !== undefined + ) { submitCommitArgs.push(closestNode, leftNeighborhoodEdge, rightNeighborhoodEdge); functionName = 'submitCommit((address,uint256,bytes,uint8,uint16,uint72,uint72,uint72))'; From dffdd031290f1347e245e3d9036f045f8a8d698f Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Tue, 30 Jan 2024 14:35:13 +0100 Subject: [PATCH 63/96] Fix submitUpdateCommit --- .../update/receiver/submit-update-commit-command.js | 6 +++--- src/modules/blockchain/implementation/web3-service.js | 11 +++++++++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/commands/protocols/update/receiver/submit-update-commit-command.js b/src/commands/protocols/update/receiver/submit-update-commit-command.js index 0213c31bce..1b26bd13c4 100644 --- a/src/commands/protocols/update/receiver/submit-update-commit-command.js +++ b/src/commands/protocols/update/receiver/submit-update-commit-command.js @@ -84,6 +84,9 @@ class SubmitUpdateCommitCommand extends Command { tokenId, keyword, hashFunctionId, + closestNode, + leftNeighborhoodEdge, + rightNeighborhoodEdge, epoch, (result) => { if (result?.error) { @@ -93,9 +96,6 @@ class SubmitUpdateCommitCommand extends Command { resolve(); }, txGasPrice, - closestNode, - leftNeighborhoodEdge, - rightNeighborhoodEdge, ); }); diff --git a/src/modules/blockchain/implementation/web3-service.js b/src/modules/blockchain/implementation/web3-service.js index a0ab460295..fcddc30f65 100644 --- a/src/modules/blockchain/implementation/web3-service.js +++ b/src/modules/blockchain/implementation/web3-service.js @@ -1029,12 +1029,19 @@ class Web3Service { gasPrice, ) { const submitCommitArgs = [assetContractAddress, tokenId, keyword, hashFunctionId, epoch]; - if (closestNode && leftNeighborhoodEdge && rightNeighborhoodEdge) { + let functionName = 'submitUpdateCommit((address,uint256,bytes,uint8,uint16))'; + if ( + closestNode !== undefined && + leftNeighborhoodEdge !== undefined && + rightNeighborhoodEdge !== undefined + ) { submitCommitArgs.push(closestNode, leftNeighborhoodEdge, rightNeighborhoodEdge); + functionName = + 'submitUpdateCommit((address,uint256,bytes,uint8,uint16,uint72,uint72,uint72))'; } return this.queueTransaction( this.CommitManagerV1U1Contract, - 'submitUpdateCommit', + functionName, [submitCommitArgs], callback, gasPrice, From db6ba3f15ffae15978710a15403d9e937887c937 Mon Sep 17 00:00:00 2001 From: NZT48 Date: Tue, 30 Jan 2024 14:40:51 +0100 Subject: [PATCH 64/96] Check if edges exist befor looking for index --- src/commands/protocols/common/epoch-check-command.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/commands/protocols/common/epoch-check-command.js b/src/commands/protocols/common/epoch-check-command.js index df5a31bb72..914002ea30 100644 --- a/src/commands/protocols/common/epoch-check-command.js +++ b/src/commands/protocols/common/epoch-check-command.js @@ -302,8 +302,8 @@ class EpochCheckCommand extends Command { agreementId: agreement.agreementId, stateIndex: agreement.stateIndex, closestNode: closestNode.index, - leftNeighborhoodEdge: neighbourhoodEdges.leftEdge.index, - rightNeighborhoodEdge: neighbourhoodEdges.rightEdge.index, + leftNeighborhoodEdge: neighbourhoodEdges?.leftEdge.index, + rightNeighborhoodEdge: neighbourhoodEdges?.rightEdge.index, }; await this.commandExecutor.add({ From dec0b0c8035e0b4430192f74807010dd715b2f4b Mon Sep 17 00:00:00 2001 From: Djordje Kovacevic Date: Tue, 30 Jan 2024 16:48:38 +0100 Subject: [PATCH 65/96] Removed logs --- src/service/sharding-table-service.js | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/service/sharding-table-service.js b/src/service/sharding-table-service.js index 847e67f484..d6fd6e5718 100644 --- a/src/service/sharding-table-service.js +++ b/src/service/sharding-table-service.js @@ -334,8 +334,6 @@ class ShardingTableService { neighbourhood[neighbourhood.length - 1][hashFunctionName], keyHash, ); - this.logger.info(`Searching for edges:`); - this.logger.info(`****************`); for (const neighbour of neighbourhood) { // eslint-disable-next-line no-await-in-loop @@ -343,13 +341,6 @@ class ShardingTableService { blockchainId, neighbour[hashFunctionName], ); - this.logger.info( - `peerId: ${neighbour.peerId} sha256 : ${ - neighbour.sha256 - }, asset position: ${assetPositionOnHashRing}, maxDistance: ${maxDistance.toString()} ring position: ${neighbourPositionOnHashRing.toString()} index: ${ - neighbour.index - }`, - ); if (assetPositionOnHashRing.lte(neighbourPositionOnHashRing)) { if (neighbourPositionOnHashRing.sub(assetPositionOnHashRing).lte(maxDistance)) { hashRing.push(neighbour); @@ -364,11 +355,6 @@ class ShardingTableService { } } } - this.logger.info('Hash ring:'); - hashRing.forEach((node) => { - this.logger.info(JSON.stringify(node, null, 2)); - }); - this.logger.info(`****************`); return { leftEdge: hashRing[0], From 4a14b9cad038ef89f4fb203e3a78acc0c853c6d6 Mon Sep 17 00:00:00 2001 From: NZT48 Date: Tue, 30 Jan 2024 17:09:27 +0100 Subject: [PATCH 66/96] Set dkg-evm-module dependecy to github repo --- package-lock.json | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9f02771c78..88af26486f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25,7 +25,7 @@ "axios": "^1.6.0", "cors": "^2.8.5", "deep-extend": "^0.6.0", - "dkg-evm-module": "^4.1.0", + "dkg-evm-module": "github:OriginTrail/dkg-evm-module#release/delegations", "dotenv": "^16.0.1", "ethers": "^5.7.2", "express": "^4.18.1", @@ -8562,9 +8562,9 @@ } }, "node_modules/dkg-evm-module": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/dkg-evm-module/-/dkg-evm-module-4.1.0.tgz", - "integrity": "sha512-80l9dIjUchBk5SOx6+hdo/I1+r1hUpxHmz3fwQuLISvof4aLSVZ2oqvkEdMPtkvBOjgT3F7uKnHSu6SanBa40A==", + "version": "4.2.0", + "resolved": "git+ssh://git@github.com/OriginTrail/dkg-evm-module.git#316b67a1ec9fbccad08331be7d4be6a3d2a310c7", + "license": "Apache-2.0", "dependencies": { "@openzeppelin/contracts": "^4.9.3", "@polkadot/api": "^10.1.4", @@ -27985,9 +27985,8 @@ "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==" }, "dkg-evm-module": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/dkg-evm-module/-/dkg-evm-module-4.1.0.tgz", - "integrity": "sha512-80l9dIjUchBk5SOx6+hdo/I1+r1hUpxHmz3fwQuLISvof4aLSVZ2oqvkEdMPtkvBOjgT3F7uKnHSu6SanBa40A==", + "version": "git+ssh://git@github.com/OriginTrail/dkg-evm-module.git#316b67a1ec9fbccad08331be7d4be6a3d2a310c7", + "from": "dkg-evm-module@github:OriginTrail/dkg-evm-module#release/delegations", "requires": { "@openzeppelin/contracts": "^4.9.3", "@polkadot/api": "^10.1.4", From 958c40f3439a10fda356d6043f8cde585780531d Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Tue, 30 Jan 2024 17:14:25 +0100 Subject: [PATCH 67/96] Remove async; from shard-repository --- .../implementation/sequelize/repositories/shard-repository.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/modules/repository/implementation/sequelize/repositories/shard-repository.js b/src/modules/repository/implementation/sequelize/repositories/shard-repository.js index 067711f03f..25a2051633 100644 --- a/src/modules/repository/implementation/sequelize/repositories/shard-repository.js +++ b/src/modules/repository/implementation/sequelize/repositories/shard-repository.js @@ -147,8 +147,6 @@ class ShardRepository { where: blockchainId ? { blockchainId } : {}, }); } - - async; } export default ShardRepository; From 2f3043ae5f0d934366b9e1802b67694d1f2d735f Mon Sep 17 00:00:00 2001 From: Nikola Todorovic Date: Tue, 30 Jan 2024 17:15:42 +0100 Subject: [PATCH 68/96] Update src/controllers/http-api/v0/bid-suggestion-http-api-controller-v0.js --- .../http-api/v0/bid-suggestion-http-api-controller-v0.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/controllers/http-api/v0/bid-suggestion-http-api-controller-v0.js b/src/controllers/http-api/v0/bid-suggestion-http-api-controller-v0.js index 9f4be4b180..ce397f483d 100644 --- a/src/controllers/http-api/v0/bid-suggestion-http-api-controller-v0.js +++ b/src/controllers/http-api/v0/bid-suggestion-http-api-controller-v0.js @@ -52,7 +52,7 @@ class BidSuggestionController extends BaseController { } = req.query; let { proximityScoreFunctionsPairId } = req.query; try { - // ADD-DOCS + // TODO: ADD-DOCS if (!proximityScoreFunctionsPairId) { if (blockchain.startsWith('otp') || blockchain.startsWith('hardhat1')) proximityScoreFunctionsPairId = 1; From aacf13e1c4336dd7deab0e53e50808661844de35 Mon Sep 17 00:00:00 2001 From: NZT48 Date: Tue, 30 Jan 2024 17:24:06 +0100 Subject: [PATCH 69/96] Set dkg-evm-module dependecy to github repo --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 83890b211d..499bd6abe6 100644 --- a/package.json +++ b/package.json @@ -80,7 +80,7 @@ "axios": "^1.6.0", "cors": "^2.8.5", "deep-extend": "^0.6.0", - "dkg-evm-module": "^4.1.0", + "dkg-evm-module": "github:OriginTrail/dkg-evm-module#release/delegations", "dotenv": "^16.0.1", "ethers": "^5.7.2", "express": "^4.18.1", From 4a2547d81ea6f19130f9ef9d16fad265e5744b85 Mon Sep 17 00:00:00 2001 From: Uladzislau Hubar Date: Tue, 30 Jan 2024 17:59:41 +0100 Subject: [PATCH 70/96] Fixed formula for LinearSum score calculation to be consistent with the blockchain implementation --- package-lock.json | 4 +- .../protocols/common/epoch-check-command.js | 14 +++- .../v1-0-0-handle-update-request-command.js | 9 +++ src/constants/constants.js | 4 + .../blockchain/implementation/web3-service.js | 5 +- src/service/proximity-scoring-service.js | 80 ++++++++++++++----- src/service/service-agreement-service.js | 15 ++++ 7 files changed, 106 insertions(+), 25 deletions(-) diff --git a/package-lock.json b/package-lock.json index 88af26486f..014c3fdcc6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25,7 +25,7 @@ "axios": "^1.6.0", "cors": "^2.8.5", "deep-extend": "^0.6.0", - "dkg-evm-module": "github:OriginTrail/dkg-evm-module#release/delegations", + "dkg-evm-module": "^4.1.0", "dotenv": "^16.0.1", "ethers": "^5.7.2", "express": "^4.18.1", @@ -27986,7 +27986,7 @@ }, "dkg-evm-module": { "version": "git+ssh://git@github.com/OriginTrail/dkg-evm-module.git#316b67a1ec9fbccad08331be7d4be6a3d2a310c7", - "from": "dkg-evm-module@github:OriginTrail/dkg-evm-module#release/delegations", + "from": "dkg-evm-module@^4.1.0", "requires": { "@openzeppelin/contracts": "^4.9.3", "@polkadot/api": "^10.1.4", diff --git a/src/commands/protocols/common/epoch-check-command.js b/src/commands/protocols/common/epoch-check-command.js index 914002ea30..477a3b9747 100644 --- a/src/commands/protocols/common/epoch-check-command.js +++ b/src/commands/protocols/common/epoch-check-command.js @@ -74,9 +74,12 @@ class EpochCheckCommand extends Command { totalTransactions -= transactionQueueLength; - const [r0, r2] = await Promise.all([ + const [r0, r2, totalNodesNumber, minStake, maxStake] = await Promise.all([ this.blockchainModuleManager.getR0(blockchain), this.blockchainModuleManager.getR2(blockchain), + this.blockchainModuleManager.getShardingTableLength(blockchain), + this.blockchainModuleManager.getMinimumStake(blockchain), + this.blockchainModuleManager.getMaximumStake(blockchain), ]); await Promise.all([ @@ -86,6 +89,9 @@ class EpochCheckCommand extends Command { commitWindowDurationPerc, r0, r2, + totalNodesNumber, + minStake, + maxStake, ), this.scheduleCalculateProofsCommands( blockchain, @@ -112,6 +118,9 @@ class EpochCheckCommand extends Command { commitWindowDurationPerc, r0, r2, + totalNodesNumber, + minStake, + maxStake, ) { const timestamp = await this.blockchainModuleManager.getBlockchainTimestamp(blockchain); const eligibleAgreementForSubmitCommit = @@ -159,6 +168,9 @@ class EpochCheckCommand extends Command { r2, neighbourhood, neighbourhoodEdges, + totalNodesNumber, + minStake, + maxStake, ); updateServiceAgreementsLastCommitEpoch.push( diff --git a/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js b/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js index 00ec748806..fb75499c09 100644 --- a/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js +++ b/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js @@ -96,6 +96,12 @@ class HandleUpdateRequestCommand extends HandleProtocolMessageCommand { throw Error('Unable to find neighbourhood edges for asset'); } + const totalNodesNumber = await this.blockchainModuleManager.getShardingTableLength( + blockchain, + ); + const minStake = await this.blockchainModuleManager.getMinimumStake(blockchain); + const maxStake = await this.blockchainModuleManager.getMaximumStake(blockchain); + const rank = await this.serviceAgreementService.calculateRank( blockchain, keyword, @@ -104,6 +110,9 @@ class HandleUpdateRequestCommand extends HandleProtocolMessageCommand { r2, neighbourhood, neighbourhoodEdges, + totalNodesNumber, + minStake, + maxStake, ); if (rank != null) { this.logger.trace(`Calculated rank: ${rank + 1} for agreement id: ${agreementId}`); diff --git a/src/constants/constants.js b/src/constants/constants.js index 211066d0a5..474741c599 100644 --- a/src/constants/constants.js +++ b/src/constants/constants.js @@ -12,6 +12,10 @@ export const UINT256_MAX_BN = BigNumber.from(2).pow(256).sub(1); export const UINT128_MAX_BN = BigNumber.from(2).pow(128).sub(1); +export const UINT64_MAX_BN = BigNumber.from(2).pow(64).sub(1); + +export const UINT40_MAX_BN = BigNumber.from(2).pow(40).sub(1); + export const UINT32_MAX_BN = BigNumber.from(2).pow(32).sub(1); export const HASH_RING_SIZE = BigNumber.from(2).pow(256); diff --git a/src/modules/blockchain/implementation/web3-service.js b/src/modules/blockchain/implementation/web3-service.js index fcddc30f65..c5cbc1be3d 100644 --- a/src/modules/blockchain/implementation/web3-service.js +++ b/src/modules/blockchain/implementation/web3-service.js @@ -1283,8 +1283,9 @@ class Web3Service { ); return { distanceScaleFactor: linearSumParams[0], - w1: linearSumParams[1], - w2: linearSumParams[2], + stakeScaleFactor: linearSumParams[1], + w1: linearSumParams[2], + w2: linearSumParams[3], }; } } diff --git a/src/service/proximity-scoring-service.js b/src/service/proximity-scoring-service.js index 8b02fbad77..fb6c09a732 100644 --- a/src/service/proximity-scoring-service.js +++ b/src/service/proximity-scoring-service.js @@ -1,5 +1,10 @@ import { xor as uint8ArrayXor } from 'uint8arrays/xor'; -import { HASH_RING_SIZE, UINT128_MAX_BN } from '../constants/constants.js'; +import { + HASH_RING_SIZE, + UINT40_MAX_BN, + UINT64_MAX_BN, + UINT256_MAX_BN, +} from '../constants/constants.js'; class ProximityScoringService { constructor(ctx) { @@ -9,7 +14,7 @@ class ProximityScoringService { this.blockchainModuleManager = ctx.blockchainModuleManager; this.proximityScoreFunctionsPairs = { - 1: [this.calculateBinaryXOR.bind(this), this.Log2PLDSF.bind(this)], + 1: [this.calculateBinaryXOR.bind(this), this.log2PLDSF.bind(this)], 2: [ this.calculateBidirectionalProximityOnHashRing.bind(this), this.linearSum.bind(this), @@ -73,7 +78,7 @@ class ProximityScoringService { return directDistance.lt(wraparoundDistance) ? directDistance : wraparoundDistance; } - async Log2PLDSF(blockchain, distance, stake) { + async log2PLDSF(blockchain, distance, stake) { const log2PLDSFParams = await this.blockchainModuleManager.getLog2PLDSFParams(blockchain); const { @@ -110,29 +115,64 @@ class ProximityScoringService { }; } - async linearSum(blockchain, distance, stake, maxNeighborhoodDistance) { + async linearSum( + blockchain, + distance, + stake, + maxNeighborhoodDistance, + r2, + nodesNumber, + minStake, + maxStake, + ) { const linearSumParams = await this.blockchainModuleManager.getLinearSumParams(blockchain); - const { distanceScaleFactor, w1, w2 } = linearSumParams; - const minimumStake = await this.blockchainModuleManager.getMinimumStake(blockchain); - const maximumStake = await this.blockchainModuleManager.getMaximumStake(blockchain); - - let dividend = distance; - let divisor = maxNeighborhoodDistance; - if (dividend.gt(UINT128_MAX_BN) || divisor.gt(UINT128_MAX_BN)) { - dividend = dividend.div(distanceScaleFactor); - divisor = divisor.div(distanceScaleFactor); + const { distanceScaleFactor, stakeScaleFactor, w1, w2 } = linearSumParams; + + const idealMaxDistanceInNeighborhood = HASH_RING_SIZE.div(nodesNumber).mul( + Math.ceil(r2 / 2), + ); + const divisor = + maxNeighborhoodDistance <= idealMaxDistanceInNeighborhood + ? maxNeighborhoodDistance + : idealMaxDistanceInNeighborhood; + + const maxMultiplier = UINT256_MAX_BN.div(distance); + + let scaledDistanceScaleFactor = distanceScaleFactor; + let compensationFactor = 1; + + if (scaledDistanceScaleFactor.gt(maxMultiplier)) { + compensationFactor = scaledDistanceScaleFactor.div(maxMultiplier); + scaledDistanceScaleFactor = maxMultiplier; } - const divResult = dividend.mul(distanceScaleFactor).div(divisor); + const scaledDistance = distance.mul(scaledDistanceScaleFactor); + const adjustedDivisor = divisor.div(compensationFactor); - const mappedDistance = - parseFloat(divResult.toString()) / parseFloat(distanceScaleFactor.toString()); - const mappedStake = (stake - minimumStake) / (maximumStake - minimumStake); + let normalizedDistance = scaledDistance.div(adjustedDivisor); + if (normalizedDistance.gt(UINT64_MAX_BN)) { + normalizedDistance = normalizedDistance.mod(UINT64_MAX_BN.add(1)); + } - const proximityScore = w1 * (1 - mappedDistance); - const stakeScore = w2 * mappedStake; + let normalizedStake = stakeScaleFactor.mul(stake - minStake).div(maxStake - minStake); + if (normalizedStake.gt(UINT64_MAX_BN)) { + normalizedStake = normalizedStake.mod(UINT64_MAX_BN.add(1)); + } + + const oneEther = await this.blockchainModuleManager.toBigNumber( + blockchain, + '1000000000000000000', + ); + + const proximityScore = oneEther.sub(normalizedDistance).mul(w1); + const stakeScore = normalizedStake.mul(w2); + + let finalScore = proximityScore.add(stakeScore); + if (finalScore.gt(UINT40_MAX_BN)) { + finalScore = finalScore.mod(UINT40_MAX_BN.add(1)); + } - return proximityScore + stakeScore; + return finalScore; } } diff --git a/src/service/service-agreement-service.js b/src/service/service-agreement-service.js index 1dc580158c..de4ecd9076 100644 --- a/src/service/service-agreement-service.js +++ b/src/service/service-agreement-service.js @@ -33,6 +33,9 @@ class ServiceAgreementService { r2, neighbourhood, neighbourhoodEdges, + totalNodesNumber, + minStake, + maxStake, ) { const peerId = this.networkModuleManager.getPeerId().toB58String(); if (!neighbourhood.some((node) => node.peerId === peerId)) { @@ -48,6 +51,10 @@ class ServiceAgreementService { hashFunctionId, proximityScoreFunctionsPairId, neighbourhoodEdges, + r2, + totalNodesNumber, + minStake, + maxStake, ), peerId: node.peerId, })), @@ -65,6 +72,10 @@ class ServiceAgreementService { hashFunctionId, proximityScoreFunctionsPairId, neighbourhoodEdges, + r2, + totalNodesNumber, + minStake, + maxStake, ) { const peerRecord = await this.repositoryModuleManager.getPeerRecord(peerId, blockchainId); const keyHash = await this.hashingService.callHashFunction(hashFunctionId, keyword); @@ -93,6 +104,10 @@ class ServiceAgreementService { distance, peerRecord.stake, maxNeighborhoodDistance, + r2, + totalNodesNumber, + minStake, + maxStake, ); } } From 2886775c4044a0bc780d4528bd0ff5df9207608a Mon Sep 17 00:00:00 2001 From: Uladzislau Hubar Date: Tue, 30 Jan 2024 18:03:31 +0100 Subject: [PATCH 71/96] Reverted back dependency for dkg-evm-module --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 014c3fdcc6..88af26486f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25,7 +25,7 @@ "axios": "^1.6.0", "cors": "^2.8.5", "deep-extend": "^0.6.0", - "dkg-evm-module": "^4.1.0", + "dkg-evm-module": "github:OriginTrail/dkg-evm-module#release/delegations", "dotenv": "^16.0.1", "ethers": "^5.7.2", "express": "^4.18.1", @@ -27986,7 +27986,7 @@ }, "dkg-evm-module": { "version": "git+ssh://git@github.com/OriginTrail/dkg-evm-module.git#316b67a1ec9fbccad08331be7d4be6a3d2a310c7", - "from": "dkg-evm-module@^4.1.0", + "from": "dkg-evm-module@github:OriginTrail/dkg-evm-module#release/delegations", "requires": { "@openzeppelin/contracts": "^4.9.3", "@polkadot/api": "^10.1.4", diff --git a/package.json b/package.json index 83890b211d..499bd6abe6 100644 --- a/package.json +++ b/package.json @@ -80,7 +80,7 @@ "axios": "^1.6.0", "cors": "^2.8.5", "deep-extend": "^0.6.0", - "dkg-evm-module": "^4.1.0", + "dkg-evm-module": "github:OriginTrail/dkg-evm-module#release/delegations", "dotenv": "^16.0.1", "ethers": "^5.7.2", "express": "^4.18.1", From 4209640c4ba580aa7ba1b7745e4974a9b5d0c3c3 Mon Sep 17 00:00:00 2001 From: Uladzislau Hubar Date: Tue, 30 Jan 2024 18:30:30 +0100 Subject: [PATCH 72/96] Added casting for LinearSum parameters --- src/modules/blockchain/implementation/web3-service.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/modules/blockchain/implementation/web3-service.js b/src/modules/blockchain/implementation/web3-service.js index c5cbc1be3d..85a3c05bcd 100644 --- a/src/modules/blockchain/implementation/web3-service.js +++ b/src/modules/blockchain/implementation/web3-service.js @@ -1282,10 +1282,10 @@ class Web3Service { [], ); return { - distanceScaleFactor: linearSumParams[0], - stakeScaleFactor: linearSumParams[1], - w1: linearSumParams[2], - w2: linearSumParams[3], + distanceScaleFactor: BigNumber.from(linearSumParams[0]), + stakeScaleFactor: BigNumber.from(linearSumParams[1]), + w1: Number(linearSumParams[2]), + w2: Number(linearSumParams[3]), }; } } From a4755a66df17226ad03de86306c87c0112320264 Mon Sep 17 00:00:00 2001 From: NZT48 Date: Tue, 30 Jan 2024 18:52:23 +0100 Subject: [PATCH 73/96] Update package-lock with latest dkg-evm-module commit --- package-lock.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 88af26486f..4b88fa9e76 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8563,7 +8563,7 @@ }, "node_modules/dkg-evm-module": { "version": "4.2.0", - "resolved": "git+ssh://git@github.com/OriginTrail/dkg-evm-module.git#316b67a1ec9fbccad08331be7d4be6a3d2a310c7", + "resolved": "git+ssh://git@github.com/OriginTrail/dkg-evm-module.git#940afd112685cfd20aad52f13274b1ce47bff409", "license": "Apache-2.0", "dependencies": { "@openzeppelin/contracts": "^4.9.3", @@ -27985,7 +27985,7 @@ "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==" }, "dkg-evm-module": { - "version": "git+ssh://git@github.com/OriginTrail/dkg-evm-module.git#316b67a1ec9fbccad08331be7d4be6a3d2a310c7", + "version": "git+ssh://git@github.com/OriginTrail/dkg-evm-module.git#940afd112685cfd20aad52f13274b1ce47bff409", "from": "dkg-evm-module@github:OriginTrail/dkg-evm-module#release/delegations", "requires": { "@openzeppelin/contracts": "^4.9.3", From e7ac43352ba89d59e726f244bb555728d1f750d9 Mon Sep 17 00:00:00 2001 From: NZT48 Date: Tue, 30 Jan 2024 19:03:16 +0100 Subject: [PATCH 74/96] Bump version to 6.2.0 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 898a889d07..7d7967bf9f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "origintrail_node", - "version": "6.1.3", + "version": "6.2.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "origintrail_node", - "version": "6.1.3", + "version": "6.2.0", "license": "ISC", "dependencies": { "@comunica/query-sparql": "^2.4.3", diff --git a/package.json b/package.json index 4be976c169..4e6c99b95e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "6.1.3", + "version": "6.2.0", "description": "OTNode V6", "main": "index.js", "type": "module", From 27aaf4a0850029e3eb9fd008c648593d76f62149 Mon Sep 17 00:00:00 2001 From: NZT48 Date: Tue, 30 Jan 2024 19:33:56 +0100 Subject: [PATCH 75/96] Fix blobl migration --- package-lock.json | 4 ++-- package.json | 2 +- .../migrations/20240126120000-shard-add-sha256blobl.js | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7d7967bf9f..cd8f2d71d6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "origintrail_node", - "version": "6.2.0", + "version": "6.2.0+hotfix1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "origintrail_node", - "version": "6.2.0", + "version": "6.2.0+hotfix1", "license": "ISC", "dependencies": { "@comunica/query-sparql": "^2.4.3", diff --git a/package.json b/package.json index 4e6c99b95e..315419b4ca 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "6.2.0", + "version": "6.2.0+hotfix1", "description": "OTNode V6", "main": "index.js", "type": "module", diff --git a/src/modules/repository/implementation/sequelize/migrations/20240126120000-shard-add-sha256blobl.js b/src/modules/repository/implementation/sequelize/migrations/20240126120000-shard-add-sha256blobl.js index 7db8aceed3..89059f3454 100644 --- a/src/modules/repository/implementation/sequelize/migrations/20240126120000-shard-add-sha256blobl.js +++ b/src/modules/repository/implementation/sequelize/migrations/20240126120000-shard-add-sha256blobl.js @@ -17,8 +17,8 @@ export async function up({ context: { queryInterface, Sequelize } }) { { replacements: { sha256Blob, - peerId: shard.peerId, - blockchainId: shard.blockchainId, + peerId: shard.peer_id, + blockchainId: shard.blockchain_id, }, type: queryInterface.sequelize.QueryTypes.UPDATE, }, From a0df5003dbce78ddd2fd6c98cc87e398bef95914 Mon Sep 17 00:00:00 2001 From: Uladzislau Hubar Date: Tue, 30 Jan 2024 20:32:32 +0100 Subject: [PATCH 76/96] Added opdb migrator rollback, fixed blobl migration to add new column only if it doesn't already exist --- package-lock.json | 4 ++-- package.json | 2 +- .../migrations/20240126120000-shard-add-sha256blobl.js | 10 +++++++--- .../implementation/sequelize/sequelize-repository.js | 8 +++++++- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/package-lock.json b/package-lock.json index cd8f2d71d6..6d2bf449b8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "origintrail_node", - "version": "6.2.0+hotfix1", + "version": "6.2.0+hotfix.2", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "origintrail_node", - "version": "6.2.0+hotfix1", + "version": "6.2.0+hotfix.2", "license": "ISC", "dependencies": { "@comunica/query-sparql": "^2.4.3", diff --git a/package.json b/package.json index 315419b4ca..f17413ff2f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "6.2.0+hotfix1", + "version": "6.2.0+hotfix.2", "description": "OTNode V6", "main": "index.js", "type": "module", diff --git a/src/modules/repository/implementation/sequelize/migrations/20240126120000-shard-add-sha256blobl.js b/src/modules/repository/implementation/sequelize/migrations/20240126120000-shard-add-sha256blobl.js index 89059f3454..fa491e6bba 100644 --- a/src/modules/repository/implementation/sequelize/migrations/20240126120000-shard-add-sha256blobl.js +++ b/src/modules/repository/implementation/sequelize/migrations/20240126120000-shard-add-sha256blobl.js @@ -1,7 +1,11 @@ export async function up({ context: { queryInterface, Sequelize } }) { - await queryInterface.addColumn('shard', 'sha256_blob', { - type: Sequelize.BLOB, - }); + const tableInfo = await queryInterface.describeTable('shard'); + + if (!tableInfo.sha256_blob) { + await queryInterface.addColumn('shard', 'sha256_blob', { + type: Sequelize.BLOB, + }); + } const shards = await queryInterface.sequelize.query( 'SELECT peer_id, blockchain_id, sha256 FROM shard', diff --git a/src/modules/repository/implementation/sequelize/sequelize-repository.js b/src/modules/repository/implementation/sequelize/sequelize-repository.js index f232c34f8d..aae1f1c648 100644 --- a/src/modules/repository/implementation/sequelize/sequelize-repository.js +++ b/src/modules/repository/implementation/sequelize/sequelize-repository.js @@ -94,7 +94,13 @@ class SequelizeRepository { async runMigrations() { const migrator = createMigrator(this.models.sequelize, this.config, this.logger); - await migrator.up(); + try { + await migrator.up(); + } catch (error) { + this.logger.error(`Failed to execute ${migrator.name} migration: ${error.message}.`); + await migrator.down(); + throw error; + } } async loadModels() { From 26bd58e46bb2a0bb93b4cf5b2666f16435770d3d Mon Sep 17 00:00:00 2001 From: Djordje Kovacevic Date: Wed, 31 Jan 2024 14:02:20 +0100 Subject: [PATCH 77/96] Added caching for linear sum get parameters call Added pull sharding table migration Removed blob calculation from operational db migration --- src/constants/constants.js | 4 ++++ src/migration/migration-executor.js | 2 +- .../blockchain/implementation/web3-service.js | 1 + .../20240126120000-shard-add-sha256blobl.js | 22 ------------------- 4 files changed, 6 insertions(+), 23 deletions(-) diff --git a/src/constants/constants.js b/src/constants/constants.js index 95924936fc..47af51d9c5 100644 --- a/src/constants/constants.js +++ b/src/constants/constants.js @@ -577,6 +577,7 @@ export const CONTRACTS = { PARAMETERS_STORAGE_CONTRACT: 'ParametersStorageContract', IDENTITY_STORAGE_CONTRACT: 'IdentityStorageContract', Log2PLDSF: 'Log2PLDSF', + LINEAR_SUM: 'LinearSum', }; export const CONTRACT_EVENTS = { @@ -646,4 +647,7 @@ export const CACHED_FUNCTIONS = { Log2PLDSF: { getParameters: CACHE_DATA_TYPES.ANY, }, + LinearSum: { + getParameters: CACHE_DATA_TYPES.ANY, + }, }; diff --git a/src/migration/migration-executor.js b/src/migration/migration-executor.js index e24dc97869..3e987f22bc 100644 --- a/src/migration/migration-executor.js +++ b/src/migration/migration-executor.js @@ -30,7 +30,7 @@ class MigrationExecutor { const validationModuleManager = container.resolve('validationModuleManager'); const migration = new PullBlockchainShardingTableMigration( - 'pullShardingTableMigrationV611', + 'pullShardingTableMigrationV612', logger, config, repositoryModuleManager, diff --git a/src/modules/blockchain/implementation/web3-service.js b/src/modules/blockchain/implementation/web3-service.js index 1b21994ae9..db44d6ce73 100644 --- a/src/modules/blockchain/implementation/web3-service.js +++ b/src/modules/blockchain/implementation/web3-service.js @@ -1294,6 +1294,7 @@ class Web3Service { this.scoringFunctionsContracts[2], 'getParameters', [], + CONTRACTS.LINEAR_SUM, ); return { distanceScaleFactor: BigNumber.from(linearSumParams[0]), diff --git a/src/modules/repository/implementation/sequelize/migrations/20240126120000-shard-add-sha256blobl.js b/src/modules/repository/implementation/sequelize/migrations/20240126120000-shard-add-sha256blobl.js index fa491e6bba..45da4b3837 100644 --- a/src/modules/repository/implementation/sequelize/migrations/20240126120000-shard-add-sha256blobl.js +++ b/src/modules/repository/implementation/sequelize/migrations/20240126120000-shard-add-sha256blobl.js @@ -6,28 +6,6 @@ export async function up({ context: { queryInterface, Sequelize } }) { type: Sequelize.BLOB, }); } - - const shards = await queryInterface.sequelize.query( - 'SELECT peer_id, blockchain_id, sha256 FROM shard', - { type: queryInterface.sequelize.QueryTypes.SELECT }, - ); - - for (const shard of shards) { - const sha256Blob = Buffer.from(shard.sha256, 'hex'); - - // eslint-disable-next-line no-await-in-loop - await queryInterface.sequelize.query( - 'UPDATE shard SET sha256_blob = :sha256Blob WHERE peer_id = :peerId AND blockchain_id = :blockchainId', - { - replacements: { - sha256Blob, - peerId: shard.peer_id, - blockchainId: shard.blockchain_id, - }, - type: queryInterface.sequelize.QueryTypes.UPDATE, - }, - ); - } } export async function down({ context: { queryInterface } }) { From 09225d819dd1bd5ec8d725ed85a5e02d9b8219d6 Mon Sep 17 00:00:00 2001 From: Djordje Kovacevic Date: Wed, 31 Jan 2024 14:43:34 +0100 Subject: [PATCH 78/96] Version bump --- package-lock.json | 4 ++-- package.json | 2 +- src/migration/migration-executor.js | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6d2bf449b8..53ac11e5c1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "origintrail_node", - "version": "6.2.0+hotfix.2", + "version": "6.2.0+hotfix.3", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "origintrail_node", - "version": "6.2.0+hotfix.2", + "version": "6.2.0+hotfix.3", "license": "ISC", "dependencies": { "@comunica/query-sparql": "^2.4.3", diff --git a/package.json b/package.json index f17413ff2f..7b1cea9e8b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "6.2.0+hotfix.2", + "version": "6.2.0+hotfix.3", "description": "OTNode V6", "main": "index.js", "type": "module", diff --git a/src/migration/migration-executor.js b/src/migration/migration-executor.js index 3e987f22bc..698f1f97d8 100644 --- a/src/migration/migration-executor.js +++ b/src/migration/migration-executor.js @@ -30,7 +30,7 @@ class MigrationExecutor { const validationModuleManager = container.resolve('validationModuleManager'); const migration = new PullBlockchainShardingTableMigration( - 'pullShardingTableMigrationV612', + 'pullShardingTableMigrationV620', logger, config, repositoryModuleManager, From 4ac736a8dc90df2f3d9735d2f069ff3b15a1217c Mon Sep 17 00:00:00 2001 From: NZT48 Date: Wed, 31 Jan 2024 14:45:43 +0100 Subject: [PATCH 79/96] Fix normalized stake calculation --- src/service/proximity-scoring-service.js | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/service/proximity-scoring-service.js b/src/service/proximity-scoring-service.js index fb6c09a732..2947e114d1 100644 --- a/src/service/proximity-scoring-service.js +++ b/src/service/proximity-scoring-service.js @@ -127,6 +127,18 @@ class ProximityScoringService { ) { const linearSumParams = await this.blockchainModuleManager.getLinearSumParams(blockchain); const { distanceScaleFactor, stakeScaleFactor, w1, w2 } = linearSumParams; + const mappedStake = this.blockchainModuleManager.toBigNumber( + blockchain, + this.blockchainModuleManager.convertToWei(blockchain, stake), + ); + const mappedMinStake = this.blockchainModuleManager.toBigNumber( + blockchain, + this.blockchainModuleManager.convertToWei(blockchain, minStake), + ); + const mappedMaxStake = this.blockchainModuleManager.toBigNumber( + blockchain, + this.blockchainModuleManager.convertToWei(blockchain, maxStake), + ); const idealMaxDistanceInNeighborhood = HASH_RING_SIZE.div(nodesNumber).mul( Math.ceil(r2 / 2), @@ -154,7 +166,9 @@ class ProximityScoringService { normalizedDistance = normalizedDistance.mod(UINT64_MAX_BN.add(1)); } - let normalizedStake = stakeScaleFactor.mul(stake - minStake).div(maxStake - minStake); + let normalizedStake = stakeScaleFactor + .mul(mappedStake.sub(mappedMinStake)) + .div(mappedMaxStake.sub(mappedMinStake)); if (normalizedStake.gt(UINT64_MAX_BN)) { normalizedStake = normalizedStake.mod(UINT64_MAX_BN.add(1)); } From 32b4e36835de62046c42dffb342a2da9c541e3cb Mon Sep 17 00:00:00 2001 From: Uladzislau Hubar Date: Wed, 31 Jan 2024 14:53:35 +0100 Subject: [PATCH 80/96] Fixed LinearSum score calculation to avoid underflows --- src/service/proximity-scoring-service.js | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/service/proximity-scoring-service.js b/src/service/proximity-scoring-service.js index fb6c09a732..a0fcb37f99 100644 --- a/src/service/proximity-scoring-service.js +++ b/src/service/proximity-scoring-service.js @@ -164,10 +164,22 @@ class ProximityScoringService { '1000000000000000000', ); - const proximityScore = oneEther.sub(normalizedDistance).mul(w1); + const isProximityScorePositive = oneEther.gte(normalizedDistance); + + const proximityScore = isProximityScorePositive + ? oneEther.sub(normalizedDistance).mul(w1) + : normalizedDistance.sub(oneEther).mul(w1); const stakeScore = normalizedStake.mul(w2); - let finalScore = proximityScore.add(stakeScore); + let finalScore; + if (isProximityScorePositive) { + finalScore = proximityScore.add(stakeScore); + } else if (stakeScore.gte(proximityScore)) { + finalScore = stakeScore.sub(proximityScore); + } else { + finalScore = await this.blockchainModuleManager.toBigNumber(blockchain, 0); + } + if (finalScore.gt(UINT40_MAX_BN)) { finalScore = finalScore.mod(UINT40_MAX_BN.add(1)); } From 0370e842fed2119c3895b19e24a0ab8507dc4daa Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Wed, 31 Jan 2024 15:09:58 +0100 Subject: [PATCH 81/96] Rename migration so it can be reexectuted --- src/migration/migration-executor.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/migration/migration-executor.js b/src/migration/migration-executor.js index 698f1f97d8..35c3ccffcd 100644 --- a/src/migration/migration-executor.js +++ b/src/migration/migration-executor.js @@ -370,7 +370,7 @@ class MigrationExecutor { const repositoryModuleManager = container.resolve('repositoryModuleManager'); const migration = new RemoveServiceAgreementsForChiadoDevnetMigration( - 'removeServiceAgreementsForChiadoDevnetMigration', + 'removeServiceAgreementsForChiadoDevnetMigrationV2', logger, config, repositoryModuleManager, From 6663e5aa17285db7add69d59309e2ab4f1a5bbc7 Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Thu, 1 Feb 2024 10:29:10 +0100 Subject: [PATCH 82/96] Remove sha256Blob --- src/migration/pull-sharding-table-migration.js | 6 ------ .../20240201100000-remove-sha256Blob.js | 17 +++++++++++++++++ .../implementation/sequelize/models/shard.js | 3 --- .../sequelize/repositories/shard-repository.js | 4 +--- .../blockchain-event-listener-service.js | 4 ---- src/service/sharding-table-service.js | 3 --- 6 files changed, 18 insertions(+), 19 deletions(-) create mode 100644 src/modules/repository/implementation/sequelize/migrations/20240201100000-remove-sha256Blob.js diff --git a/src/migration/pull-sharding-table-migration.js b/src/migration/pull-sharding-table-migration.js index 968cc8b9d9..6455a88dc2 100644 --- a/src/migration/pull-sharding-table-migration.js +++ b/src/migration/pull-sharding-table-migration.js @@ -67,11 +67,6 @@ class PullBlockchainShardingTableMigration extends BaseMigration { const sha256 = await this.hashingService.callHashFunction(1, nodeId); - const cleanHexString = sha256.startsWith('0x') - ? sha256.slice(2) - : sha256; - const sha256Blob = Buffer.from(cleanHexString, 'hex'); - return { peerId: nodeId, blockchainId, @@ -86,7 +81,6 @@ class PullBlockchainShardingTableMigration extends BaseMigration { 'ether', ), sha256, - sha256Blob, }; }), ), diff --git a/src/modules/repository/implementation/sequelize/migrations/20240201100000-remove-sha256Blob.js b/src/modules/repository/implementation/sequelize/migrations/20240201100000-remove-sha256Blob.js new file mode 100644 index 0000000000..25618e1d51 --- /dev/null +++ b/src/modules/repository/implementation/sequelize/migrations/20240201100000-remove-sha256Blob.js @@ -0,0 +1,17 @@ +export async function up({ context: { queryInterface } }) { + const tableInfo = await queryInterface.describeTable('shard'); + + if (tableInfo.sha256_blob) { + await queryInterface.removeColumn('shard', 'sha256_blob'); + } +} + +export async function down({ context: { queryInterface, Sequelize } }) { + const tableInfo = await queryInterface.describeTable('shard'); + + if (!tableInfo.sha256_blob) { + await queryInterface.addColumn('shard', 'sha256_blob', { + type: Sequelize.BLOB, + }); + } +} diff --git a/src/modules/repository/implementation/sequelize/models/shard.js b/src/modules/repository/implementation/sequelize/models/shard.js index 7e5b884e29..cd14aa539f 100644 --- a/src/modules/repository/implementation/sequelize/models/shard.js +++ b/src/modules/repository/implementation/sequelize/models/shard.js @@ -26,9 +26,6 @@ export default (sequelize, DataTypes) => { type: DataTypes.STRING, allowNull: false, }, - sha256Blob: { - type: DataTypes.BLOB, - }, }, { underscored: true }, ); diff --git a/src/modules/repository/implementation/sequelize/repositories/shard-repository.js b/src/modules/repository/implementation/sequelize/repositories/shard-repository.js index 25a2051633..86fb01460b 100644 --- a/src/modules/repository/implementation/sequelize/repositories/shard-repository.js +++ b/src/modules/repository/implementation/sequelize/repositories/shard-repository.js @@ -19,7 +19,7 @@ class ShardRepository { }); } - async createPeerRecord(peerId, blockchainId, ask, stake, lastSeen, sha256, sha256Blob) { + async createPeerRecord(peerId, blockchainId, ask, stake, lastSeen, sha256) { return this.model.create( { peerId, @@ -28,7 +28,6 @@ class ShardRepository { stake, lastSeen, sha256, - sha256Blob, }, { ignoreDuplicates: true, @@ -42,7 +41,6 @@ class ShardRepository { blockchainId, }, attributes: ['peerId', 'blockchainId', 'ask', 'stake', 'lastSeen', 'sha256'], - order: [['sha256Blob', 'ASC']], }; if (filterLastSeen) { diff --git a/src/service/blockchain-event-listener-service.js b/src/service/blockchain-event-listener-service.js index afb4b889d9..89adb006f7 100644 --- a/src/service/blockchain-event-listener-service.js +++ b/src/service/blockchain-event-listener-service.js @@ -312,9 +312,6 @@ class BlockchainEventListenerService { nodeId, ); - const cleanHexString = sha256.startsWith('0x') ? sha256.slice(2) : sha256; - const sha256Blob = Buffer.from(cleanHexString, 'hex'); - this.logger.trace(`Adding peer id: ${nodeId} to sharding table.`); return { peerId: nodeId, @@ -329,7 +326,6 @@ class BlockchainEventListenerService { ), lastSeen: new Date(0), sha256, - sha256Blob, }; }), ); diff --git a/src/service/sharding-table-service.js b/src/service/sharding-table-service.js index d6fd6e5718..4a5a3ff8cf 100644 --- a/src/service/sharding-table-service.js +++ b/src/service/sharding-table-service.js @@ -93,8 +93,6 @@ class ShardingTableService { ); const sha256 = await this.hashingService.callHashFunction(1, nodeId); - const cleanHexString = sha256.startsWith('0x') ? sha256.slice(2) : sha256; - const sha256Blob = Buffer.from(cleanHexString, 'hex'); return { peerId: nodeId, blockchainId, @@ -109,7 +107,6 @@ class ShardingTableService { 'ether', ), sha256, - sha256Blob, }; }), ), From b3da277d74872932d0ca8841a522755ee6e08274 Mon Sep 17 00:00:00 2001 From: Djordje Kovacevic Date: Thu, 1 Feb 2024 11:53:30 +0100 Subject: [PATCH 83/96] Updated epoch check command, added listening on scoring contract events --- .../protocols/common/epoch-check-command.js | 50 ++++++++++++------ .../v1-0-0-handle-update-request-command.js | 4 +- src/constants/constants.js | 6 ++- .../blockchain/implementation/web3-service.js | 9 ++-- .../blockchain-event-listener-service.js | 51 ++++++++++++++++--- 5 files changed, 88 insertions(+), 32 deletions(-) diff --git a/src/commands/protocols/common/epoch-check-command.js b/src/commands/protocols/common/epoch-check-command.js index 477a3b9747..0b144995a5 100644 --- a/src/commands/protocols/common/epoch-check-command.js +++ b/src/commands/protocols/common/epoch-check-command.js @@ -37,11 +37,11 @@ class EpochCheckCommand extends Command { !migrationExecuted ) { this.logger.info( - 'Epoch check command will be postponed until ual extension triple store migration is completed', + 'Epoch check: command will be postponed until ual extension triple store migration is completed', ); return Command.repeat(); } - this.logger.info('Starting epoch check command'); + this.logger.info('Epoch check: Starting epoch check command'); const operationId = this.operationIdService.generateId(); await Promise.all( @@ -77,7 +77,7 @@ class EpochCheckCommand extends Command { const [r0, r2, totalNodesNumber, minStake, maxStake] = await Promise.all([ this.blockchainModuleManager.getR0(blockchain), this.blockchainModuleManager.getR2(blockchain), - this.blockchainModuleManager.getShardingTableLength(blockchain), + this.repositoryModuleManager.getPeersCount(blockchain), this.blockchainModuleManager.getMinimumStake(blockchain), this.blockchainModuleManager.getMaximumStake(blockchain), ]); @@ -129,11 +129,18 @@ class EpochCheckCommand extends Command { blockchain, commitWindowDurationPerc, ); - + this.logger.info( + `Epoch check: Found ${eligibleAgreementForSubmitCommit.length} eligible agreements for submit commit for blockchain: ${blockchain}`, + ); const scheduleSubmitCommitCommands = []; const updateServiceAgreementsLastCommitEpoch = []; for (const serviceAgreement of eligibleAgreementForSubmitCommit) { - if (scheduleSubmitCommitCommands.length >= maxTransactions) break; + if (scheduleSubmitCommitCommands.length >= maxTransactions) { + this.logger.warn( + `Epoch check: not scheduling new commits. Submit commit command length: ${scheduleSubmitCommitCommands.length}, max number of transactions: ${maxTransactions} for blockchain: ${blockchain}`, + ); + break; + } const neighbourhood = await this.shardingTableService.findNeighbourhood( blockchain, @@ -156,7 +163,10 @@ class EpochCheckCommand extends Command { } if (!neighbourhoodEdges && serviceAgreement.scoreFunctionId === 2) { - throw Error('Unable to find neighbourhood edges for asset'); + this.logger.warn( + `Epoch check: unable to find neighbourhood edges for agreement id: ${serviceAgreement.agreementId} for blockchain: ${blockchain}`, + ); + continue; } try { @@ -182,28 +192,28 @@ class EpochCheckCommand extends Command { if (rank == null) { this.logger.trace( - `Node not in R2: ${r2} for the Service Agreement with the ID: ${serviceAgreement.agreementId}. Skipping scheduling submitCommitCommand.`, + `Epoch check: Node not in R2: ${r2} for the Service Agreement with the ID: ${serviceAgreement.agreementId}. Skipping scheduling submitCommitCommand for blockchain: ${blockchain}`, ); continue; } if (rank >= r0) { this.logger.trace( - `Calculated rank: ${ + `Epoch check: Calculated rank: ${ rank + 1 }. Node not in R0: ${r0} for the Service Agreement with the ID: ${ serviceAgreement.agreementId - }. Skipping scheduling submitCommitCommand.`, + }. Skipping scheduling submitCommitCommand for blockchain: ${blockchain}`, ); continue; } this.logger.trace( - `Calculated rank: ${ + `Epoch check: Calculated rank: ${ rank + 1 }. Node in R0: ${r0} for the Service Agreement with the ID: ${ serviceAgreement.agreementId - }. Scheduling submitCommitCommand.`, + }. Scheduling submitCommitCommand for blockchain: ${blockchain}`, ); const closestNode = neighbourhood[0]; scheduleSubmitCommitCommands.push( @@ -215,7 +225,7 @@ class EpochCheckCommand extends Command { ); } catch (error) { this.logger.warn( - `Failed to schedule submitCommitCommand for the Service Agreement with the ID: ${serviceAgreement.agreementId}. Error message: ${error.message}.`, + `Epoch check: Failed to schedule submitCommitCommand for the Service Agreement with the ID: ${serviceAgreement.agreementId} for blockchain: ${blockchain}. Error message: ${error.message}.`, ); continue; } @@ -239,10 +249,18 @@ class EpochCheckCommand extends Command { blockchain, proofWindowDurationPerc, ); + this.logger.info( + `Epoch check: Found ${eligibleAgreementsForSubmitProofs.length} eligible agreements for submit proof for blockchain: ${blockchain}`, + ); const scheduleSubmitProofCommands = []; const updateServiceAgreementsLastProofEpoch = []; for (const serviceAgreement of eligibleAgreementsForSubmitProofs) { - if (scheduleSubmitProofCommands.length >= maxTransactions) break; + if (scheduleSubmitProofCommands.length >= maxTransactions) { + this.logger.warn( + `Epoch check: not scheduling new proofs. Submit proofs command length: ${scheduleSubmitProofCommands.length}, max number of transactions: ${maxTransactions} for blockchain: ${blockchain}`, + ); + break; + } try { const eligibleForReward = await this.isEligibleForRewards( @@ -254,7 +272,7 @@ class EpochCheckCommand extends Command { ); if (eligibleForReward) { this.logger.trace( - `Node is eligible for rewards for the Service Agreement with the ID: ${serviceAgreement.agreementId}. Scheduling submitProofsCommand.`, + `Epoch check: Node is eligible for rewards for the Service Agreement with the ID: ${serviceAgreement.agreementId} for blockchain: ${blockchain}. Scheduling submitProofsCommand.`, ); scheduleSubmitProofCommands.push( @@ -262,7 +280,7 @@ class EpochCheckCommand extends Command { ); } else { this.logger.trace( - `Node is not eligible for rewards for the Service Agreement with the ID: ${serviceAgreement.agreementId}. Skipping scheduling submitProofsCommand.`, + `Epoch check: Node is not eligible for rewards for the Service Agreement with the ID: ${serviceAgreement.agreementId}. Skipping scheduling submitProofsCommand for blockchain: ${blockchain}`, ); } updateServiceAgreementsLastProofEpoch.push( @@ -273,7 +291,7 @@ class EpochCheckCommand extends Command { ); } catch (error) { this.logger.warn( - `Failed to schedule submitProofsCommand for the Service Agreement with the ID: ${serviceAgreement.agreementId}. Error message: ${error.message}.`, + `Epoch check: Failed to schedule submitProofsCommand for the Service Agreement with the ID: ${serviceAgreement.agreementId} for blockchain: ${blockchain}. Error message: ${error.message}.`, ); continue; } diff --git a/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js b/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js index daf2576398..4a2ab745c1 100644 --- a/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js +++ b/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js @@ -96,9 +96,7 @@ class HandleUpdateRequestCommand extends HandleProtocolMessageCommand { throw Error('Unable to find neighbourhood edges for asset'); } - const totalNodesNumber = await this.blockchainModuleManager.getShardingTableLength( - blockchain, - ); + const totalNodesNumber = await this.repositoryModuleManager.getPeersCount(blockchain); const minStake = await this.blockchainModuleManager.getMinimumStake(blockchain); const maxStake = await this.blockchainModuleManager.getMaximumStake(blockchain); diff --git a/src/constants/constants.js b/src/constants/constants.js index 47af51d9c5..ddb44ab5bc 100644 --- a/src/constants/constants.js +++ b/src/constants/constants.js @@ -576,8 +576,8 @@ export const CONTRACTS = { SERVICE_AGREEMENT_V1_CONTRACT: 'ServiceAgreementV1Contract', PARAMETERS_STORAGE_CONTRACT: 'ParametersStorageContract', IDENTITY_STORAGE_CONTRACT: 'IdentityStorageContract', - Log2PLDSF: 'Log2PLDSF', - LINEAR_SUM: 'LinearSum', + Log2PLDSF_CONTRACT: 'Log2PLDSFContract', + LINEAR_SUM_CONTRACT: 'LinearSumContract', }; export const CONTRACT_EVENTS = { @@ -588,6 +588,8 @@ export const CONTRACT_EVENTS = { COMMIT_MANAGER_V1: ['StateFinalized'], SERVICE_AGREEMENT_V1: ['ServiceAgreementV1Extended', 'ServiceAgreementV1Terminated'], PARAMETERS_STORAGE: ['ParameterChanged'], + Log2PLDSF: ['ParameterChanged'], + LINEAR_SUM: ['ParameterChanged'], }; export const NODE_ENVIRONMENTS = { diff --git a/src/modules/blockchain/implementation/web3-service.js b/src/modules/blockchain/implementation/web3-service.js index db44d6ce73..9db13b47de 100644 --- a/src/modules/blockchain/implementation/web3-service.js +++ b/src/modules/blockchain/implementation/web3-service.js @@ -710,9 +710,10 @@ class Web3Service { ) { const contract = this[contractName]; if (!contract) { - throw new Error( - `Error while getting all past events. Unknown contract: ${contractName}`, - ); + // this will happen when we have different set of contracts on different blockchains + // eg LinearSum contract is available on gnosis but not on NeuroWeb, so the node should not fetch events + // from LinearSum contract on NeuroWeb blockchain + return []; } let fromBlock; @@ -1294,7 +1295,7 @@ class Web3Service { this.scoringFunctionsContracts[2], 'getParameters', [], - CONTRACTS.LINEAR_SUM, + CONTRACTS.LINEAR_SUM_CONTRACT, ); return { distanceScaleFactor: BigNumber.from(linearSumParams[0]), diff --git a/src/service/blockchain-event-listener-service.js b/src/service/blockchain-event-listener-service.js index afb4b889d9..aca8db13f2 100644 --- a/src/service/blockchain-event-listener-service.js +++ b/src/service/blockchain-event-listener-service.js @@ -88,6 +88,18 @@ class BlockchainEventListenerService { currentBlock, CONTRACT_EVENTS.PARAMETERS_STORAGE, ), + this.getContractEvents( + blockchainId, + CONTRACTS.Log2PLDSF_CONTRACT, + currentBlock, + CONTRACT_EVENTS.Log2PLDSF, + ), + this.getContractEvents( + blockchainId, + CONTRACTS.LINEAR_SUM_CONTRACT, + currentBlock, + CONTRACT_EVENTS.LINEAR_SUM, + ), ]; if (!devEnvironment) { @@ -239,13 +251,38 @@ class BlockchainEventListenerService { async handleParameterChangedEvents(blockEvents) { for (const event of blockEvents) { - const { parameterName, parameterValue } = JSON.parse(event.data); - this.blockchainModuleManager.setContractCallCache( - event.blockchainId, - CONTRACTS.PARAMETERS_STORAGE_CONTRACT, - parameterName, - parameterValue, - ); + const { blockchainId, contract, data } = event; + const { parameterName, parameterValue } = JSON.parse(data); + switch (contract) { + case CONTRACTS.Log2PLDSF_CONTRACT: + this.blockchainModuleManager.setContractCallCache( + blockchainId, + CONTRACTS.Log2PLDSF_CONTRACT, + parameterName, + null, + ); + break; + case CONTRACTS.LINEAR_SUM_CONTRACT: + this.blockchainModuleManager.setContractCallCache( + blockchainId, + CONTRACTS.LINEAR_SUM_CONTRACT, + parameterName, + null, + ); + break; + case CONTRACTS.PARAMETERS_STORAGE_CONTRACT: + this.blockchainModuleManager.setContractCallCache( + blockchainId, + CONTRACTS.PARAMETERS_STORAGE_CONTRACT, + parameterName, + parameterValue, + ); + break; + default: + this.logger.warn( + `Unable to handle parameter changed event. Unknown contract name ${event.contract}`, + ); + } } } From ac2277a9d80336c1c1f2e7926ea317ffba12ee23 Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Thu, 1 Feb 2024 12:21:58 +0100 Subject: [PATCH 84/96] Get peers ordered by sha256 --- .../implementation/sequelize/repositories/shard-repository.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/modules/repository/implementation/sequelize/repositories/shard-repository.js b/src/modules/repository/implementation/sequelize/repositories/shard-repository.js index 86fb01460b..2b66f77b84 100644 --- a/src/modules/repository/implementation/sequelize/repositories/shard-repository.js +++ b/src/modules/repository/implementation/sequelize/repositories/shard-repository.js @@ -41,6 +41,7 @@ class ShardRepository { blockchainId, }, attributes: ['peerId', 'blockchainId', 'ask', 'stake', 'lastSeen', 'sha256'], + order: [['sha256', 'asc']], }; if (filterLastSeen) { From d19ff0b8eb15ac931d0681529b50c15977efcee0 Mon Sep 17 00:00:00 2001 From: Djordje Kovacevic Date: Thu, 1 Feb 2024 13:06:15 +0100 Subject: [PATCH 85/96] Updated migration for removing service agreements for chiado --- ot-node.js | 2 +- src/migration/migration-executor.js | 15 +++++++++------ ...ve-service-agreements-for-chiado-migration.js} | 4 ++-- 3 files changed, 12 insertions(+), 9 deletions(-) rename src/migration/{remove-service-agreements-for-chiado-devnet-migration.js => remove-service-agreements-for-chiado-migration.js} (76%) diff --git a/ot-node.js b/ot-node.js index a8916ce8b7..7a64c5f38c 100644 --- a/ot-node.js +++ b/ot-node.js @@ -49,7 +49,7 @@ class OTNode { await this.initializeModules(); - await MigrationExecutor.executeRemoveServiceAgreementsForChiadoDevnetMigration( + await MigrationExecutor.executeRemoveServiceAgreementsForChiadoMigration( this.container, this.logger, this.config, diff --git a/src/migration/migration-executor.js b/src/migration/migration-executor.js index 35c3ccffcd..5a5559f777 100644 --- a/src/migration/migration-executor.js +++ b/src/migration/migration-executor.js @@ -15,7 +15,7 @@ import ServiceAgreementsInvalidDataMigration from './service-agreements-invalid- import UalExtensionUserConfigurationMigration from './ual-extension-user-configuration-migration.js'; import UalExtensionTripleStoreMigration from './ual-extension-triple-store-migration.js'; import MarkStakingEventsAsProcessedMigration from './mark-staking-events-as-processed-migration.js'; -import RemoveServiceAgreementsForChiadoDevnetMigration from './remove-service-agreements-for-chiado-devnet-migration.js'; +import RemoveServiceAgreementsForChiadoMigration from './remove-service-agreements-for-chiado-migration.js'; class MigrationExecutor { static async executePullShardingTableMigration(container, logger, config) { @@ -365,12 +365,15 @@ class MigrationExecutor { } } - static async executeRemoveServiceAgreementsForChiadoDevnetMigration(container, logger, config) { - if (process.env.NODE_ENV === NODE_ENVIRONMENTS.DEVNET) { + static async executeRemoveServiceAgreementsForChiadoMigration(container, logger, config) { + if ( + process.env.NODE_ENV === NODE_ENVIRONMENTS.DEVNET || + process.env.NODE_ENV === NODE_ENVIRONMENTS.TESTNET + ) { const repositoryModuleManager = container.resolve('repositoryModuleManager'); - const migration = new RemoveServiceAgreementsForChiadoDevnetMigration( - 'removeServiceAgreementsForChiadoDevnetMigrationV2', + const migration = new RemoveServiceAgreementsForChiadoMigration( + 'removeServiceAgreementsForChiadoMigrationV2', logger, config, repositoryModuleManager, @@ -380,7 +383,7 @@ class MigrationExecutor { await migration.migrate(); } catch (error) { logger.error( - `Unable to execute remove service agreements for Chiado Devnet migration. Error: ${error.message}`, + `Unable to execute remove service agreements for Chiado migration. Error: ${error.message}`, ); this.exitNode(1); } diff --git a/src/migration/remove-service-agreements-for-chiado-devnet-migration.js b/src/migration/remove-service-agreements-for-chiado-migration.js similarity index 76% rename from src/migration/remove-service-agreements-for-chiado-devnet-migration.js rename to src/migration/remove-service-agreements-for-chiado-migration.js index 7c8dab87c5..680931b4f2 100644 --- a/src/migration/remove-service-agreements-for-chiado-devnet-migration.js +++ b/src/migration/remove-service-agreements-for-chiado-migration.js @@ -2,7 +2,7 @@ import BaseMigration from './base-migration.js'; const GNOSIS_DEVNET_CHAIN_ID = 'gnosis:10200'; -class RemoveServiceAgreementsForChiadoDevnetMigration extends BaseMigration { +class RemoveServiceAgreementsForChiadoMigration extends BaseMigration { constructor(migrationName, logger, config, repositoryModuleManager) { super(migrationName, logger, config); this.repositoryModuleManager = repositoryModuleManager; @@ -15,4 +15,4 @@ class RemoveServiceAgreementsForChiadoDevnetMigration extends BaseMigration { } } -export default RemoveServiceAgreementsForChiadoDevnetMigration; +export default RemoveServiceAgreementsForChiadoMigration; From 3c186f4ce669580430db7625ab2e7ebdd59d7dd8 Mon Sep 17 00:00:00 2001 From: Uladzislau Hubar Date: Thu, 1 Feb 2024 14:29:42 +0100 Subject: [PATCH 86/96] Fixed LinearSum score calculation --- src/constants/constants.js | 2 +- src/service/proximity-scoring-service.js | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/constants/constants.js b/src/constants/constants.js index ddb44ab5bc..79a666d1d0 100644 --- a/src/constants/constants.js +++ b/src/constants/constants.js @@ -18,7 +18,7 @@ export const UINT40_MAX_BN = BigNumber.from(2).pow(40).sub(1); export const UINT32_MAX_BN = BigNumber.from(2).pow(32).sub(1); -export const HASH_RING_SIZE = BigNumber.from(2).pow(256); +export const HASH_RING_SIZE = BigNumber.from(2).pow(256).sub(1); export const STAKE_UINT256_MULTIPLIER_BN = UINT256_MAX_BN.div(500000000); diff --git a/src/service/proximity-scoring-service.js b/src/service/proximity-scoring-service.js index ade26896ec..4e06f6197a 100644 --- a/src/service/proximity-scoring-service.js +++ b/src/service/proximity-scoring-service.js @@ -143,10 +143,9 @@ class ProximityScoringService { const idealMaxDistanceInNeighborhood = HASH_RING_SIZE.div(nodesNumber).mul( Math.ceil(r2 / 2), ); - const divisor = - maxNeighborhoodDistance <= idealMaxDistanceInNeighborhood - ? maxNeighborhoodDistance - : idealMaxDistanceInNeighborhood; + const divisor = maxNeighborhoodDistance.lte(idealMaxDistanceInNeighborhood) + ? maxNeighborhoodDistance + : idealMaxDistanceInNeighborhood; const maxMultiplier = UINT256_MAX_BN.div(distance); From bf36e12d6b4f07ff4587c18ae269b74eb2363280 Mon Sep 17 00:00:00 2001 From: Nikola Todorovic Date: Thu, 1 Feb 2024 14:32:54 +0100 Subject: [PATCH 87/96] Update src/service/blockchain-event-listener-service.js --- src/service/blockchain-event-listener-service.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/service/blockchain-event-listener-service.js b/src/service/blockchain-event-listener-service.js index 087854f6cd..d49436ae87 100644 --- a/src/service/blockchain-event-listener-service.js +++ b/src/service/blockchain-event-listener-service.js @@ -255,6 +255,8 @@ class BlockchainEventListenerService { const { parameterName, parameterValue } = JSON.parse(data); switch (contract) { case CONTRACTS.Log2PLDSF_CONTRACT: + // This invalidates contracts parameter + // TODO: Create function for contract call cache invalidation this.blockchainModuleManager.setContractCallCache( blockchainId, CONTRACTS.Log2PLDSF_CONTRACT, From 6c03295abddf92c0b1a101ae4b9365773fab7d02 Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Thu, 1 Feb 2024 14:36:09 +0100 Subject: [PATCH 88/96] Use ehters constant for uin256 max value --- src/constants/constants.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/constants/constants.js b/src/constants/constants.js index 79a666d1d0..24b9539962 100644 --- a/src/constants/constants.js +++ b/src/constants/constants.js @@ -8,7 +8,7 @@ export const FALLBACK_PROVIDER_QUORUM = 1; export const RPC_PROVIDER_STALL_TIMEOUT = 60 * 1000; -export const UINT256_MAX_BN = BigNumber.from(2).pow(256).sub(1); +export const UINT256_MAX_BN = ethers.constants.MaxUint256; export const UINT128_MAX_BN = BigNumber.from(2).pow(128).sub(1); @@ -18,7 +18,7 @@ export const UINT40_MAX_BN = BigNumber.from(2).pow(40).sub(1); export const UINT32_MAX_BN = BigNumber.from(2).pow(32).sub(1); -export const HASH_RING_SIZE = BigNumber.from(2).pow(256).sub(1); +export const HASH_RING_SIZE = ethers.constants.MaxUint256; export const STAKE_UINT256_MULTIPLIER_BN = UINT256_MAX_BN.div(500000000); From 01bb1acca5bd37080e3b28cb29c55013562a3dad Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Thu, 1 Feb 2024 14:55:09 +0100 Subject: [PATCH 89/96] Update commit hash for dkg-evm-module --- package-lock.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 53ac11e5c1..4a1dfdf078 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8563,7 +8563,7 @@ }, "node_modules/dkg-evm-module": { "version": "4.2.0", - "resolved": "git+ssh://git@github.com/OriginTrail/dkg-evm-module.git#940afd112685cfd20aad52f13274b1ce47bff409", + "resolved": "git+ssh://git@github.com/OriginTrail/dkg-evm-module.git#5fcbdfba99d8b8b9d23d9e2277f73a1bfa70c50c", "license": "Apache-2.0", "dependencies": { "@openzeppelin/contracts": "^4.9.3", @@ -27985,7 +27985,7 @@ "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==" }, "dkg-evm-module": { - "version": "git+ssh://git@github.com/OriginTrail/dkg-evm-module.git#940afd112685cfd20aad52f13274b1ce47bff409", + "version": "git+ssh://git@github.com/OriginTrail/dkg-evm-module.git#5fcbdfba99d8b8b9d23d9e2277f73a1bfa70c50c", "from": "dkg-evm-module@github:OriginTrail/dkg-evm-module#release/delegations", "requires": { "@openzeppelin/contracts": "^4.9.3", From ec7919b1ce39dc98935d2dac1afd60350b832b6a Mon Sep 17 00:00:00 2001 From: Nikola Todorovic Date: Thu, 1 Feb 2024 15:56:41 +0100 Subject: [PATCH 90/96] Update package-lock.json --- package-lock.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4a1dfdf078..4614741313 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "origintrail_node", - "version": "6.2.0+hotfix.3", + "version": "6.2.0+hotfix.4", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "origintrail_node", - "version": "6.2.0+hotfix.3", + "version": "6.2.0+hotfix.4", "license": "ISC", "dependencies": { "@comunica/query-sparql": "^2.4.3", From a0fa9aae2a124a61d2e1c541e0142a8dd7a40248 Mon Sep 17 00:00:00 2001 From: Nikola Todorovic Date: Thu, 1 Feb 2024 15:56:59 +0100 Subject: [PATCH 91/96] Update package.json --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 7b1cea9e8b..a7a985cfcf 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "6.2.0+hotfix.3", + "version": "6.2.0+hotfix.4", "description": "OTNode V6", "main": "index.js", "type": "module", From 45744fa94d4ca27e988ac037578fdffa2966bdab Mon Sep 17 00:00:00 2001 From: Nikola Todorovic Date: Fri, 2 Feb 2024 09:56:28 +0100 Subject: [PATCH 92/96] Update otnode.service (#2911) --- installer/data/otnode.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installer/data/otnode.service b/installer/data/otnode.service index 1d2168c8f8..92fbc8d5ed 100755 --- a/installer/data/otnode.service +++ b/installer/data/otnode.service @@ -1,7 +1,7 @@ #/lib/systemd/system/otnode.service [Unit] -Description=OriginTrail V6 Stage 1 Beta Node +Description=OriginTrail V6 Node Documentation=https://github.com/OriginTrail/ot-node/ After=network.target graphdb.service blazegraph.service From 2039e20da4754993f5e7a4992cf3b30503e97abe Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Fri, 2 Feb 2024 09:56:48 +0100 Subject: [PATCH 93/96] V6/bug fix/scoring function result (#2914) * fix score function return type * convert linearSum return type to number --------- Co-authored-by: djordjekovac --- src/service/proximity-scoring-service.js | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/service/proximity-scoring-service.js b/src/service/proximity-scoring-service.js index 4e06f6197a..3dd72c7317 100644 --- a/src/service/proximity-scoring-service.js +++ b/src/service/proximity-scoring-service.js @@ -103,16 +103,10 @@ class ProximityScoringService { const dividend = mappedStake.pow(stakeExponent).mul(a).add(b); const divisor = mappedDistance.pow(distanceExponent).mul(c).add(d); - return { - mappedDistance, - mappedStake, - score: Math.floor( - Number(multiplier) * - Math.log2( - Number(logArgumentConstant) + dividend.toNumber() / divisor.toNumber(), - ), - ), - }; + return Math.floor( + Number(multiplier) * + Math.log2(Number(logArgumentConstant) + dividend.toNumber() / divisor.toNumber()), + ); } async linearSum( @@ -197,7 +191,7 @@ class ProximityScoringService { finalScore = finalScore.mod(UINT40_MAX_BN.add(1)); } - return finalScore; + return finalScore.toNumber(); } } From 11112a4994f6d3e5fd8dd49b5909d5d9605a3342 Mon Sep 17 00:00:00 2001 From: Nikola Todorovic Date: Fri, 2 Feb 2024 10:32:42 +0100 Subject: [PATCH 94/96] Increase gas limit for set stake (#2918) --- scripts/set-stake.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/set-stake.js b/scripts/set-stake.js index 4db4be7fb0..01f01f76a0 100644 --- a/scripts/set-stake.js +++ b/scripts/set-stake.js @@ -95,7 +95,7 @@ async function setStake( // TODO: Add ABI instead of hard-coded function definition tx = await stakingContract['addStake(uint72,uint96)'](identityId, stakeWei, { gasPrice: gasPrice ? gasPrice * 100 : undefined, - gasLimit: 500_000, + gasLimit: 3_000_000, }); await provider.waitForTransaction( tx.hash, From 150735c05d8729d1fcf6506dfc881b460b2c7c6d Mon Sep 17 00:00:00 2001 From: djordjekovac Date: Fri, 2 Feb 2024 10:50:00 +0100 Subject: [PATCH 95/96] Version updated (#2917) * Version updated * Updated request for bdd tests --- package-lock.json | 4 ++-- package.json | 2 +- test/bdd/steps/api/datasets/requests.json | 22 +++++++++++----------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4614741313..9d3780f6a5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "origintrail_node", - "version": "6.2.0+hotfix.4", + "version": "6.2.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "origintrail_node", - "version": "6.2.0+hotfix.4", + "version": "6.2.0", "license": "ISC", "dependencies": { "@comunica/query-sparql": "^2.4.3", diff --git a/package.json b/package.json index a7a985cfcf..4e6c99b95e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "6.2.0+hotfix.4", + "version": "6.2.0", "description": "OTNode V6", "main": "index.js", "type": "module", diff --git a/test/bdd/steps/api/datasets/requests.json b/test/bdd/steps/api/datasets/requests.json index ac5eefe732..d0ca535d19 100644 --- a/test/bdd/steps/api/datasets/requests.json +++ b/test/bdd/steps/api/datasets/requests.json @@ -9,26 +9,26 @@ "_:c14n0 ." ], "blockchain": "hardhat1:31337", - "contract": "0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07", + "contract": "0x0aec7c174554af8aec3680bb58431f6618311510", "tokenId": 0 }, "validGetFirstStateRequestBody": { - "id": "did:dkg:blockchain/0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07/0", + "id": "did:dkg:blockchain/0x0aec7c174554af8aec3680bb58431f6618311510/0", "state": "0xe3a6733d7b999ca6f0d141afe3e38ac59223a4dfde7a5458932d2094ed4193cf" }, "validGetUpdatedStateRequestBody": { - "id": "did:dkg:blockchain/0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07/0" + "id": "did:dkg:blockchain/0x0aec7c174554af8aec3680bb58431f6618311510/0" }, "getFirstStateRequestBody": { - "id": "did:dkg:blockchain/0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07/0", + "id": "did:dkg:blockchain/0x0aec7c174554af8aec3680bb58431f6618311510/0", "state": "0xe3a6733d7b999ca6f0d141afe3e38ac59223a4dfde7a5458932d2094ed4193cf" }, "getSecondStateRequestBody": { - "id": "did:dkg:blockchain/0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07/0", + "id": "did:dkg:blockchain/0x0aec7c174554af8aec3680bb58431f6618311510/0", "state": "0x591503a1c8ba4667dd7afd203025c1bf594d817d8eec71274fe960d69fb8584f" }, "getThirdStateRequestBody": { - "id": "did:dkg:blockchain/0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07/0", + "id": "did:dkg:blockchain/0x0aec7c174554af8aec3680bb58431f6618311510/0", "state": "0x759c285786b95622dad67a6be857a4eec3d9ba0caec991ed4297629ae6abbc0d" }, "blockchainNotDefinedRequestBody": { @@ -42,21 +42,21 @@ "_:c14n0 ." ], "blockchain": null, - "contract": "0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07", + "contract": "0x0aec7c174554af8aec3680bb58431f6618311510", "tokenId": 0 }, "nonExistentUAL": { - "id": "did:dkg:hardhat1:31337/0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07/1" + "id": "did:dkg:hardhat1:31337/0x0aec7c174554af8aec3680bb58431f6618311510/1" }, "invalidUAL": { "id": "did:dkg:hardhat1:31337/0xB0D4afd8879eD9F/52b28595d31B441D079B2Ca07/1" }, "nonExistentState": { - "id": "did:dkg:hardhat1:31337/0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07/0", + "id": "did:dkg:hardhat1:31337/0x0aec7c174554af8aec3680bb58431f6618311510/0", "state": -1 }, "invalidStateHash": { - "id": "did:dkg:hardhat1:31337/0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07/0", + "id": "did:dkg:hardhat1:31337/0x0aec7c174554af8aec3680bb58431f6618311510/0", "state": "0x591503a1c8ba4667dd7afd203025c1bf594d817d8eec71274fe960d69fb8584e" }, "validUpdateRequestBody": { @@ -67,7 +67,7 @@ " ." ], "blockchain": "hardhat1:31337", - "contract": "0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07", + "contract": "0x0aec7c174554af8aec3680bb58431f6618311510", "tokenId": 0 } } From 75c373dd25563d684e3bb998babeff9db225699b Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Fri, 2 Feb 2024 11:26:51 +0100 Subject: [PATCH 96/96] Fix caching --- src/constants/constants.js | 4 ++-- src/modules/blockchain/implementation/web3-service.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/constants/constants.js b/src/constants/constants.js index 24b9539962..c12e3f5482 100644 --- a/src/constants/constants.js +++ b/src/constants/constants.js @@ -646,10 +646,10 @@ export const CACHED_FUNCTIONS = { IdentityStorageContract: { getIdentityId: CACHE_DATA_TYPES.NUMBER, }, - Log2PLDSF: { + Log2PLDSFContract: { getParameters: CACHE_DATA_TYPES.ANY, }, - LinearSum: { + LinearSumContract: { getParameters: CACHE_DATA_TYPES.ANY, }, }; diff --git a/src/modules/blockchain/implementation/web3-service.js b/src/modules/blockchain/implementation/web3-service.js index 9db13b47de..e33a0657b8 100644 --- a/src/modules/blockchain/implementation/web3-service.js +++ b/src/modules/blockchain/implementation/web3-service.js @@ -1242,7 +1242,7 @@ class Web3Service { this.scoringFunctionsContracts[1], 'getParameters', [], - CONTRACTS.Log2PLDSF, + CONTRACTS.Log2PLDSF_CONTRACT, ); const params = {};