From 316f20f5399447b06c4976744960a6316b41ad08 Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Thu, 6 Jun 2024 11:32:10 +0100 Subject: [PATCH 001/243] feat(lib): add example manifest for tdp finder --- manifests/examples/tdp-finder.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 manifests/examples/tdp-finder.yml diff --git a/manifests/examples/tdp-finder.yml b/manifests/examples/tdp-finder.yml new file mode 100644 index 000000000..f0d8751c6 --- /dev/null +++ b/manifests/examples/tdp-finder.yml @@ -0,0 +1,23 @@ +name: csv-demo +description: +tags: +initialize: + plugins: + cloud-instance-metadata: + method: CSVLookup + path: "builtin" + global-config: + filepath: https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/tdp-data-1.csv + query: + name: instance-id + output: "tdp" +tree: + children: + child: + pipeline: + - cloud-instance-metadata + inputs: + - timestamp: 2023-08-06T00:00 + duration: 3600 + cpu/energy: 0.001 + instance-id: "AMD A10-9700" From d74fa639919c1dc8be88dcb13eed5bd5185d1fbb Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Thu, 6 Jun 2024 12:45:54 +0100 Subject: [PATCH 002/243] fix(lib): rename instance to tdp-finder --- manifests/examples/tdp-finder.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/manifests/examples/tdp-finder.yml b/manifests/examples/tdp-finder.yml index f0d8751c6..5ce1b5903 100644 --- a/manifests/examples/tdp-finder.yml +++ b/manifests/examples/tdp-finder.yml @@ -3,7 +3,7 @@ description: tags: initialize: plugins: - cloud-instance-metadata: + tdp-finder: method: CSVLookup path: "builtin" global-config: @@ -15,7 +15,7 @@ tree: children: child: pipeline: - - cloud-instance-metadata + - tdp-finder inputs: - timestamp: 2023-08-06T00:00 duration: 3600 From 36b7e1e4db411e1b0f233c842c576804a4f7af5c Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Thu, 6 Jun 2024 13:02:48 +0100 Subject: [PATCH 003/243] feat(lib): add example manifest for region-metadata --- manifests/examples/region-metadata.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 manifests/examples/region-metadata.yml diff --git a/manifests/examples/region-metadata.yml b/manifests/examples/region-metadata.yml new file mode 100644 index 000000000..d43b5bc65 --- /dev/null +++ b/manifests/examples/region-metadata.yml @@ -0,0 +1,26 @@ +name: csv-demo +description: +tags: +initialize: + plugins: + cloud-region-metadata: + method: CSVLookup + path: "builtin" + global-config: + filepath: https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/region-metadata.csv + query: + cloud-provider: cloud-provider + cloud-region: cloud-region + output: "*" +tree: + children: + child: + pipeline: + - cloud-region-metadata + inputs: + - timestamp: 2023-08-06T00:00 + duration: 3600 + cpu/energy: 0.001 + instance-id: AMD A10-9700 + cloud-provider: "Google Cloud" + cloud-region: "asia-east1" From cb63b7093bc4b0265a2a51d1cf9578c15091da87 Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Thu, 6 Jun 2024 13:36:03 +0100 Subject: [PATCH 004/243] feat(lib): add example manifest for instance metadata --- manifests/examples/instance-metadata.yml | 33 ++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 manifests/examples/instance-metadata.yml diff --git a/manifests/examples/instance-metadata.yml b/manifests/examples/instance-metadata.yml new file mode 100644 index 000000000..dc4da68f1 --- /dev/null +++ b/manifests/examples/instance-metadata.yml @@ -0,0 +1,33 @@ +name: csv-demo +description: +tags: +initialize: + plugins: + cloud-instance-metadata: + method: CSVLookup + path: "builtin" + global-config: + filepath: https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-azure-instances.csv + query: + instance-class: "cloud/instance-type" + output: "*" + extract-processor-name: + method: Regex + path: "builtin" + global-config: + parameter: cpu-model-name + match: /^([^,])+/g + output: cpu/name +tree: + children: + child: + pipeline: + - cloud-instance-metadata + - extract-processor-name + inputs: + - timestamp: 2023-08-06T00:00 + duration: 3600 + cpu/energy: 0.001 + cloud/provider: gcp + cloud/region: asia-east + cloud/instance-type: Standard_A1_v2 From 1adbf1d469747b113f8e43776dabd2029118b2e4 Mon Sep 17 00:00:00 2001 From: manushak Date: Fri, 7 Jun 2024 17:47:39 +0400 Subject: [PATCH 005/243] feat(util): add function to check if the file exists in the given path --- src/util/helpers.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/util/helpers.ts b/src/util/helpers.ts index b15863cef..99291f81d 100644 --- a/src/util/helpers.ts +++ b/src/util/helpers.ts @@ -1,3 +1,5 @@ +import * as fs from 'fs/promises'; + import {createInterface} from 'node:readline/promises'; import {exec} from 'node:child_process'; import {promisify} from 'node:util'; @@ -182,3 +184,15 @@ export const parseManifestFromStdin = async () => { return match![1]; }; + +/** + * Checks if file exists with the given `filePath`. + */ +export const isFileExists = async (filePath: string) => { + try { + await fs.stat(filePath); + return true; + } catch (error) { + return false; + } +}; From 152cfe3f2c87a88d024af21c4e89cfa070c6c248 Mon Sep 17 00:00:00 2001 From: manushak Date: Fri, 7 Jun 2024 17:49:18 +0400 Subject: [PATCH 006/243] feat(config): add 'MANIFEST_NOT_FOUND' string --- src/config/strings.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/config/strings.ts b/src/config/strings.ts index 04551b9b9..d0693c120 100644 --- a/src/config/strings.ts +++ b/src/config/strings.ts @@ -56,4 +56,5 @@ Note that for the '--output' option you also need to define the output type in y TARGET_IS_NOT_YAML: 'Given target is not in yaml format.', INVALID_TARGET: 'Target is invalid.', INVALID_SOURCE: 'Source is invalid.', + MANIFEST_NOT_FOUND: 'Manifest file not found.', }; From 5fd1e79d4fccffa0952992bec6c20139faf33758 Mon Sep 17 00:00:00 2001 From: manushak Date: Fri, 7 Jun 2024 17:51:03 +0400 Subject: [PATCH 007/243] feat(types): adds IFEnvArgs interface --- src/types/process-args.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/types/process-args.ts b/src/types/process-args.ts index 7dfdcb929..e0d9bcc0f 100644 --- a/src/types/process-args.ts +++ b/src/types/process-args.ts @@ -10,6 +10,11 @@ export interface IFDiffArgs { target: string; } +export interface IFEnvArgs { + manifest?: string; + install?: boolean; +} + export interface Options { outputPath?: string; stdout?: boolean; From 33ef812286201cebe9aa99f80adeaed13f443ede Mon Sep 17 00:00:00 2001 From: manushak Date: Fri, 7 Jun 2024 17:53:20 +0400 Subject: [PATCH 008/243] feat(util): add if-env releated functions --- src/util/args.ts | 53 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/src/util/args.ts b/src/util/args.ts index 56466fd48..753145022 100644 --- a/src/util/args.ts +++ b/src/util/args.ts @@ -2,21 +2,28 @@ import * as path from 'path'; import {parse} from 'ts-command-line-args'; import {checkIfFileIsYaml} from './yaml'; +import {isFileExists} from './helpers'; import {ERRORS} from './errors'; import {logger} from './logger'; import {CONFIG, STRINGS} from '../config'; -import {IFDiffArgs, IEArgs, ProcessArgsOutputs} from '../types/process-args'; +import { + IFDiffArgs, + IEArgs, + ProcessArgsOutputs, + IFEnvArgs, +} from '../types/process-args'; import {LoadDiffParams} from '../types/util/args'; const {CliInputError} = ERRORS; -const {IE, IF_DIFF} = CONFIG; +const {IE, IF_DIFF, IF_ENV} = CONFIG; const { FILE_IS_NOT_YAML, MANIFEST_IS_MISSING, + MANIFEST_NOT_FOUND, NO_OUTPUT, SOURCE_IS_NOT_YAML, TARGET_IS_NOT_YAML, @@ -134,3 +141,45 @@ export const parseIfDiffArgs = () => { throw new CliInputError(INVALID_TARGET); }; + +/** -- IF Env -- */ + +/** + * Parses `if-env` process arguments. + */ +const validateAndParseIfEnvArgs = () => { + try { + return parse(IF_ENV.ARGS, IF_ENV.HELP); + } catch (error) { + if (error instanceof Error) { + throw new CliInputError(error.message); + } + + throw error; + } +}; + +/** + * Checks if the `manifest` command is provided and it is valid manifest file. + */ +export const parseIfEnvArgs = async () => { + const {manifest, install} = validateAndParseIfEnvArgs(); + + if (manifest) { + const isManifestFileExists = await isFileExists(manifest); + + if (!isManifestFileExists) { + throw new CliInputError(MANIFEST_NOT_FOUND); + } + + if (checkIfFileIsYaml(manifest)) { + const response = prependFullFilePath(manifest); + + return {manifest: response, install}; + } + + throw new CliInputError(FILE_IS_NOT_YAML); + } + + return; +}; From 69867927018ca8518ee95fa5d6acfdc48d061a0c Mon Sep 17 00:00:00 2001 From: manushak Date: Fri, 7 Jun 2024 17:55:49 +0400 Subject: [PATCH 009/243] test(util): add args test --- src/__tests__/unit/util/args.test.ts | 106 ++++++++++++++++++++++++++- 1 file changed, 105 insertions(+), 1 deletion(-) diff --git a/src/__tests__/unit/util/args.test.ts b/src/__tests__/unit/util/args.test.ts index d40d36cae..ac91f8a3a 100644 --- a/src/__tests__/unit/util/args.test.ts +++ b/src/__tests__/unit/util/args.test.ts @@ -1,5 +1,14 @@ const processRunningPath = process.cwd(); +jest.mock('../../../util/helpers', () => ({ + isFileExists: () => { + if (process.env.fileExists === 'true') { + return true; + } + return false; + }, +})); + jest.mock('ts-command-line-args', () => ({ __esModule: true, parse: () => { @@ -60,6 +69,22 @@ jest.mock('ts-command-line-args', () => ({ throw new Error('mock-error'); case 'diff-throw': throw 'mock-error'; + /** If-env mocks */ + // case 'env-manifest-is-missing': + // return; + case 'manifest-install-provided': + return { + install: true, + manifest: 'mock-manifest.yaml', + }; + case 'manifest-is-not-yaml': + return {manifest: 'manifest'}; + case 'manifest-path-invalid': + throw new Error(MANIFEST_NOT_FOUND); + case 'env-throw-error': + throw new Error('mock-error'); + case 'env-throw': + throw 'mock-error'; default: return { manifest: 'mock-manifest.yaml', @@ -71,7 +96,11 @@ jest.mock('ts-command-line-args', () => ({ import path = require('path'); -import {parseIEProcessArgs, parseIfDiffArgs} from '../../../util/args'; +import { + parseIEProcessArgs, + parseIfDiffArgs, + parseIfEnvArgs, +} from '../../../util/args'; import {ERRORS} from '../../../util/errors'; import {STRINGS} from '../../../config'; @@ -84,6 +113,7 @@ const { TARGET_IS_NOT_YAML, INVALID_TARGET, SOURCE_IS_NOT_YAML, + MANIFEST_NOT_FOUND, } = STRINGS; describe('util/args: ', () => { @@ -299,5 +329,79 @@ describe('util/args: ', () => { }); }); + describe('parseIfEnvArgs(): ', () => { + it('executes if `manifest` is missing.', async () => { + process.env.fileExists = 'true'; + process.env.result = 'manifest-is-missing'; + const response = await parseIfEnvArgs(); + + expect.assertions(1); + + expect(response).toEqual(undefined); + }); + + it('executes if `manifest` and `install` are provided.', async () => { + process.env.fileExists = 'true'; + process.env.result = 'manifest-install-provided'; + + const response = await parseIfEnvArgs(); + + expect.assertions(2); + expect(response).toHaveProperty('install'); + expect(response).toHaveProperty('manifest'); + }); + + it('throws an error if `manifest` is not a yaml.', async () => { + process.env.fileExists = 'true'; + process.env.result = 'manifest-is-not-yaml'; + expect.assertions(1); + + try { + await parseIfEnvArgs(); + } catch (error) { + if (error instanceof Error) { + expect(error).toEqual(new CliInputError(FILE_IS_NOT_YAML)); + } + } + }); + + it('throws an error if `manifest` path is invalid.', async () => { + process.env.fileExists = 'false'; + expect.assertions(1); + + try { + await parseIfEnvArgs(); + } catch (error) { + if (error instanceof Error) { + expect(error).toEqual(new CliInputError(MANIFEST_NOT_FOUND)); + } + } + }); + + it('throws an error if parsing failed.', async () => { + process.env.result = 'env-throw-error'; + expect.assertions(1); + + try { + await parseIfEnvArgs(); + } catch (error) { + if (error instanceof Error) { + expect(error).toEqual(new CliInputError('mock-error')); + } + } + }); + + it('throws error if parsing failed (not instance of error).', async () => { + process.env.result = 'env-throw'; + expect.assertions(1); + + try { + await parseIfEnvArgs(); + } catch (error) { + expect(error).toEqual('mock-error'); + } + }); + }); + process.env = originalEnv; }); From 13a1ab22bedd3034695929c522ad178d1cf5a93a Mon Sep 17 00:00:00 2001 From: manushak Date: Fri, 7 Jun 2024 17:56:54 +0400 Subject: [PATCH 010/243] feat(config): add `IF_ENV` config --- src/config/config.ts | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/src/config/config.ts b/src/config/config.ts index 4b23f9aa5..11f11eb86 100644 --- a/src/config/config.ts +++ b/src/config/config.ts @@ -2,7 +2,7 @@ import {ArgumentConfig, ParseOptions} from 'ts-command-line-args'; import {STRINGS} from './strings'; -import {IFDiffArgs, IEArgs} from '../types/process-args'; +import {IFDiffArgs, IEArgs, IFEnvArgs} from '../types/process-args'; const {DISCLAIMER_MESSAGE} = STRINGS; @@ -77,6 +77,36 @@ export const CONFIG = { SUCCESS_MESSAGE: 'Files match!', FAILURE_MESSAGE: 'Files do not match!', }, + IF_ENV: { + ARGS: { + manifest: { + type: String, + optional: true, + alias: 'm', + description: '[path to the manifest file]', + }, + install: { + type: Boolean, + optional: true, + alias: 'i', + description: '[commant to install package.json]', + }, + } as ArgumentConfig, + HELP: { + helpArg: 'help', + headerContentSections: [ + {header: 'Impact Framework', content: 'IF-Env Helpful keywords:'}, + ], + footerContentSections: [ + {header: 'Green Software Foundation', content: DISCLAIMER_MESSAGE}, + ], + } as ParseOptions, + SUCCESS_MESSAGE: 'The environment is successfully setup!', + FAILURE_MESSAGE: 'Faied to create the environment!', + FAILURE_MESSAGE_TEMPLATE: + 'Faied to create the environment with the template manifest!', + FAILURE_MESSAGE_DEPENDENCIES: 'Manifest dependencies are not available!', + }, GITHUB_PATH: 'https://github.com', NATIVE_PLUGIN: 'if-plugins', AGGREGATION_ADDITIONAL_PARAMS: ['timestamp', 'duration'], From 9d0d73c6c881ca3065dc37f09df073c092dfde90 Mon Sep 17 00:00:00 2001 From: manushak Date: Fri, 7 Jun 2024 17:58:07 +0400 Subject: [PATCH 011/243] feat(src): add if-env template manifest file --- src/env/template.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 src/env/template.yml diff --git a/src/env/template.yml b/src/env/template.yml new file mode 100644 index 000000000..f5fd65a6a --- /dev/null +++ b/src/env/template.yml @@ -0,0 +1,24 @@ +name: template manifest # rename me! +description: auto-generated template # update description! +tags: # add any tags that will help you to track your manifests +initialize: + outputs: + - yaml # you can add - csv to export to csv + plugins: # add more plugins for your use-case + memory-energy-from-memory-util: # you can name this any way you like! + method: Coefficient # the name of the function exported from the plugin + path: "builtin" # the import path + global-config: # anmy config required by the plugin + input-parameter: "memory/utilization" + coefficient: 0.0001 #kwH/GB + output-parameter: "memory/energy" +tree: + children: # add a chile for each distinct component you want to measure + child: + pipeline: # the pipeline is an ordered list of plugins you want to execute + - memory-energy-from-memory-util # must match the name in initialize! + config: # any plugin specific, node-level config + inputs: + - timestamp: 2023-12-12T00:00:00.000Z # ISO 8061 string + duration: 3600 # units of seconds + memory/utilization: 10 From f9d4de702955429986dd274441492d54d2ba3abd Mon Sep 17 00:00:00 2001 From: manushak Date: Fri, 7 Jun 2024 17:59:24 +0400 Subject: [PATCH 012/243] feat(src): add if-env functionality --- src/env/env.ts | 211 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 211 insertions(+) create mode 100644 src/env/env.ts diff --git a/src/env/env.ts b/src/env/env.ts new file mode 100644 index 000000000..0964d095d --- /dev/null +++ b/src/env/env.ts @@ -0,0 +1,211 @@ +#!/usr/bin/env node +/* eslint-disable no-process-exit */ +import * as fs from 'fs/promises'; +import * as path from 'path'; + +import {execPromise, isFileExists} from '../util/helpers'; +import {parseIfEnvArgs} from '../util/args'; +import {logger} from '../util/logger'; + +import {load} from '../lib/load'; + +import {CONFIG} from '../config'; + +const packageJson = require('../../package.json'); + +const {IF_ENV} = CONFIG; +const { + SUCCESS_MESSAGE, + FAILURE_MESSAGE, + FAILURE_MESSAGE_TEMPLATE, + FAILURE_MESSAGE_DEPENDENCIES, +} = IF_ENV; + +const FOLDER_NAME = 'if-environment'; + +type EnvironmentOptions = { + folderPath: string; + install: boolean; + dependencies: {[path: string]: string}; +}; + +const IfEnv = async () => { + const commandArgs = await parseIfEnvArgs(); + const options: EnvironmentOptions = { + folderPath: path.resolve(__dirname, FOLDER_NAME), + install: true, + dependencies: {'@grnsft/if': packageJson.version}, + }; + + if (commandArgs) { + const {folderPath, install, dependencies} = + await getOptionsFromArgs(commandArgs); + options.folderPath = folderPath; + options.install = !!install; + options.dependencies = {...options.dependencies, ...dependencies}; + } + + await initializeAndInstallLibs(options); + + if (!commandArgs) { + await addTemplateManifest(); + } + + console.log(SUCCESS_MESSAGE); + process.exit(0); +}; + +/** + * Gets the folder path of manifest file, dependencies from manifest file and install argument from the given arguments. + */ +const getOptionsFromArgs = async (commandArgs: { + manifest: string; + install: boolean | undefined; +}) => { + const {manifest: manifestPath, install} = commandArgs; + + const folderPath = path.dirname(manifestPath); + + const manifest = await load(manifestPath); + const plugins = manifest.rawManifest?.initialize?.plugins || {}; + const dependencies = + manifest.rawManifest?.execution?.environment.dependencies || []; + + if (!dependencies.length) { + throw new Error(FAILURE_MESSAGE_DEPENDENCIES); + } + + const pathsWithVersion = extractPathsWithVersion(plugins, dependencies); + + return { + folderPath, + dependencies: pathsWithVersion, + install, + }; +}; + +/** + * Gets depencecies with versions. + */ +const extractPathsWithVersion = ( + plugins: any, + dependencies: string[] +): {[path: string]: string} => { + const paths = Object.keys(plugins).map(plugin => plugins[plugin].path); + const uniquePaths = [...new Set(paths)].filter(path => path !== 'builtin'); + const pathsWithVersion: {[path: string]: string} = {}; + + uniquePaths.forEach(pluginPath => { + const dependency = dependencies.find((dependency: string) => + dependency.startsWith(pluginPath) + ); + + if (dependency) { + const splittedDependency = dependency.split('@'); + const version = + splittedDependency.length > 2 + ? splittedDependency[2].split(' ')[0] + : splittedDependency[1]; + + pathsWithVersion[pluginPath] = `^${version}`; + } + }); + + return pathsWithVersion; +}; + +/** + * Creates folder if not exists, installs dependencies if required, update depenedencies. + */ +async function initializeAndInstallLibs(options: EnvironmentOptions) { + try { + const {folderPath, install, dependencies} = options; + + await fs.mkdir(folderPath, {recursive: true}); + + const packageJsonPath = await ensurePackageJsonExists(folderPath); + + if (install) { + await installDependencies(folderPath, dependencies); + } else { + await updatePackageJsonDependencies(packageJsonPath, dependencies); + } + } catch (error) { + console.log(FAILURE_MESSAGE); + } +} + +/** + * Checks if the package.json is exists, if not, inisializes it. + */ +async function ensurePackageJsonExists(folderPath: string) { + const packageJsonPath = path.resolve(folderPath, 'package.json'); + const isPackageJsonExists = await isFileExists(packageJsonPath); + + if (!isPackageJsonExists) { + await execPromise('npm init -y', {cwd: folderPath}); + } + + return packageJsonPath; +} + +/** + * Installs packages from the specified dependencies in the specified folder. + */ +async function installDependencies( + folderPath: string, + dependencies: {[path: string]: string} +) { + const packages = Object.entries(dependencies).map( + ([dependency, version]) => `${dependency}@${version.replace('^', '')}` + ); + + await execPromise(`npm install ${packages.join(' ')}`, { + cwd: folderPath, + }); +} + +/** + * Updates package.json dependencies. + */ +async function updatePackageJsonDependencies( + packageJsonPath: string, + dependencies: {[path: string]: string} +) { + const packageJsonContent = await fs.readFile(packageJsonPath, 'utf8'); + const packageJson = JSON.parse(packageJsonContent); + + packageJson.dependencies = { + ...packageJson.dependencies, + ...dependencies, + }; + + await fs.writeFile(packageJsonPath, JSON.stringify(packageJson, null, 2)); +} + +/** + * Adds a manifest template to the folder where the if-env CLI command runs. + */ +async function addTemplateManifest() { + try { + const templateManifest = path.resolve(__dirname, 'template.yml'); + const destinationPath = path.resolve( + __dirname, + FOLDER_NAME, + 'manifest.yml' + ); + + const data = await fs.readFile(templateManifest, 'utf-8'); + await fs.writeFile(destinationPath, '', 'utf-8'); + await fs.writeFile(destinationPath, data, 'utf-8'); + } catch (error) { + console.log(FAILURE_MESSAGE_TEMPLATE); + } +} + +IfEnv().catch(error => { + if (error instanceof Error) { + logger.error(error); + process.exit(2); + } +}); From 41397045bef3e312414bcb96c318ee7cf6f1494a Mon Sep 17 00:00:00 2001 From: manushak Date: Fri, 7 Jun 2024 18:02:17 +0400 Subject: [PATCH 013/243] feat(package): add `if-env` into bin and scripts --- package-lock.json | 3 ++- package.json | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 0c4cd07c8..8f0f4b40f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,7 +22,8 @@ }, "bin": { "ie": "build/index.js", - "if-diff": "build/diff.js" + "if-diff": "build/diff.js", + "if-env": "build/env/env.js" }, "devDependencies": { "@babel/core": "^7.22.10", diff --git a/package.json b/package.json index ded9a8331..434e1894a 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,8 @@ }, "bin": { "ie": "./build/index.js", - "if-diff": "./build/diff.js" + "if-diff": "./build/diff.js", + "if-env": "./build/env/env.js" }, "bugs": { "url": "https://github.com/Green-Software-Foundation/if/issues/new?assignees=&labels=feedback&projects=&template=feedback.md&title=Feedback+-+" @@ -71,6 +72,7 @@ "fix:package": "fixpack", "ie": "npx ts-node src/index.ts", "if-diff": "npx ts-node src/diff.ts", + "if-env": "npx ts-node src/env/env.ts", "lint": "gts lint", "pre-commit": "lint-staged", "prepare": "husky install", @@ -81,4 +83,4 @@ }, "stability": "stable", "types": "src/index.d.ts" -} +} \ No newline at end of file From 9da09b6ba3db0af434095fde0e22266d549fc2bc Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 10 Jun 2024 01:15:59 +0400 Subject: [PATCH 014/243] feat(util): update error usage in plugin storage --- src/util/plugin-storage.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/util/plugin-storage.ts b/src/util/plugin-storage.ts index 7667d5e07..15582e9f3 100644 --- a/src/util/plugin-storage.ts +++ b/src/util/plugin-storage.ts @@ -4,7 +4,7 @@ import {STRINGS} from '../config'; import {PluginInterface} from '../types/interface'; import {PluginStorage} from '../types/plugin-storage'; -const {PluginInitalizationError} = ERRORS; +const {PluginInitializationError} = ERRORS; const {NOT_INITALIZED_PLUGIN} = STRINGS; /** @@ -21,7 +21,7 @@ export const pluginStorage = () => { const plugin = storage[name]; if (!plugin) { - throw new PluginInitalizationError(NOT_INITALIZED_PLUGIN(name)); + throw new PluginInitializationError(NOT_INITALIZED_PLUGIN(name)); } return plugin; From e769eaaed7c4d22b04399fe483fdee0caa9383b5 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 10 Jun 2024 01:16:40 +0400 Subject: [PATCH 015/243] feat(util): drop build error message helper --- src/util/helpers.ts | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/src/util/helpers.ts b/src/util/helpers.ts index b15863cef..c2b644d32 100644 --- a/src/util/helpers.ts +++ b/src/util/helpers.ts @@ -1,24 +1,16 @@ import {createInterface} from 'node:readline/promises'; import {exec} from 'node:child_process'; import {promisify} from 'node:util'; -import {ErrorFormatParams} from '../types/helpers'; + import {ERRORS} from './errors'; import {logger} from './logger'; + import {STRINGS} from '../config'; import {Difference} from '../types/lib/compare'; -const {ISSUE_TEMPLATE} = STRINGS; - -/** - * Formats given error according to class instance, scope and message. - */ -export const buildErrorMessage = - (classInstanceName: string) => (params: ErrorFormatParams) => { - const {scope, message} = params; - - return `${classInstanceName}${scope ? `(${scope})` : ''}: ${message}.`; - }; +const {ISSUE_TEMPLATE, MISSING_MANIFEST_IN_STDIN} = STRINGS; +const {CliInputError} = ERRORS; /** * Impact engine error handler. Logs errors and appends issue template if error is unknown. @@ -177,7 +169,7 @@ export const parseManifestFromStdin = async () => { const match = regex.exec(pipedSourceManifest); if (!match) { - throw new Error('Manifest not found in STDIN.'); + throw new CliInputError(MISSING_MANIFEST_IN_STDIN); } return match![1]; From 4251823d72d00dfd295841c5ded980009c65d1c4 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 10 Jun 2024 01:17:20 +0400 Subject: [PATCH 016/243] feat(util): make error classes granular --- src/util/errors.ts | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/src/util/errors.ts b/src/util/errors.ts index a65a603be..f4f067d77 100644 --- a/src/util/errors.ts +++ b/src/util/errors.ts @@ -1,19 +1,33 @@ const CUSTOM_ERRORS = [ 'CliInputError', - 'ConfigNotFoundError', - 'ConfigValidationError', - 'ExhaustError', - 'FileNotFoundError', - 'MakeDirectoryError', 'ManifestValidationError', - 'ModuleInitializationError', 'InputValidationError', - 'InvalidAggregationParamsError', 'InvalidGroupingError', - 'PluginCredentialError', - 'PluginInitalizationError', 'WriteFileError', - 'ConfigNotFoundError', + /** More specific errors */ + 'ParseCliParamsError', + 'CliSourceFileError', + 'CliTargetFileError', + 'InvalidAggregationMethodError', + 'MissingAggregationParamError', + 'MissingPluginMethodError', + 'MissingPluginPathError', + 'PluginInitializationError', + 'InvalidExhaustPluginError', + /** Plugins */ + 'GlobalConfigError', + 'MissingInputDataError', + 'ProcessExecutionError', + 'RegexMismatchError', + 'FetchingFileError', + 'ReadFileError', + 'MissingCSVColumnError', + 'QueryDataNotFoundError', + 'InvalidDateInInputError', + 'InvalidPaddingError', + 'InvalidInputError', + 'ExhaustOutputArgError', + 'CSVParseError', ] as const; type CustomErrors = { From 78972b1cd8d84fe66699462b45799f7a626ec950 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 10 Jun 2024 01:17:54 +0400 Subject: [PATCH 017/243] feat(util): update errors usage in args --- src/util/args.ts | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/util/args.ts b/src/util/args.ts index 56466fd48..06ee99e93 100644 --- a/src/util/args.ts +++ b/src/util/args.ts @@ -10,7 +10,12 @@ import {CONFIG, STRINGS} from '../config'; import {IFDiffArgs, IEArgs, ProcessArgsOutputs} from '../types/process-args'; import {LoadDiffParams} from '../types/util/args'; -const {CliInputError} = ERRORS; +const { + CliInputError, + ParseCliParamsError, + CliTargetFileError, + CliSourceFileError, +} = ERRORS; const {IE, IF_DIFF} = CONFIG; @@ -83,10 +88,10 @@ export const parseIEProcessArgs = (): ProcessArgsOutputs => { }; } - throw new CliInputError(FILE_IS_NOT_YAML); + throw new CliSourceFileError(FILE_IS_NOT_YAML); } - throw new CliInputError(MANIFEST_IS_MISSING); + throw new CliSourceFileError(MANIFEST_IS_MISSING); }; /** -- IF Diff -- */ @@ -99,7 +104,7 @@ const validateAndParseIfDiffArgs = () => { return parse(IF_DIFF.ARGS, IF_DIFF.HELP); } catch (error) { if (error instanceof Error) { - throw new CliInputError(error.message); + throw new ParseCliParamsError(error.message); } throw error; @@ -114,7 +119,7 @@ export const parseIfDiffArgs = () => { if (target) { if (source && !checkIfFileIsYaml(source)) { - throw new CliInputError(SOURCE_IS_NOT_YAML); + throw new CliSourceFileError(SOURCE_IS_NOT_YAML); } if (checkIfFileIsYaml(target)) { @@ -129,7 +134,7 @@ export const parseIfDiffArgs = () => { return response; } - throw new CliInputError(TARGET_IS_NOT_YAML); + throw new CliTargetFileError(TARGET_IS_NOT_YAML); } throw new CliInputError(INVALID_TARGET); From 62f70d5123eaa5a4fd12fed9cbe01fcdc0916ecc Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 10 Jun 2024 01:18:22 +0400 Subject: [PATCH 018/243] feat(util): update errors usage in aggregation helper --- src/util/aggregation-helper.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/util/aggregation-helper.ts b/src/util/aggregation-helper.ts index 3be503ffc..0a8b3c136 100644 --- a/src/util/aggregation-helper.ts +++ b/src/util/aggregation-helper.ts @@ -6,7 +6,7 @@ import {CONFIG, STRINGS} from '../config'; import {AggregationResult} from '../types/aggregation'; import {PluginParams} from '../types/interface'; -const {InvalidAggregationParamsError} = ERRORS; +const {InvalidAggregationMethodError, MissingAggregationParamError} = ERRORS; const {INVALID_AGGREGATION_METHOD, METRIC_MISSING} = STRINGS; const {AGGREGATION_ADDITIONAL_PARAMS} = CONFIG; @@ -19,7 +19,7 @@ const checkIfMetricsAreValid = (metrics: string[]) => { const method = parameterize.getAggregationMethod(metric); if (method === 'none') { - throw new InvalidAggregationParamsError( + throw new InvalidAggregationMethodError( INVALID_AGGREGATION_METHOD(metric) ); } @@ -41,7 +41,7 @@ export const aggregateInputsIntoOne = ( return inputs.reduce((acc, input, index) => { for (const metric of extendedMetrics) { if (!(metric in input)) { - throw new InvalidAggregationParamsError(METRIC_MISSING(metric, index)); + throw new MissingAggregationParamError(METRIC_MISSING(metric, index)); } /** Checks if metric is timestamp or duration, then adds to aggregated value. */ From 3269fc9b62347945f6f5048d06bfa75e67337ba7 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 10 Jun 2024 01:22:12 +0400 Subject: [PATCH 019/243] revert(types): drop helpers --- src/types/helpers.ts | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 src/types/helpers.ts diff --git a/src/types/helpers.ts b/src/types/helpers.ts deleted file mode 100644 index ed413174b..000000000 --- a/src/types/helpers.ts +++ /dev/null @@ -1,4 +0,0 @@ -export type ErrorFormatParams = { - scope?: string; - message: string; -}; From 1c826ea2a3e563c332fd745557e2415c1505a052 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 10 Jun 2024 01:25:44 +0400 Subject: [PATCH 020/243] feat(lib): update errors usage in load --- src/lib/load.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/load.ts b/src/lib/load.ts index 447153068..e1a0cb465 100644 --- a/src/lib/load.ts +++ b/src/lib/load.ts @@ -12,7 +12,7 @@ import {Parameters} from '../types/parameters'; import {LoadDiffParams} from '../types/util/args'; import {Manifest} from '../types/manifest'; -const {CliInputError} = ERRORS; +const {CliSourceFileError} = ERRORS; const {INVALID_SOURCE} = STRINGS; @@ -43,7 +43,7 @@ export const loadIfDiffFiles = async (params: LoadDiffParams) => { const pipedSourceManifest = await parseManifestFromStdin(); if (!sourcePath && !pipedSourceManifest) { - throw new CliInputError(INVALID_SOURCE); + throw new CliSourceFileError(INVALID_SOURCE); } const loadFromSource = From 85a66dd93e9a79b75e1774c32443b00bbf1a698e Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 10 Jun 2024 01:26:03 +0400 Subject: [PATCH 021/243] feat(lib): update errors usage in initalize --- src/lib/initialize.ts | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/lib/initialize.ts b/src/lib/initialize.ts index d6bf7c184..e20d5f53b 100644 --- a/src/lib/initialize.ts +++ b/src/lib/initialize.ts @@ -11,7 +11,11 @@ import {PluginInterface} from '../types/interface'; import {GlobalPlugins, PluginOptions} from '../types/manifest'; import {PluginStorageInterface} from '../types/plugin-storage'; -const {ModuleInitializationError, PluginCredentialError} = ERRORS; +const { + PluginInitializationError, + MissingPluginMethodError, + MissingPluginPathError, +} = ERRORS; const {GITHUB_PATH, NATIVE_PLUGIN} = CONFIG; const {MISSING_METHOD, MISSING_PATH, NOT_NATIVE_PLUGIN, INVALID_MODULE_PATH} = @@ -21,14 +25,11 @@ const {MISSING_METHOD, MISSING_PATH, NOT_NATIVE_PLUGIN, INVALID_MODULE_PATH} = * Imports module by given `path`. */ const importModuleFrom = async (path: string) => { - try { - const module = await import(path); + const module = await import(path).catch(error => { + throw new PluginInitializationError(INVALID_MODULE_PATH(path, error)); + }); - return module; - } catch (error) { - logger.error(error); - throw new ModuleInitializationError(INVALID_MODULE_PATH(path)); - } + return module; }; /** @@ -71,11 +72,11 @@ const initPlugin = async ( const {method, path, 'global-config': globalConfig} = initPluginParams; if (!method) { - throw new PluginCredentialError(MISSING_METHOD); + throw new MissingPluginMethodError(MISSING_METHOD); } if (!path) { - throw new PluginCredentialError(MISSING_PATH); + throw new MissingPluginPathError(MISSING_PATH); } const plugin = await handModule(method, path); From c39cbab19fd81a24a09e8974fd736683350a155b Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 10 Jun 2024 01:26:21 +0400 Subject: [PATCH 022/243] feat(lib): update errors usage in exhaust --- src/lib/exhaust.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/exhaust.ts b/src/lib/exhaust.ts index 5c80b0a5a..98eaf7625 100644 --- a/src/lib/exhaust.ts +++ b/src/lib/exhaust.ts @@ -14,7 +14,7 @@ import {ExhaustPluginInterface} from '../types/exhaust-plugin-interface'; import {Context} from '../types/manifest'; import {Options} from '../types/process-args'; -const {ExhaustError} = ERRORS; +const {InvalidExhaustPluginError} = ERRORS; const {INVALID_EXHAUST_PLUGIN} = STRINGS; /** @@ -35,7 +35,7 @@ const initializeExhaustPlugin = (name: string): ExhaustPluginInterface => { case 'csv-raw': return ExportCSVRaw(); default: - throw new ExhaustError(INVALID_EXHAUST_PLUGIN(name)); + throw new InvalidExhaustPluginError(INVALID_EXHAUST_PLUGIN(name)); } }; From 731670336963afd75b2e9167f9ebb46cb1c0ab71 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 10 Jun 2024 12:26:58 +0400 Subject: [PATCH 023/243] feat(config): update strings to include all messages --- src/config/strings.ts | 59 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 55 insertions(+), 4 deletions(-) diff --git a/src/config/strings.ts b/src/config/strings.ts index 04551b9b9..a96e11184 100644 --- a/src/config/strings.ts +++ b/src/config/strings.ts @@ -17,12 +17,12 @@ Incubation projects are experimental, offer no support guarantee, have minimal g NOT_NATIVE_PLUGIN: (path: string) => ` You are using plugin ${path} which is not part of the Impact Framework standard library. You should do your own research to ensure the plugins are up to date and accurate. They may not be actively maintained.`, - SOMETHING_WRONG: 'Something wrong with cli arguments. Please check docs.', ISSUE_TEMPLATE: ` Impact Framework is an alpha release from the Green Software Foundation and is released to capture early feedback. If you'd like to offer some feedback, please use this issue template: https://github.com/Green-Software-Foundation/if/issues/new?assignees=&labels=feedback&projects=&template=feedback.md&title=Feedback+-+`, - INVALID_MODULE_PATH: (path: string) => - `Provided module: '${path}' is invalid or not found.`, + INVALID_MODULE_PATH: (path: string, error?: any) => + `Provided module \`${path}\` is invalid or not found. ${error ?? ''} +`, INVALID_TIME_NORMALIZATION: 'Start time or end time is missing.', UNEXPECTED_TIME_CONFIG: 'Unexpected node-level config provided for time-sync plugin.', @@ -33,7 +33,6 @@ https://github.com/Green-Software-Foundation/if/issues/new?assignees=&labels=fee `Avoiding padding at ${ start && end ? 'start and end' : start ? 'start' : 'end' }`, - INVALID_OBSERVATION_OVERLAP: 'Observation timestamps overlap.', INVALID_AGGREGATION_METHOD: (metric: string) => `Aggregation is not possible for given ${metric} since method is 'none'.`, METRIC_MISSING: (metric: string, index: number) => @@ -56,4 +55,56 @@ Note that for the '--output' option you also need to define the output type in y TARGET_IS_NOT_YAML: 'Given target is not in yaml format.', INVALID_TARGET: 'Target is invalid.', INVALID_SOURCE: 'Source is invalid.', + MISSING_MANIFEST_IN_STDIN: 'Manifest not found in STDIN.', + /** Plugin messages */ + MISSING_GLOBAL_CONFIG: 'Global config is not provided.', + MISSING_INPUT_DATA: (param: string) => + `${param} is missing from the input array.`, + NOT_NUMERIC_VALUE: (str: any) => `${str} is not numberic.`, + MISSING_FUNCTIONAL_UNIT_CONFIG: + '`functional-unit` should be provided in your global config', + MISSING_FUNCTIONAL_UNIT_INPUT: + '`functional-unit` value is missing from input data or it is not a positive integer', + REGEX_MISMATCH: (input: any, match: string) => + `\`${input}\` does not match the ${match} regex expression`, + SCI_EMBODIED_ERROR: (unit: string) => + `invalid number. please provide it as \`${unit}\` to input`, + MISSING_MIN_MAX: 'Config is missing min or max value', + INVALID_MIN_MAX: (name: string) => + `Min value should not be greater than or equal to max value of ${name}`, + FILE_FETCH_FAILED: ( + filepath: string, + message: string + ) => `Failed fetching the file: ${filepath}. +${message}`, + FILE_READ_FAILED: ( + filepath: string, + error: string + ) => `Failed reading the file: ${filepath}. +${error}`, + MISSING_CSV_COLUMN: (columnName: string) => + `There is no column with the name: ${columnName}.`, + NO_QUERY_DATA: + 'One or more of the given query parameters are not found in the target CSV file column headers.', + INVALID_DATE_TYPE: (date: any) => + `Unexpected date datatype: ${typeof date}: ${date}`, + INVALID_OBSERVATION_OVERLAP: + 'Observation timestamps overlap, please check inputs.', + /** Exhaust messages */ + OUTPUT_REQUIRED: + 'Output path is required, please make sure output is configured properly.', + CSV_EXPORT: + 'CSV export criteria is not found in output path. Please append it after --output #.', + WRITE_CSV_ERROR: (outputPath: string, error: any) => + `Failed to write CSV file to ${outputPath}: ${error}`, + INVALID_NAME: + '`name` global config parameter is empty or contains all spaces', + START_LOWER_END: '`start-time` should be lower than `end-time`', + TIMESTAMP_REQUIRED: (index: number) => `required in input[${index}]`, + INVALID_DATETIME: (index: number) => `invalid datetime in input[${index}]`, + X_Y_EQUAL: 'The length of `x` and `y` should be equal', + ARRAY_LENGTH_NON_EMPTY: + 'the length of the input arrays must be greater than 1', + WITHIN_THE_RANGE: + 'The target x value must be within the range of the given x values', }; From d9c047ef49b59e44f360eef6b43d664369421b27 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 10 Jun 2024 16:54:20 +0400 Subject: [PATCH 024/243] feat(builtins): update errors usage in sum --- src/builtins/sum/index.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/builtins/sum/index.ts b/src/builtins/sum/index.ts index 8eec61c37..9944373a8 100644 --- a/src/builtins/sum/index.ts +++ b/src/builtins/sum/index.ts @@ -1,13 +1,15 @@ import {z} from 'zod'; -import {ExecutePlugin, PluginParams} from '../../types/interface'; - import {validate} from '../../util/validations'; import {ERRORS} from '../../util/errors'; +import {STRINGS} from '../../config'; + +import {ExecutePlugin, PluginParams} from '../../types/interface'; import {SumConfig} from './types'; -const {InputValidationError, ConfigNotFoundError} = ERRORS; +const {GlobalConfigError, MissingInputDataError} = ERRORS; +const {MISSING_INPUT_DATA, MISSING_GLOBAL_CONFIG} = STRINGS; export const Sum = (globalConfig: SumConfig): ExecutePlugin => { const metadata = { @@ -37,7 +39,7 @@ export const Sum = (globalConfig: SumConfig): ExecutePlugin => { */ const validateGlobalConfig = () => { if (!globalConfig) { - throw new ConfigNotFoundError('Global config is not provided.'); + throw new GlobalConfigError(MISSING_GLOBAL_CONFIG); } const globalConfigSchema = z.object({ @@ -60,9 +62,7 @@ export const Sum = (globalConfig: SumConfig): ExecutePlugin => { ) => { inputParameters.forEach(metricToSum => { if (!input[metricToSum]) { - throw new InputValidationError( - `${metricToSum} is missing from the input array.` - ); + throw new MissingInputDataError(MISSING_INPUT_DATA(metricToSum)); } }); From d0452663110a6b71f3f6ca1a4a1e162ccf99d79d Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 10 Jun 2024 16:54:49 +0400 Subject: [PATCH 025/243] feat(builtins): update errors usage in mock observations, do cosmetic fixes --- .../helpers/common-generator.ts | 27 ++++----- .../helpers/rand-int-generator.ts | 56 ++++++++----------- src/builtins/mock-observations/index.ts | 6 +- 3 files changed, 35 insertions(+), 54 deletions(-) diff --git a/src/builtins/mock-observations/helpers/common-generator.ts b/src/builtins/mock-observations/helpers/common-generator.ts index 24bea4c94..ee4909dcd 100644 --- a/src/builtins/mock-observations/helpers/common-generator.ts +++ b/src/builtins/mock-observations/helpers/common-generator.ts @@ -1,37 +1,30 @@ -import {KeyValuePair} from '../../../types/common'; import {ERRORS} from '../../../util/errors'; -import {buildErrorMessage} from '../../../util/helpers'; -import {Generator} from '../interfaces'; - -const {InputValidationError} = ERRORS; +import {STRINGS} from '../../../config'; -export const CommonGenerator = (config: KeyValuePair): Generator => { - const errorBuilder = buildErrorMessage(CommonGenerator.name); +import {Generator} from '../interfaces'; - /** - * Creates new copy of the given `object`. - */ - const copyObject = (object: T): T => ({...object}); +const {GlobalConfigError} = ERRORS; +const {MISSING_GLOBAL_CONFIG} = STRINGS; +export const CommonGenerator = (config: Record): Generator => { /** + * Generates next value by copying the validated config. * Validates the provided config is not null or empty. - * returns a copy of the validated config, otherwise throws an InputValidationError. + * Returns a copy of the validated config, otherwise throws an GlobalConfigError. */ const validateConfig = (config: object) => { if (!config || Object.keys(config).length === 0) { - throw new InputValidationError( - errorBuilder({message: 'Config must not be null or empty'}) - ); + throw new GlobalConfigError(MISSING_GLOBAL_CONFIG); } - return copyObject(config); + return structuredClone(config); }; /** * Generates next value by copying the validated config. */ - const next = (): Object => copyObject(validateConfig(config)); + const next = (): Object => validateConfig(config); return { next, diff --git a/src/builtins/mock-observations/helpers/rand-int-generator.ts b/src/builtins/mock-observations/helpers/rand-int-generator.ts index e27d694c3..a48e399c8 100644 --- a/src/builtins/mock-observations/helpers/rand-int-generator.ts +++ b/src/builtins/mock-observations/helpers/rand-int-generator.ts @@ -1,70 +1,57 @@ -import {KeyValuePair} from '../../../types/common'; import {ERRORS} from '../../../util/errors'; -import {buildErrorMessage} from '../../../util/helpers'; + +import {STRINGS} from '../../../config'; import {Generator} from '../interfaces'; import {RandIntGeneratorParams} from '../types'; -const {InputValidationError} = ERRORS; +const {GlobalConfigError} = ERRORS; + +const {MISSING_GLOBAL_CONFIG, MISSING_MIN_MAX, INVALID_MIN_MAX, INVALID_NAME} = + STRINGS; export const RandIntGenerator = ( name: string, - config: KeyValuePair + config: Record ): Generator => { - const errorBuilder = buildErrorMessage(RandIntGenerator.name); - const next = () => ({ [validatedName]: generateRandInt(getFieldToPopulate()), }); const validateName = (name: string | null): string => { if (!name || name.trim() === '') { - throw new InputValidationError( - errorBuilder({ - message: '`name` is empty or all spaces', - }) - ); + throw new GlobalConfigError(INVALID_NAME); } + return name; }; - const validateConfig = (config: KeyValuePair): {min: number; max: number} => { + const validateConfig = ( + config: Record + ): {min: number; max: number} => { if (!config || Object.keys(config).length === 0) { - throw new InputValidationError( - errorBuilder({ - message: 'Config must not be null or empty', - }) - ); + throw new GlobalConfigError(MISSING_GLOBAL_CONFIG); } if (!config.min || !config.max) { - throw new InputValidationError( - errorBuilder({ - message: 'Config is missing min or max', - }) - ); + throw new GlobalConfigError(MISSING_MIN_MAX); } if (config.min >= config.max) { - throw new InputValidationError( - errorBuilder({ - message: `Min value should not be greater than or equal to max value of ${validatedName}`, - }) - ); + throw new GlobalConfigError(INVALID_MIN_MAX(validatedName)); } + return {min: config.min, max: config.max}; }; const validatedName = validateName(name); const validatedConfig = validateConfig(config); - const getFieldToPopulate = () => { - return { - name: validatedName, - min: validatedConfig.min, - max: validatedConfig.max, - }; - }; + const getFieldToPopulate = () => ({ + name: validatedName, + min: validatedConfig.min, + max: validatedConfig.max, + }); const generateRandInt = ( randIntGenerator: RandIntGeneratorParams @@ -73,6 +60,7 @@ export const RandIntGenerator = ( const scaledNumber = randomNumber * (randIntGenerator.max - randIntGenerator.min) + randIntGenerator.min; + return Math.trunc(scaledNumber); }; diff --git a/src/builtins/mock-observations/index.ts b/src/builtins/mock-observations/index.ts index 2d65f4643..b535da757 100644 --- a/src/builtins/mock-observations/index.ts +++ b/src/builtins/mock-observations/index.ts @@ -1,13 +1,13 @@ import {DateTime, Duration} from 'luxon'; import {z} from 'zod'; -import {ExecutePlugin, PluginParams} from '../../types/interface'; -import {ConfigParams, KeyValuePair} from '../../types/common'; - import {validate} from '../../util/validations'; import {CommonGenerator} from './helpers/common-generator'; import {RandIntGenerator} from './helpers/rand-int-generator'; + +import {ExecutePlugin, PluginParams} from '../../types/interface'; +import {ConfigParams, KeyValuePair} from '../../types/common'; import {Generator} from './interfaces/index'; import {ObservationParams} from './types'; From c4ed3d87726c386c74194ba20a825759438c64e1 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 10 Jun 2024 16:55:20 +0400 Subject: [PATCH 026/243] feat(builtins): update errors usage in subtract --- src/builtins/subtract/index.ts | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/src/builtins/subtract/index.ts b/src/builtins/subtract/index.ts index 0af02898a..25f5d8522 100644 --- a/src/builtins/subtract/index.ts +++ b/src/builtins/subtract/index.ts @@ -1,16 +1,17 @@ import {z} from 'zod'; import {ERRORS} from '../../util/errors'; -import {buildErrorMessage} from '../../util/helpers'; import {validate} from '../../util/validations'; +import {STRINGS} from '../../config'; + import {ExecutePlugin, PluginParams} from '../../types/interface'; import {SubtractConfig} from './types'; const {InputValidationError} = ERRORS; +const {MISSING_INPUT_DATA, NOT_NUMERIC_VALUE} = STRINGS; export const Subtract = (globalConfig: SubtractConfig): ExecutePlugin => { - const errorBuilder = buildErrorMessage(Subtract.name); const metadata = { kind: 'execute', }; @@ -47,21 +48,13 @@ export const Subtract = (globalConfig: SubtractConfig): ExecutePlugin => { const validateParamExists = (input: PluginParams, param: string) => { if (input[param] === undefined) { - throw new InputValidationError( - errorBuilder({ - message: `${param} is missing from the input array`, - }) - ); + throw new InputValidationError(MISSING_INPUT_DATA(param)); } }; const validateNumericString = (str: string) => { if (isNaN(+Number(str))) { - throw new InputValidationError( - errorBuilder({ - message: `${str} is not numberic`, - }) - ); + throw new InputValidationError(NOT_NUMERIC_VALUE(str)); } }; @@ -73,6 +66,7 @@ export const Subtract = (globalConfig: SubtractConfig): ExecutePlugin => { 'input-parameters': inputParameters, 'output-parameter': outputParameter, } = validateGlobalConfig(); + return inputs.map(input => { validateSingleInput(input, inputParameters); @@ -88,6 +82,7 @@ export const Subtract = (globalConfig: SubtractConfig): ExecutePlugin => { */ const calculateDiff = (input: PluginParams, inputParameters: string[]) => { const [firstItem, ...restItems] = inputParameters; + return restItems.reduce( (accumulator, metricToSubtract) => accumulator - input[metricToSubtract], input[firstItem] // Starting accumulator with the value of the first item From c21fb0f01f3d23c7652c3e3c9e0b080697659c17 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 10 Jun 2024 16:55:42 +0400 Subject: [PATCH 027/243] feat(builtins): update errors usage in shell --- src/builtins/shell/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/builtins/shell/index.ts b/src/builtins/shell/index.ts index 43b1c224c..c78f668e2 100644 --- a/src/builtins/shell/index.ts +++ b/src/builtins/shell/index.ts @@ -8,7 +8,7 @@ import {ConfigParams} from '../../types/common'; import {validate} from '../../util/validations'; import {ERRORS} from '../../util/errors'; -const {InputValidationError} = ERRORS; +const {ProcessExecutionError} = ERRORS; export const Shell = (globalConfig: ConfigParams): ExecutePlugin => { const metadata = { @@ -55,7 +55,7 @@ export const Shell = (globalConfig: ConfigParams): ExecutePlugin => { return {outputs}; } catch (error: any) { - throw new InputValidationError(error.message); + throw new ProcessExecutionError(error.message); } }; From 8c3990189e56ac0541b1cee9f0f901d2f4bc3b9f Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 10 Jun 2024 16:55:51 +0400 Subject: [PATCH 028/243] feat(builtins): update errors usage in sci-embodied --- src/builtins/sci-embodied/index.ts | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/builtins/sci-embodied/index.ts b/src/builtins/sci-embodied/index.ts index 162099860..02f897584 100644 --- a/src/builtins/sci-embodied/index.ts +++ b/src/builtins/sci-embodied/index.ts @@ -1,8 +1,12 @@ import {z} from 'zod'; +import {validate, allDefined} from '../../util/validations'; + +import {STRINGS} from '../../config'; + import {ExecutePlugin, PluginParams} from '../../types/interface'; -import {validate, allDefined} from '../../util/validations'; +const {SCI_EMBODIED_ERROR} = STRINGS; export const SciEmbodied = (): ExecutePlugin => { const metadata = { @@ -55,9 +59,6 @@ export const SciEmbodied = (): ExecutePlugin => { * Checks for required fields in input. */ const validateInput = (input: PluginParams) => { - const errorMessage = (unit: string) => - `not a valid number in input. Please provide it as \`${unit}\``; - const commonSchemaPart = (errorMessage: (unit: string) => string) => ({ 'device/emissions-embodied': z .number({ @@ -81,13 +82,13 @@ export const SciEmbodied = (): ExecutePlugin => { const vcpusSchemaPart = { 'vcpus-allocated': z .number({ - invalid_type_error: errorMessage('count'), + invalid_type_error: SCI_EMBODIED_ERROR('count'), }) .gte(0) .min(0), 'vcpus-total': z .number({ - invalid_type_error: errorMessage('count'), + invalid_type_error: SCI_EMBODIED_ERROR('count'), }) .gte(0) .min(0), @@ -96,24 +97,24 @@ export const SciEmbodied = (): ExecutePlugin => { const resourcesSchemaPart = { 'resources-reserved': z .number({ - invalid_type_error: errorMessage('count'), + invalid_type_error: SCI_EMBODIED_ERROR('count'), }) .gte(0) .min(0), 'resources-total': z .number({ - invalid_type_error: errorMessage('count'), + invalid_type_error: SCI_EMBODIED_ERROR('count'), }) .gte(0) .min(0), }; const schemaWithVcpus = z.object({ - ...commonSchemaPart(errorMessage), + ...commonSchemaPart(SCI_EMBODIED_ERROR), ...vcpusSchemaPart, }); const schemaWithResources = z.object({ - ...commonSchemaPart(errorMessage), + ...commonSchemaPart(SCI_EMBODIED_ERROR), ...resourcesSchemaPart, }); From 0d978f48bd07c05fe8a8bd29628d22731f386437 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 10 Jun 2024 16:56:00 +0400 Subject: [PATCH 029/243] feat(builtins): update errors usage in sci --- src/builtins/sci/index.ts | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/src/builtins/sci/index.ts b/src/builtins/sci/index.ts index 7a60db439..a39e33c94 100644 --- a/src/builtins/sci/index.ts +++ b/src/builtins/sci/index.ts @@ -1,16 +1,17 @@ import {z} from 'zod'; -import {ExecutePlugin, PluginParams} from '../../types/interface'; -import {ConfigParams} from '../../types/common'; - import {validate, allDefined} from '../../util/validations'; -import {buildErrorMessage} from '../../util/helpers'; import {ERRORS} from '../../util/errors'; -const {InputValidationError} = ERRORS; +import {STRINGS} from '../../config'; + +import {ExecutePlugin, PluginParams} from '../../types/interface'; +import {ConfigParams} from '../../types/common'; + +const {MissingInputDataError} = ERRORS; +const {MISSING_FUNCTIONAL_UNIT_CONFIG, MISSING_FUNCTIONAL_UNIT_INPUT} = STRINGS; export const Sci = (globalConfig: ConfigParams): ExecutePlugin => { - const errorBuilder = buildErrorMessage(Sci.name); const metadata = { kind: 'execute', }; @@ -19,15 +20,12 @@ export const Sci = (globalConfig: ConfigParams): ExecutePlugin => { * Validates node and gloabl configs. */ const validateConfig = (config?: ConfigParams) => { - const errorMessage = - '`functional-unit` should be provided in your global config'; - const schema = z .object({ 'functional-unit': z.string(), }) .refine(data => data['functional-unit'], { - message: errorMessage, + message: MISSING_FUNCTIONAL_UNIT_CONFIG, }); return validate>(schema, config); @@ -36,19 +34,19 @@ export const Sci = (globalConfig: ConfigParams): ExecutePlugin => { /** * Calculate the total emissions for a list of inputs. */ - const execute = (inputs: PluginParams[]): PluginParams[] => { - return inputs.map(input => { + const execute = (inputs: PluginParams[]): PluginParams[] => + inputs.map(input => { const safeInput = validateInput(input); const sci = safeInput['carbon'] > 0 ? safeInput['carbon'] / input[globalConfig['functional-unit']] : 0; + return { ...input, sci, }; }); - }; /** * Checks for fields in input. @@ -64,12 +62,7 @@ export const Sci = (globalConfig: ConfigParams): ExecutePlugin => { input[validatedConfig['functional-unit']] > 0 ) ) { - throw new InputValidationError( - errorBuilder({ - message: - 'functional-unit value is missing from input data or it is not a positive integer', - }) - ); + throw new MissingInputDataError(MISSING_FUNCTIONAL_UNIT_INPUT); } const schema = z From 296734aca0ca72addea6a4acdcf3f6bb40d12fd5 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 10 Jun 2024 16:56:06 +0400 Subject: [PATCH 030/243] feat(builtins): update errors usage in regex --- src/builtins/regex/index.ts | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/src/builtins/regex/index.ts b/src/builtins/regex/index.ts index 7dc854461..5730697cc 100644 --- a/src/builtins/regex/index.ts +++ b/src/builtins/regex/index.ts @@ -1,16 +1,17 @@ import {z} from 'zod'; -import {buildErrorMessage} from '../../util/helpers'; import {ERRORS} from '../../util/errors'; import {validate} from '../../util/validations'; +import {STRINGS} from '../../config'; + import {ExecutePlugin, PluginParams} from '../../types/interface'; import {ConfigParams} from '../../types/common'; -const {InputValidationError, ConfigValidationError} = ERRORS; +const {MissingInputDataError, GlobalConfigError, RegexMismatchError} = ERRORS; +const {MISSING_GLOBAL_CONFIG, MISSING_INPUT_DATA, REGEX_MISMATCH} = STRINGS; export const Regex = (globalConfig: ConfigParams): ExecutePlugin => { - const errorBuilder = buildErrorMessage(Regex.name); const metadata = { kind: 'execute', }; @@ -20,10 +21,9 @@ export const Regex = (globalConfig: ConfigParams): ExecutePlugin => { */ const validateGlobalConfig = () => { if (!globalConfig) { - throw new ConfigValidationError( - errorBuilder({message: 'Configuration data is missing'}) - ); + throw new GlobalConfigError(MISSING_GLOBAL_CONFIG); } + const schema = z.object({ parameter: z.string().min(1), match: z.string().min(1), @@ -38,11 +38,7 @@ export const Regex = (globalConfig: ConfigParams): ExecutePlugin => { */ const validateSingleInput = (input: PluginParams, parameter: string) => { if (!input[parameter]) { - throw new InputValidationError( - errorBuilder({ - message: `\`${parameter}\` is missing from the input`, - }) - ); + throw new MissingInputDataError(MISSING_INPUT_DATA(parameter)); } return input; @@ -89,11 +85,7 @@ export const Regex = (globalConfig: ConfigParams): ExecutePlugin => { const matchedItem = input[parameter].match(regex); if (!matchedItem || !matchedItem[0]) { - throw new InputValidationError( - errorBuilder({ - message: `\`${input[parameter]}\` does not match the ${match} regex expression`, - }) - ); + throw new RegexMismatchError(REGEX_MISMATCH(input[parameter], match)); } return matchedItem[0]; From c60a1ab3504222b100c734c041168f11fa53e8fb Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 10 Jun 2024 16:56:13 +0400 Subject: [PATCH 031/243] feat(builtins): update errors usage in multiply --- src/builtins/multiply/index.ts | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/builtins/multiply/index.ts b/src/builtins/multiply/index.ts index 5666a6093..ffcf4d14c 100644 --- a/src/builtins/multiply/index.ts +++ b/src/builtins/multiply/index.ts @@ -1,16 +1,17 @@ import {z} from 'zod'; -import {buildErrorMessage} from '../../util/helpers'; import {ERRORS} from '../../util/errors'; import {validate} from '../../util/validations'; +import {STRINGS} from '../../config'; + import {ExecutePlugin, PluginParams} from '../../types/interface'; import {MultiplyConfig} from './types'; -const {InputValidationError} = ERRORS; +const {MissingInputDataError} = ERRORS; +const {MISSING_INPUT_DATA} = STRINGS; export const Multiply = (globalConfig: MultiplyConfig): ExecutePlugin => { - const errorBuilder = buildErrorMessage(Multiply.name); const metadata = { kind: 'execute', }; @@ -42,11 +43,7 @@ export const Multiply = (globalConfig: MultiplyConfig): ExecutePlugin => { input[metricToMultiply] === undefined || isNaN(input[metricToMultiply]) ) { - throw new InputValidationError( - errorBuilder({ - message: `${metricToMultiply} is missing from the input array`, - }) - ); + throw new MissingInputDataError(MISSING_INPUT_DATA(metricToMultiply)); } }); From 58bdc8e45539d4ebcd8377c7a57c0ccf243a8068 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 10 Jun 2024 16:56:20 +0400 Subject: [PATCH 032/243] feat(builtins): update errors usage in interpolation --- src/builtins/interpolation/index.ts | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/builtins/interpolation/index.ts b/src/builtins/interpolation/index.ts index 3c0e0381b..35f88788b 100644 --- a/src/builtins/interpolation/index.ts +++ b/src/builtins/interpolation/index.ts @@ -6,9 +6,17 @@ import {ExecutePlugin, PluginParams, ConfigParams} from '../../types/interface'; import {validate} from '../../util/validations'; import {ERRORS} from '../../util/errors'; +import {STRINGS} from '../../config'; + import {Method} from './types'; -const {ConfigNotFoundError} = ERRORS; +const {GlobalConfigError} = ERRORS; +const { + MISSING_GLOBAL_CONFIG, + X_Y_EQUAL, + ARRAY_LENGTH_NON_EMPTY, + WITHIN_THE_RANGE, +} = STRINGS; export const Interpolation = (globalConfig: ConfigParams): ExecutePlugin => { /** @@ -118,7 +126,7 @@ export const Interpolation = (globalConfig: ConfigParams): ExecutePlugin => { */ const validateConfig = () => { if (!globalConfig) { - throw new ConfigNotFoundError('Global config is not provided.'); + throw new GlobalConfigError(MISSING_GLOBAL_CONFIG); } const schema = z @@ -130,10 +138,10 @@ export const Interpolation = (globalConfig: ConfigParams): ExecutePlugin => { 'output-parameter': z.string(), }) .refine(data => data.x && data.y && data.x.length === data.y.length, { - message: 'The length of `x` and `y` should be equal', + message: X_Y_EQUAL, }) .refine(data => data.x.length > 1 && data.y.length > 1, { - message: 'the length of the input arrays must be greater than 1', + message: ARRAY_LENGTH_NON_EMPTY, }); const defaultMethod = globalConfig.method ?? Method.LINEAR; @@ -171,8 +179,7 @@ export const Interpolation = (globalConfig: ConfigParams): ExecutePlugin => { data[inputParameter] >= globalConfig.x[0] && data[inputParameter] <= globalConfig.x[globalConfig.x.length - 1], { - message: - 'The target x value must be within the range of the given x values', + message: WITHIN_THE_RANGE, } ); From b11aa7e1566de4e63ff2bfc5347c0e8d428dd6f1 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 10 Jun 2024 16:56:26 +0400 Subject: [PATCH 033/243] feat(builtins): update errors usage in exponent --- src/builtins/exponent/index.ts | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/src/builtins/exponent/index.ts b/src/builtins/exponent/index.ts index 404161e63..0de914fda 100644 --- a/src/builtins/exponent/index.ts +++ b/src/builtins/exponent/index.ts @@ -1,16 +1,17 @@ import {z} from 'zod'; import {ERRORS} from '../../util/errors'; -import {buildErrorMessage} from '../../util/helpers'; import {validate} from '../../util/validations'; +import {STRINGS} from '../../config'; + import {ExecutePlugin, PluginParams} from '../../types/interface'; import {ExponentConfig} from './types'; -const {InputValidationError} = ERRORS; +const {MissingInputDataError, InputValidationError} = ERRORS; +const {MISSING_INPUT_DATA, NOT_NUMERIC_VALUE} = STRINGS; export const Exponent = (globalConfig: ExponentConfig): ExecutePlugin => { - const errorBuilder = buildErrorMessage(Exponent.name); const metadata = { kind: 'execute', }; @@ -41,21 +42,13 @@ export const Exponent = (globalConfig: ExponentConfig): ExecutePlugin => { const validateParamExists = (input: PluginParams, param: string) => { if (input[param] === undefined) { - throw new InputValidationError( - errorBuilder({ - message: `${param} is missing from the input array`, - }) - ); + throw new MissingInputDataError(MISSING_INPUT_DATA(param)); } }; const validateNumericString = (str: string) => { if (isNaN(+Number(str))) { - throw new InputValidationError( - errorBuilder({ - message: `${str} is not numeric`, - }) - ); + throw new InputValidationError(NOT_NUMERIC_VALUE(str)); } }; From 2e2b07fc252f773f52f5fd3d9335489d08afad6f Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 10 Jun 2024 16:56:32 +0400 Subject: [PATCH 034/243] feat(builtins): update errors usage in divide --- src/builtins/divide/index.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/builtins/divide/index.ts b/src/builtins/divide/index.ts index b91d4c9fa..e50ca81f9 100644 --- a/src/builtins/divide/index.ts +++ b/src/builtins/divide/index.ts @@ -3,9 +3,12 @@ import {z} from 'zod'; import {ERRORS} from '../../util/errors'; import {validate} from '../../util/validations'; +import {STRINGS} from '../../config'; + import {ExecutePlugin, PluginParams, ConfigParams} from '../../types/interface'; -const {InputValidationError, ConfigNotFoundError} = ERRORS; +const {GlobalConfigError, MissingInputDataError} = ERRORS; +const {MISSING_GLOBAL_CONFIG, MISSING_INPUT_DATA} = STRINGS; export const Divide = (globalConfig: ConfigParams): ExecutePlugin => { const metadata = { @@ -38,7 +41,7 @@ export const Divide = (globalConfig: ConfigParams): ExecutePlugin => { */ const validateGlobalConfig = () => { if (!globalConfig) { - throw new ConfigNotFoundError('Global config is not provided.'); + throw new GlobalConfigError(MISSING_GLOBAL_CONFIG); } const schema = z.object({ @@ -65,9 +68,7 @@ export const Divide = (globalConfig: ConfigParams): ExecutePlugin => { }) .refine(() => { if (typeof denominator === 'string' && !input[denominator]) { - throw new InputValidationError( - `\`${denominator}\` is missing from the input.` - ); + throw new MissingInputDataError(MISSING_INPUT_DATA(denominator)); } return true; }); From c16f46c36a4e8ad258bfff11cc3256955552167d Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 10 Jun 2024 16:56:52 +0400 Subject: [PATCH 035/243] feat(builtins): update errors usage in csv lookup, optimize strategy --- src/builtins/csv-lookup/index.ts | 106 +++++++++++++++++++------------ 1 file changed, 64 insertions(+), 42 deletions(-) diff --git a/src/builtins/csv-lookup/index.ts b/src/builtins/csv-lookup/index.ts index 2539a14cf..4f7df52be 100644 --- a/src/builtins/csv-lookup/index.ts +++ b/src/builtins/csv-lookup/index.ts @@ -5,12 +5,29 @@ import axios from 'axios'; import {z} from 'zod'; import {parse} from 'csv-parse/sync'; -import {ExecutePlugin, PluginParams} from '../../types/interface'; - import {validate} from '../../util/validations'; import {ERRORS} from '../../util/errors'; -const {ConfigNotFoundError, FileNotFoundError, InputValidationError} = ERRORS; +import {STRINGS} from '../../config'; + +import {ExecutePlugin, PluginParams} from '../../types/interface'; + +const { + FILE_FETCH_FAILED, + FILE_READ_FAILED, + MISSING_CSV_COLUMN, + NO_QUERY_DATA, + MISSING_GLOBAL_CONFIG, +} = STRINGS; + +const { + FetchingFileError, + ReadFileError, + MissingCSVColumnError, + QueryDataNotFoundError, + GlobalConfigError, + CSVParseError, +} = ERRORS; export const CSVLookup = (globalConfig: any): ExecutePlugin => { const metadata = { @@ -36,18 +53,16 @@ export const CSVLookup = (globalConfig: any): ExecutePlugin => { const retrieveFile = async (filepath: string) => { if (isURL(filepath)) { const {data} = await axios.get(filepath).catch(error => { - throw new FileNotFoundError(`Something went wrong while reading the file: ${filepath}. - ${error.response.message}`); + throw new FetchingFileError( + FILE_FETCH_FAILED(filepath, error.response.message) + ); }); return data; } return readFile(filepath).catch(error => { - throw new FileNotFoundError( - `Something went wrong while reading the file: ${filepath}. -${error}` - ); + throw new ReadFileError(FILE_READ_FAILED(filepath, error)); }); }; @@ -81,7 +96,7 @@ ${error}` */ const fieldAccessor = (field: string, object: any) => { if (!(`${field}` in object)) { - throw new InputValidationError(`There is no column with name: ${field}.`); + throw new MissingCSVColumnError(MISSING_CSV_COLUMN(field)); } return nanifyEmptyValues(object[field]); @@ -149,6 +164,24 @@ ${error}` return ifMatchesCriteria.every(value => value === true); }; + /** + * Parses CSV file. + */ + const parseCSVFile = (file: string | Buffer) => { + try { + const parsedCSV: any[] = parse(file, { + columns: true, + skip_empty_lines: true, + cast: true, + }); + + return parsedCSV; + } catch (error: any) { + console.error(error); + throw new CSVParseError(error); + } + }; + /** * 1. Validates global config. * 2. Tries to retrieve given file (with url or local path). @@ -160,41 +193,30 @@ ${error}` const {filepath, query, output} = safeGlobalConfig; const file = await retrieveFile(filepath); - - try { - const parsedCSV: any[] = parse(file, { - columns: true, - skip_empty_lines: true, - cast: true, + console.log(file); + const parsedCSV = parseCSVFile(file); + + return inputs.map(input => { + /** Collects query values from input. */ + const queryData: any = {}; + const queryKeys = Object.keys(query); + queryKeys.forEach(queryKey => { + const queryValue = query[queryKey]; + queryData[queryKey] = input[queryValue]; }); - return inputs.map(input => { - /** Collects query values from input. */ - const queryData: any = {}; - const queryKeys = Object.keys(query); - queryKeys.forEach(queryKey => { - const queryValue = query[queryKey]; - queryData[queryKey] = input[queryValue]; - }); - - /** Gets related data from CSV. */ - const relatedData = parsedCSV.find(withCriteria(queryData)); + /** Gets related data from CSV. */ + const relatedData = parsedCSV.find(withCriteria(queryData)); - if (!relatedData) { - throw new InputValidationError( - 'One or more of the given query parameters are not found in the target CSV file column headers.' - ); - } + if (!relatedData) { + throw new QueryDataNotFoundError(NO_QUERY_DATA); + } - return { - ...input, - ...filterOutput(relatedData, {output, query}), - }; - }); - } catch (error) { - throw new InputValidationError(`Error happened while parsing given CSV file: ${filepath} -${error}`); - } + return { + ...input, + ...filterOutput(relatedData, {output, query}), + }; + }); }; /** @@ -202,7 +224,7 @@ ${error}`); */ const validateGlobalConfig = () => { if (!globalConfig) { - throw new ConfigNotFoundError('Global config is not provided.'); + throw new GlobalConfigError(MISSING_GLOBAL_CONFIG); } const globalConfigSchema = z.object({ From 9b3ea3ee924677649d3548307f452dbab7ac0d6f Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 10 Jun 2024 16:57:03 +0400 Subject: [PATCH 036/243] feat(builtins): update errors usage coefficient --- src/builtins/coefficient/index.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/builtins/coefficient/index.ts b/src/builtins/coefficient/index.ts index 627e5f64e..c049911ae 100644 --- a/src/builtins/coefficient/index.ts +++ b/src/builtins/coefficient/index.ts @@ -5,9 +5,12 @@ import {ExecutePlugin, PluginParams} from '../../types/interface'; import {validate} from '../../util/validations'; import {ERRORS} from '../../util/errors'; +import {STRINGS} from '../../config'; + import {CoefficientConfig} from './types'; -const {ConfigNotFoundError} = ERRORS; +const {GlobalConfigError} = ERRORS; +const {MISSING_GLOBAL_CONFIG} = STRINGS; export const Coefficient = (globalConfig: CoefficientConfig): ExecutePlugin => { const metadata = { @@ -45,7 +48,7 @@ export const Coefficient = (globalConfig: CoefficientConfig): ExecutePlugin => { */ const validateGlobalConfig = () => { if (!globalConfig) { - throw new ConfigNotFoundError('Global config is not provided.'); + throw new GlobalConfigError(MISSING_GLOBAL_CONFIG); } const globalConfigSchema = z.object({ From f088869822856a27c4873dbe1ccbb3b49a4fa738 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 10 Jun 2024 16:57:37 +0400 Subject: [PATCH 037/243] feat(builtins): update errors usage time-sync --- src/builtins/time-sync.ts | 57 ++++++++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 21 deletions(-) diff --git a/src/builtins/time-sync.ts b/src/builtins/time-sync.ts index 67d5c9079..469fdec35 100644 --- a/src/builtins/time-sync.ts +++ b/src/builtins/time-sync.ts @@ -5,6 +5,7 @@ import {z} from 'zod'; import {parameterize} from '../lib/parameterize'; import {ERRORS} from '../util/errors'; +import {validate} from '../util/validations'; import {STRINGS} from '../config'; @@ -14,16 +15,24 @@ import { TimeNormalizerConfig, TimeParams, } from '../types/time-sync'; -import {validate} from '../util/validations'; Settings.defaultZone = 'utc'; -const {InputValidationError} = ERRORS; +const { + GlobalConfigError, + InvalidDateInInputError, + InvalidPaddingError, + InvalidInputError, +} = ERRORS; const { INVALID_TIME_NORMALIZATION, INVALID_OBSERVATION_OVERLAP, AVOIDING_PADDING_BY_EDGES, + INVALID_DATE_TYPE, + START_LOWER_END, + TIMESTAMP_REQUIRED, + INVALID_DATETIME, } = STRINGS; export const TimeSync = (globalConfig: TimeNormalizerConfig): ExecutePlugin => { @@ -64,7 +73,7 @@ export const TimeSync = (globalConfig: TimeNormalizerConfig): ExecutePlugin => { seconds: previousInput.duration, }) > currentMoment ) { - throw new InputValidationError(INVALID_OBSERVATION_OVERLAP); + throw new InvalidInputError(INVALID_OBSERVATION_OVERLAP); } const compareableTime = previousInputTimestamp.plus({ @@ -106,19 +115,22 @@ export const TimeSync = (globalConfig: TimeNormalizerConfig): ExecutePlugin => { }; const parseDate = (date: Date | string) => { - if (!date) return DateTime.invalid('Invalid date'); + if (!date) { + return DateTime.invalid('Invalid date'); + } + // dates are passed to time-sync.ts both in ISO 8601 format // and as a Date object (from the deserialization of a YAML file) // if the YAML parser fails to identify as a date, it passes as a string if (isDate(date)) { return DateTime.fromJSDate(date); } + if (typeof date === 'string') { return DateTime.fromISO(date); } - throw new InputValidationError( - `Unexpected date datatype: ${typeof date}: ${date}` - ); + + throw new InvalidDateInInputError(INVALID_DATE_TYPE(date)); }; /** @@ -128,10 +140,10 @@ export const TimeSync = (globalConfig: TimeNormalizerConfig): ExecutePlugin => { const schema = z.object({ timestamp: z .string({ - required_error: `required in input[${index}]`, + required_error: TIMESTAMP_REQUIRED(index), }) .datetime({ - message: `invalid datetime in input[${index}]`, + message: INVALID_DATETIME(index), }) .or(z.date()), duration: z.number(), @@ -145,7 +157,7 @@ export const TimeSync = (globalConfig: TimeNormalizerConfig): ExecutePlugin => { */ const validateGlobalConfig = () => { if (globalConfig === undefined) { - throw new InputValidationError(INVALID_TIME_NORMALIZATION); + throw new GlobalConfigError(INVALID_TIME_NORMALIZATION); } const schema = z @@ -156,7 +168,7 @@ export const TimeSync = (globalConfig: TimeNormalizerConfig): ExecutePlugin => { 'allow-padding': z.boolean(), }) .refine(data => data['start-time'] < data['end-time'], { - message: '`start-time` should be lower than `end-time`', + message: START_LOWER_END, }); return validate>(schema, globalConfig); @@ -176,8 +188,10 @@ export const TimeSync = (globalConfig: TimeNormalizerConfig): ExecutePlugin => { i: number ) => { const thisMoment = parseDate(currentRoundMoment).startOf('second'); + return thisMoment.plus({seconds: i}); }; + /** * Breaks down input per minimal time unit. */ @@ -259,8 +273,9 @@ export const TimeSync = (globalConfig: TimeNormalizerConfig): ExecutePlugin => { const validatePadding = (pad: PaddingReceipt, params: TimeParams): void => { const {start, end} = pad; const isPaddingNeeded = start || end; + if (!params.allowPadding && isPaddingNeeded) { - throw new InputValidationError(AVOIDING_PADDING_BY_EDGES(start, end)); + throw new InvalidPaddingError(AVOIDING_PADDING_BY_EDGES(start, end)); } }; @@ -292,8 +307,8 @@ export const TimeSync = (globalConfig: TimeNormalizerConfig): ExecutePlugin => { * Iterates over given inputs frame, meanwhile checking if aggregation method is `sum`, then calculates it. * For methods is `avg` and `none` calculating average of the frame. */ - const resampleInputFrame = (inputsInTimeslot: PluginParams[]) => { - return inputsInTimeslot.reduce((acc, input, index, inputs) => { + const resampleInputFrame = (inputsInTimeslot: PluginParams[]) => + inputsInTimeslot.reduce((acc, input, index, inputs) => { const metrics = Object.keys(input); metrics.forEach(metric => { @@ -336,13 +351,12 @@ export const TimeSync = (globalConfig: TimeNormalizerConfig): ExecutePlugin => { return acc; }, {} as PluginParams); - }; /** * Takes each array frame with interval length, then aggregating them together as from units.yaml file. */ - const resampleInputs = (inputs: PluginParams[], params: TimeParams) => { - return inputs.reduce((acc: PluginParams[], _input, index, inputs) => { + const resampleInputs = (inputs: PluginParams[], params: TimeParams) => + inputs.reduce((acc: PluginParams[], _input, index, inputs) => { const frameStart = index * params.interval; const frameEnd = (index + 1) * params.interval; const inputsFrame = inputs.slice(frameStart, frameEnd); @@ -356,7 +370,6 @@ export const TimeSync = (globalConfig: TimeNormalizerConfig): ExecutePlugin => { return acc; }, [] as PluginParams[]); - }; /** * Pads zeroish inputs from the beginning or at the end of the inputs if needed. @@ -394,6 +407,7 @@ export const TimeSync = (globalConfig: TimeNormalizerConfig): ExecutePlugin => { ) ); } + return paddedArray; }; @@ -404,6 +418,7 @@ export const TimeSync = (globalConfig: TimeNormalizerConfig): ExecutePlugin => { ) => { const array: PluginParams[] = []; const dateRange = Interval.fromDateTimes(startDate, endDate); + for (const interval of dateRange.splitBy({second: 1})) { array.push( fillWithZeroishInput( @@ -415,6 +430,7 @@ export const TimeSync = (globalConfig: TimeNormalizerConfig): ExecutePlugin => { ) ); } + return array; }; @@ -424,8 +440,8 @@ export const TimeSync = (globalConfig: TimeNormalizerConfig): ExecutePlugin => { const trimInputsByGlobalTimeline = ( inputs: PluginParams[], params: TimeParams - ): PluginParams[] => { - return inputs.reduce((acc: PluginParams[], item) => { + ): PluginParams[] => + inputs.reduce((acc: PluginParams[], item) => { const {timestamp} = item; if ( @@ -437,7 +453,6 @@ export const TimeSync = (globalConfig: TimeNormalizerConfig): ExecutePlugin => { return acc; }, [] as PluginParams[]); - }; return {metadata, execute}; }; From b34df91e030b646e99e7a41fe9bc2f072a3bf233 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 10 Jun 2024 16:57:52 +0400 Subject: [PATCH 038/243] feat(builtins): update errors usage group-by --- src/builtins/group-by.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/builtins/group-by.ts b/src/builtins/group-by.ts index 6a7036adf..e4e8af958 100644 --- a/src/builtins/group-by.ts +++ b/src/builtins/group-by.ts @@ -8,9 +8,9 @@ import {GroupByConfig} from '../types/group-by'; import {ERRORS} from '../util/errors'; import {validate} from '../util/validations'; -const {InvalidGroupingError, InputValidationError} = ERRORS; +const {InvalidGroupingError, GlobalConfigError} = ERRORS; -const {INVALID_GROUP_BY} = STRINGS; +const {INVALID_GROUP_BY, MISSING_GLOBAL_CONFIG} = STRINGS; /** * Plugin for inputs grouping. @@ -79,7 +79,7 @@ export const GroupBy = (): GroupByPlugin => { */ const validateConfig = (config: GroupByConfig) => { if (!config) { - throw new InputValidationError('Config is not provided.'); + throw new GlobalConfigError(MISSING_GLOBAL_CONFIG); } const schema = z.object({ From 93adf7146ada5be7c7af1b982edc1c9450c9afbb Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 10 Jun 2024 16:58:03 +0400 Subject: [PATCH 039/243] feat(builtins): update errors usage export yaml --- src/builtins/export-yaml.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/builtins/export-yaml.ts b/src/builtins/export-yaml.ts index e242bbcc2..d4f71c3f5 100644 --- a/src/builtins/export-yaml.ts +++ b/src/builtins/export-yaml.ts @@ -1,9 +1,12 @@ import {saveYamlFileAs} from '../util/yaml'; import {ERRORS} from '../util/errors'; +import {STRINGS} from '../config'; + import {Context} from '../types/manifest'; -const {ExhaustError} = ERRORS; +const {ExhaustOutputArgError} = ERRORS; +const {OUTPUT_REQUIRED} = STRINGS; export const ExportYaml = () => { /** Takes string before hashtag. */ @@ -14,7 +17,7 @@ export const ExportYaml = () => { */ const execute = async (tree: any, context: Context, outputPath: string) => { if (!outputPath) { - throw new ExhaustError('Output path is required.'); + throw new ExhaustOutputArgError(OUTPUT_REQUIRED); } const outputFile = { From b6d9b20e570d234572eaa6819acd6a35e9ddbc61 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 10 Jun 2024 16:58:08 +0400 Subject: [PATCH 040/243] feat(builtins): update errors usage export csv --- src/builtins/export-csv.ts | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/builtins/export-csv.ts b/src/builtins/export-csv.ts index efd2cc3cb..b0f4f80e8 100644 --- a/src/builtins/export-csv.ts +++ b/src/builtins/export-csv.ts @@ -3,10 +3,13 @@ import {stringify} from 'csv-stringify/sync'; import {ERRORS} from '../util/errors'; +import {STRINGS} from '../config'; + import {Context} from '../types/manifest'; import {PluginParams} from '../types/interface'; -const {ExhaustError} = ERRORS; +const {ExhaustOutputArgError} = ERRORS; +const {CSV_EXPORT, OUTPUT_REQUIRED} = STRINGS; /** * Extension to IF that outputs the tree in a CSV format. @@ -16,15 +19,14 @@ export const ExportCSV = () => { const validatedPath = validateOutputPath(outputPath); const paths = validatedPath.split('#'); - const output = paths.slice(0, paths.length - 1).join(''); const criteria = paths[paths.length - 1]; if (paths.length <= 1 || !criteria) { - throw new ExhaustError( - 'CSV export criteria is not found in output path. Please append it after --output #.' - ); + throw new ExhaustOutputArgError(CSV_EXPORT); } + const output = paths.slice(0, paths.length - 1).join(''); + return { output, criteria, @@ -36,11 +38,7 @@ export const ExportCSV = () => { */ const validateOutputPath = (outputPath: string) => { if (!outputPath) { - throw new ExhaustError('Output path is required.'); - } - - if (!outputPath.includes('#')) { - throw new ExhaustError('Output path should contain `#`.'); + throw new ExhaustOutputArgError(OUTPUT_REQUIRED); } return outputPath; From 19f1eeccb9585d698b2f6c769a19816676dbb489 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 10 Jun 2024 16:58:14 +0400 Subject: [PATCH 041/243] feat(builtins): update errors usage export csv raw --- src/builtins/export-csv-raw.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/builtins/export-csv-raw.ts b/src/builtins/export-csv-raw.ts index 429d99c85..429990902 100644 --- a/src/builtins/export-csv-raw.ts +++ b/src/builtins/export-csv-raw.ts @@ -2,10 +2,13 @@ import * as fs from 'fs/promises'; import {ERRORS} from '../util/errors'; +import {STRINGS} from '../config'; + import {ExhaustPluginInterface} from '../types/exhaust-plugin-interface'; import {Context} from '../types/manifest'; -const {ExhaustError} = ERRORS; +const {ExhaustOutputArgError, WriteFileError} = ERRORS; +const {OUTPUT_REQUIRED, WRITE_CSV_ERROR} = STRINGS; export const ExportCSVRaw = (): ExhaustPluginInterface => { /** @@ -127,7 +130,7 @@ export const ExportCSVRaw = (): ExhaustPluginInterface => { try { await fs.writeFile(`${outputPath}.csv`, content); } catch (error) { - throw new ExhaustError(`Failed to write CSV to ${outputPath}: ${error}`); + throw new WriteFileError(WRITE_CSV_ERROR(outputPath, error)); } }; @@ -136,7 +139,7 @@ export const ExportCSVRaw = (): ExhaustPluginInterface => { */ const execute = async (tree: any, _context: Context, outputPath: string) => { if (!outputPath) { - throw new ExhaustError('Output path is required.'); + throw new ExhaustOutputArgError(OUTPUT_REQUIRED); } const [extractredFlatMap, extractedHeaders] = From cfc89dbec2bad8aaa656faaf1d62114b3edf89ef Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 10 Jun 2024 16:58:45 +0400 Subject: [PATCH 042/243] test(util): migrate tests to use new errors --- src/__tests__/unit/util/aggregation-helper.test.ts | 11 ++++++----- src/__tests__/unit/util/args.test.ts | 10 +++++----- src/__tests__/unit/util/plugin-storage.test.ts | 6 +++--- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/__tests__/unit/util/aggregation-helper.test.ts b/src/__tests__/unit/util/aggregation-helper.test.ts index 20690bc2f..7cccbfccc 100644 --- a/src/__tests__/unit/util/aggregation-helper.test.ts +++ b/src/__tests__/unit/util/aggregation-helper.test.ts @@ -5,7 +5,7 @@ import {STRINGS} from '../../../config'; import {PluginParams} from '../../../types/interface'; -const {InvalidAggregationParamsError} = ERRORS; +const {InvalidAggregationMethodError, MissingAggregationParamError} = ERRORS; const {INVALID_AGGREGATION_METHOD, METRIC_MISSING} = STRINGS; describe('util/aggregation-helper: ', () => { @@ -20,9 +20,9 @@ describe('util/aggregation-helper: ', () => { try { aggregateInputsIntoOne(inputs, metrics, isTemporal); } catch (error) { - expect(error).toBeInstanceOf(InvalidAggregationParamsError); + expect(error).toBeInstanceOf(InvalidAggregationMethodError); - if (error instanceof InvalidAggregationParamsError) { + if (error instanceof InvalidAggregationMethodError) { expect(error.message).toEqual(INVALID_AGGREGATION_METHOD(metrics[0])); } } @@ -38,9 +38,10 @@ describe('util/aggregation-helper: ', () => { try { aggregateInputsIntoOne(inputs, metrics, isTemporal); } catch (error) { - expect(error).toBeInstanceOf(InvalidAggregationParamsError); + console.log(error); + expect(error).toBeInstanceOf(MissingAggregationParamError); - if (error instanceof InvalidAggregationParamsError) { + if (error instanceof MissingAggregationParamError) { expect(error.message).toEqual(METRIC_MISSING(metrics[0], 0)); } } diff --git a/src/__tests__/unit/util/args.test.ts b/src/__tests__/unit/util/args.test.ts index d40d36cae..7da9efa37 100644 --- a/src/__tests__/unit/util/args.test.ts +++ b/src/__tests__/unit/util/args.test.ts @@ -76,7 +76,7 @@ import {ERRORS} from '../../../util/errors'; import {STRINGS} from '../../../config'; -const {CliInputError} = ERRORS; +const {CliInputError, CliSourceFileError} = ERRORS; const { MANIFEST_IS_MISSING, @@ -115,8 +115,8 @@ describe('util/args: ', () => { try { parseIEProcessArgs(); } catch (error) { - expect(error).toBeInstanceOf(CliInputError); - expect(error).toEqual(new CliInputError(MANIFEST_IS_MISSING)); + expect(error).toBeInstanceOf(CliSourceFileError); + expect(error).toEqual(new CliSourceFileError(MANIFEST_IS_MISSING)); } }); @@ -195,8 +195,8 @@ describe('util/args: ', () => { try { parseIEProcessArgs(); } catch (error) { - expect(error).toBeInstanceOf(CliInputError); - expect(error).toEqual(new CliInputError(FILE_IS_NOT_YAML)); + expect(error).toBeInstanceOf(CliSourceFileError); + expect(error).toEqual(new CliSourceFileError(FILE_IS_NOT_YAML)); } }); diff --git a/src/__tests__/unit/util/plugin-storage.test.ts b/src/__tests__/unit/util/plugin-storage.test.ts index b2bd25011..f8ac5b631 100644 --- a/src/__tests__/unit/util/plugin-storage.test.ts +++ b/src/__tests__/unit/util/plugin-storage.test.ts @@ -1,7 +1,7 @@ import {pluginStorage} from '../../../util/plugin-storage'; import {ERRORS} from '../../../util/errors'; -const {PluginInitalizationError} = ERRORS; +const {PluginInitializationError} = ERRORS; describe('util/pluginStorage: ', () => { describe('pluginStorage(): ', () => { @@ -30,9 +30,9 @@ describe('util/pluginStorage: ', () => { try { storage.get(pluginName); } catch (error) { - expect(error).toBeInstanceOf(PluginInitalizationError); + expect(error).toBeInstanceOf(PluginInitializationError); - if (error instanceof PluginInitalizationError) { + if (error instanceof PluginInitializationError) { expect(error.message).toEqual; } } From fe46a8d6542bddb9ec8419c57f881c652cba9958 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 10 Jun 2024 16:58:56 +0400 Subject: [PATCH 043/243] test(lib): migrate tests to use new errors --- src/__tests__/unit/lib/exhaust.test.ts | 22 +++++++--------- src/__tests__/unit/lib/initialize.test.ts | 32 ++++++++++++++--------- 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/src/__tests__/unit/lib/exhaust.test.ts b/src/__tests__/unit/lib/exhaust.test.ts index 97db0b9dc..a1f46595a 100644 --- a/src/__tests__/unit/lib/exhaust.test.ts +++ b/src/__tests__/unit/lib/exhaust.test.ts @@ -7,8 +7,8 @@ import {ERRORS} from '../../../util/errors'; import {STRINGS} from '../../../config'; -const {ExhaustError} = ERRORS; -const {INVALID_EXHAUST_PLUGIN} = STRINGS; +const {ExhaustOutputArgError, InvalidExhaustPluginError} = ERRORS; +const {INVALID_EXHAUST_PLUGIN, OUTPUT_REQUIRED} = STRINGS; describe('lib/exhaust: ', () => { describe('exhaust(): ', () => { @@ -50,7 +50,6 @@ describe('lib/exhaust: ', () => { outputs: ['yaml'], }, }; - const expectedMessage = 'Output path is required.'; expect.assertions(2); @@ -58,10 +57,10 @@ describe('lib/exhaust: ', () => { // @ts-ignore await exhaust(tree, context, {}); } catch (error) { - expect(error).toBeInstanceOf(ExhaustError); + expect(error).toBeInstanceOf(ExhaustOutputArgError); - if (error instanceof ExhaustError) { - expect(error.message).toEqual(expectedMessage); + if (error instanceof ExhaustOutputArgError) { + expect(error.message).toEqual(OUTPUT_REQUIRED); } } }); @@ -73,9 +72,6 @@ describe('lib/exhaust: ', () => { outputs: ['mock'], }, }; - const expectedMessage = INVALID_EXHAUST_PLUGIN( - context.initialize.outputs[0] - ); expect.assertions(2); @@ -83,10 +79,12 @@ describe('lib/exhaust: ', () => { // @ts-ignore await exhaust(tree, context, {}); } catch (error) { - expect(error).toBeInstanceOf(ExhaustError); + expect(error).toBeInstanceOf(InvalidExhaustPluginError); - if (error instanceof ExhaustError) { - expect(error.message).toEqual(expectedMessage); + if (error instanceof InvalidExhaustPluginError) { + expect(error.message).toEqual( + INVALID_EXHAUST_PLUGIN(context.initialize.outputs[0]) + ); } } }); diff --git a/src/__tests__/unit/lib/initialize.test.ts b/src/__tests__/unit/lib/initialize.test.ts index ad5541684..23ca27171 100644 --- a/src/__tests__/unit/lib/initialize.test.ts +++ b/src/__tests__/unit/lib/initialize.test.ts @@ -18,7 +18,11 @@ import {STRINGS} from '../../../config'; import {GlobalPlugins} from '../../../types/manifest'; -const {PluginCredentialError, ModuleInitializationError} = ERRORS; +const { + MissingPluginPathError, + MissingPluginMethodError, + PluginInitializationError, +} = ERRORS; const {MISSING_METHOD, MISSING_PATH, INVALID_MODULE_PATH} = STRINGS; describe('lib/initalize: ', () => { @@ -81,9 +85,9 @@ describe('lib/initalize: ', () => { try { await initialize(plugins); } catch (error) { - expect(error).toBeInstanceOf(PluginCredentialError); + expect(error).toBeInstanceOf(MissingPluginPathError); - if (error instanceof PluginCredentialError) { + if (error instanceof MissingPluginPathError) { expect(error.message).toEqual(MISSING_PATH); } } @@ -103,9 +107,9 @@ describe('lib/initalize: ', () => { try { await initialize(plugins); } catch (error) { - expect(error).toBeInstanceOf(PluginCredentialError); + expect(error).toBeInstanceOf(MissingPluginMethodError); - if (error instanceof PluginCredentialError) { + if (error instanceof MissingPluginMethodError) { expect(error.message).toEqual(MISSING_METHOD); } } @@ -160,14 +164,16 @@ describe('lib/initalize: ', () => { try { await initialize(plugins); - } catch (error) { - expect(error).toBeInstanceOf(ModuleInitializationError); - - if (error instanceof ModuleInitializationError) { - expect(error.message).toEqual( - INVALID_MODULE_PATH(plugins.mockavizta.path) - ); - } + } catch (error: any) { + expect(error).toBeInstanceOf(PluginInitializationError); + expect(error.message).toEqual( + INVALID_MODULE_PATH( + plugins.mockavizta.path, + new Error( + "Cannot find module 'failing-mock' from 'src/lib/initialize.ts'" + ) + ) + ); } }); }); From 6cca843d563fb7c3241d4704d43321a5e781d831 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 10 Jun 2024 16:59:14 +0400 Subject: [PATCH 044/243] test(config): drop strings tests --- src/__tests__/unit/config/strings.test.ts | 171 ---------------------- 1 file changed, 171 deletions(-) delete mode 100644 src/__tests__/unit/config/strings.test.ts diff --git a/src/__tests__/unit/config/strings.test.ts b/src/__tests__/unit/config/strings.test.ts deleted file mode 100644 index bdc1f871f..000000000 --- a/src/__tests__/unit/config/strings.test.ts +++ /dev/null @@ -1,171 +0,0 @@ -import {STRINGS} from '../../../config/strings'; - -const { - NOT_NATIVE_PLUGIN, - INVALID_MODULE_PATH, - INVALID_AGGREGATION_METHOD, - METRIC_MISSING, - AVOIDING_PADDING, - AVOIDING_PADDING_BY_EDGES, - INVALID_GROUP_BY, - REJECTING_OVERRIDE, - INVALID_EXHAUST_PLUGIN, - UNKNOWN_PARAM, - NOT_INITALIZED_PLUGIN, -} = STRINGS; - -describe('config/strings: ', () => { - describe('NOT_NATIVE_PLUGIN(): ', () => { - it('successfully injects path into message.', () => { - const path = 'mock/path'; - const expectedMessage = ` -You are using plugin ${path} which is not part of the Impact Framework standard library. You should do your own research to ensure the plugins are up to date and accurate. They may not be actively maintained.`; - - expect(NOT_NATIVE_PLUGIN(path)).toEqual(expectedMessage); - }); - }); - - describe('INVALID_MODULE_PATH(): ', () => { - it('successfully appends given param to message.', () => { - const param = 'mock-param'; - - const expectedMessage = `Provided module: '${param}' is invalid or not found.`; - - expect(INVALID_MODULE_PATH(param)).toEqual(expectedMessage); - }); - }); - - describe('AVOIDING_PADDING(): ', () => { - it('successfully appends given param to message.', () => { - const param = 'mock-param'; - - const expectedMessage = `Avoiding padding at ${param}`; - - expect(AVOIDING_PADDING(param)).toEqual(expectedMessage); - }); - }); - - describe('AVOIDING_PADDING_BY_EDGES(): ', () => { - it('successfully appends given start and end params.', () => { - const start = true; - const end = true; - - const expectedMessage = 'Avoiding padding at start and end'; - - expect(AVOIDING_PADDING_BY_EDGES(start, end)).toEqual(expectedMessage); - }); - - it('successfully appends given start param.', () => { - const start = true; - const end = false; - - const expectedMessage = 'Avoiding padding at start'; - - expect(AVOIDING_PADDING_BY_EDGES(start, end)).toEqual(expectedMessage); - }); - - it('successfully appends given end param.', () => { - const start = false; - const end = true; - - const expectedMessage = 'Avoiding padding at end'; - - expect(AVOIDING_PADDING_BY_EDGES(start, end)).toEqual(expectedMessage); - }); - }); - - describe('INVALID_AGGREGATION_METHOD(): ', () => { - it('successfully appends given param to message.', () => { - const param = 'mock-param'; - - const expectedMessage = `Aggregation is not possible for given ${param} since method is 'none'.`; - - expect(INVALID_AGGREGATION_METHOD(param)).toEqual(expectedMessage); - }); - }); - - describe('METRIC_MISSING(): ', () => { - it('successfully appends given param to message.', () => { - const metric = 'mock-metric'; - const index = 0; - - const expectedMessage = `Aggregation metric ${metric} is not found in inputs[${index}].`; - - expect(METRIC_MISSING(metric, index)).toEqual(expectedMessage); - }); - }); - - describe('INVALID_GROUP_BY(): ', () => { - it('injects type in given message.', () => { - const type = 'mock-type'; - const message = `Invalid group ${type}.`; - - expect(INVALID_GROUP_BY(type)).toEqual(message); - }); - }); - - describe('REJECTING_OVERRIDE(): ', () => { - it('inejcts param name into message.', () => { - const param: any = { - name: 'mock-name', - description: 'mock-description', - aggregation: 'sum', - unit: 'mock-unit', - }; - - expect(REJECTING_OVERRIDE(param)); - }); - }); - - describe('INVALID_EXHAUST_PLUGIN(): ', () => { - it('injects plugin name into message.', () => { - const pluginName = 'mock-plugin'; - const message = `Invalid exhaust plugin: ${pluginName}.`; - - expect(INVALID_EXHAUST_PLUGIN(pluginName)).toEqual(message); - }); - }); - - describe('AVOIDING_PADDING(): ', () => { - it('successfully appends given param to message.', () => { - const description_suffix = 'pad description'; - - const expectedMessage = `Avoiding padding at ${description_suffix}`; - - expect(AVOIDING_PADDING(description_suffix)).toEqual(expectedMessage); - }); - }); - - describe('UNKNOWN_PARAM(): ', () => { - it('injects name into message.', () => { - const name = 'mock-name'; - const message = `Unknown parameter: ${name}. Using 'sum' aggregation method.`; - - expect(UNKNOWN_PARAM(name)).toEqual(message); - }); - }); - - describe('NOT_INITALIZED_PLUGIN(): ', () => { - it('injects name into message.', () => { - const name = 'mock-name'; - const message = `Not initalized plugin: ${name}. Check if ${name} is in 'manifest.initalize.plugins'.`; - - expect(NOT_INITALIZED_PLUGIN(name)).toEqual(message); - }); - }); - - describe('AVOIDING_PADDING_BY_EDGES(): ', () => { - it('successfully combines boolean params into a description.', () => { - let description_suffix = 'start and end'; - let expectedMessage = `Avoiding padding at ${description_suffix}`; - - expect(AVOIDING_PADDING_BY_EDGES(true, true)).toEqual(expectedMessage); - description_suffix = 'start'; - expectedMessage = `Avoiding padding at ${description_suffix}`; - expect(AVOIDING_PADDING_BY_EDGES(true, false)).toEqual(expectedMessage); - description_suffix = 'end'; - expectedMessage = `Avoiding padding at ${description_suffix}`; - expect(AVOIDING_PADDING_BY_EDGES(false, true)).toEqual(expectedMessage); - }); - }); -}); From 55a9060ae3e0a065ebba2407246776ae40041a2d Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 10 Jun 2024 16:59:44 +0400 Subject: [PATCH 045/243] test(builtins): migrate tests to use updated error pattern --- .../unit/builtins/CommonGenerator.test.ts | 18 +++---- .../unit/builtins/RandIntGenerator.test.ts | 25 +++------ .../unit/builtins/coefficient.test.ts | 10 ++-- src/__tests__/unit/builtins/divide.test.ts | 17 +++--- src/__tests__/unit/builtins/exponent.test.ts | 30 +++++------ .../unit/builtins/export-csv-raw.test.ts | 18 ++++--- .../unit/builtins/export-csv.test.ts | 23 ++++---- .../unit/builtins/export-yaml.test.ts | 9 ++-- src/__tests__/unit/builtins/group-by.test.ts | 13 ++--- .../unit/builtins/interpolation.test.ts | 34 ++++++------ .../unit/builtins/mock-observations.test.ts | 14 ++--- src/__tests__/unit/builtins/multiply.test.ts | 11 ++-- src/__tests__/unit/builtins/regex.test.ts | 20 ++++--- .../unit/builtins/sci-embodied.test.ts | 25 ++++++--- src/__tests__/unit/builtins/sci.test.ts | 8 +-- src/__tests__/unit/builtins/shell.test.ts | 8 +-- src/__tests__/unit/builtins/subtract.test.ts | 9 ++-- src/__tests__/unit/builtins/sum.test.ts | 15 +++--- src/__tests__/unit/builtins/time-sync.test.ts | 52 ++++++++++++------- 19 files changed, 189 insertions(+), 170 deletions(-) diff --git a/src/__tests__/unit/builtins/CommonGenerator.test.ts b/src/__tests__/unit/builtins/CommonGenerator.test.ts index 881aa810e..d4d1bdac9 100644 --- a/src/__tests__/unit/builtins/CommonGenerator.test.ts +++ b/src/__tests__/unit/builtins/CommonGenerator.test.ts @@ -1,12 +1,12 @@ -import {KeyValuePair} from '../../../types/common'; - import {ERRORS} from '../../../util/errors'; import {CommonGenerator} from '../../../builtins/mock-observations/helpers/common-generator'; +import {STRINGS} from '../../../config'; -const {InputValidationError} = ERRORS; +const {GlobalConfigError} = ERRORS; +const {MISSING_GLOBAL_CONFIG} = STRINGS; -describe('lib/mock-observations/CommonGenerator: ', () => { +describe('builtins/mock-observations/CommonGenerator: ', () => { describe('initialize: ', () => { it('throws an error when config is not empty object.', async () => { const commonGenerator = CommonGenerator({}); @@ -16,18 +16,14 @@ describe('lib/mock-observations/CommonGenerator: ', () => { try { commonGenerator.next([]); } catch (error) { - expect(error).toEqual( - new InputValidationError( - 'CommonGenerator: Config must not be null or empty.' - ) - ); + expect(error).toEqual(new GlobalConfigError(MISSING_GLOBAL_CONFIG)); } }); }); describe('next(): ', () => { it('returns a result with valid data.', async () => { - const config: KeyValuePair = { + const config: Record = { key1: 'value1', key2: 'value2', }; @@ -35,7 +31,7 @@ describe('lib/mock-observations/CommonGenerator: ', () => { expect.assertions(1); - expect(commonGenerator.next([])).toStrictEqual({ + expect(commonGenerator.next([])).toEqual({ key1: 'value1', key2: 'value2', }); diff --git a/src/__tests__/unit/builtins/RandIntGenerator.test.ts b/src/__tests__/unit/builtins/RandIntGenerator.test.ts index c904f7350..1d2969695 100644 --- a/src/__tests__/unit/builtins/RandIntGenerator.test.ts +++ b/src/__tests__/unit/builtins/RandIntGenerator.test.ts @@ -4,20 +4,19 @@ import {ERRORS} from '../../../util/errors'; import {RandIntGenerator} from '../../../builtins/mock-observations/helpers/rand-int-generator'; -const {InputValidationError} = ERRORS; +import {STRINGS} from '../../../config'; -describe('lib/mock-observations/RandIntGenerator: ', () => { +const {GlobalConfigError} = ERRORS; +const {INVALID_NAME, MISSING_MIN_MAX, MISSING_GLOBAL_CONFIG} = STRINGS; + +describe('builtins/mock-observations/RandIntGenerator: ', () => { describe('initialize', () => { it('throws an error when the generator name is empty string.', async () => { expect.assertions(1); try { RandIntGenerator('', {}); } catch (error) { - expect(error).toEqual( - new InputValidationError( - 'RandIntGenerator: `name` is empty or all spaces.' - ) - ); + expect(error).toEqual(new GlobalConfigError(INVALID_NAME)); } }); @@ -26,11 +25,7 @@ describe('lib/mock-observations/RandIntGenerator: ', () => { try { RandIntGenerator('generator-name', {}); } catch (error) { - expect(error).toEqual( - new InputValidationError( - 'RandIntGenerator: Config must not be null or empty.' - ) - ); + expect(error).toEqual(new GlobalConfigError(MISSING_GLOBAL_CONFIG)); } }); @@ -42,11 +37,7 @@ describe('lib/mock-observations/RandIntGenerator: ', () => { try { RandIntGenerator('random', config); } catch (error) { - expect(error).toEqual( - new InputValidationError( - 'RandIntGenerator: Config is missing min or max.' - ) - ); + expect(error).toEqual(new GlobalConfigError(MISSING_MIN_MAX)); } }); }); diff --git a/src/__tests__/unit/builtins/coefficient.test.ts b/src/__tests__/unit/builtins/coefficient.test.ts index bf7455679..44595d774 100644 --- a/src/__tests__/unit/builtins/coefficient.test.ts +++ b/src/__tests__/unit/builtins/coefficient.test.ts @@ -2,7 +2,10 @@ import {Coefficient} from '../../../builtins/coefficient'; import {ERRORS} from '../../../util/errors'; -const {InputValidationError, ConfigNotFoundError} = ERRORS; +import {STRINGS} from '../../../config'; + +const {InputValidationError, GlobalConfigError} = ERRORS; +const {MISSING_GLOBAL_CONFIG} = STRINGS; describe('builtins/coefficient: ', () => { describe('Coefficient: ', () => { @@ -49,7 +52,6 @@ describe('builtins/coefficient: ', () => { it('throws an error when global config is not provided.', () => { const config = undefined; const coefficient = Coefficient(config!); - const expectedMessage = 'Global config is not provided.'; expect.assertions(1); @@ -62,7 +64,9 @@ describe('builtins/coefficient: ', () => { }, ]); } catch (error) { - expect(error).toStrictEqual(new ConfigNotFoundError(expectedMessage)); + expect(error).toStrictEqual( + new GlobalConfigError(MISSING_GLOBAL_CONFIG) + ); } }); diff --git a/src/__tests__/unit/builtins/divide.test.ts b/src/__tests__/unit/builtins/divide.test.ts index 8f56bb19d..9c794dae6 100644 --- a/src/__tests__/unit/builtins/divide.test.ts +++ b/src/__tests__/unit/builtins/divide.test.ts @@ -1,8 +1,10 @@ import {Divide} from '../../../builtins'; import {ERRORS} from '../../../util/errors'; +import {STRINGS} from '../../../config'; -const {InputValidationError, ConfigNotFoundError} = ERRORS; +const {InputValidationError, GlobalConfigError, MissingInputDataError} = ERRORS; +const {MISSING_GLOBAL_CONFIG, MISSING_INPUT_DATA} = STRINGS; describe('builtins/divide: ', () => { describe('Divide: ', () => { @@ -103,7 +105,6 @@ describe('builtins/divide: ', () => { }); it('throws an error on missing global config.', async () => { - const expectedMessage = 'Global config is not provided.'; const config = undefined; const divide = Divide(config!); @@ -117,7 +118,9 @@ describe('builtins/divide: ', () => { }, ]); } catch (error) { - expect(error).toStrictEqual(new ConfigNotFoundError(expectedMessage)); + expect(error).toStrictEqual( + new GlobalConfigError(MISSING_GLOBAL_CONFIG) + ); } }); @@ -148,8 +151,6 @@ describe('builtins/divide: ', () => { }); it('throws an error when `denominator` is string.', async () => { - const expectedMessage = '`10` is missing from the input.'; - const globalConfig = { numerator: 'vcpus-allocated', denominator: '10', @@ -168,7 +169,11 @@ describe('builtins/divide: ', () => { }, ]); } catch (error) { - expect(error).toStrictEqual(new InputValidationError(expectedMessage)); + expect(error).toStrictEqual( + new MissingInputDataError( + MISSING_INPUT_DATA(globalConfig.denominator) + ) + ); } }); }); diff --git a/src/__tests__/unit/builtins/exponent.test.ts b/src/__tests__/unit/builtins/exponent.test.ts index 54c69ffcb..6034fd3bd 100644 --- a/src/__tests__/unit/builtins/exponent.test.ts +++ b/src/__tests__/unit/builtins/exponent.test.ts @@ -1,10 +1,12 @@ import {Exponent} from '../../../builtins/exponent'; import {ERRORS} from '../../../util/errors'; +import {STRINGS} from '../../../config'; -const {InputValidationError} = ERRORS; +const {InputValidationError, MissingInputDataError} = ERRORS; +const {NOT_NUMERIC_VALUE, MISSING_INPUT_DATA} = STRINGS; -describe('lib/exponent: ', () => { +describe('builtins/exponent: ', () => { describe('Exponent: ', () => { const globalConfig = { 'input-parameter': 'energy/base', @@ -45,9 +47,6 @@ describe('lib/exponent: ', () => { }); it('throws an error on missing params in input.', async () => { - const expectedMessage = - 'Exponent: energy/base is missing from the input array.'; - expect.assertions(1); try { @@ -59,27 +58,26 @@ describe('lib/exponent: ', () => { ]); } catch (error) { expect(error).toStrictEqual( - new InputValidationError(expectedMessage) + new MissingInputDataError(MISSING_INPUT_DATA('energy/base')) ); } }); it('throws an error on input param value not numeric.', async () => { - const expectedMessage = 'Exponent: i-am-not-a-number is not numeric.'; - expect.assertions(1); + const input = [ + { + duration: 3600, + 'energy/base': 'i-am-not-a-number', + timestamp: '2021-01-01T00:00:00Z', + }, + ]; try { - await exponent.execute([ - { - duration: 3600, - 'energy/base': 'i-am-not-a-number', - timestamp: '2021-01-01T00:00:00Z', - }, - ]); + await exponent.execute(input); } catch (error) { expect(error).toStrictEqual( - new InputValidationError(expectedMessage) + new InputValidationError(NOT_NUMERIC_VALUE(input[0]['energy/base'])) ); } }); diff --git a/src/__tests__/unit/builtins/export-csv-raw.test.ts b/src/__tests__/unit/builtins/export-csv-raw.test.ts index 1ebf9e45e..a66a24ac2 100644 --- a/src/__tests__/unit/builtins/export-csv-raw.test.ts +++ b/src/__tests__/unit/builtins/export-csv-raw.test.ts @@ -4,9 +4,12 @@ import {jest} from '@jest/globals'; import {ExportCSVRaw} from '../../../builtins/export-csv-raw'; import {ERRORS} from '../../../util/errors'; +import {STRINGS} from '../../../config'; + import {tree, context, outputs} from '../../../__mocks__/builtins/export-csv'; -const {ExhaustError} = ERRORS; +const {ExhaustOutputArgError} = ERRORS; +const {WRITE_CSV_ERROR, OUTPUT_REQUIRED} = STRINGS; jest.mock('fs/promises', () => ({ __esModule: true, @@ -40,18 +43,17 @@ describe('builtins/export-csv-raw: ', () => { it('throws an error when the CSV file could not be created.', async () => { const outputPath = 'output#carbon'; + const expectedMessage = 'Could not write CSV file.'; expect.assertions(1); - jest - .spyOn(fs, 'writeFile') - .mockRejectedValue('Could not write CSV file.'); + jest.spyOn(fs, 'writeFile').mockRejectedValue(expectedMessage); await expect( exportCSVRaw.execute(tree, context, outputPath) ).rejects.toThrow( - new ExhaustError( - 'Failed to write CSV to output#carbon: Could not write CSV file.' + new ExhaustOutputArgError( + WRITE_CSV_ERROR(outputPath, expectedMessage) ) ); }); @@ -65,8 +67,8 @@ describe('builtins/export-csv-raw: ', () => { try { await exportCSVRaw.execute(tree, context, outputPath); } catch (error) { - expect(error).toBeInstanceOf(ExhaustError); - expect(error).toEqual(new ExhaustError('Output path is required.')); + expect(error).toBeInstanceOf(ExhaustOutputArgError); + expect(error).toEqual(new ExhaustOutputArgError(OUTPUT_REQUIRED)); } }); }); diff --git a/src/__tests__/unit/builtins/export-csv.test.ts b/src/__tests__/unit/builtins/export-csv.test.ts index 94414a970..09fd4e737 100644 --- a/src/__tests__/unit/builtins/export-csv.test.ts +++ b/src/__tests__/unit/builtins/export-csv.test.ts @@ -5,6 +5,8 @@ import {jest} from '@jest/globals'; import {ExportCSV} from '../../../builtins/export-csv'; import {ERRORS} from '../../../util/errors'; +import {STRINGS} from '../../../config'; + import { tree, context, @@ -13,7 +15,8 @@ import { aggregation, } from '../../../__mocks__/builtins/export-csv'; -const {ExhaustError} = ERRORS; +const {ExhaustOutputArgError} = ERRORS; +const {OUTPUT_REQUIRED, CSV_EXPORT} = STRINGS; jest.mock('fs/promises', () => ({ writeFile: jest.fn<() => Promise>().mockResolvedValue(), @@ -197,8 +200,8 @@ describe('builtins/export-csv: ', () => { try { await exportCSV.execute(tree, context, outputPath); } catch (error) { - expect(error).toBeInstanceOf(ExhaustError); - expect(error).toEqual(new ExhaustError('Output path is required.')); + expect(error).toBeInstanceOf(ExhaustOutputArgError); + expect(error).toEqual(new ExhaustOutputArgError(OUTPUT_REQUIRED)); } }); @@ -210,10 +213,8 @@ describe('builtins/export-csv: ', () => { try { await exportCSV.execute(tree, context, outputPath); } catch (error) { - expect(error).toBeInstanceOf(ExhaustError); - expect(error).toEqual( - new ExhaustError('Output path should contain `#`.') - ); + expect(error).toBeInstanceOf(ExhaustOutputArgError); + expect(error).toEqual(new ExhaustOutputArgError(CSV_EXPORT)); } }); @@ -225,12 +226,8 @@ describe('builtins/export-csv: ', () => { try { await exportCSV.execute(tree, context, outputPath); } catch (error) { - expect(error).toBeInstanceOf(ExhaustError); - expect(error).toEqual( - new ExhaustError( - 'CSV export criteria is not found in output path. Please append it after --output #.' - ) - ); + expect(error).toBeInstanceOf(ExhaustOutputArgError); + expect(error).toEqual(new ExhaustOutputArgError(CSV_EXPORT)); } }); }); diff --git a/src/__tests__/unit/builtins/export-yaml.test.ts b/src/__tests__/unit/builtins/export-yaml.test.ts index a2157e4ef..f17b9154e 100644 --- a/src/__tests__/unit/builtins/export-yaml.test.ts +++ b/src/__tests__/unit/builtins/export-yaml.test.ts @@ -2,13 +2,16 @@ import {ExportYaml} from '../../../builtins/export-yaml'; import {ERRORS} from '../../../util/errors'; import {saveYamlFileAs} from '../../../util/yaml'; +import {STRINGS} from '../../../config'; + import {tree, context} from '../../../__mocks__/builtins/export-csv'; jest.mock('../../../util/yaml', () => ({ saveYamlFileAs: jest.fn(), })); -const {ExhaustError} = ERRORS; +const {ExhaustOutputArgError} = ERRORS; +const {OUTPUT_REQUIRED} = STRINGS; describe('builtins/export-yaml: ', () => { describe('ExportYaml: ', () => { @@ -38,8 +41,8 @@ describe('builtins/export-yaml: ', () => { try { await exportYaml.execute({}, context, ''); } catch (error) { - expect(error).toBeInstanceOf(ExhaustError); - expect(error).toEqual(new ExhaustError('Output path is required.')); + expect(error).toBeInstanceOf(ExhaustOutputArgError); + expect(error).toEqual(new ExhaustOutputArgError(OUTPUT_REQUIRED)); } }); }); diff --git a/src/__tests__/unit/builtins/group-by.test.ts b/src/__tests__/unit/builtins/group-by.test.ts index 9c99ee700..b472125d1 100644 --- a/src/__tests__/unit/builtins/group-by.test.ts +++ b/src/__tests__/unit/builtins/group-by.test.ts @@ -1,7 +1,10 @@ import {GroupBy} from '../../../builtins/group-by'; import {ERRORS} from '../../../util/errors'; -const {InvalidGroupingError, InputValidationError} = ERRORS; +import {STRINGS} from '../../../config'; + +const {InvalidGroupingError, InputValidationError, GlobalConfigError} = ERRORS; +const {MISSING_GLOBAL_CONFIG, INVALID_GROUP_BY} = STRINGS; describe('builtins/group-by: ', () => { describe('GroupBy: ', () => { @@ -92,10 +95,8 @@ describe('builtins/group-by: ', () => { try { plugin.execute(inputs, config!); } catch (error) { - expect(error).toBeInstanceOf(InputValidationError); - expect(error).toEqual( - new InputValidationError('Config is not provided.') - ); + expect(error).toBeInstanceOf(GlobalConfigError); + expect(error).toEqual(new GlobalConfigError(MISSING_GLOBAL_CONFIG)); } }); @@ -166,7 +167,7 @@ describe('builtins/group-by: ', () => { } catch (error) { expect(error).toBeInstanceOf(InvalidGroupingError); expect(error).toEqual( - new InvalidGroupingError('Invalid group unknown.') + new InvalidGroupingError(INVALID_GROUP_BY(config.group[2])) ); } }); diff --git a/src/__tests__/unit/builtins/interpolation.test.ts b/src/__tests__/unit/builtins/interpolation.test.ts index c4bba939f..54dd2dab7 100644 --- a/src/__tests__/unit/builtins/interpolation.test.ts +++ b/src/__tests__/unit/builtins/interpolation.test.ts @@ -1,8 +1,16 @@ import {Interpolation} from '../../../builtins'; -import {Method} from '../../../builtins/interpolation/types'; import {ERRORS} from '../../../util/errors'; -const {InputValidationError, ConfigNotFoundError} = ERRORS; +import {Method} from '../../../builtins/interpolation/types'; +import {STRINGS} from '../../../config'; + +const {InputValidationError, GlobalConfigError} = ERRORS; +const { + MISSING_GLOBAL_CONFIG, + WITHIN_THE_RANGE, + ARRAY_LENGTH_NON_EMPTY, + X_Y_EQUAL, +} = STRINGS; describe('builtins/interpolation: ', () => { describe('Interpolation: ', () => { @@ -144,10 +152,8 @@ describe('builtins/interpolation: ', () => { try { plugin.execute(inputs); } catch (error) { - expect(error).toBeInstanceOf(ConfigNotFoundError); - expect(error).toEqual( - new ConfigNotFoundError('Global config is not provided.') - ); + expect(error).toBeInstanceOf(GlobalConfigError); + expect(error).toEqual(new GlobalConfigError(MISSING_GLOBAL_CONFIG)); } }); @@ -163,11 +169,7 @@ describe('builtins/interpolation: ', () => { plugin.execute(inputs); } catch (error) { expect(error).toBeInstanceOf(InputValidationError); - expect(error).toEqual( - new InputValidationError( - 'The length of `x` and `y` should be equal' - ) - ); + expect(error).toEqual(new InputValidationError(X_Y_EQUAL)); } }); @@ -184,11 +186,7 @@ describe('builtins/interpolation: ', () => { plugin.execute(inputs); } catch (error) { expect(error).toBeInstanceOf(InputValidationError); - expect(error).toEqual( - new InputValidationError( - 'The target x value must be within the range of the given x values' - ) - ); + expect(error).toEqual(new InputValidationError(WITHIN_THE_RANGE)); } }); it('throws an error when the the length of the input arrays is <2', () => { @@ -213,9 +211,7 @@ describe('builtins/interpolation: ', () => { } catch (error) { expect(error).toBeInstanceOf(InputValidationError); expect(error).toEqual( - new InputValidationError( - 'the length of the input arrays must be greater than 1' - ) + new InputValidationError(ARRAY_LENGTH_NON_EMPTY) ); } }); diff --git a/src/__tests__/unit/builtins/mock-observations.test.ts b/src/__tests__/unit/builtins/mock-observations.test.ts index 37d3a04b4..3e2ddab5b 100644 --- a/src/__tests__/unit/builtins/mock-observations.test.ts +++ b/src/__tests__/unit/builtins/mock-observations.test.ts @@ -1,10 +1,12 @@ import {MockObservations} from '../../../builtins/mock-observations'; import {ERRORS} from '../../../util/errors'; +import {STRINGS} from '../../../config'; -const {InputValidationError} = ERRORS; +const {InputValidationError, GlobalConfigError} = ERRORS; +const {INVALID_MIN_MAX} = STRINGS; -describe('lib/mock-observations: ', () => { +describe('builtins/mock-observations: ', () => { describe('init: ', () => { it('successfully initalized.', () => { const mockObservations = MockObservations({ @@ -88,8 +90,6 @@ describe('lib/mock-observations: ', () => { }); it('throws an error when the `min` is greater then `max` of `randint` config.', async () => { - const errorMessage = - 'RandIntGenerator: Min value should not be greater than or equal to max value of cpu/utilization.'; const config = { 'timestamp-from': '2023-07-06T00:00', 'timestamp-to': '2023-07-06T00:01', @@ -112,8 +112,10 @@ describe('lib/mock-observations: ', () => { try { await mockObservations.execute([]); } catch (error) { - expect(error).toBeInstanceOf(InputValidationError); - expect(error).toEqual(new InputValidationError(errorMessage)); + expect(error).toBeInstanceOf(GlobalConfigError); + expect(error).toEqual( + new GlobalConfigError(INVALID_MIN_MAX('cpu/utilization')) + ); } }); diff --git a/src/__tests__/unit/builtins/multiply.test.ts b/src/__tests__/unit/builtins/multiply.test.ts index a4451179d..7efceb325 100644 --- a/src/__tests__/unit/builtins/multiply.test.ts +++ b/src/__tests__/unit/builtins/multiply.test.ts @@ -1,10 +1,12 @@ import {Multiply} from '../../../builtins/multiply'; import {ERRORS} from '../../../util/errors'; +import {STRINGS} from '../../../config'; -const {InputValidationError} = ERRORS; +const {MissingInputDataError} = ERRORS; +const {MISSING_INPUT_DATA} = STRINGS; -describe('lib/multiply: ', () => { +describe('builtins/multiply: ', () => { describe('Multiply: ', () => { const globalConfig = { 'input-parameters': ['cpu/energy', 'network/energy', 'memory/energy'], @@ -48,9 +50,6 @@ describe('lib/multiply: ', () => { }); it('throws an error on missing params in input.', async () => { - const expectedMessage = - 'Multiply: cpu/energy is missing from the input array.'; - expect.assertions(1); try { @@ -62,7 +61,7 @@ describe('lib/multiply: ', () => { ]); } catch (error) { expect(error).toStrictEqual( - new InputValidationError(expectedMessage) + new MissingInputDataError(MISSING_INPUT_DATA('cpu/energy')) ); } }); diff --git a/src/__tests__/unit/builtins/regex.test.ts b/src/__tests__/unit/builtins/regex.test.ts index 19ccdf89d..462faad8f 100644 --- a/src/__tests__/unit/builtins/regex.test.ts +++ b/src/__tests__/unit/builtins/regex.test.ts @@ -1,10 +1,12 @@ import {Regex} from '../../../builtins/regex'; import {ERRORS} from '../../../util/errors'; +import {STRINGS} from '../../../config'; -const {InputValidationError, ConfigValidationError} = ERRORS; +const {GlobalConfigError, MissingInputDataError, RegexMismatchError} = ERRORS; +const {MISSING_GLOBAL_CONFIG, MISSING_INPUT_DATA, REGEX_MISMATCH} = STRINGS; -describe('lib/regex: ', () => { +describe('builtins/regex: ', () => { describe('Regex: ', () => { const globalConfig = { parameter: 'physical-processor', @@ -81,7 +83,6 @@ describe('lib/regex: ', () => { it('throws an error when `parameter` does not match to `match`.', async () => { const physicalProcessor = 'Intel® Xeon® Platinum 8272CL,Intel® Xeon® 8171M 2.1 GHz,Intel® Xeon® E5-2673 v4 2.3 GHz,Intel® Xeon® E5-2673 v3 2.4 GHz'; - const expectedMessage = `Regex: \`${physicalProcessor}\` does not match the /^(^:)+/ regex expression.`; const globalConfig = { parameter: 'physical-processor', @@ -102,14 +103,14 @@ describe('lib/regex: ', () => { ]); } catch (error) { expect(error).toStrictEqual( - new InputValidationError(expectedMessage) + new RegexMismatchError( + REGEX_MISMATCH(physicalProcessor, '/^(^:)+/') + ) ); } }); it('throws an error on missing global config.', async () => { - const expectedMessage = 'Regex: Configuration data is missing.'; - const config = undefined; const regex = Regex(config!); @@ -124,15 +125,12 @@ describe('lib/regex: ', () => { ]); } catch (error) { expect(error).toStrictEqual( - new ConfigValidationError(expectedMessage) + new GlobalConfigError(MISSING_GLOBAL_CONFIG) ); } }); it('throws an error on missing params in input.', async () => { - const expectedMessage = - 'Regex: `physical-processor` is missing from the input.'; - expect.assertions(1); try { @@ -144,7 +142,7 @@ describe('lib/regex: ', () => { ]); } catch (error) { expect(error).toStrictEqual( - new InputValidationError(expectedMessage) + new MissingInputDataError(MISSING_INPUT_DATA('physical-processor')) ); } }); diff --git a/src/__tests__/unit/builtins/sci-embodied.test.ts b/src/__tests__/unit/builtins/sci-embodied.test.ts index 75f9f12fd..4b9f88deb 100644 --- a/src/__tests__/unit/builtins/sci-embodied.test.ts +++ b/src/__tests__/unit/builtins/sci-embodied.test.ts @@ -1,9 +1,12 @@ import {SciEmbodied} from '../../../builtins/sci-embodied'; import {ERRORS} from '../../../util/errors'; +import {STRINGS} from '../../../config'; + const {InputValidationError} = ERRORS; +const {SCI_EMBODIED_ERROR} = STRINGS; -describe('lib/sci-embodied:', () => { +describe('builtins/sci-embodied:', () => { describe('SciEmbodied: ', () => { const sciEmbodied = SciEmbodied(); @@ -191,8 +194,6 @@ describe('lib/sci-embodied:', () => { }); it('throws an error when `device/emissions-embodied` is string.', async () => { - const errorMessage = - '"device/emissions-embodied" parameter is not a valid number in input. please provide it as `gco2e`. Error code: invalid_union.'; const inputs = [ { timestamp: '2021-01-01T00:00:00Z', @@ -216,7 +217,13 @@ describe('lib/sci-embodied:', () => { try { await sciEmbodied.execute(inputs); } catch (error) { - expect(error).toStrictEqual(new InputValidationError(errorMessage)); + expect(error).toStrictEqual( + new InputValidationError( + `"device/emissions-embodied" parameter is ${SCI_EMBODIED_ERROR( + 'gco2e' + )}. Error code: invalid_union.` + ) + ); expect(error).toBeInstanceOf(InputValidationError); } }); @@ -268,8 +275,6 @@ describe('lib/sci-embodied:', () => { }); it('throws an exception on invalid values.', async () => { - const errorMessage = - '"device/emissions-embodied" parameter is not a valid number in input. please provide it as `gco2e`. Error code: invalid_union.'; const inputs = [ { timestamp: '2021-01-01T00:00:00Z', @@ -285,8 +290,14 @@ describe('lib/sci-embodied:', () => { try { await sciEmbodied.execute(inputs); } catch (error) { - expect(error).toStrictEqual(new InputValidationError(errorMessage)); expect(error).toBeInstanceOf(InputValidationError); + expect(error).toStrictEqual( + new InputValidationError( + `"device/emissions-embodied" parameter is ${SCI_EMBODIED_ERROR( + 'gco2e' + )}. Error code: invalid_union.` + ) + ); } }); }); diff --git a/src/__tests__/unit/builtins/sci.test.ts b/src/__tests__/unit/builtins/sci.test.ts index 2b6261cda..a9401a5a8 100644 --- a/src/__tests__/unit/builtins/sci.test.ts +++ b/src/__tests__/unit/builtins/sci.test.ts @@ -2,9 +2,9 @@ import {Sci} from '../../../builtins/sci'; import {ERRORS} from '../../../util/errors'; -const {InputValidationError} = ERRORS; +const {MissingInputDataError} = ERRORS; -describe('lib/sci:', () => { +describe('builtins/sci:', () => { describe('Sci: ', () => { const sci = Sci({'functional-unit': 'users'}); @@ -113,7 +113,7 @@ describe('lib/sci:', () => { try { await sci.execute(inputs); } catch (error) { - expect(error).toBeInstanceOf(InputValidationError); + expect(error).toBeInstanceOf(MissingInputDataError); } }); @@ -136,7 +136,7 @@ describe('lib/sci:', () => { try { await sci.execute(inputs); } catch (error) { - expect(error).toBeInstanceOf(InputValidationError); + expect(error).toBeInstanceOf(MissingInputDataError); } }); }); diff --git a/src/__tests__/unit/builtins/shell.test.ts b/src/__tests__/unit/builtins/shell.test.ts index 7f1ffd58c..306dada3c 100644 --- a/src/__tests__/unit/builtins/shell.test.ts +++ b/src/__tests__/unit/builtins/shell.test.ts @@ -5,12 +5,12 @@ import {Shell} from '../../../builtins/shell'; import {ERRORS} from '../../../util/errors'; -const {InputValidationError} = ERRORS; +const {InputValidationError, ProcessExecutionError} = ERRORS; jest.mock('child_process'); jest.mock('js-yaml'); -describe('lib/shell', () => { +describe('builtins/shell', () => { describe('Shell', () => { const shell = Shell({}); @@ -87,9 +87,9 @@ describe('lib/shell', () => { try { await shell.execute(inputs); } catch (error) { - expect(error).toBeInstanceOf(InputValidationError); + expect(error).toBeInstanceOf(ProcessExecutionError); expect(error).toStrictEqual( - new InputValidationError('Could not run the command') + new ProcessExecutionError('Could not run the command') ); } }); diff --git a/src/__tests__/unit/builtins/subtract.test.ts b/src/__tests__/unit/builtins/subtract.test.ts index 7ac717142..2179adcd4 100644 --- a/src/__tests__/unit/builtins/subtract.test.ts +++ b/src/__tests__/unit/builtins/subtract.test.ts @@ -1,10 +1,12 @@ import {Subtract} from '../../../builtins/subtract'; import {ERRORS} from '../../../util/errors'; +import {STRINGS} from '../../../config'; const {InputValidationError} = ERRORS; +const {MISSING_INPUT_DATA} = STRINGS; -describe('lib/subtract: ', () => { +describe('builtins/subtract: ', () => { describe('Subtract: ', () => { const globalConfig = { 'input-parameters': ['cpu/energy', 'network/energy', 'memory/energy'], @@ -48,9 +50,6 @@ describe('lib/subtract: ', () => { }); it('throws an error on missing params in input.', async () => { - const expectedMessage = - 'Subtract: cpu/energy is missing from the input array.'; - expect.assertions(1); try { @@ -62,7 +61,7 @@ describe('lib/subtract: ', () => { ]); } catch (error) { expect(error).toStrictEqual( - new InputValidationError(expectedMessage) + new InputValidationError(MISSING_INPUT_DATA('cpu/energy')) ); } }); diff --git a/src/__tests__/unit/builtins/sum.test.ts b/src/__tests__/unit/builtins/sum.test.ts index 734ea8db2..e7b515f26 100644 --- a/src/__tests__/unit/builtins/sum.test.ts +++ b/src/__tests__/unit/builtins/sum.test.ts @@ -1,10 +1,12 @@ import {Sum} from '../../../builtins/sum'; import {ERRORS} from '../../../util/errors'; +import {STRINGS} from '../../../config'; -const {InputValidationError, ConfigNotFoundError} = ERRORS; +const {GlobalConfigError, MissingInputDataError} = ERRORS; +const {MISSING_GLOBAL_CONFIG, MISSING_INPUT_DATA} = STRINGS; -describe('lib/sum: ', () => { +describe('builtins/sum: ', () => { describe('Sum: ', () => { const globalConfig = { 'input-parameters': ['cpu/energy', 'network/energy', 'memory/energy'], @@ -48,7 +50,6 @@ describe('lib/sum: ', () => { }); it('throws an error when global config is not provided.', () => { - const expectedMessage = 'Global config is not provided.'; const config = undefined; const sum = Sum(config!); @@ -65,13 +66,13 @@ describe('lib/sum: ', () => { }, ]); } catch (error) { - expect(error).toStrictEqual(new ConfigNotFoundError(expectedMessage)); + expect(error).toStrictEqual( + new GlobalConfigError(MISSING_GLOBAL_CONFIG) + ); } }); it('throws an error on missing params in input.', () => { - const expectedMessage = 'cpu/energy is missing from the input array.'; - expect.assertions(1); try { @@ -83,7 +84,7 @@ describe('lib/sum: ', () => { ]); } catch (error) { expect(error).toStrictEqual( - new InputValidationError(expectedMessage) + new MissingInputDataError(MISSING_INPUT_DATA('cpu/energy')) ); } }); diff --git a/src/__tests__/unit/builtins/time-sync.test.ts b/src/__tests__/unit/builtins/time-sync.test.ts index 4839ad6ca..ac627fe02 100644 --- a/src/__tests__/unit/builtins/time-sync.test.ts +++ b/src/__tests__/unit/builtins/time-sync.test.ts @@ -1,11 +1,26 @@ +import {Settings, DateTime} from 'luxon'; + import {TimeSync} from '../../../builtins/time-sync'; + import {ERRORS} from '../../../util/errors'; -import {Settings, DateTime} from 'luxon'; + import {STRINGS} from '../../../config'; -Settings.defaultZone = 'utc'; -const {InputValidationError} = ERRORS; -const {INVALID_OBSERVATION_OVERLAP, INVALID_TIME_NORMALIZATION} = STRINGS; +Settings.defaultZone = 'utc'; +const { + InputValidationError, + InvalidPaddingError, + InvalidDateInInputError, + InvalidInputError, + GlobalConfigError, +} = ERRORS; + +const { + INVALID_OBSERVATION_OVERLAP, + INVALID_TIME_NORMALIZATION, + AVOIDING_PADDING_BY_EDGES, + INVALID_DATE_TYPE, +} = STRINGS; jest.mock('luxon', () => { const originalModule = jest.requireActual('luxon'); @@ -196,7 +211,7 @@ describe('execute(): ', () => { ]); } catch (error) { expect(error).toStrictEqual( - new InputValidationError(INVALID_TIME_NORMALIZATION) + new GlobalConfigError(INVALID_TIME_NORMALIZATION) ); } }); @@ -228,7 +243,7 @@ describe('execute(): ', () => { ]); } catch (error) { expect(error).toStrictEqual( - new InputValidationError(INVALID_OBSERVATION_OVERLAP) + new InvalidInputError(INVALID_OBSERVATION_OVERLAP) ); } }); @@ -258,7 +273,7 @@ describe('execute(): ', () => { ]); } catch (error) { expect(error).toStrictEqual( - new InputValidationError(INVALID_OBSERVATION_OVERLAP) + new InvalidInputError(INVALID_OBSERVATION_OVERLAP) ); } }); @@ -335,22 +350,23 @@ describe('execute(): ', () => { interval: 10, 'allow-padding': true, }; + const data = [ + { + timestamp: 45, + duration: 10, + 'cpu/utilization': 10, + }, + ]; const timeModel = TimeSync(basicConfig); expect.assertions(2); try { - await timeModel.execute([ - { - timestamp: 45, - duration: 10, - 'cpu/utilization': 10, - }, - ]); + await timeModel.execute(data); } catch (error) { - expect(error).toBeInstanceOf(InputValidationError); + expect(error).toBeInstanceOf(InvalidDateInInputError); expect(error).toStrictEqual( - new InputValidationError('Unexpected date datatype: number: 45') + new InvalidDateInInputError(INVALID_DATE_TYPE(data[0].timestamp)) ); } }); @@ -675,7 +691,7 @@ describe('execute(): ', () => { ]); } catch (error) { expect(error).toStrictEqual( - new InputValidationError('Avoiding padding at start') + new InvalidPaddingError(AVOIDING_PADDING_BY_EDGES(true, false)) ); } }); @@ -735,7 +751,7 @@ describe('execute(): ', () => { ]); } catch (error) { expect(error).toStrictEqual( - new InputValidationError('Avoiding padding at start and end') + new InvalidPaddingError(AVOIDING_PADDING_BY_EDGES(true, true)) ); } }); From 35b7657a032f9c0eb32df1492080488669a5e6e5 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 10 Jun 2024 17:00:12 +0400 Subject: [PATCH 046/243] test(builtins): migrate errors, add cases to csv lookup --- .../unit/builtins/csv-lookup.test.ts | 62 ++++++++++++++----- 1 file changed, 48 insertions(+), 14 deletions(-) diff --git a/src/__tests__/unit/builtins/csv-lookup.test.ts b/src/__tests__/unit/builtins/csv-lookup.test.ts index 4e16fd9b6..8fbdee401 100644 --- a/src/__tests__/unit/builtins/csv-lookup.test.ts +++ b/src/__tests__/unit/builtins/csv-lookup.test.ts @@ -5,9 +5,19 @@ import AxiosMockAdapter from 'axios-mock-adapter'; import {CSVLookup} from '../../../builtins'; +import {STRINGS} from '../../../config'; + import {ERRORS} from '../../../util/errors'; -const {FileNotFoundError, InputValidationError, ConfigNotFoundError} = ERRORS; +const { + GlobalConfigError, + ReadFileError, + FetchingFileError, + QueryDataNotFoundError, + MissingCSVColumnError, + CSVParseError, +} = ERRORS; +const {MISSING_GLOBAL_CONFIG, MISSING_CSV_COLUMN, NO_QUERY_DATA} = STRINGS; describe('builtins/CSVLookup: ', () => { const mock = new AxiosMockAdapter(axios); @@ -131,7 +141,7 @@ describe('builtins/CSVLookup: ', () => { await csvLookup.execute(input); } catch (error) { if (error instanceof Error) { - expect(error).toBeInstanceOf(FileNotFoundError); + expect(error).toBeInstanceOf(ReadFileError); } } }); @@ -160,7 +170,7 @@ describe('builtins/CSVLookup: ', () => { await csvLookup.execute(input); } catch (error) { if (error instanceof Error) { - expect(error).toBeInstanceOf(FileNotFoundError); + expect(error).toBeInstanceOf(ReadFileError); } } }); @@ -192,7 +202,7 @@ describe('builtins/CSVLookup: ', () => { await csvLookup.execute(input); } catch (error) { if (error instanceof Error) { - expect(error).toBeInstanceOf(FileNotFoundError); + expect(error).toBeInstanceOf(FetchingFileError); } } }); @@ -340,11 +350,8 @@ describe('builtins/CSVLookup: ', () => { await csvLookup.execute(input); } catch (error) { if (error instanceof Error) { - expect(error).toBeInstanceOf(InputValidationError); - expect(error.message).toEqual( - `Error happened while parsing given CSV file: ./file.csv -InputValidationError: One or more of the given query parameters are not found in the target CSV file column headers.` - ); + expect(error).toBeInstanceOf(QueryDataNotFoundError); + expect(error.message).toEqual(NO_QUERY_DATA); } } }); @@ -368,8 +375,8 @@ InputValidationError: One or more of the given query parameters are not found in await csvLookup.execute(input); } catch (error) { if (error instanceof Error) { - expect(error).toBeInstanceOf(ConfigNotFoundError); - expect(error.message).toEqual('Global config is not provided.'); + expect(error).toBeInstanceOf(GlobalConfigError); + expect(error.message).toEqual(MISSING_GLOBAL_CONFIG); } } }); @@ -400,10 +407,9 @@ InputValidationError: One or more of the given query parameters are not found in await csvLookup.execute(input); } catch (error) { if (error instanceof Error) { - expect(error).toBeInstanceOf(InputValidationError); + expect(error).toBeInstanceOf(MissingCSVColumnError); expect(error.message).toEqual( - `Error happened while parsing given CSV file: ./file.csv -InputValidationError: There is no column with name: mock.` + MISSING_CSV_COLUMN(globalConfig.output) ); } } @@ -477,5 +483,33 @@ InputValidationError: There is no column with name: mock.` expect(result).toStrictEqual(expectedResult); }); }); + + it('rejects with CSV parse error', async () => { + process.env.csv = 'fail'; + expect.assertions(1); + const globalConfig = { + filepath: './fail-csv-reader.csv', + query: { + 'cpu-cores-available': 'cpu/available', + 'cpu-cores-utilized': 'cpu/utilized', + 'cpu-manufacturer': 'cpu/manufacturer', + }, + output: [['gpu-count']], + }; + const csvLookup = CSVLookup(globalConfig); + + try { + await csvLookup.execute([ + { + timestamp: '2024-03-01', + 'cpu/available': 16, + 'cpu/utilized': 16, + 'cpu/manufacturer': 'AWS', + }, + ]); + } catch (error) { + expect(error).toBeInstanceOf(CSVParseError); + } + }); }); }); From efa095ef9765fafaeb2089fa36a021d2516046ed Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 10 Jun 2024 17:00:26 +0400 Subject: [PATCH 047/243] test(mocks): add invalid csv case to fs --- src/__mocks__/fs/index.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/__mocks__/fs/index.ts b/src/__mocks__/fs/index.ts index da8438c8b..ec83ff675 100644 --- a/src/__mocks__/fs/index.ts +++ b/src/__mocks__/fs/index.ts @@ -28,6 +28,13 @@ export const readFile = async (filePath: string) => { throw new Error('file not found'); } + if (filePath.includes('fail-csv-reader.csv')) { + return ` +cpu-cores-available,≈ç≈¬˚∆∑∂´®øˆ´cpu-cores-utilized, ---- cpu-manufacturer,cpu-model-name,cpu-tdp,gpu-count,gpu-model-name,Hardware Information on AWS Documentation & Comments,instance-class,instance-storage,memory-available,platform-memory,release-date,storage-drives +16,8,AWS,AWS Graviton +16,16,AWS,AWS Graviton,150.00,N/A,N/A,AWS Graviton (ARM),a1.4xlarge,EBS-Only,32,32,November 2018,`; + } + /** * Used for csv lookup plugin. */ From 10cf52abd65c2ad22d417bcb2c58d6eb54a1ea3c Mon Sep 17 00:00:00 2001 From: manushak Date: Mon, 10 Jun 2024 19:05:55 +0400 Subject: [PATCH 048/243] fix(src): move template manifest to the top level of the scr folder --- src/{env/template.yml => env-template.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/{env/template.yml => env-template.yml} (100%) diff --git a/src/env/template.yml b/src/env-template.yml similarity index 100% rename from src/env/template.yml rename to src/env-template.yml From caf68b1a610b0402fe7ff060407fc6704ebfca87 Mon Sep 17 00:00:00 2001 From: manushak Date: Mon, 10 Jun 2024 19:08:52 +0400 Subject: [PATCH 049/243] feat(config): add log strings --- src/config/strings.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/config/strings.ts b/src/config/strings.ts index d0693c120..e0fd5ee42 100644 --- a/src/config/strings.ts +++ b/src/config/strings.ts @@ -57,4 +57,6 @@ Note that for the '--output' option you also need to define the output type in y INVALID_TARGET: 'Target is invalid.', INVALID_SOURCE: 'Source is invalid.', MANIFEST_NOT_FOUND: 'Manifest file not found.', + INITIALIZING_PACKAGE_JSON: 'Initializing package.json.', + INSTALLING_NPM_PACKAGES: 'Installing npm packages...', }; From a28f05b20d192000a544bd2a8ded30a81bf4c390 Mon Sep 17 00:00:00 2001 From: manushak Date: Mon, 10 Jun 2024 19:19:20 +0400 Subject: [PATCH 050/243] feat(types): add env types --- src/types/if-env.ts | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 src/types/if-env.ts diff --git a/src/types/if-env.ts b/src/types/if-env.ts new file mode 100644 index 000000000..b8a63f7ef --- /dev/null +++ b/src/types/if-env.ts @@ -0,0 +1,7 @@ +export type EnvironmentOptions = { + folderPath: string; + install: boolean; + dependencies: {[path: string]: string}; +}; + +export type PathWithVersion = {[path: string]: string}; From ec3ce36f7a9913906ae943830016fe2581e9cb91 Mon Sep 17 00:00:00 2001 From: manushak Date: Mon, 10 Jun 2024 19:21:18 +0400 Subject: [PATCH 051/243] feat(util): move some env functions into helpers.ts --- src/util/helpers.ts | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/src/util/helpers.ts b/src/util/helpers.ts index 99291f81d..86bcfab96 100644 --- a/src/util/helpers.ts +++ b/src/util/helpers.ts @@ -1,4 +1,5 @@ import * as fs from 'fs/promises'; +import * as path from 'path'; import {createInterface} from 'node:readline/promises'; import {exec} from 'node:child_process'; @@ -10,7 +11,8 @@ import {STRINGS} from '../config'; import {Difference} from '../types/lib/compare'; -const {ISSUE_TEMPLATE} = STRINGS; +const {ISSUE_TEMPLATE, INITIALIZING_PACKAGE_JSON, INSTALLING_NPM_PACKAGES} = + STRINGS; /** * Formats given error according to class instance, scope and message. @@ -196,3 +198,35 @@ export const isFileExists = async (filePath: string) => { return false; } }; + +/** + * Checks if the package.json is exists, if not, inisializes it. + */ +export const initPackageJsonIfNotExists = async (folderPath: string) => { + const packageJsonPath = path.resolve(folderPath, 'package.json'); + const isPackageJsonExists = await isFileExists(packageJsonPath); + + if (!isPackageJsonExists) { + logger.info(INITIALIZING_PACKAGE_JSON); + await execPromise('npm init -y', {cwd: folderPath}); + } + + return packageJsonPath; +}; + +/** + * Installs packages from the specified dependencies in the specified folder. + */ +export const installDependencies = async ( + folderPath: string, + dependencies: {[path: string]: string} +) => { + const packages = Object.entries(dependencies).map( + ([dependency, version]) => `${dependency}@${version.replace('^', '')}` + ); + + logger.info(INSTALLING_NPM_PACKAGES); + await execPromise(`npm install ${packages.join(' ')}`, { + cwd: folderPath, + }); +}; From 1f0700e326cd1f50df0c45dba544a1a8963acc41 Mon Sep 17 00:00:00 2001 From: manushak Date: Mon, 10 Jun 2024 19:22:56 +0400 Subject: [PATCH 052/243] fix(src): move env to the src folder --- src/{env => }/env.ts | 78 ++++++++++++-------------------------------- 1 file changed, 21 insertions(+), 57 deletions(-) rename src/{env => }/env.ts (69%) diff --git a/src/env/env.ts b/src/env.ts similarity index 69% rename from src/env/env.ts rename to src/env.ts index 0964d095d..94821d466 100644 --- a/src/env/env.ts +++ b/src/env.ts @@ -3,15 +3,17 @@ import * as fs from 'fs/promises'; import * as path from 'path'; -import {execPromise, isFileExists} from '../util/helpers'; -import {parseIfEnvArgs} from '../util/args'; -import {logger} from '../util/logger'; +import {installDependencies, initPackageJsonIfNotExists} from './util/helpers'; +import {parseIfEnvArgs} from './util/args'; +import {logger} from './util/logger'; -import {load} from '../lib/load'; +import {load} from './lib/load'; -import {CONFIG} from '../config'; +import {CONFIG} from './config'; -const packageJson = require('../../package.json'); +import {EnvironmentOptions, PathWithVersion} from './types/if-env'; + +const packageJson = require('../package.json'); const {IF_ENV} = CONFIG; const { @@ -23,12 +25,6 @@ const { const FOLDER_NAME = 'if-environment'; -type EnvironmentOptions = { - folderPath: string; - install: boolean; - dependencies: {[path: string]: string}; -}; - const IfEnv = async () => { const commandArgs = await parseIfEnvArgs(); const options: EnvironmentOptions = { @@ -87,13 +83,10 @@ const getOptionsFromArgs = async (commandArgs: { /** * Gets depencecies with versions. */ -const extractPathsWithVersion = ( - plugins: any, - dependencies: string[] -): {[path: string]: string} => { +const extractPathsWithVersion = (plugins: any, dependencies: string[]) => { const paths = Object.keys(plugins).map(plugin => plugins[plugin].path); const uniquePaths = [...new Set(paths)].filter(path => path !== 'builtin'); - const pathsWithVersion: {[path: string]: string} = {}; + const pathsWithVersion: PathWithVersion = {}; uniquePaths.forEach(pluginPath => { const dependency = dependencies.find((dependency: string) => @@ -117,13 +110,13 @@ const extractPathsWithVersion = ( /** * Creates folder if not exists, installs dependencies if required, update depenedencies. */ -async function initializeAndInstallLibs(options: EnvironmentOptions) { +const initializeAndInstallLibs = async (options: EnvironmentOptions) => { try { const {folderPath, install, dependencies} = options; await fs.mkdir(folderPath, {recursive: true}); - const packageJsonPath = await ensurePackageJsonExists(folderPath); + const packageJsonPath = await initPackageJsonIfNotExists(folderPath); if (install) { await installDependencies(folderPath, dependencies); @@ -133,45 +126,15 @@ async function initializeAndInstallLibs(options: EnvironmentOptions) { } catch (error) { console.log(FAILURE_MESSAGE); } -} - -/** - * Checks if the package.json is exists, if not, inisializes it. - */ -async function ensurePackageJsonExists(folderPath: string) { - const packageJsonPath = path.resolve(folderPath, 'package.json'); - const isPackageJsonExists = await isFileExists(packageJsonPath); - - if (!isPackageJsonExists) { - await execPromise('npm init -y', {cwd: folderPath}); - } - - return packageJsonPath; -} - -/** - * Installs packages from the specified dependencies in the specified folder. - */ -async function installDependencies( - folderPath: string, - dependencies: {[path: string]: string} -) { - const packages = Object.entries(dependencies).map( - ([dependency, version]) => `${dependency}@${version.replace('^', '')}` - ); - - await execPromise(`npm install ${packages.join(' ')}`, { - cwd: folderPath, - }); -} +}; /** * Updates package.json dependencies. */ -async function updatePackageJsonDependencies( +const updatePackageJsonDependencies = async ( packageJsonPath: string, - dependencies: {[path: string]: string} -) { + dependencies: PathWithVersion +) => { const packageJsonContent = await fs.readFile(packageJsonPath, 'utf8'); const packageJson = JSON.parse(packageJsonContent); @@ -181,14 +144,14 @@ async function updatePackageJsonDependencies( }; await fs.writeFile(packageJsonPath, JSON.stringify(packageJson, null, 2)); -} +}; /** * Adds a manifest template to the folder where the if-env CLI command runs. */ -async function addTemplateManifest() { +const addTemplateManifest = async () => { try { - const templateManifest = path.resolve(__dirname, 'template.yml'); + const templateManifest = path.resolve(__dirname, './env-template.yml'); const destinationPath = path.resolve( __dirname, FOLDER_NAME, @@ -200,8 +163,9 @@ async function addTemplateManifest() { await fs.writeFile(destinationPath, data, 'utf-8'); } catch (error) { console.log(FAILURE_MESSAGE_TEMPLATE); + process.exit(1); } -} +}; IfEnv().catch(error => { if (error instanceof Error) { From 4583da38faa824fb3c156bec97ec68c25009400b Mon Sep 17 00:00:00 2001 From: manushak Date: Mon, 10 Jun 2024 19:29:02 +0400 Subject: [PATCH 053/243] fix(package): fix env file path --- package-lock.json | 2 +- package.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 39b301049..4baee717e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25,7 +25,7 @@ "bin": { "ie": "build/index.js", "if-diff": "build/diff.js", - "if-env": "build/env/env.js" + "if-env": "build/env.js" }, "devDependencies": { "@babel/core": "^7.22.10", diff --git a/package.json b/package.json index 96074537b..ec174390f 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "bin": { "ie": "./build/index.js", "if-diff": "./build/diff.js", - "if-env": "./build/env/env.js" + "if-env": "./build/env.js" }, "bugs": { "url": "https://github.com/Green-Software-Foundation/if/issues/new?assignees=&labels=feedback&projects=&template=feedback.md&title=Feedback+-+" @@ -75,7 +75,7 @@ "fix:package": "fixpack", "ie": "npx ts-node src/index.ts", "if-diff": "npx ts-node src/diff.ts", - "if-env": "npx ts-node src/env/env.ts", + "if-env": "npx ts-node src/env.ts", "lint": "gts lint", "pre-commit": "lint-staged", "prepare": "husky install", From 9dec82bd67574c980d7cf805adc7502b8780939a Mon Sep 17 00:00:00 2001 From: manushak Date: Tue, 11 Jun 2024 11:19:07 +0400 Subject: [PATCH 054/243] test(mocks): add fs.stat mock function --- src/__mocks__/fs/index.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/__mocks__/fs/index.ts b/src/__mocks__/fs/index.ts index da8438c8b..6ac9b66ea 100644 --- a/src/__mocks__/fs/index.ts +++ b/src/__mocks__/fs/index.ts @@ -86,3 +86,11 @@ export const writeFile = async (pathToFile: string, content: string) => { expect(pathToFile).toBe(mockPathToFile); expect(content).toBe(mockObject); }; + +export const stat = async (filePath: string) => { + if (filePath === 'true') { + return true; + } else { + throw new Error('File not found.'); + } +}; From ec43b2dadc34ff94f888bf90cb259fb72d239180 Mon Sep 17 00:00:00 2001 From: manushak Date: Tue, 11 Jun 2024 11:28:34 +0400 Subject: [PATCH 055/243] test(util): add test for npm manipulation functions in helpers.ts --- src/__tests__/unit/util/helpers.test.ts | 103 +++++++++++++++++++++++- 1 file changed, 101 insertions(+), 2 deletions(-) diff --git a/src/__tests__/unit/util/helpers.test.ts b/src/__tests__/unit/util/helpers.test.ts index d627790e4..b5e04527c 100644 --- a/src/__tests__/unit/util/helpers.test.ts +++ b/src/__tests__/unit/util/helpers.test.ts @@ -1,3 +1,9 @@ +import * as fs from 'fs'; +import * as path from 'path'; + +jest.mock('fs/promises', () => require('../../../__mocks__/fs')); + +const mockInfo = jest.fn(); const mockWarn = jest.fn(); const mockError = jest.fn(); @@ -6,10 +12,12 @@ jest.mock('node:readline/promises', () => ); jest.mock('../../../util/logger', () => ({ logger: { + info: mockInfo, warn: mockWarn, error: mockError, }, })); + import { andHandle, checkIfEqual, @@ -17,11 +25,16 @@ import { mergeObjects, oneIsPrimitive, parseManifestFromStdin, + isFileExists, + installDependencies, + initPackageJsonIfNotExists, } from '../../../util/helpers'; import {ERRORS} from '../../../util/errors'; import {Difference} from '../../../types/lib/compare'; +import {STRINGS} from '../../../config/strings'; const {WriteFileError} = ERRORS; +const {INITIALIZING_PACKAGE_JSON, INSTALLING_NPM_PACKAGES} = STRINGS; describe('util/helpers: ', () => { describe('andHandle(): ', () => { @@ -47,9 +60,7 @@ describe('util/helpers: ', () => { expect(mockError).toHaveBeenCalledTimes(1); }); }); -}); -describe('util/helpers: ', () => { describe('mergeObjects(): ', () => { it('does not override input.', () => { expect.assertions(1); @@ -413,4 +424,92 @@ description: mock-description expect(response).toBeFalsy(); }); }); + + describe('isFileExists(): ', () => { + it('returns true if the file exists.', async () => { + const result = await isFileExists('true'); + + expect.assertions(1); + expect(result).toEqual(true); + }); + + it('returns fale if the file does not exist.', async () => { + const result = await isFileExists('false'); + + expect.assertions(1); + expect(result).toEqual(false); + }); + }); + + describe('npm manipulation tests.', () => { + const helpers = require('../../../util/helpers'); + let folderPath = path.resolve(__dirname, 'npm-test'); + + beforeAll(() => { + if (!fs.existsSync(folderPath)) { + fs.mkdirSync(folderPath, {recursive: true}); + } + }); + + afterAll(() => { + if (fs.existsSync(folderPath)) { + fs.rmSync(folderPath, {recursive: true, force: true}); + } + }); + + describe('initPackageJsonIfNotExists(): ', () => { + it('initializes package.json if it does not exist.', async () => { + const spyExecPromise = jest.spyOn(helpers, 'execPromise'); + isFileExists('true'); + + await initPackageJsonIfNotExists(folderPath); + + expect.assertions(2); + expect(mockInfo).toHaveBeenCalledWith(INITIALIZING_PACKAGE_JSON); + expect(spyExecPromise).toHaveBeenCalledWith('npm init -y', { + cwd: folderPath, + }); + }); + + it('returns the package.json path if it exists.', async () => { + const packageJsonPath = path.resolve(folderPath, 'package.json'); + isFileExists('false'); + + const result = await initPackageJsonIfNotExists(folderPath); + + expect.assertions(1); + expect(result).toBe(packageJsonPath); + }); + }); + + describe('installDependencies(): ', () => { + const dependencies = { + '@grnsft/if': '^0.3.3-beta.0', + }; + + it('calls execPromise with the correct arguments.', async () => { + const spyExecPromise = jest.spyOn(helpers, 'execPromise'); + const formattedDependencies = ['@grnsft/if@0.3.3-beta.0']; + expect.assertions(1); + + await installDependencies(folderPath, dependencies); + + expect(spyExecPromise).toHaveBeenCalledWith( + `npm install ${formattedDependencies.join(' ')}`, + {cwd: folderPath} + ); + }, 30000); + + it('logs the installation message.', async () => { + const dependencies = { + '@grnsft/if': '^0.3.3-beta.0', + }; + + await installDependencies(folderPath, dependencies); + + expect.assertions(1); + expect(mockInfo).toHaveBeenCalledWith(INSTALLING_NPM_PACKAGES); + }); + }); + }); }); From 91c1bedcae20261484202021465ecea2c7794472 Mon Sep 17 00:00:00 2001 From: manushak Date: Tue, 11 Jun 2024 11:30:21 +0400 Subject: [PATCH 056/243] test(util): fix commit lint error --- src/__tests__/unit/util/helpers.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/__tests__/unit/util/helpers.test.ts b/src/__tests__/unit/util/helpers.test.ts index b5e04527c..86a0f2dd7 100644 --- a/src/__tests__/unit/util/helpers.test.ts +++ b/src/__tests__/unit/util/helpers.test.ts @@ -443,7 +443,7 @@ description: mock-description describe('npm manipulation tests.', () => { const helpers = require('../../../util/helpers'); - let folderPath = path.resolve(__dirname, 'npm-test'); + const folderPath = path.resolve(__dirname, 'npm-test'); beforeAll(() => { if (!fs.existsSync(folderPath)) { From b78d49ad8618072049538607eb09a14040013cd9 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Wed, 12 Jun 2024 14:51:34 +0400 Subject: [PATCH 057/243] revert(builtins): remove unnecessary console.log --- src/builtins/csv-lookup/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/builtins/csv-lookup/index.ts b/src/builtins/csv-lookup/index.ts index 4f7df52be..5a009ac8b 100644 --- a/src/builtins/csv-lookup/index.ts +++ b/src/builtins/csv-lookup/index.ts @@ -193,7 +193,6 @@ export const CSVLookup = (globalConfig: any): ExecutePlugin => { const {filepath, query, output} = safeGlobalConfig; const file = await retrieveFile(filepath); - console.log(file); const parsedCSV = parseCSVFile(file); return inputs.map(input => { From a75e8d7c2cea2801ef2c4f7019bb30289d5a39bc Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Wed, 12 Jun 2024 14:52:12 +0400 Subject: [PATCH 058/243] test(util): remove unnecessary console.log --- src/__tests__/unit/util/aggregation-helper.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/__tests__/unit/util/aggregation-helper.test.ts b/src/__tests__/unit/util/aggregation-helper.test.ts index 7cccbfccc..64add64e0 100644 --- a/src/__tests__/unit/util/aggregation-helper.test.ts +++ b/src/__tests__/unit/util/aggregation-helper.test.ts @@ -38,7 +38,6 @@ describe('util/aggregation-helper: ', () => { try { aggregateInputsIntoOne(inputs, metrics, isTemporal); } catch (error) { - console.log(error); expect(error).toBeInstanceOf(MissingAggregationParamError); if (error instanceof MissingAggregationParamError) { From f1b4c34822bce47af13fcb30dba5283e31886e4d Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Thu, 13 Jun 2024 10:27:14 +0100 Subject: [PATCH 059/243] feat(package): rename scirpts in package and lockfile --- package-lock.json | 4 ++-- package.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index a52e10aa3..fc7063570 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,8 +23,8 @@ "zod": "^3.22.4" }, "bin": { - "ie": "build/index.js", - "if-diff": "build/diff.js" + "if-diff": "build/diff.js", + "if-run": "build/index.js" }, "devDependencies": { "@babel/core": "^7.22.10", diff --git a/package.json b/package.json index 60e33dc0f..0ff122d85 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "email": "info@gsf.com" }, "bin": { - "ie": "./build/index.js", + "if-run": "./build/index.js", "if-diff": "./build/diff.js" }, "bugs": { @@ -72,7 +72,7 @@ "coverage": "jest --verbose --coverage --testPathPattern=src/__tests__/unit", "fix": "gts fix", "fix:package": "fixpack", - "ie": "npx ts-node src/index.ts", + "if-run": "npx ts-node src/index.ts", "if-diff": "npx ts-node src/diff.ts", "lint": "gts lint", "pre-commit": "lint-staged", From 20161b583744f68714eb19ec786eb3acff8500a6 Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Thu, 13 Jun 2024 10:27:49 +0100 Subject: [PATCH 060/243] feat(package): update package.json --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 0ff122d85..f17862f4b 100644 --- a/package.json +++ b/package.json @@ -72,8 +72,8 @@ "coverage": "jest --verbose --coverage --testPathPattern=src/__tests__/unit", "fix": "gts fix", "fix:package": "fixpack", - "if-run": "npx ts-node src/index.ts", "if-diff": "npx ts-node src/diff.ts", + "if-run": "npx ts-node src/index.ts", "lint": "gts lint", "pre-commit": "lint-staged", "prepare": "husky install", From 0b4e6c9b6ccd422369f4202f28b0a334cf483453 Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Thu, 13 Jun 2024 10:28:44 +0100 Subject: [PATCH 061/243] feat(lib): rename ie to if-run in project readme --- README.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index c34bf0069..9de6c64e7 100644 --- a/README.md +++ b/README.md @@ -42,17 +42,17 @@ Then create a `manifest` file that describes your application (see our docs for Then, run `if` using the following command: ```sh -ie --manifest +if-run --manifest ## or you can use aliases -ie -m +if-run -m ``` Note that above command will not print the final output. In order to print the final output to the console, run `if` using the optional stdout argument: ```sh -ie --manifest --stdout +if-run --manifest --stdout ## or using aliases -ie -m -s +if-run -m -s ``` You can also add a savepath for your output yaml in order to have the output stored in a file. Note that you also have to add configuration to your manifest to enable this, as follows: @@ -66,25 +66,25 @@ initialize: On the command line: ```sh -ie --manifest --output +if-run --manifest --output ## or using aliases -ie -m -o +if-run -m -o ``` -The `ie` CLI tool will configure and run the plugins defined in your input `yaml` (`manifest`) and return the results as an output `yaml` (`output`). +The `if-run` CLI tool will configure and run the plugins defined in your input `yaml` (`manifest`) and return the results as an output `yaml` (`output`). Use the `debug` command if you want to diagnose and fix errors in your plugin: ```sh -ie --manifest --debug +if-run --manifest --debug ``` Use the `help` command if you need guidance about the available commands ```sh -ie --help +if-run --help ## or using alias -ie -h +if-run -h ``` ## Documentation From 0286a34bb9ae2481e5fc73e39c5837ae47be588b Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Thu, 13 Jun 2024 10:29:27 +0100 Subject: [PATCH 062/243] fix(package): update command name in refactor guide --- Refactor-migration-guide.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Refactor-migration-guide.md b/Refactor-migration-guide.md index 80328fe9e..95269e1a5 100644 --- a/Refactor-migration-guide.md +++ b/Refactor-migration-guide.md @@ -12,13 +12,13 @@ There have been some name changes to the CLI, specifically: The command line tool has been renamed from `impact-engine` to simply `ie`. This means that to invoke the Impact Framework on the command line you simply use ``` - ie ... + if-run ... ``` - `impl` --> `manifest` We have deprecated the original `impl` and `ompl` terminology across all our repositories and on the command line. Now, to pass a manifest file to IF, you use the `--manifest` command, as follows: ```sh - ie --manifest + if-run --manifest ``` @@ -27,7 +27,7 @@ There have been some name changes to the CLI, specifically: We have deprecated the original `impl` and `ompl` terminology across all our repositories and on the command line. Now, to define a savepath for your output file, you use the `--output` command, as follows: ```sh - ie --manifest --output + if-run --manifest --output ``` ## Outputs @@ -55,13 +55,13 @@ npm i @grnsft/if Then run IF using the following command: ```sh -ie --manifest +if-run --manifest ``` This will dump the output to the console. If you want to save the output to a yaml file, provide a savepath to the `--output` command: ```sh -ie --manifest --output +if-run --manifest --output ``` From 9d5d82e5ec29cf0f795fb3dcd2e4ffd8729ea0e1 Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Thu, 13 Jun 2024 10:30:42 +0100 Subject: [PATCH 063/243] fix(lib): update ie to if-run in integration test script --- src/__tests__/integration/scenarios/sci-e.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/__tests__/integration/scenarios/sci-e.ts b/src/__tests__/integration/scenarios/sci-e.ts index 9fb808341..0342240c2 100644 --- a/src/__tests__/integration/scenarios/sci-e.ts +++ b/src/__tests__/integration/scenarios/sci-e.ts @@ -36,7 +36,7 @@ describe('integration/sci-e', () => { await saveYamlFileAs(file, absoluteManifestPath); // save yaml uses absolute path const response = ( await execPromise( - `npm run ie -- --manifest ${relativeManifestPath} --stdout` + `npm run if-run -- --manifest ${relativeManifestPath} --stdout` ) ).stdout; // exec promise uses relative path From a82b978f3ad4c6f0878661f8ce8f13df061649d0 Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Thu, 13 Jun 2024 10:31:16 +0100 Subject: [PATCH 064/243] fix(lib): update ie to if-run in test script --- scripts/run-yamls.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/run-yamls.sh b/scripts/run-yamls.sh index 11371af66..fbe394c6b 100644 --- a/scripts/run-yamls.sh +++ b/scripts/run-yamls.sh @@ -4,5 +4,5 @@ echo 'Running all manifests' for f in ./examples/manifests/*.yml; do echo "Processing $f file..."; - npm run ie -- --manifest $f + npm run if-run -- --manifest $f done From 2c97c002001c74d3f5396c8b801da4aa1d856398 Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Thu, 13 Jun 2024 10:32:16 +0100 Subject: [PATCH 065/243] fix(lib): update ie -> if-run in builtin readmes --- src/builtins/README.md | 2 +- src/builtins/coefficient/README.md | 2 +- src/builtins/csv-lookup/README.md | 2 +- src/builtins/divide/README.md | 2 +- src/builtins/exponent/README.md | 2 +- src/builtins/interpolation/README.md | 2 +- src/builtins/mock-observations/README.md | 2 +- src/builtins/multiply/README.md | 2 +- src/builtins/sci-embodied/README.md | 2 +- src/builtins/sci/README.md | 2 +- src/builtins/shell/README.md | 2 +- src/builtins/subtract/README.md | 2 +- src/builtins/sum/README.md | 2 +- 13 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/builtins/README.md b/src/builtins/README.md index b9906921d..c37e143a1 100644 --- a/src/builtins/README.md +++ b/src/builtins/README.md @@ -305,7 +305,7 @@ Then, you must select the metric you want to export to CSV. The name of that met For example, to export the `carbon` data from your tree to a CSV file: ```sh -ie --manifest example.yml --output example#carbon +if-run --manifest example.yml --output example#carbon ``` This will save a CSV file called `example.csv`. The contents will look similar to the following: diff --git a/src/builtins/coefficient/README.md b/src/builtins/coefficient/README.md index 096d20ab5..bf8808e8c 100644 --- a/src/builtins/coefficient/README.md +++ b/src/builtins/coefficient/README.md @@ -86,7 +86,7 @@ tree: You can run this example by saving it as `./examples/manifests/coefficient.yml` and executing the following command from the project root: ```sh -ie --manifest ./examples/manifests/coefficient.yml --output ./examples/outputs/coefficient.yml +if-run --manifest ./examples/manifests/coefficient.yml --output ./examples/outputs/coefficient.yml ``` The results will be saved to a new `yaml` file in `./examples/outputs` diff --git a/src/builtins/csv-lookup/README.md b/src/builtins/csv-lookup/README.md index 2c52969ae..d1aece699 100644 --- a/src/builtins/csv-lookup/README.md +++ b/src/builtins/csv-lookup/README.md @@ -136,7 +136,7 @@ You can run this example by saving it as `./examples/manifests/csv-lookup.yml` a ```sh npm i -g @grnsft/if -ie --manifest manifests/plugins/csv-lookup.yml --output manifests/outputs/csv-lookup +if-run --manifest manifests/plugins/csv-lookup.yml --output manifests/outputs/csv-lookup ``` The results will be saved to a new `yaml` file in `manifests/outputs`. diff --git a/src/builtins/divide/README.md b/src/builtins/divide/README.md index d403d0361..20ed19f55 100644 --- a/src/builtins/divide/README.md +++ b/src/builtins/divide/README.md @@ -89,7 +89,7 @@ You can run this example by saving it as `./examples/manifests/divide.yml` and e ```sh npm i -g @grnsft/if -ie --manifest ./examples/manifests/divide.yml --output ./examples/outputs/divide.yml +if-run --manifest ./examples/manifests/divide.yml --output ./examples/outputs/divide.yml ``` The results will be saved to a new `yaml` file in `./examples/outputs`. diff --git a/src/builtins/exponent/README.md b/src/builtins/exponent/README.md index 48a67965a..d84020d63 100644 --- a/src/builtins/exponent/README.md +++ b/src/builtins/exponent/README.md @@ -91,7 +91,7 @@ You can run this example by saving it as `manifests/examples/test/exponent.yml` ```sh npm i -g @grnsft/if -ie --manifest manifests/examples/test/exponent.yml --output manifests/outputs/exponent.yml +if-run --manifest manifests/examples/test/exponent.yml --output manifests/outputs/exponent.yml ``` The results will be saved to a new `yaml` file in `manifests/outputs`. diff --git a/src/builtins/interpolation/README.md b/src/builtins/interpolation/README.md index cfcca2fa1..670a65ef2 100644 --- a/src/builtins/interpolation/README.md +++ b/src/builtins/interpolation/README.md @@ -164,5 +164,5 @@ You can execute this by passing it to `ie`. Run the impact using the following c ```sh npm i -g @grnsft/if -ie --manifest ./manifests/examples/interpolation.yml --output ./manifests/outputs/interpolation.yml +if-run --manifest ./manifests/examples/interpolation.yml --output ./manifests/outputs/interpolation.yml ``` diff --git a/src/builtins/mock-observations/README.md b/src/builtins/mock-observations/README.md index 629e27f2d..1af4cd6d6 100644 --- a/src/builtins/mock-observations/README.md +++ b/src/builtins/mock-observations/README.md @@ -91,7 +91,7 @@ You can run this example `manifest` by saving it as `manifests/plugins/mock-obse ```sh npm i -g @grnsft/if -ie --manifest ./examples/manifests/test/mock-observation.yml --output ./examples/outputs/mock-observation +if-run --manifest ./examples/manifests/test/mock-observation.yml --output ./examples/outputs/mock-observation ``` The results will be saved to a new `yaml` file in `./examples/outputs`. diff --git a/src/builtins/multiply/README.md b/src/builtins/multiply/README.md index d4e27674c..695d89331 100644 --- a/src/builtins/multiply/README.md +++ b/src/builtins/multiply/README.md @@ -88,7 +88,7 @@ You can run this example by saving it as `./examples/manifests/test/multiply.yml ```sh npm i -g @grnsft/if -ie --manifest ./examples/manifests/test/multiply.yml --output ./examples/outputs/multiply.yml +if-run --manifest ./examples/manifests/test/multiply.yml --output ./examples/outputs/multiply.yml ``` The results will be saved to a new `yaml` file in `./examples/outputs` diff --git a/src/builtins/sci-embodied/README.md b/src/builtins/sci-embodied/README.md index 90fc01d58..70307fee1 100644 --- a/src/builtins/sci-embodied/README.md +++ b/src/builtins/sci-embodied/README.md @@ -104,7 +104,7 @@ You can run this example `manifest` by executing the following command from the ```sh npm i -g @grnsft/if -ie --manifest manifests/plugins/sci-embodied.yml --output manifests/outputs/sci-embodied.yml +if-run --manifest manifests/plugins/sci-embodied.yml --output manifests/outputs/sci-embodied.yml ``` The results will be saved to a new `yaml` file in `./examples/outputs`. diff --git a/src/builtins/sci/README.md b/src/builtins/sci/README.md index 477f74dc0..658209f1a 100644 --- a/src/builtins/sci/README.md +++ b/src/builtins/sci/README.md @@ -83,7 +83,7 @@ You can run this example `manifest` by saving it as `./manifests/plugins/sci.yml ```sh npm i -g @grnsft/if -ie --manifest manifests/plugins/sci.yml --output manifests/outputs/sci.yml +if-run --manifest manifests/plugins/sci.yml --output manifests/outputs/sci.yml ``` The results will be saved to a new `yaml` file. diff --git a/src/builtins/shell/README.md b/src/builtins/shell/README.md index ea813878b..54ccac285 100644 --- a/src/builtins/shell/README.md +++ b/src/builtins/shell/README.md @@ -124,7 +124,7 @@ You can run this example `manifest` by saving it as `manifests/plugins/shell.yml ```sh npm i -g @grnsft/if -ie --manifest manifests/plugins/shell.yml --output manifests/outputs/shell.yml +if-run --manifest manifests/plugins/shell.yml --output manifests/outputs/shell.yml ``` The results will be saved to a new `yaml` file. diff --git a/src/builtins/subtract/README.md b/src/builtins/subtract/README.md index 9279220b8..bb8435548 100644 --- a/src/builtins/subtract/README.md +++ b/src/builtins/subtract/README.md @@ -88,7 +88,7 @@ You can run this example by saving it as `./examples/manifests/test/subrtact.yml ```sh npm i -g @grnsft/if -ie --manifest /manifests/plugins/subtract.yml --output manifests/outputs/subtract.yml +if-run --manifest /manifests/plugins/subtract.yml --output manifests/outputs/subtract.yml ``` The results will be saved to a new `yaml` file in `manifests/outputs`. diff --git a/src/builtins/sum/README.md b/src/builtins/sum/README.md index b912c07b3..d3fd892da 100644 --- a/src/builtins/sum/README.md +++ b/src/builtins/sum/README.md @@ -85,7 +85,7 @@ tree: You can run this example by saving it as `./examples/manifests/sum.yml` and executing the following command from the project root: ```sh -ie --manifest ./examples/manifests/sum.yml --output ./examples/outputs/sum.yml +if-run --manifest ./examples/manifests/sum.yml --output ./examples/outputs/sum.yml ``` The results will be saved to a new `yaml` file in `./examples/outputs`. From 76c6e80fdf6167edcc1c659cd5ab6268e26b31d0 Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 13 Jun 2024 18:26:53 +0400 Subject: [PATCH 066/243] fix(src): update return data of parseIfEnvArgs --- src/util/args.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/args.ts b/src/util/args.ts index 753145022..9e9ef5e95 100644 --- a/src/util/args.ts +++ b/src/util/args.ts @@ -181,5 +181,5 @@ export const parseIfEnvArgs = async () => { throw new CliInputError(FILE_IS_NOT_YAML); } - return; + return {install}; }; From 2d2ca0e2324d9dc3eca1024d1941a77254f44d7c Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 13 Jun 2024 18:29:19 +0400 Subject: [PATCH 067/243] test(util): update test of parseIfEnvArgs function --- src/__tests__/unit/util/args.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/__tests__/unit/util/args.test.ts b/src/__tests__/unit/util/args.test.ts index ac91f8a3a..07620013d 100644 --- a/src/__tests__/unit/util/args.test.ts +++ b/src/__tests__/unit/util/args.test.ts @@ -337,7 +337,7 @@ describe('util/args: ', () => { expect.assertions(1); - expect(response).toEqual(undefined); + expect(response).toEqual({install: undefined}); }); it('executes if `manifest` and `install` are provided.', async () => { From be6412a73721533b92baf6293bb27146897051df Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 13 Jun 2024 18:30:26 +0400 Subject: [PATCH 068/243] fix(src): remove functioanlity that create sepereate folder --- src/env.ts | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/env.ts b/src/env.ts index 94821d466..5128f58d4 100644 --- a/src/env.ts +++ b/src/env.ts @@ -23,17 +23,15 @@ const { FAILURE_MESSAGE_DEPENDENCIES, } = IF_ENV; -const FOLDER_NAME = 'if-environment'; - const IfEnv = async () => { const commandArgs = await parseIfEnvArgs(); const options: EnvironmentOptions = { - folderPath: path.resolve(__dirname, FOLDER_NAME), - install: true, + folderPath: __dirname, + install: !!commandArgs.install, dependencies: {'@grnsft/if': packageJson.version}, }; - if (commandArgs) { + if (commandArgs && commandArgs.manifest) { const {folderPath, install, dependencies} = await getOptionsFromArgs(commandArgs); options.folderPath = folderPath; @@ -43,7 +41,7 @@ const IfEnv = async () => { await initializeAndInstallLibs(options); - if (!commandArgs) { + if (!commandArgs || !commandArgs.manifest) { await addTemplateManifest(); } @@ -152,11 +150,7 @@ const updatePackageJsonDependencies = async ( const addTemplateManifest = async () => { try { const templateManifest = path.resolve(__dirname, './env-template.yml'); - const destinationPath = path.resolve( - __dirname, - FOLDER_NAME, - 'manifest.yml' - ); + const destinationPath = path.resolve(__dirname, 'manifest.yml'); const data = await fs.readFile(templateManifest, 'utf-8'); await fs.writeFile(destinationPath, '', 'utf-8'); From d5d1ea9e15f2890b057a3e19948585b81d66f5a6 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Thu, 13 Jun 2024 19:52:02 +0400 Subject: [PATCH 069/243] revert(config): remove missing manifest from stdin message --- src/config/strings.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/config/strings.ts b/src/config/strings.ts index 6ef27ac5e..83283e8fb 100644 --- a/src/config/strings.ts +++ b/src/config/strings.ts @@ -55,7 +55,6 @@ Note that for the '--output' option you also need to define the output type in y TARGET_IS_NOT_YAML: 'Given target is not in yaml format.', INVALID_TARGET: 'Target is invalid.', INVALID_SOURCE: 'Source is invalid.', - MISSING_MANIFEST_IN_STDIN: 'Manifest not found in STDIN.', /** Plugin messages */ MISSING_GLOBAL_CONFIG: 'Global config is not provided.', MISSING_INPUT_DATA: (param: string) => From f932b7dbdab916e0bebd49d2c517a1f5fbb8c166 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Thu, 13 Jun 2024 19:52:40 +0400 Subject: [PATCH 070/243] revert(util): remove unused error class and message --- src/util/helpers.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/util/helpers.ts b/src/util/helpers.ts index 6b94c9397..d3b5a0a3c 100644 --- a/src/util/helpers.ts +++ b/src/util/helpers.ts @@ -9,8 +9,7 @@ import {STRINGS} from '../config'; import {Difference} from '../types/lib/compare'; -const {ISSUE_TEMPLATE, MISSING_MANIFEST_IN_STDIN} = STRINGS; -const {CliInputError} = ERRORS; +const {ISSUE_TEMPLATE} = STRINGS; /** * Impact engine error handler. Logs errors and appends issue template if error is unknown. From 6c40decedeb8c81d46f55bd256f534a05a39fa49 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Thu, 13 Jun 2024 20:05:58 +0400 Subject: [PATCH 071/243] feat(util): introduce if-core for errors --- src/util/aggregation-helper.ts | 3 ++- src/util/args.ts | 3 ++- src/util/helpers.ts | 3 ++- src/util/plugin-storage.ts | 3 ++- src/util/validations.ts | 3 +-- 5 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/util/aggregation-helper.ts b/src/util/aggregation-helper.ts index 0a8b3c136..3caff8c6d 100644 --- a/src/util/aggregation-helper.ts +++ b/src/util/aggregation-helper.ts @@ -1,4 +1,5 @@ -import {ERRORS} from '../util/errors'; +import {ERRORS} from '@grnsft/if-core'; + import {parameterize} from '../lib/parameterize'; import {CONFIG, STRINGS} from '../config'; diff --git a/src/util/args.ts b/src/util/args.ts index e7a042094..ee090c1ef 100644 --- a/src/util/args.ts +++ b/src/util/args.ts @@ -1,8 +1,9 @@ import * as path from 'path'; + import {parse} from 'ts-command-line-args'; +import {ERRORS} from '@grnsft/if-core'; import {checkIfFileIsYaml} from './yaml'; -import {ERRORS} from './errors'; import {logger} from './logger'; import {CONFIG, STRINGS} from '../config'; diff --git a/src/util/helpers.ts b/src/util/helpers.ts index d3b5a0a3c..45af8b50c 100644 --- a/src/util/helpers.ts +++ b/src/util/helpers.ts @@ -2,7 +2,8 @@ import {createInterface} from 'node:readline/promises'; import {exec} from 'node:child_process'; import {promisify} from 'node:util'; -import {ERRORS} from './errors'; +import {ERRORS} from '@grnsft/if-core'; + import {logger} from './logger'; import {STRINGS} from '../config'; diff --git a/src/util/plugin-storage.ts b/src/util/plugin-storage.ts index 15582e9f3..da1e45970 100644 --- a/src/util/plugin-storage.ts +++ b/src/util/plugin-storage.ts @@ -1,4 +1,5 @@ -import {ERRORS} from '../util/errors'; +import {ERRORS} from '@grnsft/if-core'; + import {STRINGS} from '../config'; import {PluginInterface} from '../types/interface'; diff --git a/src/util/validations.ts b/src/util/validations.ts index 21695f3ee..30cb1e930 100644 --- a/src/util/validations.ts +++ b/src/util/validations.ts @@ -1,6 +1,5 @@ import {ZodIssue, ZodIssueCode, ZodSchema, z} from 'zod'; - -import {ERRORS} from './errors'; +import {ERRORS} from '@grnsft/if-core'; import {AGGREGATION_METHODS} from '../types/aggregation'; import {AGGREGATION_TYPES} from '../types/parameters'; From 56abea628fa1c32358299d5bba5e5b9dbd246d53 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Thu, 13 Jun 2024 20:07:48 +0400 Subject: [PATCH 072/243] feat(lib): introduce if-core for errors --- src/lib/exhaust.ts | 3 ++- src/lib/initialize.ts | 25 +++++++++++++------------ src/lib/load.ts | 2 +- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/lib/exhaust.ts b/src/lib/exhaust.ts index 002818c70..d60268698 100644 --- a/src/lib/exhaust.ts +++ b/src/lib/exhaust.ts @@ -1,3 +1,5 @@ +import {ERRORS} from '@grnsft/if-core'; + /** * @todo This is temporary solution, will be refactored to support dynamic plugins. */ @@ -6,7 +8,6 @@ import {ExportCSVRaw} from '../builtins/export-csv-raw'; import {ExportLog} from '../builtins/export-log'; import {ExportYaml} from '../builtins/export-yaml'; -import {ERRORS} from '../util/errors'; import {STRINGS} from '../config'; diff --git a/src/lib/initialize.ts b/src/lib/initialize.ts index e9c4f27a0..ac01e8e8e 100644 --- a/src/lib/initialize.ts +++ b/src/lib/initialize.ts @@ -1,6 +1,7 @@ -import pathLib = require('path'); +import * as path from 'node:path'; + +import {ERRORS} from '@grnsft/if-core'; -import {ERRORS} from '../util/errors'; import {logger} from '../util/logger'; import {memoizedLog} from '../util/log-memoize'; import {pluginStorage} from '../util/plugin-storage'; @@ -53,23 +54,23 @@ const importAndVerifyModule = async (method: string, path: string) => { * Then checks if `path` is starting with github, then grabs the repository name. * Imports module, then checks if it's a valid plugin. */ -const handModule = (method: string, path: string) => { - console.debug(LOADING_PLUGIN_FROM_PATH(method, path)); +const handModule = (method: string, pluginPath: string) => { + console.debug(LOADING_PLUGIN_FROM_PATH(method, pluginPath)); - if (path === 'builtin') { - path = pathLib.normalize(`${__dirname}/../builtins`); + if (pluginPath === 'builtin') { + pluginPath = path.normalize(`${__dirname}/../builtins`); } else { - if (path?.startsWith(GITHUB_PATH)) { - const parts = path.split('/'); - path = parts[parts.length - 1]; + if (pluginPath?.startsWith(GITHUB_PATH)) { + const parts = pluginPath.split('/'); + pluginPath = parts[parts.length - 1]; } - if (!path.includes(NATIVE_PLUGIN)) { - memoizedLog(logger.warn, NOT_NATIVE_PLUGIN(path)); + if (!pluginPath.includes(NATIVE_PLUGIN)) { + memoizedLog(logger.warn, NOT_NATIVE_PLUGIN(pluginPath)); } } - return importAndVerifyModule(method, path); + return importAndVerifyModule(method, pluginPath); }; /** diff --git a/src/lib/load.ts b/src/lib/load.ts index 0a65d4476..72620e982 100644 --- a/src/lib/load.ts +++ b/src/lib/load.ts @@ -1,6 +1,6 @@ import * as YAML from 'js-yaml'; +import {ERRORS} from '@grnsft/if-core'; -import {ERRORS} from '../util/errors'; import {openYamlFileAsObject} from '../util/yaml'; import {readAndParseJson} from '../util/json'; From 5ea7b14909c94a665bb97d6280361a2696b8f2fd Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Thu, 13 Jun 2024 20:08:03 +0400 Subject: [PATCH 073/243] feat(builtins): introduce if-core for errors --- src/builtins/coefficient/index.ts | 2 +- src/builtins/csv-lookup/index.ts | 2 +- src/builtins/divide/index.ts | 2 +- src/builtins/exponent/index.ts | 2 +- src/builtins/export-csv-raw.ts | 2 +- src/builtins/export-csv.ts | 2 +- src/builtins/export-yaml.ts | 2 +- src/builtins/group-by.ts | 2 +- src/builtins/interpolation/index.ts | 2 +- src/builtins/mock-observations/helpers/common-generator.ts | 2 +- src/builtins/mock-observations/helpers/rand-int-generator.ts | 2 +- src/builtins/multiply/index.ts | 2 +- src/builtins/regex/index.ts | 2 +- src/builtins/sci/index.ts | 2 +- src/builtins/shell/index.ts | 2 +- src/builtins/subtract/index.ts | 2 +- src/builtins/sum/index.ts | 2 +- src/builtins/time-sync.ts | 2 +- 18 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/builtins/coefficient/index.ts b/src/builtins/coefficient/index.ts index c049911ae..65404172f 100644 --- a/src/builtins/coefficient/index.ts +++ b/src/builtins/coefficient/index.ts @@ -3,7 +3,7 @@ import {z} from 'zod'; import {ExecutePlugin, PluginParams} from '../../types/interface'; import {validate} from '../../util/validations'; -import {ERRORS} from '../../util/errors'; +import {ERRORS} from '@grnsft/if-core'; import {STRINGS} from '../../config'; diff --git a/src/builtins/csv-lookup/index.ts b/src/builtins/csv-lookup/index.ts index 5a009ac8b..47232e115 100644 --- a/src/builtins/csv-lookup/index.ts +++ b/src/builtins/csv-lookup/index.ts @@ -6,7 +6,7 @@ import {z} from 'zod'; import {parse} from 'csv-parse/sync'; import {validate} from '../../util/validations'; -import {ERRORS} from '../../util/errors'; +import {ERRORS} from '@grnsft/if-core'; import {STRINGS} from '../../config'; diff --git a/src/builtins/divide/index.ts b/src/builtins/divide/index.ts index e50ca81f9..14e8b4d44 100644 --- a/src/builtins/divide/index.ts +++ b/src/builtins/divide/index.ts @@ -1,6 +1,6 @@ import {z} from 'zod'; -import {ERRORS} from '../../util/errors'; +import {ERRORS} from '@grnsft/if-core'; import {validate} from '../../util/validations'; import {STRINGS} from '../../config'; diff --git a/src/builtins/exponent/index.ts b/src/builtins/exponent/index.ts index 0de914fda..3a417329b 100644 --- a/src/builtins/exponent/index.ts +++ b/src/builtins/exponent/index.ts @@ -1,6 +1,6 @@ import {z} from 'zod'; -import {ERRORS} from '../../util/errors'; +import {ERRORS} from '@grnsft/if-core'; import {validate} from '../../util/validations'; import {STRINGS} from '../../config'; diff --git a/src/builtins/export-csv-raw.ts b/src/builtins/export-csv-raw.ts index 7adc4281b..9ce1edae3 100644 --- a/src/builtins/export-csv-raw.ts +++ b/src/builtins/export-csv-raw.ts @@ -1,6 +1,6 @@ import * as fs from 'fs/promises'; -import {ERRORS} from '../util/errors'; +import {ERRORS} from '@grnsft/if-core'; import {STRINGS} from '../config'; diff --git a/src/builtins/export-csv.ts b/src/builtins/export-csv.ts index e92de3ce8..9eb03d9cc 100644 --- a/src/builtins/export-csv.ts +++ b/src/builtins/export-csv.ts @@ -1,7 +1,7 @@ import {writeFile} from 'fs/promises'; import {stringify} from 'csv-stringify/sync'; -import {ERRORS} from '../util/errors'; +import {ERRORS} from '@grnsft/if-core'; import {STRINGS} from '../config'; diff --git a/src/builtins/export-yaml.ts b/src/builtins/export-yaml.ts index b3822c619..f6a3d71d7 100644 --- a/src/builtins/export-yaml.ts +++ b/src/builtins/export-yaml.ts @@ -1,5 +1,5 @@ import {saveYamlFileAs} from '../util/yaml'; -import {ERRORS} from '../util/errors'; +import {ERRORS} from '@grnsft/if-core'; import {STRINGS} from '../config'; diff --git a/src/builtins/group-by.ts b/src/builtins/group-by.ts index e4e8af958..3d38d444a 100644 --- a/src/builtins/group-by.ts +++ b/src/builtins/group-by.ts @@ -5,7 +5,7 @@ import {STRINGS} from '../config'; import {GroupByPlugin, PluginParams} from '../types/interface'; import {GroupByConfig} from '../types/group-by'; -import {ERRORS} from '../util/errors'; +import {ERRORS} from '@grnsft/if-core'; import {validate} from '../util/validations'; const {InvalidGroupingError, GlobalConfigError} = ERRORS; diff --git a/src/builtins/interpolation/index.ts b/src/builtins/interpolation/index.ts index 35f88788b..dfe507c42 100644 --- a/src/builtins/interpolation/index.ts +++ b/src/builtins/interpolation/index.ts @@ -4,7 +4,7 @@ import {z} from 'zod'; import {ExecutePlugin, PluginParams, ConfigParams} from '../../types/interface'; import {validate} from '../../util/validations'; -import {ERRORS} from '../../util/errors'; +import {ERRORS} from '@grnsft/if-core'; import {STRINGS} from '../../config'; diff --git a/src/builtins/mock-observations/helpers/common-generator.ts b/src/builtins/mock-observations/helpers/common-generator.ts index ee4909dcd..732e7320e 100644 --- a/src/builtins/mock-observations/helpers/common-generator.ts +++ b/src/builtins/mock-observations/helpers/common-generator.ts @@ -1,4 +1,4 @@ -import {ERRORS} from '../../../util/errors'; +import {ERRORS} from '@grnsft/if-core'; import {STRINGS} from '../../../config'; diff --git a/src/builtins/mock-observations/helpers/rand-int-generator.ts b/src/builtins/mock-observations/helpers/rand-int-generator.ts index a48e399c8..6c0365d35 100644 --- a/src/builtins/mock-observations/helpers/rand-int-generator.ts +++ b/src/builtins/mock-observations/helpers/rand-int-generator.ts @@ -1,4 +1,4 @@ -import {ERRORS} from '../../../util/errors'; +import {ERRORS} from '@grnsft/if-core'; import {STRINGS} from '../../../config'; diff --git a/src/builtins/multiply/index.ts b/src/builtins/multiply/index.ts index ffcf4d14c..5ceb44c3c 100644 --- a/src/builtins/multiply/index.ts +++ b/src/builtins/multiply/index.ts @@ -1,6 +1,6 @@ import {z} from 'zod'; -import {ERRORS} from '../../util/errors'; +import {ERRORS} from '@grnsft/if-core'; import {validate} from '../../util/validations'; import {STRINGS} from '../../config'; diff --git a/src/builtins/regex/index.ts b/src/builtins/regex/index.ts index 5730697cc..cdcbd13b8 100644 --- a/src/builtins/regex/index.ts +++ b/src/builtins/regex/index.ts @@ -1,6 +1,6 @@ import {z} from 'zod'; -import {ERRORS} from '../../util/errors'; +import {ERRORS} from '@grnsft/if-core'; import {validate} from '../../util/validations'; import {STRINGS} from '../../config'; diff --git a/src/builtins/sci/index.ts b/src/builtins/sci/index.ts index a39e33c94..c7802213d 100644 --- a/src/builtins/sci/index.ts +++ b/src/builtins/sci/index.ts @@ -1,7 +1,7 @@ import {z} from 'zod'; import {validate, allDefined} from '../../util/validations'; -import {ERRORS} from '../../util/errors'; +import {ERRORS} from '@grnsft/if-core'; import {STRINGS} from '../../config'; diff --git a/src/builtins/shell/index.ts b/src/builtins/shell/index.ts index c78f668e2..0b894d64b 100644 --- a/src/builtins/shell/index.ts +++ b/src/builtins/shell/index.ts @@ -6,7 +6,7 @@ import {ExecutePlugin, PluginParams} from '../../types/interface'; import {ConfigParams} from '../../types/common'; import {validate} from '../../util/validations'; -import {ERRORS} from '../../util/errors'; +import {ERRORS} from '@grnsft/if-core'; const {ProcessExecutionError} = ERRORS; diff --git a/src/builtins/subtract/index.ts b/src/builtins/subtract/index.ts index 25f5d8522..87523082e 100644 --- a/src/builtins/subtract/index.ts +++ b/src/builtins/subtract/index.ts @@ -1,6 +1,6 @@ import {z} from 'zod'; -import {ERRORS} from '../../util/errors'; +import {ERRORS} from '@grnsft/if-core'; import {validate} from '../../util/validations'; import {STRINGS} from '../../config'; diff --git a/src/builtins/sum/index.ts b/src/builtins/sum/index.ts index 9944373a8..676829162 100644 --- a/src/builtins/sum/index.ts +++ b/src/builtins/sum/index.ts @@ -1,7 +1,7 @@ import {z} from 'zod'; import {validate} from '../../util/validations'; -import {ERRORS} from '../../util/errors'; +import {ERRORS} from '@grnsft/if-core'; import {STRINGS} from '../../config'; diff --git a/src/builtins/time-sync.ts b/src/builtins/time-sync.ts index 469fdec35..1542b7854 100644 --- a/src/builtins/time-sync.ts +++ b/src/builtins/time-sync.ts @@ -4,7 +4,7 @@ import {z} from 'zod'; import {parameterize} from '../lib/parameterize'; -import {ERRORS} from '../util/errors'; +import {ERRORS} from '@grnsft/if-core'; import {validate} from '../util/validations'; import {STRINGS} from '../config'; From 896553e0f077ec9b4b0482271884e15f57d6643f Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Thu, 13 Jun 2024 20:09:56 +0400 Subject: [PATCH 074/243] test(util): introduce if-core for errors --- src/__tests__/unit/util/aggregation-helper.test.ts | 3 ++- src/__tests__/unit/util/args.test.ts | 4 ++-- src/__tests__/unit/util/errors.test.ts | 2 +- src/__tests__/unit/util/helpers.test.ts | 2 +- src/__tests__/unit/util/plugin-storage.test.ts | 3 ++- 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/__tests__/unit/util/aggregation-helper.test.ts b/src/__tests__/unit/util/aggregation-helper.test.ts index 64add64e0..eb379e10f 100644 --- a/src/__tests__/unit/util/aggregation-helper.test.ts +++ b/src/__tests__/unit/util/aggregation-helper.test.ts @@ -1,5 +1,6 @@ +import {ERRORS} from '@grnsft/if-core'; + import {aggregateInputsIntoOne} from '../../../util/aggregation-helper'; -import {ERRORS} from '../../../util/errors'; import {STRINGS} from '../../../config'; diff --git a/src/__tests__/unit/util/args.test.ts b/src/__tests__/unit/util/args.test.ts index 7da9efa37..52e468987 100644 --- a/src/__tests__/unit/util/args.test.ts +++ b/src/__tests__/unit/util/args.test.ts @@ -69,10 +69,10 @@ jest.mock('ts-command-line-args', () => ({ }, })); -import path = require('path'); +import * as path from 'node:path'; +import {ERRORS} from '@grnsft/if-core'; import {parseIEProcessArgs, parseIfDiffArgs} from '../../../util/args'; -import {ERRORS} from '../../../util/errors'; import {STRINGS} from '../../../config'; diff --git a/src/__tests__/unit/util/errors.test.ts b/src/__tests__/unit/util/errors.test.ts index 47e3e9e9a..15d6ea52a 100644 --- a/src/__tests__/unit/util/errors.test.ts +++ b/src/__tests__/unit/util/errors.test.ts @@ -1,4 +1,4 @@ -import {ERRORS} from '../../../util/errors'; +import {ERRORS} from '@grnsft/if-core'; describe('util/errors: ', () => { describe('ERRORS: ', () => { diff --git a/src/__tests__/unit/util/helpers.test.ts b/src/__tests__/unit/util/helpers.test.ts index 58ed0525b..4aaccc5f5 100644 --- a/src/__tests__/unit/util/helpers.test.ts +++ b/src/__tests__/unit/util/helpers.test.ts @@ -10,6 +10,7 @@ jest.mock('../../../util/logger', () => ({ error: mockError, }, })); +import {ERRORS} from '@grnsft/if-core'; import { andHandle, checkIfEqual, @@ -18,7 +19,6 @@ import { oneIsPrimitive, parseManifestFromStdin, } from '../../../util/helpers'; -import {ERRORS} from '../../../util/errors'; import {Difference} from '../../../types/lib/compare'; const {WriteFileError} = ERRORS; diff --git a/src/__tests__/unit/util/plugin-storage.test.ts b/src/__tests__/unit/util/plugin-storage.test.ts index f8ac5b631..0555b1db5 100644 --- a/src/__tests__/unit/util/plugin-storage.test.ts +++ b/src/__tests__/unit/util/plugin-storage.test.ts @@ -1,5 +1,6 @@ +import {ERRORS} from '@grnsft/if-core'; + import {pluginStorage} from '../../../util/plugin-storage'; -import {ERRORS} from '../../../util/errors'; const {PluginInitializationError} = ERRORS; From 395737ac0017eb5b7044982d9c8cec2be5614fd2 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Thu, 13 Jun 2024 20:10:26 +0400 Subject: [PATCH 075/243] test(lib): introduce if-core for errors --- src/__tests__/unit/lib/exhaust.test.ts | 4 ++-- src/__tests__/unit/lib/initialize.test.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/__tests__/unit/lib/exhaust.test.ts b/src/__tests__/unit/lib/exhaust.test.ts index a1f46595a..1fa652bb3 100644 --- a/src/__tests__/unit/lib/exhaust.test.ts +++ b/src/__tests__/unit/lib/exhaust.test.ts @@ -1,9 +1,9 @@ /* eslint-disable @typescript-eslint/ban-ts-comment */ jest.mock('fs', () => require('../../../__mocks__/fs')); -import {exhaust} from '../../../lib/exhaust'; +import {ERRORS} from '@grnsft/if-core'; -import {ERRORS} from '../../../util/errors'; +import {exhaust} from '../../../lib/exhaust'; import {STRINGS} from '../../../config'; diff --git a/src/__tests__/unit/lib/initialize.test.ts b/src/__tests__/unit/lib/initialize.test.ts index 23ca27171..51d15f0d7 100644 --- a/src/__tests__/unit/lib/initialize.test.ts +++ b/src/__tests__/unit/lib/initialize.test.ts @@ -10,9 +10,9 @@ jest.mock('../../../util/log-memoize', () => ({ memoizedLog: mockLog, })); -import {initialize} from '../../../lib/initialize'; +import {ERRORS} from '@grnsft/if-core'; -import {ERRORS} from '../../../util/errors'; +import {initialize} from '../../../lib/initialize'; import {STRINGS} from '../../../config'; From e39478132bd643b56334f8db4bfe479e693989f8 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Thu, 13 Jun 2024 20:13:14 +0400 Subject: [PATCH 076/243] test(builtins): introduce if-core for errors --- src/__tests__/unit/builtins/CommonGenerator.test.ts | 3 ++- src/__tests__/unit/builtins/RandIntGenerator.test.ts | 6 +++--- src/__tests__/unit/builtins/coefficient.test.ts | 4 ++-- src/__tests__/unit/builtins/csv-lookup.test.ts | 2 +- src/__tests__/unit/builtins/divide.test.ts | 3 ++- src/__tests__/unit/builtins/exponent.test.ts | 3 ++- src/__tests__/unit/builtins/export-csv-raw.test.ts | 2 +- src/__tests__/unit/builtins/export-csv.test.ts | 2 +- src/__tests__/unit/builtins/export-yaml.test.ts | 3 ++- src/__tests__/unit/builtins/group-by.test.ts | 3 ++- src/__tests__/unit/builtins/interpolation.test.ts | 3 ++- src/__tests__/unit/builtins/mock-observations.test.ts | 3 ++- src/__tests__/unit/builtins/multiply.test.ts | 3 ++- src/__tests__/unit/builtins/regex.test.ts | 3 ++- src/__tests__/unit/builtins/sci-embodied.test.ts | 3 ++- src/__tests__/unit/builtins/sci.test.ts | 4 ++-- src/__tests__/unit/builtins/shell.test.ts | 2 +- src/__tests__/unit/builtins/subtract.test.ts | 3 ++- src/__tests__/unit/builtins/sum.test.ts | 3 ++- src/__tests__/unit/builtins/time-sync.test.ts | 3 +-- 20 files changed, 36 insertions(+), 25 deletions(-) diff --git a/src/__tests__/unit/builtins/CommonGenerator.test.ts b/src/__tests__/unit/builtins/CommonGenerator.test.ts index d4d1bdac9..23109c117 100644 --- a/src/__tests__/unit/builtins/CommonGenerator.test.ts +++ b/src/__tests__/unit/builtins/CommonGenerator.test.ts @@ -1,6 +1,7 @@ -import {ERRORS} from '../../../util/errors'; +import {ERRORS} from '@grnsft/if-core'; import {CommonGenerator} from '../../../builtins/mock-observations/helpers/common-generator'; + import {STRINGS} from '../../../config'; const {GlobalConfigError} = ERRORS; diff --git a/src/__tests__/unit/builtins/RandIntGenerator.test.ts b/src/__tests__/unit/builtins/RandIntGenerator.test.ts index 1d2969695..ca1b60183 100644 --- a/src/__tests__/unit/builtins/RandIntGenerator.test.ts +++ b/src/__tests__/unit/builtins/RandIntGenerator.test.ts @@ -1,11 +1,11 @@ -import {KeyValuePair} from '../../../types/common'; - -import {ERRORS} from '../../../util/errors'; +import {ERRORS} from '@grnsft/if-core'; import {RandIntGenerator} from '../../../builtins/mock-observations/helpers/rand-int-generator'; import {STRINGS} from '../../../config'; +import {KeyValuePair} from '../../../types/common'; + const {GlobalConfigError} = ERRORS; const {INVALID_NAME, MISSING_MIN_MAX, MISSING_GLOBAL_CONFIG} = STRINGS; diff --git a/src/__tests__/unit/builtins/coefficient.test.ts b/src/__tests__/unit/builtins/coefficient.test.ts index 44595d774..e7e92adf5 100644 --- a/src/__tests__/unit/builtins/coefficient.test.ts +++ b/src/__tests__/unit/builtins/coefficient.test.ts @@ -1,6 +1,6 @@ -import {Coefficient} from '../../../builtins/coefficient'; +import {ERRORS} from '@grnsft/if-core'; -import {ERRORS} from '../../../util/errors'; +import {Coefficient} from '../../../builtins/coefficient'; import {STRINGS} from '../../../config'; diff --git a/src/__tests__/unit/builtins/csv-lookup.test.ts b/src/__tests__/unit/builtins/csv-lookup.test.ts index 8fbdee401..e887f5e4b 100644 --- a/src/__tests__/unit/builtins/csv-lookup.test.ts +++ b/src/__tests__/unit/builtins/csv-lookup.test.ts @@ -2,12 +2,12 @@ jest.mock('fs/promises', () => require('../../../__mocks__/fs')); import axios from 'axios'; import AxiosMockAdapter from 'axios-mock-adapter'; +import {ERRORS} from '@grnsft/if-core'; import {CSVLookup} from '../../../builtins'; import {STRINGS} from '../../../config'; -import {ERRORS} from '../../../util/errors'; const { GlobalConfigError, diff --git a/src/__tests__/unit/builtins/divide.test.ts b/src/__tests__/unit/builtins/divide.test.ts index 9c794dae6..c75c926be 100644 --- a/src/__tests__/unit/builtins/divide.test.ts +++ b/src/__tests__/unit/builtins/divide.test.ts @@ -1,6 +1,7 @@ +import {ERRORS} from '@grnsft/if-core'; + import {Divide} from '../../../builtins'; -import {ERRORS} from '../../../util/errors'; import {STRINGS} from '../../../config'; const {InputValidationError, GlobalConfigError, MissingInputDataError} = ERRORS; diff --git a/src/__tests__/unit/builtins/exponent.test.ts b/src/__tests__/unit/builtins/exponent.test.ts index 6034fd3bd..04d9f7e9b 100644 --- a/src/__tests__/unit/builtins/exponent.test.ts +++ b/src/__tests__/unit/builtins/exponent.test.ts @@ -1,6 +1,7 @@ +import {ERRORS} from '@grnsft/if-core'; + import {Exponent} from '../../../builtins/exponent'; -import {ERRORS} from '../../../util/errors'; import {STRINGS} from '../../../config'; const {InputValidationError, MissingInputDataError} = ERRORS; diff --git a/src/__tests__/unit/builtins/export-csv-raw.test.ts b/src/__tests__/unit/builtins/export-csv-raw.test.ts index a66a24ac2..5ff8d20cd 100644 --- a/src/__tests__/unit/builtins/export-csv-raw.test.ts +++ b/src/__tests__/unit/builtins/export-csv-raw.test.ts @@ -1,8 +1,8 @@ import * as fs from 'fs/promises'; import {jest} from '@jest/globals'; +import {ERRORS} from '@grnsft/if-core'; import {ExportCSVRaw} from '../../../builtins/export-csv-raw'; -import {ERRORS} from '../../../util/errors'; import {STRINGS} from '../../../config'; diff --git a/src/__tests__/unit/builtins/export-csv.test.ts b/src/__tests__/unit/builtins/export-csv.test.ts index 09fd4e737..942732e2e 100644 --- a/src/__tests__/unit/builtins/export-csv.test.ts +++ b/src/__tests__/unit/builtins/export-csv.test.ts @@ -1,9 +1,9 @@ import * as fs from 'fs/promises'; import {stringify} from 'csv-stringify/sync'; import {jest} from '@jest/globals'; +import {ERRORS} from '@grnsft/if-core'; import {ExportCSV} from '../../../builtins/export-csv'; -import {ERRORS} from '../../../util/errors'; import {STRINGS} from '../../../config'; diff --git a/src/__tests__/unit/builtins/export-yaml.test.ts b/src/__tests__/unit/builtins/export-yaml.test.ts index f17b9154e..56b155cba 100644 --- a/src/__tests__/unit/builtins/export-yaml.test.ts +++ b/src/__tests__/unit/builtins/export-yaml.test.ts @@ -1,5 +1,6 @@ +import {ERRORS} from '@grnsft/if-core'; + import {ExportYaml} from '../../../builtins/export-yaml'; -import {ERRORS} from '../../../util/errors'; import {saveYamlFileAs} from '../../../util/yaml'; import {STRINGS} from '../../../config'; diff --git a/src/__tests__/unit/builtins/group-by.test.ts b/src/__tests__/unit/builtins/group-by.test.ts index b472125d1..22ef77f56 100644 --- a/src/__tests__/unit/builtins/group-by.test.ts +++ b/src/__tests__/unit/builtins/group-by.test.ts @@ -1,5 +1,6 @@ +import {ERRORS} from '@grnsft/if-core'; + import {GroupBy} from '../../../builtins/group-by'; -import {ERRORS} from '../../../util/errors'; import {STRINGS} from '../../../config'; diff --git a/src/__tests__/unit/builtins/interpolation.test.ts b/src/__tests__/unit/builtins/interpolation.test.ts index 54dd2dab7..ead5880bb 100644 --- a/src/__tests__/unit/builtins/interpolation.test.ts +++ b/src/__tests__/unit/builtins/interpolation.test.ts @@ -1,5 +1,6 @@ +import {ERRORS} from '@grnsft/if-core'; + import {Interpolation} from '../../../builtins'; -import {ERRORS} from '../../../util/errors'; import {Method} from '../../../builtins/interpolation/types'; import {STRINGS} from '../../../config'; diff --git a/src/__tests__/unit/builtins/mock-observations.test.ts b/src/__tests__/unit/builtins/mock-observations.test.ts index 3e2ddab5b..7a46cf79f 100644 --- a/src/__tests__/unit/builtins/mock-observations.test.ts +++ b/src/__tests__/unit/builtins/mock-observations.test.ts @@ -1,6 +1,7 @@ +import {ERRORS} from '@grnsft/if-core'; + import {MockObservations} from '../../../builtins/mock-observations'; -import {ERRORS} from '../../../util/errors'; import {STRINGS} from '../../../config'; const {InputValidationError, GlobalConfigError} = ERRORS; diff --git a/src/__tests__/unit/builtins/multiply.test.ts b/src/__tests__/unit/builtins/multiply.test.ts index 7efceb325..ff168110e 100644 --- a/src/__tests__/unit/builtins/multiply.test.ts +++ b/src/__tests__/unit/builtins/multiply.test.ts @@ -1,6 +1,7 @@ +import {ERRORS} from '@grnsft/if-core'; + import {Multiply} from '../../../builtins/multiply'; -import {ERRORS} from '../../../util/errors'; import {STRINGS} from '../../../config'; const {MissingInputDataError} = ERRORS; diff --git a/src/__tests__/unit/builtins/regex.test.ts b/src/__tests__/unit/builtins/regex.test.ts index 462faad8f..57cdc5e0c 100644 --- a/src/__tests__/unit/builtins/regex.test.ts +++ b/src/__tests__/unit/builtins/regex.test.ts @@ -1,6 +1,7 @@ +import {ERRORS} from '@grnsft/if-core'; + import {Regex} from '../../../builtins/regex'; -import {ERRORS} from '../../../util/errors'; import {STRINGS} from '../../../config'; const {GlobalConfigError, MissingInputDataError, RegexMismatchError} = ERRORS; diff --git a/src/__tests__/unit/builtins/sci-embodied.test.ts b/src/__tests__/unit/builtins/sci-embodied.test.ts index 4b9f88deb..1d8f05438 100644 --- a/src/__tests__/unit/builtins/sci-embodied.test.ts +++ b/src/__tests__/unit/builtins/sci-embodied.test.ts @@ -1,5 +1,6 @@ +import {ERRORS} from '@grnsft/if-core'; + import {SciEmbodied} from '../../../builtins/sci-embodied'; -import {ERRORS} from '../../../util/errors'; import {STRINGS} from '../../../config'; diff --git a/src/__tests__/unit/builtins/sci.test.ts b/src/__tests__/unit/builtins/sci.test.ts index a9401a5a8..751804f17 100644 --- a/src/__tests__/unit/builtins/sci.test.ts +++ b/src/__tests__/unit/builtins/sci.test.ts @@ -1,6 +1,6 @@ -import {Sci} from '../../../builtins/sci'; +import {ERRORS} from '@grnsft/if-core'; -import {ERRORS} from '../../../util/errors'; +import {Sci} from '../../../builtins/sci'; const {MissingInputDataError} = ERRORS; diff --git a/src/__tests__/unit/builtins/shell.test.ts b/src/__tests__/unit/builtins/shell.test.ts index 306dada3c..ce5b4aea7 100644 --- a/src/__tests__/unit/builtins/shell.test.ts +++ b/src/__tests__/unit/builtins/shell.test.ts @@ -1,9 +1,9 @@ import {spawnSync} from 'child_process'; import {loadAll} from 'js-yaml'; +import {ERRORS} from '@grnsft/if-core'; import {Shell} from '../../../builtins/shell'; -import {ERRORS} from '../../../util/errors'; const {InputValidationError, ProcessExecutionError} = ERRORS; diff --git a/src/__tests__/unit/builtins/subtract.test.ts b/src/__tests__/unit/builtins/subtract.test.ts index 2179adcd4..a1c1e3302 100644 --- a/src/__tests__/unit/builtins/subtract.test.ts +++ b/src/__tests__/unit/builtins/subtract.test.ts @@ -1,6 +1,7 @@ +import {ERRORS} from '@grnsft/if-core'; + import {Subtract} from '../../../builtins/subtract'; -import {ERRORS} from '../../../util/errors'; import {STRINGS} from '../../../config'; const {InputValidationError} = ERRORS; diff --git a/src/__tests__/unit/builtins/sum.test.ts b/src/__tests__/unit/builtins/sum.test.ts index e7b515f26..7a09958d1 100644 --- a/src/__tests__/unit/builtins/sum.test.ts +++ b/src/__tests__/unit/builtins/sum.test.ts @@ -1,6 +1,7 @@ +import {ERRORS} from '@grnsft/if-core'; + import {Sum} from '../../../builtins/sum'; -import {ERRORS} from '../../../util/errors'; import {STRINGS} from '../../../config'; const {GlobalConfigError, MissingInputDataError} = ERRORS; diff --git a/src/__tests__/unit/builtins/time-sync.test.ts b/src/__tests__/unit/builtins/time-sync.test.ts index ac627fe02..63174743a 100644 --- a/src/__tests__/unit/builtins/time-sync.test.ts +++ b/src/__tests__/unit/builtins/time-sync.test.ts @@ -1,9 +1,8 @@ +import {ERRORS} from '@grnsft/if-core'; import {Settings, DateTime} from 'luxon'; import {TimeSync} from '../../../builtins/time-sync'; -import {ERRORS} from '../../../util/errors'; - import {STRINGS} from '../../../config'; Settings.defaultZone = 'utc'; From f95d21c7589381921c1a387b1cf857344b602471 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Thu, 13 Jun 2024 20:13:40 +0400 Subject: [PATCH 077/243] feat(package): install `@grnsft/if-core` dependency --- package-lock.json | 13 +++++++++++++ package.json | 1 + 2 files changed, 14 insertions(+) diff --git a/package-lock.json b/package-lock.json index 1c2c47735..2ec34c515 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "dependencies": { "@commitlint/cli": "^18.6.0", "@commitlint/config-conventional": "^18.6.0", + "@grnsft/if-core": "^0.0.3", "axios": "^1.7.2", "csv-parse": "^5.5.6", "csv-stringify": "^6.4.6", @@ -1179,6 +1180,18 @@ "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, + "node_modules/@grnsft/if-core": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/@grnsft/if-core/-/if-core-0.0.3.tgz", + "integrity": "sha512-ieuUcadEgA4EztIzCbCW5ifK2a6SUU2ioZtLTce8pJ+imgeQ47lQM3UtChtbaUm52rEwvgl1Psp0ysgOjqssFA==", + "dependencies": { + "typescript": "^5.1.6" + }, + "engines": { + "node": ">=18", + "npm": ">=8" + } + }, "node_modules/@humanwhocodes/config-array": { "version": "0.11.13", "dev": true, diff --git a/package.json b/package.json index c6db34155..5093d2235 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "dependencies": { "@commitlint/cli": "^18.6.0", "@commitlint/config-conventional": "^18.6.0", + "@grnsft/if-core": "^0.0.3", "axios": "^1.7.2", "csv-parse": "^5.5.6", "csv-stringify": "^6.4.6", From 163df83594b4ab6be79297e7fee70750089c48f1 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Thu, 13 Jun 2024 20:16:01 +0400 Subject: [PATCH 078/243] revert(util): drop errors support --- src/util/errors.ts | 49 ---------------------------------------------- 1 file changed, 49 deletions(-) delete mode 100644 src/util/errors.ts diff --git a/src/util/errors.ts b/src/util/errors.ts deleted file mode 100644 index f4f067d77..000000000 --- a/src/util/errors.ts +++ /dev/null @@ -1,49 +0,0 @@ -const CUSTOM_ERRORS = [ - 'CliInputError', - 'ManifestValidationError', - 'InputValidationError', - 'InvalidGroupingError', - 'WriteFileError', - /** More specific errors */ - 'ParseCliParamsError', - 'CliSourceFileError', - 'CliTargetFileError', - 'InvalidAggregationMethodError', - 'MissingAggregationParamError', - 'MissingPluginMethodError', - 'MissingPluginPathError', - 'PluginInitializationError', - 'InvalidExhaustPluginError', - /** Plugins */ - 'GlobalConfigError', - 'MissingInputDataError', - 'ProcessExecutionError', - 'RegexMismatchError', - 'FetchingFileError', - 'ReadFileError', - 'MissingCSVColumnError', - 'QueryDataNotFoundError', - 'InvalidDateInInputError', - 'InvalidPaddingError', - 'InvalidInputError', - 'ExhaustOutputArgError', - 'CSVParseError', -] as const; - -type CustomErrors = { - [K in (typeof CUSTOM_ERRORS)[number]]: ErrorConstructor; -}; - -export const ERRORS = CUSTOM_ERRORS.reduce((acc, className) => { - acc = { - ...acc, - [className]: class extends Error { - constructor(message: string) { - super(message); - this.name = this.constructor.name; - } - }, - }; - - return acc; -}, {} as CustomErrors); From 98e825a02f7995dfaaa727fe0baf683812e5b90b Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Thu, 13 Jun 2024 20:17:02 +0400 Subject: [PATCH 079/243] chore(builtins): remove redundant newlines --- src/__tests__/unit/builtins/csv-lookup.test.ts | 1 - src/__tests__/unit/builtins/shell.test.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/src/__tests__/unit/builtins/csv-lookup.test.ts b/src/__tests__/unit/builtins/csv-lookup.test.ts index e887f5e4b..9e718d537 100644 --- a/src/__tests__/unit/builtins/csv-lookup.test.ts +++ b/src/__tests__/unit/builtins/csv-lookup.test.ts @@ -8,7 +8,6 @@ import {CSVLookup} from '../../../builtins'; import {STRINGS} from '../../../config'; - const { GlobalConfigError, ReadFileError, diff --git a/src/__tests__/unit/builtins/shell.test.ts b/src/__tests__/unit/builtins/shell.test.ts index ce5b4aea7..95f44fb12 100644 --- a/src/__tests__/unit/builtins/shell.test.ts +++ b/src/__tests__/unit/builtins/shell.test.ts @@ -4,7 +4,6 @@ import {ERRORS} from '@grnsft/if-core'; import {Shell} from '../../../builtins/shell'; - const {InputValidationError, ProcessExecutionError} = ERRORS; jest.mock('child_process'); From 434cde45f5f3c5421f51754a0398ec9582227908 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Thu, 13 Jun 2024 20:17:23 +0400 Subject: [PATCH 080/243] test(util): drop errors --- src/__tests__/unit/util/errors.test.ts | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 src/__tests__/unit/util/errors.test.ts diff --git a/src/__tests__/unit/util/errors.test.ts b/src/__tests__/unit/util/errors.test.ts deleted file mode 100644 index 15d6ea52a..000000000 --- a/src/__tests__/unit/util/errors.test.ts +++ /dev/null @@ -1,13 +0,0 @@ -import {ERRORS} from '@grnsft/if-core'; - -describe('util/errors: ', () => { - describe('ERRORS: ', () => { - it('checks for properties.', () => { - const errors = Object.values(ERRORS); - - errors.forEach(error => { - expect(error).toBeInstanceOf(Function); - }); - }); - }); -}); From 6743cba3cc25f505b689fc0ad6df9efe7e112f89 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Thu, 13 Jun 2024 20:17:53 +0400 Subject: [PATCH 081/243] fix(lib): remove redundant exhaust newline --- src/lib/exhaust.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/lib/exhaust.ts b/src/lib/exhaust.ts index d60268698..a4982681d 100644 --- a/src/lib/exhaust.ts +++ b/src/lib/exhaust.ts @@ -8,7 +8,6 @@ import {ExportCSVRaw} from '../builtins/export-csv-raw'; import {ExportLog} from '../builtins/export-log'; import {ExportYaml} from '../builtins/export-yaml'; - import {STRINGS} from '../config'; import {ExhaustPluginInterface} from '../types/exhaust-plugin-interface'; From 474a46589612b4312df38a28ca25a8284befec41 Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Fri, 14 Jun 2024 09:23:39 +0100 Subject: [PATCH 082/243] fix(package): rm if-plugins from readme --- README.md | 44 ++++++++++++-------------------------------- 1 file changed, 12 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index c34bf0069..2be7911fa 100644 --- a/README.md +++ b/README.md @@ -5,10 +5,6 @@ > [!IMPORTANT] > Incubation Project: This project is an incubation project being run inside the Green Software Foundation; as such, we DON’T recommend using it in any critical use case. Incubation projects are experimental, offer no support guarantee, have minimal governance and process, and may be retired at any moment. This project may one day Graduate, in which case this disclaimer will be removed. -**Note** We have recently (March 2024) refactored the IF codebase and introduced some changes affecting both users and developers. You can read our migration guide [HERE](./Refactor-migration-guide.md) to help you update to the latest version! - -[Impact Framework](https://greensoftwarefoundation.atlassian.net/wiki/spaces/~612dd45e45cd76006a84071a/pages/17072136/Opensource+Impact+Engine+Framework) (IF) is an [Incubation](https://oc.greensoftware.foundation/project-lifecycle.html#incubation) project from the [Open Source Working Group](https://greensoftwarefoundation.atlassian.net/wiki/spaces/~612dd45e45cd76006a84071a/pages/852049/Open+Source+Working+Group) in the [Green Software Foundation](https://greensoftware.foundation/). - **Our documentation is online at [if.greensoftware.foundation](https://if.greensoftware.foundation/)** **IF** is a framework to **M**odel, **M**easure, si**M**ulate and **M**onitor the environmental impacts of software @@ -23,39 +19,31 @@ Read the [specification and design docs](https://if.greensoftware.foundation) to ## Get started -The first thing to understand is that IF is a framework for running plugins. This means that in order to do some calculations, you need to load some plugins from some external resource. We provide a [standard library of plugins](https://github.com/Green-Software-Foundation/if-plugins) and a repository of [community plugins](https://github.com/Green-Software-Foundation/if-unofficial-plugins) to get you started. +IF is a framework for running pipelines of plugins that operate on a set of observations. This is all configured using a manifest file. We provide a standard library of plugins that come bundled with IF - we refer to these as `builtins`. We also have an [Explorer](https://explorer.if.greensoftware.foundation) where anyone can list third party plugins you can install. -Start by installing framework itself: +Start by installing the latest version of IF: ```sh npm install -g "@grnsft/if" ``` -Then installing some plugins: - -```sh -npm install -g "@grnsft/if-plugins" -``` - Then create a `manifest` file that describes your application (see our docs for a detailed explanation). Then, run `if` using the following command: ```sh -ie --manifest +if-run --manifest --stdout ## or you can use aliases -ie -m +if-run -m --stdout ``` -Note that above command will not print the final output. In order to print the final output to the console, run `if` using the optional stdout argument: +Note that above command will print your outputs to the console. You can also provide the `--output` command to save your outputs to a yaml file: ```sh -ie --manifest --stdout -## or using aliases -ie -m -s +if-run -m -o ``` -You can also add a savepath for your output yaml in order to have the output stored in a file. Note that you also have to add configuration to your manifest to enable this, as follows: +Note that you also have to add configuration to your manifest to enable this, as follows: ```yaml initialize: @@ -63,34 +51,27 @@ initialize: - yaml ``` -On the command line: - -```sh -ie --manifest --output -## or using aliases -ie -m -o -``` - The `ie` CLI tool will configure and run the plugins defined in your input `yaml` (`manifest`) and return the results as an output `yaml` (`output`). Use the `debug` command if you want to diagnose and fix errors in your plugin: ```sh -ie --manifest --debug +if-run --manifest --debug ``` Use the `help` command if you need guidance about the available commands ```sh -ie --help +if-run --help ## or using alias -ie -h +if-run -h ``` ## Documentation Please read our documentation at [if.greensoftware.foundation](https://if.greensoftware.foundation/) + ## Video walk-through Watch this video to learn how to create and run a `manifest`. @@ -107,11 +88,10 @@ We have a public mailing list at [if-community@greensoftware.foundation](https:/ To contribute to IF, please fork this repository and raise a pull request from your fork. -You can check our issue board for issues tagged `help-wanted`. These are issues that are not currently, actively being worked on by the core team but are well-scoped enough for someone to pick up. We recommend commenting on the issue to start a chat with the core team, then start working on the issue when you have been assigned to it. This process helps to ensure your work is aligned with our roadmap and makes it much more likely that your changes will get merged compared to unsolicited PRs. +You can check our issue board for issues. We mark some issues `core-only` if they are somehow sensitive and we want one of our core developers to handle it. Any other issues are open for the community to work on. We recommend commenting on the issue to start a chat with the core team, then start working on the issue when you have been assigned to it. This process helps to ensure your work is aligned with our roadmap and makes it much more likely that your changes will get merged compared to unsolicited PRs. Please read the full contribution guidelines at [if.greensoftware.foundation](https://if.greensoftware.foundation/Contributing) -The same guidelines also apply to `if-docs`, `if-plugins` and `if-unofficial-plugins`. ## Bug reports From 5ec4d5a25739f5f7458a7b65c4ffad6c8e8d4c1a Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Fri, 14 Jun 2024 10:25:00 +0100 Subject: [PATCH 083/243] fix(lib): small fix to contributing guide --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3974becb0..841d69272 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,7 +24,7 @@ The following document is a rule set of guidelines for contributing. ## What and when to contribute -You can contribute anything to the IF, but we are likely to close out unsolicited PRs without merging them. Our issue board is completely open and we have tags (`help-wanted`, `good-first-issue`) to help contributors to choose tasks to work on. We recommend speaking to the core team on Github before starting working on an issue. You can do this by raising an issue or commenting on an existing issue. This helps us to direct your energy in directions that are aligned with our roadmap, prevent multiple people working on the same task, and better manage our board. This all makes it much more likely that your work will get merged. +You can contribute anything to the IF, but we are likely to close out unsolicited PRs without merging them. Our issue board is completely open and we have tags (`core-only`, `good-first-issue`) to help contributors to choose tasks to work on. If an issue is unassigned and does not have the `core-only` label, it is available to work on. We recommend speaking to the core team on Github before starting working on an issue. You can do this by commenting on an existing issue or discussion thread or starting a new one if appropriate. This helps us to direct your energy in directions that are aligned with our roadmap, prevent multiple people working on the same task, and better manage our board. This all makes it much more likely that your work will get merged. You can also contribute by participating in discussions on our mailing list at [if-community@greensoftware.foundation](https://groups.google.com/u/1/a/greensoftware.foundation/g/if-community). We send out weekly updates that includes what we've shipped, what we're working on and how you can get involved each week. From 1ddb2f2bec24d759261898843736fca1d0a70856 Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Fri, 14 Jun 2024 10:30:33 +0100 Subject: [PATCH 084/243] fix(package): remove refs to if-plugins --- CONTRIBUTING.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 841d69272..d1ea0db53 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,9 +30,9 @@ You can also contribute by participating in discussions on our mailing list at [ ## Reporting bugs -We appreciate bug reports! If you experience an issue with IF or one of our plugins, you can report it using our bug reporting template. To do this: +We appreciate bug reports! If you experience an issue with IF, you can report it using our bug reporting template. To do this: -1. Go to the [IF repository](https://github.com/Green-Software-Foundation/if) (or [plugin repository](https://github.com/Green-Software-Foundation/if-plugins) if you bug relates to a specific plugin) +1. Go to the [IF repository](https://github.com/Green-Software-Foundation/if) 2. Click on the `Issues` tab 3. Click on `Create New Issue` and select the `Bug Report` template. 4. Fill out the requested information. @@ -46,8 +46,7 @@ The assessment rubric is as follows: | | Consequence | Severity | | ---------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------- | -------- | | Bugs in IF core leading to incorrect calculations | unusable framework | 5 | -| Bugs in if-plugins leading to incorrect calculations | core pathways fail, IF very limited in functionality | 5 | -| Bugs in if-unofficial-pluginsd leading to incorrect calculations | Third party plugins harder to use, limits IF to standard lib | 3 | +| Bugs in builtins leading to incorrect calculations | core pathways fail, IF very limited in functionality | 5 | | Bugs in template | Harder to build plugins, ecosystem growth is impacted | 2 | | Bugs in docs | product does not match expectation, hard to debug, frustration, loss of adoption | 2 | | Security flaw: privacy related | leak user data, unlikely to achieve adoption in serious orgs | 5 | From 015bf60b8cd4460ec69d8b9bbf46a5a7e57d36bc Mon Sep 17 00:00:00 2001 From: Joseph Cook <33655003+jmcook1186@users.noreply.github.com> Date: Fri, 14 Jun 2024 11:02:48 +0100 Subject: [PATCH 085/243] Update README.md Co-authored-by: Manushak Keramyan Signed-off-by: Joseph Cook <33655003+jmcook1186@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2be7911fa..c72600b91 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ initialize: - yaml ``` -The `ie` CLI tool will configure and run the plugins defined in your input `yaml` (`manifest`) and return the results as an output `yaml` (`output`). +The `if-run` CLI tool will configure and run the plugins defined in your input `yaml` (`manifest`) and return the results as an output `yaml` (`output`). Use the `debug` command if you want to diagnose and fix errors in your plugin: From 2fdb5dcdf6c6b31005ef79ceafe004feb011156e Mon Sep 17 00:00:00 2001 From: Joseph Cook <33655003+jmcook1186@users.noreply.github.com> Date: Fri, 14 Jun 2024 11:09:02 +0100 Subject: [PATCH 086/243] Update README.md Co-authored-by: Narek Hovhannisyan Signed-off-by: Joseph Cook <33655003+jmcook1186@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d44dd1cd6..6fe0f67a4 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ Then, run `if` using the following command: ```sh if-run --manifest --stdout ## or you can use aliases -if-run -m --stdout +if-run -m -s ``` From 4e1ee62015ecfd63afaf005e48d45a57888498a3 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Fri, 14 Jun 2024 16:52:29 +0400 Subject: [PATCH 087/243] feat(util): introduce if-core utils and types --- src/util/aggregation-helper.ts | 4 ++-- src/util/args.ts | 2 +- src/util/helpers.ts | 2 +- src/util/plugin-storage.ts | 2 +- src/util/validations.ts | 6 +++--- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/util/aggregation-helper.ts b/src/util/aggregation-helper.ts index 3caff8c6d..408a4e9f7 100644 --- a/src/util/aggregation-helper.ts +++ b/src/util/aggregation-helper.ts @@ -1,11 +1,11 @@ -import {ERRORS} from '@grnsft/if-core'; +import {ERRORS} from '@grnsft/if-core/utils'; +import {PluginParams} from '@grnsft/if-core/types'; import {parameterize} from '../lib/parameterize'; import {CONFIG, STRINGS} from '../config'; import {AggregationResult} from '../types/aggregation'; -import {PluginParams} from '../types/interface'; const {InvalidAggregationMethodError, MissingAggregationParamError} = ERRORS; const {INVALID_AGGREGATION_METHOD, METRIC_MISSING} = STRINGS; diff --git a/src/util/args.ts b/src/util/args.ts index ee090c1ef..c54117c75 100644 --- a/src/util/args.ts +++ b/src/util/args.ts @@ -1,7 +1,7 @@ import * as path from 'path'; import {parse} from 'ts-command-line-args'; -import {ERRORS} from '@grnsft/if-core'; +import {ERRORS} from '@grnsft/if-core/utils'; import {checkIfFileIsYaml} from './yaml'; import {logger} from './logger'; diff --git a/src/util/helpers.ts b/src/util/helpers.ts index 45af8b50c..fcab2e27a 100644 --- a/src/util/helpers.ts +++ b/src/util/helpers.ts @@ -2,7 +2,7 @@ import {createInterface} from 'node:readline/promises'; import {exec} from 'node:child_process'; import {promisify} from 'node:util'; -import {ERRORS} from '@grnsft/if-core'; +import {ERRORS} from '@grnsft/if-core/utils'; import {logger} from './logger'; diff --git a/src/util/plugin-storage.ts b/src/util/plugin-storage.ts index da1e45970..1f9711943 100644 --- a/src/util/plugin-storage.ts +++ b/src/util/plugin-storage.ts @@ -1,4 +1,4 @@ -import {ERRORS} from '@grnsft/if-core'; +import {ERRORS} from '@grnsft/if-core/utils'; import {STRINGS} from '../config'; diff --git a/src/util/validations.ts b/src/util/validations.ts index 30cb1e930..920e7cbdb 100644 --- a/src/util/validations.ts +++ b/src/util/validations.ts @@ -1,11 +1,11 @@ import {ZodIssue, ZodIssueCode, ZodSchema, z} from 'zod'; -import {ERRORS} from '@grnsft/if-core'; +import {ERRORS} from '@grnsft/if-core/utils'; + +import {STRINGS} from '../config/strings'; import {AGGREGATION_METHODS} from '../types/aggregation'; import {AGGREGATION_TYPES} from '../types/parameters'; -import {STRINGS} from '../config/strings'; - const {ManifestValidationError, InputValidationError} = ERRORS; const {VALIDATING_MANIFEST} = STRINGS; From 8d1244ac8770c0291ee2d15521a7f4ddfa194d87 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Fri, 14 Jun 2024 16:53:01 +0400 Subject: [PATCH 088/243] revert(types): drop common and group-by --- src/types/common.ts | 5 ----- src/types/group-by.ts | 3 --- 2 files changed, 8 deletions(-) delete mode 100644 src/types/common.ts delete mode 100644 src/types/group-by.ts diff --git a/src/types/common.ts b/src/types/common.ts deleted file mode 100644 index f7d337392..000000000 --- a/src/types/common.ts +++ /dev/null @@ -1,5 +0,0 @@ -export type KeyValuePair = { - [key: string]: any; -}; - -export type ConfigParams = Record; diff --git a/src/types/group-by.ts b/src/types/group-by.ts deleted file mode 100644 index 72dd6d61b..000000000 --- a/src/types/group-by.ts +++ /dev/null @@ -1,3 +0,0 @@ -export type GroupByConfig = { - group: string[]; -}; From 9ee5885b886731673866c46f3f7b1cffa95e496b Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Fri, 14 Jun 2024 16:53:22 +0400 Subject: [PATCH 089/243] feat(types): introduce if-core types --- src/types/compute.ts | 3 ++- src/types/interface.ts | 25 +------------------------ 2 files changed, 3 insertions(+), 25 deletions(-) diff --git a/src/types/compute.ts b/src/types/compute.ts index 739d86742..2de3e6e33 100644 --- a/src/types/compute.ts +++ b/src/types/compute.ts @@ -1,4 +1,5 @@ -import {PluginParams} from './interface'; +import {PluginParams} from '@grnsft/if-core/types'; + import {Context} from './manifest'; import {PluginStorageInterface} from './plugin-storage'; diff --git a/src/types/interface.ts b/src/types/interface.ts index f06449bc6..6834a3ebb 100644 --- a/src/types/interface.ts +++ b/src/types/interface.ts @@ -1,27 +1,4 @@ -import {GroupByConfig} from './group-by'; - -export type PluginParams = Record; - -export type ConfigParams = Record; - -export type ExecutePlugin = { - execute: ( - inputs: PluginParams[], - config?: Record - ) => PluginParams[] | Promise; - metadata: { - kind: string; - }; - [key: string]: any; -}; - -export type GroupByPlugin = { - execute: (inputs: PluginParams[], config: GroupByConfig) => {children: any}; - metadata: { - kind: string; - }; - [key: string]: any; -}; +import {ExecutePlugin, GroupByPlugin} from '@grnsft/if-core/types'; export type PluginInterface = ExecutePlugin | GroupByPlugin; From 27c74503d78e79ece685d43c4fff7dcc255e03ba Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Fri, 14 Jun 2024 17:43:28 +0400 Subject: [PATCH 090/243] feat(lib): introduce if-core utils and types --- src/lib/aggregate.ts | 7 ++++--- src/lib/compute.ts | 5 +++-- src/lib/exhaust.ts | 2 +- src/lib/initialize.ts | 2 +- src/lib/load.ts | 2 +- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/lib/aggregate.ts b/src/lib/aggregate.ts index 977f30300..c6eef11cc 100644 --- a/src/lib/aggregate.ts +++ b/src/lib/aggregate.ts @@ -1,10 +1,11 @@ -import {aggregateInputsIntoOne} from '../util/aggregation-helper'; +import {PluginParams} from '@grnsft/if-core/types'; -import {PluginParams} from '../types/interface'; -import {AggregationParams, AggregationParamsSure} from '../types/manifest'; +import {aggregateInputsIntoOne} from '../util/aggregation-helper'; import {STRINGS} from '../config/strings'; +import {AggregationParams, AggregationParamsSure} from '../types/manifest'; + const {AGGREGATING_NODE, AGGREGATING_OUTPUTS} = STRINGS; /** diff --git a/src/lib/compute.ts b/src/lib/compute.ts index 40db1a42a..b83a22d2e 100644 --- a/src/lib/compute.ts +++ b/src/lib/compute.ts @@ -1,9 +1,10 @@ +import {PluginParams, GroupByConfig} from '@grnsft/if-core/types'; + import {debugLogger} from '../util/debug-logger'; import {mergeObjects} from '../util/helpers'; import {ComputeParams, Node, Params} from '../types/compute'; -import {PluginParams, isExecute, isGroupBy} from '../types/interface'; -import {GroupByConfig} from '../types/group-by'; +import {isExecute, isGroupBy} from '../types/interface'; import {STRINGS} from '../config/strings'; diff --git a/src/lib/exhaust.ts b/src/lib/exhaust.ts index a4982681d..1b65ca898 100644 --- a/src/lib/exhaust.ts +++ b/src/lib/exhaust.ts @@ -1,4 +1,4 @@ -import {ERRORS} from '@grnsft/if-core'; +import {ERRORS} from '@grnsft/if-core/utils'; /** * @todo This is temporary solution, will be refactored to support dynamic plugins. diff --git a/src/lib/initialize.ts b/src/lib/initialize.ts index ac01e8e8e..212d98e37 100644 --- a/src/lib/initialize.ts +++ b/src/lib/initialize.ts @@ -1,6 +1,6 @@ import * as path from 'node:path'; -import {ERRORS} from '@grnsft/if-core'; +import {ERRORS} from '@grnsft/if-core/utils'; import {logger} from '../util/logger'; import {memoizedLog} from '../util/log-memoize'; diff --git a/src/lib/load.ts b/src/lib/load.ts index 72620e982..61f2b6cf5 100644 --- a/src/lib/load.ts +++ b/src/lib/load.ts @@ -1,5 +1,5 @@ import * as YAML from 'js-yaml'; -import {ERRORS} from '@grnsft/if-core'; +import {ERRORS} from '@grnsft/if-core/utils'; import {openYamlFileAsObject} from '../util/yaml'; import {readAndParseJson} from '../util/json'; From fa18a5ee72ffe73a56f0aa8a16e2b9980e0c0e95 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Fri, 14 Jun 2024 17:44:00 +0400 Subject: [PATCH 091/243] revert(uiltins): drop local types support --- src/builtins/coefficient/types.ts | 5 ----- src/builtins/exponent/types.ts | 5 ----- src/builtins/interpolation/types.ts | 5 ----- src/builtins/mock-observations/types.ts | 16 ---------------- src/builtins/multiply/types.ts | 4 ---- src/builtins/subtract/types.ts | 4 ---- src/builtins/sum/types.ts | 4 ---- 7 files changed, 43 deletions(-) delete mode 100644 src/builtins/coefficient/types.ts delete mode 100644 src/builtins/exponent/types.ts delete mode 100644 src/builtins/interpolation/types.ts delete mode 100644 src/builtins/mock-observations/types.ts delete mode 100644 src/builtins/multiply/types.ts delete mode 100644 src/builtins/subtract/types.ts delete mode 100644 src/builtins/sum/types.ts diff --git a/src/builtins/coefficient/types.ts b/src/builtins/coefficient/types.ts deleted file mode 100644 index 9cbb78b7d..000000000 --- a/src/builtins/coefficient/types.ts +++ /dev/null @@ -1,5 +0,0 @@ -export type CoefficientConfig = { - 'input-parameter': string; - coefficient: number; - 'output-parameter': string; -}; diff --git a/src/builtins/exponent/types.ts b/src/builtins/exponent/types.ts deleted file mode 100644 index 54e8b2efc..000000000 --- a/src/builtins/exponent/types.ts +++ /dev/null @@ -1,5 +0,0 @@ -export type ExponentConfig = { - 'input-parameter': string; - exponent: number; - 'output-parameter': string; -}; diff --git a/src/builtins/interpolation/types.ts b/src/builtins/interpolation/types.ts deleted file mode 100644 index 09b7abb5e..000000000 --- a/src/builtins/interpolation/types.ts +++ /dev/null @@ -1,5 +0,0 @@ -export enum Method { - LINEAR = 'linear', - SPLINE = 'spline', - POLYNOMIAL = 'polynomial', -} diff --git a/src/builtins/mock-observations/types.ts b/src/builtins/mock-observations/types.ts deleted file mode 100644 index 82591207c..000000000 --- a/src/builtins/mock-observations/types.ts +++ /dev/null @@ -1,16 +0,0 @@ -import {DateTime} from 'luxon'; - -import {Generator} from './interfaces/index'; - -export type ObservationParams = { - duration: number; - timeBucket: DateTime; - component: Record; - generators: Generator[]; -}; - -export type RandIntGeneratorParams = { - name: string; - min: number; - max: number; -}; diff --git a/src/builtins/multiply/types.ts b/src/builtins/multiply/types.ts deleted file mode 100644 index 88f4cae49..000000000 --- a/src/builtins/multiply/types.ts +++ /dev/null @@ -1,4 +0,0 @@ -export type MultiplyConfig = { - 'input-parameters': string[]; - 'output-parameter': string; -}; diff --git a/src/builtins/subtract/types.ts b/src/builtins/subtract/types.ts deleted file mode 100644 index 4dc6775d2..000000000 --- a/src/builtins/subtract/types.ts +++ /dev/null @@ -1,4 +0,0 @@ -export type SubtractConfig = { - 'input-parameters': string[]; - 'output-parameter': string; -}; diff --git a/src/builtins/sum/types.ts b/src/builtins/sum/types.ts deleted file mode 100644 index e30c990e5..000000000 --- a/src/builtins/sum/types.ts +++ /dev/null @@ -1,4 +0,0 @@ -export type SumConfig = { - 'input-parameters': string[]; - 'output-parameter': string; -}; From 89a19ee055e4775ba07a31e542fb733a7e3d94cb Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Fri, 14 Jun 2024 17:44:15 +0400 Subject: [PATCH 092/243] feat(builtins): introduce if-core utils and types --- src/builtins/coefficient/index.ts | 11 ++++++----- src/builtins/csv-lookup/index.ts | 5 ++--- src/builtins/divide/index.ts | 5 ++--- src/builtins/exponent/index.ts | 10 ++++++---- src/builtins/export-csv-raw.ts | 6 ++---- src/builtins/export-csv.ts | 6 +++--- src/builtins/export-log.ts | 1 + src/builtins/export-yaml.ts | 3 ++- src/builtins/group-by.ts | 10 ++++++---- src/builtins/interpolation/index.ts | 12 +++++++----- .../helpers/common-generator.ts | 5 +++-- .../helpers/rand-int-generator.ts | 10 ++++------ src/builtins/mock-observations/index.ts | 16 +++++++++------- src/builtins/multiply/index.ts | 10 ++++++---- src/builtins/regex/index.ts | 6 ++---- src/builtins/sci-embodied/index.ts | 3 +-- src/builtins/sci/index.ts | 6 ++---- src/builtins/shell/index.ts | 7 +++---- src/builtins/subtract/index.ts | 10 ++++++---- src/builtins/sum/index.ts | 6 ++---- src/builtins/time-sync.ts | 17 +++++++++-------- 21 files changed, 84 insertions(+), 81 deletions(-) diff --git a/src/builtins/coefficient/index.ts b/src/builtins/coefficient/index.ts index 65404172f..aed444ad3 100644 --- a/src/builtins/coefficient/index.ts +++ b/src/builtins/coefficient/index.ts @@ -1,14 +1,15 @@ import {z} from 'zod'; - -import {ExecutePlugin, PluginParams} from '../../types/interface'; +import {ERRORS} from '@grnsft/if-core/utils'; +import { + CoefficientConfig, + ExecutePlugin, + PluginParams, +} from '@grnsft/if-core/types'; import {validate} from '../../util/validations'; -import {ERRORS} from '@grnsft/if-core'; import {STRINGS} from '../../config'; -import {CoefficientConfig} from './types'; - const {GlobalConfigError} = ERRORS; const {MISSING_GLOBAL_CONFIG} = STRINGS; diff --git a/src/builtins/csv-lookup/index.ts b/src/builtins/csv-lookup/index.ts index 47232e115..8243a6391 100644 --- a/src/builtins/csv-lookup/index.ts +++ b/src/builtins/csv-lookup/index.ts @@ -4,14 +4,13 @@ import {readFile} from 'fs/promises'; import axios from 'axios'; import {z} from 'zod'; import {parse} from 'csv-parse/sync'; +import {ERRORS} from '@grnsft/if-core/utils'; +import {ExecutePlugin, PluginParams} from '@grnsft/if-core/types'; import {validate} from '../../util/validations'; -import {ERRORS} from '@grnsft/if-core'; import {STRINGS} from '../../config'; -import {ExecutePlugin, PluginParams} from '../../types/interface'; - const { FILE_FETCH_FAILED, FILE_READ_FAILED, diff --git a/src/builtins/divide/index.ts b/src/builtins/divide/index.ts index 14e8b4d44..a38e3601b 100644 --- a/src/builtins/divide/index.ts +++ b/src/builtins/divide/index.ts @@ -1,12 +1,11 @@ import {z} from 'zod'; +import {ERRORS} from '@grnsft/if-core/utils'; +import {ExecutePlugin, PluginParams, ConfigParams} from '@grnsft/if-core/types'; -import {ERRORS} from '@grnsft/if-core'; import {validate} from '../../util/validations'; import {STRINGS} from '../../config'; -import {ExecutePlugin, PluginParams, ConfigParams} from '../../types/interface'; - const {GlobalConfigError, MissingInputDataError} = ERRORS; const {MISSING_GLOBAL_CONFIG, MISSING_INPUT_DATA} = STRINGS; diff --git a/src/builtins/exponent/index.ts b/src/builtins/exponent/index.ts index 3a417329b..90d66cd72 100644 --- a/src/builtins/exponent/index.ts +++ b/src/builtins/exponent/index.ts @@ -1,13 +1,15 @@ import {z} from 'zod'; +import {ERRORS} from '@grnsft/if-core/utils'; +import { + ExecutePlugin, + PluginParams, + ExponentConfig, +} from '@grnsft/if-core/types'; -import {ERRORS} from '@grnsft/if-core'; import {validate} from '../../util/validations'; import {STRINGS} from '../../config'; -import {ExecutePlugin, PluginParams} from '../../types/interface'; -import {ExponentConfig} from './types'; - const {MissingInputDataError, InputValidationError} = ERRORS; const {MISSING_INPUT_DATA, NOT_NUMERIC_VALUE} = STRINGS; diff --git a/src/builtins/export-csv-raw.ts b/src/builtins/export-csv-raw.ts index 9ce1edae3..7afc21cb2 100644 --- a/src/builtins/export-csv-raw.ts +++ b/src/builtins/export-csv-raw.ts @@ -1,16 +1,14 @@ import * as fs from 'fs/promises'; - -import {ERRORS} from '@grnsft/if-core'; +import {ERRORS} from '@grnsft/if-core/utils'; import {STRINGS} from '../config'; -import {ExhaustPluginInterface} from '../types/exhaust-plugin-interface'; import {Context} from '../types/manifest'; const {ExhaustOutputArgError, WriteFileError} = ERRORS; const {OUTPUT_REQUIRED, WRITE_CSV_ERROR, EXPORTING_RAW_CSV_FILE} = STRINGS; -export const ExportCSVRaw = (): ExhaustPluginInterface => { +export const ExportCSVRaw = () => { /** * handle a tree leaf, where there are no child nodes, by adding it as key->value pair to the flat map * and capturing key as a header diff --git a/src/builtins/export-csv.ts b/src/builtins/export-csv.ts index 9eb03d9cc..53a105733 100644 --- a/src/builtins/export-csv.ts +++ b/src/builtins/export-csv.ts @@ -1,12 +1,12 @@ import {writeFile} from 'fs/promises'; -import {stringify} from 'csv-stringify/sync'; -import {ERRORS} from '@grnsft/if-core'; +import {stringify} from 'csv-stringify/sync'; +import {ERRORS} from '@grnsft/if-core/utils'; +import {PluginParams} from '@grnsft/if-core/types'; import {STRINGS} from '../config'; import {Context} from '../types/manifest'; -import {PluginParams} from '../types/interface'; const {ExhaustOutputArgError} = ERRORS; const {CSV_EXPORT, OUTPUT_REQUIRED, EXPORTING_TO_CSV_FILE} = STRINGS; diff --git a/src/builtins/export-log.ts b/src/builtins/export-log.ts index 3120f3386..fee267dc2 100644 --- a/src/builtins/export-log.ts +++ b/src/builtins/export-log.ts @@ -1,4 +1,5 @@ import * as YAML from 'js-yaml'; + import {Context} from '../types/manifest'; export const ExportLog = () => { diff --git a/src/builtins/export-yaml.ts b/src/builtins/export-yaml.ts index f6a3d71d7..624e1298f 100644 --- a/src/builtins/export-yaml.ts +++ b/src/builtins/export-yaml.ts @@ -1,5 +1,6 @@ +import {ERRORS} from '@grnsft/if-core/utils'; + import {saveYamlFileAs} from '../util/yaml'; -import {ERRORS} from '@grnsft/if-core'; import {STRINGS} from '../config'; diff --git a/src/builtins/group-by.ts b/src/builtins/group-by.ts index 3d38d444a..63c4cd5c5 100644 --- a/src/builtins/group-by.ts +++ b/src/builtins/group-by.ts @@ -1,11 +1,13 @@ import {z} from 'zod'; +import {ERRORS} from '@grnsft/if-core/utils'; +import { + GroupByPlugin, + PluginParams, + GroupByConfig, +} from '@grnsft/if-core/types'; import {STRINGS} from '../config'; -import {GroupByPlugin, PluginParams} from '../types/interface'; -import {GroupByConfig} from '../types/group-by'; - -import {ERRORS} from '@grnsft/if-core'; import {validate} from '../util/validations'; const {InvalidGroupingError, GlobalConfigError} = ERRORS; diff --git a/src/builtins/interpolation/index.ts b/src/builtins/interpolation/index.ts index dfe507c42..98a84ad8a 100644 --- a/src/builtins/interpolation/index.ts +++ b/src/builtins/interpolation/index.ts @@ -1,15 +1,17 @@ import Spline from 'typescript-cubic-spline'; import {z} from 'zod'; - -import {ExecutePlugin, PluginParams, ConfigParams} from '../../types/interface'; +import {ERRORS} from '@grnsft/if-core/utils'; +import { + ExecutePlugin, + PluginParams, + ConfigParams, + Method, +} from '@grnsft/if-core/types'; import {validate} from '../../util/validations'; -import {ERRORS} from '@grnsft/if-core'; import {STRINGS} from '../../config'; -import {Method} from './types'; - const {GlobalConfigError} = ERRORS; const { MISSING_GLOBAL_CONFIG, diff --git a/src/builtins/mock-observations/helpers/common-generator.ts b/src/builtins/mock-observations/helpers/common-generator.ts index 732e7320e..8283cf1bd 100644 --- a/src/builtins/mock-observations/helpers/common-generator.ts +++ b/src/builtins/mock-observations/helpers/common-generator.ts @@ -1,4 +1,5 @@ -import {ERRORS} from '@grnsft/if-core'; +import {ERRORS} from '@grnsft/if-core/utils'; +import {ConfigParams} from '@grnsft/if-core/types'; import {STRINGS} from '../../../config'; @@ -7,7 +8,7 @@ import {Generator} from '../interfaces'; const {GlobalConfigError} = ERRORS; const {MISSING_GLOBAL_CONFIG} = STRINGS; -export const CommonGenerator = (config: Record): Generator => { +export const CommonGenerator = (config: ConfigParams): Generator => { /** * Generates next value by copying the validated config. * Validates the provided config is not null or empty. diff --git a/src/builtins/mock-observations/helpers/rand-int-generator.ts b/src/builtins/mock-observations/helpers/rand-int-generator.ts index 6c0365d35..1cb4d161f 100644 --- a/src/builtins/mock-observations/helpers/rand-int-generator.ts +++ b/src/builtins/mock-observations/helpers/rand-int-generator.ts @@ -1,9 +1,9 @@ -import {ERRORS} from '@grnsft/if-core'; +import {ERRORS} from '@grnsft/if-core/utils'; +import {RandIntGeneratorParams, ConfigParams} from '@grnsft/if-core/types'; import {STRINGS} from '../../../config'; import {Generator} from '../interfaces'; -import {RandIntGeneratorParams} from '../types'; const {GlobalConfigError} = ERRORS; @@ -12,7 +12,7 @@ const {MISSING_GLOBAL_CONFIG, MISSING_MIN_MAX, INVALID_MIN_MAX, INVALID_NAME} = export const RandIntGenerator = ( name: string, - config: Record + config: ConfigParams ): Generator => { const next = () => ({ [validatedName]: generateRandInt(getFieldToPopulate()), @@ -26,9 +26,7 @@ export const RandIntGenerator = ( return name; }; - const validateConfig = ( - config: Record - ): {min: number; max: number} => { + const validateConfig = (config: ConfigParams): {min: number; max: number} => { if (!config || Object.keys(config).length === 0) { throw new GlobalConfigError(MISSING_GLOBAL_CONFIG); } diff --git a/src/builtins/mock-observations/index.ts b/src/builtins/mock-observations/index.ts index b535da757..c7488de0f 100644 --- a/src/builtins/mock-observations/index.ts +++ b/src/builtins/mock-observations/index.ts @@ -1,15 +1,18 @@ import {DateTime, Duration} from 'luxon'; import {z} from 'zod'; +import { + ExecutePlugin, + PluginParams, + ConfigParams, + ObservationParams, +} from '@grnsft/if-core/types'; import {validate} from '../../util/validations'; import {CommonGenerator} from './helpers/common-generator'; import {RandIntGenerator} from './helpers/rand-int-generator'; -import {ExecutePlugin, PluginParams} from '../../types/interface'; -import {ConfigParams, KeyValuePair} from '../../types/common'; import {Generator} from './interfaces/index'; -import {ObservationParams} from './types'; export const MockObservations = (globalConfig: ConfigParams): ExecutePlugin => { const metadata = { @@ -122,11 +125,10 @@ export const MockObservations = (globalConfig: ConfigParams): ExecutePlugin => { CommonGenerator(config), ]; - const createRandIntGenerators = (config: any): Generator[] => { - return Object.entries(config).map(([fieldToPopulate, value]) => - RandIntGenerator(fieldToPopulate, value as KeyValuePair) + const createRandIntGenerators = (config: any): Generator[] => + Object.entries(config).map(([fieldToPopulate, value]) => + RandIntGenerator(fieldToPopulate, value as Record) ); - }; return Object.entries(generatorsConfig).flatMap(([key, value]) => key === 'randint' diff --git a/src/builtins/multiply/index.ts b/src/builtins/multiply/index.ts index 5ceb44c3c..717c68507 100644 --- a/src/builtins/multiply/index.ts +++ b/src/builtins/multiply/index.ts @@ -1,13 +1,15 @@ import {z} from 'zod'; +import {ERRORS} from '@grnsft/if-core/utils'; +import { + ExecutePlugin, + PluginParams, + MultiplyConfig, +} from '@grnsft/if-core/types'; -import {ERRORS} from '@grnsft/if-core'; import {validate} from '../../util/validations'; import {STRINGS} from '../../config'; -import {ExecutePlugin, PluginParams} from '../../types/interface'; -import {MultiplyConfig} from './types'; - const {MissingInputDataError} = ERRORS; const {MISSING_INPUT_DATA} = STRINGS; diff --git a/src/builtins/regex/index.ts b/src/builtins/regex/index.ts index cdcbd13b8..8d382ec74 100644 --- a/src/builtins/regex/index.ts +++ b/src/builtins/regex/index.ts @@ -1,13 +1,11 @@ import {z} from 'zod'; +import {ERRORS} from '@grnsft/if-core/utils'; +import {ExecutePlugin, PluginParams, ConfigParams} from '@grnsft/if-core/types'; -import {ERRORS} from '@grnsft/if-core'; import {validate} from '../../util/validations'; import {STRINGS} from '../../config'; -import {ExecutePlugin, PluginParams} from '../../types/interface'; -import {ConfigParams} from '../../types/common'; - const {MissingInputDataError, GlobalConfigError, RegexMismatchError} = ERRORS; const {MISSING_GLOBAL_CONFIG, MISSING_INPUT_DATA, REGEX_MISMATCH} = STRINGS; diff --git a/src/builtins/sci-embodied/index.ts b/src/builtins/sci-embodied/index.ts index 02f897584..678d41336 100644 --- a/src/builtins/sci-embodied/index.ts +++ b/src/builtins/sci-embodied/index.ts @@ -1,11 +1,10 @@ import {z} from 'zod'; +import {ExecutePlugin, PluginParams} from '@grnsft/if-core/types'; import {validate, allDefined} from '../../util/validations'; import {STRINGS} from '../../config'; -import {ExecutePlugin, PluginParams} from '../../types/interface'; - const {SCI_EMBODIED_ERROR} = STRINGS; export const SciEmbodied = (): ExecutePlugin => { diff --git a/src/builtins/sci/index.ts b/src/builtins/sci/index.ts index c7802213d..223493f3a 100644 --- a/src/builtins/sci/index.ts +++ b/src/builtins/sci/index.ts @@ -1,13 +1,11 @@ import {z} from 'zod'; +import {ERRORS} from '@grnsft/if-core/utils'; +import {ExecutePlugin, PluginParams, ConfigParams} from '@grnsft/if-core/types'; import {validate, allDefined} from '../../util/validations'; -import {ERRORS} from '@grnsft/if-core'; import {STRINGS} from '../../config'; -import {ExecutePlugin, PluginParams} from '../../types/interface'; -import {ConfigParams} from '../../types/common'; - const {MissingInputDataError} = ERRORS; const {MISSING_FUNCTIONAL_UNIT_CONFIG, MISSING_FUNCTIONAL_UNIT_INPUT} = STRINGS; diff --git a/src/builtins/shell/index.ts b/src/builtins/shell/index.ts index 0b894d64b..352ee6bcd 100644 --- a/src/builtins/shell/index.ts +++ b/src/builtins/shell/index.ts @@ -1,12 +1,11 @@ import {spawnSync, SpawnSyncReturns} from 'child_process'; + import {loadAll, dump} from 'js-yaml'; import {z} from 'zod'; - -import {ExecutePlugin, PluginParams} from '../../types/interface'; -import {ConfigParams} from '../../types/common'; +import {ERRORS} from '@grnsft/if-core/utils'; +import {ExecutePlugin, PluginParams, ConfigParams} from '@grnsft/if-core/types'; import {validate} from '../../util/validations'; -import {ERRORS} from '@grnsft/if-core'; const {ProcessExecutionError} = ERRORS; diff --git a/src/builtins/subtract/index.ts b/src/builtins/subtract/index.ts index 87523082e..20a61b5b6 100644 --- a/src/builtins/subtract/index.ts +++ b/src/builtins/subtract/index.ts @@ -1,13 +1,15 @@ import {z} from 'zod'; +import {ERRORS} from '@grnsft/if-core/utils'; +import { + ExecutePlugin, + PluginParams, + SubtractConfig, +} from '@grnsft/if-core/types'; -import {ERRORS} from '@grnsft/if-core'; import {validate} from '../../util/validations'; import {STRINGS} from '../../config'; -import {ExecutePlugin, PluginParams} from '../../types/interface'; -import {SubtractConfig} from './types'; - const {InputValidationError} = ERRORS; const {MISSING_INPUT_DATA, NOT_NUMERIC_VALUE} = STRINGS; diff --git a/src/builtins/sum/index.ts b/src/builtins/sum/index.ts index 676829162..ecce47cf3 100644 --- a/src/builtins/sum/index.ts +++ b/src/builtins/sum/index.ts @@ -1,13 +1,11 @@ import {z} from 'zod'; +import {ERRORS} from '@grnsft/if-core/utils'; +import {ExecutePlugin, PluginParams, SumConfig} from '@grnsft/if-core/types'; import {validate} from '../../util/validations'; -import {ERRORS} from '@grnsft/if-core'; import {STRINGS} from '../../config'; -import {ExecutePlugin, PluginParams} from '../../types/interface'; -import {SumConfig} from './types'; - const {GlobalConfigError, MissingInputDataError} = ERRORS; const {MISSING_INPUT_DATA, MISSING_GLOBAL_CONFIG} = STRINGS; diff --git a/src/builtins/time-sync.ts b/src/builtins/time-sync.ts index 1542b7854..8b37241b8 100644 --- a/src/builtins/time-sync.ts +++ b/src/builtins/time-sync.ts @@ -1,21 +1,22 @@ import {isDate} from 'node:util/types'; + import {Settings, DateTime, DateTimeMaybeValid, Interval} from 'luxon'; import {z} from 'zod'; +import {ERRORS} from '@grnsft/if-core/utils'; +import { + ExecutePlugin, + PluginParams, + PaddingReceipt, + TimeNormalizerConfig, + TimeParams, +} from '@grnsft/if-core/types'; import {parameterize} from '../lib/parameterize'; -import {ERRORS} from '@grnsft/if-core'; import {validate} from '../util/validations'; import {STRINGS} from '../config'; -import {ExecutePlugin, PluginParams} from '../types/interface'; -import { - PaddingReceipt, - TimeNormalizerConfig, - TimeParams, -} from '../types/time-sync'; - Settings.defaultZone = 'utc'; const { From 710b2a32e333cc5e9e8cb0001158ee684c775b0c Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Fri, 14 Jun 2024 17:45:25 +0400 Subject: [PATCH 093/243] test(util): introduce if-core util and types --- src/__tests__/unit/util/aggregation-helper.test.ts | 5 ++--- src/__tests__/unit/util/args.test.ts | 2 +- src/__tests__/unit/util/helpers.test.ts | 2 +- src/__tests__/unit/util/plugin-storage.test.ts | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/__tests__/unit/util/aggregation-helper.test.ts b/src/__tests__/unit/util/aggregation-helper.test.ts index eb379e10f..1a23bc3c1 100644 --- a/src/__tests__/unit/util/aggregation-helper.test.ts +++ b/src/__tests__/unit/util/aggregation-helper.test.ts @@ -1,11 +1,10 @@ -import {ERRORS} from '@grnsft/if-core'; +import {ERRORS} from '@grnsft/if-core/utils'; +import {PluginParams} from '@grnsft/if-core/types'; import {aggregateInputsIntoOne} from '../../../util/aggregation-helper'; import {STRINGS} from '../../../config'; -import {PluginParams} from '../../../types/interface'; - const {InvalidAggregationMethodError, MissingAggregationParamError} = ERRORS; const {INVALID_AGGREGATION_METHOD, METRIC_MISSING} = STRINGS; diff --git a/src/__tests__/unit/util/args.test.ts b/src/__tests__/unit/util/args.test.ts index 52e468987..045bf2807 100644 --- a/src/__tests__/unit/util/args.test.ts +++ b/src/__tests__/unit/util/args.test.ts @@ -70,7 +70,7 @@ jest.mock('ts-command-line-args', () => ({ })); import * as path from 'node:path'; -import {ERRORS} from '@grnsft/if-core'; +import {ERRORS} from '@grnsft/if-core/utils'; import {parseIEProcessArgs, parseIfDiffArgs} from '../../../util/args'; diff --git a/src/__tests__/unit/util/helpers.test.ts b/src/__tests__/unit/util/helpers.test.ts index 4aaccc5f5..86ea702a4 100644 --- a/src/__tests__/unit/util/helpers.test.ts +++ b/src/__tests__/unit/util/helpers.test.ts @@ -10,7 +10,7 @@ jest.mock('../../../util/logger', () => ({ error: mockError, }, })); -import {ERRORS} from '@grnsft/if-core'; +import {ERRORS} from '@grnsft/if-core/utils'; import { andHandle, checkIfEqual, diff --git a/src/__tests__/unit/util/plugin-storage.test.ts b/src/__tests__/unit/util/plugin-storage.test.ts index 0555b1db5..8b76f36cd 100644 --- a/src/__tests__/unit/util/plugin-storage.test.ts +++ b/src/__tests__/unit/util/plugin-storage.test.ts @@ -1,4 +1,4 @@ -import {ERRORS} from '@grnsft/if-core'; +import {ERRORS} from '@grnsft/if-core/utils'; import {pluginStorage} from '../../../util/plugin-storage'; From 21c8d0bf00ba33a66884c7642b36a827c45f0302 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Fri, 14 Jun 2024 17:45:55 +0400 Subject: [PATCH 094/243] test(lib): drop unused manifests --- src/__tests__/unit/lib/manifest/basic.ts | 48 ------- src/__tests__/unit/lib/manifest/nested.ts | 146 ---------------------- 2 files changed, 194 deletions(-) delete mode 100644 src/__tests__/unit/lib/manifest/basic.ts delete mode 100644 src/__tests__/unit/lib/manifest/nested.ts diff --git a/src/__tests__/unit/lib/manifest/basic.ts b/src/__tests__/unit/lib/manifest/basic.ts deleted file mode 100644 index e5da7c642..000000000 --- a/src/__tests__/unit/lib/manifest/basic.ts +++ /dev/null @@ -1,48 +0,0 @@ -import {Manifest} from '../../../../types/manifest'; - -export const manifest: Manifest = { - name: 'gsf-demo', - description: 'Hello', - tags: { - kind: 'web', - complexity: 'moderate', - category: 'cloud', - }, - initialize: { - plugins: { - 'mock-name': { - model: 'MockaviztaModel', - path: 'mock-path', - 'global-config': { - allocation: 'LINEAR', - verbose: true, - }, - }, - }, - }, - tree: { - children: { - 'front-end': { - pipeline: ['mock-name'], - config: { - 'mock-name': { - 'core-units': 24, - processor: 'Intel® Core™ i7-1185G7', - }, - }, - inputs: [ - { - timestamp: '2023-07-06T00:00', - duration: 3600, - 'cpu/utilization': 18.392, - }, - { - timestamp: '2023-08-06T00:00', - duration: 3600, - 'cpu/utilization': 16, - }, - ], - }, - }, - }, -}; diff --git a/src/__tests__/unit/lib/manifest/nested.ts b/src/__tests__/unit/lib/manifest/nested.ts deleted file mode 100644 index 0c27fad12..000000000 --- a/src/__tests__/unit/lib/manifest/nested.ts +++ /dev/null @@ -1,146 +0,0 @@ -import {Manifest} from '../../../../types/manifest'; - -export const manifestNested: Manifest = { - name: 'nesting-demo', - description: null, - tags: { - kind: 'web', - complexity: 'moderate', - category: 'on-premise', - }, - initialize: { - plugins: { - mockavizta: { - model: 'MockaviztaModel', - path: 'mock-path', - 'global-config': { - allocation: 'LINEAR', - verbose: true, - }, - }, - }, - }, - tree: { - children: { - 'child-0': { - config: { - mockavizta: { - allocation: 'LINEAR', - verbose: true, - }, - }, - pipeline: ['mockavizta'], - children: { - 'child-0-1': { - children: { - 'child-0-1-1': { - pipeline: ['mockavizta'], - inputs: [ - { - timestamp: '2023-07-06T00:00', - duration: 10, - 'cpu/utilization': 50, - 'energy-network': 0.000811, - carbon: 10, - }, - ], - }, - 'child-0-1-2': { - children: { - 'child-1-2-1': { - pipeline: ['mockavizta'], - config: { - mockavizta: { - allocation: 'mock-allocation', - verbose: false, - }, - }, - inputs: [ - { - timestamp: '2023-07-06T00:00', - duration: 10, - 'cpu/utilization': 50, - 'energy-network': 0.000811, - carbon: 10, - }, - ], - }, - }, - }, - }, - }, - }, - }, - }, - }, -}; - -export const manifestNestedNoConfig: Manifest = { - name: 'nesting-demo', - description: null, - tags: { - kind: 'web', - complexity: 'moderate', - category: 'on-premise', - }, - initialize: { - plugins: { - mockavizta: { - model: 'MockaviztaModel', - path: 'mock-path', - 'global-config': { - allocation: 'LINEAR', - verbose: true, - }, - }, - }, - }, - tree: { - children: { - 'child-0': { - config: { - mockavizta: { - allocation: 'LINEAR', - verbose: true, - }, - }, - pipeline: ['mockavizta'], - children: { - 'child-0-1': { - children: { - 'child-0-1-1': { - pipeline: ['mockavizta'], - inputs: [ - { - timestamp: '2023-07-06T00:00', - duration: 10, - 'cpu/utilization': 50, - 'energy-network': 0.000811, - carbon: 10, - }, - ], - }, - 'child-0-1-2': { - children: { - 'child-1-2-1': { - pipeline: ['mockavizta'], - config: {}, - inputs: [ - { - timestamp: '2023-07-06T00:00', - duration: 10, - 'cpu/utilization': 50, - 'energy-network': 0.000811, - carbon: 10, - }, - ], - }, - }, - }, - }, - }, - }, - }, - }, - }, -}; From 6709d12de37aa0582d2d2f722ca0e02412346ca5 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Fri, 14 Jun 2024 17:46:10 +0400 Subject: [PATCH 095/243] test(lib): introduce if-core utils and types --- src/__tests__/unit/lib/exhaust.test.ts | 2 +- src/__tests__/unit/lib/initialize.test.ts | 2 +- src/__tests__/unit/lib/load.test.ts | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/__tests__/unit/lib/exhaust.test.ts b/src/__tests__/unit/lib/exhaust.test.ts index 1fa652bb3..49db14370 100644 --- a/src/__tests__/unit/lib/exhaust.test.ts +++ b/src/__tests__/unit/lib/exhaust.test.ts @@ -1,7 +1,7 @@ /* eslint-disable @typescript-eslint/ban-ts-comment */ jest.mock('fs', () => require('../../../__mocks__/fs')); -import {ERRORS} from '@grnsft/if-core'; +import {ERRORS} from '@grnsft/if-core/utils'; import {exhaust} from '../../../lib/exhaust'; diff --git a/src/__tests__/unit/lib/initialize.test.ts b/src/__tests__/unit/lib/initialize.test.ts index 51d15f0d7..ff3c10c45 100644 --- a/src/__tests__/unit/lib/initialize.test.ts +++ b/src/__tests__/unit/lib/initialize.test.ts @@ -10,7 +10,7 @@ jest.mock('../../../util/log-memoize', () => ({ memoizedLog: mockLog, })); -import {ERRORS} from '@grnsft/if-core'; +import {ERRORS} from '@grnsft/if-core/utils'; import {initialize} from '../../../lib/initialize'; diff --git a/src/__tests__/unit/lib/load.test.ts b/src/__tests__/unit/lib/load.test.ts index 95ae6b26b..ba3bca902 100644 --- a/src/__tests__/unit/lib/load.test.ts +++ b/src/__tests__/unit/lib/load.test.ts @@ -38,14 +38,14 @@ jest.mock('../../../util/yaml', () => ({ }, })); -import {load, loadIfDiffFiles} from '../../../lib/load'; +import {PluginParams} from '@grnsft/if-core/types'; -import {PARAMETERS} from '../../../config'; +import {load, loadIfDiffFiles} from '../../../lib/load'; -import {PluginParams} from '../../../types/interface'; +import {PARAMETERS, STRINGS} from '../../../config'; -import {STRINGS} from '../../../config'; import {parseManifestFromStdin} from '../../../util/helpers'; + import {LoadDiffParams} from '../../../types/util/args'; const {INVALID_SOURCE} = STRINGS; From ec2d8c0b57f3650742d2bbf94d01d7280d8cebdf Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Fri, 14 Jun 2024 17:46:23 +0400 Subject: [PATCH 096/243] test(builtins): introduce if-core utils and types --- src/__tests__/unit/builtins/CommonGenerator.test.ts | 2 +- src/__tests__/unit/builtins/RandIntGenerator.test.ts | 6 ++---- src/__tests__/unit/builtins/coefficient.test.ts | 2 +- src/__tests__/unit/builtins/csv-lookup.test.ts | 2 +- src/__tests__/unit/builtins/divide.test.ts | 2 +- src/__tests__/unit/builtins/exponent.test.ts | 2 +- src/__tests__/unit/builtins/export-csv-raw.test.ts | 3 ++- src/__tests__/unit/builtins/export-csv.test.ts | 3 ++- src/__tests__/unit/builtins/export-yaml.test.ts | 2 +- src/__tests__/unit/builtins/group-by.test.ts | 2 +- src/__tests__/unit/builtins/interpolation.test.ts | 4 ++-- src/__tests__/unit/builtins/mock-observations.test.ts | 2 +- src/__tests__/unit/builtins/multiply.test.ts | 2 +- src/__tests__/unit/builtins/regex.test.ts | 2 +- src/__tests__/unit/builtins/sci-embodied.test.ts | 2 +- src/__tests__/unit/builtins/sci.test.ts | 2 +- src/__tests__/unit/builtins/shell.test.ts | 2 +- src/__tests__/unit/builtins/subtract.test.ts | 2 +- src/__tests__/unit/builtins/sum.test.ts | 2 +- src/__tests__/unit/builtins/time-sync.test.ts | 2 +- 20 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/__tests__/unit/builtins/CommonGenerator.test.ts b/src/__tests__/unit/builtins/CommonGenerator.test.ts index 23109c117..b0c165d1a 100644 --- a/src/__tests__/unit/builtins/CommonGenerator.test.ts +++ b/src/__tests__/unit/builtins/CommonGenerator.test.ts @@ -1,4 +1,4 @@ -import {ERRORS} from '@grnsft/if-core'; +import {ERRORS} from '@grnsft/if-core/utils'; import {CommonGenerator} from '../../../builtins/mock-observations/helpers/common-generator'; diff --git a/src/__tests__/unit/builtins/RandIntGenerator.test.ts b/src/__tests__/unit/builtins/RandIntGenerator.test.ts index ca1b60183..328f7d6f4 100644 --- a/src/__tests__/unit/builtins/RandIntGenerator.test.ts +++ b/src/__tests__/unit/builtins/RandIntGenerator.test.ts @@ -1,11 +1,9 @@ -import {ERRORS} from '@grnsft/if-core'; +import {ERRORS} from '@grnsft/if-core/utils'; import {RandIntGenerator} from '../../../builtins/mock-observations/helpers/rand-int-generator'; import {STRINGS} from '../../../config'; -import {KeyValuePair} from '../../../types/common'; - const {GlobalConfigError} = ERRORS; const {INVALID_NAME, MISSING_MIN_MAX, MISSING_GLOBAL_CONFIG} = STRINGS; @@ -44,7 +42,7 @@ describe('builtins/mock-observations/RandIntGenerator: ', () => { describe('next(): ', () => { it('returns a result with valid data.', async () => { - const config: KeyValuePair = { + const config: Record = { min: 10, max: 90, }; diff --git a/src/__tests__/unit/builtins/coefficient.test.ts b/src/__tests__/unit/builtins/coefficient.test.ts index e7e92adf5..034d20f95 100644 --- a/src/__tests__/unit/builtins/coefficient.test.ts +++ b/src/__tests__/unit/builtins/coefficient.test.ts @@ -1,4 +1,4 @@ -import {ERRORS} from '@grnsft/if-core'; +import {ERRORS} from '@grnsft/if-core/utils'; import {Coefficient} from '../../../builtins/coefficient'; diff --git a/src/__tests__/unit/builtins/csv-lookup.test.ts b/src/__tests__/unit/builtins/csv-lookup.test.ts index 9e718d537..81e0498b4 100644 --- a/src/__tests__/unit/builtins/csv-lookup.test.ts +++ b/src/__tests__/unit/builtins/csv-lookup.test.ts @@ -2,7 +2,7 @@ jest.mock('fs/promises', () => require('../../../__mocks__/fs')); import axios from 'axios'; import AxiosMockAdapter from 'axios-mock-adapter'; -import {ERRORS} from '@grnsft/if-core'; +import {ERRORS} from '@grnsft/if-core/utils'; import {CSVLookup} from '../../../builtins'; diff --git a/src/__tests__/unit/builtins/divide.test.ts b/src/__tests__/unit/builtins/divide.test.ts index c75c926be..79fc6c0b5 100644 --- a/src/__tests__/unit/builtins/divide.test.ts +++ b/src/__tests__/unit/builtins/divide.test.ts @@ -1,4 +1,4 @@ -import {ERRORS} from '@grnsft/if-core'; +import {ERRORS} from '@grnsft/if-core/utils'; import {Divide} from '../../../builtins'; diff --git a/src/__tests__/unit/builtins/exponent.test.ts b/src/__tests__/unit/builtins/exponent.test.ts index 04d9f7e9b..c139a5608 100644 --- a/src/__tests__/unit/builtins/exponent.test.ts +++ b/src/__tests__/unit/builtins/exponent.test.ts @@ -1,4 +1,4 @@ -import {ERRORS} from '@grnsft/if-core'; +import {ERRORS} from '@grnsft/if-core/utils'; import {Exponent} from '../../../builtins/exponent'; diff --git a/src/__tests__/unit/builtins/export-csv-raw.test.ts b/src/__tests__/unit/builtins/export-csv-raw.test.ts index 5ff8d20cd..ab71e0e34 100644 --- a/src/__tests__/unit/builtins/export-csv-raw.test.ts +++ b/src/__tests__/unit/builtins/export-csv-raw.test.ts @@ -1,6 +1,7 @@ import * as fs from 'fs/promises'; + import {jest} from '@jest/globals'; -import {ERRORS} from '@grnsft/if-core'; +import {ERRORS} from '@grnsft/if-core/utils'; import {ExportCSVRaw} from '../../../builtins/export-csv-raw'; diff --git a/src/__tests__/unit/builtins/export-csv.test.ts b/src/__tests__/unit/builtins/export-csv.test.ts index 942732e2e..78feaf534 100644 --- a/src/__tests__/unit/builtins/export-csv.test.ts +++ b/src/__tests__/unit/builtins/export-csv.test.ts @@ -1,7 +1,8 @@ import * as fs from 'fs/promises'; + import {stringify} from 'csv-stringify/sync'; import {jest} from '@jest/globals'; -import {ERRORS} from '@grnsft/if-core'; +import {ERRORS} from '@grnsft/if-core/utils'; import {ExportCSV} from '../../../builtins/export-csv'; diff --git a/src/__tests__/unit/builtins/export-yaml.test.ts b/src/__tests__/unit/builtins/export-yaml.test.ts index 56b155cba..fb7954c1e 100644 --- a/src/__tests__/unit/builtins/export-yaml.test.ts +++ b/src/__tests__/unit/builtins/export-yaml.test.ts @@ -1,4 +1,4 @@ -import {ERRORS} from '@grnsft/if-core'; +import {ERRORS} from '@grnsft/if-core/utils'; import {ExportYaml} from '../../../builtins/export-yaml'; import {saveYamlFileAs} from '../../../util/yaml'; diff --git a/src/__tests__/unit/builtins/group-by.test.ts b/src/__tests__/unit/builtins/group-by.test.ts index 22ef77f56..3d676148b 100644 --- a/src/__tests__/unit/builtins/group-by.test.ts +++ b/src/__tests__/unit/builtins/group-by.test.ts @@ -1,4 +1,4 @@ -import {ERRORS} from '@grnsft/if-core'; +import {ERRORS} from '@grnsft/if-core/utils'; import {GroupBy} from '../../../builtins/group-by'; diff --git a/src/__tests__/unit/builtins/interpolation.test.ts b/src/__tests__/unit/builtins/interpolation.test.ts index ead5880bb..6219e01ec 100644 --- a/src/__tests__/unit/builtins/interpolation.test.ts +++ b/src/__tests__/unit/builtins/interpolation.test.ts @@ -1,8 +1,8 @@ -import {ERRORS} from '@grnsft/if-core'; +import {ERRORS} from '@grnsft/if-core/utils'; +import {Method} from '@grnsft/if-core/types'; import {Interpolation} from '../../../builtins'; -import {Method} from '../../../builtins/interpolation/types'; import {STRINGS} from '../../../config'; const {InputValidationError, GlobalConfigError} = ERRORS; diff --git a/src/__tests__/unit/builtins/mock-observations.test.ts b/src/__tests__/unit/builtins/mock-observations.test.ts index 7a46cf79f..fd28d5595 100644 --- a/src/__tests__/unit/builtins/mock-observations.test.ts +++ b/src/__tests__/unit/builtins/mock-observations.test.ts @@ -1,4 +1,4 @@ -import {ERRORS} from '@grnsft/if-core'; +import {ERRORS} from '@grnsft/if-core/utils'; import {MockObservations} from '../../../builtins/mock-observations'; diff --git a/src/__tests__/unit/builtins/multiply.test.ts b/src/__tests__/unit/builtins/multiply.test.ts index ff168110e..f191ae169 100644 --- a/src/__tests__/unit/builtins/multiply.test.ts +++ b/src/__tests__/unit/builtins/multiply.test.ts @@ -1,4 +1,4 @@ -import {ERRORS} from '@grnsft/if-core'; +import {ERRORS} from '@grnsft/if-core/utils'; import {Multiply} from '../../../builtins/multiply'; diff --git a/src/__tests__/unit/builtins/regex.test.ts b/src/__tests__/unit/builtins/regex.test.ts index 57cdc5e0c..54a70ed9e 100644 --- a/src/__tests__/unit/builtins/regex.test.ts +++ b/src/__tests__/unit/builtins/regex.test.ts @@ -1,4 +1,4 @@ -import {ERRORS} from '@grnsft/if-core'; +import {ERRORS} from '@grnsft/if-core/utils'; import {Regex} from '../../../builtins/regex'; diff --git a/src/__tests__/unit/builtins/sci-embodied.test.ts b/src/__tests__/unit/builtins/sci-embodied.test.ts index 1d8f05438..a3f5bcbff 100644 --- a/src/__tests__/unit/builtins/sci-embodied.test.ts +++ b/src/__tests__/unit/builtins/sci-embodied.test.ts @@ -1,4 +1,4 @@ -import {ERRORS} from '@grnsft/if-core'; +import {ERRORS} from '@grnsft/if-core/utils'; import {SciEmbodied} from '../../../builtins/sci-embodied'; diff --git a/src/__tests__/unit/builtins/sci.test.ts b/src/__tests__/unit/builtins/sci.test.ts index 751804f17..c12085d4e 100644 --- a/src/__tests__/unit/builtins/sci.test.ts +++ b/src/__tests__/unit/builtins/sci.test.ts @@ -1,4 +1,4 @@ -import {ERRORS} from '@grnsft/if-core'; +import {ERRORS} from '@grnsft/if-core/utils'; import {Sci} from '../../../builtins/sci'; diff --git a/src/__tests__/unit/builtins/shell.test.ts b/src/__tests__/unit/builtins/shell.test.ts index 95f44fb12..96de944f1 100644 --- a/src/__tests__/unit/builtins/shell.test.ts +++ b/src/__tests__/unit/builtins/shell.test.ts @@ -1,6 +1,6 @@ import {spawnSync} from 'child_process'; import {loadAll} from 'js-yaml'; -import {ERRORS} from '@grnsft/if-core'; +import {ERRORS} from '@grnsft/if-core/utils'; import {Shell} from '../../../builtins/shell'; diff --git a/src/__tests__/unit/builtins/subtract.test.ts b/src/__tests__/unit/builtins/subtract.test.ts index a1c1e3302..b84b05db8 100644 --- a/src/__tests__/unit/builtins/subtract.test.ts +++ b/src/__tests__/unit/builtins/subtract.test.ts @@ -1,4 +1,4 @@ -import {ERRORS} from '@grnsft/if-core'; +import {ERRORS} from '@grnsft/if-core/utils'; import {Subtract} from '../../../builtins/subtract'; diff --git a/src/__tests__/unit/builtins/sum.test.ts b/src/__tests__/unit/builtins/sum.test.ts index 7a09958d1..dbd05a783 100644 --- a/src/__tests__/unit/builtins/sum.test.ts +++ b/src/__tests__/unit/builtins/sum.test.ts @@ -1,4 +1,4 @@ -import {ERRORS} from '@grnsft/if-core'; +import {ERRORS} from '@grnsft/if-core/utils'; import {Sum} from '../../../builtins/sum'; diff --git a/src/__tests__/unit/builtins/time-sync.test.ts b/src/__tests__/unit/builtins/time-sync.test.ts index 63174743a..6dd984659 100644 --- a/src/__tests__/unit/builtins/time-sync.test.ts +++ b/src/__tests__/unit/builtins/time-sync.test.ts @@ -1,4 +1,4 @@ -import {ERRORS} from '@grnsft/if-core'; +import {ERRORS} from '@grnsft/if-core/utils'; import {Settings, DateTime} from 'luxon'; import {TimeSync} from '../../../builtins/time-sync'; From 5ee388aa73e6dffbb3d12d5a017e74ed1d59d38b Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Fri, 14 Jun 2024 17:46:36 +0400 Subject: [PATCH 097/243] test(mocks): introduce if-core types --- src/__mocks__/plugin/lib/mockavizta/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/__mocks__/plugin/lib/mockavizta/index.ts b/src/__mocks__/plugin/lib/mockavizta/index.ts index 163a0cd24..54feae6a2 100644 --- a/src/__mocks__/plugin/lib/mockavizta/index.ts +++ b/src/__mocks__/plugin/lib/mockavizta/index.ts @@ -1,4 +1,4 @@ -import {PluginParams} from '../../../../types/interface'; +import {PluginParams} from '@grnsft/if-core/types'; /** * Mock model for testing. From b7a54a1fdbc59381dc17dc8cd07a4df626d281e6 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Fri, 14 Jun 2024 17:46:53 +0400 Subject: [PATCH 098/243] fix(src): fix imports order in diff --- src/diff.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/diff.ts b/src/diff.ts index 71a57ed17..aad02482c 100644 --- a/src/diff.ts +++ b/src/diff.ts @@ -7,10 +7,11 @@ import {parseIfDiffArgs} from './util/args'; import {formatNotMatchingLog, parseManifestFromStdin} from './util/helpers'; import {validateManifest} from './util/validations'; -import {CONFIG} from './config'; import {logger} from './util/logger'; import {debugLogger} from './util/debug-logger'; +import {CONFIG} from './config'; + const {IF_DIFF} = CONFIG; const {SUCCESS_MESSAGE, FAILURE_MESSAGE} = IF_DIFF; From df7374ad8683ddf20b6368fa3971f5fa4b919c5b Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Fri, 14 Jun 2024 17:47:07 +0400 Subject: [PATCH 099/243] feat(package): update if-core version --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2ec34c515..885d05d11 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,7 @@ "dependencies": { "@commitlint/cli": "^18.6.0", "@commitlint/config-conventional": "^18.6.0", - "@grnsft/if-core": "^0.0.3", + "@grnsft/if-core": "^0.0.7", "axios": "^1.7.2", "csv-parse": "^5.5.6", "csv-stringify": "^6.4.6", @@ -1181,9 +1181,9 @@ } }, "node_modules/@grnsft/if-core": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/@grnsft/if-core/-/if-core-0.0.3.tgz", - "integrity": "sha512-ieuUcadEgA4EztIzCbCW5ifK2a6SUU2ioZtLTce8pJ+imgeQ47lQM3UtChtbaUm52rEwvgl1Psp0ysgOjqssFA==", + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/@grnsft/if-core/-/if-core-0.0.7.tgz", + "integrity": "sha512-+4w8Sq1KRPDb+Jy638wgrTXlXIZzewOBceT+rAy3Oaov1M/veY3gu3AV15SXcPHrsBoFmZ6QeI9g1rF3RKB0ww==", "dependencies": { "typescript": "^5.1.6" }, diff --git a/package.json b/package.json index 5093d2235..1846ec684 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "dependencies": { "@commitlint/cli": "^18.6.0", "@commitlint/config-conventional": "^18.6.0", - "@grnsft/if-core": "^0.0.3", + "@grnsft/if-core": "^0.0.7", "axios": "^1.7.2", "csv-parse": "^5.5.6", "csv-stringify": "^6.4.6", From 1d39e6b885d974232d9c884dbfbcd0997cc64947 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Fri, 14 Jun 2024 17:47:58 +0400 Subject: [PATCH 100/243] chore(config): drop unused module path from jest --- jest.config.js | 1 - 1 file changed, 1 deletion(-) diff --git a/jest.config.js b/jest.config.js index ed24da708..2e770bcad 100644 --- a/jest.config.js +++ b/jest.config.js @@ -12,7 +12,6 @@ module.exports = { }, modulePathIgnorePatterns: [ './build', - './src/__tests__/unit/lib/manifest', './src/__tests__/integration/helpers', './src/__tests__/integration/test-data', ], From 8857737bdf2aee115fe9d4b3c35c16158d92903a Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Fri, 14 Jun 2024 19:13:08 +0400 Subject: [PATCH 101/243] revert(config): drop issue template message --- src/config/strings.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/config/strings.ts b/src/config/strings.ts index 83283e8fb..7efa57a76 100644 --- a/src/config/strings.ts +++ b/src/config/strings.ts @@ -17,9 +17,6 @@ Incubation projects are experimental, offer no support guarantee, have minimal g NOT_NATIVE_PLUGIN: (path: string) => ` You are using plugin ${path} which is not part of the Impact Framework standard library. You should do your own research to ensure the plugins are up to date and accurate. They may not be actively maintained.`, - ISSUE_TEMPLATE: ` -Impact Framework is an alpha release from the Green Software Foundation and is released to capture early feedback. If you'd like to offer some feedback, please use this issue template: -https://github.com/Green-Software-Foundation/if/issues/new?assignees=&labels=feedback&projects=&template=feedback.md&title=Feedback+-+`, INVALID_MODULE_PATH: (path: string, error?: any) => `Provided module \`${path}\` is invalid or not found. ${error ?? ''} `, @@ -55,6 +52,8 @@ Note that for the '--output' option you also need to define the output type in y TARGET_IS_NOT_YAML: 'Given target is not in yaml format.', INVALID_TARGET: 'Target is invalid.', INVALID_SOURCE: 'Source is invalid.', + UNSUPPORTED_ERROR: (errorName: string) => + `UnsupportedErrorClass: plugin threw error class: ${errorName} that is not recognized by Impact Framework`, /** Plugin messages */ MISSING_GLOBAL_CONFIG: 'Global config is not provided.', MISSING_INPUT_DATA: (param: string) => From 00f0dafda6bed1ce30180a5549de4e7f25618c29 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Fri, 14 Jun 2024 19:13:34 +0400 Subject: [PATCH 102/243] test(util): remove changed case --- src/__tests__/unit/util/helpers.test.ts | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/__tests__/unit/util/helpers.test.ts b/src/__tests__/unit/util/helpers.test.ts index 86ea702a4..d8dfe2096 100644 --- a/src/__tests__/unit/util/helpers.test.ts +++ b/src/__tests__/unit/util/helpers.test.ts @@ -30,15 +30,6 @@ describe('util/helpers: ', () => { mockError.mockReset(); }); - it('logs error and warn in case of error is unknown.', () => { - const message = 'mock-message'; - const MockError = class extends Error {}; - - andHandle(new MockError(message)); - expect(mockWarn).toHaveBeenCalledTimes(1); - expect(mockError).toHaveBeenCalledTimes(1); - }); - it('logs error in case of error is unknown.', () => { const message = 'mock-message'; From 3000cc9981c39a006f97e7f25a4cee2a3f57a3e1 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Fri, 14 Jun 2024 19:14:32 +0400 Subject: [PATCH 103/243] feat(util): exit on unknown error --- src/util/helpers.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/util/helpers.ts b/src/util/helpers.ts index fcab2e27a..0c422c96f 100644 --- a/src/util/helpers.ts +++ b/src/util/helpers.ts @@ -10,7 +10,7 @@ import {STRINGS} from '../config'; import {Difference} from '../types/lib/compare'; -const {ISSUE_TEMPLATE} = STRINGS; +const {UNSUPPORTED_ERROR} = STRINGS; /** * Impact engine error handler. Logs errors and appends issue template if error is unknown. @@ -21,7 +21,9 @@ export const andHandle = (error: Error) => { logger.error(error); if (!knownErrors.includes(error.name)) { - logger.warn(ISSUE_TEMPLATE); + logger.error(UNSUPPORTED_ERROR(error.name)); + // eslint-disable-next-line no-process-exit + process.exit(2); } }; From 1e04a0ecf6dd3acccb1a2e4ae6210d0ab62f570a Mon Sep 17 00:00:00 2001 From: manushak Date: Mon, 17 Jun 2024 15:28:19 +0400 Subject: [PATCH 104/243] fix(src): get all dependencies from the manifest file --- src/env.ts | 70 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 47 insertions(+), 23 deletions(-) diff --git a/src/env.ts b/src/env.ts index 5128f58d4..5e2dba498 100644 --- a/src/env.ts +++ b/src/env.ts @@ -26,9 +26,9 @@ const { const IfEnv = async () => { const commandArgs = await parseIfEnvArgs(); const options: EnvironmentOptions = { - folderPath: __dirname, + folderPath: process.cwd(), install: !!commandArgs.install, - dependencies: {'@grnsft/if': packageJson.version}, + dependencies: {}, }; if (commandArgs && commandArgs.manifest) { @@ -57,11 +57,8 @@ const getOptionsFromArgs = async (commandArgs: { install: boolean | undefined; }) => { const {manifest: manifestPath, install} = commandArgs; - const folderPath = path.dirname(manifestPath); - const manifest = await load(manifestPath); - const plugins = manifest.rawManifest?.initialize?.plugins || {}; const dependencies = manifest.rawManifest?.execution?.environment.dependencies || []; @@ -69,7 +66,7 @@ const getOptionsFromArgs = async (commandArgs: { throw new Error(FAILURE_MESSAGE_DEPENDENCIES); } - const pathsWithVersion = extractPathsWithVersion(plugins, dependencies); + const pathsWithVersion = extractPathsWithVersion(dependencies); return { folderPath, @@ -81,25 +78,21 @@ const getOptionsFromArgs = async (commandArgs: { /** * Gets depencecies with versions. */ -const extractPathsWithVersion = (plugins: any, dependencies: string[]) => { - const paths = Object.keys(plugins).map(plugin => plugins[plugin].path); - const uniquePaths = [...new Set(paths)].filter(path => path !== 'builtin'); +const extractPathsWithVersion = (dependencies: string[]) => { const pathsWithVersion: PathWithVersion = {}; - uniquePaths.forEach(pluginPath => { - const dependency = dependencies.find((dependency: string) => - dependency.startsWith(pluginPath) - ); - - if (dependency) { - const splittedDependency = dependency.split('@'); - const version = - splittedDependency.length > 2 - ? splittedDependency[2].split(' ')[0] - : splittedDependency[1]; - - pathsWithVersion[pluginPath] = `^${version}`; - } + dependencies.forEach(dependency => { + const splittedDependency = dependency.split('@'); + const packageName = + splittedDependency.length > 2 + ? `@${splittedDependency[1]}` + : `@${splittedDependency[0]}`; + const version = + splittedDependency.length > 2 + ? splittedDependency[2].split(' ')[0] + : splittedDependency[1]; + + pathsWithVersion[packageName] = `^${version}`; }); return pathsWithVersion; @@ -112,9 +105,14 @@ const initializeAndInstallLibs = async (options: EnvironmentOptions) => { try { const {folderPath, install, dependencies} = options; + if (!Object.keys(dependencies).length) { + throw new Error(FAILURE_MESSAGE_DEPENDENCIES); + } + await fs.mkdir(folderPath, {recursive: true}); const packageJsonPath = await initPackageJsonIfNotExists(folderPath); + await updatePackageJsonProperties(packageJsonPath); if (install) { await installDependencies(folderPath, dependencies); @@ -123,9 +121,35 @@ const initializeAndInstallLibs = async (options: EnvironmentOptions) => { } } catch (error) { console.log(FAILURE_MESSAGE); + process.exit(2); } }; +/** + * Update the package.json properties. + */ +const updatePackageJsonProperties = async (packageJsonPath: string) => { + const packageJsonContent = await fs.readFile(packageJsonPath, 'utf8'); + const parsedPackageJsonContent = JSON.parse(packageJsonContent); + + const properties = { + name: 'if-environment', + description: packageJson.description, + author: packageJson.author, + bugs: packageJson.bugs, + engines: packageJson.engines, + homepage: packageJson.homepage, + }; + + const newPackageJson = Object.assign( + {}, + parsedPackageJsonContent, + properties + ); + + await fs.writeFile(packageJsonPath, JSON.stringify(newPackageJson, null, 2)); +}; + /** * Updates package.json dependencies. */ From 22febb1eda96d131ba4149378d82f573c28fae60 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Tue, 18 Jun 2024 11:17:02 +0400 Subject: [PATCH 105/243] chore(src): fetch changes from remote --- manifests/plugins/csv-lookup/success.yml | 2 +- manifests/plugins/divide/success.yml | 2 +- manifests/plugins/sum/success.yml | 2 +- package-lock.json | 19 +++++++++++++++++++ package.json | 2 +- src/env.ts | 1 + src/util/helpers.ts | 1 - 7 files changed, 24 insertions(+), 5 deletions(-) diff --git a/manifests/plugins/csv-lookup/success.yml b/manifests/plugins/csv-lookup/success.yml index f57b4718a..c6fc07d7b 100644 --- a/manifests/plugins/csv-lookup/success.yml +++ b/manifests/plugins/csv-lookup/success.yml @@ -23,4 +23,4 @@ tree: - timestamp: 2023-08-06T00:00 duration: 3600 cloud/provider: Google Cloud - cloud/region: asia-east-1 + cloud/region: asia-east1 diff --git a/manifests/plugins/divide/success.yml b/manifests/plugins/divide/success.yml index ed75a7c5c..d9840205a 100644 --- a/manifests/plugins/divide/success.yml +++ b/manifests/plugins/divide/success.yml @@ -2,7 +2,7 @@ name: divide description: success path tags: initialize: -# outputs: ['yaml'] + outputs: ['yaml'] plugins: cloud-metadata: method: CloudMetadata diff --git a/manifests/plugins/sum/success.yml b/manifests/plugins/sum/success.yml index fc0661bf5..454efc9e0 100644 --- a/manifests/plugins/sum/success.yml +++ b/manifests/plugins/sum/success.yml @@ -2,7 +2,7 @@ name: sum description: successful path tags: initialize: - outputs: ['yaml'] + # outputs: ['yaml'] plugins: sum: method: Sum diff --git a/package-lock.json b/package-lock.json index e42ff4db5..73687b54f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -37,6 +37,7 @@ "@types/luxon": "^3.4.2", "@types/node": "^20.8.9", "axios-mock-adapter": "^1.22.0", + "cross-env": "7.0.3", "fixpack": "^4.0.0", "gts": "^5.0.0", "husky": "^8.0.0", @@ -3772,6 +3773,24 @@ "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, + "node_modules/cross-env": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-7.0.3.tgz", + "integrity": "sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw==", + "dev": true, + "dependencies": { + "cross-spawn": "^7.0.1" + }, + "bin": { + "cross-env": "src/bin/cross-env.js", + "cross-env-shell": "src/bin/cross-env-shell.js" + }, + "engines": { + "node": ">=10.14", + "npm": ">=6", + "yarn": ">=1" + } + }, "node_modules/cross-spawn": { "version": "7.0.3", "license": "MIT", diff --git a/package.json b/package.json index 3bb7cb5c4..a9d044f33 100644 --- a/package.json +++ b/package.json @@ -76,7 +76,7 @@ "fix:package": "fixpack", "if-diff": "npx ts-node src/diff.ts", "if-run": "npx ts-node src/index.ts", - "if-env": "npx ts-node src/env.ts", + "if-env": "cross-env CURRENT_DIR=$(node -p \"process.env.INIT_CWD\") npx ts-node src/env.ts", "lint": "gts lint", "pre-commit": "lint-staged", "prepare": "husky install", diff --git a/src/env.ts b/src/env.ts index 5128f58d4..92f563e38 100644 --- a/src/env.ts +++ b/src/env.ts @@ -24,6 +24,7 @@ const { } = IF_ENV; const IfEnv = async () => { + console.log(process.env.CURRENT_DIR); const commandArgs = await parseIfEnvArgs(); const options: EnvironmentOptions = { folderPath: __dirname, diff --git a/src/util/helpers.ts b/src/util/helpers.ts index a2c69e0ea..9cac1f9d9 100644 --- a/src/util/helpers.ts +++ b/src/util/helpers.ts @@ -1,6 +1,5 @@ import * as fs from 'fs/promises'; import * as path from 'path'; - import {createInterface} from 'node:readline/promises'; import {exec} from 'node:child_process'; import {promisify} from 'node:util'; From be0862bfd373107d8baeabf88df7f19e94e81120 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Tue, 18 Jun 2024 11:22:21 +0400 Subject: [PATCH 106/243] feat(package): init cross env --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index a9d044f33..81a4d33e1 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "@types/luxon": "^3.4.2", "@types/node": "^20.8.9", "axios-mock-adapter": "^1.22.0", + "cross-env": "7.0.3", "fixpack": "^4.0.0", "gts": "^5.0.0", "husky": "^8.0.0", @@ -75,8 +76,8 @@ "fix": "gts fix", "fix:package": "fixpack", "if-diff": "npx ts-node src/diff.ts", - "if-run": "npx ts-node src/index.ts", "if-env": "cross-env CURRENT_DIR=$(node -p \"process.env.INIT_CWD\") npx ts-node src/env.ts", + "if-run": "npx ts-node src/index.ts", "lint": "gts lint", "pre-commit": "lint-staged", "prepare": "husky install", From 4a8a962341ab409e39e54823db3a82c6a7f5c279 Mon Sep 17 00:00:00 2001 From: manushak Date: Tue, 18 Jun 2024 14:04:33 +0400 Subject: [PATCH 107/243] fix(src): fix getting command working directory --- package-lock.json | 19 +++++++++++++++++++ package.json | 3 ++- src/env.ts | 10 ++++++++-- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index e42ff4db5..73687b54f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -37,6 +37,7 @@ "@types/luxon": "^3.4.2", "@types/node": "^20.8.9", "axios-mock-adapter": "^1.22.0", + "cross-env": "7.0.3", "fixpack": "^4.0.0", "gts": "^5.0.0", "husky": "^8.0.0", @@ -3772,6 +3773,24 @@ "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, + "node_modules/cross-env": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-7.0.3.tgz", + "integrity": "sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw==", + "dev": true, + "dependencies": { + "cross-spawn": "^7.0.1" + }, + "bin": { + "cross-env": "src/bin/cross-env.js", + "cross-env-shell": "src/bin/cross-env-shell.js" + }, + "engines": { + "node": ">=10.14", + "npm": ">=6", + "yarn": ">=1" + } + }, "node_modules/cross-spawn": { "version": "7.0.3", "license": "MIT", diff --git a/package.json b/package.json index 3bb7cb5c4..81a4d33e1 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "@types/luxon": "^3.4.2", "@types/node": "^20.8.9", "axios-mock-adapter": "^1.22.0", + "cross-env": "7.0.3", "fixpack": "^4.0.0", "gts": "^5.0.0", "husky": "^8.0.0", @@ -75,8 +76,8 @@ "fix": "gts fix", "fix:package": "fixpack", "if-diff": "npx ts-node src/diff.ts", + "if-env": "cross-env CURRENT_DIR=$(node -p \"process.env.INIT_CWD\") npx ts-node src/env.ts", "if-run": "npx ts-node src/index.ts", - "if-env": "npx ts-node src/env.ts", "lint": "gts lint", "pre-commit": "lint-staged", "prepare": "husky install", diff --git a/src/env.ts b/src/env.ts index 5e2dba498..12f29d903 100644 --- a/src/env.ts +++ b/src/env.ts @@ -24,9 +24,10 @@ const { } = IF_ENV; const IfEnv = async () => { + console.log(process.env.CURRENT_DIR); const commandArgs = await parseIfEnvArgs(); const options: EnvironmentOptions = { - folderPath: process.cwd(), + folderPath: process.env.CURRENT_DIR || process.cwd(), install: !!commandArgs.install, dependencies: {}, }; @@ -36,7 +37,12 @@ const IfEnv = async () => { await getOptionsFromArgs(commandArgs); options.folderPath = folderPath; options.install = !!install; - options.dependencies = {...options.dependencies, ...dependencies}; + options.dependencies = {...dependencies}; + } else { + options.dependencies = { + ...packageJson.depencecies, + ...packageJson.devDependencies, + }; } await initializeAndInstallLibs(options); From aebd849a617bbd0b600727ceb72bafe7766c5c7b Mon Sep 17 00:00:00 2001 From: manushak Date: Wed, 19 Jun 2024 14:39:58 +0400 Subject: [PATCH 108/243] feat(types): add `cmd` property to IFEnvArgs type --- src/types/process-args.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/types/process-args.ts b/src/types/process-args.ts index 6b8cfc050..83561e55c 100644 --- a/src/types/process-args.ts +++ b/src/types/process-args.ts @@ -14,6 +14,7 @@ export interface IFDiffArgs { export interface IFEnvArgs { manifest?: string; install?: boolean; + cmd?: boolean; } export interface Options { From d6f4280e366d3296434a2ab5f1784ab5c3ea2eaa Mon Sep 17 00:00:00 2001 From: manushak Date: Wed, 19 Jun 2024 14:41:27 +0400 Subject: [PATCH 109/243] feat(config): add `cmd` flag for if-env --- src/config/config.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/config/config.ts b/src/config/config.ts index cbd389ff2..13acbd9b0 100644 --- a/src/config/config.ts +++ b/src/config/config.ts @@ -95,7 +95,14 @@ export const CONFIG = { type: Boolean, optional: true, alias: 'i', - description: '[commant to install package.json]', + description: '[command to install package.json]', + }, + cmd: { + type: Boolean, + optional: true, + alias: 'c', + description: + '[command to generate the package.json in the command working directory]', }, } as ArgumentConfig, HELP: { From c9fc62b59fa62547a80dadc681d3c793fa444128 Mon Sep 17 00:00:00 2001 From: manushak Date: Wed, 19 Jun 2024 14:49:21 +0400 Subject: [PATCH 110/243] feat(util): update parseIfEnvArgs function to add cmd flag --- src/util/args.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/util/args.ts b/src/util/args.ts index 013b94a83..3f7d2c066 100644 --- a/src/util/args.ts +++ b/src/util/args.ts @@ -173,7 +173,7 @@ const validateAndParseIfEnvArgs = () => { * Checks if the `manifest` command is provided and it is valid manifest file. */ export const parseIfEnvArgs = async () => { - const {manifest, install} = validateAndParseIfEnvArgs(); + const {manifest, install, cmd} = validateAndParseIfEnvArgs(); if (manifest) { const isManifestFileExists = await isFileExists(manifest); @@ -185,11 +185,11 @@ export const parseIfEnvArgs = async () => { if (checkIfFileIsYaml(manifest)) { const response = prependFullFilePath(manifest); - return {manifest: response, install}; + return {manifest: response, install, cmd}; } throw new CliInputError(FILE_IS_NOT_YAML); } - return {install}; + return {install, cmd}; }; From a2dd0227af2d1bcfc70452201b59a6b6c5a98da2 Mon Sep 17 00:00:00 2001 From: manushak Date: Wed, 19 Jun 2024 14:51:17 +0400 Subject: [PATCH 111/243] feat(types): add cmd property into EnvironmentOptions type and add ManifestPlugin type --- src/types/if-env.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/types/if-env.ts b/src/types/if-env.ts index b8a63f7ef..547787cea 100644 --- a/src/types/if-env.ts +++ b/src/types/if-env.ts @@ -1,7 +1,10 @@ export type EnvironmentOptions = { folderPath: string; install: boolean; + cmd: boolean; dependencies: {[path: string]: string}; }; export type PathWithVersion = {[path: string]: string}; + +export type ManifestPlugin = {[key: string]: {path: string; method: string}}; From ff443915a3b3eeec8a36b7b0d9268e51306b3bdb Mon Sep 17 00:00:00 2001 From: manushak Date: Wed, 19 Jun 2024 14:53:21 +0400 Subject: [PATCH 112/243] fix(src): add cmd flag, fix template manifest directory and package.json depencencies --- src/env.ts | 84 +++++++++++++++++++++++++++++------------------------- 1 file changed, 45 insertions(+), 39 deletions(-) diff --git a/src/env.ts b/src/env.ts index 12f29d903..5a9f3bd8f 100644 --- a/src/env.ts +++ b/src/env.ts @@ -11,7 +11,11 @@ import {load} from './lib/load'; import {CONFIG} from './config'; -import {EnvironmentOptions, PathWithVersion} from './types/if-env'; +import { + EnvironmentOptions, + ManifestPlugin, + PathWithVersion, +} from './types/if-env'; const packageJson = require('../package.json'); @@ -24,31 +28,26 @@ const { } = IF_ENV; const IfEnv = async () => { - console.log(process.env.CURRENT_DIR); const commandArgs = await parseIfEnvArgs(); const options: EnvironmentOptions = { folderPath: process.env.CURRENT_DIR || process.cwd(), install: !!commandArgs.install, dependencies: {}, + cmd: !!commandArgs.cmd, }; if (commandArgs && commandArgs.manifest) { const {folderPath, install, dependencies} = await getOptionsFromArgs(commandArgs); - options.folderPath = folderPath; + options.folderPath = commandArgs.cmd ? options.folderPath : folderPath; options.install = !!install; options.dependencies = {...dependencies}; - } else { - options.dependencies = { - ...packageJson.depencecies, - ...packageJson.devDependencies, - }; } await initializeAndInstallLibs(options); if (!commandArgs || !commandArgs.manifest) { - await addTemplateManifest(); + await addTemplateManifest(options.folderPath); } console.log(SUCCESS_MESSAGE); @@ -65,6 +64,7 @@ const getOptionsFromArgs = async (commandArgs: { const {manifest: manifestPath, install} = commandArgs; const folderPath = path.dirname(manifestPath); const manifest = await load(manifestPath); + const plugins = manifest.rawManifest?.initialize?.plugins || {}; const dependencies = manifest.rawManifest?.execution?.environment.dependencies || []; @@ -72,7 +72,7 @@ const getOptionsFromArgs = async (commandArgs: { throw new Error(FAILURE_MESSAGE_DEPENDENCIES); } - const pathsWithVersion = extractPathsWithVersion(dependencies); + const pathsWithVersion = extractPathsWithVersion(plugins, dependencies); return { folderPath, @@ -84,21 +84,28 @@ const getOptionsFromArgs = async (commandArgs: { /** * Gets depencecies with versions. */ -const extractPathsWithVersion = (dependencies: string[]) => { +const extractPathsWithVersion = ( + plugins: ManifestPlugin, + dependencies: string[] +) => { + const paths = Object.keys(plugins).map(plugin => plugins[plugin].path); + const uniquePaths = [...new Set(paths)].filter(path => path !== 'builtin'); const pathsWithVersion: PathWithVersion = {}; - dependencies.forEach(dependency => { - const splittedDependency = dependency.split('@'); - const packageName = - splittedDependency.length > 2 - ? `@${splittedDependency[1]}` - : `@${splittedDependency[0]}`; - const version = - splittedDependency.length > 2 - ? splittedDependency[2].split(' ')[0] - : splittedDependency[1]; - - pathsWithVersion[packageName] = `^${version}`; + uniquePaths.forEach(pluginPath => { + const dependency = dependencies.find((dependency: string) => + dependency.startsWith(pluginPath) + ); + + if (dependency) { + const splittedDependency = dependency.split('@'); + const version = + splittedDependency.length > 2 + ? splittedDependency[2].split(' ')[0] + : splittedDependency[1]; + + pathsWithVersion[pluginPath] = `^${version}`; + } }); return pathsWithVersion; @@ -109,21 +116,15 @@ const extractPathsWithVersion = (dependencies: string[]) => { */ const initializeAndInstallLibs = async (options: EnvironmentOptions) => { try { - const {folderPath, install, dependencies} = options; - - if (!Object.keys(dependencies).length) { - throw new Error(FAILURE_MESSAGE_DEPENDENCIES); - } - - await fs.mkdir(folderPath, {recursive: true}); - + const {folderPath, install, cmd, dependencies} = options; const packageJsonPath = await initPackageJsonIfNotExists(folderPath); + await updatePackageJsonProperties(packageJsonPath); if (install) { await installDependencies(folderPath, dependencies); } else { - await updatePackageJsonDependencies(packageJsonPath, dependencies); + await updatePackageJsonDependencies(packageJsonPath, dependencies, cmd); } } catch (error) { console.log(FAILURE_MESSAGE); @@ -161,15 +162,20 @@ const updatePackageJsonProperties = async (packageJsonPath: string) => { */ const updatePackageJsonDependencies = async ( packageJsonPath: string, - dependencies: PathWithVersion + dependencies: PathWithVersion, + cmd: boolean ) => { const packageJsonContent = await fs.readFile(packageJsonPath, 'utf8'); const packageJson = JSON.parse(packageJsonContent); - packageJson.dependencies = { - ...packageJson.dependencies, - ...dependencies, - }; + if (cmd) { + packageJson.dependencies = { + ...packageJson.dependencies, + ...dependencies, + }; + } else { + packageJson.dependencies = {...dependencies}; + } await fs.writeFile(packageJsonPath, JSON.stringify(packageJson, null, 2)); }; @@ -177,10 +183,10 @@ const updatePackageJsonDependencies = async ( /** * Adds a manifest template to the folder where the if-env CLI command runs. */ -const addTemplateManifest = async () => { +const addTemplateManifest = async (destinationDir: string) => { try { const templateManifest = path.resolve(__dirname, './env-template.yml'); - const destinationPath = path.resolve(__dirname, 'manifest.yml'); + const destinationPath = path.resolve(destinationDir, 'manifest.yml'); const data = await fs.readFile(templateManifest, 'utf-8'); await fs.writeFile(destinationPath, '', 'utf-8'); From f1b204aaa0df73a70e7da1b32fa5d93576ebe951 Mon Sep 17 00:00:00 2001 From: manushak Date: Wed, 19 Jun 2024 15:13:43 +0400 Subject: [PATCH 113/243] fix(src): rename `cmd` to `cwd` --- src/config/config.ts | 2 +- src/env.ts | 12 ++++++------ src/types/if-env.ts | 2 +- src/types/process-args.ts | 2 +- src/util/args.ts | 6 +++--- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/config/config.ts b/src/config/config.ts index 13acbd9b0..0f96019f2 100644 --- a/src/config/config.ts +++ b/src/config/config.ts @@ -97,7 +97,7 @@ export const CONFIG = { alias: 'i', description: '[command to install package.json]', }, - cmd: { + cwd: { type: Boolean, optional: true, alias: 'c', diff --git a/src/env.ts b/src/env.ts index 5a9f3bd8f..e2232d8f4 100644 --- a/src/env.ts +++ b/src/env.ts @@ -33,13 +33,13 @@ const IfEnv = async () => { folderPath: process.env.CURRENT_DIR || process.cwd(), install: !!commandArgs.install, dependencies: {}, - cmd: !!commandArgs.cmd, + cwd: !!commandArgs.cwd, }; if (commandArgs && commandArgs.manifest) { const {folderPath, install, dependencies} = await getOptionsFromArgs(commandArgs); - options.folderPath = commandArgs.cmd ? options.folderPath : folderPath; + options.folderPath = commandArgs.cwd ? options.folderPath : folderPath; options.install = !!install; options.dependencies = {...dependencies}; } @@ -116,7 +116,7 @@ const extractPathsWithVersion = ( */ const initializeAndInstallLibs = async (options: EnvironmentOptions) => { try { - const {folderPath, install, cmd, dependencies} = options; + const {folderPath, install, cwd, dependencies} = options; const packageJsonPath = await initPackageJsonIfNotExists(folderPath); await updatePackageJsonProperties(packageJsonPath); @@ -124,7 +124,7 @@ const initializeAndInstallLibs = async (options: EnvironmentOptions) => { if (install) { await installDependencies(folderPath, dependencies); } else { - await updatePackageJsonDependencies(packageJsonPath, dependencies, cmd); + await updatePackageJsonDependencies(packageJsonPath, dependencies, cwd); } } catch (error) { console.log(FAILURE_MESSAGE); @@ -163,12 +163,12 @@ const updatePackageJsonProperties = async (packageJsonPath: string) => { const updatePackageJsonDependencies = async ( packageJsonPath: string, dependencies: PathWithVersion, - cmd: boolean + cwd: boolean ) => { const packageJsonContent = await fs.readFile(packageJsonPath, 'utf8'); const packageJson = JSON.parse(packageJsonContent); - if (cmd) { + if (cwd) { packageJson.dependencies = { ...packageJson.dependencies, ...dependencies, diff --git a/src/types/if-env.ts b/src/types/if-env.ts index 547787cea..796fd0ca8 100644 --- a/src/types/if-env.ts +++ b/src/types/if-env.ts @@ -1,7 +1,7 @@ export type EnvironmentOptions = { folderPath: string; install: boolean; - cmd: boolean; + cwd: boolean; dependencies: {[path: string]: string}; }; diff --git a/src/types/process-args.ts b/src/types/process-args.ts index 83561e55c..40dece54d 100644 --- a/src/types/process-args.ts +++ b/src/types/process-args.ts @@ -14,7 +14,7 @@ export interface IFDiffArgs { export interface IFEnvArgs { manifest?: string; install?: boolean; - cmd?: boolean; + cwd?: boolean; } export interface Options { diff --git a/src/util/args.ts b/src/util/args.ts index 3f7d2c066..fce459016 100644 --- a/src/util/args.ts +++ b/src/util/args.ts @@ -173,7 +173,7 @@ const validateAndParseIfEnvArgs = () => { * Checks if the `manifest` command is provided and it is valid manifest file. */ export const parseIfEnvArgs = async () => { - const {manifest, install, cmd} = validateAndParseIfEnvArgs(); + const {manifest, install, cwd} = validateAndParseIfEnvArgs(); if (manifest) { const isManifestFileExists = await isFileExists(manifest); @@ -185,11 +185,11 @@ export const parseIfEnvArgs = async () => { if (checkIfFileIsYaml(manifest)) { const response = prependFullFilePath(manifest); - return {manifest: response, install, cmd}; + return {manifest: response, install, cwd}; } throw new CliInputError(FILE_IS_NOT_YAML); } - return {install, cmd}; + return {install, cwd}; }; From 6819c5c487a47dd5a68ae64aec41875d8f0e395d Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Wed, 19 Jun 2024 13:23:07 +0100 Subject: [PATCH 114/243] fix(lib): fix import for error strings --- src/config/strings.ts | 8 +++++--- src/util/helpers.ts | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/config/strings.ts b/src/config/strings.ts index 7efa57a76..caf8d2bb0 100644 --- a/src/config/strings.ts +++ b/src/config/strings.ts @@ -1,4 +1,4 @@ -import {ManifestParameter} from '../types/manifest'; +import { ManifestParameter } from '../types/manifest'; export const STRINGS = { FILE_IS_NOT_YAML: 'Provided manifest is not in yaml format.', @@ -27,8 +27,7 @@ You are using plugin ${path} which is not part of the Impact Framework standard AVOIDING_PADDING: (description: string) => `Avoiding padding at ${description}`, AVOIDING_PADDING_BY_EDGES: (start: boolean, end: boolean) => - `Avoiding padding at ${ - start && end ? 'start and end' : start ? 'start' : 'end' + `Avoiding padding at ${start && end ? 'start and end' : start ? 'start' : 'end' }`, INVALID_AGGREGATION_METHOD: (metric: string) => `Aggregation is not possible for given ${metric} since method is 'none'.`, @@ -58,6 +57,9 @@ Note that for the '--output' option you also need to define the output type in y MISSING_GLOBAL_CONFIG: 'Global config is not provided.', MISSING_INPUT_DATA: (param: string) => `${param} is missing from the input array.`, + MANIFEST_NOT_FOUND: 'Manifest file not found.', + INITIALIZING_PACKAGE_JSON: 'Initializing package.json.', + INSTALLING_NPM_PACKAGES: 'Installing npm packages...', NOT_NUMERIC_VALUE: (str: any) => `${str} is not numberic.`, MISSING_FUNCTIONAL_UNIT_CONFIG: '`functional-unit` should be provided in your global config', diff --git a/src/util/helpers.ts b/src/util/helpers.ts index 98af5209d..cc61f93ac 100644 --- a/src/util/helpers.ts +++ b/src/util/helpers.ts @@ -1,18 +1,18 @@ import * as fs from 'fs/promises'; import * as path from 'path'; -import {createInterface} from 'node:readline/promises'; -import {exec} from 'node:child_process'; -import {promisify} from 'node:util'; +import { createInterface } from 'node:readline/promises'; +import { exec } from 'node:child_process'; +import { promisify } from 'node:util'; -import {ERRORS} from '@grnsft/if-core/utils'; +import { ERRORS } from '@grnsft/if-core/utils'; -import {logger} from './logger'; +import { logger } from './logger'; -import {STRINGS} from '../config'; +import { STRINGS } from '../config'; -import {Difference} from '../types/lib/compare'; +import { Difference } from '../types/lib/compare'; -const {UNSUPPORTED_ERROR} = STRINGS; +const { UNSUPPORTED_ERROR, INITIALIZING_PACKAGE_JSON, INSTALLING_NPM_PACKAGES } = STRINGS; /** * Impact engine error handler. Logs errors and appends issue template if error is unknown. @@ -200,7 +200,7 @@ export const initPackageJsonIfNotExists = async (folderPath: string) => { if (!isPackageJsonExists) { logger.info(INITIALIZING_PACKAGE_JSON); - await execPromise('npm init -y', {cwd: folderPath}); + await execPromise('npm init -y', { cwd: folderPath }); } return packageJsonPath; @@ -211,7 +211,7 @@ export const initPackageJsonIfNotExists = async (folderPath: string) => { */ export const installDependencies = async ( folderPath: string, - dependencies: {[path: string]: string} + dependencies: { [path: string]: string } ) => { const packages = Object.entries(dependencies).map( ([dependency, version]) => `${dependency}@${version.replace('^', '')}` From c9b6db6eba04f4b3e3943584e4c3549451962a0d Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Wed, 19 Jun 2024 13:23:54 +0100 Subject: [PATCH 115/243] fix(lib): apply linter --- src/config/strings.ts | 5 +++-- src/util/helpers.ts | 21 +++++++++++---------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/config/strings.ts b/src/config/strings.ts index caf8d2bb0..bf57a11d5 100644 --- a/src/config/strings.ts +++ b/src/config/strings.ts @@ -1,4 +1,4 @@ -import { ManifestParameter } from '../types/manifest'; +import {ManifestParameter} from '../types/manifest'; export const STRINGS = { FILE_IS_NOT_YAML: 'Provided manifest is not in yaml format.', @@ -27,7 +27,8 @@ You are using plugin ${path} which is not part of the Impact Framework standard AVOIDING_PADDING: (description: string) => `Avoiding padding at ${description}`, AVOIDING_PADDING_BY_EDGES: (start: boolean, end: boolean) => - `Avoiding padding at ${start && end ? 'start and end' : start ? 'start' : 'end' + `Avoiding padding at ${ + start && end ? 'start and end' : start ? 'start' : 'end' }`, INVALID_AGGREGATION_METHOD: (metric: string) => `Aggregation is not possible for given ${metric} since method is 'none'.`, diff --git a/src/util/helpers.ts b/src/util/helpers.ts index cc61f93ac..9abcf9ba4 100644 --- a/src/util/helpers.ts +++ b/src/util/helpers.ts @@ -1,18 +1,19 @@ import * as fs from 'fs/promises'; import * as path from 'path'; -import { createInterface } from 'node:readline/promises'; -import { exec } from 'node:child_process'; -import { promisify } from 'node:util'; +import {createInterface} from 'node:readline/promises'; +import {exec} from 'node:child_process'; +import {promisify} from 'node:util'; -import { ERRORS } from '@grnsft/if-core/utils'; +import {ERRORS} from '@grnsft/if-core/utils'; -import { logger } from './logger'; +import {logger} from './logger'; -import { STRINGS } from '../config'; +import {STRINGS} from '../config'; -import { Difference } from '../types/lib/compare'; +import {Difference} from '../types/lib/compare'; -const { UNSUPPORTED_ERROR, INITIALIZING_PACKAGE_JSON, INSTALLING_NPM_PACKAGES } = STRINGS; +const {UNSUPPORTED_ERROR, INITIALIZING_PACKAGE_JSON, INSTALLING_NPM_PACKAGES} = + STRINGS; /** * Impact engine error handler. Logs errors and appends issue template if error is unknown. @@ -200,7 +201,7 @@ export const initPackageJsonIfNotExists = async (folderPath: string) => { if (!isPackageJsonExists) { logger.info(INITIALIZING_PACKAGE_JSON); - await execPromise('npm init -y', { cwd: folderPath }); + await execPromise('npm init -y', {cwd: folderPath}); } return packageJsonPath; @@ -211,7 +212,7 @@ export const initPackageJsonIfNotExists = async (folderPath: string) => { */ export const installDependencies = async ( folderPath: string, - dependencies: { [path: string]: string } + dependencies: {[path: string]: string} ) => { const packages = Object.entries(dependencies).map( ([dependency, version]) => `${dependency}@${version.replace('^', '')}` From 39931f516b08bb6a50bdf39a4ff24743857d3d7a Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Wed, 19 Jun 2024 17:22:31 +0400 Subject: [PATCH 116/243] fix(util): optimize error classes usage in args --- src/util/args.ts | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/util/args.ts b/src/util/args.ts index c54117c75..523791f2a 100644 --- a/src/util/args.ts +++ b/src/util/args.ts @@ -11,17 +11,11 @@ import {CONFIG, STRINGS} from '../config'; import {IFDiffArgs, IEArgs, ProcessArgsOutputs} from '../types/process-args'; import {LoadDiffParams} from '../types/util/args'; -const { - CliInputError, - ParseCliParamsError, - CliTargetFileError, - CliSourceFileError, -} = ERRORS; +const {ParseCliParamsError, CliTargetFileError, CliSourceFileError} = ERRORS; const {IE, IF_DIFF} = CONFIG; const { - FILE_IS_NOT_YAML, MANIFEST_IS_MISSING, NO_OUTPUT, SOURCE_IS_NOT_YAML, @@ -37,7 +31,7 @@ const validateAndParseProcessArgs = () => { return parse(IE.ARGS, IE.HELP); } catch (error) { if (error instanceof Error) { - throw new CliInputError(error.message); + throw new ParseCliParamsError(error.message); } throw error; @@ -63,7 +57,7 @@ const prependFullFilePath = (filePath: string) => { * 3. If output params are missing, warns user about it. * 3. Otherwise checks if `manifest` param is there, then processes with checking if it's a yaml file. * If it is, then returns object containing full path. - * 4. If params are missing or invalid, then rejects with `CliInputError`. + * 4. If params are missing or invalid, then rejects with `ParseCliParamsError`. */ export const parseIEProcessArgs = (): ProcessArgsOutputs => { const { @@ -91,7 +85,7 @@ export const parseIEProcessArgs = (): ProcessArgsOutputs => { }; } - throw new CliSourceFileError(FILE_IS_NOT_YAML); + throw new CliSourceFileError(SOURCE_IS_NOT_YAML); } throw new CliSourceFileError(MANIFEST_IS_MISSING); @@ -140,5 +134,5 @@ export const parseIfDiffArgs = () => { throw new CliTargetFileError(TARGET_IS_NOT_YAML); } - throw new CliInputError(INVALID_TARGET); + throw new ParseCliParamsError(INVALID_TARGET); }; From d235e6fe1cc68b76ecd1d45bfdf913fadad4a18d Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Wed, 19 Jun 2024 18:11:21 +0400 Subject: [PATCH 117/243] fix(config): drop unused string, fix typo --- src/config/strings.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/config/strings.ts b/src/config/strings.ts index 7efa57a76..82e6d388b 100644 --- a/src/config/strings.ts +++ b/src/config/strings.ts @@ -1,7 +1,6 @@ import {ManifestParameter} from '../types/manifest'; export const STRINGS = { - FILE_IS_NOT_YAML: 'Provided manifest is not in yaml format.', MANIFEST_IS_MISSING: 'Manifest is missing.', MISSING_METHOD: "Initalization param 'method' is missing.", MISSING_PATH: "Initalization param 'path' is missing.", @@ -49,7 +48,7 @@ You have not selected an output method. To see your output data, you can choose --output : this will save your output data to the given filepath (do not provide file extension) Note that for the '--output' option you also need to define the output type in your manifest file. See https://if.greensoftware.foundation/major-concepts/manifest-file#initialize`, SOURCE_IS_NOT_YAML: 'Given source file is not in yaml format.', - TARGET_IS_NOT_YAML: 'Given target is not in yaml format.', + TARGET_IS_NOT_YAML: 'Given target file is not in yaml format.', INVALID_TARGET: 'Target is invalid.', INVALID_SOURCE: 'Source is invalid.', UNSUPPORTED_ERROR: (errorName: string) => From 990a7c7016ba578f20db28c43573560c665875f0 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Wed, 19 Jun 2024 18:11:53 +0400 Subject: [PATCH 118/243] test(util): apply specific errors to args --- src/__tests__/unit/util/args.test.ts | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/__tests__/unit/util/args.test.ts b/src/__tests__/unit/util/args.test.ts index 045bf2807..61c8c4577 100644 --- a/src/__tests__/unit/util/args.test.ts +++ b/src/__tests__/unit/util/args.test.ts @@ -76,11 +76,10 @@ import {parseIEProcessArgs, parseIfDiffArgs} from '../../../util/args'; import {STRINGS} from '../../../config'; -const {CliInputError, CliSourceFileError} = ERRORS; +const {CliSourceFileError, ParseCliParamsError} = ERRORS; const { MANIFEST_IS_MISSING, - FILE_IS_NOT_YAML, TARGET_IS_NOT_YAML, INVALID_TARGET, SOURCE_IS_NOT_YAML, @@ -106,8 +105,8 @@ describe('util/args: ', () => { try { parseIEProcessArgs(); } catch (error) { - expect(error).toBeInstanceOf(CliInputError); - expect(error).toEqual(new CliInputError(MANIFEST_IS_MISSING)); + expect(error).toBeInstanceOf(ParseCliParamsError); + expect(error).toEqual(new ParseCliParamsError(MANIFEST_IS_MISSING)); } process.env.result = 'manifest-is-missing'; @@ -196,7 +195,7 @@ describe('util/args: ', () => { parseIEProcessArgs(); } catch (error) { expect(error).toBeInstanceOf(CliSourceFileError); - expect(error).toEqual(new CliSourceFileError(FILE_IS_NOT_YAML)); + expect(error).toEqual(new CliSourceFileError(SOURCE_IS_NOT_YAML)); } }); @@ -226,7 +225,7 @@ describe('util/args: ', () => { parseIfDiffArgs(); } catch (error) { if (error instanceof Error) { - expect(error).toEqual(new CliInputError(INVALID_TARGET)); + expect(error).toEqual(new ParseCliParamsError(INVALID_TARGET)); } } }); @@ -239,7 +238,7 @@ describe('util/args: ', () => { parseIfDiffArgs(); } catch (error) { if (error instanceof Error) { - expect(error).toEqual(new CliInputError(TARGET_IS_NOT_YAML)); + expect(error).toEqual(new ParseCliParamsError(TARGET_IS_NOT_YAML)); } } }); @@ -260,7 +259,7 @@ describe('util/args: ', () => { parseIfDiffArgs(); } catch (error) { if (error instanceof Error) { - expect(error).toEqual(new CliInputError(SOURCE_IS_NOT_YAML)); + expect(error).toEqual(new ParseCliParamsError(SOURCE_IS_NOT_YAML)); } } }); @@ -282,7 +281,7 @@ describe('util/args: ', () => { parseIfDiffArgs(); } catch (error) { if (error instanceof Error) { - expect(error).toEqual(new CliInputError('mock-error')); + expect(error).toEqual(new ParseCliParamsError('mock-error')); } } }); From 3c09a5002723ea4ab1f094b903b26472e56f344d Mon Sep 17 00:00:00 2001 From: manushak Date: Wed, 19 Jun 2024 18:56:42 +0400 Subject: [PATCH 119/243] fix(config): move template manifest into config directory --- src/{ => config}/env-template.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/{ => config}/env-template.yml (100%) diff --git a/src/env-template.yml b/src/config/env-template.yml similarity index 100% rename from src/env-template.yml rename to src/config/env-template.yml From 14b2b38bf20a8b63ba1bf736024bbb12777f8092 Mon Sep 17 00:00:00 2001 From: manushak Date: Wed, 19 Jun 2024 18:57:49 +0400 Subject: [PATCH 120/243] fix(src): update template manifest path --- src/env.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/env.ts b/src/env.ts index e2232d8f4..759ae0be5 100644 --- a/src/env.ts +++ b/src/env.ts @@ -185,7 +185,10 @@ const updatePackageJsonDependencies = async ( */ const addTemplateManifest = async (destinationDir: string) => { try { - const templateManifest = path.resolve(__dirname, './env-template.yml'); + const templateManifest = path.resolve( + __dirname, + './config/env-template.yml' + ); const destinationPath = path.resolve(destinationDir, 'manifest.yml'); const data = await fs.readFile(templateManifest, 'utf-8'); From a6a8b320068584d68bb8fe860240c230dbcb6d38 Mon Sep 17 00:00:00 2001 From: MariamKhalatova Date: Thu, 20 Jun 2024 12:40:47 +0400 Subject: [PATCH 121/243] fix(manifests): update bugs to use builtins --- .../bugs/aggregation-error-wrong-metric.yml | 24 ++++++++++++------- .../bugs/azure-importer-ignoring-defaults.yml | 9 ++++--- .../azure-importer-incorrect-calculation.yml | 9 ++++--- manifests/bugs/pipeline-ordering-error.yml | 11 +++++---- 4 files changed, 35 insertions(+), 18 deletions(-) diff --git a/manifests/bugs/aggregation-error-wrong-metric.yml b/manifests/bugs/aggregation-error-wrong-metric.yml index e9a9af6ea..f95a9eed7 100644 --- a/manifests/bugs/aggregation-error-wrong-metric.yml +++ b/manifests/bugs/aggregation-error-wrong-metric.yml @@ -12,15 +12,23 @@ initialize: method: TeadsCurve global-config: interpolation: spline - "sci-e": - path: "@grnsft/if-plugins" - method: SciE + "sum-energy-components": + path: "builtin" + method: Sum + global-config: + input-parameters: + - cpu/energy + - network/energy + output-parameter: energy "sci-embodied": path: "builtin" method: SciEmbodied - "sci-o": - path: "@grnsft/if-plugins" - method: SciO + "operational-carbon": + method: Multiply + path: builtin + global-config: + input-parameters: ["energy", "grid/carbon-intensity"] + output-parameter: "carbon-operational" "sci": path: "builtin" method: Sci @@ -43,9 +51,9 @@ tree: child-1: pipeline: - teads-curve - - sci-e + - sum-energy-components - sci-embodied - - sci-o + - operational-carbon - time-sync - sci config: diff --git a/manifests/bugs/azure-importer-ignoring-defaults.yml b/manifests/bugs/azure-importer-ignoring-defaults.yml index 424be5e84..b6749ee70 100644 --- a/manifests/bugs/azure-importer-ignoring-defaults.yml +++ b/manifests/bugs/azure-importer-ignoring-defaults.yml @@ -28,9 +28,12 @@ initialize: input-parameter: network/energy coefficient: 1000 output-parameter: network/energy - "sci-o": - method: SciO - path: "@grnsft/if-plugins" + "operational-carbon": + method: Multiply + path: builtin + global-config: + input-parameters: ["energy", "grid/carbon-intensity"] + output-parameter: "carbon-operational" "group-by": path: "builtin" method: GroupBy diff --git a/manifests/bugs/azure-importer-incorrect-calculation.yml b/manifests/bugs/azure-importer-incorrect-calculation.yml index fa4cc17d0..bbac70ae2 100644 --- a/manifests/bugs/azure-importer-incorrect-calculation.yml +++ b/manifests/bugs/azure-importer-incorrect-calculation.yml @@ -20,9 +20,12 @@ initialize: input-parameter: network/energy coefficient: 1000 output-parameter: network/energy - sci-o: - path: '@grnsft/if-plugins' - method: SciO + "operational-carbon": + method: Multiply + path: builtin + global-config: + input-parameters: ["energy", "grid/carbon-intensity"] + output-parameter: "carbon-operational" group-by: path: builtin method: GroupBy diff --git a/manifests/bugs/pipeline-ordering-error.yml b/manifests/bugs/pipeline-ordering-error.yml index 3538465ea..554134287 100644 --- a/manifests/bugs/pipeline-ordering-error.yml +++ b/manifests/bugs/pipeline-ordering-error.yml @@ -19,9 +19,12 @@ initialize: "sci-embodied": path: "builtin" method: SciEmbodied - "sci-o": - path: "@grnsft/if-plugins" - method: SciO + "operational-carbon": + method: Multiply + path: builtin + global-config: + input-parameters: ["cpu/energy", "grid/carbon-intensity"] + output-parameter: "carbon-operational" "sci": path: "builtin" method: Sci @@ -41,7 +44,7 @@ tree: child-1: pipeline: - teads-curve - - sci-o + - operational-carbon - sum - sci-embodied - time-sync From ec099f839f0c273bd3c47af36c8da7939f4fa099 Mon Sep 17 00:00:00 2001 From: MariamKhalatova Date: Thu, 20 Jun 2024 12:42:45 +0400 Subject: [PATCH 122/243] fix(manifests): update examples to use builtins --- manifests/examples/basic.yml | 28 --- manifests/examples/instance-metadata.yml | 25 +-- .../examples/mock-cpu-util-to-carbon.yml | 10 +- manifests/examples/nesting.yml | 11 +- manifests/examples/pipeline-teads-sci.yml | 11 +- .../examples/pipeline-with-aggregate.yml | 178 +++++++++++------- manifests/examples/pipeline-with-mocks.yml | 169 ++++++++++------- 7 files changed, 240 insertions(+), 192 deletions(-) delete mode 100644 manifests/examples/basic.yml diff --git a/manifests/examples/basic.yml b/manifests/examples/basic.yml deleted file mode 100644 index e737f5d5b..000000000 --- a/manifests/examples/basic.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: basic -description: a minimal manifest executing a single plugin on a single component for a single timestep -tags: -initialize: - plugins: - teads-curve: - path: '@grnsft/if-unofficial-plugins' - method: TeadsCurve - global-config: - interpolation: spline - outputs: ['yaml'] -tree: - children: - child-0: - defaults: - cpu/thermal-design-power: 100 - pipeline: - - teads-curve - inputs: - - timestamp: 2023-07-06T00:00 - duration: 1 - cpu/utilization: 20 - - timestamp: 2023-07-06T00:01 - duration: 1 - cpu/utilization: 80 - - timestamp: 2023-07-06T00:02 - duration: 1 - cpu/utilization: 20 diff --git a/manifests/examples/instance-metadata.yml b/manifests/examples/instance-metadata.yml index c3e514773..ea0998241 100644 --- a/manifests/examples/instance-metadata.yml +++ b/manifests/examples/instance-metadata.yml @@ -1,23 +1,6 @@ name: csv-demo -<<<<<<< add-tdp-manifest description: tags: -initialize: - plugins: - cloud-instance-metadata: - method: CSVLookup - path: "builtin" - global-config: - filepath: https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-azure-instances.csv - query: - instance-class: "cloud/instance-type" - output: "*" - extract-processor-name: - method: Regex - path: "builtin" -======= -description: null -tags: null initialize: plugins: cloud-instance-metadata: @@ -32,16 +15,12 @@ initialize: extract-processor-name: path: builtin method: Regex ->>>>>>> main global-config: parameter: cpu-model-name match: /^([^,])+/g output: cpu/name -<<<<<<< add-tdp-manifest -======= - outputs: - - yaml ->>>>>>> main + # outputs: + # - yaml tree: children: child: diff --git a/manifests/examples/mock-cpu-util-to-carbon.yml b/manifests/examples/mock-cpu-util-to-carbon.yml index 35f7cea26..64e6d6719 100644 --- a/manifests/examples/mock-cpu-util-to-carbon.yml +++ b/manifests/examples/mock-cpu-util-to-carbon.yml @@ -21,8 +21,14 @@ initialize: global-config: interpolation: spline cloud-metadata: - method: CloudMetadata - path: "@grnsft/if-plugins" + path: builtin + method: CSVLookup + global-config: + filepath: >- + https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-azure-instances.csv + query: + instance-class: cloud/instance-type + output: ['cpu-cores-utilized', 'vcpus-allocated'] mock-observations: path: 'builtin' method: MockObservations diff --git a/manifests/examples/nesting.yml b/manifests/examples/nesting.yml index e56e8bb18..a5bf89230 100644 --- a/manifests/examples/nesting.yml +++ b/manifests/examples/nesting.yml @@ -29,9 +29,12 @@ initialize: sci-embodied: path: "builtin" method: SciEmbodied - sci-o: - path: "@grnsft/if-plugins" - method: SciO + "operational-carbon": + method: Multiply + path: builtin + global-config: + input-parameters: ["energy", "grid/carbon-intensity"] + output-parameter: "carbon-operational" sci: path: "builtin" method: Sci @@ -68,7 +71,7 @@ tree: - teads-curve - sum - sci-embodied - - sci-o + - operational-carbon - sum-carbon - time-sync - sci diff --git a/manifests/examples/pipeline-teads-sci.yml b/manifests/examples/pipeline-teads-sci.yml index 01c491cdc..bf20cc251 100644 --- a/manifests/examples/pipeline-teads-sci.yml +++ b/manifests/examples/pipeline-teads-sci.yml @@ -19,9 +19,12 @@ initialize: "sci-embodied": path: "builtin" method: SciEmbodied - "sci-o": - path: "@grnsft/if-plugins" - method: SciO + "operational-carbon": + method: Multiply + path: builtin + global-config: + input-parameters: ["cpu/energy", "grid/carbon-intensity"] + output-parameter: "carbon-operational" "sci": path: "builtin" method: Sci @@ -50,7 +53,7 @@ tree: - teads-curve - sum - sci-embodied - - sci-o + - operational-carbon - sum-carbon - sci config: diff --git a/manifests/examples/pipeline-with-aggregate.yml b/manifests/examples/pipeline-with-aggregate.yml index bd3e637ac..d1b90b152 100644 --- a/manifests/examples/pipeline-with-aggregate.yml +++ b/manifests/examples/pipeline-with-aggregate.yml @@ -7,25 +7,63 @@ aggregation: type: "both" initialize: plugins: - "teads-curve": - path: "@grnsft/if-unofficial-plugins" - method: TeadsCurve + "interpolate": + method: Interpolation + path: 'builtin' global-config: - interpolation: spline - "sci-e": - path: "@grnsft/if-plugins" - method: SciE + method: linear + x: [0, 10, 50, 100] + y: [0.12, 0.32, 0.75, 1.02] + input-parameter: 'cpu/utilization' + output-parameter: 'cpu-factor' + "cpu-factor-to-wattage": + method: Multiply + path: builtin + global-config: + input-parameters: ["cpu-factor", "cpu/thermal-design-power"] + output-parameter: "cpu-wattage" + "wattage-times-duration": + method: Multiply + path: builtin + global-config: + input-parameters: ["cpu-wattage", "duration"] + output-parameter: "cpu-wattage-times-duration" + "wattage-to-energy-kwh": + method: Divide + path: "builtin" + global-config: + numerator: cpu-wattage-times-duration + denominator: 3600000 + output: cpu-energy-raw + "calculate-vcpu-ratio": + method: Divide + path: "builtin" + global-config: + numerator: vcpus-total + denominator: vcpus-allocated + output: vcpu-ratio + "correct-cpu-energy-for-vcpu-ratio": + method: Divide + path: "builtin" + global-config: + numerator: cpu-energy-raw + denominator: vcpu-ratio + output: cpu-energy-kwh "sci-embodied": path: "builtin" method: SciEmbodied - "sci-o": - path: "@grnsft/if-plugins" - method: SciO + "operational-carbon": + method: Multiply + path: builtin + global-config: + input-parameters: ["cpu-energy-kwh", "grid/carbon-intensity"] + output-parameter: "carbon-operational" "sci": path: "builtin" method: Sci global-config: - functional-unit: "requests" + functional-unit-time: 1 sec + functional-unit: requests # factor to convert per time to per f.unit "sum-carbon": path: "builtin" method: Sum @@ -42,20 +80,21 @@ initialize: end-time: "2023-12-12T00:01:00.000Z" interval: 5 allow-padding: true - "group-by": - path: builtin - method: GroupBy tree: children: child-1: pipeline: - - teads-curve - - sci-e + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio - sci-embodied - - sci-o + - operational-carbon - sum-carbon - time-sync - - sci + # - sci config: group-by: group: @@ -67,8 +106,8 @@ tree: device/emissions-embodied: 1533.120 # gCO2eq time-reserved: 3600 # 1hr in seconds device/expected-lifespan: 94608000 # 3 years in seconds - resources-reserved: 1 - resources-total: 8 + vcpus-total: 8 + vcpus-allocated: 1 inputs: - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 @@ -94,50 +133,55 @@ tree: cloud/region: uk-west cpu/utilization: 15 requests: 30 - child-2: - pipeline: - - teads-curve - - sci-e - - sci-embodied - - sci-o - - sum-carbon - - time-sync - - sci - config: - group-by: - group: - - cloud/region - - cloud/instance-type - defaults: - cpu/thermal-design-power: 100 - grid/carbon-intensity: 800 - device/emissions-embodied: 1533.120 # gCO2eq - time-reserved: 3600 # 1hr in seconds - device/expected-lifespan: 94608000 # 3 years in seconds - resources-reserved: 1 - resources-total: 8 - inputs: - - timestamp: "2023-12-12T00:00:00.000Z" - duration: 1 - cpu/utilization: 30 - cloud/instance-type: A1 - cloud/region: uk-west - requests: 100 - - timestamp: "2023-12-12T00:00:01.000Z" - duration: 5 - cpu/utilization: 28 - cloud/instance-type: A1 - cloud/region: uk-west - requests: 150 - - timestamp: "2023-12-12T00:00:06.000Z" - duration: 7 - cpu/utilization: 40 - cloud/instance-type: A1 - cloud/region: uk-west - requests: 110 - - timestamp: "2023-12-12T00:00:13.000Z" - duration: 30 - cpu/utilization: 33 - cloud/instance-type: A1 - cloud/region: uk-west - requests: 180 + # child-2: + # pipeline: + # - interpolate + # - cpu-factor-to-wattage + # - wattage-times-duration + # - wattage-to-energy-kwh + # - calculate-vcpu-ratio + # - correct-cpu-energy-for-vcpu-ratio + # - sci-e + # - sci-embodied + # - sci-o + # - sum-carbon + # - time-sync + # - sci + # config: + # group-by: + # group: + # - cloud/region + # - cloud/instance-type + # defaults: + # cpu/thermal-design-power: 100 + # grid/carbon-intensity: 800 + # device/emissions-embodied: 1533.120 # gCO2eq + # time-reserved: 3600 # 1hr in seconds + # device/expected-lifespan: 94608000 # 3 years in seconds + # resources-reserved: 1 + # resources-total: 8 + # inputs: + # - timestamp: "2023-12-12T00:00:00.000Z" + # duration: 1 + # cpu/utilization: 30 + # cloud/instance-type: A1 + # cloud/region: uk-west + # requests: 100 + # - timestamp: "2023-12-12T00:00:01.000Z" + # duration: 5 + # cpu/utilization: 28 + # cloud/instance-type: A1 + # cloud/region: uk-west + # requests: 150 + # - timestamp: "2023-12-12T00:00:06.000Z" + # duration: 7 + # cpu/utilization: 40 + # cloud/instance-type: A1 + # cloud/region: uk-west + # requests: 110 + # - timestamp: "2023-12-12T00:00:13.000Z" + # duration: 30 + # cpu/utilization: 33 + # cloud/instance-type: A1 + # cloud/region: uk-west + # requests: 180 diff --git a/manifests/examples/pipeline-with-mocks.yml b/manifests/examples/pipeline-with-mocks.yml index 008e319b1..fde8963ef 100644 --- a/manifests/examples/pipeline-with-mocks.yml +++ b/manifests/examples/pipeline-with-mocks.yml @@ -25,20 +25,57 @@ initialize: cpu/utilization: min: 1 max: 99 - "teads-curve": - path: "@grnsft/if-unofficial-plugins" - method: TeadsCurve + "interpolate": + method: Interpolation + path: 'builtin' global-config: - interpolation: spline - "sci-e": - path: "@grnsft/if-plugins" - method: SciE + method: linear + x: [0, 10, 50, 100] + y: [0.12, 0.32, 0.75, 1.02] + input-parameter: 'cpu/utilization' + output-parameter: 'cpu-factor' + "cpu-factor-to-wattage": + method: Multiply + path: builtin + global-config: + input-parameters: ["cpu-factor", "cpu/thermal-design-power"] + output-parameter: "cpu-wattage" + "wattage-times-duration": + method: Multiply + path: builtin + global-config: + input-parameters: ["cpu-wattage", "duration"] + output-parameter: "cpu-wattage-times-duration" + "wattage-to-energy-kwh": + method: Divide + path: "builtin" + global-config: + numerator: cpu-wattage-times-duration + denominator: 3600000 + output: cpu-energy-raw + "calculate-vcpu-ratio": + method: Divide + path: "builtin" + global-config: + numerator: vcpus-total + denominator: vcpus-allocated + output: vcpu-ratio + "correct-cpu-energy-for-vcpu-ratio": + method: Divide + path: "builtin" + global-config: + numerator: cpu-energy-raw + denominator: vcpu-ratio + output: cpu-energy-kwh "sci-embodied": path: "builtin" method: SciEmbodied - "sci-o": - path: "@grnsft/if-plugins" - method: SciO + "operational-carbon": + method: Multiply + path: builtin + global-config: + input-parameters: ["cpu-energy-kwh", "grid/carbon-intensity"] + output-parameter: "carbon-operational" "sum-carbon": path: "builtin" method: Sum @@ -68,13 +105,17 @@ tree: children: child-1: pipeline: - - teads-curve - - sci-e + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio - sci-embodied - - sci-o + - operational-carbon - sum-carbon - time-sync - - sci + # - sci config: group-by: group: @@ -86,8 +127,8 @@ tree: device/emissions-embodied: 1533.120 # gCO2eq time-reserved: 3600 # 1hr in seconds device/expected-lifespan: 94608000 # 3 years in seconds - resources-reserved: 1 - resources-total: 8 + vcpus-total: 8 + vcpus-allocated: 1 functional-unit-time: "1 min" inputs: - timestamp: "2023-12-12T00:00:00.000Z" @@ -114,51 +155,51 @@ tree: cloud/region: uk-west cpu/utilization: 15 requests: 50 - child-2: - pipeline: - - teads-curve - - sci-e - - sci-embodied - - sci-o - - sum-carbon - - time-sync - - sci - config: - group-by: - group: - - cloud/region - - cloud/instance-type - defaults: - cpu/thermal-design-power: 100 - grid/carbon-intensity: 800 - device/emissions-embodied: 1533.120 # gCO2eq - time-reserved: 3600 # 1hr in seconds - device/expected-lifespan: 94608000 # 3 years in seconds - resources-reserved: 1 - resources-total: 8 - functional-unit-time: "1 min" - inputs: - - timestamp: "2023-12-12T00:00:00.000Z" - duration: 1 - cpu/utilization: 30 - cloud/instance-type: A1 - cloud/region: uk-west - requests: 30 - - timestamp: "2023-12-12T00:00:01.000Z" - duration: 5 - cpu/utilization: 28 - cloud/instance-type: A1 - cloud/region: uk-west - requests: 40 - - timestamp: "2023-12-12T00:00:06.000Z" - duration: 7 - cpu/utilization: 40 - cloud/instance-type: A1 - cloud/region: uk-west - requests: 50 - - timestamp: "2023-12-12T00:00:13.000Z" - duration: 30 - cpu/utilization: 33 - cloud/instance-type: A1 - cloud/region: uk-west - requests: 60 + # child-2: + # pipeline: + # - teads-curve + # - sci-e + # - sci-embodied + # - sci-o + # - sum-carbon + # - time-sync + # - sci + # config: + # group-by: + # group: + # - cloud/region + # - cloud/instance-type + # defaults: + # cpu/thermal-design-power: 100 + # grid/carbon-intensity: 800 + # device/emissions-embodied: 1533.120 # gCO2eq + # time-reserved: 3600 # 1hr in seconds + # device/expected-lifespan: 94608000 # 3 years in seconds + # resources-reserved: 1 + # resources-total: 8 + # functional-unit-time: "1 min" + # inputs: + # - timestamp: "2023-12-12T00:00:00.000Z" + # duration: 1 + # cpu/utilization: 30 + # cloud/instance-type: A1 + # cloud/region: uk-west + # requests: 30 + # - timestamp: "2023-12-12T00:00:01.000Z" + # duration: 5 + # cpu/utilization: 28 + # cloud/instance-type: A1 + # cloud/region: uk-west + # requests: 40 + # - timestamp: "2023-12-12T00:00:06.000Z" + # duration: 7 + # cpu/utilization: 40 + # cloud/instance-type: A1 + # cloud/region: uk-west + # requests: 50 + # - timestamp: "2023-12-12T00:00:13.000Z" + # duration: 30 + # cpu/utilization: 33 + # cloud/instance-type: A1 + # cloud/region: uk-west + # requests: 60 From 918fde8c1b1c4e0aa20afcd1112f4319b5ef29a4 Mon Sep 17 00:00:00 2001 From: MariamKhalatova Date: Thu, 20 Jun 2024 12:43:54 +0400 Subject: [PATCH 123/243] fix(manifests): update features to use builtins --- ...ics.yml => aggregate-failure-invalid-metrics.yml} | 12 +++++++++--- .../aggregate-failure-missing-metric-in-inputs.yml | 12 +++++++++--- manifests/features/aggregate-horizontal.yml | 10 ++++++++-- manifests/features/aggregate-vertical.yml | 10 ++++++++-- manifests/features/aggregate.yml | 10 ++++++++-- 5 files changed, 42 insertions(+), 12 deletions(-) rename manifests/features/{aggregate-failure-inalid-metrics.yml => aggregate-failure-invalid-metrics.yml} (75%) diff --git a/manifests/features/aggregate-failure-inalid-metrics.yml b/manifests/features/aggregate-failure-invalid-metrics.yml similarity index 75% rename from manifests/features/aggregate-failure-inalid-metrics.yml rename to manifests/features/aggregate-failure-invalid-metrics.yml index cf2e31bce..7e4692509 100644 --- a/manifests/features/aggregate-failure-inalid-metrics.yml +++ b/manifests/features/aggregate-failure-invalid-metrics.yml @@ -1,5 +1,5 @@ name: Aggregation -description: Apply both `horizontal` and `vertical` aggregations +description: Fails with invalid metric. aggregation: metrics: - 'test' @@ -7,8 +7,14 @@ aggregation: initialize: plugins: cloud-metadata: - method: CloudMetadata - path: "@grnsft/if-plugins" + path: builtin + method: CSVLookup + global-config: + filepath: >- + https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv + query: + instance-class: cloud/instance-type + output: ['cpu-cores-utilized', 'vcpus-allocated'] tree: children: application: diff --git a/manifests/features/aggregate-failure-missing-metric-in-inputs.yml b/manifests/features/aggregate-failure-missing-metric-in-inputs.yml index 3857d0f76..eeff85290 100644 --- a/manifests/features/aggregate-failure-missing-metric-in-inputs.yml +++ b/manifests/features/aggregate-failure-missing-metric-in-inputs.yml @@ -1,5 +1,5 @@ name: Aggregation -description: Apply both `horizontal` and `vertical` aggregations +description: Fails with missing metric in inputs. aggregation: metrics: - 'cpu/utilization' @@ -7,8 +7,14 @@ aggregation: initialize: plugins: cloud-metadata: - method: CloudMetadata - path: "@grnsft/if-plugins" + path: builtin + method: CSVLookup + global-config: + filepath: >- + https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv + query: + instance-class: cloud/instance-type + output: ['cpu-cores-utilized', 'vcpus-allocated'] tree: children: application: diff --git a/manifests/features/aggregate-horizontal.yml b/manifests/features/aggregate-horizontal.yml index 114074795..8003c4214 100644 --- a/manifests/features/aggregate-horizontal.yml +++ b/manifests/features/aggregate-horizontal.yml @@ -7,8 +7,14 @@ aggregation: initialize: plugins: cloud-metadata: - method: CloudMetadata - path: "@grnsft/if-plugins" + path: builtin + method: CSVLookup + global-config: + filepath: >- + https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv + query: + instance-class: cloud/instance-type + output: ['cpu-cores-utilized', 'vcpus-allocated'] tree: children: application: diff --git a/manifests/features/aggregate-vertical.yml b/manifests/features/aggregate-vertical.yml index b37f8e22a..6308a01d0 100644 --- a/manifests/features/aggregate-vertical.yml +++ b/manifests/features/aggregate-vertical.yml @@ -7,8 +7,14 @@ aggregation: initialize: plugins: cloud-metadata: - method: CloudMetadata - path: "@grnsft/if-plugins" + path: builtin + method: CSVLookup + global-config: + filepath: >- + https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv + query: + instance-class: cloud/instance-type + output: ['cpu-cores-utilized', 'vcpus-allocated'] tree: children: application: diff --git a/manifests/features/aggregate.yml b/manifests/features/aggregate.yml index f320a13ea..04672cf38 100644 --- a/manifests/features/aggregate.yml +++ b/manifests/features/aggregate.yml @@ -7,8 +7,14 @@ aggregation: initialize: plugins: cloud-metadata: - method: CloudMetadata - path: "@grnsft/if-plugins" + path: builtin + method: CSVLookup + global-config: + filepath: >- + https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv + query: + instance-class: cloud/instance-type + output: ['cpu-cores-utilized', 'vcpus-allocated'] tree: children: application: From b80ab76e34f5901615c9424702d3717d01c7fdb7 Mon Sep 17 00:00:00 2001 From: MariamKhalatova Date: Thu, 20 Jun 2024 12:45:11 +0400 Subject: [PATCH 124/243] fix(manifests): update integrations to use builtins --- .../cloud-metadata-divide-boavizta.yml | 10 ++++-- .../failure-invalid-instance-type.yaml | 33 +++++++++++++++++++ .../failure-invalid-provider.yaml | 33 +++++++++++++++++++ .../failure-missing-queried-input.yml | 33 +++++++++++++++++++ .../instance-metadata/success.yml | 33 +++++++++++++++++++ .../mock-obs-group-by-cloud-meta.yml | 10 ++++-- 6 files changed, 148 insertions(+), 4 deletions(-) create mode 100644 manifests/integrations/instance-metadata/failure-invalid-instance-type.yaml create mode 100644 manifests/integrations/instance-metadata/failure-invalid-provider.yaml create mode 100644 manifests/integrations/instance-metadata/failure-missing-queried-input.yml create mode 100644 manifests/integrations/instance-metadata/success.yml diff --git a/manifests/integrations/cloud-metadata-divide-boavizta.yml b/manifests/integrations/cloud-metadata-divide-boavizta.yml index cafd1467b..efb48f0ab 100644 --- a/manifests/integrations/cloud-metadata-divide-boavizta.yml +++ b/manifests/integrations/cloud-metadata-divide-boavizta.yml @@ -4,8 +4,14 @@ tags: initialize: plugins: cloud-metadata: - method: CloudMetadata - path: "@grnsft/if-plugins" + path: builtin + method: CSVLookup + global-config: + filepath: >- + https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv + query: + instance-class: cloud/instance-type + output: ['cpu-cores-utilized', 'vcpus-allocated'] divide: method: Divide path: "builtin" diff --git a/manifests/integrations/instance-metadata/failure-invalid-instance-type.yaml b/manifests/integrations/instance-metadata/failure-invalid-instance-type.yaml new file mode 100644 index 000000000..810dee072 --- /dev/null +++ b/manifests/integrations/instance-metadata/failure-invalid-instance-type.yaml @@ -0,0 +1,33 @@ +name: instance-metadata +description: +tags: +initialize: + plugins: + cloud-instance-metadata: + method: CSVLookup + path: "builtin" + global-config: + filepath: https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-azure-instances.csv + query: + instance-class: "cloud/instance-type" + output: "*" + extract-processor-name: + method: Regex + path: "builtin" + global-config: + parameter: cpu-model-name + match: /^([^,])+/g + output: cpu/name +tree: + children: + child: + pipeline: + - cloud-instance-metadata + - extract-processor-name + inputs: + - timestamp: 2023-08-06T00:00 + duration: 3600 + cpu/energy: 0.001 + cloud/provider: gcp + cloud/region: asia-east + cloud/instance-type: m6 \ No newline at end of file diff --git a/manifests/integrations/instance-metadata/failure-invalid-provider.yaml b/manifests/integrations/instance-metadata/failure-invalid-provider.yaml new file mode 100644 index 000000000..a362132e7 --- /dev/null +++ b/manifests/integrations/instance-metadata/failure-invalid-provider.yaml @@ -0,0 +1,33 @@ +name: instance-metadata +description: +tags: +initialize: + plugins: + cloud-instance-metadata: + method: CSVLookup + path: "builtin" + global-config: + filepath: https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-azure-instances.csv + query: + instance-class: "cloud/provider" + output: "*" + extract-processor-name: + method: Regex + path: "builtin" + global-config: + parameter: cpu-model-name + match: /^([^,])+/g + output: cpu/name +tree: + children: + child: + pipeline: + - cloud-instance-metadata + - extract-processor-name + inputs: + - timestamp: 2023-08-06T00:00 + duration: 3600 + cpu/energy: 0.001 + cloud/provider: 6568 + cloud/region: asia-east + cloud/instance-type: Standard_A1_v2 diff --git a/manifests/integrations/instance-metadata/failure-missing-queried-input.yml b/manifests/integrations/instance-metadata/failure-missing-queried-input.yml new file mode 100644 index 000000000..37f4001ae --- /dev/null +++ b/manifests/integrations/instance-metadata/failure-missing-queried-input.yml @@ -0,0 +1,33 @@ +name: instance-metadata +description: +tags: +initialize: + plugins: + cloud-instance-metadata: + method: CSVLookup + path: "builtin" + global-config: + filepath: https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-azure-instances.csv + query: + instance-class: "cloud/provider" + output: "*" + extract-processor-name: + method: Regex + path: "builtin" + global-config: + parameter: cpu-model-name + match: /^([^,])+/g + output: cpu/name +tree: + children: + child: + pipeline: + - cloud-instance-metadata + - extract-processor-name + inputs: + - timestamp: 2023-08-06T00:00 + duration: 3600 + cpu/energy: 0.001 + # cloud/provider: gcp + cloud/region: asia-east + cloud/instance-type: Standard_A1_v2 \ No newline at end of file diff --git a/manifests/integrations/instance-metadata/success.yml b/manifests/integrations/instance-metadata/success.yml new file mode 100644 index 000000000..d95115896 --- /dev/null +++ b/manifests/integrations/instance-metadata/success.yml @@ -0,0 +1,33 @@ +name: instance-metadata +description: +tags: +initialize: + plugins: + cloud-instance-metadata: + method: CSVLookup + path: "builtin" + global-config: + filepath: https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-azure-instances.csv + query: + instance-class: "cloud/instance-type" + output: "*" + extract-processor-name: + method: Regex + path: "builtin" + global-config: + parameter: cpu-model-name + match: /^([^,])+/g + output: cpu/name +tree: + children: + child: + pipeline: + - cloud-instance-metadata + - extract-processor-name + inputs: + - timestamp: 2023-08-06T00:00 + duration: 3600 + cpu/energy: 0.001 + cloud/provider: gcp + cloud/region: asia-east + cloud/instance-type: Standard_A1_v2 \ No newline at end of file diff --git a/manifests/integrations/mock-obs-group-by-cloud-meta.yml b/manifests/integrations/mock-obs-group-by-cloud-meta.yml index b0d43f2a5..6b6196cab 100644 --- a/manifests/integrations/mock-obs-group-by-cloud-meta.yml +++ b/manifests/integrations/mock-obs-group-by-cloud-meta.yml @@ -14,8 +14,14 @@ initialize: input-parameters: ['cpu/energy', 'grid/carbon-intensity'] output-parameter: 'carbon' cloud-metadata: - method: CloudMetadata - path: "@grnsft/if-plugins" + path: builtin + method: CSVLookup + global-config: + filepath: >- + https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-azure-instances.csv + query: + instance-class: cloud/instance-type + output: ['cpu-tdp'] mock-observations: path: 'builtin' method: MockObservations From ae53a1cc1fc13449b1707e2c6243377e8395d080 Mon Sep 17 00:00:00 2001 From: MariamKhalatova Date: Thu, 20 Jun 2024 12:46:12 +0400 Subject: [PATCH 125/243] fix(manifests): update plugins to use builtins --- .../failure-invalid-instance-type.yaml | 12 ++++++--- .../failure-invalid-vendor.yaml | 10 +++++-- .../failure-missing-cloud-vendor.yml | 12 ++++++--- .../cloud-metadata/success.yml | 10 +++++-- .../failure-missing-column.yml | 0 .../failure-missing-output.yml | 0 .../success-renaming.yml | 0 .../{ => region-metadata}/success.yml | 0 .../failure-missing-input-param.yml | 23 ++++++++++++++++ ...failure-unsupported-physical-processor.yml | 24 +++++++++++++++++ .../plugins/csv-lookup/tdp-finder/success.yml | 24 +++++++++++++++++ .../divide/failure-denominator-equal-zero.yml | 10 +++++-- .../failure-invalid-config-denominator.yml | 10 +++++-- .../divide/failure-missing-numerator.yml | 10 +++++-- manifests/plugins/divide/success.yml | 10 +++++-- manifests/plugins/exponent/success.yml | 26 +++++++++++++++++++ manifests/plugins/interpolation/success.yml | 24 +++++++++++++++++ manifests/plugins/subtract/success.yml | 25 ++++++++++++++++++ .../failure-missing-input-param.yml | 18 ------------- ...failure-unsupported-physical-processor.yml | 19 -------------- manifests/plugins/tdp-finder/success.yml | 19 -------------- 21 files changed, 212 insertions(+), 74 deletions(-) rename manifests/plugins/{ => csv-lookup}/cloud-metadata/failure-invalid-instance-type.yaml (57%) rename manifests/plugins/{ => csv-lookup}/cloud-metadata/failure-invalid-vendor.yaml (59%) rename manifests/plugins/{ => csv-lookup}/cloud-metadata/failure-missing-cloud-vendor.yml (56%) rename manifests/plugins/{ => csv-lookup}/cloud-metadata/success.yml (57%) rename manifests/plugins/csv-lookup/{ => region-metadata}/failure-missing-column.yml (100%) rename manifests/plugins/csv-lookup/{ => region-metadata}/failure-missing-output.yml (100%) rename manifests/plugins/csv-lookup/{ => region-metadata}/success-renaming.yml (100%) rename manifests/plugins/csv-lookup/{ => region-metadata}/success.yml (100%) create mode 100644 manifests/plugins/csv-lookup/tdp-finder/failure-missing-input-param.yml create mode 100644 manifests/plugins/csv-lookup/tdp-finder/failure-unsupported-physical-processor.yml create mode 100644 manifests/plugins/csv-lookup/tdp-finder/success.yml create mode 100644 manifests/plugins/exponent/success.yml create mode 100644 manifests/plugins/interpolation/success.yml create mode 100644 manifests/plugins/subtract/success.yml delete mode 100644 manifests/plugins/tdp-finder/failure-missing-input-param.yml delete mode 100644 manifests/plugins/tdp-finder/failure-unsupported-physical-processor.yml delete mode 100644 manifests/plugins/tdp-finder/success.yml diff --git a/manifests/plugins/cloud-metadata/failure-invalid-instance-type.yaml b/manifests/plugins/csv-lookup/cloud-metadata/failure-invalid-instance-type.yaml similarity index 57% rename from manifests/plugins/cloud-metadata/failure-invalid-instance-type.yaml rename to manifests/plugins/csv-lookup/cloud-metadata/failure-invalid-instance-type.yaml index 9bb4581ef..fa55623c2 100644 --- a/manifests/plugins/cloud-metadata/failure-invalid-instance-type.yaml +++ b/manifests/plugins/csv-lookup/cloud-metadata/failure-invalid-instance-type.yaml @@ -2,11 +2,17 @@ name: cloud-metadata description: cloud/instance-type instance type is not supported in the cloud vendor tags: initialize: - outputs: ['yaml'] + # outputs: ['yaml'] plugins: cloud-metadata: - method: CloudMetadata - path: "@grnsft/if-plugins" + path: builtin + method: CSVLookup + global-config: + filepath: >- + https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv + query: + instance-class: cloud/instance-type + output: ['cpu-cores-utilized', 'vcpus-allocated'] tree: children: child: diff --git a/manifests/plugins/cloud-metadata/failure-invalid-vendor.yaml b/manifests/plugins/csv-lookup/cloud-metadata/failure-invalid-vendor.yaml similarity index 59% rename from manifests/plugins/cloud-metadata/failure-invalid-vendor.yaml rename to manifests/plugins/csv-lookup/cloud-metadata/failure-invalid-vendor.yaml index 9e2ebe4f2..c474c91fe 100644 --- a/manifests/plugins/cloud-metadata/failure-invalid-vendor.yaml +++ b/manifests/plugins/csv-lookup/cloud-metadata/failure-invalid-vendor.yaml @@ -5,8 +5,14 @@ initialize: #outputs: ['yaml'] plugins: cloud-metadata: - method: CloudMetadata - path: "@grnsft/if-plugins" + path: builtin + method: CSVLookup + global-config: + filepath: >- + https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv + query: + instance-class: cloud/instance-type + output: ['cpu-cores-utilized', 'vcpus-allocated'] tree: children: child: diff --git a/manifests/plugins/cloud-metadata/failure-missing-cloud-vendor.yml b/manifests/plugins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yml similarity index 56% rename from manifests/plugins/cloud-metadata/failure-missing-cloud-vendor.yml rename to manifests/plugins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yml index 9b80c4301..33c5cc4b1 100644 --- a/manifests/plugins/cloud-metadata/failure-missing-cloud-vendor.yml +++ b/manifests/plugins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yml @@ -2,11 +2,17 @@ name: cloud-metadata description: failing because cloud/vendor is not provided tags: initialize: - outputs: ['yaml'] + # outputs: ['yaml'] plugins: cloud-metadata: - method: CloudMetadata - path: "@grnsft/if-plugins" + path: builtin + method: CSVLookup + global-config: + filepath: >- + https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv + query: + instance-class: cloud/instance-type + output: ['cpu-cores-utilized', 'vcpus-allocated'] tree: children: child: diff --git a/manifests/plugins/cloud-metadata/success.yml b/manifests/plugins/csv-lookup/cloud-metadata/success.yml similarity index 57% rename from manifests/plugins/cloud-metadata/success.yml rename to manifests/plugins/csv-lookup/cloud-metadata/success.yml index dec29460d..bd3cfdbb7 100644 --- a/manifests/plugins/cloud-metadata/success.yml +++ b/manifests/plugins/csv-lookup/cloud-metadata/success.yml @@ -5,8 +5,14 @@ initialize: # outputs: ['yaml'] plugins: cloud-metadata: - method: CloudMetadata - path: "@grnsft/if-plugins" + path: builtin + method: CSVLookup + global-config: + filepath: >- + https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv + query: + instance-class: cloud/instance-type + output: ['cpu-cores-utilized', 'vcpus-allocated'] tree: children: child: diff --git a/manifests/plugins/csv-lookup/failure-missing-column.yml b/manifests/plugins/csv-lookup/region-metadata/failure-missing-column.yml similarity index 100% rename from manifests/plugins/csv-lookup/failure-missing-column.yml rename to manifests/plugins/csv-lookup/region-metadata/failure-missing-column.yml diff --git a/manifests/plugins/csv-lookup/failure-missing-output.yml b/manifests/plugins/csv-lookup/region-metadata/failure-missing-output.yml similarity index 100% rename from manifests/plugins/csv-lookup/failure-missing-output.yml rename to manifests/plugins/csv-lookup/region-metadata/failure-missing-output.yml diff --git a/manifests/plugins/csv-lookup/success-renaming.yml b/manifests/plugins/csv-lookup/region-metadata/success-renaming.yml similarity index 100% rename from manifests/plugins/csv-lookup/success-renaming.yml rename to manifests/plugins/csv-lookup/region-metadata/success-renaming.yml diff --git a/manifests/plugins/csv-lookup/success.yml b/manifests/plugins/csv-lookup/region-metadata/success.yml similarity index 100% rename from manifests/plugins/csv-lookup/success.yml rename to manifests/plugins/csv-lookup/region-metadata/success.yml diff --git a/manifests/plugins/csv-lookup/tdp-finder/failure-missing-input-param.yml b/manifests/plugins/csv-lookup/tdp-finder/failure-missing-input-param.yml new file mode 100644 index 000000000..b14e57d1d --- /dev/null +++ b/manifests/plugins/csv-lookup/tdp-finder/failure-missing-input-param.yml @@ -0,0 +1,23 @@ +name: tdp-finder +description: failure with `inputs` missing `physical-processor` param +tags: +initialize: + # outputs: ['yaml'] + plugins: + tdp-finder: + method: CSVLookup + path: "builtin" + global-config: + filepath: https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/tdp-data-1.csv + query: + name: physical-processor + output: "tdp" +tree: + children: + child: + pipeline: + - tdp-finder + config: + inputs: + - timestamp: 2023-07-06T00:00 + duration: 300 diff --git a/manifests/plugins/csv-lookup/tdp-finder/failure-unsupported-physical-processor.yml b/manifests/plugins/csv-lookup/tdp-finder/failure-unsupported-physical-processor.yml new file mode 100644 index 000000000..47f392fcc --- /dev/null +++ b/manifests/plugins/csv-lookup/tdp-finder/failure-unsupported-physical-processor.yml @@ -0,0 +1,24 @@ +name: tdp-finder +description: successful path +tags: +initialize: + # outputs: ['yaml'] + plugins: + tdp-finder: + method: CSVLookup + path: "builtin" + global-config: + filepath: https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/tdp-data-1.csv + query: + name: physical-processor + output: "tdp" +tree: + children: + child: + pipeline: + - tdp-finder + config: + inputs: + - timestamp: 2023-07-06T00:00 + duration: 300 + physical-processor: AMD 302 diff --git a/manifests/plugins/csv-lookup/tdp-finder/success.yml b/manifests/plugins/csv-lookup/tdp-finder/success.yml new file mode 100644 index 000000000..42545b0df --- /dev/null +++ b/manifests/plugins/csv-lookup/tdp-finder/success.yml @@ -0,0 +1,24 @@ +name: tdp-finder +description: successful path +tags: +initialize: + # outputs: ['yaml'] + plugins: + tdp-finder: + method: CSVLookup + path: "builtin" + global-config: + filepath: https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/tdp-data-1.csv + query: + name: physical-processor + output: "tdp" +tree: + children: + child: + pipeline: + - tdp-finder + config: + inputs: + - timestamp: 2023-07-06T00:00 + duration: 300 + physical-processor: AMD 3020e diff --git a/manifests/plugins/divide/failure-denominator-equal-zero.yml b/manifests/plugins/divide/failure-denominator-equal-zero.yml index af5ed2ae0..8c48e42b9 100644 --- a/manifests/plugins/divide/failure-denominator-equal-zero.yml +++ b/manifests/plugins/divide/failure-denominator-equal-zero.yml @@ -5,8 +5,14 @@ initialize: # outputs: ['yaml'] plugins: cloud-metadata: - method: CloudMetadata - path: "@grnsft/if-plugins" + path: builtin + method: CSVLookup + global-config: + filepath: >- + https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv + query: + instance-class: cloud/instance-type + output: ['cpu-cores-utilized', 'vcpus-allocated'] divide: method: Divide path: "builtin" diff --git a/manifests/plugins/divide/failure-invalid-config-denominator.yml b/manifests/plugins/divide/failure-invalid-config-denominator.yml index 063274bc0..bc93f4f5f 100644 --- a/manifests/plugins/divide/failure-invalid-config-denominator.yml +++ b/manifests/plugins/divide/failure-invalid-config-denominator.yml @@ -5,8 +5,14 @@ initialize: outputs: ['yaml'] plugins: cloud-metadata: - method: CloudMetadata - path: "@grnsft/if-plugins" + path: builtin + method: CSVLookup + global-config: + filepath: >- + https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv + query: + instance-class: cloud/instance-type + output: ['cpu-cores-utilized', 'vcpus-allocated'] divide: method: Divide path: "builtin" diff --git a/manifests/plugins/divide/failure-missing-numerator.yml b/manifests/plugins/divide/failure-missing-numerator.yml index 7f03d5eba..ebf250961 100644 --- a/manifests/plugins/divide/failure-missing-numerator.yml +++ b/manifests/plugins/divide/failure-missing-numerator.yml @@ -5,8 +5,14 @@ initialize: # outputs: ['yaml'] plugins: cloud-metadata: - method: CloudMetadata - path: "@grnsft/if-plugins" + path: builtin + method: CSVLookup + global-config: + filepath: >- + https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv + query: + instance-class: cloud/instance-type + output: ['cpu-cores-utilized', 'vcpus-allocated'] divide: method: Divide path: "builtin" diff --git a/manifests/plugins/divide/success.yml b/manifests/plugins/divide/success.yml index ed75a7c5c..24626c8f4 100644 --- a/manifests/plugins/divide/success.yml +++ b/manifests/plugins/divide/success.yml @@ -5,8 +5,14 @@ initialize: # outputs: ['yaml'] plugins: cloud-metadata: - method: CloudMetadata - path: "@grnsft/if-plugins" + path: builtin + method: CSVLookup + global-config: + filepath: >- + https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv + query: + instance-class: cloud/instance-type + output: ['cpu-cores-utilized', 'vcpus-allocated'] divide: method: Divide path: "builtin" diff --git a/manifests/plugins/exponent/success.yml b/manifests/plugins/exponent/success.yml new file mode 100644 index 000000000..c123b3a24 --- /dev/null +++ b/manifests/plugins/exponent/success.yml @@ -0,0 +1,26 @@ +name: exponent demo +description: +tags: +initialize: + # outputs: + # - yaml + plugins: + exponent: + method: Exponent + path: 'builtin' + global-config: + input-parameter: 'cpu/energy' + exponent: 2 + output-parameter: 'energy' +tree: + children: + child: + pipeline: + - exponent + config: + exponent: + inputs: + - timestamp: 2023-08-06T00:00 + duration: 3600 + cpu/energy: 0.001 + network/energy: 0.001 diff --git a/manifests/plugins/interpolation/success.yml b/manifests/plugins/interpolation/success.yml new file mode 100644 index 000000000..35d750e48 --- /dev/null +++ b/manifests/plugins/interpolation/success.yml @@ -0,0 +1,24 @@ +name: interpolation-demo +description: simple demo of interpolation plugin +tags: +initialize: + plugins: + interpolation: + method: Interpolation + path: "builtin" + global-config: + method: linear + x: [0, 10, 50, 100] + y: [0.12, 0.32, 0.75, 1.02] + input-parameter: "cpu/utilization" + output-parameter: "result" + +tree: + children: + child: + pipeline: + - interpolation + inputs: + - timestamp: 2023-07-06T00:00 + duration: 3600 + cpu/utilization: 45 diff --git a/manifests/plugins/subtract/success.yml b/manifests/plugins/subtract/success.yml new file mode 100644 index 000000000..276128171 --- /dev/null +++ b/manifests/plugins/subtract/success.yml @@ -0,0 +1,25 @@ +name: subtract demo +description: +tags: +initialize: + outputs: + - yaml + plugins: + subtract: + method: Subtract + path: 'builtin' + global-config: + input-parameters: ['cpu/energy', 'network/energy'] + output-parameter: 'energy/diff' +tree: + children: + child: + pipeline: + - subtract + config: + subtract: + inputs: + - timestamp: 2023-08-06T00:00 + duration: 3600 + cpu/energy: 0.003 + network/energy: 0.001 \ No newline at end of file diff --git a/manifests/plugins/tdp-finder/failure-missing-input-param.yml b/manifests/plugins/tdp-finder/failure-missing-input-param.yml deleted file mode 100644 index 7877b56ce..000000000 --- a/manifests/plugins/tdp-finder/failure-missing-input-param.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: tdp-finder -description: failure with `inputs` missing `physical-processor` param -tags: -initialize: - outputs: ['yaml'] - plugins: - tdp-finder: # a model that returns an embodied value given the sci embodied attribution equation. - method: TdpFinder - path: "@grnsft/if-plugins" -tree: - children: - child: - pipeline: - - tdp-finder - config: - inputs: - - timestamp: 2023-07-06T00:00 - duration: 300 diff --git a/manifests/plugins/tdp-finder/failure-unsupported-physical-processor.yml b/manifests/plugins/tdp-finder/failure-unsupported-physical-processor.yml deleted file mode 100644 index 56c5c93a5..000000000 --- a/manifests/plugins/tdp-finder/failure-unsupported-physical-processor.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: tdp-finder -description: successful path -tags: -initialize: - outputs: ['yaml'] - plugins: - tdp-finder: # a model that returns an embodied value given the sci embodied attribution equation. - method: TdpFinder - path: "@grnsft/if-plugins" -tree: - children: - child: - pipeline: - - tdp-finder - config: - inputs: - - timestamp: 2023-07-06T00:00 - duration: 300 - physical-processor: AMD 302 diff --git a/manifests/plugins/tdp-finder/success.yml b/manifests/plugins/tdp-finder/success.yml deleted file mode 100644 index 26f905ecd..000000000 --- a/manifests/plugins/tdp-finder/success.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: tdp-finder -description: successful path -tags: -initialize: - outputs: ['yaml'] - plugins: - tdp-finder: # a model that returns an embodied value given the sci embodied attribution equation. - method: TdpFinder - path: "@grnsft/if-plugins" -tree: - children: - child: - pipeline: - - tdp-finder - config: - inputs: - - timestamp: 2023-07-06T00:00 - duration: 300 - physical-processor: AMD 3020e From ef26d4ba4e8e5a277ce913b45c6c27d148b1a5dc Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Thu, 20 Jun 2024 14:09:20 +0100 Subject: [PATCH 126/243] feat(lib): add copy-param README --- src/builtins/copy-param/README.md | 104 ++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 src/builtins/copy-param/README.md diff --git a/src/builtins/copy-param/README.md b/src/builtins/copy-param/README.md new file mode 100644 index 000000000..1ec04f249 --- /dev/null +++ b/src/builtins/copy-param/README.md @@ -0,0 +1,104 @@ +# Copy-param + +`copy-param` is a generic plugin that duplicates an existing parameter in the `input` data and assigns it to a new key. You can either keep or delete the original copied parameter. A common use case for this is to rename parameters in the `inputs` array. + +You provide the name of the value you want to copy, and a name to assign the copy to. You also toggle a `keep-existing` parameter to either persist or del;et the original copied value. + +For example, you could copy `energy` into `energy-copy`, with `keep-existing=true`. In this case your inputs: + +```yaml +- timestamp: "2023-12-12T00:00:13.000Z", + duration: 30, + energy: 30 +``` + +would become + +```yaml +- timestamp: "2023-12-12T00:00:13.000Z", + duration: 30, + energy: 30 + energy-copy: 30 +``` + +but with `keep-existing=false`, the same inputs would yield: + +```yaml +- timestamp: "2023-12-12T00:00:13.000Z", + duration: 30, + energy-copy: 30 +``` + +## Parameters + +### Config + +Three parameters are required in config: `from` and `to` and `keep-existing`. + +`from`: an array of strings. Each string should match an existing key in the `inputs` array +`to`: a string defining the name to use to add the result of summing the input parameters to the output array. +`keep-existing`: toggles whether to keep or delete the copied parameter (defined in `to`) + +### Inputs + +As with all plugins, `timestamp` and `duration` are required. The key passed to `from` must exist in the `input` data. + +## Returns + +The plugin adds a new parameter with the name defined in `to` to the `input` data. + + +## Implementation + +To run the plugin, you must first create an instance of `Copy`. Then, you can call `execute()`. + +```typescript +import { Copy } from "."; + +const plugin = Copy({ 'keep-existing': true, from: 'from-param', to: 'to-param' }); + +const result = plugin.execute([{ + timestamp: "2023-12-12T00:00:13.000Z", + duration: 30, + 'from-param': 'hello', +}]) + +console.log(result) + +``` + +## Example manifest + +IF users will typically call the plugin as part of a pipeline defined in a manifest file. In this case, instantiating the plugin is handled by and does not have to be done explicitly by the user. The following is an example manifest that calls `sum`: + +```yaml +name: copy-param +description: +tags: +initialize: + plugins: + copy-param: + path: builtin + method: Copy + global-config: + keep-existing: true + from: original + to: copy +tree: + children: + child-1: + pipeline: + - copy-param + inputs: + - timestamp: "2023-12-12T00:00:00.000Z" + original: 'hello' + +``` + +You can run this example by saving it as `./manifests/examples/copy.yml` and executing the following command from the project root: + +```sh +if-run --manifest ./manifests/examples/copy.yml -s +``` + +The results will be displayed in the console. From 4fefa36c75c076dab96a9a89c1a5f40cb6b9de6c Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Thu, 20 Jun 2024 14:09:54 +0100 Subject: [PATCH 127/243] feat(lib): export copy-param --- src/builtins/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/builtins/index.ts b/src/builtins/index.ts index 1a98e3273..d22491aca 100644 --- a/src/builtins/index.ts +++ b/src/builtins/index.ts @@ -13,3 +13,4 @@ export {Exponent} from './exponent'; export {CSVLookup} from './csv-lookup'; export {Shell} from './shell'; export {Regex} from './regex'; +export {Copy} from './copy-param'; From b95c7e1baf210f2cb66177be4c33bc3b2e6f70b3 Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Thu, 20 Jun 2024 14:10:28 +0100 Subject: [PATCH 128/243] feat(lib): add copy-param source code --- src/builtins/copy-param/index.ts | 78 ++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 src/builtins/copy-param/index.ts diff --git a/src/builtins/copy-param/index.ts b/src/builtins/copy-param/index.ts new file mode 100644 index 000000000..3f3e1f5ae --- /dev/null +++ b/src/builtins/copy-param/index.ts @@ -0,0 +1,78 @@ +import {z} from 'zod'; +import {ERRORS} from '@grnsft/if-core/utils'; +import {ExecutePlugin, PluginParams} from '@grnsft/if-core/types'; + +import {validate} from '../../util/validations'; + +import {STRINGS} from '../../config'; + +const {MISSING_GLOBAL_CONFIG, MISSING_INPUT_DATA} = STRINGS; +const {GlobalConfigError, MissingInputDataError} = ERRORS; +// global-config: +// keep-existing: true/false (whether to remove the parameter you are copying from) +// from-param: the parameter you are copying from (e.g. cpu/name) +// to-field: the parameter you are copying to (e.g. cpu/processor-name) + +export const Copy = (globalConfig: Record): ExecutePlugin => { + const metadata = { + kind: 'execute', + }; + + /** + * Checks global config value are valid. + */ + const validateGlobalConfig = () => { + if (!globalConfig) { + throw new GlobalConfigError(MISSING_GLOBAL_CONFIG); + } + + const globalConfigSchema = z.object({ + 'keep-existing': z.boolean(), + from: z.string().min(1), + to: z.string().min(1), + }); + + return validate>( + globalConfigSchema, + globalConfig + ); + }; + + /** + * Checks for required fields in input. + */ + const validateSingleInput = (input: PluginParams, parameter: string) => { + if (!input[parameter]) { + throw new MissingInputDataError(MISSING_INPUT_DATA(parameter)); + } + + return input; + }; + + const execute = (inputs: PluginParams[]) => { + const safeGlobalConfig = validateGlobalConfig(); + const keepExisting = safeGlobalConfig['keep-existing'] === true; + const from = safeGlobalConfig['from']; + const to = safeGlobalConfig['to']; + + return inputs.map(input => { + const safeInput = validateSingleInput(input, from); + + const outputValue = safeInput[from]; + if (safeInput[from]) { + if (!keepExisting) { + delete safeInput[from]; + } + } + return { + ...safeInput, // need to return or what you provide won't be outputted, don't be evil! + [to]: outputValue, + }; + }); + }; + + return { + metadata, + execute, + }; +}; From 57e64df04ce4be5819c19291dae1a5bd88e225df Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Thu, 20 Jun 2024 14:10:58 +0100 Subject: [PATCH 129/243] feat(lib): add unit tests for copy-param --- .../unit/builtins/copy-param.test.ts | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 src/__tests__/unit/builtins/copy-param.test.ts diff --git a/src/__tests__/unit/builtins/copy-param.test.ts b/src/__tests__/unit/builtins/copy-param.test.ts new file mode 100644 index 000000000..c3fcd9639 --- /dev/null +++ b/src/__tests__/unit/builtins/copy-param.test.ts @@ -0,0 +1,95 @@ +import {ERRORS} from '@grnsft/if-core/utils'; + +import {Copy} from '../../../builtins/copy-param'; + +import {STRINGS} from '../../../config'; + +const {GlobalConfigError, MissingInputDataError} = ERRORS; +const {MISSING_GLOBAL_CONFIG, MISSING_INPUT_DATA} = STRINGS; + +describe('builtins/copy: ', () => { + describe('Copy: ', () => { + const globalConfig = { + 'keep-existing': true, + from: 'original', + to: 'copy', + }; + const copy = Copy(globalConfig); + + describe('init: ', () => { + it('successfully initalized.', () => { + expect(copy).toHaveProperty('metadata'); + expect(copy).toHaveProperty('execute'); + }); + }); + + describe('execute(): ', () => { + it('successfully applies Copy strategy to given input.', () => { + expect.assertions(1); + + const expectedResult = [ + { + duration: 3600, + original: 'hello', + copy: 'hello', + timestamp: '2021-01-01T00:00:00Z', + }, + ]; + + const result = copy.execute([ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + original: 'hello', + }, + ]); + + expect(result).toStrictEqual(expectedResult); + }); + + it('throws an error when global config is not provided.', () => { + const config = undefined; + const copy = Copy(config!); + + expect.assertions(1); + + try { + copy.execute([ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + original: 1, + }, + ]); + } catch (error) { + expect(error).toStrictEqual( + new GlobalConfigError(MISSING_GLOBAL_CONFIG) + ); + } + }); + + it('throws an error on missing params in input.', () => { + const globalConfig = { + 'keep-existing': true, + from: 'original', + to: 'copy', + }; + const copy = Copy(globalConfig); + expect.assertions(1); + + try { + copy.execute([ + { + duration: 3600, + timestamp: '2021-01-01T00:00:00Z', + }, + ]); + } catch (error) { + expect(error).toStrictEqual( + new MissingInputDataError(MISSING_INPUT_DATA('original')) + ); + } + }); + }); + }); +}); From 22a34001b723d1f05e9b2a2946f5c1597f92391d Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Thu, 20 Jun 2024 14:12:15 +0100 Subject: [PATCH 130/243] feat(manifests): add example manifest for copy-param --- manifests/examples/copy.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 manifests/examples/copy.yaml diff --git a/manifests/examples/copy.yaml b/manifests/examples/copy.yaml new file mode 100644 index 000000000..f2ee816e4 --- /dev/null +++ b/manifests/examples/copy.yaml @@ -0,0 +1,20 @@ +name: copy-param +description: +tags: +initialize: + plugins: + copy-param: + path: builtin + method: Copy + global-config: + keep-existing: true + from: original + to: copy +tree: + children: + child-1: + pipeline: + - copy-param + inputs: + - timestamp: "2023-12-12T00:00:00.000Z" + original: 'hello' From a2bb35a3844eb9709c63fd911eb8340eb78b7758 Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Thu, 20 Jun 2024 15:27:25 +0100 Subject: [PATCH 131/243] feat(lib): add errors to coefficient readme --- src/builtins/coefficient/README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/builtins/coefficient/README.md b/src/builtins/coefficient/README.md index bf8808e8c..629059799 100644 --- a/src/builtins/coefficient/README.md +++ b/src/builtins/coefficient/README.md @@ -90,3 +90,19 @@ if-run --manifest ./examples/manifests/coefficient.yml --output ./examples/outpu ``` The results will be saved to a new `yaml` file in `./examples/outputs` + + +## Errors + +`Coefficient exposes one of the IF error classes. + +### GlobalConfigError + +You will receive an error starting `GlobalConfigError: ` if you have not provided the expected configuration data in the plugin's `initialize` block. + +The required parameters are: +- `input-parameter`: this must be a string +- `coefficient`: this must be a number +- `output-parameter`: this must be a string + +You can fix this error by checking you are providing valid values for each parameter in the config. From dcca6efc4e0602cd671d05088426cdc80ac99a65 Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Thu, 20 Jun 2024 15:28:04 +0100 Subject: [PATCH 132/243] feat(lib): add errors to csv readme --- src/builtins/csv-lookup/README.md | 36 +++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src/builtins/csv-lookup/README.md b/src/builtins/csv-lookup/README.md index d1aece699..19d8642ec 100644 --- a/src/builtins/csv-lookup/README.md +++ b/src/builtins/csv-lookup/README.md @@ -140,3 +140,39 @@ if-run --manifest manifests/plugins/csv-lookup.yml --output manifests/outputs/cs ``` The results will be saved to a new `yaml` file in `manifests/outputs`. + + +## Errors + +`Coefficient exposes six of the IF error classes. + +### FetchingFileError + +This error is caused by problems finding the file at the path provided in the `filepath`. If the file is on your local filesystem, you can check that the file is definitely there. For a remote file, check your internet connection. You can check your connection to the server using a tool such as `ping` or `curl`. if you still experience problems, you could retrieve the remote file onto your local filesystem using a tool such as `wget`. + +### ReadFileError, + +This error is caused by problems reading the CSV file provided in the `filepath`. To fix it, check that the file contains valid CSV data. The file should have a `.csv` file extension and the data inside should be formatted correctly. + +### MissingCSVColumnError, + +This error is caused by `CsvLookup` failing to find a column in the CSV file whose name matches what was provided in `query`. To debug, check that you do not have any typos in your `query` and confirm that the requested column name definitely exists in the target file. + +### QueryDataNotFoundError, + +This error is caused by the `CsvLookup` plugin failing to find data that matches your query. Try revising your query parameters. + +### CSVParseError, + +This error arises due to problems parsing CSV data into IF. This can occur when the CSV data is incorrectly formatted or contains unexpected characters that IF does not recognize. These errors are expected to be unusual edge cases as incorrectly formatted data will usually be identified during file loading and cause a `ReadFileError`. To debug, check your CSV file for any unexpected formatting or unusual characters. + +### GlobalConfigError + +You will receive an error starting `GlobalConfigError: ` if you have not provided the expected configuration data in the plugin's `initialize` block. + +The required parameters are: +- `filepath`: This must be a path to a csv file +- `query`: this must be an array of key-value pairs where the key is a string containing a column name an the value is a string containing the name of a value in `inputs` +- `output`: this must be a string containing a name or a wildcard character (`"*"`) + +You can fix this error by checking you are providing valid values for each parameter in the config. From 12312e55c46972d4544205c880bc03f5260e6ca7 Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Thu, 20 Jun 2024 15:32:19 +0100 Subject: [PATCH 133/243] feat(lib): add errors for divide --- src/builtins/divide/README.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/builtins/divide/README.md b/src/builtins/divide/README.md index 20ed19f55..a80108d04 100644 --- a/src/builtins/divide/README.md +++ b/src/builtins/divide/README.md @@ -93,3 +93,28 @@ if-run --manifest ./examples/manifests/divide.yml --output ./examples/outputs/di ``` The results will be saved to a new `yaml` file in `./examples/outputs`. + +## Errors + +`Divide` exposes two of IF's error classes. + +### GlobalConfigError + +You will receive an error starting `GlobalConfigError: ` if you have not provided the expected configuration data in the plugin's `initialize` block. + +The required parameters are: +- `numerator`: a string containing the name of the input parameter whose value should be divided by `denominator` +- `denominator`: a number to use as the denominator +- ``output`: a string containing the name to assign the result of the division + +You can fix this error by checking you are providing valid values for each parameter in the config. + +### `MissingInputDataError` + +This error arises when a necessary piece of input data is missing from the `inputs` array. +Every element in the ``inputs` array must contain: +- `timestamp` +- `duration` +- whatever value you passed to `numerator` + + From e5cc8d67cbf73ae5ca545298f95ce771b2ba70aa Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Thu, 20 Jun 2024 15:36:35 +0100 Subject: [PATCH 134/243] feat(lib): add errors to exponent readme --- src/builtins/exponent/README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/builtins/exponent/README.md b/src/builtins/exponent/README.md index d84020d63..9a263e4be 100644 --- a/src/builtins/exponent/README.md +++ b/src/builtins/exponent/README.md @@ -95,3 +95,21 @@ if-run --manifest manifests/examples/test/exponent.yml --output manifests/output ``` The results will be saved to a new `yaml` file in `manifests/outputs`. + +## Errors + +`Exponent` exposes two of IF's error classes. + +### `MissingInputDataError` + +This error arises when a necessary piece of input data is missing from the `inputs` array. +Every element in the ``inputs` array must contain: + +- `timestamp` +- `duration` +- whatever value you passed to `input-parameter` + + +### `InputValidationError` + +This error arises when an invalid value is passed to `Exponent`. Typically, this can occur when a non-numeric value (such as a string made of alphabetic characters) is passed where a number or numeric string is expected. Please check that the types are correct for all the relevant fields in your `inputs` array. From 0f91086da064de4cc2b805c640f6796dbe83d507 Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Thu, 20 Jun 2024 15:42:10 +0100 Subject: [PATCH 135/243] feat(lib): add errors for interpolate --- src/builtins/interpolation/README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/builtins/interpolation/README.md b/src/builtins/interpolation/README.md index 670a65ef2..cdb832818 100644 --- a/src/builtins/interpolation/README.md +++ b/src/builtins/interpolation/README.md @@ -166,3 +166,29 @@ You can execute this by passing it to `ie`. Run the impact using the following c npm i -g @grnsft/if if-run --manifest ./manifests/examples/interpolation.yml --output ./manifests/outputs/interpolation.yml ``` + +## Errors + +`Interpolation` exposes one of IF's error classes. + +## `GlobalConfigError` + +### GlobalConfigError + +You will receive an error starting `GlobalConfigError: ` if you have not provided the expected configuration data in the plugin's `initialize` block. + +The required parameters are: +- `method`: a string containing either `linear`, `spline` or `polynomial` +- `x`: an array of numbers +- `y`: an array of numbers +- `input-parameter`: a string containing the name of a value present in the `inputs` array' +- `output-parameter`: a string + +You can fix this error by checking you are providing valid values for each parameter in the config. + +### Validation errors + +There are also several validation errors that can arise, including: +- if the lengths of `x` and `y` are not equal +- if `x` or `y` are empty +- if the requested point to interpolate at is outside the range of `x` From e8e2c5a70d9d59d8953e8c6c394706a1c1ae1518 Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Thu, 20 Jun 2024 15:44:12 +0100 Subject: [PATCH 136/243] feat(lib): add errors to Multiply --- src/builtins/multiply/README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/builtins/multiply/README.md b/src/builtins/multiply/README.md index 695d89331..f0e5d6dad 100644 --- a/src/builtins/multiply/README.md +++ b/src/builtins/multiply/README.md @@ -92,3 +92,17 @@ if-run --manifest ./examples/manifests/test/multiply.yml --output ./examples/out ``` The results will be saved to a new `yaml` file in `./examples/outputs` + + +## Errors + +`Multiply` uses one of the IF error classes. + +### `MissingInputDataError` + +This error arises when a necessary piece of input data is missing from the `inputs` array. +Every element in the ``inputs` array must contain: +- `timestamp` +- `duration` +- whatever values you passed to `input-parameters` + From f6c6ce18d19975ba85c061a940a95875e9765a41 Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 20 Jun 2024 18:44:46 +0400 Subject: [PATCH 137/243] feat(util): move the isFileExists funtion to the relevant file --- src/util/fs.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 src/util/fs.ts diff --git a/src/util/fs.ts b/src/util/fs.ts new file mode 100644 index 000000000..92ca9684a --- /dev/null +++ b/src/util/fs.ts @@ -0,0 +1,13 @@ +import * as fs from 'fs/promises'; + +/** + * Checks if file exists with the given `filePath`. + */ +export const isFileExists = async (filePath: string) => { + try { + await fs.stat(filePath); + return true; + } catch (error) { + return false; + } +}; From 9ff31c85e9557da6247c3e14186de992305d6cdd Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 20 Jun 2024 18:47:12 +0400 Subject: [PATCH 138/243] feat(util): move the npm related funtions to the relevant file --- src/types/npm.ts | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 src/types/npm.ts diff --git a/src/types/npm.ts b/src/types/npm.ts new file mode 100644 index 000000000..cbb3e0680 --- /dev/null +++ b/src/types/npm.ts @@ -0,0 +1,3 @@ +export type PathWithVersion = {[path: string]: string}; + +export type ManifestPlugin = {[key: string]: {path: string; method: string}}; From 5e382707b2545e6289c5cb479f3d3b3c44376e8b Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 20 Jun 2024 18:48:41 +0400 Subject: [PATCH 139/243] fix(util): move some functions to the coresponding files --- src/util/helpers.ts | 49 +-------------------------------------------- 1 file changed, 1 insertion(+), 48 deletions(-) diff --git a/src/util/helpers.ts b/src/util/helpers.ts index 9abcf9ba4..0c422c96f 100644 --- a/src/util/helpers.ts +++ b/src/util/helpers.ts @@ -1,5 +1,3 @@ -import * as fs from 'fs/promises'; -import * as path from 'path'; import {createInterface} from 'node:readline/promises'; import {exec} from 'node:child_process'; import {promisify} from 'node:util'; @@ -12,8 +10,7 @@ import {STRINGS} from '../config'; import {Difference} from '../types/lib/compare'; -const {UNSUPPORTED_ERROR, INITIALIZING_PACKAGE_JSON, INSTALLING_NPM_PACKAGES} = - STRINGS; +const {UNSUPPORTED_ERROR} = STRINGS; /** * Impact engine error handler. Logs errors and appends issue template if error is unknown. @@ -179,47 +176,3 @@ export const parseManifestFromStdin = async () => { return match![1]; }; - -/** - * Checks if file exists with the given `filePath`. - */ -export const isFileExists = async (filePath: string) => { - try { - await fs.stat(filePath); - return true; - } catch (error) { - return false; - } -}; - -/** - * Checks if the package.json is exists, if not, inisializes it. - */ -export const initPackageJsonIfNotExists = async (folderPath: string) => { - const packageJsonPath = path.resolve(folderPath, 'package.json'); - const isPackageJsonExists = await isFileExists(packageJsonPath); - - if (!isPackageJsonExists) { - logger.info(INITIALIZING_PACKAGE_JSON); - await execPromise('npm init -y', {cwd: folderPath}); - } - - return packageJsonPath; -}; - -/** - * Installs packages from the specified dependencies in the specified folder. - */ -export const installDependencies = async ( - folderPath: string, - dependencies: {[path: string]: string} -) => { - const packages = Object.entries(dependencies).map( - ([dependency, version]) => `${dependency}@${version.replace('^', '')}` - ); - - logger.info(INSTALLING_NPM_PACKAGES); - await execPromise(`npm install ${packages.join(' ')}`, { - cwd: folderPath, - }); -}; From e6440049d18dd428207fdf2ca87a50de15002332 Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Thu, 20 Jun 2024 15:49:25 +0100 Subject: [PATCH 140/243] feat(lib): add errors for regex to readme --- src/builtins/regex/README.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/builtins/regex/README.md b/src/builtins/regex/README.md index 52a99db9f..a0e88c7cc 100644 --- a/src/builtins/regex/README.md +++ b/src/builtins/regex/README.md @@ -89,3 +89,33 @@ if --manifest manifests/examples/regex.yml --output manifests/outputs/regex.yml ``` The results will be saved to a new `yaml` file in `manifests/outputs`. + +## Errors + +`Regex` uses three of IF's error classes: + +### `MissingInputDataError` + +This error arises when a necessary piece of input data is missing from the `inputs` array. +Every element in the ``inputs` array must contain: +- `timestamp` +- `duration` +- whatever value you passed to `parameter` + + +### `GlobalConfigError` + +You will receive an error starting `GlobalConfigError: ` if you have not provided the expected configuration data in the plugin's `initialize` block. + +The required parameters are: + +- `parameter`: a string containing the name of a value in the inputs array +- `match`: a valid regex pattern +- `output`: a string + +You can fix this error by checking you are providing valid values for each parameter in the config. + + +### `RegexMismatchError` + +This error arises when the requested regex cannot find any matches in the given data. If there are multiple matches, the plugin returns the first, but if there are none, it throws this error. From 7a764ae96ff58888cd6520a04f09d198aae82455 Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Thu, 20 Jun 2024 15:53:02 +0100 Subject: [PATCH 141/243] feat(lib): add error info to sci readme --- src/builtins/sci/README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/builtins/sci/README.md b/src/builtins/sci/README.md index 658209f1a..21dca0e19 100644 --- a/src/builtins/sci/README.md +++ b/src/builtins/sci/README.md @@ -87,3 +87,22 @@ if-run --manifest manifests/plugins/sci.yml --output manifests/outputs/sci.yml ``` The results will be saved to a new `yaml` file. + +## Errors + +`SCI` uses one of the IF error classes. + +### `MissingInputDataError` + +This error arises when a necessary piece of input data is missing from the `inputs` array. + +Every element in the `inputs` array must contain: +- `timestamp` +- `duration` +- `carbon`: a numeric value named `carbon` must exist in the inputs array +- whatever value you passed to `functional-unit` + + +### Validation errors + +There is also a validation step that checks that the `functional-unit` was provided in the plugin config. If you see an error reporting this value as missing, please check you have provided it. From 5483ad71f4f153f2cc723aa2222e1682cfc31c4a Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 20 Jun 2024 18:59:22 +0400 Subject: [PATCH 142/243] fix(util): update isFileExists path --- src/util/args.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/util/args.ts b/src/util/args.ts index 031c2250e..67e38890c 100644 --- a/src/util/args.ts +++ b/src/util/args.ts @@ -5,7 +5,7 @@ import {ERRORS} from '@grnsft/if-core/utils'; import {checkIfFileIsYaml} from './yaml'; -import {isFileExists} from './helpers'; +import {isFileExists} from './fs'; import {logger} from './logger'; @@ -176,15 +176,14 @@ export const parseIfEnvArgs = async () => { const {manifest, install, cwd} = validateAndParseIfEnvArgs(); if (manifest) { - const isManifestFileExists = await isFileExists(manifest); + const response = prependFullFilePath(manifest); + const isManifestFileExists = await isFileExists(response); if (!isManifestFileExists) { throw new CliInputError(MANIFEST_NOT_FOUND); } if (checkIfFileIsYaml(manifest)) { - const response = prependFullFilePath(manifest); - return {manifest: response, install, cwd}; } From 43a79aa22741265c25adc779d3c85a060706572e Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Thu, 20 Jun 2024 16:01:30 +0100 Subject: [PATCH 143/243] feat(lib): add error info to sci-embodied and shell --- src/builtins/sci-embodied/README.md | 11 +++++++++++ src/builtins/shell/README.md | 9 +++++++++ 2 files changed, 20 insertions(+) diff --git a/src/builtins/sci-embodied/README.md b/src/builtins/sci-embodied/README.md index 70307fee1..3fd169211 100644 --- a/src/builtins/sci-embodied/README.md +++ b/src/builtins/sci-embodied/README.md @@ -108,3 +108,14 @@ if-run --manifest manifests/plugins/sci-embodied.yml --output manifests/outputs/ ``` The results will be saved to a new `yaml` file in `./examples/outputs`. + + +## Errors + +`SciEmbodied` uses one of IF's error classes + +### `SciEmbodiedError` + +This error class is used to describe a problem with one of the input values to `sci-embodied`. This is typically due to an incorrect type or a reference to a value that is not available. + +You will receive a specific error message explaining which parameter is problematic, and you can check and replace where appropriate. diff --git a/src/builtins/shell/README.md b/src/builtins/shell/README.md index 54ccac285..61987958e 100644 --- a/src/builtins/shell/README.md +++ b/src/builtins/shell/README.md @@ -128,3 +128,12 @@ if-run --manifest manifests/plugins/shell.yml --output manifests/outputs/shell.y ``` The results will be saved to a new `yaml` file. + + +## Errors + +`Shell` uses one of the error classes provided by IF + +### `ProcessExecutionError` + +This error is thrown when the program invoked in the spawned shell fails for some reason that is not known to IF. Since the shell executes arbitrary code, it is difficult for IF to provide specific errors - this is delegated to the developers of the executed program. From 2a6f242ed73f48e0fd66c68724b6fc9273a99ca7 Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 20 Jun 2024 19:02:37 +0400 Subject: [PATCH 144/243] feat(util): move the npm related funtions to the relevant file --- src/util/npm.ts | 135 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 src/util/npm.ts diff --git a/src/util/npm.ts b/src/util/npm.ts new file mode 100644 index 000000000..bfdf82ca6 --- /dev/null +++ b/src/util/npm.ts @@ -0,0 +1,135 @@ +#!/usr/bin/env node +/* eslint-disable no-process-exit */ +import * as fs from 'fs/promises'; +import * as path from 'path'; + +import {execPromise} from './helpers'; +import {isFileExists} from './fs'; +import {logger} from './logger'; + +import {STRINGS} from '../config'; +import {ManifestPlugin, PathWithVersion} from '../types/npm'; + +const packageJson = require('../../package.json'); + +const {INITIALIZING_PACKAGE_JSON, INSTALLING_NPM_PACKAGES} = STRINGS; + +/** + * Checks if the package.json is exists, if not, inisializes it. + */ +export const initPackageJsonIfNotExists = async (folderPath: string) => { + const packageJsonPath = path.resolve(folderPath, 'package.json'); + const isPackageJsonExists = await isFileExists(packageJsonPath); + + if (!isPackageJsonExists) { + logger.info(INITIALIZING_PACKAGE_JSON); + await execPromise('npm init -y', {cwd: folderPath}); + } + + return packageJsonPath; +}; + +/** + * Installs packages from the specified dependencies in the specified folder. + */ +export const installDependencies = async ( + folderPath: string, + dependencies: {[path: string]: string} +) => { + const packages = Object.entries(dependencies).map( + ([dependency, version]) => `${dependency}@${version.replace('^', '')}` + ); + + logger.info(INSTALLING_NPM_PACKAGES); + await execPromise(`npm install ${packages.join(' ')}`, { + cwd: folderPath, + }); +}; + +/** + * Updates package.json dependencies. + */ +export const updatePackageJsonDependencies = async ( + packageJsonPath: string, + dependencies: PathWithVersion, + cwd: boolean +) => { + const packageJsonContent = await fs.readFile(packageJsonPath, 'utf8'); + + const parsedPackageJson = JSON.parse(packageJsonContent); + + if (cwd) { + parsedPackageJson.dependencies = { + ...parsedPackageJson.dependencies, + ...dependencies, + }; + } else { + parsedPackageJson.dependencies = {...dependencies}; + } + + await fs.writeFile( + packageJsonPath, + JSON.stringify(parsedPackageJson, null, 2) + ); +}; + +/** + * Gets depencecies with versions. + */ +export const extractPathsWithVersion = ( + plugins: ManifestPlugin, + dependencies: string[] +) => { + const paths = Object.keys(plugins).map(plugin => plugins[plugin].path); + const uniquePaths = [...new Set(paths)].filter(path => path !== 'builtin'); + const pathsWithVersion: PathWithVersion = {}; + + uniquePaths.forEach(pluginPath => { + const dependency = dependencies.find((dependency: string) => + dependency.startsWith(pluginPath) + ); + + if (dependency) { + const splittedDependency = dependency.split('@'); + const version = + splittedDependency.length > 2 + ? splittedDependency[2].split(' ')[0] + : splittedDependency[1]; + + pathsWithVersion[pluginPath] = `^${version}`; + } + }); + + return pathsWithVersion; +}; + +/** + * Update the package.json properties. + */ +export const updatePackageJsonProperties = async ( + newPackageJsonPath: string +) => { + const packageJsonContent = await fs.readFile(newPackageJsonPath, 'utf8'); + const parsedPackageJsonContent = JSON.parse(packageJsonContent); + + const properties = { + name: 'if-environment', + description: packageJson.description, + author: packageJson.author, + bugs: packageJson.bugs, + engines: packageJson.engines, + homepage: packageJson.homepage, + dependencies: {}, + }; + + const newPackageJson = Object.assign( + {}, + parsedPackageJsonContent, + properties + ); + + await fs.writeFile( + newPackageJsonPath, + JSON.stringify(newPackageJson, null, 2) + ); +}; From ad1cbcb768c76760c9b06cb52c2d87306c3d67d4 Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Thu, 20 Jun 2024 16:03:13 +0100 Subject: [PATCH 145/243] feat(lib): add error info to subtract readme --- src/builtins/subtract/README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/builtins/subtract/README.md b/src/builtins/subtract/README.md index bb8435548..5869d93a7 100644 --- a/src/builtins/subtract/README.md +++ b/src/builtins/subtract/README.md @@ -92,3 +92,12 @@ if-run --manifest /manifests/plugins/subtract.yml --output manifests/outputs/sub ``` The results will be saved to a new `yaml` file in `manifests/outputs`. + + +## Errors + +`Subtract` uses one of IF's error classes: + +### `InputValidationError` + +This error arises when an invalid value is passed to `Subtract`. Typically, this can occur when a non-numeric value (such as a string made of alphabetic characters) is passed where a number or numeric string is expected. Please check that the types are correct for all the relevant fields in your `inputs` array. From 0b403b2f04e481c9782747e36af526a92f0b05a5 Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 20 Jun 2024 19:03:41 +0400 Subject: [PATCH 146/243] test(util): fix isFileExists path --- src/__tests__/unit/util/args.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/__tests__/unit/util/args.test.ts b/src/__tests__/unit/util/args.test.ts index 61874dd35..b0605260e 100644 --- a/src/__tests__/unit/util/args.test.ts +++ b/src/__tests__/unit/util/args.test.ts @@ -1,6 +1,6 @@ const processRunningPath = process.cwd(); -jest.mock('../../../util/helpers', () => ({ +jest.mock('../../../util/fs', () => ({ isFileExists: () => { if (process.env.fileExists === 'true') { return true; From ab544404eb40c86c9a42d5e07e0ffbb2b6eb5ce1 Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 20 Jun 2024 19:05:18 +0400 Subject: [PATCH 147/243] feat(mocks): add mocks for util/npm functionality --- src/__mocks__/fs/index.ts | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/src/__mocks__/fs/index.ts b/src/__mocks__/fs/index.ts index eab90532e..b7731b4b8 100644 --- a/src/__mocks__/fs/index.ts +++ b/src/__mocks__/fs/index.ts @@ -1,10 +1,19 @@ import * as YAML from 'js-yaml'; +import * as fs from 'fs'; +import * as fsAsync from 'fs/promises'; export const readFile = async (filePath: string) => { + /** mock for util/npm */ + if (filePath.includes('package.json-npm')) { + const updatedPath = filePath.replace('-npm', ''); + return fs.readFileSync(updatedPath, 'utf8'); + } + /** mock for util/json */ if (filePath.includes('json-reject')) { return Promise.reject(new Error('rejected')); } + if (filePath.includes('json')) { if (filePath.includes('param')) { return JSON.stringify({ @@ -80,18 +89,25 @@ cpu-cores-available,cpu-cores-utilized,cpu-manufacturer,cpu-model-name,cpu-tdp,g export const mkdir = (dirPath: string) => dirPath; export const writeFile = async (pathToFile: string, content: string) => { - if (pathToFile === 'reject') { - throw new Error('Wrong file path'); - } + if (pathToFile.includes('package.json-npm')) { + const updatedPath = pathToFile.replace('-npm', ''); - const mockPathToFile = 'mock-pathToFile'; - const mockContent = { - name: 'mock-name', - }; - const mockObject = YAML.dump(mockContent, {noRefs: true}); + const content = await fsAsync.readFile(updatedPath, 'utf8'); + expect(content).toBe(content); + } else { + if (pathToFile === 'reject') { + throw new Error('Wrong file path'); + } - expect(pathToFile).toBe(mockPathToFile); - expect(content).toBe(mockObject); + const mockPathToFile = 'mock-pathToFile'; + const mockContent = { + name: 'mock-name', + }; + const mockObject = YAML.dump(mockContent, {noRefs: true}); + + expect(pathToFile).toBe(mockPathToFile); + expect(content).toBe(mockObject); + } }; export const stat = async (filePath: string) => { From 549230e3f92d25bef420e43da1ce390a5d874bcc Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Thu, 20 Jun 2024 16:06:32 +0100 Subject: [PATCH 148/243] feat(lib): add error info to sum --- src/builtins/coefficient/README.md | 2 +- src/builtins/sum/README.md | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/builtins/coefficient/README.md b/src/builtins/coefficient/README.md index 629059799..4f4d2b839 100644 --- a/src/builtins/coefficient/README.md +++ b/src/builtins/coefficient/README.md @@ -94,7 +94,7 @@ The results will be saved to a new `yaml` file in `./examples/outputs` ## Errors -`Coefficient exposes one of the IF error classes. +`Coefficient` exposes one of the IF error classes. ### GlobalConfigError diff --git a/src/builtins/sum/README.md b/src/builtins/sum/README.md index d3fd892da..5ffc85b37 100644 --- a/src/builtins/sum/README.md +++ b/src/builtins/sum/README.md @@ -89,3 +89,27 @@ if-run --manifest ./examples/manifests/sum.yml --output ./examples/outputs/sum.y ``` The results will be saved to a new `yaml` file in `./examples/outputs`. + + +## Errors + +`Sum` exposes two of the IF error classes. + +### GlobalConfigError + +You will receive an error starting `GlobalConfigError: ` if you have not provided the expected configuration data in the plugin's `initialize` block. + +The required parameters are: +- `input-parameters`: this must be an array of strings, each being the name of a value in the `inputs` array +- `output-parameter`: this must be a string + +You can fix this error by checking you are providing valid values for each parameter in the config. + +### `MissingInputDataError` + +This error arises when a necessary piece of input data is missing from the `inputs` array. +Every element in the ``inputs` array must contain: + +- `timestamp` +- `duration` +- whatever values you passed to `input-parameters` From 6a3ab7b7b1bd5e8cfeab722f365987061a40d0e9 Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 20 Jun 2024 19:06:52 +0400 Subject: [PATCH 149/243] test(util): add test file for util/fs --- src/__tests__/unit/util/fs.test.ts | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 src/__tests__/unit/util/fs.test.ts diff --git a/src/__tests__/unit/util/fs.test.ts b/src/__tests__/unit/util/fs.test.ts new file mode 100644 index 000000000..82f28bc75 --- /dev/null +++ b/src/__tests__/unit/util/fs.test.ts @@ -0,0 +1,21 @@ +import {isFileExists} from '../../../util/fs'; + +jest.mock('fs/promises', () => require('../../../__mocks__/fs')); + +describe('util/fs: ', () => { + describe('isFileExists(): ', () => { + it('returns true if the file exists.', async () => { + const result = await isFileExists('true'); + + expect.assertions(1); + expect(result).toEqual(true); + }); + + it('returns fale if the file does not exist.', async () => { + const result = await isFileExists('false'); + + expect.assertions(1); + expect(result).toEqual(false); + }); + }); +}); From 83fd07200096be4234bfab8caed81e54d0a65b75 Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 20 Jun 2024 19:08:14 +0400 Subject: [PATCH 150/243] test(util): add test file for util/npm --- src/__tests__/unit/util/npm.test.ts | 230 ++++++++++++++++++++++++++++ 1 file changed, 230 insertions(+) create mode 100644 src/__tests__/unit/util/npm.test.ts diff --git a/src/__tests__/unit/util/npm.test.ts b/src/__tests__/unit/util/npm.test.ts new file mode 100644 index 000000000..5c904aa23 --- /dev/null +++ b/src/__tests__/unit/util/npm.test.ts @@ -0,0 +1,230 @@ +import * as fs from 'fs/promises'; +import * as fsSync from 'fs'; +import * as path from 'path'; + +jest.mock('fs/promises', () => require('../../../__mocks__/fs')); + +const mockInfo = jest.fn(); + +jest.mock('node:readline/promises', () => + require('../../../__mocks__/readline') +); +jest.mock('../../../util/logger', () => ({ + logger: { + info: mockInfo, + }, +})); + +import { + installDependencies, + initPackageJsonIfNotExists, + updatePackageJsonDependencies, + extractPathsWithVersion, + updatePackageJsonProperties, +} from '../../../util/npm'; +import {isFileExists} from '../../../util/fs'; + +import {STRINGS} from '../../../config/strings'; +import {ManifestPlugin} from '../../../types/npm'; + +const {INITIALIZING_PACKAGE_JSON, INSTALLING_NPM_PACKAGES} = STRINGS; + +describe('util/npm: ', () => { + const helpers = require('../../../util/helpers'); + const folderPath = path.resolve(__dirname, 'npm-test'); + + beforeAll(() => { + if (!fsSync.existsSync(folderPath)) { + fsSync.mkdirSync(folderPath, {recursive: true}); + } + }); + + afterAll(() => { + if (fsSync.existsSync(folderPath)) { + fsSync.rmSync(folderPath, {recursive: true, force: true}); + } + }); + + describe('initPackageJsonIfNotExists(): ', () => { + it('initializes package.json if it does not exist.', async () => { + const spyExecPromise = jest.spyOn(helpers, 'execPromise'); + isFileExists('true'); + + await initPackageJsonIfNotExists(folderPath); + + expect.assertions(2); + expect(mockInfo).toHaveBeenCalledWith(INITIALIZING_PACKAGE_JSON); + expect(spyExecPromise).toHaveBeenCalledWith('npm init -y', { + cwd: folderPath, + }); + }); + + it('returns the package.json path if it exists.', async () => { + const packageJsonPath = path.resolve(folderPath, 'package.json'); + isFileExists('false'); + + const result = await initPackageJsonIfNotExists(folderPath); + + expect.assertions(1); + expect(result).toBe(packageJsonPath); + }); + }); + + describe('installDependencies(): ', () => { + const dependencies = { + '@grnsft/if': '^0.3.3-beta.0', + }; + + it('calls execPromise with the correct arguments.', async () => { + const spyExecPromise = jest.spyOn(helpers, 'execPromise'); + const formattedDependencies = ['@grnsft/if@0.3.3-beta.0']; + expect.assertions(1); + + await installDependencies(folderPath, dependencies); + + expect(spyExecPromise).toHaveBeenCalledWith( + `npm install ${formattedDependencies.join(' ')}`, + {cwd: folderPath} + ); + }, 30000); + + it('logs the installation message.', async () => { + const dependencies = { + '@grnsft/if': '^0.3.3-beta.0', + }; + + await installDependencies(folderPath, dependencies); + + expect.assertions(1); + expect(mockInfo).toHaveBeenCalledWith(INSTALLING_NPM_PACKAGES); + }); + }); + + describe('updatePackageJsonDependencies(): ', () => { + it('successfully updates the package.json dependencies when cwd is false.', async () => { + const dependencies = { + '@grnsft/if-plugins': '^0.3.3-beta.0', + }; + const packageJsonPath = path.join(folderPath, 'package.json-npm'); + + const expectedPackageJsonContent = JSON.stringify( + { + dependencies: { + '@grnsft/if': '^0.3.3-beta.0', + '@grnsft/if-plugins': '^0.3.3-beta.0', + }, + }, + null, + 2 + ); + + const fsReadSpy = jest + .spyOn(fs, 'readFile') + .mockResolvedValue(expectedPackageJsonContent); + await updatePackageJsonDependencies(packageJsonPath, dependencies, false); + + expect.assertions(2); + + expect(fsReadSpy).toHaveBeenCalledWith(packageJsonPath, 'utf8'); + }); + + it('successfully updates the package.json dependencies when cwd is true.', async () => { + const dependencies = { + '@grnsft/if-plugins': '^0.3.3-beta.0', + }; + const packageJsonPath = path.join(folderPath, 'package.json-npm'); + + const expectedPackageJsonContent = JSON.stringify( + { + dependencies: { + '@grnsft/if': '^0.3.3-beta.0', + '@grnsft/if-plugins': '^0.3.3-beta.0', + }, + }, + null, + 2 + ); + + const fsReadSpy = jest + .spyOn(fs, 'readFile') + .mockResolvedValue(expectedPackageJsonContent); + await updatePackageJsonDependencies(packageJsonPath, dependencies, true); + + expect.assertions(2); + + expect(fsReadSpy).toHaveBeenCalledWith(packageJsonPath, 'utf8'); + }); + }); + + describe('extractPathsWithVersion(): ', () => { + it('extracts paths with correct versions.', () => { + const plugins: ManifestPlugin = { + 'cloud-metadata': { + path: '@grnsft/if-plugins', + method: 'CloudMetadata', + }, + divide: { + path: 'builtin', + method: 'Divide', + }, + 'boavizta-cpu': { + path: '@grnsft/if-unofficial-plugins', + method: 'BoaviztaCpuOutput', + }, + }; + const dependencies = [ + '@babel/core@7.22.10', + '@babel/preset-typescript@7.23.3', + '@commitlint/cli@18.6.0', + '@commitlint/config-conventional@18.6.0', + '@grnsft/if-core@0.0.7', + '@grnsft/if-plugins@v0.3.2 extraneous -> file:../../../if-models', + '@grnsft/if-unofficial-plugins@v0.3.0 extraneous -> file:../../../if-unofficial-models', + '@jest/globals@29.7.0', + ]; + + const result = extractPathsWithVersion(plugins, dependencies); + + expect.assertions(1); + expect(result).toEqual({ + '@grnsft/if-plugins': '^v0.3.2', + '@grnsft/if-unofficial-plugins': '^v0.3.0', + }); + }); + + it('returns an empty object if no matches found', () => { + const plugins: ManifestPlugin = { + 'cloud-metadata': { + path: '@grnsft/if-plugins', + method: 'CloudMetadata', + }, + divide: { + path: 'builtin', + method: 'Divide', + }, + 'boavizta-cpu': { + path: '@grnsft/if-unofficial-plugins', + method: 'BoaviztaCpuOutput', + }, + }; + const dependencies = [ + '@babel/core@7.22.10', + '@babel/preset-typescript@7.23.3', + ]; + + expect.assertions(1); + const result = extractPathsWithVersion(plugins, dependencies); + expect(result).toEqual({}); + }); + }); + + describe('updatePackageJsonProperties(): ', () => { + it('updates the package.json properties correctly.', async () => { + const newPackageJsonPath = path.resolve(folderPath, '/package.json-npm'); + await updatePackageJsonProperties(newPackageJsonPath); + + expect.assertions(2); + expect(fs.readFile).toHaveBeenCalledWith(newPackageJsonPath, 'utf8'); + }); + }); +}); From 46afe5a65c9b4d21f061a2b8e24cdcc6bbcd72ae Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 20 Jun 2024 19:09:47 +0400 Subject: [PATCH 151/243] fix(types): move some type to the corresponding file --- src/types/if-env.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/types/if-env.ts b/src/types/if-env.ts index 796fd0ca8..03ace8ec9 100644 --- a/src/types/if-env.ts +++ b/src/types/if-env.ts @@ -4,7 +4,3 @@ export type EnvironmentOptions = { cwd: boolean; dependencies: {[path: string]: string}; }; - -export type PathWithVersion = {[path: string]: string}; - -export type ManifestPlugin = {[key: string]: {path: string; method: string}}; From 07d91a4e204d52c26c51b6f584425ccb9ea3a800 Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 20 Jun 2024 19:11:14 +0400 Subject: [PATCH 152/243] test(util): move not related tests --- src/__tests__/unit/util/helpers.test.ts | 100 ------------------------ 1 file changed, 100 deletions(-) diff --git a/src/__tests__/unit/util/helpers.test.ts b/src/__tests__/unit/util/helpers.test.ts index e093a3468..ba89850a3 100644 --- a/src/__tests__/unit/util/helpers.test.ts +++ b/src/__tests__/unit/util/helpers.test.ts @@ -1,9 +1,3 @@ -import * as fs from 'fs'; -import * as path from 'path'; - -jest.mock('fs/promises', () => require('../../../__mocks__/fs')); - -const mockInfo = jest.fn(); const mockWarn = jest.fn(); const mockError = jest.fn(); @@ -12,7 +6,6 @@ jest.mock('node:readline/promises', () => ); jest.mock('../../../util/logger', () => ({ logger: { - info: mockInfo, warn: mockWarn, error: mockError, }, @@ -27,15 +20,10 @@ import { mergeObjects, oneIsPrimitive, parseManifestFromStdin, - isFileExists, - installDependencies, - initPackageJsonIfNotExists, } from '../../../util/helpers'; import {Difference} from '../../../types/lib/compare'; -import {STRINGS} from '../../../config/strings'; const {WriteFileError} = ERRORS; -const {INITIALIZING_PACKAGE_JSON, INSTALLING_NPM_PACKAGES} = STRINGS; describe('util/helpers: ', () => { describe('andHandle(): ', () => { @@ -411,92 +399,4 @@ description: mock-description expect(response).toBeFalsy(); }); }); - - describe('isFileExists(): ', () => { - it('returns true if the file exists.', async () => { - const result = await isFileExists('true'); - - expect.assertions(1); - expect(result).toEqual(true); - }); - - it('returns fale if the file does not exist.', async () => { - const result = await isFileExists('false'); - - expect.assertions(1); - expect(result).toEqual(false); - }); - }); - - describe('npm manipulation tests.', () => { - const helpers = require('../../../util/helpers'); - const folderPath = path.resolve(__dirname, 'npm-test'); - - beforeAll(() => { - if (!fs.existsSync(folderPath)) { - fs.mkdirSync(folderPath, {recursive: true}); - } - }); - - afterAll(() => { - if (fs.existsSync(folderPath)) { - fs.rmSync(folderPath, {recursive: true, force: true}); - } - }); - - describe('initPackageJsonIfNotExists(): ', () => { - it('initializes package.json if it does not exist.', async () => { - const spyExecPromise = jest.spyOn(helpers, 'execPromise'); - isFileExists('true'); - - await initPackageJsonIfNotExists(folderPath); - - expect.assertions(2); - expect(mockInfo).toHaveBeenCalledWith(INITIALIZING_PACKAGE_JSON); - expect(spyExecPromise).toHaveBeenCalledWith('npm init -y', { - cwd: folderPath, - }); - }); - - it('returns the package.json path if it exists.', async () => { - const packageJsonPath = path.resolve(folderPath, 'package.json'); - isFileExists('false'); - - const result = await initPackageJsonIfNotExists(folderPath); - - expect.assertions(1); - expect(result).toBe(packageJsonPath); - }); - }); - - describe('installDependencies(): ', () => { - const dependencies = { - '@grnsft/if': '^0.3.3-beta.0', - }; - - it('calls execPromise with the correct arguments.', async () => { - const spyExecPromise = jest.spyOn(helpers, 'execPromise'); - const formattedDependencies = ['@grnsft/if@0.3.3-beta.0']; - expect.assertions(1); - - await installDependencies(folderPath, dependencies); - - expect(spyExecPromise).toHaveBeenCalledWith( - `npm install ${formattedDependencies.join(' ')}`, - {cwd: folderPath} - ); - }, 30000); - - it('logs the installation message.', async () => { - const dependencies = { - '@grnsft/if': '^0.3.3-beta.0', - }; - - await installDependencies(folderPath, dependencies); - - expect.assertions(1); - expect(mockInfo).toHaveBeenCalledWith(INSTALLING_NPM_PACKAGES); - }); - }); - }); }); From d8a61815df19e4b9451288c6afd8cfea27d5aad4 Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Thu, 20 Jun 2024 16:11:40 +0100 Subject: [PATCH 153/243] feat(lib): add link out to docs to all readmes --- src/builtins/coefficient/README.md | 2 ++ src/builtins/csv-lookup/README.md | 2 ++ src/builtins/divide/README.md | 1 + src/builtins/exponent/README.md | 2 ++ src/builtins/interpolation/README.md | 2 ++ src/builtins/multiply/README.md | 1 + src/builtins/regex/README.md | 2 ++ src/builtins/sci-embodied/README.md | 3 +++ src/builtins/sci/README.md | 2 ++ src/builtins/shell/README.md | 2 ++ src/builtins/subtract/README.md | 2 ++ src/builtins/sum/README.md | 3 +++ 12 files changed, 24 insertions(+) diff --git a/src/builtins/coefficient/README.md b/src/builtins/coefficient/README.md index 4f4d2b839..4579cd5b8 100644 --- a/src/builtins/coefficient/README.md +++ b/src/builtins/coefficient/README.md @@ -106,3 +106,5 @@ The required parameters are: - `output-parameter`: this must be a string You can fix this error by checking you are providing valid values for each parameter in the config. + +For more information on our error classes, please visit [our docs](https://if.greensoftware.foundation/reference/errors diff --git a/src/builtins/csv-lookup/README.md b/src/builtins/csv-lookup/README.md index 19d8642ec..eb62f8c7e 100644 --- a/src/builtins/csv-lookup/README.md +++ b/src/builtins/csv-lookup/README.md @@ -176,3 +176,5 @@ The required parameters are: - `output`: this must be a string containing a name or a wildcard character (`"*"`) You can fix this error by checking you are providing valid values for each parameter in the config. + +For more information on our error classes, please visit [our docs](https://if.greensoftware.foundation/reference/errors diff --git a/src/builtins/divide/README.md b/src/builtins/divide/README.md index a80108d04..cb98c7141 100644 --- a/src/builtins/divide/README.md +++ b/src/builtins/divide/README.md @@ -118,3 +118,4 @@ Every element in the ``inputs` array must contain: - whatever value you passed to `numerator` +For more information on our error classes, please visit [our docs](https://if.greensoftware.foundation/reference/errors diff --git a/src/builtins/exponent/README.md b/src/builtins/exponent/README.md index 9a263e4be..e484ea74d 100644 --- a/src/builtins/exponent/README.md +++ b/src/builtins/exponent/README.md @@ -113,3 +113,5 @@ Every element in the ``inputs` array must contain: ### `InputValidationError` This error arises when an invalid value is passed to `Exponent`. Typically, this can occur when a non-numeric value (such as a string made of alphabetic characters) is passed where a number or numeric string is expected. Please check that the types are correct for all the relevant fields in your `inputs` array. + +For more information on our error classes, please visit [our docs](https://if.greensoftware.foundation/reference/errors diff --git a/src/builtins/interpolation/README.md b/src/builtins/interpolation/README.md index cdb832818..498c26c6e 100644 --- a/src/builtins/interpolation/README.md +++ b/src/builtins/interpolation/README.md @@ -192,3 +192,5 @@ There are also several validation errors that can arise, including: - if the lengths of `x` and `y` are not equal - if `x` or `y` are empty - if the requested point to interpolate at is outside the range of `x` + +For more information on our error classes, please visit [our docs](https://if.greensoftware.foundation/reference/errors diff --git a/src/builtins/multiply/README.md b/src/builtins/multiply/README.md index f0e5d6dad..ae410bba5 100644 --- a/src/builtins/multiply/README.md +++ b/src/builtins/multiply/README.md @@ -106,3 +106,4 @@ Every element in the ``inputs` array must contain: - `duration` - whatever values you passed to `input-parameters` +For more information on our error classes, please visit [our docs](https://if.greensoftware.foundation/reference/errors diff --git a/src/builtins/regex/README.md b/src/builtins/regex/README.md index a0e88c7cc..62f05bca9 100644 --- a/src/builtins/regex/README.md +++ b/src/builtins/regex/README.md @@ -119,3 +119,5 @@ You can fix this error by checking you are providing valid values for each param ### `RegexMismatchError` This error arises when the requested regex cannot find any matches in the given data. If there are multiple matches, the plugin returns the first, but if there are none, it throws this error. + +For more information on our error classes, please visit [our docs](https://if.greensoftware.foundation/reference/errors diff --git a/src/builtins/sci-embodied/README.md b/src/builtins/sci-embodied/README.md index 3fd169211..97b6ce4af 100644 --- a/src/builtins/sci-embodied/README.md +++ b/src/builtins/sci-embodied/README.md @@ -119,3 +119,6 @@ The results will be saved to a new `yaml` file in `./examples/outputs`. This error class is used to describe a problem with one of the input values to `sci-embodied`. This is typically due to an incorrect type or a reference to a value that is not available. You will receive a specific error message explaining which parameter is problematic, and you can check and replace where appropriate. + + +For more information on our error classes, please visit [our docs](https://if.greensoftware.foundation/reference/errors diff --git a/src/builtins/sci/README.md b/src/builtins/sci/README.md index 21dca0e19..d9a117d60 100644 --- a/src/builtins/sci/README.md +++ b/src/builtins/sci/README.md @@ -106,3 +106,5 @@ Every element in the `inputs` array must contain: ### Validation errors There is also a validation step that checks that the `functional-unit` was provided in the plugin config. If you see an error reporting this value as missing, please check you have provided it. + +For more information on our error classes, please visit [our docs](https://if.greensoftware.foundation/reference/errors diff --git a/src/builtins/shell/README.md b/src/builtins/shell/README.md index 61987958e..bacba1255 100644 --- a/src/builtins/shell/README.md +++ b/src/builtins/shell/README.md @@ -137,3 +137,5 @@ The results will be saved to a new `yaml` file. ### `ProcessExecutionError` This error is thrown when the program invoked in the spawned shell fails for some reason that is not known to IF. Since the shell executes arbitrary code, it is difficult for IF to provide specific errors - this is delegated to the developers of the executed program. + +For more information on our error classes, please visit [our docs](https://if.greensoftware.foundation/reference/errors diff --git a/src/builtins/subtract/README.md b/src/builtins/subtract/README.md index 5869d93a7..8ff9a922f 100644 --- a/src/builtins/subtract/README.md +++ b/src/builtins/subtract/README.md @@ -101,3 +101,5 @@ The results will be saved to a new `yaml` file in `manifests/outputs`. ### `InputValidationError` This error arises when an invalid value is passed to `Subtract`. Typically, this can occur when a non-numeric value (such as a string made of alphabetic characters) is passed where a number or numeric string is expected. Please check that the types are correct for all the relevant fields in your `inputs` array. + +For more information on our error classes, please visit [our docs](https://if.greensoftware.foundation/reference/errors diff --git a/src/builtins/sum/README.md b/src/builtins/sum/README.md index 5ffc85b37..ea0f0113d 100644 --- a/src/builtins/sum/README.md +++ b/src/builtins/sum/README.md @@ -113,3 +113,6 @@ Every element in the ``inputs` array must contain: - `timestamp` - `duration` - whatever values you passed to `input-parameters` + + +For more information on our error classes, please visit [our docs](https://if.greensoftware.foundation/reference/errors). From 041098c8d20f42b316c23e85062e1fd9e651d59e Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 20 Jun 2024 19:13:00 +0400 Subject: [PATCH 154/243] feat(src): move some function to the corresponding files --- src/env.ts | 94 +++++------------------------------------------------- 1 file changed, 8 insertions(+), 86 deletions(-) diff --git a/src/env.ts b/src/env.ts index 759ae0be5..e1fd42a57 100644 --- a/src/env.ts +++ b/src/env.ts @@ -3,7 +3,13 @@ import * as fs from 'fs/promises'; import * as path from 'path'; -import {installDependencies, initPackageJsonIfNotExists} from './util/helpers'; +import { + installDependencies, + initPackageJsonIfNotExists, + updatePackageJsonDependencies, + extractPathsWithVersion, + updatePackageJsonProperties, +} from './util/npm'; import {parseIfEnvArgs} from './util/args'; import {logger} from './util/logger'; @@ -11,13 +17,7 @@ import {load} from './lib/load'; import {CONFIG} from './config'; -import { - EnvironmentOptions, - ManifestPlugin, - PathWithVersion, -} from './types/if-env'; - -const packageJson = require('../package.json'); +import {EnvironmentOptions} from './types/if-env'; const {IF_ENV} = CONFIG; const { @@ -81,36 +81,6 @@ const getOptionsFromArgs = async (commandArgs: { }; }; -/** - * Gets depencecies with versions. - */ -const extractPathsWithVersion = ( - plugins: ManifestPlugin, - dependencies: string[] -) => { - const paths = Object.keys(plugins).map(plugin => plugins[plugin].path); - const uniquePaths = [...new Set(paths)].filter(path => path !== 'builtin'); - const pathsWithVersion: PathWithVersion = {}; - - uniquePaths.forEach(pluginPath => { - const dependency = dependencies.find((dependency: string) => - dependency.startsWith(pluginPath) - ); - - if (dependency) { - const splittedDependency = dependency.split('@'); - const version = - splittedDependency.length > 2 - ? splittedDependency[2].split(' ')[0] - : splittedDependency[1]; - - pathsWithVersion[pluginPath] = `^${version}`; - } - }); - - return pathsWithVersion; -}; - /** * Creates folder if not exists, installs dependencies if required, update depenedencies. */ @@ -132,54 +102,6 @@ const initializeAndInstallLibs = async (options: EnvironmentOptions) => { } }; -/** - * Update the package.json properties. - */ -const updatePackageJsonProperties = async (packageJsonPath: string) => { - const packageJsonContent = await fs.readFile(packageJsonPath, 'utf8'); - const parsedPackageJsonContent = JSON.parse(packageJsonContent); - - const properties = { - name: 'if-environment', - description: packageJson.description, - author: packageJson.author, - bugs: packageJson.bugs, - engines: packageJson.engines, - homepage: packageJson.homepage, - }; - - const newPackageJson = Object.assign( - {}, - parsedPackageJsonContent, - properties - ); - - await fs.writeFile(packageJsonPath, JSON.stringify(newPackageJson, null, 2)); -}; - -/** - * Updates package.json dependencies. - */ -const updatePackageJsonDependencies = async ( - packageJsonPath: string, - dependencies: PathWithVersion, - cwd: boolean -) => { - const packageJsonContent = await fs.readFile(packageJsonPath, 'utf8'); - const packageJson = JSON.parse(packageJsonContent); - - if (cwd) { - packageJson.dependencies = { - ...packageJson.dependencies, - ...dependencies, - }; - } else { - packageJson.dependencies = {...dependencies}; - } - - await fs.writeFile(packageJsonPath, JSON.stringify(packageJson, null, 2)); -}; - /** * Adds a manifest template to the folder where the if-env CLI command runs. */ From e361448f5f62be3295e2fa52ea524ca8ce0f1ae8 Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Fri, 21 Jun 2024 09:38:00 +0100 Subject: [PATCH 155/243] feat(lib): add unit test to cover keep-existing==false --- .../unit/builtins/copy-param.test.ts | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/__tests__/unit/builtins/copy-param.test.ts b/src/__tests__/unit/builtins/copy-param.test.ts index c3fcd9639..cf5166107 100644 --- a/src/__tests__/unit/builtins/copy-param.test.ts +++ b/src/__tests__/unit/builtins/copy-param.test.ts @@ -90,6 +90,33 @@ describe('builtins/copy: ', () => { ); } }); + it('does not persist the original value when keep-existing==false.', () => { + expect.assertions(1); + const globalConfig = { + 'keep-existing': false, + from: 'original', + to: 'copy', + }; + const copy = Copy(globalConfig); + + const expectedResult = [ + { + duration: 3600, + copy: 'hello', + timestamp: '2021-01-01T00:00:00Z', + }, + ]; + + const result = copy.execute([ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + original: 'hello', + }, + ]); + + expect(result).toStrictEqual(expectedResult); + }); }); }); }); From cf78e06f3ded72786331628b34aaf3efdb4d5bc7 Mon Sep 17 00:00:00 2001 From: manushak Date: Fri, 21 Jun 2024 16:51:34 +0400 Subject: [PATCH 156/243] fix(util): add boolean argument to check if need to update package dependencies --- src/util/npm.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/util/npm.ts b/src/util/npm.ts index bfdf82ca6..0cbfe27e1 100644 --- a/src/util/npm.ts +++ b/src/util/npm.ts @@ -107,7 +107,8 @@ export const extractPathsWithVersion = ( * Update the package.json properties. */ export const updatePackageJsonProperties = async ( - newPackageJsonPath: string + newPackageJsonPath: string, + resetDependencies: boolean ) => { const packageJsonContent = await fs.readFile(newPackageJsonPath, 'utf8'); const parsedPackageJsonContent = JSON.parse(packageJsonContent); @@ -119,7 +120,7 @@ export const updatePackageJsonProperties = async ( bugs: packageJson.bugs, engines: packageJson.engines, homepage: packageJson.homepage, - dependencies: {}, + dependencies: resetDependencies ? {} : packageJson.dependencies, }; const newPackageJson = Object.assign( From 1545fcdc92facbf885cb694d5fdd32b1400098ae Mon Sep 17 00:00:00 2001 From: manushak Date: Fri, 21 Jun 2024 16:52:53 +0400 Subject: [PATCH 157/243] test(util): update the test according to changes --- src/__tests__/unit/util/npm.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/__tests__/unit/util/npm.test.ts b/src/__tests__/unit/util/npm.test.ts index 5c904aa23..9680bf722 100644 --- a/src/__tests__/unit/util/npm.test.ts +++ b/src/__tests__/unit/util/npm.test.ts @@ -221,7 +221,7 @@ describe('util/npm: ', () => { describe('updatePackageJsonProperties(): ', () => { it('updates the package.json properties correctly.', async () => { const newPackageJsonPath = path.resolve(folderPath, '/package.json-npm'); - await updatePackageJsonProperties(newPackageJsonPath); + await updatePackageJsonProperties(newPackageJsonPath, false); expect.assertions(2); expect(fs.readFile).toHaveBeenCalledWith(newPackageJsonPath, 'utf8'); From 8e78685f2d44e83b20e5476631c39116da4dcdfc Mon Sep 17 00:00:00 2001 From: manushak Date: Fri, 21 Jun 2024 16:54:45 +0400 Subject: [PATCH 158/243] fix(src): update package depenedencies depend on cwd flag --- src/env.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/env.ts b/src/env.ts index e1fd42a57..1775c3fd3 100644 --- a/src/env.ts +++ b/src/env.ts @@ -89,7 +89,7 @@ const initializeAndInstallLibs = async (options: EnvironmentOptions) => { const {folderPath, install, cwd, dependencies} = options; const packageJsonPath = await initPackageJsonIfNotExists(folderPath); - await updatePackageJsonProperties(packageJsonPath); + await updatePackageJsonProperties(packageJsonPath, !cwd); if (install) { await installDependencies(folderPath, dependencies); From 9a816dddba19af4cb37ec886701a849d0bd00e8a Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Sun, 23 Jun 2024 20:00:51 +0400 Subject: [PATCH 159/243] fix(config): drop unused non numeric value message --- src/config/strings.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/config/strings.ts b/src/config/strings.ts index 82e6d388b..ba6e8a918 100644 --- a/src/config/strings.ts +++ b/src/config/strings.ts @@ -56,8 +56,7 @@ Note that for the '--output' option you also need to define the output type in y /** Plugin messages */ MISSING_GLOBAL_CONFIG: 'Global config is not provided.', MISSING_INPUT_DATA: (param: string) => - `${param} is missing from the input array.`, - NOT_NUMERIC_VALUE: (str: any) => `${str} is not numberic.`, + `${param} is missing from the input array, or has nullish value.`, MISSING_FUNCTIONAL_UNIT_CONFIG: '`functional-unit` should be provided in your global config', MISSING_FUNCTIONAL_UNIT_INPUT: From 33ffea161f73ef5ee30514d1fb7b07767b3465ce Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Sun, 23 Jun 2024 20:07:32 +0400 Subject: [PATCH 160/243] fix(builtins): tune sum plugin validation --- src/builtins/sum/index.ts | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/builtins/sum/index.ts b/src/builtins/sum/index.ts index ecce47cf3..57437ba31 100644 --- a/src/builtins/sum/index.ts +++ b/src/builtins/sum/index.ts @@ -6,8 +6,8 @@ import {validate} from '../../util/validations'; import {STRINGS} from '../../config'; -const {GlobalConfigError, MissingInputDataError} = ERRORS; -const {MISSING_INPUT_DATA, MISSING_GLOBAL_CONFIG} = STRINGS; +const {GlobalConfigError} = ERRORS; +const {MISSING_GLOBAL_CONFIG} = STRINGS; export const Sum = (globalConfig: SumConfig): ExecutePlugin => { const metadata = { @@ -23,11 +23,11 @@ export const Sum = (globalConfig: SumConfig): ExecutePlugin => { const outputParameter = safeGlobalConfig['output-parameter']; return inputs.map(input => { - const safeInput = validateSingleInput(input, inputParameters); + validateSingleInput(input, inputParameters); return { ...input, - [outputParameter]: calculateSum(safeInput, inputParameters), + [outputParameter]: calculateSum(input, inputParameters), }; }); }; @@ -58,11 +58,16 @@ export const Sum = (globalConfig: SumConfig): ExecutePlugin => { input: PluginParams, inputParameters: string[] ) => { - inputParameters.forEach(metricToSum => { - if (!input[metricToSum]) { - throw new MissingInputDataError(MISSING_INPUT_DATA(metricToSum)); - } - }); + const inputData = inputParameters.reduce( + (acc, param) => { + acc[param] = input[param]; + + return acc; + }, + {} as Record + ); + const validationSchema = z.record(z.string(), z.number()); + validate(validationSchema, inputData); return input; }; From fe8f0fbe88a504c2998695f5b8b8311a0591db8c Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Sun, 23 Jun 2024 20:15:16 +0400 Subject: [PATCH 161/243] fix(builtins): tune subtract plugin validation --- src/builtins/subtract/index.ts | 32 +++++++++++--------------------- 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/src/builtins/subtract/index.ts b/src/builtins/subtract/index.ts index 20a61b5b6..797b259c1 100644 --- a/src/builtins/subtract/index.ts +++ b/src/builtins/subtract/index.ts @@ -1,5 +1,4 @@ import {z} from 'zod'; -import {ERRORS} from '@grnsft/if-core/utils'; import { ExecutePlugin, PluginParams, @@ -8,11 +7,6 @@ import { import {validate} from '../../util/validations'; -import {STRINGS} from '../../config'; - -const {InputValidationError} = ERRORS; -const {MISSING_INPUT_DATA, NOT_NUMERIC_VALUE} = STRINGS; - export const Subtract = (globalConfig: SubtractConfig): ExecutePlugin => { const metadata = { kind: 'execute', @@ -40,24 +34,20 @@ export const Subtract = (globalConfig: SubtractConfig): ExecutePlugin => { input: PluginParams, inputParameters: string[] ) => { - inputParameters.forEach(metricToSubtract => { - validateParamExists(input, metricToSubtract); - validateNumericString(input[metricToSubtract]); - }); + const inputData = inputParameters.reduce( + (acc, param) => { + acc[param] = input[param]; - return input; - }; + return acc; + }, + {} as Record + ); - const validateParamExists = (input: PluginParams, param: string) => { - if (input[param] === undefined) { - throw new InputValidationError(MISSING_INPUT_DATA(param)); - } - }; + const validationSchema = z.record(z.string(), z.number()); - const validateNumericString = (str: string) => { - if (isNaN(+Number(str))) { - throw new InputValidationError(NOT_NUMERIC_VALUE(str)); - } + validate(validationSchema, inputData); + + return input; }; /** From 3a0cc9c1c3b7174236839d85ab5aeb51573c974f Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Sun, 23 Jun 2024 20:16:06 +0400 Subject: [PATCH 162/243] fix(builtins): simplify execute in sci embodied --- src/builtins/sci-embodied/index.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/builtins/sci-embodied/index.ts b/src/builtins/sci-embodied/index.ts index 678d41336..a88017e9f 100644 --- a/src/builtins/sci-embodied/index.ts +++ b/src/builtins/sci-embodied/index.ts @@ -24,8 +24,8 @@ export const SciEmbodied = (): ExecutePlugin => { /** * Calculate the Embodied carbon for a list of inputs. */ - const execute = (inputs: PluginParams[]) => { - return inputs.map(input => { + const execute = (inputs: PluginParams[]) => + inputs.map(input => { const safeInput = validateInput(input); return { @@ -33,7 +33,6 @@ export const SciEmbodied = (): ExecutePlugin => { 'carbon-embodied': calculateEmbodiedCarbon(safeInput), }; }); - }; /** * Calculate the Embodied carbon for the input. From edc02c9ef853434852f75443a8a4edd776408f8c Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Sun, 23 Jun 2024 20:20:42 +0400 Subject: [PATCH 163/243] feat(config): add zero division message to strings --- src/config/strings.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/config/strings.ts b/src/config/strings.ts index ba6e8a918..b89822794 100644 --- a/src/config/strings.ts +++ b/src/config/strings.ts @@ -128,4 +128,6 @@ ${error}`, `Exporting to csv file: ${savepath}`, EXPORTING_RAW_CSV_FILE: (savepath: string) => `Exporting raw csv file: ${savepath}`, + ZERO_DIVISION: (moduleName: string, index: number) => + `-- SKIPPING -- DivisionByZero: you are attempting to divide by zero in ${moduleName} plugin : inputs[${index}]\n`, }; From 402bfbe66b67aabc593d434e5d4bfa3b88ad869a Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Sun, 23 Jun 2024 20:22:38 +0400 Subject: [PATCH 164/243] feat(builtins): tune zero devision in sci --- src/builtins/sci/index.ts | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/src/builtins/sci/index.ts b/src/builtins/sci/index.ts index 223493f3a..f119f667a 100644 --- a/src/builtins/sci/index.ts +++ b/src/builtins/sci/index.ts @@ -7,7 +7,11 @@ import {validate, allDefined} from '../../util/validations'; import {STRINGS} from '../../config'; const {MissingInputDataError} = ERRORS; -const {MISSING_FUNCTIONAL_UNIT_CONFIG, MISSING_FUNCTIONAL_UNIT_INPUT} = STRINGS; +const { + MISSING_FUNCTIONAL_UNIT_CONFIG, + MISSING_FUNCTIONAL_UNIT_INPUT, + ZERO_DIVISION, +} = STRINGS; export const Sci = (globalConfig: ConfigParams): ExecutePlugin => { const metadata = { @@ -33,12 +37,22 @@ export const Sci = (globalConfig: ConfigParams): ExecutePlugin => { * Calculate the total emissions for a list of inputs. */ const execute = (inputs: PluginParams[]): PluginParams[] => - inputs.map(input => { + inputs.map((input, index) => { const safeInput = validateInput(input); - const sci = - safeInput['carbon'] > 0 - ? safeInput['carbon'] / input[globalConfig['functional-unit']] - : 0; + const functionalUnit = input[globalConfig['functional-unit']]; + + let sci: any = {}; + + if (safeInput['carbon'] > 0) { + if (functionalUnit === 0) { + console.warn(ZERO_DIVISION(Sci.name, index)); + sci = safeInput['carbon']; + } else { + sci = safeInput['carbon'] / functionalUnit; + } + } else { + sci = 0; + } return { ...input, @@ -57,7 +71,7 @@ export const Sci = (globalConfig: ConfigParams): ExecutePlugin => { if ( !( validatedConfig['functional-unit'] in input && - input[validatedConfig['functional-unit']] > 0 + input[validatedConfig['functional-unit']] >= 0 ) ) { throw new MissingInputDataError(MISSING_FUNCTIONAL_UNIT_INPUT); From 0bffd851a6410413e782197d143bba526f45e42a Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Sun, 23 Jun 2024 20:23:09 +0400 Subject: [PATCH 165/243] feat(builtins): tune validation in multiply --- src/builtins/multiply/index.ts | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/src/builtins/multiply/index.ts b/src/builtins/multiply/index.ts index 717c68507..cd2a74837 100644 --- a/src/builtins/multiply/index.ts +++ b/src/builtins/multiply/index.ts @@ -1,5 +1,4 @@ import {z} from 'zod'; -import {ERRORS} from '@grnsft/if-core/utils'; import { ExecutePlugin, PluginParams, @@ -8,11 +7,6 @@ import { import {validate} from '../../util/validations'; -import {STRINGS} from '../../config'; - -const {MissingInputDataError} = ERRORS; -const {MISSING_INPUT_DATA} = STRINGS; - export const Multiply = (globalConfig: MultiplyConfig): ExecutePlugin => { const metadata = { kind: 'execute', @@ -40,14 +34,18 @@ export const Multiply = (globalConfig: MultiplyConfig): ExecutePlugin => { input: PluginParams, inputParameters: string[] ) => { - inputParameters.forEach(metricToMultiply => { - if ( - input[metricToMultiply] === undefined || - isNaN(input[metricToMultiply]) - ) { - throw new MissingInputDataError(MISSING_INPUT_DATA(metricToMultiply)); - } - }); + const inputData = inputParameters.reduce( + (acc, param) => { + acc[param] = input[param]; + + return acc; + }, + {} as Record + ); + + const validationSchema = z.record(z.string(), z.number()); + + validate(validationSchema, inputData); return input; }; @@ -61,17 +59,17 @@ export const Multiply = (globalConfig: MultiplyConfig): ExecutePlugin => { const outputParameter = safeGlobalConfig['output-parameter']; return inputs.map(input => { - const safeInput = validateSingleInput(input, inputParameters); + validateSingleInput(input, inputParameters); return { ...input, - [outputParameter]: calculateProduct(safeInput, inputParameters), + [outputParameter]: calculateProduct(input, inputParameters), }; }); }; /** - * Calculates the product of the energy components. + * Calculates the product of the components. */ const calculateProduct = (input: PluginParams, inputParameters: string[]) => inputParameters.reduce( From 887d91210ce9298c66c72f27b26529ca71a0116c Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Sun, 23 Jun 2024 20:23:42 +0400 Subject: [PATCH 166/243] feat(builtins): tune validation in exponent --- src/builtins/exponent/index.ts | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/src/builtins/exponent/index.ts b/src/builtins/exponent/index.ts index 90d66cd72..493993fc9 100644 --- a/src/builtins/exponent/index.ts +++ b/src/builtins/exponent/index.ts @@ -1,5 +1,4 @@ import {z} from 'zod'; -import {ERRORS} from '@grnsft/if-core/utils'; import { ExecutePlugin, PluginParams, @@ -8,11 +7,6 @@ import { import {validate} from '../../util/validations'; -import {STRINGS} from '../../config'; - -const {MissingInputDataError, InputValidationError} = ERRORS; -const {MISSING_INPUT_DATA, NOT_NUMERIC_VALUE} = STRINGS; - export const Exponent = (globalConfig: ExponentConfig): ExecutePlugin => { const metadata = { kind: 'execute', @@ -38,20 +32,13 @@ export const Exponent = (globalConfig: ExponentConfig): ExecutePlugin => { * Checks for required fields in input. */ const validateSingleInput = (input: PluginParams, inputParameter: string) => { - validateParamExists(input, inputParameter); - validateNumericString(input[inputParameter]); - }; + const inputData = { + 'input-parameter': input[inputParameter], + }; + const validationSchema = z.record(z.string(), z.number()); + validate(validationSchema, inputData); - const validateParamExists = (input: PluginParams, param: string) => { - if (input[param] === undefined) { - throw new MissingInputDataError(MISSING_INPUT_DATA(param)); - } - }; - - const validateNumericString = (str: string) => { - if (isNaN(+Number(str))) { - throw new InputValidationError(NOT_NUMERIC_VALUE(str)); - } + return input; }; /** @@ -60,9 +47,10 @@ export const Exponent = (globalConfig: ExponentConfig): ExecutePlugin => { const execute = (inputs: PluginParams[]): PluginParams[] => { const { 'input-parameter': inputParameter, - exponent: exponent, + exponent, 'output-parameter': outputParameter, } = validateGlobalConfig(); + return inputs.map(input => { validateSingleInput(input, inputParameter); From a15c28edc3eeaf9fcc0c23d6cd8ae3afaaac4502 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Sun, 23 Jun 2024 20:24:48 +0400 Subject: [PATCH 167/243] feat(builtins): tune validation and zero devision in divide --- src/builtins/divide/index.ts | 38 ++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/src/builtins/divide/index.ts b/src/builtins/divide/index.ts index a38e3601b..453a50921 100644 --- a/src/builtins/divide/index.ts +++ b/src/builtins/divide/index.ts @@ -7,7 +7,7 @@ import {validate} from '../../util/validations'; import {STRINGS} from '../../config'; const {GlobalConfigError, MissingInputDataError} = ERRORS; -const {MISSING_GLOBAL_CONFIG, MISSING_INPUT_DATA} = STRINGS; +const {MISSING_GLOBAL_CONFIG, MISSING_INPUT_DATA, ZERO_DIVISION} = STRINGS; export const Divide = (globalConfig: ConfigParams): ExecutePlugin => { const metadata = { @@ -21,16 +21,16 @@ export const Divide = (globalConfig: ConfigParams): ExecutePlugin => { const safeGlobalConfig = validateGlobalConfig(); const {numerator, denominator, output} = safeGlobalConfig; - return inputs.map(input => { + return inputs.map((input, index) => { const safeInput = Object.assign( {}, input, - validateSingleInput(input, numerator, denominator) + validateSingleInput(input, {numerator, denominator}) ); return { ...input, - [output]: calculateDivide(safeInput, numerator, denominator), + [output]: calculateDivide(safeInput, index, {numerator, denominator}), }; }); }; @@ -45,7 +45,7 @@ export const Divide = (globalConfig: ConfigParams): ExecutePlugin => { const schema = z.object({ numerator: z.string().min(1), - denominator: z.string().or(z.number().gt(0)), + denominator: z.string().or(z.number()), output: z.string(), }); @@ -57,9 +57,13 @@ export const Divide = (globalConfig: ConfigParams): ExecutePlugin => { */ const validateSingleInput = ( input: PluginParams, - numerator: string, - denominator: number | string + params: { + numerator: string; + denominator: number | string; + } ) => { + const {numerator, denominator} = params; + const schema = z .object({ [numerator]: z.number(), @@ -69,6 +73,7 @@ export const Divide = (globalConfig: ConfigParams): ExecutePlugin => { if (typeof denominator === 'string' && !input[denominator]) { throw new MissingInputDataError(MISSING_INPUT_DATA(denominator)); } + return true; }); @@ -80,9 +85,22 @@ export const Divide = (globalConfig: ConfigParams): ExecutePlugin => { */ const calculateDivide = ( input: PluginParams, - numerator: string, - denominator: number | string - ) => input[numerator] / (input[denominator] || denominator); + index: number, + params: { + numerator: string; + denominator: number | string; + } + ) => { + const {denominator, numerator} = params; + const finalDenominator = input[denominator] || denominator; + + if (finalDenominator === 0) { + console.warn(ZERO_DIVISION(Divide.name, index)); + return input[numerator]; + } + + return input[numerator] / finalDenominator; + }; return { metadata, From 11e832327f70046235124c6b6f96169ea89c053c Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Sun, 23 Jun 2024 20:27:38 +0400 Subject: [PATCH 168/243] feat(builtins): tune validation in coefficient --- src/builtins/coefficient/index.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/builtins/coefficient/index.ts b/src/builtins/coefficient/index.ts index aed444ad3..2d4f6a1c9 100644 --- a/src/builtins/coefficient/index.ts +++ b/src/builtins/coefficient/index.ts @@ -28,6 +28,8 @@ export const Coefficient = (globalConfig: CoefficientConfig): ExecutePlugin => { const coefficient = safeGlobalConfig['coefficient']; return inputs.map(input => { + validateSingleInput(input, inputParameter); + return { ...input, [outputParameter]: calculateProduct(input, inputParameter, coefficient), @@ -35,6 +37,19 @@ export const Coefficient = (globalConfig: CoefficientConfig): ExecutePlugin => { }); }; + /** + * Checks for required fields in input. + */ + const validateSingleInput = (input: PluginParams, inputParameter: string) => { + const inputData = { + 'input-parameter': input[inputParameter], + }; + const validationSchema = z.record(z.string(), z.number()); + validate(validationSchema, inputData); + + return input; + }; + /** * Calculates the product of the energy components. */ From 9df90fbd245e2fb0a7f059919f0da237733a76f5 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Sun, 23 Jun 2024 20:30:10 +0400 Subject: [PATCH 169/243] test(builtins): fix units after refactor --- src/__tests__/unit/builtins/divide.test.ts | 30 +++++++++++--------- src/__tests__/unit/builtins/exponent.test.ts | 13 +++++---- src/__tests__/unit/builtins/multiply.test.ts | 11 ++++--- src/__tests__/unit/builtins/subtract.test.ts | 7 ++--- src/__tests__/unit/builtins/sum.test.ts | 8 ++++-- 5 files changed, 36 insertions(+), 33 deletions(-) diff --git a/src/__tests__/unit/builtins/divide.test.ts b/src/__tests__/unit/builtins/divide.test.ts index 79fc6c0b5..ba75e3879 100644 --- a/src/__tests__/unit/builtins/divide.test.ts +++ b/src/__tests__/unit/builtins/divide.test.ts @@ -126,9 +126,6 @@ describe('builtins/divide: ', () => { }); it('throws an error when `denominator` is 0.', async () => { - const expectedMessage = - '"denominator" parameter is number must be greater than 0. Error code: too_small.'; - const globalConfig = { numerator: 'vcpus-allocated', denominator: 0, @@ -138,17 +135,22 @@ describe('builtins/divide: ', () => { expect.assertions(1); - try { - await divide.execute([ - { - timestamp: '2021-01-01T00:00:00Z', - duration: 3600, - 'vcpus-allocated': 24, - }, - ]); - } catch (error) { - expect(error).toStrictEqual(new InputValidationError(expectedMessage)); - } + const response = await divide.execute([ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + 'vcpus-allocated': 24, + }, + ]); + + expect(response).toEqual([ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + 'vcpus-allocated': 24, + 'vcpus-allocated-per-second': 24, + }, + ]); }); it('throws an error when `denominator` is string.', async () => { diff --git a/src/__tests__/unit/builtins/exponent.test.ts b/src/__tests__/unit/builtins/exponent.test.ts index c139a5608..ad438179d 100644 --- a/src/__tests__/unit/builtins/exponent.test.ts +++ b/src/__tests__/unit/builtins/exponent.test.ts @@ -2,10 +2,7 @@ import {ERRORS} from '@grnsft/if-core/utils'; import {Exponent} from '../../../builtins/exponent'; -import {STRINGS} from '../../../config'; - -const {InputValidationError, MissingInputDataError} = ERRORS; -const {NOT_NUMERIC_VALUE, MISSING_INPUT_DATA} = STRINGS; +const {InputValidationError} = ERRORS; describe('builtins/exponent: ', () => { describe('Exponent: ', () => { @@ -59,7 +56,9 @@ describe('builtins/exponent: ', () => { ]); } catch (error) { expect(error).toStrictEqual( - new MissingInputDataError(MISSING_INPUT_DATA('energy/base')) + new InputValidationError( + '"input-parameter" parameter is required. Error code: invalid_type.' + ) ); } }); @@ -78,7 +77,9 @@ describe('builtins/exponent: ', () => { await exponent.execute(input); } catch (error) { expect(error).toStrictEqual( - new InputValidationError(NOT_NUMERIC_VALUE(input[0]['energy/base'])) + new InputValidationError( + '"input-parameter" parameter is expected number, received string. Error code: invalid_type.' + ) ); } }); diff --git a/src/__tests__/unit/builtins/multiply.test.ts b/src/__tests__/unit/builtins/multiply.test.ts index f191ae169..d15a7e8f9 100644 --- a/src/__tests__/unit/builtins/multiply.test.ts +++ b/src/__tests__/unit/builtins/multiply.test.ts @@ -2,10 +2,7 @@ import {ERRORS} from '@grnsft/if-core/utils'; import {Multiply} from '../../../builtins/multiply'; -import {STRINGS} from '../../../config'; - -const {MissingInputDataError} = ERRORS; -const {MISSING_INPUT_DATA} = STRINGS; +const {InputValidationError} = ERRORS; describe('builtins/multiply: ', () => { describe('Multiply: ', () => { @@ -62,7 +59,9 @@ describe('builtins/multiply: ', () => { ]); } catch (error) { expect(error).toStrictEqual( - new MissingInputDataError(MISSING_INPUT_DATA('cpu/energy')) + new InputValidationError( + '"cpu/energy" parameter is required. Error code: invalid_type.,"network/energy" parameter is required. Error code: invalid_type.,"memory/energy" parameter is required. Error code: invalid_type.' + ) ); } }); @@ -88,10 +87,10 @@ describe('builtins/multiply: ', () => { const expectedResult = [ { duration: 3600, + timestamp: '2021-01-01T00:00:00Z', carbon: 3, 'other-carbon': 2, 'carbon-product': 6, - timestamp: '2021-01-01T00:00:00Z', }, ]; diff --git a/src/__tests__/unit/builtins/subtract.test.ts b/src/__tests__/unit/builtins/subtract.test.ts index b84b05db8..fd4cf7c96 100644 --- a/src/__tests__/unit/builtins/subtract.test.ts +++ b/src/__tests__/unit/builtins/subtract.test.ts @@ -2,10 +2,7 @@ import {ERRORS} from '@grnsft/if-core/utils'; import {Subtract} from '../../../builtins/subtract'; -import {STRINGS} from '../../../config'; - const {InputValidationError} = ERRORS; -const {MISSING_INPUT_DATA} = STRINGS; describe('builtins/subtract: ', () => { describe('Subtract: ', () => { @@ -62,7 +59,9 @@ describe('builtins/subtract: ', () => { ]); } catch (error) { expect(error).toStrictEqual( - new InputValidationError(MISSING_INPUT_DATA('cpu/energy')) + new InputValidationError( + '"cpu/energy" parameter is required. Error code: invalid_type.,"network/energy" parameter is required. Error code: invalid_type.,"memory/energy" parameter is required. Error code: invalid_type.' + ) ); } }); diff --git a/src/__tests__/unit/builtins/sum.test.ts b/src/__tests__/unit/builtins/sum.test.ts index dbd05a783..1810fc13a 100644 --- a/src/__tests__/unit/builtins/sum.test.ts +++ b/src/__tests__/unit/builtins/sum.test.ts @@ -4,8 +4,8 @@ import {Sum} from '../../../builtins/sum'; import {STRINGS} from '../../../config'; -const {GlobalConfigError, MissingInputDataError} = ERRORS; -const {MISSING_GLOBAL_CONFIG, MISSING_INPUT_DATA} = STRINGS; +const {GlobalConfigError, InputValidationError} = ERRORS; +const {MISSING_GLOBAL_CONFIG} = STRINGS; describe('builtins/sum: ', () => { describe('Sum: ', () => { @@ -85,7 +85,9 @@ describe('builtins/sum: ', () => { ]); } catch (error) { expect(error).toStrictEqual( - new MissingInputDataError(MISSING_INPUT_DATA('cpu/energy')) + new InputValidationError( + '"cpu/energy" parameter is required. Error code: invalid_type.,"network/energy" parameter is required. Error code: invalid_type.,"memory/energy" parameter is required. Error code: invalid_type.' + ) ); } }); From 8c090cc7233209e83f1d75a01af1694deda5ff04 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Sun, 23 Jun 2024 20:49:15 +0400 Subject: [PATCH 170/243] chore(config): add coverage ignore patterns --- jest.config.js | 1 + 1 file changed, 1 insertion(+) diff --git a/jest.config.js b/jest.config.js index 2e770bcad..016be5863 100644 --- a/jest.config.js +++ b/jest.config.js @@ -15,4 +15,5 @@ module.exports = { './src/__tests__/integration/helpers', './src/__tests__/integration/test-data', ], + coveragePathIgnorePatterns: ['src/config', 'src/types'], }; From 6f3089d1623bba9a8ce41d06d4f82a8a3825e4c1 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 24 Jun 2024 00:15:21 +0400 Subject: [PATCH 171/243] fix(builtins): sci logic --- src/builtins/sci/index.ts | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/builtins/sci/index.ts b/src/builtins/sci/index.ts index f119f667a..3bc4fedf0 100644 --- a/src/builtins/sci/index.ts +++ b/src/builtins/sci/index.ts @@ -41,22 +41,18 @@ export const Sci = (globalConfig: ConfigParams): ExecutePlugin => { const safeInput = validateInput(input); const functionalUnit = input[globalConfig['functional-unit']]; - let sci: any = {}; - - if (safeInput['carbon'] > 0) { - if (functionalUnit === 0) { - console.warn(ZERO_DIVISION(Sci.name, index)); - sci = safeInput['carbon']; - } else { - sci = safeInput['carbon'] / functionalUnit; - } - } else { - sci = 0; + if (functionalUnit === 0) { + console.warn(ZERO_DIVISION(Sci.name, index)); + + return { + ...input, + sci: safeInput['carbon'], + }; } return { ...input, - sci, + sci: safeInput['carbon'] / functionalUnit, }; }); From 317119c5f05c063ab685a745962e57ad4167cbd7 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 24 Jun 2024 00:15:54 +0400 Subject: [PATCH 172/243] test(builtins): add zero case to sci --- src/__tests__/unit/builtins/sci.test.ts | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/__tests__/unit/builtins/sci.test.ts b/src/__tests__/unit/builtins/sci.test.ts index c12085d4e..6a8ddc8db 100644 --- a/src/__tests__/unit/builtins/sci.test.ts +++ b/src/__tests__/unit/builtins/sci.test.ts @@ -140,5 +140,26 @@ describe('builtins/sci:', () => { } }); }); + + it('fallbacks to carbon value, if functional unit is 0.', async () => { + const sci = Sci({ + 'functional-unit': 'requests', + }); + const inputs = [ + { + timestamp: '2021-01-01T00:00:00Z', + 'carbon-operational': 0.2, + 'carbon-embodied': 0.05, + carbon: 0.205, + duration: 1, + requests: 0, + }, + ]; + const result = await sci.execute(inputs); + + expect.assertions(1); + + expect(result).toStrictEqual([{...inputs[0], sci: inputs[0].carbon}]); + }); }); }); From 791b38f03a28f71f2c78fdf6dd8d26996a668f51 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 24 Jun 2024 00:40:57 +0400 Subject: [PATCH 173/243] fix(builtins): exponent limitation --- src/builtins/exponent/index.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/builtins/exponent/index.ts b/src/builtins/exponent/index.ts index 493993fc9..ce4e959ef 100644 --- a/src/builtins/exponent/index.ts +++ b/src/builtins/exponent/index.ts @@ -18,7 +18,7 @@ export const Exponent = (globalConfig: ExponentConfig): ExecutePlugin => { const validateGlobalConfig = () => { const globalConfigSchema = z.object({ 'input-parameter': z.string().min(1), - exponent: z.number().min(1), + exponent: z.number(), 'output-parameter': z.string().min(1), }); @@ -70,6 +70,7 @@ export const Exponent = (globalConfig: ExponentConfig): ExecutePlugin => { exponent: number ) => { const base = input[inputParameter]; + return Math.pow(base, exponent); }; From fbd34a62436a1c3fe8e41ee5759fd731826950b6 Mon Sep 17 00:00:00 2001 From: Joseph Cook <33655003+jmcook1186@users.noreply.github.com> Date: Mon, 24 Jun 2024 09:01:23 +0100 Subject: [PATCH 174/243] Update src/builtins/copy-param/index.ts Signed-off-by: Joseph Cook <33655003+jmcook1186@users.noreply.github.com> --- src/builtins/copy-param/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/builtins/copy-param/index.ts b/src/builtins/copy-param/index.ts index 3f3e1f5ae..4501dccc8 100644 --- a/src/builtins/copy-param/index.ts +++ b/src/builtins/copy-param/index.ts @@ -8,7 +8,6 @@ import {STRINGS} from '../../config'; const {MISSING_GLOBAL_CONFIG, MISSING_INPUT_DATA} = STRINGS; const {GlobalConfigError, MissingInputDataError} = ERRORS; -// global-config: // keep-existing: true/false (whether to remove the parameter you are copying from) // from-param: the parameter you are copying from (e.g. cpu/name) // to-field: the parameter you are copying to (e.g. cpu/processor-name) From 332bc6b35aa37b9fd717d213b45bcc2c2ce448a8 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 24 Jun 2024 12:16:55 +0400 Subject: [PATCH 175/243] fix(builtins): fix typo in csv lookup readme Signed-off-by: Narek Hovhannisyan --- src/builtins/csv-lookup/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/builtins/csv-lookup/README.md b/src/builtins/csv-lookup/README.md index eb62f8c7e..3663f4123 100644 --- a/src/builtins/csv-lookup/README.md +++ b/src/builtins/csv-lookup/README.md @@ -144,7 +144,7 @@ The results will be saved to a new `yaml` file in `manifests/outputs`. ## Errors -`Coefficient exposes six of the IF error classes. +Coefficient exposes six of the IF error classes. ### FetchingFileError From 85c89f25e03809b59c5dde49d89d842107c57da8 Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Mon, 24 Jun 2024 09:35:24 +0100 Subject: [PATCH 176/243] fix(lib): use zod to validate single inputs --- src/builtins/copy-param/index.ts | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/src/builtins/copy-param/index.ts b/src/builtins/copy-param/index.ts index 4501dccc8..6301bbf25 100644 --- a/src/builtins/copy-param/index.ts +++ b/src/builtins/copy-param/index.ts @@ -6,8 +6,8 @@ import {validate} from '../../util/validations'; import {STRINGS} from '../../config'; -const {MISSING_GLOBAL_CONFIG, MISSING_INPUT_DATA} = STRINGS; -const {GlobalConfigError, MissingInputDataError} = ERRORS; +const {MISSING_GLOBAL_CONFIG} = STRINGS; +const {GlobalConfigError} = ERRORS; // keep-existing: true/false (whether to remove the parameter you are copying from) // from-param: the parameter you are copying from (e.g. cpu/name) // to-field: the parameter you are copying to (e.g. cpu/processor-name) @@ -40,10 +40,22 @@ export const Copy = (globalConfig: Record): ExecutePlugin => { /** * Checks for required fields in input. */ - const validateSingleInput = (input: PluginParams, parameter: string) => { - if (!input[parameter]) { - throw new MissingInputDataError(MISSING_INPUT_DATA(parameter)); - } + const validateSingleInput = ( + input: PluginParams, + inputParameters: string[] + ) => { + const inputData = inputParameters.reduce( + (acc, param) => { + acc[param] = input[param]; + + return acc; + }, + {} as Record + ); + + const validationSchema = z.record(z.string(), z.string()); + + validate(validationSchema, inputData); return input; }; @@ -55,7 +67,7 @@ export const Copy = (globalConfig: Record): ExecutePlugin => { const to = safeGlobalConfig['to']; return inputs.map(input => { - const safeInput = validateSingleInput(input, from); + const safeInput = validateSingleInput(input, [from]); const outputValue = safeInput[from]; if (safeInput[from]) { From 46440a8c78fd8162e5bcf31d3276e629a862589f Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Mon, 24 Jun 2024 09:37:59 +0100 Subject: [PATCH 177/243] fix(lib): update unit tests to expect errors to surface from zod --- src/__tests__/unit/builtins/copy-param.test.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/__tests__/unit/builtins/copy-param.test.ts b/src/__tests__/unit/builtins/copy-param.test.ts index cf5166107..64809e44a 100644 --- a/src/__tests__/unit/builtins/copy-param.test.ts +++ b/src/__tests__/unit/builtins/copy-param.test.ts @@ -4,8 +4,8 @@ import {Copy} from '../../../builtins/copy-param'; import {STRINGS} from '../../../config'; -const {GlobalConfigError, MissingInputDataError} = ERRORS; -const {MISSING_GLOBAL_CONFIG, MISSING_INPUT_DATA} = STRINGS; +const {GlobalConfigError, InputValidationError} = ERRORS; +const {MISSING_GLOBAL_CONFIG} = STRINGS; describe('builtins/copy: ', () => { describe('Copy: ', () => { @@ -86,7 +86,9 @@ describe('builtins/copy: ', () => { ]); } catch (error) { expect(error).toStrictEqual( - new MissingInputDataError(MISSING_INPUT_DATA('original')) + new InputValidationError( + '"original" parameter is required. Error code: invalid_type.' + ) ); } }); From 3e6c970277e8e86b7f28bbdf45879f56e772e58d Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 24 Jun 2024 14:23:35 +0400 Subject: [PATCH 178/243] fix(config): add sci missing functional unit to strings --- src/config/strings.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/config/strings.ts b/src/config/strings.ts index b89822794..31e485b24 100644 --- a/src/config/strings.ts +++ b/src/config/strings.ts @@ -86,6 +86,8 @@ ${error}`, `Unexpected date datatype: ${typeof date}: ${date}`, INVALID_OBSERVATION_OVERLAP: 'Observation timestamps overlap, please check inputs.', + SCI_MISSING_FN_UNIT: (functionalUnit: string) => + `'carbon' and ${functionalUnit} should be present in your input data.`, /** Exhaust messages */ OUTPUT_REQUIRED: 'Output path is required, please make sure output is configured properly.', From b12e968062e647bc59bba2289aca48afa44b3c1a Mon Sep 17 00:00:00 2001 From: Joseph Cook <33655003+jmcook1186@users.noreply.github.com> Date: Mon, 24 Jun 2024 11:48:44 +0100 Subject: [PATCH 179/243] Update src/builtins/regex/README.md Co-authored-by: Narek Hovhannisyan Signed-off-by: Joseph Cook <33655003+jmcook1186@users.noreply.github.com> --- src/builtins/regex/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/builtins/regex/README.md b/src/builtins/regex/README.md index 62f05bca9..f079b0461 100644 --- a/src/builtins/regex/README.md +++ b/src/builtins/regex/README.md @@ -97,7 +97,7 @@ The results will be saved to a new `yaml` file in `manifests/outputs`. ### `MissingInputDataError` This error arises when a necessary piece of input data is missing from the `inputs` array. -Every element in the ``inputs` array must contain: +Every element in the `inputs` array must contain: - `timestamp` - `duration` - whatever value you passed to `parameter` From c561cda14a8c74598516f13c27d8f061fda5658f Mon Sep 17 00:00:00 2001 From: Joseph Cook <33655003+jmcook1186@users.noreply.github.com> Date: Mon, 24 Jun 2024 11:48:51 +0100 Subject: [PATCH 180/243] Update src/builtins/multiply/README.md Co-authored-by: Narek Hovhannisyan Signed-off-by: Joseph Cook <33655003+jmcook1186@users.noreply.github.com> --- src/builtins/multiply/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/builtins/multiply/README.md b/src/builtins/multiply/README.md index ae410bba5..7e9c2dfbd 100644 --- a/src/builtins/multiply/README.md +++ b/src/builtins/multiply/README.md @@ -101,7 +101,7 @@ The results will be saved to a new `yaml` file in `./examples/outputs` ### `MissingInputDataError` This error arises when a necessary piece of input data is missing from the `inputs` array. -Every element in the ``inputs` array must contain: +Every element in the `inputs` array must contain: - `timestamp` - `duration` - whatever values you passed to `input-parameters` From 2c1b26be991d9dc2a1fd3a1244b4b3e32599736c Mon Sep 17 00:00:00 2001 From: Joseph Cook <33655003+jmcook1186@users.noreply.github.com> Date: Mon, 24 Jun 2024 11:48:56 +0100 Subject: [PATCH 181/243] Update src/builtins/exponent/README.md Co-authored-by: Narek Hovhannisyan Signed-off-by: Joseph Cook <33655003+jmcook1186@users.noreply.github.com> --- src/builtins/exponent/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/builtins/exponent/README.md b/src/builtins/exponent/README.md index e484ea74d..91d938cef 100644 --- a/src/builtins/exponent/README.md +++ b/src/builtins/exponent/README.md @@ -103,7 +103,7 @@ The results will be saved to a new `yaml` file in `manifests/outputs`. ### `MissingInputDataError` This error arises when a necessary piece of input data is missing from the `inputs` array. -Every element in the ``inputs` array must contain: +Every element in the `inputs` array must contain: - `timestamp` - `duration` From 04ef096e4aed3be1071075604dd3342217e727eb Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 24 Jun 2024 15:17:21 +0400 Subject: [PATCH 182/243] docs(builtins): add note for zero devision --- src/builtins/divide/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/builtins/divide/README.md b/src/builtins/divide/README.md index 20ed19f55..0d753dd53 100644 --- a/src/builtins/divide/README.md +++ b/src/builtins/divide/README.md @@ -26,6 +26,8 @@ For example, `boavizta-cpu` need `cpu/number-cores` to work, however `cloud-meta The plugin throws an exception if the division result is not a number. +>Note: Plugin will warn and return `numerator` value in case if `denominator` is zero. + ## Calculation ```pseudocode From 34832d1e3860c03f900d84d25ee0b2c5add2c1fd Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 24 Jun 2024 16:49:10 +0400 Subject: [PATCH 183/243] fix(builtins): use error message from config in sci --- src/builtins/sci/index.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/builtins/sci/index.ts b/src/builtins/sci/index.ts index 3bc4fedf0..47d495351 100644 --- a/src/builtins/sci/index.ts +++ b/src/builtins/sci/index.ts @@ -10,6 +10,7 @@ const {MissingInputDataError} = ERRORS; const { MISSING_FUNCTIONAL_UNIT_CONFIG, MISSING_FUNCTIONAL_UNIT_INPUT, + SCI_MISSING_FN_UNIT, ZERO_DIVISION, } = STRINGS; @@ -60,8 +61,6 @@ export const Sci = (globalConfig: ConfigParams): ExecutePlugin => { * Checks for fields in input. */ const validateInput = (input: PluginParams) => { - const message = `'carbon' and ${globalConfig['functional-unit']} should be present in your input data.`; - const validatedConfig = validateConfig(globalConfig); if ( @@ -78,7 +77,9 @@ export const Sci = (globalConfig: ConfigParams): ExecutePlugin => { carbon: z.number().gte(0), duration: z.number().gte(1), }) - .refine(allDefined, {message}); + .refine(allDefined, { + message: SCI_MISSING_FN_UNIT(globalConfig['functional-unit']), + }); return validate>(schema, input); }; From 6df9a5cdddc0193c9812224d3f8b0f9a3e4fc185 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 24 Jun 2024 16:49:45 +0400 Subject: [PATCH 184/243] docs(builtins): add note for zero devision in sci --- src/builtins/sci/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/builtins/sci/README.md b/src/builtins/sci/README.md index 658209f1a..f46bb2b03 100644 --- a/src/builtins/sci/README.md +++ b/src/builtins/sci/README.md @@ -19,6 +19,8 @@ - `sci`: carbon expressed in terms of the given functional unit +>Note: Plugin will warn and return `carbon` value in case if `functional-unit`'s value is zero. + ## Calculation SCI is calculated as: From 67e1595e668b4de45866d84e7fd25419a40fa079 Mon Sep 17 00:00:00 2001 From: manushak Date: Mon, 24 Jun 2024 19:28:10 +0400 Subject: [PATCH 185/243] fix(util): add some function into helpers.ts from env.ts --- src/util/helpers.ts | 102 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 99 insertions(+), 3 deletions(-) diff --git a/src/util/helpers.ts b/src/util/helpers.ts index 0c422c96f..612ee6e85 100644 --- a/src/util/helpers.ts +++ b/src/util/helpers.ts @@ -1,15 +1,38 @@ +#!/usr/bin/env node +/* eslint-disable no-process-exit */ import {createInterface} from 'node:readline/promises'; import {exec} from 'node:child_process'; import {promisify} from 'node:util'; -import {ERRORS} from '@grnsft/if-core/utils'; +import * as fs from 'fs/promises'; +import * as path from 'path'; -import {logger} from './logger'; +import {ERRORS} from '@grnsft/if-core/utils'; -import {STRINGS} from '../config'; +import {STRINGS, CONFIG} from '../config'; import {Difference} from '../types/lib/compare'; +import {load} from '../lib/load'; + +import { + installDependencies, + initPackageJsonIfNotExists, + updatePackageJsonDependencies, + extractPathsWithVersion, + updatePackageJsonProperties, +} from './npm'; + +import {logger} from './logger'; +import {EnvironmentOptions} from '../types/if-env'; + +const {IF_ENV} = CONFIG; +const { + FAILURE_MESSAGE, + FAILURE_MESSAGE_TEMPLATE, + FAILURE_MESSAGE_DEPENDENCIES, +} = IF_ENV; + const {UNSUPPORTED_ERROR} = STRINGS; /** @@ -176,3 +199,76 @@ export const parseManifestFromStdin = async () => { return match![1]; }; + +/** + * Gets the folder path of the manifest file, dependencies from manifest file and install argument from the given arguments. + */ +export const getOptionsFromArgs = async (commandArgs: { + manifest: string; + install: boolean | undefined; +}) => { + const {manifest, install} = commandArgs; + const folderPath = path.dirname(manifest); + const loadedManifest = await load(manifest); + const rawManifest = loadedManifest.rawManifest; + const plugins = rawManifest?.initialize?.plugins || {}; + const dependencies = rawManifest?.execution?.environment.dependencies || []; + + if (!dependencies.length) { + throw new Error(FAILURE_MESSAGE_DEPENDENCIES); + } + + const pathsWithVersion = extractPathsWithVersion(plugins, dependencies); + + return { + folderPath, + dependencies: pathsWithVersion, + install, + }; +}; + +/** + * Creates folder if not exists, installs dependencies if required, update depenedencies. + */ +export const initializeAndInstallLibs = async (options: EnvironmentOptions) => { + try { + const {folderPath, install, cwd, dependencies} = options; + const packageJsonPath = await initPackageJsonIfNotExists(folderPath); + + await updatePackageJsonProperties(packageJsonPath, !cwd); + + if (install) { + await installDependencies(folderPath, dependencies); + } else { + await updatePackageJsonDependencies(packageJsonPath, dependencies, cwd); + } + } catch (error) { + console.log(FAILURE_MESSAGE); + process.exit(2); + } +}; + +/** + * Adds a manifest template to the folder where the if-env CLI command runs. + */ +export const addTemplateManifest = async (destinationDir: string) => { + try { + const templateManifest = path.resolve( + __dirname, + '../config/env-template.yml' + ); + + console.log('--templateManifest', templateManifest); + + const destinationPath = path.resolve(destinationDir, 'manifest.yml'); + + const data = await fs.readFile(templateManifest, 'utf-8'); + console.log('--after read'); + await fs.writeFile(destinationPath, '', 'utf-8'); + await fs.writeFile(destinationPath, data, 'utf-8'); + } catch (error) { + console.log('---errr', error); + console.log(FAILURE_MESSAGE_TEMPLATE); + process.exit(1); + } +}; From b3e49bb629be161efea00601239c0d3874f3cc1b Mon Sep 17 00:00:00 2001 From: manushak Date: Mon, 24 Jun 2024 19:29:57 +0400 Subject: [PATCH 186/243] fix(util): remove unnecessary logs --- src/util/helpers.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/util/helpers.ts b/src/util/helpers.ts index 612ee6e85..3912b63e0 100644 --- a/src/util/helpers.ts +++ b/src/util/helpers.ts @@ -258,16 +258,12 @@ export const addTemplateManifest = async (destinationDir: string) => { '../config/env-template.yml' ); - console.log('--templateManifest', templateManifest); - const destinationPath = path.resolve(destinationDir, 'manifest.yml'); - const data = await fs.readFile(templateManifest, 'utf-8'); - console.log('--after read'); + await fs.writeFile(destinationPath, '', 'utf-8'); await fs.writeFile(destinationPath, data, 'utf-8'); } catch (error) { - console.log('---errr', error); console.log(FAILURE_MESSAGE_TEMPLATE); process.exit(1); } From 3bfa2858025cda8ae9eab4bc604ebc22ced0c1cf Mon Sep 17 00:00:00 2001 From: manushak Date: Mon, 24 Jun 2024 19:39:16 +0400 Subject: [PATCH 187/243] test(util): add test for helpers function --- src/__tests__/unit/util/helpers.test.ts | 106 ++++++++++++++++++++++++ 1 file changed, 106 insertions(+) diff --git a/src/__tests__/unit/util/helpers.test.ts b/src/__tests__/unit/util/helpers.test.ts index ba89850a3..5757628ad 100644 --- a/src/__tests__/unit/util/helpers.test.ts +++ b/src/__tests__/unit/util/helpers.test.ts @@ -1,3 +1,5 @@ +import {jest} from '@jest/globals'; + const mockWarn = jest.fn(); const mockError = jest.fn(); @@ -11,6 +13,67 @@ jest.mock('../../../util/logger', () => ({ }, })); +jest.mock('path', () => { + const actualPath = jest.requireActual('path') as Record; + return { + __esModule: true, + ...actualPath, + dirname: jest.fn(() => './mock-path'), + }; +}); + +jest.mock('fs/promises', () => require('../../../__mocks__/fs')); + +jest.mock('../../../lib/load', () => ({ + load: jest.fn(() => { + if (process.env.manifest === 'true') { + return { + rawManifest: { + name: 'divide', + initialize: { + plugins: { + 'cloud-metadata': { + path: '@grnsft/if-plugins', + method: 'CloudMetadata', + }, + divide: { + path: 'builtin', + method: 'Divide', + 'global-config': { + numerator: 'vcpus-allocated', + denominator: 2, + output: 'cpu/number-cores', + }, + }, + }, + }, + execution: { + environment: { + dependencies: [ + '@grnsft/if-core@0.0.7', + '@grnsft/if-plugins@v0.3.2 extraneous -> file:../../../if-models', + '@grnsft/if-unofficial-plugins@v0.3.0 extraneous -> file:../../../if-unofficial-models', + ], + }, + }, + }, + }; + } + return { + rawManifest: { + initialize: { + plugins: {'@grnsft/if-plugins': '1.0.0'}, + }, + execution: { + environment: { + dependencies: [], + }, + }, + }, + }; + }), +})); + import {ERRORS} from '@grnsft/if-core/utils'; import { @@ -20,8 +83,13 @@ import { mergeObjects, oneIsPrimitive, parseManifestFromStdin, + getOptionsFromArgs, } from '../../../util/helpers'; import {Difference} from '../../../types/lib/compare'; +import {CONFIG} from '../../../config'; + +const {IF_ENV} = CONFIG; +const {FAILURE_MESSAGE_DEPENDENCIES} = IF_ENV; const {WriteFileError} = ERRORS; @@ -399,4 +467,42 @@ description: mock-description expect(response).toBeFalsy(); }); }); + + describe('getOptionsFromArgs(): ', () => { + it('returns the correct options when dependencies are present.', async () => { + const commandArgs = { + manifest: '/path/to/mock-manifest.json', + install: false, + }; + + process.env.manifest = 'true'; + + const result = await getOptionsFromArgs(commandArgs); + expect.assertions(1); + + expect(result).toEqual({ + folderPath: './mock-path', + dependencies: { + '@grnsft/if-plugins': '^v0.3.2', + }, + install: false, + }); + }); + + it('throws an error when there are no dependencies.', async () => { + const commandArgs = { + manifest: '/path/to/mock-manifest.json', + install: false, + }; + + process.env.manifest = 'false'; + + expect.assertions(1); + try { + await getOptionsFromArgs(commandArgs); + } catch (error) { + expect(error).toEqual(new Error(FAILURE_MESSAGE_DEPENDENCIES)); + } + }); + }); }); From 276589f69d0d16c09d4b27992e695d18ce5ffd4e Mon Sep 17 00:00:00 2001 From: manushak Date: Mon, 24 Jun 2024 19:41:09 +0400 Subject: [PATCH 188/243] fix(src): move some funcions into helpers --- src/env.ts | 92 ++++-------------------------------------------------- 1 file changed, 6 insertions(+), 86 deletions(-) diff --git a/src/env.ts b/src/env.ts index 1775c3fd3..48b0b76b9 100644 --- a/src/env.ts +++ b/src/env.ts @@ -1,31 +1,19 @@ #!/usr/bin/env node /* eslint-disable no-process-exit */ -import * as fs from 'fs/promises'; -import * as path from 'path'; - -import { - installDependencies, - initPackageJsonIfNotExists, - updatePackageJsonDependencies, - extractPathsWithVersion, - updatePackageJsonProperties, -} from './util/npm'; import {parseIfEnvArgs} from './util/args'; import {logger} from './util/logger'; -import {load} from './lib/load'; - import {CONFIG} from './config'; import {EnvironmentOptions} from './types/if-env'; +import { + addTemplateManifest, + getOptionsFromArgs, + initializeAndInstallLibs, +} from './util/helpers'; const {IF_ENV} = CONFIG; -const { - SUCCESS_MESSAGE, - FAILURE_MESSAGE, - FAILURE_MESSAGE_TEMPLATE, - FAILURE_MESSAGE_DEPENDENCIES, -} = IF_ENV; +const {SUCCESS_MESSAGE} = IF_ENV; const IfEnv = async () => { const commandArgs = await parseIfEnvArgs(); @@ -54,74 +42,6 @@ const IfEnv = async () => { process.exit(0); }; -/** - * Gets the folder path of manifest file, dependencies from manifest file and install argument from the given arguments. - */ -const getOptionsFromArgs = async (commandArgs: { - manifest: string; - install: boolean | undefined; -}) => { - const {manifest: manifestPath, install} = commandArgs; - const folderPath = path.dirname(manifestPath); - const manifest = await load(manifestPath); - const plugins = manifest.rawManifest?.initialize?.plugins || {}; - const dependencies = - manifest.rawManifest?.execution?.environment.dependencies || []; - - if (!dependencies.length) { - throw new Error(FAILURE_MESSAGE_DEPENDENCIES); - } - - const pathsWithVersion = extractPathsWithVersion(plugins, dependencies); - - return { - folderPath, - dependencies: pathsWithVersion, - install, - }; -}; - -/** - * Creates folder if not exists, installs dependencies if required, update depenedencies. - */ -const initializeAndInstallLibs = async (options: EnvironmentOptions) => { - try { - const {folderPath, install, cwd, dependencies} = options; - const packageJsonPath = await initPackageJsonIfNotExists(folderPath); - - await updatePackageJsonProperties(packageJsonPath, !cwd); - - if (install) { - await installDependencies(folderPath, dependencies); - } else { - await updatePackageJsonDependencies(packageJsonPath, dependencies, cwd); - } - } catch (error) { - console.log(FAILURE_MESSAGE); - process.exit(2); - } -}; - -/** - * Adds a manifest template to the folder where the if-env CLI command runs. - */ -const addTemplateManifest = async (destinationDir: string) => { - try { - const templateManifest = path.resolve( - __dirname, - './config/env-template.yml' - ); - const destinationPath = path.resolve(destinationDir, 'manifest.yml'); - - const data = await fs.readFile(templateManifest, 'utf-8'); - await fs.writeFile(destinationPath, '', 'utf-8'); - await fs.writeFile(destinationPath, data, 'utf-8'); - } catch (error) { - console.log(FAILURE_MESSAGE_TEMPLATE); - process.exit(1); - } -}; - IfEnv().catch(error => { if (error instanceof Error) { logger.error(error); From 13a7019d59f4d23062c38399c27108bdc8261811 Mon Sep 17 00:00:00 2001 From: manushak Date: Tue, 25 Jun 2024 15:49:20 +0400 Subject: [PATCH 189/243] fix(util): change `CliInputError` to `ParseCliParamsError` error --- src/util/args.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/util/args.ts b/src/util/args.ts index 4f537328d..5d9524390 100644 --- a/src/util/args.ts +++ b/src/util/args.ts @@ -156,7 +156,7 @@ const validateAndParseIfEnvArgs = () => { return parse(IF_ENV.ARGS, IF_ENV.HELP); } catch (error) { if (error instanceof Error) { - throw new CliInputError(error.message); + throw new ParseCliParamsError(error.message); } throw error; @@ -174,14 +174,14 @@ export const parseIfEnvArgs = async () => { const isManifestFileExists = await isFileExists(response); if (!isManifestFileExists) { - throw new CliInputError(MANIFEST_NOT_FOUND); + throw new ParseCliParamsError(MANIFEST_NOT_FOUND); } if (checkIfFileIsYaml(manifest)) { return {manifest: response, install, cwd}; } - throw new CliInputError(FILE_IS_NOT_YAML); + throw new CliSourceFileError(SOURCE_IS_NOT_YAML); } return {install, cwd}; From 66a1c4c49e8bf3c4155508f3e031fb22fdcac701 Mon Sep 17 00:00:00 2001 From: manushak Date: Tue, 25 Jun 2024 15:50:07 +0400 Subject: [PATCH 190/243] test(util): fix npm test --- src/__tests__/unit/util/npm.test.ts | 31 +++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/src/__tests__/unit/util/npm.test.ts b/src/__tests__/unit/util/npm.test.ts index 9680bf722..403a1138c 100644 --- a/src/__tests__/unit/util/npm.test.ts +++ b/src/__tests__/unit/util/npm.test.ts @@ -110,7 +110,6 @@ describe('util/npm: ', () => { const expectedPackageJsonContent = JSON.stringify( { dependencies: { - '@grnsft/if': '^0.3.3-beta.0', '@grnsft/if-plugins': '^0.3.3-beta.0', }, }, @@ -137,7 +136,6 @@ describe('util/npm: ', () => { const expectedPackageJsonContent = JSON.stringify( { dependencies: { - '@grnsft/if': '^0.3.3-beta.0', '@grnsft/if-plugins': '^0.3.3-beta.0', }, }, @@ -220,11 +218,32 @@ describe('util/npm: ', () => { describe('updatePackageJsonProperties(): ', () => { it('updates the package.json properties correctly.', async () => { - const newPackageJsonPath = path.resolve(folderPath, '/package.json-npm'); - await updatePackageJsonProperties(newPackageJsonPath, false); + const packageJsonPath = path.join(folderPath, 'package.json-npm1'); - expect.assertions(2); - expect(fs.readFile).toHaveBeenCalledWith(newPackageJsonPath, 'utf8'); + const expectedPackageJsonContent = JSON.stringify( + { + name: 'if-environment', + description: 'mock-description', + author: {}, + bugs: {}, + engines: {}, + homepage: 'mock-homepage', + dependencies: { + '@grnsft/if-plugins': '^0.3.3-beta.0', + }, + }, + null, + 2 + ); + + const fsReadSpy = jest + .spyOn(fs, 'readFile') + .mockResolvedValue(expectedPackageJsonContent); + await updatePackageJsonProperties(packageJsonPath, true); + + expect.assertions(8); + + expect(fsReadSpy).toHaveBeenCalledWith(packageJsonPath, 'utf8'); }); }); }); From 0e789b16468f65b38197ea88fb7acb1372b09d8f Mon Sep 17 00:00:00 2001 From: manushak Date: Tue, 25 Jun 2024 15:51:20 +0400 Subject: [PATCH 191/243] test(mocks): add mocks for fs.writeFile --- src/__mocks__/fs/index.ts | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/src/__mocks__/fs/index.ts b/src/__mocks__/fs/index.ts index b7731b4b8..3853ad7ee 100644 --- a/src/__mocks__/fs/index.ts +++ b/src/__mocks__/fs/index.ts @@ -1,6 +1,7 @@ import * as YAML from 'js-yaml'; import * as fs from 'fs'; import * as fsAsync from 'fs/promises'; +import * as path from 'path'; export const readFile = async (filePath: string) => { /** mock for util/npm */ @@ -89,11 +90,28 @@ cpu-cores-available,cpu-cores-utilized,cpu-manufacturer,cpu-model-name,cpu-tdp,g export const mkdir = (dirPath: string) => dirPath; export const writeFile = async (pathToFile: string, content: string) => { - if (pathToFile.includes('package.json-npm')) { + if (pathToFile.includes('package.json-npm1')) { + const updatedPath = pathToFile.replace('-npm1', ''); + const fileContent = await fsAsync.readFile(updatedPath, 'utf8'); + const fileContentObject = JSON.parse(fileContent); + const parsedContent = JSON.parse(content); + + for (const property in fileContentObject) { + expect(parsedContent).toHaveProperty(property); + } + } else if (pathToFile.includes('package.json-npm')) { const updatedPath = pathToFile.replace('-npm', ''); + const fileContent = await fsAsync.readFile(updatedPath, 'utf8'); + + expect(content).toBe(fileContent); + } else if (pathToFile.includes('/manifest.yml')) { + const templateManifest = path.resolve( + __dirname, + '../../config/env-template.yml' + ); + const fileContent = await fsAsync.readFile(templateManifest, 'utf8'); - const content = await fsAsync.readFile(updatedPath, 'utf8'); - expect(content).toBe(content); + expect(content).toBe(fileContent); } else { if (pathToFile === 'reject') { throw new Error('Wrong file path'); From accf32f3d7f957c22b610a9b78f0177af7a9e1ca Mon Sep 17 00:00:00 2001 From: manushak Date: Tue, 25 Jun 2024 15:56:06 +0400 Subject: [PATCH 192/243] test(util): add missied tests --- src/__tests__/unit/util/helpers.test.ts | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/__tests__/unit/util/helpers.test.ts b/src/__tests__/unit/util/helpers.test.ts index 5757628ad..24ef57a2d 100644 --- a/src/__tests__/unit/util/helpers.test.ts +++ b/src/__tests__/unit/util/helpers.test.ts @@ -84,12 +84,14 @@ import { oneIsPrimitive, parseManifestFromStdin, getOptionsFromArgs, + addTemplateManifest, + // initializeAndInstallLibs, } from '../../../util/helpers'; import {Difference} from '../../../types/lib/compare'; import {CONFIG} from '../../../config'; const {IF_ENV} = CONFIG; -const {FAILURE_MESSAGE_DEPENDENCIES} = IF_ENV; +const {FAILURE_MESSAGE_DEPENDENCIES, FAILURE_MESSAGE} = IF_ENV; const {WriteFileError} = ERRORS; @@ -505,4 +507,23 @@ description: mock-description } }); }); + + describe('addTemplateManifest(): ', () => { + it('successfully adds the template manifest to the directory.', async () => { + await addTemplateManifest('./'); + + expect.assertions(1); + }); + + it('throws an error when the manifest is not added into the directory.', async () => { + expect.assertions(1); + + try { + await addTemplateManifest(''); + } catch (error) { + const logSpy = jest.spyOn(global.console, 'log'); + expect(logSpy).toEqual(FAILURE_MESSAGE); + } + }); + }); }); From d46d71c3833c519c25088bfcc366ac3917a5ffae Mon Sep 17 00:00:00 2001 From: manushak Date: Tue, 25 Jun 2024 15:57:22 +0400 Subject: [PATCH 193/243] fix(util): add `MissingPluginDependenciesError` custom error class --- src/util/helpers.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/util/helpers.ts b/src/util/helpers.ts index 3912b63e0..b000af60e 100644 --- a/src/util/helpers.ts +++ b/src/util/helpers.ts @@ -34,6 +34,7 @@ const { } = IF_ENV; const {UNSUPPORTED_ERROR} = STRINGS; +const {MissingPluginDependenciesError} = ERRORS; /** * Impact engine error handler. Logs errors and appends issue template if error is unknown. @@ -215,7 +216,7 @@ export const getOptionsFromArgs = async (commandArgs: { const dependencies = rawManifest?.execution?.environment.dependencies || []; if (!dependencies.length) { - throw new Error(FAILURE_MESSAGE_DEPENDENCIES); + throw new MissingPluginDependenciesError(FAILURE_MESSAGE_DEPENDENCIES); } const pathsWithVersion = extractPathsWithVersion(plugins, dependencies); @@ -261,7 +262,6 @@ export const addTemplateManifest = async (destinationDir: string) => { const destinationPath = path.resolve(destinationDir, 'manifest.yml'); const data = await fs.readFile(templateManifest, 'utf-8'); - await fs.writeFile(destinationPath, '', 'utf-8'); await fs.writeFile(destinationPath, data, 'utf-8'); } catch (error) { console.log(FAILURE_MESSAGE_TEMPLATE); From 04e514a33186adf5f37bd3d6abdb95dcd0773dbe Mon Sep 17 00:00:00 2001 From: manushak Date: Tue, 25 Jun 2024 16:46:52 +0400 Subject: [PATCH 194/243] feat(package): update if-core version --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8eb3478c2..3e66c802a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,7 @@ "dependencies": { "@commitlint/cli": "^18.6.0", "@commitlint/config-conventional": "^18.6.0", - "@grnsft/if-core": "^0.0.7", + "@grnsft/if-core": "^0.0.9", "axios": "^1.7.2", "csv-parse": "^5.5.6", "csv-stringify": "^6.4.6", @@ -1183,9 +1183,9 @@ } }, "node_modules/@grnsft/if-core": { - "version": "0.0.7", - "resolved": "https://registry.npmjs.org/@grnsft/if-core/-/if-core-0.0.7.tgz", - "integrity": "sha512-+4w8Sq1KRPDb+Jy638wgrTXlXIZzewOBceT+rAy3Oaov1M/veY3gu3AV15SXcPHrsBoFmZ6QeI9g1rF3RKB0ww==", + "version": "0.0.9", + "resolved": "https://registry.npmjs.org/@grnsft/if-core/-/if-core-0.0.9.tgz", + "integrity": "sha512-F0niYe1j+NfhH+okz5sjP/CD7w/9BeXXe13bHkCsdOnk0WfXrq0DGKwpqh9TiLbKc9f1P3/XuojeFANMQu5nig==", "dependencies": { "typescript": "^5.1.6" }, diff --git a/package.json b/package.json index 2f3257216..0e03be3ee 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "dependencies": { "@commitlint/cli": "^18.6.0", "@commitlint/config-conventional": "^18.6.0", - "@grnsft/if-core": "^0.0.7", + "@grnsft/if-core": "^0.0.9", "axios": "^1.7.2", "csv-parse": "^5.5.6", "csv-stringify": "^6.4.6", From ad2f87da76d53933afc5589d46739225d76cfec9 Mon Sep 17 00:00:00 2001 From: manushak Date: Tue, 25 Jun 2024 16:55:08 +0400 Subject: [PATCH 195/243] fix(util): fix dependencies append logic --- src/util/helpers.ts | 2 +- src/util/npm.ts | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/util/helpers.ts b/src/util/helpers.ts index b000af60e..e69b9e0e9 100644 --- a/src/util/helpers.ts +++ b/src/util/helpers.ts @@ -236,7 +236,7 @@ export const initializeAndInstallLibs = async (options: EnvironmentOptions) => { const {folderPath, install, cwd, dependencies} = options; const packageJsonPath = await initPackageJsonIfNotExists(folderPath); - await updatePackageJsonProperties(packageJsonPath, !cwd); + await updatePackageJsonProperties(packageJsonPath, cwd); if (install) { await installDependencies(folderPath, dependencies); diff --git a/src/util/npm.ts b/src/util/npm.ts index 0cbfe27e1..e7fd8d62b 100644 --- a/src/util/npm.ts +++ b/src/util/npm.ts @@ -108,7 +108,7 @@ export const extractPathsWithVersion = ( */ export const updatePackageJsonProperties = async ( newPackageJsonPath: string, - resetDependencies: boolean + appendDependencies: boolean ) => { const packageJsonContent = await fs.readFile(newPackageJsonPath, 'utf8'); const parsedPackageJsonContent = JSON.parse(packageJsonContent); @@ -120,7 +120,9 @@ export const updatePackageJsonProperties = async ( bugs: packageJson.bugs, engines: packageJson.engines, homepage: packageJson.homepage, - dependencies: resetDependencies ? {} : packageJson.dependencies, + dependencies: appendDependencies + ? parsedPackageJsonContent.dependencies + : {}, }; const newPackageJson = Object.assign( From 621a311092255dd135525828e8edb307b5903efb Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Wed, 26 Jun 2024 10:01:19 +0400 Subject: [PATCH 196/243] fix(util): typo in npm --- src/util/npm.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/npm.ts b/src/util/npm.ts index e7fd8d62b..e453b771b 100644 --- a/src/util/npm.ts +++ b/src/util/npm.ts @@ -15,7 +15,7 @@ const packageJson = require('../../package.json'); const {INITIALIZING_PACKAGE_JSON, INSTALLING_NPM_PACKAGES} = STRINGS; /** - * Checks if the package.json is exists, if not, inisializes it. + * Checks if the package.json is exists, if not, initializes it. */ export const initPackageJsonIfNotExists = async (folderPath: string) => { const packageJsonPath = path.resolve(folderPath, 'package.json'); From 136f062d45788f8ac06a92bd171d5afbf2a152dc Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Wed, 26 Jun 2024 10:01:53 +0400 Subject: [PATCH 197/243] test(util): add cases for initializeAndInstallLibs --- src/__tests__/unit/util/helpers.test.ts | 144 +++++++++++++++++++++++- 1 file changed, 141 insertions(+), 3 deletions(-) diff --git a/src/__tests__/unit/util/helpers.test.ts b/src/__tests__/unit/util/helpers.test.ts index 24ef57a2d..cef643247 100644 --- a/src/__tests__/unit/util/helpers.test.ts +++ b/src/__tests__/unit/util/helpers.test.ts @@ -1,5 +1,4 @@ -import {jest} from '@jest/globals'; - +/* eslint-disable @typescript-eslint/ban-ts-comment */ const mockWarn = jest.fn(); const mockError = jest.fn(); @@ -74,6 +73,67 @@ jest.mock('../../../lib/load', () => ({ }), })); +const initPackage = jest.fn(() => Promise.resolve('mock-path')); +const updatePackage = jest.fn(() => Promise.resolve(true)); +const installdeps = jest.fn(); +const updatedeps = jest.fn(); +jest.mock('../../../util/npm', () => { + const actualNPMUtil = jest.requireActual('../../../util/npm'); + + return { + ...actualNPMUtil, + initPackageJsonIfNotExists: (folderPath: string) => { + if (process.env.NPM_MOCK === 'true') { + return initPackage(); + } + + if (process.env.NPM_MOCK === 'error') { + throw new Error('mock-error'); + } + + return actualNPMUtil.initPackageJsonIfNotExists(folderPath); + }, + updatePackageJsonProperties: ( + newPackageJsonPath: string, + appendDependencies: boolean + ) => { + if (process.env.NPM_MOCK === 'true') { + return updatePackage(); + } + + return actualNPMUtil.updatePackageJsonProperties( + newPackageJsonPath, + appendDependencies + ); + }, + installDependencies: ( + folderPath: string, + dependencies: {[path: string]: string} + ) => { + if (process.env.NPM_MOCK === 'true') { + return installdeps(); + } + + return actualNPMUtil.installDependencies(folderPath, dependencies); + }, + updatePackageJsonDependencies: ( + packageJsonPath: string, + dependencies: any, + cwd: boolean + ) => { + if (process.env.NPM_MOCK === 'true') { + return updatedeps(); + } + + return actualNPMUtil.updatePackageJsonDependencies( + packageJsonPath, + dependencies, + cwd + ); + }, + }; +}); + import {ERRORS} from '@grnsft/if-core/utils'; import { @@ -85,10 +145,11 @@ import { parseManifestFromStdin, getOptionsFromArgs, addTemplateManifest, + initializeAndInstallLibs, // initializeAndInstallLibs, } from '../../../util/helpers'; -import {Difference} from '../../../types/lib/compare'; import {CONFIG} from '../../../config'; +import {Difference} from '../../../types/lib/compare'; const {IF_ENV} = CONFIG; const {FAILURE_MESSAGE_DEPENDENCIES, FAILURE_MESSAGE} = IF_ENV; @@ -526,4 +587,81 @@ description: mock-description } }); }); + + describe('initializeAndInstallLibs(): ', () => { + beforeEach(() => { + initPackage.mockReset(); + updatePackage.mockReset(); + installdeps.mockReset(); + updatedeps.mockReset(); + }); + + it('installs dependencies if install flag is truthy.', async () => { + process.env.NPM_MOCK = 'true'; + // @ts-ignore + process.exit = (code: any) => code; + const options = { + folderPath: 'mock-folderPath', + install: true, + cwd: true, + dependencies: { + mock: 'mock-dependencies', + }, + }; + + expect.assertions(4); + await initializeAndInstallLibs(options); + + expect(initPackage).toHaveBeenCalledTimes(1); + expect(updatePackage).toHaveBeenCalledTimes(1); + expect(installdeps).toHaveBeenCalledTimes(1); + expect(updatedeps).toHaveBeenCalledTimes(0); + }); + + it('updates dependencies if install flag is falsy.', async () => { + process.env.NPM_MOCK = 'true'; + // @ts-ignore + process.exit = (code: any) => code; + const options = { + folderPath: 'mock-folderPath', + install: false, + cwd: true, + dependencies: { + mock: 'mock-dependencies', + }, + }; + + expect.assertions(4); + await initializeAndInstallLibs(options); + + expect(initPackage).toHaveBeenCalledTimes(1); + expect(updatePackage).toHaveBeenCalledTimes(1); + expect(installdeps).toHaveBeenCalledTimes(0); + expect(updatedeps).toHaveBeenCalledTimes(1); + }); + + it('exits process if error is thrown.', async () => { + process.env.NPM_MOCK = 'error'; + const mockExit = jest.fn(); + // @ts-ignore + process.exit = mockExit; + const options = { + folderPath: 'mock-folderPath', + install: false, + cwd: true, + dependencies: { + mock: 'mock-dependencies', + }, + }; + + expect.assertions(5); + await initializeAndInstallLibs(options); + + expect(initPackage).toHaveBeenCalledTimes(0); + expect(updatePackage).toHaveBeenCalledTimes(0); + expect(installdeps).toHaveBeenCalledTimes(0); + expect(updatedeps).toHaveBeenCalledTimes(0); + expect(mockExit).toHaveBeenCalledTimes(1); + }); + }); }); From 7a466ccccd16dba64133d9b79a2b73c09fbedb0c Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Wed, 26 Jun 2024 10:04:47 +0400 Subject: [PATCH 198/243] test(util): put back original process exit --- src/__tests__/unit/util/helpers.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/__tests__/unit/util/helpers.test.ts b/src/__tests__/unit/util/helpers.test.ts index cef643247..d04ad43d2 100644 --- a/src/__tests__/unit/util/helpers.test.ts +++ b/src/__tests__/unit/util/helpers.test.ts @@ -642,6 +642,7 @@ description: mock-description it('exits process if error is thrown.', async () => { process.env.NPM_MOCK = 'error'; + const originalProcessExit = process.exit; const mockExit = jest.fn(); // @ts-ignore process.exit = mockExit; @@ -662,6 +663,8 @@ description: mock-description expect(installdeps).toHaveBeenCalledTimes(0); expect(updatedeps).toHaveBeenCalledTimes(0); expect(mockExit).toHaveBeenCalledTimes(1); + + process.exit = originalProcessExit; }); }); }); From ed31d52cb1320ede6ac2c8f3f96b3a6f22c5bd3a Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Wed, 26 Jun 2024 10:14:16 +0400 Subject: [PATCH 199/243] fix(src): add newline to package.json Signed-off-by: Narek Hovhannisyan --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 0e03be3ee..e9f2f2777 100644 --- a/package.json +++ b/package.json @@ -88,4 +88,4 @@ }, "stability": "stable", "types": "src/index.d.ts" -} \ No newline at end of file +} From d56cd86d07246546fdf7259ab8768afeea486bae Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Wed, 26 Jun 2024 10:59:27 +0100 Subject: [PATCH 200/243] fix(doc): update config in refactor guide --- Refactor-migration-guide.md | 1 - 1 file changed, 1 deletion(-) diff --git a/Refactor-migration-guide.md b/Refactor-migration-guide.md index 95269e1a5..d315d5c41 100644 --- a/Refactor-migration-guide.md +++ b/Refactor-migration-guide.md @@ -186,7 +186,6 @@ There have also been some changes to the structure of manifest files. Some of th device/expected-lifespan: 94608000 # 3 years in seconds resources-reserved: 1 resources-total: 8 - functional-unit-time: "1 min" ``` From dc85178880cb7ee65897f79b38de16bcd941fa13 Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Wed, 26 Jun 2024 11:00:57 +0100 Subject: [PATCH 201/243] fix(manifests): fix sci config in example manifests --- manifests/bugs/aggregation-error-wrong-metric.yml | 3 --- manifests/bugs/pipeline-ordering-error.yml | 1 - manifests/examples/generics.yml | 1 - manifests/examples/pipeline-with-aggregate.yml | 1 - manifests/examples/pipeline-with-mocks.yml | 2 -- manifests/plugins/sci/failure-missing-input-param.yml | 8 ++------ manifests/plugins/sci/success.yml | 3 --- 7 files changed, 2 insertions(+), 17 deletions(-) diff --git a/manifests/bugs/aggregation-error-wrong-metric.yml b/manifests/bugs/aggregation-error-wrong-metric.yml index f95a9eed7..536508b59 100644 --- a/manifests/bugs/aggregation-error-wrong-metric.yml +++ b/manifests/bugs/aggregation-error-wrong-metric.yml @@ -34,7 +34,6 @@ initialize: method: Sci global-config: functional-unit: "requests" - functional-unit-time: "1 minute" "time-sync": method: TimeSync path: "builtin" @@ -69,7 +68,6 @@ tree: device/expected-lifespan: 94608000 # 3 years in seconds resources-reserved: 1 resources-total: 8 - functional-unit-time: "1 min" inputs: - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 @@ -112,7 +110,6 @@ tree: device/expected-lifespan: 94608000 # 3 years in seconds resources-reserved: 1 resources-total: 8 - functional-unit-time: "1 min" inputs: - timestamp: "2023-12-12T00:00:00.000Z" duration: 1 diff --git a/manifests/bugs/pipeline-ordering-error.yml b/manifests/bugs/pipeline-ordering-error.yml index 554134287..a561ea24d 100644 --- a/manifests/bugs/pipeline-ordering-error.yml +++ b/manifests/bugs/pipeline-ordering-error.yml @@ -30,7 +30,6 @@ initialize: method: Sci global-config: functional-unit: "" - functional-unit-time: "1-day" "time-sync": method: TimeSync path: "builtin" diff --git a/manifests/examples/generics.yml b/manifests/examples/generics.yml index c3ffa52e2..b3de5e007 100644 --- a/manifests/examples/generics.yml +++ b/manifests/examples/generics.yml @@ -33,7 +33,6 @@ tree: children: child-1: pipeline: - - teads-curve - sum - coefficient - multiply diff --git a/manifests/examples/pipeline-with-aggregate.yml b/manifests/examples/pipeline-with-aggregate.yml index d1b90b152..65d6c9d26 100644 --- a/manifests/examples/pipeline-with-aggregate.yml +++ b/manifests/examples/pipeline-with-aggregate.yml @@ -62,7 +62,6 @@ initialize: path: "builtin" method: Sci global-config: - functional-unit-time: 1 sec functional-unit: requests # factor to convert per time to per f.unit "sum-carbon": path: "builtin" diff --git a/manifests/examples/pipeline-with-mocks.yml b/manifests/examples/pipeline-with-mocks.yml index fde8963ef..801b7bfd9 100644 --- a/manifests/examples/pipeline-with-mocks.yml +++ b/manifests/examples/pipeline-with-mocks.yml @@ -129,7 +129,6 @@ tree: device/expected-lifespan: 94608000 # 3 years in seconds vcpus-total: 8 vcpus-allocated: 1 - functional-unit-time: "1 min" inputs: - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 @@ -177,7 +176,6 @@ tree: # device/expected-lifespan: 94608000 # 3 years in seconds # resources-reserved: 1 # resources-total: 8 - # functional-unit-time: "1 min" # inputs: # - timestamp: "2023-12-12T00:00:00.000Z" # duration: 1 diff --git a/manifests/plugins/sci/failure-missing-input-param.yml b/manifests/plugins/sci/failure-missing-input-param.yml index 44ced7374..54514a8e5 100644 --- a/manifests/plugins/sci/failure-missing-input-param.yml +++ b/manifests/plugins/sci/failure-missing-input-param.yml @@ -8,17 +8,13 @@ initialize: kind: plugin method: Sci path: "builtin" - # global-config: - # functional-unit: requests + global-config: + functional-unit: requests tree: children: child: pipeline: - sci - config: - sci: - functional-unit-time: 1 sec - functional-unit: requests # factor to convert per time to per f.unit inputs: - timestamp: 2023-07-06T00:00 duration: 3600 diff --git a/manifests/plugins/sci/success.yml b/manifests/plugins/sci/success.yml index c7e6dbde6..cddfe5755 100644 --- a/manifests/plugins/sci/success.yml +++ b/manifests/plugins/sci/success.yml @@ -16,9 +16,6 @@ tree: pipeline: - sci config: - sci: - functional-unit-time: 1 sec - functional-unit: requests # factor to convert per time to per f.unit inputs: - timestamp: 2023-07-06T00:00 duration: 3600 From a853e10f074516e57840888c27ec15d7b71baf5d Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Wed, 26 Jun 2024 11:03:33 +0100 Subject: [PATCH 202/243] fix(lib): fix sci config in csv export test --- src/__mocks__/builtins/export-csv.ts | 5 ----- src/__tests__/unit/builtins/export-csv-raw.test.ts | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/src/__mocks__/builtins/export-csv.ts b/src/__mocks__/builtins/export-csv.ts index e8fc2a26f..1f0e00e73 100644 --- a/src/__mocks__/builtins/export-csv.ts +++ b/src/__mocks__/builtins/export-csv.ts @@ -14,7 +14,6 @@ export const tree = { 'device/expected-lifespan': 94608000, 'resources-reserved': 1, 'resources-total': 8, - 'functional-unit-time': '1 min', }, inputs: [ { @@ -43,7 +42,6 @@ export const tree = { 'device/expected-lifespan': 94608000, 'resources-reserved': 1, 'resources-total': 8, - 'functional-unit-time': '1 min', 'cpu/energy': 0.000008888888888888888, "carbon-plus-energy'": 10.000008888888889, 'carbon-embodied': 0.0000020256215119228817, @@ -65,7 +63,6 @@ export const tree = { 'device/expected-lifespan': 94608000, 'resources-reserved': 1, 'resources-total': 8, - 'functional-unit-time': '1 min', }, inputs: [ { @@ -94,7 +91,6 @@ export const tree = { 'device/expected-lifespan': 94608000, 'resources-reserved': 1, 'resources-total': 8, - 'functional-unit-time': '1 min', 'cpu/energy': 0.00001650338753387534, "carbon-plus-energy'": 10.000016503387533, 'carbon-embodied': 0.0000020256215119228817, @@ -154,7 +150,6 @@ export const context: Context = { method: 'Sci', 'global-config': { 'functional-unit': 'requests', - 'functional-unit-time': '1 minute', }, }, }, diff --git a/src/__tests__/unit/builtins/export-csv-raw.test.ts b/src/__tests__/unit/builtins/export-csv-raw.test.ts index ab71e0e34..e97242a4d 100644 --- a/src/__tests__/unit/builtins/export-csv-raw.test.ts +++ b/src/__tests__/unit/builtins/export-csv-raw.test.ts @@ -35,7 +35,7 @@ describe('builtins/export-csv-raw: ', () => { it('generates CSV file with correct data.', async () => { const outputPath = 'output#carbon'; const content = - "id,timestamp,cloud/instance-type,region,duration,cpu/utilization,network/energy,energy,cpu/thermal-design-power,grid/carbon-intensity,device/emissions-embodied,time-reserved,device/expected-lifespan,resources-reserved,resources-total,functional-unit-time,cpu/energy,carbon-plus-energy',carbon-embodied,carbon-operational,carbon,sci\nchildren.child-1.outputs.0,2023-12-12T00:00:00.000Z,A1,uk-west,1,10,10,5,100,800,1533.12,3600,94608000,1,8,1 min,0.000008888888888888888,10.000008888888889,0.0000020256215119228817,4000,4000.0000020256216,240000.0001215373\nchildren.child-2.outputs.0,2023-12-12T00:00:00.000Z,A1,uk-west,1,30,10,5,100,800,1533.12,3600,94608000,1,8,1 min,0.00001650338753387534,10.000016503387533,0.0000020256215119228817,4000,4000.0000020256216,240000.0001215373\noutputs.0,2023-12-12T00:00:00.000Z,,,1,,,,,,,,,,,,,,,,8000.000004051243,"; + "id,timestamp,cloud/instance-type,region,duration,cpu/utilization,network/energy,energy,cpu/thermal-design-power,grid/carbon-intensity,device/emissions-embodied,time-reserved,device/expected-lifespan,resources-reserved,resources-total,cpu/energy,carbon-plus-energy',carbon-embodied,carbon-operational,carbon,sci\nchildren.child-1.outputs.0,2023-12-12T00:00:00.000Z,A1,uk-west,1,10,10,5,100,800,1533.12,3600,94608000,1,8,0.000008888888888888888,10.000008888888889,0.0000020256215119228817,4000,4000.0000020256216,240000.0001215373\nchildren.child-2.outputs.0,2023-12-12T00:00:00.000Z,A1,uk-west,1,30,10,5,100,800,1533.12,3600,94608000,1,8,0.00001650338753387534,10.000016503387533,0.0000020256215119228817,4000,4000.0000020256216,240000.0001215373\noutputs.0,2023-12-12T00:00:00.000Z,,,1,,,,,,,,,,,,,,,8000.000004051243,"; await exportCSVRaw.execute(tree, context, outputPath); From 992739668db3d399172c732a45855d3b9f796cd6 Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Wed, 26 Jun 2024 11:04:09 +0100 Subject: [PATCH 203/243] fix(lib): rm functional-unit-time from params --- src/config/params.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/config/params.ts b/src/config/params.ts index 377538391..76732e57b 100644 --- a/src/config/params.ts +++ b/src/config/params.ts @@ -62,12 +62,6 @@ export const PARAMETERS: Parameters = { unit: 'none', aggregation: 'sum', }, - 'functional-unit-time': { - description: - 'string describing the unit of time in which the final SCI calculation should be expressed, e.g. "1-min"', - unit: 'none', - aggregation: 'none', - }, 'gpu-util': { description: 'refers to CPU utilization.', unit: 'percentage', From 93769573c2741e9d8463e5edcface259f060a88a Mon Sep 17 00:00:00 2001 From: Joseph Cook <33655003+jmcook1186@users.noreply.github.com> Date: Wed, 26 Jun 2024 14:48:21 +0100 Subject: [PATCH 204/243] Update src/builtins/copy-param/README.md Co-authored-by: Manushak Keramyan Signed-off-by: Joseph Cook <33655003+jmcook1186@users.noreply.github.com> --- src/builtins/copy-param/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/builtins/copy-param/README.md b/src/builtins/copy-param/README.md index 1ec04f249..634b45448 100644 --- a/src/builtins/copy-param/README.md +++ b/src/builtins/copy-param/README.md @@ -2,7 +2,7 @@ `copy-param` is a generic plugin that duplicates an existing parameter in the `input` data and assigns it to a new key. You can either keep or delete the original copied parameter. A common use case for this is to rename parameters in the `inputs` array. -You provide the name of the value you want to copy, and a name to assign the copy to. You also toggle a `keep-existing` parameter to either persist or del;et the original copied value. +You provide the name of the value you want to copy, and a name to assign the copy to. You also toggle a `keep-existing` parameter to either persist or delete the original copied value. For example, you could copy `energy` into `energy-copy`, with `keep-existing=true`. In this case your inputs: From 700138c2035ca3403d49dd30361561bdf6097a80 Mon Sep 17 00:00:00 2001 From: Joseph Cook <33655003+jmcook1186@users.noreply.github.com> Date: Wed, 26 Jun 2024 14:48:28 +0100 Subject: [PATCH 205/243] Update src/builtins/copy-param/README.md Co-authored-by: Manushak Keramyan Signed-off-by: Joseph Cook <33655003+jmcook1186@users.noreply.github.com> --- src/builtins/copy-param/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/builtins/copy-param/README.md b/src/builtins/copy-param/README.md index 634b45448..78ab5f656 100644 --- a/src/builtins/copy-param/README.md +++ b/src/builtins/copy-param/README.md @@ -69,7 +69,7 @@ console.log(result) ## Example manifest -IF users will typically call the plugin as part of a pipeline defined in a manifest file. In this case, instantiating the plugin is handled by and does not have to be done explicitly by the user. The following is an example manifest that calls `sum`: +IF users will typically call the plugin as part of a pipeline defined in a manifest file. In this case, instantiating the plugin is handled by and does not have to be done explicitly by the user. The following is an example manifest that calls `copy-param`: ```yaml name: copy-param From e7034c50f1b6f4fb546b275f273d5d3f3ab623ee Mon Sep 17 00:00:00 2001 From: manushak Date: Wed, 26 Jun 2024 17:53:43 +0400 Subject: [PATCH 206/243] feat(util): add isDirectoryExists function --- src/util/fs.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/util/fs.ts b/src/util/fs.ts index 92ca9684a..70cec75e4 100644 --- a/src/util/fs.ts +++ b/src/util/fs.ts @@ -11,3 +11,15 @@ export const isFileExists = async (filePath: string) => { return false; } }; + +/** + * Checks if the directory exists with the given `filePath`. + */ +export const isDirectoryExists = async (directoryPath: string) => { + try { + await fs.access(directoryPath); + return true; + } catch (error) { + return false; + } +}; From ac56ed1150516f77a3df00a241484c15bb8d3fd7 Mon Sep 17 00:00:00 2001 From: manushak Date: Wed, 26 Jun 2024 18:12:46 +0400 Subject: [PATCH 207/243] fix(util): remove the node_modules folder if the package.json does not persist --- src/util/npm.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/util/npm.ts b/src/util/npm.ts index e453b771b..61baf75b8 100644 --- a/src/util/npm.ts +++ b/src/util/npm.ts @@ -4,7 +4,7 @@ import * as fs from 'fs/promises'; import * as path from 'path'; import {execPromise} from './helpers'; -import {isFileExists} from './fs'; +import {isDirectoryExists, isFileExists} from './fs'; import {logger} from './logger'; import {STRINGS} from '../config'; @@ -23,6 +23,14 @@ export const initPackageJsonIfNotExists = async (folderPath: string) => { if (!isPackageJsonExists) { logger.info(INITIALIZING_PACKAGE_JSON); + + const nodeModulesPath = path.resolve(folderPath, 'node_modules'); + const isNodeModulesExists = await isDirectoryExists(nodeModulesPath); + + if (isNodeModulesExists) { + await fs.rm(nodeModulesPath, {recursive: true}); + } + await execPromise('npm init -y', {cwd: folderPath}); } From 0258ec68afc31141a83fd87053eeb69118631ab7 Mon Sep 17 00:00:00 2001 From: MariamKhalatova Date: Thu, 27 Jun 2024 01:13:27 +0400 Subject: [PATCH 208/243] fix(manifests): update teads usage in bugs --- .../bugs/aggregation-error-wrong-metric.yml | 81 ++++++++++++++----- manifests/bugs/initialize-error-no-config.yml | 9 +-- manifests/bugs/initialize-error-no-path.yml | 14 ++-- .../bugs/input-error-missing-duration.yml | 16 ++-- .../bugs/pipeline-error-naming-mismatch.yml | 12 ++- .../pipeline-error-uninitialized-plugin.yml | 14 ++-- manifests/bugs/pipeline-ordering-error.yml | 75 +++++++++-------- 7 files changed, 140 insertions(+), 81 deletions(-) diff --git a/manifests/bugs/aggregation-error-wrong-metric.yml b/manifests/bugs/aggregation-error-wrong-metric.yml index 536508b59..60a8dd8dd 100644 --- a/manifests/bugs/aggregation-error-wrong-metric.yml +++ b/manifests/bugs/aggregation-error-wrong-metric.yml @@ -7,19 +7,48 @@ aggregation: type: "both" initialize: plugins: - "teads-curve": - path: "@grnsft/if-unofficial-plugins" - method: TeadsCurve + "interpolate": + method: Interpolation + path: 'builtin' global-config: - interpolation: spline - "sum-energy-components": + method: linear + x: [0, 10, 50, 100] + y: [0.12, 0.32, 0.75, 1.02] + input-parameter: 'cpu/utilization' + output-parameter: 'cpu-factor' + "cpu-factor-to-wattage": + method: Multiply + path: builtin + global-config: + input-parameters: ["cpu-factor", "cpu/thermal-design-power"] + output-parameter: "cpu-wattage" + "wattage-times-duration": + method: Multiply + path: builtin + global-config: + input-parameters: ["cpu-wattage", "duration"] + output-parameter: "cpu-wattage-times-duration" + "wattage-to-energy-kwh": + method: Divide path: "builtin" - method: Sum global-config: - input-parameters: - - cpu/energy - - network/energy - output-parameter: energy + numerator: cpu-wattage-times-duration + denominator: 3600000 + output: cpu-energy-raw + "calculate-vcpu-ratio": + method: Divide + path: "builtin" + global-config: + numerator: vcpus-total + denominator: vcpus-allocated + output: vcpu-ratio + "correct-cpu-energy-for-vcpu-ratio": + method: Divide + path: "builtin" + global-config: + numerator: cpu-energy-raw + denominator: vcpu-ratio + output: cpu-energy-kwh "sci-embodied": path: "builtin" method: SciEmbodied @@ -27,7 +56,7 @@ initialize: method: Multiply path: builtin global-config: - input-parameters: ["energy", "grid/carbon-intensity"] + input-parameters: ["cpu-energy-kwh", "grid/carbon-intensity"] output-parameter: "carbon-operational" "sci": path: "builtin" @@ -49,12 +78,16 @@ tree: children: child-1: pipeline: - - teads-curve - - sum-energy-components + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio - sci-embodied - operational-carbon - time-sync - - sci + # - sci config: group-by: group: @@ -66,8 +99,8 @@ tree: device/emissions-embodied: 1533.120 # gCO2eq time-reserved: 3600 # 1hr in seconds device/expected-lifespan: 94608000 # 3 years in seconds - resources-reserved: 1 - resources-total: 8 + vcpus-allocated: 1 + vcpus-total: 8 inputs: - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 @@ -91,12 +124,16 @@ tree: cpu/utilization: 15 child-2: pipeline: - - teads-curve - - sci-e + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio - sci-embodied - - sci-o + - operational-carbon - time-sync - - sci + # - sci config: group-by: group: @@ -108,8 +145,8 @@ tree: device/emissions-embodied: 1533.120 # gCO2eq time-reserved: 3600 # 1hr in seconds device/expected-lifespan: 94608000 # 3 years in seconds - resources-reserved: 1 - resources-total: 8 + vcpus-allocated: 1 + vcpus-total: 8 inputs: - timestamp: "2023-12-12T00:00:00.000Z" duration: 1 diff --git a/manifests/bugs/initialize-error-no-config.yml b/manifests/bugs/initialize-error-no-config.yml index 67973b53f..466c7f5f9 100644 --- a/manifests/bugs/initialize-error-no-config.yml +++ b/manifests/bugs/initialize-error-no-config.yml @@ -3,18 +3,17 @@ description: a negative test case that fails due to plugin initialization missin tags: initialize: plugins: - teads-curve: - path: '@grnsft/if-unofficial-plugins' - method: TeadsCurve + "interpolate": + method: Interpolation + path: 'builtin' global-config: - outputs: ['yaml'] tree: children: child-0: defaults: cpu/thermal-design-power: 100 pipeline: - - teads-curve + - interpolate inputs: - timestamp: 2023-07-06T00:00 duration: 1 diff --git a/manifests/bugs/initialize-error-no-path.yml b/manifests/bugs/initialize-error-no-path.yml index 21210c544..279f304e6 100644 --- a/manifests/bugs/initialize-error-no-path.yml +++ b/manifests/bugs/initialize-error-no-path.yml @@ -3,19 +3,23 @@ description: a negative test case that fails because the path is mising in a plu tags: initialize: plugins: - teads-curve: + "interpolate": + method: Interpolation path: - method: TeadsCurve global-config: - interpolation: spline - outputs: ['yaml'] + method: linear + x: [0, 10, 50, 100] + y: [0.12, 0.32, 0.75, 1.02] + input-parameter: 'cpu/utilization' + output-parameter: 'cpu-factor' + # outputs: ['yaml'] tree: children: child-0: defaults: cpu/thermal-design-power: 100 pipeline: - - teads-curve + - interpolate inputs: - timestamp: 2023-07-06T00:00 duration: 1 diff --git a/manifests/bugs/input-error-missing-duration.yml b/manifests/bugs/input-error-missing-duration.yml index 1a4a5d9ef..a2e7f3ca4 100644 --- a/manifests/bugs/input-error-missing-duration.yml +++ b/manifests/bugs/input-error-missing-duration.yml @@ -3,19 +3,23 @@ description: a negative test case that fails due to the required `duration` fiel tags: initialize: plugins: - teads-curve: - path: '@grnsft/if-unofficial-plugins' - method: TeadsCurve + "interpolate": + method: Interpolation + path: builtin global-config: - interpolation: spline - outputs: ['yaml'] + method: linear + x: [0, 10, 50, 100] + y: [0.12, 0.32, 0.75, 1.02] + input-parameter: 'cpu/utilization' + output-parameter: 'cpu-factor' + # outputs: ['yaml'] tree: children: child-0: defaults: cpu/thermal-design-power: 100 pipeline: - - teads-curve + - interpolate inputs: - timestamp: 2023-07-06T00:00 cpu/utilization: 20 diff --git a/manifests/bugs/pipeline-error-naming-mismatch.yml b/manifests/bugs/pipeline-error-naming-mismatch.yml index 22bb804a4..982e89088 100644 --- a/manifests/bugs/pipeline-error-naming-mismatch.yml +++ b/manifests/bugs/pipeline-error-naming-mismatch.yml @@ -3,11 +3,15 @@ description: a negative test case that fails due to the plugin name in the pipel tags: initialize: plugins: - teads-curve: - path: '@grnsft/if-unofficial-plugins' - method: TeadsCurve + "interpolate": + method: Interpolation + path: builtin global-config: - interpolation: spline + method: linear + x: [0, 10, 50, 100] + y: [0.12, 0.32, 0.75, 1.02] + input-parameter: 'cpu/utilization' + output-parameter: 'cpu-factor' outputs: ['yaml'] tree: children: diff --git a/manifests/bugs/pipeline-error-uninitialized-plugin.yml b/manifests/bugs/pipeline-error-uninitialized-plugin.yml index 95afc196b..7bead556b 100644 --- a/manifests/bugs/pipeline-error-uninitialized-plugin.yml +++ b/manifests/bugs/pipeline-error-uninitialized-plugin.yml @@ -4,18 +4,22 @@ tags: initialize: # outputs: ['yaml'] plugins: - teads-curve: - path: '@grnsft/if-unofficial-plugins' - method: TeadsCurve + "interpolate": + method: Interpolation + path: builtin global-config: - interpolation: spline + method: linear + x: [0, 10, 50, 100] + y: [0.12, 0.32, 0.75, 1.02] + input-parameter: 'cpu/utilization' + output-parameter: 'cpu-factor' tree: children: child-0: defaults: cpu/thermal-design-power: 100 pipeline: - - teads-curve + - interpolate - multiply inputs: - timestamp: 2023-07-06T00:00 diff --git a/manifests/bugs/pipeline-ordering-error.yml b/manifests/bugs/pipeline-ordering-error.yml index a561ea24d..7fa8836c1 100644 --- a/manifests/bugs/pipeline-ordering-error.yml +++ b/manifests/bugs/pipeline-ordering-error.yml @@ -3,51 +3,58 @@ description: a negative test case that fails because sci-o is invoked too early tags: initialize: plugins: - "teads-curve": - path: "@grnsft/if-unofficial-plugins" - method: TeadsCurve + "interpolate": + method: Interpolation + path: 'builtin' global-config: - interpolation: spline - "sum": - path: "builtin" - method: Sum + method: linear + x: [0, 10, 50, 100] + y: [0.12, 0.32, 0.75, 1.02] + input-parameter: 'cpu/utilization' + output-parameter: 'cpu-factor' + "cpu-factor-to-wattage": + method: Multiply + path: builtin global-config: - input-parameters: - - cpu/energy - - network/energy - output-parameter: energy - "sci-embodied": - path: "builtin" - method: SciEmbodied - "operational-carbon": + input-parameters: ["cpu-factor", "cpu/thermal-design-power"] + output-parameter: "cpu-wattage" + "wattage-times-duration": method: Multiply path: builtin global-config: - input-parameters: ["cpu/energy", "grid/carbon-intensity"] - output-parameter: "carbon-operational" - "sci": + input-parameters: ["cpu-wattage", "duration"] + output-parameter: "cpu-wattage-times-duration" + "wattage-to-energy-kwh": + method: Divide + path: "builtin" + global-config: + numerator: cpu-wattage-times-duration + denominator: 3600000 + output: cpu-energy-raw + "calculate-vcpu-ratio": + method: Divide path: "builtin" - method: Sci global-config: - functional-unit: "" - "time-sync": - method: TimeSync + numerator: vcpus-total + denominator: vcpus-allocated + output: vcpu-ratio + "correct-cpu-energy-for-vcpu-ratio": + method: Divide path: "builtin" global-config: - start-time: "2023-12-12T00:00:00.000Z" - end-time: "2023-12-12T00:01:00.000Z" - interval: 5 - allow-padding: true + numerator: cpu-energy-raw + denominator: vcpu-ratio + output: cpu-energy-kwh tree: children: child-1: pipeline: - - teads-curve - - operational-carbon - - sum - - sci-embodied - - time-sync - - sci + - interpolate + - correct-cpu-energy-for-vcpu-ratio + - calculate-vcpu-ratio + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh config: defaults: cpu/thermal-design-power: 100 @@ -55,8 +62,8 @@ tree: device/emissions-embodied: 1533.120 # gCO2eq time-reserved: 3600 # 1hr in seconds device/expected-lifespan: 94608000 # 3 years in seconds - resources-reserved: 1 - resources-total: 8 + vcpus-allocated: 1 + vcpus-total: 8 inputs: - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 From 4aff1f315ba9061ee13f29f03fe8d76d1ed32f16 Mon Sep 17 00:00:00 2001 From: MariamKhalatova Date: Thu, 27 Jun 2024 01:14:08 +0400 Subject: [PATCH 209/243] fix(manifests): update teads usage in examples --- manifests/examples/generics.yml | 60 ++++++++--- manifests/examples/nesting.yml | 116 +++++++++++++++------- manifests/examples/pipeline-teads-sci.yml | 63 +++++++++--- 3 files changed, 176 insertions(+), 63 deletions(-) diff --git a/manifests/examples/generics.yml b/manifests/examples/generics.yml index b3de5e007..c319967f0 100644 --- a/manifests/examples/generics.yml +++ b/manifests/examples/generics.yml @@ -3,24 +3,53 @@ description: a pipeline that does arbitrary calculations using our generic arith tags: initialize: plugins: - teads-curve: - path: "@grnsft/if-unofficial-plugins" - method: TeadsCurve + "interpolate": + method: Interpolation + path: 'builtin' global-config: - interpolation: spline - "sum": + method: linear + x: [0, 10, 50, 100] + y: [0.12, 0.32, 0.75, 1.02] + input-parameter: 'cpu/utilization' + output-parameter: 'cpu-factor' + "cpu-factor-to-wattage": + method: Multiply + path: builtin + global-config: + input-parameters: ["cpu-factor", "cpu/thermal-design-power"] + output-parameter: "cpu-wattage" + "wattage-times-duration": + method: Multiply + path: builtin + global-config: + input-parameters: ["cpu-wattage", "duration"] + output-parameter: "cpu-wattage-times-duration" + "wattage-to-energy-kwh": + method: Divide + path: "builtin" + global-config: + numerator: cpu-wattage-times-duration + denominator: 3600000 + output: cpu-energy-raw + "calculate-vcpu-ratio": + method: Divide + path: "builtin" + global-config: + numerator: vcpus-total + denominator: vcpus-allocated + output: vcpu-ratio + "correct-cpu-energy-for-vcpu-ratio": + method: Divide path: "builtin" - method: Sum global-config: - input-parameters: - - cpu/energy - - network/energy - output-parameter: energy-sum + numerator: cpu-energy-raw + denominator: vcpu-ratio + output: cpu-energy-kwh "coefficient": path: "builtin" method: Coefficient global-config: - input-parameter: energy + input-parameter: cpu-energy-kwh coefficient: 2 output-parameter: energy-doubled "multiply": @@ -33,12 +62,19 @@ tree: children: child-1: pipeline: - - sum + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio - coefficient - multiply config: defaults: cpu/thermal-design-power: 100 + vcpus-allocated: 1 + vcpus-total: 8 inputs: - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 diff --git a/manifests/examples/nesting.yml b/manifests/examples/nesting.yml index a5bf89230..b274ae7f0 100644 --- a/manifests/examples/nesting.yml +++ b/manifests/examples/nesting.yml @@ -7,25 +7,53 @@ tags: aggregation: metrics: - "carbon" - - "energy" type: "both" params: initialize: - outputs: ['yaml'] + # outputs: ['yaml'] plugins: - teads-curve: - path: "@grnsft/if-unofficial-plugins" - method: TeadsCurve + "interpolate": + method: Interpolation + path: 'builtin' global-config: - interpolation: spline - sum: + method: linear + x: [0, 10, 50, 100] + y: [0.12, 0.32, 0.75, 1.02] + input-parameter: 'cpu/utilization' + output-parameter: 'cpu-factor' + "cpu-factor-to-wattage": + method: Multiply + path: builtin + global-config: + input-parameters: ["cpu-factor", "cpu/thermal-design-power"] + output-parameter: "cpu-wattage" + "wattage-times-duration": + method: Multiply + path: builtin + global-config: + input-parameters: ["cpu-wattage", "duration"] + output-parameter: "cpu-wattage-times-duration" + "wattage-to-energy-kwh": + method: Divide path: "builtin" - method: Sum global-config: - input-parameters: - - cpu/energy - - network/energy - output-parameter: energy + numerator: cpu-wattage-times-duration + denominator: 3600000 + output: cpu-energy-raw + "calculate-vcpu-ratio": + method: Divide + path: "builtin" + global-config: + numerator: vcpus-total + denominator: vcpus-allocated + output: vcpu-ratio + "correct-cpu-energy-for-vcpu-ratio": + method: Divide + path: "builtin" + global-config: + numerator: cpu-energy-raw + denominator: vcpu-ratio + output: cpu-energy-kwh sci-embodied: path: "builtin" method: SciEmbodied @@ -33,7 +61,7 @@ initialize: method: Multiply path: builtin global-config: - input-parameters: ["energy", "grid/carbon-intensity"] + input-parameters: ["cpu-energy-kwh", "grid/carbon-intensity"] output-parameter: "carbon-operational" sci: path: "builtin" @@ -65,16 +93,20 @@ tree: device/emissions-embodied: 1533.120 # gCO2eq time-reserved: 3600 # 1hr in seconds device/expected-lifespan: 94608000 # 3 years in seconds - resources-reserved: 1 - resources-total: 8 + vcpus-allocated: 1 + vcpus-total: 8 pipeline: - - teads-curve - - sum + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio - sci-embodied - operational-carbon - sum-carbon - time-sync - - sci + # - sci inputs: - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 @@ -111,16 +143,20 @@ tree: device/emissions-embodied: 1533.120 # gCO2eq time-reserved: 3600 # 1hr in seconds device/expected-lifespan: 94608000 # 3 years in seconds - resources-reserved: 1 - resources-total: 8 + vcpus-allocated: 1 + vcpus-total: 8 pipeline: - - teads-curve - - sum + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio - sci-embodied - - sci-o + - operational-carbon - sum-carbon - time-sync - - sci + # - sci inputs: - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 @@ -159,16 +195,20 @@ tree: device/emissions-embodied: 1533.120 # gCO2eq time-reserved: 3600 # 1hr in seconds device/expected-lifespan: 94608000 # 3 years in seconds - resources-reserved: 1 - resources-total: 8 + vcpus-allocated: 1 + vcpus-total: 8 pipeline: - - teads-curve - - sum + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio - sci-embodied - - sci-o + - operational-carbon - sum-carbon - time-sync - - sci + # - sci inputs: - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 @@ -205,16 +245,20 @@ tree: device/emissions-embodied: 1533.120 # gCO2eq time-reserved: 3600 # 1hr in seconds device/expected-lifespan: 94608000 # 3 years in seconds - resources-reserved: 1 - resources-total: 8 + vcpus-allocated: 1 + vcpus-total: 8 pipeline: - - teads-curve - - sum + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio - sci-embodied - - sci-o + - operational-carbon - sum-carbon - time-sync - - sci + # - sci inputs: - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 diff --git a/manifests/examples/pipeline-teads-sci.yml b/manifests/examples/pipeline-teads-sci.yml index bf20cc251..c07b093bd 100644 --- a/manifests/examples/pipeline-teads-sci.yml +++ b/manifests/examples/pipeline-teads-sci.yml @@ -3,19 +3,48 @@ description: a full pipeline seeded with some hardcoded input data and yielding tags: initialize: plugins: - "teads-curve": - path: "@grnsft/if-unofficial-plugins" - method: TeadsCurve + "interpolate": + method: Interpolation + path: 'builtin' global-config: - interpolation: spline - "sum": + method: linear + x: [0, 10, 50, 100] + y: [0.12, 0.32, 0.75, 1.02] + input-parameter: 'cpu/utilization' + output-parameter: 'cpu-factor' + "cpu-factor-to-wattage": + method: Multiply + path: builtin + global-config: + input-parameters: ["cpu-factor", "cpu/thermal-design-power"] + output-parameter: "cpu-wattage" + "wattage-times-duration": + method: Multiply + path: builtin + global-config: + input-parameters: ["cpu-wattage", "duration"] + output-parameter: "cpu-wattage-times-duration" + "wattage-to-energy-kwh": + method: Divide path: "builtin" - method: Sum global-config: - input-parameters: - - cpu/energy - - network/energy - output-parameter: energy + numerator: cpu-wattage-times-duration + denominator: 3600000 + output: cpu-energy-raw + "calculate-vcpu-ratio": + method: Divide + path: "builtin" + global-config: + numerator: vcpus-total + denominator: vcpus-allocated + output: vcpu-ratio + "correct-cpu-energy-for-vcpu-ratio": + method: Divide + path: "builtin" + global-config: + numerator: cpu-energy-raw + denominator: vcpu-ratio + output: cpu-energy-kwh "sci-embodied": path: "builtin" method: SciEmbodied @@ -23,7 +52,7 @@ initialize: method: Multiply path: builtin global-config: - input-parameters: ["cpu/energy", "grid/carbon-intensity"] + input-parameters: ["cpu-energy-kwh", "grid/carbon-intensity"] output-parameter: "carbon-operational" "sci": path: "builtin" @@ -50,8 +79,12 @@ tree: children: child-1: pipeline: - - teads-curve - - sum + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio - sci-embodied - operational-carbon - sum-carbon @@ -63,8 +96,8 @@ tree: device/emissions-embodied: 1533.120 # gCO2eq time-reserved: 3600 # 1hr in seconds device/expected-lifespan: 94608000 # 3 years in seconds - resources-reserved: 1 - resources-total: 8 + vcpus-total: 8 + vcpus-allocated: 1 component: 1 inputs: - timestamp: "2023-12-12T00:00:00.000Z" From 366899e2ca29f7e73d7ec581826852dc29209014 Mon Sep 17 00:00:00 2001 From: MariamKhalatova Date: Thu, 27 Jun 2024 01:14:19 +0400 Subject: [PATCH 210/243] fix(manifests): update teads usage in integrations --- manifests/integrations/mock-obs-time-sync.yml | 55 +++++++++++++++++-- 1 file changed, 50 insertions(+), 5 deletions(-) diff --git a/manifests/integrations/mock-obs-time-sync.yml b/manifests/integrations/mock-obs-time-sync.yml index 727caf363..f1f176eb3 100644 --- a/manifests/integrations/mock-obs-time-sync.yml +++ b/manifests/integrations/mock-obs-time-sync.yml @@ -21,11 +21,48 @@ initialize: cpu/utilization: min: 1 max: 99 - "teads-curve": - path: "@grnsft/if-unofficial-plugins" - method: TeadsCurve + "interpolate": + method: Interpolation + path: 'builtin' global-config: - interpolation: spline + method: linear + x: [0, 10, 50, 100] + y: [0.12, 0.32, 0.75, 1.02] + input-parameter: 'cpu/utilization' + output-parameter: 'cpu-factor' + "cpu-factor-to-wattage": + method: Multiply + path: builtin + global-config: + input-parameters: ["cpu-factor", "cpu/thermal-design-power"] + output-parameter: "cpu-wattage" + "wattage-times-duration": + method: Multiply + path: builtin + global-config: + input-parameters: ["cpu-wattage", "duration"] + output-parameter: "cpu-wattage-times-duration" + "wattage-to-energy-kwh": + method: Divide + path: "builtin" + global-config: + numerator: cpu-wattage-times-duration + denominator: 3600000 + output: cpu-energy-raw + "calculate-vcpu-ratio": + method: Divide + path: "builtin" + global-config: + numerator: vcpus-total + denominator: vcpus-allocated + output: vcpu-ratio + "correct-cpu-energy-for-vcpu-ratio": + method: Divide + path: "builtin" + global-config: + numerator: cpu-energy-raw + denominator: vcpu-ratio + output: cpu-energy-kwh "time-sync": method: TimeSync path: "builtin" @@ -38,10 +75,18 @@ tree: children: child-1: pipeline: - - teads-curve + # - mock-observations + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio - time-sync defaults: cpu/thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 1 inputs: - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 From 448ecbad0a6207951adf6a61df5560f56aeee5d4 Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 27 Jun 2024 12:21:59 +0400 Subject: [PATCH 211/243] feat(types): add IFCheckArgs type --- src/types/process-args.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/types/process-args.ts b/src/types/process-args.ts index 40dece54d..0f3799f5a 100644 --- a/src/types/process-args.ts +++ b/src/types/process-args.ts @@ -17,6 +17,11 @@ export interface IFEnvArgs { cwd?: boolean; } +export interface IFCheckArgs { + manifest?: string; + directory?: string; +} + export interface Options { outputPath?: string; stdout?: boolean; From 61d727862bf71e083da21f9e95b69a359563f46c Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 27 Jun 2024 12:24:38 +0400 Subject: [PATCH 212/243] feat(config): add IF_CHECK config --- src/config/config.ts | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/src/config/config.ts b/src/config/config.ts index 0f96019f2..b8f676924 100644 --- a/src/config/config.ts +++ b/src/config/config.ts @@ -2,7 +2,12 @@ import {ArgumentConfig, ParseOptions} from 'ts-command-line-args'; import {STRINGS} from './strings'; -import {IFDiffArgs, IEArgs, IFEnvArgs} from '../types/process-args'; +import { + IFDiffArgs, + IEArgs, + IFEnvArgs, + IFCheckArgs, +} from '../types/process-args'; const {DISCLAIMER_MESSAGE} = STRINGS; @@ -120,6 +125,31 @@ export const CONFIG = { 'Faied to create the environment with the template manifest!', FAILURE_MESSAGE_DEPENDENCIES: 'Manifest dependencies are not available!', }, + IF_CHECK: { + ARGS: { + manifest: { + type: String, + optional: true, + alias: 'm', + description: '[path to the manifest file]', + }, + directory: { + type: String, + optional: true, + alias: 'd', + description: '[path to the manifests directory]', + }, + } as ArgumentConfig, + HELP: { + helpArg: 'help', + headerContentSections: [ + {header: 'Impact Framework', content: 'IF-Check Helpful keywords:'}, + ], + footerContentSections: [ + {header: 'Green Software Foundation', content: DISCLAIMER_MESSAGE}, + ], + } as ParseOptions, + }, GITHUB_PATH: 'https://github.com', NATIVE_PLUGIN: 'if-plugins', AGGREGATION_ADDITIONAL_PARAMS: ['timestamp', 'duration'], From 5dfde08d23a0fe7e4c8b6f1b2c975f1334e43c04 Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 27 Jun 2024 12:26:10 +0400 Subject: [PATCH 213/243] test(mocks): add mock functions for fs --- src/__mocks__/fs/index.ts | 50 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/src/__mocks__/fs/index.ts b/src/__mocks__/fs/index.ts index 3853ad7ee..6d4f61b8b 100644 --- a/src/__mocks__/fs/index.ts +++ b/src/__mocks__/fs/index.ts @@ -135,3 +135,53 @@ export const stat = async (filePath: string) => { throw new Error('File not found.'); } }; + +export const access = async (directoryPath: string) => { + if (directoryPath === 'true') { + return true; + } else { + throw new Error('Directory not found.'); + } +}; + +export const unlink = async (filePath: string) => { + if (filePath === 'true') { + return; + } else { + throw new Error('File not found.'); + } +}; + +export const readdir = (directoryPath: string) => { + if (directoryPath.includes('mock-empty-directory')) { + return []; + } + + if (directoryPath.includes('mock-directory')) { + return ['file1.yaml', 'file2.yml', 'file3.txt']; + } + + if (directoryPath.includes('mock-sub-directory')) { + return ['subdir/file2.yml', 'file1.yaml']; + } + + return []; +}; + +export const lstat = (filePath: string) => { + if ( + filePath.includes('mock-directory') || + filePath.includes('mock-sub-directory/subdir') + ) { + return { + isDirectory: () => true, + }; + } + + if (filePath.includes('mock-file')) { + return { + isDirectory: () => false, + }; + } + return; +}; From e18cf4ec063f91b0f7c15d127c0518700168193e Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 27 Jun 2024 12:28:45 +0400 Subject: [PATCH 214/243] feat(util): add helpers functions --- src/util/fs.ts | 34 ++++++++++++++++++++++++++++++++++ src/util/helpers.ts | 11 +++++++++++ src/util/npm.ts | 42 +++++++++++++++++++++++++++++++++++++++++- 3 files changed, 86 insertions(+), 1 deletion(-) diff --git a/src/util/fs.ts b/src/util/fs.ts index 70cec75e4..7438ced61 100644 --- a/src/util/fs.ts +++ b/src/util/fs.ts @@ -1,4 +1,5 @@ import * as fs from 'fs/promises'; +import * as path from 'path'; /** * Checks if file exists with the given `filePath`. @@ -23,3 +24,36 @@ export const isDirectoryExists = async (directoryPath: string) => { return false; } }; + +/** + * Gets all files that have either .yml or .yaml extension in the given directory. + */ +export const getYamlFiles = async (directory: string) => { + let yamlFiles: string[] = []; + + const files = await fs.readdir(directory); + + for (const file of files) { + const fullPath = path.join(directory, file); + const stat = await fs.lstat(fullPath); + + if (stat.isDirectory()) { + yamlFiles = yamlFiles.concat(await getYamlFiles(fullPath)); + } else { + if (file.endsWith('.yml') || file.endsWith('.yaml')) { + yamlFiles.push(fullPath); + } + } + } + + return yamlFiles; +}; + +/** + * Gets fileName from the given path without an extension. + */ +export const getFileName = (filePath: string) => { + const baseName = path.basename(filePath); + const extension = path.extname(filePath); + return baseName.replace(extension, ''); +}; diff --git a/src/util/helpers.ts b/src/util/helpers.ts index e69b9e0e9..3273fa8bc 100644 --- a/src/util/helpers.ts +++ b/src/util/helpers.ts @@ -268,3 +268,14 @@ export const addTemplateManifest = async (destinationDir: string) => { process.exit(1); } }; + +/** + * Logs the failure message from the stdout of an error. + */ +export const logStdoutFailMessage = (error: any) => { + const stdout = error.stdout; + const logs = stdout.split('\n\n'); + const failMessage = logs[logs.length - 1]; + + console.log(failMessage); +}; diff --git a/src/util/npm.ts b/src/util/npm.ts index 61baf75b8..8c66d37b5 100644 --- a/src/util/npm.ts +++ b/src/util/npm.ts @@ -4,7 +4,7 @@ import * as fs from 'fs/promises'; import * as path from 'path'; import {execPromise} from './helpers'; -import {isDirectoryExists, isFileExists} from './fs'; +import {isDirectoryExists, getFileName, isFileExists} from './fs'; import {logger} from './logger'; import {STRINGS} from '../config'; @@ -144,3 +144,43 @@ export const updatePackageJsonProperties = async ( JSON.stringify(newPackageJson, null, 2) ); }; + +/** + * Executes a series of npm commands based on the provided manifest file. + */ +export const executeCommands = async (manifest: string, cwd: boolean) => { + // TODO: After release remove isGlobal and appropriate checks + const isGlobal = !!process.env.npm_config_global; + const manifestDirPath = path.dirname(manifest); + const manifestFileName = getFileName(manifest); + const executedManifest = path.join(manifestDirPath, `re-${manifestFileName}`); + const ifEnv = `${isGlobal ? 'if-env' : 'npm run if-env --'} -m ${manifest}`; + const ifEnvCommand = cwd ? `${ifEnv} -c` : ifEnv; + const ifRunCommand = `${ + isGlobal ? 'if-run' : 'npm run if-run --' + } -m ${manifest} -o ${executedManifest}`; + const ifDiffCommand = `${ + isGlobal ? 'if-diff' : 'npm run if-diff --' + } -s ${executedManifest}.yaml -t ${manifest}`; + const ttyCommand = " node -p 'Boolean(process.stdout.isTTY)'"; + + const result = await execPromise( + `${ifEnvCommand} && ${ifRunCommand} && ${ttyCommand} | ${ifDiffCommand}`, + { + cwd: process.env.CURRENT_DIR || process.cwd(), + } + ); + + if (!cwd) { + await fs.unlink(`${manifestDirPath}/package.json`); + } + + await fs.unlink(`${executedManifest}.yaml`); + + if (result.stdout) { + const logs = result.stdout.split('\n\n'); + const successMessage = logs[logs.length - 1]; + + console.log(successMessage); + } +}; From 94c3a4413381b43cd74938a03141111742f31ddc Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 27 Jun 2024 12:31:43 +0400 Subject: [PATCH 215/243] test(util): add tests for utils functions --- src/__tests__/unit/util/args.test.ts | 122 +++++++++++++++++++++++- src/__tests__/unit/util/fs.test.ts | 106 +++++++++++++++++++- src/__tests__/unit/util/helpers.test.ts | 14 ++- src/__tests__/unit/util/npm.test.ts | 29 +++++- 4 files changed, 265 insertions(+), 6 deletions(-) diff --git a/src/__tests__/unit/util/args.test.ts b/src/__tests__/unit/util/args.test.ts index 03752acc4..f4238cbce 100644 --- a/src/__tests__/unit/util/args.test.ts +++ b/src/__tests__/unit/util/args.test.ts @@ -7,6 +7,12 @@ jest.mock('../../../util/fs', () => ({ } return false; }, + isDirectoryExists: () => { + if (process.env.directoryExists === 'true') { + return true; + } + return false; + }, })); jest.mock('ts-command-line-args', () => ({ @@ -70,8 +76,6 @@ jest.mock('ts-command-line-args', () => ({ case 'diff-throw': throw 'mock-error'; /** If-env mocks */ - // case 'env-manifest-is-missing': - // return; case 'manifest-install-provided': return { install: true, @@ -85,6 +89,13 @@ jest.mock('ts-command-line-args', () => ({ throw new Error('mock-error'); case 'env-throw': throw 'mock-error'; + /** If-check */ + case 'manifest-is-provided': + return {manifest: 'mock-manifest.yaml'}; + case 'directory-is-provided': + return {directory: '/mock-directory'}; + case 'flags-are-not-provided': + return {manifest: undefined, directory: undefined}; default: return { manifest: 'mock-manifest.yaml', @@ -99,13 +110,19 @@ import {ERRORS} from '@grnsft/if-core/utils'; import { parseIEProcessArgs, + parseIfCheckArgs, parseIfDiffArgs, parseIfEnvArgs, } from '../../../util/args'; import {STRINGS} from '../../../config'; -const {CliSourceFileError, ParseCliParamsError} = ERRORS; +const { + CliSourceFileError, + ParseCliParamsError, + InvalidDirectoryError, + MissingCliFlagsError, +} = ERRORS; const { MANIFEST_IS_MISSING, @@ -113,6 +130,8 @@ const { INVALID_TARGET, SOURCE_IS_NOT_YAML, MANIFEST_NOT_FOUND, + DIRECTORY_NOT_FOUND, + IF_CHECK_FLAGS_MISSING, } = STRINGS; describe('util/args: ', () => { @@ -402,5 +421,102 @@ describe('util/args: ', () => { }); }); + describe('parseIfCheckArgs(): ', () => { + it('executes when `manifest` is provided.', async () => { + process.env.fileExists = 'true'; + process.env.result = 'manifest-is-provided'; + const response = await parseIfCheckArgs(); + + expect.assertions(1); + + expect(response).toEqual({manifest: 'mock-manifest.yaml'}); + }); + + it('executes when the `directory` is provided.', async () => { + process.env.directoryExists = 'true'; + process.env.result = 'directory-is-provided'; + + const response = await parseIfCheckArgs(); + + expect.assertions(1); + + expect(response).toEqual({directory: '/mock-directory'}); + }); + + it('throws an error when the `directory` does not exist.', async () => { + process.env.directoryExists = 'false'; + process.env.result = 'directory-is-provided'; + expect.assertions(1); + + try { + await parseIfCheckArgs(); + } catch (error) { + expect(error).toEqual(new InvalidDirectoryError(DIRECTORY_NOT_FOUND)); + } + }); + + it('throws an error when both `manifest` and `directory` flags are not provided.', async () => { + process.env.result = 'flags-are-not-provided'; + expect.assertions(1); + + try { + await parseIfCheckArgs(); + } catch (error) { + expect(error).toEqual(new MissingCliFlagsError(IF_CHECK_FLAGS_MISSING)); + } + }); + + it('throws an error if `manifest` is not a yaml.', async () => { + process.env.fileExists = 'true'; + process.env.result = 'manifest-is-not-yaml'; + expect.assertions(1); + + try { + await parseIfCheckArgs(); + } catch (error) { + if (error instanceof Error) { + expect(error).toEqual(new CliSourceFileError(SOURCE_IS_NOT_YAML)); + } + } + }); + + it('throws an error if `manifest` path is invalid.', async () => { + process.env.fileExists = 'false'; + expect.assertions(1); + + try { + await parseIfCheckArgs(); + } catch (error) { + if (error instanceof Error) { + expect(error).toEqual(new ParseCliParamsError(MANIFEST_NOT_FOUND)); + } + } + }); + + it('throws an error if parsing failed.', async () => { + process.env.result = 'env-throw-error'; + expect.assertions(1); + + try { + await parseIfCheckArgs(); + } catch (error) { + if (error instanceof Error) { + expect(error).toEqual(new ParseCliParamsError('mock-error')); + } + } + }); + + it('throws error if parsing failed (not instance of error).', async () => { + process.env.result = 'env-throw'; + expect.assertions(1); + + try { + await parseIfCheckArgs(); + } catch (error) { + expect(error).toEqual('mock-error'); + } + }); + }); + process.env = originalEnv; }); diff --git a/src/__tests__/unit/util/fs.test.ts b/src/__tests__/unit/util/fs.test.ts index 82f28bc75..15cb0103d 100644 --- a/src/__tests__/unit/util/fs.test.ts +++ b/src/__tests__/unit/util/fs.test.ts @@ -1,4 +1,11 @@ -import {isFileExists} from '../../../util/fs'; +import * as fs from 'fs/promises'; + +import { + getFileName, + isDirectoryExists, + isFileExists, + getYamlFiles, +} from '../../../util/fs'; jest.mock('fs/promises', () => require('../../../__mocks__/fs')); @@ -18,4 +25,101 @@ describe('util/fs: ', () => { expect(result).toEqual(false); }); }); + + describe('isDirectoryExists(): ', () => { + it('returns true if directory exists.', async () => { + const result = await isDirectoryExists('true'); + + expect.assertions(1); + expect(result).toEqual(true); + }); + + it('returns false if directory does not exist.', async () => { + const result = await isDirectoryExists('false'); + + expect.assertions(1); + expect(result).toEqual(false); + }); + }); + + describe('getFileName(): ', () => { + it('returns the file name without extension for a file with an extension.', () => { + const filePath = '/path/to/file/example.yaml'; + const result = getFileName(filePath); + + expect.assertions(1); + expect(result).toBe('example'); + }); + + it('returns the file name without extension for a file with multiple dots.', () => { + const filePath = '/path/to/file/example.test.yaml'; + const result = getFileName(filePath); + expect(result).toBe('example.test'); + }); + + it('returns the file name as is if there is no extension.', () => { + const filePath = '/path/to/file/example'; + const result = getFileName(filePath); + expect(result).toBe('example'); + }); + + it('handles file names with special characters.', () => { + const filePath = + '/path/to/file/complex-file.name.with-multiple.parts.yaml'; + const result = getFileName(filePath); + expect(result).toBe('complex-file.name.with-multiple.parts'); + }); + + it('handles file names with no path.', () => { + const filePath = 'example.yaml'; + const result = getFileName(filePath); + expect(result).toBe('example'); + }); + + it('handles empty string as file path.', () => { + const filePath = ''; + const result = getFileName(filePath); + expect(result).toBe(''); + }); + }); + + describe('getYamlFiles(): ', () => { + it('returns an empty array if the directory is empty.', async () => { + const fsReaddirSpy = jest.spyOn(fs, 'readdir'); + const result = await getYamlFiles('/mock-empty-directory'); + + expect(result).toEqual([]); + expect(fsReaddirSpy).toHaveBeenCalledWith('/mock-empty-directory'); + }); + + it('returns YAML files in the directory', async () => { + const fsReaddirSpy = jest.spyOn(fs, 'readdir'); + jest + .spyOn(fs, 'lstat') + .mockResolvedValue({isDirectory: () => false} as any); + + const result = await getYamlFiles('/mock-directory'); + expect.assertions(2); + expect(result).toEqual([ + '/mock-directory/file1.yaml', + '/mock-directory/file2.yml', + ]); + expect(fsReaddirSpy).toHaveBeenCalledWith('/mock-directory'); + }); + + it('recursively finds YAML files in nested directories.', async () => { + const fsReaddirSpy = jest.spyOn(fs, 'readdir'); + jest + .spyOn(fs, 'lstat') + .mockResolvedValue({isDirectory: () => false} as any); + const result = await getYamlFiles('/mock-sub-directory'); + + expect.assertions(2); + expect(result).toEqual([ + '/mock-sub-directory/subdir/file2.yml', + '/mock-sub-directory/file1.yaml', + ]); + expect(fsReaddirSpy).toHaveBeenCalledWith('/mock-directory'); + }); + }); }); diff --git a/src/__tests__/unit/util/helpers.test.ts b/src/__tests__/unit/util/helpers.test.ts index d04ad43d2..42ff9450b 100644 --- a/src/__tests__/unit/util/helpers.test.ts +++ b/src/__tests__/unit/util/helpers.test.ts @@ -146,7 +146,7 @@ import { getOptionsFromArgs, addTemplateManifest, initializeAndInstallLibs, - // initializeAndInstallLibs, + logStdoutFailMessage, } from '../../../util/helpers'; import {CONFIG} from '../../../config'; import {Difference} from '../../../types/lib/compare'; @@ -667,4 +667,16 @@ description: mock-description process.exit = originalProcessExit; }); }); + + describe('logStdoutFailMessage(): ', () => { + it('successfully logs the failed message.', () => { + const errorMessage = {stdout: '\n\nmock error message'}; + const logSpy = jest.spyOn(global.console, 'log'); + logStdoutFailMessage(errorMessage); + + expect.assertions(1); + + expect(logSpy).toHaveBeenCalledWith('mock error message'); + }); + }); }); diff --git a/src/__tests__/unit/util/npm.test.ts b/src/__tests__/unit/util/npm.test.ts index 403a1138c..0cf801583 100644 --- a/src/__tests__/unit/util/npm.test.ts +++ b/src/__tests__/unit/util/npm.test.ts @@ -21,6 +21,7 @@ import { updatePackageJsonDependencies, extractPathsWithVersion, updatePackageJsonProperties, + executeCommands, } from '../../../util/npm'; import {isFileExists} from '../../../util/fs'; @@ -176,7 +177,7 @@ describe('util/npm: ', () => { '@commitlint/cli@18.6.0', '@commitlint/config-conventional@18.6.0', '@grnsft/if-core@0.0.7', - '@grnsft/if-plugins@v0.3.2 extraneous -> file:../../../if-models', + '@grnsft/if-plugins@v0.3.2', '@grnsft/if-unofficial-plugins@v0.3.0 extraneous -> file:../../../if-unofficial-models', '@jest/globals@29.7.0', ]; @@ -246,4 +247,30 @@ describe('util/npm: ', () => { expect(fsReadSpy).toHaveBeenCalledWith(packageJsonPath, 'utf8'); }); }); + + describe('executeCommands(): ', () => { + it('successfully executes with correct commands.', async () => { + const manifest = './src/__mocks__/mock-manifest.yaml'; + const reManifest = 'src/__mocks__/re-mock-manifest.yaml'; + const command = `npm run if-env -- -m ${manifest} && npm run if-run -- -m ${manifest} -o ${reManifest.replace( + '.yaml', + '' + )} && node -p 'Boolean(process.stdout.isTTY)' | npm run if-diff -- -s ${reManifest} -t ${manifest}`; + const logSpy = jest.spyOn(global.console, 'log'); + const spyExecPromise = jest.spyOn(helpers, 'execPromise'); + jest.spyOn(fs, 'unlink').mockResolvedValue(); + + await executeCommands(manifest, false); + + expect.assertions(2); + expect(spyExecPromise).toHaveBeenCalledWith(command, { + cwd: process.cwd(), + }); + expect(logSpy).toHaveBeenCalledWith('Files match!\n'); + + const packageJsonPath = 'src/__mocks__/package.json'; + fsSync.unlink(path.resolve(process.cwd(), reManifest), () => {}); + fsSync.unlink(path.resolve(process.cwd(), packageJsonPath), () => {}); + }, 70000); + }); }); From f6296f41dd277824760d6cdd35b07f9ed8e7ae20 Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 27 Jun 2024 12:33:33 +0400 Subject: [PATCH 216/243] chore(config): add string --- src/config/strings.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/config/strings.ts b/src/config/strings.ts index 4da6defae..3a2fc84fe 100644 --- a/src/config/strings.ts +++ b/src/config/strings.ts @@ -132,4 +132,8 @@ ${error}`, `Exporting to csv file: ${savepath}`, EXPORTING_RAW_CSV_FILE: (savepath: string) => `Exporting raw csv file: ${savepath}`, + CHECKING: 'Checking...', + IF_CHECK_FLAGS_MISSING: + 'Either the `--manifest` or `--directory` command should be provided with a path', + DIRECTORY_NOT_FOUND: 'Directory not found.', }; From 24148df470471848188da93b1243a7bc79d82cdb Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 27 Jun 2024 12:35:46 +0400 Subject: [PATCH 217/243] feat(mocks): add mock manifest file --- src/__mocks__/mock-manifest.yaml | 72 ++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 src/__mocks__/mock-manifest.yaml diff --git a/src/__mocks__/mock-manifest.yaml b/src/__mocks__/mock-manifest.yaml new file mode 100644 index 000000000..1e38c1b13 --- /dev/null +++ b/src/__mocks__/mock-manifest.yaml @@ -0,0 +1,72 @@ +name: template manifest +description: auto-generated template +tags: null +initialize: + plugins: + memory-energy-from-memory-util: + path: builtin + method: Coefficient + global-config: + input-parameter: memory/utilization + coefficient: 0.0001 + output-parameter: memory/energy + outputs: + - yaml +execution: + command: >- + /Users/manushak/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/manushak/Documents/Projects/Green-Software/if/src/index.ts -m + ./src/env-template.yml -o ./manifests/outputs/template + environment: + if-version: 0.4.0 + os: macOS + os-version: 13.6.6 + node-version: 20.12.2 + date-time: 2024-06-18T08:39:55.771Z (UTC) + dependencies: + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.3" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" + - axios-mock-adapter@1.22.0 + - axios@1.7.2 + - cross-env@7.0.3 + - csv-parse@5.5.6 + - csv-stringify@6.4.6 + - fixpack@4.0.0 + - gts@5.2.0 + - husky@8.0.3 + - jest@29.7.0 + - js-yaml@4.1.0 + - lint-staged@15.2.2 + - luxon@3.4.4 + - release-it@16.3.0 + - rimraf@5.0.5 + - ts-command-line-args@2.5.1 + - ts-jest@29.1.1 + - typescript-cubic-spline@1.0.1 + - typescript@5.2.2 + - winston@3.11.0 + - zod@3.22.4 + status: success +tree: + children: + child: + pipeline: + - memory-energy-from-memory-util + config: null + inputs: + - timestamp: 2023-12-12T00:00:00.000Z + duration: 3600 + memory/utilization: 10 + outputs: + - timestamp: 2023-12-12T00:00:00.000Z + duration: 3600 + memory/utilization: 10 + memory/energy: 0.001 From 5e9ae199a6299b116419ce14cb9e3b0edd32172f Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 27 Jun 2024 13:00:55 +0400 Subject: [PATCH 218/243] feat(util): add if-check argument parsing functions --- src/util/args.ts | 66 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 63 insertions(+), 3 deletions(-) diff --git a/src/util/args.ts b/src/util/args.ts index 5d9524390..31da90461 100644 --- a/src/util/args.ts +++ b/src/util/args.ts @@ -5,7 +5,7 @@ import {ERRORS} from '@grnsft/if-core/utils'; import {checkIfFileIsYaml} from './yaml'; -import {isFileExists} from './fs'; +import {isDirectoryExists, isFileExists} from './fs'; import {logger} from './logger'; @@ -16,12 +16,19 @@ import { IEArgs, ProcessArgsOutputs, IFEnvArgs, + IFCheckArgs, } from '../types/process-args'; import {LoadDiffParams} from '../types/util/args'; -const {ParseCliParamsError, CliTargetFileError, CliSourceFileError} = ERRORS; +const { + ParseCliParamsError, + CliTargetFileError, + CliSourceFileError, + InvalidDirectoryError, + MissingCliFlagsError, +} = ERRORS; -const {IE, IF_DIFF, IF_ENV} = CONFIG; +const {IE, IF_DIFF, IF_ENV, IF_CHECK} = CONFIG; const { MANIFEST_IS_MISSING, @@ -30,6 +37,8 @@ const { SOURCE_IS_NOT_YAML, TARGET_IS_NOT_YAML, INVALID_TARGET, + IF_CHECK_FLAGS_MISSING, + DIRECTORY_NOT_FOUND, } = STRINGS; /** @@ -186,3 +195,54 @@ export const parseIfEnvArgs = async () => { return {install, cwd}; }; + +/** -- IF Check -- */ + +/** + * Parses `if-check` process arguments. + */ +const validateAndParseIfCheckArgs = () => { + try { + return parse(IF_CHECK.ARGS, IF_CHECK.HELP); + } catch (error) { + if (error instanceof Error) { + throw new ParseCliParamsError(error.message); + } + + throw error; + } +}; + +/** + * Checks if either `manifest` or `directory` command is provided. + */ +export const parseIfCheckArgs = async () => { + const {manifest, directory} = validateAndParseIfCheckArgs(); + + if (manifest) { + const response = prependFullFilePath(manifest); + const isManifestFileExists = await isFileExists(response); + + if (!isManifestFileExists) { + throw new ParseCliParamsError(MANIFEST_NOT_FOUND); + } + + if (checkIfFileIsYaml(manifest)) { + return {manifest}; + } + + throw new CliSourceFileError(SOURCE_IS_NOT_YAML); + } else if (directory) { + const isDirExists = await isDirectoryExists(directory); + + if (!isDirExists) { + throw new InvalidDirectoryError(DIRECTORY_NOT_FOUND); + } + + const response = prependFullFilePath(directory); + + return {directory: response}; + } + + throw new MissingCliFlagsError(IF_CHECK_FLAGS_MISSING); +}; From 5948600ebee8441d8c3fcfd54fcf2a26ba5970d8 Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 27 Jun 2024 13:03:09 +0400 Subject: [PATCH 219/243] feat(src): add if-check logic --- src/check.ts | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 src/check.ts diff --git a/src/check.ts b/src/check.ts new file mode 100644 index 000000000..56e659efb --- /dev/null +++ b/src/check.ts @@ -0,0 +1,62 @@ +#!/usr/bin/env node +/* eslint-disable no-process-exit */ +import * as fs from 'fs/promises'; +import * as path from 'path'; + +import {logger} from './util/logger'; +import {parseIfCheckArgs} from './util/args'; +import {logStdoutFailMessage} from './util/helpers'; +import {getYamlFiles} from './util/fs'; + +import {STRINGS} from './config'; +import {executeCommands} from './util/npm'; + +const {CHECKING} = STRINGS; + +const IfCheck = async () => { + const commandArgs = await parseIfCheckArgs(); + + console.log(`${CHECKING}\n`); + + if (commandArgs.manifest) { + const manifest = commandArgs.manifest; + + try { + await executeCommands(manifest, false); + } catch (error: any) { + const fileName = path.basename(manifest); + const executedFile = manifest.replace(fileName, `re-${fileName}`); + const manifestDirPath = path.dirname(manifest); + + logStdoutFailMessage(error); + + await fs.unlink(`${manifestDirPath}/package.json`); + await fs.unlink(executedFile); + } + } else { + const directory = commandArgs.directory; + const files = await getYamlFiles(directory!); + + for await (const file of files) { + const fileName = path.basename(file); + console.log(fileName); + + try { + await executeCommands(file, true); + } catch (error: any) { + const fileName = path.basename(file); + const executedFile = file.replace(fileName, `re-${fileName}`); + + logStdoutFailMessage(error); + await fs.unlink(executedFile); + } + } + } +}; + +IfCheck().catch(error => { + if (error instanceof Error) { + logger.error(error); + process.exit(2); + } +}); From 850ea050c2b61d2098815aed1455251184832e8b Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 27 Jun 2024 13:06:32 +0400 Subject: [PATCH 220/243] feat(package): add if-check script and update if-core version --- package-lock.json | 9 +++++---- package.json | 6 ++++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3e66c802a..9d64c6b7b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,7 @@ "dependencies": { "@commitlint/cli": "^18.6.0", "@commitlint/config-conventional": "^18.6.0", - "@grnsft/if-core": "^0.0.9", + "@grnsft/if-core": "^0.0.10", "axios": "^1.7.2", "csv-parse": "^5.5.6", "csv-stringify": "^6.4.6", @@ -24,6 +24,7 @@ "zod": "^3.22.4" }, "bin": { + "if-check": "build/check.js", "if-diff": "build/diff.js", "if-env": "build/env.js", "if-run": "build/index.js" @@ -1183,9 +1184,9 @@ } }, "node_modules/@grnsft/if-core": { - "version": "0.0.9", - "resolved": "https://registry.npmjs.org/@grnsft/if-core/-/if-core-0.0.9.tgz", - "integrity": "sha512-F0niYe1j+NfhH+okz5sjP/CD7w/9BeXXe13bHkCsdOnk0WfXrq0DGKwpqh9TiLbKc9f1P3/XuojeFANMQu5nig==", + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/@grnsft/if-core/-/if-core-0.0.10.tgz", + "integrity": "sha512-WHCdr7H/dFO9gT5fbjrthjOU+4RoLZ5P1F84pbGwJiKLmcU7dvYRuNQKDVIQQ7YJfZl76KSaS7sYgqA+QG8Wpw==", "dependencies": { "typescript": "^5.1.6" }, diff --git a/package.json b/package.json index e9f2f2777..0d965de37 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,8 @@ "bin": { "if-diff": "./build/diff.js", "if-run": "./build/index.js", - "if-env": "./build/env.js" + "if-env": "./build/env.js", + "if-check": "./build/check.js" }, "bugs": { "url": "https://github.com/Green-Software-Foundation/if/issues/new?assignees=&labels=feedback&projects=&template=feedback.md&title=Feedback+-+" @@ -17,7 +18,7 @@ "dependencies": { "@commitlint/cli": "^18.6.0", "@commitlint/config-conventional": "^18.6.0", - "@grnsft/if-core": "^0.0.9", + "@grnsft/if-core": "^0.0.10", "axios": "^1.7.2", "csv-parse": "^5.5.6", "csv-stringify": "^6.4.6", @@ -75,6 +76,7 @@ "coverage": "jest --verbose --coverage --testPathPattern=src/__tests__/unit", "fix": "gts fix", "fix:package": "fixpack", + "if-check": "cross-env CURRENT_DIR=$(node -p \"process.env.INIT_CWD\") npx ts-node src/check.ts", "if-diff": "npx ts-node src/diff.ts", "if-env": "cross-env CURRENT_DIR=$(node -p \"process.env.INIT_CWD\") npx ts-node src/env.ts", "if-run": "npx ts-node src/index.ts", From d51b9a077e70bce59dfe55e02021677bbd11c250 Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 27 Jun 2024 16:26:27 +0400 Subject: [PATCH 221/243] fix(src): fix issue realted to remove created temp re-manifest file during if-check execution --- src/check.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/check.ts b/src/check.ts index 56e659efb..887ff13f6 100644 --- a/src/check.ts +++ b/src/check.ts @@ -25,7 +25,9 @@ const IfCheck = async () => { await executeCommands(manifest, false); } catch (error: any) { const fileName = path.basename(manifest); - const executedFile = manifest.replace(fileName, `re-${fileName}`); + const executedFile = manifest + .replace(fileName, `re-${fileName}`) + .replace('yml', 'yaml'); const manifestDirPath = path.dirname(manifest); logStdoutFailMessage(error); @@ -45,7 +47,9 @@ const IfCheck = async () => { await executeCommands(file, true); } catch (error: any) { const fileName = path.basename(file); - const executedFile = file.replace(fileName, `re-${fileName}`); + const executedFile = file + .replace(fileName, `re-${fileName}`) + .replace('yml', 'yaml'); logStdoutFailMessage(error); await fs.unlink(executedFile); From f2a371ac44d70abaa4c792925bf915245c8fab06 Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 27 Jun 2024 18:43:51 +0400 Subject: [PATCH 222/243] feat(config): add messages for if-check fail and verify --- src/config/strings.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/config/strings.ts b/src/config/strings.ts index 3a2fc84fe..e55eb1b9c 100644 --- a/src/config/strings.ts +++ b/src/config/strings.ts @@ -136,4 +136,8 @@ ${error}`, IF_CHECK_FLAGS_MISSING: 'Either the `--manifest` or `--directory` command should be provided with a path', DIRECTORY_NOT_FOUND: 'Directory not found.', + IF_CHECK_FAILED: (filename: string) => + `if-check could not verify <${filename}>. The re-executed file does not match the original.\n`, + IF_CHECK_VERIFIED: (filename: string) => + `if-check successfully verified <${filename}>\n`, }; From 9f1c32ecdb432ec6e3aa00659daa0f1836a57ae9 Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 27 Jun 2024 18:45:45 +0400 Subject: [PATCH 223/243] feat(util): remove logStdoutFailMessage function --- src/__tests__/unit/util/helpers.test.ts | 13 ------------- src/util/helpers.ts | 11 ----------- 2 files changed, 24 deletions(-) diff --git a/src/__tests__/unit/util/helpers.test.ts b/src/__tests__/unit/util/helpers.test.ts index 42ff9450b..4e0422403 100644 --- a/src/__tests__/unit/util/helpers.test.ts +++ b/src/__tests__/unit/util/helpers.test.ts @@ -146,7 +146,6 @@ import { getOptionsFromArgs, addTemplateManifest, initializeAndInstallLibs, - logStdoutFailMessage, } from '../../../util/helpers'; import {CONFIG} from '../../../config'; import {Difference} from '../../../types/lib/compare'; @@ -667,16 +666,4 @@ description: mock-description process.exit = originalProcessExit; }); }); - - describe('logStdoutFailMessage(): ', () => { - it('successfully logs the failed message.', () => { - const errorMessage = {stdout: '\n\nmock error message'}; - const logSpy = jest.spyOn(global.console, 'log'); - logStdoutFailMessage(errorMessage); - - expect.assertions(1); - - expect(logSpy).toHaveBeenCalledWith('mock error message'); - }); - }); }); diff --git a/src/util/helpers.ts b/src/util/helpers.ts index 3273fa8bc..e69b9e0e9 100644 --- a/src/util/helpers.ts +++ b/src/util/helpers.ts @@ -268,14 +268,3 @@ export const addTemplateManifest = async (destinationDir: string) => { process.exit(1); } }; - -/** - * Logs the failure message from the stdout of an error. - */ -export const logStdoutFailMessage = (error: any) => { - const stdout = error.stdout; - const logs = stdout.split('\n\n'); - const failMessage = logs[logs.length - 1]; - - console.log(failMessage); -}; From 03ce50b7fec0f9529a0dbae592828c43f7493518 Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 27 Jun 2024 18:48:32 +0400 Subject: [PATCH 224/243] test(util): update log message --- src/__tests__/unit/util/npm.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/__tests__/unit/util/npm.test.ts b/src/__tests__/unit/util/npm.test.ts index 0cf801583..d73095d93 100644 --- a/src/__tests__/unit/util/npm.test.ts +++ b/src/__tests__/unit/util/npm.test.ts @@ -266,7 +266,9 @@ describe('util/npm: ', () => { expect(spyExecPromise).toHaveBeenCalledWith(command, { cwd: process.cwd(), }); - expect(logSpy).toHaveBeenCalledWith('Files match!\n'); + expect(logSpy).toHaveBeenCalledWith( + 'if-check successfully verified \n' + ); const packageJsonPath = 'src/__mocks__/package.json'; fsSync.unlink(path.resolve(process.cwd(), reManifest), () => {}); From f894878f23c787c60d9eac581d343e7f461299fa Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 27 Jun 2024 18:50:41 +0400 Subject: [PATCH 225/243] feat(util): update if-check verify log --- src/util/npm.ts | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/util/npm.ts b/src/util/npm.ts index 8c66d37b5..16c2cb4d2 100644 --- a/src/util/npm.ts +++ b/src/util/npm.ts @@ -12,7 +12,8 @@ import {ManifestPlugin, PathWithVersion} from '../types/npm'; const packageJson = require('../../package.json'); -const {INITIALIZING_PACKAGE_JSON, INSTALLING_NPM_PACKAGES} = STRINGS; +const {INITIALIZING_PACKAGE_JSON, INSTALLING_NPM_PACKAGES, IF_CHECK_VERIFIED} = + STRINGS; /** * Checks if the package.json is exists, if not, initializes it. @@ -164,7 +165,7 @@ export const executeCommands = async (manifest: string, cwd: boolean) => { } -s ${executedManifest}.yaml -t ${manifest}`; const ttyCommand = " node -p 'Boolean(process.stdout.isTTY)'"; - const result = await execPromise( + await execPromise( `${ifEnvCommand} && ${ifRunCommand} && ${ttyCommand} | ${ifDiffCommand}`, { cwd: process.env.CURRENT_DIR || process.cwd(), @@ -177,10 +178,5 @@ export const executeCommands = async (manifest: string, cwd: boolean) => { await fs.unlink(`${executedManifest}.yaml`); - if (result.stdout) { - const logs = result.stdout.split('\n\n'); - const successMessage = logs[logs.length - 1]; - - console.log(successMessage); - } + console.log(IF_CHECK_VERIFIED(path.basename(manifest))); }; From eca2d2ff41d7cd28b6e76325158381e49d89d9f4 Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 27 Jun 2024 18:52:28 +0400 Subject: [PATCH 226/243] feat(src): update fail log --- src/check.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/check.ts b/src/check.ts index 887ff13f6..530072b04 100644 --- a/src/check.ts +++ b/src/check.ts @@ -5,13 +5,12 @@ import * as path from 'path'; import {logger} from './util/logger'; import {parseIfCheckArgs} from './util/args'; -import {logStdoutFailMessage} from './util/helpers'; import {getYamlFiles} from './util/fs'; import {STRINGS} from './config'; import {executeCommands} from './util/npm'; -const {CHECKING} = STRINGS; +const {CHECKING, IF_CHECK_FAILED} = STRINGS; const IfCheck = async () => { const commandArgs = await parseIfCheckArgs(); @@ -30,7 +29,7 @@ const IfCheck = async () => { .replace('yml', 'yaml'); const manifestDirPath = path.dirname(manifest); - logStdoutFailMessage(error); + console.log(IF_CHECK_FAILED(fileName)); await fs.unlink(`${manifestDirPath}/package.json`); await fs.unlink(executedFile); @@ -51,7 +50,7 @@ const IfCheck = async () => { .replace(fileName, `re-${fileName}`) .replace('yml', 'yaml'); - logStdoutFailMessage(error); + console.log(IF_CHECK_FAILED(fileName)); await fs.unlink(executedFile); } } From 68cbf5b8c1529db489a8e66309557e259f3d8ded Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 27 Jun 2024 19:51:10 +0400 Subject: [PATCH 227/243] fix(config): update srtings of if-check logs --- src/__tests__/unit/util/npm.test.ts | 2 +- src/config/strings.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/__tests__/unit/util/npm.test.ts b/src/__tests__/unit/util/npm.test.ts index d73095d93..3fd4fac18 100644 --- a/src/__tests__/unit/util/npm.test.ts +++ b/src/__tests__/unit/util/npm.test.ts @@ -267,7 +267,7 @@ describe('util/npm: ', () => { cwd: process.cwd(), }); expect(logSpy).toHaveBeenCalledWith( - 'if-check successfully verified \n' + 'if-check successfully verified mock-manifest.yaml\n' ); const packageJsonPath = 'src/__mocks__/package.json'; diff --git a/src/config/strings.ts b/src/config/strings.ts index e55eb1b9c..b1d58a7b4 100644 --- a/src/config/strings.ts +++ b/src/config/strings.ts @@ -137,7 +137,7 @@ ${error}`, 'Either the `--manifest` or `--directory` command should be provided with a path', DIRECTORY_NOT_FOUND: 'Directory not found.', IF_CHECK_FAILED: (filename: string) => - `if-check could not verify <${filename}>. The re-executed file does not match the original.\n`, + `if-check could not verify ${filename}. The re-executed file does not match the original.\n`, IF_CHECK_VERIFIED: (filename: string) => - `if-check successfully verified <${filename}>\n`, + `if-check successfully verified ${filename}\n`, }; From 2c0e2fcb45d198f693824ebc2f85836e2f039ed8 Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 27 Jun 2024 20:14:03 +0400 Subject: [PATCH 228/243] fix(util): bring back logStdoutFailMessage function --- src/__tests__/unit/util/helpers.test.ts | 18 ++++++++++++++++++ src/util/helpers.ts | 15 ++++++++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/src/__tests__/unit/util/helpers.test.ts b/src/__tests__/unit/util/helpers.test.ts index 4e0422403..7b6bc2eec 100644 --- a/src/__tests__/unit/util/helpers.test.ts +++ b/src/__tests__/unit/util/helpers.test.ts @@ -146,6 +146,7 @@ import { getOptionsFromArgs, addTemplateManifest, initializeAndInstallLibs, + logStdoutFailMessage, } from '../../../util/helpers'; import {CONFIG} from '../../../config'; import {Difference} from '../../../types/lib/compare'; @@ -666,4 +667,21 @@ description: mock-description process.exit = originalProcessExit; }); }); + + describe('logStdoutFailMessage(): ', () => { + it('successfully logs the failed message.', () => { + const errorMessage = {stdout: '\n\nmock error message'}; + const mockFilename = 'mock-filename.yaml'; + const logSpy = jest.spyOn(global.console, 'log'); + logStdoutFailMessage(errorMessage, mockFilename); + + expect.assertions(2); + + expect(logSpy).toHaveBeenCalledWith( + `if-check could not verify ${mockFilename}. The re-executed file does not match the original.\n` + ); + + expect(logSpy).toHaveBeenCalledWith('mock error message'); + }); + }); }); diff --git a/src/util/helpers.ts b/src/util/helpers.ts index e69b9e0e9..0fcdda963 100644 --- a/src/util/helpers.ts +++ b/src/util/helpers.ts @@ -33,7 +33,7 @@ const { FAILURE_MESSAGE_DEPENDENCIES, } = IF_ENV; -const {UNSUPPORTED_ERROR} = STRINGS; +const {UNSUPPORTED_ERROR, IF_CHECK_FAILED} = STRINGS; const {MissingPluginDependenciesError} = ERRORS; /** @@ -268,3 +268,16 @@ export const addTemplateManifest = async (destinationDir: string) => { process.exit(1); } }; + +/** + * Logs the failure message from the stdout of an error. + */ +export const logStdoutFailMessage = (error: any, fileName: string) => { + console.log(IF_CHECK_FAILED(fileName)); + + const stdout = error.stdout; + const logs = stdout.split('\n\n'); + const failMessage = logs[logs.length - 1]; + + console.log(failMessage); +}; From 0cc011bc9c7cc0425f57793135b29854446d4a2b Mon Sep 17 00:00:00 2001 From: manushak Date: Thu, 27 Jun 2024 20:15:12 +0400 Subject: [PATCH 229/243] fix(src): move fail logs into logStdoutFailMessage function --- src/check.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/check.ts b/src/check.ts index 530072b04..a7e561b04 100644 --- a/src/check.ts +++ b/src/check.ts @@ -4,13 +4,14 @@ import * as fs from 'fs/promises'; import * as path from 'path'; import {logger} from './util/logger'; +import {logStdoutFailMessage} from './util/helpers'; import {parseIfCheckArgs} from './util/args'; import {getYamlFiles} from './util/fs'; import {STRINGS} from './config'; import {executeCommands} from './util/npm'; -const {CHECKING, IF_CHECK_FAILED} = STRINGS; +const {CHECKING} = STRINGS; const IfCheck = async () => { const commandArgs = await parseIfCheckArgs(); @@ -29,7 +30,7 @@ const IfCheck = async () => { .replace('yml', 'yaml'); const manifestDirPath = path.dirname(manifest); - console.log(IF_CHECK_FAILED(fileName)); + logStdoutFailMessage(error, fileName); await fs.unlink(`${manifestDirPath}/package.json`); await fs.unlink(executedFile); @@ -50,7 +51,8 @@ const IfCheck = async () => { .replace(fileName, `re-${fileName}`) .replace('yml', 'yaml'); - console.log(IF_CHECK_FAILED(fileName)); + logStdoutFailMessage(error, fileName); + await fs.unlink(executedFile); } } From ae7e4c89958052f1b33921f20dcceda49b621052 Mon Sep 17 00:00:00 2001 From: MariamKhalatova Date: Thu, 27 Jun 2024 22:48:25 +0400 Subject: [PATCH 230/243] fix(manifests): uncomment sci from aggregation error wrong metric --- manifests/bugs/aggregation-error-wrong-metric.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/manifests/bugs/aggregation-error-wrong-metric.yml b/manifests/bugs/aggregation-error-wrong-metric.yml index 60a8dd8dd..2c3141c13 100644 --- a/manifests/bugs/aggregation-error-wrong-metric.yml +++ b/manifests/bugs/aggregation-error-wrong-metric.yml @@ -57,7 +57,7 @@ initialize: path: builtin global-config: input-parameters: ["cpu-energy-kwh", "grid/carbon-intensity"] - output-parameter: "carbon-operational" + output-parameter: "carbon" "sci": path: "builtin" method: Sci @@ -87,7 +87,7 @@ tree: - sci-embodied - operational-carbon - time-sync - # - sci + - sci config: group-by: group: @@ -107,21 +107,25 @@ tree: cloud/region: uk-west duration: 1 cpu/utilization: 10 + requests: 100 - timestamp: "2023-12-12T00:00:01.000Z" duration: 5 cpu/utilization: 20 cloud/instance-type: A1 cloud/region: uk-west + requests: 100 - timestamp: "2023-12-12T00:00:06.000Z" duration: 7 cpu/utilization: 15 cloud/instance-type: A1 cloud/region: uk-west + requests: 100 - timestamp: "2023-12-12T00:00:13.000Z" duration: 30 cloud/instance-type: A1 cloud/region: uk-west cpu/utilization: 15 + requests: 100 child-2: pipeline: - interpolate @@ -133,7 +137,7 @@ tree: - sci-embodied - operational-carbon - time-sync - # - sci + - sci config: group-by: group: @@ -153,18 +157,22 @@ tree: cpu/utilization: 30 cloud/instance-type: A1 cloud/region: uk-west + requests: 100 - timestamp: "2023-12-12T00:00:01.000Z" duration: 5 cpu/utilization: 28 cloud/instance-type: A1 cloud/region: uk-west + requests: 100 - timestamp: "2023-12-12T00:00:06.000Z" duration: 7 cpu/utilization: 40 cloud/instance-type: A1 cloud/region: uk-west + requests: 100 - timestamp: "2023-12-12T00:00:13.000Z" duration: 30 cpu/utilization: 33 cloud/instance-type: A1 cloud/region: uk-west + requests: 100 From 084da9dd1753690c30e1b50288194a44a11812de Mon Sep 17 00:00:00 2001 From: MariamKhalatova Date: Thu, 27 Jun 2024 22:53:22 +0400 Subject: [PATCH 231/243] fix(manifests): uncomment sci from nesting example --- manifests/examples/nesting.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/manifests/examples/nesting.yml b/manifests/examples/nesting.yml index b274ae7f0..978510f48 100644 --- a/manifests/examples/nesting.yml +++ b/manifests/examples/nesting.yml @@ -106,7 +106,7 @@ tree: - operational-carbon - sum-carbon - time-sync - # - sci + - sci inputs: - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 @@ -156,7 +156,7 @@ tree: - operational-carbon - sum-carbon - time-sync - # - sci + - sci inputs: - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 @@ -208,7 +208,7 @@ tree: - operational-carbon - sum-carbon - time-sync - # - sci + - sci inputs: - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 @@ -258,7 +258,7 @@ tree: - operational-carbon - sum-carbon - time-sync - # - sci + - sci inputs: - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 From 5e1b13ff032007234c9eec9896fd11e7e5bd5310 Mon Sep 17 00:00:00 2001 From: MariamKhalatova Date: Thu, 27 Jun 2024 22:57:04 +0400 Subject: [PATCH 232/243] fix(manifests): uncomment sci from pipeline with aggregate, fix second child --- .../examples/pipeline-with-aggregate.yml | 105 +++++++++--------- 1 file changed, 52 insertions(+), 53 deletions(-) diff --git a/manifests/examples/pipeline-with-aggregate.yml b/manifests/examples/pipeline-with-aggregate.yml index 65d6c9d26..73203676b 100644 --- a/manifests/examples/pipeline-with-aggregate.yml +++ b/manifests/examples/pipeline-with-aggregate.yml @@ -93,7 +93,7 @@ tree: - operational-carbon - sum-carbon - time-sync - # - sci + - sci config: group-by: group: @@ -132,55 +132,54 @@ tree: cloud/region: uk-west cpu/utilization: 15 requests: 30 - # child-2: - # pipeline: - # - interpolate - # - cpu-factor-to-wattage - # - wattage-times-duration - # - wattage-to-energy-kwh - # - calculate-vcpu-ratio - # - correct-cpu-energy-for-vcpu-ratio - # - sci-e - # - sci-embodied - # - sci-o - # - sum-carbon - # - time-sync - # - sci - # config: - # group-by: - # group: - # - cloud/region - # - cloud/instance-type - # defaults: - # cpu/thermal-design-power: 100 - # grid/carbon-intensity: 800 - # device/emissions-embodied: 1533.120 # gCO2eq - # time-reserved: 3600 # 1hr in seconds - # device/expected-lifespan: 94608000 # 3 years in seconds - # resources-reserved: 1 - # resources-total: 8 - # inputs: - # - timestamp: "2023-12-12T00:00:00.000Z" - # duration: 1 - # cpu/utilization: 30 - # cloud/instance-type: A1 - # cloud/region: uk-west - # requests: 100 - # - timestamp: "2023-12-12T00:00:01.000Z" - # duration: 5 - # cpu/utilization: 28 - # cloud/instance-type: A1 - # cloud/region: uk-west - # requests: 150 - # - timestamp: "2023-12-12T00:00:06.000Z" - # duration: 7 - # cpu/utilization: 40 - # cloud/instance-type: A1 - # cloud/region: uk-west - # requests: 110 - # - timestamp: "2023-12-12T00:00:13.000Z" - # duration: 30 - # cpu/utilization: 33 - # cloud/instance-type: A1 - # cloud/region: uk-west - # requests: 180 + child-2: + pipeline: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci + config: + group-by: + group: + - cloud/region + - cloud/instance-type + defaults: + cpu/thermal-design-power: 100 + grid/carbon-intensity: 800 + device/emissions-embodied: 1533.120 # gCO2eq + time-reserved: 3600 # 1hr in seconds + device/expected-lifespan: 94608000 # 3 years in seconds + vcpus-total: 8 + vcpus-allocated: 1 + inputs: + - timestamp: "2023-12-12T00:00:00.000Z" + duration: 1 + cpu/utilization: 30 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 100 + - timestamp: "2023-12-12T00:00:01.000Z" + duration: 5 + cpu/utilization: 28 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 150 + - timestamp: "2023-12-12T00:00:06.000Z" + duration: 7 + cpu/utilization: 40 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 110 + - timestamp: "2023-12-12T00:00:13.000Z" + duration: 30 + cpu/utilization: 33 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 180 From a4e2757890bb8295975db72ccabc09ba51387700 Mon Sep 17 00:00:00 2001 From: MariamKhalatova Date: Thu, 27 Jun 2024 22:58:23 +0400 Subject: [PATCH 233/243] fix(manifests): uncomment sci from pipeline with mocks, fix second child --- manifests/examples/pipeline-with-mocks.yml | 100 +++++++++++---------- 1 file changed, 52 insertions(+), 48 deletions(-) diff --git a/manifests/examples/pipeline-with-mocks.yml b/manifests/examples/pipeline-with-mocks.yml index 801b7bfd9..168ebd7fe 100644 --- a/manifests/examples/pipeline-with-mocks.yml +++ b/manifests/examples/pipeline-with-mocks.yml @@ -115,7 +115,7 @@ tree: - operational-carbon - sum-carbon - time-sync - # - sci + - sci config: group-by: group: @@ -154,50 +154,54 @@ tree: cloud/region: uk-west cpu/utilization: 15 requests: 50 - # child-2: - # pipeline: - # - teads-curve - # - sci-e - # - sci-embodied - # - sci-o - # - sum-carbon - # - time-sync - # - sci - # config: - # group-by: - # group: - # - cloud/region - # - cloud/instance-type - # defaults: - # cpu/thermal-design-power: 100 - # grid/carbon-intensity: 800 - # device/emissions-embodied: 1533.120 # gCO2eq - # time-reserved: 3600 # 1hr in seconds - # device/expected-lifespan: 94608000 # 3 years in seconds - # resources-reserved: 1 - # resources-total: 8 - # inputs: - # - timestamp: "2023-12-12T00:00:00.000Z" - # duration: 1 - # cpu/utilization: 30 - # cloud/instance-type: A1 - # cloud/region: uk-west - # requests: 30 - # - timestamp: "2023-12-12T00:00:01.000Z" - # duration: 5 - # cpu/utilization: 28 - # cloud/instance-type: A1 - # cloud/region: uk-west - # requests: 40 - # - timestamp: "2023-12-12T00:00:06.000Z" - # duration: 7 - # cpu/utilization: 40 - # cloud/instance-type: A1 - # cloud/region: uk-west - # requests: 50 - # - timestamp: "2023-12-12T00:00:13.000Z" - # duration: 30 - # cpu/utilization: 33 - # cloud/instance-type: A1 - # cloud/region: uk-west - # requests: 60 + child-2: + pipeline: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci + config: + group-by: + group: + - cloud/region + - cloud/instance-type + defaults: + cpu/thermal-design-power: 100 + grid/carbon-intensity: 800 + device/emissions-embodied: 1533.120 # gCO2eq + time-reserved: 3600 # 1hr in seconds + device/expected-lifespan: 94608000 # 3 years in seconds + vcpus-total: 8 + vcpus-allocated: 1 + inputs: + - timestamp: "2023-12-12T00:00:00.000Z" + duration: 1 + cpu/utilization: 30 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 30 + - timestamp: "2023-12-12T00:00:01.000Z" + duration: 5 + cpu/utilization: 28 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 40 + - timestamp: "2023-12-12T00:00:06.000Z" + duration: 7 + cpu/utilization: 40 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 50 + - timestamp: "2023-12-12T00:00:13.000Z" + duration: 30 + cpu/utilization: 33 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 60 From 5313ff19495a85beabd983b8bbacb4ffa8546163 Mon Sep 17 00:00:00 2001 From: MariamKhalatova Date: Thu, 27 Jun 2024 23:19:07 +0400 Subject: [PATCH 234/243] feat(manifests): init zeros example --- manifests/examples/zeros.yaml | 130 ++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 manifests/examples/zeros.yaml diff --git a/manifests/examples/zeros.yaml b/manifests/examples/zeros.yaml new file mode 100644 index 000000000..da5541789 --- /dev/null +++ b/manifests/examples/zeros.yaml @@ -0,0 +1,130 @@ +name: generics +description: a pipeline that does arbitrary calculations using our generic arithmetic builtins +tags: +initialize: + plugins: + "sum-zero-and-one": + path: "builtin" + method: Sum + global-config: + input-parameters: + - some-value + - zero-value + output-parameter: one-plus-zero + "sum-zero-and-zero": + path: "builtin" + method: Sum + global-config: + input-parameters: + - zero-value + - zero-value + output-parameter: zero-plus-zero + "subtract-one-and-zero": + path: "builtin" + method: Subtract + global-config: + input-parameters: + - some-value + - zero-value + output-parameter: one-minus-zero + "subtract-zero-and-zero": + path: "builtin" + method: Sum + global-config: + input-parameters: + - zero-value + - zero-value + output-parameter: zero-minus-zero + "subtract-zero-and-one": + path: "builtin" + method: Subtract + global-config: + input-parameters: + - zero-value + - some-value + output-parameter: zero-minus-one + "coefficient-one-times-zero": + path: "builtin" + method: Coefficient + global-config: + input-parameter: zero-value + coefficient: 1 + output-parameter: zero-times-one-coefficient + "coefficient-zero-times-one": + path: "builtin" + method: Coefficient + global-config: + input-parameter: some-value + coefficient: 0 + output-parameter: one-times-zero-coefficient + "coefficient-zero-times-zero": + path: "builtin" + method: Coefficient + global-config: + input-parameter: zero-value + coefficient: 0 + output-parameter: zero-times-zero-coefficient + "multiply-one-times-zero": + path: "builtin" + method: Multiply + global-config: + input-parameters: ["some-value", "zero-value"] + output-parameter: "one-times-zero" + "multiply-zero-times-one": + path: "builtin" + method: Multiply + global-config: + input-parameters: ["zero-value", "zero-value"] + output-parameter: "zero-times-one" + exponent-one-to-zero: + method: Exponent + path: 'builtin' + global-config: + input-parameter: 'some-value' + exponent: 0 + output-parameter: 'one-raised-to-zero-power' + exponent-zero-to-zero: + method: Exponent + path: 'builtin' + global-config: + input-parameter: 'zero-value' + exponent: 0 + output-parameter: 'zero-raised-to-zero-power' + exponent-zero-to-one: + method: Exponent + path: 'builtin' + global-config: + input-parameter: 'zero-value' + exponent: 1 + output-parameter: 'zero-raised-to-first-power' + "sci": + path: "builtin" + method: Sci + global-config: + functional-unit: "zero-value" +tree: + children: + child-1: + pipeline: + - sum-zero-and-one + - sum-zero-and-zero + - subtract-one-and-zero + - subtract-zero-and-zero + - subtract-zero-and-one + - coefficient-one-times-zero + - coefficient-zero-times-one + - coefficient-zero-times-zero + - multiply-one-times-zero + - multiply-zero-times-one + - exponent-one-to-zero + - exponent-zero-to-one + - exponent-zero-to-zero + - sci + config: + defaults: + inputs: + - timestamp: "2023-12-12T00:00:00.000Z" + duration: 1 + some-value: 1 + zero-value: 0 + carbon: 10 From 629c488d05caaf67e272be34b762c36de303b8c1 Mon Sep 17 00:00:00 2001 From: MariamKhalatova Date: Thu, 27 Jun 2024 23:36:47 +0400 Subject: [PATCH 235/243] fix(manifests): drop boavizta from pipeline --- ...adata-divide-boavizta.yml => cloud-metadata-divide.yml} | 7 ------- 1 file changed, 7 deletions(-) rename manifests/integrations/{cloud-metadata-divide-boavizta.yml => cloud-metadata-divide.yml} (83%) diff --git a/manifests/integrations/cloud-metadata-divide-boavizta.yml b/manifests/integrations/cloud-metadata-divide.yml similarity index 83% rename from manifests/integrations/cloud-metadata-divide-boavizta.yml rename to manifests/integrations/cloud-metadata-divide.yml index efb48f0ab..032206474 100644 --- a/manifests/integrations/cloud-metadata-divide-boavizta.yml +++ b/manifests/integrations/cloud-metadata-divide.yml @@ -19,19 +19,12 @@ initialize: numerator: vcpus-allocated denominator: 2 output: cpu/number-cores - boavizta-cpu: - method: BoaviztaCpuOutput - path: "@grnsft/if-unofficial-plugins" - global-config: - allocation: LINEAR - verbose: true tree: children: child: pipeline: - cloud-metadata - divide - - boavizta-cpu config: divide: defaults: From a33ff08741b21fc9bdb28d01e9748fb678942971 Mon Sep 17 00:00:00 2001 From: MariamKhalatova Date: Thu, 27 Jun 2024 23:37:13 +0400 Subject: [PATCH 236/243] fix(manifests): drop boavizta from divide --- .../plugins/divide/failure-denominator-equal-zero.yml | 7 ------- .../divide/failure-invalid-config-denominator.yml | 7 ------- manifests/plugins/divide/failure-missing-numerator.yml | 7 ------- manifests/plugins/divide/success.yml | 9 +-------- 4 files changed, 1 insertion(+), 29 deletions(-) diff --git a/manifests/plugins/divide/failure-denominator-equal-zero.yml b/manifests/plugins/divide/failure-denominator-equal-zero.yml index 8c48e42b9..4022d4797 100644 --- a/manifests/plugins/divide/failure-denominator-equal-zero.yml +++ b/manifests/plugins/divide/failure-denominator-equal-zero.yml @@ -20,19 +20,12 @@ initialize: numerator: vcpus-allocated denominator: 0 output: cpu/number-cores - boavizta-cpu: - method: BoaviztaCpuOutput - path: "@grnsft/if-unofficial-plugins" - global-config: - allocation: LINEAR - verbose: true tree: children: child: pipeline: - cloud-metadata - divide - - boavizta-cpu config: divide: defaults: diff --git a/manifests/plugins/divide/failure-invalid-config-denominator.yml b/manifests/plugins/divide/failure-invalid-config-denominator.yml index bc93f4f5f..91129714a 100644 --- a/manifests/plugins/divide/failure-invalid-config-denominator.yml +++ b/manifests/plugins/divide/failure-invalid-config-denominator.yml @@ -20,19 +20,12 @@ initialize: numerator: vcpus-allocated denominator: 'vcpus' output: cpu/number-cores - boavizta-cpu: - method: BoaviztaCpuOutput - path: "@grnsft/if-unofficial-plugins" - global-config: - allocation: LINEAR - verbose: true tree: children: child: pipeline: - cloud-metadata - divide - - boavizta-cpu config: divide: defaults: diff --git a/manifests/plugins/divide/failure-missing-numerator.yml b/manifests/plugins/divide/failure-missing-numerator.yml index ebf250961..7cf123976 100644 --- a/manifests/plugins/divide/failure-missing-numerator.yml +++ b/manifests/plugins/divide/failure-missing-numerator.yml @@ -20,19 +20,12 @@ initialize: #numerator: vcpus-allocated denominator: 2 output: cpu/number-cores - boavizta-cpu: - method: BoaviztaCpuOutput - path: "@grnsft/if-unofficial-plugins" - global-config: - allocation: LINEAR - verbose: true tree: children: child: pipeline: - cloud-metadata - divide - - boavizta-cpu config: divide: defaults: diff --git a/manifests/plugins/divide/success.yml b/manifests/plugins/divide/success.yml index 4d35e5fc1..6a32d7e2c 100644 --- a/manifests/plugins/divide/success.yml +++ b/manifests/plugins/divide/success.yml @@ -2,7 +2,7 @@ name: divide description: success path tags: initialize: - outputs: ['yaml'] + # outputs: ['yaml'] plugins: cloud-metadata: path: builtin @@ -20,19 +20,12 @@ initialize: numerator: vcpus-allocated denominator: 2 output: cpu/number-cores - boavizta-cpu: - method: BoaviztaCpuOutput - path: "@grnsft/if-unofficial-plugins" - global-config: - allocation: LINEAR - verbose: true tree: children: child: pipeline: - cloud-metadata - divide - - boavizta-cpu config: divide: defaults: From d1bc0377607ade0eb48575fffed2215fc957de9f Mon Sep 17 00:00:00 2001 From: MariamKhalatova Date: Fri, 28 Jun 2024 00:19:56 +0400 Subject: [PATCH 237/243] fix(doc): update divide readme --- src/builtins/divide/README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/builtins/divide/README.md b/src/builtins/divide/README.md index 35f7d867c..9a4d04f36 100644 --- a/src/builtins/divide/README.md +++ b/src/builtins/divide/README.md @@ -4,8 +4,6 @@ You provide the names of the values you want to divide, and a name to use to add the divide to the output array. -For example, `boavizta-cpu` need `cpu/number-cores` to work, however `cloud-metadata` returns `vcpus-allocated`, to get number of cores you divide `vcpus-allocated` by 2. - ## Parameters ### Plugin config From 4eed69cd14e493a147a4195a0023f46f2dba44a9 Mon Sep 17 00:00:00 2001 From: MariamKhalatova Date: Fri, 28 Jun 2024 00:20:09 +0400 Subject: [PATCH 238/243] fix(doc): update regex readme --- src/builtins/regex/README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/builtins/regex/README.md b/src/builtins/regex/README.md index f079b0461..a0fa5d624 100644 --- a/src/builtins/regex/README.md +++ b/src/builtins/regex/README.md @@ -4,8 +4,6 @@ You provide the name of the value you want to match, and a name to use to add the regex to the output array. -For example, `boavizta-cpu` need `cpu/name` to work, however `cloud-metadata` returns `physical-processor` which usually contains a long string of processors that the instance could be separated by `,`, like so: - ``` Intel® Xeon® Platinum 8272CL,Intel® Xeon® 8171M 2.1 GHz,Intel® Xeon® E5-2673 v4 2.3 GHz,Intel® Xeon® E5-2673 v3 2.4 GHz ``` From f8ca3b4219df585814fbe3e39c67a696c9c451b3 Mon Sep 17 00:00:00 2001 From: manushak Date: Fri, 28 Jun 2024 12:47:48 +0400 Subject: [PATCH 239/243] feat(config): add string for if-check --- src/config/strings.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/config/strings.ts b/src/config/strings.ts index b6aaf1de5..6a8a586f3 100644 --- a/src/config/strings.ts +++ b/src/config/strings.ts @@ -138,6 +138,8 @@ ${error}`, IF_CHECK_FLAGS_MISSING: 'Either the `--manifest` or `--directory` command should be provided with a path', DIRECTORY_NOT_FOUND: 'Directory not found.', + DIRECTORY_YAML_FILES_NOT_FOUND: + 'The directory does not contain any YAML/YML files.\n', IF_CHECK_FAILED: (filename: string) => `if-check could not verify ${filename}. The re-executed file does not match the original.\n`, IF_CHECK_VERIFIED: (filename: string) => From 9606c446a50bc8c181c4a4de8da82a7a3578080d Mon Sep 17 00:00:00 2001 From: manushak Date: Fri, 28 Jun 2024 14:46:07 +0400 Subject: [PATCH 240/243] feat(util): add removeFileIfExists function to fs.ts --- src/__tests__/unit/util/fs.test.ts | 19 +++++++++++++++++++ src/util/fs.ts | 9 +++++++++ 2 files changed, 28 insertions(+) diff --git a/src/__tests__/unit/util/fs.test.ts b/src/__tests__/unit/util/fs.test.ts index 15cb0103d..77b3c0370 100644 --- a/src/__tests__/unit/util/fs.test.ts +++ b/src/__tests__/unit/util/fs.test.ts @@ -5,6 +5,7 @@ import { isDirectoryExists, isFileExists, getYamlFiles, + removeFileIfExists, } from '../../../util/fs'; jest.mock('fs/promises', () => require('../../../__mocks__/fs')); @@ -122,4 +123,22 @@ describe('util/fs: ', () => { expect(fsReaddirSpy).toHaveBeenCalledWith('/mock-directory'); }); }); + + describe('removeFileIfExists(): ', () => { + it('successfully delete file if exists.', async () => { + await isFileExists('true'); + const result = await removeFileIfExists('mock-path'); + + expect.assertions(1); + expect(result).toEqual(undefined); + }); + + it('does not throw an error if the file not exists.', async () => { + await isFileExists('false'); + const result = await removeFileIfExists('mock-path'); + + expect.assertions(1); + expect(result).toEqual(undefined); + }); + }); }); diff --git a/src/util/fs.ts b/src/util/fs.ts index 7438ced61..bf10af9ba 100644 --- a/src/util/fs.ts +++ b/src/util/fs.ts @@ -57,3 +57,12 @@ export const getFileName = (filePath: string) => { const extension = path.extname(filePath); return baseName.replace(extension, ''); }; + +/** + * Removes the given file if exists. + */ +export const removeFileIfExists = async (filePath: string) => { + if (await isFileExists(filePath)) { + await fs.unlink(filePath); + } +}; From c8d8879823a8ac76b319d4bdabb1dad23c7b99a7 Mon Sep 17 00:00:00 2001 From: manushak Date: Fri, 28 Jun 2024 15:09:27 +0400 Subject: [PATCH 241/243] fix(util): fix to take account the --prefix flag --- src/__tests__/unit/util/npm.test.ts | 11 ++--------- src/util/npm.ts | 23 +++++++++++++++++------ 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/__tests__/unit/util/npm.test.ts b/src/__tests__/unit/util/npm.test.ts index 3fd4fac18..b7f028fa4 100644 --- a/src/__tests__/unit/util/npm.test.ts +++ b/src/__tests__/unit/util/npm.test.ts @@ -252,20 +252,13 @@ describe('util/npm: ', () => { it('successfully executes with correct commands.', async () => { const manifest = './src/__mocks__/mock-manifest.yaml'; const reManifest = 'src/__mocks__/re-mock-manifest.yaml'; - const command = `npm run if-env -- -m ${manifest} && npm run if-run -- -m ${manifest} -o ${reManifest.replace( - '.yaml', - '' - )} && node -p 'Boolean(process.stdout.isTTY)' | npm run if-diff -- -s ${reManifest} -t ${manifest}`; const logSpy = jest.spyOn(global.console, 'log'); - const spyExecPromise = jest.spyOn(helpers, 'execPromise'); + jest.spyOn(fs, 'unlink').mockResolvedValue(); await executeCommands(manifest, false); - expect.assertions(2); - expect(spyExecPromise).toHaveBeenCalledWith(command, { - cwd: process.cwd(), - }); + expect.assertions(1); expect(logSpy).toHaveBeenCalledWith( 'if-check successfully verified mock-manifest.yaml\n' ); diff --git a/src/util/npm.ts b/src/util/npm.ts index 16c2cb4d2..de27ae4de 100644 --- a/src/util/npm.ts +++ b/src/util/npm.ts @@ -4,7 +4,12 @@ import * as fs from 'fs/promises'; import * as path from 'path'; import {execPromise} from './helpers'; -import {isDirectoryExists, getFileName, isFileExists} from './fs'; +import { + isDirectoryExists, + getFileName, + isFileExists, + removeFileIfExists, +} from './fs'; import {logger} from './logger'; import {STRINGS} from '../config'; @@ -155,13 +160,19 @@ export const executeCommands = async (manifest: string, cwd: boolean) => { const manifestDirPath = path.dirname(manifest); const manifestFileName = getFileName(manifest); const executedManifest = path.join(manifestDirPath, `re-${manifestFileName}`); - const ifEnv = `${isGlobal ? 'if-env' : 'npm run if-env --'} -m ${manifest}`; + const prefixFlag = + process.env.CURRENT_DIR && process.env.CURRENT_DIR !== process.cwd() + ? `--prefix=${path.relative(process.env.CURRENT_DIR!, process.cwd())}` + : ''; + const ifEnv = `${ + isGlobal ? `if-env ${prefixFlag}` : `npm run if-env ${prefixFlag} --` + } -m ${manifest}`; const ifEnvCommand = cwd ? `${ifEnv} -c` : ifEnv; const ifRunCommand = `${ - isGlobal ? 'if-run' : 'npm run if-run --' + isGlobal ? `if-run ${prefixFlag}` : `npm run if-run ${prefixFlag} --` } -m ${manifest} -o ${executedManifest}`; const ifDiffCommand = `${ - isGlobal ? 'if-diff' : 'npm run if-diff --' + isGlobal ? `if-diff ${prefixFlag}` : `npm run if-diff ${prefixFlag} --` } -s ${executedManifest}.yaml -t ${manifest}`; const ttyCommand = " node -p 'Boolean(process.stdout.isTTY)'"; @@ -173,10 +184,10 @@ export const executeCommands = async (manifest: string, cwd: boolean) => { ); if (!cwd) { - await fs.unlink(`${manifestDirPath}/package.json`); + await removeFileIfExists(`${manifestDirPath}/package.json`); } - await fs.unlink(`${executedManifest}.yaml`); + await removeFileIfExists(`${executedManifest}.yaml`); console.log(IF_CHECK_VERIFIED(path.basename(manifest))); }; From 86da2e60b8f5b9c896c9f063f5156e534f16967f Mon Sep 17 00:00:00 2001 From: manushak Date: Fri, 28 Jun 2024 15:11:31 +0400 Subject: [PATCH 242/243] fix(util): add log when in the directory no any yaml files --- src/check.ts | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/check.ts b/src/check.ts index a7e561b04..3d53bc516 100644 --- a/src/check.ts +++ b/src/check.ts @@ -1,17 +1,16 @@ #!/usr/bin/env node /* eslint-disable no-process-exit */ -import * as fs from 'fs/promises'; import * as path from 'path'; import {logger} from './util/logger'; import {logStdoutFailMessage} from './util/helpers'; import {parseIfCheckArgs} from './util/args'; -import {getYamlFiles} from './util/fs'; +import {getYamlFiles, removeFileIfExists} from './util/fs'; import {STRINGS} from './config'; import {executeCommands} from './util/npm'; -const {CHECKING} = STRINGS; +const {CHECKING, DIRECTORY_YAML_FILES_NOT_FOUND} = STRINGS; const IfCheck = async () => { const commandArgs = await parseIfCheckArgs(); @@ -32,13 +31,18 @@ const IfCheck = async () => { logStdoutFailMessage(error, fileName); - await fs.unlink(`${manifestDirPath}/package.json`); - await fs.unlink(executedFile); + await removeFileIfExists(`${manifestDirPath}/package.json`); + await removeFileIfExists(executedFile); } } else { const directory = commandArgs.directory; const files = await getYamlFiles(directory!); + if (files.length === 0) { + console.log(DIRECTORY_YAML_FILES_NOT_FOUND); + process.exit(1); + } + for await (const file of files) { const fileName = path.basename(file); console.log(fileName); @@ -53,7 +57,7 @@ const IfCheck = async () => { logStdoutFailMessage(error, fileName); - await fs.unlink(executedFile); + await removeFileIfExists(executedFile); } } } From 2479a2741fae4dc24d94c6b989ff8c9684b7b9da Mon Sep 17 00:00:00 2001 From: Release commit workflow Date: Fri, 28 Jun 2024 13:09:09 +0000 Subject: [PATCH 243/243] =?UTF-8?q?chore(release):=20v0.5.0-beta.0=20?= =?UTF-8?q?=E2=9C=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9d64c6b7b..0cd6305c8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@grnsft/if", - "version": "0.4.0", + "version": "0.5.0-beta.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@grnsft/if", - "version": "0.4.0", + "version": "0.5.0-beta.0", "license": "MIT", "dependencies": { "@commitlint/cli": "^18.6.0", diff --git a/package.json b/package.json index 0d965de37..e01715c14 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@grnsft/if", "description": "Impact Framework", - "version": "0.4.0", + "version": "0.5.0-beta.0", "author": { "name": "Green Software Foundation", "email": "info@gsf.com"