From 2dc1a23d5284d3eb03fcb58c9172dc60da11f7cf Mon Sep 17 00:00:00 2001 From: Joshua Lochner Date: Sun, 17 Nov 2024 00:21:59 +0100 Subject: [PATCH 1/4] Add support for Split pretokenizer w/ `behavior=removed` & `invert=false` --- .github/workflows/format-check.yml | 29 + .prettierignore | 4 + .prettierrc | 4 +- jest.config.mjs | 263 +- jsconfig.json | 24 +- package-lock.json | 22220 ++++++++-------- package.json | 198 +- src/backends/onnx.js | 76 +- src/configs.js | 318 +- src/env.js | 58 +- src/generation/configuration_utils.js | 6 +- src/generation/logits_process.js | 109 +- src/generation/logits_sampler.js | 29 +- src/generation/parameters.js | 1 - src/generation/stopping_criteria.js | 21 +- src/generation/streamers.js | 85 +- src/models.js | 2687 +- src/models/whisper/common_whisper.js | 7 +- src/models/whisper/generation_whisper.js | 1 - src/ops/registry.js | 93 +- src/pipelines.js | 1379 +- src/processors.js | 852 +- src/tokenizers.js | 1255 +- src/transformers.js | 29 +- src/utils/audio.js | 206 +- src/utils/constants.js | 3 +- src/utils/core.js | 30 +- src/utils/data-structures.js | 24 +- src/utils/devices.js | 23 +- src/utils/dtypes.js | 34 +- src/utils/generic.js | 51 +- src/utils/hub.js | 238 +- src/utils/image.js | 237 +- src/utils/maths.js | 137 +- src/utils/tensor.js | 237 +- tests/bundles.test.js | 30 +- tests/configs.test.js | 22 +- tests/init.js | 66 +- tests/models.test.js | 223 +- tests/models/albert/tokenization.js | 352 +- tests/models/bert/tokenization.js | 2650 +- tests/models/blenderbot_small/tokenization.js | 314 +- tests/models/bloom/tokenization.js | 374 +- tests/models/clip/tokenization.js | 318 +- tests/models/deberta-v2/tokenization.js | 594 +- tests/models/distilbert/tokenization.js | 598 +- tests/models/esm/tokenization.js | 630 +- tests/models/falcon/tokenization.js | 474 +- tests/models/gemma/tokenization.js | 448 +- tests/models/gpt2/tokenization.js | 910 +- tests/models/llama/tokenization.js | 2548 +- tests/models/m2m_100/tokenization.js | 338 +- tests/models/mpnet/tokenization.js | 302 +- tests/models/nllb/tokenization.js | 302 +- tests/models/qwen2/tokenization.js | 302 +- tests/models/roberta/tokenization.js | 1390 +- tests/models/t5/tokenization.js | 556 +- tests/models/test_strings.js | 148 +- tests/models/vits/tokenization.js | 136 +- tests/models/wav2vec2/tokenization.js | 930 +- tests/models/whisper/tokenization.js | 1512 +- tests/models/xlm-roberta/tokenization.js | 650 +- tests/pipelines.test.js | 3200 +-- tests/processors.test.js | 1966 +- tests/test_utils.js | 92 +- tests/tiny_random.test.js | 5580 ++-- tests/tokenizers.test.js | 966 +- tests/utils/data_structures.test.js | 52 +- tests/utils/generation.test.js | 377 +- tests/utils/hub.test.js | 60 +- tests/utils/logits_process.test.js | 122 +- tests/utils/maths.test.js | 448 +- tests/utils/tensor.test.js | 270 +- tests/utils/tensor_ops.test.js | 338 +- tests/utils/utils.test.js | 102 +- webpack.config.js | 155 +- 76 files changed, 31003 insertions(+), 30810 deletions(-) create mode 100644 .github/workflows/format-check.yml diff --git a/.github/workflows/format-check.yml b/.github/workflows/format-check.yml new file mode 100644 index 000000000..ba3fda0c0 --- /dev/null +++ b/.github/workflows/format-check.yml @@ -0,0 +1,29 @@ +name: Prettier Check + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + format: + runs-on: ubuntu-latest + strategy: + matrix: + node-version: [22] + + steps: + - uses: actions/checkout@v4 + + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: format:check + run: npm run format:check diff --git a/.prettierignore b/.prettierignore index bd1927ab2..264b95b18 100644 --- a/.prettierignore +++ b/.prettierignore @@ -6,3 +6,7 @@ examples scripts types *.md +# Ignore python related files +.venv +# Mac files +.DS_STORE diff --git a/.prettierrc b/.prettierrc index 57d5ce89a..d423bc3b3 100644 --- a/.prettierrc +++ b/.prettierrc @@ -1,9 +1,11 @@ { + "tabWidth": 4, + "printWidth": 120, "overrides": [ { "files": ["tests/**/*.js"], "options": { - "printWidth": 10000000 + "printWidth": 10000000 } } ] diff --git a/jest.config.mjs b/jest.config.mjs index 0d15ce842..e52067082 100644 --- a/jest.config.mjs +++ b/jest.config.mjs @@ -4,188 +4,185 @@ */ export default { - // All imported modules in your tests should be mocked automatically - // automock: false, + // All imported modules in your tests should be mocked automatically + // automock: false, - // Stop running tests after `n` failures - // bail: 0, + // Stop running tests after `n` failures + // bail: 0, - // Automatically clear mock calls, instances, contexts and results before every test - clearMocks: true, + // Automatically clear mock calls, instances, contexts and results before every test + clearMocks: true, - // Indicates whether the coverage information should be collected while executing the test - collectCoverage: true, + // Indicates whether the coverage information should be collected while executing the test + collectCoverage: true, - // An array of glob patterns indicating a set of files for which coverage information should be collected - // collectCoverageFrom: undefined, + // An array of glob patterns indicating a set of files for which coverage information should be collected + // collectCoverageFrom: undefined, - // The directory where Jest should output its coverage files - coverageDirectory: "coverage", + // The directory where Jest should output its coverage files + coverageDirectory: "coverage", - // An array of regexp pattern strings used to skip coverage collection - coveragePathIgnorePatterns: [ - "node_modules", - "tests", - ], + // An array of regexp pattern strings used to skip coverage collection + coveragePathIgnorePatterns: ["node_modules", "tests"], - // Indicates which provider should be used to instrument code for coverage - coverageProvider: "v8", + // Indicates which provider should be used to instrument code for coverage + coverageProvider: "v8", - // A list of reporter names that Jest uses when writing coverage reports - // coverageReporters: [ - // "json", - // "text", - // "lcov", - // "clover" - // ], + // A list of reporter names that Jest uses when writing coverage reports + // coverageReporters: [ + // "json", + // "text", + // "lcov", + // "clover" + // ], - // An object that configures minimum threshold enforcement for coverage results - // coverageThreshold: undefined, + // An object that configures minimum threshold enforcement for coverage results + // coverageThreshold: undefined, - // A path to a custom dependency extractor - // dependencyExtractor: undefined, + // A path to a custom dependency extractor + // dependencyExtractor: undefined, - // Make calling deprecated APIs throw helpful error messages - // errorOnDeprecated: false, + // Make calling deprecated APIs throw helpful error messages + // errorOnDeprecated: false, - // The default configuration for fake timers - // fakeTimers: { - // "enableGlobally": false - // }, + // The default configuration for fake timers + // fakeTimers: { + // "enableGlobally": false + // }, - // Force coverage collection from ignored files using an array of glob patterns - // forceCoverageMatch: [], + // Force coverage collection from ignored files using an array of glob patterns + // forceCoverageMatch: [], - // A path to a module which exports an async function that is triggered once before all test suites - // globalSetup: undefined, + // A path to a module which exports an async function that is triggered once before all test suites + // globalSetup: undefined, - // A path to a module which exports an async function that is triggered once after all test suites - // globalTeardown: undefined, + // A path to a module which exports an async function that is triggered once after all test suites + // globalTeardown: undefined, - // A set of global variables that need to be available in all test environments - // globals: {}, + // A set of global variables that need to be available in all test environments + // globals: {}, - // The maximum amount of workers used to run your tests. Can be specified as % or a number. E.g. maxWorkers: 10% will use 10% of your CPU amount + 1 as the maximum worker number. maxWorkers: 2 will use a maximum of 2 workers. - // maxWorkers: "50%", + // The maximum amount of workers used to run your tests. Can be specified as % or a number. E.g. maxWorkers: 10% will use 10% of your CPU amount + 1 as the maximum worker number. maxWorkers: 2 will use a maximum of 2 workers. + // maxWorkers: "50%", - // An array of directory names to be searched recursively up from the requiring module's location - // moduleDirectories: [ - // "node_modules" - // ], + // An array of directory names to be searched recursively up from the requiring module's location + // moduleDirectories: [ + // "node_modules" + // ], - // An array of file extensions your modules use - // moduleFileExtensions: [ - // "js", - // "mjs", - // "cjs", - // "jsx", - // "ts", - // "tsx", - // "json", - // "node" - // ], + // An array of file extensions your modules use + // moduleFileExtensions: [ + // "js", + // "mjs", + // "cjs", + // "jsx", + // "ts", + // "tsx", + // "json", + // "node" + // ], - // A map from regular expressions to module names or to arrays of module names that allow to stub out resources with a single module - // moduleNameMapper: {}, + // A map from regular expressions to module names or to arrays of module names that allow to stub out resources with a single module + // moduleNameMapper: {}, - // An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader - // modulePathIgnorePatterns: [], + // An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader + // modulePathIgnorePatterns: [], - // Activates notifications for test results - // notify: false, + // Activates notifications for test results + // notify: false, - // An enum that specifies notification mode. Requires { notify: true } - // notifyMode: "failure-change", + // An enum that specifies notification mode. Requires { notify: true } + // notifyMode: "failure-change", - // A preset that is used as a base for Jest's configuration - // preset: undefined, + // A preset that is used as a base for Jest's configuration + // preset: undefined, - // Run tests from one or more projects - // projects: undefined, + // Run tests from one or more projects + // projects: undefined, - // Use this configuration option to add custom reporters to Jest - // reporters: undefined, + // Use this configuration option to add custom reporters to Jest + // reporters: undefined, - // Automatically reset mock state before every test - // resetMocks: false, + // Automatically reset mock state before every test + // resetMocks: false, - // Reset the module registry before running each individual test - // resetModules: false, + // Reset the module registry before running each individual test + // resetModules: false, - // A path to a custom resolver - // resolver: undefined, + // A path to a custom resolver + // resolver: undefined, - // Automatically restore mock state and implementation before every test - // restoreMocks: false, + // Automatically restore mock state and implementation before every test + // restoreMocks: false, - // The root directory that Jest should scan for tests and modules within - // rootDir: undefined, + // The root directory that Jest should scan for tests and modules within + // rootDir: undefined, - // A list of paths to directories that Jest should use to search for files in - roots: ["./tests/"], + // A list of paths to directories that Jest should use to search for files in + roots: ["./tests/"], - // Allows you to use a custom runner instead of Jest's default test runner - // runner: "jest-runner", + // Allows you to use a custom runner instead of Jest's default test runner + // runner: "jest-runner", - // The paths to modules that run some code to configure or set up the testing environment before each test - // setupFiles: [], + // The paths to modules that run some code to configure or set up the testing environment before each test + // setupFiles: [], - // A list of paths to modules that run some code to configure or set up the testing framework before each test - // setupFilesAfterEnv: [], + // A list of paths to modules that run some code to configure or set up the testing framework before each test + // setupFilesAfterEnv: [], - // The number of seconds after which a test is considered as slow and reported as such in the results. - // slowTestThreshold: 5, + // The number of seconds after which a test is considered as slow and reported as such in the results. + // slowTestThreshold: 5, - // A list of paths to snapshot serializer modules Jest should use for snapshot testing - // snapshotSerializers: [], + // A list of paths to snapshot serializer modules Jest should use for snapshot testing + // snapshotSerializers: [], - // The test environment that will be used for testing - // testEnvironment: "jest-environment-node", + // The test environment that will be used for testing + // testEnvironment: "jest-environment-node", - // Options that will be passed to the testEnvironment - // testEnvironmentOptions: {}, + // Options that will be passed to the testEnvironment + // testEnvironmentOptions: {}, - // Adds a location field to test results - // testLocationInResults: false, + // Adds a location field to test results + // testLocationInResults: false, - // The glob patterns Jest uses to detect test files - // testMatch: [ - // "**/__tests__/**/*.[jt]s?(x)", - // "**/?(*.)+(spec|test).[tj]s?(x)" - // ], + // The glob patterns Jest uses to detect test files + // testMatch: [ + // "**/__tests__/**/*.[jt]s?(x)", + // "**/?(*.)+(spec|test).[tj]s?(x)" + // ], - // An array of regexp pattern strings that are matched against all test paths, matched tests are skipped - // testPathIgnorePatterns: [ - // "\\\\node_modules\\\\" - // ], + // An array of regexp pattern strings that are matched against all test paths, matched tests are skipped + // testPathIgnorePatterns: [ + // "\\\\node_modules\\\\" + // ], - // The regexp pattern or array of patterns that Jest uses to detect test files - // testRegex: [], + // The regexp pattern or array of patterns that Jest uses to detect test files + // testRegex: [], - // This option allows the use of a custom results processor - // testResultsProcessor: undefined, + // This option allows the use of a custom results processor + // testResultsProcessor: undefined, - // This option allows use of a custom test runner - // testRunner: "jest-circus/runner", + // This option allows use of a custom test runner + // testRunner: "jest-circus/runner", - // A map from regular expressions to paths to transformers - transform: {}, + // A map from regular expressions to paths to transformers + transform: {}, - // An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation - // transformIgnorePatterns: [ - // "\\\\node_modules\\\\", - // "\\.pnp\\.[^\\\\]+$" - // ], + // An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation + // transformIgnorePatterns: [ + // "\\\\node_modules\\\\", + // "\\.pnp\\.[^\\\\]+$" + // ], - // An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them - // unmockedModulePathPatterns: undefined, + // An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them + // unmockedModulePathPatterns: undefined, - // Indicates whether each individual test should be reported during the run - // verbose: undefined, + // Indicates whether each individual test should be reported during the run + // verbose: undefined, - // An array of regexp patterns that are matched against all source file paths before re-running tests in watch mode - // watchPathIgnorePatterns: [], + // An array of regexp patterns that are matched against all source file paths before re-running tests in watch mode + // watchPathIgnorePatterns: [], - // Whether to use watchman for file crawling - // watchman: true, + // Whether to use watchman for file crawling + // watchman: true, }; diff --git a/jsconfig.json b/jsconfig.json index 9af7d54be..8a21e8292 100644 --- a/jsconfig.json +++ b/jsconfig.json @@ -1,14 +1,14 @@ { - // Only include files in the src directory - "include": ["src/**/*"], - "compilerOptions": { - // Tells the compiler to check JS files - "checkJs": true, - "target": "esnext", - "module": "nodenext", - "moduleResolution": "nodenext" - }, - "typeAcquisition": { - "include": ["jest"] - } + // Only include files in the src directory + "include": ["src/**/*"], + "compilerOptions": { + // Tells the compiler to check JS files + "checkJs": true, + "target": "esnext", + "module": "nodenext", + "moduleResolution": "nodenext" + }, + "typeAcquisition": { + "include": ["jest"] + } } diff --git a/package-lock.json b/package-lock.json index 410ae35a7..6c5958a3e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,11114 +1,11114 @@ { - "name": "@huggingface/transformers", - "version": "3.0.2", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "@huggingface/transformers", - "version": "3.0.2", - "license": "Apache-2.0", - "dependencies": { - "@huggingface/jinja": "^0.3.0", - "onnxruntime-node": "1.19.2", - "onnxruntime-web": "1.21.0-dev.20241024-d9ca84ef96", - "sharp": "^0.33.5" - }, - "devDependencies": { - "@types/jest": "^29.5.14", - "@webgpu/types": "^0.1.44", - "catharsis": "github:xenova/catharsis", - "jest": "^30.0.0-alpha.6", - "jest-environment-node": "^30.0.0-alpha.6", - "jsdoc-to-markdown": "^8.0.1", - "prettier": "3.3.3", - "typescript": "^5.2.2", - "wavefile": "^11.0.0", - "webpack": "^5.80.0", - "webpack-cli": "^5.0.2", - "webpack-dev-server": "^4.13.3" - } - }, - "node_modules/@ampproject/remapping": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.3.0.tgz", - "integrity": "sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@jridgewell/gen-mapping": "^0.3.5", - "@jridgewell/trace-mapping": "^0.3.24" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@babel/code-frame": { - "version": "7.26.2", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.26.2.tgz", - "integrity": "sha512-RJlIHRueQgwWitWgF8OdFYGZX328Ax5BCemNGlqHfplnRT9ESi8JkFlvaVYbS+UubVY6dpv87Fs2u5M29iNFVQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-validator-identifier": "^7.25.9", - "js-tokens": "^4.0.0", - "picocolors": "^1.0.0" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/compat-data": { - "version": "7.26.2", - "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.26.2.tgz", - "integrity": "sha512-Z0WgzSEa+aUcdiJuCIqgujCshpMWgUpgOxXotrYPSA53hA3qopNaqcJpyr0hVb1FeWdnqFA35/fUtXgBK8srQg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/core": { - "version": "7.26.0", - "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.26.0.tgz", - "integrity": "sha512-i1SLeK+DzNnQ3LL/CswPCa/E5u4lh1k6IAEphON8F+cXt0t9euTshDru0q7/IqMa1PMPz5RnHuHscF8/ZJsStg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@ampproject/remapping": "^2.2.0", - "@babel/code-frame": "^7.26.0", - "@babel/generator": "^7.26.0", - "@babel/helper-compilation-targets": "^7.25.9", - "@babel/helper-module-transforms": "^7.26.0", - "@babel/helpers": "^7.26.0", - "@babel/parser": "^7.26.0", - "@babel/template": "^7.25.9", - "@babel/traverse": "^7.25.9", - "@babel/types": "^7.26.0", - "convert-source-map": "^2.0.0", - "debug": "^4.1.0", - "gensync": "^1.0.0-beta.2", - "json5": "^2.2.3", - "semver": "^6.3.1" - }, - "engines": { - "node": ">=6.9.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/babel" - } - }, - "node_modules/@babel/core/node_modules/debug": { - "version": "4.3.7", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", - "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/@babel/core/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true, - "license": "MIT" - }, - "node_modules/@babel/core/node_modules/semver": { - "version": "6.3.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", - "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - } - }, - "node_modules/@babel/generator": { - "version": "7.26.2", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.26.2.tgz", - "integrity": "sha512-zevQbhbau95nkoxSq3f/DC/SC+EEOUZd3DYqfSkMhY2/wfSeaHV1Ew4vk8e+x8lja31IbyuUa2uQ3JONqKbysw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/parser": "^7.26.2", - "@babel/types": "^7.26.0", - "@jridgewell/gen-mapping": "^0.3.5", - "@jridgewell/trace-mapping": "^0.3.25", - "jsesc": "^3.0.2" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-compilation-targets": { - "version": "7.25.9", - "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.25.9.tgz", - "integrity": "sha512-j9Db8Suy6yV/VHa4qzrj9yZfZxhLWQdVnRlXxmKLYlhWUVB1sB2G5sxuWYXk/whHD9iW76PmNzxZ4UCnTQTVEQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/compat-data": "^7.25.9", - "@babel/helper-validator-option": "^7.25.9", - "browserslist": "^4.24.0", - "lru-cache": "^5.1.1", - "semver": "^6.3.1" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-compilation-targets/node_modules/semver": { - "version": "6.3.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", - "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - } - }, - "node_modules/@babel/helper-module-imports": { - "version": "7.25.9", - "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.25.9.tgz", - "integrity": "sha512-tnUA4RsrmflIM6W6RFTLFSXITtl0wKjgpnLgXyowocVPrbYrLUXSBXDgTs8BlbmIzIdlBySRQjINYs2BAkiLtw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/traverse": "^7.25.9", - "@babel/types": "^7.25.9" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-module-transforms": { - "version": "7.26.0", - "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.26.0.tgz", - "integrity": "sha512-xO+xu6B5K2czEnQye6BHA7DolFFmS3LB7stHZFaOLb1pAwO1HWLS8fXA+eh0A2yIvltPVmx3eNNDBJA2SLHXFw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-module-imports": "^7.25.9", - "@babel/helper-validator-identifier": "^7.25.9", - "@babel/traverse": "^7.25.9" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0" - } - }, - "node_modules/@babel/helper-plugin-utils": { - "version": "7.25.9", - "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.25.9.tgz", - "integrity": "sha512-kSMlyUVdWe25rEsRGviIgOWnoT/nfABVWlqt9N19/dIPWViAOW2s9wznP5tURbs/IDuNk4gPy3YdYRgH3uxhBw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-string-parser": { - "version": "7.25.9", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.25.9.tgz", - "integrity": "sha512-4A/SCr/2KLd5jrtOMFzaKjVtAei3+2r/NChoBNoZ3EyP/+GlhoaEGoWOZUmFmoITP7zOJyHIMm+DYRd8o3PvHA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-validator-identifier": { - "version": "7.25.9", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.25.9.tgz", - "integrity": "sha512-Ed61U6XJc3CVRfkERJWDz4dJwKe7iLmmJsbOGu9wSloNSFttHV0I8g6UAgb7qnK5ly5bGLPd4oXZlxCdANBOWQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-validator-option": { - "version": "7.25.9", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.25.9.tgz", - "integrity": "sha512-e/zv1co8pp55dNdEcCynfj9X7nyUKUXoUEwfXqaZt0omVOmDe9oOTdKStH4GmAw6zxMFs50ZayuMfHDKlO7Tfw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helpers": { - "version": "7.26.0", - "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.26.0.tgz", - "integrity": "sha512-tbhNuIxNcVb21pInl3ZSjksLCvgdZy9KwJ8brv993QtIVKJBBkYXz4q4ZbAv31GdnC+R90np23L5FbEBlthAEw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/template": "^7.25.9", - "@babel/types": "^7.26.0" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/parser": { - "version": "7.26.2", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.26.2.tgz", - "integrity": "sha512-DWMCZH9WA4Maitz2q21SRKHo9QXZxkDsbNZoVD62gusNtNBBqDg9i7uOhASfTfIGNzW+O+r7+jAlM8dwphcJKQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/types": "^7.26.0" - }, - "bin": { - "parser": "bin/babel-parser.js" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@babel/plugin-syntax-async-generators": { - "version": "7.8.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz", - "integrity": "sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.8.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-bigint": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-bigint/-/plugin-syntax-bigint-7.8.3.tgz", - "integrity": "sha512-wnTnFlG+YxQm3vDxpGE57Pj0srRU4sHE/mDkt1qv2YJJSeUAec2ma4WLUnUPeKjyrfntVwe/N6dCXpU+zL3Npg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.8.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-class-properties": { - "version": "7.12.13", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-properties/-/plugin-syntax-class-properties-7.12.13.tgz", - "integrity": "sha512-fm4idjKla0YahUNgFNLCB0qySdsoPiZP3iQE3rky0mBUtMZ23yDJ9SJdg6dXTSDnulOVqiF3Hgr9nbXvXTQZYA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.12.13" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-class-static-block": { - "version": "7.14.5", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-static-block/-/plugin-syntax-class-static-block-7.14.5.tgz", - "integrity": "sha512-b+YyPmr6ldyNnM6sqYeMWE+bgJcJpO6yS4QD7ymxgH34GBPNDM/THBh8iunyvKIZztiwLH4CJZ0RxTk9emgpjw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.14.5" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-import-attributes": { - "version": "7.26.0", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-attributes/-/plugin-syntax-import-attributes-7.26.0.tgz", - "integrity": "sha512-e2dttdsJ1ZTpi3B9UYGLw41hifAubg19AtCu/2I/F1QNVclOBr1dYpTdmdyZ84Xiz43BS/tCUkMAZNLv12Pi+A==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.25.9" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-import-meta": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-meta/-/plugin-syntax-import-meta-7.10.4.tgz", - "integrity": "sha512-Yqfm+XDx0+Prh3VSeEQCPU81yC+JWZ2pDPFSS4ZdpfZhp4MkFMaDC1UqseovEKwSUpnIL7+vK+Clp7bfh0iD7g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.10.4" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-json-strings": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-json-strings/-/plugin-syntax-json-strings-7.8.3.tgz", - "integrity": "sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.8.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-jsx": { - "version": "7.25.9", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.25.9.tgz", - "integrity": "sha512-ld6oezHQMZsZfp6pWtbjaNDF2tiiCYYDqQszHt5VV437lewP9aSi2Of99CK0D0XB21k7FLgnLcmQKyKzynfeAA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.25.9" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-logical-assignment-operators": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-logical-assignment-operators/-/plugin-syntax-logical-assignment-operators-7.10.4.tgz", - "integrity": "sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.10.4" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-nullish-coalescing-operator": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-nullish-coalescing-operator/-/plugin-syntax-nullish-coalescing-operator-7.8.3.tgz", - "integrity": "sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.8.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-numeric-separator": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-numeric-separator/-/plugin-syntax-numeric-separator-7.10.4.tgz", - "integrity": "sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.10.4" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-object-rest-spread": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.8.3.tgz", - "integrity": "sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.8.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-optional-catch-binding": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.8.3.tgz", - "integrity": "sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.8.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-optional-chaining": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-chaining/-/plugin-syntax-optional-chaining-7.8.3.tgz", - "integrity": "sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.8.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-private-property-in-object": { - "version": "7.14.5", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-private-property-in-object/-/plugin-syntax-private-property-in-object-7.14.5.tgz", - "integrity": "sha512-0wVnp9dxJ72ZUJDV27ZfbSj6iHLoytYZmh3rFcxNnvsJF3ktkzLDZPy/mA17HGsaQT3/DQsWYX1f1QGWkCoVUg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.14.5" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-top-level-await": { - "version": "7.14.5", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.14.5.tgz", - "integrity": "sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.14.5" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-typescript": { - "version": "7.25.9", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.25.9.tgz", - "integrity": "sha512-hjMgRy5hb8uJJjUcdWunWVcoi9bGpJp8p5Ol1229PoN6aytsLwNMgmdftO23wnCLMfVmTwZDWMPNq/D1SY60JQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.25.9" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/template": { - "version": "7.25.9", - "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.25.9.tgz", - "integrity": "sha512-9DGttpmPvIxBb/2uwpVo3dqJ+O6RooAFOS+lB+xDqoE2PVCE8nfoHMdZLpfCQRLwvohzXISPZcgxt80xLfsuwg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.25.9", - "@babel/parser": "^7.25.9", - "@babel/types": "^7.25.9" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/traverse": { - "version": "7.25.9", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.25.9.tgz", - "integrity": "sha512-ZCuvfwOwlz/bawvAuvcj8rrithP2/N55Tzz342AkTvq4qaWbGfmCk/tKhNaV2cthijKrPAA8SRJV5WWe7IBMJw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.25.9", - "@babel/generator": "^7.25.9", - "@babel/parser": "^7.25.9", - "@babel/template": "^7.25.9", - "@babel/types": "^7.25.9", - "debug": "^4.3.1", - "globals": "^11.1.0" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/traverse/node_modules/debug": { - "version": "4.3.7", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", - "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/@babel/traverse/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true, - "license": "MIT" - }, - "node_modules/@babel/types": { - "version": "7.26.0", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.26.0.tgz", - "integrity": "sha512-Z/yiTPj+lDVnF7lWeKCIJzaIkI0vYO87dMpZ4bg4TDrFe4XXLFWL1TbXU27gBP3QccxV9mZICCrnjnYlJjXHOA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-string-parser": "^7.25.9", - "@babel/helper-validator-identifier": "^7.25.9" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@bcoe/v8-coverage": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz", - "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", - "dev": true, - "license": "MIT" - }, - "node_modules/@discoveryjs/json-ext": { - "version": "0.5.7", - "resolved": "https://registry.npmjs.org/@discoveryjs/json-ext/-/json-ext-0.5.7.tgz", - "integrity": "sha512-dBVuXR082gk3jsFp7Rd/JI4kytwGHecnCoTtXFb7DB6CNHp4rg5k1bhg0nWdLGLnOV71lmDzGQaLMy8iPLY0pw==", - "dev": true, - "engines": { - "node": ">=10.0.0" - } - }, - "node_modules/@emnapi/runtime": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.2.0.tgz", - "integrity": "sha512-bV21/9LQmcQeCPEg3BDFtvwL6cwiTMksYNWQQ4KOxCZikEGalWtenoZ0wCiukJINlGCIi2KXx01g4FoH/LxpzQ==", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, - "node_modules/@huggingface/jinja": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/@huggingface/jinja/-/jinja-0.3.0.tgz", - "integrity": "sha512-GLJzso0M07ZncFkrJMIXVU4os6GFbPocD4g8fMQPMGJubf48FtGOsUORH2rtFdXPIPelz8SLBMn8ZRmOTwXm9Q==", - "engines": { - "node": ">=18" - } - }, - "node_modules/@img/sharp-darwin-arm64": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.33.5.tgz", - "integrity": "sha512-UT4p+iz/2H4twwAoLCqfA9UH5pI6DggwKEGuaPy7nCVQ8ZsiY5PIcrRvD1DzuY3qYL07NtIQcWnBSY/heikIFQ==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-darwin-arm64": "1.0.4" - } - }, - "node_modules/@img/sharp-darwin-x64": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.33.5.tgz", - "integrity": "sha512-fyHac4jIc1ANYGRDxtiqelIbdWkIuQaI84Mv45KvGRRxSAa7o7d1ZKAOBaYbnepLC1WqxfpimdeWfvqqSGwR2Q==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-darwin-x64": "1.0.4" - } - }, - "node_modules/@img/sharp-libvips-darwin-arm64": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.0.4.tgz", - "integrity": "sha512-XblONe153h0O2zuFfTAbQYAX2JhYmDHeWikp1LM9Hul9gVPjFY427k6dFEcOL72O01QxQsWi761svJ/ev9xEDg==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "darwin" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-darwin-x64": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.0.4.tgz", - "integrity": "sha512-xnGR8YuZYfJGmWPvmlunFaWJsb9T/AO2ykoP3Fz/0X5XV2aoYBPkX6xqCQvUTKKiLddarLaxpzNe+b1hjeWHAQ==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "darwin" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-arm": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.0.5.tgz", - "integrity": "sha512-gvcC4ACAOPRNATg/ov8/MnbxFDJqf/pDePbBnuBDcjsI8PssmjoKMAz4LtLaVi+OnSb5FK/yIOamqDwGmXW32g==", - "cpu": [ - "arm" - ], - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-arm64": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.0.4.tgz", - "integrity": "sha512-9B+taZ8DlyyqzZQnoeIvDVR/2F4EbMepXMc/NdVbkzsJbzkUjhXv/70GQJ7tdLA4YJgNP25zukcxpX2/SueNrA==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-s390x": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.0.4.tgz", - "integrity": "sha512-u7Wz6ntiSSgGSGcjZ55im6uvTrOxSIS8/dgoVMoiGE9I6JAfU50yH5BoDlYA1tcuGS7g/QNtetJnxA6QEsCVTA==", - "cpu": [ - "s390x" - ], - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-x64": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.0.4.tgz", - "integrity": "sha512-MmWmQ3iPFZr0Iev+BAgVMb3ZyC4KeFc3jFxnNbEPas60e1cIfevbtuyf9nDGIzOaW9PdnDciJm+wFFaTlj5xYw==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linuxmusl-arm64": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.0.4.tgz", - "integrity": "sha512-9Ti+BbTYDcsbp4wfYib8Ctm1ilkugkA/uscUn6UXK1ldpC1JjiXbLfFZtRlBhjPZ5o1NCLiDbg8fhUPKStHoTA==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linuxmusl-x64": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.0.4.tgz", - "integrity": "sha512-viYN1KX9m+/hGkJtvYYp+CCLgnJXwiQB39damAO7WMdKWlIhmYTfHjwSbQeUK/20vY154mwezd9HflVFM1wVSw==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-linux-arm": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.33.5.tgz", - "integrity": "sha512-JTS1eldqZbJxjvKaAkxhZmBqPRGmxgu+qFKSInv8moZ2AmT5Yib3EQ1c6gp493HvrvV8QgdOXdyaIBrhvFhBMQ==", - "cpu": [ - "arm" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-arm": "1.0.5" - } - }, - "node_modules/@img/sharp-linux-arm64": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.33.5.tgz", - "integrity": "sha512-JMVv+AMRyGOHtO1RFBiJy/MBsgz0x4AWrT6QoEVVTyh1E39TrCUpTRI7mx9VksGX4awWASxqCYLCV4wBZHAYxA==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-arm64": "1.0.4" - } - }, - "node_modules/@img/sharp-linux-s390x": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.33.5.tgz", - "integrity": "sha512-y/5PCd+mP4CA/sPDKl2961b+C9d+vPAveS33s6Z3zfASk2j5upL6fXVPZi7ztePZ5CuH+1kW8JtvxgbuXHRa4Q==", - "cpu": [ - "s390x" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-s390x": "1.0.4" - } - }, - "node_modules/@img/sharp-linux-x64": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.33.5.tgz", - "integrity": "sha512-opC+Ok5pRNAzuvq1AG0ar+1owsu842/Ab+4qvU879ippJBHvyY5n2mxF1izXqkPYlGuP/M556uh53jRLJmzTWA==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-x64": "1.0.4" - } - }, - "node_modules/@img/sharp-linuxmusl-arm64": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.33.5.tgz", - "integrity": "sha512-XrHMZwGQGvJg2V/oRSUfSAfjfPxO+4DkiRh6p2AFjLQztWUuY/o8Mq0eMQVIY7HJ1CDQUJlxGGZRw1a5bqmd1g==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-arm64": "1.0.4" - } - }, - "node_modules/@img/sharp-linuxmusl-x64": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.33.5.tgz", - "integrity": "sha512-WT+d/cgqKkkKySYmqoZ8y3pxx7lx9vVejxW/W4DOFMYVSkErR+w7mf2u8m/y4+xHe7yY9DAXQMWQhpnMuFfScw==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-x64": "1.0.4" - } - }, - "node_modules/@img/sharp-wasm32": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.33.5.tgz", - "integrity": "sha512-ykUW4LVGaMcU9lu9thv85CbRMAwfeadCJHRsg2GmeRa/cJxsVY9Rbd57JcMxBkKHag5U/x7TSBpScF4U8ElVzg==", - "cpu": [ - "wasm32" - ], - "optional": true, - "dependencies": { - "@emnapi/runtime": "^1.2.0" - }, - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-win32-ia32": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.33.5.tgz", - "integrity": "sha512-T36PblLaTwuVJ/zw/LaH0PdZkRz5rd3SmMHX8GSmR7vtNSP5Z6bQkExdSK7xGWyxLw4sUknBuugTelgw2faBbQ==", - "cpu": [ - "ia32" - ], - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-win32-x64": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.33.5.tgz", - "integrity": "sha512-MpY/o8/8kj+EcnxwvrP4aTJSWw/aZ7JIGR4aBeZkZw5B7/Jn+tY9/VNwtcoGmdT7GfggGIU4kygOMSbYnOrAbg==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@isaacs/cliui": { - "version": "8.0.2", - "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", - "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", - "dependencies": { - "string-width": "^5.1.2", - "string-width-cjs": "npm:string-width@^4.2.0", - "strip-ansi": "^7.0.1", - "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", - "wrap-ansi": "^8.1.0", - "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/@isaacs/cliui/node_modules/ansi-regex": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", - "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-regex?sponsor=1" - } - }, - "node_modules/@isaacs/cliui/node_modules/ansi-styles": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", - "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/@isaacs/cliui/node_modules/emoji-regex": { - "version": "9.2.2", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", - "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" - }, - "node_modules/@isaacs/cliui/node_modules/string-width": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", - "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", - "dependencies": { - "eastasianwidth": "^0.2.0", - "emoji-regex": "^9.2.2", - "strip-ansi": "^7.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/@isaacs/cliui/node_modules/strip-ansi": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", - "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", - "dependencies": { - "ansi-regex": "^6.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/strip-ansi?sponsor=1" - } - }, - "node_modules/@isaacs/cliui/node_modules/wrap-ansi": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", - "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", - "dependencies": { - "ansi-styles": "^6.1.0", - "string-width": "^5.0.1", - "strip-ansi": "^7.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/@isaacs/fs-minipass": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", - "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==", - "dependencies": { - "minipass": "^7.0.4" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@istanbuljs/load-nyc-config": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", - "integrity": "sha512-VjeHSlIzpv/NyD3N0YuHfXOPDIixcA1q2ZV98wsMqcYlPmv2n3Yb2lYP9XMElnaFVXg5A7YLTeLu6V84uQDjmQ==", - "dev": true, - "license": "ISC", - "dependencies": { - "camelcase": "^5.3.1", - "find-up": "^4.1.0", - "get-package-type": "^0.1.0", - "js-yaml": "^3.13.1", - "resolve-from": "^5.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/@istanbuljs/schema": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.3.tgz", - "integrity": "sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/@jest/console": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/console/-/console-30.0.0-alpha.6.tgz", - "integrity": "sha512-08BeAnuabmauj5B+Xa4GNPAotQUGm3PLKSE3rnpnmxniZzR4tXhx8+AA2+HGTri4bbVRY/r3Jl0vJnkhvHTkeQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "30.0.0-alpha.6", - "@types/node": "*", - "chalk": "^4.0.0", - "jest-message-util": "30.0.0-alpha.6", - "jest-util": "30.0.0-alpha.6", - "slash": "^3.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/console/node_modules/@jest/schemas": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", - "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@sinclair/typebox": "^0.33.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/console/node_modules/@jest/types": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", - "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/pattern": "30.0.0-alpha.6", - "@jest/schemas": "30.0.0-alpha.6", - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^3.0.0", - "@types/node": "*", - "@types/yargs": "^17.0.8", - "chalk": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/console/node_modules/@sinclair/typebox": { - "version": "0.33.17", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", - "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", - "dev": true, - "license": "MIT" - }, - "node_modules/@jest/console/node_modules/ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/@jest/console/node_modules/ci-info": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.0.0.tgz", - "integrity": "sha512-TdHqgGf9odd8SXNuxtUBVx8Nv+qZOejE6qyqiy5NtbYYQOeFa6zmHkxlPzmaLxWWHsU6nJmB7AETdVPi+2NBUg==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/sibiraj-s" - } - ], - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/@jest/console/node_modules/jest-message-util": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-30.0.0-alpha.6.tgz", - "integrity": "sha512-XAGJqkrBo7m3bFxWqiNqL0PyAWGf1XHR6bTve90MjBKJuIzhJsounGTzBNUw8JoU7Uzcj5Z6ZmEhaE3CDnwjfw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.12.13", - "@jest/types": "30.0.0-alpha.6", - "@types/stack-utils": "^2.0.0", - "chalk": "^4.0.0", - "graceful-fs": "^4.2.9", - "micromatch": "^4.0.7", - "pretty-format": "30.0.0-alpha.6", - "slash": "^3.0.0", - "stack-utils": "^2.0.3" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/console/node_modules/jest-util": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.0.0-alpha.6.tgz", - "integrity": "sha512-JlimakOVDyoMC8TEG+knoufxUqLG+Btihf1G8o5sHxz54C6oL54Wetfepp+Nhuj/1hSL0sQtkovvjlEycf9i0w==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "30.0.0-alpha.6", - "@types/node": "*", - "chalk": "^4.0.0", - "ci-info": "^4.0.0", - "graceful-fs": "^4.2.9", - "picomatch": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/console/node_modules/picomatch": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", - "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/@jest/console/node_modules/pretty-format": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.0.0-alpha.6.tgz", - "integrity": "sha512-xkeffkZoqQmRrcNewpOsUCKNOl+CkPqjt3Ld749uz1S7/O7GuPNPv2fZk3v/1U/FE8/B4Zz0llVL80MKON1tOQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/schemas": "30.0.0-alpha.6", - "ansi-styles": "^5.0.0", - "react-is": "^18.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/core": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/core/-/core-30.0.0-alpha.6.tgz", - "integrity": "sha512-Qsvu9/I0hUOpeelp3jlTmg6cg3C+w18v4hxWVGchCRJAChvuxmsomB1Cm+DKB6NiMy2EvUvpwdT8X31lERtemw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/console": "30.0.0-alpha.6", - "@jest/pattern": "30.0.0-alpha.6", - "@jest/reporters": "30.0.0-alpha.6", - "@jest/test-result": "30.0.0-alpha.6", - "@jest/transform": "30.0.0-alpha.6", - "@jest/types": "30.0.0-alpha.6", - "@types/node": "*", - "ansi-escapes": "^4.2.1", - "chalk": "^4.0.0", - "ci-info": "^4.0.0", - "exit": "^0.1.2", - "graceful-fs": "^4.2.9", - "jest-changed-files": "30.0.0-alpha.6", - "jest-config": "30.0.0-alpha.6", - "jest-haste-map": "30.0.0-alpha.6", - "jest-message-util": "30.0.0-alpha.6", - "jest-regex-util": "30.0.0-alpha.6", - "jest-resolve": "30.0.0-alpha.6", - "jest-resolve-dependencies": "30.0.0-alpha.6", - "jest-runner": "30.0.0-alpha.6", - "jest-runtime": "30.0.0-alpha.6", - "jest-snapshot": "30.0.0-alpha.6", - "jest-util": "30.0.0-alpha.6", - "jest-validate": "30.0.0-alpha.6", - "jest-watcher": "30.0.0-alpha.6", - "micromatch": "^4.0.7", - "pretty-format": "30.0.0-alpha.6", - "slash": "^3.0.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - }, - "peerDependencies": { - "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" - }, - "peerDependenciesMeta": { - "node-notifier": { - "optional": true - } - } - }, - "node_modules/@jest/core/node_modules/@jest/schemas": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", - "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@sinclair/typebox": "^0.33.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/core/node_modules/@jest/types": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", - "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/pattern": "30.0.0-alpha.6", - "@jest/schemas": "30.0.0-alpha.6", - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^3.0.0", - "@types/node": "*", - "@types/yargs": "^17.0.8", - "chalk": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/core/node_modules/@sinclair/typebox": { - "version": "0.33.17", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", - "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", - "dev": true, - "license": "MIT" - }, - "node_modules/@jest/core/node_modules/ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/@jest/core/node_modules/ci-info": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.0.0.tgz", - "integrity": "sha512-TdHqgGf9odd8SXNuxtUBVx8Nv+qZOejE6qyqiy5NtbYYQOeFa6zmHkxlPzmaLxWWHsU6nJmB7AETdVPi+2NBUg==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/sibiraj-s" - } - ], - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/@jest/core/node_modules/jest-message-util": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-30.0.0-alpha.6.tgz", - "integrity": "sha512-XAGJqkrBo7m3bFxWqiNqL0PyAWGf1XHR6bTve90MjBKJuIzhJsounGTzBNUw8JoU7Uzcj5Z6ZmEhaE3CDnwjfw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.12.13", - "@jest/types": "30.0.0-alpha.6", - "@types/stack-utils": "^2.0.0", - "chalk": "^4.0.0", - "graceful-fs": "^4.2.9", - "micromatch": "^4.0.7", - "pretty-format": "30.0.0-alpha.6", - "slash": "^3.0.0", - "stack-utils": "^2.0.3" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/core/node_modules/jest-util": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.0.0-alpha.6.tgz", - "integrity": "sha512-JlimakOVDyoMC8TEG+knoufxUqLG+Btihf1G8o5sHxz54C6oL54Wetfepp+Nhuj/1hSL0sQtkovvjlEycf9i0w==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "30.0.0-alpha.6", - "@types/node": "*", - "chalk": "^4.0.0", - "ci-info": "^4.0.0", - "graceful-fs": "^4.2.9", - "picomatch": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/core/node_modules/picomatch": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", - "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/@jest/core/node_modules/pretty-format": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.0.0-alpha.6.tgz", - "integrity": "sha512-xkeffkZoqQmRrcNewpOsUCKNOl+CkPqjt3Ld749uz1S7/O7GuPNPv2fZk3v/1U/FE8/B4Zz0llVL80MKON1tOQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/schemas": "30.0.0-alpha.6", - "ansi-styles": "^5.0.0", - "react-is": "^18.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/environment": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-30.0.0-alpha.6.tgz", - "integrity": "sha512-pjNYNkzq761hh8D2grrG77L6nhe2VBCFFM+G1hyqhaJ2MAzhp2Gh+G94uF3px7luSzLh8GYvGJQGYy197EUOGQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/fake-timers": "30.0.0-alpha.6", - "@jest/types": "30.0.0-alpha.6", - "@types/node": "*", - "jest-mock": "30.0.0-alpha.6" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/environment/node_modules/@jest/schemas": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", - "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@sinclair/typebox": "^0.33.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/environment/node_modules/@jest/types": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", - "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/pattern": "30.0.0-alpha.6", - "@jest/schemas": "30.0.0-alpha.6", - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^3.0.0", - "@types/node": "*", - "@types/yargs": "^17.0.8", - "chalk": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/environment/node_modules/@sinclair/typebox": { - "version": "0.33.17", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", - "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", - "dev": true, - "license": "MIT" - }, - "node_modules/@jest/expect": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/expect/-/expect-30.0.0-alpha.6.tgz", - "integrity": "sha512-3O74pygTwUBzUjO958IgNwmp0WrjASbiWdMEfUMePVqtiGoyS4Nxj9hsx4uKsNVivNJSZiiayYoP6dLhWerJXQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "expect": "30.0.0-alpha.6", - "jest-snapshot": "30.0.0-alpha.6" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/expect-utils": { - "version": "29.6.1", - "resolved": "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-29.6.1.tgz", - "integrity": "sha512-o319vIf5pEMx0LmzSxxkYYxo4wrRLKHq9dP1yJU7FoPTB0LfAKSz8SWD6D/6U3v/O52t9cF5t+MeJiRsfk7zMw==", - "dev": true, - "dependencies": { - "jest-get-type": "^29.4.3" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/expect/node_modules/@jest/expect-utils": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-30.0.0-alpha.6.tgz", - "integrity": "sha512-QMySMhaCUl0ZQd7Tx5X3fVWY5jtQxZNrTll0OyavdQ70ZTLgk0kU9K+XovcMWO26MK9R5EX7bBgD/j7w9hUM4w==", - "dev": true, - "license": "MIT", - "dependencies": { - "jest-get-type": "30.0.0-alpha.6" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/expect/node_modules/@jest/schemas": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", - "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@sinclair/typebox": "^0.33.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/expect/node_modules/@jest/types": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", - "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/pattern": "30.0.0-alpha.6", - "@jest/schemas": "30.0.0-alpha.6", - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^3.0.0", - "@types/node": "*", - "@types/yargs": "^17.0.8", - "chalk": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/expect/node_modules/@sinclair/typebox": { - "version": "0.33.17", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", - "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", - "dev": true, - "license": "MIT" - }, - "node_modules/@jest/expect/node_modules/ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/@jest/expect/node_modules/ci-info": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.0.0.tgz", - "integrity": "sha512-TdHqgGf9odd8SXNuxtUBVx8Nv+qZOejE6qyqiy5NtbYYQOeFa6zmHkxlPzmaLxWWHsU6nJmB7AETdVPi+2NBUg==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/sibiraj-s" - } - ], - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/@jest/expect/node_modules/diff-sequences": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-30.0.0-alpha.6.tgz", - "integrity": "sha512-DVGt3/yzbneMUTuupsMqyfSXMnU2fE0lVsC9uFsJmRpluvSi7ZhrS0GX5tnMna6Ta788FGfOUx+irI/+cAZ4EA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/expect/node_modules/expect": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/expect/-/expect-30.0.0-alpha.6.tgz", - "integrity": "sha512-WVi2V4iHKw/vHEyye00Q9CSZz7KHDbJkJyteUI8kTih9jiyMl3bIk7wLYFcY9D1Blnadlyb5w5NBuNjQBow99g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/expect-utils": "30.0.0-alpha.6", - "jest-get-type": "30.0.0-alpha.6", - "jest-matcher-utils": "30.0.0-alpha.6", - "jest-message-util": "30.0.0-alpha.6", - "jest-mock": "30.0.0-alpha.6", - "jest-util": "30.0.0-alpha.6" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/expect/node_modules/jest-diff": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-30.0.0-alpha.6.tgz", - "integrity": "sha512-43j1DoYwVKrkbB67a2gC0ijjIY9biF0JSPXv7H6zlOkzNlqYg8hSDzrurLNo6zGKatW4JSBLE79LmXPJPj1m6A==", - "dev": true, - "license": "MIT", - "dependencies": { - "chalk": "^4.0.0", - "diff-sequences": "30.0.0-alpha.6", - "jest-get-type": "30.0.0-alpha.6", - "pretty-format": "30.0.0-alpha.6" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/expect/node_modules/jest-get-type": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-30.0.0-alpha.6.tgz", - "integrity": "sha512-lJEoQdCY4ICN6+T0lJ9BODKuqPOEpCv2NnJsEO1nmsK0fbWZmN/pgOPHVqLfK8i3jZpUmgupJ1w8r36mc8iiBQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/expect/node_modules/jest-matcher-utils": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-30.0.0-alpha.6.tgz", - "integrity": "sha512-jaq7+HznsK54G0qzu96ZwfMEKHmlPiDqg6qG2p/hVQzr6Y/qVMRh8abI9Y1lX6SSXkr+S9mPAkmOsuJNLTLYmQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "chalk": "^4.0.0", - "jest-diff": "30.0.0-alpha.6", - "jest-get-type": "30.0.0-alpha.6", - "pretty-format": "30.0.0-alpha.6" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/expect/node_modules/jest-message-util": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-30.0.0-alpha.6.tgz", - "integrity": "sha512-XAGJqkrBo7m3bFxWqiNqL0PyAWGf1XHR6bTve90MjBKJuIzhJsounGTzBNUw8JoU7Uzcj5Z6ZmEhaE3CDnwjfw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.12.13", - "@jest/types": "30.0.0-alpha.6", - "@types/stack-utils": "^2.0.0", - "chalk": "^4.0.0", - "graceful-fs": "^4.2.9", - "micromatch": "^4.0.7", - "pretty-format": "30.0.0-alpha.6", - "slash": "^3.0.0", - "stack-utils": "^2.0.3" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/expect/node_modules/jest-util": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.0.0-alpha.6.tgz", - "integrity": "sha512-JlimakOVDyoMC8TEG+knoufxUqLG+Btihf1G8o5sHxz54C6oL54Wetfepp+Nhuj/1hSL0sQtkovvjlEycf9i0w==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "30.0.0-alpha.6", - "@types/node": "*", - "chalk": "^4.0.0", - "ci-info": "^4.0.0", - "graceful-fs": "^4.2.9", - "picomatch": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/expect/node_modules/picomatch": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", - "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/@jest/expect/node_modules/pretty-format": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.0.0-alpha.6.tgz", - "integrity": "sha512-xkeffkZoqQmRrcNewpOsUCKNOl+CkPqjt3Ld749uz1S7/O7GuPNPv2fZk3v/1U/FE8/B4Zz0llVL80MKON1tOQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/schemas": "30.0.0-alpha.6", - "ansi-styles": "^5.0.0", - "react-is": "^18.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/fake-timers": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/fake-timers/-/fake-timers-30.0.0-alpha.6.tgz", - "integrity": "sha512-deka0RmhJgEKPJM6cXPd4TJQ6fLczErdMN7Oxzr16UTDFHxtFd79tduJ8uP92dQyO4zy63N/dlFK6d+FHyWUDw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "30.0.0-alpha.6", - "@sinonjs/fake-timers": "^11.1.0", - "@types/node": "*", - "jest-message-util": "30.0.0-alpha.6", - "jest-mock": "30.0.0-alpha.6", - "jest-util": "30.0.0-alpha.6" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/fake-timers/node_modules/@jest/schemas": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", - "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@sinclair/typebox": "^0.33.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/fake-timers/node_modules/@jest/types": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", - "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/pattern": "30.0.0-alpha.6", - "@jest/schemas": "30.0.0-alpha.6", - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^3.0.0", - "@types/node": "*", - "@types/yargs": "^17.0.8", - "chalk": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/fake-timers/node_modules/@sinclair/typebox": { - "version": "0.33.17", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", - "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", - "dev": true, - "license": "MIT" - }, - "node_modules/@jest/fake-timers/node_modules/ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/@jest/fake-timers/node_modules/ci-info": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.0.0.tgz", - "integrity": "sha512-TdHqgGf9odd8SXNuxtUBVx8Nv+qZOejE6qyqiy5NtbYYQOeFa6zmHkxlPzmaLxWWHsU6nJmB7AETdVPi+2NBUg==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/sibiraj-s" - } - ], - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/@jest/fake-timers/node_modules/jest-message-util": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-30.0.0-alpha.6.tgz", - "integrity": "sha512-XAGJqkrBo7m3bFxWqiNqL0PyAWGf1XHR6bTve90MjBKJuIzhJsounGTzBNUw8JoU7Uzcj5Z6ZmEhaE3CDnwjfw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.12.13", - "@jest/types": "30.0.0-alpha.6", - "@types/stack-utils": "^2.0.0", - "chalk": "^4.0.0", - "graceful-fs": "^4.2.9", - "micromatch": "^4.0.7", - "pretty-format": "30.0.0-alpha.6", - "slash": "^3.0.0", - "stack-utils": "^2.0.3" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/fake-timers/node_modules/jest-util": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.0.0-alpha.6.tgz", - "integrity": "sha512-JlimakOVDyoMC8TEG+knoufxUqLG+Btihf1G8o5sHxz54C6oL54Wetfepp+Nhuj/1hSL0sQtkovvjlEycf9i0w==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "30.0.0-alpha.6", - "@types/node": "*", - "chalk": "^4.0.0", - "ci-info": "^4.0.0", - "graceful-fs": "^4.2.9", - "picomatch": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/fake-timers/node_modules/picomatch": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", - "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/@jest/fake-timers/node_modules/pretty-format": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.0.0-alpha.6.tgz", - "integrity": "sha512-xkeffkZoqQmRrcNewpOsUCKNOl+CkPqjt3Ld749uz1S7/O7GuPNPv2fZk3v/1U/FE8/B4Zz0llVL80MKON1tOQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/schemas": "30.0.0-alpha.6", - "ansi-styles": "^5.0.0", - "react-is": "^18.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/globals": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/globals/-/globals-30.0.0-alpha.6.tgz", - "integrity": "sha512-+uJMoPUos9RH6+52iNgKJBbx1Hd2QsCZjExi5XpVvTjJ/gE4eJ1X7irUMt+14sH0QkeZ3GnjeTJFopyjOCsu+Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/environment": "30.0.0-alpha.6", - "@jest/expect": "30.0.0-alpha.6", - "@jest/types": "30.0.0-alpha.6", - "jest-mock": "30.0.0-alpha.6" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/globals/node_modules/@jest/schemas": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", - "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@sinclair/typebox": "^0.33.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/globals/node_modules/@jest/types": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", - "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/pattern": "30.0.0-alpha.6", - "@jest/schemas": "30.0.0-alpha.6", - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^3.0.0", - "@types/node": "*", - "@types/yargs": "^17.0.8", - "chalk": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/globals/node_modules/@sinclair/typebox": { - "version": "0.33.17", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", - "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", - "dev": true, - "license": "MIT" - }, - "node_modules/@jest/pattern": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/pattern/-/pattern-30.0.0-alpha.6.tgz", - "integrity": "sha512-eoV3sjS1M5k3YdrFWezqdndfgepwB86gwyXC0BzV2saZdJ42ySUoEDBGKuwta8A6Zh3w8tVHNFxz1BqiFraHCQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*", - "jest-regex-util": "30.0.0-alpha.6" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/reporters": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/reporters/-/reporters-30.0.0-alpha.6.tgz", - "integrity": "sha512-jzW0t2OtEzBYwlG4EMJKG4q5RPaVvLPDm/nBS08hd+XPoLJJ9b5thyo/MoThIqJfdi0lHqFlDQUmlL205CMoSw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@bcoe/v8-coverage": "^0.2.3", - "@jest/console": "30.0.0-alpha.6", - "@jest/test-result": "30.0.0-alpha.6", - "@jest/transform": "30.0.0-alpha.6", - "@jest/types": "30.0.0-alpha.6", - "@jridgewell/trace-mapping": "^0.3.18", - "@types/node": "*", - "chalk": "^4.0.0", - "collect-v8-coverage": "^1.0.0", - "exit": "^0.1.2", - "glob": "^10.3.10", - "graceful-fs": "^4.2.9", - "istanbul-lib-coverage": "^3.0.0", - "istanbul-lib-instrument": "^6.0.0", - "istanbul-lib-report": "^3.0.0", - "istanbul-lib-source-maps": "^5.0.0", - "istanbul-reports": "^3.1.3", - "jest-message-util": "30.0.0-alpha.6", - "jest-util": "30.0.0-alpha.6", - "jest-worker": "30.0.0-alpha.6", - "slash": "^3.0.0", - "string-length": "^4.0.1", - "strip-ansi": "^6.0.0", - "v8-to-istanbul": "^9.0.1" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - }, - "peerDependencies": { - "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" - }, - "peerDependenciesMeta": { - "node-notifier": { - "optional": true - } - } - }, - "node_modules/@jest/reporters/node_modules/@jest/schemas": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", - "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@sinclair/typebox": "^0.33.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/reporters/node_modules/@jest/types": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", - "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/pattern": "30.0.0-alpha.6", - "@jest/schemas": "30.0.0-alpha.6", - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^3.0.0", - "@types/node": "*", - "@types/yargs": "^17.0.8", - "chalk": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/reporters/node_modules/@sinclair/typebox": { - "version": "0.33.17", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", - "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", - "dev": true, - "license": "MIT" - }, - "node_modules/@jest/reporters/node_modules/ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/@jest/reporters/node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0" - } - }, - "node_modules/@jest/reporters/node_modules/ci-info": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.0.0.tgz", - "integrity": "sha512-TdHqgGf9odd8SXNuxtUBVx8Nv+qZOejE6qyqiy5NtbYYQOeFa6zmHkxlPzmaLxWWHsU6nJmB7AETdVPi+2NBUg==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/sibiraj-s" - } - ], - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/@jest/reporters/node_modules/glob": { - "version": "10.4.5", - "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz", - "integrity": "sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==", - "dev": true, - "license": "ISC", - "dependencies": { - "foreground-child": "^3.1.0", - "jackspeak": "^3.1.2", - "minimatch": "^9.0.4", - "minipass": "^7.1.2", - "package-json-from-dist": "^1.0.0", - "path-scurry": "^1.11.1" - }, - "bin": { - "glob": "dist/esm/bin.mjs" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/@jest/reporters/node_modules/jest-message-util": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-30.0.0-alpha.6.tgz", - "integrity": "sha512-XAGJqkrBo7m3bFxWqiNqL0PyAWGf1XHR6bTve90MjBKJuIzhJsounGTzBNUw8JoU7Uzcj5Z6ZmEhaE3CDnwjfw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.12.13", - "@jest/types": "30.0.0-alpha.6", - "@types/stack-utils": "^2.0.0", - "chalk": "^4.0.0", - "graceful-fs": "^4.2.9", - "micromatch": "^4.0.7", - "pretty-format": "30.0.0-alpha.6", - "slash": "^3.0.0", - "stack-utils": "^2.0.3" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/reporters/node_modules/jest-util": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.0.0-alpha.6.tgz", - "integrity": "sha512-JlimakOVDyoMC8TEG+knoufxUqLG+Btihf1G8o5sHxz54C6oL54Wetfepp+Nhuj/1hSL0sQtkovvjlEycf9i0w==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "30.0.0-alpha.6", - "@types/node": "*", - "chalk": "^4.0.0", - "ci-info": "^4.0.0", - "graceful-fs": "^4.2.9", - "picomatch": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/reporters/node_modules/jest-worker": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-30.0.0-alpha.6.tgz", - "integrity": "sha512-qlzX7zFT/QdUV/LWsJwZBlaIBaJ+E2VH3d1gArGVP+9hUHGpJkEzCSBK7yuZrkt+M/U0Jre5+maPRmkinEF4DA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*", - "@ungap/structured-clone": "^1.2.0", - "jest-util": "30.0.0-alpha.6", - "merge-stream": "^2.0.0", - "supports-color": "^8.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/reporters/node_modules/minimatch": { - "version": "9.0.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", - "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", - "dev": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^2.0.1" - }, - "engines": { - "node": ">=16 || 14 >=14.17" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/@jest/reporters/node_modules/picomatch": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", - "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/@jest/reporters/node_modules/pretty-format": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.0.0-alpha.6.tgz", - "integrity": "sha512-xkeffkZoqQmRrcNewpOsUCKNOl+CkPqjt3Ld749uz1S7/O7GuPNPv2fZk3v/1U/FE8/B4Zz0llVL80MKON1tOQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/schemas": "30.0.0-alpha.6", - "ansi-styles": "^5.0.0", - "react-is": "^18.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/schemas": { - "version": "29.6.0", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-29.6.0.tgz", - "integrity": "sha512-rxLjXyJBTL4LQeJW3aKo0M/+GkCOXsO+8i9Iu7eDb6KwtP65ayoDsitrdPBtujxQ88k4wI2FNYfa6TOGwSn6cQ==", - "dev": true, - "dependencies": { - "@sinclair/typebox": "^0.27.8" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/snapshot-utils": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/snapshot-utils/-/snapshot-utils-30.0.0-alpha.6.tgz", - "integrity": "sha512-iDtIFCyRT8ZyLmz6kYbS8GR/MBXKA6uZPBfdTcnd2y0T987DV3GVlvwkAC+iFTc1w3HgwQe8LTf+y3i+O2ISCw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "30.0.0-alpha.6", - "chalk": "^4.0.0", - "graceful-fs": "^4.2.9", - "natural-compare": "^1.4.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/snapshot-utils/node_modules/@jest/schemas": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", - "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@sinclair/typebox": "^0.33.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/snapshot-utils/node_modules/@jest/types": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", - "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/pattern": "30.0.0-alpha.6", - "@jest/schemas": "30.0.0-alpha.6", - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^3.0.0", - "@types/node": "*", - "@types/yargs": "^17.0.8", - "chalk": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/snapshot-utils/node_modules/@sinclair/typebox": { - "version": "0.33.17", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", - "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", - "dev": true, - "license": "MIT" - }, - "node_modules/@jest/source-map": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/source-map/-/source-map-30.0.0-alpha.6.tgz", - "integrity": "sha512-7rSrxehVyzqw5O+F2ds7wLAm9f6QxqYsJU42LNyUpaFlJqtWz3PeQ2Wu3DVoPzGu0C66EhDHKYmeN0mXnRDZmQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/trace-mapping": "^0.3.18", - "callsites": "^3.0.0", - "graceful-fs": "^4.2.9" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/test-result": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/test-result/-/test-result-30.0.0-alpha.6.tgz", - "integrity": "sha512-Jlg8lCm7VQ6YvQ0eZx2nQEtej/ng+ulV8cXH7Nj5i33hNZq8EZvWM4gQDWDzRe1X7cVE3Bd42On5f6s2rqqIjw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/console": "30.0.0-alpha.6", - "@jest/types": "30.0.0-alpha.6", - "@types/istanbul-lib-coverage": "^2.0.0", - "collect-v8-coverage": "^1.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/test-result/node_modules/@jest/schemas": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", - "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@sinclair/typebox": "^0.33.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/test-result/node_modules/@jest/types": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", - "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/pattern": "30.0.0-alpha.6", - "@jest/schemas": "30.0.0-alpha.6", - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^3.0.0", - "@types/node": "*", - "@types/yargs": "^17.0.8", - "chalk": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/test-result/node_modules/@sinclair/typebox": { - "version": "0.33.17", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", - "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", - "dev": true, - "license": "MIT" - }, - "node_modules/@jest/test-sequencer": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/test-sequencer/-/test-sequencer-30.0.0-alpha.6.tgz", - "integrity": "sha512-5M89jbSQWkBjGlFrRk2wXjRJVxR+uN553sFN0q2TglH0/a4OMSVxRBcCmnIqqDMDizGAlYTxW6BaXxHGHpvrRQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/test-result": "30.0.0-alpha.6", - "graceful-fs": "^4.2.9", - "jest-haste-map": "30.0.0-alpha.6", - "slash": "^3.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/transform": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/transform/-/transform-30.0.0-alpha.6.tgz", - "integrity": "sha512-4L8BZm38BJASswsqruc4c3F0AExYLvp0xq8067e7fIyg4hfwa4zUA+N2idf+eTTjDWevVVdIBfELzJ8b7nvO4Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/core": "^7.11.6", - "@jest/types": "30.0.0-alpha.6", - "@jridgewell/trace-mapping": "^0.3.18", - "babel-plugin-istanbul": "^7.0.0", - "chalk": "^4.0.0", - "convert-source-map": "^2.0.0", - "fast-json-stable-stringify": "^2.1.0", - "graceful-fs": "^4.2.9", - "jest-haste-map": "30.0.0-alpha.6", - "jest-regex-util": "30.0.0-alpha.6", - "jest-util": "30.0.0-alpha.6", - "micromatch": "^4.0.7", - "pirates": "^4.0.4", - "slash": "^3.0.0", - "write-file-atomic": "^5.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/transform/node_modules/@jest/schemas": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", - "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@sinclair/typebox": "^0.33.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/transform/node_modules/@jest/types": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", - "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/pattern": "30.0.0-alpha.6", - "@jest/schemas": "30.0.0-alpha.6", - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^3.0.0", - "@types/node": "*", - "@types/yargs": "^17.0.8", - "chalk": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/transform/node_modules/@sinclair/typebox": { - "version": "0.33.17", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", - "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", - "dev": true, - "license": "MIT" - }, - "node_modules/@jest/transform/node_modules/ci-info": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.0.0.tgz", - "integrity": "sha512-TdHqgGf9odd8SXNuxtUBVx8Nv+qZOejE6qyqiy5NtbYYQOeFa6zmHkxlPzmaLxWWHsU6nJmB7AETdVPi+2NBUg==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/sibiraj-s" - } - ], - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/@jest/transform/node_modules/jest-util": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.0.0-alpha.6.tgz", - "integrity": "sha512-JlimakOVDyoMC8TEG+knoufxUqLG+Btihf1G8o5sHxz54C6oL54Wetfepp+Nhuj/1hSL0sQtkovvjlEycf9i0w==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "30.0.0-alpha.6", - "@types/node": "*", - "chalk": "^4.0.0", - "ci-info": "^4.0.0", - "graceful-fs": "^4.2.9", - "picomatch": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/@jest/transform/node_modules/picomatch": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", - "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/@jest/types": { - "version": "29.6.1", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-29.6.1.tgz", - "integrity": "sha512-tPKQNMPuXgvdOn2/Lg9HNfUvjYVGolt04Hp03f5hAk878uwOLikN+JzeLY0HcVgKgFl9Hs3EIqpu3WX27XNhnw==", - "dev": true, - "dependencies": { - "@jest/schemas": "^29.6.0", - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^3.0.0", - "@types/node": "*", - "@types/yargs": "^17.0.8", - "chalk": "^4.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jridgewell/gen-mapping": { - "version": "0.3.5", - "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.5.tgz", - "integrity": "sha512-IzL8ZoEDIBRWEzlCcRhOaCupYyN5gdIK+Q6fbFdPDg6HqX6jpkItn7DFIpW9LQzXG6Df9sA7+OKnq0qlz/GaQg==", - "dev": true, - "dependencies": { - "@jridgewell/set-array": "^1.2.1", - "@jridgewell/sourcemap-codec": "^1.4.10", - "@jridgewell/trace-mapping": "^0.3.24" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@jridgewell/resolve-uri": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz", - "integrity": "sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w==", - "dev": true, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@jridgewell/set-array": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.2.1.tgz", - "integrity": "sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A==", - "dev": true, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@jridgewell/source-map": { - "version": "0.3.6", - "resolved": "https://registry.npmjs.org/@jridgewell/source-map/-/source-map-0.3.6.tgz", - "integrity": "sha512-1ZJTZebgqllO79ue2bm3rIGud/bOe0pP5BjSRCRxxYkEZS8STV7zN84UBbiYu7jy+eCKSnVIUgoWWE/tt+shMQ==", - "dev": true, - "dependencies": { - "@jridgewell/gen-mapping": "^0.3.5", - "@jridgewell/trace-mapping": "^0.3.25" - } - }, - "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.4.15", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz", - "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==", - "dev": true - }, - "node_modules/@jridgewell/trace-mapping": { - "version": "0.3.25", - "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz", - "integrity": "sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==", - "dev": true, - "dependencies": { - "@jridgewell/resolve-uri": "^3.1.0", - "@jridgewell/sourcemap-codec": "^1.4.14" - } - }, - "node_modules/@jsdoc/salty": { - "version": "0.2.5", - "resolved": "https://registry.npmjs.org/@jsdoc/salty/-/salty-0.2.5.tgz", - "integrity": "sha512-TfRP53RqunNe2HBobVBJ0VLhK1HbfvBYeTC1ahnN64PWvyYyGebmMiPkuwvD9fpw2ZbkoPb8Q7mwy0aR8Z9rvw==", - "dev": true, - "dependencies": { - "lodash": "^4.17.21" - }, - "engines": { - "node": ">=v12.0.0" - } - }, - "node_modules/@leichtgewicht/ip-codec": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/@leichtgewicht/ip-codec/-/ip-codec-2.0.4.tgz", - "integrity": "sha512-Hcv+nVC0kZnQ3tD9GVu5xSMR4VVYOteQIr/hwFPVEvPdlXqgGEuRjiheChHgdM+JyqdgNcmzZOX/tnl0JOiI7A==", - "dev": true - }, - "node_modules/@pkgjs/parseargs": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", - "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", - "optional": true, - "engines": { - "node": ">=14" - } - }, - "node_modules/@pkgr/core": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/@pkgr/core/-/core-0.1.1.tgz", - "integrity": "sha512-cq8o4cWH0ibXh9VGi5P20Tu9XF/0fFXl9EUinr9QfTM7a7p0oTA4iJRCQWppXR1Pg8dSM0UCItCkPwsk9qWWYA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^12.20.0 || ^14.18.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/unts" - } - }, - "node_modules/@protobufjs/aspromise": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", - "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==" - }, - "node_modules/@protobufjs/base64": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", - "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==" - }, - "node_modules/@protobufjs/codegen": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", - "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==" - }, - "node_modules/@protobufjs/eventemitter": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", - "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==" - }, - "node_modules/@protobufjs/fetch": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", - "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", - "dependencies": { - "@protobufjs/aspromise": "^1.1.1", - "@protobufjs/inquire": "^1.1.0" - } - }, - "node_modules/@protobufjs/float": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", - "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==" - }, - "node_modules/@protobufjs/inquire": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", - "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==" - }, - "node_modules/@protobufjs/path": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", - "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==" - }, - "node_modules/@protobufjs/pool": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", - "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==" - }, - "node_modules/@protobufjs/utf8": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", - "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" - }, - "node_modules/@sinclair/typebox": { - "version": "0.27.8", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz", - "integrity": "sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==", - "dev": true - }, - "node_modules/@sinonjs/commons": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.1.tgz", - "integrity": "sha512-K3mCHKQ9sVh8o1C9cxkwxaOmXoAMlDxC1mYyHrjqOWEcBjYr76t96zL2zlj5dUGZ3HSw240X1qgH3Mjf1yJWpQ==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "type-detect": "4.0.8" - } - }, - "node_modules/@sinonjs/fake-timers": { - "version": "11.3.1", - "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-11.3.1.tgz", - "integrity": "sha512-EVJO7nW5M/F5Tur0Rf2z/QoMo+1Ia963RiMtapiQrEWvY0iBUvADo8Beegwjpnle5BHkyHuoxSTW3jF43H1XRA==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "@sinonjs/commons": "^3.0.1" - } - }, - "node_modules/@types/babel__core": { - "version": "7.20.5", - "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", - "integrity": "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/parser": "^7.20.7", - "@babel/types": "^7.20.7", - "@types/babel__generator": "*", - "@types/babel__template": "*", - "@types/babel__traverse": "*" - } - }, - "node_modules/@types/babel__generator": { - "version": "7.6.8", - "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.6.8.tgz", - "integrity": "sha512-ASsj+tpEDsEiFr1arWrlN6V3mdfjRMZt6LtK/Vp/kreFLnr5QH5+DhvD5nINYZXzwJvXeGq+05iUXcAzVrqWtw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/types": "^7.0.0" - } - }, - "node_modules/@types/babel__template": { - "version": "7.4.4", - "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.4.tgz", - "integrity": "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/parser": "^7.1.0", - "@babel/types": "^7.0.0" - } - }, - "node_modules/@types/babel__traverse": { - "version": "7.20.6", - "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.20.6.tgz", - "integrity": "sha512-r1bzfrm0tomOI8g1SzvCaQHo6Lcv6zu0EA+W2kHrt8dyrHQxGzBBL4kdkzIS+jBMV+EYcMAEAqXqYaLJq5rOZg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/types": "^7.20.7" - } - }, - "node_modules/@types/body-parser": { - "version": "1.19.2", - "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.2.tgz", - "integrity": "sha512-ALYone6pm6QmwZoAgeyNksccT9Q4AWZQ6PvfwR37GT6r6FWUPguq6sUmNGSMV2Wr761oQoBxwGGa6DR5o1DC9g==", - "dev": true, - "dependencies": { - "@types/connect": "*", - "@types/node": "*" - } - }, - "node_modules/@types/bonjour": { - "version": "3.5.10", - "resolved": "https://registry.npmjs.org/@types/bonjour/-/bonjour-3.5.10.tgz", - "integrity": "sha512-p7ienRMiS41Nu2/igbJxxLDWrSZ0WxM8UQgCeO9KhoVF7cOVFkrKsiDr1EsJIla8vV3oEEjGcz11jc5yimhzZw==", - "dev": true, - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/@types/connect": { - "version": "3.4.35", - "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz", - "integrity": "sha512-cdeYyv4KWoEgpBISTxWvqYsVy444DOqehiF3fM3ne10AmJ62RSyNkUnxMJXHQWRQQX2eR94m5y1IZyDwBjV9FQ==", - "dev": true, - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/@types/connect-history-api-fallback": { - "version": "1.3.5", - "resolved": "https://registry.npmjs.org/@types/connect-history-api-fallback/-/connect-history-api-fallback-1.3.5.tgz", - "integrity": "sha512-h8QJa8xSb1WD4fpKBDcATDNGXghFj6/3GRWG6dhmRcu0RX1Ubasur2Uvx5aeEwlf0MwblEC2bMzzMQntxnw/Cw==", - "dev": true, - "dependencies": { - "@types/express-serve-static-core": "*", - "@types/node": "*" - } - }, - "node_modules/@types/estree": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.5.tgz", - "integrity": "sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw==", - "dev": true - }, - "node_modules/@types/express": { - "version": "4.17.17", - "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.17.tgz", - "integrity": "sha512-Q4FmmuLGBG58btUnfS1c1r/NQdlp3DMfGDGig8WhfpA2YRUtEkxAjkZb0yvplJGYdF1fsQ81iMDcH24sSCNC/Q==", - "dev": true, - "dependencies": { - "@types/body-parser": "*", - "@types/express-serve-static-core": "^4.17.33", - "@types/qs": "*", - "@types/serve-static": "*" - } - }, - "node_modules/@types/express-serve-static-core": { - "version": "4.17.33", - "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.33.tgz", - "integrity": "sha512-TPBqmR/HRYI3eC2E5hmiivIzv+bidAfXofM+sbonAGvyDhySGw9/PQZFt2BLOrjUUR++4eJVpx6KnLQK1Fk9tA==", - "dev": true, - "dependencies": { - "@types/node": "*", - "@types/qs": "*", - "@types/range-parser": "*" - } - }, - "node_modules/@types/http-proxy": { - "version": "1.17.10", - "resolved": "https://registry.npmjs.org/@types/http-proxy/-/http-proxy-1.17.10.tgz", - "integrity": "sha512-Qs5aULi+zV1bwKAg5z1PWnDXWmsn+LxIvUGv6E2+OOMYhclZMO+OXd9pYVf2gLykf2I7IV2u7oTHwChPNsvJ7g==", - "dev": true, - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/@types/istanbul-lib-coverage": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.4.tgz", - "integrity": "sha512-z/QT1XN4K4KYuslS23k62yDIDLwLFkzxOuMplDtObz0+y7VqJCaO2o+SPwHCvLFZh7xazvvoor2tA/hPz9ee7g==", - "dev": true - }, - "node_modules/@types/istanbul-lib-report": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@types/istanbul-lib-report/-/istanbul-lib-report-3.0.0.tgz", - "integrity": "sha512-plGgXAPfVKFoYfa9NpYDAkseG+g6Jr294RqeqcqDixSbU34MZVJRi/P+7Y8GDpzkEwLaGZZOpKIEmeVZNtKsrg==", - "dev": true, - "dependencies": { - "@types/istanbul-lib-coverage": "*" - } - }, - "node_modules/@types/istanbul-reports": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.1.tgz", - "integrity": "sha512-c3mAZEuK0lvBp8tmuL74XRKn1+y2dcwOUpH7x4WrF6gk1GIgiluDRgMYQtw2OFcBvAJWlt6ASU3tSqxp0Uu0Aw==", - "dev": true, - "dependencies": { - "@types/istanbul-lib-report": "*" - } - }, - "node_modules/@types/jest": { - "version": "29.5.14", - "resolved": "https://registry.npmjs.org/@types/jest/-/jest-29.5.14.tgz", - "integrity": "sha512-ZN+4sdnLUbo8EVvVc2ao0GFW6oVrQRPn4K2lglySj7APvSrgzxHiNNK99us4WDMi57xxA2yggblIAMNhXOotLQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "expect": "^29.0.0", - "pretty-format": "^29.0.0" - } - }, - "node_modules/@types/json-schema": { - "version": "7.0.11", - "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.11.tgz", - "integrity": "sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ==", - "dev": true - }, - "node_modules/@types/linkify-it": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@types/linkify-it/-/linkify-it-3.0.2.tgz", - "integrity": "sha512-HZQYqbiFVWufzCwexrvh694SOim8z2d+xJl5UNamcvQFejLY/2YUtzXHYi3cHdI7PMlS8ejH2slRAOJQ32aNbA==", - "dev": true - }, - "node_modules/@types/markdown-it": { - "version": "12.2.3", - "resolved": "https://registry.npmjs.org/@types/markdown-it/-/markdown-it-12.2.3.tgz", - "integrity": "sha512-GKMHFfv3458yYy+v/N8gjufHO6MSZKCOXpZc5GXIWWy8uldwfmPn98vp81gZ5f9SVw8YYBctgfJ22a2d7AOMeQ==", - "dev": true, - "dependencies": { - "@types/linkify-it": "*", - "@types/mdurl": "*" - } - }, - "node_modules/@types/mdurl": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@types/mdurl/-/mdurl-1.0.2.tgz", - "integrity": "sha512-eC4U9MlIcu2q0KQmXszyn5Akca/0jrQmwDRgpAMJai7qBWq4amIQhZyNau4VYGtCeALvW1/NtjzJJ567aZxfKA==", - "dev": true - }, - "node_modules/@types/mime": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@types/mime/-/mime-3.0.1.tgz", - "integrity": "sha512-Y4XFY5VJAuw0FgAqPNd6NNoV44jbq9Bz2L7Rh/J6jLTiHBSBJa9fxqQIvkIld4GsoDOcCbvzOUAbLPsSKKg+uA==", - "dev": true - }, - "node_modules/@types/node": { - "version": "18.14.4", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.14.4.tgz", - "integrity": "sha512-VhCw7I7qO2X49+jaKcAUwi3rR+hbxT5VcYF493+Z5kMLI0DL568b7JI4IDJaxWFH0D/xwmGJNoXisyX+w7GH/g==" - }, - "node_modules/@types/qs": { - "version": "6.9.7", - "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.7.tgz", - "integrity": "sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==", - "dev": true - }, - "node_modules/@types/range-parser": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.4.tgz", - "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==", - "dev": true - }, - "node_modules/@types/retry": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz", - "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==", - "dev": true - }, - "node_modules/@types/serve-index": { - "version": "1.9.1", - "resolved": "https://registry.npmjs.org/@types/serve-index/-/serve-index-1.9.1.tgz", - "integrity": "sha512-d/Hs3nWDxNL2xAczmOVZNj92YZCS6RGxfBPjKzuu/XirCgXdpKEb88dYNbrYGint6IVWLNP+yonwVAuRC0T2Dg==", - "dev": true, - "dependencies": { - "@types/express": "*" - } - }, - "node_modules/@types/serve-static": { - "version": "1.15.1", - "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.1.tgz", - "integrity": "sha512-NUo5XNiAdULrJENtJXZZ3fHtfMolzZwczzBbnAeBbqBwG+LaG6YaJtuwzwGSQZ2wsCrxjEhNNjAkKigy3n8teQ==", - "dev": true, - "dependencies": { - "@types/mime": "*", - "@types/node": "*" - } - }, - "node_modules/@types/sockjs": { - "version": "0.3.33", - "resolved": "https://registry.npmjs.org/@types/sockjs/-/sockjs-0.3.33.tgz", - "integrity": "sha512-f0KEEe05NvUnat+boPTZ0dgaLZ4SfSouXUgv5noUiefG2ajgKjmETo9ZJyuqsl7dfl2aHlLJUiki6B4ZYldiiw==", - "dev": true, - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/@types/stack-utils": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.1.tgz", - "integrity": "sha512-Hl219/BT5fLAaz6NDkSuhzasy49dwQS/DSdu4MdggFB8zcXv7vflBI3xp7FEmkmdDkBUI2bPUNeMttp2knYdxw==", - "dev": true - }, - "node_modules/@types/ws": { - "version": "8.5.4", - "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.4.tgz", - "integrity": "sha512-zdQDHKUgcX/zBc4GrwsE/7dVdAD8JR4EuiAXiiUhhfyIJXXb2+PrGshFyeXWQPMmmZ2XxgaqclgpIC7eTXc1mg==", - "dev": true, - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/@types/yargs": { - "version": "17.0.24", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.24.tgz", - "integrity": "sha512-6i0aC7jV6QzQB8ne1joVZ0eSFIstHsCrobmOtghM11yGlH0j43FKL2UhWdELkyps0zuf7qVTUVCCR+tgSlyLLw==", - "dev": true, - "dependencies": { - "@types/yargs-parser": "*" - } - }, - "node_modules/@types/yargs-parser": { - "version": "21.0.0", - "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-21.0.0.tgz", - "integrity": "sha512-iO9ZQHkZxHn4mSakYV0vFHAVDyEOIJQrV2uZ06HxEPcx+mt8swXoZHIbaaJ2crJYFfErySgktuTZ3BeLz+XmFA==", - "dev": true - }, - "node_modules/@ungap/structured-clone": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz", - "integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==", - "dev": true, - "license": "ISC" - }, - "node_modules/@webassemblyjs/ast": { - "version": "1.12.1", - "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.12.1.tgz", - "integrity": "sha512-EKfMUOPRRUTy5UII4qJDGPpqfwjOmZ5jeGFwid9mnoqIFK+e0vqoi1qH56JpmZSzEL53jKnNzScdmftJyG5xWg==", - "dev": true, - "dependencies": { - "@webassemblyjs/helper-numbers": "1.11.6", - "@webassemblyjs/helper-wasm-bytecode": "1.11.6" - } - }, - "node_modules/@webassemblyjs/floating-point-hex-parser": { - "version": "1.11.6", - "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.11.6.tgz", - "integrity": "sha512-ejAj9hfRJ2XMsNHk/v6Fu2dGS+i4UaXBXGemOfQ/JfQ6mdQg/WXtwleQRLLS4OvfDhv8rYnVwH27YJLMyYsxhw==", - "dev": true - }, - "node_modules/@webassemblyjs/helper-api-error": { - "version": "1.11.6", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.11.6.tgz", - "integrity": "sha512-o0YkoP4pVu4rN8aTJgAyj9hC2Sv5UlkzCHhxqWj8butaLvnpdc2jOwh4ewE6CX0txSfLn/UYaV/pheS2Txg//Q==", - "dev": true - }, - "node_modules/@webassemblyjs/helper-buffer": { - "version": "1.12.1", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.12.1.tgz", - "integrity": "sha512-nzJwQw99DNDKr9BVCOZcLuJJUlqkJh+kVzVl6Fmq/tI5ZtEyWT1KZMyOXltXLZJmDtvLCDgwsyrkohEtopTXCw==", - "dev": true - }, - "node_modules/@webassemblyjs/helper-numbers": { - "version": "1.11.6", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-numbers/-/helper-numbers-1.11.6.tgz", - "integrity": "sha512-vUIhZ8LZoIWHBohiEObxVm6hwP034jwmc9kuq5GdHZH0wiLVLIPcMCdpJzG4C11cHoQ25TFIQj9kaVADVX7N3g==", - "dev": true, - "dependencies": { - "@webassemblyjs/floating-point-hex-parser": "1.11.6", - "@webassemblyjs/helper-api-error": "1.11.6", - "@xtuc/long": "4.2.2" - } - }, - "node_modules/@webassemblyjs/helper-wasm-bytecode": { - "version": "1.11.6", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.11.6.tgz", - "integrity": "sha512-sFFHKwcmBprO9e7Icf0+gddyWYDViL8bpPjJJl0WHxCdETktXdmtWLGVzoHbqUcY4Be1LkNfwTmXOJUFZYSJdA==", - "dev": true - }, - "node_modules/@webassemblyjs/helper-wasm-section": { - "version": "1.12.1", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.12.1.tgz", - "integrity": "sha512-Jif4vfB6FJlUlSbgEMHUyk1j234GTNG9dBJ4XJdOySoj518Xj0oGsNi59cUQF4RRMS9ouBUxDDdyBVfPTypa5g==", - "dev": true, - "dependencies": { - "@webassemblyjs/ast": "1.12.1", - "@webassemblyjs/helper-buffer": "1.12.1", - "@webassemblyjs/helper-wasm-bytecode": "1.11.6", - "@webassemblyjs/wasm-gen": "1.12.1" - } - }, - "node_modules/@webassemblyjs/ieee754": { - "version": "1.11.6", - "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.11.6.tgz", - "integrity": "sha512-LM4p2csPNvbij6U1f19v6WR56QZ8JcHg3QIJTlSwzFcmx6WSORicYj6I63f9yU1kEUtrpG+kjkiIAkevHpDXrg==", - "dev": true, - "dependencies": { - "@xtuc/ieee754": "^1.2.0" - } - }, - "node_modules/@webassemblyjs/leb128": { - "version": "1.11.6", - "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.11.6.tgz", - "integrity": "sha512-m7a0FhE67DQXgouf1tbN5XQcdWoNgaAuoULHIfGFIEVKA6tu/edls6XnIlkmS6FrXAquJRPni3ZZKjw6FSPjPQ==", - "dev": true, - "dependencies": { - "@xtuc/long": "4.2.2" - } - }, - "node_modules/@webassemblyjs/utf8": { - "version": "1.11.6", - "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.11.6.tgz", - "integrity": "sha512-vtXf2wTQ3+up9Zsg8sa2yWiQpzSsMyXj0qViVP6xKGCUT8p8YJ6HqI7l5eCnWx1T/FYdsv07HQs2wTFbbof/RA==", - "dev": true - }, - "node_modules/@webassemblyjs/wasm-edit": { - "version": "1.12.1", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.12.1.tgz", - "integrity": "sha512-1DuwbVvADvS5mGnXbE+c9NfA8QRcZ6iKquqjjmR10k6o+zzsRVesil54DKexiowcFCPdr/Q0qaMgB01+SQ1u6g==", - "dev": true, - "dependencies": { - "@webassemblyjs/ast": "1.12.1", - "@webassemblyjs/helper-buffer": "1.12.1", - "@webassemblyjs/helper-wasm-bytecode": "1.11.6", - "@webassemblyjs/helper-wasm-section": "1.12.1", - "@webassemblyjs/wasm-gen": "1.12.1", - "@webassemblyjs/wasm-opt": "1.12.1", - "@webassemblyjs/wasm-parser": "1.12.1", - "@webassemblyjs/wast-printer": "1.12.1" - } - }, - "node_modules/@webassemblyjs/wasm-gen": { - "version": "1.12.1", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.12.1.tgz", - "integrity": "sha512-TDq4Ojh9fcohAw6OIMXqiIcTq5KUXTGRkVxbSo1hQnSy6lAM5GSdfwWeSxpAo0YzgsgF182E/U0mDNhuA0tW7w==", - "dev": true, - "dependencies": { - "@webassemblyjs/ast": "1.12.1", - "@webassemblyjs/helper-wasm-bytecode": "1.11.6", - "@webassemblyjs/ieee754": "1.11.6", - "@webassemblyjs/leb128": "1.11.6", - "@webassemblyjs/utf8": "1.11.6" - } - }, - "node_modules/@webassemblyjs/wasm-opt": { - "version": "1.12.1", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.12.1.tgz", - "integrity": "sha512-Jg99j/2gG2iaz3hijw857AVYekZe2SAskcqlWIZXjji5WStnOpVoat3gQfT/Q5tb2djnCjBtMocY/Su1GfxPBg==", - "dev": true, - "dependencies": { - "@webassemblyjs/ast": "1.12.1", - "@webassemblyjs/helper-buffer": "1.12.1", - "@webassemblyjs/wasm-gen": "1.12.1", - "@webassemblyjs/wasm-parser": "1.12.1" - } - }, - "node_modules/@webassemblyjs/wasm-parser": { - "version": "1.12.1", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.12.1.tgz", - "integrity": "sha512-xikIi7c2FHXysxXe3COrVUPSheuBtpcfhbpFj4gmu7KRLYOzANztwUU0IbsqvMqzuNK2+glRGWCEqZo1WCLyAQ==", - "dev": true, - "dependencies": { - "@webassemblyjs/ast": "1.12.1", - "@webassemblyjs/helper-api-error": "1.11.6", - "@webassemblyjs/helper-wasm-bytecode": "1.11.6", - "@webassemblyjs/ieee754": "1.11.6", - "@webassemblyjs/leb128": "1.11.6", - "@webassemblyjs/utf8": "1.11.6" - } - }, - "node_modules/@webassemblyjs/wast-printer": { - "version": "1.12.1", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.12.1.tgz", - "integrity": "sha512-+X4WAlOisVWQMikjbcvY2e0rwPsKQ9F688lksZhBcPycBBuii3O7m8FACbDMWDojpAqvjIncrG8J0XHKyQfVeA==", - "dev": true, - "dependencies": { - "@webassemblyjs/ast": "1.12.1", - "@xtuc/long": "4.2.2" - } - }, - "node_modules/@webgpu/types": { - "version": "0.1.44", - "resolved": "https://registry.npmjs.org/@webgpu/types/-/types-0.1.44.tgz", - "integrity": "sha512-JDpYJN5E/asw84LTYhKyvPpxGnD+bAKPtpW9Ilurf7cZpxaTbxkQcGwOd7jgB9BPBrTYQ+32ufo4HiuomTjHNQ==", - "dev": true - }, - "node_modules/@webpack-cli/configtest": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/@webpack-cli/configtest/-/configtest-2.0.1.tgz", - "integrity": "sha512-njsdJXJSiS2iNbQVS0eT8A/KPnmyH4pv1APj2K0d1wrZcBLw+yppxOy4CGqa0OxDJkzfL/XELDhD8rocnIwB5A==", - "dev": true, - "engines": { - "node": ">=14.15.0" - }, - "peerDependencies": { - "webpack": "5.x.x", - "webpack-cli": "5.x.x" - } - }, - "node_modules/@webpack-cli/info": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/@webpack-cli/info/-/info-2.0.1.tgz", - "integrity": "sha512-fE1UEWTwsAxRhrJNikE7v4EotYflkEhBL7EbajfkPlf6E37/2QshOy/D48Mw8G5XMFlQtS6YV42vtbG9zBpIQA==", - "dev": true, - "engines": { - "node": ">=14.15.0" - }, - "peerDependencies": { - "webpack": "5.x.x", - "webpack-cli": "5.x.x" - } - }, - "node_modules/@webpack-cli/serve": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/@webpack-cli/serve/-/serve-2.0.2.tgz", - "integrity": "sha512-S9h3GmOmzUseyeFW3tYNnWS7gNUuwxZ3mmMq0JyW78Vx1SGKPSkt5bT4pB0rUnVfHjP0EL9gW2bOzmtiTfQt0A==", - "dev": true, - "engines": { - "node": ">=14.15.0" - }, - "peerDependencies": { - "webpack": "5.x.x", - "webpack-cli": "5.x.x" - }, - "peerDependenciesMeta": { - "webpack-dev-server": { - "optional": true - } - } - }, - "node_modules/@xtuc/ieee754": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@xtuc/ieee754/-/ieee754-1.2.0.tgz", - "integrity": "sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA==", - "dev": true - }, - "node_modules/@xtuc/long": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/@xtuc/long/-/long-4.2.2.tgz", - "integrity": "sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==", - "dev": true - }, - "node_modules/accepts": { - "version": "1.3.8", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", - "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", - "dev": true, - "dependencies": { - "mime-types": "~2.1.34", - "negotiator": "0.6.3" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/acorn": { - "version": "8.12.1", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.12.1.tgz", - "integrity": "sha512-tcpGyI9zbizT9JbV6oYE477V6mTlXvvi0T0G3SNIYE2apm/G5huBa1+K89VGeovbg+jycCrfhl3ADxErOuO6Jg==", - "dev": true, - "bin": { - "acorn": "bin/acorn" - }, - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/acorn-import-attributes": { - "version": "1.9.5", - "resolved": "https://registry.npmjs.org/acorn-import-attributes/-/acorn-import-attributes-1.9.5.tgz", - "integrity": "sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ==", - "dev": true, - "peerDependencies": { - "acorn": "^8" - } - }, - "node_modules/ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", - "dev": true, - "dependencies": { - "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/ajv-formats": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-2.1.1.tgz", - "integrity": "sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==", - "dev": true, - "dependencies": { - "ajv": "^8.0.0" - }, - "peerDependencies": { - "ajv": "^8.0.0" - }, - "peerDependenciesMeta": { - "ajv": { - "optional": true - } - } - }, - "node_modules/ajv-formats/node_modules/ajv": { - "version": "8.12.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz", - "integrity": "sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==", - "dev": true, - "dependencies": { - "fast-deep-equal": "^3.1.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.2.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/ajv-formats/node_modules/json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", - "dev": true - }, - "node_modules/ajv-keywords": { - "version": "3.5.2", - "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz", - "integrity": "sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==", - "dev": true, - "peerDependencies": { - "ajv": "^6.9.1" - } - }, - "node_modules/ansi-escape-sequences": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/ansi-escape-sequences/-/ansi-escape-sequences-4.1.0.tgz", - "integrity": "sha512-dzW9kHxH011uBsidTXd14JXgzye/YLb2LzeKZ4bsgl/Knwx8AtbSFkkGxagdNOoh0DlqHCmfiEjWKBaqjOanVw==", - "dev": true, - "dependencies": { - "array-back": "^3.0.1" - }, - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/ansi-escape-sequences/node_modules/array-back": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/array-back/-/array-back-3.1.0.tgz", - "integrity": "sha512-TkuxA4UCOvxuDK6NZYXCalszEzj+TLszyASooky+i742l9TqsOdYCMJJupxRic61hwquNtppB3hgcuq9SVSH1Q==", - "dev": true, - "engines": { - "node": ">=6" - } - }, - "node_modules/ansi-escapes": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", - "integrity": "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "type-fest": "^0.21.3" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/ansi-html-community": { - "version": "0.0.8", - "resolved": "https://registry.npmjs.org/ansi-html-community/-/ansi-html-community-0.0.8.tgz", - "integrity": "sha512-1APHAyr3+PCamwNw3bXCPp4HFLONZt/yIH0sZp0/469KWNTEy+qN5jQ3GVX6DMZ1UXAi34yVwtTeaG/HpBuuzw==", - "dev": true, - "engines": [ - "node >= 0.8.0" - ], - "bin": { - "ansi-html": "bin/ansi-html" - } - }, - "node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "engines": { - "node": ">=8" - } - }, - "node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/anymatch": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", - "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", - "dev": true, - "dependencies": { - "normalize-path": "^3.0.0", - "picomatch": "^2.0.4" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true - }, - "node_modules/array-back": { - "version": "6.2.2", - "resolved": "https://registry.npmjs.org/array-back/-/array-back-6.2.2.tgz", - "integrity": "sha512-gUAZ7HPyb4SJczXAMUXMGAvI976JoK3qEx9v1FTmeYuJj0IBiaKttG1ydtGKdkfqWkIkouke7nG8ufGy77+Cvw==", - "dev": true, - "engines": { - "node": ">=12.17" - } - }, - "node_modules/array-flatten": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-2.1.2.tgz", - "integrity": "sha512-hNfzcOV8W4NdualtqBFPyVO+54DSJuZGY9qT4pRroB6S9e3iiido2ISIC5h9R2sPJ8H3FHCIiEnsv1lPXO3KtQ==", - "dev": true - }, - "node_modules/babel-jest": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-30.0.0-alpha.6.tgz", - "integrity": "sha512-WOQkqpBz2q8d/AT6D6rZXW5xnKHDmk3kIukaXlzUyoBBIvLh1SEvi2RGS4fboEtS0kNkyL+zf1rSfkt5OCIgmw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/transform": "30.0.0-alpha.6", - "@types/babel__core": "^7.1.14", - "babel-plugin-istanbul": "^7.0.0", - "babel-preset-jest": "30.0.0-alpha.6", - "chalk": "^4.0.0", - "graceful-fs": "^4.2.9", - "slash": "^3.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - }, - "peerDependencies": { - "@babel/core": "^7.11.0" - } - }, - "node_modules/babel-plugin-istanbul": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/babel-plugin-istanbul/-/babel-plugin-istanbul-7.0.0.tgz", - "integrity": "sha512-C5OzENSx/A+gt7t4VH1I2XsflxyPUmXRFPKBxt33xncdOmq7oROVM3bZv9Ysjjkv8OJYDMa+tKuKMvqU/H3xdw==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "@babel/helper-plugin-utils": "^7.0.0", - "@istanbuljs/load-nyc-config": "^1.0.0", - "@istanbuljs/schema": "^0.1.3", - "istanbul-lib-instrument": "^6.0.2", - "test-exclude": "^6.0.0" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/babel-plugin-jest-hoist": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/babel-plugin-jest-hoist/-/babel-plugin-jest-hoist-30.0.0-alpha.6.tgz", - "integrity": "sha512-e/aPv0pmnvJqXM5SfCBpyMwZFEZrKW1Mb4unwTkxewk6/0TjwBk6l3B3F9H9OKZ3ErhkH4b+Epd3IIM5E53I2g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/template": "^7.3.3", - "@babel/types": "^7.3.3", - "@types/babel__core": "^7.1.14" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/babel-preset-current-node-syntax": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/babel-preset-current-node-syntax/-/babel-preset-current-node-syntax-1.1.0.tgz", - "integrity": "sha512-ldYss8SbBlWva1bs28q78Ju5Zq1F+8BrqBZZ0VFhLBvhh6lCpC2o3gDJi/5DRLs9FgYZCnmPYIVFU4lRXCkyUw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/plugin-syntax-async-generators": "^7.8.4", - "@babel/plugin-syntax-bigint": "^7.8.3", - "@babel/plugin-syntax-class-properties": "^7.12.13", - "@babel/plugin-syntax-class-static-block": "^7.14.5", - "@babel/plugin-syntax-import-attributes": "^7.24.7", - "@babel/plugin-syntax-import-meta": "^7.10.4", - "@babel/plugin-syntax-json-strings": "^7.8.3", - "@babel/plugin-syntax-logical-assignment-operators": "^7.10.4", - "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.3", - "@babel/plugin-syntax-numeric-separator": "^7.10.4", - "@babel/plugin-syntax-object-rest-spread": "^7.8.3", - "@babel/plugin-syntax-optional-catch-binding": "^7.8.3", - "@babel/plugin-syntax-optional-chaining": "^7.8.3", - "@babel/plugin-syntax-private-property-in-object": "^7.14.5", - "@babel/plugin-syntax-top-level-await": "^7.14.5" - }, - "peerDependencies": { - "@babel/core": "^7.0.0" - } - }, - "node_modules/babel-preset-jest": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/babel-preset-jest/-/babel-preset-jest-30.0.0-alpha.6.tgz", - "integrity": "sha512-Xsis7RI2oT2zlyCIEzMtjDiES0wKoQxTUo5BGzx1q3ZemnDE1/7xTC4/lI4eBLmAtwk/hpZLRYwltvbQEvyRWw==", - "dev": true, - "license": "MIT", - "dependencies": { - "babel-plugin-jest-hoist": "30.0.0-alpha.6", - "babel-preset-current-node-syntax": "^1.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - }, - "peerDependencies": { - "@babel/core": "^7.11.0" - } - }, - "node_modules/balanced-match": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" - }, - "node_modules/batch": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/batch/-/batch-0.6.1.tgz", - "integrity": "sha512-x+VAiMRL6UPkx+kudNvxTl6hB2XNNCG2r+7wixVfIYwu/2HKRXimwQyaumLjMveWvT2Hkd/cAJw+QBMfJ/EKVw==", - "dev": true - }, - "node_modules/binary-extensions": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", - "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/bluebird": { - "version": "3.7.2", - "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", - "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==", - "dev": true - }, - "node_modules/body-parser": { - "version": "1.20.3", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", - "integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==", - "dev": true, - "dependencies": { - "bytes": "3.1.2", - "content-type": "~1.0.5", - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "1.2.0", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "on-finished": "2.4.1", - "qs": "6.13.0", - "raw-body": "2.5.2", - "type-is": "~1.6.18", - "unpipe": "1.0.0" - }, - "engines": { - "node": ">= 0.8", - "npm": "1.2.8000 || >= 1.4.16" - } - }, - "node_modules/body-parser/node_modules/bytes": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", - "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", - "dev": true, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/bonjour-service": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/bonjour-service/-/bonjour-service-1.1.1.tgz", - "integrity": "sha512-Z/5lQRMOG9k7W+FkeGTNjh7htqn/2LMnfOvBZ8pynNZCM9MwkQkI3zeI4oz09uWdcgmgHugVvBqxGg4VQJ5PCg==", - "dev": true, - "dependencies": { - "array-flatten": "^2.1.2", - "dns-equal": "^1.0.0", - "fast-deep-equal": "^3.1.3", - "multicast-dns": "^7.2.5" - } - }, - "node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "dev": true, - "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "node_modules/braces": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", - "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", - "dev": true, - "dependencies": { - "fill-range": "^7.1.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/browserslist": { - "version": "4.24.2", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.24.2.tgz", - "integrity": "sha512-ZIc+Q62revdMcqC6aChtW4jz3My3klmCO1fEmINZY/8J3EpBg5/A/D0AKmBveUh6pgoeycoMkVMko84tuYS+Gg==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/browserslist" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/browserslist" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "dependencies": { - "caniuse-lite": "^1.0.30001669", - "electron-to-chromium": "^1.5.41", - "node-releases": "^2.0.18", - "update-browserslist-db": "^1.1.1" - }, - "bin": { - "browserslist": "cli.js" - }, - "engines": { - "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" - } - }, - "node_modules/bser": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/bser/-/bser-2.1.1.tgz", - "integrity": "sha512-gQxTNE/GAfIIrmHLUE3oJyp5FO6HRBfhjnw4/wMmA63ZGDJnWBmgY/lyQBpnDUkGmAhbSe39tx2d/iTOAfglwQ==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "node-int64": "^0.4.0" - } - }, - "node_modules/buffer-from": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", - "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", - "dev": true - }, - "node_modules/bytes": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", - "integrity": "sha512-pMhOfFDPiv9t5jjIXkHosWmkSyQbvsgEVNkz0ERHbuLh2T/7j4Mqqpz523Fe8MVY89KC6Sh/QfS2sM+SjgFDcw==", - "dev": true, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/cache-point": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/cache-point/-/cache-point-2.0.0.tgz", - "integrity": "sha512-4gkeHlFpSKgm3vm2gJN5sPqfmijYRFYCQ6tv5cLw0xVmT6r1z1vd4FNnpuOREco3cBs1G709sZ72LdgddKvL5w==", - "dev": true, - "dependencies": { - "array-back": "^4.0.1", - "fs-then-native": "^2.0.0", - "mkdirp2": "^1.0.4" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/cache-point/node_modules/array-back": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/array-back/-/array-back-4.0.2.tgz", - "integrity": "sha512-NbdMezxqf94cnNfWLL7V/im0Ub+Anbb0IoZhvzie8+4HJ4nMQuzHuy49FkGYCJK2yAloZ3meiB6AVMClbrI1vg==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/call-bind": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.7.tgz", - "integrity": "sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==", - "dev": true, - "dependencies": { - "es-define-property": "^1.0.0", - "es-errors": "^1.3.0", - "function-bind": "^1.1.2", - "get-intrinsic": "^1.2.4", - "set-function-length": "^1.2.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/callsites": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", - "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/camelcase": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", - "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/caniuse-lite": { - "version": "1.0.30001677", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001677.tgz", - "integrity": "sha512-fmfjsOlJUpMWu+mAAtZZZHz7UEwsUxIIvu1TJfO1HqFQvB/B+ii0xr9B5HpbZY/mC4XZ8SvjHJqtAY6pDPQEog==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/browserslist" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/caniuse-lite" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "CC-BY-4.0" - }, - "node_modules/catharsis": { - "version": "0.9.0", - "resolved": "git+ssh://git@github.com/xenova/catharsis.git#dda824bd3b6c08d418e8c8a03f20446cfb25aa1e", - "dev": true, - "license": "MIT", - "dependencies": { - "lodash": "^4.17.21" - }, - "engines": { - "node": ">= v18.16.0" - } - }, - "node_modules/chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "dev": true, - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, - "node_modules/chalk/node_modules/supports-color": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dev": true, - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/char-regex": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/char-regex/-/char-regex-1.0.2.tgz", - "integrity": "sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - } - }, - "node_modules/chokidar": { - "version": "3.5.3", - "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", - "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", - "dev": true, - "funding": [ - { - "type": "individual", - "url": "https://paulmillr.com/funding/" - } - ], - "dependencies": { - "anymatch": "~3.1.2", - "braces": "~3.0.2", - "glob-parent": "~5.1.2", - "is-binary-path": "~2.1.0", - "is-glob": "~4.0.1", - "normalize-path": "~3.0.0", - "readdirp": "~3.6.0" - }, - "engines": { - "node": ">= 8.10.0" - }, - "optionalDependencies": { - "fsevents": "~2.3.2" - } - }, - "node_modules/chownr": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", - "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==", - "engines": { - "node": ">=18" - } - }, - "node_modules/chrome-trace-event": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/chrome-trace-event/-/chrome-trace-event-1.0.3.tgz", - "integrity": "sha512-p3KULyQg4S7NIHixdwbGX+nFHkoBiA4YQmyWtjb8XngSKV124nJmRysgAeujbUVb15vh+RvFUfCPqU7rXk+hZg==", - "dev": true, - "engines": { - "node": ">=6.0" - } - }, - "node_modules/ci-info": { - "version": "3.8.0", - "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.8.0.tgz", - "integrity": "sha512-eXTggHWSooYhq49F2opQhuHWgzucfF2YgODK4e1566GQs5BIfP30B0oenwBJHfWxAs2fyPB1s7Mg949zLf61Yw==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/sibiraj-s" - } - ], - "engines": { - "node": ">=8" - } - }, - "node_modules/cjs-module-lexer": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.4.1.tgz", - "integrity": "sha512-cuSVIHi9/9E/+821Qjdvngor+xpnlwnuwIyZOaLmHBVdXL+gP+I6QQB9VkO7RI77YIcTV+S1W9AreJ5eN63JBA==", - "dev": true, - "license": "MIT" - }, - "node_modules/cliui": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", - "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", - "dev": true, - "license": "ISC", - "dependencies": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.1", - "wrap-ansi": "^7.0.0" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/clone-deep": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-4.0.1.tgz", - "integrity": "sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ==", - "dev": true, - "dependencies": { - "is-plain-object": "^2.0.4", - "kind-of": "^6.0.2", - "shallow-clone": "^3.0.0" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/co": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", - "integrity": "sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==", - "dev": true, - "license": "MIT", - "engines": { - "iojs": ">= 1.0.0", - "node": ">= 0.12.0" - } - }, - "node_modules/collect-all": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/collect-all/-/collect-all-1.0.4.tgz", - "integrity": "sha512-RKZhRwJtJEP5FWul+gkSMEnaK6H3AGPTTWOiRimCcs+rc/OmQE3Yhy1Q7A7KsdkG3ZXVdZq68Y6ONSdvkeEcKA==", - "dev": true, - "dependencies": { - "stream-connect": "^1.0.2", - "stream-via": "^1.0.4" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/collect-v8-coverage": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/collect-v8-coverage/-/collect-v8-coverage-1.0.2.tgz", - "integrity": "sha512-lHl4d5/ONEbLlJvaJNtsF/Lz+WvB07u2ycqTYbdrq7UypDXailES4valYb2eWiJFxZlVmpGekfqoxQhzyFdT4Q==", - "dev": true, - "license": "MIT" - }, - "node_modules/color": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz", - "integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==", - "dependencies": { - "color-convert": "^2.0.1", - "color-string": "^1.9.0" - }, - "engines": { - "node": ">=12.5.0" - } - }, - "node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" - }, - "node_modules/color-string": { - "version": "1.9.1", - "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", - "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", - "dependencies": { - "color-name": "^1.0.0", - "simple-swizzle": "^0.2.2" - } - }, - "node_modules/colorette": { - "version": "2.0.20", - "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.20.tgz", - "integrity": "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==", - "dev": true - }, - "node_modules/command-line-args": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/command-line-args/-/command-line-args-5.2.1.tgz", - "integrity": "sha512-H4UfQhZyakIjC74I9d34fGYDwk3XpSr17QhEd0Q3I9Xq1CETHo4Hcuo87WyWHpAF1aSLjLRf5lD9ZGX2qStUvg==", - "dev": true, - "dependencies": { - "array-back": "^3.1.0", - "find-replace": "^3.0.0", - "lodash.camelcase": "^4.3.0", - "typical": "^4.0.0" - }, - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/command-line-args/node_modules/array-back": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/array-back/-/array-back-3.1.0.tgz", - "integrity": "sha512-TkuxA4UCOvxuDK6NZYXCalszEzj+TLszyASooky+i742l9TqsOdYCMJJupxRic61hwquNtppB3hgcuq9SVSH1Q==", - "dev": true, - "engines": { - "node": ">=6" - } - }, - "node_modules/command-line-args/node_modules/typical": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/typical/-/typical-4.0.0.tgz", - "integrity": "sha512-VAH4IvQ7BDFYglMd7BPRDfLgxZZX4O4TFcRDA6EN5X7erNJJq+McIEp8np9aVtxrCJ6qx4GTYVfOWNjcqwZgRw==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/command-line-tool": { - "version": "0.8.0", - "resolved": "https://registry.npmjs.org/command-line-tool/-/command-line-tool-0.8.0.tgz", - "integrity": "sha512-Xw18HVx/QzQV3Sc5k1vy3kgtOeGmsKIqwtFFoyjI4bbcpSgnw2CWVULvtakyw4s6fhyAdI6soQQhXc2OzJy62g==", - "dev": true, - "dependencies": { - "ansi-escape-sequences": "^4.0.0", - "array-back": "^2.0.0", - "command-line-args": "^5.0.0", - "command-line-usage": "^4.1.0", - "typical": "^2.6.1" - }, - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/command-line-tool/node_modules/array-back": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/array-back/-/array-back-2.0.0.tgz", - "integrity": "sha512-eJv4pLLufP3g5kcZry0j6WXpIbzYw9GUB4mVJZno9wfwiBxbizTnHCw3VJb07cBihbFX48Y7oSrW9y+gt4glyw==", - "dev": true, - "dependencies": { - "typical": "^2.6.1" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/command-line-usage": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/command-line-usage/-/command-line-usage-4.1.0.tgz", - "integrity": "sha512-MxS8Ad995KpdAC0Jopo/ovGIroV/m0KHwzKfXxKag6FHOkGsH8/lv5yjgablcRxCJJC0oJeUMuO/gmaq+Wq46g==", - "dev": true, - "dependencies": { - "ansi-escape-sequences": "^4.0.0", - "array-back": "^2.0.0", - "table-layout": "^0.4.2", - "typical": "^2.6.1" - }, - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/command-line-usage/node_modules/array-back": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/array-back/-/array-back-2.0.0.tgz", - "integrity": "sha512-eJv4pLLufP3g5kcZry0j6WXpIbzYw9GUB4mVJZno9wfwiBxbizTnHCw3VJb07cBihbFX48Y7oSrW9y+gt4glyw==", - "dev": true, - "dependencies": { - "typical": "^2.6.1" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/commander": { - "version": "2.20.3", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", - "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", - "dev": true - }, - "node_modules/common-sequence": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/common-sequence/-/common-sequence-2.0.2.tgz", - "integrity": "sha512-jAg09gkdkrDO9EWTdXfv80WWH3yeZl5oT69fGfedBNS9pXUKYInVJ1bJ+/ht2+Moeei48TmSbQDYMc8EOx9G0g==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/compressible": { - "version": "2.0.18", - "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz", - "integrity": "sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==", - "dev": true, - "dependencies": { - "mime-db": ">= 1.43.0 < 2" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/compression": { - "version": "1.7.4", - "resolved": "https://registry.npmjs.org/compression/-/compression-1.7.4.tgz", - "integrity": "sha512-jaSIDzP9pZVS4ZfQ+TzvtiWhdpFhE2RDHz8QJkpX9SIpLq88VueF5jJw6t+6CUQcAoA6t+x89MLrWAqpfDE8iQ==", - "dev": true, - "dependencies": { - "accepts": "~1.3.5", - "bytes": "3.0.0", - "compressible": "~2.0.16", - "debug": "2.6.9", - "on-headers": "~1.0.2", - "safe-buffer": "5.1.2", - "vary": "~1.1.2" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/compression/node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", - "dev": true - }, - "node_modules/concat-map": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", - "dev": true - }, - "node_modules/config-master": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/config-master/-/config-master-3.1.0.tgz", - "integrity": "sha512-n7LBL1zBzYdTpF1mx5DNcZnZn05CWIdsdvtPL4MosvqbBUK3Rq6VWEtGUuF3Y0s9/CIhMejezqlSkP6TnCJ/9g==", - "dev": true, - "dependencies": { - "walk-back": "^2.0.1" - } - }, - "node_modules/config-master/node_modules/walk-back": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/walk-back/-/walk-back-2.0.1.tgz", - "integrity": "sha512-Nb6GvBR8UWX1D+Le+xUq0+Q1kFmRBIWVrfLnQAOmcpEzA9oAxwJ9gIr36t9TWYfzvWRvuMtjHiVsJYEkXWaTAQ==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/connect-history-api-fallback": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/connect-history-api-fallback/-/connect-history-api-fallback-2.0.0.tgz", - "integrity": "sha512-U73+6lQFmfiNPrYbXqr6kZ1i1wiRqXnp2nhMsINseWXO8lDau0LGEffJ8kQi4EjLZympVgRdvqjAgiZ1tgzDDA==", - "dev": true, - "engines": { - "node": ">=0.8" - } - }, - "node_modules/content-disposition": { - "version": "0.5.4", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", - "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", - "dev": true, - "dependencies": { - "safe-buffer": "5.2.1" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/content-type": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", - "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", - "dev": true, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/convert-source-map": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", - "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", - "dev": true, - "license": "MIT" - }, - "node_modules/cookie": { - "version": "0.7.1", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.1.tgz", - "integrity": "sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==", - "dev": true, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/cookie-signature": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", - "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==", - "dev": true - }, - "node_modules/core-util-is": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", - "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", - "dev": true - }, - "node_modules/cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", - "dependencies": { - "path-key": "^3.1.0", - "shebang-command": "^2.0.0", - "which": "^2.0.1" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "dev": true, - "dependencies": { - "ms": "2.0.0" - } - }, - "node_modules/dedent": { - "version": "1.5.3", - "resolved": "https://registry.npmjs.org/dedent/-/dedent-1.5.3.tgz", - "integrity": "sha512-NHQtfOOW68WD8lgypbLA5oT+Bt0xXJhiYvoR6SmmNXZfpzOGXwdKWmcwG8N7PwVVWV3eF/68nmD9BaJSsTBhyQ==", - "dev": true, - "license": "MIT", - "peerDependencies": { - "babel-plugin-macros": "^3.1.0" - }, - "peerDependenciesMeta": { - "babel-plugin-macros": { - "optional": true - } - } - }, - "node_modules/deep-extend": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", - "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", - "dev": true, - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/deepmerge": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", - "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/default-gateway": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/default-gateway/-/default-gateway-6.0.3.tgz", - "integrity": "sha512-fwSOJsbbNzZ/CUFpqFBqYfYNLj1NbMPm8MMCIzHjC83iSJRBEGmDUxU+WP661BaBQImeC2yHwXtz+P/O9o+XEg==", - "dev": true, - "dependencies": { - "execa": "^5.0.0" - }, - "engines": { - "node": ">= 10" - } - }, - "node_modules/define-data-property": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", - "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", - "dev": true, - "dependencies": { - "es-define-property": "^1.0.0", - "es-errors": "^1.3.0", - "gopd": "^1.0.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/define-lazy-prop": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-2.0.0.tgz", - "integrity": "sha512-Ds09qNh8yw3khSjiJjiUInaGX9xlqZDY7JVryGxdxV7NPeuqQfplOpQ66yJFZut3jLa5zOwkXw1g9EI2uKh4Og==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/depd": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", - "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", - "dev": true, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/destroy": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", - "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", - "dev": true, - "engines": { - "node": ">= 0.8", - "npm": "1.2.8000 || >= 1.4.16" - } - }, - "node_modules/detect-libc": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.3.tgz", - "integrity": "sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==", - "engines": { - "node": ">=8" - } - }, - "node_modules/detect-newline": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", - "integrity": "sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/detect-node": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz", - "integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==", - "dev": true - }, - "node_modules/diff-sequences": { - "version": "29.4.3", - "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-29.4.3.tgz", - "integrity": "sha512-ofrBgwpPhCD85kMKtE9RYFFq6OC1A89oW2vvgWZNCwxrUpRUILopY7lsYyMDSjc8g6U6aiO0Qubg6r4Wgt5ZnA==", - "dev": true, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/dmd": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/dmd/-/dmd-6.2.0.tgz", - "integrity": "sha512-uXWxLF1H7TkUAuoHK59/h/ts5cKavm2LnhrIgJWisip4BVzPoXavlwyoprFFn2CzcahKYgvkfaebS6oxzgflkg==", - "dev": true, - "dependencies": { - "array-back": "^6.2.2", - "cache-point": "^2.0.0", - "common-sequence": "^2.0.2", - "file-set": "^4.0.2", - "handlebars": "^4.7.7", - "marked": "^4.2.3", - "object-get": "^2.1.1", - "reduce-flatten": "^3.0.1", - "reduce-unique": "^2.0.1", - "reduce-without": "^1.0.1", - "test-value": "^3.0.0", - "walk-back": "^5.1.0" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/dns-equal": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/dns-equal/-/dns-equal-1.0.0.tgz", - "integrity": "sha512-z+paD6YUQsk+AbGCEM4PrOXSss5gd66QfcVBFTKR/HpFL9jCqikS94HYwKww6fQyO7IxrIIyUu+g0Ka9tUS2Cg==", - "dev": true - }, - "node_modules/dns-packet": { - "version": "5.6.0", - "resolved": "https://registry.npmjs.org/dns-packet/-/dns-packet-5.6.0.tgz", - "integrity": "sha512-rza3UH1LwdHh9qyPXp8lkwpjSNk/AMD3dPytUoRoqnypDUhY0xvbdmVhWOfxO68frEfV9BU8V12Ez7ZsHGZpCQ==", - "dev": true, - "dependencies": { - "@leichtgewicht/ip-codec": "^2.0.1" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/eastasianwidth": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", - "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==" - }, - "node_modules/ee-first": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", - "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", - "dev": true - }, - "node_modules/electron-to-chromium": { - "version": "1.5.50", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.50.tgz", - "integrity": "sha512-eMVObiUQ2LdgeO1F/ySTXsvqvxb6ZH2zPGaMYsWzRDdOddUa77tdmI0ltg+L16UpbWdhPmuF3wIQYyQq65WfZw==", - "dev": true, - "license": "ISC" - }, - "node_modules/emittery": { - "version": "0.13.1", - "resolved": "https://registry.npmjs.org/emittery/-/emittery-0.13.1.tgz", - "integrity": "sha512-DeWwawk6r5yR9jFgnDKYt4sLS0LmHJJi3ZOnb5/JdbYwj3nW+FxQnHIjhBKz8YLC7oRNPVM9NQ47I3CVx34eqQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sindresorhus/emittery?sponsor=1" - } - }, - "node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" - }, - "node_modules/encodeurl": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", - "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", - "dev": true, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/enhanced-resolve": { - "version": "5.17.1", - "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.17.1.tgz", - "integrity": "sha512-LMHl3dXhTcfv8gM4kEzIUeTQ+7fpdA0l2tUf34BddXPkz2A5xJ5L/Pchd5BL6rdccM9QGvu0sWZzK1Z1t4wwyg==", - "dev": true, - "dependencies": { - "graceful-fs": "^4.2.4", - "tapable": "^2.2.0" - }, - "engines": { - "node": ">=10.13.0" - } - }, - "node_modules/entities": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/entities/-/entities-2.1.0.tgz", - "integrity": "sha512-hCx1oky9PFrJ611mf0ifBLBRW8lUUVRlFolb5gWRfIELabBlbp9xZvrqZLZAs+NxFnbfQoeGd8wDkygjg7U85w==", - "dev": true, - "funding": { - "url": "https://github.com/fb55/entities?sponsor=1" - } - }, - "node_modules/envinfo": { - "version": "7.8.1", - "resolved": "https://registry.npmjs.org/envinfo/-/envinfo-7.8.1.tgz", - "integrity": "sha512-/o+BXHmB7ocbHEAs6F2EnG0ogybVVUdkRunTT2glZU9XAaGmhqskrvKwqXuDfNjEO0LZKWdejEEpnq8aM0tOaw==", - "dev": true, - "bin": { - "envinfo": "dist/cli.js" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/error-ex": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", - "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-arrayish": "^0.2.1" - } - }, - "node_modules/error-ex/node_modules/is-arrayish": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", - "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", - "dev": true, - "license": "MIT" - }, - "node_modules/es-define-property": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz", - "integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==", - "dev": true, - "dependencies": { - "get-intrinsic": "^1.2.4" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-errors": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", - "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", - "dev": true, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-module-lexer": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.2.1.tgz", - "integrity": "sha512-9978wrXM50Y4rTMmW5kXIC09ZdXQZqkE4mxhwkd8VbzsGkXGPgV4zWuqQJgCEzYngdo2dYDa0l8xhX4fkSwJSg==", - "dev": true - }, - "node_modules/escalade": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", - "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/escape-html": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", - "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", - "dev": true - }, - "node_modules/escape-string-regexp": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz", - "integrity": "sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/eslint-scope": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz", - "integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==", - "dev": true, - "dependencies": { - "esrecurse": "^4.3.0", - "estraverse": "^4.1.1" - }, - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/esprima": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", - "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", - "dev": true, - "license": "BSD-2-Clause", - "bin": { - "esparse": "bin/esparse.js", - "esvalidate": "bin/esvalidate.js" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/esrecurse": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", - "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", - "dev": true, - "dependencies": { - "estraverse": "^5.2.0" - }, - "engines": { - "node": ">=4.0" - } - }, - "node_modules/esrecurse/node_modules/estraverse": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", - "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", - "dev": true, - "engines": { - "node": ">=4.0" - } - }, - "node_modules/estraverse": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", - "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", - "dev": true, - "engines": { - "node": ">=4.0" - } - }, - "node_modules/etag": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", - "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", - "dev": true, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/eventemitter3": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", - "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", - "dev": true - }, - "node_modules/events": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", - "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", - "dev": true, - "engines": { - "node": ">=0.8.x" - } - }, - "node_modules/execa": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", - "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==", - "dev": true, - "dependencies": { - "cross-spawn": "^7.0.3", - "get-stream": "^6.0.0", - "human-signals": "^2.1.0", - "is-stream": "^2.0.0", - "merge-stream": "^2.0.0", - "npm-run-path": "^4.0.1", - "onetime": "^5.1.2", - "signal-exit": "^3.0.3", - "strip-final-newline": "^2.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sindresorhus/execa?sponsor=1" - } - }, - "node_modules/exit": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz", - "integrity": "sha512-Zk/eNKV2zbjpKzrsQ+n1G6poVbErQxJ0LBOJXaKZ1EViLzH+hrLu9cdXI4zw9dBQJslwBEpbQ2P1oS7nDxs6jQ==", - "dev": true, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/expect": { - "version": "29.6.1", - "resolved": "https://registry.npmjs.org/expect/-/expect-29.6.1.tgz", - "integrity": "sha512-XEdDLonERCU1n9uR56/Stx9OqojaLAQtZf9PrCHH9Hl8YXiEIka3H4NXJ3NOIBmQJTg7+j7buh34PMHfJujc8g==", - "dev": true, - "dependencies": { - "@jest/expect-utils": "^29.6.1", - "@types/node": "*", - "jest-get-type": "^29.4.3", - "jest-matcher-utils": "^29.6.1", - "jest-message-util": "^29.6.1", - "jest-util": "^29.6.1" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/express": { - "version": "4.21.1", - "resolved": "https://registry.npmjs.org/express/-/express-4.21.1.tgz", - "integrity": "sha512-YSFlK1Ee0/GC8QaO91tHcDxJiE/X4FbpAyQWkxAvG6AXCuR65YzK8ua6D9hvi/TzUfZMpc+BwuM1IPw8fmQBiQ==", - "dev": true, - "dependencies": { - "accepts": "~1.3.8", - "array-flatten": "1.1.1", - "body-parser": "1.20.3", - "content-disposition": "0.5.4", - "content-type": "~1.0.4", - "cookie": "0.7.1", - "cookie-signature": "1.0.6", - "debug": "2.6.9", - "depd": "2.0.0", - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "finalhandler": "1.3.1", - "fresh": "0.5.2", - "http-errors": "2.0.0", - "merge-descriptors": "1.0.3", - "methods": "~1.1.2", - "on-finished": "2.4.1", - "parseurl": "~1.3.3", - "path-to-regexp": "0.1.10", - "proxy-addr": "~2.0.7", - "qs": "6.13.0", - "range-parser": "~1.2.1", - "safe-buffer": "5.2.1", - "send": "0.19.0", - "serve-static": "1.16.2", - "setprototypeof": "1.2.0", - "statuses": "2.0.1", - "type-is": "~1.6.18", - "utils-merge": "1.0.1", - "vary": "~1.1.2" - }, - "engines": { - "node": ">= 0.10.0" - } - }, - "node_modules/express/node_modules/array-flatten": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", - "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", - "dev": true - }, - "node_modules/fast-deep-equal": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", - "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "dev": true - }, - "node_modules/fast-json-stable-stringify": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", - "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", - "dev": true - }, - "node_modules/fastest-levenshtein": { - "version": "1.0.16", - "resolved": "https://registry.npmjs.org/fastest-levenshtein/-/fastest-levenshtein-1.0.16.tgz", - "integrity": "sha512-eRnCtTTtGZFpQCwhJiUOuxPQWRXVKYDn0b2PeHfXL6/Zi53SLAzAHfVhVWK2AryC/WH05kGfxhFIPvTF0SXQzg==", - "dev": true, - "engines": { - "node": ">= 4.9.1" - } - }, - "node_modules/faye-websocket": { - "version": "0.11.4", - "resolved": "https://registry.npmjs.org/faye-websocket/-/faye-websocket-0.11.4.tgz", - "integrity": "sha512-CzbClwlXAuiRQAlUyfqPgvPoNKTckTPGfwZV4ZdAhVcP2lh9KUxJg2b5GkE7XbjKQ3YJnQ9z6D9ntLAlB+tP8g==", - "dev": true, - "dependencies": { - "websocket-driver": ">=0.5.1" - }, - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/fb-watchman": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/fb-watchman/-/fb-watchman-2.0.2.tgz", - "integrity": "sha512-p5161BqbuCaSnB8jIbzQHOlpgsPmK5rJVDfDKO91Axs5NC1uu3HRQm6wt9cd9/+GtQQIO53JdGXXoyDpTAsgYA==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "bser": "2.1.1" - } - }, - "node_modules/file-set": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/file-set/-/file-set-4.0.2.tgz", - "integrity": "sha512-fuxEgzk4L8waGXaAkd8cMr73Pm0FxOVkn8hztzUW7BAHhOGH90viQNXbiOsnecCWmfInqU6YmAMwxRMdKETceQ==", - "dev": true, - "dependencies": { - "array-back": "^5.0.0", - "glob": "^7.1.6" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/file-set/node_modules/array-back": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/array-back/-/array-back-5.0.0.tgz", - "integrity": "sha512-kgVWwJReZWmVuWOQKEOohXKJX+nD02JAZ54D1RRWlv8L0NebauKAaFxACKzB74RTclt1+WNz5KHaLRDAPZbDEw==", - "dev": true, - "engines": { - "node": ">=10" - } - }, - "node_modules/fill-range": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", - "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", - "dev": true, - "dependencies": { - "to-regex-range": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/finalhandler": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz", - "integrity": "sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==", - "dev": true, - "dependencies": { - "debug": "2.6.9", - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "on-finished": "2.4.1", - "parseurl": "~1.3.3", - "statuses": "2.0.1", - "unpipe": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/find-replace": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/find-replace/-/find-replace-3.0.0.tgz", - "integrity": "sha512-6Tb2myMioCAgv5kfvP5/PkZZ/ntTpVK39fHY7WkWBgvbeE+VHd/tZuZ4mrC+bxh4cfOZeYKVPaJIZtZXV7GNCQ==", - "dev": true, - "dependencies": { - "array-back": "^3.0.1" - }, - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/find-replace/node_modules/array-back": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/array-back/-/array-back-3.1.0.tgz", - "integrity": "sha512-TkuxA4UCOvxuDK6NZYXCalszEzj+TLszyASooky+i742l9TqsOdYCMJJupxRic61hwquNtppB3hgcuq9SVSH1Q==", - "dev": true, - "engines": { - "node": ">=6" - } - }, - "node_modules/find-up": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", - "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", - "dev": true, - "dependencies": { - "locate-path": "^5.0.0", - "path-exists": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/flatbuffers": { - "version": "1.12.0", - "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-1.12.0.tgz", - "integrity": "sha512-c7CZADjRcl6j0PlvFy0ZqXQ67qSEZfrVPynmnL+2zPc+NtMvrF8Y0QceMo7QqnSPc7+uWjUIAbvCQ5WIKlMVdQ==" - }, - "node_modules/follow-redirects": { - "version": "1.15.6", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", - "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==", - "dev": true, - "funding": [ - { - "type": "individual", - "url": "https://github.com/sponsors/RubenVerborgh" - } - ], - "engines": { - "node": ">=4.0" - }, - "peerDependenciesMeta": { - "debug": { - "optional": true - } - } - }, - "node_modules/foreground-child": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz", - "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==", - "dependencies": { - "cross-spawn": "^7.0.0", - "signal-exit": "^4.0.1" - }, - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/foreground-child/node_modules/signal-exit": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", - "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/forwarded": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", - "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", - "dev": true, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/fresh": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", - "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", - "dev": true, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/fs-monkey": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/fs-monkey/-/fs-monkey-1.0.3.tgz", - "integrity": "sha512-cybjIfiiE+pTWicSCLFHSrXZ6EilF30oh91FDP9S2B051prEa7QWfrVTQm10/dDpswBDXZugPa1Ogu8Yh+HV0Q==", - "dev": true - }, - "node_modules/fs-then-native": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/fs-then-native/-/fs-then-native-2.0.0.tgz", - "integrity": "sha512-X712jAOaWXkemQCAmWeg5rOT2i+KOpWz1Z/txk/cW0qlOu2oQ9H61vc5w3X/iyuUEfq/OyaFJ78/cZAQD1/bgA==", - "dev": true, - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/fs.realpath": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", - "dev": true - }, - "node_modules/fsevents": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", - "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "dev": true, - "hasInstallScript": true, - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, - "node_modules/function-bind": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", - "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "dev": true, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/gensync": { - "version": "1.0.0-beta.2", - "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", - "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/get-caller-file": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", - "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - "dev": true, - "license": "ISC", - "engines": { - "node": "6.* || 8.* || >= 10.*" - } - }, - "node_modules/get-intrinsic": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", - "integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==", - "dev": true, - "dependencies": { - "es-errors": "^1.3.0", - "function-bind": "^1.1.2", - "has-proto": "^1.0.1", - "has-symbols": "^1.0.3", - "hasown": "^2.0.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/get-package-type": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", - "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/get-stream": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", - "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", - "dev": true, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/glob": { - "version": "7.2.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", - "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", - "dev": true, - "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.1.1", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - }, - "engines": { - "node": "*" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/glob-parent": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", - "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - "dev": true, - "dependencies": { - "is-glob": "^4.0.1" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/glob-to-regexp": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz", - "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==", - "dev": true - }, - "node_modules/globals": { - "version": "11.12.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", - "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/gopd": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", - "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", - "dev": true, - "dependencies": { - "get-intrinsic": "^1.1.3" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/graceful-fs": { - "version": "4.2.11", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", - "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", - "dev": true - }, - "node_modules/guid-typescript": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz", - "integrity": "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==" - }, - "node_modules/handle-thing": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/handle-thing/-/handle-thing-2.0.1.tgz", - "integrity": "sha512-9Qn4yBxelxoh2Ow62nP+Ka/kMnOXRi8BXnRaUwezLNhqelnN49xKz4F/dPP8OYLxLxq6JDtZb2i9XznUQbNPTg==", - "dev": true - }, - "node_modules/handlebars": { - "version": "4.7.7", - "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.7.tgz", - "integrity": "sha512-aAcXm5OAfE/8IXkcZvCepKU3VzW1/39Fb5ZuqMtgI/hT8X2YgoMvBY5dLhq/cpOvw7Lk1nK/UF71aLG/ZnVYRA==", - "dev": true, - "dependencies": { - "minimist": "^1.2.5", - "neo-async": "^2.6.0", - "source-map": "^0.6.1", - "wordwrap": "^1.0.0" - }, - "bin": { - "handlebars": "bin/handlebars" - }, - "engines": { - "node": ">=0.4.7" - }, - "optionalDependencies": { - "uglify-js": "^3.1.4" - } - }, - "node_modules/has": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", - "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", - "dev": true, - "dependencies": { - "function-bind": "^1.1.1" - }, - "engines": { - "node": ">= 0.4.0" - } - }, - "node_modules/has-flag": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", - "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/has-property-descriptors": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", - "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", - "dev": true, - "dependencies": { - "es-define-property": "^1.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-proto": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.3.tgz", - "integrity": "sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q==", - "dev": true, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-symbols": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", - "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", - "dev": true, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/hasown": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", - "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", - "dev": true, - "dependencies": { - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/hpack.js": { - "version": "2.1.6", - "resolved": "https://registry.npmjs.org/hpack.js/-/hpack.js-2.1.6.tgz", - "integrity": "sha512-zJxVehUdMGIKsRaNt7apO2Gqp0BdqW5yaiGHXXmbpvxgBYVZnAql+BJb4RO5ad2MgpbZKn5G6nMnegrH1FcNYQ==", - "dev": true, - "dependencies": { - "inherits": "^2.0.1", - "obuf": "^1.0.0", - "readable-stream": "^2.0.1", - "wbuf": "^1.1.0" - } - }, - "node_modules/hpack.js/node_modules/readable-stream": { - "version": "2.3.8", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", - "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", - "dev": true, - "dependencies": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "node_modules/hpack.js/node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", - "dev": true - }, - "node_modules/hpack.js/node_modules/string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "dev": true, - "dependencies": { - "safe-buffer": "~5.1.0" - } - }, - "node_modules/html-entities": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/html-entities/-/html-entities-2.3.3.tgz", - "integrity": "sha512-DV5Ln36z34NNTDgnz0EWGBLZENelNAtkiFA4kyNOG2tDI6Mz1uSWiq1wAKdyjnJwyDiDO7Fa2SO1CTxPXL8VxA==", - "dev": true - }, - "node_modules/html-escaper": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", - "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", - "dev": true, - "license": "MIT" - }, - "node_modules/http-deceiver": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/http-deceiver/-/http-deceiver-1.2.7.tgz", - "integrity": "sha512-LmpOGxTfbpgtGVxJrj5k7asXHCgNZp5nLfp+hWc8QQRqtb7fUy6kRY3BO1h9ddF6yIPYUARgxGOwB42DnxIaNw==", - "dev": true - }, - "node_modules/http-errors": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", - "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", - "dev": true, - "dependencies": { - "depd": "2.0.0", - "inherits": "2.0.4", - "setprototypeof": "1.2.0", - "statuses": "2.0.1", - "toidentifier": "1.0.1" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/http-parser-js": { - "version": "0.5.8", - "resolved": "https://registry.npmjs.org/http-parser-js/-/http-parser-js-0.5.8.tgz", - "integrity": "sha512-SGeBX54F94Wgu5RH3X5jsDtf4eHyRogWX1XGT3b4HuW3tQPM4AaBzoUji/4AAJNXCEOWZ5O0DgZmJw1947gD5Q==", - "dev": true - }, - "node_modules/http-proxy": { - "version": "1.18.1", - "resolved": "https://registry.npmjs.org/http-proxy/-/http-proxy-1.18.1.tgz", - "integrity": "sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ==", - "dev": true, - "dependencies": { - "eventemitter3": "^4.0.0", - "follow-redirects": "^1.0.0", - "requires-port": "^1.0.0" - }, - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/http-proxy-middleware": { - "version": "2.0.7", - "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.7.tgz", - "integrity": "sha512-fgVY8AV7qU7z/MmXJ/rxwbrtQH4jBQ9m7kp3llF0liB7glmFeVZFBepQb32T3y8n8k2+AEYuMPCpinYW+/CuRA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/http-proxy": "^1.17.8", - "http-proxy": "^1.18.1", - "is-glob": "^4.0.1", - "is-plain-obj": "^3.0.0", - "micromatch": "^4.0.2" - }, - "engines": { - "node": ">=12.0.0" - }, - "peerDependencies": { - "@types/express": "^4.17.13" - }, - "peerDependenciesMeta": { - "@types/express": { - "optional": true - } - } - }, - "node_modules/human-signals": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", - "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", - "dev": true, - "engines": { - "node": ">=10.17.0" - } - }, - "node_modules/iconv-lite": { - "version": "0.4.24", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", - "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", - "dev": true, - "dependencies": { - "safer-buffer": ">= 2.1.2 < 3" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/import-local": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.1.0.tgz", - "integrity": "sha512-ASB07uLtnDs1o6EHjKpX34BKYDSqnFerfTOJL2HvMqF70LnxpjkzDB8J44oT9pu4AMPkQwf8jl6szgvNd2tRIg==", - "dev": true, - "dependencies": { - "pkg-dir": "^4.2.0", - "resolve-cwd": "^3.0.0" - }, - "bin": { - "import-local-fixture": "fixtures/cli.js" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/imurmurhash": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", - "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.8.19" - } - }, - "node_modules/inflight": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", - "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", - "dev": true, - "dependencies": { - "once": "^1.3.0", - "wrappy": "1" - } - }, - "node_modules/inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true - }, - "node_modules/interpret": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/interpret/-/interpret-3.1.1.tgz", - "integrity": "sha512-6xwYfHbajpoF0xLW+iwLkhwgvLoZDfjYfoFNu8ftMoXINzwuymNLd9u/KmwtdT2GbR+/Cz66otEGEVVUHX9QLQ==", - "dev": true, - "engines": { - "node": ">=10.13.0" - } - }, - "node_modules/ipaddr.js": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.0.1.tgz", - "integrity": "sha512-1qTgH9NG+IIJ4yfKs2e6Pp1bZg8wbDbKHT21HrLIeYBTRLgMYKnMTPAuI3Lcs61nfx5h1xlXnbJtH1kX5/d/ng==", - "dev": true, - "engines": { - "node": ">= 10" - } - }, - "node_modules/is-arrayish": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", - "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" - }, - "node_modules/is-binary-path": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", - "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", - "dev": true, - "dependencies": { - "binary-extensions": "^2.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/is-core-module": { - "version": "2.12.0", - "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.12.0.tgz", - "integrity": "sha512-RECHCBCd/viahWmwj6enj19sKbHfJrddi/6cBDsNTKbNq0f7VeaUkBo60BqzvPqo/W54ChS62Z5qyun7cfOMqQ==", - "dev": true, - "dependencies": { - "has": "^1.0.3" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-docker": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.2.1.tgz", - "integrity": "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==", - "dev": true, - "bin": { - "is-docker": "cli.js" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/is-extglob": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", - "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "engines": { - "node": ">=8" - } - }, - "node_modules/is-generator-fn": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/is-generator-fn/-/is-generator-fn-2.1.0.tgz", - "integrity": "sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/is-glob": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", - "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", - "dev": true, - "dependencies": { - "is-extglob": "^2.1.1" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/is-number": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", - "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "dev": true, - "engines": { - "node": ">=0.12.0" - } - }, - "node_modules/is-plain-obj": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-3.0.0.tgz", - "integrity": "sha512-gwsOE28k+23GP1B6vFl1oVh/WOzmawBrKwo5Ev6wMKzPkaXaCDIQKzLnvsA42DRlbVTWorkgTKIviAKCWkfUwA==", - "dev": true, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/is-plain-object": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", - "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", - "dev": true, - "dependencies": { - "isobject": "^3.0.1" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/is-stream": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", - "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", - "dev": true, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/is-wsl": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz", - "integrity": "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==", - "dev": true, - "dependencies": { - "is-docker": "^2.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/isarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", - "dev": true - }, - "node_modules/isexe": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" - }, - "node_modules/isobject": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", - "integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/istanbul-lib-coverage": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", - "integrity": "sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==", - "dev": true, - "license": "BSD-3-Clause", - "engines": { - "node": ">=8" - } - }, - "node_modules/istanbul-lib-instrument": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-6.0.3.tgz", - "integrity": "sha512-Vtgk7L/R2JHyyGW07spoFlB8/lpjiOLTjMdms6AFMraYt3BaJauod/NGrfnVG/y4Ix1JEuMRPDPEj2ua+zz1/Q==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "@babel/core": "^7.23.9", - "@babel/parser": "^7.23.9", - "@istanbuljs/schema": "^0.1.3", - "istanbul-lib-coverage": "^3.2.0", - "semver": "^7.5.4" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/istanbul-lib-report": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz", - "integrity": "sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "istanbul-lib-coverage": "^3.0.0", - "make-dir": "^4.0.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/istanbul-lib-report/node_modules/supports-color": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dev": true, - "license": "MIT", - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/istanbul-lib-source-maps": { - "version": "5.0.6", - "resolved": "https://registry.npmjs.org/istanbul-lib-source-maps/-/istanbul-lib-source-maps-5.0.6.tgz", - "integrity": "sha512-yg2d+Em4KizZC5niWhQaIomgf5WlL4vOOjZ5xGCmF8SnPE/mDWWXgvRExdcpCgh9lLRRa1/fSYp2ymmbJ1pI+A==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "@jridgewell/trace-mapping": "^0.3.23", - "debug": "^4.1.1", - "istanbul-lib-coverage": "^3.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/istanbul-lib-source-maps/node_modules/debug": { - "version": "4.3.7", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", - "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/istanbul-lib-source-maps/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true, - "license": "MIT" - }, - "node_modules/istanbul-reports": { - "version": "3.1.7", - "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.1.7.tgz", - "integrity": "sha512-BewmUXImeuRk2YY0PVbxgKAysvhRPUQE0h5QRM++nVWyubKGV0l8qQ5op8+B2DOmwSe63Jivj0BjkPQVf8fP5g==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "html-escaper": "^2.0.0", - "istanbul-lib-report": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/jackspeak": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.1.2.tgz", - "integrity": "sha512-kWmLKn2tRtfYMF/BakihVVRzBKOxz4gJMiL2Rj91WnAB5TPZumSH99R/Yf1qE1u4uRimvCSJfm6hnxohXeEXjQ==", - "dependencies": { - "@isaacs/cliui": "^8.0.2" - }, - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - }, - "optionalDependencies": { - "@pkgjs/parseargs": "^0.11.0" - } - }, - "node_modules/jest": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest/-/jest-30.0.0-alpha.6.tgz", - "integrity": "sha512-9T3nAcIAcEpCX2MdxcjG2IDfG/0tjumnCkVNGh+AKkRXcWF4Er5jLROKvXsgXUJCmr/nMqLF6LG0GrDJ0kjFag==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/core": "30.0.0-alpha.6", - "@jest/types": "30.0.0-alpha.6", - "import-local": "^3.0.2", - "jest-cli": "30.0.0-alpha.6" - }, - "bin": { - "jest": "bin/jest.js" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - }, - "peerDependencies": { - "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" - }, - "peerDependenciesMeta": { - "node-notifier": { - "optional": true - } - } - }, - "node_modules/jest-changed-files": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-changed-files/-/jest-changed-files-30.0.0-alpha.6.tgz", - "integrity": "sha512-Fmyt6W27L4fRBl/gReFC4WU+3XIqB7ySHu+a9QxrERapfCb43o7y81TCvTwJHSw5dxGzXLOObVB0tRMDWMafnw==", - "dev": true, - "license": "MIT", - "dependencies": { - "execa": "^5.0.0", - "jest-util": "30.0.0-alpha.6", - "p-limit": "^3.1.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-changed-files/node_modules/@jest/schemas": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", - "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@sinclair/typebox": "^0.33.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-changed-files/node_modules/@jest/types": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", - "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/pattern": "30.0.0-alpha.6", - "@jest/schemas": "30.0.0-alpha.6", - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^3.0.0", - "@types/node": "*", - "@types/yargs": "^17.0.8", - "chalk": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-changed-files/node_modules/@sinclair/typebox": { - "version": "0.33.17", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", - "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", - "dev": true, - "license": "MIT" - }, - "node_modules/jest-changed-files/node_modules/ci-info": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.0.0.tgz", - "integrity": "sha512-TdHqgGf9odd8SXNuxtUBVx8Nv+qZOejE6qyqiy5NtbYYQOeFa6zmHkxlPzmaLxWWHsU6nJmB7AETdVPi+2NBUg==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/sibiraj-s" - } - ], - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/jest-changed-files/node_modules/jest-util": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.0.0-alpha.6.tgz", - "integrity": "sha512-JlimakOVDyoMC8TEG+knoufxUqLG+Btihf1G8o5sHxz54C6oL54Wetfepp+Nhuj/1hSL0sQtkovvjlEycf9i0w==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "30.0.0-alpha.6", - "@types/node": "*", - "chalk": "^4.0.0", - "ci-info": "^4.0.0", - "graceful-fs": "^4.2.9", - "picomatch": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-changed-files/node_modules/p-limit": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", - "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "yocto-queue": "^0.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/jest-changed-files/node_modules/picomatch": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", - "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/jest-circus": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-circus/-/jest-circus-30.0.0-alpha.6.tgz", - "integrity": "sha512-1C62WeTyWinn6zR61syYKe5yqVbV+ftf21vOgj8AtTxGfMUAlGCpeZ5zh4Kc9Qk7r/PiPiHWZtgZmeT4oe9Dug==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/environment": "30.0.0-alpha.6", - "@jest/expect": "30.0.0-alpha.6", - "@jest/test-result": "30.0.0-alpha.6", - "@jest/types": "30.0.0-alpha.6", - "@types/node": "*", - "chalk": "^4.0.0", - "co": "^4.6.0", - "dedent": "^1.0.0", - "is-generator-fn": "^2.0.0", - "jest-each": "30.0.0-alpha.6", - "jest-matcher-utils": "30.0.0-alpha.6", - "jest-message-util": "30.0.0-alpha.6", - "jest-runtime": "30.0.0-alpha.6", - "jest-snapshot": "30.0.0-alpha.6", - "jest-util": "30.0.0-alpha.6", - "p-limit": "^3.1.0", - "pretty-format": "30.0.0-alpha.6", - "pure-rand": "^6.0.0", - "slash": "^3.0.0", - "stack-utils": "^2.0.3" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-circus/node_modules/@jest/schemas": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", - "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@sinclair/typebox": "^0.33.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-circus/node_modules/@jest/types": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", - "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/pattern": "30.0.0-alpha.6", - "@jest/schemas": "30.0.0-alpha.6", - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^3.0.0", - "@types/node": "*", - "@types/yargs": "^17.0.8", - "chalk": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-circus/node_modules/@sinclair/typebox": { - "version": "0.33.17", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", - "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", - "dev": true, - "license": "MIT" - }, - "node_modules/jest-circus/node_modules/ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/jest-circus/node_modules/ci-info": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.0.0.tgz", - "integrity": "sha512-TdHqgGf9odd8SXNuxtUBVx8Nv+qZOejE6qyqiy5NtbYYQOeFa6zmHkxlPzmaLxWWHsU6nJmB7AETdVPi+2NBUg==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/sibiraj-s" - } - ], - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/jest-circus/node_modules/diff-sequences": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-30.0.0-alpha.6.tgz", - "integrity": "sha512-DVGt3/yzbneMUTuupsMqyfSXMnU2fE0lVsC9uFsJmRpluvSi7ZhrS0GX5tnMna6Ta788FGfOUx+irI/+cAZ4EA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-circus/node_modules/jest-diff": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-30.0.0-alpha.6.tgz", - "integrity": "sha512-43j1DoYwVKrkbB67a2gC0ijjIY9biF0JSPXv7H6zlOkzNlqYg8hSDzrurLNo6zGKatW4JSBLE79LmXPJPj1m6A==", - "dev": true, - "license": "MIT", - "dependencies": { - "chalk": "^4.0.0", - "diff-sequences": "30.0.0-alpha.6", - "jest-get-type": "30.0.0-alpha.6", - "pretty-format": "30.0.0-alpha.6" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-circus/node_modules/jest-get-type": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-30.0.0-alpha.6.tgz", - "integrity": "sha512-lJEoQdCY4ICN6+T0lJ9BODKuqPOEpCv2NnJsEO1nmsK0fbWZmN/pgOPHVqLfK8i3jZpUmgupJ1w8r36mc8iiBQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-circus/node_modules/jest-matcher-utils": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-30.0.0-alpha.6.tgz", - "integrity": "sha512-jaq7+HznsK54G0qzu96ZwfMEKHmlPiDqg6qG2p/hVQzr6Y/qVMRh8abI9Y1lX6SSXkr+S9mPAkmOsuJNLTLYmQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "chalk": "^4.0.0", - "jest-diff": "30.0.0-alpha.6", - "jest-get-type": "30.0.0-alpha.6", - "pretty-format": "30.0.0-alpha.6" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-circus/node_modules/jest-message-util": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-30.0.0-alpha.6.tgz", - "integrity": "sha512-XAGJqkrBo7m3bFxWqiNqL0PyAWGf1XHR6bTve90MjBKJuIzhJsounGTzBNUw8JoU7Uzcj5Z6ZmEhaE3CDnwjfw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.12.13", - "@jest/types": "30.0.0-alpha.6", - "@types/stack-utils": "^2.0.0", - "chalk": "^4.0.0", - "graceful-fs": "^4.2.9", - "micromatch": "^4.0.7", - "pretty-format": "30.0.0-alpha.6", - "slash": "^3.0.0", - "stack-utils": "^2.0.3" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-circus/node_modules/jest-util": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.0.0-alpha.6.tgz", - "integrity": "sha512-JlimakOVDyoMC8TEG+knoufxUqLG+Btihf1G8o5sHxz54C6oL54Wetfepp+Nhuj/1hSL0sQtkovvjlEycf9i0w==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "30.0.0-alpha.6", - "@types/node": "*", - "chalk": "^4.0.0", - "ci-info": "^4.0.0", - "graceful-fs": "^4.2.9", - "picomatch": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-circus/node_modules/p-limit": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", - "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "yocto-queue": "^0.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/jest-circus/node_modules/picomatch": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", - "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/jest-circus/node_modules/pretty-format": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.0.0-alpha.6.tgz", - "integrity": "sha512-xkeffkZoqQmRrcNewpOsUCKNOl+CkPqjt3Ld749uz1S7/O7GuPNPv2fZk3v/1U/FE8/B4Zz0llVL80MKON1tOQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/schemas": "30.0.0-alpha.6", - "ansi-styles": "^5.0.0", - "react-is": "^18.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-cli": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-cli/-/jest-cli-30.0.0-alpha.6.tgz", - "integrity": "sha512-3VYzI2KgpMNAsf+LdRAQtAbhH3IDyFnT36U6URXot+2JWwoCGQQ6w4HIfqyOSlH4aejKgTPSfxki2shRPDFtlQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/core": "30.0.0-alpha.6", - "@jest/test-result": "30.0.0-alpha.6", - "@jest/types": "30.0.0-alpha.6", - "chalk": "^4.0.0", - "exit": "^0.1.2", - "import-local": "^3.0.2", - "jest-config": "30.0.0-alpha.6", - "jest-util": "30.0.0-alpha.6", - "jest-validate": "30.0.0-alpha.6", - "yargs": "^17.3.1" - }, - "bin": { - "jest": "bin/jest.js" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - }, - "peerDependencies": { - "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" - }, - "peerDependenciesMeta": { - "node-notifier": { - "optional": true - } - } - }, - "node_modules/jest-cli/node_modules/@jest/schemas": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", - "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@sinclair/typebox": "^0.33.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-cli/node_modules/@jest/types": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", - "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/pattern": "30.0.0-alpha.6", - "@jest/schemas": "30.0.0-alpha.6", - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^3.0.0", - "@types/node": "*", - "@types/yargs": "^17.0.8", - "chalk": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-cli/node_modules/@sinclair/typebox": { - "version": "0.33.17", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", - "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", - "dev": true, - "license": "MIT" - }, - "node_modules/jest-cli/node_modules/ci-info": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.0.0.tgz", - "integrity": "sha512-TdHqgGf9odd8SXNuxtUBVx8Nv+qZOejE6qyqiy5NtbYYQOeFa6zmHkxlPzmaLxWWHsU6nJmB7AETdVPi+2NBUg==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/sibiraj-s" - } - ], - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/jest-cli/node_modules/jest-util": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.0.0-alpha.6.tgz", - "integrity": "sha512-JlimakOVDyoMC8TEG+knoufxUqLG+Btihf1G8o5sHxz54C6oL54Wetfepp+Nhuj/1hSL0sQtkovvjlEycf9i0w==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "30.0.0-alpha.6", - "@types/node": "*", - "chalk": "^4.0.0", - "ci-info": "^4.0.0", - "graceful-fs": "^4.2.9", - "picomatch": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-cli/node_modules/picomatch": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", - "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/jest-config": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-config/-/jest-config-30.0.0-alpha.6.tgz", - "integrity": "sha512-Tq9rH1mg9+nlIhh3efGwMSogFVKZ9z7c6P33ZlK74iJlnqqIAKYERZL2nNmNC5+5p8uxlTPSFZfBz9O8NGKotw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/core": "^7.11.6", - "@jest/pattern": "30.0.0-alpha.6", - "@jest/test-sequencer": "30.0.0-alpha.6", - "@jest/types": "30.0.0-alpha.6", - "babel-jest": "30.0.0-alpha.6", - "chalk": "^4.0.0", - "ci-info": "^4.0.0", - "deepmerge": "^4.2.2", - "glob": "^10.3.10", - "graceful-fs": "^4.2.9", - "jest-circus": "30.0.0-alpha.6", - "jest-docblock": "30.0.0-alpha.6", - "jest-environment-node": "30.0.0-alpha.6", - "jest-get-type": "30.0.0-alpha.6", - "jest-regex-util": "30.0.0-alpha.6", - "jest-resolve": "30.0.0-alpha.6", - "jest-runner": "30.0.0-alpha.6", - "jest-util": "30.0.0-alpha.6", - "jest-validate": "30.0.0-alpha.6", - "micromatch": "^4.0.7", - "parse-json": "^5.2.0", - "pretty-format": "30.0.0-alpha.6", - "slash": "^3.0.0", - "strip-json-comments": "^3.1.1" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - }, - "peerDependencies": { - "@types/node": "*", - "esbuild-register": ">=3.4.0", - "ts-node": ">=9.0.0" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - }, - "esbuild-register": { - "optional": true - }, - "ts-node": { - "optional": true - } - } - }, - "node_modules/jest-config/node_modules/@jest/schemas": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", - "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@sinclair/typebox": "^0.33.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-config/node_modules/@jest/types": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", - "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/pattern": "30.0.0-alpha.6", - "@jest/schemas": "30.0.0-alpha.6", - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^3.0.0", - "@types/node": "*", - "@types/yargs": "^17.0.8", - "chalk": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-config/node_modules/@sinclair/typebox": { - "version": "0.33.17", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", - "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", - "dev": true, - "license": "MIT" - }, - "node_modules/jest-config/node_modules/ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/jest-config/node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0" - } - }, - "node_modules/jest-config/node_modules/ci-info": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.0.0.tgz", - "integrity": "sha512-TdHqgGf9odd8SXNuxtUBVx8Nv+qZOejE6qyqiy5NtbYYQOeFa6zmHkxlPzmaLxWWHsU6nJmB7AETdVPi+2NBUg==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/sibiraj-s" - } - ], - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/jest-config/node_modules/glob": { - "version": "10.4.5", - "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz", - "integrity": "sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==", - "dev": true, - "license": "ISC", - "dependencies": { - "foreground-child": "^3.1.0", - "jackspeak": "^3.1.2", - "minimatch": "^9.0.4", - "minipass": "^7.1.2", - "package-json-from-dist": "^1.0.0", - "path-scurry": "^1.11.1" - }, - "bin": { - "glob": "dist/esm/bin.mjs" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/jest-config/node_modules/jest-get-type": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-30.0.0-alpha.6.tgz", - "integrity": "sha512-lJEoQdCY4ICN6+T0lJ9BODKuqPOEpCv2NnJsEO1nmsK0fbWZmN/pgOPHVqLfK8i3jZpUmgupJ1w8r36mc8iiBQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-config/node_modules/jest-util": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.0.0-alpha.6.tgz", - "integrity": "sha512-JlimakOVDyoMC8TEG+knoufxUqLG+Btihf1G8o5sHxz54C6oL54Wetfepp+Nhuj/1hSL0sQtkovvjlEycf9i0w==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "30.0.0-alpha.6", - "@types/node": "*", - "chalk": "^4.0.0", - "ci-info": "^4.0.0", - "graceful-fs": "^4.2.9", - "picomatch": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-config/node_modules/minimatch": { - "version": "9.0.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", - "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", - "dev": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^2.0.1" - }, - "engines": { - "node": ">=16 || 14 >=14.17" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/jest-config/node_modules/picomatch": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", - "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/jest-config/node_modules/pretty-format": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.0.0-alpha.6.tgz", - "integrity": "sha512-xkeffkZoqQmRrcNewpOsUCKNOl+CkPqjt3Ld749uz1S7/O7GuPNPv2fZk3v/1U/FE8/B4Zz0llVL80MKON1tOQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/schemas": "30.0.0-alpha.6", - "ansi-styles": "^5.0.0", - "react-is": "^18.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-diff": { - "version": "29.6.1", - "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-29.6.1.tgz", - "integrity": "sha512-FsNCvinvl8oVxpNLttNQX7FAq7vR+gMDGj90tiP7siWw1UdakWUGqrylpsYrpvj908IYckm5Y0Q7azNAozU1Kg==", - "dev": true, - "dependencies": { - "chalk": "^4.0.0", - "diff-sequences": "^29.4.3", - "jest-get-type": "^29.4.3", - "pretty-format": "^29.6.1" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-docblock": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-docblock/-/jest-docblock-30.0.0-alpha.6.tgz", - "integrity": "sha512-KXRLgRo7/rF1wqxQupsFCZa6wOp1qrDg4GdSXKfIHODYQb0dpi4rYaYA8xV5l2g9KwYc9/zV7l1tPe9TOr27ew==", - "dev": true, - "license": "MIT", - "dependencies": { - "detect-newline": "^3.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-each": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-each/-/jest-each-30.0.0-alpha.6.tgz", - "integrity": "sha512-snLI2JNYkoBMlZRrNk67XiauUy+uEzRCszKdj+cqHyZ4/MU8fz7gCxbn3g0zmiGUxr0RX0534UxMjc82Sk++tg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "30.0.0-alpha.6", - "chalk": "^4.0.0", - "jest-get-type": "30.0.0-alpha.6", - "jest-util": "30.0.0-alpha.6", - "pretty-format": "30.0.0-alpha.6" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-each/node_modules/@jest/schemas": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", - "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@sinclair/typebox": "^0.33.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-each/node_modules/@jest/types": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", - "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/pattern": "30.0.0-alpha.6", - "@jest/schemas": "30.0.0-alpha.6", - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^3.0.0", - "@types/node": "*", - "@types/yargs": "^17.0.8", - "chalk": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-each/node_modules/@sinclair/typebox": { - "version": "0.33.17", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", - "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", - "dev": true, - "license": "MIT" - }, - "node_modules/jest-each/node_modules/ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/jest-each/node_modules/ci-info": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.0.0.tgz", - "integrity": "sha512-TdHqgGf9odd8SXNuxtUBVx8Nv+qZOejE6qyqiy5NtbYYQOeFa6zmHkxlPzmaLxWWHsU6nJmB7AETdVPi+2NBUg==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/sibiraj-s" - } - ], - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/jest-each/node_modules/jest-get-type": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-30.0.0-alpha.6.tgz", - "integrity": "sha512-lJEoQdCY4ICN6+T0lJ9BODKuqPOEpCv2NnJsEO1nmsK0fbWZmN/pgOPHVqLfK8i3jZpUmgupJ1w8r36mc8iiBQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-each/node_modules/jest-util": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.0.0-alpha.6.tgz", - "integrity": "sha512-JlimakOVDyoMC8TEG+knoufxUqLG+Btihf1G8o5sHxz54C6oL54Wetfepp+Nhuj/1hSL0sQtkovvjlEycf9i0w==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "30.0.0-alpha.6", - "@types/node": "*", - "chalk": "^4.0.0", - "ci-info": "^4.0.0", - "graceful-fs": "^4.2.9", - "picomatch": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-each/node_modules/picomatch": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", - "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/jest-each/node_modules/pretty-format": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.0.0-alpha.6.tgz", - "integrity": "sha512-xkeffkZoqQmRrcNewpOsUCKNOl+CkPqjt3Ld749uz1S7/O7GuPNPv2fZk3v/1U/FE8/B4Zz0llVL80MKON1tOQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/schemas": "30.0.0-alpha.6", - "ansi-styles": "^5.0.0", - "react-is": "^18.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-environment-node": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-30.0.0-alpha.6.tgz", - "integrity": "sha512-UN9W3dFzO150Bqj1x+1pq7dMUqw/QhpqhdtmC3B1P6GD9eKEMFGuRw3EButx5SGzrZOqRNlF+tUNC8CoWGW2Og==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/environment": "30.0.0-alpha.6", - "@jest/fake-timers": "30.0.0-alpha.6", - "@jest/types": "30.0.0-alpha.6", - "@types/node": "*", - "jest-mock": "30.0.0-alpha.6", - "jest-util": "30.0.0-alpha.6" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-environment-node/node_modules/@jest/schemas": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", - "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@sinclair/typebox": "^0.33.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-environment-node/node_modules/@jest/types": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", - "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/pattern": "30.0.0-alpha.6", - "@jest/schemas": "30.0.0-alpha.6", - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^3.0.0", - "@types/node": "*", - "@types/yargs": "^17.0.8", - "chalk": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-environment-node/node_modules/@sinclair/typebox": { - "version": "0.33.17", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", - "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", - "dev": true, - "license": "MIT" - }, - "node_modules/jest-environment-node/node_modules/ci-info": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.0.0.tgz", - "integrity": "sha512-TdHqgGf9odd8SXNuxtUBVx8Nv+qZOejE6qyqiy5NtbYYQOeFa6zmHkxlPzmaLxWWHsU6nJmB7AETdVPi+2NBUg==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/sibiraj-s" - } - ], - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/jest-environment-node/node_modules/jest-util": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.0.0-alpha.6.tgz", - "integrity": "sha512-JlimakOVDyoMC8TEG+knoufxUqLG+Btihf1G8o5sHxz54C6oL54Wetfepp+Nhuj/1hSL0sQtkovvjlEycf9i0w==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "30.0.0-alpha.6", - "@types/node": "*", - "chalk": "^4.0.0", - "ci-info": "^4.0.0", - "graceful-fs": "^4.2.9", - "picomatch": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-environment-node/node_modules/picomatch": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", - "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/jest-get-type": { - "version": "29.4.3", - "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-29.4.3.tgz", - "integrity": "sha512-J5Xez4nRRMjk8emnTpWrlkyb9pfRQQanDrvWHhsR1+VUfbwxi30eVcZFlcdGInRibU4G5LwHXpI7IRHU0CY+gg==", - "dev": true, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-haste-map": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-haste-map/-/jest-haste-map-30.0.0-alpha.6.tgz", - "integrity": "sha512-NR/Kw8HyOkuWIdT8ynsp9KnsTDvWnlz8WSOmtQxySTIzOWbZaeJ2FJi9LoDL6+vhKpdlLfUvhgZVtnFJSLCzew==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "30.0.0-alpha.6", - "@types/node": "*", - "anymatch": "^3.0.3", - "fb-watchman": "^2.0.0", - "graceful-fs": "^4.2.9", - "jest-regex-util": "30.0.0-alpha.6", - "jest-util": "30.0.0-alpha.6", - "jest-worker": "30.0.0-alpha.6", - "micromatch": "^4.0.7", - "walker": "^1.0.8" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - }, - "optionalDependencies": { - "fsevents": "^2.3.2" - } - }, - "node_modules/jest-haste-map/node_modules/@jest/schemas": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", - "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@sinclair/typebox": "^0.33.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-haste-map/node_modules/@jest/types": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", - "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/pattern": "30.0.0-alpha.6", - "@jest/schemas": "30.0.0-alpha.6", - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^3.0.0", - "@types/node": "*", - "@types/yargs": "^17.0.8", - "chalk": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-haste-map/node_modules/@sinclair/typebox": { - "version": "0.33.17", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", - "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", - "dev": true, - "license": "MIT" - }, - "node_modules/jest-haste-map/node_modules/ci-info": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.0.0.tgz", - "integrity": "sha512-TdHqgGf9odd8SXNuxtUBVx8Nv+qZOejE6qyqiy5NtbYYQOeFa6zmHkxlPzmaLxWWHsU6nJmB7AETdVPi+2NBUg==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/sibiraj-s" - } - ], - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/jest-haste-map/node_modules/jest-util": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.0.0-alpha.6.tgz", - "integrity": "sha512-JlimakOVDyoMC8TEG+knoufxUqLG+Btihf1G8o5sHxz54C6oL54Wetfepp+Nhuj/1hSL0sQtkovvjlEycf9i0w==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "30.0.0-alpha.6", - "@types/node": "*", - "chalk": "^4.0.0", - "ci-info": "^4.0.0", - "graceful-fs": "^4.2.9", - "picomatch": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-haste-map/node_modules/jest-worker": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-30.0.0-alpha.6.tgz", - "integrity": "sha512-qlzX7zFT/QdUV/LWsJwZBlaIBaJ+E2VH3d1gArGVP+9hUHGpJkEzCSBK7yuZrkt+M/U0Jre5+maPRmkinEF4DA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*", - "@ungap/structured-clone": "^1.2.0", - "jest-util": "30.0.0-alpha.6", - "merge-stream": "^2.0.0", - "supports-color": "^8.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-haste-map/node_modules/picomatch": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", - "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/jest-leak-detector": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-leak-detector/-/jest-leak-detector-30.0.0-alpha.6.tgz", - "integrity": "sha512-a6fh/6h6dCDyj+aplGqkajVqzmi+qYHs5X8orMZv+u56++gUezJZJf8GCiQqw2vtxcsWVPUuQXa3kF33tAYzNQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "jest-get-type": "30.0.0-alpha.6", - "pretty-format": "30.0.0-alpha.6" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-leak-detector/node_modules/@jest/schemas": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", - "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@sinclair/typebox": "^0.33.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-leak-detector/node_modules/@sinclair/typebox": { - "version": "0.33.17", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", - "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", - "dev": true, - "license": "MIT" - }, - "node_modules/jest-leak-detector/node_modules/ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/jest-leak-detector/node_modules/jest-get-type": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-30.0.0-alpha.6.tgz", - "integrity": "sha512-lJEoQdCY4ICN6+T0lJ9BODKuqPOEpCv2NnJsEO1nmsK0fbWZmN/pgOPHVqLfK8i3jZpUmgupJ1w8r36mc8iiBQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-leak-detector/node_modules/pretty-format": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.0.0-alpha.6.tgz", - "integrity": "sha512-xkeffkZoqQmRrcNewpOsUCKNOl+CkPqjt3Ld749uz1S7/O7GuPNPv2fZk3v/1U/FE8/B4Zz0llVL80MKON1tOQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/schemas": "30.0.0-alpha.6", - "ansi-styles": "^5.0.0", - "react-is": "^18.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-matcher-utils": { - "version": "29.6.1", - "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-29.6.1.tgz", - "integrity": "sha512-SLaztw9d2mfQQKHmJXKM0HCbl2PPVld/t9Xa6P9sgiExijviSp7TnZZpw2Fpt+OI3nwUO/slJbOfzfUMKKC5QA==", - "dev": true, - "dependencies": { - "chalk": "^4.0.0", - "jest-diff": "^29.6.1", - "jest-get-type": "^29.4.3", - "pretty-format": "^29.6.1" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-message-util": { - "version": "29.6.1", - "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-29.6.1.tgz", - "integrity": "sha512-KoAW2zAmNSd3Gk88uJ56qXUWbFk787QKmjjJVOjtGFmmGSZgDBrlIL4AfQw1xyMYPNVD7dNInfIbur9B2rd/wQ==", - "dev": true, - "dependencies": { - "@babel/code-frame": "^7.12.13", - "@jest/types": "^29.6.1", - "@types/stack-utils": "^2.0.0", - "chalk": "^4.0.0", - "graceful-fs": "^4.2.9", - "micromatch": "^4.0.4", - "pretty-format": "^29.6.1", - "slash": "^3.0.0", - "stack-utils": "^2.0.3" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-mock": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-mock/-/jest-mock-30.0.0-alpha.6.tgz", - "integrity": "sha512-ezW02IXiKyFYAgDuxfAlONWULitSaB66t411fq2BJxQtgyMGtv59CsnhgbKb0gQp+9vig5MO5ytDCUPalTbarg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "30.0.0-alpha.6", - "@types/node": "*", - "jest-util": "30.0.0-alpha.6" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-mock/node_modules/@jest/schemas": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", - "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@sinclair/typebox": "^0.33.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-mock/node_modules/@jest/types": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", - "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/pattern": "30.0.0-alpha.6", - "@jest/schemas": "30.0.0-alpha.6", - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^3.0.0", - "@types/node": "*", - "@types/yargs": "^17.0.8", - "chalk": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-mock/node_modules/@sinclair/typebox": { - "version": "0.33.17", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", - "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", - "dev": true, - "license": "MIT" - }, - "node_modules/jest-mock/node_modules/ci-info": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.0.0.tgz", - "integrity": "sha512-TdHqgGf9odd8SXNuxtUBVx8Nv+qZOejE6qyqiy5NtbYYQOeFa6zmHkxlPzmaLxWWHsU6nJmB7AETdVPi+2NBUg==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/sibiraj-s" - } - ], - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/jest-mock/node_modules/jest-util": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.0.0-alpha.6.tgz", - "integrity": "sha512-JlimakOVDyoMC8TEG+knoufxUqLG+Btihf1G8o5sHxz54C6oL54Wetfepp+Nhuj/1hSL0sQtkovvjlEycf9i0w==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "30.0.0-alpha.6", - "@types/node": "*", - "chalk": "^4.0.0", - "ci-info": "^4.0.0", - "graceful-fs": "^4.2.9", - "picomatch": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-mock/node_modules/picomatch": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", - "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/jest-pnp-resolver": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/jest-pnp-resolver/-/jest-pnp-resolver-1.2.3.tgz", - "integrity": "sha512-+3NpwQEnRoIBtx4fyhblQDPgJI0H1IEIkX7ShLUjPGA7TtUTvI1oiKi3SR4oBR0hQhQR80l4WAe5RrXBwWMA8w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - }, - "peerDependencies": { - "jest-resolve": "*" - }, - "peerDependenciesMeta": { - "jest-resolve": { - "optional": true - } - } - }, - "node_modules/jest-regex-util": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-regex-util/-/jest-regex-util-30.0.0-alpha.6.tgz", - "integrity": "sha512-XcsAVaqc69QyMz1/FChyhWSoAMaKcDPhFOuWJz/H51LppsyZRAJPXkPnMopsS+qfut8cggExr9QLcsYaX6hqqA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-resolve": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-resolve/-/jest-resolve-30.0.0-alpha.6.tgz", - "integrity": "sha512-0EyeId+RFng52qHvuxOzKjZd2uDF/2Hdzpzt54+biGgY/VVAvf8mYE9UV7g6154Ozpq6KLztSqqMCfPgVs4CbA==", - "dev": true, - "license": "MIT", - "dependencies": { - "chalk": "^4.0.0", - "graceful-fs": "^4.2.9", - "jest-haste-map": "30.0.0-alpha.6", - "jest-pnp-resolver": "^1.2.2", - "jest-util": "30.0.0-alpha.6", - "jest-validate": "30.0.0-alpha.6", - "resolve": "^1.20.0", - "resolve.exports": "^2.0.0", - "slash": "^3.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-resolve-dependencies": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-resolve-dependencies/-/jest-resolve-dependencies-30.0.0-alpha.6.tgz", - "integrity": "sha512-G+st0nBR4FNIvVCHq8YNJBiG6t7u0+cxM099lbtOoJNJU+ZTdIxSyzPnnmp/C+YHd1QOlDNlplvL+xe1KHhPUA==", - "dev": true, - "license": "MIT", - "dependencies": { - "jest-regex-util": "30.0.0-alpha.6", - "jest-snapshot": "30.0.0-alpha.6" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-resolve/node_modules/@jest/schemas": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", - "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@sinclair/typebox": "^0.33.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-resolve/node_modules/@jest/types": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", - "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/pattern": "30.0.0-alpha.6", - "@jest/schemas": "30.0.0-alpha.6", - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^3.0.0", - "@types/node": "*", - "@types/yargs": "^17.0.8", - "chalk": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-resolve/node_modules/@sinclair/typebox": { - "version": "0.33.17", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", - "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", - "dev": true, - "license": "MIT" - }, - "node_modules/jest-resolve/node_modules/ci-info": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.0.0.tgz", - "integrity": "sha512-TdHqgGf9odd8SXNuxtUBVx8Nv+qZOejE6qyqiy5NtbYYQOeFa6zmHkxlPzmaLxWWHsU6nJmB7AETdVPi+2NBUg==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/sibiraj-s" - } - ], - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/jest-resolve/node_modules/jest-util": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.0.0-alpha.6.tgz", - "integrity": "sha512-JlimakOVDyoMC8TEG+knoufxUqLG+Btihf1G8o5sHxz54C6oL54Wetfepp+Nhuj/1hSL0sQtkovvjlEycf9i0w==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "30.0.0-alpha.6", - "@types/node": "*", - "chalk": "^4.0.0", - "ci-info": "^4.0.0", - "graceful-fs": "^4.2.9", - "picomatch": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-resolve/node_modules/picomatch": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", - "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/jest-runner": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-runner/-/jest-runner-30.0.0-alpha.6.tgz", - "integrity": "sha512-SoADy4YnspMpXLNnRCXNIoinm1N5SMci+iF6Y29Duv3wnWhcL14XjEOcyUKBB+AIL52YwouLeUHkCyCspbBk1Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/console": "30.0.0-alpha.6", - "@jest/environment": "30.0.0-alpha.6", - "@jest/test-result": "30.0.0-alpha.6", - "@jest/transform": "30.0.0-alpha.6", - "@jest/types": "30.0.0-alpha.6", - "@types/node": "*", - "chalk": "^4.0.0", - "emittery": "^0.13.1", - "graceful-fs": "^4.2.9", - "jest-docblock": "30.0.0-alpha.6", - "jest-environment-node": "30.0.0-alpha.6", - "jest-haste-map": "30.0.0-alpha.6", - "jest-leak-detector": "30.0.0-alpha.6", - "jest-message-util": "30.0.0-alpha.6", - "jest-resolve": "30.0.0-alpha.6", - "jest-runtime": "30.0.0-alpha.6", - "jest-util": "30.0.0-alpha.6", - "jest-watcher": "30.0.0-alpha.6", - "jest-worker": "30.0.0-alpha.6", - "p-limit": "^3.1.0", - "source-map-support": "0.5.13" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-runner/node_modules/@jest/schemas": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", - "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@sinclair/typebox": "^0.33.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-runner/node_modules/@jest/types": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", - "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/pattern": "30.0.0-alpha.6", - "@jest/schemas": "30.0.0-alpha.6", - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^3.0.0", - "@types/node": "*", - "@types/yargs": "^17.0.8", - "chalk": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-runner/node_modules/@sinclair/typebox": { - "version": "0.33.17", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", - "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", - "dev": true, - "license": "MIT" - }, - "node_modules/jest-runner/node_modules/ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/jest-runner/node_modules/ci-info": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.0.0.tgz", - "integrity": "sha512-TdHqgGf9odd8SXNuxtUBVx8Nv+qZOejE6qyqiy5NtbYYQOeFa6zmHkxlPzmaLxWWHsU6nJmB7AETdVPi+2NBUg==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/sibiraj-s" - } - ], - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/jest-runner/node_modules/jest-message-util": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-30.0.0-alpha.6.tgz", - "integrity": "sha512-XAGJqkrBo7m3bFxWqiNqL0PyAWGf1XHR6bTve90MjBKJuIzhJsounGTzBNUw8JoU7Uzcj5Z6ZmEhaE3CDnwjfw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.12.13", - "@jest/types": "30.0.0-alpha.6", - "@types/stack-utils": "^2.0.0", - "chalk": "^4.0.0", - "graceful-fs": "^4.2.9", - "micromatch": "^4.0.7", - "pretty-format": "30.0.0-alpha.6", - "slash": "^3.0.0", - "stack-utils": "^2.0.3" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-runner/node_modules/jest-util": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.0.0-alpha.6.tgz", - "integrity": "sha512-JlimakOVDyoMC8TEG+knoufxUqLG+Btihf1G8o5sHxz54C6oL54Wetfepp+Nhuj/1hSL0sQtkovvjlEycf9i0w==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "30.0.0-alpha.6", - "@types/node": "*", - "chalk": "^4.0.0", - "ci-info": "^4.0.0", - "graceful-fs": "^4.2.9", - "picomatch": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-runner/node_modules/jest-worker": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-30.0.0-alpha.6.tgz", - "integrity": "sha512-qlzX7zFT/QdUV/LWsJwZBlaIBaJ+E2VH3d1gArGVP+9hUHGpJkEzCSBK7yuZrkt+M/U0Jre5+maPRmkinEF4DA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*", - "@ungap/structured-clone": "^1.2.0", - "jest-util": "30.0.0-alpha.6", - "merge-stream": "^2.0.0", - "supports-color": "^8.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-runner/node_modules/p-limit": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", - "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "yocto-queue": "^0.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/jest-runner/node_modules/picomatch": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", - "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/jest-runner/node_modules/pretty-format": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.0.0-alpha.6.tgz", - "integrity": "sha512-xkeffkZoqQmRrcNewpOsUCKNOl+CkPqjt3Ld749uz1S7/O7GuPNPv2fZk3v/1U/FE8/B4Zz0llVL80MKON1tOQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/schemas": "30.0.0-alpha.6", - "ansi-styles": "^5.0.0", - "react-is": "^18.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-runner/node_modules/source-map-support": { - "version": "0.5.13", - "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.13.tgz", - "integrity": "sha512-SHSKFHadjVA5oR4PPqhtAVdcBWwRYVd6g6cAXnIbRiIwc2EhPrTuKUBdSLvlEKyIP3GCf89fltvcZiP9MMFA1w==", - "dev": true, - "license": "MIT", - "dependencies": { - "buffer-from": "^1.0.0", - "source-map": "^0.6.0" - } - }, - "node_modules/jest-runtime": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-runtime/-/jest-runtime-30.0.0-alpha.6.tgz", - "integrity": "sha512-p7w7DSFFzwHyR4HsNXca/p32VpL9MLT1c71+VplFJIEgeRHvyqxrARentlul6uJniwtlqvZrVVf5baCQ5a5GUw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/environment": "30.0.0-alpha.6", - "@jest/fake-timers": "30.0.0-alpha.6", - "@jest/globals": "30.0.0-alpha.6", - "@jest/source-map": "30.0.0-alpha.6", - "@jest/test-result": "30.0.0-alpha.6", - "@jest/transform": "30.0.0-alpha.6", - "@jest/types": "30.0.0-alpha.6", - "@types/node": "*", - "chalk": "^4.0.0", - "cjs-module-lexer": "^1.0.0", - "collect-v8-coverage": "^1.0.0", - "glob": "^10.3.10", - "graceful-fs": "^4.2.9", - "jest-haste-map": "30.0.0-alpha.6", - "jest-message-util": "30.0.0-alpha.6", - "jest-mock": "30.0.0-alpha.6", - "jest-regex-util": "30.0.0-alpha.6", - "jest-resolve": "30.0.0-alpha.6", - "jest-snapshot": "30.0.0-alpha.6", - "jest-util": "30.0.0-alpha.6", - "slash": "^3.0.0", - "strip-bom": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-runtime/node_modules/@jest/schemas": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", - "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@sinclair/typebox": "^0.33.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-runtime/node_modules/@jest/types": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", - "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/pattern": "30.0.0-alpha.6", - "@jest/schemas": "30.0.0-alpha.6", - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^3.0.0", - "@types/node": "*", - "@types/yargs": "^17.0.8", - "chalk": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-runtime/node_modules/@sinclair/typebox": { - "version": "0.33.17", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", - "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", - "dev": true, - "license": "MIT" - }, - "node_modules/jest-runtime/node_modules/ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/jest-runtime/node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0" - } - }, - "node_modules/jest-runtime/node_modules/ci-info": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.0.0.tgz", - "integrity": "sha512-TdHqgGf9odd8SXNuxtUBVx8Nv+qZOejE6qyqiy5NtbYYQOeFa6zmHkxlPzmaLxWWHsU6nJmB7AETdVPi+2NBUg==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/sibiraj-s" - } - ], - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/jest-runtime/node_modules/glob": { - "version": "10.4.5", - "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz", - "integrity": "sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==", - "dev": true, - "license": "ISC", - "dependencies": { - "foreground-child": "^3.1.0", - "jackspeak": "^3.1.2", - "minimatch": "^9.0.4", - "minipass": "^7.1.2", - "package-json-from-dist": "^1.0.0", - "path-scurry": "^1.11.1" - }, - "bin": { - "glob": "dist/esm/bin.mjs" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/jest-runtime/node_modules/jest-message-util": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-30.0.0-alpha.6.tgz", - "integrity": "sha512-XAGJqkrBo7m3bFxWqiNqL0PyAWGf1XHR6bTve90MjBKJuIzhJsounGTzBNUw8JoU7Uzcj5Z6ZmEhaE3CDnwjfw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.12.13", - "@jest/types": "30.0.0-alpha.6", - "@types/stack-utils": "^2.0.0", - "chalk": "^4.0.0", - "graceful-fs": "^4.2.9", - "micromatch": "^4.0.7", - "pretty-format": "30.0.0-alpha.6", - "slash": "^3.0.0", - "stack-utils": "^2.0.3" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-runtime/node_modules/jest-util": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.0.0-alpha.6.tgz", - "integrity": "sha512-JlimakOVDyoMC8TEG+knoufxUqLG+Btihf1G8o5sHxz54C6oL54Wetfepp+Nhuj/1hSL0sQtkovvjlEycf9i0w==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "30.0.0-alpha.6", - "@types/node": "*", - "chalk": "^4.0.0", - "ci-info": "^4.0.0", - "graceful-fs": "^4.2.9", - "picomatch": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-runtime/node_modules/minimatch": { - "version": "9.0.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", - "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", - "dev": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^2.0.1" - }, - "engines": { - "node": ">=16 || 14 >=14.17" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/jest-runtime/node_modules/picomatch": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", - "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/jest-runtime/node_modules/pretty-format": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.0.0-alpha.6.tgz", - "integrity": "sha512-xkeffkZoqQmRrcNewpOsUCKNOl+CkPqjt3Ld749uz1S7/O7GuPNPv2fZk3v/1U/FE8/B4Zz0llVL80MKON1tOQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/schemas": "30.0.0-alpha.6", - "ansi-styles": "^5.0.0", - "react-is": "^18.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-snapshot": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-snapshot/-/jest-snapshot-30.0.0-alpha.6.tgz", - "integrity": "sha512-YCBUxSNJ9YGch3tyQdxQkOUitbmXahHL6UhSQeSMERFfX1UMrHyEDHggglocCUg4G3jdU8YzshxOJ/oaR6Ph8w==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/core": "^7.11.6", - "@babel/generator": "^7.7.2", - "@babel/plugin-syntax-jsx": "^7.7.2", - "@babel/plugin-syntax-typescript": "^7.7.2", - "@babel/types": "^7.3.3", - "@jest/expect-utils": "30.0.0-alpha.6", - "@jest/snapshot-utils": "30.0.0-alpha.6", - "@jest/transform": "30.0.0-alpha.6", - "@jest/types": "30.0.0-alpha.6", - "babel-preset-current-node-syntax": "^1.0.0", - "chalk": "^4.0.0", - "expect": "30.0.0-alpha.6", - "graceful-fs": "^4.2.9", - "jest-diff": "30.0.0-alpha.6", - "jest-get-type": "30.0.0-alpha.6", - "jest-matcher-utils": "30.0.0-alpha.6", - "jest-message-util": "30.0.0-alpha.6", - "jest-util": "30.0.0-alpha.6", - "pretty-format": "30.0.0-alpha.6", - "semver": "^7.5.3", - "synckit": "^0.9.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-snapshot/node_modules/@jest/expect-utils": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-30.0.0-alpha.6.tgz", - "integrity": "sha512-QMySMhaCUl0ZQd7Tx5X3fVWY5jtQxZNrTll0OyavdQ70ZTLgk0kU9K+XovcMWO26MK9R5EX7bBgD/j7w9hUM4w==", - "dev": true, - "license": "MIT", - "dependencies": { - "jest-get-type": "30.0.0-alpha.6" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-snapshot/node_modules/@jest/schemas": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", - "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@sinclair/typebox": "^0.33.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-snapshot/node_modules/@jest/types": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", - "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/pattern": "30.0.0-alpha.6", - "@jest/schemas": "30.0.0-alpha.6", - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^3.0.0", - "@types/node": "*", - "@types/yargs": "^17.0.8", - "chalk": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-snapshot/node_modules/@sinclair/typebox": { - "version": "0.33.17", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", - "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", - "dev": true, - "license": "MIT" - }, - "node_modules/jest-snapshot/node_modules/ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/jest-snapshot/node_modules/ci-info": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.0.0.tgz", - "integrity": "sha512-TdHqgGf9odd8SXNuxtUBVx8Nv+qZOejE6qyqiy5NtbYYQOeFa6zmHkxlPzmaLxWWHsU6nJmB7AETdVPi+2NBUg==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/sibiraj-s" - } - ], - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/jest-snapshot/node_modules/diff-sequences": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-30.0.0-alpha.6.tgz", - "integrity": "sha512-DVGt3/yzbneMUTuupsMqyfSXMnU2fE0lVsC9uFsJmRpluvSi7ZhrS0GX5tnMna6Ta788FGfOUx+irI/+cAZ4EA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-snapshot/node_modules/expect": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/expect/-/expect-30.0.0-alpha.6.tgz", - "integrity": "sha512-WVi2V4iHKw/vHEyye00Q9CSZz7KHDbJkJyteUI8kTih9jiyMl3bIk7wLYFcY9D1Blnadlyb5w5NBuNjQBow99g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/expect-utils": "30.0.0-alpha.6", - "jest-get-type": "30.0.0-alpha.6", - "jest-matcher-utils": "30.0.0-alpha.6", - "jest-message-util": "30.0.0-alpha.6", - "jest-mock": "30.0.0-alpha.6", - "jest-util": "30.0.0-alpha.6" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-snapshot/node_modules/jest-diff": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-30.0.0-alpha.6.tgz", - "integrity": "sha512-43j1DoYwVKrkbB67a2gC0ijjIY9biF0JSPXv7H6zlOkzNlqYg8hSDzrurLNo6zGKatW4JSBLE79LmXPJPj1m6A==", - "dev": true, - "license": "MIT", - "dependencies": { - "chalk": "^4.0.0", - "diff-sequences": "30.0.0-alpha.6", - "jest-get-type": "30.0.0-alpha.6", - "pretty-format": "30.0.0-alpha.6" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-snapshot/node_modules/jest-get-type": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-30.0.0-alpha.6.tgz", - "integrity": "sha512-lJEoQdCY4ICN6+T0lJ9BODKuqPOEpCv2NnJsEO1nmsK0fbWZmN/pgOPHVqLfK8i3jZpUmgupJ1w8r36mc8iiBQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-snapshot/node_modules/jest-matcher-utils": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-30.0.0-alpha.6.tgz", - "integrity": "sha512-jaq7+HznsK54G0qzu96ZwfMEKHmlPiDqg6qG2p/hVQzr6Y/qVMRh8abI9Y1lX6SSXkr+S9mPAkmOsuJNLTLYmQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "chalk": "^4.0.0", - "jest-diff": "30.0.0-alpha.6", - "jest-get-type": "30.0.0-alpha.6", - "pretty-format": "30.0.0-alpha.6" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-snapshot/node_modules/jest-message-util": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-30.0.0-alpha.6.tgz", - "integrity": "sha512-XAGJqkrBo7m3bFxWqiNqL0PyAWGf1XHR6bTve90MjBKJuIzhJsounGTzBNUw8JoU7Uzcj5Z6ZmEhaE3CDnwjfw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.12.13", - "@jest/types": "30.0.0-alpha.6", - "@types/stack-utils": "^2.0.0", - "chalk": "^4.0.0", - "graceful-fs": "^4.2.9", - "micromatch": "^4.0.7", - "pretty-format": "30.0.0-alpha.6", - "slash": "^3.0.0", - "stack-utils": "^2.0.3" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-snapshot/node_modules/jest-util": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.0.0-alpha.6.tgz", - "integrity": "sha512-JlimakOVDyoMC8TEG+knoufxUqLG+Btihf1G8o5sHxz54C6oL54Wetfepp+Nhuj/1hSL0sQtkovvjlEycf9i0w==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "30.0.0-alpha.6", - "@types/node": "*", - "chalk": "^4.0.0", - "ci-info": "^4.0.0", - "graceful-fs": "^4.2.9", - "picomatch": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-snapshot/node_modules/picomatch": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", - "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/jest-snapshot/node_modules/pretty-format": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.0.0-alpha.6.tgz", - "integrity": "sha512-xkeffkZoqQmRrcNewpOsUCKNOl+CkPqjt3Ld749uz1S7/O7GuPNPv2fZk3v/1U/FE8/B4Zz0llVL80MKON1tOQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/schemas": "30.0.0-alpha.6", - "ansi-styles": "^5.0.0", - "react-is": "^18.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-util": { - "version": "29.6.1", - "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-29.6.1.tgz", - "integrity": "sha512-NRFCcjc+/uO3ijUVyNOQJluf8PtGCe/W6cix36+M3cTFgiYqFOOW5MgN4JOOcvbUhcKTYVd1CvHz/LWi8d16Mg==", - "dev": true, - "dependencies": { - "@jest/types": "^29.6.1", - "@types/node": "*", - "chalk": "^4.0.0", - "ci-info": "^3.2.0", - "graceful-fs": "^4.2.9", - "picomatch": "^2.2.3" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-validate": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-30.0.0-alpha.6.tgz", - "integrity": "sha512-sINLwCenOUeJVzS5p+o1NhwKsY0de5Es0J7bsaSuZJQGRY67W20idceflr+aZ2akrKgvvqU8Tsg6lkFQyq+a6Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "30.0.0-alpha.6", - "camelcase": "^6.2.0", - "chalk": "^4.0.0", - "jest-get-type": "30.0.0-alpha.6", - "leven": "^3.1.0", - "pretty-format": "30.0.0-alpha.6" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-validate/node_modules/@jest/schemas": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", - "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@sinclair/typebox": "^0.33.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-validate/node_modules/@jest/types": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", - "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/pattern": "30.0.0-alpha.6", - "@jest/schemas": "30.0.0-alpha.6", - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^3.0.0", - "@types/node": "*", - "@types/yargs": "^17.0.8", - "chalk": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-validate/node_modules/@sinclair/typebox": { - "version": "0.33.17", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", - "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", - "dev": true, - "license": "MIT" - }, - "node_modules/jest-validate/node_modules/ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/jest-validate/node_modules/camelcase": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", - "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/jest-validate/node_modules/jest-get-type": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-30.0.0-alpha.6.tgz", - "integrity": "sha512-lJEoQdCY4ICN6+T0lJ9BODKuqPOEpCv2NnJsEO1nmsK0fbWZmN/pgOPHVqLfK8i3jZpUmgupJ1w8r36mc8iiBQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-validate/node_modules/pretty-format": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.0.0-alpha.6.tgz", - "integrity": "sha512-xkeffkZoqQmRrcNewpOsUCKNOl+CkPqjt3Ld749uz1S7/O7GuPNPv2fZk3v/1U/FE8/B4Zz0llVL80MKON1tOQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/schemas": "30.0.0-alpha.6", - "ansi-styles": "^5.0.0", - "react-is": "^18.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-watcher": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-watcher/-/jest-watcher-30.0.0-alpha.6.tgz", - "integrity": "sha512-+zL1y3GSJG8EOxVSc2p0dndis0rNDcwKTs4b1bpNTI0XneeTiZlCpRBNYI+sqBl/eZtJBrQdiBRSYz7kJqg7NQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/test-result": "30.0.0-alpha.6", - "@jest/types": "30.0.0-alpha.6", - "@types/node": "*", - "ansi-escapes": "^4.2.1", - "chalk": "^4.0.0", - "emittery": "^0.13.1", - "jest-util": "30.0.0-alpha.6", - "string-length": "^4.0.1" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-watcher/node_modules/@jest/schemas": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", - "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@sinclair/typebox": "^0.33.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-watcher/node_modules/@jest/types": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", - "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/pattern": "30.0.0-alpha.6", - "@jest/schemas": "30.0.0-alpha.6", - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^3.0.0", - "@types/node": "*", - "@types/yargs": "^17.0.8", - "chalk": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-watcher/node_modules/@sinclair/typebox": { - "version": "0.33.17", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", - "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", - "dev": true, - "license": "MIT" - }, - "node_modules/jest-watcher/node_modules/ci-info": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.0.0.tgz", - "integrity": "sha512-TdHqgGf9odd8SXNuxtUBVx8Nv+qZOejE6qyqiy5NtbYYQOeFa6zmHkxlPzmaLxWWHsU6nJmB7AETdVPi+2NBUg==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/sibiraj-s" - } - ], - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/jest-watcher/node_modules/jest-util": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.0.0-alpha.6.tgz", - "integrity": "sha512-JlimakOVDyoMC8TEG+knoufxUqLG+Btihf1G8o5sHxz54C6oL54Wetfepp+Nhuj/1hSL0sQtkovvjlEycf9i0w==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "30.0.0-alpha.6", - "@types/node": "*", - "chalk": "^4.0.0", - "ci-info": "^4.0.0", - "graceful-fs": "^4.2.9", - "picomatch": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest-watcher/node_modules/picomatch": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", - "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/jest-worker": { - "version": "27.5.1", - "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-27.5.1.tgz", - "integrity": "sha512-7vuh85V5cdDofPyxn58nrPjBktZo0u9x1g8WtjQol+jZDaE+fhN+cIvTj11GndBnMnyfrUOG1sZQxCdjKh+DKg==", - "dev": true, - "dependencies": { - "@types/node": "*", - "merge-stream": "^2.0.0", - "supports-color": "^8.0.0" - }, - "engines": { - "node": ">= 10.13.0" - } - }, - "node_modules/jest/node_modules/@jest/schemas": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", - "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@sinclair/typebox": "^0.33.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest/node_modules/@jest/types": { - "version": "30.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", - "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/pattern": "30.0.0-alpha.6", - "@jest/schemas": "30.0.0-alpha.6", - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^3.0.0", - "@types/node": "*", - "@types/yargs": "^17.0.8", - "chalk": "^4.0.0" - }, - "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" - } - }, - "node_modules/jest/node_modules/@sinclair/typebox": { - "version": "0.33.17", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", - "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", - "dev": true, - "license": "MIT" - }, - "node_modules/js-tokens": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", - "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/js-yaml": { - "version": "3.14.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", - "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", - "dev": true, - "license": "MIT", - "dependencies": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, - "node_modules/js-yaml/node_modules/argparse": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", - "dev": true, - "license": "MIT", - "dependencies": { - "sprintf-js": "~1.0.2" - } - }, - "node_modules/js2xmlparser": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/js2xmlparser/-/js2xmlparser-4.0.2.tgz", - "integrity": "sha512-6n4D8gLlLf1n5mNLQPRfViYzu9RATblzPEtm1SthMX1Pjao0r9YI9nw7ZIfRxQMERS87mcswrg+r/OYrPRX6jA==", - "dev": true, - "dependencies": { - "xmlcreate": "^2.0.4" - } - }, - "node_modules/jsdoc": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/jsdoc/-/jsdoc-4.0.2.tgz", - "integrity": "sha512-e8cIg2z62InH7azBBi3EsSEqrKx+nUtAS5bBcYTSpZFA+vhNPyhv8PTFZ0WsjOPDj04/dOLlm08EDcQJDqaGQg==", - "dev": true, - "dependencies": { - "@babel/parser": "^7.20.15", - "@jsdoc/salty": "^0.2.1", - "@types/markdown-it": "^12.2.3", - "bluebird": "^3.7.2", - "catharsis": "^0.9.0", - "escape-string-regexp": "^2.0.0", - "js2xmlparser": "^4.0.2", - "klaw": "^3.0.0", - "markdown-it": "^12.3.2", - "markdown-it-anchor": "^8.4.1", - "marked": "^4.0.10", - "mkdirp": "^1.0.4", - "requizzle": "^0.2.3", - "strip-json-comments": "^3.1.0", - "underscore": "~1.13.2" - }, - "bin": { - "jsdoc": "jsdoc.js" - }, - "engines": { - "node": ">=12.0.0" - } - }, - "node_modules/jsdoc-api": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/jsdoc-api/-/jsdoc-api-8.0.0.tgz", - "integrity": "sha512-Rnhor0suB1Ds1abjmFkFfKeD+kSMRN9oHMTMZoJVUrmtCGDwXty+sWMA9sa4xbe4UyxuPjhC7tavZ40mDKK6QQ==", - "dev": true, - "dependencies": { - "array-back": "^6.2.2", - "cache-point": "^2.0.0", - "collect-all": "^1.0.4", - "file-set": "^4.0.2", - "fs-then-native": "^2.0.0", - "jsdoc": "^4.0.0", - "object-to-spawn-args": "^2.0.1", - "temp-path": "^1.0.0", - "walk-back": "^5.1.0" - }, - "engines": { - "node": ">=12.17" - } - }, - "node_modules/jsdoc-parse": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/jsdoc-parse/-/jsdoc-parse-6.2.1.tgz", - "integrity": "sha512-9viGRUUtWOk/G4V0+nQ6rfLucz5plxh5I74WbNSNm9h9NWugCDVX4jbG8hZP9QqKGpdTPDE+qJXzaYNos3wqTA==", - "dev": true, - "dependencies": { - "array-back": "^6.2.2", - "lodash.omit": "^4.5.0", - "reduce-extract": "^1.0.0", - "sort-array": "^4.1.5", - "test-value": "^3.0.0" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/jsdoc-to-markdown": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/jsdoc-to-markdown/-/jsdoc-to-markdown-8.0.1.tgz", - "integrity": "sha512-qJfNJhkq2C26UYoOdj8L1yheTJlk1veCsxwRejRmj07XZKCn7oSkuPErx6+JoNi8afCaUKdIM5oUu0uF2/T8iw==", - "dev": true, - "dependencies": { - "array-back": "^6.2.2", - "command-line-tool": "^0.8.0", - "config-master": "^3.1.0", - "dmd": "^6.2.0", - "jsdoc-api": "^8.0.0", - "jsdoc-parse": "^6.2.1", - "walk-back": "^5.1.0" - }, - "bin": { - "jsdoc2md": "bin/cli.js" - }, - "engines": { - "node": ">=12.17" - } - }, - "node_modules/jsesc": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.0.2.tgz", - "integrity": "sha512-xKqzzWXDttJuOcawBt4KnKHHIf5oQ/Cxax+0PWFG+DFDgHNAdi+TXECADI+RYiFUMmx8792xsMbbgXj4CwnP4g==", - "dev": true, - "license": "MIT", - "bin": { - "jsesc": "bin/jsesc" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/json-parse-even-better-errors": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", - "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", - "dev": true - }, - "node_modules/json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", - "dev": true - }, - "node_modules/json5": { - "version": "2.2.3", - "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", - "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", - "dev": true, - "license": "MIT", - "bin": { - "json5": "lib/cli.js" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/klaw": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/klaw/-/klaw-3.0.0.tgz", - "integrity": "sha512-0Fo5oir+O9jnXu5EefYbVK+mHMBeEVEy2cmctR1O1NECcCkPRreJKrS6Qt/j3KC2C148Dfo9i3pCmCMsdqGr0g==", - "dev": true, - "dependencies": { - "graceful-fs": "^4.1.9" - } - }, - "node_modules/launch-editor": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/launch-editor/-/launch-editor-2.6.0.tgz", - "integrity": "sha512-JpDCcQnyAAzZZaZ7vEiSqL690w7dAEyLao+KC96zBplnYbJS7TYNjvM3M7y3dGz+v7aIsJk3hllWuc0kWAjyRQ==", - "dev": true, - "dependencies": { - "picocolors": "^1.0.0", - "shell-quote": "^1.7.3" - } - }, - "node_modules/leven": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", - "integrity": "sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/lines-and-columns": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", - "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", - "dev": true, - "license": "MIT" - }, - "node_modules/linkify-it": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-3.0.3.tgz", - "integrity": "sha512-ynTsyrFSdE5oZ/O9GEf00kPngmOfVwazR5GKDq6EYfhlpFug3J2zybX56a2PRRpc9P+FuSoGNAwjlbDs9jJBPQ==", - "dev": true, - "dependencies": { - "uc.micro": "^1.0.1" - } - }, - "node_modules/loader-runner": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.3.0.tgz", - "integrity": "sha512-3R/1M+yS3j5ou80Me59j7F9IMs4PXs3VqRrm0TU3AbKPxlmpoY1TNscJV/oGJXo8qCatFGTfDbY6W6ipGOYXfg==", - "dev": true, - "engines": { - "node": ">=6.11.5" - } - }, - "node_modules/locate-path": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", - "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", - "dev": true, - "dependencies": { - "p-locate": "^4.1.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/lodash": { - "version": "4.17.21", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", - "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", - "dev": true - }, - "node_modules/lodash.camelcase": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", - "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", - "dev": true - }, - "node_modules/lodash.omit": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/lodash.omit/-/lodash.omit-4.5.0.tgz", - "integrity": "sha512-XeqSp49hNGmlkj2EJlfrQFIzQ6lXdNro9sddtQzcJY8QaoC2GO0DT7xaIokHeyM+mIT0mPMlPvkYzg2xCuHdZg==", - "dev": true - }, - "node_modules/lodash.padend": { - "version": "4.6.1", - "resolved": "https://registry.npmjs.org/lodash.padend/-/lodash.padend-4.6.1.tgz", - "integrity": "sha512-sOQs2aqGpbl27tmCS1QNZA09Uqp01ZzWfDUoD+xzTii0E7dSQfRKcRetFwa+uXaxaqL+TKm7CgD2JdKP7aZBSw==", - "dev": true - }, - "node_modules/long": { - "version": "5.2.3", - "resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz", - "integrity": "sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==" - }, - "node_modules/lru-cache": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", - "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", - "dev": true, - "license": "ISC", - "dependencies": { - "yallist": "^3.0.2" - } - }, - "node_modules/make-dir": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", - "integrity": "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==", - "dev": true, - "license": "MIT", - "dependencies": { - "semver": "^7.5.3" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/makeerror": { - "version": "1.0.12", - "resolved": "https://registry.npmjs.org/makeerror/-/makeerror-1.0.12.tgz", - "integrity": "sha512-JmqCvUhmt43madlpFzG4BQzG2Z3m6tvQDNKdClZnO3VbIudJYmxsT0FNJMeiB2+JTSlTQTSbU8QdesVmwJcmLg==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "tmpl": "1.0.5" - } - }, - "node_modules/markdown-it": { - "version": "12.3.2", - "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-12.3.2.tgz", - "integrity": "sha512-TchMembfxfNVpHkbtriWltGWc+m3xszaRD0CZup7GFFhzIgQqxIfn3eGj1yZpfuflzPvfkt611B2Q/Bsk1YnGg==", - "dev": true, - "dependencies": { - "argparse": "^2.0.1", - "entities": "~2.1.0", - "linkify-it": "^3.0.1", - "mdurl": "^1.0.1", - "uc.micro": "^1.0.5" - }, - "bin": { - "markdown-it": "bin/markdown-it.js" - } - }, - "node_modules/markdown-it-anchor": { - "version": "8.6.7", - "resolved": "https://registry.npmjs.org/markdown-it-anchor/-/markdown-it-anchor-8.6.7.tgz", - "integrity": "sha512-FlCHFwNnutLgVTflOYHPW2pPcl2AACqVzExlkGQNsi4CJgqOHN7YTgDd4LuhgN1BFO3TS0vLAruV1Td6dwWPJA==", - "dev": true, - "peerDependencies": { - "@types/markdown-it": "*", - "markdown-it": "*" - } - }, - "node_modules/marked": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/marked/-/marked-4.3.0.tgz", - "integrity": "sha512-PRsaiG84bK+AMvxziE/lCFss8juXjNaWzVbN5tXAm4XjeaS9NAHhop+PjQxz2A9h8Q4M/xGmzP8vqNwy6JeK0A==", - "dev": true, - "bin": { - "marked": "bin/marked.js" - }, - "engines": { - "node": ">= 12" - } - }, - "node_modules/mdurl": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-1.0.1.tgz", - "integrity": "sha512-/sKlQJCBYVY9Ers9hqzKou4H6V5UWc/M59TH2dvkt+84itfnq7uFOMLpOiOS4ujvHP4etln18fmIxA5R5fll0g==", - "dev": true - }, - "node_modules/media-typer": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", - "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", - "dev": true, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/memfs": { - "version": "3.5.1", - "resolved": "https://registry.npmjs.org/memfs/-/memfs-3.5.1.tgz", - "integrity": "sha512-UWbFJKvj5k+nETdteFndTpYxdeTMox/ULeqX5k/dpaQJCCFmj5EeKv3dBcyO2xmkRAx2vppRu5dVG7SOtsGOzA==", - "dev": true, - "dependencies": { - "fs-monkey": "^1.0.3" - }, - "engines": { - "node": ">= 4.0.0" - } - }, - "node_modules/merge-descriptors": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", - "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", - "dev": true, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/merge-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", - "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", - "dev": true - }, - "node_modules/methods": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", - "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", - "dev": true, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/micromatch": { - "version": "4.0.8", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", - "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", - "dev": true, - "dependencies": { - "braces": "^3.0.3", - "picomatch": "^2.3.1" - }, - "engines": { - "node": ">=8.6" - } - }, - "node_modules/mime": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", - "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", - "dev": true, - "bin": { - "mime": "cli.js" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "dev": true, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime-types": { - "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "dev": true, - "dependencies": { - "mime-db": "1.52.0" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mimic-fn": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", - "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", - "dev": true, - "engines": { - "node": ">=6" - } - }, - "node_modules/minimalistic-assert": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz", - "integrity": "sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==", - "dev": true - }, - "node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, - "node_modules/minimist": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", - "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", - "dev": true, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/minipass": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", - "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, - "node_modules/minizlib": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.0.1.tgz", - "integrity": "sha512-umcy022ILvb5/3Djuu8LWeqUa8D68JaBzlttKeMWen48SjabqS3iY5w/vzeMzMUNhLDifyhbOwKDSznB1vvrwg==", - "dependencies": { - "minipass": "^7.0.4", - "rimraf": "^5.0.5" - }, - "engines": { - "node": ">= 18" - } - }, - "node_modules/minizlib/node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", - "dependencies": { - "balanced-match": "^1.0.0" - } - }, - "node_modules/minizlib/node_modules/glob": { - "version": "10.4.1", - "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.1.tgz", - "integrity": "sha512-2jelhlq3E4ho74ZyVLN03oKdAZVUa6UDZzFLVH1H7dnoax+y9qyaq8zBkfDIggjniU19z0wU18y16jMB2eyVIw==", - "dependencies": { - "foreground-child": "^3.1.0", - "jackspeak": "^3.1.2", - "minimatch": "^9.0.4", - "minipass": "^7.1.2", - "path-scurry": "^1.11.1" - }, - "bin": { - "glob": "dist/esm/bin.mjs" - }, - "engines": { - "node": ">=16 || 14 >=14.18" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/minizlib/node_modules/minimatch": { - "version": "9.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.4.tgz", - "integrity": "sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw==", - "dependencies": { - "brace-expansion": "^2.0.1" - }, - "engines": { - "node": ">=16 || 14 >=14.17" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/minizlib/node_modules/rimraf": { - "version": "5.0.7", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.7.tgz", - "integrity": "sha512-nV6YcJo5wbLW77m+8KjH8aB/7/rxQy9SZ0HY5shnwULfS+9nmTtVXAJET5NdZmCzA4fPI/Hm1wo/Po/4mopOdg==", - "dependencies": { - "glob": "^10.3.7" - }, - "bin": { - "rimraf": "dist/esm/bin.mjs" - }, - "engines": { - "node": ">=14.18" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/mkdirp": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", - "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", - "dev": true, - "bin": { - "mkdirp": "bin/cmd.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/mkdirp2": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/mkdirp2/-/mkdirp2-1.0.5.tgz", - "integrity": "sha512-xOE9xbICroUDmG1ye2h4bZ8WBie9EGmACaco8K8cx6RlkJJrxGIqjGqztAI+NMhexXBcdGbSEzI6N3EJPevxZw==", - "dev": true - }, - "node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "dev": true - }, - "node_modules/multicast-dns": { - "version": "7.2.5", - "resolved": "https://registry.npmjs.org/multicast-dns/-/multicast-dns-7.2.5.tgz", - "integrity": "sha512-2eznPJP8z2BFLX50tf0LuODrpINqP1RVIm/CObbTcBRITQgmC/TjcREF1NeTBzIcR5XO/ukWo+YHOjBbFwIupg==", - "dev": true, - "dependencies": { - "dns-packet": "^5.2.2", - "thunky": "^1.0.2" - }, - "bin": { - "multicast-dns": "cli.js" - } - }, - "node_modules/natural-compare": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", - "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", - "dev": true, - "license": "MIT" - }, - "node_modules/negotiator": { - "version": "0.6.3", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", - "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", - "dev": true, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/neo-async": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz", - "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", - "dev": true - }, - "node_modules/node-forge": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.1.tgz", - "integrity": "sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==", - "dev": true, - "engines": { - "node": ">= 6.13.0" - } - }, - "node_modules/node-int64": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", - "integrity": "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==", - "dev": true, - "license": "MIT" - }, - "node_modules/node-releases": { - "version": "2.0.18", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.18.tgz", - "integrity": "sha512-d9VeXT4SJ7ZeOqGX6R5EM022wpL+eWPooLI+5UpWn2jCT1aosUQEhQP214x33Wkwx3JQMvIm+tIoVOdodFS40g==", - "dev": true - }, - "node_modules/normalize-path": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", - "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/npm-run-path": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", - "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", - "dev": true, - "dependencies": { - "path-key": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/object-get": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/object-get/-/object-get-2.1.1.tgz", - "integrity": "sha512-7n4IpLMzGGcLEMiQKsNR7vCe+N5E9LORFrtNUVy4sO3dj9a3HedZCxEL2T7QuLhcHN1NBuBsMOKaOsAYI9IIvg==", - "dev": true - }, - "node_modules/object-inspect": { - "version": "1.13.2", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.2.tgz", - "integrity": "sha512-IRZSRuzJiynemAXPYtPe5BoI/RESNYR7TYm50MC5Mqbd3Jmw5y790sErYw3V6SryFJD64b74qQQs9wn5Bg/k3g==", - "dev": true, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/object-to-spawn-args": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/object-to-spawn-args/-/object-to-spawn-args-2.0.1.tgz", - "integrity": "sha512-6FuKFQ39cOID+BMZ3QaphcC8Y4cw6LXBLyIgPU+OhIYwviJamPAn+4mITapnSBQrejB+NNp+FMskhD8Cq+Ys3w==", - "dev": true, - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/obuf": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/obuf/-/obuf-1.1.2.tgz", - "integrity": "sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==", - "dev": true - }, - "node_modules/on-finished": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", - "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", - "dev": true, - "dependencies": { - "ee-first": "1.1.1" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/on-headers": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.2.tgz", - "integrity": "sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA==", - "dev": true, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/once": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "dev": true, - "dependencies": { - "wrappy": "1" - } - }, - "node_modules/onetime": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", - "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", - "dev": true, - "dependencies": { - "mimic-fn": "^2.1.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/onnxruntime-common": { - "version": "1.19.2", - "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.19.2.tgz", - "integrity": "sha512-a4R7wYEVFbZBlp0BfhpbFWqe4opCor3KM+5Wm22Az3NGDcQMiU2hfG/0MfnBs+1ZrlSGmlgWeMcXQkDk1UFb8Q==" - }, - "node_modules/onnxruntime-node": { - "version": "1.19.2", - "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.19.2.tgz", - "integrity": "sha512-9eHMP/HKbbeUcqte1JYzaaRC8JPn7ojWeCeoyShO86TOR97OCyIyAIOGX3V95ErjslVhJRXY8Em/caIUc0hm1Q==", - "hasInstallScript": true, - "os": [ - "win32", - "darwin", - "linux" - ], - "dependencies": { - "onnxruntime-common": "1.19.2", - "tar": "^7.0.1" - } - }, - "node_modules/onnxruntime-web": { - "version": "1.21.0-dev.20241024-d9ca84ef96", - "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.21.0-dev.20241024-d9ca84ef96.tgz", - "integrity": "sha512-ANSQfMALvCviN3Y4tvTViKofKToV1WUb2r2VjZVCi3uUBPaK15oNJyIxhsNyEckBr/Num3JmSXlkHOD8HfVzSQ==", - "license": "MIT", - "dependencies": { - "flatbuffers": "^1.12.0", - "guid-typescript": "^1.0.9", - "long": "^5.2.3", - "onnxruntime-common": "1.20.0-dev.20241016-2b8fc5529b", - "platform": "^1.3.6", - "protobufjs": "^7.2.4" - } - }, - "node_modules/onnxruntime-web/node_modules/onnxruntime-common": { - "version": "1.20.0-dev.20241016-2b8fc5529b", - "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.20.0-dev.20241016-2b8fc5529b.tgz", - "integrity": "sha512-KZK8b6zCYGZFjd4ANze0pqBnqnFTS3GIVeclQpa2qseDpXrCQJfkWBixRcrZShNhm3LpFOZ8qJYFC5/qsJK9WQ==" - }, - "node_modules/open": { - "version": "8.4.2", - "resolved": "https://registry.npmjs.org/open/-/open-8.4.2.tgz", - "integrity": "sha512-7x81NCL719oNbsq/3mh+hVrAWmFuEYUqrq/Iw3kUzH8ReypT9QQ0BLoJS7/G9k6N81XjW4qHWtjWwe/9eLy1EQ==", - "dev": true, - "dependencies": { - "define-lazy-prop": "^2.0.0", - "is-docker": "^2.1.1", - "is-wsl": "^2.2.0" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-limit": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", - "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", - "dev": true, - "dependencies": { - "p-try": "^2.0.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-locate": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", - "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", - "dev": true, - "dependencies": { - "p-limit": "^2.2.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/p-retry": { - "version": "4.6.2", - "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz", - "integrity": "sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==", - "dev": true, - "dependencies": { - "@types/retry": "0.12.0", - "retry": "^0.13.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/p-try": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", - "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", - "dev": true, - "engines": { - "node": ">=6" - } - }, - "node_modules/package-json-from-dist": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", - "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", - "dev": true, - "license": "BlueOak-1.0.0" - }, - "node_modules/parse-json": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", - "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.0.0", - "error-ex": "^1.3.1", - "json-parse-even-better-errors": "^2.3.0", - "lines-and-columns": "^1.1.6" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/parseurl": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", - "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", - "dev": true, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/path-exists": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", - "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/path-is-absolute": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/path-key": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", - "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", - "engines": { - "node": ">=8" - } - }, - "node_modules/path-parse": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", - "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", - "dev": true - }, - "node_modules/path-scurry": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", - "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", - "dependencies": { - "lru-cache": "^10.2.0", - "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" - }, - "engines": { - "node": ">=16 || 14 >=14.18" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/path-scurry/node_modules/lru-cache": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.2.tgz", - "integrity": "sha512-9hp3Vp2/hFQUiIwKo8XCeFVnrg8Pk3TYNPIR7tJADKi5YfcF7vEaK7avFHTlSy3kOKYaJQaalfEo6YuXdceBOQ==", - "engines": { - "node": "14 || >=16.14" - } - }, - "node_modules/path-to-regexp": { - "version": "0.1.10", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.10.tgz", - "integrity": "sha512-7lf7qcQidTku0Gu3YDPc8DJ1q7OOucfa/BSsIwjuh56VU7katFvuM8hULfkwB3Fns/rsVF7PwPKVw1sl5KQS9w==", - "dev": true - }, - "node_modules/picocolors": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", - "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", - "dev": true, - "license": "ISC" - }, - "node_modules/picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", - "dev": true, - "engines": { - "node": ">=8.6" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/pirates": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.6.tgz", - "integrity": "sha512-saLsH7WeYYPiD25LDuLRRY/i+6HaPYr6G1OUlN39otzkSTxKnubR9RTxS3/Kk50s1g2JTgFwWQDQyplC5/SHZg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 6" - } - }, - "node_modules/pkg-dir": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", - "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", - "dev": true, - "dependencies": { - "find-up": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/platform": { - "version": "1.3.6", - "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz", - "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==" - }, - "node_modules/prettier": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.3.3.tgz", - "integrity": "sha512-i2tDNA0O5IrMO757lfrdQZCc2jPNDVntV0m/+4whiDfWaTKfMNgR7Qz0NAeGz/nRqF4m5/6CLzbP4/liHt12Ew==", - "dev": true, - "bin": { - "prettier": "bin/prettier.cjs" - }, - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/prettier/prettier?sponsor=1" - } - }, - "node_modules/pretty-format": { - "version": "29.6.1", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.6.1.tgz", - "integrity": "sha512-7jRj+yXO0W7e4/tSJKoR7HRIHLPPjtNaUGG2xxKQnGvPNRkgWcQ0AZX6P4KBRJN4FcTBWb3sa7DVUJmocYuoog==", - "dev": true, - "dependencies": { - "@jest/schemas": "^29.6.0", - "ansi-styles": "^5.0.0", - "react-is": "^18.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/pretty-format/node_modules/ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "dev": true, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/process-nextick-args": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", - "dev": true - }, - "node_modules/protobufjs": { - "version": "7.2.6", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.2.6.tgz", - "integrity": "sha512-dgJaEDDL6x8ASUZ1YqWciTRrdOuYNzoOf27oHNfdyvKqHr5i0FV7FSLU+aIeFjyFgVxrpTOtQUi0BLLBymZaBw==", - "hasInstallScript": true, - "dependencies": { - "@protobufjs/aspromise": "^1.1.2", - "@protobufjs/base64": "^1.1.2", - "@protobufjs/codegen": "^2.0.4", - "@protobufjs/eventemitter": "^1.1.0", - "@protobufjs/fetch": "^1.1.0", - "@protobufjs/float": "^1.0.2", - "@protobufjs/inquire": "^1.1.0", - "@protobufjs/path": "^1.1.2", - "@protobufjs/pool": "^1.1.0", - "@protobufjs/utf8": "^1.1.0", - "@types/node": ">=13.7.0", - "long": "^5.0.0" - }, - "engines": { - "node": ">=12.0.0" - } - }, - "node_modules/proxy-addr": { - "version": "2.0.7", - "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", - "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", - "dev": true, - "dependencies": { - "forwarded": "0.2.0", - "ipaddr.js": "1.9.1" - }, - "engines": { - "node": ">= 0.10" - } - }, - "node_modules/proxy-addr/node_modules/ipaddr.js": { - "version": "1.9.1", - "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", - "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", - "dev": true, - "engines": { - "node": ">= 0.10" - } - }, - "node_modules/punycode": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz", - "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==", - "dev": true, - "engines": { - "node": ">=6" - } - }, - "node_modules/pure-rand": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-6.1.0.tgz", - "integrity": "sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA==", - "dev": true, - "funding": [ - { - "type": "individual", - "url": "https://github.com/sponsors/dubzzz" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fast-check" - } - ], - "license": "MIT" - }, - "node_modules/qs": { - "version": "6.13.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", - "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", - "dev": true, - "dependencies": { - "side-channel": "^1.0.6" - }, - "engines": { - "node": ">=0.6" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/randombytes": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", - "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", - "dev": true, - "dependencies": { - "safe-buffer": "^5.1.0" - } - }, - "node_modules/range-parser": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", - "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", - "dev": true, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/raw-body": { - "version": "2.5.2", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", - "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", - "dev": true, - "dependencies": { - "bytes": "3.1.2", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "unpipe": "1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/raw-body/node_modules/bytes": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", - "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", - "dev": true, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/react-is": { - "version": "18.2.0", - "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.2.0.tgz", - "integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==", - "dev": true - }, - "node_modules/readable-stream": { - "version": "3.6.1", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.1.tgz", - "integrity": "sha512-+rQmrWMYGA90yenhTYsLWAsLsqVC8osOw6PKE1HDYiO0gdPeKe/xDHNzIAIn4C91YQ6oenEhfYqqc1883qHbjQ==", - "dev": true, - "dependencies": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/readdirp": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", - "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", - "dev": true, - "dependencies": { - "picomatch": "^2.2.1" - }, - "engines": { - "node": ">=8.10.0" - } - }, - "node_modules/rechoir": { - "version": "0.8.0", - "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.8.0.tgz", - "integrity": "sha512-/vxpCXddiX8NGfGO/mTafwjq4aFa/71pvamip0++IQk3zG8cbCj0fifNPrjjF1XMXUne91jL9OoxmdykoEtifQ==", - "dev": true, - "dependencies": { - "resolve": "^1.20.0" - }, - "engines": { - "node": ">= 10.13.0" - } - }, - "node_modules/reduce-extract": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/reduce-extract/-/reduce-extract-1.0.0.tgz", - "integrity": "sha512-QF8vjWx3wnRSL5uFMyCjDeDc5EBMiryoT9tz94VvgjKfzecHAVnqmXAwQDcr7X4JmLc2cjkjFGCVzhMqDjgR9g==", - "dev": true, - "dependencies": { - "test-value": "^1.0.1" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/reduce-extract/node_modules/array-back": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/array-back/-/array-back-1.0.4.tgz", - "integrity": "sha512-1WxbZvrmyhkNoeYcizokbmh5oiOCIfyvGtcqbK3Ls1v1fKcquzxnQSceOx6tzq7jmai2kFLWIpGND2cLhH6TPw==", - "dev": true, - "dependencies": { - "typical": "^2.6.0" - }, - "engines": { - "node": ">=0.12.0" - } - }, - "node_modules/reduce-extract/node_modules/test-value": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/test-value/-/test-value-1.1.0.tgz", - "integrity": "sha512-wrsbRo7qP+2Je8x8DsK8ovCGyxe3sYfQwOraIY/09A2gFXU9DYKiTF14W4ki/01AEh56kMzAmlj9CaHGDDUBJA==", - "dev": true, - "dependencies": { - "array-back": "^1.0.2", - "typical": "^2.4.2" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/reduce-flatten": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/reduce-flatten/-/reduce-flatten-3.0.1.tgz", - "integrity": "sha512-bYo+97BmUUOzg09XwfkwALt4PQH1M5L0wzKerBt6WLm3Fhdd43mMS89HiT1B9pJIqko/6lWx3OnV4J9f2Kqp5Q==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/reduce-unique": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/reduce-unique/-/reduce-unique-2.0.1.tgz", - "integrity": "sha512-x4jH/8L1eyZGR785WY+ePtyMNhycl1N2XOLxhCbzZFaqF4AXjLzqSxa2UHgJ2ZVR/HHyPOvl1L7xRnW8ye5MdA==", - "dev": true, - "engines": { - "node": ">=6" - } - }, - "node_modules/reduce-without": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/reduce-without/-/reduce-without-1.0.1.tgz", - "integrity": "sha512-zQv5y/cf85sxvdrKPlfcRzlDn/OqKFThNimYmsS3flmkioKvkUGn2Qg9cJVoQiEvdxFGLE0MQER/9fZ9sUqdxg==", - "dev": true, - "dependencies": { - "test-value": "^2.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/reduce-without/node_modules/array-back": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/array-back/-/array-back-1.0.4.tgz", - "integrity": "sha512-1WxbZvrmyhkNoeYcizokbmh5oiOCIfyvGtcqbK3Ls1v1fKcquzxnQSceOx6tzq7jmai2kFLWIpGND2cLhH6TPw==", - "dev": true, - "dependencies": { - "typical": "^2.6.0" - }, - "engines": { - "node": ">=0.12.0" - } - }, - "node_modules/reduce-without/node_modules/test-value": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/test-value/-/test-value-2.1.0.tgz", - "integrity": "sha512-+1epbAxtKeXttkGFMTX9H42oqzOTufR1ceCF+GYA5aOmvaPq9wd4PUS8329fn2RRLGNeUkgRLnVpycjx8DsO2w==", - "dev": true, - "dependencies": { - "array-back": "^1.0.3", - "typical": "^2.6.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/require-directory": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", - "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/require-from-string": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", - "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/requires-port": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", - "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==", - "dev": true - }, - "node_modules/requizzle": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/requizzle/-/requizzle-0.2.4.tgz", - "integrity": "sha512-JRrFk1D4OQ4SqovXOgdav+K8EAhSB/LJZqCz8tbX0KObcdeM15Ss59ozWMBWmmINMagCwmqn4ZNryUGpBsl6Jw==", - "dev": true, - "dependencies": { - "lodash": "^4.17.21" - } - }, - "node_modules/resolve": { - "version": "1.22.2", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.2.tgz", - "integrity": "sha512-Sb+mjNHOULsBv818T40qSPeRiuWLyaGMa5ewydRLFimneixmVy2zdivRl+AF6jaYPC8ERxGDmFSiqui6SfPd+g==", - "dev": true, - "dependencies": { - "is-core-module": "^2.11.0", - "path-parse": "^1.0.7", - "supports-preserve-symlinks-flag": "^1.0.0" - }, - "bin": { - "resolve": "bin/resolve" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/resolve-cwd": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-3.0.0.tgz", - "integrity": "sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==", - "dev": true, - "dependencies": { - "resolve-from": "^5.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/resolve-from": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", - "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/resolve.exports": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/resolve.exports/-/resolve.exports-2.0.2.tgz", - "integrity": "sha512-X2UW6Nw3n/aMgDVy+0rSqgHlv39WZAlZrXCdnbyEiKm17DSqHX4MmQMaST3FbeWR5FTuRcUwYAziZajji0Y7mg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - } - }, - "node_modules/retry": { - "version": "0.13.1", - "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", - "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", - "dev": true, - "engines": { - "node": ">= 4" - } - }, - "node_modules/rimraf": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", - "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", - "dev": true, - "dependencies": { - "glob": "^7.1.3" - }, - "bin": { - "rimraf": "bin.js" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] - }, - "node_modules/safer-buffer": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", - "dev": true - }, - "node_modules/schema-utils": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.3.0.tgz", - "integrity": "sha512-pN/yOAvcC+5rQ5nERGuwrjLlYvLTbCibnZ1I7B1LaiAz9BRBlE9GMgE/eqV30P7aJQUf7Ddimy/RsbYO/GrVGg==", - "dev": true, - "dependencies": { - "@types/json-schema": "^7.0.8", - "ajv": "^6.12.5", - "ajv-keywords": "^3.5.2" - }, - "engines": { - "node": ">= 10.13.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/webpack" - } - }, - "node_modules/select-hose": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/select-hose/-/select-hose-2.0.0.tgz", - "integrity": "sha512-mEugaLK+YfkijB4fx0e6kImuJdCIt2LxCRcbEYPqRGCs4F2ogyfZU5IAZRdjCP8JPq2AtdNoC/Dux63d9Kiryg==", - "dev": true - }, - "node_modules/selfsigned": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/selfsigned/-/selfsigned-2.1.1.tgz", - "integrity": "sha512-GSL3aowiF7wa/WtSFwnUrludWFoNhftq8bUkH9pkzjpN2XSPOAYEgg6e0sS9s0rZwgJzJiQRPU18A6clnoW5wQ==", - "dev": true, - "dependencies": { - "node-forge": "^1" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/semver": { - "version": "7.6.3", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", - "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/send": { - "version": "0.19.0", - "resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz", - "integrity": "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==", - "dev": true, - "dependencies": { - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "1.2.0", - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "fresh": "0.5.2", - "http-errors": "2.0.0", - "mime": "1.6.0", - "ms": "2.1.3", - "on-finished": "2.4.1", - "range-parser": "~1.2.1", - "statuses": "2.0.1" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/send/node_modules/encodeurl": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", - "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", - "dev": true, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/send/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true - }, - "node_modules/serialize-javascript": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.2.tgz", - "integrity": "sha512-Saa1xPByTTq2gdeFZYLLo+RFE35NHZkAbqZeWNd3BpzppeVisAqpDjcp8dyf6uIvEqJRd46jemmyA4iFIeVk8g==", - "dev": true, - "dependencies": { - "randombytes": "^2.1.0" - } - }, - "node_modules/serve-index": { - "version": "1.9.1", - "resolved": "https://registry.npmjs.org/serve-index/-/serve-index-1.9.1.tgz", - "integrity": "sha512-pXHfKNP4qujrtteMrSBb0rc8HJ9Ms/GrXwcUtUtD5s4ewDJI8bT3Cz2zTVRMKtri49pLx2e0Ya8ziP5Ya2pZZw==", - "dev": true, - "dependencies": { - "accepts": "~1.3.4", - "batch": "0.6.1", - "debug": "2.6.9", - "escape-html": "~1.0.3", - "http-errors": "~1.6.2", - "mime-types": "~2.1.17", - "parseurl": "~1.3.2" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/serve-index/node_modules/depd": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", - "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==", - "dev": true, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/serve-index/node_modules/http-errors": { - "version": "1.6.3", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", - "integrity": "sha512-lks+lVC8dgGyh97jxvxeYTWQFvh4uw4yC12gVl63Cg30sjPX4wuGcdkICVXDAESr6OJGjqGA8Iz5mkeN6zlD7A==", - "dev": true, - "dependencies": { - "depd": "~1.1.2", - "inherits": "2.0.3", - "setprototypeof": "1.1.0", - "statuses": ">= 1.4.0 < 2" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/serve-index/node_modules/inherits": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", - "integrity": "sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==", - "dev": true - }, - "node_modules/serve-index/node_modules/setprototypeof": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz", - "integrity": "sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ==", - "dev": true - }, - "node_modules/serve-index/node_modules/statuses": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", - "integrity": "sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==", - "dev": true, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/serve-static": { - "version": "1.16.2", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.2.tgz", - "integrity": "sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==", - "dev": true, - "dependencies": { - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "parseurl": "~1.3.3", - "send": "0.19.0" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/set-function-length": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", - "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==", - "dev": true, - "dependencies": { - "define-data-property": "^1.1.4", - "es-errors": "^1.3.0", - "function-bind": "^1.1.2", - "get-intrinsic": "^1.2.4", - "gopd": "^1.0.1", - "has-property-descriptors": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/setprototypeof": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", - "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", - "dev": true - }, - "node_modules/shallow-clone": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-3.0.1.tgz", - "integrity": "sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA==", - "dev": true, - "dependencies": { - "kind-of": "^6.0.2" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/sharp": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.33.5.tgz", - "integrity": "sha512-haPVm1EkS9pgvHrQ/F3Xy+hgcuMV0Wm9vfIBSiwZ05k+xgb0PkBQpGsAA/oWdDobNaZTH5ppvHtzCFbnSEwHVw==", - "hasInstallScript": true, - "dependencies": { - "color": "^4.2.3", - "detect-libc": "^2.0.3", - "semver": "^7.6.3" - }, - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-darwin-arm64": "0.33.5", - "@img/sharp-darwin-x64": "0.33.5", - "@img/sharp-libvips-darwin-arm64": "1.0.4", - "@img/sharp-libvips-darwin-x64": "1.0.4", - "@img/sharp-libvips-linux-arm": "1.0.5", - "@img/sharp-libvips-linux-arm64": "1.0.4", - "@img/sharp-libvips-linux-s390x": "1.0.4", - "@img/sharp-libvips-linux-x64": "1.0.4", - "@img/sharp-libvips-linuxmusl-arm64": "1.0.4", - "@img/sharp-libvips-linuxmusl-x64": "1.0.4", - "@img/sharp-linux-arm": "0.33.5", - "@img/sharp-linux-arm64": "0.33.5", - "@img/sharp-linux-s390x": "0.33.5", - "@img/sharp-linux-x64": "0.33.5", - "@img/sharp-linuxmusl-arm64": "0.33.5", - "@img/sharp-linuxmusl-x64": "0.33.5", - "@img/sharp-wasm32": "0.33.5", - "@img/sharp-win32-ia32": "0.33.5", - "@img/sharp-win32-x64": "0.33.5" - } - }, - "node_modules/shebang-command": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", - "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "dependencies": { - "shebang-regex": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/shebang-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", - "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", - "engines": { - "node": ">=8" - } - }, - "node_modules/shell-quote": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.8.1.tgz", - "integrity": "sha512-6j1W9l1iAs/4xYBI1SYOVZyFcCis9b4KCLQ8fgAGG07QvzaRLVVRQvAy85yNmmZSjYjg4MWh4gNvlPujU/5LpA==", - "dev": true, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/side-channel": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz", - "integrity": "sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==", - "dev": true, - "dependencies": { - "call-bind": "^1.0.7", - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.4", - "object-inspect": "^1.13.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/signal-exit": { - "version": "3.0.7", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", - "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", - "dev": true - }, - "node_modules/simple-swizzle": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", - "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==", - "dependencies": { - "is-arrayish": "^0.3.1" - } - }, - "node_modules/slash": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", - "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/sockjs": { - "version": "0.3.24", - "resolved": "https://registry.npmjs.org/sockjs/-/sockjs-0.3.24.tgz", - "integrity": "sha512-GJgLTZ7vYb/JtPSSZ10hsOYIvEYsjbNU+zPdIHcUaWVNUEPivzxku31865sSSud0Da0W4lEeOPlmw93zLQchuQ==", - "dev": true, - "dependencies": { - "faye-websocket": "^0.11.3", - "uuid": "^8.3.2", - "websocket-driver": "^0.7.4" - } - }, - "node_modules/sort-array": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/sort-array/-/sort-array-4.1.5.tgz", - "integrity": "sha512-Ya4peoS1fgFN42RN1REk2FgdNOeLIEMKFGJvs7VTP3OklF8+kl2SkpVliZ4tk/PurWsrWRsdNdU+tgyOBkB9sA==", - "dev": true, - "dependencies": { - "array-back": "^5.0.0", - "typical": "^6.0.1" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/sort-array/node_modules/array-back": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/array-back/-/array-back-5.0.0.tgz", - "integrity": "sha512-kgVWwJReZWmVuWOQKEOohXKJX+nD02JAZ54D1RRWlv8L0NebauKAaFxACKzB74RTclt1+WNz5KHaLRDAPZbDEw==", - "dev": true, - "engines": { - "node": ">=10" - } - }, - "node_modules/sort-array/node_modules/typical": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/typical/-/typical-6.0.1.tgz", - "integrity": "sha512-+g3NEp7fJLe9DPa1TArHm9QAA7YciZmWnfAqEaFrBihQ7epOv9i99rjtgb6Iz0wh3WuQDjsCTDfgRoGnmHN81A==", - "dev": true, - "engines": { - "node": ">=10" - } - }, - "node_modules/source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/source-map-support": { - "version": "0.5.21", - "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", - "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", - "dev": true, - "dependencies": { - "buffer-from": "^1.0.0", - "source-map": "^0.6.0" - } - }, - "node_modules/spdy": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/spdy/-/spdy-4.0.2.tgz", - "integrity": "sha512-r46gZQZQV+Kl9oItvl1JZZqJKGr+oEkB08A6BzkiR7593/7IbtuncXHd2YoYeTsG4157ZssMu9KYvUHLcjcDoA==", - "dev": true, - "dependencies": { - "debug": "^4.1.0", - "handle-thing": "^2.0.0", - "http-deceiver": "^1.2.7", - "select-hose": "^2.0.0", - "spdy-transport": "^3.0.0" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/spdy-transport": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/spdy-transport/-/spdy-transport-3.0.0.tgz", - "integrity": "sha512-hsLVFE5SjA6TCisWeJXFKniGGOpBgMLmerfO2aCyCU5s7nJ/rpAepqmFifv/GCbSbueEeAJJnmSQ2rKC/g8Fcw==", - "dev": true, - "dependencies": { - "debug": "^4.1.0", - "detect-node": "^2.0.4", - "hpack.js": "^2.1.6", - "obuf": "^1.1.2", - "readable-stream": "^3.0.6", - "wbuf": "^1.7.3" - } - }, - "node_modules/spdy-transport/node_modules/debug": { - "version": "4.3.4", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", - "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", - "dev": true, - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/spdy-transport/node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - }, - "node_modules/spdy/node_modules/debug": { - "version": "4.3.4", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", - "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", - "dev": true, - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/spdy/node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - }, - "node_modules/sprintf-js": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", - "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", - "dev": true, - "license": "BSD-3-Clause" - }, - "node_modules/stack-utils": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-2.0.6.tgz", - "integrity": "sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ==", - "dev": true, - "dependencies": { - "escape-string-regexp": "^2.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/statuses": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", - "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", - "dev": true, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/stream-connect": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/stream-connect/-/stream-connect-1.0.2.tgz", - "integrity": "sha512-68Kl+79cE0RGKemKkhxTSg8+6AGrqBt+cbZAXevg2iJ6Y3zX4JhA/sZeGzLpxW9cXhmqAcE7KnJCisUmIUfnFQ==", - "dev": true, - "dependencies": { - "array-back": "^1.0.2" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/stream-connect/node_modules/array-back": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/array-back/-/array-back-1.0.4.tgz", - "integrity": "sha512-1WxbZvrmyhkNoeYcizokbmh5oiOCIfyvGtcqbK3Ls1v1fKcquzxnQSceOx6tzq7jmai2kFLWIpGND2cLhH6TPw==", - "dev": true, - "dependencies": { - "typical": "^2.6.0" - }, - "engines": { - "node": ">=0.12.0" - } - }, - "node_modules/stream-via": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/stream-via/-/stream-via-1.0.4.tgz", - "integrity": "sha512-DBp0lSvX5G9KGRDTkR/R+a29H+Wk2xItOF+MpZLLNDWbEV9tGPnqLPxHEYjmiz8xGtJHRIqmI+hCjmNzqoA4nQ==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/string_decoder": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", - "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", - "dev": true, - "dependencies": { - "safe-buffer": "~5.2.0" - } - }, - "node_modules/string-length": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/string-length/-/string-length-4.0.2.tgz", - "integrity": "sha512-+l6rNN5fYHNhZZy41RXsYptCjA2Igmq4EG7kZAYFQI1E1VTXarr6ZPXBg6eq7Y6eK4FEhY6AJlyuFIb/v/S0VQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "char-regex": "^1.0.2", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/string-width-cjs": { - "name": "string-width", - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi-cjs": { - "name": "strip-ansi", - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-bom": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-4.0.0.tgz", - "integrity": "sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-final-newline": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", - "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", - "dev": true, - "engines": { - "node": ">=6" - } - }, - "node_modules/strip-json-comments": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", - "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/supports-color": { - "version": "8.1.1", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", - "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", - "dev": true, - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/supports-color?sponsor=1" - } - }, - "node_modules/supports-preserve-symlinks-flag": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", - "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", - "dev": true, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/synckit": { - "version": "0.9.2", - "resolved": "https://registry.npmjs.org/synckit/-/synckit-0.9.2.tgz", - "integrity": "sha512-vrozgXDQwYO72vHjUb/HnFbQx1exDjoKzqx23aXEg2a9VIg2TSFZ8FmeZpTjUCFMYw7mpX4BE2SFu8wI7asYsw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@pkgr/core": "^0.1.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": "^14.18.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/unts" - } - }, - "node_modules/table-layout": { - "version": "0.4.5", - "resolved": "https://registry.npmjs.org/table-layout/-/table-layout-0.4.5.tgz", - "integrity": "sha512-zTvf0mcggrGeTe/2jJ6ECkJHAQPIYEwDoqsiqBjI24mvRmQbInK5jq33fyypaCBxX08hMkfmdOqj6haT33EqWw==", - "dev": true, - "dependencies": { - "array-back": "^2.0.0", - "deep-extend": "~0.6.0", - "lodash.padend": "^4.6.1", - "typical": "^2.6.1", - "wordwrapjs": "^3.0.0" - }, - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/table-layout/node_modules/array-back": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/array-back/-/array-back-2.0.0.tgz", - "integrity": "sha512-eJv4pLLufP3g5kcZry0j6WXpIbzYw9GUB4mVJZno9wfwiBxbizTnHCw3VJb07cBihbFX48Y7oSrW9y+gt4glyw==", - "dev": true, - "dependencies": { - "typical": "^2.6.1" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/tapable": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.2.1.tgz", - "integrity": "sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ==", - "dev": true, - "engines": { - "node": ">=6" - } - }, - "node_modules/tar": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/tar/-/tar-7.2.0.tgz", - "integrity": "sha512-hctwP0Nb4AB60bj8WQgRYaMOuJYRAPMGiQUAotms5igN8ppfQM+IvjQ5HcKu1MaZh2Wy2KWVTe563Yj8dfc14w==", - "dependencies": { - "@isaacs/fs-minipass": "^4.0.0", - "chownr": "^3.0.0", - "minipass": "^7.1.0", - "minizlib": "^3.0.1", - "mkdirp": "^3.0.1", - "yallist": "^5.0.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/tar/node_modules/mkdirp": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz", - "integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==", - "bin": { - "mkdirp": "dist/cjs/src/bin.js" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/tar/node_modules/yallist": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", - "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==", - "engines": { - "node": ">=18" - } - }, - "node_modules/temp-path": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/temp-path/-/temp-path-1.0.0.tgz", - "integrity": "sha512-TvmyH7kC6ZVTYkqCODjJIbgvu0FKiwQpZ4D1aknE7xpcDf/qEOB8KZEK5ef2pfbVoiBhNWs3yx4y+ESMtNYmlg==", - "dev": true - }, - "node_modules/terser": { - "version": "5.31.6", - "resolved": "https://registry.npmjs.org/terser/-/terser-5.31.6.tgz", - "integrity": "sha512-PQ4DAriWzKj+qgehQ7LK5bQqCFNMmlhjR2PFFLuqGCpuCAauxemVBWwWOxo3UIwWQx8+Pr61Df++r76wDmkQBg==", - "dev": true, - "dependencies": { - "@jridgewell/source-map": "^0.3.3", - "acorn": "^8.8.2", - "commander": "^2.20.0", - "source-map-support": "~0.5.20" - }, - "bin": { - "terser": "bin/terser" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/terser-webpack-plugin": { - "version": "5.3.10", - "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.3.10.tgz", - "integrity": "sha512-BKFPWlPDndPs+NGGCr1U59t0XScL5317Y0UReNrHaw9/FwhPENlq6bfgs+4yPfyP51vqC1bQ4rp1EfXW5ZSH9w==", - "dev": true, - "dependencies": { - "@jridgewell/trace-mapping": "^0.3.20", - "jest-worker": "^27.4.5", - "schema-utils": "^3.1.1", - "serialize-javascript": "^6.0.1", - "terser": "^5.26.0" - }, - "engines": { - "node": ">= 10.13.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/webpack" - }, - "peerDependencies": { - "webpack": "^5.1.0" - }, - "peerDependenciesMeta": { - "@swc/core": { - "optional": true - }, - "esbuild": { - "optional": true - }, - "uglify-js": { - "optional": true - } - } - }, - "node_modules/test-exclude": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", - "integrity": "sha512-cAGWPIyOHU6zlmg88jwm7VRyXnMN7iV68OGAbYDk/Mh/xC/pzVPlQtY6ngoIH/5/tciuhGfvESU8GrHrcxD56w==", - "dev": true, - "license": "ISC", - "dependencies": { - "@istanbuljs/schema": "^0.1.2", - "glob": "^7.1.4", - "minimatch": "^3.0.4" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/test-value": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/test-value/-/test-value-3.0.0.tgz", - "integrity": "sha512-sVACdAWcZkSU9x7AOmJo5TqE+GyNJknHaHsMrR6ZnhjVlVN9Yx6FjHrsKZ3BjIpPCT68zYesPWkakrNupwfOTQ==", - "dev": true, - "dependencies": { - "array-back": "^2.0.0", - "typical": "^2.6.1" - }, - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/test-value/node_modules/array-back": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/array-back/-/array-back-2.0.0.tgz", - "integrity": "sha512-eJv4pLLufP3g5kcZry0j6WXpIbzYw9GUB4mVJZno9wfwiBxbizTnHCw3VJb07cBihbFX48Y7oSrW9y+gt4glyw==", - "dev": true, - "dependencies": { - "typical": "^2.6.1" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/thunky": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/thunky/-/thunky-1.1.0.tgz", - "integrity": "sha512-eHY7nBftgThBqOyHGVN+l8gF0BucP09fMo0oO/Lb0w1OF80dJv+lDVpXG60WMQvkcxAkNybKsrEIE3ZtKGmPrA==", - "dev": true - }, - "node_modules/tmpl": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.5.tgz", - "integrity": "sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw==", - "dev": true, - "license": "BSD-3-Clause" - }, - "node_modules/to-regex-range": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", - "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "dev": true, - "dependencies": { - "is-number": "^7.0.0" - }, - "engines": { - "node": ">=8.0" - } - }, - "node_modules/toidentifier": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", - "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", - "dev": true, - "engines": { - "node": ">=0.6" - } - }, - "node_modules/tslib": { - "version": "2.6.3", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz", - "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==", - "devOptional": true - }, - "node_modules/type-detect": { - "version": "4.0.8", - "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", - "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/type-fest": { - "version": "0.21.3", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", - "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", - "dev": true, - "license": "(MIT OR CC0-1.0)", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/type-is": { - "version": "1.6.18", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", - "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", - "dev": true, - "dependencies": { - "media-typer": "0.3.0", - "mime-types": "~2.1.24" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/typescript": { - "version": "5.2.2", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.2.2.tgz", - "integrity": "sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==", - "dev": true, - "bin": { - "tsc": "bin/tsc", - "tsserver": "bin/tsserver" - }, - "engines": { - "node": ">=14.17" - } - }, - "node_modules/typical": { - "version": "2.6.1", - "resolved": "https://registry.npmjs.org/typical/-/typical-2.6.1.tgz", - "integrity": "sha512-ofhi8kjIje6npGozTip9Fr8iecmYfEbS06i0JnIg+rh51KakryWF4+jX8lLKZVhy6N+ID45WYSFCxPOdTWCzNg==", - "dev": true - }, - "node_modules/uc.micro": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-1.0.6.tgz", - "integrity": "sha512-8Y75pvTYkLJW2hWQHXxoqRgV7qb9B+9vFEtidML+7koHUFapnVJAZ6cKs+Qjz5Aw3aZWHMC6u0wJE3At+nSGwA==", - "dev": true - }, - "node_modules/uglify-js": { - "version": "3.17.4", - "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.17.4.tgz", - "integrity": "sha512-T9q82TJI9e/C1TAxYvfb16xO120tMVFZrGA3f9/P4424DNu6ypK103y0GPFVa17yotwSyZW5iYXgjYHkGrJW/g==", - "dev": true, - "optional": true, - "bin": { - "uglifyjs": "bin/uglifyjs" - }, - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/underscore": { - "version": "1.13.6", - "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.6.tgz", - "integrity": "sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==", - "dev": true - }, - "node_modules/unpipe": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", - "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", - "dev": true, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/update-browserslist-db": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.1.tgz", - "integrity": "sha512-R8UzCaa9Az+38REPiJ1tXlImTJXlVfgHZsglwBD/k6nj76ctsH1E3q4doGrukiLQd3sGQYu56r5+lo5r94l29A==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/browserslist" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/browserslist" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "dependencies": { - "escalade": "^3.2.0", - "picocolors": "^1.1.0" - }, - "bin": { - "update-browserslist-db": "cli.js" - }, - "peerDependencies": { - "browserslist": ">= 4.21.0" - } - }, - "node_modules/uri-js": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", - "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", - "dev": true, - "dependencies": { - "punycode": "^2.1.0" - } - }, - "node_modules/util-deprecate": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", - "dev": true - }, - "node_modules/utils-merge": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", - "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", - "dev": true, - "engines": { - "node": ">= 0.4.0" - } - }, - "node_modules/uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", - "dev": true, - "bin": { - "uuid": "dist/bin/uuid" - } - }, - "node_modules/v8-to-istanbul": { - "version": "9.3.0", - "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.3.0.tgz", - "integrity": "sha512-kiGUalWN+rgBJ/1OHZsBtU4rXZOfj/7rKQxULKlIzwzQSvMJUUNgPwJEEh7gU6xEVxC0ahoOBvN2YI8GH6FNgA==", - "dev": true, - "license": "ISC", - "dependencies": { - "@jridgewell/trace-mapping": "^0.3.12", - "@types/istanbul-lib-coverage": "^2.0.1", - "convert-source-map": "^2.0.0" - }, - "engines": { - "node": ">=10.12.0" - } - }, - "node_modules/vary": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", - "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", - "dev": true, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/walk-back": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/walk-back/-/walk-back-5.1.0.tgz", - "integrity": "sha512-Uhxps5yZcVNbLEAnb+xaEEMdgTXl9qAQDzKYejG2AZ7qPwRQ81lozY9ECDbjLPNWm7YsO1IK5rsP1KoQzXAcGA==", - "dev": true, - "engines": { - "node": ">=12.17" - } - }, - "node_modules/walker": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/walker/-/walker-1.0.8.tgz", - "integrity": "sha512-ts/8E8l5b7kY0vlWLewOkDXMmPdLcVV4GmOQLyxuSswIJsweeFZtAsMF7k1Nszz+TYBQrlYRmzOnr398y1JemQ==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "makeerror": "1.0.12" - } - }, - "node_modules/watchpack": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.4.2.tgz", - "integrity": "sha512-TnbFSbcOCcDgjZ4piURLCbJ3nJhznVh9kw6F6iokjiFPl8ONxe9A6nMDVXDiNbrSfLILs6vB07F7wLBrwPYzJw==", - "dev": true, - "dependencies": { - "glob-to-regexp": "^0.4.1", - "graceful-fs": "^4.1.2" - }, - "engines": { - "node": ">=10.13.0" - } - }, - "node_modules/wavefile": { - "version": "11.0.0", - "resolved": "https://registry.npmjs.org/wavefile/-/wavefile-11.0.0.tgz", - "integrity": "sha512-/OBiAALgWU24IG7sC84cDO/KfFuvajWc5Uec0oV2zrpOOZZDgGdOwHwgEzOrwh8jkubBk7PtZfQBIcI1OaE5Ng==", - "dev": true, - "bin": { - "wavefile": "bin/wavefile.js" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/wbuf": { - "version": "1.7.3", - "resolved": "https://registry.npmjs.org/wbuf/-/wbuf-1.7.3.tgz", - "integrity": "sha512-O84QOnr0icsbFGLS0O3bI5FswxzRr8/gHwWkDlQFskhSPryQXvrTMxjxGP4+iWYoauLoBvfDpkrOauZ+0iZpDA==", - "dev": true, - "dependencies": { - "minimalistic-assert": "^1.0.0" - } - }, - "node_modules/webpack": { - "version": "5.94.0", - "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.94.0.tgz", - "integrity": "sha512-KcsGn50VT+06JH/iunZJedYGUJS5FGjow8wb9c0v5n1Om8O1g4L6LjtfxwlXIATopoQu+vOXXa7gYisWxCoPyg==", - "dev": true, - "dependencies": { - "@types/estree": "^1.0.5", - "@webassemblyjs/ast": "^1.12.1", - "@webassemblyjs/wasm-edit": "^1.12.1", - "@webassemblyjs/wasm-parser": "^1.12.1", - "acorn": "^8.7.1", - "acorn-import-attributes": "^1.9.5", - "browserslist": "^4.21.10", - "chrome-trace-event": "^1.0.2", - "enhanced-resolve": "^5.17.1", - "es-module-lexer": "^1.2.1", - "eslint-scope": "5.1.1", - "events": "^3.2.0", - "glob-to-regexp": "^0.4.1", - "graceful-fs": "^4.2.11", - "json-parse-even-better-errors": "^2.3.1", - "loader-runner": "^4.2.0", - "mime-types": "^2.1.27", - "neo-async": "^2.6.2", - "schema-utils": "^3.2.0", - "tapable": "^2.1.1", - "terser-webpack-plugin": "^5.3.10", - "watchpack": "^2.4.1", - "webpack-sources": "^3.2.3" - }, - "bin": { - "webpack": "bin/webpack.js" - }, - "engines": { - "node": ">=10.13.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/webpack" - }, - "peerDependenciesMeta": { - "webpack-cli": { - "optional": true - } - } - }, - "node_modules/webpack-cli": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/webpack-cli/-/webpack-cli-5.0.2.tgz", - "integrity": "sha512-4y3W5Dawri5+8dXm3+diW6Mn1Ya+Dei6eEVAdIduAmYNLzv1koKVAqsfgrrc9P2mhrYHQphx5htnGkcNwtubyQ==", - "dev": true, - "dependencies": { - "@discoveryjs/json-ext": "^0.5.0", - "@webpack-cli/configtest": "^2.0.1", - "@webpack-cli/info": "^2.0.1", - "@webpack-cli/serve": "^2.0.2", - "colorette": "^2.0.14", - "commander": "^10.0.1", - "cross-spawn": "^7.0.3", - "envinfo": "^7.7.3", - "fastest-levenshtein": "^1.0.12", - "import-local": "^3.0.2", - "interpret": "^3.1.1", - "rechoir": "^0.8.0", - "webpack-merge": "^5.7.3" - }, - "bin": { - "webpack-cli": "bin/cli.js" - }, - "engines": { - "node": ">=14.15.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/webpack" - }, - "peerDependencies": { - "webpack": "5.x.x" - }, - "peerDependenciesMeta": { - "@webpack-cli/generators": { - "optional": true - }, - "webpack-bundle-analyzer": { - "optional": true - }, - "webpack-dev-server": { - "optional": true - } - } - }, - "node_modules/webpack-cli/node_modules/commander": { - "version": "10.0.1", - "resolved": "https://registry.npmjs.org/commander/-/commander-10.0.1.tgz", - "integrity": "sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==", - "dev": true, - "engines": { - "node": ">=14" - } - }, - "node_modules/webpack-dev-middleware": { - "version": "5.3.4", - "resolved": "https://registry.npmjs.org/webpack-dev-middleware/-/webpack-dev-middleware-5.3.4.tgz", - "integrity": "sha512-BVdTqhhs+0IfoeAf7EoH5WE+exCmqGerHfDM0IL096Px60Tq2Mn9MAbnaGUe6HiMa41KMCYF19gyzZmBcq/o4Q==", - "dev": true, - "dependencies": { - "colorette": "^2.0.10", - "memfs": "^3.4.3", - "mime-types": "^2.1.31", - "range-parser": "^1.2.1", - "schema-utils": "^4.0.0" - }, - "engines": { - "node": ">= 12.13.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/webpack" - }, - "peerDependencies": { - "webpack": "^4.0.0 || ^5.0.0" - } - }, - "node_modules/webpack-dev-middleware/node_modules/ajv": { - "version": "8.12.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz", - "integrity": "sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==", - "dev": true, - "dependencies": { - "fast-deep-equal": "^3.1.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.2.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/webpack-dev-middleware/node_modules/ajv-keywords": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-5.1.0.tgz", - "integrity": "sha512-YCS/JNFAUyr5vAuhk1DWm1CBxRHW9LbJ2ozWeemrIqpbsqKjHVxYPyi5GC0rjZIT5JxJ3virVTS8wk4i/Z+krw==", - "dev": true, - "dependencies": { - "fast-deep-equal": "^3.1.3" - }, - "peerDependencies": { - "ajv": "^8.8.2" - } - }, - "node_modules/webpack-dev-middleware/node_modules/json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", - "dev": true - }, - "node_modules/webpack-dev-middleware/node_modules/schema-utils": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-4.0.1.tgz", - "integrity": "sha512-lELhBAAly9NowEsX0yZBlw9ahZG+sK/1RJ21EpzdYHKEs13Vku3LJ+MIPhh4sMs0oCCeufZQEQbMekiA4vuVIQ==", - "dev": true, - "dependencies": { - "@types/json-schema": "^7.0.9", - "ajv": "^8.9.0", - "ajv-formats": "^2.1.1", - "ajv-keywords": "^5.1.0" - }, - "engines": { - "node": ">= 12.13.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/webpack" - } - }, - "node_modules/webpack-dev-server": { - "version": "4.13.3", - "resolved": "https://registry.npmjs.org/webpack-dev-server/-/webpack-dev-server-4.13.3.tgz", - "integrity": "sha512-KqqzrzMRSRy5ePz10VhjyL27K2dxqwXQLP5rAKwRJBPUahe7Z2bBWzHw37jeb8GCPKxZRO79ZdQUAPesMh/Nug==", - "dev": true, - "dependencies": { - "@types/bonjour": "^3.5.9", - "@types/connect-history-api-fallback": "^1.3.5", - "@types/express": "^4.17.13", - "@types/serve-index": "^1.9.1", - "@types/serve-static": "^1.13.10", - "@types/sockjs": "^0.3.33", - "@types/ws": "^8.5.1", - "ansi-html-community": "^0.0.8", - "bonjour-service": "^1.0.11", - "chokidar": "^3.5.3", - "colorette": "^2.0.10", - "compression": "^1.7.4", - "connect-history-api-fallback": "^2.0.0", - "default-gateway": "^6.0.3", - "express": "^4.17.3", - "graceful-fs": "^4.2.6", - "html-entities": "^2.3.2", - "http-proxy-middleware": "^2.0.3", - "ipaddr.js": "^2.0.1", - "launch-editor": "^2.6.0", - "open": "^8.0.9", - "p-retry": "^4.5.0", - "rimraf": "^3.0.2", - "schema-utils": "^4.0.0", - "selfsigned": "^2.1.1", - "serve-index": "^1.9.1", - "sockjs": "^0.3.24", - "spdy": "^4.0.2", - "webpack-dev-middleware": "^5.3.1", - "ws": "^8.13.0" - }, - "bin": { - "webpack-dev-server": "bin/webpack-dev-server.js" - }, - "engines": { - "node": ">= 12.13.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/webpack" - }, - "peerDependencies": { - "webpack": "^4.37.0 || ^5.0.0" - }, - "peerDependenciesMeta": { - "webpack": { - "optional": true - }, - "webpack-cli": { - "optional": true - } - } - }, - "node_modules/webpack-dev-server/node_modules/ajv": { - "version": "8.12.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz", - "integrity": "sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==", - "dev": true, - "dependencies": { - "fast-deep-equal": "^3.1.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.2.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/webpack-dev-server/node_modules/ajv-keywords": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-5.1.0.tgz", - "integrity": "sha512-YCS/JNFAUyr5vAuhk1DWm1CBxRHW9LbJ2ozWeemrIqpbsqKjHVxYPyi5GC0rjZIT5JxJ3virVTS8wk4i/Z+krw==", - "dev": true, - "dependencies": { - "fast-deep-equal": "^3.1.3" - }, - "peerDependencies": { - "ajv": "^8.8.2" - } - }, - "node_modules/webpack-dev-server/node_modules/json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", - "dev": true - }, - "node_modules/webpack-dev-server/node_modules/schema-utils": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-4.0.1.tgz", - "integrity": "sha512-lELhBAAly9NowEsX0yZBlw9ahZG+sK/1RJ21EpzdYHKEs13Vku3LJ+MIPhh4sMs0oCCeufZQEQbMekiA4vuVIQ==", - "dev": true, - "dependencies": { - "@types/json-schema": "^7.0.9", - "ajv": "^8.9.0", - "ajv-formats": "^2.1.1", - "ajv-keywords": "^5.1.0" - }, - "engines": { - "node": ">= 12.13.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/webpack" - } - }, - "node_modules/webpack-merge": { - "version": "5.8.0", - "resolved": "https://registry.npmjs.org/webpack-merge/-/webpack-merge-5.8.0.tgz", - "integrity": "sha512-/SaI7xY0831XwP6kzuwhKWVKDP9t1QY1h65lAFLbZqMPIuYcD9QAW4u9STIbU9kaJbPBB/geU/gLr1wDjOhQ+Q==", - "dev": true, - "dependencies": { - "clone-deep": "^4.0.1", - "wildcard": "^2.0.0" - }, - "engines": { - "node": ">=10.0.0" - } - }, - "node_modules/webpack-sources": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-3.2.3.tgz", - "integrity": "sha512-/DyMEOrDgLKKIG0fmvtz+4dUX/3Ghozwgm6iPp8KRhvn+eQf9+Q7GWxVNMk3+uCPWfdXYC4ExGBckIXdFEfH1w==", - "dev": true, - "engines": { - "node": ">=10.13.0" - } - }, - "node_modules/websocket-driver": { - "version": "0.7.4", - "resolved": "https://registry.npmjs.org/websocket-driver/-/websocket-driver-0.7.4.tgz", - "integrity": "sha512-b17KeDIQVjvb0ssuSDF2cYXSg2iztliJ4B9WdsuB6J952qCPKmnVq4DyW5motImXHDC1cBT/1UezrJVsKw5zjg==", - "dev": true, - "dependencies": { - "http-parser-js": ">=0.5.1", - "safe-buffer": ">=5.1.0", - "websocket-extensions": ">=0.1.1" - }, - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/websocket-extensions": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/websocket-extensions/-/websocket-extensions-0.1.4.tgz", - "integrity": "sha512-OqedPIGOfsDlo31UNwYbCFMSaO9m9G/0faIHj5/dZFDMFqPTcx6UwqyOy3COEaEOg/9VsGIpdqn62W5KhoKSpg==", - "dev": true, - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/which": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", - "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "dependencies": { - "isexe": "^2.0.0" - }, - "bin": { - "node-which": "bin/node-which" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/wildcard": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/wildcard/-/wildcard-2.0.0.tgz", - "integrity": "sha512-JcKqAHLPxcdb9KM49dufGXn2x3ssnfjbcaQdLlfZsL9rH9wgDQjUtDxbo8NE0F6SFvydeu1VhZe7hZuHsB2/pw==", - "dev": true - }, - "node_modules/wordwrap": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", - "integrity": "sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==", - "dev": true - }, - "node_modules/wordwrapjs": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/wordwrapjs/-/wordwrapjs-3.0.0.tgz", - "integrity": "sha512-mO8XtqyPvykVCsrwj5MlOVWvSnCdT+C+QVbm6blradR7JExAhbkZ7hZ9A+9NUtwzSqrlUo9a67ws0EiILrvRpw==", - "dev": true, - "dependencies": { - "reduce-flatten": "^1.0.1", - "typical": "^2.6.1" - }, - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/wordwrapjs/node_modules/reduce-flatten": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/reduce-flatten/-/reduce-flatten-1.0.1.tgz", - "integrity": "sha512-j5WfFJfc9CoXv/WbwVLHq74i/hdTUpy+iNC534LxczMRP67vJeK3V9JOdnL0N1cIRbn9mYhE2yVjvvKXDxvNXQ==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/wrap-ansi": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/wrap-ansi-cjs": { - "name": "wrap-ansi", - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/wrappy": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "dev": true - }, - "node_modules/write-file-atomic": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-5.0.1.tgz", - "integrity": "sha512-+QU2zd6OTD8XWIJCbffaiQeH9U73qIqafo1x6V1snCWYGJf6cVE0cDR4D8xRzcEnfI21IFrUPzPGtcPf8AC+Rw==", - "dev": true, - "license": "ISC", - "dependencies": { - "imurmurhash": "^0.1.4", - "signal-exit": "^4.0.1" - }, - "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" - } - }, - "node_modules/write-file-atomic/node_modules/signal-exit": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", - "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/ws": { - "version": "8.17.1", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.1.tgz", - "integrity": "sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==", - "dev": true, - "engines": { - "node": ">=10.0.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": ">=5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true + "name": "@huggingface/transformers", + "version": "3.0.2", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@huggingface/transformers", + "version": "3.0.2", + "license": "Apache-2.0", + "dependencies": { + "@huggingface/jinja": "^0.3.0", + "onnxruntime-node": "1.19.2", + "onnxruntime-web": "1.21.0-dev.20241024-d9ca84ef96", + "sharp": "^0.33.5" + }, + "devDependencies": { + "@types/jest": "^29.5.14", + "@webgpu/types": "^0.1.44", + "catharsis": "github:xenova/catharsis", + "jest": "^30.0.0-alpha.6", + "jest-environment-node": "^30.0.0-alpha.6", + "jsdoc-to-markdown": "^8.0.1", + "prettier": "3.3.3", + "typescript": "^5.2.2", + "wavefile": "^11.0.0", + "webpack": "^5.80.0", + "webpack-cli": "^5.0.2", + "webpack-dev-server": "^4.13.3" + } + }, + "node_modules/@ampproject/remapping": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.3.0.tgz", + "integrity": "sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.26.2", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.26.2.tgz", + "integrity": "sha512-RJlIHRueQgwWitWgF8OdFYGZX328Ax5BCemNGlqHfplnRT9ESi8JkFlvaVYbS+UubVY6dpv87Fs2u5M29iNFVQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-validator-identifier": "^7.25.9", + "js-tokens": "^4.0.0", + "picocolors": "^1.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/compat-data": { + "version": "7.26.2", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.26.2.tgz", + "integrity": "sha512-Z0WgzSEa+aUcdiJuCIqgujCshpMWgUpgOxXotrYPSA53hA3qopNaqcJpyr0hVb1FeWdnqFA35/fUtXgBK8srQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/core": { + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.26.0.tgz", + "integrity": "sha512-i1SLeK+DzNnQ3LL/CswPCa/E5u4lh1k6IAEphON8F+cXt0t9euTshDru0q7/IqMa1PMPz5RnHuHscF8/ZJsStg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@ampproject/remapping": "^2.2.0", + "@babel/code-frame": "^7.26.0", + "@babel/generator": "^7.26.0", + "@babel/helper-compilation-targets": "^7.25.9", + "@babel/helper-module-transforms": "^7.26.0", + "@babel/helpers": "^7.26.0", + "@babel/parser": "^7.26.0", + "@babel/template": "^7.25.9", + "@babel/traverse": "^7.25.9", + "@babel/types": "^7.26.0", + "convert-source-map": "^2.0.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.3", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/babel" + } + }, + "node_modules/@babel/core/node_modules/debug": { + "version": "4.3.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", + "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@babel/core/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@babel/core/node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/@babel/generator": { + "version": "7.26.2", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.26.2.tgz", + "integrity": "sha512-zevQbhbau95nkoxSq3f/DC/SC+EEOUZd3DYqfSkMhY2/wfSeaHV1Ew4vk8e+x8lja31IbyuUa2uQ3JONqKbysw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.26.2", + "@babel/types": "^7.26.0", + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.25", + "jsesc": "^3.0.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.25.9.tgz", + "integrity": "sha512-j9Db8Suy6yV/VHa4qzrj9yZfZxhLWQdVnRlXxmKLYlhWUVB1sB2G5sxuWYXk/whHD9iW76PmNzxZ4UCnTQTVEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/compat-data": "^7.25.9", + "@babel/helper-validator-option": "^7.25.9", + "browserslist": "^4.24.0", + "lru-cache": "^5.1.1", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets/node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/@babel/helper-module-imports": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.25.9.tgz", + "integrity": "sha512-tnUA4RsrmflIM6W6RFTLFSXITtl0wKjgpnLgXyowocVPrbYrLUXSBXDgTs8BlbmIzIdlBySRQjINYs2BAkiLtw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/traverse": "^7.25.9", + "@babel/types": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-transforms": { + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.26.0.tgz", + "integrity": "sha512-xO+xu6B5K2czEnQye6BHA7DolFFmS3LB7stHZFaOLb1pAwO1HWLS8fXA+eh0A2yIvltPVmx3eNNDBJA2SLHXFw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-module-imports": "^7.25.9", + "@babel/helper-validator-identifier": "^7.25.9", + "@babel/traverse": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-plugin-utils": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.25.9.tgz", + "integrity": "sha512-kSMlyUVdWe25rEsRGviIgOWnoT/nfABVWlqt9N19/dIPWViAOW2s9wznP5tURbs/IDuNk4gPy3YdYRgH3uxhBw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.25.9.tgz", + "integrity": "sha512-4A/SCr/2KLd5jrtOMFzaKjVtAei3+2r/NChoBNoZ3EyP/+GlhoaEGoWOZUmFmoITP7zOJyHIMm+DYRd8o3PvHA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.25.9.tgz", + "integrity": "sha512-Ed61U6XJc3CVRfkERJWDz4dJwKe7iLmmJsbOGu9wSloNSFttHV0I8g6UAgb7qnK5ly5bGLPd4oXZlxCdANBOWQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-option": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.25.9.tgz", + "integrity": "sha512-e/zv1co8pp55dNdEcCynfj9X7nyUKUXoUEwfXqaZt0omVOmDe9oOTdKStH4GmAw6zxMFs50ZayuMfHDKlO7Tfw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helpers": { + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.26.0.tgz", + "integrity": "sha512-tbhNuIxNcVb21pInl3ZSjksLCvgdZy9KwJ8brv993QtIVKJBBkYXz4q4ZbAv31GdnC+R90np23L5FbEBlthAEw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/template": "^7.25.9", + "@babel/types": "^7.26.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.26.2", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.26.2.tgz", + "integrity": "sha512-DWMCZH9WA4Maitz2q21SRKHo9QXZxkDsbNZoVD62gusNtNBBqDg9i7uOhASfTfIGNzW+O+r7+jAlM8dwphcJKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.26.0" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/plugin-syntax-async-generators": { + "version": "7.8.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz", + "integrity": "sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-bigint": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-bigint/-/plugin-syntax-bigint-7.8.3.tgz", + "integrity": "sha512-wnTnFlG+YxQm3vDxpGE57Pj0srRU4sHE/mDkt1qv2YJJSeUAec2ma4WLUnUPeKjyrfntVwe/N6dCXpU+zL3Npg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-class-properties": { + "version": "7.12.13", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-properties/-/plugin-syntax-class-properties-7.12.13.tgz", + "integrity": "sha512-fm4idjKla0YahUNgFNLCB0qySdsoPiZP3iQE3rky0mBUtMZ23yDJ9SJdg6dXTSDnulOVqiF3Hgr9nbXvXTQZYA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.12.13" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-class-static-block": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-static-block/-/plugin-syntax-class-static-block-7.14.5.tgz", + "integrity": "sha512-b+YyPmr6ldyNnM6sqYeMWE+bgJcJpO6yS4QD7ymxgH34GBPNDM/THBh8iunyvKIZztiwLH4CJZ0RxTk9emgpjw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-import-attributes": { + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-attributes/-/plugin-syntax-import-attributes-7.26.0.tgz", + "integrity": "sha512-e2dttdsJ1ZTpi3B9UYGLw41hifAubg19AtCu/2I/F1QNVclOBr1dYpTdmdyZ84Xiz43BS/tCUkMAZNLv12Pi+A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-import-meta": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-meta/-/plugin-syntax-import-meta-7.10.4.tgz", + "integrity": "sha512-Yqfm+XDx0+Prh3VSeEQCPU81yC+JWZ2pDPFSS4ZdpfZhp4MkFMaDC1UqseovEKwSUpnIL7+vK+Clp7bfh0iD7g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-json-strings": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-json-strings/-/plugin-syntax-json-strings-7.8.3.tgz", + "integrity": "sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-jsx": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.25.9.tgz", + "integrity": "sha512-ld6oezHQMZsZfp6pWtbjaNDF2tiiCYYDqQszHt5VV437lewP9aSi2Of99CK0D0XB21k7FLgnLcmQKyKzynfeAA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-logical-assignment-operators": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-logical-assignment-operators/-/plugin-syntax-logical-assignment-operators-7.10.4.tgz", + "integrity": "sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-nullish-coalescing-operator": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-nullish-coalescing-operator/-/plugin-syntax-nullish-coalescing-operator-7.8.3.tgz", + "integrity": "sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-numeric-separator": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-numeric-separator/-/plugin-syntax-numeric-separator-7.10.4.tgz", + "integrity": "sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-object-rest-spread": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.8.3.tgz", + "integrity": "sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-optional-catch-binding": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.8.3.tgz", + "integrity": "sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-optional-chaining": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-chaining/-/plugin-syntax-optional-chaining-7.8.3.tgz", + "integrity": "sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-private-property-in-object": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-private-property-in-object/-/plugin-syntax-private-property-in-object-7.14.5.tgz", + "integrity": "sha512-0wVnp9dxJ72ZUJDV27ZfbSj6iHLoytYZmh3rFcxNnvsJF3ktkzLDZPy/mA17HGsaQT3/DQsWYX1f1QGWkCoVUg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-top-level-await": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.14.5.tgz", + "integrity": "sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-typescript": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.25.9.tgz", + "integrity": "sha512-hjMgRy5hb8uJJjUcdWunWVcoi9bGpJp8p5Ol1229PoN6aytsLwNMgmdftO23wnCLMfVmTwZDWMPNq/D1SY60JQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/template": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.25.9.tgz", + "integrity": "sha512-9DGttpmPvIxBb/2uwpVo3dqJ+O6RooAFOS+lB+xDqoE2PVCE8nfoHMdZLpfCQRLwvohzXISPZcgxt80xLfsuwg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.25.9", + "@babel/parser": "^7.25.9", + "@babel/types": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.25.9.tgz", + "integrity": "sha512-ZCuvfwOwlz/bawvAuvcj8rrithP2/N55Tzz342AkTvq4qaWbGfmCk/tKhNaV2cthijKrPAA8SRJV5WWe7IBMJw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.25.9", + "@babel/generator": "^7.25.9", + "@babel/parser": "^7.25.9", + "@babel/template": "^7.25.9", + "@babel/types": "^7.25.9", + "debug": "^4.3.1", + "globals": "^11.1.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse/node_modules/debug": { + "version": "4.3.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", + "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@babel/traverse/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@babel/types": { + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.26.0.tgz", + "integrity": "sha512-Z/yiTPj+lDVnF7lWeKCIJzaIkI0vYO87dMpZ4bg4TDrFe4XXLFWL1TbXU27gBP3QccxV9mZICCrnjnYlJjXHOA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.25.9", + "@babel/helper-validator-identifier": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@bcoe/v8-coverage": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz", + "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@discoveryjs/json-ext": { + "version": "0.5.7", + "resolved": "https://registry.npmjs.org/@discoveryjs/json-ext/-/json-ext-0.5.7.tgz", + "integrity": "sha512-dBVuXR082gk3jsFp7Rd/JI4kytwGHecnCoTtXFb7DB6CNHp4rg5k1bhg0nWdLGLnOV71lmDzGQaLMy8iPLY0pw==", + "dev": true, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.2.0.tgz", + "integrity": "sha512-bV21/9LQmcQeCPEg3BDFtvwL6cwiTMksYNWQQ4KOxCZikEGalWtenoZ0wCiukJINlGCIi2KXx01g4FoH/LxpzQ==", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@huggingface/jinja": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@huggingface/jinja/-/jinja-0.3.0.tgz", + "integrity": "sha512-GLJzso0M07ZncFkrJMIXVU4os6GFbPocD4g8fMQPMGJubf48FtGOsUORH2rtFdXPIPelz8SLBMn8ZRmOTwXm9Q==", + "engines": { + "node": ">=18" + } + }, + "node_modules/@img/sharp-darwin-arm64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.33.5.tgz", + "integrity": "sha512-UT4p+iz/2H4twwAoLCqfA9UH5pI6DggwKEGuaPy7nCVQ8ZsiY5PIcrRvD1DzuY3qYL07NtIQcWnBSY/heikIFQ==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.0.4" + } + }, + "node_modules/@img/sharp-darwin-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.33.5.tgz", + "integrity": "sha512-fyHac4jIc1ANYGRDxtiqelIbdWkIuQaI84Mv45KvGRRxSAa7o7d1ZKAOBaYbnepLC1WqxfpimdeWfvqqSGwR2Q==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.0.4" + } + }, + "node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.0.4.tgz", + "integrity": "sha512-XblONe153h0O2zuFfTAbQYAX2JhYmDHeWikp1LM9Hul9gVPjFY427k6dFEcOL72O01QxQsWi761svJ/ev9xEDg==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.0.4.tgz", + "integrity": "sha512-xnGR8YuZYfJGmWPvmlunFaWJsb9T/AO2ykoP3Fz/0X5XV2aoYBPkX6xqCQvUTKKiLddarLaxpzNe+b1hjeWHAQ==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.0.5.tgz", + "integrity": "sha512-gvcC4ACAOPRNATg/ov8/MnbxFDJqf/pDePbBnuBDcjsI8PssmjoKMAz4LtLaVi+OnSb5FK/yIOamqDwGmXW32g==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.0.4.tgz", + "integrity": "sha512-9B+taZ8DlyyqzZQnoeIvDVR/2F4EbMepXMc/NdVbkzsJbzkUjhXv/70GQJ7tdLA4YJgNP25zukcxpX2/SueNrA==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.0.4.tgz", + "integrity": "sha512-u7Wz6ntiSSgGSGcjZ55im6uvTrOxSIS8/dgoVMoiGE9I6JAfU50yH5BoDlYA1tcuGS7g/QNtetJnxA6QEsCVTA==", + "cpu": [ + "s390x" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.0.4.tgz", + "integrity": "sha512-MmWmQ3iPFZr0Iev+BAgVMb3ZyC4KeFc3jFxnNbEPas60e1cIfevbtuyf9nDGIzOaW9PdnDciJm+wFFaTlj5xYw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.0.4.tgz", + "integrity": "sha512-9Ti+BbTYDcsbp4wfYib8Ctm1ilkugkA/uscUn6UXK1ldpC1JjiXbLfFZtRlBhjPZ5o1NCLiDbg8fhUPKStHoTA==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.0.4.tgz", + "integrity": "sha512-viYN1KX9m+/hGkJtvYYp+CCLgnJXwiQB39damAO7WMdKWlIhmYTfHjwSbQeUK/20vY154mwezd9HflVFM1wVSw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-linux-arm": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.33.5.tgz", + "integrity": "sha512-JTS1eldqZbJxjvKaAkxhZmBqPRGmxgu+qFKSInv8moZ2AmT5Yib3EQ1c6gp493HvrvV8QgdOXdyaIBrhvFhBMQ==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.0.5" + } + }, + "node_modules/@img/sharp-linux-arm64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.33.5.tgz", + "integrity": "sha512-JMVv+AMRyGOHtO1RFBiJy/MBsgz0x4AWrT6QoEVVTyh1E39TrCUpTRI7mx9VksGX4awWASxqCYLCV4wBZHAYxA==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.0.4" + } + }, + "node_modules/@img/sharp-linux-s390x": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.33.5.tgz", + "integrity": "sha512-y/5PCd+mP4CA/sPDKl2961b+C9d+vPAveS33s6Z3zfASk2j5upL6fXVPZi7ztePZ5CuH+1kW8JtvxgbuXHRa4Q==", + "cpu": [ + "s390x" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.0.4" + } + }, + "node_modules/@img/sharp-linux-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.33.5.tgz", + "integrity": "sha512-opC+Ok5pRNAzuvq1AG0ar+1owsu842/Ab+4qvU879ippJBHvyY5n2mxF1izXqkPYlGuP/M556uh53jRLJmzTWA==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.0.4" + } + }, + "node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.33.5.tgz", + "integrity": "sha512-XrHMZwGQGvJg2V/oRSUfSAfjfPxO+4DkiRh6p2AFjLQztWUuY/o8Mq0eMQVIY7HJ1CDQUJlxGGZRw1a5bqmd1g==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.0.4" + } + }, + "node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.33.5.tgz", + "integrity": "sha512-WT+d/cgqKkkKySYmqoZ8y3pxx7lx9vVejxW/W4DOFMYVSkErR+w7mf2u8m/y4+xHe7yY9DAXQMWQhpnMuFfScw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.0.4" + } + }, + "node_modules/@img/sharp-wasm32": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.33.5.tgz", + "integrity": "sha512-ykUW4LVGaMcU9lu9thv85CbRMAwfeadCJHRsg2GmeRa/cJxsVY9Rbd57JcMxBkKHag5U/x7TSBpScF4U8ElVzg==", + "cpu": [ + "wasm32" + ], + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.2.0" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-ia32": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.33.5.tgz", + "integrity": "sha512-T36PblLaTwuVJ/zw/LaH0PdZkRz5rd3SmMHX8GSmR7vtNSP5Z6bQkExdSK7xGWyxLw4sUknBuugTelgw2faBbQ==", + "cpu": [ + "ia32" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.33.5.tgz", + "integrity": "sha512-MpY/o8/8kj+EcnxwvrP4aTJSWw/aZ7JIGR4aBeZkZw5B7/Jn+tY9/VNwtcoGmdT7GfggGIU4kygOMSbYnOrAbg==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@isaacs/cliui/node_modules/ansi-regex": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", + "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/ansi-styles": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" + }, + "node_modules/@isaacs/cliui/node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@isaacs/cliui/node_modules/strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "dependencies": { + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/@isaacs/fs-minipass": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", + "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==", + "dependencies": { + "minipass": "^7.0.4" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@istanbuljs/load-nyc-config": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", + "integrity": "sha512-VjeHSlIzpv/NyD3N0YuHfXOPDIixcA1q2ZV98wsMqcYlPmv2n3Yb2lYP9XMElnaFVXg5A7YLTeLu6V84uQDjmQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "camelcase": "^5.3.1", + "find-up": "^4.1.0", + "get-package-type": "^0.1.0", + "js-yaml": "^3.13.1", + "resolve-from": "^5.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@istanbuljs/schema": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.3.tgz", + "integrity": "sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/@jest/console": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/console/-/console-30.0.0-alpha.6.tgz", + "integrity": "sha512-08BeAnuabmauj5B+Xa4GNPAotQUGm3PLKSE3rnpnmxniZzR4tXhx8+AA2+HGTri4bbVRY/r3Jl0vJnkhvHTkeQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "30.0.0-alpha.6", + "@types/node": "*", + "chalk": "^4.0.0", + "jest-message-util": "30.0.0-alpha.6", + "jest-util": "30.0.0-alpha.6", + "slash": "^3.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/console/node_modules/@jest/schemas": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", + "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.33.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/console/node_modules/@jest/types": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", + "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/pattern": "30.0.0-alpha.6", + "@jest/schemas": "30.0.0-alpha.6", + "@types/istanbul-lib-coverage": "^2.0.0", + "@types/istanbul-reports": "^3.0.0", + "@types/node": "*", + "@types/yargs": "^17.0.8", + "chalk": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/console/node_modules/@sinclair/typebox": { + "version": "0.33.17", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", + "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jest/console/node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/@jest/console/node_modules/ci-info": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.0.0.tgz", + "integrity": "sha512-TdHqgGf9odd8SXNuxtUBVx8Nv+qZOejE6qyqiy5NtbYYQOeFa6zmHkxlPzmaLxWWHsU6nJmB7AETdVPi+2NBUg==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/sibiraj-s" + } + ], + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/@jest/console/node_modules/jest-message-util": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-30.0.0-alpha.6.tgz", + "integrity": "sha512-XAGJqkrBo7m3bFxWqiNqL0PyAWGf1XHR6bTve90MjBKJuIzhJsounGTzBNUw8JoU7Uzcj5Z6ZmEhaE3CDnwjfw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.12.13", + "@jest/types": "30.0.0-alpha.6", + "@types/stack-utils": "^2.0.0", + "chalk": "^4.0.0", + "graceful-fs": "^4.2.9", + "micromatch": "^4.0.7", + "pretty-format": "30.0.0-alpha.6", + "slash": "^3.0.0", + "stack-utils": "^2.0.3" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/console/node_modules/jest-util": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.0.0-alpha.6.tgz", + "integrity": "sha512-JlimakOVDyoMC8TEG+knoufxUqLG+Btihf1G8o5sHxz54C6oL54Wetfepp+Nhuj/1hSL0sQtkovvjlEycf9i0w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "30.0.0-alpha.6", + "@types/node": "*", + "chalk": "^4.0.0", + "ci-info": "^4.0.0", + "graceful-fs": "^4.2.9", + "picomatch": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/console/node_modules/picomatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", + "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/@jest/console/node_modules/pretty-format": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.0.0-alpha.6.tgz", + "integrity": "sha512-xkeffkZoqQmRrcNewpOsUCKNOl+CkPqjt3Ld749uz1S7/O7GuPNPv2fZk3v/1U/FE8/B4Zz0llVL80MKON1tOQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/schemas": "30.0.0-alpha.6", + "ansi-styles": "^5.0.0", + "react-is": "^18.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/core": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/core/-/core-30.0.0-alpha.6.tgz", + "integrity": "sha512-Qsvu9/I0hUOpeelp3jlTmg6cg3C+w18v4hxWVGchCRJAChvuxmsomB1Cm+DKB6NiMy2EvUvpwdT8X31lERtemw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/console": "30.0.0-alpha.6", + "@jest/pattern": "30.0.0-alpha.6", + "@jest/reporters": "30.0.0-alpha.6", + "@jest/test-result": "30.0.0-alpha.6", + "@jest/transform": "30.0.0-alpha.6", + "@jest/types": "30.0.0-alpha.6", + "@types/node": "*", + "ansi-escapes": "^4.2.1", + "chalk": "^4.0.0", + "ci-info": "^4.0.0", + "exit": "^0.1.2", + "graceful-fs": "^4.2.9", + "jest-changed-files": "30.0.0-alpha.6", + "jest-config": "30.0.0-alpha.6", + "jest-haste-map": "30.0.0-alpha.6", + "jest-message-util": "30.0.0-alpha.6", + "jest-regex-util": "30.0.0-alpha.6", + "jest-resolve": "30.0.0-alpha.6", + "jest-resolve-dependencies": "30.0.0-alpha.6", + "jest-runner": "30.0.0-alpha.6", + "jest-runtime": "30.0.0-alpha.6", + "jest-snapshot": "30.0.0-alpha.6", + "jest-util": "30.0.0-alpha.6", + "jest-validate": "30.0.0-alpha.6", + "jest-watcher": "30.0.0-alpha.6", + "micromatch": "^4.0.7", + "pretty-format": "30.0.0-alpha.6", + "slash": "^3.0.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + }, + "peerDependencies": { + "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" + }, + "peerDependenciesMeta": { + "node-notifier": { + "optional": true + } + } + }, + "node_modules/@jest/core/node_modules/@jest/schemas": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", + "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.33.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/core/node_modules/@jest/types": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", + "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/pattern": "30.0.0-alpha.6", + "@jest/schemas": "30.0.0-alpha.6", + "@types/istanbul-lib-coverage": "^2.0.0", + "@types/istanbul-reports": "^3.0.0", + "@types/node": "*", + "@types/yargs": "^17.0.8", + "chalk": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/core/node_modules/@sinclair/typebox": { + "version": "0.33.17", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", + "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jest/core/node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/@jest/core/node_modules/ci-info": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.0.0.tgz", + "integrity": "sha512-TdHqgGf9odd8SXNuxtUBVx8Nv+qZOejE6qyqiy5NtbYYQOeFa6zmHkxlPzmaLxWWHsU6nJmB7AETdVPi+2NBUg==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/sibiraj-s" + } + ], + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/@jest/core/node_modules/jest-message-util": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-30.0.0-alpha.6.tgz", + "integrity": "sha512-XAGJqkrBo7m3bFxWqiNqL0PyAWGf1XHR6bTve90MjBKJuIzhJsounGTzBNUw8JoU7Uzcj5Z6ZmEhaE3CDnwjfw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.12.13", + "@jest/types": "30.0.0-alpha.6", + "@types/stack-utils": "^2.0.0", + "chalk": "^4.0.0", + "graceful-fs": "^4.2.9", + "micromatch": "^4.0.7", + "pretty-format": "30.0.0-alpha.6", + "slash": "^3.0.0", + "stack-utils": "^2.0.3" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/core/node_modules/jest-util": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.0.0-alpha.6.tgz", + "integrity": "sha512-JlimakOVDyoMC8TEG+knoufxUqLG+Btihf1G8o5sHxz54C6oL54Wetfepp+Nhuj/1hSL0sQtkovvjlEycf9i0w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "30.0.0-alpha.6", + "@types/node": "*", + "chalk": "^4.0.0", + "ci-info": "^4.0.0", + "graceful-fs": "^4.2.9", + "picomatch": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/core/node_modules/picomatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", + "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/@jest/core/node_modules/pretty-format": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.0.0-alpha.6.tgz", + "integrity": "sha512-xkeffkZoqQmRrcNewpOsUCKNOl+CkPqjt3Ld749uz1S7/O7GuPNPv2fZk3v/1U/FE8/B4Zz0llVL80MKON1tOQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/schemas": "30.0.0-alpha.6", + "ansi-styles": "^5.0.0", + "react-is": "^18.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/environment": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-30.0.0-alpha.6.tgz", + "integrity": "sha512-pjNYNkzq761hh8D2grrG77L6nhe2VBCFFM+G1hyqhaJ2MAzhp2Gh+G94uF3px7luSzLh8GYvGJQGYy197EUOGQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/fake-timers": "30.0.0-alpha.6", + "@jest/types": "30.0.0-alpha.6", + "@types/node": "*", + "jest-mock": "30.0.0-alpha.6" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/environment/node_modules/@jest/schemas": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", + "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.33.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/environment/node_modules/@jest/types": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", + "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/pattern": "30.0.0-alpha.6", + "@jest/schemas": "30.0.0-alpha.6", + "@types/istanbul-lib-coverage": "^2.0.0", + "@types/istanbul-reports": "^3.0.0", + "@types/node": "*", + "@types/yargs": "^17.0.8", + "chalk": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/environment/node_modules/@sinclair/typebox": { + "version": "0.33.17", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", + "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jest/expect": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/expect/-/expect-30.0.0-alpha.6.tgz", + "integrity": "sha512-3O74pygTwUBzUjO958IgNwmp0WrjASbiWdMEfUMePVqtiGoyS4Nxj9hsx4uKsNVivNJSZiiayYoP6dLhWerJXQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "expect": "30.0.0-alpha.6", + "jest-snapshot": "30.0.0-alpha.6" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/expect-utils": { + "version": "29.6.1", + "resolved": "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-29.6.1.tgz", + "integrity": "sha512-o319vIf5pEMx0LmzSxxkYYxo4wrRLKHq9dP1yJU7FoPTB0LfAKSz8SWD6D/6U3v/O52t9cF5t+MeJiRsfk7zMw==", + "dev": true, + "dependencies": { + "jest-get-type": "^29.4.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/expect/node_modules/@jest/expect-utils": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-30.0.0-alpha.6.tgz", + "integrity": "sha512-QMySMhaCUl0ZQd7Tx5X3fVWY5jtQxZNrTll0OyavdQ70ZTLgk0kU9K+XovcMWO26MK9R5EX7bBgD/j7w9hUM4w==", + "dev": true, + "license": "MIT", + "dependencies": { + "jest-get-type": "30.0.0-alpha.6" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/expect/node_modules/@jest/schemas": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", + "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.33.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/expect/node_modules/@jest/types": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", + "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/pattern": "30.0.0-alpha.6", + "@jest/schemas": "30.0.0-alpha.6", + "@types/istanbul-lib-coverage": "^2.0.0", + "@types/istanbul-reports": "^3.0.0", + "@types/node": "*", + "@types/yargs": "^17.0.8", + "chalk": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/expect/node_modules/@sinclair/typebox": { + "version": "0.33.17", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", + "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jest/expect/node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/@jest/expect/node_modules/ci-info": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.0.0.tgz", + "integrity": "sha512-TdHqgGf9odd8SXNuxtUBVx8Nv+qZOejE6qyqiy5NtbYYQOeFa6zmHkxlPzmaLxWWHsU6nJmB7AETdVPi+2NBUg==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/sibiraj-s" + } + ], + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/@jest/expect/node_modules/diff-sequences": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-30.0.0-alpha.6.tgz", + "integrity": "sha512-DVGt3/yzbneMUTuupsMqyfSXMnU2fE0lVsC9uFsJmRpluvSi7ZhrS0GX5tnMna6Ta788FGfOUx+irI/+cAZ4EA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/expect/node_modules/expect": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/expect/-/expect-30.0.0-alpha.6.tgz", + "integrity": "sha512-WVi2V4iHKw/vHEyye00Q9CSZz7KHDbJkJyteUI8kTih9jiyMl3bIk7wLYFcY9D1Blnadlyb5w5NBuNjQBow99g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/expect-utils": "30.0.0-alpha.6", + "jest-get-type": "30.0.0-alpha.6", + "jest-matcher-utils": "30.0.0-alpha.6", + "jest-message-util": "30.0.0-alpha.6", + "jest-mock": "30.0.0-alpha.6", + "jest-util": "30.0.0-alpha.6" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/expect/node_modules/jest-diff": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-30.0.0-alpha.6.tgz", + "integrity": "sha512-43j1DoYwVKrkbB67a2gC0ijjIY9biF0JSPXv7H6zlOkzNlqYg8hSDzrurLNo6zGKatW4JSBLE79LmXPJPj1m6A==", + "dev": true, + "license": "MIT", + "dependencies": { + "chalk": "^4.0.0", + "diff-sequences": "30.0.0-alpha.6", + "jest-get-type": "30.0.0-alpha.6", + "pretty-format": "30.0.0-alpha.6" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/expect/node_modules/jest-get-type": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-30.0.0-alpha.6.tgz", + "integrity": "sha512-lJEoQdCY4ICN6+T0lJ9BODKuqPOEpCv2NnJsEO1nmsK0fbWZmN/pgOPHVqLfK8i3jZpUmgupJ1w8r36mc8iiBQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/expect/node_modules/jest-matcher-utils": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-30.0.0-alpha.6.tgz", + "integrity": "sha512-jaq7+HznsK54G0qzu96ZwfMEKHmlPiDqg6qG2p/hVQzr6Y/qVMRh8abI9Y1lX6SSXkr+S9mPAkmOsuJNLTLYmQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "chalk": "^4.0.0", + "jest-diff": "30.0.0-alpha.6", + "jest-get-type": "30.0.0-alpha.6", + "pretty-format": "30.0.0-alpha.6" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/expect/node_modules/jest-message-util": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-30.0.0-alpha.6.tgz", + "integrity": "sha512-XAGJqkrBo7m3bFxWqiNqL0PyAWGf1XHR6bTve90MjBKJuIzhJsounGTzBNUw8JoU7Uzcj5Z6ZmEhaE3CDnwjfw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.12.13", + "@jest/types": "30.0.0-alpha.6", + "@types/stack-utils": "^2.0.0", + "chalk": "^4.0.0", + "graceful-fs": "^4.2.9", + "micromatch": "^4.0.7", + "pretty-format": "30.0.0-alpha.6", + "slash": "^3.0.0", + "stack-utils": "^2.0.3" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/expect/node_modules/jest-util": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.0.0-alpha.6.tgz", + "integrity": "sha512-JlimakOVDyoMC8TEG+knoufxUqLG+Btihf1G8o5sHxz54C6oL54Wetfepp+Nhuj/1hSL0sQtkovvjlEycf9i0w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "30.0.0-alpha.6", + "@types/node": "*", + "chalk": "^4.0.0", + "ci-info": "^4.0.0", + "graceful-fs": "^4.2.9", + "picomatch": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/expect/node_modules/picomatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", + "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/@jest/expect/node_modules/pretty-format": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.0.0-alpha.6.tgz", + "integrity": "sha512-xkeffkZoqQmRrcNewpOsUCKNOl+CkPqjt3Ld749uz1S7/O7GuPNPv2fZk3v/1U/FE8/B4Zz0llVL80MKON1tOQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/schemas": "30.0.0-alpha.6", + "ansi-styles": "^5.0.0", + "react-is": "^18.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/fake-timers": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/fake-timers/-/fake-timers-30.0.0-alpha.6.tgz", + "integrity": "sha512-deka0RmhJgEKPJM6cXPd4TJQ6fLczErdMN7Oxzr16UTDFHxtFd79tduJ8uP92dQyO4zy63N/dlFK6d+FHyWUDw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "30.0.0-alpha.6", + "@sinonjs/fake-timers": "^11.1.0", + "@types/node": "*", + "jest-message-util": "30.0.0-alpha.6", + "jest-mock": "30.0.0-alpha.6", + "jest-util": "30.0.0-alpha.6" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/fake-timers/node_modules/@jest/schemas": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", + "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.33.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/fake-timers/node_modules/@jest/types": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", + "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/pattern": "30.0.0-alpha.6", + "@jest/schemas": "30.0.0-alpha.6", + "@types/istanbul-lib-coverage": "^2.0.0", + "@types/istanbul-reports": "^3.0.0", + "@types/node": "*", + "@types/yargs": "^17.0.8", + "chalk": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/fake-timers/node_modules/@sinclair/typebox": { + "version": "0.33.17", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", + "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jest/fake-timers/node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/@jest/fake-timers/node_modules/ci-info": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.0.0.tgz", + "integrity": "sha512-TdHqgGf9odd8SXNuxtUBVx8Nv+qZOejE6qyqiy5NtbYYQOeFa6zmHkxlPzmaLxWWHsU6nJmB7AETdVPi+2NBUg==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/sibiraj-s" + } + ], + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/@jest/fake-timers/node_modules/jest-message-util": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-30.0.0-alpha.6.tgz", + "integrity": "sha512-XAGJqkrBo7m3bFxWqiNqL0PyAWGf1XHR6bTve90MjBKJuIzhJsounGTzBNUw8JoU7Uzcj5Z6ZmEhaE3CDnwjfw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.12.13", + "@jest/types": "30.0.0-alpha.6", + "@types/stack-utils": "^2.0.0", + "chalk": "^4.0.0", + "graceful-fs": "^4.2.9", + "micromatch": "^4.0.7", + "pretty-format": "30.0.0-alpha.6", + "slash": "^3.0.0", + "stack-utils": "^2.0.3" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/fake-timers/node_modules/jest-util": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.0.0-alpha.6.tgz", + "integrity": "sha512-JlimakOVDyoMC8TEG+knoufxUqLG+Btihf1G8o5sHxz54C6oL54Wetfepp+Nhuj/1hSL0sQtkovvjlEycf9i0w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "30.0.0-alpha.6", + "@types/node": "*", + "chalk": "^4.0.0", + "ci-info": "^4.0.0", + "graceful-fs": "^4.2.9", + "picomatch": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/fake-timers/node_modules/picomatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", + "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/@jest/fake-timers/node_modules/pretty-format": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.0.0-alpha.6.tgz", + "integrity": "sha512-xkeffkZoqQmRrcNewpOsUCKNOl+CkPqjt3Ld749uz1S7/O7GuPNPv2fZk3v/1U/FE8/B4Zz0llVL80MKON1tOQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/schemas": "30.0.0-alpha.6", + "ansi-styles": "^5.0.0", + "react-is": "^18.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/globals": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/globals/-/globals-30.0.0-alpha.6.tgz", + "integrity": "sha512-+uJMoPUos9RH6+52iNgKJBbx1Hd2QsCZjExi5XpVvTjJ/gE4eJ1X7irUMt+14sH0QkeZ3GnjeTJFopyjOCsu+Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/environment": "30.0.0-alpha.6", + "@jest/expect": "30.0.0-alpha.6", + "@jest/types": "30.0.0-alpha.6", + "jest-mock": "30.0.0-alpha.6" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/globals/node_modules/@jest/schemas": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", + "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.33.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/globals/node_modules/@jest/types": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", + "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/pattern": "30.0.0-alpha.6", + "@jest/schemas": "30.0.0-alpha.6", + "@types/istanbul-lib-coverage": "^2.0.0", + "@types/istanbul-reports": "^3.0.0", + "@types/node": "*", + "@types/yargs": "^17.0.8", + "chalk": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/globals/node_modules/@sinclair/typebox": { + "version": "0.33.17", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", + "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jest/pattern": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/pattern/-/pattern-30.0.0-alpha.6.tgz", + "integrity": "sha512-eoV3sjS1M5k3YdrFWezqdndfgepwB86gwyXC0BzV2saZdJ42ySUoEDBGKuwta8A6Zh3w8tVHNFxz1BqiFraHCQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "jest-regex-util": "30.0.0-alpha.6" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/reporters": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/reporters/-/reporters-30.0.0-alpha.6.tgz", + "integrity": "sha512-jzW0t2OtEzBYwlG4EMJKG4q5RPaVvLPDm/nBS08hd+XPoLJJ9b5thyo/MoThIqJfdi0lHqFlDQUmlL205CMoSw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@bcoe/v8-coverage": "^0.2.3", + "@jest/console": "30.0.0-alpha.6", + "@jest/test-result": "30.0.0-alpha.6", + "@jest/transform": "30.0.0-alpha.6", + "@jest/types": "30.0.0-alpha.6", + "@jridgewell/trace-mapping": "^0.3.18", + "@types/node": "*", + "chalk": "^4.0.0", + "collect-v8-coverage": "^1.0.0", + "exit": "^0.1.2", + "glob": "^10.3.10", + "graceful-fs": "^4.2.9", + "istanbul-lib-coverage": "^3.0.0", + "istanbul-lib-instrument": "^6.0.0", + "istanbul-lib-report": "^3.0.0", + "istanbul-lib-source-maps": "^5.0.0", + "istanbul-reports": "^3.1.3", + "jest-message-util": "30.0.0-alpha.6", + "jest-util": "30.0.0-alpha.6", + "jest-worker": "30.0.0-alpha.6", + "slash": "^3.0.0", + "string-length": "^4.0.1", + "strip-ansi": "^6.0.0", + "v8-to-istanbul": "^9.0.1" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + }, + "peerDependencies": { + "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" + }, + "peerDependenciesMeta": { + "node-notifier": { + "optional": true + } + } + }, + "node_modules/@jest/reporters/node_modules/@jest/schemas": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", + "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.33.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/reporters/node_modules/@jest/types": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", + "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/pattern": "30.0.0-alpha.6", + "@jest/schemas": "30.0.0-alpha.6", + "@types/istanbul-lib-coverage": "^2.0.0", + "@types/istanbul-reports": "^3.0.0", + "@types/node": "*", + "@types/yargs": "^17.0.8", + "chalk": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/reporters/node_modules/@sinclair/typebox": { + "version": "0.33.17", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", + "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jest/reporters/node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/@jest/reporters/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/@jest/reporters/node_modules/ci-info": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.0.0.tgz", + "integrity": "sha512-TdHqgGf9odd8SXNuxtUBVx8Nv+qZOejE6qyqiy5NtbYYQOeFa6zmHkxlPzmaLxWWHsU6nJmB7AETdVPi+2NBUg==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/sibiraj-s" + } + ], + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/@jest/reporters/node_modules/glob": { + "version": "10.4.5", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz", + "integrity": "sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==", + "dev": true, + "license": "ISC", + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^1.11.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@jest/reporters/node_modules/jest-message-util": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-30.0.0-alpha.6.tgz", + "integrity": "sha512-XAGJqkrBo7m3bFxWqiNqL0PyAWGf1XHR6bTve90MjBKJuIzhJsounGTzBNUw8JoU7Uzcj5Z6ZmEhaE3CDnwjfw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.12.13", + "@jest/types": "30.0.0-alpha.6", + "@types/stack-utils": "^2.0.0", + "chalk": "^4.0.0", + "graceful-fs": "^4.2.9", + "micromatch": "^4.0.7", + "pretty-format": "30.0.0-alpha.6", + "slash": "^3.0.0", + "stack-utils": "^2.0.3" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/reporters/node_modules/jest-util": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.0.0-alpha.6.tgz", + "integrity": "sha512-JlimakOVDyoMC8TEG+knoufxUqLG+Btihf1G8o5sHxz54C6oL54Wetfepp+Nhuj/1hSL0sQtkovvjlEycf9i0w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "30.0.0-alpha.6", + "@types/node": "*", + "chalk": "^4.0.0", + "ci-info": "^4.0.0", + "graceful-fs": "^4.2.9", + "picomatch": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/reporters/node_modules/jest-worker": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-30.0.0-alpha.6.tgz", + "integrity": "sha512-qlzX7zFT/QdUV/LWsJwZBlaIBaJ+E2VH3d1gArGVP+9hUHGpJkEzCSBK7yuZrkt+M/U0Jre5+maPRmkinEF4DA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@ungap/structured-clone": "^1.2.0", + "jest-util": "30.0.0-alpha.6", + "merge-stream": "^2.0.0", + "supports-color": "^8.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/reporters/node_modules/minimatch": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@jest/reporters/node_modules/picomatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", + "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/@jest/reporters/node_modules/pretty-format": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.0.0-alpha.6.tgz", + "integrity": "sha512-xkeffkZoqQmRrcNewpOsUCKNOl+CkPqjt3Ld749uz1S7/O7GuPNPv2fZk3v/1U/FE8/B4Zz0llVL80MKON1tOQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/schemas": "30.0.0-alpha.6", + "ansi-styles": "^5.0.0", + "react-is": "^18.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/schemas": { + "version": "29.6.0", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-29.6.0.tgz", + "integrity": "sha512-rxLjXyJBTL4LQeJW3aKo0M/+GkCOXsO+8i9Iu7eDb6KwtP65ayoDsitrdPBtujxQ88k4wI2FNYfa6TOGwSn6cQ==", + "dev": true, + "dependencies": { + "@sinclair/typebox": "^0.27.8" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/snapshot-utils": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/snapshot-utils/-/snapshot-utils-30.0.0-alpha.6.tgz", + "integrity": "sha512-iDtIFCyRT8ZyLmz6kYbS8GR/MBXKA6uZPBfdTcnd2y0T987DV3GVlvwkAC+iFTc1w3HgwQe8LTf+y3i+O2ISCw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "30.0.0-alpha.6", + "chalk": "^4.0.0", + "graceful-fs": "^4.2.9", + "natural-compare": "^1.4.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/snapshot-utils/node_modules/@jest/schemas": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", + "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.33.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/snapshot-utils/node_modules/@jest/types": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", + "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/pattern": "30.0.0-alpha.6", + "@jest/schemas": "30.0.0-alpha.6", + "@types/istanbul-lib-coverage": "^2.0.0", + "@types/istanbul-reports": "^3.0.0", + "@types/node": "*", + "@types/yargs": "^17.0.8", + "chalk": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/snapshot-utils/node_modules/@sinclair/typebox": { + "version": "0.33.17", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", + "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jest/source-map": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/source-map/-/source-map-30.0.0-alpha.6.tgz", + "integrity": "sha512-7rSrxehVyzqw5O+F2ds7wLAm9f6QxqYsJU42LNyUpaFlJqtWz3PeQ2Wu3DVoPzGu0C66EhDHKYmeN0mXnRDZmQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.18", + "callsites": "^3.0.0", + "graceful-fs": "^4.2.9" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/test-result": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/test-result/-/test-result-30.0.0-alpha.6.tgz", + "integrity": "sha512-Jlg8lCm7VQ6YvQ0eZx2nQEtej/ng+ulV8cXH7Nj5i33hNZq8EZvWM4gQDWDzRe1X7cVE3Bd42On5f6s2rqqIjw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/console": "30.0.0-alpha.6", + "@jest/types": "30.0.0-alpha.6", + "@types/istanbul-lib-coverage": "^2.0.0", + "collect-v8-coverage": "^1.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/test-result/node_modules/@jest/schemas": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", + "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.33.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/test-result/node_modules/@jest/types": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", + "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/pattern": "30.0.0-alpha.6", + "@jest/schemas": "30.0.0-alpha.6", + "@types/istanbul-lib-coverage": "^2.0.0", + "@types/istanbul-reports": "^3.0.0", + "@types/node": "*", + "@types/yargs": "^17.0.8", + "chalk": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/test-result/node_modules/@sinclair/typebox": { + "version": "0.33.17", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", + "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jest/test-sequencer": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/test-sequencer/-/test-sequencer-30.0.0-alpha.6.tgz", + "integrity": "sha512-5M89jbSQWkBjGlFrRk2wXjRJVxR+uN553sFN0q2TglH0/a4OMSVxRBcCmnIqqDMDizGAlYTxW6BaXxHGHpvrRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/test-result": "30.0.0-alpha.6", + "graceful-fs": "^4.2.9", + "jest-haste-map": "30.0.0-alpha.6", + "slash": "^3.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/transform": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/transform/-/transform-30.0.0-alpha.6.tgz", + "integrity": "sha512-4L8BZm38BJASswsqruc4c3F0AExYLvp0xq8067e7fIyg4hfwa4zUA+N2idf+eTTjDWevVVdIBfELzJ8b7nvO4Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.11.6", + "@jest/types": "30.0.0-alpha.6", + "@jridgewell/trace-mapping": "^0.3.18", + "babel-plugin-istanbul": "^7.0.0", + "chalk": "^4.0.0", + "convert-source-map": "^2.0.0", + "fast-json-stable-stringify": "^2.1.0", + "graceful-fs": "^4.2.9", + "jest-haste-map": "30.0.0-alpha.6", + "jest-regex-util": "30.0.0-alpha.6", + "jest-util": "30.0.0-alpha.6", + "micromatch": "^4.0.7", + "pirates": "^4.0.4", + "slash": "^3.0.0", + "write-file-atomic": "^5.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/transform/node_modules/@jest/schemas": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", + "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.33.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/transform/node_modules/@jest/types": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", + "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/pattern": "30.0.0-alpha.6", + "@jest/schemas": "30.0.0-alpha.6", + "@types/istanbul-lib-coverage": "^2.0.0", + "@types/istanbul-reports": "^3.0.0", + "@types/node": "*", + "@types/yargs": "^17.0.8", + "chalk": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/transform/node_modules/@sinclair/typebox": { + "version": "0.33.17", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", + "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jest/transform/node_modules/ci-info": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.0.0.tgz", + "integrity": "sha512-TdHqgGf9odd8SXNuxtUBVx8Nv+qZOejE6qyqiy5NtbYYQOeFa6zmHkxlPzmaLxWWHsU6nJmB7AETdVPi+2NBUg==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/sibiraj-s" + } + ], + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/@jest/transform/node_modules/jest-util": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.0.0-alpha.6.tgz", + "integrity": "sha512-JlimakOVDyoMC8TEG+knoufxUqLG+Btihf1G8o5sHxz54C6oL54Wetfepp+Nhuj/1hSL0sQtkovvjlEycf9i0w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "30.0.0-alpha.6", + "@types/node": "*", + "chalk": "^4.0.0", + "ci-info": "^4.0.0", + "graceful-fs": "^4.2.9", + "picomatch": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/@jest/transform/node_modules/picomatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", + "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/@jest/types": { + "version": "29.6.1", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-29.6.1.tgz", + "integrity": "sha512-tPKQNMPuXgvdOn2/Lg9HNfUvjYVGolt04Hp03f5hAk878uwOLikN+JzeLY0HcVgKgFl9Hs3EIqpu3WX27XNhnw==", + "dev": true, + "dependencies": { + "@jest/schemas": "^29.6.0", + "@types/istanbul-lib-coverage": "^2.0.0", + "@types/istanbul-reports": "^3.0.0", + "@types/node": "*", + "@types/yargs": "^17.0.8", + "chalk": "^4.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.5.tgz", + "integrity": "sha512-IzL8ZoEDIBRWEzlCcRhOaCupYyN5gdIK+Q6fbFdPDg6HqX6jpkItn7DFIpW9LQzXG6Df9sA7+OKnq0qlz/GaQg==", + "dev": true, + "dependencies": { + "@jridgewell/set-array": "^1.2.1", + "@jridgewell/sourcemap-codec": "^1.4.10", + "@jridgewell/trace-mapping": "^0.3.24" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz", + "integrity": "sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w==", + "dev": true, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/set-array": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.2.1.tgz", + "integrity": "sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A==", + "dev": true, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/source-map": { + "version": "0.3.6", + "resolved": "https://registry.npmjs.org/@jridgewell/source-map/-/source-map-0.3.6.tgz", + "integrity": "sha512-1ZJTZebgqllO79ue2bm3rIGud/bOe0pP5BjSRCRxxYkEZS8STV7zN84UBbiYu7jy+eCKSnVIUgoWWE/tt+shMQ==", + "dev": true, + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.25" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.4.15", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz", + "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==", + "dev": true + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.25", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz", + "integrity": "sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==", + "dev": true, + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@jsdoc/salty": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/@jsdoc/salty/-/salty-0.2.5.tgz", + "integrity": "sha512-TfRP53RqunNe2HBobVBJ0VLhK1HbfvBYeTC1ahnN64PWvyYyGebmMiPkuwvD9fpw2ZbkoPb8Q7mwy0aR8Z9rvw==", + "dev": true, + "dependencies": { + "lodash": "^4.17.21" + }, + "engines": { + "node": ">=v12.0.0" + } + }, + "node_modules/@leichtgewicht/ip-codec": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@leichtgewicht/ip-codec/-/ip-codec-2.0.4.tgz", + "integrity": "sha512-Hcv+nVC0kZnQ3tD9GVu5xSMR4VVYOteQIr/hwFPVEvPdlXqgGEuRjiheChHgdM+JyqdgNcmzZOX/tnl0JOiI7A==", + "dev": true + }, + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "optional": true, + "engines": { + "node": ">=14" + } + }, + "node_modules/@pkgr/core": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@pkgr/core/-/core-0.1.1.tgz", + "integrity": "sha512-cq8o4cWH0ibXh9VGi5P20Tu9XF/0fFXl9EUinr9QfTM7a7p0oTA4iJRCQWppXR1Pg8dSM0UCItCkPwsk9qWWYA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.18.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/unts" + } + }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==" + }, + "node_modules/@protobufjs/inquire": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", + "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", + "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" + }, + "node_modules/@sinclair/typebox": { + "version": "0.27.8", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz", + "integrity": "sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==", + "dev": true + }, + "node_modules/@sinonjs/commons": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.1.tgz", + "integrity": "sha512-K3mCHKQ9sVh8o1C9cxkwxaOmXoAMlDxC1mYyHrjqOWEcBjYr76t96zL2zlj5dUGZ3HSw240X1qgH3Mjf1yJWpQ==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "type-detect": "4.0.8" + } + }, + "node_modules/@sinonjs/fake-timers": { + "version": "11.3.1", + "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-11.3.1.tgz", + "integrity": "sha512-EVJO7nW5M/F5Tur0Rf2z/QoMo+1Ia963RiMtapiQrEWvY0iBUvADo8Beegwjpnle5BHkyHuoxSTW3jF43H1XRA==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@sinonjs/commons": "^3.0.1" + } + }, + "node_modules/@types/babel__core": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", + "integrity": "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.20.7", + "@babel/types": "^7.20.7", + "@types/babel__generator": "*", + "@types/babel__template": "*", + "@types/babel__traverse": "*" + } + }, + "node_modules/@types/babel__generator": { + "version": "7.6.8", + "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.6.8.tgz", + "integrity": "sha512-ASsj+tpEDsEiFr1arWrlN6V3mdfjRMZt6LtK/Vp/kreFLnr5QH5+DhvD5nINYZXzwJvXeGq+05iUXcAzVrqWtw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__template": { + "version": "7.4.4", + "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.4.tgz", + "integrity": "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.1.0", + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__traverse": { + "version": "7.20.6", + "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.20.6.tgz", + "integrity": "sha512-r1bzfrm0tomOI8g1SzvCaQHo6Lcv6zu0EA+W2kHrt8dyrHQxGzBBL4kdkzIS+jBMV+EYcMAEAqXqYaLJq5rOZg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.20.7" + } + }, + "node_modules/@types/body-parser": { + "version": "1.19.2", + "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.2.tgz", + "integrity": "sha512-ALYone6pm6QmwZoAgeyNksccT9Q4AWZQ6PvfwR37GT6r6FWUPguq6sUmNGSMV2Wr761oQoBxwGGa6DR5o1DC9g==", + "dev": true, + "dependencies": { + "@types/connect": "*", + "@types/node": "*" + } + }, + "node_modules/@types/bonjour": { + "version": "3.5.10", + "resolved": "https://registry.npmjs.org/@types/bonjour/-/bonjour-3.5.10.tgz", + "integrity": "sha512-p7ienRMiS41Nu2/igbJxxLDWrSZ0WxM8UQgCeO9KhoVF7cOVFkrKsiDr1EsJIla8vV3oEEjGcz11jc5yimhzZw==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/connect": { + "version": "3.4.35", + "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz", + "integrity": "sha512-cdeYyv4KWoEgpBISTxWvqYsVy444DOqehiF3fM3ne10AmJ62RSyNkUnxMJXHQWRQQX2eR94m5y1IZyDwBjV9FQ==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/connect-history-api-fallback": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/@types/connect-history-api-fallback/-/connect-history-api-fallback-1.3.5.tgz", + "integrity": "sha512-h8QJa8xSb1WD4fpKBDcATDNGXghFj6/3GRWG6dhmRcu0RX1Ubasur2Uvx5aeEwlf0MwblEC2bMzzMQntxnw/Cw==", + "dev": true, + "dependencies": { + "@types/express-serve-static-core": "*", + "@types/node": "*" + } + }, + "node_modules/@types/estree": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.5.tgz", + "integrity": "sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw==", + "dev": true + }, + "node_modules/@types/express": { + "version": "4.17.17", + "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.17.tgz", + "integrity": "sha512-Q4FmmuLGBG58btUnfS1c1r/NQdlp3DMfGDGig8WhfpA2YRUtEkxAjkZb0yvplJGYdF1fsQ81iMDcH24sSCNC/Q==", + "dev": true, + "dependencies": { + "@types/body-parser": "*", + "@types/express-serve-static-core": "^4.17.33", + "@types/qs": "*", + "@types/serve-static": "*" + } + }, + "node_modules/@types/express-serve-static-core": { + "version": "4.17.33", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.33.tgz", + "integrity": "sha512-TPBqmR/HRYI3eC2E5hmiivIzv+bidAfXofM+sbonAGvyDhySGw9/PQZFt2BLOrjUUR++4eJVpx6KnLQK1Fk9tA==", + "dev": true, + "dependencies": { + "@types/node": "*", + "@types/qs": "*", + "@types/range-parser": "*" + } + }, + "node_modules/@types/http-proxy": { + "version": "1.17.10", + "resolved": "https://registry.npmjs.org/@types/http-proxy/-/http-proxy-1.17.10.tgz", + "integrity": "sha512-Qs5aULi+zV1bwKAg5z1PWnDXWmsn+LxIvUGv6E2+OOMYhclZMO+OXd9pYVf2gLykf2I7IV2u7oTHwChPNsvJ7g==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/istanbul-lib-coverage": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.4.tgz", + "integrity": "sha512-z/QT1XN4K4KYuslS23k62yDIDLwLFkzxOuMplDtObz0+y7VqJCaO2o+SPwHCvLFZh7xazvvoor2tA/hPz9ee7g==", + "dev": true + }, + "node_modules/@types/istanbul-lib-report": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@types/istanbul-lib-report/-/istanbul-lib-report-3.0.0.tgz", + "integrity": "sha512-plGgXAPfVKFoYfa9NpYDAkseG+g6Jr294RqeqcqDixSbU34MZVJRi/P+7Y8GDpzkEwLaGZZOpKIEmeVZNtKsrg==", + "dev": true, + "dependencies": { + "@types/istanbul-lib-coverage": "*" + } + }, + "node_modules/@types/istanbul-reports": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.1.tgz", + "integrity": "sha512-c3mAZEuK0lvBp8tmuL74XRKn1+y2dcwOUpH7x4WrF6gk1GIgiluDRgMYQtw2OFcBvAJWlt6ASU3tSqxp0Uu0Aw==", + "dev": true, + "dependencies": { + "@types/istanbul-lib-report": "*" + } + }, + "node_modules/@types/jest": { + "version": "29.5.14", + "resolved": "https://registry.npmjs.org/@types/jest/-/jest-29.5.14.tgz", + "integrity": "sha512-ZN+4sdnLUbo8EVvVc2ao0GFW6oVrQRPn4K2lglySj7APvSrgzxHiNNK99us4WDMi57xxA2yggblIAMNhXOotLQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "expect": "^29.0.0", + "pretty-format": "^29.0.0" + } + }, + "node_modules/@types/json-schema": { + "version": "7.0.11", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.11.tgz", + "integrity": "sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ==", + "dev": true + }, + "node_modules/@types/linkify-it": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/linkify-it/-/linkify-it-3.0.2.tgz", + "integrity": "sha512-HZQYqbiFVWufzCwexrvh694SOim8z2d+xJl5UNamcvQFejLY/2YUtzXHYi3cHdI7PMlS8ejH2slRAOJQ32aNbA==", + "dev": true + }, + "node_modules/@types/markdown-it": { + "version": "12.2.3", + "resolved": "https://registry.npmjs.org/@types/markdown-it/-/markdown-it-12.2.3.tgz", + "integrity": "sha512-GKMHFfv3458yYy+v/N8gjufHO6MSZKCOXpZc5GXIWWy8uldwfmPn98vp81gZ5f9SVw8YYBctgfJ22a2d7AOMeQ==", + "dev": true, + "dependencies": { + "@types/linkify-it": "*", + "@types/mdurl": "*" + } + }, + "node_modules/@types/mdurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@types/mdurl/-/mdurl-1.0.2.tgz", + "integrity": "sha512-eC4U9MlIcu2q0KQmXszyn5Akca/0jrQmwDRgpAMJai7qBWq4amIQhZyNau4VYGtCeALvW1/NtjzJJ567aZxfKA==", + "dev": true + }, + "node_modules/@types/mime": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@types/mime/-/mime-3.0.1.tgz", + "integrity": "sha512-Y4XFY5VJAuw0FgAqPNd6NNoV44jbq9Bz2L7Rh/J6jLTiHBSBJa9fxqQIvkIld4GsoDOcCbvzOUAbLPsSKKg+uA==", + "dev": true + }, + "node_modules/@types/node": { + "version": "18.14.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.14.4.tgz", + "integrity": "sha512-VhCw7I7qO2X49+jaKcAUwi3rR+hbxT5VcYF493+Z5kMLI0DL568b7JI4IDJaxWFH0D/xwmGJNoXisyX+w7GH/g==" + }, + "node_modules/@types/qs": { + "version": "6.9.7", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.7.tgz", + "integrity": "sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==", + "dev": true + }, + "node_modules/@types/range-parser": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.4.tgz", + "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==", + "dev": true + }, + "node_modules/@types/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==", + "dev": true + }, + "node_modules/@types/serve-index": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@types/serve-index/-/serve-index-1.9.1.tgz", + "integrity": "sha512-d/Hs3nWDxNL2xAczmOVZNj92YZCS6RGxfBPjKzuu/XirCgXdpKEb88dYNbrYGint6IVWLNP+yonwVAuRC0T2Dg==", + "dev": true, + "dependencies": { + "@types/express": "*" + } + }, + "node_modules/@types/serve-static": { + "version": "1.15.1", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.1.tgz", + "integrity": "sha512-NUo5XNiAdULrJENtJXZZ3fHtfMolzZwczzBbnAeBbqBwG+LaG6YaJtuwzwGSQZ2wsCrxjEhNNjAkKigy3n8teQ==", + "dev": true, + "dependencies": { + "@types/mime": "*", + "@types/node": "*" + } + }, + "node_modules/@types/sockjs": { + "version": "0.3.33", + "resolved": "https://registry.npmjs.org/@types/sockjs/-/sockjs-0.3.33.tgz", + "integrity": "sha512-f0KEEe05NvUnat+boPTZ0dgaLZ4SfSouXUgv5noUiefG2ajgKjmETo9ZJyuqsl7dfl2aHlLJUiki6B4ZYldiiw==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/stack-utils": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.1.tgz", + "integrity": "sha512-Hl219/BT5fLAaz6NDkSuhzasy49dwQS/DSdu4MdggFB8zcXv7vflBI3xp7FEmkmdDkBUI2bPUNeMttp2knYdxw==", + "dev": true + }, + "node_modules/@types/ws": { + "version": "8.5.4", + "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.4.tgz", + "integrity": "sha512-zdQDHKUgcX/zBc4GrwsE/7dVdAD8JR4EuiAXiiUhhfyIJXXb2+PrGshFyeXWQPMmmZ2XxgaqclgpIC7eTXc1mg==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/yargs": { + "version": "17.0.24", + "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.24.tgz", + "integrity": "sha512-6i0aC7jV6QzQB8ne1joVZ0eSFIstHsCrobmOtghM11yGlH0j43FKL2UhWdELkyps0zuf7qVTUVCCR+tgSlyLLw==", + "dev": true, + "dependencies": { + "@types/yargs-parser": "*" + } + }, + "node_modules/@types/yargs-parser": { + "version": "21.0.0", + "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-21.0.0.tgz", + "integrity": "sha512-iO9ZQHkZxHn4mSakYV0vFHAVDyEOIJQrV2uZ06HxEPcx+mt8swXoZHIbaaJ2crJYFfErySgktuTZ3BeLz+XmFA==", + "dev": true + }, + "node_modules/@ungap/structured-clone": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz", + "integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/@webassemblyjs/ast": { + "version": "1.12.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.12.1.tgz", + "integrity": "sha512-EKfMUOPRRUTy5UII4qJDGPpqfwjOmZ5jeGFwid9mnoqIFK+e0vqoi1qH56JpmZSzEL53jKnNzScdmftJyG5xWg==", + "dev": true, + "dependencies": { + "@webassemblyjs/helper-numbers": "1.11.6", + "@webassemblyjs/helper-wasm-bytecode": "1.11.6" + } + }, + "node_modules/@webassemblyjs/floating-point-hex-parser": { + "version": "1.11.6", + "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.11.6.tgz", + "integrity": "sha512-ejAj9hfRJ2XMsNHk/v6Fu2dGS+i4UaXBXGemOfQ/JfQ6mdQg/WXtwleQRLLS4OvfDhv8rYnVwH27YJLMyYsxhw==", + "dev": true + }, + "node_modules/@webassemblyjs/helper-api-error": { + "version": "1.11.6", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.11.6.tgz", + "integrity": "sha512-o0YkoP4pVu4rN8aTJgAyj9hC2Sv5UlkzCHhxqWj8butaLvnpdc2jOwh4ewE6CX0txSfLn/UYaV/pheS2Txg//Q==", + "dev": true + }, + "node_modules/@webassemblyjs/helper-buffer": { + "version": "1.12.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.12.1.tgz", + "integrity": "sha512-nzJwQw99DNDKr9BVCOZcLuJJUlqkJh+kVzVl6Fmq/tI5ZtEyWT1KZMyOXltXLZJmDtvLCDgwsyrkohEtopTXCw==", + "dev": true + }, + "node_modules/@webassemblyjs/helper-numbers": { + "version": "1.11.6", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-numbers/-/helper-numbers-1.11.6.tgz", + "integrity": "sha512-vUIhZ8LZoIWHBohiEObxVm6hwP034jwmc9kuq5GdHZH0wiLVLIPcMCdpJzG4C11cHoQ25TFIQj9kaVADVX7N3g==", + "dev": true, + "dependencies": { + "@webassemblyjs/floating-point-hex-parser": "1.11.6", + "@webassemblyjs/helper-api-error": "1.11.6", + "@xtuc/long": "4.2.2" + } + }, + "node_modules/@webassemblyjs/helper-wasm-bytecode": { + "version": "1.11.6", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.11.6.tgz", + "integrity": "sha512-sFFHKwcmBprO9e7Icf0+gddyWYDViL8bpPjJJl0WHxCdETktXdmtWLGVzoHbqUcY4Be1LkNfwTmXOJUFZYSJdA==", + "dev": true + }, + "node_modules/@webassemblyjs/helper-wasm-section": { + "version": "1.12.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.12.1.tgz", + "integrity": "sha512-Jif4vfB6FJlUlSbgEMHUyk1j234GTNG9dBJ4XJdOySoj518Xj0oGsNi59cUQF4RRMS9ouBUxDDdyBVfPTypa5g==", + "dev": true, + "dependencies": { + "@webassemblyjs/ast": "1.12.1", + "@webassemblyjs/helper-buffer": "1.12.1", + "@webassemblyjs/helper-wasm-bytecode": "1.11.6", + "@webassemblyjs/wasm-gen": "1.12.1" + } + }, + "node_modules/@webassemblyjs/ieee754": { + "version": "1.11.6", + "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.11.6.tgz", + "integrity": "sha512-LM4p2csPNvbij6U1f19v6WR56QZ8JcHg3QIJTlSwzFcmx6WSORicYj6I63f9yU1kEUtrpG+kjkiIAkevHpDXrg==", + "dev": true, + "dependencies": { + "@xtuc/ieee754": "^1.2.0" + } + }, + "node_modules/@webassemblyjs/leb128": { + "version": "1.11.6", + "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.11.6.tgz", + "integrity": "sha512-m7a0FhE67DQXgouf1tbN5XQcdWoNgaAuoULHIfGFIEVKA6tu/edls6XnIlkmS6FrXAquJRPni3ZZKjw6FSPjPQ==", + "dev": true, + "dependencies": { + "@xtuc/long": "4.2.2" + } + }, + "node_modules/@webassemblyjs/utf8": { + "version": "1.11.6", + "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.11.6.tgz", + "integrity": "sha512-vtXf2wTQ3+up9Zsg8sa2yWiQpzSsMyXj0qViVP6xKGCUT8p8YJ6HqI7l5eCnWx1T/FYdsv07HQs2wTFbbof/RA==", + "dev": true + }, + "node_modules/@webassemblyjs/wasm-edit": { + "version": "1.12.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.12.1.tgz", + "integrity": "sha512-1DuwbVvADvS5mGnXbE+c9NfA8QRcZ6iKquqjjmR10k6o+zzsRVesil54DKexiowcFCPdr/Q0qaMgB01+SQ1u6g==", + "dev": true, + "dependencies": { + "@webassemblyjs/ast": "1.12.1", + "@webassemblyjs/helper-buffer": "1.12.1", + "@webassemblyjs/helper-wasm-bytecode": "1.11.6", + "@webassemblyjs/helper-wasm-section": "1.12.1", + "@webassemblyjs/wasm-gen": "1.12.1", + "@webassemblyjs/wasm-opt": "1.12.1", + "@webassemblyjs/wasm-parser": "1.12.1", + "@webassemblyjs/wast-printer": "1.12.1" + } + }, + "node_modules/@webassemblyjs/wasm-gen": { + "version": "1.12.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.12.1.tgz", + "integrity": "sha512-TDq4Ojh9fcohAw6OIMXqiIcTq5KUXTGRkVxbSo1hQnSy6lAM5GSdfwWeSxpAo0YzgsgF182E/U0mDNhuA0tW7w==", + "dev": true, + "dependencies": { + "@webassemblyjs/ast": "1.12.1", + "@webassemblyjs/helper-wasm-bytecode": "1.11.6", + "@webassemblyjs/ieee754": "1.11.6", + "@webassemblyjs/leb128": "1.11.6", + "@webassemblyjs/utf8": "1.11.6" + } + }, + "node_modules/@webassemblyjs/wasm-opt": { + "version": "1.12.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.12.1.tgz", + "integrity": "sha512-Jg99j/2gG2iaz3hijw857AVYekZe2SAskcqlWIZXjji5WStnOpVoat3gQfT/Q5tb2djnCjBtMocY/Su1GfxPBg==", + "dev": true, + "dependencies": { + "@webassemblyjs/ast": "1.12.1", + "@webassemblyjs/helper-buffer": "1.12.1", + "@webassemblyjs/wasm-gen": "1.12.1", + "@webassemblyjs/wasm-parser": "1.12.1" + } + }, + "node_modules/@webassemblyjs/wasm-parser": { + "version": "1.12.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.12.1.tgz", + "integrity": "sha512-xikIi7c2FHXysxXe3COrVUPSheuBtpcfhbpFj4gmu7KRLYOzANztwUU0IbsqvMqzuNK2+glRGWCEqZo1WCLyAQ==", + "dev": true, + "dependencies": { + "@webassemblyjs/ast": "1.12.1", + "@webassemblyjs/helper-api-error": "1.11.6", + "@webassemblyjs/helper-wasm-bytecode": "1.11.6", + "@webassemblyjs/ieee754": "1.11.6", + "@webassemblyjs/leb128": "1.11.6", + "@webassemblyjs/utf8": "1.11.6" + } + }, + "node_modules/@webassemblyjs/wast-printer": { + "version": "1.12.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.12.1.tgz", + "integrity": "sha512-+X4WAlOisVWQMikjbcvY2e0rwPsKQ9F688lksZhBcPycBBuii3O7m8FACbDMWDojpAqvjIncrG8J0XHKyQfVeA==", + "dev": true, + "dependencies": { + "@webassemblyjs/ast": "1.12.1", + "@xtuc/long": "4.2.2" + } + }, + "node_modules/@webgpu/types": { + "version": "0.1.44", + "resolved": "https://registry.npmjs.org/@webgpu/types/-/types-0.1.44.tgz", + "integrity": "sha512-JDpYJN5E/asw84LTYhKyvPpxGnD+bAKPtpW9Ilurf7cZpxaTbxkQcGwOd7jgB9BPBrTYQ+32ufo4HiuomTjHNQ==", + "dev": true + }, + "node_modules/@webpack-cli/configtest": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@webpack-cli/configtest/-/configtest-2.0.1.tgz", + "integrity": "sha512-njsdJXJSiS2iNbQVS0eT8A/KPnmyH4pv1APj2K0d1wrZcBLw+yppxOy4CGqa0OxDJkzfL/XELDhD8rocnIwB5A==", + "dev": true, + "engines": { + "node": ">=14.15.0" + }, + "peerDependencies": { + "webpack": "5.x.x", + "webpack-cli": "5.x.x" + } + }, + "node_modules/@webpack-cli/info": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@webpack-cli/info/-/info-2.0.1.tgz", + "integrity": "sha512-fE1UEWTwsAxRhrJNikE7v4EotYflkEhBL7EbajfkPlf6E37/2QshOy/D48Mw8G5XMFlQtS6YV42vtbG9zBpIQA==", + "dev": true, + "engines": { + "node": ">=14.15.0" + }, + "peerDependencies": { + "webpack": "5.x.x", + "webpack-cli": "5.x.x" + } + }, + "node_modules/@webpack-cli/serve": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@webpack-cli/serve/-/serve-2.0.2.tgz", + "integrity": "sha512-S9h3GmOmzUseyeFW3tYNnWS7gNUuwxZ3mmMq0JyW78Vx1SGKPSkt5bT4pB0rUnVfHjP0EL9gW2bOzmtiTfQt0A==", + "dev": true, + "engines": { + "node": ">=14.15.0" + }, + "peerDependencies": { + "webpack": "5.x.x", + "webpack-cli": "5.x.x" + }, + "peerDependenciesMeta": { + "webpack-dev-server": { + "optional": true + } + } + }, + "node_modules/@xtuc/ieee754": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@xtuc/ieee754/-/ieee754-1.2.0.tgz", + "integrity": "sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA==", + "dev": true + }, + "node_modules/@xtuc/long": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/@xtuc/long/-/long-4.2.2.tgz", + "integrity": "sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==", + "dev": true + }, + "node_modules/accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "dev": true, + "dependencies": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/acorn": { + "version": "8.12.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.12.1.tgz", + "integrity": "sha512-tcpGyI9zbizT9JbV6oYE477V6mTlXvvi0T0G3SNIYE2apm/G5huBa1+K89VGeovbg+jycCrfhl3ADxErOuO6Jg==", + "dev": true, + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-import-attributes": { + "version": "1.9.5", + "resolved": "https://registry.npmjs.org/acorn-import-attributes/-/acorn-import-attributes-1.9.5.tgz", + "integrity": "sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ==", + "dev": true, + "peerDependencies": { + "acorn": "^8" + } + }, + "node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-2.1.1.tgz", + "integrity": "sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==", + "dev": true, + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/ajv-formats/node_modules/ajv": { + "version": "8.12.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz", + "integrity": "sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==", + "dev": true, + "dependencies": { + "fast-deep-equal": "^3.1.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats/node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true + }, + "node_modules/ajv-keywords": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz", + "integrity": "sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==", + "dev": true, + "peerDependencies": { + "ajv": "^6.9.1" + } + }, + "node_modules/ansi-escape-sequences": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-escape-sequences/-/ansi-escape-sequences-4.1.0.tgz", + "integrity": "sha512-dzW9kHxH011uBsidTXd14JXgzye/YLb2LzeKZ4bsgl/Knwx8AtbSFkkGxagdNOoh0DlqHCmfiEjWKBaqjOanVw==", + "dev": true, + "dependencies": { + "array-back": "^3.0.1" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/ansi-escape-sequences/node_modules/array-back": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-3.1.0.tgz", + "integrity": "sha512-TkuxA4UCOvxuDK6NZYXCalszEzj+TLszyASooky+i742l9TqsOdYCMJJupxRic61hwquNtppB3hgcuq9SVSH1Q==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/ansi-escapes": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", + "integrity": "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "type-fest": "^0.21.3" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ansi-html-community": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/ansi-html-community/-/ansi-html-community-0.0.8.tgz", + "integrity": "sha512-1APHAyr3+PCamwNw3bXCPp4HFLONZt/yIH0sZp0/469KWNTEy+qN5jQ3GVX6DMZ1UXAi34yVwtTeaG/HpBuuzw==", + "dev": true, + "engines": [ + "node >= 0.8.0" + ], + "bin": { + "ansi-html": "bin/ansi-html" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/anymatch": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", + "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", + "dev": true, + "dependencies": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true + }, + "node_modules/array-back": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-6.2.2.tgz", + "integrity": "sha512-gUAZ7HPyb4SJczXAMUXMGAvI976JoK3qEx9v1FTmeYuJj0IBiaKttG1ydtGKdkfqWkIkouke7nG8ufGy77+Cvw==", + "dev": true, + "engines": { + "node": ">=12.17" + } + }, + "node_modules/array-flatten": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-2.1.2.tgz", + "integrity": "sha512-hNfzcOV8W4NdualtqBFPyVO+54DSJuZGY9qT4pRroB6S9e3iiido2ISIC5h9R2sPJ8H3FHCIiEnsv1lPXO3KtQ==", + "dev": true + }, + "node_modules/babel-jest": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-30.0.0-alpha.6.tgz", + "integrity": "sha512-WOQkqpBz2q8d/AT6D6rZXW5xnKHDmk3kIukaXlzUyoBBIvLh1SEvi2RGS4fboEtS0kNkyL+zf1rSfkt5OCIgmw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/transform": "30.0.0-alpha.6", + "@types/babel__core": "^7.1.14", + "babel-plugin-istanbul": "^7.0.0", + "babel-preset-jest": "30.0.0-alpha.6", + "chalk": "^4.0.0", + "graceful-fs": "^4.2.9", + "slash": "^3.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + }, + "peerDependencies": { + "@babel/core": "^7.11.0" + } + }, + "node_modules/babel-plugin-istanbul": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/babel-plugin-istanbul/-/babel-plugin-istanbul-7.0.0.tgz", + "integrity": "sha512-C5OzENSx/A+gt7t4VH1I2XsflxyPUmXRFPKBxt33xncdOmq7oROVM3bZv9Ysjjkv8OJYDMa+tKuKMvqU/H3xdw==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@babel/helper-plugin-utils": "^7.0.0", + "@istanbuljs/load-nyc-config": "^1.0.0", + "@istanbuljs/schema": "^0.1.3", + "istanbul-lib-instrument": "^6.0.2", + "test-exclude": "^6.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/babel-plugin-jest-hoist": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/babel-plugin-jest-hoist/-/babel-plugin-jest-hoist-30.0.0-alpha.6.tgz", + "integrity": "sha512-e/aPv0pmnvJqXM5SfCBpyMwZFEZrKW1Mb4unwTkxewk6/0TjwBk6l3B3F9H9OKZ3ErhkH4b+Epd3IIM5E53I2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/template": "^7.3.3", + "@babel/types": "^7.3.3", + "@types/babel__core": "^7.1.14" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/babel-preset-current-node-syntax": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/babel-preset-current-node-syntax/-/babel-preset-current-node-syntax-1.1.0.tgz", + "integrity": "sha512-ldYss8SbBlWva1bs28q78Ju5Zq1F+8BrqBZZ0VFhLBvhh6lCpC2o3gDJi/5DRLs9FgYZCnmPYIVFU4lRXCkyUw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/plugin-syntax-async-generators": "^7.8.4", + "@babel/plugin-syntax-bigint": "^7.8.3", + "@babel/plugin-syntax-class-properties": "^7.12.13", + "@babel/plugin-syntax-class-static-block": "^7.14.5", + "@babel/plugin-syntax-import-attributes": "^7.24.7", + "@babel/plugin-syntax-import-meta": "^7.10.4", + "@babel/plugin-syntax-json-strings": "^7.8.3", + "@babel/plugin-syntax-logical-assignment-operators": "^7.10.4", + "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.3", + "@babel/plugin-syntax-numeric-separator": "^7.10.4", + "@babel/plugin-syntax-object-rest-spread": "^7.8.3", + "@babel/plugin-syntax-optional-catch-binding": "^7.8.3", + "@babel/plugin-syntax-optional-chaining": "^7.8.3", + "@babel/plugin-syntax-private-property-in-object": "^7.14.5", + "@babel/plugin-syntax-top-level-await": "^7.14.5" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/babel-preset-jest": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/babel-preset-jest/-/babel-preset-jest-30.0.0-alpha.6.tgz", + "integrity": "sha512-Xsis7RI2oT2zlyCIEzMtjDiES0wKoQxTUo5BGzx1q3ZemnDE1/7xTC4/lI4eBLmAtwk/hpZLRYwltvbQEvyRWw==", + "dev": true, + "license": "MIT", + "dependencies": { + "babel-plugin-jest-hoist": "30.0.0-alpha.6", + "babel-preset-current-node-syntax": "^1.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + }, + "peerDependencies": { + "@babel/core": "^7.11.0" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" + }, + "node_modules/batch": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/batch/-/batch-0.6.1.tgz", + "integrity": "sha512-x+VAiMRL6UPkx+kudNvxTl6hB2XNNCG2r+7wixVfIYwu/2HKRXimwQyaumLjMveWvT2Hkd/cAJw+QBMfJ/EKVw==", + "dev": true + }, + "node_modules/binary-extensions": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", + "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/bluebird": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", + "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==", + "dev": true + }, + "node_modules/body-parser": { + "version": "1.20.3", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", + "integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==", + "dev": true, + "dependencies": { + "bytes": "3.1.2", + "content-type": "~1.0.5", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "on-finished": "2.4.1", + "qs": "6.13.0", + "raw-body": "2.5.2", + "type-is": "~1.6.18", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/body-parser/node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/bonjour-service": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/bonjour-service/-/bonjour-service-1.1.1.tgz", + "integrity": "sha512-Z/5lQRMOG9k7W+FkeGTNjh7htqn/2LMnfOvBZ8pynNZCM9MwkQkI3zeI4oz09uWdcgmgHugVvBqxGg4VQJ5PCg==", + "dev": true, + "dependencies": { + "array-flatten": "^2.1.2", + "dns-equal": "^1.0.0", + "fast-deep-equal": "^3.1.3", + "multicast-dns": "^7.2.5" + } + }, + "node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "dev": true, + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/browserslist": { + "version": "4.24.2", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.24.2.tgz", + "integrity": "sha512-ZIc+Q62revdMcqC6aChtW4jz3My3klmCO1fEmINZY/8J3EpBg5/A/D0AKmBveUh6pgoeycoMkVMko84tuYS+Gg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "caniuse-lite": "^1.0.30001669", + "electron-to-chromium": "^1.5.41", + "node-releases": "^2.0.18", + "update-browserslist-db": "^1.1.1" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/bser": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/bser/-/bser-2.1.1.tgz", + "integrity": "sha512-gQxTNE/GAfIIrmHLUE3oJyp5FO6HRBfhjnw4/wMmA63ZGDJnWBmgY/lyQBpnDUkGmAhbSe39tx2d/iTOAfglwQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "node-int64": "^0.4.0" + } + }, + "node_modules/buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "dev": true + }, + "node_modules/bytes": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", + "integrity": "sha512-pMhOfFDPiv9t5jjIXkHosWmkSyQbvsgEVNkz0ERHbuLh2T/7j4Mqqpz523Fe8MVY89KC6Sh/QfS2sM+SjgFDcw==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/cache-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/cache-point/-/cache-point-2.0.0.tgz", + "integrity": "sha512-4gkeHlFpSKgm3vm2gJN5sPqfmijYRFYCQ6tv5cLw0xVmT6r1z1vd4FNnpuOREco3cBs1G709sZ72LdgddKvL5w==", + "dev": true, + "dependencies": { + "array-back": "^4.0.1", + "fs-then-native": "^2.0.0", + "mkdirp2": "^1.0.4" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cache-point/node_modules/array-back": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-4.0.2.tgz", + "integrity": "sha512-NbdMezxqf94cnNfWLL7V/im0Ub+Anbb0IoZhvzie8+4HJ4nMQuzHuy49FkGYCJK2yAloZ3meiB6AVMClbrI1vg==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/call-bind": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.7.tgz", + "integrity": "sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==", + "dev": true, + "dependencies": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.4", + "set-function-length": "^1.2.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/camelcase": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001677", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001677.tgz", + "integrity": "sha512-fmfjsOlJUpMWu+mAAtZZZHz7UEwsUxIIvu1TJfO1HqFQvB/B+ii0xr9B5HpbZY/mC4XZ8SvjHJqtAY6pDPQEog==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "CC-BY-4.0" + }, + "node_modules/catharsis": { + "version": "0.9.0", + "resolved": "git+ssh://git@github.com/xenova/catharsis.git#dda824bd3b6c08d418e8c8a03f20446cfb25aa1e", + "dev": true, + "license": "MIT", + "dependencies": { + "lodash": "^4.17.21" + }, + "engines": { + "node": ">= v18.16.0" + } + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/chalk/node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/char-regex": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/char-regex/-/char-regex-1.0.2.tgz", + "integrity": "sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/chokidar": { + "version": "3.5.3", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", + "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } + ], + "dependencies": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + }, + "engines": { + "node": ">= 8.10.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + } + }, + "node_modules/chownr": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", + "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==", + "engines": { + "node": ">=18" + } + }, + "node_modules/chrome-trace-event": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/chrome-trace-event/-/chrome-trace-event-1.0.3.tgz", + "integrity": "sha512-p3KULyQg4S7NIHixdwbGX+nFHkoBiA4YQmyWtjb8XngSKV124nJmRysgAeujbUVb15vh+RvFUfCPqU7rXk+hZg==", + "dev": true, + "engines": { + "node": ">=6.0" + } + }, + "node_modules/ci-info": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.8.0.tgz", + "integrity": "sha512-eXTggHWSooYhq49F2opQhuHWgzucfF2YgODK4e1566GQs5BIfP30B0oenwBJHfWxAs2fyPB1s7Mg949zLf61Yw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/sibiraj-s" + } + ], + "engines": { + "node": ">=8" + } + }, + "node_modules/cjs-module-lexer": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.4.1.tgz", + "integrity": "sha512-cuSVIHi9/9E/+821Qjdvngor+xpnlwnuwIyZOaLmHBVdXL+gP+I6QQB9VkO7RI77YIcTV+S1W9AreJ5eN63JBA==", + "dev": true, + "license": "MIT" + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/clone-deep": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-4.0.1.tgz", + "integrity": "sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ==", + "dev": true, + "dependencies": { + "is-plain-object": "^2.0.4", + "kind-of": "^6.0.2", + "shallow-clone": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/co": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", + "integrity": "sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==", + "dev": true, + "license": "MIT", + "engines": { + "iojs": ">= 1.0.0", + "node": ">= 0.12.0" + } + }, + "node_modules/collect-all": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/collect-all/-/collect-all-1.0.4.tgz", + "integrity": "sha512-RKZhRwJtJEP5FWul+gkSMEnaK6H3AGPTTWOiRimCcs+rc/OmQE3Yhy1Q7A7KsdkG3ZXVdZq68Y6ONSdvkeEcKA==", + "dev": true, + "dependencies": { + "stream-connect": "^1.0.2", + "stream-via": "^1.0.4" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/collect-v8-coverage": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/collect-v8-coverage/-/collect-v8-coverage-1.0.2.tgz", + "integrity": "sha512-lHl4d5/ONEbLlJvaJNtsF/Lz+WvB07u2ycqTYbdrq7UypDXailES4valYb2eWiJFxZlVmpGekfqoxQhzyFdT4Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/color": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz", + "integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==", + "dependencies": { + "color-convert": "^2.0.1", + "color-string": "^1.9.0" + }, + "engines": { + "node": ">=12.5.0" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, + "node_modules/color-string": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", + "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", + "dependencies": { + "color-name": "^1.0.0", + "simple-swizzle": "^0.2.2" + } + }, + "node_modules/colorette": { + "version": "2.0.20", + "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.20.tgz", + "integrity": "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==", + "dev": true + }, + "node_modules/command-line-args": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/command-line-args/-/command-line-args-5.2.1.tgz", + "integrity": "sha512-H4UfQhZyakIjC74I9d34fGYDwk3XpSr17QhEd0Q3I9Xq1CETHo4Hcuo87WyWHpAF1aSLjLRf5lD9ZGX2qStUvg==", + "dev": true, + "dependencies": { + "array-back": "^3.1.0", + "find-replace": "^3.0.0", + "lodash.camelcase": "^4.3.0", + "typical": "^4.0.0" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/command-line-args/node_modules/array-back": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-3.1.0.tgz", + "integrity": "sha512-TkuxA4UCOvxuDK6NZYXCalszEzj+TLszyASooky+i742l9TqsOdYCMJJupxRic61hwquNtppB3hgcuq9SVSH1Q==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/command-line-args/node_modules/typical": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/typical/-/typical-4.0.0.tgz", + "integrity": "sha512-VAH4IvQ7BDFYglMd7BPRDfLgxZZX4O4TFcRDA6EN5X7erNJJq+McIEp8np9aVtxrCJ6qx4GTYVfOWNjcqwZgRw==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/command-line-tool": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/command-line-tool/-/command-line-tool-0.8.0.tgz", + "integrity": "sha512-Xw18HVx/QzQV3Sc5k1vy3kgtOeGmsKIqwtFFoyjI4bbcpSgnw2CWVULvtakyw4s6fhyAdI6soQQhXc2OzJy62g==", + "dev": true, + "dependencies": { + "ansi-escape-sequences": "^4.0.0", + "array-back": "^2.0.0", + "command-line-args": "^5.0.0", + "command-line-usage": "^4.1.0", + "typical": "^2.6.1" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/command-line-tool/node_modules/array-back": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-2.0.0.tgz", + "integrity": "sha512-eJv4pLLufP3g5kcZry0j6WXpIbzYw9GUB4mVJZno9wfwiBxbizTnHCw3VJb07cBihbFX48Y7oSrW9y+gt4glyw==", + "dev": true, + "dependencies": { + "typical": "^2.6.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/command-line-usage": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/command-line-usage/-/command-line-usage-4.1.0.tgz", + "integrity": "sha512-MxS8Ad995KpdAC0Jopo/ovGIroV/m0KHwzKfXxKag6FHOkGsH8/lv5yjgablcRxCJJC0oJeUMuO/gmaq+Wq46g==", + "dev": true, + "dependencies": { + "ansi-escape-sequences": "^4.0.0", + "array-back": "^2.0.0", + "table-layout": "^0.4.2", + "typical": "^2.6.1" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/command-line-usage/node_modules/array-back": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-2.0.0.tgz", + "integrity": "sha512-eJv4pLLufP3g5kcZry0j6WXpIbzYw9GUB4mVJZno9wfwiBxbizTnHCw3VJb07cBihbFX48Y7oSrW9y+gt4glyw==", + "dev": true, + "dependencies": { + "typical": "^2.6.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", + "dev": true + }, + "node_modules/common-sequence": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/common-sequence/-/common-sequence-2.0.2.tgz", + "integrity": "sha512-jAg09gkdkrDO9EWTdXfv80WWH3yeZl5oT69fGfedBNS9pXUKYInVJ1bJ+/ht2+Moeei48TmSbQDYMc8EOx9G0g==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/compressible": { + "version": "2.0.18", + "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz", + "integrity": "sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==", + "dev": true, + "dependencies": { + "mime-db": ">= 1.43.0 < 2" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/compression": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/compression/-/compression-1.7.4.tgz", + "integrity": "sha512-jaSIDzP9pZVS4ZfQ+TzvtiWhdpFhE2RDHz8QJkpX9SIpLq88VueF5jJw6t+6CUQcAoA6t+x89MLrWAqpfDE8iQ==", + "dev": true, + "dependencies": { + "accepts": "~1.3.5", + "bytes": "3.0.0", + "compressible": "~2.0.16", + "debug": "2.6.9", + "on-headers": "~1.0.2", + "safe-buffer": "5.1.2", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/compression/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true + }, + "node_modules/config-master": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/config-master/-/config-master-3.1.0.tgz", + "integrity": "sha512-n7LBL1zBzYdTpF1mx5DNcZnZn05CWIdsdvtPL4MosvqbBUK3Rq6VWEtGUuF3Y0s9/CIhMejezqlSkP6TnCJ/9g==", + "dev": true, + "dependencies": { + "walk-back": "^2.0.1" + } + }, + "node_modules/config-master/node_modules/walk-back": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/walk-back/-/walk-back-2.0.1.tgz", + "integrity": "sha512-Nb6GvBR8UWX1D+Le+xUq0+Q1kFmRBIWVrfLnQAOmcpEzA9oAxwJ9gIr36t9TWYfzvWRvuMtjHiVsJYEkXWaTAQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/connect-history-api-fallback": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/connect-history-api-fallback/-/connect-history-api-fallback-2.0.0.tgz", + "integrity": "sha512-U73+6lQFmfiNPrYbXqr6kZ1i1wiRqXnp2nhMsINseWXO8lDau0LGEffJ8kQi4EjLZympVgRdvqjAgiZ1tgzDDA==", + "dev": true, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "dev": true, + "dependencies": { + "safe-buffer": "5.2.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true, + "license": "MIT" + }, + "node_modules/cookie": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.1.tgz", + "integrity": "sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==", + "dev": true + }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "dev": true + }, + "node_modules/cross-spawn": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/dedent": { + "version": "1.5.3", + "resolved": "https://registry.npmjs.org/dedent/-/dedent-1.5.3.tgz", + "integrity": "sha512-NHQtfOOW68WD8lgypbLA5oT+Bt0xXJhiYvoR6SmmNXZfpzOGXwdKWmcwG8N7PwVVWV3eF/68nmD9BaJSsTBhyQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "babel-plugin-macros": "^3.1.0" + }, + "peerDependenciesMeta": { + "babel-plugin-macros": { + "optional": true + } + } + }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "dev": true, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/deepmerge": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", + "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/default-gateway": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/default-gateway/-/default-gateway-6.0.3.tgz", + "integrity": "sha512-fwSOJsbbNzZ/CUFpqFBqYfYNLj1NbMPm8MMCIzHjC83iSJRBEGmDUxU+WP661BaBQImeC2yHwXtz+P/O9o+XEg==", + "dev": true, + "dependencies": { + "execa": "^5.0.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/define-data-property": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", + "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", + "dev": true, + "dependencies": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/define-lazy-prop": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-2.0.0.tgz", + "integrity": "sha512-Ds09qNh8yw3khSjiJjiUInaGX9xlqZDY7JVryGxdxV7NPeuqQfplOpQ66yJFZut3jLa5zOwkXw1g9EI2uKh4Og==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/destroy": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", + "dev": true, + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/detect-libc": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.3.tgz", + "integrity": "sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==", + "engines": { + "node": ">=8" + } + }, + "node_modules/detect-newline": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", + "integrity": "sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/detect-node": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz", + "integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==", + "dev": true + }, + "node_modules/diff-sequences": { + "version": "29.4.3", + "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-29.4.3.tgz", + "integrity": "sha512-ofrBgwpPhCD85kMKtE9RYFFq6OC1A89oW2vvgWZNCwxrUpRUILopY7lsYyMDSjc8g6U6aiO0Qubg6r4Wgt5ZnA==", + "dev": true, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/dmd": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/dmd/-/dmd-6.2.0.tgz", + "integrity": "sha512-uXWxLF1H7TkUAuoHK59/h/ts5cKavm2LnhrIgJWisip4BVzPoXavlwyoprFFn2CzcahKYgvkfaebS6oxzgflkg==", + "dev": true, + "dependencies": { + "array-back": "^6.2.2", + "cache-point": "^2.0.0", + "common-sequence": "^2.0.2", + "file-set": "^4.0.2", + "handlebars": "^4.7.7", + "marked": "^4.2.3", + "object-get": "^2.1.1", + "reduce-flatten": "^3.0.1", + "reduce-unique": "^2.0.1", + "reduce-without": "^1.0.1", + "test-value": "^3.0.0", + "walk-back": "^5.1.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/dns-equal": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/dns-equal/-/dns-equal-1.0.0.tgz", + "integrity": "sha512-z+paD6YUQsk+AbGCEM4PrOXSss5gd66QfcVBFTKR/HpFL9jCqikS94HYwKww6fQyO7IxrIIyUu+g0Ka9tUS2Cg==", + "dev": true + }, + "node_modules/dns-packet": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/dns-packet/-/dns-packet-5.6.0.tgz", + "integrity": "sha512-rza3UH1LwdHh9qyPXp8lkwpjSNk/AMD3dPytUoRoqnypDUhY0xvbdmVhWOfxO68frEfV9BU8V12Ez7ZsHGZpCQ==", + "dev": true, + "dependencies": { + "@leichtgewicht/ip-codec": "^2.0.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==" + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "dev": true + }, + "node_modules/electron-to-chromium": { + "version": "1.5.50", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.50.tgz", + "integrity": "sha512-eMVObiUQ2LdgeO1F/ySTXsvqvxb6ZH2zPGaMYsWzRDdOddUa77tdmI0ltg+L16UpbWdhPmuF3wIQYyQq65WfZw==", + "dev": true, + "license": "ISC" + }, + "node_modules/emittery": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/emittery/-/emittery-0.13.1.tgz", + "integrity": "sha512-DeWwawk6r5yR9jFgnDKYt4sLS0LmHJJi3ZOnb5/JdbYwj3nW+FxQnHIjhBKz8YLC7oRNPVM9NQ47I3CVx34eqQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sindresorhus/emittery?sponsor=1" + } + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/enhanced-resolve": { + "version": "5.17.1", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.17.1.tgz", + "integrity": "sha512-LMHl3dXhTcfv8gM4kEzIUeTQ+7fpdA0l2tUf34BddXPkz2A5xJ5L/Pchd5BL6rdccM9QGvu0sWZzK1Z1t4wwyg==", + "dev": true, + "dependencies": { + "graceful-fs": "^4.2.4", + "tapable": "^2.2.0" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/entities": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-2.1.0.tgz", + "integrity": "sha512-hCx1oky9PFrJ611mf0ifBLBRW8lUUVRlFolb5gWRfIELabBlbp9xZvrqZLZAs+NxFnbfQoeGd8wDkygjg7U85w==", + "dev": true, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/envinfo": { + "version": "7.8.1", + "resolved": "https://registry.npmjs.org/envinfo/-/envinfo-7.8.1.tgz", + "integrity": "sha512-/o+BXHmB7ocbHEAs6F2EnG0ogybVVUdkRunTT2glZU9XAaGmhqskrvKwqXuDfNjEO0LZKWdejEEpnq8aM0tOaw==", + "dev": true, + "bin": { + "envinfo": "dist/cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/error-ex": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", + "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-arrayish": "^0.2.1" + } + }, + "node_modules/error-ex/node_modules/is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", + "dev": true, + "license": "MIT" + }, + "node_modules/es-define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz", + "integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==", + "dev": true, + "dependencies": { + "get-intrinsic": "^1.2.4" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "dev": true, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-module-lexer": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.2.1.tgz", + "integrity": "sha512-9978wrXM50Y4rTMmW5kXIC09ZdXQZqkE4mxhwkd8VbzsGkXGPgV4zWuqQJgCEzYngdo2dYDa0l8xhX4fkSwJSg==", + "dev": true + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "dev": true + }, + "node_modules/escape-string-regexp": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz", + "integrity": "sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/eslint-scope": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz", + "integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==", + "dev": true, + "dependencies": { + "esrecurse": "^4.3.0", + "estraverse": "^4.1.1" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true, + "license": "BSD-2-Clause", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, + "dependencies": { + "estraverse": "^5.2.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esrecurse/node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estraverse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", + "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", + "dev": true, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/eventemitter3": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", + "dev": true + }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "dev": true, + "engines": { + "node": ">=0.8.x" + } + }, + "node_modules/execa": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", + "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==", + "dev": true, + "dependencies": { + "cross-spawn": "^7.0.3", + "get-stream": "^6.0.0", + "human-signals": "^2.1.0", + "is-stream": "^2.0.0", + "merge-stream": "^2.0.0", + "npm-run-path": "^4.0.1", + "onetime": "^5.1.2", + "signal-exit": "^3.0.3", + "strip-final-newline": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sindresorhus/execa?sponsor=1" + } + }, + "node_modules/exit": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz", + "integrity": "sha512-Zk/eNKV2zbjpKzrsQ+n1G6poVbErQxJ0LBOJXaKZ1EViLzH+hrLu9cdXI4zw9dBQJslwBEpbQ2P1oS7nDxs6jQ==", + "dev": true, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/expect": { + "version": "29.6.1", + "resolved": "https://registry.npmjs.org/expect/-/expect-29.6.1.tgz", + "integrity": "sha512-XEdDLonERCU1n9uR56/Stx9OqojaLAQtZf9PrCHH9Hl8YXiEIka3H4NXJ3NOIBmQJTg7+j7buh34PMHfJujc8g==", + "dev": true, + "dependencies": { + "@jest/expect-utils": "^29.6.1", + "@types/node": "*", + "jest-get-type": "^29.4.3", + "jest-matcher-utils": "^29.6.1", + "jest-message-util": "^29.6.1", + "jest-util": "^29.6.1" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/express": { + "version": "4.21.1", + "resolved": "https://registry.npmjs.org/express/-/express-4.21.1.tgz", + "integrity": "sha512-YSFlK1Ee0/GC8QaO91tHcDxJiE/X4FbpAyQWkxAvG6AXCuR65YzK8ua6D9hvi/TzUfZMpc+BwuM1IPw8fmQBiQ==", + "dev": true, + "dependencies": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "1.20.3", + "content-disposition": "0.5.4", + "content-type": "~1.0.4", + "cookie": "0.7.1", + "cookie-signature": "1.0.6", + "debug": "2.6.9", + "depd": "2.0.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "1.3.1", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "merge-descriptors": "1.0.3", + "methods": "~1.1.2", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "path-to-regexp": "0.1.10", + "proxy-addr": "~2.0.7", + "qs": "6.13.0", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "0.19.0", + "serve-static": "1.16.2", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.10.0" + } + }, + "node_modules/express/node_modules/array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", + "dev": true + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true + }, + "node_modules/fastest-levenshtein": { + "version": "1.0.16", + "resolved": "https://registry.npmjs.org/fastest-levenshtein/-/fastest-levenshtein-1.0.16.tgz", + "integrity": "sha512-eRnCtTTtGZFpQCwhJiUOuxPQWRXVKYDn0b2PeHfXL6/Zi53SLAzAHfVhVWK2AryC/WH05kGfxhFIPvTF0SXQzg==", + "dev": true, + "engines": { + "node": ">= 4.9.1" + } + }, + "node_modules/faye-websocket": { + "version": "0.11.4", + "resolved": "https://registry.npmjs.org/faye-websocket/-/faye-websocket-0.11.4.tgz", + "integrity": "sha512-CzbClwlXAuiRQAlUyfqPgvPoNKTckTPGfwZV4ZdAhVcP2lh9KUxJg2b5GkE7XbjKQ3YJnQ9z6D9ntLAlB+tP8g==", + "dev": true, + "dependencies": { + "websocket-driver": ">=0.5.1" + }, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/fb-watchman": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/fb-watchman/-/fb-watchman-2.0.2.tgz", + "integrity": "sha512-p5161BqbuCaSnB8jIbzQHOlpgsPmK5rJVDfDKO91Axs5NC1uu3HRQm6wt9cd9/+GtQQIO53JdGXXoyDpTAsgYA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "bser": "2.1.1" + } + }, + "node_modules/file-set": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/file-set/-/file-set-4.0.2.tgz", + "integrity": "sha512-fuxEgzk4L8waGXaAkd8cMr73Pm0FxOVkn8hztzUW7BAHhOGH90viQNXbiOsnecCWmfInqU6YmAMwxRMdKETceQ==", + "dev": true, + "dependencies": { + "array-back": "^5.0.0", + "glob": "^7.1.6" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/file-set/node_modules/array-back": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-5.0.0.tgz", + "integrity": "sha512-kgVWwJReZWmVuWOQKEOohXKJX+nD02JAZ54D1RRWlv8L0NebauKAaFxACKzB74RTclt1+WNz5KHaLRDAPZbDEw==", + "dev": true, + "engines": { + "node": ">=10" + } + }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "dev": true, + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/finalhandler": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz", + "integrity": "sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==", + "dev": true, + "dependencies": { + "debug": "2.6.9", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "statuses": "2.0.1", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/find-replace": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-replace/-/find-replace-3.0.0.tgz", + "integrity": "sha512-6Tb2myMioCAgv5kfvP5/PkZZ/ntTpVK39fHY7WkWBgvbeE+VHd/tZuZ4mrC+bxh4cfOZeYKVPaJIZtZXV7GNCQ==", + "dev": true, + "dependencies": { + "array-back": "^3.0.1" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/find-replace/node_modules/array-back": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-3.1.0.tgz", + "integrity": "sha512-TkuxA4UCOvxuDK6NZYXCalszEzj+TLszyASooky+i742l9TqsOdYCMJJupxRic61hwquNtppB3hgcuq9SVSH1Q==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, + "dependencies": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/flatbuffers": { + "version": "1.12.0", + "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-1.12.0.tgz", + "integrity": "sha512-c7CZADjRcl6j0PlvFy0ZqXQ67qSEZfrVPynmnL+2zPc+NtMvrF8Y0QceMo7QqnSPc7+uWjUIAbvCQ5WIKlMVdQ==" + }, + "node_modules/follow-redirects": { + "version": "1.15.6", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", + "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/foreground-child": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz", + "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==", + "dependencies": { + "cross-spawn": "^7.0.0", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/foreground-child/node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fs-monkey": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/fs-monkey/-/fs-monkey-1.0.3.tgz", + "integrity": "sha512-cybjIfiiE+pTWicSCLFHSrXZ6EilF30oh91FDP9S2B051prEa7QWfrVTQm10/dDpswBDXZugPa1Ogu8Yh+HV0Q==", + "dev": true + }, + "node_modules/fs-then-native": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fs-then-native/-/fs-then-native-2.0.0.tgz", + "integrity": "sha512-X712jAOaWXkemQCAmWeg5rOT2i+KOpWz1Z/txk/cW0qlOu2oQ9H61vc5w3X/iyuUEfq/OyaFJ78/cZAQD1/bgA==", + "dev": true, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "dev": true + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "dev": true, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gensync": { + "version": "1.0.0-beta.2", + "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", + "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true, + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-intrinsic": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", + "integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==", + "dev": true, + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "has-proto": "^1.0.1", + "has-symbols": "^1.0.3", + "hasown": "^2.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-package-type": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", + "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/get-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", + "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/glob-to-regexp": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz", + "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==", + "dev": true + }, + "node_modules/globals": { + "version": "11.12.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", + "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/gopd": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", + "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", + "dev": true, + "dependencies": { + "get-intrinsic": "^1.1.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "dev": true + }, + "node_modules/guid-typescript": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz", + "integrity": "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==" + }, + "node_modules/handle-thing": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/handle-thing/-/handle-thing-2.0.1.tgz", + "integrity": "sha512-9Qn4yBxelxoh2Ow62nP+Ka/kMnOXRi8BXnRaUwezLNhqelnN49xKz4F/dPP8OYLxLxq6JDtZb2i9XznUQbNPTg==", + "dev": true + }, + "node_modules/handlebars": { + "version": "4.7.7", + "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.7.tgz", + "integrity": "sha512-aAcXm5OAfE/8IXkcZvCepKU3VzW1/39Fb5ZuqMtgI/hT8X2YgoMvBY5dLhq/cpOvw7Lk1nK/UF71aLG/ZnVYRA==", + "dev": true, + "dependencies": { + "minimist": "^1.2.5", + "neo-async": "^2.6.0", + "source-map": "^0.6.1", + "wordwrap": "^1.0.0" + }, + "bin": { + "handlebars": "bin/handlebars" + }, + "engines": { + "node": ">=0.4.7" + }, + "optionalDependencies": { + "uglify-js": "^3.1.4" + } + }, + "node_modules/has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "dev": true, + "dependencies": { + "function-bind": "^1.1.1" + }, + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/has-property-descriptors": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", + "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", + "dev": true, + "dependencies": { + "es-define-property": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-proto": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.3.tgz", + "integrity": "sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q==", + "dev": true, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "dev": true, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "dev": true, + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/hpack.js": { + "version": "2.1.6", + "resolved": "https://registry.npmjs.org/hpack.js/-/hpack.js-2.1.6.tgz", + "integrity": "sha512-zJxVehUdMGIKsRaNt7apO2Gqp0BdqW5yaiGHXXmbpvxgBYVZnAql+BJb4RO5ad2MgpbZKn5G6nMnegrH1FcNYQ==", + "dev": true, + "dependencies": { + "inherits": "^2.0.1", + "obuf": "^1.0.0", + "readable-stream": "^2.0.1", + "wbuf": "^1.1.0" + } + }, + "node_modules/hpack.js/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "dev": true, + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/hpack.js/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true + }, + "node_modules/hpack.js/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/html-entities": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/html-entities/-/html-entities-2.3.3.tgz", + "integrity": "sha512-DV5Ln36z34NNTDgnz0EWGBLZENelNAtkiFA4kyNOG2tDI6Mz1uSWiq1wAKdyjnJwyDiDO7Fa2SO1CTxPXL8VxA==", + "dev": true + }, + "node_modules/html-escaper": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", + "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", + "dev": true, + "license": "MIT" + }, + "node_modules/http-deceiver": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/http-deceiver/-/http-deceiver-1.2.7.tgz", + "integrity": "sha512-LmpOGxTfbpgtGVxJrj5k7asXHCgNZp5nLfp+hWc8QQRqtb7fUy6kRY3BO1h9ddF6yIPYUARgxGOwB42DnxIaNw==", + "dev": true + }, + "node_modules/http-errors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", + "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", + "dev": true, + "dependencies": { + "depd": "2.0.0", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "toidentifier": "1.0.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/http-parser-js": { + "version": "0.5.8", + "resolved": "https://registry.npmjs.org/http-parser-js/-/http-parser-js-0.5.8.tgz", + "integrity": "sha512-SGeBX54F94Wgu5RH3X5jsDtf4eHyRogWX1XGT3b4HuW3tQPM4AaBzoUji/4AAJNXCEOWZ5O0DgZmJw1947gD5Q==", + "dev": true + }, + "node_modules/http-proxy": { + "version": "1.18.1", + "resolved": "https://registry.npmjs.org/http-proxy/-/http-proxy-1.18.1.tgz", + "integrity": "sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ==", + "dev": true, + "dependencies": { + "eventemitter3": "^4.0.0", + "follow-redirects": "^1.0.0", + "requires-port": "^1.0.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/http-proxy-middleware": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.7.tgz", + "integrity": "sha512-fgVY8AV7qU7z/MmXJ/rxwbrtQH4jBQ9m7kp3llF0liB7glmFeVZFBepQb32T3y8n8k2+AEYuMPCpinYW+/CuRA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/http-proxy": "^1.17.8", + "http-proxy": "^1.18.1", + "is-glob": "^4.0.1", + "is-plain-obj": "^3.0.0", + "micromatch": "^4.0.2" + }, + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "@types/express": "^4.17.13" + }, + "peerDependenciesMeta": { + "@types/express": { + "optional": true + } + } + }, + "node_modules/human-signals": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", + "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", + "dev": true, + "engines": { + "node": ">=10.17.0" + } + }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "dev": true, + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/import-local": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.1.0.tgz", + "integrity": "sha512-ASB07uLtnDs1o6EHjKpX34BKYDSqnFerfTOJL2HvMqF70LnxpjkzDB8J44oT9pu4AMPkQwf8jl6szgvNd2tRIg==", + "dev": true, + "dependencies": { + "pkg-dir": "^4.2.0", + "resolve-cwd": "^3.0.0" + }, + "bin": { + "import-local-fixture": "fixtures/cli.js" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "dev": true, + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true + }, + "node_modules/interpret": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/interpret/-/interpret-3.1.1.tgz", + "integrity": "sha512-6xwYfHbajpoF0xLW+iwLkhwgvLoZDfjYfoFNu8ftMoXINzwuymNLd9u/KmwtdT2GbR+/Cz66otEGEVVUHX9QLQ==", + "dev": true, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/ipaddr.js": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.0.1.tgz", + "integrity": "sha512-1qTgH9NG+IIJ4yfKs2e6Pp1bZg8wbDbKHT21HrLIeYBTRLgMYKnMTPAuI3Lcs61nfx5h1xlXnbJtH1kX5/d/ng==", + "dev": true, + "engines": { + "node": ">= 10" + } + }, + "node_modules/is-arrayish": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", + "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" + }, + "node_modules/is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dev": true, + "dependencies": { + "binary-extensions": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-core-module": { + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.12.0.tgz", + "integrity": "sha512-RECHCBCd/viahWmwj6enj19sKbHfJrddi/6cBDsNTKbNq0f7VeaUkBo60BqzvPqo/W54ChS62Z5qyun7cfOMqQ==", + "dev": true, + "dependencies": { + "has": "^1.0.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-docker": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.2.1.tgz", + "integrity": "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==", + "dev": true, + "bin": { + "is-docker": "cli.js" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-generator-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-generator-fn/-/is-generator-fn-2.1.0.tgz", + "integrity": "sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/is-plain-obj": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-3.0.0.tgz", + "integrity": "sha512-gwsOE28k+23GP1B6vFl1oVh/WOzmawBrKwo5Ev6wMKzPkaXaCDIQKzLnvsA42DRlbVTWorkgTKIviAKCWkfUwA==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-plain-object": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", + "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "dev": true, + "dependencies": { + "isobject": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "dev": true, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-wsl": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz", + "integrity": "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==", + "dev": true, + "dependencies": { + "is-docker": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "dev": true + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" + }, + "node_modules/isobject": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", + "integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/istanbul-lib-coverage": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", + "integrity": "sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=8" + } + }, + "node_modules/istanbul-lib-instrument": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-6.0.3.tgz", + "integrity": "sha512-Vtgk7L/R2JHyyGW07spoFlB8/lpjiOLTjMdms6AFMraYt3BaJauod/NGrfnVG/y4Ix1JEuMRPDPEj2ua+zz1/Q==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@babel/core": "^7.23.9", + "@babel/parser": "^7.23.9", + "@istanbuljs/schema": "^0.1.3", + "istanbul-lib-coverage": "^3.2.0", + "semver": "^7.5.4" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-lib-report": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz", + "integrity": "sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "istanbul-lib-coverage": "^3.0.0", + "make-dir": "^4.0.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-lib-report/node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/istanbul-lib-source-maps": { + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/istanbul-lib-source-maps/-/istanbul-lib-source-maps-5.0.6.tgz", + "integrity": "sha512-yg2d+Em4KizZC5niWhQaIomgf5WlL4vOOjZ5xGCmF8SnPE/mDWWXgvRExdcpCgh9lLRRa1/fSYp2ymmbJ1pI+A==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.23", + "debug": "^4.1.1", + "istanbul-lib-coverage": "^3.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-lib-source-maps/node_modules/debug": { + "version": "4.3.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", + "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/istanbul-lib-source-maps/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/istanbul-reports": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.1.7.tgz", + "integrity": "sha512-BewmUXImeuRk2YY0PVbxgKAysvhRPUQE0h5QRM++nVWyubKGV0l8qQ5op8+B2DOmwSe63Jivj0BjkPQVf8fP5g==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "html-escaper": "^2.0.0", + "istanbul-lib-report": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/jackspeak": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.1.2.tgz", + "integrity": "sha512-kWmLKn2tRtfYMF/BakihVVRzBKOxz4gJMiL2Rj91WnAB5TPZumSH99R/Yf1qE1u4uRimvCSJfm6hnxohXeEXjQ==", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, + "node_modules/jest": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest/-/jest-30.0.0-alpha.6.tgz", + "integrity": "sha512-9T3nAcIAcEpCX2MdxcjG2IDfG/0tjumnCkVNGh+AKkRXcWF4Er5jLROKvXsgXUJCmr/nMqLF6LG0GrDJ0kjFag==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/core": "30.0.0-alpha.6", + "@jest/types": "30.0.0-alpha.6", + "import-local": "^3.0.2", + "jest-cli": "30.0.0-alpha.6" + }, + "bin": { + "jest": "bin/jest.js" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + }, + "peerDependencies": { + "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" + }, + "peerDependenciesMeta": { + "node-notifier": { + "optional": true + } + } + }, + "node_modules/jest-changed-files": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-changed-files/-/jest-changed-files-30.0.0-alpha.6.tgz", + "integrity": "sha512-Fmyt6W27L4fRBl/gReFC4WU+3XIqB7ySHu+a9QxrERapfCb43o7y81TCvTwJHSw5dxGzXLOObVB0tRMDWMafnw==", + "dev": true, + "license": "MIT", + "dependencies": { + "execa": "^5.0.0", + "jest-util": "30.0.0-alpha.6", + "p-limit": "^3.1.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-changed-files/node_modules/@jest/schemas": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", + "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.33.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-changed-files/node_modules/@jest/types": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", + "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/pattern": "30.0.0-alpha.6", + "@jest/schemas": "30.0.0-alpha.6", + "@types/istanbul-lib-coverage": "^2.0.0", + "@types/istanbul-reports": "^3.0.0", + "@types/node": "*", + "@types/yargs": "^17.0.8", + "chalk": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-changed-files/node_modules/@sinclair/typebox": { + "version": "0.33.17", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", + "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", + "dev": true, + "license": "MIT" + }, + "node_modules/jest-changed-files/node_modules/ci-info": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.0.0.tgz", + "integrity": "sha512-TdHqgGf9odd8SXNuxtUBVx8Nv+qZOejE6qyqiy5NtbYYQOeFa6zmHkxlPzmaLxWWHsU6nJmB7AETdVPi+2NBUg==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/sibiraj-s" + } + ], + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/jest-changed-files/node_modules/jest-util": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.0.0-alpha.6.tgz", + "integrity": "sha512-JlimakOVDyoMC8TEG+knoufxUqLG+Btihf1G8o5sHxz54C6oL54Wetfepp+Nhuj/1hSL0sQtkovvjlEycf9i0w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "30.0.0-alpha.6", + "@types/node": "*", + "chalk": "^4.0.0", + "ci-info": "^4.0.0", + "graceful-fs": "^4.2.9", + "picomatch": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-changed-files/node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/jest-changed-files/node_modules/picomatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", + "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/jest-circus": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-circus/-/jest-circus-30.0.0-alpha.6.tgz", + "integrity": "sha512-1C62WeTyWinn6zR61syYKe5yqVbV+ftf21vOgj8AtTxGfMUAlGCpeZ5zh4Kc9Qk7r/PiPiHWZtgZmeT4oe9Dug==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/environment": "30.0.0-alpha.6", + "@jest/expect": "30.0.0-alpha.6", + "@jest/test-result": "30.0.0-alpha.6", + "@jest/types": "30.0.0-alpha.6", + "@types/node": "*", + "chalk": "^4.0.0", + "co": "^4.6.0", + "dedent": "^1.0.0", + "is-generator-fn": "^2.0.0", + "jest-each": "30.0.0-alpha.6", + "jest-matcher-utils": "30.0.0-alpha.6", + "jest-message-util": "30.0.0-alpha.6", + "jest-runtime": "30.0.0-alpha.6", + "jest-snapshot": "30.0.0-alpha.6", + "jest-util": "30.0.0-alpha.6", + "p-limit": "^3.1.0", + "pretty-format": "30.0.0-alpha.6", + "pure-rand": "^6.0.0", + "slash": "^3.0.0", + "stack-utils": "^2.0.3" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-circus/node_modules/@jest/schemas": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", + "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.33.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-circus/node_modules/@jest/types": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", + "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/pattern": "30.0.0-alpha.6", + "@jest/schemas": "30.0.0-alpha.6", + "@types/istanbul-lib-coverage": "^2.0.0", + "@types/istanbul-reports": "^3.0.0", + "@types/node": "*", + "@types/yargs": "^17.0.8", + "chalk": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-circus/node_modules/@sinclair/typebox": { + "version": "0.33.17", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", + "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", + "dev": true, + "license": "MIT" + }, + "node_modules/jest-circus/node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/jest-circus/node_modules/ci-info": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.0.0.tgz", + "integrity": "sha512-TdHqgGf9odd8SXNuxtUBVx8Nv+qZOejE6qyqiy5NtbYYQOeFa6zmHkxlPzmaLxWWHsU6nJmB7AETdVPi+2NBUg==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/sibiraj-s" + } + ], + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/jest-circus/node_modules/diff-sequences": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-30.0.0-alpha.6.tgz", + "integrity": "sha512-DVGt3/yzbneMUTuupsMqyfSXMnU2fE0lVsC9uFsJmRpluvSi7ZhrS0GX5tnMna6Ta788FGfOUx+irI/+cAZ4EA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-circus/node_modules/jest-diff": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-30.0.0-alpha.6.tgz", + "integrity": "sha512-43j1DoYwVKrkbB67a2gC0ijjIY9biF0JSPXv7H6zlOkzNlqYg8hSDzrurLNo6zGKatW4JSBLE79LmXPJPj1m6A==", + "dev": true, + "license": "MIT", + "dependencies": { + "chalk": "^4.0.0", + "diff-sequences": "30.0.0-alpha.6", + "jest-get-type": "30.0.0-alpha.6", + "pretty-format": "30.0.0-alpha.6" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-circus/node_modules/jest-get-type": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-30.0.0-alpha.6.tgz", + "integrity": "sha512-lJEoQdCY4ICN6+T0lJ9BODKuqPOEpCv2NnJsEO1nmsK0fbWZmN/pgOPHVqLfK8i3jZpUmgupJ1w8r36mc8iiBQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-circus/node_modules/jest-matcher-utils": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-30.0.0-alpha.6.tgz", + "integrity": "sha512-jaq7+HznsK54G0qzu96ZwfMEKHmlPiDqg6qG2p/hVQzr6Y/qVMRh8abI9Y1lX6SSXkr+S9mPAkmOsuJNLTLYmQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "chalk": "^4.0.0", + "jest-diff": "30.0.0-alpha.6", + "jest-get-type": "30.0.0-alpha.6", + "pretty-format": "30.0.0-alpha.6" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-circus/node_modules/jest-message-util": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-30.0.0-alpha.6.tgz", + "integrity": "sha512-XAGJqkrBo7m3bFxWqiNqL0PyAWGf1XHR6bTve90MjBKJuIzhJsounGTzBNUw8JoU7Uzcj5Z6ZmEhaE3CDnwjfw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.12.13", + "@jest/types": "30.0.0-alpha.6", + "@types/stack-utils": "^2.0.0", + "chalk": "^4.0.0", + "graceful-fs": "^4.2.9", + "micromatch": "^4.0.7", + "pretty-format": "30.0.0-alpha.6", + "slash": "^3.0.0", + "stack-utils": "^2.0.3" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-circus/node_modules/jest-util": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.0.0-alpha.6.tgz", + "integrity": "sha512-JlimakOVDyoMC8TEG+knoufxUqLG+Btihf1G8o5sHxz54C6oL54Wetfepp+Nhuj/1hSL0sQtkovvjlEycf9i0w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "30.0.0-alpha.6", + "@types/node": "*", + "chalk": "^4.0.0", + "ci-info": "^4.0.0", + "graceful-fs": "^4.2.9", + "picomatch": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-circus/node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/jest-circus/node_modules/picomatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", + "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/jest-circus/node_modules/pretty-format": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.0.0-alpha.6.tgz", + "integrity": "sha512-xkeffkZoqQmRrcNewpOsUCKNOl+CkPqjt3Ld749uz1S7/O7GuPNPv2fZk3v/1U/FE8/B4Zz0llVL80MKON1tOQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/schemas": "30.0.0-alpha.6", + "ansi-styles": "^5.0.0", + "react-is": "^18.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-cli": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-cli/-/jest-cli-30.0.0-alpha.6.tgz", + "integrity": "sha512-3VYzI2KgpMNAsf+LdRAQtAbhH3IDyFnT36U6URXot+2JWwoCGQQ6w4HIfqyOSlH4aejKgTPSfxki2shRPDFtlQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/core": "30.0.0-alpha.6", + "@jest/test-result": "30.0.0-alpha.6", + "@jest/types": "30.0.0-alpha.6", + "chalk": "^4.0.0", + "exit": "^0.1.2", + "import-local": "^3.0.2", + "jest-config": "30.0.0-alpha.6", + "jest-util": "30.0.0-alpha.6", + "jest-validate": "30.0.0-alpha.6", + "yargs": "^17.3.1" + }, + "bin": { + "jest": "bin/jest.js" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + }, + "peerDependencies": { + "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" + }, + "peerDependenciesMeta": { + "node-notifier": { + "optional": true + } + } + }, + "node_modules/jest-cli/node_modules/@jest/schemas": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", + "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.33.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-cli/node_modules/@jest/types": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", + "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/pattern": "30.0.0-alpha.6", + "@jest/schemas": "30.0.0-alpha.6", + "@types/istanbul-lib-coverage": "^2.0.0", + "@types/istanbul-reports": "^3.0.0", + "@types/node": "*", + "@types/yargs": "^17.0.8", + "chalk": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-cli/node_modules/@sinclair/typebox": { + "version": "0.33.17", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", + "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", + "dev": true, + "license": "MIT" + }, + "node_modules/jest-cli/node_modules/ci-info": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.0.0.tgz", + "integrity": "sha512-TdHqgGf9odd8SXNuxtUBVx8Nv+qZOejE6qyqiy5NtbYYQOeFa6zmHkxlPzmaLxWWHsU6nJmB7AETdVPi+2NBUg==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/sibiraj-s" + } + ], + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/jest-cli/node_modules/jest-util": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.0.0-alpha.6.tgz", + "integrity": "sha512-JlimakOVDyoMC8TEG+knoufxUqLG+Btihf1G8o5sHxz54C6oL54Wetfepp+Nhuj/1hSL0sQtkovvjlEycf9i0w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "30.0.0-alpha.6", + "@types/node": "*", + "chalk": "^4.0.0", + "ci-info": "^4.0.0", + "graceful-fs": "^4.2.9", + "picomatch": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-cli/node_modules/picomatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", + "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/jest-config": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-config/-/jest-config-30.0.0-alpha.6.tgz", + "integrity": "sha512-Tq9rH1mg9+nlIhh3efGwMSogFVKZ9z7c6P33ZlK74iJlnqqIAKYERZL2nNmNC5+5p8uxlTPSFZfBz9O8NGKotw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.11.6", + "@jest/pattern": "30.0.0-alpha.6", + "@jest/test-sequencer": "30.0.0-alpha.6", + "@jest/types": "30.0.0-alpha.6", + "babel-jest": "30.0.0-alpha.6", + "chalk": "^4.0.0", + "ci-info": "^4.0.0", + "deepmerge": "^4.2.2", + "glob": "^10.3.10", + "graceful-fs": "^4.2.9", + "jest-circus": "30.0.0-alpha.6", + "jest-docblock": "30.0.0-alpha.6", + "jest-environment-node": "30.0.0-alpha.6", + "jest-get-type": "30.0.0-alpha.6", + "jest-regex-util": "30.0.0-alpha.6", + "jest-resolve": "30.0.0-alpha.6", + "jest-runner": "30.0.0-alpha.6", + "jest-util": "30.0.0-alpha.6", + "jest-validate": "30.0.0-alpha.6", + "micromatch": "^4.0.7", + "parse-json": "^5.2.0", + "pretty-format": "30.0.0-alpha.6", + "slash": "^3.0.0", + "strip-json-comments": "^3.1.1" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + }, + "peerDependencies": { + "@types/node": "*", + "esbuild-register": ">=3.4.0", + "ts-node": ">=9.0.0" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "esbuild-register": { + "optional": true + }, + "ts-node": { + "optional": true + } + } + }, + "node_modules/jest-config/node_modules/@jest/schemas": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", + "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.33.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-config/node_modules/@jest/types": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", + "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/pattern": "30.0.0-alpha.6", + "@jest/schemas": "30.0.0-alpha.6", + "@types/istanbul-lib-coverage": "^2.0.0", + "@types/istanbul-reports": "^3.0.0", + "@types/node": "*", + "@types/yargs": "^17.0.8", + "chalk": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-config/node_modules/@sinclair/typebox": { + "version": "0.33.17", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", + "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", + "dev": true, + "license": "MIT" + }, + "node_modules/jest-config/node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/jest-config/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/jest-config/node_modules/ci-info": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.0.0.tgz", + "integrity": "sha512-TdHqgGf9odd8SXNuxtUBVx8Nv+qZOejE6qyqiy5NtbYYQOeFa6zmHkxlPzmaLxWWHsU6nJmB7AETdVPi+2NBUg==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/sibiraj-s" + } + ], + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/jest-config/node_modules/glob": { + "version": "10.4.5", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz", + "integrity": "sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==", + "dev": true, + "license": "ISC", + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^1.11.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/jest-config/node_modules/jest-get-type": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-30.0.0-alpha.6.tgz", + "integrity": "sha512-lJEoQdCY4ICN6+T0lJ9BODKuqPOEpCv2NnJsEO1nmsK0fbWZmN/pgOPHVqLfK8i3jZpUmgupJ1w8r36mc8iiBQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-config/node_modules/jest-util": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.0.0-alpha.6.tgz", + "integrity": "sha512-JlimakOVDyoMC8TEG+knoufxUqLG+Btihf1G8o5sHxz54C6oL54Wetfepp+Nhuj/1hSL0sQtkovvjlEycf9i0w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "30.0.0-alpha.6", + "@types/node": "*", + "chalk": "^4.0.0", + "ci-info": "^4.0.0", + "graceful-fs": "^4.2.9", + "picomatch": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-config/node_modules/minimatch": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/jest-config/node_modules/picomatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", + "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/jest-config/node_modules/pretty-format": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.0.0-alpha.6.tgz", + "integrity": "sha512-xkeffkZoqQmRrcNewpOsUCKNOl+CkPqjt3Ld749uz1S7/O7GuPNPv2fZk3v/1U/FE8/B4Zz0llVL80MKON1tOQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/schemas": "30.0.0-alpha.6", + "ansi-styles": "^5.0.0", + "react-is": "^18.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-diff": { + "version": "29.6.1", + "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-29.6.1.tgz", + "integrity": "sha512-FsNCvinvl8oVxpNLttNQX7FAq7vR+gMDGj90tiP7siWw1UdakWUGqrylpsYrpvj908IYckm5Y0Q7azNAozU1Kg==", + "dev": true, + "dependencies": { + "chalk": "^4.0.0", + "diff-sequences": "^29.4.3", + "jest-get-type": "^29.4.3", + "pretty-format": "^29.6.1" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-docblock": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-docblock/-/jest-docblock-30.0.0-alpha.6.tgz", + "integrity": "sha512-KXRLgRo7/rF1wqxQupsFCZa6wOp1qrDg4GdSXKfIHODYQb0dpi4rYaYA8xV5l2g9KwYc9/zV7l1tPe9TOr27ew==", + "dev": true, + "license": "MIT", + "dependencies": { + "detect-newline": "^3.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-each": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-each/-/jest-each-30.0.0-alpha.6.tgz", + "integrity": "sha512-snLI2JNYkoBMlZRrNk67XiauUy+uEzRCszKdj+cqHyZ4/MU8fz7gCxbn3g0zmiGUxr0RX0534UxMjc82Sk++tg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "30.0.0-alpha.6", + "chalk": "^4.0.0", + "jest-get-type": "30.0.0-alpha.6", + "jest-util": "30.0.0-alpha.6", + "pretty-format": "30.0.0-alpha.6" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-each/node_modules/@jest/schemas": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", + "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.33.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-each/node_modules/@jest/types": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", + "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/pattern": "30.0.0-alpha.6", + "@jest/schemas": "30.0.0-alpha.6", + "@types/istanbul-lib-coverage": "^2.0.0", + "@types/istanbul-reports": "^3.0.0", + "@types/node": "*", + "@types/yargs": "^17.0.8", + "chalk": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-each/node_modules/@sinclair/typebox": { + "version": "0.33.17", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", + "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", + "dev": true, + "license": "MIT" + }, + "node_modules/jest-each/node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/jest-each/node_modules/ci-info": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.0.0.tgz", + "integrity": "sha512-TdHqgGf9odd8SXNuxtUBVx8Nv+qZOejE6qyqiy5NtbYYQOeFa6zmHkxlPzmaLxWWHsU6nJmB7AETdVPi+2NBUg==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/sibiraj-s" + } + ], + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/jest-each/node_modules/jest-get-type": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-30.0.0-alpha.6.tgz", + "integrity": "sha512-lJEoQdCY4ICN6+T0lJ9BODKuqPOEpCv2NnJsEO1nmsK0fbWZmN/pgOPHVqLfK8i3jZpUmgupJ1w8r36mc8iiBQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-each/node_modules/jest-util": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.0.0-alpha.6.tgz", + "integrity": "sha512-JlimakOVDyoMC8TEG+knoufxUqLG+Btihf1G8o5sHxz54C6oL54Wetfepp+Nhuj/1hSL0sQtkovvjlEycf9i0w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "30.0.0-alpha.6", + "@types/node": "*", + "chalk": "^4.0.0", + "ci-info": "^4.0.0", + "graceful-fs": "^4.2.9", + "picomatch": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-each/node_modules/picomatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", + "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/jest-each/node_modules/pretty-format": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.0.0-alpha.6.tgz", + "integrity": "sha512-xkeffkZoqQmRrcNewpOsUCKNOl+CkPqjt3Ld749uz1S7/O7GuPNPv2fZk3v/1U/FE8/B4Zz0llVL80MKON1tOQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/schemas": "30.0.0-alpha.6", + "ansi-styles": "^5.0.0", + "react-is": "^18.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-environment-node": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-30.0.0-alpha.6.tgz", + "integrity": "sha512-UN9W3dFzO150Bqj1x+1pq7dMUqw/QhpqhdtmC3B1P6GD9eKEMFGuRw3EButx5SGzrZOqRNlF+tUNC8CoWGW2Og==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/environment": "30.0.0-alpha.6", + "@jest/fake-timers": "30.0.0-alpha.6", + "@jest/types": "30.0.0-alpha.6", + "@types/node": "*", + "jest-mock": "30.0.0-alpha.6", + "jest-util": "30.0.0-alpha.6" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-environment-node/node_modules/@jest/schemas": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", + "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.33.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-environment-node/node_modules/@jest/types": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", + "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/pattern": "30.0.0-alpha.6", + "@jest/schemas": "30.0.0-alpha.6", + "@types/istanbul-lib-coverage": "^2.0.0", + "@types/istanbul-reports": "^3.0.0", + "@types/node": "*", + "@types/yargs": "^17.0.8", + "chalk": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-environment-node/node_modules/@sinclair/typebox": { + "version": "0.33.17", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", + "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", + "dev": true, + "license": "MIT" + }, + "node_modules/jest-environment-node/node_modules/ci-info": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.0.0.tgz", + "integrity": "sha512-TdHqgGf9odd8SXNuxtUBVx8Nv+qZOejE6qyqiy5NtbYYQOeFa6zmHkxlPzmaLxWWHsU6nJmB7AETdVPi+2NBUg==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/sibiraj-s" + } + ], + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/jest-environment-node/node_modules/jest-util": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.0.0-alpha.6.tgz", + "integrity": "sha512-JlimakOVDyoMC8TEG+knoufxUqLG+Btihf1G8o5sHxz54C6oL54Wetfepp+Nhuj/1hSL0sQtkovvjlEycf9i0w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "30.0.0-alpha.6", + "@types/node": "*", + "chalk": "^4.0.0", + "ci-info": "^4.0.0", + "graceful-fs": "^4.2.9", + "picomatch": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-environment-node/node_modules/picomatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", + "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/jest-get-type": { + "version": "29.4.3", + "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-29.4.3.tgz", + "integrity": "sha512-J5Xez4nRRMjk8emnTpWrlkyb9pfRQQanDrvWHhsR1+VUfbwxi30eVcZFlcdGInRibU4G5LwHXpI7IRHU0CY+gg==", + "dev": true, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-haste-map": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-haste-map/-/jest-haste-map-30.0.0-alpha.6.tgz", + "integrity": "sha512-NR/Kw8HyOkuWIdT8ynsp9KnsTDvWnlz8WSOmtQxySTIzOWbZaeJ2FJi9LoDL6+vhKpdlLfUvhgZVtnFJSLCzew==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "30.0.0-alpha.6", + "@types/node": "*", + "anymatch": "^3.0.3", + "fb-watchman": "^2.0.0", + "graceful-fs": "^4.2.9", + "jest-regex-util": "30.0.0-alpha.6", + "jest-util": "30.0.0-alpha.6", + "jest-worker": "30.0.0-alpha.6", + "micromatch": "^4.0.7", + "walker": "^1.0.8" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + }, + "optionalDependencies": { + "fsevents": "^2.3.2" + } + }, + "node_modules/jest-haste-map/node_modules/@jest/schemas": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", + "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.33.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-haste-map/node_modules/@jest/types": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", + "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/pattern": "30.0.0-alpha.6", + "@jest/schemas": "30.0.0-alpha.6", + "@types/istanbul-lib-coverage": "^2.0.0", + "@types/istanbul-reports": "^3.0.0", + "@types/node": "*", + "@types/yargs": "^17.0.8", + "chalk": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-haste-map/node_modules/@sinclair/typebox": { + "version": "0.33.17", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", + "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", + "dev": true, + "license": "MIT" + }, + "node_modules/jest-haste-map/node_modules/ci-info": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.0.0.tgz", + "integrity": "sha512-TdHqgGf9odd8SXNuxtUBVx8Nv+qZOejE6qyqiy5NtbYYQOeFa6zmHkxlPzmaLxWWHsU6nJmB7AETdVPi+2NBUg==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/sibiraj-s" + } + ], + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/jest-haste-map/node_modules/jest-util": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.0.0-alpha.6.tgz", + "integrity": "sha512-JlimakOVDyoMC8TEG+knoufxUqLG+Btihf1G8o5sHxz54C6oL54Wetfepp+Nhuj/1hSL0sQtkovvjlEycf9i0w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "30.0.0-alpha.6", + "@types/node": "*", + "chalk": "^4.0.0", + "ci-info": "^4.0.0", + "graceful-fs": "^4.2.9", + "picomatch": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-haste-map/node_modules/jest-worker": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-30.0.0-alpha.6.tgz", + "integrity": "sha512-qlzX7zFT/QdUV/LWsJwZBlaIBaJ+E2VH3d1gArGVP+9hUHGpJkEzCSBK7yuZrkt+M/U0Jre5+maPRmkinEF4DA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@ungap/structured-clone": "^1.2.0", + "jest-util": "30.0.0-alpha.6", + "merge-stream": "^2.0.0", + "supports-color": "^8.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-haste-map/node_modules/picomatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", + "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/jest-leak-detector": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-leak-detector/-/jest-leak-detector-30.0.0-alpha.6.tgz", + "integrity": "sha512-a6fh/6h6dCDyj+aplGqkajVqzmi+qYHs5X8orMZv+u56++gUezJZJf8GCiQqw2vtxcsWVPUuQXa3kF33tAYzNQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "jest-get-type": "30.0.0-alpha.6", + "pretty-format": "30.0.0-alpha.6" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-leak-detector/node_modules/@jest/schemas": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", + "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.33.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-leak-detector/node_modules/@sinclair/typebox": { + "version": "0.33.17", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", + "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", + "dev": true, + "license": "MIT" + }, + "node_modules/jest-leak-detector/node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/jest-leak-detector/node_modules/jest-get-type": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-30.0.0-alpha.6.tgz", + "integrity": "sha512-lJEoQdCY4ICN6+T0lJ9BODKuqPOEpCv2NnJsEO1nmsK0fbWZmN/pgOPHVqLfK8i3jZpUmgupJ1w8r36mc8iiBQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-leak-detector/node_modules/pretty-format": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.0.0-alpha.6.tgz", + "integrity": "sha512-xkeffkZoqQmRrcNewpOsUCKNOl+CkPqjt3Ld749uz1S7/O7GuPNPv2fZk3v/1U/FE8/B4Zz0llVL80MKON1tOQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/schemas": "30.0.0-alpha.6", + "ansi-styles": "^5.0.0", + "react-is": "^18.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-matcher-utils": { + "version": "29.6.1", + "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-29.6.1.tgz", + "integrity": "sha512-SLaztw9d2mfQQKHmJXKM0HCbl2PPVld/t9Xa6P9sgiExijviSp7TnZZpw2Fpt+OI3nwUO/slJbOfzfUMKKC5QA==", + "dev": true, + "dependencies": { + "chalk": "^4.0.0", + "jest-diff": "^29.6.1", + "jest-get-type": "^29.4.3", + "pretty-format": "^29.6.1" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-message-util": { + "version": "29.6.1", + "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-29.6.1.tgz", + "integrity": "sha512-KoAW2zAmNSd3Gk88uJ56qXUWbFk787QKmjjJVOjtGFmmGSZgDBrlIL4AfQw1xyMYPNVD7dNInfIbur9B2rd/wQ==", + "dev": true, + "dependencies": { + "@babel/code-frame": "^7.12.13", + "@jest/types": "^29.6.1", + "@types/stack-utils": "^2.0.0", + "chalk": "^4.0.0", + "graceful-fs": "^4.2.9", + "micromatch": "^4.0.4", + "pretty-format": "^29.6.1", + "slash": "^3.0.0", + "stack-utils": "^2.0.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-mock": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-mock/-/jest-mock-30.0.0-alpha.6.tgz", + "integrity": "sha512-ezW02IXiKyFYAgDuxfAlONWULitSaB66t411fq2BJxQtgyMGtv59CsnhgbKb0gQp+9vig5MO5ytDCUPalTbarg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "30.0.0-alpha.6", + "@types/node": "*", + "jest-util": "30.0.0-alpha.6" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-mock/node_modules/@jest/schemas": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", + "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.33.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-mock/node_modules/@jest/types": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", + "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/pattern": "30.0.0-alpha.6", + "@jest/schemas": "30.0.0-alpha.6", + "@types/istanbul-lib-coverage": "^2.0.0", + "@types/istanbul-reports": "^3.0.0", + "@types/node": "*", + "@types/yargs": "^17.0.8", + "chalk": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-mock/node_modules/@sinclair/typebox": { + "version": "0.33.17", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", + "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", + "dev": true, + "license": "MIT" + }, + "node_modules/jest-mock/node_modules/ci-info": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.0.0.tgz", + "integrity": "sha512-TdHqgGf9odd8SXNuxtUBVx8Nv+qZOejE6qyqiy5NtbYYQOeFa6zmHkxlPzmaLxWWHsU6nJmB7AETdVPi+2NBUg==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/sibiraj-s" + } + ], + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/jest-mock/node_modules/jest-util": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.0.0-alpha.6.tgz", + "integrity": "sha512-JlimakOVDyoMC8TEG+knoufxUqLG+Btihf1G8o5sHxz54C6oL54Wetfepp+Nhuj/1hSL0sQtkovvjlEycf9i0w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "30.0.0-alpha.6", + "@types/node": "*", + "chalk": "^4.0.0", + "ci-info": "^4.0.0", + "graceful-fs": "^4.2.9", + "picomatch": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-mock/node_modules/picomatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", + "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/jest-pnp-resolver": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/jest-pnp-resolver/-/jest-pnp-resolver-1.2.3.tgz", + "integrity": "sha512-+3NpwQEnRoIBtx4fyhblQDPgJI0H1IEIkX7ShLUjPGA7TtUTvI1oiKi3SR4oBR0hQhQR80l4WAe5RrXBwWMA8w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + }, + "peerDependencies": { + "jest-resolve": "*" + }, + "peerDependenciesMeta": { + "jest-resolve": { + "optional": true + } + } + }, + "node_modules/jest-regex-util": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-regex-util/-/jest-regex-util-30.0.0-alpha.6.tgz", + "integrity": "sha512-XcsAVaqc69QyMz1/FChyhWSoAMaKcDPhFOuWJz/H51LppsyZRAJPXkPnMopsS+qfut8cggExr9QLcsYaX6hqqA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-resolve": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-resolve/-/jest-resolve-30.0.0-alpha.6.tgz", + "integrity": "sha512-0EyeId+RFng52qHvuxOzKjZd2uDF/2Hdzpzt54+biGgY/VVAvf8mYE9UV7g6154Ozpq6KLztSqqMCfPgVs4CbA==", + "dev": true, + "license": "MIT", + "dependencies": { + "chalk": "^4.0.0", + "graceful-fs": "^4.2.9", + "jest-haste-map": "30.0.0-alpha.6", + "jest-pnp-resolver": "^1.2.2", + "jest-util": "30.0.0-alpha.6", + "jest-validate": "30.0.0-alpha.6", + "resolve": "^1.20.0", + "resolve.exports": "^2.0.0", + "slash": "^3.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-resolve-dependencies": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-resolve-dependencies/-/jest-resolve-dependencies-30.0.0-alpha.6.tgz", + "integrity": "sha512-G+st0nBR4FNIvVCHq8YNJBiG6t7u0+cxM099lbtOoJNJU+ZTdIxSyzPnnmp/C+YHd1QOlDNlplvL+xe1KHhPUA==", + "dev": true, + "license": "MIT", + "dependencies": { + "jest-regex-util": "30.0.0-alpha.6", + "jest-snapshot": "30.0.0-alpha.6" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-resolve/node_modules/@jest/schemas": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", + "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.33.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-resolve/node_modules/@jest/types": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", + "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/pattern": "30.0.0-alpha.6", + "@jest/schemas": "30.0.0-alpha.6", + "@types/istanbul-lib-coverage": "^2.0.0", + "@types/istanbul-reports": "^3.0.0", + "@types/node": "*", + "@types/yargs": "^17.0.8", + "chalk": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-resolve/node_modules/@sinclair/typebox": { + "version": "0.33.17", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", + "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", + "dev": true, + "license": "MIT" + }, + "node_modules/jest-resolve/node_modules/ci-info": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.0.0.tgz", + "integrity": "sha512-TdHqgGf9odd8SXNuxtUBVx8Nv+qZOejE6qyqiy5NtbYYQOeFa6zmHkxlPzmaLxWWHsU6nJmB7AETdVPi+2NBUg==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/sibiraj-s" + } + ], + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/jest-resolve/node_modules/jest-util": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.0.0-alpha.6.tgz", + "integrity": "sha512-JlimakOVDyoMC8TEG+knoufxUqLG+Btihf1G8o5sHxz54C6oL54Wetfepp+Nhuj/1hSL0sQtkovvjlEycf9i0w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "30.0.0-alpha.6", + "@types/node": "*", + "chalk": "^4.0.0", + "ci-info": "^4.0.0", + "graceful-fs": "^4.2.9", + "picomatch": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-resolve/node_modules/picomatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", + "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/jest-runner": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-runner/-/jest-runner-30.0.0-alpha.6.tgz", + "integrity": "sha512-SoADy4YnspMpXLNnRCXNIoinm1N5SMci+iF6Y29Duv3wnWhcL14XjEOcyUKBB+AIL52YwouLeUHkCyCspbBk1Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/console": "30.0.0-alpha.6", + "@jest/environment": "30.0.0-alpha.6", + "@jest/test-result": "30.0.0-alpha.6", + "@jest/transform": "30.0.0-alpha.6", + "@jest/types": "30.0.0-alpha.6", + "@types/node": "*", + "chalk": "^4.0.0", + "emittery": "^0.13.1", + "graceful-fs": "^4.2.9", + "jest-docblock": "30.0.0-alpha.6", + "jest-environment-node": "30.0.0-alpha.6", + "jest-haste-map": "30.0.0-alpha.6", + "jest-leak-detector": "30.0.0-alpha.6", + "jest-message-util": "30.0.0-alpha.6", + "jest-resolve": "30.0.0-alpha.6", + "jest-runtime": "30.0.0-alpha.6", + "jest-util": "30.0.0-alpha.6", + "jest-watcher": "30.0.0-alpha.6", + "jest-worker": "30.0.0-alpha.6", + "p-limit": "^3.1.0", + "source-map-support": "0.5.13" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-runner/node_modules/@jest/schemas": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", + "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.33.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-runner/node_modules/@jest/types": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", + "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/pattern": "30.0.0-alpha.6", + "@jest/schemas": "30.0.0-alpha.6", + "@types/istanbul-lib-coverage": "^2.0.0", + "@types/istanbul-reports": "^3.0.0", + "@types/node": "*", + "@types/yargs": "^17.0.8", + "chalk": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-runner/node_modules/@sinclair/typebox": { + "version": "0.33.17", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", + "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", + "dev": true, + "license": "MIT" + }, + "node_modules/jest-runner/node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/jest-runner/node_modules/ci-info": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.0.0.tgz", + "integrity": "sha512-TdHqgGf9odd8SXNuxtUBVx8Nv+qZOejE6qyqiy5NtbYYQOeFa6zmHkxlPzmaLxWWHsU6nJmB7AETdVPi+2NBUg==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/sibiraj-s" + } + ], + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/jest-runner/node_modules/jest-message-util": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-30.0.0-alpha.6.tgz", + "integrity": "sha512-XAGJqkrBo7m3bFxWqiNqL0PyAWGf1XHR6bTve90MjBKJuIzhJsounGTzBNUw8JoU7Uzcj5Z6ZmEhaE3CDnwjfw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.12.13", + "@jest/types": "30.0.0-alpha.6", + "@types/stack-utils": "^2.0.0", + "chalk": "^4.0.0", + "graceful-fs": "^4.2.9", + "micromatch": "^4.0.7", + "pretty-format": "30.0.0-alpha.6", + "slash": "^3.0.0", + "stack-utils": "^2.0.3" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-runner/node_modules/jest-util": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.0.0-alpha.6.tgz", + "integrity": "sha512-JlimakOVDyoMC8TEG+knoufxUqLG+Btihf1G8o5sHxz54C6oL54Wetfepp+Nhuj/1hSL0sQtkovvjlEycf9i0w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "30.0.0-alpha.6", + "@types/node": "*", + "chalk": "^4.0.0", + "ci-info": "^4.0.0", + "graceful-fs": "^4.2.9", + "picomatch": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-runner/node_modules/jest-worker": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-30.0.0-alpha.6.tgz", + "integrity": "sha512-qlzX7zFT/QdUV/LWsJwZBlaIBaJ+E2VH3d1gArGVP+9hUHGpJkEzCSBK7yuZrkt+M/U0Jre5+maPRmkinEF4DA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@ungap/structured-clone": "^1.2.0", + "jest-util": "30.0.0-alpha.6", + "merge-stream": "^2.0.0", + "supports-color": "^8.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-runner/node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/jest-runner/node_modules/picomatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", + "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/jest-runner/node_modules/pretty-format": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.0.0-alpha.6.tgz", + "integrity": "sha512-xkeffkZoqQmRrcNewpOsUCKNOl+CkPqjt3Ld749uz1S7/O7GuPNPv2fZk3v/1U/FE8/B4Zz0llVL80MKON1tOQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/schemas": "30.0.0-alpha.6", + "ansi-styles": "^5.0.0", + "react-is": "^18.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-runner/node_modules/source-map-support": { + "version": "0.5.13", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.13.tgz", + "integrity": "sha512-SHSKFHadjVA5oR4PPqhtAVdcBWwRYVd6g6cAXnIbRiIwc2EhPrTuKUBdSLvlEKyIP3GCf89fltvcZiP9MMFA1w==", + "dev": true, + "license": "MIT", + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "node_modules/jest-runtime": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-runtime/-/jest-runtime-30.0.0-alpha.6.tgz", + "integrity": "sha512-p7w7DSFFzwHyR4HsNXca/p32VpL9MLT1c71+VplFJIEgeRHvyqxrARentlul6uJniwtlqvZrVVf5baCQ5a5GUw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/environment": "30.0.0-alpha.6", + "@jest/fake-timers": "30.0.0-alpha.6", + "@jest/globals": "30.0.0-alpha.6", + "@jest/source-map": "30.0.0-alpha.6", + "@jest/test-result": "30.0.0-alpha.6", + "@jest/transform": "30.0.0-alpha.6", + "@jest/types": "30.0.0-alpha.6", + "@types/node": "*", + "chalk": "^4.0.0", + "cjs-module-lexer": "^1.0.0", + "collect-v8-coverage": "^1.0.0", + "glob": "^10.3.10", + "graceful-fs": "^4.2.9", + "jest-haste-map": "30.0.0-alpha.6", + "jest-message-util": "30.0.0-alpha.6", + "jest-mock": "30.0.0-alpha.6", + "jest-regex-util": "30.0.0-alpha.6", + "jest-resolve": "30.0.0-alpha.6", + "jest-snapshot": "30.0.0-alpha.6", + "jest-util": "30.0.0-alpha.6", + "slash": "^3.0.0", + "strip-bom": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-runtime/node_modules/@jest/schemas": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", + "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.33.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-runtime/node_modules/@jest/types": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", + "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/pattern": "30.0.0-alpha.6", + "@jest/schemas": "30.0.0-alpha.6", + "@types/istanbul-lib-coverage": "^2.0.0", + "@types/istanbul-reports": "^3.0.0", + "@types/node": "*", + "@types/yargs": "^17.0.8", + "chalk": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-runtime/node_modules/@sinclair/typebox": { + "version": "0.33.17", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", + "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", + "dev": true, + "license": "MIT" + }, + "node_modules/jest-runtime/node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/jest-runtime/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/jest-runtime/node_modules/ci-info": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.0.0.tgz", + "integrity": "sha512-TdHqgGf9odd8SXNuxtUBVx8Nv+qZOejE6qyqiy5NtbYYQOeFa6zmHkxlPzmaLxWWHsU6nJmB7AETdVPi+2NBUg==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/sibiraj-s" + } + ], + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/jest-runtime/node_modules/glob": { + "version": "10.4.5", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz", + "integrity": "sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==", + "dev": true, + "license": "ISC", + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^1.11.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/jest-runtime/node_modules/jest-message-util": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-30.0.0-alpha.6.tgz", + "integrity": "sha512-XAGJqkrBo7m3bFxWqiNqL0PyAWGf1XHR6bTve90MjBKJuIzhJsounGTzBNUw8JoU7Uzcj5Z6ZmEhaE3CDnwjfw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.12.13", + "@jest/types": "30.0.0-alpha.6", + "@types/stack-utils": "^2.0.0", + "chalk": "^4.0.0", + "graceful-fs": "^4.2.9", + "micromatch": "^4.0.7", + "pretty-format": "30.0.0-alpha.6", + "slash": "^3.0.0", + "stack-utils": "^2.0.3" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-runtime/node_modules/jest-util": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.0.0-alpha.6.tgz", + "integrity": "sha512-JlimakOVDyoMC8TEG+knoufxUqLG+Btihf1G8o5sHxz54C6oL54Wetfepp+Nhuj/1hSL0sQtkovvjlEycf9i0w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "30.0.0-alpha.6", + "@types/node": "*", + "chalk": "^4.0.0", + "ci-info": "^4.0.0", + "graceful-fs": "^4.2.9", + "picomatch": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-runtime/node_modules/minimatch": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/jest-runtime/node_modules/picomatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", + "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/jest-runtime/node_modules/pretty-format": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.0.0-alpha.6.tgz", + "integrity": "sha512-xkeffkZoqQmRrcNewpOsUCKNOl+CkPqjt3Ld749uz1S7/O7GuPNPv2fZk3v/1U/FE8/B4Zz0llVL80MKON1tOQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/schemas": "30.0.0-alpha.6", + "ansi-styles": "^5.0.0", + "react-is": "^18.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-snapshot": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-snapshot/-/jest-snapshot-30.0.0-alpha.6.tgz", + "integrity": "sha512-YCBUxSNJ9YGch3tyQdxQkOUitbmXahHL6UhSQeSMERFfX1UMrHyEDHggglocCUg4G3jdU8YzshxOJ/oaR6Ph8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.11.6", + "@babel/generator": "^7.7.2", + "@babel/plugin-syntax-jsx": "^7.7.2", + "@babel/plugin-syntax-typescript": "^7.7.2", + "@babel/types": "^7.3.3", + "@jest/expect-utils": "30.0.0-alpha.6", + "@jest/snapshot-utils": "30.0.0-alpha.6", + "@jest/transform": "30.0.0-alpha.6", + "@jest/types": "30.0.0-alpha.6", + "babel-preset-current-node-syntax": "^1.0.0", + "chalk": "^4.0.0", + "expect": "30.0.0-alpha.6", + "graceful-fs": "^4.2.9", + "jest-diff": "30.0.0-alpha.6", + "jest-get-type": "30.0.0-alpha.6", + "jest-matcher-utils": "30.0.0-alpha.6", + "jest-message-util": "30.0.0-alpha.6", + "jest-util": "30.0.0-alpha.6", + "pretty-format": "30.0.0-alpha.6", + "semver": "^7.5.3", + "synckit": "^0.9.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-snapshot/node_modules/@jest/expect-utils": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-30.0.0-alpha.6.tgz", + "integrity": "sha512-QMySMhaCUl0ZQd7Tx5X3fVWY5jtQxZNrTll0OyavdQ70ZTLgk0kU9K+XovcMWO26MK9R5EX7bBgD/j7w9hUM4w==", + "dev": true, + "license": "MIT", + "dependencies": { + "jest-get-type": "30.0.0-alpha.6" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-snapshot/node_modules/@jest/schemas": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", + "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.33.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-snapshot/node_modules/@jest/types": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", + "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/pattern": "30.0.0-alpha.6", + "@jest/schemas": "30.0.0-alpha.6", + "@types/istanbul-lib-coverage": "^2.0.0", + "@types/istanbul-reports": "^3.0.0", + "@types/node": "*", + "@types/yargs": "^17.0.8", + "chalk": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-snapshot/node_modules/@sinclair/typebox": { + "version": "0.33.17", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", + "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", + "dev": true, + "license": "MIT" + }, + "node_modules/jest-snapshot/node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/jest-snapshot/node_modules/ci-info": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.0.0.tgz", + "integrity": "sha512-TdHqgGf9odd8SXNuxtUBVx8Nv+qZOejE6qyqiy5NtbYYQOeFa6zmHkxlPzmaLxWWHsU6nJmB7AETdVPi+2NBUg==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/sibiraj-s" + } + ], + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/jest-snapshot/node_modules/diff-sequences": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-30.0.0-alpha.6.tgz", + "integrity": "sha512-DVGt3/yzbneMUTuupsMqyfSXMnU2fE0lVsC9uFsJmRpluvSi7ZhrS0GX5tnMna6Ta788FGfOUx+irI/+cAZ4EA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-snapshot/node_modules/expect": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/expect/-/expect-30.0.0-alpha.6.tgz", + "integrity": "sha512-WVi2V4iHKw/vHEyye00Q9CSZz7KHDbJkJyteUI8kTih9jiyMl3bIk7wLYFcY9D1Blnadlyb5w5NBuNjQBow99g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/expect-utils": "30.0.0-alpha.6", + "jest-get-type": "30.0.0-alpha.6", + "jest-matcher-utils": "30.0.0-alpha.6", + "jest-message-util": "30.0.0-alpha.6", + "jest-mock": "30.0.0-alpha.6", + "jest-util": "30.0.0-alpha.6" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-snapshot/node_modules/jest-diff": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-30.0.0-alpha.6.tgz", + "integrity": "sha512-43j1DoYwVKrkbB67a2gC0ijjIY9biF0JSPXv7H6zlOkzNlqYg8hSDzrurLNo6zGKatW4JSBLE79LmXPJPj1m6A==", + "dev": true, + "license": "MIT", + "dependencies": { + "chalk": "^4.0.0", + "diff-sequences": "30.0.0-alpha.6", + "jest-get-type": "30.0.0-alpha.6", + "pretty-format": "30.0.0-alpha.6" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-snapshot/node_modules/jest-get-type": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-30.0.0-alpha.6.tgz", + "integrity": "sha512-lJEoQdCY4ICN6+T0lJ9BODKuqPOEpCv2NnJsEO1nmsK0fbWZmN/pgOPHVqLfK8i3jZpUmgupJ1w8r36mc8iiBQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-snapshot/node_modules/jest-matcher-utils": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-30.0.0-alpha.6.tgz", + "integrity": "sha512-jaq7+HznsK54G0qzu96ZwfMEKHmlPiDqg6qG2p/hVQzr6Y/qVMRh8abI9Y1lX6SSXkr+S9mPAkmOsuJNLTLYmQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "chalk": "^4.0.0", + "jest-diff": "30.0.0-alpha.6", + "jest-get-type": "30.0.0-alpha.6", + "pretty-format": "30.0.0-alpha.6" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-snapshot/node_modules/jest-message-util": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-30.0.0-alpha.6.tgz", + "integrity": "sha512-XAGJqkrBo7m3bFxWqiNqL0PyAWGf1XHR6bTve90MjBKJuIzhJsounGTzBNUw8JoU7Uzcj5Z6ZmEhaE3CDnwjfw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.12.13", + "@jest/types": "30.0.0-alpha.6", + "@types/stack-utils": "^2.0.0", + "chalk": "^4.0.0", + "graceful-fs": "^4.2.9", + "micromatch": "^4.0.7", + "pretty-format": "30.0.0-alpha.6", + "slash": "^3.0.0", + "stack-utils": "^2.0.3" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-snapshot/node_modules/jest-util": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.0.0-alpha.6.tgz", + "integrity": "sha512-JlimakOVDyoMC8TEG+knoufxUqLG+Btihf1G8o5sHxz54C6oL54Wetfepp+Nhuj/1hSL0sQtkovvjlEycf9i0w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "30.0.0-alpha.6", + "@types/node": "*", + "chalk": "^4.0.0", + "ci-info": "^4.0.0", + "graceful-fs": "^4.2.9", + "picomatch": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-snapshot/node_modules/picomatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", + "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/jest-snapshot/node_modules/pretty-format": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.0.0-alpha.6.tgz", + "integrity": "sha512-xkeffkZoqQmRrcNewpOsUCKNOl+CkPqjt3Ld749uz1S7/O7GuPNPv2fZk3v/1U/FE8/B4Zz0llVL80MKON1tOQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/schemas": "30.0.0-alpha.6", + "ansi-styles": "^5.0.0", + "react-is": "^18.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-util": { + "version": "29.6.1", + "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-29.6.1.tgz", + "integrity": "sha512-NRFCcjc+/uO3ijUVyNOQJluf8PtGCe/W6cix36+M3cTFgiYqFOOW5MgN4JOOcvbUhcKTYVd1CvHz/LWi8d16Mg==", + "dev": true, + "dependencies": { + "@jest/types": "^29.6.1", + "@types/node": "*", + "chalk": "^4.0.0", + "ci-info": "^3.2.0", + "graceful-fs": "^4.2.9", + "picomatch": "^2.2.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-validate": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-30.0.0-alpha.6.tgz", + "integrity": "sha512-sINLwCenOUeJVzS5p+o1NhwKsY0de5Es0J7bsaSuZJQGRY67W20idceflr+aZ2akrKgvvqU8Tsg6lkFQyq+a6Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "30.0.0-alpha.6", + "camelcase": "^6.2.0", + "chalk": "^4.0.0", + "jest-get-type": "30.0.0-alpha.6", + "leven": "^3.1.0", + "pretty-format": "30.0.0-alpha.6" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-validate/node_modules/@jest/schemas": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", + "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.33.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-validate/node_modules/@jest/types": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", + "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/pattern": "30.0.0-alpha.6", + "@jest/schemas": "30.0.0-alpha.6", + "@types/istanbul-lib-coverage": "^2.0.0", + "@types/istanbul-reports": "^3.0.0", + "@types/node": "*", + "@types/yargs": "^17.0.8", + "chalk": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-validate/node_modules/@sinclair/typebox": { + "version": "0.33.17", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", + "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", + "dev": true, + "license": "MIT" + }, + "node_modules/jest-validate/node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/jest-validate/node_modules/camelcase": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", + "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/jest-validate/node_modules/jest-get-type": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-30.0.0-alpha.6.tgz", + "integrity": "sha512-lJEoQdCY4ICN6+T0lJ9BODKuqPOEpCv2NnJsEO1nmsK0fbWZmN/pgOPHVqLfK8i3jZpUmgupJ1w8r36mc8iiBQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-validate/node_modules/pretty-format": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.0.0-alpha.6.tgz", + "integrity": "sha512-xkeffkZoqQmRrcNewpOsUCKNOl+CkPqjt3Ld749uz1S7/O7GuPNPv2fZk3v/1U/FE8/B4Zz0llVL80MKON1tOQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/schemas": "30.0.0-alpha.6", + "ansi-styles": "^5.0.0", + "react-is": "^18.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-watcher": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-watcher/-/jest-watcher-30.0.0-alpha.6.tgz", + "integrity": "sha512-+zL1y3GSJG8EOxVSc2p0dndis0rNDcwKTs4b1bpNTI0XneeTiZlCpRBNYI+sqBl/eZtJBrQdiBRSYz7kJqg7NQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/test-result": "30.0.0-alpha.6", + "@jest/types": "30.0.0-alpha.6", + "@types/node": "*", + "ansi-escapes": "^4.2.1", + "chalk": "^4.0.0", + "emittery": "^0.13.1", + "jest-util": "30.0.0-alpha.6", + "string-length": "^4.0.1" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-watcher/node_modules/@jest/schemas": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", + "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.33.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-watcher/node_modules/@jest/types": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", + "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/pattern": "30.0.0-alpha.6", + "@jest/schemas": "30.0.0-alpha.6", + "@types/istanbul-lib-coverage": "^2.0.0", + "@types/istanbul-reports": "^3.0.0", + "@types/node": "*", + "@types/yargs": "^17.0.8", + "chalk": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-watcher/node_modules/@sinclair/typebox": { + "version": "0.33.17", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", + "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", + "dev": true, + "license": "MIT" + }, + "node_modules/jest-watcher/node_modules/ci-info": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.0.0.tgz", + "integrity": "sha512-TdHqgGf9odd8SXNuxtUBVx8Nv+qZOejE6qyqiy5NtbYYQOeFa6zmHkxlPzmaLxWWHsU6nJmB7AETdVPi+2NBUg==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/sibiraj-s" + } + ], + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/jest-watcher/node_modules/jest-util": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.0.0-alpha.6.tgz", + "integrity": "sha512-JlimakOVDyoMC8TEG+knoufxUqLG+Btihf1G8o5sHxz54C6oL54Wetfepp+Nhuj/1hSL0sQtkovvjlEycf9i0w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "30.0.0-alpha.6", + "@types/node": "*", + "chalk": "^4.0.0", + "ci-info": "^4.0.0", + "graceful-fs": "^4.2.9", + "picomatch": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest-watcher/node_modules/picomatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", + "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/jest-worker": { + "version": "27.5.1", + "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-27.5.1.tgz", + "integrity": "sha512-7vuh85V5cdDofPyxn58nrPjBktZo0u9x1g8WtjQol+jZDaE+fhN+cIvTj11GndBnMnyfrUOG1sZQxCdjKh+DKg==", + "dev": true, + "dependencies": { + "@types/node": "*", + "merge-stream": "^2.0.0", + "supports-color": "^8.0.0" + }, + "engines": { + "node": ">= 10.13.0" + } + }, + "node_modules/jest/node_modules/@jest/schemas": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.0-alpha.6.tgz", + "integrity": "sha512-Ukr3kR/VsBq8+JHU92xArhSJeFQHVHs5T1laPO00GrrNzv3DvoHn3/EVVagGn9CHbLeAyJHXFRHYxq3+520kiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.33.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest/node_modules/@jest/types": { + "version": "30.0.0-alpha.6", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.0.0-alpha.6.tgz", + "integrity": "sha512-qUjAm8uvIR7oExn/Fp7/bvn58HSZng5itQDM9x0vaxXWxxGH/8MDmqX/h7OUBz9ka+KfYRaTxe4Y6wiM8+nphw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/pattern": "30.0.0-alpha.6", + "@jest/schemas": "30.0.0-alpha.6", + "@types/istanbul-lib-coverage": "^2.0.0", + "@types/istanbul-reports": "^3.0.0", + "@types/node": "*", + "@types/yargs": "^17.0.8", + "chalk": "^4.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + } + }, + "node_modules/jest/node_modules/@sinclair/typebox": { + "version": "0.33.17", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", + "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", + "dev": true, + "license": "MIT" + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "dev": true, + "license": "MIT", + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/js-yaml/node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "license": "MIT", + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "node_modules/js2xmlparser": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/js2xmlparser/-/js2xmlparser-4.0.2.tgz", + "integrity": "sha512-6n4D8gLlLf1n5mNLQPRfViYzu9RATblzPEtm1SthMX1Pjao0r9YI9nw7ZIfRxQMERS87mcswrg+r/OYrPRX6jA==", + "dev": true, + "dependencies": { + "xmlcreate": "^2.0.4" + } + }, + "node_modules/jsdoc": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/jsdoc/-/jsdoc-4.0.2.tgz", + "integrity": "sha512-e8cIg2z62InH7azBBi3EsSEqrKx+nUtAS5bBcYTSpZFA+vhNPyhv8PTFZ0WsjOPDj04/dOLlm08EDcQJDqaGQg==", + "dev": true, + "dependencies": { + "@babel/parser": "^7.20.15", + "@jsdoc/salty": "^0.2.1", + "@types/markdown-it": "^12.2.3", + "bluebird": "^3.7.2", + "catharsis": "^0.9.0", + "escape-string-regexp": "^2.0.0", + "js2xmlparser": "^4.0.2", + "klaw": "^3.0.0", + "markdown-it": "^12.3.2", + "markdown-it-anchor": "^8.4.1", + "marked": "^4.0.10", + "mkdirp": "^1.0.4", + "requizzle": "^0.2.3", + "strip-json-comments": "^3.1.0", + "underscore": "~1.13.2" + }, + "bin": { + "jsdoc": "jsdoc.js" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/jsdoc-api": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/jsdoc-api/-/jsdoc-api-8.0.0.tgz", + "integrity": "sha512-Rnhor0suB1Ds1abjmFkFfKeD+kSMRN9oHMTMZoJVUrmtCGDwXty+sWMA9sa4xbe4UyxuPjhC7tavZ40mDKK6QQ==", + "dev": true, + "dependencies": { + "array-back": "^6.2.2", + "cache-point": "^2.0.0", + "collect-all": "^1.0.4", + "file-set": "^4.0.2", + "fs-then-native": "^2.0.0", + "jsdoc": "^4.0.0", + "object-to-spawn-args": "^2.0.1", + "temp-path": "^1.0.0", + "walk-back": "^5.1.0" + }, + "engines": { + "node": ">=12.17" + } + }, + "node_modules/jsdoc-parse": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/jsdoc-parse/-/jsdoc-parse-6.2.1.tgz", + "integrity": "sha512-9viGRUUtWOk/G4V0+nQ6rfLucz5plxh5I74WbNSNm9h9NWugCDVX4jbG8hZP9QqKGpdTPDE+qJXzaYNos3wqTA==", + "dev": true, + "dependencies": { + "array-back": "^6.2.2", + "lodash.omit": "^4.5.0", + "reduce-extract": "^1.0.0", + "sort-array": "^4.1.5", + "test-value": "^3.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/jsdoc-to-markdown": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/jsdoc-to-markdown/-/jsdoc-to-markdown-8.0.1.tgz", + "integrity": "sha512-qJfNJhkq2C26UYoOdj8L1yheTJlk1veCsxwRejRmj07XZKCn7oSkuPErx6+JoNi8afCaUKdIM5oUu0uF2/T8iw==", + "dev": true, + "dependencies": { + "array-back": "^6.2.2", + "command-line-tool": "^0.8.0", + "config-master": "^3.1.0", + "dmd": "^6.2.0", + "jsdoc-api": "^8.0.0", + "jsdoc-parse": "^6.2.1", + "walk-back": "^5.1.0" + }, + "bin": { + "jsdoc2md": "bin/cli.js" + }, + "engines": { + "node": ">=12.17" + } + }, + "node_modules/jsesc": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.0.2.tgz", + "integrity": "sha512-xKqzzWXDttJuOcawBt4KnKHHIf5oQ/Cxax+0PWFG+DFDgHNAdi+TXECADI+RYiFUMmx8792xsMbbgXj4CwnP4g==", + "dev": true, + "license": "MIT", + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/json-parse-even-better-errors": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", + "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", + "dev": true + }, + "node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true + }, + "node_modules/json5": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "dev": true, + "license": "MIT", + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/klaw": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/klaw/-/klaw-3.0.0.tgz", + "integrity": "sha512-0Fo5oir+O9jnXu5EefYbVK+mHMBeEVEy2cmctR1O1NECcCkPRreJKrS6Qt/j3KC2C148Dfo9i3pCmCMsdqGr0g==", + "dev": true, + "dependencies": { + "graceful-fs": "^4.1.9" + } + }, + "node_modules/launch-editor": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/launch-editor/-/launch-editor-2.6.0.tgz", + "integrity": "sha512-JpDCcQnyAAzZZaZ7vEiSqL690w7dAEyLao+KC96zBplnYbJS7TYNjvM3M7y3dGz+v7aIsJk3hllWuc0kWAjyRQ==", + "dev": true, + "dependencies": { + "picocolors": "^1.0.0", + "shell-quote": "^1.7.3" + } + }, + "node_modules/leven": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", + "integrity": "sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/lines-and-columns": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", + "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", + "dev": true, + "license": "MIT" + }, + "node_modules/linkify-it": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-3.0.3.tgz", + "integrity": "sha512-ynTsyrFSdE5oZ/O9GEf00kPngmOfVwazR5GKDq6EYfhlpFug3J2zybX56a2PRRpc9P+FuSoGNAwjlbDs9jJBPQ==", + "dev": true, + "dependencies": { + "uc.micro": "^1.0.1" + } + }, + "node_modules/loader-runner": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.3.0.tgz", + "integrity": "sha512-3R/1M+yS3j5ou80Me59j7F9IMs4PXs3VqRrm0TU3AbKPxlmpoY1TNscJV/oGJXo8qCatFGTfDbY6W6ipGOYXfg==", + "dev": true, + "engines": { + "node": ">=6.11.5" + } + }, + "node_modules/locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, + "dependencies": { + "p-locate": "^4.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", + "dev": true + }, + "node_modules/lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", + "dev": true + }, + "node_modules/lodash.omit": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.omit/-/lodash.omit-4.5.0.tgz", + "integrity": "sha512-XeqSp49hNGmlkj2EJlfrQFIzQ6lXdNro9sddtQzcJY8QaoC2GO0DT7xaIokHeyM+mIT0mPMlPvkYzg2xCuHdZg==", + "dev": true + }, + "node_modules/lodash.padend": { + "version": "4.6.1", + "resolved": "https://registry.npmjs.org/lodash.padend/-/lodash.padend-4.6.1.tgz", + "integrity": "sha512-sOQs2aqGpbl27tmCS1QNZA09Uqp01ZzWfDUoD+xzTii0E7dSQfRKcRetFwa+uXaxaqL+TKm7CgD2JdKP7aZBSw==", + "dev": true + }, + "node_modules/long": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz", + "integrity": "sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==" + }, + "node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dev": true, + "license": "ISC", + "dependencies": { + "yallist": "^3.0.2" + } + }, + "node_modules/make-dir": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", + "integrity": "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==", + "dev": true, + "license": "MIT", + "dependencies": { + "semver": "^7.5.3" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/makeerror": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/makeerror/-/makeerror-1.0.12.tgz", + "integrity": "sha512-JmqCvUhmt43madlpFzG4BQzG2Z3m6tvQDNKdClZnO3VbIudJYmxsT0FNJMeiB2+JTSlTQTSbU8QdesVmwJcmLg==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "tmpl": "1.0.5" + } + }, + "node_modules/markdown-it": { + "version": "12.3.2", + "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-12.3.2.tgz", + "integrity": "sha512-TchMembfxfNVpHkbtriWltGWc+m3xszaRD0CZup7GFFhzIgQqxIfn3eGj1yZpfuflzPvfkt611B2Q/Bsk1YnGg==", + "dev": true, + "dependencies": { + "argparse": "^2.0.1", + "entities": "~2.1.0", + "linkify-it": "^3.0.1", + "mdurl": "^1.0.1", + "uc.micro": "^1.0.5" + }, + "bin": { + "markdown-it": "bin/markdown-it.js" + } + }, + "node_modules/markdown-it-anchor": { + "version": "8.6.7", + "resolved": "https://registry.npmjs.org/markdown-it-anchor/-/markdown-it-anchor-8.6.7.tgz", + "integrity": "sha512-FlCHFwNnutLgVTflOYHPW2pPcl2AACqVzExlkGQNsi4CJgqOHN7YTgDd4LuhgN1BFO3TS0vLAruV1Td6dwWPJA==", + "dev": true, + "peerDependencies": { + "@types/markdown-it": "*", + "markdown-it": "*" + } + }, + "node_modules/marked": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/marked/-/marked-4.3.0.tgz", + "integrity": "sha512-PRsaiG84bK+AMvxziE/lCFss8juXjNaWzVbN5tXAm4XjeaS9NAHhop+PjQxz2A9h8Q4M/xGmzP8vqNwy6JeK0A==", + "dev": true, + "bin": { + "marked": "bin/marked.js" + }, + "engines": { + "node": ">= 12" + } + }, + "node_modules/mdurl": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-1.0.1.tgz", + "integrity": "sha512-/sKlQJCBYVY9Ers9hqzKou4H6V5UWc/M59TH2dvkt+84itfnq7uFOMLpOiOS4ujvHP4etln18fmIxA5R5fll0g==", + "dev": true + }, + "node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/memfs": { + "version": "3.5.1", + "resolved": "https://registry.npmjs.org/memfs/-/memfs-3.5.1.tgz", + "integrity": "sha512-UWbFJKvj5k+nETdteFndTpYxdeTMox/ULeqX5k/dpaQJCCFmj5EeKv3dBcyO2xmkRAx2vppRu5dVG7SOtsGOzA==", + "dev": true, + "dependencies": { + "fs-monkey": "^1.0.3" + }, + "engines": { + "node": ">= 4.0.0" + } + }, + "node_modules/merge-descriptors": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", + "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", + "dev": true, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/merge-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", + "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", + "dev": true + }, + "node_modules/methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/micromatch": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", + "dev": true, + "dependencies": { + "braces": "^3.0.3", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "dev": true, + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dev": true, + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/minimalistic-assert": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz", + "integrity": "sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==", + "dev": true + }, + "node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "dev": true, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/minipass": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", + "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/minizlib": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.0.1.tgz", + "integrity": "sha512-umcy022ILvb5/3Djuu8LWeqUa8D68JaBzlttKeMWen48SjabqS3iY5w/vzeMzMUNhLDifyhbOwKDSznB1vvrwg==", + "dependencies": { + "minipass": "^7.0.4", + "rimraf": "^5.0.5" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/minizlib/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/minizlib/node_modules/glob": { + "version": "10.4.1", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.1.tgz", + "integrity": "sha512-2jelhlq3E4ho74ZyVLN03oKdAZVUa6UDZzFLVH1H7dnoax+y9qyaq8zBkfDIggjniU19z0wU18y16jMB2eyVIw==", + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "path-scurry": "^1.11.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "engines": { + "node": ">=16 || 14 >=14.18" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/minizlib/node_modules/minimatch": { + "version": "9.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.4.tgz", + "integrity": "sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw==", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/minizlib/node_modules/rimraf": { + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.7.tgz", + "integrity": "sha512-nV6YcJo5wbLW77m+8KjH8aB/7/rxQy9SZ0HY5shnwULfS+9nmTtVXAJET5NdZmCzA4fPI/Hm1wo/Po/4mopOdg==", + "dependencies": { + "glob": "^10.3.7" + }, + "bin": { + "rimraf": "dist/esm/bin.mjs" + }, + "engines": { + "node": ">=14.18" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "dev": true, + "bin": { + "mkdirp": "bin/cmd.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/mkdirp2": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/mkdirp2/-/mkdirp2-1.0.5.tgz", + "integrity": "sha512-xOE9xbICroUDmG1ye2h4bZ8WBie9EGmACaco8K8cx6RlkJJrxGIqjGqztAI+NMhexXBcdGbSEzI6N3EJPevxZw==", + "dev": true + }, + "node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "dev": true + }, + "node_modules/multicast-dns": { + "version": "7.2.5", + "resolved": "https://registry.npmjs.org/multicast-dns/-/multicast-dns-7.2.5.tgz", + "integrity": "sha512-2eznPJP8z2BFLX50tf0LuODrpINqP1RVIm/CObbTcBRITQgmC/TjcREF1NeTBzIcR5XO/ukWo+YHOjBbFwIupg==", + "dev": true, + "dependencies": { + "dns-packet": "^5.2.2", + "thunky": "^1.0.2" + }, + "bin": { + "multicast-dns": "cli.js" + } + }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", + "dev": true, + "license": "MIT" + }, + "node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/neo-async": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz", + "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", + "dev": true + }, + "node_modules/node-forge": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.1.tgz", + "integrity": "sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==", + "dev": true, + "engines": { + "node": ">= 6.13.0" + } + }, + "node_modules/node-int64": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", + "integrity": "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==", + "dev": true, + "license": "MIT" + }, + "node_modules/node-releases": { + "version": "2.0.18", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.18.tgz", + "integrity": "sha512-d9VeXT4SJ7ZeOqGX6R5EM022wpL+eWPooLI+5UpWn2jCT1aosUQEhQP214x33Wkwx3JQMvIm+tIoVOdodFS40g==", + "dev": true + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/npm-run-path": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", + "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", + "dev": true, + "dependencies": { + "path-key": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/object-get": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/object-get/-/object-get-2.1.1.tgz", + "integrity": "sha512-7n4IpLMzGGcLEMiQKsNR7vCe+N5E9LORFrtNUVy4sO3dj9a3HedZCxEL2T7QuLhcHN1NBuBsMOKaOsAYI9IIvg==", + "dev": true + }, + "node_modules/object-inspect": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.2.tgz", + "integrity": "sha512-IRZSRuzJiynemAXPYtPe5BoI/RESNYR7TYm50MC5Mqbd3Jmw5y790sErYw3V6SryFJD64b74qQQs9wn5Bg/k3g==", + "dev": true, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/object-to-spawn-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/object-to-spawn-args/-/object-to-spawn-args-2.0.1.tgz", + "integrity": "sha512-6FuKFQ39cOID+BMZ3QaphcC8Y4cw6LXBLyIgPU+OhIYwviJamPAn+4mITapnSBQrejB+NNp+FMskhD8Cq+Ys3w==", + "dev": true, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/obuf": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/obuf/-/obuf-1.1.2.tgz", + "integrity": "sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==", + "dev": true + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "dev": true, + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/on-headers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.2.tgz", + "integrity": "sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/onetime": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", + "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", + "dev": true, + "dependencies": { + "mimic-fn": "^2.1.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/onnxruntime-common": { + "version": "1.19.2", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.19.2.tgz", + "integrity": "sha512-a4R7wYEVFbZBlp0BfhpbFWqe4opCor3KM+5Wm22Az3NGDcQMiU2hfG/0MfnBs+1ZrlSGmlgWeMcXQkDk1UFb8Q==" + }, + "node_modules/onnxruntime-node": { + "version": "1.19.2", + "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.19.2.tgz", + "integrity": "sha512-9eHMP/HKbbeUcqte1JYzaaRC8JPn7ojWeCeoyShO86TOR97OCyIyAIOGX3V95ErjslVhJRXY8Em/caIUc0hm1Q==", + "hasInstallScript": true, + "os": [ + "win32", + "darwin", + "linux" + ], + "dependencies": { + "onnxruntime-common": "1.19.2", + "tar": "^7.0.1" + } + }, + "node_modules/onnxruntime-web": { + "version": "1.21.0-dev.20241024-d9ca84ef96", + "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.21.0-dev.20241024-d9ca84ef96.tgz", + "integrity": "sha512-ANSQfMALvCviN3Y4tvTViKofKToV1WUb2r2VjZVCi3uUBPaK15oNJyIxhsNyEckBr/Num3JmSXlkHOD8HfVzSQ==", + "license": "MIT", + "dependencies": { + "flatbuffers": "^1.12.0", + "guid-typescript": "^1.0.9", + "long": "^5.2.3", + "onnxruntime-common": "1.20.0-dev.20241016-2b8fc5529b", + "platform": "^1.3.6", + "protobufjs": "^7.2.4" + } + }, + "node_modules/onnxruntime-web/node_modules/onnxruntime-common": { + "version": "1.20.0-dev.20241016-2b8fc5529b", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.20.0-dev.20241016-2b8fc5529b.tgz", + "integrity": "sha512-KZK8b6zCYGZFjd4ANze0pqBnqnFTS3GIVeclQpa2qseDpXrCQJfkWBixRcrZShNhm3LpFOZ8qJYFC5/qsJK9WQ==" + }, + "node_modules/open": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/open/-/open-8.4.2.tgz", + "integrity": "sha512-7x81NCL719oNbsq/3mh+hVrAWmFuEYUqrq/Iw3kUzH8ReypT9QQ0BLoJS7/G9k6N81XjW4qHWtjWwe/9eLy1EQ==", + "dev": true, + "dependencies": { + "define-lazy-prop": "^2.0.0", + "is-docker": "^2.1.1", + "is-wsl": "^2.2.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, + "dependencies": { + "p-limit": "^2.2.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/p-retry": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz", + "integrity": "sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==", + "dev": true, + "dependencies": { + "@types/retry": "0.12.0", + "retry": "^0.13.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/package-json-from-dist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", + "dev": true, + "license": "BlueOak-1.0.0" + }, + "node_modules/parse-json": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", + "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.0.0", + "error-ex": "^1.3.1", + "json-parse-even-better-errors": "^2.3.0", + "lines-and-columns": "^1.1.6" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", + "dev": true + }, + "node_modules/path-scurry": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", + "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.18" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/path-scurry/node_modules/lru-cache": { + "version": "10.2.2", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.2.tgz", + "integrity": "sha512-9hp3Vp2/hFQUiIwKo8XCeFVnrg8Pk3TYNPIR7tJADKi5YfcF7vEaK7avFHTlSy3kOKYaJQaalfEo6YuXdceBOQ==", + "engines": { + "node": "14 || >=16.14" + } + }, + "node_modules/path-to-regexp": { + "version": "0.1.10", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.10.tgz", + "integrity": "sha512-7lf7qcQidTku0Gu3YDPc8DJ1q7OOucfa/BSsIwjuh56VU7katFvuM8hULfkwB3Fns/rsVF7PwPKVw1sl5KQS9w==", + "dev": true + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "dev": true, + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/pirates": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.6.tgz", + "integrity": "sha512-saLsH7WeYYPiD25LDuLRRY/i+6HaPYr6G1OUlN39otzkSTxKnubR9RTxS3/Kk50s1g2JTgFwWQDQyplC5/SHZg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 6" + } + }, + "node_modules/pkg-dir": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", + "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", + "dev": true, + "dependencies": { + "find-up": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/platform": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz", + "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==" + }, + "node_modules/prettier": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.3.3.tgz", + "integrity": "sha512-i2tDNA0O5IrMO757lfrdQZCc2jPNDVntV0m/+4whiDfWaTKfMNgR7Qz0NAeGz/nRqF4m5/6CLzbP4/liHt12Ew==", + "dev": true, + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, + "node_modules/pretty-format": { + "version": "29.6.1", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.6.1.tgz", + "integrity": "sha512-7jRj+yXO0W7e4/tSJKoR7HRIHLPPjtNaUGG2xxKQnGvPNRkgWcQ0AZX6P4KBRJN4FcTBWb3sa7DVUJmocYuoog==", + "dev": true, + "dependencies": { + "@jest/schemas": "^29.6.0", + "ansi-styles": "^5.0.0", + "react-is": "^18.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/pretty-format/node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "dev": true + }, + "node_modules/protobufjs": { + "version": "7.2.6", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.2.6.tgz", + "integrity": "sha512-dgJaEDDL6x8ASUZ1YqWciTRrdOuYNzoOf27oHNfdyvKqHr5i0FV7FSLU+aIeFjyFgVxrpTOtQUi0BLLBymZaBw==", + "hasInstallScript": true, + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/node": ">=13.7.0", + "long": "^5.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "dev": true, + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/proxy-addr/node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "dev": true, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/punycode": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz", + "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/pure-rand": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-6.1.0.tgz", + "integrity": "sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/dubzzz" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fast-check" + } + ], + "license": "MIT" + }, + "node_modules/qs": { + "version": "6.13.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", + "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", + "dev": true, + "dependencies": { + "side-channel": "^1.0.6" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/randombytes": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", + "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", + "dev": true, + "dependencies": { + "safe-buffer": "^5.1.0" + } + }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", + "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", + "dev": true, + "dependencies": { + "bytes": "3.1.2", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/raw-body/node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/react-is": { + "version": "18.2.0", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.2.0.tgz", + "integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==", + "dev": true + }, + "node_modules/readable-stream": { + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.1.tgz", + "integrity": "sha512-+rQmrWMYGA90yenhTYsLWAsLsqVC8osOw6PKE1HDYiO0gdPeKe/xDHNzIAIn4C91YQ6oenEhfYqqc1883qHbjQ==", + "dev": true, + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "dev": true, + "dependencies": { + "picomatch": "^2.2.1" + }, + "engines": { + "node": ">=8.10.0" + } + }, + "node_modules/rechoir": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.8.0.tgz", + "integrity": "sha512-/vxpCXddiX8NGfGO/mTafwjq4aFa/71pvamip0++IQk3zG8cbCj0fifNPrjjF1XMXUne91jL9OoxmdykoEtifQ==", + "dev": true, + "dependencies": { + "resolve": "^1.20.0" + }, + "engines": { + "node": ">= 10.13.0" + } + }, + "node_modules/reduce-extract": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/reduce-extract/-/reduce-extract-1.0.0.tgz", + "integrity": "sha512-QF8vjWx3wnRSL5uFMyCjDeDc5EBMiryoT9tz94VvgjKfzecHAVnqmXAwQDcr7X4JmLc2cjkjFGCVzhMqDjgR9g==", + "dev": true, + "dependencies": { + "test-value": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/reduce-extract/node_modules/array-back": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-1.0.4.tgz", + "integrity": "sha512-1WxbZvrmyhkNoeYcizokbmh5oiOCIfyvGtcqbK3Ls1v1fKcquzxnQSceOx6tzq7jmai2kFLWIpGND2cLhH6TPw==", + "dev": true, + "dependencies": { + "typical": "^2.6.0" + }, + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/reduce-extract/node_modules/test-value": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/test-value/-/test-value-1.1.0.tgz", + "integrity": "sha512-wrsbRo7qP+2Je8x8DsK8ovCGyxe3sYfQwOraIY/09A2gFXU9DYKiTF14W4ki/01AEh56kMzAmlj9CaHGDDUBJA==", + "dev": true, + "dependencies": { + "array-back": "^1.0.2", + "typical": "^2.4.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/reduce-flatten": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/reduce-flatten/-/reduce-flatten-3.0.1.tgz", + "integrity": "sha512-bYo+97BmUUOzg09XwfkwALt4PQH1M5L0wzKerBt6WLm3Fhdd43mMS89HiT1B9pJIqko/6lWx3OnV4J9f2Kqp5Q==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/reduce-unique": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/reduce-unique/-/reduce-unique-2.0.1.tgz", + "integrity": "sha512-x4jH/8L1eyZGR785WY+ePtyMNhycl1N2XOLxhCbzZFaqF4AXjLzqSxa2UHgJ2ZVR/HHyPOvl1L7xRnW8ye5MdA==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/reduce-without": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/reduce-without/-/reduce-without-1.0.1.tgz", + "integrity": "sha512-zQv5y/cf85sxvdrKPlfcRzlDn/OqKFThNimYmsS3flmkioKvkUGn2Qg9cJVoQiEvdxFGLE0MQER/9fZ9sUqdxg==", + "dev": true, + "dependencies": { + "test-value": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/reduce-without/node_modules/array-back": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-1.0.4.tgz", + "integrity": "sha512-1WxbZvrmyhkNoeYcizokbmh5oiOCIfyvGtcqbK3Ls1v1fKcquzxnQSceOx6tzq7jmai2kFLWIpGND2cLhH6TPw==", + "dev": true, + "dependencies": { + "typical": "^2.6.0" + }, + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/reduce-without/node_modules/test-value": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/test-value/-/test-value-2.1.0.tgz", + "integrity": "sha512-+1epbAxtKeXttkGFMTX9H42oqzOTufR1ceCF+GYA5aOmvaPq9wd4PUS8329fn2RRLGNeUkgRLnVpycjx8DsO2w==", + "dev": true, + "dependencies": { + "array-back": "^1.0.3", + "typical": "^2.6.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==", + "dev": true + }, + "node_modules/requizzle": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/requizzle/-/requizzle-0.2.4.tgz", + "integrity": "sha512-JRrFk1D4OQ4SqovXOgdav+K8EAhSB/LJZqCz8tbX0KObcdeM15Ss59ozWMBWmmINMagCwmqn4ZNryUGpBsl6Jw==", + "dev": true, + "dependencies": { + "lodash": "^4.17.21" + } + }, + "node_modules/resolve": { + "version": "1.22.2", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.2.tgz", + "integrity": "sha512-Sb+mjNHOULsBv818T40qSPeRiuWLyaGMa5ewydRLFimneixmVy2zdivRl+AF6jaYPC8ERxGDmFSiqui6SfPd+g==", + "dev": true, + "dependencies": { + "is-core-module": "^2.11.0", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/resolve-cwd": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-3.0.0.tgz", + "integrity": "sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==", + "dev": true, + "dependencies": { + "resolve-from": "^5.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/resolve-from": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", + "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/resolve.exports": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/resolve.exports/-/resolve.exports-2.0.2.tgz", + "integrity": "sha512-X2UW6Nw3n/aMgDVy+0rSqgHlv39WZAlZrXCdnbyEiKm17DSqHX4MmQMaST3FbeWR5FTuRcUwYAziZajji0Y7mg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "dev": true, + "engines": { + "node": ">= 4" + } + }, + "node_modules/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "dev": true, + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "dev": true + }, + "node_modules/schema-utils": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.3.0.tgz", + "integrity": "sha512-pN/yOAvcC+5rQ5nERGuwrjLlYvLTbCibnZ1I7B1LaiAz9BRBlE9GMgE/eqV30P7aJQUf7Ddimy/RsbYO/GrVGg==", + "dev": true, + "dependencies": { + "@types/json-schema": "^7.0.8", + "ajv": "^6.12.5", + "ajv-keywords": "^3.5.2" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + } + }, + "node_modules/select-hose": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/select-hose/-/select-hose-2.0.0.tgz", + "integrity": "sha512-mEugaLK+YfkijB4fx0e6kImuJdCIt2LxCRcbEYPqRGCs4F2ogyfZU5IAZRdjCP8JPq2AtdNoC/Dux63d9Kiryg==", + "dev": true + }, + "node_modules/selfsigned": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/selfsigned/-/selfsigned-2.1.1.tgz", + "integrity": "sha512-GSL3aowiF7wa/WtSFwnUrludWFoNhftq8bUkH9pkzjpN2XSPOAYEgg6e0sS9s0rZwgJzJiQRPU18A6clnoW5wQ==", + "dev": true, + "dependencies": { + "node-forge": "^1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/semver": { + "version": "7.6.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", + "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/send": { + "version": "0.19.0", + "resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz", + "integrity": "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==", + "dev": true, + "dependencies": { + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "2.4.1", + "range-parser": "~1.2.1", + "statuses": "2.0.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/send/node_modules/encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/send/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + }, + "node_modules/serialize-javascript": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.2.tgz", + "integrity": "sha512-Saa1xPByTTq2gdeFZYLLo+RFE35NHZkAbqZeWNd3BpzppeVisAqpDjcp8dyf6uIvEqJRd46jemmyA4iFIeVk8g==", + "dev": true, + "dependencies": { + "randombytes": "^2.1.0" + } + }, + "node_modules/serve-index": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/serve-index/-/serve-index-1.9.1.tgz", + "integrity": "sha512-pXHfKNP4qujrtteMrSBb0rc8HJ9Ms/GrXwcUtUtD5s4ewDJI8bT3Cz2zTVRMKtri49pLx2e0Ya8ziP5Ya2pZZw==", + "dev": true, + "dependencies": { + "accepts": "~1.3.4", + "batch": "0.6.1", + "debug": "2.6.9", + "escape-html": "~1.0.3", + "http-errors": "~1.6.2", + "mime-types": "~2.1.17", + "parseurl": "~1.3.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/serve-index/node_modules/depd": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", + "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/serve-index/node_modules/http-errors": { + "version": "1.6.3", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", + "integrity": "sha512-lks+lVC8dgGyh97jxvxeYTWQFvh4uw4yC12gVl63Cg30sjPX4wuGcdkICVXDAESr6OJGjqGA8Iz5mkeN6zlD7A==", + "dev": true, + "dependencies": { + "depd": "~1.1.2", + "inherits": "2.0.3", + "setprototypeof": "1.1.0", + "statuses": ">= 1.4.0 < 2" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/serve-index/node_modules/inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==", + "dev": true + }, + "node_modules/serve-index/node_modules/setprototypeof": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz", + "integrity": "sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ==", + "dev": true + }, + "node_modules/serve-index/node_modules/statuses": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", + "integrity": "sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/serve-static": { + "version": "1.16.2", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.2.tgz", + "integrity": "sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==", + "dev": true, + "dependencies": { + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "0.19.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/set-function-length": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", + "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==", + "dev": true, + "dependencies": { + "define-data-property": "^1.1.4", + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.4", + "gopd": "^1.0.1", + "has-property-descriptors": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "dev": true + }, + "node_modules/shallow-clone": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-3.0.1.tgz", + "integrity": "sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA==", + "dev": true, + "dependencies": { + "kind-of": "^6.0.2" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/sharp": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.33.5.tgz", + "integrity": "sha512-haPVm1EkS9pgvHrQ/F3Xy+hgcuMV0Wm9vfIBSiwZ05k+xgb0PkBQpGsAA/oWdDobNaZTH5ppvHtzCFbnSEwHVw==", + "hasInstallScript": true, + "dependencies": { + "color": "^4.2.3", + "detect-libc": "^2.0.3", + "semver": "^7.6.3" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.33.5", + "@img/sharp-darwin-x64": "0.33.5", + "@img/sharp-libvips-darwin-arm64": "1.0.4", + "@img/sharp-libvips-darwin-x64": "1.0.4", + "@img/sharp-libvips-linux-arm": "1.0.5", + "@img/sharp-libvips-linux-arm64": "1.0.4", + "@img/sharp-libvips-linux-s390x": "1.0.4", + "@img/sharp-libvips-linux-x64": "1.0.4", + "@img/sharp-libvips-linuxmusl-arm64": "1.0.4", + "@img/sharp-libvips-linuxmusl-x64": "1.0.4", + "@img/sharp-linux-arm": "0.33.5", + "@img/sharp-linux-arm64": "0.33.5", + "@img/sharp-linux-s390x": "0.33.5", + "@img/sharp-linux-x64": "0.33.5", + "@img/sharp-linuxmusl-arm64": "0.33.5", + "@img/sharp-linuxmusl-x64": "0.33.5", + "@img/sharp-wasm32": "0.33.5", + "@img/sharp-win32-ia32": "0.33.5", + "@img/sharp-win32-x64": "0.33.5" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "engines": { + "node": ">=8" + } + }, + "node_modules/shell-quote": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.8.1.tgz", + "integrity": "sha512-6j1W9l1iAs/4xYBI1SYOVZyFcCis9b4KCLQ8fgAGG07QvzaRLVVRQvAy85yNmmZSjYjg4MWh4gNvlPujU/5LpA==", + "dev": true, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz", + "integrity": "sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.7", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.4", + "object-inspect": "^1.13.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "dev": true + }, + "node_modules/simple-swizzle": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", + "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==", + "dependencies": { + "is-arrayish": "^0.3.1" + } + }, + "node_modules/slash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", + "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/sockjs": { + "version": "0.3.24", + "resolved": "https://registry.npmjs.org/sockjs/-/sockjs-0.3.24.tgz", + "integrity": "sha512-GJgLTZ7vYb/JtPSSZ10hsOYIvEYsjbNU+zPdIHcUaWVNUEPivzxku31865sSSud0Da0W4lEeOPlmw93zLQchuQ==", + "dev": true, + "dependencies": { + "faye-websocket": "^0.11.3", + "uuid": "^8.3.2", + "websocket-driver": "^0.7.4" + } + }, + "node_modules/sort-array": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/sort-array/-/sort-array-4.1.5.tgz", + "integrity": "sha512-Ya4peoS1fgFN42RN1REk2FgdNOeLIEMKFGJvs7VTP3OklF8+kl2SkpVliZ4tk/PurWsrWRsdNdU+tgyOBkB9sA==", + "dev": true, + "dependencies": { + "array-back": "^5.0.0", + "typical": "^6.0.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/sort-array/node_modules/array-back": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-5.0.0.tgz", + "integrity": "sha512-kgVWwJReZWmVuWOQKEOohXKJX+nD02JAZ54D1RRWlv8L0NebauKAaFxACKzB74RTclt1+WNz5KHaLRDAPZbDEw==", + "dev": true, + "engines": { + "node": ">=10" + } + }, + "node_modules/sort-array/node_modules/typical": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/typical/-/typical-6.0.1.tgz", + "integrity": "sha512-+g3NEp7fJLe9DPa1TArHm9QAA7YciZmWnfAqEaFrBihQ7epOv9i99rjtgb6Iz0wh3WuQDjsCTDfgRoGnmHN81A==", + "dev": true, + "engines": { + "node": ">=10" + } + }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-support": { + "version": "0.5.21", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", + "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", + "dev": true, + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "node_modules/spdy": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/spdy/-/spdy-4.0.2.tgz", + "integrity": "sha512-r46gZQZQV+Kl9oItvl1JZZqJKGr+oEkB08A6BzkiR7593/7IbtuncXHd2YoYeTsG4157ZssMu9KYvUHLcjcDoA==", + "dev": true, + "dependencies": { + "debug": "^4.1.0", + "handle-thing": "^2.0.0", + "http-deceiver": "^1.2.7", + "select-hose": "^2.0.0", + "spdy-transport": "^3.0.0" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/spdy-transport": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/spdy-transport/-/spdy-transport-3.0.0.tgz", + "integrity": "sha512-hsLVFE5SjA6TCisWeJXFKniGGOpBgMLmerfO2aCyCU5s7nJ/rpAepqmFifv/GCbSbueEeAJJnmSQ2rKC/g8Fcw==", + "dev": true, + "dependencies": { + "debug": "^4.1.0", + "detect-node": "^2.0.4", + "hpack.js": "^2.1.6", + "obuf": "^1.1.2", + "readable-stream": "^3.0.6", + "wbuf": "^1.7.3" + } + }, + "node_modules/spdy-transport/node_modules/debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/spdy-transport/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "node_modules/spdy/node_modules/debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/spdy/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "node_modules/sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/stack-utils": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-2.0.6.tgz", + "integrity": "sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ==", + "dev": true, + "dependencies": { + "escape-string-regexp": "^2.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/statuses": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", + "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/stream-connect": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/stream-connect/-/stream-connect-1.0.2.tgz", + "integrity": "sha512-68Kl+79cE0RGKemKkhxTSg8+6AGrqBt+cbZAXevg2iJ6Y3zX4JhA/sZeGzLpxW9cXhmqAcE7KnJCisUmIUfnFQ==", + "dev": true, + "dependencies": { + "array-back": "^1.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/stream-connect/node_modules/array-back": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-1.0.4.tgz", + "integrity": "sha512-1WxbZvrmyhkNoeYcizokbmh5oiOCIfyvGtcqbK3Ls1v1fKcquzxnQSceOx6tzq7jmai2kFLWIpGND2cLhH6TPw==", + "dev": true, + "dependencies": { + "typical": "^2.6.0" + }, + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/stream-via": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/stream-via/-/stream-via-1.0.4.tgz", + "integrity": "sha512-DBp0lSvX5G9KGRDTkR/R+a29H+Wk2xItOF+MpZLLNDWbEV9tGPnqLPxHEYjmiz8xGtJHRIqmI+hCjmNzqoA4nQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/string-length": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/string-length/-/string-length-4.0.2.tgz", + "integrity": "sha512-+l6rNN5fYHNhZZy41RXsYptCjA2Igmq4EG7kZAYFQI1E1VTXarr6ZPXBg6eq7Y6eK4FEhY6AJlyuFIb/v/S0VQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "char-regex": "^1.0.2", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs": { + "name": "string-width", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-bom": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-4.0.0.tgz", + "integrity": "sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-final-newline": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", + "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/supports-color": { + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", + "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", + "dev": true, + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/supports-color?sponsor=1" + } + }, + "node_modules/supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "dev": true, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/synckit": { + "version": "0.9.2", + "resolved": "https://registry.npmjs.org/synckit/-/synckit-0.9.2.tgz", + "integrity": "sha512-vrozgXDQwYO72vHjUb/HnFbQx1exDjoKzqx23aXEg2a9VIg2TSFZ8FmeZpTjUCFMYw7mpX4BE2SFu8wI7asYsw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@pkgr/core": "^0.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": "^14.18.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/unts" + } + }, + "node_modules/table-layout": { + "version": "0.4.5", + "resolved": "https://registry.npmjs.org/table-layout/-/table-layout-0.4.5.tgz", + "integrity": "sha512-zTvf0mcggrGeTe/2jJ6ECkJHAQPIYEwDoqsiqBjI24mvRmQbInK5jq33fyypaCBxX08hMkfmdOqj6haT33EqWw==", + "dev": true, + "dependencies": { + "array-back": "^2.0.0", + "deep-extend": "~0.6.0", + "lodash.padend": "^4.6.1", + "typical": "^2.6.1", + "wordwrapjs": "^3.0.0" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/table-layout/node_modules/array-back": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-2.0.0.tgz", + "integrity": "sha512-eJv4pLLufP3g5kcZry0j6WXpIbzYw9GUB4mVJZno9wfwiBxbizTnHCw3VJb07cBihbFX48Y7oSrW9y+gt4glyw==", + "dev": true, + "dependencies": { + "typical": "^2.6.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/tapable": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.2.1.tgz", + "integrity": "sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/tar": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.2.0.tgz", + "integrity": "sha512-hctwP0Nb4AB60bj8WQgRYaMOuJYRAPMGiQUAotms5igN8ppfQM+IvjQ5HcKu1MaZh2Wy2KWVTe563Yj8dfc14w==", + "dependencies": { + "@isaacs/fs-minipass": "^4.0.0", + "chownr": "^3.0.0", + "minipass": "^7.1.0", + "minizlib": "^3.0.1", + "mkdirp": "^3.0.1", + "yallist": "^5.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/tar/node_modules/mkdirp": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz", + "integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==", + "bin": { + "mkdirp": "dist/cjs/src/bin.js" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/tar/node_modules/yallist": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", + "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==", + "engines": { + "node": ">=18" + } + }, + "node_modules/temp-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/temp-path/-/temp-path-1.0.0.tgz", + "integrity": "sha512-TvmyH7kC6ZVTYkqCODjJIbgvu0FKiwQpZ4D1aknE7xpcDf/qEOB8KZEK5ef2pfbVoiBhNWs3yx4y+ESMtNYmlg==", + "dev": true + }, + "node_modules/terser": { + "version": "5.31.6", + "resolved": "https://registry.npmjs.org/terser/-/terser-5.31.6.tgz", + "integrity": "sha512-PQ4DAriWzKj+qgehQ7LK5bQqCFNMmlhjR2PFFLuqGCpuCAauxemVBWwWOxo3UIwWQx8+Pr61Df++r76wDmkQBg==", + "dev": true, + "dependencies": { + "@jridgewell/source-map": "^0.3.3", + "acorn": "^8.8.2", + "commander": "^2.20.0", + "source-map-support": "~0.5.20" + }, + "bin": { + "terser": "bin/terser" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/terser-webpack-plugin": { + "version": "5.3.10", + "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.3.10.tgz", + "integrity": "sha512-BKFPWlPDndPs+NGGCr1U59t0XScL5317Y0UReNrHaw9/FwhPENlq6bfgs+4yPfyP51vqC1bQ4rp1EfXW5ZSH9w==", + "dev": true, + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.20", + "jest-worker": "^27.4.5", + "schema-utils": "^3.1.1", + "serialize-javascript": "^6.0.1", + "terser": "^5.26.0" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^5.1.0" + }, + "peerDependenciesMeta": { + "@swc/core": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "uglify-js": { + "optional": true + } + } + }, + "node_modules/test-exclude": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", + "integrity": "sha512-cAGWPIyOHU6zlmg88jwm7VRyXnMN7iV68OGAbYDk/Mh/xC/pzVPlQtY6ngoIH/5/tciuhGfvESU8GrHrcxD56w==", + "dev": true, + "license": "ISC", + "dependencies": { + "@istanbuljs/schema": "^0.1.2", + "glob": "^7.1.4", + "minimatch": "^3.0.4" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/test-value": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/test-value/-/test-value-3.0.0.tgz", + "integrity": "sha512-sVACdAWcZkSU9x7AOmJo5TqE+GyNJknHaHsMrR6ZnhjVlVN9Yx6FjHrsKZ3BjIpPCT68zYesPWkakrNupwfOTQ==", + "dev": true, + "dependencies": { + "array-back": "^2.0.0", + "typical": "^2.6.1" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/test-value/node_modules/array-back": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-2.0.0.tgz", + "integrity": "sha512-eJv4pLLufP3g5kcZry0j6WXpIbzYw9GUB4mVJZno9wfwiBxbizTnHCw3VJb07cBihbFX48Y7oSrW9y+gt4glyw==", + "dev": true, + "dependencies": { + "typical": "^2.6.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/thunky": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/thunky/-/thunky-1.1.0.tgz", + "integrity": "sha512-eHY7nBftgThBqOyHGVN+l8gF0BucP09fMo0oO/Lb0w1OF80dJv+lDVpXG60WMQvkcxAkNybKsrEIE3ZtKGmPrA==", + "dev": true + }, + "node_modules/tmpl": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.5.tgz", + "integrity": "sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "dev": true, + "engines": { + "node": ">=0.6" + } + }, + "node_modules/tslib": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz", + "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==", + "devOptional": true + }, + "node_modules/type-detect": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", + "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/type-fest": { + "version": "0.21.3", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", + "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", + "dev": true, + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "dev": true, + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/typescript": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.2.2.tgz", + "integrity": "sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==", + "dev": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/typical": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/typical/-/typical-2.6.1.tgz", + "integrity": "sha512-ofhi8kjIje6npGozTip9Fr8iecmYfEbS06i0JnIg+rh51KakryWF4+jX8lLKZVhy6N+ID45WYSFCxPOdTWCzNg==", + "dev": true + }, + "node_modules/uc.micro": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-1.0.6.tgz", + "integrity": "sha512-8Y75pvTYkLJW2hWQHXxoqRgV7qb9B+9vFEtidML+7koHUFapnVJAZ6cKs+Qjz5Aw3aZWHMC6u0wJE3At+nSGwA==", + "dev": true + }, + "node_modules/uglify-js": { + "version": "3.17.4", + "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.17.4.tgz", + "integrity": "sha512-T9q82TJI9e/C1TAxYvfb16xO120tMVFZrGA3f9/P4424DNu6ypK103y0GPFVa17yotwSyZW5iYXgjYHkGrJW/g==", + "dev": true, + "optional": true, + "bin": { + "uglifyjs": "bin/uglifyjs" + }, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/underscore": { + "version": "1.13.6", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.6.tgz", + "integrity": "sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==", + "dev": true + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/update-browserslist-db": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.1.tgz", + "integrity": "sha512-R8UzCaa9Az+38REPiJ1tXlImTJXlVfgHZsglwBD/k6nj76ctsH1E3q4doGrukiLQd3sGQYu56r5+lo5r94l29A==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "escalade": "^3.2.0", + "picocolors": "^1.1.0" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dev": true, + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "dev": true + }, + "node_modules/utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", + "dev": true, + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "dev": true, + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/v8-to-istanbul": { + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.3.0.tgz", + "integrity": "sha512-kiGUalWN+rgBJ/1OHZsBtU4rXZOfj/7rKQxULKlIzwzQSvMJUUNgPwJEEh7gU6xEVxC0ahoOBvN2YI8GH6FNgA==", + "dev": true, + "license": "ISC", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.12", + "@types/istanbul-lib-coverage": "^2.0.1", + "convert-source-map": "^2.0.0" + }, + "engines": { + "node": ">=10.12.0" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/walk-back": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/walk-back/-/walk-back-5.1.0.tgz", + "integrity": "sha512-Uhxps5yZcVNbLEAnb+xaEEMdgTXl9qAQDzKYejG2AZ7qPwRQ81lozY9ECDbjLPNWm7YsO1IK5rsP1KoQzXAcGA==", + "dev": true, + "engines": { + "node": ">=12.17" + } + }, + "node_modules/walker": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/walker/-/walker-1.0.8.tgz", + "integrity": "sha512-ts/8E8l5b7kY0vlWLewOkDXMmPdLcVV4GmOQLyxuSswIJsweeFZtAsMF7k1Nszz+TYBQrlYRmzOnr398y1JemQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "makeerror": "1.0.12" + } + }, + "node_modules/watchpack": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.4.2.tgz", + "integrity": "sha512-TnbFSbcOCcDgjZ4piURLCbJ3nJhznVh9kw6F6iokjiFPl8ONxe9A6nMDVXDiNbrSfLILs6vB07F7wLBrwPYzJw==", + "dev": true, + "dependencies": { + "glob-to-regexp": "^0.4.1", + "graceful-fs": "^4.1.2" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/wavefile": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/wavefile/-/wavefile-11.0.0.tgz", + "integrity": "sha512-/OBiAALgWU24IG7sC84cDO/KfFuvajWc5Uec0oV2zrpOOZZDgGdOwHwgEzOrwh8jkubBk7PtZfQBIcI1OaE5Ng==", + "dev": true, + "bin": { + "wavefile": "bin/wavefile.js" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wbuf": { + "version": "1.7.3", + "resolved": "https://registry.npmjs.org/wbuf/-/wbuf-1.7.3.tgz", + "integrity": "sha512-O84QOnr0icsbFGLS0O3bI5FswxzRr8/gHwWkDlQFskhSPryQXvrTMxjxGP4+iWYoauLoBvfDpkrOauZ+0iZpDA==", + "dev": true, + "dependencies": { + "minimalistic-assert": "^1.0.0" + } + }, + "node_modules/webpack": { + "version": "5.94.0", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.94.0.tgz", + "integrity": "sha512-KcsGn50VT+06JH/iunZJedYGUJS5FGjow8wb9c0v5n1Om8O1g4L6LjtfxwlXIATopoQu+vOXXa7gYisWxCoPyg==", + "dev": true, + "dependencies": { + "@types/estree": "^1.0.5", + "@webassemblyjs/ast": "^1.12.1", + "@webassemblyjs/wasm-edit": "^1.12.1", + "@webassemblyjs/wasm-parser": "^1.12.1", + "acorn": "^8.7.1", + "acorn-import-attributes": "^1.9.5", + "browserslist": "^4.21.10", + "chrome-trace-event": "^1.0.2", + "enhanced-resolve": "^5.17.1", + "es-module-lexer": "^1.2.1", + "eslint-scope": "5.1.1", + "events": "^3.2.0", + "glob-to-regexp": "^0.4.1", + "graceful-fs": "^4.2.11", + "json-parse-even-better-errors": "^2.3.1", + "loader-runner": "^4.2.0", + "mime-types": "^2.1.27", + "neo-async": "^2.6.2", + "schema-utils": "^3.2.0", + "tapable": "^2.1.1", + "terser-webpack-plugin": "^5.3.10", + "watchpack": "^2.4.1", + "webpack-sources": "^3.2.3" + }, + "bin": { + "webpack": "bin/webpack.js" + }, + "engines": { + "node": ">=10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependenciesMeta": { + "webpack-cli": { + "optional": true + } + } + }, + "node_modules/webpack-cli": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/webpack-cli/-/webpack-cli-5.0.2.tgz", + "integrity": "sha512-4y3W5Dawri5+8dXm3+diW6Mn1Ya+Dei6eEVAdIduAmYNLzv1koKVAqsfgrrc9P2mhrYHQphx5htnGkcNwtubyQ==", + "dev": true, + "dependencies": { + "@discoveryjs/json-ext": "^0.5.0", + "@webpack-cli/configtest": "^2.0.1", + "@webpack-cli/info": "^2.0.1", + "@webpack-cli/serve": "^2.0.2", + "colorette": "^2.0.14", + "commander": "^10.0.1", + "cross-spawn": "^7.0.3", + "envinfo": "^7.7.3", + "fastest-levenshtein": "^1.0.12", + "import-local": "^3.0.2", + "interpret": "^3.1.1", + "rechoir": "^0.8.0", + "webpack-merge": "^5.7.3" + }, + "bin": { + "webpack-cli": "bin/cli.js" + }, + "engines": { + "node": ">=14.15.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "5.x.x" + }, + "peerDependenciesMeta": { + "@webpack-cli/generators": { + "optional": true + }, + "webpack-bundle-analyzer": { + "optional": true + }, + "webpack-dev-server": { + "optional": true + } + } + }, + "node_modules/webpack-cli/node_modules/commander": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-10.0.1.tgz", + "integrity": "sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==", + "dev": true, + "engines": { + "node": ">=14" + } + }, + "node_modules/webpack-dev-middleware": { + "version": "5.3.4", + "resolved": "https://registry.npmjs.org/webpack-dev-middleware/-/webpack-dev-middleware-5.3.4.tgz", + "integrity": "sha512-BVdTqhhs+0IfoeAf7EoH5WE+exCmqGerHfDM0IL096Px60Tq2Mn9MAbnaGUe6HiMa41KMCYF19gyzZmBcq/o4Q==", + "dev": true, + "dependencies": { + "colorette": "^2.0.10", + "memfs": "^3.4.3", + "mime-types": "^2.1.31", + "range-parser": "^1.2.1", + "schema-utils": "^4.0.0" + }, + "engines": { + "node": ">= 12.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^4.0.0 || ^5.0.0" + } + }, + "node_modules/webpack-dev-middleware/node_modules/ajv": { + "version": "8.12.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz", + "integrity": "sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==", + "dev": true, + "dependencies": { + "fast-deep-equal": "^3.1.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/webpack-dev-middleware/node_modules/ajv-keywords": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-5.1.0.tgz", + "integrity": "sha512-YCS/JNFAUyr5vAuhk1DWm1CBxRHW9LbJ2ozWeemrIqpbsqKjHVxYPyi5GC0rjZIT5JxJ3virVTS8wk4i/Z+krw==", + "dev": true, + "dependencies": { + "fast-deep-equal": "^3.1.3" + }, + "peerDependencies": { + "ajv": "^8.8.2" + } + }, + "node_modules/webpack-dev-middleware/node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true + }, + "node_modules/webpack-dev-middleware/node_modules/schema-utils": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-4.0.1.tgz", + "integrity": "sha512-lELhBAAly9NowEsX0yZBlw9ahZG+sK/1RJ21EpzdYHKEs13Vku3LJ+MIPhh4sMs0oCCeufZQEQbMekiA4vuVIQ==", + "dev": true, + "dependencies": { + "@types/json-schema": "^7.0.9", + "ajv": "^8.9.0", + "ajv-formats": "^2.1.1", + "ajv-keywords": "^5.1.0" + }, + "engines": { + "node": ">= 12.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + } + }, + "node_modules/webpack-dev-server": { + "version": "4.13.3", + "resolved": "https://registry.npmjs.org/webpack-dev-server/-/webpack-dev-server-4.13.3.tgz", + "integrity": "sha512-KqqzrzMRSRy5ePz10VhjyL27K2dxqwXQLP5rAKwRJBPUahe7Z2bBWzHw37jeb8GCPKxZRO79ZdQUAPesMh/Nug==", + "dev": true, + "dependencies": { + "@types/bonjour": "^3.5.9", + "@types/connect-history-api-fallback": "^1.3.5", + "@types/express": "^4.17.13", + "@types/serve-index": "^1.9.1", + "@types/serve-static": "^1.13.10", + "@types/sockjs": "^0.3.33", + "@types/ws": "^8.5.1", + "ansi-html-community": "^0.0.8", + "bonjour-service": "^1.0.11", + "chokidar": "^3.5.3", + "colorette": "^2.0.10", + "compression": "^1.7.4", + "connect-history-api-fallback": "^2.0.0", + "default-gateway": "^6.0.3", + "express": "^4.17.3", + "graceful-fs": "^4.2.6", + "html-entities": "^2.3.2", + "http-proxy-middleware": "^2.0.3", + "ipaddr.js": "^2.0.1", + "launch-editor": "^2.6.0", + "open": "^8.0.9", + "p-retry": "^4.5.0", + "rimraf": "^3.0.2", + "schema-utils": "^4.0.0", + "selfsigned": "^2.1.1", + "serve-index": "^1.9.1", + "sockjs": "^0.3.24", + "spdy": "^4.0.2", + "webpack-dev-middleware": "^5.3.1", + "ws": "^8.13.0" + }, + "bin": { + "webpack-dev-server": "bin/webpack-dev-server.js" + }, + "engines": { + "node": ">= 12.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^4.37.0 || ^5.0.0" + }, + "peerDependenciesMeta": { + "webpack": { + "optional": true + }, + "webpack-cli": { + "optional": true + } + } + }, + "node_modules/webpack-dev-server/node_modules/ajv": { + "version": "8.12.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz", + "integrity": "sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==", + "dev": true, + "dependencies": { + "fast-deep-equal": "^3.1.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/webpack-dev-server/node_modules/ajv-keywords": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-5.1.0.tgz", + "integrity": "sha512-YCS/JNFAUyr5vAuhk1DWm1CBxRHW9LbJ2ozWeemrIqpbsqKjHVxYPyi5GC0rjZIT5JxJ3virVTS8wk4i/Z+krw==", + "dev": true, + "dependencies": { + "fast-deep-equal": "^3.1.3" + }, + "peerDependencies": { + "ajv": "^8.8.2" + } + }, + "node_modules/webpack-dev-server/node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true + }, + "node_modules/webpack-dev-server/node_modules/schema-utils": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-4.0.1.tgz", + "integrity": "sha512-lELhBAAly9NowEsX0yZBlw9ahZG+sK/1RJ21EpzdYHKEs13Vku3LJ+MIPhh4sMs0oCCeufZQEQbMekiA4vuVIQ==", + "dev": true, + "dependencies": { + "@types/json-schema": "^7.0.9", + "ajv": "^8.9.0", + "ajv-formats": "^2.1.1", + "ajv-keywords": "^5.1.0" + }, + "engines": { + "node": ">= 12.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + } + }, + "node_modules/webpack-merge": { + "version": "5.8.0", + "resolved": "https://registry.npmjs.org/webpack-merge/-/webpack-merge-5.8.0.tgz", + "integrity": "sha512-/SaI7xY0831XwP6kzuwhKWVKDP9t1QY1h65lAFLbZqMPIuYcD9QAW4u9STIbU9kaJbPBB/geU/gLr1wDjOhQ+Q==", + "dev": true, + "dependencies": { + "clone-deep": "^4.0.1", + "wildcard": "^2.0.0" + }, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/webpack-sources": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-3.2.3.tgz", + "integrity": "sha512-/DyMEOrDgLKKIG0fmvtz+4dUX/3Ghozwgm6iPp8KRhvn+eQf9+Q7GWxVNMk3+uCPWfdXYC4ExGBckIXdFEfH1w==", + "dev": true, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/websocket-driver": { + "version": "0.7.4", + "resolved": "https://registry.npmjs.org/websocket-driver/-/websocket-driver-0.7.4.tgz", + "integrity": "sha512-b17KeDIQVjvb0ssuSDF2cYXSg2iztliJ4B9WdsuB6J952qCPKmnVq4DyW5motImXHDC1cBT/1UezrJVsKw5zjg==", + "dev": true, + "dependencies": { + "http-parser-js": ">=0.5.1", + "safe-buffer": ">=5.1.0", + "websocket-extensions": ">=0.1.1" + }, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/websocket-extensions": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/websocket-extensions/-/websocket-extensions-0.1.4.tgz", + "integrity": "sha512-OqedPIGOfsDlo31UNwYbCFMSaO9m9G/0faIHj5/dZFDMFqPTcx6UwqyOy3COEaEOg/9VsGIpdqn62W5KhoKSpg==", + "dev": true, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/wildcard": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/wildcard/-/wildcard-2.0.0.tgz", + "integrity": "sha512-JcKqAHLPxcdb9KM49dufGXn2x3ssnfjbcaQdLlfZsL9rH9wgDQjUtDxbo8NE0F6SFvydeu1VhZe7hZuHsB2/pw==", + "dev": true + }, + "node_modules/wordwrap": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", + "integrity": "sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==", + "dev": true + }, + "node_modules/wordwrapjs": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/wordwrapjs/-/wordwrapjs-3.0.0.tgz", + "integrity": "sha512-mO8XtqyPvykVCsrwj5MlOVWvSnCdT+C+QVbm6blradR7JExAhbkZ7hZ9A+9NUtwzSqrlUo9a67ws0EiILrvRpw==", + "dev": true, + "dependencies": { + "reduce-flatten": "^1.0.1", + "typical": "^2.6.1" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/wordwrapjs/node_modules/reduce-flatten": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/reduce-flatten/-/reduce-flatten-1.0.1.tgz", + "integrity": "sha512-j5WfFJfc9CoXv/WbwVLHq74i/hdTUpy+iNC534LxczMRP67vJeK3V9JOdnL0N1cIRbn9mYhE2yVjvvKXDxvNXQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true + }, + "node_modules/write-file-atomic": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-5.0.1.tgz", + "integrity": "sha512-+QU2zd6OTD8XWIJCbffaiQeH9U73qIqafo1x6V1snCWYGJf6cVE0cDR4D8xRzcEnfI21IFrUPzPGtcPf8AC+Rw==", + "dev": true, + "license": "ISC", + "dependencies": { + "imurmurhash": "^0.1.4", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + } + }, + "node_modules/write-file-atomic/node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/ws": { + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.1.tgz", + "integrity": "sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==", + "dev": true, + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/xmlcreate": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/xmlcreate/-/xmlcreate-2.0.4.tgz", + "integrity": "sha512-nquOebG4sngPmGPICTS5EnxqhKbCmz5Ox5hsszI2T6U5qdrJizBc+0ilYSEjTSzU0yZcmvppztXe/5Al5fUwdg==", + "dev": true + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "dev": true, + "license": "ISC" + }, + "node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } } - } - }, - "node_modules/xmlcreate": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/xmlcreate/-/xmlcreate-2.0.4.tgz", - "integrity": "sha512-nquOebG4sngPmGPICTS5EnxqhKbCmz5Ox5hsszI2T6U5qdrJizBc+0ilYSEjTSzU0yZcmvppztXe/5Al5fUwdg==", - "dev": true - }, - "node_modules/y18n": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", - "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=10" - } - }, - "node_modules/yallist": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", - "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", - "dev": true, - "license": "ISC" - }, - "node_modules/yargs": { - "version": "17.7.2", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", - "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", - "dev": true, - "license": "MIT", - "dependencies": { - "cliui": "^8.0.1", - "escalade": "^3.1.1", - "get-caller-file": "^2.0.5", - "require-directory": "^2.1.1", - "string-width": "^4.2.3", - "y18n": "^5.0.5", - "yargs-parser": "^21.1.1" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/yargs-parser": { - "version": "21.1.1", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", - "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=12" - } - }, - "node_modules/yocto-queue": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", - "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } } - } } diff --git a/package.json b/package.json index 94e59b866..2eeb1a775 100644 --- a/package.json +++ b/package.json @@ -1,102 +1,102 @@ { - "name": "@huggingface/transformers", - "version": "3.0.2", - "description": "State-of-the-art Machine Learning for the web. Run 🤗 Transformers directly in your browser, with no need for a server!", - "main": "./src/transformers.js", - "types": "./types/transformers.d.ts", - "type": "module", - "exports": { - "node": { - "import": { - "types": "./types/transformers.d.ts", - "default": "./dist/transformers.mjs" - }, - "require": { - "types": "./types/transformers.d.ts", - "default": "./dist/transformers.cjs" - } + "name": "@huggingface/transformers", + "version": "3.0.2", + "description": "State-of-the-art Machine Learning for the web. Run 🤗 Transformers directly in your browser, with no need for a server!", + "main": "./src/transformers.js", + "types": "./types/transformers.d.ts", + "type": "module", + "exports": { + "node": { + "import": { + "types": "./types/transformers.d.ts", + "default": "./dist/transformers.mjs" + }, + "require": { + "types": "./types/transformers.d.ts", + "default": "./dist/transformers.cjs" + } + }, + "default": { + "types": "./types/transformers.d.ts", + "default": "./dist/transformers.js" + } }, - "default": { - "types": "./types/transformers.d.ts", - "default": "./dist/transformers.js" - } - }, - "imports": { - "#onnxruntime-webgpu": { - "node": "onnxruntime-web", - "default": "onnxruntime-web/webgpu" - } - }, - "scripts": { - "format": "prettier --write .", - "format:check": "prettier --check .", - "typegen": "tsc ./src/transformers.js --allowJs --declaration --emitDeclarationOnly --declarationMap --outDir types", - "dev": "webpack serve --no-client-overlay", - "build": "webpack && npm run typegen", - "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js --verbose", - "readme": "python ./docs/scripts/build_readme.py", - "docs-api": "node ./docs/scripts/generate.js", - "docs-preview": "doc-builder preview transformers.js ./docs/source/ --not_python_module", - "docs-build": "doc-builder build transformers.js ./docs/source/ --not_python_module --build_dir ./docs/build/" - }, - "repository": { - "type": "git", - "url": "git+https://github.com/huggingface/transformers.js.git" - }, - "keywords": [ - "transformers", - "transformers.js", - "huggingface", - "hugging face", - "machine learning", - "deep learning", - "artificial intelligence", - "AI", - "ML" - ], - "author": "Hugging Face", - "license": "Apache-2.0", - "bugs": { - "url": "https://github.com/huggingface/transformers.js/issues" - }, - "homepage": "https://github.com/huggingface/transformers.js#readme", - "dependencies": { - "@huggingface/jinja": "^0.3.0", - "onnxruntime-node": "1.19.2", - "onnxruntime-web": "1.21.0-dev.20241024-d9ca84ef96", - "sharp": "^0.33.5" - }, - "devDependencies": { - "@types/jest": "^29.5.14", - "@webgpu/types": "^0.1.44", - "catharsis": "github:xenova/catharsis", - "jest": "^30.0.0-alpha.6", - "jest-environment-node": "^30.0.0-alpha.6", - "jsdoc-to-markdown": "^8.0.1", - "prettier": "3.3.3", - "typescript": "^5.2.2", - "wavefile": "^11.0.0", - "webpack": "^5.80.0", - "webpack-cli": "^5.0.2", - "webpack-dev-server": "^4.13.3" - }, - "files": [ - "src", - "dist", - "types", - "README.md", - "LICENSE" - ], - "browser": { - "fs": false, - "path": false, - "url": false, - "sharp": false, - "onnxruntime-node": false - }, - "publishConfig": { - "access": "public" - }, - "jsdelivr": "./dist/transformers.min.js", - "unpkg": "./dist/transformers.min.js" + "imports": { + "#onnxruntime-webgpu": { + "node": "onnxruntime-web", + "default": "onnxruntime-web/webgpu" + } + }, + "scripts": { + "format": "prettier --write .", + "format:check": "prettier --check .", + "typegen": "tsc ./src/transformers.js --allowJs --declaration --emitDeclarationOnly --declarationMap --outDir types", + "dev": "webpack serve --no-client-overlay", + "build": "webpack && npm run typegen", + "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js --verbose", + "readme": "python ./docs/scripts/build_readme.py", + "docs-api": "node ./docs/scripts/generate.js", + "docs-preview": "doc-builder preview transformers.js ./docs/source/ --not_python_module", + "docs-build": "doc-builder build transformers.js ./docs/source/ --not_python_module --build_dir ./docs/build/" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/huggingface/transformers.js.git" + }, + "keywords": [ + "transformers", + "transformers.js", + "huggingface", + "hugging face", + "machine learning", + "deep learning", + "artificial intelligence", + "AI", + "ML" + ], + "author": "Hugging Face", + "license": "Apache-2.0", + "bugs": { + "url": "https://github.com/huggingface/transformers.js/issues" + }, + "homepage": "https://github.com/huggingface/transformers.js#readme", + "dependencies": { + "@huggingface/jinja": "^0.3.0", + "onnxruntime-node": "1.19.2", + "onnxruntime-web": "1.21.0-dev.20241024-d9ca84ef96", + "sharp": "^0.33.5" + }, + "devDependencies": { + "@types/jest": "^29.5.14", + "@webgpu/types": "^0.1.44", + "catharsis": "github:xenova/catharsis", + "jest": "^30.0.0-alpha.6", + "jest-environment-node": "^30.0.0-alpha.6", + "jsdoc-to-markdown": "^8.0.1", + "prettier": "3.3.3", + "typescript": "^5.2.2", + "wavefile": "^11.0.0", + "webpack": "^5.80.0", + "webpack-cli": "^5.0.2", + "webpack-dev-server": "^4.13.3" + }, + "files": [ + "src", + "dist", + "types", + "README.md", + "LICENSE" + ], + "browser": { + "fs": false, + "path": false, + "url": false, + "sharp": false, + "onnxruntime-node": false + }, + "publishConfig": { + "access": "public" + }, + "jsdelivr": "./dist/transformers.min.js", + "unpkg": "./dist/transformers.min.js" } diff --git a/src/backends/onnx.js b/src/backends/onnx.js index de89da037..26a689bff 100644 --- a/src/backends/onnx.js +++ b/src/backends/onnx.js @@ -6,29 +6,29 @@ * So, we just import both packages, and use the appropriate one based on the environment: * - When running in node, we use `onnxruntime-node`. * - When running in the browser, we use `onnxruntime-web` (`onnxruntime-node` is not bundled). - * + * * This module is not directly exported, but can be accessed through the environment variables: * ```javascript * import { env } from '@huggingface/transformers'; * console.log(env.backends.onnx); * ``` - * + * * @module backends/onnx */ -import { env, apis } from '../env.js'; +import { env, apis } from "../env.js"; // NOTE: Import order matters here. We need to import `onnxruntime-node` before `onnxruntime-web`. // In either case, we select the default export if it exists, otherwise we use the named export. -import * as ONNX_NODE from 'onnxruntime-node'; +import * as ONNX_NODE from "onnxruntime-node"; // Use subpath-imports to ensure Node.js and browser interoperability. // See package.json and https://nodejs.org/api/packages.html#subpath-imports // for more information. // @ts-ignore -import * as ONNX_WEB from '#onnxruntime-webgpu'; +import * as ONNX_WEB from "#onnxruntime-webgpu"; -export { Tensor } from 'onnxruntime-common'; +export { Tensor } from "onnxruntime-common"; /** * @typedef {import('onnxruntime-common').InferenceSession.ExecutionProviderConfig} ONNXExecutionProviders @@ -38,19 +38,19 @@ export { Tensor } from 'onnxruntime-common'; const DEVICE_TO_EXECUTION_PROVIDER_MAPPING = Object.freeze({ auto: null, // Auto-detect based on device and environment gpu: null, // Auto-detect GPU - cpu: 'cpu', // CPU - wasm: 'wasm', // WebAssembly - webgpu: 'webgpu', // WebGPU - cuda: 'cuda', // CUDA - dml: 'dml', // DirectML - - webnn: { name: 'webnn', deviceType: 'cpu' }, // WebNN (default) - 'webnn-npu': { name: 'webnn', deviceType: 'npu' }, // WebNN NPU - 'webnn-gpu': { name: 'webnn', deviceType: 'gpu' }, // WebNN GPU - 'webnn-cpu': { name: 'webnn', deviceType: 'cpu' }, // WebNN CPU + cpu: "cpu", // CPU + wasm: "wasm", // WebAssembly + webgpu: "webgpu", // WebGPU + cuda: "cuda", // CUDA + dml: "dml", // DirectML + + webnn: { name: "webnn", deviceType: "cpu" }, // WebNN (default) + "webnn-npu": { name: "webnn", deviceType: "npu" }, // WebNN NPU + "webnn-gpu": { name: "webnn", deviceType: "gpu" }, // WebNN GPU + "webnn-cpu": { name: "webnn", deviceType: "cpu" }, // WebNN CPU }); -/** +/** * The list of supported devices, sorted by priority/performance. * @type {import("../utils/devices.js").DeviceType[]} */ @@ -59,12 +59,11 @@ const supportedDevices = []; /** @type {ONNXExecutionProviders[]} */ let defaultDevices; let ONNX; -const ORT_SYMBOL = Symbol.for('onnxruntime'); +const ORT_SYMBOL = Symbol.for("onnxruntime"); if (ORT_SYMBOL in globalThis) { - // If the JS runtime exposes their own ONNX runtime, use it - ONNX = globalThis[ORT_SYMBOL]; - + // If the JS runtime exposes their own ONNX runtime, use it + ONNX = globalThis[ORT_SYMBOL]; } else if (apis.IS_NODE_ENV) { ONNX = ONNX_NODE.default ?? ONNX_NODE; @@ -76,34 +75,34 @@ if (ORT_SYMBOL in globalThis) { // | DirectML | ✔️ | ✔️ | ❌ | ❌ | ❌ | ❌ | // | CUDA | ❌ | ❌ | ✔️ (CUDA v11.8) | ❌ | ❌ | ❌ | switch (process.platform) { - case 'win32': // Windows x64 and Windows arm64 - supportedDevices.push('dml'); + case "win32": // Windows x64 and Windows arm64 + supportedDevices.push("dml"); break; - case 'linux': // Linux x64 and Linux arm64 - if (process.arch === 'x64') { - supportedDevices.push('cuda'); + case "linux": // Linux x64 and Linux arm64 + if (process.arch === "x64") { + supportedDevices.push("cuda"); } break; - case 'darwin': // MacOS x64 and MacOS arm64 + case "darwin": // MacOS x64 and MacOS arm64 break; } - supportedDevices.push('cpu'); - defaultDevices = ['cpu']; + supportedDevices.push("cpu"); + defaultDevices = ["cpu"]; } else { ONNX = ONNX_WEB; if (apis.IS_WEBNN_AVAILABLE) { // TODO: Only push supported providers (depending on available hardware) - supportedDevices.push('webnn-npu', 'webnn-gpu', 'webnn-cpu', 'webnn'); + supportedDevices.push("webnn-npu", "webnn-gpu", "webnn-cpu", "webnn"); } if (apis.IS_WEBGPU_AVAILABLE) { - supportedDevices.push('webgpu'); + supportedDevices.push("webgpu"); } - supportedDevices.push('wasm'); - defaultDevices = ['wasm']; + supportedDevices.push("wasm"); + defaultDevices = ["wasm"]; } // @ts-ignore @@ -123,19 +122,16 @@ export function deviceToExecutionProviders(device = null) { case "auto": return supportedDevices; case "gpu": - return supportedDevices.filter(x => - ["webgpu", "cuda", "dml", "webnn-gpu"].includes(x), - ); + return supportedDevices.filter((x) => ["webgpu", "cuda", "dml", "webnn-gpu"].includes(x)); } if (supportedDevices.includes(device)) { return [DEVICE_TO_EXECUTION_PROVIDER_MAPPING[device] ?? device]; } - throw new Error(`Unsupported device: "${device}". Should be one of: ${supportedDevices.join(', ')}.`) + throw new Error(`Unsupported device: "${device}". Should be one of: ${supportedDevices.join(", ")}.`); } - /** * To prevent multiple calls to `initWasm()`, we store the first call in a Promise * that is resolved when the first InferenceSession is created. Subsequent calls @@ -194,13 +190,13 @@ if (ONNX_ENV?.wasm) { ONNX_ENV.wasm.proxy = false; // https://developer.mozilla.org/en-US/docs/Web/API/crossOriginIsolated - if (typeof crossOriginIsolated === 'undefined' || !crossOriginIsolated) { + if (typeof crossOriginIsolated === "undefined" || !crossOriginIsolated) { ONNX_ENV.wasm.numThreads = 1; } } if (ONNX_ENV?.webgpu) { - ONNX_ENV.webgpu.powerPreference = 'high-performance'; + ONNX_ENV.webgpu.powerPreference = "high-performance"; } /** diff --git a/src/configs.js b/src/configs.js index 6329cc137..fcaa15e9c 100644 --- a/src/configs.js +++ b/src/configs.js @@ -1,10 +1,9 @@ - /** * @file Helper module for using model configs. For more information, see the corresponding * [Python documentation](https://huggingface.co/docs/transformers/main/en/model_doc/auto#transformers.AutoConfig). - * + * * **Example:** Load an `AutoConfig`. - * + * * ```javascript * import { AutoConfig } from '@huggingface/transformers'; * const config = await AutoConfig.from_pretrained('bert-base-uncased'); @@ -23,20 +22,17 @@ * // ... * // } * ``` - * + * * @module configs */ -import { pick } from './utils/core.js'; -import { - getModelJSON, -} from './utils/hub.js'; +import { pick } from "./utils/core.js"; +import { getModelJSON } from "./utils/hub.js"; /** * @typedef {import('./utils/hub.js').PretrainedOptions} PretrainedOptions */ - /** * Loads a config from the specified path. * @param {string} pretrained_model_name_or_path The path to the config directory. @@ -44,12 +40,12 @@ import { * @returns {Promise} A promise that resolves with information about the loaded config. */ async function loadConfig(pretrained_model_name_or_path, options) { - return await getModelJSON(pretrained_model_name_or_path, 'config.json', true, options); + return await getModelJSON(pretrained_model_name_or_path, "config.json", true, options); } /** - * - * @param {PretrainedConfig} config + * + * @param {PretrainedConfig} config * @returns {Object} The normalized configuration. */ function getNormalizedConfig(config) { @@ -58,129 +54,129 @@ function getNormalizedConfig(config) { let init_normalized_config = {}; switch (config.model_type) { // Sub-configs - case 'llava': - case 'paligemma': - case 'florence2': + case "llava": + case "paligemma": + case "florence2": init_normalized_config = getNormalizedConfig(config.text_config); break; - case 'moondream1': + case "moondream1": init_normalized_config = getNormalizedConfig(config.phi_config); break; - case 'musicgen': + case "musicgen": init_normalized_config = getNormalizedConfig(config.decoder); break; // Decoder-only models - case 'gpt2': - case 'gptj': - case 'jais': - case 'codegen': - case 'gpt_bigcode': - mapping['num_heads'] = 'n_head'; - mapping['num_layers'] = 'n_layer'; - mapping['hidden_size'] = 'n_embd'; + case "gpt2": + case "gptj": + case "jais": + case "codegen": + case "gpt_bigcode": + mapping["num_heads"] = "n_head"; + mapping["num_layers"] = "n_layer"; + mapping["hidden_size"] = "n_embd"; break; - case 'gpt_neox': - case 'stablelm': - case 'opt': - case 'phi': - case 'phi3': - case 'falcon': - mapping['num_heads'] = 'num_attention_heads'; - mapping['num_layers'] = 'num_hidden_layers'; - mapping['hidden_size'] = 'hidden_size'; + case "gpt_neox": + case "stablelm": + case "opt": + case "phi": + case "phi3": + case "falcon": + mapping["num_heads"] = "num_attention_heads"; + mapping["num_layers"] = "num_hidden_layers"; + mapping["hidden_size"] = "hidden_size"; break; - case 'llama': - case 'olmo': - case 'mobilellm': - case 'granite': - case 'cohere': - case 'mistral': - case 'starcoder2': - case 'qwen2': - mapping['num_heads'] = 'num_key_value_heads'; - mapping['num_layers'] = 'num_hidden_layers'; - mapping['hidden_size'] = 'hidden_size'; - mapping['num_attention_heads'] = 'num_attention_heads'; + case "llama": + case "olmo": + case "mobilellm": + case "granite": + case "cohere": + case "mistral": + case "starcoder2": + case "qwen2": + mapping["num_heads"] = "num_key_value_heads"; + mapping["num_layers"] = "num_hidden_layers"; + mapping["hidden_size"] = "hidden_size"; + mapping["num_attention_heads"] = "num_attention_heads"; break; - case 'gemma': - case 'gemma2': - mapping['num_heads'] = 'num_key_value_heads'; - mapping['num_layers'] = 'num_hidden_layers'; - mapping['dim_kv'] = 'head_dim'; + case "gemma": + case "gemma2": + mapping["num_heads"] = "num_key_value_heads"; + mapping["num_layers"] = "num_hidden_layers"; + mapping["dim_kv"] = "head_dim"; break; - case 'openelm': - mapping['num_heads'] = 'num_kv_heads'; - mapping['num_layers'] = 'num_transformer_layers'; - mapping['dim_kv'] = 'head_dim'; + case "openelm": + mapping["num_heads"] = "num_kv_heads"; + mapping["num_layers"] = "num_transformer_layers"; + mapping["dim_kv"] = "head_dim"; break; - case 'gpt_neo': - case 'donut-swin': - mapping['num_heads'] = 'num_heads'; - mapping['num_layers'] = 'num_layers'; - mapping['hidden_size'] = 'hidden_size'; + case "gpt_neo": + case "donut-swin": + mapping["num_heads"] = "num_heads"; + mapping["num_layers"] = "num_layers"; + mapping["hidden_size"] = "hidden_size"; break; - case 'bloom': - mapping['num_heads'] = 'n_head'; - mapping['num_layers'] = 'n_layer'; - mapping['hidden_size'] = 'hidden_size'; + case "bloom": + mapping["num_heads"] = "n_head"; + mapping["num_layers"] = "n_layer"; + mapping["hidden_size"] = "hidden_size"; break; - case 'mpt': - mapping['num_heads'] = 'n_heads'; - mapping['num_layers'] = 'n_layers'; - mapping['hidden_size'] = 'd_model'; + case "mpt": + mapping["num_heads"] = "n_heads"; + mapping["num_layers"] = "n_layers"; + mapping["hidden_size"] = "d_model"; break; // Encoder-decoder models - case 't5': - case 'mt5': - case 'longt5': - mapping['num_decoder_layers'] = 'num_decoder_layers'; - mapping['num_decoder_heads'] = 'num_heads'; - mapping['decoder_dim_kv'] = 'd_kv'; - mapping['num_encoder_layers'] = 'num_layers'; - mapping['num_encoder_heads'] = 'num_heads'; - mapping['encoder_dim_kv'] = 'd_kv'; + case "t5": + case "mt5": + case "longt5": + mapping["num_decoder_layers"] = "num_decoder_layers"; + mapping["num_decoder_heads"] = "num_heads"; + mapping["decoder_dim_kv"] = "d_kv"; + mapping["num_encoder_layers"] = "num_layers"; + mapping["num_encoder_heads"] = "num_heads"; + mapping["encoder_dim_kv"] = "d_kv"; break; - case 'bart': - case 'mbart': - case 'marian': - case 'whisper': - case 'm2m_100': - case 'blenderbot': - case 'blenderbot-small': - case 'florence2_language': - mapping['num_decoder_layers'] = 'decoder_layers'; - mapping['num_decoder_heads'] = 'decoder_attention_heads'; - mapping['decoder_hidden_size'] = 'd_model'; - mapping['num_encoder_layers'] = 'encoder_layers'; - mapping['num_encoder_heads'] = 'encoder_attention_heads'; - mapping['encoder_hidden_size'] = 'd_model'; + case "bart": + case "mbart": + case "marian": + case "whisper": + case "m2m_100": + case "blenderbot": + case "blenderbot-small": + case "florence2_language": + mapping["num_decoder_layers"] = "decoder_layers"; + mapping["num_decoder_heads"] = "decoder_attention_heads"; + mapping["decoder_hidden_size"] = "d_model"; + mapping["num_encoder_layers"] = "encoder_layers"; + mapping["num_encoder_heads"] = "encoder_attention_heads"; + mapping["encoder_hidden_size"] = "d_model"; break; - case 'speecht5': - mapping['num_decoder_layers'] = 'decoder_layers'; - mapping['num_decoder_heads'] = 'decoder_attention_heads'; - mapping['decoder_hidden_size'] = 'hidden_size'; - mapping['num_encoder_layers'] = 'encoder_layers'; - mapping['num_encoder_heads'] = 'encoder_attention_heads'; - mapping['encoder_hidden_size'] = 'hidden_size'; + case "speecht5": + mapping["num_decoder_layers"] = "decoder_layers"; + mapping["num_decoder_heads"] = "decoder_attention_heads"; + mapping["decoder_hidden_size"] = "hidden_size"; + mapping["num_encoder_layers"] = "encoder_layers"; + mapping["num_encoder_heads"] = "encoder_attention_heads"; + mapping["encoder_hidden_size"] = "hidden_size"; break; - case 'trocr': - mapping['num_encoder_layers'] = mapping['num_decoder_layers'] = 'decoder_layers'; - mapping['num_encoder_heads'] = mapping['num_decoder_heads'] = 'decoder_attention_heads'; - mapping['encoder_hidden_size'] = mapping['decoder_hidden_size'] = 'd_model'; + case "trocr": + mapping["num_encoder_layers"] = mapping["num_decoder_layers"] = "decoder_layers"; + mapping["num_encoder_heads"] = mapping["num_decoder_heads"] = "decoder_attention_heads"; + mapping["encoder_hidden_size"] = mapping["decoder_hidden_size"] = "d_model"; break; - case 'musicgen_decoder': - mapping['num_encoder_layers'] = mapping['num_decoder_layers'] = 'num_hidden_layers'; - mapping['num_encoder_heads'] = mapping['num_decoder_heads'] = 'num_attention_heads'; - mapping['encoder_hidden_size'] = mapping['decoder_hidden_size'] = 'hidden_size'; + case "musicgen_decoder": + mapping["num_encoder_layers"] = mapping["num_decoder_layers"] = "num_hidden_layers"; + mapping["num_encoder_heads"] = mapping["num_decoder_heads"] = "num_attention_heads"; + mapping["encoder_hidden_size"] = mapping["decoder_hidden_size"] = "hidden_size"; break; - case 'vision-encoder-decoder': + case "vision-encoder-decoder": const decoderConfig = getNormalizedConfig(config.decoder); - const add_encoder_pkv = 'num_decoder_layers' in decoderConfig; - const result = pick(config, ['model_type', 'is_encoder_decoder']); + const add_encoder_pkv = "num_decoder_layers" in decoderConfig; + const result = pick(config, ["model_type", "is_encoder_decoder"]); if (add_encoder_pkv) { // Decoder is part of an encoder-decoder model result.num_decoder_layers = decoderConfig.num_decoder_layers; @@ -197,13 +193,12 @@ function getNormalizedConfig(config) { result.hidden_size = decoderConfig.hidden_size; } return result; - } // NOTE: If `num_attention_heads` is not set, it is assumed to be equal to `num_heads` const normalized_config = { ...init_normalized_config, - ...pick(config, ['model_type', 'multi_query', 'is_encoder_decoder']), + ...pick(config, ["model_type", "multi_query", "is_encoder_decoder"]), }; for (const key in mapping) { normalized_config[key] = config[mapping[key]]; @@ -212,13 +207,11 @@ function getNormalizedConfig(config) { } /** - * - * @param {PretrainedConfig} config + * + * @param {PretrainedConfig} config * @returns {Record} */ -export function getKeyValueShapes(config, { - prefix = 'past_key_values', -} = {}) { +export function getKeyValueShapes(config, { prefix = "past_key_values" } = {}) { /** @type {Record} */ const decoderFeeds = {}; const normalized_config = config.normalized_config; @@ -226,15 +219,17 @@ export function getKeyValueShapes(config, { // TODO support batches (i.e., batch_size > 1) const batch_size = 1; - if (normalized_config.is_encoder_decoder && ( - 'num_encoder_heads' in normalized_config && 'num_decoder_heads' in normalized_config - )) { - const encoder_dim_kv = normalized_config.encoder_dim_kv ?? ( - normalized_config.encoder_hidden_size / normalized_config.num_encoder_heads - ); - const decoder_dim_kv = normalized_config.decoder_dim_kv ?? ( - normalized_config.decoder_hidden_size / normalized_config.num_decoder_heads - ); + if ( + normalized_config.is_encoder_decoder && + "num_encoder_heads" in normalized_config && + "num_decoder_heads" in normalized_config + ) { + const encoder_dim_kv = + normalized_config.encoder_dim_kv ?? + normalized_config.encoder_hidden_size / normalized_config.num_encoder_heads; + const decoder_dim_kv = + normalized_config.decoder_dim_kv ?? + normalized_config.decoder_hidden_size / normalized_config.num_decoder_heads; const encoder_dims = [batch_size, normalized_config.num_encoder_heads, 0, encoder_dim_kv]; const decoder_dims = [batch_size, normalized_config.num_decoder_heads, 0, decoder_dim_kv]; @@ -244,45 +239,47 @@ export function getKeyValueShapes(config, { decoderFeeds[`${prefix}.${i}.decoder.key`] = decoder_dims; decoderFeeds[`${prefix}.${i}.decoder.value`] = decoder_dims; } - } else { // Decoders + } else { + // Decoders const num_heads = normalized_config.num_heads; const num_layers = normalized_config.num_layers; - const dim_kv = normalized_config.dim_kv ?? ( - normalized_config.hidden_size / - (normalized_config.num_attention_heads ?? num_heads) - ); + const dim_kv = + normalized_config.dim_kv ?? + normalized_config.hidden_size / (normalized_config.num_attention_heads ?? num_heads); - if (normalized_config.model_type === 'falcon') { + if (normalized_config.model_type === "falcon") { // NOTE: Custom implementation for Falcon - const dims = [batch_size * num_heads, 0, dim_kv] + const dims = [batch_size * num_heads, 0, dim_kv]; for (let i = 0; i < num_layers; ++i) { decoderFeeds[`${prefix}.${i}.key`] = dims; decoderFeeds[`${prefix}.${i}.value`] = dims; } - } else if (normalized_config.multi_query) { // e.g., for `gpt_bigcode` - const dims = [batch_size * num_heads, 0, 2 * dim_kv] + } else if (normalized_config.multi_query) { + // e.g., for `gpt_bigcode` + const dims = [batch_size * num_heads, 0, 2 * dim_kv]; for (let i = 0; i < num_layers; ++i) { decoderFeeds[`${prefix}.${i}.key_value`] = dims; } - } else if (normalized_config.model_type === 'bloom') { + } else if (normalized_config.model_type === "bloom") { // NOTE: Custom implementation for Bloom - const keyDims = [batch_size * num_heads, dim_kv, 0] // [batch_size x num_heads,64,past_sequence_length] - const valueDims = [batch_size * num_heads, 0, dim_kv] // [batch_size x num_heads,past_sequence_length,64] + const keyDims = [batch_size * num_heads, dim_kv, 0]; // [batch_size x num_heads,64,past_sequence_length] + const valueDims = [batch_size * num_heads, 0, dim_kv]; // [batch_size x num_heads,past_sequence_length,64] for (let i = 0; i < num_layers; ++i) { decoderFeeds[`${prefix}.${i}.key`] = keyDims; decoderFeeds[`${prefix}.${i}.value`] = valueDims; } - } else if (normalized_config.model_type === 'openelm') { + } else if (normalized_config.model_type === "openelm") { for (let i = 0; i < num_layers; ++i) { - const dims = [batch_size, num_heads[i], 0, dim_kv] + const dims = [batch_size, num_heads[i], 0, dim_kv]; decoderFeeds[`${prefix}.${i}.key`] = dims; decoderFeeds[`${prefix}.${i}.value`] = dims; } - } else { // Decoder-only - const dims = [batch_size, num_heads, 0, dim_kv] + } else { + // Decoder-only + const dims = [batch_size, num_heads, 0, dim_kv]; for (let i = 0; i < num_layers; ++i) { decoderFeeds[`${prefix}.${i}.key`] = dims; decoderFeeds[`${prefix}.${i}.value`] = dims; @@ -309,7 +306,7 @@ export class PretrainedConfig { max_position_embeddings; /** @type {TransformersJSConfig} */ - 'transformers.js_config'; + "transformers.js_config"; /** * Create a new PreTrainedTokenizer instance. @@ -321,41 +318,40 @@ export class PretrainedConfig { } /** - * Loads a pre-trained config from the given `pretrained_model_name_or_path`. - * + * Loads a pre-trained config from the given `pretrained_model_name_or_path`. + * * @param {string} pretrained_model_name_or_path The path to the pre-trained config. * @param {PretrainedOptions} options Additional options for loading the config. * @throws {Error} Throws an error if the config.json is not found in the `pretrained_model_name_or_path`. - * + * * @returns {Promise} A new instance of the `PretrainedConfig` class. */ - static async from_pretrained(pretrained_model_name_or_path, { - progress_callback = null, - config = null, - cache_dir = null, - local_files_only = false, - revision = 'main', - } = {}) { + static async from_pretrained( + pretrained_model_name_or_path, + { progress_callback = null, config = null, cache_dir = null, local_files_only = false, revision = "main" } = {}, + ) { if (config && !(config instanceof PretrainedConfig)) { config = new PretrainedConfig(config); } - const data = config ?? await loadConfig(pretrained_model_name_or_path, { - progress_callback, - config, - cache_dir, - local_files_only, - revision, - }) + const data = + config ?? + (await loadConfig(pretrained_model_name_or_path, { + progress_callback, + config, + cache_dir, + local_files_only, + revision, + })); return new this(data); } } /** * Helper class which is used to instantiate pretrained configs with the `from_pretrained` function. - * + * * @example - * const config = await AutoConfig.from_pretrained('Xenova/bert-base-uncased'); + * const config = await AutoConfig.from_pretrained('Xenova/bert-base-uncased'); */ export class AutoConfig { /** @type {typeof PretrainedConfig.from_pretrained} */ diff --git a/src/env.js b/src/env.js index b0cd97aef..b3c8fc258 100644 --- a/src/env.js +++ b/src/env.js @@ -1,42 +1,42 @@ /** * @file Module used to configure Transformers.js. - * + * * **Example:** Disable remote models. * ```javascript * import { env } from '@huggingface/transformers'; * env.allowRemoteModels = false; * ``` - * + * * **Example:** Set local model path. * ```javascript * import { env } from '@huggingface/transformers'; * env.localModelPath = '/path/to/local/models/'; * ``` - * + * * **Example:** Set cache directory. * ```javascript * import { env } from '@huggingface/transformers'; * env.cacheDir = '/path/to/cache/directory/'; * ``` - * + * * @module env */ -import fs from 'fs'; -import path from 'path'; -import url from 'url'; +import fs from "fs"; +import path from "path"; +import url from "url"; -const VERSION = '3.0.2'; +const VERSION = "3.0.2"; // Check if various APIs are available (depends on environment) -const IS_BROWSER_ENV = typeof self !== 'undefined'; -const IS_WEBWORKER_ENV = IS_BROWSER_ENV && self.constructor.name === 'DedicatedWorkerGlobalScope'; -const IS_WEB_CACHE_AVAILABLE = IS_BROWSER_ENV && 'caches' in self; -const IS_WEBGPU_AVAILABLE = typeof navigator !== 'undefined' && 'gpu' in navigator; -const IS_WEBNN_AVAILABLE = typeof navigator !== 'undefined' && 'ml' in navigator; - -const IS_PROCESS_AVAILABLE = typeof process !== 'undefined'; -const IS_NODE_ENV = IS_PROCESS_AVAILABLE && process?.release?.name === 'node'; +const IS_BROWSER_ENV = typeof self !== "undefined"; +const IS_WEBWORKER_ENV = IS_BROWSER_ENV && self.constructor.name === "DedicatedWorkerGlobalScope"; +const IS_WEB_CACHE_AVAILABLE = IS_BROWSER_ENV && "caches" in self; +const IS_WEBGPU_AVAILABLE = typeof navigator !== "undefined" && "gpu" in navigator; +const IS_WEBNN_AVAILABLE = typeof navigator !== "undefined" && "ml" in navigator; + +const IS_PROCESS_AVAILABLE = typeof process !== "undefined"; +const IS_NODE_ENV = IS_PROCESS_AVAILABLE && process?.release?.name === "node"; const IS_FS_AVAILABLE = !isEmpty(fs); const IS_PATH_AVAILABLE = !isEmpty(path); @@ -74,7 +74,7 @@ export const apis = Object.freeze({ const RUNNING_LOCALLY = IS_FS_AVAILABLE && IS_PATH_AVAILABLE; -let dirname__ = './'; +let dirname__ = "./"; if (RUNNING_LOCALLY) { // NOTE: We wrap `import.meta` in a call to `Object` to prevent Webpack from trying to bundle it in CommonJS. // Although we get the warning: "Accessing import.meta directly is unsupported (only property access or destructuring is supported)", @@ -82,22 +82,18 @@ if (RUNNING_LOCALLY) { const _import_meta_url = Object(import.meta).url; if (_import_meta_url) { - dirname__ = path.dirname(path.dirname(url.fileURLToPath(_import_meta_url))) // ESM - } else if (typeof __dirname !== 'undefined') { - dirname__ = path.dirname(__dirname) // CommonJS + dirname__ = path.dirname(path.dirname(url.fileURLToPath(_import_meta_url))); // ESM + } else if (typeof __dirname !== "undefined") { + dirname__ = path.dirname(__dirname); // CommonJS } } // Only used for environments with access to file system -const DEFAULT_CACHE_DIR = RUNNING_LOCALLY - ? path.join(dirname__, '/.cache/') - : null; +const DEFAULT_CACHE_DIR = RUNNING_LOCALLY ? path.join(dirname__, "/.cache/") : null; // Set local model path, based on available APIs -const DEFAULT_LOCAL_MODEL_PATH = '/models/'; -const localModelPath = RUNNING_LOCALLY - ? path.join(dirname__, DEFAULT_LOCAL_MODEL_PATH) - : DEFAULT_LOCAL_MODEL_PATH; +const DEFAULT_LOCAL_MODEL_PATH = "/models/"; +const localModelPath = RUNNING_LOCALLY ? path.join(dirname__, DEFAULT_LOCAL_MODEL_PATH) : DEFAULT_LOCAL_MODEL_PATH; /** * Global variable given visible to users to control execution. This provides users a simple way to configure Transformers.js. @@ -134,8 +130,8 @@ export const env = { /////////////////// Model settings /////////////////// allowRemoteModels: true, - remoteHost: 'https://huggingface.co/', - remotePathTemplate: '{model}/resolve/{revision}/', + remoteHost: "https://huggingface.co/", + remotePathTemplate: "{model}/resolve/{revision}/", allowLocalModels: !IS_BROWSER_ENV, localModelPath: localModelPath, @@ -150,8 +146,7 @@ export const env = { useCustomCache: false, customCache: null, ////////////////////////////////////////////////////// -} - +}; /** * @param {Object} obj @@ -160,4 +155,3 @@ export const env = { function isEmpty(obj) { return Object.keys(obj).length === 0; } - diff --git a/src/generation/configuration_utils.js b/src/generation/configuration_utils.js index 33a6fbe81..94a0e822f 100644 --- a/src/generation/configuration_utils.js +++ b/src/generation/configuration_utils.js @@ -1,4 +1,3 @@ - /** * @module generation/configuration_utils */ @@ -371,11 +370,10 @@ export class GenerationConfig { generation_kwargs = {}; /** - * - * @param {GenerationConfig|import('../configs.js').PretrainedConfig} config + * + * @param {GenerationConfig|import('../configs.js').PretrainedConfig} config */ constructor(config) { Object.assign(this, pick(config, Object.getOwnPropertyNames(this))); } } - diff --git a/src/generation/logits_process.js b/src/generation/logits_process.js index 732af4f3f..b647a1077 100644 --- a/src/generation/logits_process.js +++ b/src/generation/logits_process.js @@ -1,4 +1,3 @@ - /** * @module generation/logits_process */ @@ -21,11 +20,10 @@ export class LogitsProcessor extends Callable { * @throws {Error} Throws an error if `_call` is not implemented in the subclass. */ _call(input_ids, logits) { - throw Error("`_call` should be implemented in a subclass") + throw Error("`_call` should be implemented in a subclass"); } } - /** * Abstract base class for all logit warpers that can be applied during generation with multinomial sampling. */ @@ -39,11 +37,10 @@ export class LogitsWarper extends Callable { * @throws {Error} Throws an error if `_call` is not implemented in the subclass. */ _call(input_ids, logits) { - throw Error("`_call` should be implemented in a subclass") + throw Error("`_call` should be implemented in a subclass"); } } - /** * A class representing a list of logits processors. A logits processor is a function that modifies the logits * output of a language model. This class provides methods for adding new processors and applying all processors to a @@ -103,7 +100,7 @@ export class LogitsProcessorList extends Callable { // export class ForceTokensLogitsProcessor extends LogitsProcessor { // /** // * Constructs a new instance of `ForceTokensLogitsProcessor`. -// * +// * // * @param {[number, number][]} forced_decoder_ids The ids of tokens that should be forced. // */ // constructor(forced_decoder_ids) { @@ -156,7 +153,7 @@ export class ForcedBOSTokenLogitsProcessor extends LogitsProcessor { _call(input_ids, logits) { for (let i = 0; i < input_ids.length; ++i) { if (input_ids[i].length === 1) { - const batch_logits_data = /** @type {Float32Array} */(logits[i].data); + const batch_logits_data = /** @type {Float32Array} */ (logits[i].data); batch_logits_data.fill(-Infinity); batch_logits_data[this.bos_token_id] = 0; } @@ -182,14 +179,14 @@ export class ForcedEOSTokenLogitsProcessor extends LogitsProcessor { /** * Apply the processor to input_ids and logits. - * + * * @param {bigint[][]} input_ids The input ids. * @param {Tensor} logits The logits tensor. */ _call(input_ids, logits) { for (let i = 0; i < input_ids.length; ++i) { if (input_ids[i].length === this.max_length - 1) { - const batch_logits_data = /** @type {Float32Array} */(logits[i].data); + const batch_logits_data = /** @type {Float32Array} */ (logits[i].data); batch_logits_data.fill(-Infinity); for (const eos_token of this.eos_token_id) { batch_logits_data[eos_token] = 0; @@ -226,7 +223,7 @@ export class SuppressTokensAtBeginLogitsProcessor extends LogitsProcessor { _call(input_ids, logits) { for (let i = 0; i < input_ids.length; ++i) { if (input_ids[i].length === this.begin_index) { - const batch_logits_data = /** @type {Float32Array} */(logits[i].data); + const batch_logits_data = /** @type {Float32Array} */ (logits[i].data); for (const token_id of this.begin_suppress_tokens) { batch_logits_data[token_id] = -Infinity; } @@ -247,10 +244,9 @@ export class WhisperTimeStampLogitsProcessor extends LogitsProcessor { */ constructor(generate_config, init_tokens) { super(); - this.eos_token_id = - Array.isArray(generate_config.eos_token_id) - ? generate_config.eos_token_id[0] - : generate_config.eos_token_id; + this.eos_token_id = Array.isArray(generate_config.eos_token_id) + ? generate_config.eos_token_id[0] + : generate_config.eos_token_id; this.no_timestamps_token_id = generate_config.no_timestamps_token_id; this.timestamp_begin = this.no_timestamps_token_id + 1; @@ -270,7 +266,7 @@ export class WhisperTimeStampLogitsProcessor extends LogitsProcessor { */ _call(input_ids, logits) { for (let i = 0; i < input_ids.length; ++i) { - const batch_logits_data = /** @type {Float32Array} */(logits[i].data); + const batch_logits_data = /** @type {Float32Array} */ (logits[i].data); // suppress <|notimestamps|> which is handled by without_timestamps batch_logits_data[this.no_timestamps_token_id] = -Infinity; @@ -287,9 +283,11 @@ export class WhisperTimeStampLogitsProcessor extends LogitsProcessor { const penultimate_was_timestamp = seq.length < 2 || seq[seq.length - 2] >= this.timestamp_begin; if (last_was_timestamp) { - if (penultimate_was_timestamp) { // has to be non-timestamp + if (penultimate_was_timestamp) { + // has to be non-timestamp batch_logits_data.subarray(this.timestamp_begin).fill(-Infinity); - } else { // cannot be normal text tokens + } else { + // cannot be normal text tokens batch_logits_data.subarray(0, this.eos_token_id).fill(-Infinity); } } @@ -302,7 +300,12 @@ export class WhisperTimeStampLogitsProcessor extends LogitsProcessor { // if sum of probability over timestamps is above any other token, sample timestamp const logprobs = log_softmax(batch_logits_data); - const timestamp_logprob = Math.log(logprobs.subarray(this.timestamp_begin).map(Math.exp).reduce((a, b) => a + b)); + const timestamp_logprob = Math.log( + logprobs + .subarray(this.timestamp_begin) + .map(Math.exp) + .reduce((a, b) => a + b), + ); const max_text_token_logprob = max(logprobs.subarray(0, this.timestamp_begin))[0]; if (timestamp_logprob > max_text_token_logprob) { @@ -379,7 +382,6 @@ export class NoRepeatNGramLogitsProcessor extends LogitsProcessor { if (prevInputIds.length + 1 < this.no_repeat_ngram_size) { // return no banned tokens if we haven't generated no_repeat_ngram_size tokens yet return bannedTokens; - } else { const generatedNgrams = this.getNgrams(prevInputIds); const bannedTokens = this.getGeneratedNgrams(generatedNgrams, prevInputIds); @@ -395,7 +397,7 @@ export class NoRepeatNGramLogitsProcessor extends LogitsProcessor { */ _call(input_ids, logits) { for (let i = 0; i < input_ids.length; ++i) { - const batch_logits_data = /** @type {Float32Array} */(logits[i].data); + const batch_logits_data = /** @type {Float32Array} */ (logits[i].data); const bannedTokens = this.calcBannedNgramTokens(input_ids[i]); for (const token of bannedTokens) { batch_logits_data[token] = -Infinity; @@ -430,7 +432,7 @@ export class RepetitionPenaltyLogitsProcessor extends LogitsProcessor { // many times in the output will be penalised more. for (let i = 0; i < input_ids.length; ++i) { - const batch_logits_data = /** @type {Float32Array} */(logits[i].data); + const batch_logits_data = /** @type {Float32Array} */ (logits[i].data); for (const input_id of input_ids[i]) { const token = Number(input_id); if (batch_logits_data[token] < 0) { @@ -441,7 +443,7 @@ export class RepetitionPenaltyLogitsProcessor extends LogitsProcessor { } } - return logits + return logits; } } @@ -469,7 +471,7 @@ export class MinLengthLogitsProcessor extends LogitsProcessor { _call(input_ids, logits) { for (let i = 0; i < input_ids.length; ++i) { if (input_ids[i].length < this.min_length) { - const batch_logits_data = /** @type {Float32Array} */(logits[i].data); + const batch_logits_data = /** @type {Float32Array} */ (logits[i].data); for (const eos_token of this.eos_token_id) { batch_logits_data[eos_token] = -Infinity; @@ -477,7 +479,7 @@ export class MinLengthLogitsProcessor extends LogitsProcessor { } } - return logits + return logits; } } @@ -508,14 +510,14 @@ export class MinNewTokensLengthLogitsProcessor extends LogitsProcessor { for (let i = 0; i < input_ids.length; ++i) { const new_tokens_length = input_ids[i].length - this.prompt_length_to_skip; if (new_tokens_length < this.min_new_tokens) { - const batch_logits_data = /** @type {Float32Array} */(logits[i].data); + const batch_logits_data = /** @type {Float32Array} */ (logits[i].data); for (const eos_token of this.eos_token_id) { batch_logits_data[eos_token] = -Infinity; } } } - return logits + return logits; } } @@ -539,7 +541,7 @@ export class NoBadWordsLogitsProcessor extends LogitsProcessor { */ _call(input_ids, logits) { for (let i = 0; i < input_ids.length; ++i) { - const batch_logits_data = /** @type {Float32Array} */(logits[i].data); + const batch_logits_data = /** @type {Float32Array} */ (logits[i].data); const ids = input_ids[i]; for (const bad_word_ids of this.bad_words_ids) { // Whether to modify the logits of the last token in the bad word id sequence @@ -548,7 +550,6 @@ export class NoBadWordsLogitsProcessor extends LogitsProcessor { // For each bad word in the list, if the current sequence of input ids ends with this sequence (excluding the last), // then we set the logits of the last bad word id to -Infinity. for (let j = 1; j <= bad_word_ids.length - 1 && bad_word_ids.length < ids.length; ++j) { - // NOTE: We use != instead of !== to compare bigint and number // @ts-ignore if (bad_word_ids.at(-j - 1) != ids.at(-j)) { @@ -562,7 +563,7 @@ export class NoBadWordsLogitsProcessor extends LogitsProcessor { } } } - return logits + return logits; } } @@ -571,11 +572,10 @@ export class NoBadWordsLogitsProcessor extends LogitsProcessor { * where the first half correspond to the conditional logits (predicted from the input prompt) and the second half * correspond to the unconditional logits (predicted from an empty or 'null' prompt). The processor computes a * weighted average across the conditional and unconditional logits, parameterised by the `guidance_scale`. - * + * * See [the paper](https://arxiv.org/abs/2306.05284) for more information. */ export class ClassifierFreeGuidanceLogitsProcessor extends LogitsProcessor { - /** * Create a `ClassifierFreeGuidanceLogitsProcessor`. * @param {number} guidance_scale The guidance scale for classifier free guidance (CFG). CFG is enabled by setting `guidance_scale > 1`. @@ -586,8 +586,8 @@ export class ClassifierFreeGuidanceLogitsProcessor extends LogitsProcessor { super(); if (guidance_scale <= 1) { throw new Error( - `Require guidance scale >1 to use the classifier free guidance processor, got guidance scale ${guidance_scale}.` - ) + `Require guidance scale >1 to use the classifier free guidance processor, got guidance scale ${guidance_scale}.`, + ); } this.guidance_scale = guidance_scale; } @@ -602,9 +602,9 @@ export class ClassifierFreeGuidanceLogitsProcessor extends LogitsProcessor { if (logits.dims[0] !== 2 * input_ids.length) { throw new Error( `Logits should have twice the batch size of the input ids, the first half of batches corresponding to ` + - `the conditional inputs, and the second half of batches corresponding to the unconditional inputs. Got ` + - `batch size ${logits.dims[0]} for the logits and ${input_ids.length} for the input ids.` - ) + `the conditional inputs, and the second half of batches corresponding to the unconditional inputs. Got ` + + `batch size ${logits.dims[0]} for the logits and ${input_ids.length} for the input ids.`, + ); } const unguided_bsz = input_ids.length; @@ -635,12 +635,11 @@ export class TemperatureLogitsWarper extends LogitsWarper { constructor(temperature) { super(); - if (typeof temperature !== 'number' || temperature <= 0) { - let errorMessage = - `\`temperature\` (=${temperature}) must be a strictly positive float, otherwise your next token scores will be invalid.`; + if (typeof temperature !== "number" || temperature <= 0) { + let errorMessage = `\`temperature\` (=${temperature}) must be a strictly positive float, otherwise your next token scores will be invalid.`; if (temperature === 0) { - errorMessage += " If you're looking for greedy decoding strategies, set `do_sample=false`." + errorMessage += " If you're looking for greedy decoding strategies, set `do_sample=false`."; } } this.temperature = temperature; @@ -653,7 +652,7 @@ export class TemperatureLogitsWarper extends LogitsWarper { * @returns {Object} The processed logits. */ _call(input_ids, logits) { - const batch_logits_data = /** @type {Float32Array} */(logits.data); + const batch_logits_data = /** @type {Float32Array} */ (logits.data); for (let i = 0; i < batch_logits_data.length; ++i) { batch_logits_data[i] /= this.temperature; } @@ -674,21 +673,18 @@ export class TopPLogitsWarper extends LogitsWarper { * @param {number} [options.filter_value=-Infinity] All filtered values will be set to this float value. * @param {number} [options.min_tokens_to_keep=1] Minimum number of tokens that cannot be filtered. */ - constructor(top_p, { - filter_value = -Infinity, - min_tokens_to_keep = 1, - } = {}) { + constructor(top_p, { filter_value = -Infinity, min_tokens_to_keep = 1 } = {}) { super(); if (top_p < 0 || top_p > 1.0) { - throw new Error(`\`top_p\` must be a float > 0 and < 1, but is ${top_p}`) + throw new Error(`\`top_p\` must be a float > 0 and < 1, but is ${top_p}`); } if (!Number.isInteger(min_tokens_to_keep) || min_tokens_to_keep < 1) { - throw new Error(`\`min_tokens_to_keep\` must be a positive integer, but is ${min_tokens_to_keep}`) + throw new Error(`\`min_tokens_to_keep\` must be a positive integer, but is ${min_tokens_to_keep}`); } - this.top_p = top_p - this.filter_value = filter_value - this.min_tokens_to_keep = min_tokens_to_keep + this.top_p = top_p; + this.filter_value = filter_value; + this.min_tokens_to_keep = min_tokens_to_keep; } } @@ -704,16 +700,13 @@ export class TopKLogitsWarper extends LogitsWarper { * @param {number} [options.filter_value=-Infinity] All filtered values will be set to this float value. * @param {number} [options.min_tokens_to_keep=1] Minimum number of tokens that cannot be filtered. */ - constructor(top_k, { - filter_value = -Infinity, - min_tokens_to_keep = 1, - } = {}) { + constructor(top_k, { filter_value = -Infinity, min_tokens_to_keep = 1 } = {}) { super(); if (!Number.isInteger(top_k) || top_k < 0) { - throw new Error(`\`top_k\` must be a positive integer, but is ${top_k}`) + throw new Error(`\`top_k\` must be a positive integer, but is ${top_k}`); } - this.top_k = Math.max(top_k, min_tokens_to_keep) - this.filter_value = filter_value + this.top_k = Math.max(top_k, min_tokens_to_keep); + this.filter_value = filter_value; } -} \ No newline at end of file +} diff --git a/src/generation/logits_sampler.js b/src/generation/logits_sampler.js index 46b74e081..94e2cf0f6 100644 --- a/src/generation/logits_sampler.js +++ b/src/generation/logits_sampler.js @@ -1,4 +1,3 @@ - /** * @module generation/logits_sampler */ @@ -6,11 +5,8 @@ import { Callable } from "../utils/generic.js"; import { Tensor, topk } from "../utils/tensor.js"; -import { - max, - softmax, -} from '../utils/maths.js'; -import { GenerationConfig } from '../generation/configuration_utils.js'; +import { max, softmax } from "../utils/maths.js"; +import { GenerationConfig } from "../generation/configuration_utils.js"; /** * Sampler is a base class for all sampling methods used for text generation. @@ -43,7 +39,7 @@ export class LogitsSampler extends Callable { * @returns {Promise<[bigint, number][]>} */ async sample(logits) { - throw Error("sample should be implemented in subclasses.") + throw Error("sample should be implemented in subclasses."); } /** @@ -55,7 +51,7 @@ export class LogitsSampler extends Callable { getLogits(logits, index) { let vocabSize = logits.dims.at(-1); - let logs = /** @type {Float32Array} */(logits.data); + let logs = /** @type {Float32Array} */ (logits.data); if (index === -1) { logs = logs.slice(-vocabSize); @@ -105,13 +101,13 @@ export class LogitsSampler extends Callable { // NOTE: beam search is implemented directly into the generation function if (generation_config.do_sample) { return new MultinomialSampler(generation_config); - } else if (generation_config.num_beams > 1) { return new BeamSearchSampler(generation_config); - } else { if (generation_config.num_return_sequences > 1) { - throw Error(`num_return_sequences has to be 1 when doing greedy search, but is ${generation_config.num_return_sequences}.`) + throw Error( + `num_return_sequences has to be 1 when doing greedy search, but is ${generation_config.num_return_sequences}.`, + ); } return new GreedySampler(generation_config); } @@ -133,9 +129,7 @@ class GreedySampler extends LogitsSampler { // Note: score is meaningless in this context, since we are performing // greedy search (p = 1 => log(p) = 0) - return [ - [BigInt(argmax), 0] - ]; + return [[BigInt(argmax), 0]]; } } @@ -143,7 +137,6 @@ class GreedySampler extends LogitsSampler { * Class representing a MultinomialSampler. */ class MultinomialSampler extends LogitsSampler { - /** * Sample from the logits. * @param {Tensor} logits @@ -159,7 +152,7 @@ class MultinomialSampler extends LogitsSampler { const [v, i] = await topk(logits, k); // Compute softmax over logits - const probabilities = softmax(/** @type {Float32Array} */(v.data)); + const probabilities = softmax(/** @type {Float32Array} */ (v.data)); return Array.from({ length: this.generation_config.num_beams }, () => { const sampledIndex = this.randomSelect(probabilities); @@ -171,12 +164,10 @@ class MultinomialSampler extends LogitsSampler { } } - /** * Class representing a BeamSearchSampler. */ class BeamSearchSampler extends LogitsSampler { - /** * Sample from the logits. * @param {Tensor} logits @@ -192,7 +183,7 @@ class BeamSearchSampler extends LogitsSampler { const [v, i] = await topk(logits, k); // Compute softmax over logits - const probabilities = softmax(/** @type {Float32Array} */(v.data)); + const probabilities = softmax(/** @type {Float32Array} */ (v.data)); return Array.from({ length: this.generation_config.num_beams }, (_, x) => { return [ diff --git a/src/generation/parameters.js b/src/generation/parameters.js index 1e2f2def3..d73ea642d 100644 --- a/src/generation/parameters.js +++ b/src/generation/parameters.js @@ -1,4 +1,3 @@ - /** * @module generation/parameters */ diff --git a/src/generation/stopping_criteria.js b/src/generation/stopping_criteria.js index 08434f2b4..d328a6f35 100644 --- a/src/generation/stopping_criteria.js +++ b/src/generation/stopping_criteria.js @@ -1,4 +1,3 @@ - /** * @module generation/stopping_criteria */ @@ -13,7 +12,7 @@ import { Callable } from "../utils/generic.js"; */ export class StoppingCriteria extends Callable { /** - * + * * @param {number[][]} input_ids (`number[][]` of shape `(batch_size, sequence_length)`): * Indices of input sequence tokens in the vocabulary. * @param {number[][]} scores scores (`number[][]` of shape `(batch_size, config.vocab_size)`): @@ -80,9 +79,8 @@ export class StoppingCriteriaList extends Callable { * Keep in mind for decoder-only type of transformers, this will include the initial prompted tokens. */ export class MaxLengthCriteria extends StoppingCriteria { - /** - * + * * @param {number} max_length The maximum length that the output sequence can have in number of tokens. * @param {number} [max_position_embeddings=null] The maximum model length, as defined by the model's `config.max_position_embeddings` attribute. */ @@ -93,7 +91,7 @@ export class MaxLengthCriteria extends StoppingCriteria { } _call(input_ids) { - return input_ids.map(ids => ids.length >= this.max_length); + return input_ids.map((ids) => ids.length >= this.max_length); } } @@ -104,9 +102,8 @@ export class MaxLengthCriteria extends StoppingCriteria { * By default, it uses the `model.generation_config.eos_token_id`. */ export class EosTokenCriteria extends StoppingCriteria { - /** - * + * * @param {number|number[]} eos_token_id The id of the *end-of-sequence* token. * Optionally, use a list to set multiple *end-of-sequence* tokens. */ @@ -119,16 +116,16 @@ export class EosTokenCriteria extends StoppingCriteria { } /** - * - * @param {number[][]} input_ids - * @param {number[][]} scores + * + * @param {number[][]} input_ids + * @param {number[][]} scores * @returns {boolean[]} */ _call(input_ids, scores) { - return input_ids.map(ids => { + return input_ids.map((ids) => { const last = ids.at(-1); // NOTE: We use == instead of === to allow for number/bigint comparison - return this.eos_token_id.some(eos_id => last == eos_id); + return this.eos_token_id.some((eos_id) => last == eos_id); }); } } diff --git a/src/generation/streamers.js b/src/generation/streamers.js index 64afc71c7..d5046987e 100644 --- a/src/generation/streamers.js +++ b/src/generation/streamers.js @@ -1,48 +1,48 @@ - /** * @module generation/streamers */ -import { mergeArrays } from '../utils/core.js'; -import { is_chinese_char } from '../tokenizers.js'; -import { apis } from '../env.js'; +import { mergeArrays } from "../utils/core.js"; +import { is_chinese_char } from "../tokenizers.js"; +import { apis } from "../env.js"; export class BaseStreamer { /** * Function that is called by `.generate()` to push new tokens - * @param {bigint[][]} value + * @param {bigint[][]} value */ put(value) { - throw Error('Not implemented'); + throw Error("Not implemented"); } /** * Function that is called by `.generate()` to signal the end of generation */ end() { - throw Error('Not implemented'); + throw Error("Not implemented"); } } -const stdout_write = apis.IS_PROCESS_AVAILABLE - ? x => process.stdout.write(x) - : x => console.log(x); +const stdout_write = apis.IS_PROCESS_AVAILABLE ? (x) => process.stdout.write(x) : (x) => console.log(x); /** * Simple text streamer that prints the token(s) to stdout as soon as entire words are formed. */ export class TextStreamer extends BaseStreamer { /** - * - * @param {import('../tokenizers.js').PreTrainedTokenizer} tokenizer + * + * @param {import('../tokenizers.js').PreTrainedTokenizer} tokenizer */ - constructor(tokenizer, { - skip_prompt = false, - callback_function = null, - token_callback_function = null, - decode_kwargs = {}, - ...kwargs - } = {}) { + constructor( + tokenizer, + { + skip_prompt = false, + callback_function = null, + token_callback_function = null, + decode_kwargs = {}, + ...kwargs + } = {}, + ) { super(); this.tokenizer = tokenizer; this.skip_prompt = skip_prompt; @@ -58,11 +58,11 @@ export class TextStreamer extends BaseStreamer { /** * Receives tokens, decodes them, and prints them to stdout as soon as they form entire words. - * @param {bigint[][]} value + * @param {bigint[][]} value */ put(value) { if (value.length > 1) { - throw Error('TextStreamer only supports batch size of 1'); + throw Error("TextStreamer only supports batch size of 1"); } if (this.skip_prompt && this.next_tokens_are_prompt) { @@ -71,14 +71,14 @@ export class TextStreamer extends BaseStreamer { } const tokens = value[0]; - this.token_callback_function?.(tokens) + this.token_callback_function?.(tokens); // Add the new token to the cache and decodes the entire thing. this.token_cache = mergeArrays(this.token_cache, tokens); const text = this.tokenizer.decode(this.token_cache, this.decode_kwargs); let printable_text; - if (text.endsWith('\n')) { + if (text.endsWith("\n")) { // After the symbol for a new line, we flush the cache. printable_text = text.slice(this.print_len); this.token_cache = []; @@ -90,7 +90,7 @@ export class TextStreamer extends BaseStreamer { } else { // Otherwise, prints until the last space char (simple heuristic to avoid printing incomplete words, // which may change with the subsequent token -- there are probably smarter ways to do this!) - printable_text = text.slice(this.print_len, text.lastIndexOf(' ') + 1); + printable_text = text.slice(this.print_len, text.lastIndexOf(" ") + 1); this.print_len += printable_text.length; } @@ -108,7 +108,7 @@ export class TextStreamer extends BaseStreamer { this.token_cache = []; this.print_len = 0; } else { - printable_text = ''; + printable_text = ""; } this.next_tokens_are_prompt = true; this.on_finalized_text(printable_text, true); @@ -116,15 +116,15 @@ export class TextStreamer extends BaseStreamer { /** * Prints the new text to stdout. If the stream is ending, also prints a newline. - * @param {string} text - * @param {boolean} stream_end + * @param {string} text + * @param {boolean} stream_end */ on_finalized_text(text, stream_end) { if (text.length > 0) { this.callback_function?.(text); } if (stream_end && this.callback_function === stdout_write && apis.IS_PROCESS_AVAILABLE) { - this.callback_function?.('\n'); + this.callback_function?.("\n"); } } } @@ -151,17 +151,20 @@ export class WhisperTextStreamer extends TextStreamer { * @param {boolean} [options.skip_special_tokens=true] Whether to skip special tokens when decoding * @param {Object} [options.decode_kwargs={}] Additional keyword arguments to pass to the tokenizer's decode method */ - constructor(tokenizer, { - skip_prompt = false, - callback_function = null, - token_callback_function = null, - on_chunk_start = null, - on_chunk_end = null, - on_finalize = null, - time_precision = 0.02, - skip_special_tokens = true, - decode_kwargs = {}, - } = {}) { + constructor( + tokenizer, + { + skip_prompt = false, + callback_function = null, + token_callback_function = null, + on_chunk_start = null, + on_chunk_end = null, + on_finalize = null, + time_precision = 0.02, + skip_special_tokens = true, + decode_kwargs = {}, + } = {}, + ) { super(tokenizer, { skip_prompt, callback_function, @@ -180,11 +183,11 @@ export class WhisperTextStreamer extends TextStreamer { } /** - * @param {bigint[][]} value + * @param {bigint[][]} value */ put(value) { if (value.length > 1) { - throw Error('WhisperTextStreamer only supports batch size of 1'); + throw Error("WhisperTextStreamer only supports batch size of 1"); } const tokens = value[0]; diff --git a/src/models.js b/src/models.js index 031b85f01..f701cbc9f 100644 --- a/src/models.js +++ b/src/models.js @@ -1,15 +1,14 @@ - /** * @file Definitions of all models available in Transformers.js. - * + * * **Example:** Load and run an `AutoModel`. - * + * * ```javascript * import { AutoModel, AutoTokenizer } from '@huggingface/transformers'; - * + * * let tokenizer = await AutoTokenizer.from_pretrained('Xenova/bert-base-uncased'); * let model = await AutoModel.from_pretrained('Xenova/bert-base-uncased'); - * + * * let inputs = await tokenizer('I love transformers!'); * let { logits } = await model(inputs); * // Tensor { @@ -19,61 +18,42 @@ * // size: 183132, * // } * ``` - * + * * We also provide other `AutoModel`s (listed below), which you can use in the same way as the Python library. For example: - * + * * **Example:** Load and run an `AutoModelForSeq2SeqLM`. * ```javascript * import { AutoModelForSeq2SeqLM, AutoTokenizer } from '@huggingface/transformers'; - * + * * let tokenizer = await AutoTokenizer.from_pretrained('Xenova/t5-small'); * let model = await AutoModelForSeq2SeqLM.from_pretrained('Xenova/t5-small'); - * + * * let { input_ids } = await tokenizer('translate English to German: I love transformers!'); * let outputs = await model.generate(input_ids); * let decoded = tokenizer.decode(outputs[0], { skip_special_tokens: true }); * // 'Ich liebe Transformatoren!' * ``` - * + * * @module models */ -import { - AutoConfig, - getKeyValueShapes, -} from './configs.js'; +import { AutoConfig, getKeyValueShapes } from "./configs.js"; -import { - deviceToExecutionProviders, - createInferenceSession, - isONNXTensor, - isONNXProxy, -} from './backends/onnx.js'; +import { deviceToExecutionProviders, createInferenceSession, isONNXTensor, isONNXProxy } from "./backends/onnx.js"; import { DATA_TYPES, DEFAULT_DEVICE_DTYPE_MAPPING, DEFAULT_DTYPE_SUFFIX_MAPPING, isWebGpuFp16Supported, -} from './utils/dtypes.js'; +} from "./utils/dtypes.js"; -import { - Callable, -} from './utils/generic.js'; +import { Callable } from "./utils/generic.js"; -import { - isIntegralNumber, - mergeArrays, - pick, -} from './utils/core.js'; +import { isIntegralNumber, mergeArrays, pick } from "./utils/core.js"; -import { - getModelFile, - getModelJSON, -} from './utils/hub.js'; +import { getModelFile, getModelJSON } from "./utils/hub.js"; -import { - GITHUB_ISSUE_URL, -} from './utils/constants.js'; +import { GITHUB_ISSUE_URL } from "./utils/constants.js"; import { LogitsProcessorList, @@ -86,36 +66,23 @@ import { NoBadWordsLogitsProcessor, MinLengthLogitsProcessor, MinNewTokensLengthLogitsProcessor, - TemperatureLogitsWarper, TopKLogitsWarper, TopPLogitsWarper, ClassifierFreeGuidanceLogitsProcessor, -} from './generation/logits_process.js'; +} from "./generation/logits_process.js"; -import { - GenerationConfig, -} from './generation/configuration_utils.js'; +import { GenerationConfig } from "./generation/configuration_utils.js"; -import { - cat, - full_like, - mean, - ones, - ones_like, - stack, - std_mean, - Tensor, - zeros_like, -} from './utils/tensor.js'; - -import { dynamic_time_warping, medianFilter } from './utils/maths.js'; -import { EosTokenCriteria, MaxLengthCriteria, StoppingCriteriaList } from './generation/stopping_criteria.js'; -import { LogitsSampler } from './generation/logits_sampler.js'; -import { apis } from './env.js'; - -import { WhisperGenerationConfig } from './models/whisper/generation_whisper.js'; -import { whisper_language_to_code } from './models/whisper/common_whisper.js'; +import { cat, full_like, mean, ones, ones_like, stack, std_mean, Tensor, zeros_like } from "./utils/tensor.js"; + +import { dynamic_time_warping, medianFilter } from "./utils/maths.js"; +import { EosTokenCriteria, MaxLengthCriteria, StoppingCriteriaList } from "./generation/stopping_criteria.js"; +import { LogitsSampler } from "./generation/logits_sampler.js"; +import { apis } from "./env.js"; + +import { WhisperGenerationConfig } from "./models/whisper/generation_whisper.js"; +import { whisper_language_to_code } from "./models/whisper/common_whisper.js"; ////////////////////////////////////////////////// // Model types: used internally @@ -128,10 +95,9 @@ const MODEL_TYPES = { MaskGeneration: 5, ImageTextToText: 6, Musicgen: 7, -} +}; ////////////////////////////////////////////////// - ////////////////////////////////////////////////// // Helper functions @@ -140,7 +106,6 @@ const MODEL_TYPE_MAPPING = new Map(); const MODEL_NAME_TO_CLASS_MAPPING = new Map(); const MODEL_CLASS_TO_NAME_MAPPING = new Map(); - /** * Constructs an InferenceSession using a model file located at the specified path. * @param {string} pretrained_model_name_or_path The path to the directory containing the model file. @@ -150,9 +115,9 @@ const MODEL_CLASS_TO_NAME_MAPPING = new Map(); * @private */ async function getSession(pretrained_model_name_or_path, fileName, options) { - const custom_config = options.config?.['transformers.js_config'] ?? {}; + const custom_config = options.config?.["transformers.js_config"] ?? {}; let device = options.device ?? custom_config.device; - if (device && typeof device !== 'string') { + if (device && typeof device !== "string") { if (device.hasOwnProperty(fileName)) { device = device[fileName]; } else { @@ -162,50 +127,52 @@ async function getSession(pretrained_model_name_or_path, fileName, options) { } // If the device is not specified, we use the default (supported) execution providers. - const selectedDevice = /** @type {import("./utils/devices.js").DeviceType} */( - device ?? (apis.IS_NODE_ENV ? 'cpu' : 'wasm') + const selectedDevice = /** @type {import("./utils/devices.js").DeviceType} */ ( + device ?? (apis.IS_NODE_ENV ? "cpu" : "wasm") ); const executionProviders = deviceToExecutionProviders(selectedDevice); // If options.dtype is specified, we use it to choose the suffix for the model file. // Otherwise, we use the default dtype for the device. let dtype = options.dtype ?? custom_config.dtype; - if (typeof dtype !== 'string') { + if (typeof dtype !== "string") { if (dtype && dtype.hasOwnProperty(fileName)) { dtype = dtype[fileName]; } else { dtype = DEFAULT_DEVICE_DTYPE_MAPPING[selectedDevice] ?? DATA_TYPES.fp32; - console.warn(`dtype not specified for "${fileName}". Using the default dtype (${dtype}) for this device (${selectedDevice}).`); + console.warn( + `dtype not specified for "${fileName}". Using the default dtype (${dtype}) for this device (${selectedDevice}).`, + ); } } - const selectedDtype = /** @type {import("./utils/dtypes.js").DataType} */(dtype); + const selectedDtype = /** @type {import("./utils/dtypes.js").DataType} */ (dtype); if (!DEFAULT_DTYPE_SUFFIX_MAPPING.hasOwnProperty(selectedDtype)) { - throw new Error(`Invalid dtype: ${selectedDtype}. Should be one of: ${Object.keys(DATA_TYPES).join(', ')}`); - } else if (selectedDtype === DATA_TYPES.fp16 && selectedDevice === 'webgpu' && !(await isWebGpuFp16Supported())) { + throw new Error(`Invalid dtype: ${selectedDtype}. Should be one of: ${Object.keys(DATA_TYPES).join(", ")}`); + } else if (selectedDtype === DATA_TYPES.fp16 && selectedDevice === "webgpu" && !(await isWebGpuFp16Supported())) { throw new Error(`The device (${selectedDevice}) does not support fp16.`); } // Only valid for models with a decoder const kv_cache_dtype = custom_config.kv_cache_dtype - ? (typeof custom_config.kv_cache_dtype === 'string' + ? typeof custom_config.kv_cache_dtype === "string" ? custom_config.kv_cache_dtype - : custom_config.kv_cache_dtype[selectedDtype] ?? 'float32') + : (custom_config.kv_cache_dtype[selectedDtype] ?? "float32") : undefined; - if (kv_cache_dtype && !['float32', 'float16'].includes(kv_cache_dtype)) { + if (kv_cache_dtype && !["float32", "float16"].includes(kv_cache_dtype)) { throw new Error(`Invalid kv_cache_dtype: ${kv_cache_dtype}. Should be one of: float32, float16`); } const session_config = { dtype: selectedDtype, kv_cache_dtype, - } + }; // Construct the model file name const suffix = DEFAULT_DTYPE_SUFFIX_MAPPING[selectedDtype]; - const modelFileName = `${options.subfolder ?? ''}/${fileName}${suffix}.onnx`; + const modelFileName = `${options.subfolder ?? ""}/${fileName}${suffix}.onnx`; const session_options = { ...options.session_options }; @@ -216,10 +183,10 @@ async function getSession(pretrained_model_name_or_path, fileName, options) { const free_dimension_overrides = custom_config.free_dimension_overrides; if (free_dimension_overrides) { session_options.freeDimensionOverrides ??= free_dimension_overrides; - } else if (selectedDevice.startsWith('webnn') && !session_options.freeDimensionOverrides) { + } else if (selectedDevice.startsWith("webnn") && !session_options.freeDimensionOverrides) { console.warn( 'WebNN does not currently support dynamic shapes and requires `free_dimension_overrides` to be set in config.json as a field within "transformers.js_config". ' + - 'When `free_dimension_overrides` is not set, you may experience significant performance degradation.' + "When `free_dimension_overrides` is not set, you may experience significant performance degradation.", ); } @@ -229,24 +196,24 @@ async function getSession(pretrained_model_name_or_path, fileName, options) { const use_external_data_format = options.use_external_data_format ?? custom_config.use_external_data_format; /** @type {Promise<{path: string, data: Uint8Array}>[]} */ let externalDataPromises = []; - if (use_external_data_format && ( - use_external_data_format === true || - ( - typeof use_external_data_format === 'object' && - use_external_data_format.hasOwnProperty(fileName) && - use_external_data_format[fileName] === true - ) - )) { + if ( + use_external_data_format && + (use_external_data_format === true || + (typeof use_external_data_format === "object" && + use_external_data_format.hasOwnProperty(fileName) && + use_external_data_format[fileName] === true)) + ) { if (apis.IS_NODE_ENV) { - throw new Error('External data format is not yet supported in Node.js'); + throw new Error("External data format is not yet supported in Node.js"); } const path = `${fileName}${suffix}.onnx_data`; - const fullPath = `${options.subfolder ?? ''}/${path}`; - externalDataPromises.push(new Promise(async (resolve, reject) => { - const data = await getModelFile(pretrained_model_name_or_path, fullPath, true, options); - resolve({ path, data }) - })); - + const fullPath = `${options.subfolder ?? ""}/${path}`; + externalDataPromises.push( + new Promise(async (resolve, reject) => { + const data = await getModelFile(pretrained_model_name_or_path, fullPath, true, options); + resolve({ path, data }); + }), + ); } else if (session_options.externalData !== undefined) { externalDataPromises = session_options.externalData.map(async (ext) => { // if the external data is a string, fetch the file and replace the string with its content @@ -262,16 +229,16 @@ async function getSession(pretrained_model_name_or_path, fileName, options) { session_options.externalData = await Promise.all(externalDataPromises); } - if (selectedDevice === 'webgpu') { + if (selectedDevice === "webgpu") { const shapes = getKeyValueShapes(options.config, { - prefix: 'present', + prefix: "present", }); if (Object.keys(shapes).length > 0 && !isONNXProxy()) { // Only set preferredOutputLocation if shapes are present and we aren't proxying ONNX /** @type {Record} */ const preferredOutputLocation = {}; for (const key in shapes) { - preferredOutputLocation[key] = 'gpu-buffer'; + preferredOutputLocation[key] = "gpu-buffer"; } session_options.preferredOutputLocation = preferredOutputLocation; } @@ -284,7 +251,7 @@ async function getSession(pretrained_model_name_or_path, fileName, options) { /** * Helper function to create multiple InferenceSession objects. - * + * * @param {string} pretrained_model_name_or_path The path to the directory containing the model file. * @param {Record} names The names of the model files to load. * @param {import('./utils/hub.js').PretrainedModelOptions} options Additional options for loading the model. @@ -292,13 +259,19 @@ async function getSession(pretrained_model_name_or_path, fileName, options) { * @private */ async function constructSessions(pretrained_model_name_or_path, names, options) { - return Object.fromEntries(await Promise.all( - Object.keys(names).map(async (name) => { - const { buffer, session_options, session_config } = await getSession(pretrained_model_name_or_path, names[name], options); - const session = await createInferenceSession(buffer, session_options, session_config); - return [name, session]; - }) - )); + return Object.fromEntries( + await Promise.all( + Object.keys(names).map(async (name) => { + const { buffer, session_options, session_config } = await getSession( + pretrained_model_name_or_path, + names[name], + options, + ); + const session = await createInferenceSession(buffer, session_options, session_config); + return [name, session]; + }), + ), + ); } /** @@ -310,12 +283,14 @@ async function constructSessions(pretrained_model_name_or_path, names, options) * @private */ async function getOptionalConfigs(pretrained_model_name_or_path, names, options) { - return Object.fromEntries(await Promise.all( - Object.keys(names).map(async (name) => { - const config = await getModelJSON(pretrained_model_name_or_path, names[name], false, options); - return [name, config]; - }) - )); + return Object.fromEntries( + await Promise.all( + Object.keys(names).map(async (name) => { + const config = await getModelJSON(pretrained_model_name_or_path, names[name], false, options); + return [name, config]; + }), + ), + ); } /** @@ -349,7 +324,8 @@ function validateInputs(session, inputs) { } if (missingInputs.length > 0) { throw new Error( - `An error occurred during model execution: "Missing the following inputs: ${missingInputs.join(', ')}.`); + `An error occurred during model execution: "Missing the following inputs: ${missingInputs.join(", ")}.`, + ); } const numInputsProvided = Object.keys(inputs).length; @@ -357,8 +333,10 @@ function validateInputs(session, inputs) { if (numInputsProvided > numInputsNeeded) { // No missing inputs, but too many inputs were provided. // Warn the user and ignore the extra inputs. - let ignored = Object.keys(inputs).filter(inputName => !session.inputNames.includes(inputName)); - console.warn(`WARNING: Too many inputs were provided (${numInputsProvided} > ${numInputsNeeded}). The following inputs will be ignored: "${ignored.join(', ')}".`); + let ignored = Object.keys(inputs).filter((inputName) => !session.inputNames.includes(inputName)); + console.warn( + `WARNING: Too many inputs were provided (${numInputsProvided} > ${numInputsNeeded}). The following inputs will be ignored: "${ignored.join(", ")}".`, + ); } return checkedInputs; @@ -369,7 +347,7 @@ function validateInputs(session, inputs) { * NOTE: `inputs` must contain at least the input names of the model. * - If additional inputs are passed, they will be ignored. * - If inputs are missing, an error will be thrown. - * + * * @param {Object} session The InferenceSession object to run. * @param {Object} inputs An object that maps input names to input tensors. * @returns {Promise} A Promise that resolves to an object that maps output names to output tensors. @@ -386,7 +364,7 @@ async function sessionRun(session, inputs) { } catch (e) { // This usually occurs when the inputs are of the wrong type. console.error(`An error occurred during model execution: "${e}".`); - console.error('Inputs given to model:', checkedInputs); + console.error("Inputs given to model:", checkedInputs); throw e; } } @@ -401,14 +379,13 @@ function replaceTensors(obj) { for (let prop in obj) { if (isONNXTensor(obj[prop])) { obj[prop] = new Tensor(obj[prop]); - } else if (typeof obj[prop] === 'object') { + } else if (typeof obj[prop] === "object") { replaceTensors(obj[prop]); } } return obj; } - /** * Converts an array or Tensor of integers to an int64 Tensor. * @param {any[]|Tensor} items The input integers to be converted. @@ -427,20 +404,19 @@ function toI64Tensor(items) { if (Array.isArray(items[0])) { // batched - if (items.some(x => x.length !== items[0].length)) { - throw Error("Unable to create tensor, you should probably activate truncation and/or padding with 'padding=True' and/or 'truncation=True' to have batched tensors with the same length.") + if (items.some((x) => x.length !== items[0].length)) { + throw Error( + "Unable to create tensor, you should probably activate truncation and/or padding with 'padding=True' and/or 'truncation=True' to have batched tensors with the same length.", + ); } - return new Tensor('int64', - BigInt64Array.from(items.flat().map(x => BigInt(x))), - [items.length, items[0].length] - ); + return new Tensor("int64", BigInt64Array.from(items.flat().map((x) => BigInt(x))), [ + items.length, + items[0].length, + ]); } else { //flat - return new Tensor('int64', - BigInt64Array.from(items.map(x => BigInt(x))), - [1, items.length] - ); + return new Tensor("int64", BigInt64Array.from(items.map((x) => BigInt(x))), [1, items.length]); } } @@ -451,7 +427,7 @@ function toI64Tensor(items) { * @private */ function boolTensor(value) { - return new Tensor('bool', [value], [1]); + return new Tensor("bool", [value], [1]); } // JS doesn't support mixins, so we define some reused functions here, and allow "this" to be passed in @@ -466,7 +442,7 @@ async function seq2seqForward(self, model_inputs) { let { encoder_outputs, input_ids, decoder_input_ids, ...other_decoder_inputs } = model_inputs; // Encode if needed if (!encoder_outputs) { - const encoder_inputs = pick(model_inputs, self.sessions['model'].inputNames); + const encoder_inputs = pick(model_inputs, self.sessions["model"].inputNames); // Encoder outputs are not given, so we must compute them. encoder_outputs = (await encoderForward(self, encoder_inputs)).last_hidden_state; } @@ -474,8 +450,8 @@ async function seq2seqForward(self, model_inputs) { other_decoder_inputs.input_ids = decoder_input_ids; other_decoder_inputs.encoder_hidden_states = encoder_outputs; - if (self.sessions['decoder_model_merged'].inputNames.includes('encoder_attention_mask')) { - other_decoder_inputs.encoder_attention_mask = model_inputs.attention_mask + if (self.sessions["decoder_model_merged"].inputNames.includes("encoder_attention_mask")) { + other_decoder_inputs.encoder_attention_mask = model_inputs.attention_mask; } const decoderResults = await decoderForward(self, other_decoder_inputs, true); @@ -491,23 +467,23 @@ async function seq2seqForward(self, model_inputs) { * @private */ async function encoderForward(self, model_inputs) { - const session = self.sessions['model']; + const session = self.sessions["model"]; const encoderFeeds = pick(model_inputs, session.inputNames); - if (session.inputNames.includes('inputs_embeds') && !encoderFeeds.inputs_embeds) { + if (session.inputNames.includes("inputs_embeds") && !encoderFeeds.inputs_embeds) { if (!model_inputs.input_ids) { - throw new Error('Both `input_ids` and `inputs_embeds` are missing in the model inputs.'); + throw new Error("Both `input_ids` and `inputs_embeds` are missing in the model inputs."); } encoderFeeds.inputs_embeds = await self.encode_text({ input_ids: model_inputs.input_ids }); } - if (session.inputNames.includes('token_type_ids') && !encoderFeeds.token_type_ids) { + if (session.inputNames.includes("token_type_ids") && !encoderFeeds.token_type_ids) { // Assign default `token_type_ids` (all zeroes) to the `encoderFeeds` if the model expects it, // but they weren't created by the tokenizer. encoderFeeds.token_type_ids = new Tensor( - 'int64', + "int64", new BigInt64Array(encoderFeeds.input_ids.data.length), - encoderFeeds.input_ids.dims - ) + encoderFeeds.input_ids.dims, + ); } return await sessionRun(session, encoderFeeds); } @@ -520,17 +496,18 @@ async function encoderForward(self, model_inputs) { * @private */ async function decoderForward(self, model_inputs, is_encoder_decoder = false) { - - const session = self.sessions[ - is_encoder_decoder ? 'decoder_model_merged' : 'model' - ] + const session = self.sessions[is_encoder_decoder ? "decoder_model_merged" : "model"]; const { past_key_values, ...new_model_inputs } = model_inputs; - if (session.inputNames.includes('use_cache_branch')) { + if (session.inputNames.includes("use_cache_branch")) { new_model_inputs.use_cache_branch = boolTensor(!!past_key_values); } - if (session.inputNames.includes('position_ids') && new_model_inputs.attention_mask && !new_model_inputs.position_ids) { + if ( + session.inputNames.includes("position_ids") && + new_model_inputs.attention_mask && + !new_model_inputs.position_ids + ) { new_model_inputs.position_ids = createPositionIds(new_model_inputs, past_key_values); } @@ -542,7 +519,6 @@ async function decoderForward(self, model_inputs, is_encoder_decoder = false) { return await sessionRun(session, fixed); } - /** * Forward pass of an image-text-to-text model. * @param {Object} self The image-text-to-text model model. @@ -558,25 +534,27 @@ async function decoderForward(self, model_inputs, is_encoder_decoder = false) { * @returns {Promise} The model's output tensor * @private */ -async function imageTextToTextForward(self, { - // Produced by the tokenizer/processor: - input_ids = null, - attention_mask = null, - pixel_values = null, - - // Used during generation: - position_ids = null, - inputs_embeds = null, - past_key_values = null, +async function imageTextToTextForward( + self, + { + // Produced by the tokenizer/processor: + input_ids = null, + attention_mask = null, + pixel_values = null, - // Generic generation parameters - generation_config = null, - logits_processor = null, + // Used during generation: + position_ids = null, + inputs_embeds = null, + past_key_values = null, - // TODO: needed? - ...kwargs -}) { + // Generic generation parameters + generation_config = null, + logits_processor = null, + // TODO: needed? + ...kwargs + }, +) { if (!inputs_embeds) { // 1. Extract the input embeddings inputs_embeds = await self.encode_text({ input_ids }); @@ -591,34 +569,40 @@ async function imageTextToTextForward(self, { input_ids, attention_mask, })); - } else if (past_key_values && pixel_values && input_ids.dims[1] === 1) { // This is the case when we are generating with cache const target_length = input_ids.dims[1]; // always 1 const past_length = Object.values(past_key_values)[0].dims.at(-2); - attention_mask = cat([ - ones([input_ids.dims[0], past_length]), - attention_mask.slice(null, [attention_mask.dims[1] - target_length, attention_mask.dims[1]]), - ], 1); + attention_mask = cat( + [ + ones([input_ids.dims[0], past_length]), + attention_mask.slice(null, [attention_mask.dims[1] - target_length, attention_mask.dims[1]]), + ], + 1, + ); } } - const outputs = await decoderForward(self, { - inputs_embeds, - past_key_values, - attention_mask, - position_ids, - generation_config, - logits_processor, - }, true); + const outputs = await decoderForward( + self, + { + inputs_embeds, + past_key_values, + attention_mask, + position_ids, + generation_config, + logits_processor, + }, + true, + ); return outputs; } function createPositionIds(model_inputs, past_key_values = null) { // If the model supports providing position_ids, we create position_ids on the fly for batch generation, // by computing the cumulative sum of the attention mask along the sequence length dimension. - // + // // Equivalent to: // position_ids = attention_mask.long().cumsum(-1) - 1 // position_ids.masked_fill_(attention_mask == 0, 1) @@ -635,14 +619,15 @@ function createPositionIds(model_inputs, past_key_values = null) { const index = start + j; if (attention_mask.data[index] === 0n) { data[index] = BigInt(1); - } else { // === 1n + } else { + // === 1n data[index] = sum; sum += attention_mask.data[index]; } } } - let position_ids = new Tensor('int64', data, attention_mask.dims); + let position_ids = new Tensor("int64", data, attention_mask.dims); if (past_key_values) { const offset = -(input_ids ?? inputs_embeds).dims.at(1); position_ids = position_ids.slice(null, [offset, null]); @@ -677,12 +662,12 @@ function decoder_prepare_inputs_for_generation(self, input_ids, model_inputs, ge // NOTE: Only used by VLMs (!= so that null matches undefined) self.config.image_token_index != null && // Equivalent to `self.config.image_token_index in input_ids` (== so that int matches bigint) - input_ids.data.some(x => x == self.config.image_token_index) + input_ids.data.some((x) => x == self.config.image_token_index) ) { // TODO: Support multiple image tokens const num_image_tokens = self.config.num_image_tokens; if (!num_image_tokens) { - throw new Error('`num_image_tokens` is missing in the model configuration.'); + throw new Error("`num_image_tokens` is missing in the model configuration."); } const num_new_tokens = input_ids.dims[1] - (past_length - num_image_tokens); @@ -699,7 +684,7 @@ function decoder_prepare_inputs_for_generation(self, input_ids, model_inputs, ge function encoder_decoder_prepare_inputs_for_generation(self, input_ids, model_inputs, generation_config) { if (model_inputs.past_key_values) { - input_ids = input_ids.map(x => [x.at(-1)]); + input_ids = input_ids.map((x) => [x.at(-1)]); } return { @@ -723,8 +708,8 @@ function image_text_to_text_prepare_inputs_for_generation(self, ...args) { * A base class for pre-trained models that provides the model configuration and an ONNX session. */ export class PreTrainedModel extends Callable { - main_input_name = 'input_ids'; - forward_params = ['input_ids', 'attention_mask']; + main_input_name = "input_ids"; + forward_params = ["input_ids", "attention_mask"]; /** * Creates a new instance of the `PreTrainedModel` class. * @param {import('./configs.js').PretrainedConfig} config The model configuration. @@ -776,23 +761,23 @@ export class PreTrainedModel extends Callable { } if (this.can_generate) { - this.forward_params.push('past_key_values'); + this.forward_params.push("past_key_values"); } /** @type {import('./configs.js').TransformersJSConfig} */ - this.custom_config = this.config['transformers.js_config'] ?? {}; + this.custom_config = this.config["transformers.js_config"] ?? {}; } /** - * Disposes of all the ONNX sessions that were created during inference. - * @returns {Promise} An array of promises, one for each ONNX session that is being disposed. - * @todo Use https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/FinalizationRegistry - */ + * Disposes of all the ONNX sessions that were created during inference. + * @returns {Promise} An array of promises, one for each ONNX session that is being disposed. + * @todo Use https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/FinalizationRegistry + */ async dispose() { const promises = []; for (const session of Object.values(this.sessions)) { if (session?.handler?.dispose) { - promises.push(session.handler.dispose()) + promises.push(session.handler.dispose()); } } return await Promise.all(promises); @@ -800,33 +785,35 @@ export class PreTrainedModel extends Callable { /** * Instantiate one of the model classes of the library from a pretrained model. - * + * * The model class to instantiate is selected based on the `model_type` property of the config object * (either passed as an argument or loaded from `pretrained_model_name_or_path` if possible) - * + * * @param {string} pretrained_model_name_or_path The name or path of the pretrained model. Can be either: * - A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co. * Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a * user or organization name, like `dbmdz/bert-base-german-cased`. * - A path to a *directory* containing model weights, e.g., `./my_model_directory/`. * @param {import('./utils/hub.js').PretrainedModelOptions} options Additional options for loading the model. - * + * * @returns {Promise} A new instance of the `PreTrainedModel` class. */ - static async from_pretrained(pretrained_model_name_or_path, { - progress_callback = null, - config = null, - cache_dir = null, - local_files_only = false, - revision = 'main', - model_file_name = null, - subfolder = 'onnx', - device = null, - dtype = null, - use_external_data_format = null, - session_options = {}, - } = {}) { - + static async from_pretrained( + pretrained_model_name_or_path, + { + progress_callback = null, + config = null, + cache_dir = null, + local_files_only = false, + revision = "main", + model_file_name = null, + subfolder = "onnx", + device = null, + dtype = null, + use_external_data_format = null, + session_options = {}, + } = {}, + ) { let options = { progress_callback, config, @@ -839,7 +826,7 @@ export class PreTrainedModel extends Callable { dtype, use_external_data_format, session_options, - } + }; const modelName = MODEL_CLASS_TO_NAME_MAPPING.get(this); const modelType = MODEL_TYPE_MAPPING.get(modelName); @@ -849,77 +836,114 @@ export class PreTrainedModel extends Callable { let info; if (modelType === MODEL_TYPES.DecoderOnly) { info = await Promise.all([ - constructSessions(pretrained_model_name_or_path, { - model: options.model_file_name ?? 'model', - }, options), - getOptionalConfigs(pretrained_model_name_or_path, { - generation_config: 'generation_config.json', - }, options), + constructSessions( + pretrained_model_name_or_path, + { + model: options.model_file_name ?? "model", + }, + options, + ), + getOptionalConfigs( + pretrained_model_name_or_path, + { + generation_config: "generation_config.json", + }, + options, + ), ]); - } else if (modelType === MODEL_TYPES.Seq2Seq || modelType === MODEL_TYPES.Vision2Seq) { info = await Promise.all([ - constructSessions(pretrained_model_name_or_path, { - model: 'encoder_model', - decoder_model_merged: 'decoder_model_merged', - }, options), - getOptionalConfigs(pretrained_model_name_or_path, { - generation_config: 'generation_config.json', - }, options), + constructSessions( + pretrained_model_name_or_path, + { + model: "encoder_model", + decoder_model_merged: "decoder_model_merged", + }, + options, + ), + getOptionalConfigs( + pretrained_model_name_or_path, + { + generation_config: "generation_config.json", + }, + options, + ), ]); - } else if (modelType === MODEL_TYPES.MaskGeneration) { info = await Promise.all([ - constructSessions(pretrained_model_name_or_path, { - model: 'vision_encoder', - prompt_encoder_mask_decoder: 'prompt_encoder_mask_decoder', - }, options), + constructSessions( + pretrained_model_name_or_path, + { + model: "vision_encoder", + prompt_encoder_mask_decoder: "prompt_encoder_mask_decoder", + }, + options, + ), ]); - } else if (modelType === MODEL_TYPES.EncoderDecoder) { info = await Promise.all([ - constructSessions(pretrained_model_name_or_path, { - model: 'encoder_model', - decoder_model_merged: 'decoder_model_merged', - }, options), + constructSessions( + pretrained_model_name_or_path, + { + model: "encoder_model", + decoder_model_merged: "decoder_model_merged", + }, + options, + ), ]); - } else if (modelType === MODEL_TYPES.ImageTextToText) { const sessions = { - embed_tokens: 'embed_tokens', - vision_encoder: 'vision_encoder', - decoder_model_merged: 'decoder_model_merged', - } + embed_tokens: "embed_tokens", + vision_encoder: "vision_encoder", + decoder_model_merged: "decoder_model_merged", + }; if (config.is_encoder_decoder) { - sessions['model'] = 'encoder_model'; + sessions["model"] = "encoder_model"; } info = await Promise.all([ constructSessions(pretrained_model_name_or_path, sessions, options), - getOptionalConfigs(pretrained_model_name_or_path, { - generation_config: 'generation_config.json', - }, options), + getOptionalConfigs( + pretrained_model_name_or_path, + { + generation_config: "generation_config.json", + }, + options, + ), ]); - } else if (modelType === MODEL_TYPES.Musicgen) { info = await Promise.all([ - constructSessions(pretrained_model_name_or_path, { - model: 'text_encoder', - decoder_model_merged: 'decoder_model_merged', - encodec_decode: 'encodec_decode', - }, options), - getOptionalConfigs(pretrained_model_name_or_path, { - generation_config: 'generation_config.json', - }, options), + constructSessions( + pretrained_model_name_or_path, + { + model: "text_encoder", + decoder_model_merged: "decoder_model_merged", + encodec_decode: "encodec_decode", + }, + options, + ), + getOptionalConfigs( + pretrained_model_name_or_path, + { + generation_config: "generation_config.json", + }, + options, + ), ]); - - } else { // should be MODEL_TYPES.EncoderOnly + } else { + // should be MODEL_TYPES.EncoderOnly if (modelType !== MODEL_TYPES.EncoderOnly) { - console.warn(`Model type for '${modelName ?? config?.model_type}' not found, assuming encoder-only architecture. Please report this at ${GITHUB_ISSUE_URL}.`) + console.warn( + `Model type for '${modelName ?? config?.model_type}' not found, assuming encoder-only architecture. Please report this at ${GITHUB_ISSUE_URL}.`, + ); } info = await Promise.all([ - constructSessions(pretrained_model_name_or_path, { - model: options.model_file_name ?? 'model', - }, options), + constructSessions( + pretrained_model_name_or_path, + { + model: options.model_file_name ?? "model", + }, + options, + ), ]); } @@ -959,10 +983,9 @@ export class PreTrainedModel extends Callable { * This function returns a [`LogitsProcessorList`] list object that contains all relevant [`LogitsWarper`] * instances used for multinomial sampling. * @param {GenerationConfig} generation_config The generation config. - * @returns {LogitsProcessorList} generation_config + * @returns {LogitsProcessorList} generation_config */ _get_logits_warper(generation_config) { - // instantiate warpers list const warpers = new LogitsProcessorList(); @@ -982,7 +1005,7 @@ export class PreTrainedModel extends Callable { } /** - * @param {GenerationConfig} generation_config + * @param {GenerationConfig} generation_config * @param {number} input_ids_seq_length The starting sequence length for the input ids. * @returns {LogitsProcessorList} * @private @@ -992,7 +1015,7 @@ export class PreTrainedModel extends Callable { input_ids_seq_length, // encoder_input_ids, TODO // prefix_allowed_tokens_fn, TODO - logits_processor = null + logits_processor = null, ) { const processors = new LogitsProcessorList(); @@ -1031,19 +1054,31 @@ export class PreTrainedModel extends Callable { // } if (generation_config.bad_words_ids !== null) { - processors.push(new NoBadWordsLogitsProcessor(generation_config.bad_words_ids, generation_config.eos_token_id)); + processors.push( + new NoBadWordsLogitsProcessor(generation_config.bad_words_ids, generation_config.eos_token_id), + ); } - if (generation_config.min_length !== null && generation_config.eos_token_id !== null && generation_config.min_length > 0) { + if ( + generation_config.min_length !== null && + generation_config.eos_token_id !== null && + generation_config.min_length > 0 + ) { processors.push(new MinLengthLogitsProcessor(generation_config.min_length, generation_config.eos_token_id)); } - if (generation_config.min_new_tokens !== null && generation_config.eos_token_id !== null && generation_config.min_new_tokens > 0) { - processors.push(new MinNewTokensLengthLogitsProcessor( - input_ids_seq_length, - generation_config.min_new_tokens, - generation_config.eos_token_id - )); + if ( + generation_config.min_new_tokens !== null && + generation_config.eos_token_id !== null && + generation_config.min_new_tokens > 0 + ) { + processors.push( + new MinNewTokensLengthLogitsProcessor( + input_ids_seq_length, + generation_config.min_new_tokens, + generation_config.eos_token_id, + ), + ); } // if (prefix_allowed_tokens_fn !== null) { @@ -1053,16 +1088,14 @@ export class PreTrainedModel extends Callable { // )); // } - if (generation_config.forced_bos_token_id !== null) { processors.push(new ForcedBOSTokenLogitsProcessor(generation_config.forced_bos_token_id)); } if (generation_config.forced_eos_token_id !== null) { - processors.push(new ForcedEOSTokenLogitsProcessor( - generation_config.max_length, - generation_config.forced_eos_token_id - )); + processors.push( + new ForcedEOSTokenLogitsProcessor(generation_config.max_length, generation_config.forced_eos_token_id), + ); } // if (generation_config.remove_invalid_values === true) { @@ -1082,11 +1115,14 @@ export class PreTrainedModel extends Callable { // } if (generation_config.begin_suppress_tokens !== null) { - const begin_index = (input_ids_seq_length > 1 || generation_config.forced_bos_token_id === null) - ? input_ids_seq_length - : input_ids_seq_length + 1; + const begin_index = + input_ids_seq_length > 1 || generation_config.forced_bos_token_id === null + ? input_ids_seq_length + : input_ids_seq_length + 1; - processors.push(new SuppressTokensAtBeginLogitsProcessor(generation_config.begin_suppress_tokens, begin_index)); + processors.push( + new SuppressTokensAtBeginLogitsProcessor(generation_config.begin_suppress_tokens, begin_index), + ); } // DEPRECATED: https://github.com/huggingface/transformers/pull/29485 @@ -1094,14 +1130,13 @@ export class PreTrainedModel extends Callable { // processors.push(new ForceTokensLogitsProcessor(generation_config.forced_decoder_ids)); // } - // 8. prepare batched CFG externally if (generation_config.guidance_scale !== null && generation_config.guidance_scale > 1) { processors.push(new ClassifierFreeGuidanceLogitsProcessor(generation_config.guidance_scale)); } if (logits_processor !== null) { - processors.extend(logits_processor) + processors.extend(logits_processor); } // `LogitNormalization` should always be the last logit processor, when present @@ -1151,18 +1186,17 @@ export class PreTrainedModel extends Callable { } /** - * - * @param {GenerationConfig} generation_config - * @param {StoppingCriteriaList} [stopping_criteria=null] + * + * @param {GenerationConfig} generation_config + * @param {StoppingCriteriaList} [stopping_criteria=null] */ _get_stopping_criteria(generation_config, stopping_criteria = null) { const criteria = new StoppingCriteriaList(); if (generation_config.max_length !== null) { - criteria.push(new MaxLengthCriteria( - generation_config.max_length, - this.config.max_position_embeddings ?? null, - )); + criteria.push( + new MaxLengthCriteria(generation_config.max_length, this.config.max_position_embeddings ?? null), + ); } // if (generation_config.max_time !== null) { // criteria.push(new MaxTimeCriteria(generation_config.max_time)); @@ -1175,7 +1209,6 @@ export class PreTrainedModel extends Callable { criteria.extend(stopping_criteria); } return criteria; - } /** @@ -1203,9 +1236,9 @@ export class PreTrainedModel extends Callable { } } - let errorMessage = `The current model class (${modelName}) is not compatible with \`.generate()\`, as it doesn't have a language model head.` + let errorMessage = `The current model class (${modelName}) is not compatible with \`.generate()\`, as it doesn't have a language model head.`; if (generate_compatible_classes.size > 0) { - errorMessage += ` Please use the following class instead: ${[...generate_compatible_classes].join(', ')}`; + errorMessage += ` Please use the following class instead: ${[...generate_compatible_classes].join(", ")}`; } throw Error(errorMessage); } @@ -1216,7 +1249,7 @@ export class PreTrainedModel extends Callable { } /** - * + * * @param {Object} inputs * @param {bigint[][]} inputs.generated_input_ids * @param {Object} inputs.outputs @@ -1226,25 +1259,23 @@ export class PreTrainedModel extends Callable { */ _update_model_kwargs_for_generation({ generated_input_ids, outputs, model_inputs, is_encoder_decoder }) { // update past_key_values - model_inputs['past_key_values'] = this.getPastKeyValues(outputs, model_inputs.past_key_values); + model_inputs["past_key_values"] = this.getPastKeyValues(outputs, model_inputs.past_key_values); // update inputs for next run - model_inputs['input_ids'] = new Tensor('int64', generated_input_ids.flat(), [generated_input_ids.length, 1]); + model_inputs["input_ids"] = new Tensor("int64", generated_input_ids.flat(), [generated_input_ids.length, 1]); if (!is_encoder_decoder) { // update attention mask model_inputs.attention_mask = cat( - [ - model_inputs.attention_mask, - ones([model_inputs.attention_mask.dims[0], 1]), - ], 1 + [model_inputs.attention_mask, ones([model_inputs.attention_mask.dims[0], 1])], + 1, ); - } else if ('decoder_attention_mask' in model_inputs) { + } else if ("decoder_attention_mask" in model_inputs) { // TODO: update decoder attention mask if the model requires it } // force recreate position_ids in next iteration - model_inputs['position_ids'] = null; + model_inputs["position_ids"] = null; return model_inputs; } @@ -1264,7 +1295,7 @@ export class PreTrainedModel extends Callable { if (inputs) { throw new Error( "`inputs`: {inputs}` were passed alongside {input_name} which is not allowed. " + - "Make sure to either pass {inputs} or {input_name}=..." + "Make sure to either pass {inputs} or {input_name}=...", ); } } else { @@ -1276,11 +1307,16 @@ export class PreTrainedModel extends Callable { return { inputs_tensor, model_inputs, model_input_name: input_name }; } - async _prepare_encoder_decoder_kwargs_for_generation({ inputs_tensor, model_inputs, model_input_name, generation_config }) { + async _prepare_encoder_decoder_kwargs_for_generation({ + inputs_tensor, + model_inputs, + model_input_name, + generation_config, + }) { if ( - this.sessions['model'].inputNames.includes('inputs_embeds') - && !model_inputs.inputs_embeds - && '_prepare_inputs_embeds' in this + this.sessions["model"].inputNames.includes("inputs_embeds") && + !model_inputs.inputs_embeds && + "_prepare_inputs_embeds" in this ) { // Encoder expects `inputs_embeds` instead of `input_ids` const { input_ids, pixel_values, attention_mask, ...kwargs } = model_inputs; @@ -1288,26 +1324,21 @@ export class PreTrainedModel extends Callable { const prepared_inputs = await this._prepare_inputs_embeds(model_inputs); model_inputs = { ...kwargs, - ...pick(prepared_inputs, ['inputs_embeds', 'attention_mask']), + ...pick(prepared_inputs, ["inputs_embeds", "attention_mask"]), }; } let { last_hidden_state } = await encoderForward(this, model_inputs); // for classifier free guidance we need to add a 'null' input to our encoder hidden states if (generation_config.guidance_scale !== null && generation_config.guidance_scale > 1) { + last_hidden_state = cat([last_hidden_state, full_like(last_hidden_state, 0.0)], 0); - last_hidden_state = cat([ - last_hidden_state, - full_like(last_hidden_state, 0.0), - ], 0); - - if ('attention_mask' in model_inputs) { - model_inputs['attention_mask'] = cat([ - model_inputs['attention_mask'], - zeros_like(model_inputs['attention_mask']), - ], 0); + if ("attention_mask" in model_inputs) { + model_inputs["attention_mask"] = cat( + [model_inputs["attention_mask"], zeros_like(model_inputs["attention_mask"])], + 0, + ); } - } else if (model_inputs.decoder_input_ids) { // Ensure that the encoder outputs have the same batch size as the decoder inputs, // allowing for more efficient batched generation for single inputs @@ -1315,22 +1346,32 @@ export class PreTrainedModel extends Callable { if (decoder_input_ids_batch_size !== last_hidden_state.dims[0]) { if (last_hidden_state.dims[0] !== 1) { throw new Error( - `The encoder outputs have a different batch size (${last_hidden_state.dims[0]}) than the decoder inputs (${decoder_input_ids_batch_size}).` - ) + `The encoder outputs have a different batch size (${last_hidden_state.dims[0]}) than the decoder inputs (${decoder_input_ids_batch_size}).`, + ); } - last_hidden_state = cat(Array.from({ length: decoder_input_ids_batch_size }, () => last_hidden_state), 0); + last_hidden_state = cat( + Array.from({ length: decoder_input_ids_batch_size }, () => last_hidden_state), + 0, + ); } } - model_inputs['encoder_outputs'] = last_hidden_state; + model_inputs["encoder_outputs"] = last_hidden_state; return model_inputs; } /** * Prepares `decoder_input_ids` for generation with encoder-decoder models - * @param {*} param0 - */ - _prepare_decoder_input_ids_for_generation({ batch_size, model_input_name, model_kwargs, decoder_start_token_id, bos_token_id, generation_config }) { + * @param {*} param0 + */ + _prepare_decoder_input_ids_for_generation({ + batch_size, + model_input_name, + model_kwargs, + decoder_start_token_id, + bos_token_id, + generation_config, + }) { let { decoder_input_ids, ...model_inputs } = model_kwargs; // Prepare input ids if the user has not defined `decoder_input_ids` manually. @@ -1338,34 +1379,42 @@ export class PreTrainedModel extends Callable { if (!decoder_input_ids) { decoder_start_token_id ??= bos_token_id; - if (this.config.model_type === 'musicgen') { + if (this.config.model_type === "musicgen") { // Custom logic (TODO: move to Musicgen class) - decoder_input_ids = Array.from({ - length: batch_size * this.config.decoder.num_codebooks - }, () => [decoder_start_token_id]); - + decoder_input_ids = Array.from( + { + length: batch_size * this.config.decoder.num_codebooks, + }, + () => [decoder_start_token_id], + ); } else if (Array.isArray(decoder_start_token_id)) { if (decoder_start_token_id.length !== batch_size) { throw new Error( - `\`decoder_start_token_id\` expcted to have length ${batch_size} but got ${decoder_start_token_id.length}` - ) + `\`decoder_start_token_id\` expcted to have length ${batch_size} but got ${decoder_start_token_id.length}`, + ); } decoder_input_ids = decoder_start_token_id; } else { - decoder_input_ids = Array.from({ - length: batch_size, - }, () => [decoder_start_token_id]); + decoder_input_ids = Array.from( + { + length: batch_size, + }, + () => [decoder_start_token_id], + ); } } else if (!Array.isArray(decoder_input_ids[0])) { // Correct batch size - decoder_input_ids = Array.from({ - length: batch_size, - }, () => decoder_input_ids); + decoder_input_ids = Array.from( + { + length: batch_size, + }, + () => decoder_input_ids, + ); } decoder_input_ids = toI64Tensor(decoder_input_ids); } - model_kwargs['decoder_attention_mask'] = ones_like(decoder_input_ids); + model_kwargs["decoder_attention_mask"] = ones_like(decoder_input_ids); return { input_ids: decoder_input_ids, model_inputs }; } @@ -1401,12 +1450,15 @@ export class PreTrainedModel extends Callable { // 4. Define other model kwargs if (!is_encoder_decoder) { // decoder-only models should use left-padding for generation - } else if (!('encoder_outputs' in model_inputs)) { + } else if (!("encoder_outputs" in model_inputs)) { // if model is encoder decoder encoder_outputs are created // and added to `model_kwargs` - model_inputs = await this._prepare_encoder_decoder_kwargs_for_generation( - { inputs_tensor, model_inputs, model_input_name, generation_config } - ) + model_inputs = await this._prepare_encoder_decoder_kwargs_for_generation({ + inputs_tensor, + model_inputs, + model_input_name, + generation_config, + }); } // 5. Prepare `input_ids` which will be used for auto-regressive generation @@ -1423,7 +1475,7 @@ export class PreTrainedModel extends Callable { generation_config, })); } else { - input_ids = model_inputs[model_input_name] + input_ids = model_inputs[model_input_name]; } // 6. Prepare `max_length` depending on other stopping criteria. @@ -1453,12 +1505,10 @@ export class PreTrainedModel extends Callable { generation_config, input_ids_length, logits_processor, - ) + ); // 9. prepare stopping criteria - const prepared_stopping_criteria = this._get_stopping_criteria( - generation_config, stopping_criteria - ) + const prepared_stopping_criteria = this._get_stopping_criteria(generation_config, stopping_criteria); // /** @type {number[]} */ // let eos_token_ids = generation_config.eos_token_id; @@ -1547,12 +1597,15 @@ export class PreTrainedModel extends Callable { } const stop = prepared_stopping_criteria(all_input_ids); - if (stop.every(x => x)) { + if (stop.every((x) => x)) { break; } model_inputs = this._update_model_kwargs_for_generation({ - generated_input_ids, outputs, model_inputs, is_encoder_decoder, + generated_input_ids, + outputs, + model_inputs, + is_encoder_decoder, }); } @@ -1564,7 +1617,7 @@ export class PreTrainedModel extends Callable { const past_key_values = this.getPastKeyValues(outputs, model_inputs.past_key_values, true); // TODO: ensure all_input_ids is padded correctly... - const sequences = new Tensor('int64', all_input_ids.flat(), [all_input_ids.length, all_input_ids[0].length]); + const sequences = new Tensor("int64", all_input_ids.flat(), [all_input_ids.length, all_input_ids[0].length]); if (generation_config.return_dict_in_generate) { return { @@ -1574,11 +1627,11 @@ export class PreTrainedModel extends Callable { // TODO: // scores, // logits, - } + }; } else { // Dispose all remaining tensors for (const tensor of Object.values(outputs)) { - if (tensor.location === 'gpu-buffer') { + if (tensor.location === "gpu-buffer") { tensor.dispose(); } } @@ -1597,15 +1650,16 @@ export class PreTrainedModel extends Callable { const pkvs = Object.create(null); for (const name in decoderResults) { - if (name.startsWith('present')) { - const newName = name.replace('present', 'past_key_values'); - const is_encoder_pkv = name.includes('encoder'); + if (name.startsWith("present")) { + const newName = name.replace("present", "past_key_values"); + const is_encoder_pkv = name.includes("encoder"); if (is_encoder_pkv && pastKeyValues) { // Optimization introduced by optimum to reuse past key values. // So, we just replace the constant outputs (`decoderResults[name]`) with the previous past key values. // https://github.com/huggingface/optimum/blob/0bf2c05fb7e1182b52d21b703cfc95fd9e4ea3dc/optimum/onnxruntime/base.py#L677-L704 pkvs[newName] = pastKeyValues[newName]; - } else { // decoder or using first encoder PKVs + } else { + // decoder or using first encoder PKVs pkvs[newName] = decoderResults[name]; } @@ -1613,7 +1667,7 @@ export class PreTrainedModel extends Callable { // - Always dispose decoder PKVs // - Only dispose encoder past key values when requested (after generation) const t = pastKeyValues[newName]; - if (t.location === 'gpu-buffer') { + if (t.location === "gpu-buffer") { t.dispose(); } } @@ -1631,7 +1685,7 @@ export class PreTrainedModel extends Callable { getAttentions(model_output) { const attentions = {}; - for (const attnName of ['cross_attentions', 'encoder_attentions', 'decoder_attentions']) { + for (const attnName of ["cross_attentions", "encoder_attentions", "decoder_attentions"]) { for (const name in model_output) { if (name.startsWith(attnName)) { if (!(attnName in attentions)) { @@ -1652,11 +1706,11 @@ export class PreTrainedModel extends Callable { */ addPastKeyValues(decoderFeeds, pastKeyValues) { if (pastKeyValues) { - Object.assign(decoderFeeds, pastKeyValues) + Object.assign(decoderFeeds, pastKeyValues); } else { - const session = this.sessions['decoder_model_merged'] ?? this.sessions['model']; - const dtype = session?.config?.kv_cache_dtype ?? 'float32'; - const empty = (dtype === 'float16') ? new Uint16Array() : []; + const session = this.sessions["decoder_model_merged"] ?? this.sessions["model"]; + const dtype = session?.config?.kv_cache_dtype ?? "float32"; + const empty = dtype === "float16" ? new Uint16Array() : []; const shapes = getKeyValueShapes(this.config); @@ -1668,12 +1722,12 @@ export class PreTrainedModel extends Callable { async encode_image({ pixel_values }) { // image_inputs === { pixel_values } - const features = (await sessionRun(this.sessions['vision_encoder'], { pixel_values })).image_features; + const features = (await sessionRun(this.sessions["vision_encoder"], { pixel_values })).image_features; if (!this.config.num_image_tokens) { console.warn( - 'The number of image tokens was not set in the model configuration. ' + - `Setting it to the number of features detected by the vision encoder (${features.dims[1]}).` - ) + "The number of image tokens was not set in the model configuration. " + + `Setting it to the number of features detected by the vision encoder (${features.dims[1]}).`, + ); this.config.num_image_tokens = features.dims[1]; } return features; @@ -1681,13 +1735,13 @@ export class PreTrainedModel extends Callable { async encode_text({ input_ids }) { // text_inputs === { input_ids, attention_mask } - return (await sessionRun(this.sessions['embed_tokens'], { input_ids })).inputs_embeds; + return (await sessionRun(this.sessions["embed_tokens"], { input_ids })).inputs_embeds; } } ////////////////////////////////////////////////// // Base model output class -export class ModelOutput { } +export class ModelOutput {} /** * Base class for model's outputs, with potential hidden states and attentions. @@ -1708,8 +1762,8 @@ export class BaseModelOutput extends ModelOutput { } ////////////////////////////////////////////////// // Bert models -export class BertPreTrainedModel extends PreTrainedModel { } -export class BertModel extends BertPreTrainedModel { } +export class BertPreTrainedModel extends PreTrainedModel {} +export class BertModel extends BertPreTrainedModel {} /** * BertForMaskedLM is a class representing a BERT model for masked language modeling. @@ -1774,18 +1828,18 @@ export class BertForQuestionAnswering extends BertPreTrainedModel { ////////////////////////////////////////////////// // NomicBert models -export class NomicBertPreTrainedModel extends PreTrainedModel { } -export class NomicBertModel extends NomicBertPreTrainedModel { } +export class NomicBertPreTrainedModel extends PreTrainedModel {} +export class NomicBertModel extends NomicBertPreTrainedModel {} ////////////////////////////////////////////////// ////////////////////////////////////////////////// // RoFormer models -export class RoFormerPreTrainedModel extends PreTrainedModel { } +export class RoFormerPreTrainedModel extends PreTrainedModel {} /** * The bare RoFormer Model transformer outputting raw hidden-states without any specific head on top. */ -export class RoFormerModel extends RoFormerPreTrainedModel { } +export class RoFormerModel extends RoFormerPreTrainedModel {} /** * RoFormer Model with a `language modeling` head on top. @@ -1853,12 +1907,12 @@ export class RoFormerForQuestionAnswering extends RoFormerPreTrainedModel { ////////////////////////////////////////////////// // ConvBert models -export class ConvBertPreTrainedModel extends PreTrainedModel { } +export class ConvBertPreTrainedModel extends PreTrainedModel {} /** * The bare ConvBERT Model transformer outputting raw hidden-states without any specific head on top. */ -export class ConvBertModel extends ConvBertPreTrainedModel { } +export class ConvBertModel extends ConvBertPreTrainedModel {} /** * ConvBERT Model with a language modeling head on top. @@ -1923,17 +1977,16 @@ export class ConvBertForQuestionAnswering extends ConvBertPreTrainedModel { } ////////////////////////////////////////////////// - ////////////////////////////////////////////////// // Electra models -export class ElectraPreTrainedModel extends PreTrainedModel { } +export class ElectraPreTrainedModel extends PreTrainedModel {} /** * The bare Electra Model transformer outputting raw hidden-states without any specific head on top. * Identical to the BERT model except that it uses an additional linear layer between the embedding * layer and the encoder if the hidden size and embedding size are different. */ -export class ElectraModel extends ElectraPreTrainedModel { } +export class ElectraModel extends ElectraPreTrainedModel {} // TODO add ElectraForPreTraining /** * Electra model with a language modeling head on top. @@ -1997,15 +2050,14 @@ export class ElectraForQuestionAnswering extends ElectraPreTrainedModel { } ////////////////////////////////////////////////// - ////////////////////////////////////////////////// // CamemBERT models -export class CamembertPreTrainedModel extends PreTrainedModel { } +export class CamembertPreTrainedModel extends PreTrainedModel {} /** * The bare CamemBERT Model transformer outputting raw hidden-states without any specific head on top. */ -export class CamembertModel extends CamembertPreTrainedModel { } +export class CamembertModel extends CamembertPreTrainedModel {} /** * CamemBERT Model with a `language modeling` head on top. @@ -2070,12 +2122,12 @@ export class CamembertForQuestionAnswering extends CamembertPreTrainedModel { ////////////////////////////////////////////////// // DeBERTa models -export class DebertaPreTrainedModel extends PreTrainedModel { } +export class DebertaPreTrainedModel extends PreTrainedModel {} /** * The bare DeBERTa Model transformer outputting raw hidden-states without any specific head on top. */ -export class DebertaModel extends DebertaPreTrainedModel { } +export class DebertaModel extends DebertaPreTrainedModel {} /** * DeBERTa Model with a `language modeling` head on top. @@ -2141,12 +2193,12 @@ export class DebertaForQuestionAnswering extends DebertaPreTrainedModel { ////////////////////////////////////////////////// // DeBERTa-v2 models -export class DebertaV2PreTrainedModel extends PreTrainedModel { } +export class DebertaV2PreTrainedModel extends PreTrainedModel {} /** * The bare DeBERTa-V2 Model transformer outputting raw hidden-states without any specific head on top. */ -export class DebertaV2Model extends DebertaV2PreTrainedModel { } +export class DebertaV2Model extends DebertaV2PreTrainedModel {} /** * DeBERTa-V2 Model with a `language modeling` head on top. @@ -2212,8 +2264,8 @@ export class DebertaV2ForQuestionAnswering extends DebertaV2PreTrainedModel { ////////////////////////////////////////////////// // DistilBert models -export class DistilBertPreTrainedModel extends PreTrainedModel { } -export class DistilBertModel extends DistilBertPreTrainedModel { } +export class DistilBertPreTrainedModel extends PreTrainedModel {} +export class DistilBertModel extends DistilBertPreTrainedModel {} /** * DistilBertForSequenceClassification is a class representing a DistilBERT model for sequence classification. @@ -2245,7 +2297,6 @@ export class DistilBertForTokenClassification extends DistilBertPreTrainedModel } } - /** * DistilBertForQuestionAnswering is a class representing a DistilBERT model for question answering. */ @@ -2277,15 +2328,14 @@ export class DistilBertForMaskedLM extends DistilBertPreTrainedModel { } ////////////////////////////////////////////////// - ////////////////////////////////////////////////// // ESM models -export class EsmPreTrainedModel extends PreTrainedModel { } +export class EsmPreTrainedModel extends PreTrainedModel {} /** * The bare ESM Model transformer outputting raw hidden-states without any specific head on top. */ -export class EsmModel extends EsmPreTrainedModel { } +export class EsmModel extends EsmPreTrainedModel {} /** * ESM Model with a `language modeling` head on top. @@ -2334,11 +2384,10 @@ export class EsmForTokenClassification extends EsmPreTrainedModel { } ////////////////////////////////////////////////// - ////////////////////////////////////////////////// // MobileBert models -export class MobileBertPreTrainedModel extends PreTrainedModel { } -export class MobileBertModel extends MobileBertPreTrainedModel { } +export class MobileBertPreTrainedModel extends PreTrainedModel {} +export class MobileBertModel extends MobileBertPreTrainedModel {} /** * MobileBertForMaskedLM is a class representing a MobileBERT model for masking task. @@ -2388,12 +2437,12 @@ export class MobileBertForQuestionAnswering extends MobileBertPreTrainedModel { ////////////////////////////////////////////////// // MPNet models -export class MPNetPreTrainedModel extends PreTrainedModel { } +export class MPNetPreTrainedModel extends PreTrainedModel {} /** * The bare MPNet Model transformer outputting raw hidden-states without any specific head on top. */ -export class MPNetModel extends MPNetPreTrainedModel { } +export class MPNetModel extends MPNetPreTrainedModel {} /** * MPNetForMaskedLM is a class representing a MPNet model for masked language modeling. @@ -2456,11 +2505,10 @@ export class MPNetForQuestionAnswering extends MPNetPreTrainedModel { } ////////////////////////////////////////////////// - ////////////////////////////////////////////////// // SqueezeBert models -export class SqueezeBertPreTrainedModel extends PreTrainedModel { } -export class SqueezeBertModel extends SqueezeBertPreTrainedModel { } +export class SqueezeBertPreTrainedModel extends PreTrainedModel {} +export class SqueezeBertModel extends SqueezeBertPreTrainedModel {} export class SqueezeBertForMaskedLM extends SqueezeBertPreTrainedModel { /** * Calls the model on new inputs. @@ -2496,11 +2544,10 @@ export class SqueezeBertForQuestionAnswering extends SqueezeBertPreTrainedModel } ////////////////////////////////////////////////// - ////////////////////////////////////////////////// // Albert models -export class AlbertPreTrainedModel extends PreTrainedModel { } -export class AlbertModel extends AlbertPreTrainedModel { } +export class AlbertPreTrainedModel extends PreTrainedModel {} +export class AlbertModel extends AlbertPreTrainedModel {} export class AlbertForSequenceClassification extends AlbertPreTrainedModel { /** * Calls the model on new inputs. @@ -2536,73 +2583,70 @@ export class AlbertForMaskedLM extends AlbertPreTrainedModel { } ////////////////////////////////////////////////// - ////////////////////////////////////////////////// // T5 models export class T5PreTrainedModel extends PreTrainedModel { forward_params = [ - 'input_ids', - 'attention_mask', - 'encoder_outputs', - 'decoder_input_ids', - 'decoder_attention_mask', - 'past_key_values', + "input_ids", + "attention_mask", + "encoder_outputs", + "decoder_input_ids", + "decoder_attention_mask", + "past_key_values", ]; -}; +} -export class T5Model extends T5PreTrainedModel { } +export class T5Model extends T5PreTrainedModel {} /** * T5Model is a class representing a T5 model for conditional generation. */ -export class T5ForConditionalGeneration extends T5PreTrainedModel { } +export class T5ForConditionalGeneration extends T5PreTrainedModel {} ////////////////////////////////////////////////// - ////////////////////////////////////////////////// // LONGT5 models /** * An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. */ -export class LongT5PreTrainedModel extends PreTrainedModel { }; +export class LongT5PreTrainedModel extends PreTrainedModel {} /** * The bare LONGT5 Model transformer outputting raw hidden-states without any specific head on top. */ -export class LongT5Model extends LongT5PreTrainedModel { } +export class LongT5Model extends LongT5PreTrainedModel {} /** * LONGT5 Model with a `language modeling` head on top. */ -export class LongT5ForConditionalGeneration extends LongT5PreTrainedModel { } +export class LongT5ForConditionalGeneration extends LongT5PreTrainedModel {} ////////////////////////////////////////////////// - ////////////////////////////////////////////////// // MT5 models -export class MT5PreTrainedModel extends PreTrainedModel { }; +export class MT5PreTrainedModel extends PreTrainedModel {} -export class MT5Model extends MT5PreTrainedModel { } +export class MT5Model extends MT5PreTrainedModel {} /** * A class representing a conditional sequence-to-sequence model based on the MT5 architecture. */ -export class MT5ForConditionalGeneration extends MT5PreTrainedModel { } +export class MT5ForConditionalGeneration extends MT5PreTrainedModel {} ////////////////////////////////////////////////// ////////////////////////////////////////////////// // Bart models -export class BartPretrainedModel extends PreTrainedModel { }; +export class BartPretrainedModel extends PreTrainedModel {} /** * The bare BART Model outputting raw hidden-states without any specific head on top. */ -export class BartModel extends BartPretrainedModel { } +export class BartModel extends BartPretrainedModel {} /** * The BART Model with a language modeling head. Can be used for summarization. */ -export class BartForConditionalGeneration extends BartPretrainedModel { } +export class BartForConditionalGeneration extends BartPretrainedModel {} /** * Bart model with a sequence classification/head on top (a linear layer on top of the pooled output) @@ -2623,17 +2667,17 @@ export class BartForSequenceClassification extends BartPretrainedModel { ////////////////////////////////////////////////// // MBart models -export class MBartPreTrainedModel extends PreTrainedModel { }; +export class MBartPreTrainedModel extends PreTrainedModel {} /** * The bare MBART Model outputting raw hidden-states without any specific head on top. */ -export class MBartModel extends MBartPreTrainedModel { } +export class MBartModel extends MBartPreTrainedModel {} /** * The MBART Model with a language modeling head. Can be used for summarization, after fine-tuning the pretrained models. */ -export class MBartForConditionalGeneration extends MBartPreTrainedModel { } +export class MBartForConditionalGeneration extends MBartPreTrainedModel {} /** * MBart model with a sequence classification/head on top (a linear layer on top of the pooled output). @@ -2650,47 +2694,43 @@ export class MBartForSequenceClassification extends MBartPreTrainedModel { } } - -export class MBartForCausalLM extends MBartPreTrainedModel { } +export class MBartForCausalLM extends MBartPreTrainedModel {} ////////////////////////////////////////////////// - ////////////////////////////////////////////////// // Blenderbot models -export class BlenderbotPreTrainedModel extends PreTrainedModel { }; +export class BlenderbotPreTrainedModel extends PreTrainedModel {} /** * The bare Blenderbot Model outputting raw hidden-states without any specific head on top. */ -export class BlenderbotModel extends BlenderbotPreTrainedModel { } +export class BlenderbotModel extends BlenderbotPreTrainedModel {} /** * The Blenderbot Model with a language modeling head. Can be used for summarization. */ -export class BlenderbotForConditionalGeneration extends BlenderbotPreTrainedModel { } +export class BlenderbotForConditionalGeneration extends BlenderbotPreTrainedModel {} ////////////////////////////////////////////////// - ////////////////////////////////////////////////// // Blenderbot models -export class BlenderbotSmallPreTrainedModel extends PreTrainedModel { }; +export class BlenderbotSmallPreTrainedModel extends PreTrainedModel {} /** * The bare BlenderbotSmall Model outputting raw hidden-states without any specific head on top. */ -export class BlenderbotSmallModel extends BlenderbotSmallPreTrainedModel { } +export class BlenderbotSmallModel extends BlenderbotSmallPreTrainedModel {} /** * The BlenderbotSmall Model with a language modeling head. Can be used for summarization. */ -export class BlenderbotSmallForConditionalGeneration extends BlenderbotSmallPreTrainedModel { } +export class BlenderbotSmallForConditionalGeneration extends BlenderbotSmallPreTrainedModel {} ////////////////////////////////////////////////// - ////////////////////////////////////////////////// // Roberta models -export class RobertaPreTrainedModel extends PreTrainedModel { } -export class RobertaModel extends RobertaPreTrainedModel { } +export class RobertaPreTrainedModel extends PreTrainedModel {} +export class RobertaModel extends RobertaPreTrainedModel {} /** * RobertaForMaskedLM class for performing masked language modeling on Roberta models. @@ -2753,18 +2793,17 @@ export class RobertaForQuestionAnswering extends RobertaPreTrainedModel { } ////////////////////////////////////////////////// - ////////////////////////////////////////////////// // XLM models /** * An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. */ -export class XLMPreTrainedModel extends PreTrainedModel { } +export class XLMPreTrainedModel extends PreTrainedModel {} /** * The bare XLM Model transformer outputting raw hidden-states without any specific head on top. */ -export class XLMModel extends XLMPreTrainedModel { } +export class XLMModel extends XLMPreTrainedModel {} /** * The XLM Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). @@ -2829,8 +2868,8 @@ export class XLMForQuestionAnswering extends XLMPreTrainedModel { ////////////////////////////////////////////////// // XLMRoberta models -export class XLMRobertaPreTrainedModel extends PreTrainedModel { } -export class XLMRobertaModel extends XLMRobertaPreTrainedModel { } +export class XLMRobertaPreTrainedModel extends PreTrainedModel {} +export class XLMRobertaModel extends XLMRobertaPreTrainedModel {} /** * XLMRobertaForMaskedLM class for performing masked language modeling on XLMRoberta models. @@ -2895,56 +2934,55 @@ export class XLMRobertaForQuestionAnswering extends XLMRobertaPreTrainedModel { ////////////////////////////////////////////////// // Audio Spectrogram Transformer (AST) models -export class ASTPreTrainedModel extends PreTrainedModel { }; +export class ASTPreTrainedModel extends PreTrainedModel {} /** * The bare AST Model transformer outputting raw hidden-states without any specific head on top. */ -export class ASTModel extends ASTPreTrainedModel { } +export class ASTModel extends ASTPreTrainedModel {} /** * Audio Spectrogram Transformer model with an audio classification head on top * (a linear layer on top of the pooled output) e.g. for datasets like AudioSet, Speech Commands v2. */ -export class ASTForAudioClassification extends ASTPreTrainedModel { } +export class ASTForAudioClassification extends ASTPreTrainedModel {} ////////////////////////////////////////////////// ////////////////////////////////////////////////// // Whisper models export class WhisperPreTrainedModel extends PreTrainedModel { - requires_attention_mask = false; - main_input_name = 'input_features'; + main_input_name = "input_features"; forward_params = [ - 'input_features', - 'attention_mask', - 'decoder_input_ids', - 'decoder_attention_mask', - 'past_key_values', + "input_features", + "attention_mask", + "decoder_input_ids", + "decoder_attention_mask", + "past_key_values", ]; -}; +} /** * WhisperModel class for training Whisper models without a language model head. */ -export class WhisperModel extends WhisperPreTrainedModel { } - +export class WhisperModel extends WhisperPreTrainedModel {} /** * WhisperForConditionalGeneration class for generating conditional outputs from Whisper models. */ export class WhisperForConditionalGeneration extends WhisperPreTrainedModel { - _prepare_generation_config(generation_config, kwargs) { - return /** @type {WhisperGenerationConfig} */ (super._prepare_generation_config(generation_config, kwargs, WhisperGenerationConfig)); + return /** @type {WhisperGenerationConfig} */ ( + super._prepare_generation_config(generation_config, kwargs, WhisperGenerationConfig) + ); } /** - * - * @param {WhisperGenerationConfig} generation_config + * + * @param {WhisperGenerationConfig} generation_config */ _retrieve_init_tokens(generation_config) { - // prefix tokens are of the form: + // prefix tokens are of the form: // - Multilingual: <|startoftranscript|> <|lang_id|> <|task|> [<|notimestamps|>] // - English-only: <|startoftranscript|> [<|notimestamps|>] @@ -2957,43 +2995,43 @@ export class WhisperForConditionalGeneration extends WhisperPreTrainedModel { if (generation_config.is_multilingual) { if (!language) { // TODO: Implement language detection - console.warn('No language specified - defaulting to English (en).'); - language = 'en'; + console.warn("No language specified - defaulting to English (en)."); + language = "en"; } // Add language token const language_code = whisper_language_to_code(language); const language_token = `<|${language_code}|>`; - init_tokens.push(generation_config.lang_to_id[language_token]) + init_tokens.push(generation_config.lang_to_id[language_token]); // Add task token // NOTE: Defaults to 'transcribe' if no task is specified - init_tokens.push(generation_config.task_to_id[task ?? 'transcribe']); - + init_tokens.push(generation_config.task_to_id[task ?? "transcribe"]); } else if (language || task) { throw new Error( - "Cannot specify `task` or `language` for an English-only model. If the model is intended to be multilingual, pass `is_multilingual=true` to generate, or update the generation config." - ) + "Cannot specify `task` or `language` for an English-only model. If the model is intended to be multilingual, pass `is_multilingual=true` to generate, or update the generation config.", + ); } // 3. Handle <|notimestamps|> token if ( - !generation_config.return_timestamps - && generation_config.no_timestamps_token_id - && init_tokens.at(-1) !== generation_config.no_timestamps_token_id + !generation_config.return_timestamps && + generation_config.no_timestamps_token_id && + init_tokens.at(-1) !== generation_config.no_timestamps_token_id ) { init_tokens.push(generation_config.no_timestamps_token_id); } else if ( - generation_config.return_timestamps - && + generation_config.return_timestamps && init_tokens.at(-1) === generation_config.no_timestamps_token_id ) { - console.warn("<|notimestamps|> prompt token is removed from generation_config since `return_timestamps` is set to `true`."); + console.warn( + "<|notimestamps|> prompt token is removed from generation_config since `return_timestamps` is set to `true`.", + ); init_tokens.pop(); } // let's make sure we don't pass `null` tokens as prompt tokens - return init_tokens.filter(token => token != null); + return init_tokens.filter((token) => token != null); } /** @@ -3020,15 +3058,13 @@ export class WhisperForConditionalGeneration extends WhisperPreTrainedModel { if (generation_config.return_timestamps) { logits_processor ??= new LogitsProcessorList(); - logits_processor.push( - new WhisperTimeStampLogitsProcessor(generation_config, init_tokens) - ); + logits_processor.push(new WhisperTimeStampLogitsProcessor(generation_config, init_tokens)); } if (generation_config.begin_suppress_tokens) { logits_processor ??= new LogitsProcessorList(); logits_processor.push( - new SuppressTokensAtBeginLogitsProcessor(generation_config.begin_suppress_tokens, init_tokens.length) + new SuppressTokensAtBeginLogitsProcessor(generation_config.begin_suppress_tokens, init_tokens.length), ); } @@ -3036,12 +3072,12 @@ export class WhisperForConditionalGeneration extends WhisperPreTrainedModel { if (!generation_config.alignment_heads) { throw new Error( "Model generation config has no `alignment_heads`, token-level timestamps not available. " + - "See https://gist.github.com/hollance/42e32852f24243b748ae6bc1f985b13a on how to add this property to the generation config." - ) + "See https://gist.github.com/hollance/42e32852f24243b748ae6bc1f985b13a on how to add this property to the generation config.", + ); } - if (generation_config.task === 'translate') { - console.warn("Token-level timestamps may not be reliable for task 'translate'.") + if (generation_config.task === "translate") { + console.warn("Token-level timestamps may not be reliable for task 'translate'."); } generation_config.output_attentions = true; @@ -3053,7 +3089,7 @@ export class WhisperForConditionalGeneration extends WhisperPreTrainedModel { generation_config, logits_processor, decoder_input_ids: init_tokens, - ...kwargs + ...kwargs, }); if (generation_config.return_token_timestamps) { @@ -3083,19 +3119,19 @@ export class WhisperForConditionalGeneration extends WhisperPreTrainedModel { if (!generate_outputs.cross_attentions) { throw new Error( "Model outputs must contain cross attentions to extract timestamps. " + - "This is most likely because the model was not exported with `output_attentions=True`." - ) + "This is most likely because the model was not exported with `output_attentions=True`.", + ); } if (num_frames == null) { console.warn( "`num_frames` has not been set, meaning the entire audio will be analyzed. " + - "This may lead to inaccurate token-level timestamps for short audios (< 30 seconds)." + "This may lead to inaccurate token-level timestamps for short audios (< 30 seconds).", ); } let median_filter_width = this.config.median_filter_width; if (median_filter_width === undefined) { - console.warn("Model config has no `median_filter_width`, using default value of 7.") + console.warn("Model config has no `median_filter_width`, using default value of 7."); median_filter_width = 7; } @@ -3103,19 +3139,28 @@ export class WhisperForConditionalGeneration extends WhisperPreTrainedModel { const batch = generate_outputs.cross_attentions; // Create a list with `decoder_layers` elements, each a tensor of shape // (batch size, attention_heads, output length, input length). - const cross_attentions = Array.from({ length: this.config.decoder_layers }, + const cross_attentions = Array.from( + { length: this.config.decoder_layers }, // Concatenate the cross attentions for each layer across sequence length dimension. - (_, i) => cat(batch.map(x => x[i]), 2) + (_, i) => + cat( + batch.map((x) => x[i]), + 2, + ), ); - const weights = stack(alignment_heads.map(([l, h]) => { - if (l >= cross_attentions.length) { - throw new Error(`Layer index ${l} is out of bounds for cross attentions (length ${cross_attentions.length}).`) - } - return num_frames - ? cross_attentions[l].slice(null, h, null, [0, num_frames]) - : cross_attentions[l].slice(null, h); - })).transpose(1, 0, 2, 3); + const weights = stack( + alignment_heads.map(([l, h]) => { + if (l >= cross_attentions.length) { + throw new Error( + `Layer index ${l} is out of bounds for cross attentions (length ${cross_attentions.length}).`, + ); + } + return num_frames + ? cross_attentions[l].slice(null, h, null, [0, num_frames]) + : cross_attentions[l].slice(null, h); + }), + ).transpose(1, 0, 2, 3); const [std, calculatedMean] = std_mean(weights, -2, 0, true); @@ -3132,14 +3177,13 @@ export class WhisperForConditionalGeneration extends WhisperPreTrainedModel { const meanTensorData = calculatedMean[a][b][0].data; // [1500] for (let c = 0; c < bTensor.dims[0]; ++c) { - let cTensorData = bTensor[c].data; // [1500] for (let d = 0; d < cTensorData.length; ++d) { - cTensorData[d] = (cTensorData[d] - meanTensorData[d]) / stdTensorData[d] + cTensorData[d] = (cTensorData[d] - meanTensorData[d]) / stdTensorData[d]; } // Apply median filter. - cTensorData.set(medianFilter(cTensorData, median_filter_width)) + cTensorData.set(medianFilter(cTensorData, median_filter_width)); } } } @@ -3150,9 +3194,9 @@ export class WhisperForConditionalGeneration extends WhisperPreTrainedModel { const timestampsShape = generate_outputs.sequences.dims; const timestamps = new Tensor( - 'float32', + "float32", new Float32Array(timestampsShape[0] * timestampsShape[1]), - timestampsShape + timestampsShape, ); // Perform dynamic time warping on each element of the batch. @@ -3162,8 +3206,11 @@ export class WhisperForConditionalGeneration extends WhisperPreTrainedModel { const matrix = batchedMatrices[batch_idx].neg().squeeze_(0); const [text_indices, time_indices] = dynamic_time_warping(matrix.tolist()); - const diffs = Array.from({ length: text_indices.length - 1 }, (v, i) => text_indices[i + 1] - text_indices[i]); - const jumps = mergeArrays([1], diffs).map(x => !!x); // convert to boolean + const diffs = Array.from( + { length: text_indices.length - 1 }, + (v, i) => text_indices[i + 1] - text_indices[i], + ); + const jumps = mergeArrays([1], diffs).map((x) => !!x); // convert to boolean const jump_times = []; for (let i = 0; i < jumps.length; ++i) { @@ -3172,7 +3219,7 @@ export class WhisperForConditionalGeneration extends WhisperPreTrainedModel { jump_times.push(time_indices[i] * time_precision); } } - timestamps[batch_idx].data.set(jump_times, 1) + timestamps[batch_idx].data.set(jump_times, 1); } return timestamps; @@ -3185,63 +3232,49 @@ export class WhisperForConditionalGeneration extends WhisperPreTrainedModel { * Vision Encoder-Decoder model based on OpenAI's GPT architecture for image captioning and other vision tasks */ export class VisionEncoderDecoderModel extends PreTrainedModel { - main_input_name = 'pixel_values'; + main_input_name = "pixel_values"; forward_params = [ // Encoder inputs - 'pixel_values', + "pixel_values", // Decoder inpputs - 'decoder_input_ids', - 'encoder_hidden_states', - 'past_key_values', + "decoder_input_ids", + "encoder_hidden_states", + "past_key_values", ]; } ////////////////////////////////////////////////// - ////////////////////////////////////////////////// // LLaVa Models export class LlavaPreTrainedModel extends PreTrainedModel { - forward_params = [ - 'input_ids', - 'pixel_values', - 'attention_mask', - 'position_ids', - 'past_key_values', - ]; + forward_params = ["input_ids", "pixel_values", "attention_mask", "position_ids", "past_key_values"]; } /** * The LLAVA model which consists of a vision backbone and a language model. */ export class LlavaForConditionalGeneration extends LlavaPreTrainedModel { - - _merge_input_ids_with_image_features({ - inputs_embeds, - image_features, - input_ids, - attention_mask, - }) { - + _merge_input_ids_with_image_features({ inputs_embeds, image_features, input_ids, attention_mask }) { const image_token_index = this.config.image_token_index; const idsList = input_ids.tolist(); // NOTE: we use .findIndex instead of .indexOf to perform weak comparison (==) between BigInt and Number - const indexOfImage = idsList.map(x => x.findIndex(x => x == image_token_index)); + const indexOfImage = idsList.map((x) => x.findIndex((x) => x == image_token_index)); - const noImages = indexOfImage.every(x => x === -1); - const allImages = indexOfImage.every(x => x !== -1); + const noImages = indexOfImage.every((x) => x === -1); + const allImages = indexOfImage.every((x) => x !== -1); if (!noImages && !allImages) { // Check for padding reasons - throw new Error('Every input should contain either 0 or 1 image token.'); + throw new Error("Every input should contain either 0 or 1 image token."); } if (noImages) { return { inputs_embeds, attention_mask, - } + }; } const stacked = []; @@ -3252,74 +3285,64 @@ export class LlavaForConditionalGeneration extends LlavaPreTrainedModel { const e = inputs_embeds[i]; const im = image_features[i]; const am = attention_mask[i]; - stacked.push( - cat([ - e.slice([0, index]), - im, - e.slice([index + 1, e.dims[0]]), - ], 0) - ); + stacked.push(cat([e.slice([0, index]), im, e.slice([index + 1, e.dims[0]])], 0)); stacked_attention_mask.push( - cat([ - am.slice([0, index]), - ones([im.dims[0]]), - am.slice([index + 1, am.dims[0]]) - ], 0) - ) + cat([am.slice([0, index]), ones([im.dims[0]]), am.slice([index + 1, am.dims[0]])], 0), + ); } return { inputs_embeds: stack(stacked, 0), attention_mask: stack(stacked_attention_mask, 0), - } + }; } } ////////////////////////////////////////////////// -export class Moondream1ForConditionalGeneration extends LlavaForConditionalGeneration { } // NOTE: extends LlavaForConditionalGeneration +export class Moondream1ForConditionalGeneration extends LlavaForConditionalGeneration {} // NOTE: extends LlavaForConditionalGeneration export class Florence2PreTrainedModel extends PreTrainedModel { forward_params = [ // Encoder inputs - 'input_ids', - 'inputs_embeds', - 'attention_mask', - 'pixel_values', + "input_ids", + "inputs_embeds", + "attention_mask", + "pixel_values", // Decoder inputs - 'encoder_outputs', - 'decoder_input_ids', - 'decoder_inputs_embeds', - 'decoder_attention_mask', - 'past_key_values', + "encoder_outputs", + "decoder_input_ids", + "decoder_inputs_embeds", + "decoder_attention_mask", + "past_key_values", ]; - main_input_name = 'inputs_embeds'; + main_input_name = "inputs_embeds"; } export class Florence2ForConditionalGeneration extends Florence2PreTrainedModel { - - _merge_input_ids_with_image_features({ - inputs_embeds, - image_features, - input_ids, - attention_mask, - }) { + _merge_input_ids_with_image_features({ inputs_embeds, image_features, input_ids, attention_mask }) { return { - inputs_embeds: cat([ - image_features, // image embeds - inputs_embeds, // task prefix embeds - ], 1), - attention_mask: cat([ - ones(image_features.dims.slice(0, 2)), // image attention mask - attention_mask, // task prefix attention mask - ], 1), - } + inputs_embeds: cat( + [ + image_features, // image embeds + inputs_embeds, // task prefix embeds + ], + 1, + ), + attention_mask: cat( + [ + ones(image_features.dims.slice(0, 2)), // image attention mask + attention_mask, // task prefix attention mask + ], + 1, + ), + }; } async _prepare_inputs_embeds({ input_ids, pixel_values, inputs_embeds, attention_mask }) { if (!input_ids && !pixel_values) { - throw new Error('Either `input_ids` or `pixel_values` should be provided.'); + throw new Error("Either `input_ids` or `pixel_values` should be provided."); } // 1. Possibly, extract the input embeddings @@ -3359,7 +3382,12 @@ export class Florence2ForConditionalGeneration extends Florence2PreTrainedModel decoder_inputs_embeds, }) { if (!inputs_embeds) { - ({ inputs_embeds, attention_mask } = await this._prepare_inputs_embeds({ input_ids, pixel_values, inputs_embeds, attention_mask })); + ({ inputs_embeds, attention_mask } = await this._prepare_inputs_embeds({ + input_ids, + pixel_values, + inputs_embeds, + attention_mask, + })); } if (!encoder_outputs) { @@ -3370,7 +3398,7 @@ export class Florence2ForConditionalGeneration extends Florence2PreTrainedModel if (!decoder_inputs_embeds) { if (!decoder_input_ids) { - throw new Error('Either `decoder_input_ids` or `decoder_inputs_embeds` should be provided.'); + throw new Error("Either `decoder_input_ids` or `decoder_inputs_embeds` should be provided."); } decoder_inputs_embeds = await this.encode_text({ input_ids: decoder_input_ids }); } @@ -3386,29 +3414,29 @@ export class Florence2ForConditionalGeneration extends Florence2PreTrainedModel return decoder_outputs; } } -export class CLIPPreTrainedModel extends PreTrainedModel { } +export class CLIPPreTrainedModel extends PreTrainedModel {} /** * CLIP Text and Vision Model with a projection layers on top - * + * * **Example:** Perform zero-shot image classification with a `CLIPModel`. - * + * * ```javascript * import { AutoTokenizer, AutoProcessor, CLIPModel, RawImage } from '@huggingface/transformers'; - * + * * // Load tokenizer, processor, and model * let tokenizer = await AutoTokenizer.from_pretrained('Xenova/clip-vit-base-patch16'); * let processor = await AutoProcessor.from_pretrained('Xenova/clip-vit-base-patch16'); * let model = await CLIPModel.from_pretrained('Xenova/clip-vit-base-patch16'); - * + * * // Run tokenization * let texts = ['a photo of a car', 'a photo of a football match'] * let text_inputs = tokenizer(texts, { padding: true, truncation: true }); - * + * * // Read image and run processor * let image = await RawImage.read('https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/football-match.jpg'); * let image_inputs = await processor(image); - * + * * // Run model with both text and pixel inputs * let output = await model({ ...text_inputs, ...image_inputs }); * // { @@ -3431,7 +3459,7 @@ export class CLIPPreTrainedModel extends PreTrainedModel { } * // } * ``` */ -export class CLIPModel extends CLIPPreTrainedModel { } +export class CLIPModel extends CLIPPreTrainedModel {} /** * The text model from CLIP without any head or projection on top. @@ -3440,27 +3468,27 @@ export class CLIPTextModel extends CLIPPreTrainedModel { /** @type {PreTrainedModel.from_pretrained} */ static async from_pretrained(pretrained_model_name_or_path, options = {}) { // Update default model file name if not provided - options.model_file_name ??= 'text_model'; + options.model_file_name ??= "text_model"; return super.from_pretrained(pretrained_model_name_or_path, options); } } /** * CLIP Text Model with a projection layer on top (a linear layer on top of the pooled output) - * + * * **Example:** Compute text embeddings with `CLIPTextModelWithProjection`. - * + * * ```javascript * import { AutoTokenizer, CLIPTextModelWithProjection } from '@huggingface/transformers'; - * + * * // Load tokenizer and text model * const tokenizer = await AutoTokenizer.from_pretrained('Xenova/clip-vit-base-patch16'); * const text_model = await CLIPTextModelWithProjection.from_pretrained('Xenova/clip-vit-base-patch16'); - * + * * // Run tokenization * let texts = ['a photo of a car', 'a photo of a football match']; * let text_inputs = tokenizer(texts, { padding: true, truncation: true }); - * + * * // Compute embeddings * const { text_embeds } = await text_model(text_inputs); * // Tensor { @@ -3475,7 +3503,7 @@ export class CLIPTextModelWithProjection extends CLIPPreTrainedModel { /** @type {PreTrainedModel.from_pretrained} */ static async from_pretrained(pretrained_model_name_or_path, options = {}) { // Update default model file name if not provided - options.model_file_name ??= 'text_model'; + options.model_file_name ??= "text_model"; return super.from_pretrained(pretrained_model_name_or_path, options); } } @@ -3487,27 +3515,27 @@ export class CLIPVisionModel extends CLIPPreTrainedModel { /** @type {PreTrainedModel.from_pretrained} */ static async from_pretrained(pretrained_model_name_or_path, options = {}) { // Update default model file name if not provided - options.model_file_name ??= 'vision_model'; + options.model_file_name ??= "vision_model"; return super.from_pretrained(pretrained_model_name_or_path, options); } } /** * CLIP Vision Model with a projection layer on top (a linear layer on top of the pooled output) - * + * * **Example:** Compute vision embeddings with `CLIPVisionModelWithProjection`. - * + * * ```javascript * import { AutoProcessor, CLIPVisionModelWithProjection, RawImage} from '@huggingface/transformers'; - * + * * // Load processor and vision model * const processor = await AutoProcessor.from_pretrained('Xenova/clip-vit-base-patch16'); * const vision_model = await CLIPVisionModelWithProjection.from_pretrained('Xenova/clip-vit-base-patch16'); - * + * * // Read image and run processor * let image = await RawImage.read('https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/football-match.jpg'); * let image_inputs = await processor(image); - * + * * // Compute embeddings * const { image_embeds } = await vision_model(image_inputs); * // Tensor { @@ -3522,38 +3550,37 @@ export class CLIPVisionModelWithProjection extends CLIPPreTrainedModel { /** @type {PreTrainedModel.from_pretrained} */ static async from_pretrained(pretrained_model_name_or_path, options = {}) { // Update default model file name if not provided - options.model_file_name ??= 'vision_model'; + options.model_file_name ??= "vision_model"; return super.from_pretrained(pretrained_model_name_or_path, options); } } ////////////////////////////////////////////////// - ////////////////////////////////////////////////// // SigLIP models -export class SiglipPreTrainedModel extends PreTrainedModel { } +export class SiglipPreTrainedModel extends PreTrainedModel {} /** * SigLIP Text and Vision Model with a projection layers on top - * + * * **Example:** Perform zero-shot image classification with a `SiglipModel`. - * + * * ```javascript * import { AutoTokenizer, AutoProcessor, SiglipModel, RawImage } from '@huggingface/transformers'; - * + * * // Load tokenizer, processor, and model * const tokenizer = await AutoTokenizer.from_pretrained('Xenova/siglip-base-patch16-224'); * const processor = await AutoProcessor.from_pretrained('Xenova/siglip-base-patch16-224'); * const model = await SiglipModel.from_pretrained('Xenova/siglip-base-patch16-224'); - * + * * // Run tokenization * const texts = ['a photo of 2 cats', 'a photo of 2 dogs']; * const text_inputs = tokenizer(texts, { padding: 'max_length', truncation: true }); - * + * * // Read image and run processor * const image = await RawImage.read('http://images.cocodataset.org/val2017/000000039769.jpg'); * const image_inputs = await processor(image); - * + * * // Run model with both text and pixel inputs * const output = await model({ ...text_inputs, ...image_inputs }); * // { @@ -3576,24 +3603,24 @@ export class SiglipPreTrainedModel extends PreTrainedModel { } * // } * ``` */ -export class SiglipModel extends SiglipPreTrainedModel { } +export class SiglipModel extends SiglipPreTrainedModel {} /** * The text model from SigLIP without any head or projection on top. - * + * * **Example:** Compute text embeddings with `SiglipTextModel`. - * + * * ```javascript * import { AutoTokenizer, SiglipTextModel } from '@huggingface/transformers'; - * + * * // Load tokenizer and text model * const tokenizer = await AutoTokenizer.from_pretrained('Xenova/siglip-base-patch16-224'); * const text_model = await SiglipTextModel.from_pretrained('Xenova/siglip-base-patch16-224'); - * + * * // Run tokenization * const texts = ['a photo of 2 cats', 'a photo of 2 dogs']; * const text_inputs = tokenizer(texts, { padding: 'max_length', truncation: true }); - * + * * // Compute embeddings * const { pooler_output } = await text_model(text_inputs); * // Tensor { @@ -3605,31 +3632,30 @@ export class SiglipModel extends SiglipPreTrainedModel { } * ``` */ export class SiglipTextModel extends SiglipPreTrainedModel { - /** @type {PreTrainedModel.from_pretrained} */ static async from_pretrained(pretrained_model_name_or_path, options = {}) { // Update default model file name if not provided - options.model_file_name ??= 'text_model'; + options.model_file_name ??= "text_model"; return super.from_pretrained(pretrained_model_name_or_path, options); } } /** * The vision model from SigLIP without any head or projection on top. - * + * * **Example:** Compute vision embeddings with `SiglipVisionModel`. - * + * * ```javascript * import { AutoProcessor, SiglipVisionModel, RawImage} from '@huggingface/transformers'; - * + * * // Load processor and vision model * const processor = await AutoProcessor.from_pretrained('Xenova/siglip-base-patch16-224'); * const vision_model = await SiglipVisionModel.from_pretrained('Xenova/siglip-base-patch16-224'); - * + * * // Read image and run processor * const image = await RawImage.read('https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/football-match.jpg'); * const image_inputs = await processor(image); - * + * * // Compute embeddings * const { pooler_output } = await vision_model(image_inputs); * // Tensor { @@ -3644,45 +3670,44 @@ export class SiglipVisionModel extends CLIPPreTrainedModel { /** @type {PreTrainedModel.from_pretrained} */ static async from_pretrained(pretrained_model_name_or_path, options = {}) { // Update default model file name if not provided - options.model_file_name ??= 'vision_model'; + options.model_file_name ??= "vision_model"; return super.from_pretrained(pretrained_model_name_or_path, options); } } ////////////////////////////////////////////////// // ChineseCLIP models -export class ChineseCLIPPreTrainedModel extends PreTrainedModel { } +export class ChineseCLIPPreTrainedModel extends PreTrainedModel {} -export class ChineseCLIPModel extends ChineseCLIPPreTrainedModel { } +export class ChineseCLIPModel extends ChineseCLIPPreTrainedModel {} ////////////////////////////////////////////////// - ////////////////////////////////////////////////// // CLIPSeg models -export class CLIPSegPreTrainedModel extends PreTrainedModel { } +export class CLIPSegPreTrainedModel extends PreTrainedModel {} -export class CLIPSegModel extends CLIPSegPreTrainedModel { } +export class CLIPSegModel extends CLIPSegPreTrainedModel {} /** * CLIPSeg model with a Transformer-based decoder on top for zero-shot and one-shot image segmentation. - * + * * **Example:** Perform zero-shot image segmentation with a `CLIPSegForImageSegmentation` model. - * + * * ```javascript * import { AutoTokenizer, AutoProcessor, CLIPSegForImageSegmentation, RawImage } from '@huggingface/transformers'; - * + * * // Load tokenizer, processor, and model * const tokenizer = await AutoTokenizer.from_pretrained('Xenova/clipseg-rd64-refined'); * const processor = await AutoProcessor.from_pretrained('Xenova/clipseg-rd64-refined'); * const model = await CLIPSegForImageSegmentation.from_pretrained('Xenova/clipseg-rd64-refined'); - * + * * // Run tokenization * const texts = ['a glass', 'something to fill', 'wood', 'a jar']; * const text_inputs = tokenizer(texts, { padding: true, truncation: true }); - * + * * // Read image and run processor * const image = await RawImage.read('https://github.com/timojl/clipseg/blob/master/example_image.jpg?raw=true'); * const image_inputs = await processor(image); - * + * * // Run model with both text and pixel inputs * const { logits } = await model({ ...text_inputs, ...image_inputs }); * // logits: Tensor { @@ -3692,7 +3717,7 @@ export class CLIPSegModel extends CLIPSegPreTrainedModel { } * // size: 495616 * // } * ``` - * + * * You can visualize the predictions as follows: * ```javascript * const preds = logits @@ -3701,27 +3726,26 @@ export class CLIPSegModel extends CLIPSegPreTrainedModel { } * .mul_(255) * .round_() * .to('uint8'); - * + * * for (let i = 0; i < preds.dims[0]; ++i) { * const img = RawImage.fromTensor(preds[i]); * img.save(`prediction_${i}.png`); * } * ``` */ -export class CLIPSegForImageSegmentation extends CLIPSegPreTrainedModel { } +export class CLIPSegForImageSegmentation extends CLIPSegPreTrainedModel {} ////////////////////////////////////////////////// - ////////////////////////////////////////////////// // GPT2 models -export class GPT2PreTrainedModel extends PreTrainedModel { } +export class GPT2PreTrainedModel extends PreTrainedModel {} -export class GPT2Model extends GPT2PreTrainedModel { } +export class GPT2Model extends GPT2PreTrainedModel {} /** * GPT-2 language model head on top of the GPT-2 base model. This model is suitable for text generation tasks. */ -export class GPT2LMHeadModel extends GPT2PreTrainedModel { } +export class GPT2LMHeadModel extends GPT2PreTrainedModel {} // export class GPT2ForSequenceClassification extends GPT2PreTrainedModel { // TODO // } @@ -3729,121 +3753,113 @@ export class GPT2LMHeadModel extends GPT2PreTrainedModel { } ////////////////////////////////////////////////// // JAIS models -export class JAISPreTrainedModel extends PreTrainedModel { } +export class JAISPreTrainedModel extends PreTrainedModel {} /** * The bare JAIS Model transformer outputting raw hidden-states without any specific head on top. */ -export class JAISModel extends JAISPreTrainedModel { } +export class JAISModel extends JAISPreTrainedModel {} /** * The JAIS Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). */ -export class JAISLMHeadModel extends JAISPreTrainedModel { } +export class JAISLMHeadModel extends JAISPreTrainedModel {} ////////////////////////////////////////////////// - ////////////////////////////////////////////////// // GPTNeo models -export class GPTNeoPreTrainedModel extends PreTrainedModel { } -export class GPTNeoModel extends GPTNeoPreTrainedModel { } +export class GPTNeoPreTrainedModel extends PreTrainedModel {} +export class GPTNeoModel extends GPTNeoPreTrainedModel {} -export class GPTNeoForCausalLM extends GPTNeoPreTrainedModel { } +export class GPTNeoForCausalLM extends GPTNeoPreTrainedModel {} ////////////////////////////////////////////////// ////////////////////////////////////////////////// // GPTNeoX models -export class GPTNeoXPreTrainedModel extends PreTrainedModel { } -export class GPTNeoXModel extends GPTNeoXPreTrainedModel { } +export class GPTNeoXPreTrainedModel extends PreTrainedModel {} +export class GPTNeoXModel extends GPTNeoXPreTrainedModel {} -export class GPTNeoXForCausalLM extends GPTNeoXPreTrainedModel { } +export class GPTNeoXForCausalLM extends GPTNeoXPreTrainedModel {} ////////////////////////////////////////////////// - ////////////////////////////////////////////////// // GPT-J models -export class GPTJPreTrainedModel extends PreTrainedModel { } +export class GPTJPreTrainedModel extends PreTrainedModel {} -export class GPTJModel extends GPTJPreTrainedModel { } +export class GPTJModel extends GPTJPreTrainedModel {} -export class GPTJForCausalLM extends GPTJPreTrainedModel { } +export class GPTJForCausalLM extends GPTJPreTrainedModel {} ////////////////////////////////////////////////// - ////////////////////////////////////////////////// // GPTBigCode models -export class GPTBigCodePreTrainedModel extends PreTrainedModel { } +export class GPTBigCodePreTrainedModel extends PreTrainedModel {} -export class GPTBigCodeModel extends GPTBigCodePreTrainedModel { } +export class GPTBigCodeModel extends GPTBigCodePreTrainedModel {} -export class GPTBigCodeForCausalLM extends GPTBigCodePreTrainedModel { } +export class GPTBigCodeForCausalLM extends GPTBigCodePreTrainedModel {} ////////////////////////////////////////////////// ////////////////////////////////////////////////// // CodeGen models -export class CodeGenPreTrainedModel extends PreTrainedModel { } +export class CodeGenPreTrainedModel extends PreTrainedModel {} /** * CodeGenModel is a class representing a code generation model without a language model head. */ -export class CodeGenModel extends CodeGenPreTrainedModel { } +export class CodeGenModel extends CodeGenPreTrainedModel {} /** * CodeGenForCausalLM is a class that represents a code generation model based on the GPT-2 architecture. It extends the `CodeGenPreTrainedModel` class. */ -export class CodeGenForCausalLM extends CodeGenPreTrainedModel { } +export class CodeGenForCausalLM extends CodeGenPreTrainedModel {} ////////////////////////////////////////////////// - ////////////////////////////////////////////////// // LLama models /** * The bare LLama Model outputting raw hidden-states without any specific head on top. */ -export class LlamaPreTrainedModel extends PreTrainedModel { } +export class LlamaPreTrainedModel extends PreTrainedModel {} /** * The bare LLaMA Model outputting raw hidden-states without any specific head on top. */ -export class LlamaModel extends LlamaPreTrainedModel { } +export class LlamaModel extends LlamaPreTrainedModel {} -export class LlamaForCausalLM extends LlamaPreTrainedModel { } +export class LlamaForCausalLM extends LlamaPreTrainedModel {} ////////////////////////////////////////////////// - ////////////////////////////////////////////////// // MobileLLM models -export class MobileLLMPreTrainedModel extends PreTrainedModel { } -export class MobileLLMModel extends MobileLLMPreTrainedModel { } -export class MobileLLMForCausalLM extends MobileLLMPreTrainedModel { } +export class MobileLLMPreTrainedModel extends PreTrainedModel {} +export class MobileLLMModel extends MobileLLMPreTrainedModel {} +export class MobileLLMForCausalLM extends MobileLLMPreTrainedModel {} ////////////////////////////////////////////////// - ////////////////////////////////////////////////// // OLMo models -export class OlmoPreTrainedModel extends PreTrainedModel { } -export class OlmoModel extends OlmoPreTrainedModel { } -export class OlmoForCausalLM extends OlmoPreTrainedModel { } +export class OlmoPreTrainedModel extends PreTrainedModel {} +export class OlmoModel extends OlmoPreTrainedModel {} +export class OlmoForCausalLM extends OlmoPreTrainedModel {} ////////////////////////////////////////////////// - ////////////////////////////////////////////////// // Granite models -export class GranitePreTrainedModel extends PreTrainedModel { } -export class GraniteModel extends GranitePreTrainedModel { } -export class GraniteForCausalLM extends GranitePreTrainedModel { } +export class GranitePreTrainedModel extends PreTrainedModel {} +export class GraniteModel extends GranitePreTrainedModel {} +export class GraniteForCausalLM extends GranitePreTrainedModel {} ////////////////////////////////////////////////// - ////////////////////////////////////////////////// // Cohere models /** * The bare Cohere Model outputting raw hidden-states without any specific head on top. */ -export class CoherePreTrainedModel extends PreTrainedModel { } -export class CohereModel extends CoherePreTrainedModel { } +export class CoherePreTrainedModel extends PreTrainedModel {} +export class CohereModel extends CoherePreTrainedModel {} -export class CohereForCausalLM extends CoherePreTrainedModel { } +export class CohereForCausalLM extends CoherePreTrainedModel {} ////////////////////////////////////////////////// ////////////////////////////////////////////////// @@ -3852,13 +3868,13 @@ export class CohereForCausalLM extends CoherePreTrainedModel { } /** * The bare Gemma Model outputting raw hidden-states without any specific head on top. */ -export class GemmaPreTrainedModel extends PreTrainedModel { } +export class GemmaPreTrainedModel extends PreTrainedModel {} /** * The bare Gemma Model outputting raw hidden-states without any specific head on top. */ -export class GemmaModel extends GemmaPreTrainedModel { } +export class GemmaModel extends GemmaPreTrainedModel {} -export class GemmaForCausalLM extends GemmaPreTrainedModel { } +export class GemmaForCausalLM extends GemmaPreTrainedModel {} ////////////////////////////////////////////////// ////////////////////////////////////////////////// @@ -3867,21 +3883,20 @@ export class GemmaForCausalLM extends GemmaPreTrainedModel { } /** * The bare Gemma2 Model outputting raw hidden-states without any specific head on top. */ -export class Gemma2PreTrainedModel extends PreTrainedModel { } +export class Gemma2PreTrainedModel extends PreTrainedModel {} /** * The bare Gemma2 Model outputting raw hidden-states without any specific head on top. */ -export class Gemma2Model extends Gemma2PreTrainedModel { } +export class Gemma2Model extends Gemma2PreTrainedModel {} -export class Gemma2ForCausalLM extends Gemma2PreTrainedModel { } +export class Gemma2ForCausalLM extends Gemma2PreTrainedModel {} ////////////////////////////////////////////////// ////////////////////////////////////////////////// -export class OpenELMPreTrainedModel extends PreTrainedModel { } -export class OpenELMModel extends OpenELMPreTrainedModel { } - -export class OpenELMForCausalLM extends OpenELMPreTrainedModel { } +export class OpenELMPreTrainedModel extends PreTrainedModel {} +export class OpenELMModel extends OpenELMPreTrainedModel {} +export class OpenELMForCausalLM extends OpenELMPreTrainedModel {} ////////////////////////////////////////////////// // Qwen2 models @@ -3889,92 +3904,89 @@ export class OpenELMForCausalLM extends OpenELMPreTrainedModel { } /** * The bare Qwen2 Model outputting raw hidden-states without any specific head on top. */ -export class Qwen2PreTrainedModel extends PreTrainedModel { } +export class Qwen2PreTrainedModel extends PreTrainedModel {} /** * The bare Qwen2 Model outputting raw hidden-states without any specific head on top. */ -export class Qwen2Model extends Qwen2PreTrainedModel { } +export class Qwen2Model extends Qwen2PreTrainedModel {} -export class Qwen2ForCausalLM extends Qwen2PreTrainedModel { } +export class Qwen2ForCausalLM extends Qwen2PreTrainedModel {} ////////////////////////////////////////////////// - ////////////////////////////////////////////////// // Phi models -export class PhiPreTrainedModel extends PreTrainedModel { } +export class PhiPreTrainedModel extends PreTrainedModel {} /** * The bare Phi Model outputting raw hidden-states without any specific head on top. */ -export class PhiModel extends PhiPreTrainedModel { } +export class PhiModel extends PhiPreTrainedModel {} -export class PhiForCausalLM extends PhiPreTrainedModel { } +export class PhiForCausalLM extends PhiPreTrainedModel {} ////////////////////////////////////////////////// ////////////////////////////////////////////////// // Phi3 models -export class Phi3PreTrainedModel extends PreTrainedModel { } +export class Phi3PreTrainedModel extends PreTrainedModel {} /** * The bare Phi3 Model outputting raw hidden-states without any specific head on top. */ -export class Phi3Model extends Phi3PreTrainedModel { } +export class Phi3Model extends Phi3PreTrainedModel {} -export class Phi3ForCausalLM extends Phi3PreTrainedModel { } +export class Phi3ForCausalLM extends Phi3PreTrainedModel {} ////////////////////////////////////////////////// - ////////////////////////////////////////////////// // Bloom models /** * The Bloom Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). */ -export class BloomPreTrainedModel extends PreTrainedModel { } +export class BloomPreTrainedModel extends PreTrainedModel {} /** * The bare Bloom Model transformer outputting raw hidden-states without any specific head on top. */ -export class BloomModel extends BloomPreTrainedModel { } +export class BloomModel extends BloomPreTrainedModel {} /** * The Bloom Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). */ -export class BloomForCausalLM extends BloomPreTrainedModel { } +export class BloomForCausalLM extends BloomPreTrainedModel {} ////////////////////////////////////////////////// ////////////////////////////////////////////////// // MPT models -export class MptPreTrainedModel extends PreTrainedModel { } +export class MptPreTrainedModel extends PreTrainedModel {} /** * The bare Mpt Model transformer outputting raw hidden-states without any specific head on top. */ -export class MptModel extends MptPreTrainedModel { } +export class MptModel extends MptPreTrainedModel {} /** * The MPT Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). */ -export class MptForCausalLM extends MptPreTrainedModel { } +export class MptForCausalLM extends MptPreTrainedModel {} ////////////////////////////////////////////////// - ////////////////////////////////////////////////// // OPT models -export class OPTPreTrainedModel extends PreTrainedModel { } +export class OPTPreTrainedModel extends PreTrainedModel {} /** * The bare OPT Model outputting raw hidden-states without any specific head on top. */ -export class OPTModel extends OPTPreTrainedModel { } +export class OPTModel extends OPTPreTrainedModel {} /** * The OPT Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). */ -export class OPTForCausalLM extends OPTPreTrainedModel { } +export class OPTForCausalLM extends OPTPreTrainedModel {} ////////////////////////////////////////////////// ////////////////////////////////////////////////// -export class ViTPreTrainedModel extends PreTrainedModel { } -export class ViTModel extends ViTPreTrainedModel { } +export class ViTPreTrainedModel extends PreTrainedModel {} +export class ViTModel extends ViTPreTrainedModel {} export class ViTForImageClassification extends ViTPreTrainedModel { /** * @param {any} model_inputs @@ -3986,8 +3998,8 @@ export class ViTForImageClassification extends ViTPreTrainedModel { ////////////////////////////////////////////////// ////////////////////////////////////////////////// -export class PvtPreTrainedModel extends PreTrainedModel { } -export class PvtModel extends PvtPreTrainedModel { } +export class PvtPreTrainedModel extends PreTrainedModel {} +export class PvtModel extends PvtPreTrainedModel {} export class PvtForImageClassification extends PvtPreTrainedModel { /** * @param {any} model_inputs @@ -3999,14 +4011,13 @@ export class PvtForImageClassification extends PvtPreTrainedModel { ////////////////////////////////////////////////// ////////////////////////////////////////////////// -export class ViTMAEPreTrainedModel extends PreTrainedModel { } -export class ViTMAEModel extends ViTMAEPreTrainedModel { } +export class ViTMAEPreTrainedModel extends PreTrainedModel {} +export class ViTMAEModel extends ViTMAEPreTrainedModel {} ////////////////////////////////////////////////// - ////////////////////////////////////////////////// -export class ViTMSNPreTrainedModel extends PreTrainedModel { } -export class ViTMSNModel extends ViTMSNPreTrainedModel { } +export class ViTMSNPreTrainedModel extends PreTrainedModel {} +export class ViTMSNModel extends ViTMSNPreTrainedModel {} export class ViTMSNForImageClassification extends ViTMSNPreTrainedModel { /** * @param {any} model_inputs @@ -4018,14 +4029,13 @@ export class ViTMSNForImageClassification extends ViTMSNPreTrainedModel { ////////////////////////////////////////////////// ////////////////////////////////////////////////// -export class GroupViTPreTrainedModel extends PreTrainedModel { } -export class GroupViTModel extends GroupViTPreTrainedModel { } +export class GroupViTPreTrainedModel extends PreTrainedModel {} +export class GroupViTModel extends GroupViTPreTrainedModel {} ////////////////////////////////////////////////// - ////////////////////////////////////////////////// -export class FastViTPreTrainedModel extends PreTrainedModel { } -export class FastViTModel extends FastViTPreTrainedModel { } +export class FastViTPreTrainedModel extends PreTrainedModel {} +export class FastViTModel extends FastViTPreTrainedModel {} export class FastViTForImageClassification extends FastViTPreTrainedModel { /** * @param {any} model_inputs @@ -4037,26 +4047,26 @@ export class FastViTForImageClassification extends FastViTPreTrainedModel { ////////////////////////////////////////////////// ////////////////////////////////////////////////// -export class VitMattePreTrainedModel extends PreTrainedModel { } +export class VitMattePreTrainedModel extends PreTrainedModel {} /** * ViTMatte framework leveraging any vision backbone e.g. for ADE20k, CityScapes. - * + * * **Example:** Perform image matting with a `VitMatteForImageMatting` model. * ```javascript * import { AutoProcessor, VitMatteForImageMatting, RawImage } from '@huggingface/transformers'; - * + * * // Load processor and model * const processor = await AutoProcessor.from_pretrained('Xenova/vitmatte-small-distinctions-646'); * const model = await VitMatteForImageMatting.from_pretrained('Xenova/vitmatte-small-distinctions-646'); - * + * * // Load image and trimap * const image = await RawImage.fromURL('https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/vitmatte_image.png'); * const trimap = await RawImage.fromURL('https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/vitmatte_trimap.png'); - * + * * // Prepare image + trimap for the model * const inputs = await processor(image, trimap); - * + * * // Predict alpha matte * const { alphas } = await model(inputs); * // Tensor { @@ -4066,14 +4076,14 @@ export class VitMattePreTrainedModel extends PreTrainedModel { } * // data: Float32Array(614400) [ 0.9894027709960938, 0.9970508813858032, ... ] * // } * ``` - * + * * You can visualize the alpha matte as follows: * ```javascript * import { Tensor, cat } from '@huggingface/transformers'; - * + * * // Visualize predicted alpha matte * const imageTensor = image.toTensor(); - * + * * // Convert float (0-1) alpha matte to uint8 (0-255) * const alphaChannel = alphas * .squeeze(0) @@ -4081,10 +4091,10 @@ export class VitMattePreTrainedModel extends PreTrainedModel { } * .clamp_(0, 255) * .round_() * .to('uint8'); - * + * * // Concatenate original image with predicted alpha * const imageData = cat([imageTensor, alphaChannel], 0); - * + * * // Save output image * const outputImage = RawImage.fromTensor(imageData); * outputImage.save('output.png'); @@ -4101,8 +4111,8 @@ export class VitMatteForImageMatting extends VitMattePreTrainedModel { ////////////////////////////////////////////////// ////////////////////////////////////////////////// -export class MobileViTPreTrainedModel extends PreTrainedModel { } -export class MobileViTModel extends MobileViTPreTrainedModel { } +export class MobileViTPreTrainedModel extends PreTrainedModel {} +export class MobileViTModel extends MobileViTPreTrainedModel {} export class MobileViTForImageClassification extends MobileViTPreTrainedModel { /** * @param {any} model_inputs @@ -4116,8 +4126,8 @@ export class MobileViTForImageClassification extends MobileViTPreTrainedModel { ////////////////////////////////////////////////// ////////////////////////////////////////////////// -export class MobileViTV2PreTrainedModel extends PreTrainedModel { } -export class MobileViTV2Model extends MobileViTV2PreTrainedModel { } +export class MobileViTV2PreTrainedModel extends PreTrainedModel {} +export class MobileViTV2Model extends MobileViTV2PreTrainedModel {} export class MobileViTV2ForImageClassification extends MobileViTV2PreTrainedModel { /** * @param {any} model_inputs @@ -4131,21 +4141,21 @@ export class MobileViTV2ForImageClassification extends MobileViTV2PreTrainedMode ////////////////////////////////////////////////// ////////////////////////////////////////////////// -export class OwlViTPreTrainedModel extends PreTrainedModel { } -export class OwlViTModel extends OwlViTPreTrainedModel { } -export class OwlViTForObjectDetection extends OwlViTPreTrainedModel { } +export class OwlViTPreTrainedModel extends PreTrainedModel {} +export class OwlViTModel extends OwlViTPreTrainedModel {} +export class OwlViTForObjectDetection extends OwlViTPreTrainedModel {} ////////////////////////////////////////////////// ////////////////////////////////////////////////// -export class Owlv2PreTrainedModel extends PreTrainedModel { } -export class Owlv2Model extends Owlv2PreTrainedModel { } -export class Owlv2ForObjectDetection extends Owlv2PreTrainedModel { } +export class Owlv2PreTrainedModel extends PreTrainedModel {} +export class Owlv2Model extends Owlv2PreTrainedModel {} +export class Owlv2ForObjectDetection extends Owlv2PreTrainedModel {} ////////////////////////////////////////////////// ////////////////////////////////////////////////// // Beit Models -export class BeitPreTrainedModel extends PreTrainedModel { } -export class BeitModel extends BeitPreTrainedModel { } +export class BeitPreTrainedModel extends PreTrainedModel {} +export class BeitModel extends BeitPreTrainedModel {} export class BeitForImageClassification extends BeitPreTrainedModel { /** * @param {any} model_inputs @@ -4156,10 +4166,9 @@ export class BeitForImageClassification extends BeitPreTrainedModel { } ////////////////////////////////////////////////// - ////////////////////////////////////////////////// -export class DetrPreTrainedModel extends PreTrainedModel { } -export class DetrModel extends DetrPreTrainedModel { } +export class DetrPreTrainedModel extends PreTrainedModel {} +export class DetrModel extends DetrPreTrainedModel {} export class DetrForObjectDetection extends DetrPreTrainedModel { /** * @param {any} model_inputs @@ -4211,8 +4220,8 @@ export class DetrSegmentationOutput extends ModelOutput { ////////////////////////////////////////////////// ////////////////////////////////////////////////// -export class RTDetrPreTrainedModel extends PreTrainedModel { } -export class RTDetrModel extends RTDetrPreTrainedModel { } +export class RTDetrPreTrainedModel extends PreTrainedModel {} +export class RTDetrModel extends RTDetrPreTrainedModel {} export class RTDetrForObjectDetection extends RTDetrPreTrainedModel { /** * @param {any} model_inputs @@ -4238,13 +4247,13 @@ export class RTDetrObjectDetectionOutput extends ModelOutput { ////////////////////////////////////////////////// ////////////////////////////////////////////////// -export class TableTransformerPreTrainedModel extends PreTrainedModel { } +export class TableTransformerPreTrainedModel extends PreTrainedModel {} /** * The bare Table Transformer Model (consisting of a backbone and encoder-decoder Transformer) * outputting raw hidden-states without any specific head on top. */ -export class TableTransformerModel extends TableTransformerPreTrainedModel { } +export class TableTransformerModel extends TableTransformerPreTrainedModel {} /** * Table Transformer Model (consisting of a backbone and encoder-decoder Transformer) @@ -4258,13 +4267,12 @@ export class TableTransformerForObjectDetection extends TableTransformerPreTrain return new TableTransformerObjectDetectionOutput(await super._call(model_inputs)); } } -export class TableTransformerObjectDetectionOutput extends DetrObjectDetectionOutput { } +export class TableTransformerObjectDetectionOutput extends DetrObjectDetectionOutput {} ////////////////////////////////////////////////// - ////////////////////////////////////////////////// -export class DeiTPreTrainedModel extends PreTrainedModel { } -export class DeiTModel extends DeiTPreTrainedModel { } +export class DeiTPreTrainedModel extends PreTrainedModel {} +export class DeiTModel extends DeiTPreTrainedModel {} export class DeiTForImageClassification extends DeiTPreTrainedModel { /** * @param {any} model_inputs @@ -4276,8 +4284,8 @@ export class DeiTForImageClassification extends DeiTPreTrainedModel { ////////////////////////////////////////////////// ////////////////////////////////////////////////// -export class HieraPreTrainedModel extends PreTrainedModel { } -export class HieraModel extends HieraPreTrainedModel { } +export class HieraPreTrainedModel extends PreTrainedModel {} +export class HieraModel extends HieraPreTrainedModel {} export class HieraForImageClassification extends HieraPreTrainedModel { /** * @param {any} model_inputs @@ -4288,17 +4296,16 @@ export class HieraForImageClassification extends HieraPreTrainedModel { } ////////////////////////////////////////////////// - ////////////////////////////////////////////////// /** * An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. */ -export class ResNetPreTrainedModel extends PreTrainedModel { } +export class ResNetPreTrainedModel extends PreTrainedModel {} /** * The bare ResNet model outputting raw features without any specific head on top. */ -export class ResNetModel extends ResNetPreTrainedModel { } +export class ResNetModel extends ResNetPreTrainedModel {} /** * ResNet Model with an image classification head on top (a linear layer on top of the pooled features), e.g. for ImageNet. @@ -4313,10 +4320,9 @@ export class ResNetForImageClassification extends ResNetPreTrainedModel { } ////////////////////////////////////////////////// - ////////////////////////////////////////////////// -export class SwinPreTrainedModel extends PreTrainedModel { } -export class SwinModel extends SwinPreTrainedModel { } +export class SwinPreTrainedModel extends PreTrainedModel {} +export class SwinModel extends SwinPreTrainedModel {} export class SwinForImageClassification extends SwinPreTrainedModel { /** * @param {any} model_inputs @@ -4328,34 +4334,34 @@ export class SwinForImageClassification extends SwinPreTrainedModel { ////////////////////////////////////////////////// ////////////////////////////////////////////////// -export class Swin2SRPreTrainedModel extends PreTrainedModel { } +export class Swin2SRPreTrainedModel extends PreTrainedModel {} /** * The bare Swin2SR Model transformer outputting raw hidden-states without any specific head on top. */ -export class Swin2SRModel extends Swin2SRPreTrainedModel { } +export class Swin2SRModel extends Swin2SRPreTrainedModel {} /** * Swin2SR Model transformer with an upsampler head on top for image super resolution and restoration. - * + * * **Example:** Super-resolution w/ `Xenova/swin2SR-classical-sr-x2-64`. - * + * * ```javascript * import { AutoProcessor, Swin2SRForImageSuperResolution, RawImage } from '@huggingface/transformers'; - * + * * // Load processor and model * const model_id = 'Xenova/swin2SR-classical-sr-x2-64'; * const processor = await AutoProcessor.from_pretrained(model_id); * const model = await Swin2SRForImageSuperResolution.from_pretrained(model_id); - * + * * // Prepare model inputs * const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/butterfly.jpg'; * const image = await RawImage.fromURL(url); * const inputs = await processor(image); - * + * * // Run model * const outputs = await model(inputs); - * + * * // Convert Tensor to RawImage * const output = outputs.reconstruction.squeeze().clamp_(0, 1).mul_(255).round_().to('uint8'); * const outputImage = RawImage.fromTensor(output); @@ -4367,42 +4373,42 @@ export class Swin2SRModel extends Swin2SRPreTrainedModel { } * // } * ``` */ -export class Swin2SRForImageSuperResolution extends Swin2SRPreTrainedModel { } +export class Swin2SRForImageSuperResolution extends Swin2SRPreTrainedModel {} ////////////////////////////////////////////////// ////////////////////////////////////////////////// -export class DPTPreTrainedModel extends PreTrainedModel { } +export class DPTPreTrainedModel extends PreTrainedModel {} /** * The bare DPT Model transformer outputting raw hidden-states without any specific head on top. */ -export class DPTModel extends DPTPreTrainedModel { } +export class DPTModel extends DPTPreTrainedModel {} /** * DPT Model with a depth estimation head on top (consisting of 3 convolutional layers) e.g. for KITTI, NYUv2. - * + * * **Example:** Depth estimation w/ `Xenova/dpt-hybrid-midas`. * ```javascript * import { DPTForDepthEstimation, AutoProcessor, RawImage, interpolate, max } from '@huggingface/transformers'; - * + * * // Load model and processor * const model_id = 'Xenova/dpt-hybrid-midas'; * const model = await DPTForDepthEstimation.from_pretrained(model_id); * const processor = await AutoProcessor.from_pretrained(model_id); - * + * * // Load image from URL * const url = 'http://images.cocodataset.org/val2017/000000039769.jpg'; * const image = await RawImage.fromURL(url); - * + * * // Prepare image for the model * const inputs = await processor(image); - * + * * // Run model * const { predicted_depth } = await model(inputs); - * + * * // Interpolate to original size * const prediction = interpolate(predicted_depth, image.size.reverse(), 'bilinear', false); - * + * * // Visualize the prediction * const formatted = prediction.mul_(255 / max(prediction.data)[0]).to('uint8'); * const depth = RawImage.fromTensor(formatted); @@ -4414,70 +4420,69 @@ export class DPTModel extends DPTPreTrainedModel { } * // } * ``` */ -export class DPTForDepthEstimation extends DPTPreTrainedModel { } +export class DPTForDepthEstimation extends DPTPreTrainedModel {} ////////////////////////////////////////////////// ////////////////////////////////////////////////// -export class DepthAnythingPreTrainedModel extends PreTrainedModel { } +export class DepthAnythingPreTrainedModel extends PreTrainedModel {} /** * Depth Anything Model with a depth estimation head on top (consisting of 3 convolutional layers) e.g. for KITTI, NYUv2. */ -export class DepthAnythingForDepthEstimation extends DepthAnythingPreTrainedModel { } +export class DepthAnythingForDepthEstimation extends DepthAnythingPreTrainedModel {} ////////////////////////////////////////////////// - ////////////////////////////////////////////////// -export class SapiensPreTrainedModel extends PreTrainedModel { } -export class SapiensForSemanticSegmentation extends SapiensPreTrainedModel { } -export class SapiensForDepthEstimation extends SapiensPreTrainedModel { } -export class SapiensForNormalEstimation extends SapiensPreTrainedModel { } +export class SapiensPreTrainedModel extends PreTrainedModel {} +export class SapiensForSemanticSegmentation extends SapiensPreTrainedModel {} +export class SapiensForDepthEstimation extends SapiensPreTrainedModel {} +export class SapiensForNormalEstimation extends SapiensPreTrainedModel {} ////////////////////////////////////////////////// ////////////////////////////////////////////////// -export class DepthProPreTrainedModel extends PreTrainedModel { } -export class DepthProForDepthEstimation extends DepthProPreTrainedModel { } +export class DepthProPreTrainedModel extends PreTrainedModel {} +export class DepthProForDepthEstimation extends DepthProPreTrainedModel {} ////////////////////////////////////////////////// ////////////////////////////////////////////////// -export class MaskFormerPreTrainedModel extends PreTrainedModel { } -export class MaskFormerModel extends MaskFormerPreTrainedModel { } -export class MaskFormerForInstanceSegmentation extends MaskFormerPreTrainedModel { } +export class MaskFormerPreTrainedModel extends PreTrainedModel {} +export class MaskFormerModel extends MaskFormerPreTrainedModel {} +export class MaskFormerForInstanceSegmentation extends MaskFormerPreTrainedModel {} ////////////////////////////////////////////////// ////////////////////////////////////////////////// -export class GLPNPreTrainedModel extends PreTrainedModel { } +export class GLPNPreTrainedModel extends PreTrainedModel {} /** * The bare GLPN encoder (Mix-Transformer) outputting raw hidden-states without any specific head on top. */ -export class GLPNModel extends GLPNPreTrainedModel { } +export class GLPNModel extends GLPNPreTrainedModel {} /** * GLPN Model transformer with a lightweight depth estimation head on top e.g. for KITTI, NYUv2. - * + * * **Example:** Depth estimation w/ `Xenova/glpn-kitti`. * ```javascript * import { GLPNForDepthEstimation, AutoProcessor, RawImage, interpolate, max } from '@huggingface/transformers'; - * + * * // Load model and processor * const model_id = 'Xenova/glpn-kitti'; * const model = await GLPNForDepthEstimation.from_pretrained(model_id); * const processor = await AutoProcessor.from_pretrained(model_id); - * + * * // Load image from URL * const url = 'http://images.cocodataset.org/val2017/000000039769.jpg'; * const image = await RawImage.fromURL(url); - * + * * // Prepare image for the model * const inputs = await processor(image); - * + * * // Run model * const { predicted_depth } = await model(inputs); - * + * * // Interpolate to original size * const prediction = interpolate(predicted_depth, image.size.reverse(), 'bilinear', false); - * + * * // Visualize the prediction * const formatted = prediction.mul_(255 / max(prediction.data)[0]).to('uint8'); * const depth = RawImage.fromTensor(formatted); @@ -4489,64 +4494,64 @@ export class GLPNModel extends GLPNPreTrainedModel { } * // } * ``` */ -export class GLPNForDepthEstimation extends GLPNPreTrainedModel { } +export class GLPNForDepthEstimation extends GLPNPreTrainedModel {} ////////////////////////////////////////////////// ////////////////////////////////////////////////// -export class DonutSwinPreTrainedModel extends PreTrainedModel { } +export class DonutSwinPreTrainedModel extends PreTrainedModel {} /** * The bare Donut Swin Model transformer outputting raw hidden-states without any specific head on top. - * + * * **Example:** Step-by-step Document Parsing. - * + * * ```javascript * import { AutoProcessor, AutoTokenizer, AutoModelForVision2Seq, RawImage } from '@huggingface/transformers'; - * + * * // Choose model to use * const model_id = 'Xenova/donut-base-finetuned-cord-v2'; - * + * * // Prepare image inputs * const processor = await AutoProcessor.from_pretrained(model_id); * const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/receipt.png'; * const image = await RawImage.read(url); * const image_inputs = await processor(image); - * + * * // Prepare decoder inputs * const tokenizer = await AutoTokenizer.from_pretrained(model_id); * const task_prompt = ''; * const decoder_input_ids = tokenizer(task_prompt, { * add_special_tokens: false, * }).input_ids; - * + * * // Create the model * const model = await AutoModelForVision2Seq.from_pretrained(model_id); - * + * * // Run inference * const output = await model.generate(image_inputs.pixel_values, { * decoder_input_ids, * max_length: model.config.decoder.max_position_embeddings, * }); - * + * * // Decode output * const decoded = tokenizer.batch_decode(output)[0]; * // CINNAMON SUGAR 17,000 1 x 17,000 17,000 17,000 20,000 3,000 * ``` - * + * * **Example:** Step-by-step Document Visual Question Answering (DocVQA) - * + * * ```javascript * import { AutoProcessor, AutoTokenizer, AutoModelForVision2Seq, RawImage } from '@huggingface/transformers'; - * + * * // Choose model to use * const model_id = 'Xenova/donut-base-finetuned-docvqa'; - * + * * // Prepare image inputs * const processor = await AutoProcessor.from_pretrained(model_id); * const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/invoice.png'; * const image = await RawImage.read(url); * const image_inputs = await processor(image); - * + * * // Prepare decoder inputs * const tokenizer = await AutoTokenizer.from_pretrained(model_id); * const question = 'What is the invoice number?'; @@ -4554,32 +4559,31 @@ export class DonutSwinPreTrainedModel extends PreTrainedModel { } * const decoder_input_ids = tokenizer(task_prompt, { * add_special_tokens: false, * }).input_ids; - * + * * // Create the model * const model = await AutoModelForVision2Seq.from_pretrained(model_id); - * + * * // Run inference * const output = await model.generate(image_inputs.pixel_values, { * decoder_input_ids, * max_length: model.config.decoder.max_position_embeddings, * }); - * + * * // Decode output * const decoded = tokenizer.batch_decode(output)[0]; * // What is the invoice number? us-001 * ``` */ -export class DonutSwinModel extends DonutSwinPreTrainedModel { } +export class DonutSwinModel extends DonutSwinPreTrainedModel {} ////////////////////////////////////////////////// - ////////////////////////////////////////////////// -export class ConvNextPreTrainedModel extends PreTrainedModel { } +export class ConvNextPreTrainedModel extends PreTrainedModel {} /** * The bare ConvNext model outputting raw features without any specific head on top. */ -export class ConvNextModel extends ConvNextPreTrainedModel { } +export class ConvNextModel extends ConvNextPreTrainedModel {} /** * ConvNext Model with an image classification head on top (a linear layer on top of the pooled features), e.g. for ImageNet. @@ -4594,14 +4598,13 @@ export class ConvNextForImageClassification extends ConvNextPreTrainedModel { } ////////////////////////////////////////////////// - ////////////////////////////////////////////////// -export class ConvNextV2PreTrainedModel extends PreTrainedModel { } +export class ConvNextV2PreTrainedModel extends PreTrainedModel {} /** * The bare ConvNextV2 model outputting raw features without any specific head on top. */ -export class ConvNextV2Model extends ConvNextV2PreTrainedModel { } +export class ConvNextV2Model extends ConvNextV2PreTrainedModel {} /** * ConvNextV2 Model with an image classification head on top (a linear layer on top of the pooled features), e.g. for ImageNet. @@ -4617,12 +4620,12 @@ export class ConvNextV2ForImageClassification extends ConvNextV2PreTrainedModel ////////////////////////////////////////////////// ////////////////////////////////////////////////// -export class Dinov2PreTrainedModel extends PreTrainedModel { } +export class Dinov2PreTrainedModel extends PreTrainedModel {} /** * The bare DINOv2 Model transformer outputting raw hidden-states without any specific head on top. */ -export class Dinov2Model extends Dinov2PreTrainedModel { } +export class Dinov2Model extends Dinov2PreTrainedModel {} /** * Dinov2 Model transformer with an image classification head on top (a linear layer on top of the final hidden state of the [CLS] token) e.g. for ImageNet. @@ -4637,10 +4640,9 @@ export class Dinov2ForImageClassification extends Dinov2PreTrainedModel { } ////////////////////////////////////////////////// - ////////////////////////////////////////////////// -export class YolosPreTrainedModel extends PreTrainedModel { } -export class YolosModel extends YolosPreTrainedModel { } +export class YolosPreTrainedModel extends PreTrainedModel {} +export class YolosModel extends YolosPreTrainedModel {} export class YolosForObjectDetection extends YolosPreTrainedModel { /** * @param {any} model_inputs @@ -4665,30 +4667,27 @@ export class YolosObjectDetectionOutput extends ModelOutput { } ////////////////////////////////////////////////// - - - ////////////////////////////////////////////////// -export class SamPreTrainedModel extends PreTrainedModel { } +export class SamPreTrainedModel extends PreTrainedModel {} /** * Segment Anything Model (SAM) for generating segmentation masks, given an input image * and optional 2D location and bounding boxes. - * + * * **Example:** Perform mask generation w/ `Xenova/sam-vit-base`. * ```javascript * import { SamModel, AutoProcessor, RawImage } from '@huggingface/transformers'; - * + * * const model = await SamModel.from_pretrained('Xenova/sam-vit-base'); * const processor = await AutoProcessor.from_pretrained('Xenova/sam-vit-base'); - * + * * const img_url = 'https://huggingface.co/ybelkada/segment-anything/resolve/main/assets/car.png'; * const raw_image = await RawImage.read(img_url); * const input_points = [[[450, 600]]] // 2D localization of a window - * + * * const inputs = await processor(raw_image, { input_points }); * const outputs = await model(inputs); - * + * * const masks = await processor.post_process_masks(outputs.pred_masks, inputs.original_sizes, inputs.reshaped_input_sizes); * // [ * // Tensor { @@ -4712,7 +4711,6 @@ export class SamPreTrainedModel extends PreTrainedModel { } * ``` */ export class SamModel extends SamPreTrainedModel { - /** * Compute image embeddings and positional image embeddings, given the pixel values of an image. * @param {Object} model_inputs Object containing the model inputs. @@ -4722,11 +4720,11 @@ export class SamModel extends SamPreTrainedModel { async get_image_embeddings({ pixel_values }) { // in: // - pixel_values: tensor.float32[batch_size,3,1024,1024] - // + // // out: // - image_embeddings: tensor.float32[batch_size,256,64,64] // - image_positional_embeddings: tensor.float32[batch_size,256,64,64] - return await encoderForward(this, { pixel_values }) + return await encoderForward(this, { pixel_values }); } /** @@ -4755,19 +4753,15 @@ export class SamModel extends SamPreTrainedModel { // Compute the image embeddings if they are missing model_inputs = { ...model_inputs, - ...(await this.get_image_embeddings(model_inputs)) - } + ...(await this.get_image_embeddings(model_inputs)), + }; } if (!model_inputs.input_labels && model_inputs.input_points) { // Set default input labels if they are missing const shape = model_inputs.input_points.dims.slice(0, -1); const numElements = shape.reduce((a, b) => a * b, 1); - model_inputs.input_labels = new Tensor( - 'int64', - new BigInt64Array(numElements).fill(1n), - shape - ); + model_inputs.input_labels = new Tensor("int64", new BigInt64Array(numElements).fill(1n), shape); } const decoder_inputs = { @@ -4787,7 +4781,7 @@ export class SamModel extends SamPreTrainedModel { // Returns: // - iou_scores: tensor.float32[batch_size,point_batch_size,3] // - pred_masks: tensor.float32[batch_size,point_batch_size,3,256,256] - return await sessionRun(this.sessions['prompt_encoder_mask_decoder'], decoder_inputs); + return await sessionRun(this.sessions["prompt_encoder_mask_decoder"], decoder_inputs); } /** @@ -4800,7 +4794,6 @@ export class SamModel extends SamPreTrainedModel { } } - /** * Base class for Segment-Anything model's output. */ @@ -4818,42 +4811,41 @@ export class SamImageSegmentationOutput extends ModelOutput { } ////////////////////////////////////////////////// - ////////////////////////////////////////////////// // MarianMT models -export class MarianPreTrainedModel extends PreTrainedModel { }; +export class MarianPreTrainedModel extends PreTrainedModel {} -export class MarianModel extends MarianPreTrainedModel { } +export class MarianModel extends MarianPreTrainedModel {} -export class MarianMTModel extends MarianPreTrainedModel { } +export class MarianMTModel extends MarianPreTrainedModel {} ////////////////////////////////////////////////// ////////////////////////////////////////////////// // M2M100 models -export class M2M100PreTrainedModel extends PreTrainedModel { }; +export class M2M100PreTrainedModel extends PreTrainedModel {} -export class M2M100Model extends M2M100PreTrainedModel { } +export class M2M100Model extends M2M100PreTrainedModel {} -export class M2M100ForConditionalGeneration extends M2M100PreTrainedModel { } +export class M2M100ForConditionalGeneration extends M2M100PreTrainedModel {} ////////////////////////////////////////////////// ////////////////////////////////////////////////// // Wav2Vec2 models -export class Wav2Vec2PreTrainedModel extends PreTrainedModel { }; +export class Wav2Vec2PreTrainedModel extends PreTrainedModel {} /** * The bare Wav2Vec2 Model transformer outputting raw hidden-states without any specific head on top. - * + * * **Example:** Load and run a `Wav2Vec2Model` for feature extraction. - * + * * ```javascript * import { AutoProcessor, AutoModel, read_audio } from '@huggingface/transformers'; - * + * * // Read and preprocess audio * const processor = await AutoProcessor.from_pretrained('Xenova/mms-300m'); * const audio = await read_audio('https://huggingface.co/datasets/Narsil/asr_dummy/resolve/main/mlk.flac', 16000); * const inputs = await processor(audio); - * + * * // Run model with inputs * const model = await AutoModel.from_pretrained('Xenova/mms-300m'); * const output = await model(inputs); @@ -4867,7 +4859,7 @@ export class Wav2Vec2PreTrainedModel extends PreTrainedModel { }; * // } * ``` */ -export class Wav2Vec2Model extends Wav2Vec2PreTrainedModel { } +export class Wav2Vec2Model extends Wav2Vec2PreTrainedModel {} export class Wav2Vec2ForCTC extends Wav2Vec2PreTrainedModel { /** @@ -4906,34 +4898,33 @@ export class Wav2Vec2ForAudioFrameClassification extends Wav2Vec2PreTrainedModel } ////////////////////////////////////////////////// - ////////////////////////////////////////////////// // PyAnnote models -export class PyAnnotePreTrainedModel extends PreTrainedModel { }; +export class PyAnnotePreTrainedModel extends PreTrainedModel {} /** * The bare PyAnnote Model transformer outputting raw hidden-states without any specific head on top. */ -export class PyAnnoteModel extends PyAnnotePreTrainedModel { } +export class PyAnnoteModel extends PyAnnotePreTrainedModel {} /** * PyAnnote Model with a frame classification head on top for tasks like Speaker Diarization. - * + * * **Example:** Load and run a `PyAnnoteForAudioFrameClassification` for speaker diarization. - * + * * ```javascript * import { AutoProcessor, AutoModelForAudioFrameClassification, read_audio } from '@huggingface/transformers'; - * + * * // Load model and processor * const model_id = 'onnx-community/pyannote-segmentation-3.0'; * const model = await AutoModelForAudioFrameClassification.from_pretrained(model_id); * const processor = await AutoProcessor.from_pretrained(model_id); - * + * * // Read and preprocess audio * const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/mlk.wav'; * const audio = await read_audio(url, processor.feature_extractor.config.sampling_rate); * const inputs = await processor(audio); - * + * * // Run model with inputs * const { logits } = await model(inputs); * // { @@ -4944,7 +4935,7 @@ export class PyAnnoteModel extends PyAnnotePreTrainedModel { } * // size: 5369 * // } * // } - * + * * const result = processor.post_process_speaker_diarization(logits, audio.length); * // [ * // [ @@ -4953,7 +4944,7 @@ export class PyAnnoteModel extends PyAnnotePreTrainedModel { } * // ... * // ] * // ] - * + * * // Display result * console.table(result[0], ['start', 'end', 'id', 'confidence']); * // ┌─────────┬────────────────────┬────────────────────┬────┬─────────────────────┐ @@ -4987,19 +4978,18 @@ export class PyAnnoteForAudioFrameClassification extends PyAnnotePreTrainedModel ////////////////////////////////////////////////// // WeSpeakerResNet models -export class WeSpeakerResNetPreTrainedModel extends PreTrainedModel { }; -export class WeSpeakerResNetModel extends WeSpeakerResNetPreTrainedModel { } +export class WeSpeakerResNetPreTrainedModel extends PreTrainedModel {} +export class WeSpeakerResNetModel extends WeSpeakerResNetPreTrainedModel {} ////////////////////////////////////////////////// - ////////////////////////////////////////////////// // UniSpeech models -export class UniSpeechPreTrainedModel extends PreTrainedModel { }; +export class UniSpeechPreTrainedModel extends PreTrainedModel {} /** * The bare UniSpeech Model transformer outputting raw hidden-states without any specific head on top. */ -export class UniSpeechModel extends UniSpeechPreTrainedModel { } +export class UniSpeechModel extends UniSpeechPreTrainedModel {} /** * UniSpeech Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC). @@ -5032,12 +5022,12 @@ export class UniSpeechForSequenceClassification extends UniSpeechPreTrainedModel ////////////////////////////////////////////////// // UniSpeechSat models -export class UniSpeechSatPreTrainedModel extends PreTrainedModel { }; +export class UniSpeechSatPreTrainedModel extends PreTrainedModel {} /** * The bare UniSpeechSat Model transformer outputting raw hidden-states without any specific head on top. */ -export class UniSpeechSatModel extends UniSpeechSatPreTrainedModel { } +export class UniSpeechSatModel extends UniSpeechSatPreTrainedModel {} /** * UniSpeechSat Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC). @@ -5084,12 +5074,12 @@ export class UniSpeechSatForAudioFrameClassification extends UniSpeechSatPreTrai ////////////////////////////////////////////////// // Wav2Vec2Bert models -export class Wav2Vec2BertPreTrainedModel extends PreTrainedModel { }; +export class Wav2Vec2BertPreTrainedModel extends PreTrainedModel {} /** * The bare Wav2Vec2Bert Model transformer outputting raw hidden-states without any specific head on top. */ -export class Wav2Vec2BertModel extends Wav2Vec2BertPreTrainedModel { } +export class Wav2Vec2BertModel extends Wav2Vec2BertPreTrainedModel {} /** * Wav2Vec2Bert Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC). @@ -5122,21 +5112,21 @@ export class Wav2Vec2BertForSequenceClassification extends Wav2Vec2BertPreTraine ////////////////////////////////////////////////// // Hubert models -export class HubertPreTrainedModel extends PreTrainedModel { } +export class HubertPreTrainedModel extends PreTrainedModel {} /** * The bare Hubert Model transformer outputting raw hidden-states without any specific head on top. - * + * * **Example:** Load and run a `HubertModel` for feature extraction. - * + * * ```javascript * import { AutoProcessor, AutoModel, read_audio } from '@huggingface/transformers'; - * + * * // Read and preprocess audio * const processor = await AutoProcessor.from_pretrained('Xenova/hubert-base-ls960'); * const audio = await read_audio('https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/jfk.wav', 16000); * const inputs = await processor(audio); - * + * * // Load and run model with inputs * const model = await AutoModel.from_pretrained('Xenova/hubert-base-ls960'); * const output = await model(inputs); @@ -5150,7 +5140,7 @@ export class HubertPreTrainedModel extends PreTrainedModel { } * // } * ``` */ -export class HubertModel extends Wav2Vec2PreTrainedModel { } +export class HubertModel extends Wav2Vec2PreTrainedModel {} /** * Hubert Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC). @@ -5186,21 +5176,21 @@ export class HubertForSequenceClassification extends Wav2Vec2PreTrainedModel { /** * An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. */ -export class WavLMPreTrainedModel extends PreTrainedModel { }; +export class WavLMPreTrainedModel extends PreTrainedModel {} /** * The bare WavLM Model transformer outputting raw hidden-states without any specific head on top. - * + * * **Example:** Load and run a `WavLMModel` for feature extraction. - * + * * ```javascript * import { AutoProcessor, AutoModel, read_audio } from '@huggingface/transformers'; - * + * * // Read and preprocess audio * const processor = await AutoProcessor.from_pretrained('Xenova/wavlm-base'); * const audio = await read_audio('https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/jfk.wav', 16000); * const inputs = await processor(audio); - * + * * // Run model with inputs * const model = await AutoModel.from_pretrained('Xenova/wavlm-base'); * const output = await model(inputs); @@ -5214,7 +5204,7 @@ export class WavLMPreTrainedModel extends PreTrainedModel { }; * // } * ``` */ -export class WavLMModel extends WavLMPreTrainedModel { } +export class WavLMModel extends WavLMPreTrainedModel {} /** * WavLM Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC). @@ -5246,17 +5236,17 @@ export class WavLMForSequenceClassification extends WavLMPreTrainedModel { /** * WavLM Model with an XVector feature extraction head on top for tasks like Speaker Verification. - * + * * **Example:** Extract speaker embeddings with `WavLMForXVector`. * ```javascript * import { AutoProcessor, AutoModel, read_audio } from '@huggingface/transformers'; - * + * * // Read and preprocess audio * const processor = await AutoProcessor.from_pretrained('Xenova/wavlm-base-plus-sv'); * const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/jfk.wav'; * const audio = await read_audio(url, 16000); * const inputs = await processor(audio); - * + * * // Run model with inputs * const model = await AutoModel.from_pretrained('Xenova/wavlm-base-plus-sv'); * const outputs = await model(inputs); @@ -5289,17 +5279,17 @@ export class WavLMForXVector extends WavLMPreTrainedModel { /** * WavLM Model with a frame classification head on top for tasks like Speaker Diarization. - * + * * **Example:** Perform speaker diarization with `WavLMForAudioFrameClassification`. * ```javascript * import { AutoProcessor, AutoModelForAudioFrameClassification, read_audio } from '@huggingface/transformers'; - * + * * // Read and preprocess audio * const processor = await AutoProcessor.from_pretrained('Xenova/wavlm-base-plus-sd'); * const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/jfk.wav'; * const audio = await read_audio(url, 16000); * const inputs = await processor(audio); - * + * * // Run model with inputs * const model = await AutoModelForAudioFrameClassification.from_pretrained('Xenova/wavlm-base-plus-sd'); * const { logits } = await model(inputs); @@ -5311,7 +5301,7 @@ export class WavLMForXVector extends WavLMPreTrainedModel { * // size: 1098 * // } * // } - * + * * const labels = logits[0].sigmoid().tolist().map( * frames => frames.map(speaker => speaker > 0.5 ? 1 : 0) * ); @@ -5340,29 +5330,29 @@ export class WavLMForAudioFrameClassification extends WavLMPreTrainedModel { /** * An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. */ -export class SpeechT5PreTrainedModel extends PreTrainedModel { }; +export class SpeechT5PreTrainedModel extends PreTrainedModel {} /** * The bare SpeechT5 Encoder-Decoder Model outputting raw hidden-states without any specific pre- or post-nets. */ -export class SpeechT5Model extends SpeechT5PreTrainedModel { }; +export class SpeechT5Model extends SpeechT5PreTrainedModel {} /** * SpeechT5 Model with a speech encoder and a text decoder. - * + * * **Example:** Generate speech from text with `SpeechT5ForSpeechToText`. * ```javascript * import { AutoTokenizer, AutoProcessor, SpeechT5ForTextToSpeech, SpeechT5HifiGan, Tensor } from '@huggingface/transformers'; - * + * * // Load the tokenizer and processor * const tokenizer = await AutoTokenizer.from_pretrained('Xenova/speecht5_tts'); * const processor = await AutoProcessor.from_pretrained('Xenova/speecht5_tts'); - * + * * // Load the models * // NOTE: We use the full-precision versions as they are more accurate * const model = await SpeechT5ForTextToSpeech.from_pretrained('Xenova/speecht5_tts', { dtype: 'fp32' }); * const vocoder = await SpeechT5HifiGan.from_pretrained('Xenova/speecht5_hifigan', { dtype: 'fp32' }); - * + * * // Load speaker embeddings from URL * const speaker_embeddings_data = new Float32Array( * await (await fetch('https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/speaker_embeddings.bin')).arrayBuffer() @@ -5372,10 +5362,10 @@ export class SpeechT5Model extends SpeechT5PreTrainedModel { }; * speaker_embeddings_data, * [1, speaker_embeddings_data.length] * ) - * + * * // Run tokenization * const { input_ids } = tokenizer('Hello, my dog is cute'); - * + * * // Generate waveform * const { waveform } = await model.generate_speech(input_ids, speaker_embeddings, { vocoder }); * console.log(waveform) @@ -5387,13 +5377,12 @@ export class SpeechT5Model extends SpeechT5PreTrainedModel { }; * // } * ``` */ -export class SpeechT5ForSpeechToText extends SpeechT5PreTrainedModel { } +export class SpeechT5ForSpeechToText extends SpeechT5PreTrainedModel {} /** * SpeechT5 Model with a text encoder and a speech decoder. */ export class SpeechT5ForTextToSpeech extends SpeechT5PreTrainedModel { - /** * @typedef {Object} SpeechOutput * @property {Tensor} [spectrogram] The predicted log-mel spectrogram of shape @@ -5415,17 +5404,20 @@ export class SpeechT5ForTextToSpeech extends SpeechT5PreTrainedModel { * @param {boolean} [options.output_cross_attentions=false] Whether or not to return the attentions tensors of the decoder's cross-attention layers. * @returns {Promise} A promise which resolves to an object containing the spectrogram, waveform, and cross-attention tensors. */ - async generate_speech(input_values, speaker_embeddings, { - threshold = 0.5, - minlenratio = 0.0, - maxlenratio = 20.0, - vocoder = null, - // output_cross_attentions = false, // TODO add - } = {}) { - + async generate_speech( + input_values, + speaker_embeddings, + { + threshold = 0.5, + minlenratio = 0.0, + maxlenratio = 20.0, + vocoder = null, + // output_cross_attentions = false, // TODO add + } = {}, + ) { const model_inputs = { - input_ids: input_values - } + input_ids: input_values, + }; const { encoder_outputs, encoder_attention_mask } = await encoderForward(this, model_inputs); @@ -5448,11 +5440,7 @@ export class SpeechT5ForTextToSpeech extends SpeechT5PreTrainedModel { if (decoder_outputs) { output_sequence = decoder_outputs.output_sequence_out; } else { - output_sequence = new Tensor( - 'float32', - new Float32Array(num_mel_bins), - [1, 1, num_mel_bins], - ) + output_sequence = new Tensor("float32", new Float32Array(num_mel_bins), [1, 1, num_mel_bins]); } let decoderFeeds = { use_cache_branch, @@ -5463,115 +5451,111 @@ export class SpeechT5ForTextToSpeech extends SpeechT5PreTrainedModel { }; this.addPastKeyValues(decoderFeeds, past_key_values); - decoder_outputs = await sessionRun(this.sessions['decoder_model_merged'], decoderFeeds); + decoder_outputs = await sessionRun(this.sessions["decoder_model_merged"], decoderFeeds); past_key_values = this.getPastKeyValues(decoder_outputs, past_key_values); const { prob, spectrum } = decoder_outputs; spectrogramParts.push(spectrum); - if (idx >= minlen && ( + if ( + idx >= minlen && // Finished when stop token or maximum length is reached. - Array.from(prob.data).filter(p => p >= threshold).length > 0 || idx >= maxlen - )) { + (Array.from(prob.data).filter((p) => p >= threshold).length > 0 || idx >= maxlen) + ) { break; } } const spectrogram = cat(spectrogramParts); - const { waveform } = await sessionRun(vocoder.sessions['model'], { spectrogram }); + const { waveform } = await sessionRun(vocoder.sessions["model"], { spectrogram }); return { spectrogram, waveform, // cross_attentions: null, // TODO add - } + }; } } /** * HiFi-GAN vocoder. - * + * * See [SpeechT5ForSpeechToText](./models#module_models.SpeechT5ForSpeechToText) for example usage. */ export class SpeechT5HifiGan extends PreTrainedModel { - main_input_name = 'spectrogram'; + main_input_name = "spectrogram"; } ////////////////////////////////////////////////// - ////////////////////////////////////////////////// // TrOCR models -export class TrOCRPreTrainedModel extends PreTrainedModel { } +export class TrOCRPreTrainedModel extends PreTrainedModel {} /** * The TrOCR Decoder with a language modeling head. */ -export class TrOCRForCausalLM extends TrOCRPreTrainedModel { } +export class TrOCRForCausalLM extends TrOCRPreTrainedModel {} ////////////////////////////////////////////////// - ////////////////////////////////////////////////// // Mistral models /** * The bare Mistral Model outputting raw hidden-states without any specific head on top. */ -export class MistralPreTrainedModel extends PreTrainedModel { } +export class MistralPreTrainedModel extends PreTrainedModel {} -export class MistralModel extends MistralPreTrainedModel { } +export class MistralModel extends MistralPreTrainedModel {} -export class MistralForCausalLM extends MistralPreTrainedModel { } +export class MistralForCausalLM extends MistralPreTrainedModel {} ////////////////////////////////////////////////// - ////////////////////////////////////////////////// // Starcoder2 models /** * The bare Starcoder2 Model outputting raw hidden-states without any specific head on top. */ -export class Starcoder2PreTrainedModel extends PreTrainedModel { } +export class Starcoder2PreTrainedModel extends PreTrainedModel {} -export class Starcoder2Model extends Starcoder2PreTrainedModel { } +export class Starcoder2Model extends Starcoder2PreTrainedModel {} -export class Starcoder2ForCausalLM extends Starcoder2PreTrainedModel { } +export class Starcoder2ForCausalLM extends Starcoder2PreTrainedModel {} ////////////////////////////////////////////////// - ////////////////////////////////////////////////// // Falcon models /** * The bare Falcon Model outputting raw hidden-states without any specific head on top. */ -export class FalconPreTrainedModel extends PreTrainedModel { } +export class FalconPreTrainedModel extends PreTrainedModel {} -export class FalconModel extends FalconPreTrainedModel { } +export class FalconModel extends FalconPreTrainedModel {} -export class FalconForCausalLM extends FalconPreTrainedModel { } +export class FalconForCausalLM extends FalconPreTrainedModel {} ////////////////////////////////////////////////// - ////////////////////////////////////////////////// // CLAP models -export class ClapPreTrainedModel extends PreTrainedModel { } +export class ClapPreTrainedModel extends PreTrainedModel {} -export class ClapModel extends ClapPreTrainedModel { } +export class ClapModel extends ClapPreTrainedModel {} /** * CLAP Text Model with a projection layer on top (a linear layer on top of the pooled output). - * + * * **Example:** Compute text embeddings with `ClapTextModelWithProjection`. - * + * * ```javascript * import { AutoTokenizer, ClapTextModelWithProjection } from '@huggingface/transformers'; - * + * * // Load tokenizer and text model * const tokenizer = await AutoTokenizer.from_pretrained('Xenova/clap-htsat-unfused'); * const text_model = await ClapTextModelWithProjection.from_pretrained('Xenova/clap-htsat-unfused'); - * + * * // Run tokenization * const texts = ['a sound of a cat', 'a sound of a dog']; * const text_inputs = tokenizer(texts, { padding: true, truncation: true }); - * + * * // Compute embeddings * const { text_embeds } = await text_model(text_inputs); * // Tensor { @@ -5583,31 +5567,30 @@ export class ClapModel extends ClapPreTrainedModel { } * ``` */ export class ClapTextModelWithProjection extends ClapPreTrainedModel { - /** @type {PreTrainedModel.from_pretrained} */ static async from_pretrained(pretrained_model_name_or_path, options = {}) { // Update default model file name if not provided - options.model_file_name ??= 'text_model'; + options.model_file_name ??= "text_model"; return super.from_pretrained(pretrained_model_name_or_path, options); } } /** * CLAP Audio Model with a projection layer on top (a linear layer on top of the pooled output). - * + * * **Example:** Compute audio embeddings with `ClapAudioModelWithProjection`. - * + * * ```javascript * import { AutoProcessor, ClapAudioModelWithProjection, read_audio } from '@huggingface/transformers'; - * + * * // Load processor and audio model * const processor = await AutoProcessor.from_pretrained('Xenova/clap-htsat-unfused'); * const audio_model = await ClapAudioModelWithProjection.from_pretrained('Xenova/clap-htsat-unfused'); - * + * * // Read audio and run processor * const audio = await read_audio('https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/cat_meow.wav'); * const audio_inputs = await processor(audio); - * + * * // Compute embeddings * const { audio_embeds } = await audio_model(audio_inputs); * // Tensor { @@ -5622,31 +5605,30 @@ export class ClapAudioModelWithProjection extends ClapPreTrainedModel { /** @type {PreTrainedModel.from_pretrained} */ static async from_pretrained(pretrained_model_name_or_path, options = {}) { // Update default model file name if not provided - options.model_file_name ??= 'audio_model'; + options.model_file_name ??= "audio_model"; return super.from_pretrained(pretrained_model_name_or_path, options); } } ////////////////////////////////////////////////// - ////////////////////////////////////////////////// // VITS models -export class VitsPreTrainedModel extends PreTrainedModel { } +export class VitsPreTrainedModel extends PreTrainedModel {} /** * The complete VITS model, for text-to-speech synthesis. - * + * * **Example:** Generate speech from text with `VitsModel`. * ```javascript * import { AutoTokenizer, VitsModel } from '@huggingface/transformers'; - * + * * // Load the tokenizer and model * const tokenizer = await AutoTokenizer.from_pretrained('Xenova/mms-tts-eng'); * const model = await VitsModel.from_pretrained('Xenova/mms-tts-eng'); - * + * * // Run tokenization * const inputs = tokenizer('I love transformers'); - * + * * // Generate waveform * const { waveform } = await model(inputs); * // Tensor { @@ -5671,48 +5653,47 @@ export class VitsModel extends VitsPreTrainedModel { ////////////////////////////////////////////////// // Segformer models -export class SegformerPreTrainedModel extends PreTrainedModel { } +export class SegformerPreTrainedModel extends PreTrainedModel {} /** * The bare SegFormer encoder (Mix-Transformer) outputting raw hidden-states without any specific head on top. */ -export class SegformerModel extends SegformerPreTrainedModel { } +export class SegformerModel extends SegformerPreTrainedModel {} /** * SegFormer Model transformer with an image classification head on top (a linear layer on top of the final hidden states) e.g. for ImageNet. */ -export class SegformerForImageClassification extends SegformerPreTrainedModel { } +export class SegformerForImageClassification extends SegformerPreTrainedModel {} /** * SegFormer Model transformer with an all-MLP decode head on top e.g. for ADE20k, CityScapes. */ -export class SegformerForSemanticSegmentation extends SegformerPreTrainedModel { } +export class SegformerForSemanticSegmentation extends SegformerPreTrainedModel {} ////////////////////////////////////////////////// ////////////////////////////////////////////////// // StableLm models -export class StableLmPreTrainedModel extends PreTrainedModel { } +export class StableLmPreTrainedModel extends PreTrainedModel {} /** * The bare StableLm Model transformer outputting raw hidden-states without any specific head on top. */ -export class StableLmModel extends StableLmPreTrainedModel { } +export class StableLmModel extends StableLmPreTrainedModel {} /** * StableLm Model with a `language modeling` head on top for Causal Language Modeling (with past). */ -export class StableLmForCausalLM extends StableLmPreTrainedModel { } +export class StableLmForCausalLM extends StableLmPreTrainedModel {} ////////////////////////////////////////////////// - ////////////////////////////////////////////////// -export class EfficientNetPreTrainedModel extends PreTrainedModel { } +export class EfficientNetPreTrainedModel extends PreTrainedModel {} /** * The bare EfficientNet model outputting raw features without any specific head on top. */ -export class EfficientNetModel extends EfficientNetPreTrainedModel { } +export class EfficientNetModel extends EfficientNetPreTrainedModel {} /** * EfficientNet Model with an image classification head on top (a linear layer on top of the pooled features). @@ -5729,36 +5710,36 @@ export class EfficientNetForImageClassification extends EfficientNetPreTrainedMo ////////////////////////////////////////////////// // Musicgen models -export class MusicgenPreTrainedModel extends PreTrainedModel { } +export class MusicgenPreTrainedModel extends PreTrainedModel {} /** * The bare Musicgen decoder model outputting raw hidden-states without any specific head on top. */ -export class MusicgenModel extends MusicgenPreTrainedModel { } +export class MusicgenModel extends MusicgenPreTrainedModel {} /** * The MusicGen decoder model with a language modelling head on top. */ -export class MusicgenForCausalLM extends MusicgenPreTrainedModel { } +export class MusicgenForCausalLM extends MusicgenPreTrainedModel {} /** * The composite MusicGen model with a text encoder, audio encoder and Musicgen decoder, * for music generation tasks with one or both of text and audio prompts. - * + * * **Example:** Generate music from text with `Xenova/musicgen-small`. * ```javascript * import { AutoTokenizer, MusicgenForConditionalGeneration } from '@huggingface/transformers'; - * + * * // Load tokenizer and model * const tokenizer = await AutoTokenizer.from_pretrained('Xenova/musicgen-small'); * const model = await MusicgenForConditionalGeneration.from_pretrained( * 'Xenova/musicgen-small', { dtype: 'fp32' } * ); - * + * * // Prepare text input * const prompt = '80s pop track with bassy drums and synth'; * const inputs = tokenizer(prompt); - * + * * // Generate audio * const audio_values = await model.generate({ * ...inputs, @@ -5766,24 +5747,25 @@ export class MusicgenForCausalLM extends MusicgenPreTrainedModel { } * do_sample: true, * guidance_scale: 3, * }); - * + * * // (Optional) Write the output to a WAV file * import wavefile from 'wavefile'; * import fs from 'fs'; - * + * * const wav = new wavefile.WaveFile(); * wav.fromScratch(1, model.config.audio_encoder.sampling_rate, '32f', audio_values.data); * fs.writeFileSync('musicgen_out.wav', wav.toBuffer()); * ``` */ -export class MusicgenForConditionalGeneration extends PreTrainedModel { // NOTE: not MusicgenPreTrainedModel +export class MusicgenForConditionalGeneration extends PreTrainedModel { + // NOTE: not MusicgenPreTrainedModel forward_params = [ - 'input_ids', - 'attention_mask', - 'encoder_outputs', - 'decoder_input_ids', - 'decoder_attention_mask', - 'past_key_values', + "input_ids", + "attention_mask", + "encoder_outputs", + "decoder_input_ids", + "decoder_attention_mask", + "past_key_values", ]; /** @@ -5795,7 +5777,7 @@ export class MusicgenForConditionalGeneration extends PreTrainedModel { // NOTE: _apply_and_filter_by_delay_pattern_mask(outputs) { const [bs_x_codebooks, seqLength] = outputs.dims; const num_codebooks = this.config.decoder.num_codebooks; - const upperBound = (seqLength - num_codebooks); + const upperBound = seqLength - num_codebooks; let newDataSize = 0; for (let i = 0; i < outputs.size; ++i) { @@ -5803,7 +5785,7 @@ export class MusicgenForConditionalGeneration extends PreTrainedModel { // NOTE: continue; } - const row = (i % seqLength); + const row = i % seqLength; const col = Math.floor(i / seqLength) % num_codebooks; const diff = row - col; @@ -5815,20 +5797,15 @@ export class MusicgenForConditionalGeneration extends PreTrainedModel { // NOTE: const batch_size = Math.floor(bs_x_codebooks / num_codebooks); const inferred = newDataSize / (batch_size * num_codebooks); // TODO: assert `inferred` is an integer - return new Tensor( - outputs.type, - outputs.data.slice(0, newDataSize), - [batch_size, num_codebooks, inferred] - ); + return new Tensor(outputs.type, outputs.data.slice(0, newDataSize), [batch_size, num_codebooks, inferred]); } - prepare_inputs_for_generation(input_ids, model_inputs, generation_config) { // apply the delay pattern mask let clonedInputIds = structuredClone(input_ids); for (let i = 0; i < clonedInputIds.length; ++i) { for (let j = 0; j < clonedInputIds[i].length; ++j) { - if ((i % this.config.decoder.num_codebooks) >= j) { + if (i % this.config.decoder.num_codebooks >= j) { clonedInputIds[i][j] = BigInt(this.config.decoder.pad_token_id); } } @@ -5850,16 +5827,15 @@ export class MusicgenForConditionalGeneration extends PreTrainedModel { // NOTE: * @returns {Promise} The output of the model, which can contain the generated token ids, attentions, and scores. */ async generate(options) { - const output_ids = await super.generate(options); // apply the pattern mask to the final ids // tensor: int64[1,batch_size,4,chunk_length] - const audio_codes = this._apply_and_filter_by_delay_pattern_mask( - /** @type {Tensor} */(output_ids) - ).unsqueeze_(0); // append the frame dimension back to the audio codes + const audio_codes = this._apply_and_filter_by_delay_pattern_mask(/** @type {Tensor} */ (output_ids)).unsqueeze_( + 0, + ); // append the frame dimension back to the audio codes - const { audio_values } = await sessionRun(this.sessions['encodec_decode'], { audio_codes }) + const { audio_values } = await sessionRun(this.sessions["encodec_decode"], { audio_codes }); return audio_values; } @@ -5868,12 +5844,12 @@ export class MusicgenForConditionalGeneration extends PreTrainedModel { // NOTE: ////////////////////////////////////////////////// // MobileNetV1 models -export class MobileNetV1PreTrainedModel extends PreTrainedModel { } +export class MobileNetV1PreTrainedModel extends PreTrainedModel {} /** * The bare MobileNetV1 model outputting raw hidden-states without any specific head on top. */ -export class MobileNetV1Model extends MobileNetV1PreTrainedModel { } +export class MobileNetV1Model extends MobileNetV1PreTrainedModel {} /** * MobileNetV1 model with an image classification head on top (a linear layer on top of the pooled features), @@ -5891,12 +5867,12 @@ export class MobileNetV1ForImageClassification extends MobileNetV1PreTrainedMode ////////////////////////////////////////////////// // MobileNetV2 models -export class MobileNetV2PreTrainedModel extends PreTrainedModel { } +export class MobileNetV2PreTrainedModel extends PreTrainedModel {} /** * The bare MobileNetV2 model outputting raw hidden-states without any specific head on top. */ -export class MobileNetV2Model extends MobileNetV2PreTrainedModel { } +export class MobileNetV2Model extends MobileNetV2PreTrainedModel {} /** * MobileNetV2 model with an image classification head on top (a linear layer on top of the pooled features), @@ -5914,12 +5890,12 @@ export class MobileNetV2ForImageClassification extends MobileNetV2PreTrainedMode ////////////////////////////////////////////////// // MobileNetV3 models -export class MobileNetV3PreTrainedModel extends PreTrainedModel { } +export class MobileNetV3PreTrainedModel extends PreTrainedModel {} /** * The bare MobileNetV3 model outputting raw hidden-states without any specific head on top. */ -export class MobileNetV3Model extends MobileNetV3PreTrainedModel { } +export class MobileNetV3Model extends MobileNetV3PreTrainedModel {} /** * MobileNetV3 model with an image classification head on top (a linear layer on top of the pooled features), @@ -5937,12 +5913,12 @@ export class MobileNetV3ForImageClassification extends MobileNetV3PreTrainedMode ////////////////////////////////////////////////// // MobileNetV4 models -export class MobileNetV4PreTrainedModel extends PreTrainedModel { } +export class MobileNetV4PreTrainedModel extends PreTrainedModel {} /** * The bare MobileNetV4 model outputting raw hidden-states without any specific head on top. */ -export class MobileNetV4Model extends MobileNetV4PreTrainedModel { } +export class MobileNetV4Model extends MobileNetV4PreTrainedModel {} /** * MobileNetV4 model with an image classification head on top (a linear layer on top of the pooled features), @@ -5960,13 +5936,13 @@ export class MobileNetV4ForImageClassification extends MobileNetV4PreTrainedMode ////////////////////////////////////////////////// // Decision Transformer models -export class DecisionTransformerPreTrainedModel extends PreTrainedModel { } +export class DecisionTransformerPreTrainedModel extends PreTrainedModel {} /** * The model builds upon the GPT2 architecture to perform autoregressive prediction of actions in an offline RL setting. * Refer to the paper for more details: https://arxiv.org/abs/2106.01345 */ -export class DecisionTransformerModel extends DecisionTransformerPreTrainedModel { } +export class DecisionTransformerModel extends DecisionTransformerPreTrainedModel {} ////////////////////////////////////////////////// @@ -5986,27 +5962,28 @@ export class PretrainedMixin { static MODEL_CLASS_MAPPINGS = null; /** - * Whether to attempt to instantiate the base class (`PretrainedModel`) if + * Whether to attempt to instantiate the base class (`PretrainedModel`) if * the model type is not found in the mapping. */ static BASE_IF_FAIL = false; - /** @type {typeof PreTrainedModel.from_pretrained} */ - static async from_pretrained(pretrained_model_name_or_path, { - progress_callback = null, - config = null, - cache_dir = null, - local_files_only = false, - revision = 'main', - model_file_name = null, - subfolder = 'onnx', - device = null, - dtype = null, - use_external_data_format = null, - session_options = {}, - } = {}) { - + static async from_pretrained( + pretrained_model_name_or_path, + { + progress_callback = null, + config = null, + cache_dir = null, + local_files_only = false, + revision = "main", + model_file_name = null, + subfolder = "onnx", + device = null, + dtype = null, + use_external_data_format = null, + session_options = {}, + } = {}, + ) { const options = { progress_callback, config, @@ -6019,7 +5996,7 @@ export class PretrainedMixin { dtype, use_external_data_format, session_options, - } + }; options.config = await AutoConfig.from_pretrained(pretrained_model_name_or_path, options); if (!this.MODEL_CLASS_MAPPINGS) { @@ -6035,385 +6012,382 @@ export class PretrainedMixin { } if (this.BASE_IF_FAIL) { - console.warn(`Unknown model class "${options.config.model_type}", attempting to construct from base class.`); + console.warn( + `Unknown model class "${options.config.model_type}", attempting to construct from base class.`, + ); return await PreTrainedModel.from_pretrained(pretrained_model_name_or_path, options); } else { - throw Error(`Unsupported model type: ${options.config.model_type}`) + throw Error(`Unsupported model type: ${options.config.model_type}`); } } } const MODEL_MAPPING_NAMES_ENCODER_ONLY = new Map([ - ['bert', ['BertModel', BertModel]], - ['nomic_bert', ['NomicBertModel', NomicBertModel]], - ['roformer', ['RoFormerModel', RoFormerModel]], - ['electra', ['ElectraModel', ElectraModel]], - ['esm', ['EsmModel', EsmModel]], - ['convbert', ['ConvBertModel', ConvBertModel]], - ['camembert', ['CamembertModel', CamembertModel]], - ['deberta', ['DebertaModel', DebertaModel]], - ['deberta-v2', ['DebertaV2Model', DebertaV2Model]], - ['mpnet', ['MPNetModel', MPNetModel]], - ['albert', ['AlbertModel', AlbertModel]], - ['distilbert', ['DistilBertModel', DistilBertModel]], - ['roberta', ['RobertaModel', RobertaModel]], - ['xlm', ['XLMModel', XLMModel]], - ['xlm-roberta', ['XLMRobertaModel', XLMRobertaModel]], - ['clap', ['ClapModel', ClapModel]], - ['clip', ['CLIPModel', CLIPModel]], - ['clipseg', ['CLIPSegModel', CLIPSegModel]], - ['chinese_clip', ['ChineseCLIPModel', ChineseCLIPModel]], - ['siglip', ['SiglipModel', SiglipModel]], - ['mobilebert', ['MobileBertModel', MobileBertModel]], - ['squeezebert', ['SqueezeBertModel', SqueezeBertModel]], - ['wav2vec2', ['Wav2Vec2Model', Wav2Vec2Model]], - ['wav2vec2-bert', ['Wav2Vec2BertModel', Wav2Vec2BertModel]], - ['unispeech', ['UniSpeechModel', UniSpeechModel]], - ['unispeech-sat', ['UniSpeechSatModel', UniSpeechSatModel]], - ['hubert', ['HubertModel', HubertModel]], - ['wavlm', ['WavLMModel', WavLMModel]], - ['audio-spectrogram-transformer', ['ASTModel', ASTModel]], - ['vits', ['VitsModel', VitsModel]], - ['pyannote', ['PyAnnoteModel', PyAnnoteModel]], - ['wespeaker-resnet', ['WeSpeakerResNetModel', WeSpeakerResNetModel]], - - ['detr', ['DetrModel', DetrModel]], - ['rt_detr', ['RTDetrModel', RTDetrModel]], - ['table-transformer', ['TableTransformerModel', TableTransformerModel]], - ['vit', ['ViTModel', ViTModel]], - ['pvt', ['PvtModel', PvtModel]], - ['vit_msn', ['ViTMSNModel', ViTMSNModel]], - ['vit_mae', ['ViTMAEModel', ViTMAEModel]], - ['groupvit', ['GroupViTModel', GroupViTModel]], - ['fastvit', ['FastViTModel', FastViTModel]], - ['mobilevit', ['MobileViTModel', MobileViTModel]], - ['mobilevitv2', ['MobileViTV2Model', MobileViTV2Model]], - ['owlvit', ['OwlViTModel', OwlViTModel]], - ['owlv2', ['Owlv2Model', Owlv2Model]], - ['beit', ['BeitModel', BeitModel]], - ['deit', ['DeiTModel', DeiTModel]], - ['hiera', ['HieraModel', HieraModel]], - ['convnext', ['ConvNextModel', ConvNextModel]], - ['convnextv2', ['ConvNextV2Model', ConvNextV2Model]], - ['dinov2', ['Dinov2Model', Dinov2Model]], - ['resnet', ['ResNetModel', ResNetModel]], - ['swin', ['SwinModel', SwinModel]], - ['swin2sr', ['Swin2SRModel', Swin2SRModel]], - ['donut-swin', ['DonutSwinModel', DonutSwinModel]], - ['yolos', ['YolosModel', YolosModel]], - ['dpt', ['DPTModel', DPTModel]], - ['glpn', ['GLPNModel', GLPNModel]], - - ['hifigan', ['SpeechT5HifiGan', SpeechT5HifiGan]], - ['efficientnet', ['EfficientNetModel', EfficientNetModel]], - - ['decision_transformer', ['DecisionTransformerModel', DecisionTransformerModel]], - - ['mobilenet_v1', ['MobileNetV1Model', MobileNetV1Model]], - ['mobilenet_v2', ['MobileNetV2Model', MobileNetV2Model]], - ['mobilenet_v3', ['MobileNetV3Model', MobileNetV3Model]], - ['mobilenet_v4', ['MobileNetV4Model', MobileNetV4Model]], - - ['maskformer', ['MaskFormerModel', MaskFormerModel]], + ["bert", ["BertModel", BertModel]], + ["nomic_bert", ["NomicBertModel", NomicBertModel]], + ["roformer", ["RoFormerModel", RoFormerModel]], + ["electra", ["ElectraModel", ElectraModel]], + ["esm", ["EsmModel", EsmModel]], + ["convbert", ["ConvBertModel", ConvBertModel]], + ["camembert", ["CamembertModel", CamembertModel]], + ["deberta", ["DebertaModel", DebertaModel]], + ["deberta-v2", ["DebertaV2Model", DebertaV2Model]], + ["mpnet", ["MPNetModel", MPNetModel]], + ["albert", ["AlbertModel", AlbertModel]], + ["distilbert", ["DistilBertModel", DistilBertModel]], + ["roberta", ["RobertaModel", RobertaModel]], + ["xlm", ["XLMModel", XLMModel]], + ["xlm-roberta", ["XLMRobertaModel", XLMRobertaModel]], + ["clap", ["ClapModel", ClapModel]], + ["clip", ["CLIPModel", CLIPModel]], + ["clipseg", ["CLIPSegModel", CLIPSegModel]], + ["chinese_clip", ["ChineseCLIPModel", ChineseCLIPModel]], + ["siglip", ["SiglipModel", SiglipModel]], + ["mobilebert", ["MobileBertModel", MobileBertModel]], + ["squeezebert", ["SqueezeBertModel", SqueezeBertModel]], + ["wav2vec2", ["Wav2Vec2Model", Wav2Vec2Model]], + ["wav2vec2-bert", ["Wav2Vec2BertModel", Wav2Vec2BertModel]], + ["unispeech", ["UniSpeechModel", UniSpeechModel]], + ["unispeech-sat", ["UniSpeechSatModel", UniSpeechSatModel]], + ["hubert", ["HubertModel", HubertModel]], + ["wavlm", ["WavLMModel", WavLMModel]], + ["audio-spectrogram-transformer", ["ASTModel", ASTModel]], + ["vits", ["VitsModel", VitsModel]], + ["pyannote", ["PyAnnoteModel", PyAnnoteModel]], + ["wespeaker-resnet", ["WeSpeakerResNetModel", WeSpeakerResNetModel]], + + ["detr", ["DetrModel", DetrModel]], + ["rt_detr", ["RTDetrModel", RTDetrModel]], + ["table-transformer", ["TableTransformerModel", TableTransformerModel]], + ["vit", ["ViTModel", ViTModel]], + ["pvt", ["PvtModel", PvtModel]], + ["vit_msn", ["ViTMSNModel", ViTMSNModel]], + ["vit_mae", ["ViTMAEModel", ViTMAEModel]], + ["groupvit", ["GroupViTModel", GroupViTModel]], + ["fastvit", ["FastViTModel", FastViTModel]], + ["mobilevit", ["MobileViTModel", MobileViTModel]], + ["mobilevitv2", ["MobileViTV2Model", MobileViTV2Model]], + ["owlvit", ["OwlViTModel", OwlViTModel]], + ["owlv2", ["Owlv2Model", Owlv2Model]], + ["beit", ["BeitModel", BeitModel]], + ["deit", ["DeiTModel", DeiTModel]], + ["hiera", ["HieraModel", HieraModel]], + ["convnext", ["ConvNextModel", ConvNextModel]], + ["convnextv2", ["ConvNextV2Model", ConvNextV2Model]], + ["dinov2", ["Dinov2Model", Dinov2Model]], + ["resnet", ["ResNetModel", ResNetModel]], + ["swin", ["SwinModel", SwinModel]], + ["swin2sr", ["Swin2SRModel", Swin2SRModel]], + ["donut-swin", ["DonutSwinModel", DonutSwinModel]], + ["yolos", ["YolosModel", YolosModel]], + ["dpt", ["DPTModel", DPTModel]], + ["glpn", ["GLPNModel", GLPNModel]], + + ["hifigan", ["SpeechT5HifiGan", SpeechT5HifiGan]], + ["efficientnet", ["EfficientNetModel", EfficientNetModel]], + + ["decision_transformer", ["DecisionTransformerModel", DecisionTransformerModel]], + + ["mobilenet_v1", ["MobileNetV1Model", MobileNetV1Model]], + ["mobilenet_v2", ["MobileNetV2Model", MobileNetV2Model]], + ["mobilenet_v3", ["MobileNetV3Model", MobileNetV3Model]], + ["mobilenet_v4", ["MobileNetV4Model", MobileNetV4Model]], + + ["maskformer", ["MaskFormerModel", MaskFormerModel]], ]); const MODEL_MAPPING_NAMES_ENCODER_DECODER = new Map([ - ['t5', ['T5Model', T5Model]], - ['longt5', ['LongT5Model', LongT5Model]], - ['mt5', ['MT5Model', MT5Model]], - ['bart', ['BartModel', BartModel]], - ['mbart', ['MBartModel', MBartModel]], - ['marian', ['MarianModel', MarianModel]], - ['whisper', ['WhisperModel', WhisperModel]], - ['m2m_100', ['M2M100Model', M2M100Model]], - ['blenderbot', ['BlenderbotModel', BlenderbotModel]], - ['blenderbot-small', ['BlenderbotSmallModel', BlenderbotSmallModel]], + ["t5", ["T5Model", T5Model]], + ["longt5", ["LongT5Model", LongT5Model]], + ["mt5", ["MT5Model", MT5Model]], + ["bart", ["BartModel", BartModel]], + ["mbart", ["MBartModel", MBartModel]], + ["marian", ["MarianModel", MarianModel]], + ["whisper", ["WhisperModel", WhisperModel]], + ["m2m_100", ["M2M100Model", M2M100Model]], + ["blenderbot", ["BlenderbotModel", BlenderbotModel]], + ["blenderbot-small", ["BlenderbotSmallModel", BlenderbotSmallModel]], ]); - const MODEL_MAPPING_NAMES_DECODER_ONLY = new Map([ - ['bloom', ['BloomModel', BloomModel]], - ['jais', ['JAISModel', JAISModel]], - ['gpt2', ['GPT2Model', GPT2Model]], - ['gptj', ['GPTJModel', GPTJModel]], - ['gpt_bigcode', ['GPTBigCodeModel', GPTBigCodeModel]], - ['gpt_neo', ['GPTNeoModel', GPTNeoModel]], - ['gpt_neox', ['GPTNeoXModel', GPTNeoXModel]], - ['codegen', ['CodeGenModel', CodeGenModel]], - ['llama', ['LlamaModel', LlamaModel]], - ['olmo', ['OlmoModel', OlmoModel]], - ['mobilellm', ['MobileLLMModel', MobileLLMModel]], - ['granite', ['GraniteModel', GraniteModel]], - ['cohere', ['CohereModel', CohereModel]], - ['gemma', ['GemmaModel', GemmaModel]], - ['gemma2', ['Gemma2Model', Gemma2Model]], - ['openelm', ['OpenELMModel', OpenELMModel]], - ['qwen2', ['Qwen2Model', Qwen2Model]], - ['phi', ['PhiModel', PhiModel]], - ['phi3', ['Phi3Model', Phi3Model]], - ['mpt', ['MptModel', MptModel]], - ['opt', ['OPTModel', OPTModel]], - ['mistral', ['MistralModel', MistralModel]], - ['starcoder2', ['Starcoder2Model', Starcoder2Model]], - ['falcon', ['FalconModel', FalconModel]], - ['stablelm', ['StableLmModel', StableLmModel]], + ["bloom", ["BloomModel", BloomModel]], + ["jais", ["JAISModel", JAISModel]], + ["gpt2", ["GPT2Model", GPT2Model]], + ["gptj", ["GPTJModel", GPTJModel]], + ["gpt_bigcode", ["GPTBigCodeModel", GPTBigCodeModel]], + ["gpt_neo", ["GPTNeoModel", GPTNeoModel]], + ["gpt_neox", ["GPTNeoXModel", GPTNeoXModel]], + ["codegen", ["CodeGenModel", CodeGenModel]], + ["llama", ["LlamaModel", LlamaModel]], + ["olmo", ["OlmoModel", OlmoModel]], + ["mobilellm", ["MobileLLMModel", MobileLLMModel]], + ["granite", ["GraniteModel", GraniteModel]], + ["cohere", ["CohereModel", CohereModel]], + ["gemma", ["GemmaModel", GemmaModel]], + ["gemma2", ["Gemma2Model", Gemma2Model]], + ["openelm", ["OpenELMModel", OpenELMModel]], + ["qwen2", ["Qwen2Model", Qwen2Model]], + ["phi", ["PhiModel", PhiModel]], + ["phi3", ["Phi3Model", Phi3Model]], + ["mpt", ["MptModel", MptModel]], + ["opt", ["OPTModel", OPTModel]], + ["mistral", ["MistralModel", MistralModel]], + ["starcoder2", ["Starcoder2Model", Starcoder2Model]], + ["falcon", ["FalconModel", FalconModel]], + ["stablelm", ["StableLmModel", StableLmModel]], ]); const MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES = new Map([ - ['speecht5', ['SpeechT5ForSpeechToText', SpeechT5ForSpeechToText]], - ['whisper', ['WhisperForConditionalGeneration', WhisperForConditionalGeneration]], + ["speecht5", ["SpeechT5ForSpeechToText", SpeechT5ForSpeechToText]], + ["whisper", ["WhisperForConditionalGeneration", WhisperForConditionalGeneration]], ]); const MODEL_FOR_TEXT_TO_SPECTROGRAM_MAPPING_NAMES = new Map([ - ['speecht5', ['SpeechT5ForTextToSpeech', SpeechT5ForTextToSpeech]], + ["speecht5", ["SpeechT5ForTextToSpeech", SpeechT5ForTextToSpeech]], ]); const MODEL_FOR_TEXT_TO_WAVEFORM_MAPPING_NAMES = new Map([ - ['vits', ['VitsModel', VitsModel]], - ['musicgen', ['MusicgenForConditionalGeneration', MusicgenForConditionalGeneration]], + ["vits", ["VitsModel", VitsModel]], + ["musicgen", ["MusicgenForConditionalGeneration", MusicgenForConditionalGeneration]], ]); const MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES = new Map([ - ['bert', ['BertForSequenceClassification', BertForSequenceClassification]], - ['roformer', ['RoFormerForSequenceClassification', RoFormerForSequenceClassification]], - ['electra', ['ElectraForSequenceClassification', ElectraForSequenceClassification]], - ['esm', ['EsmForSequenceClassification', EsmForSequenceClassification]], - ['convbert', ['ConvBertForSequenceClassification', ConvBertForSequenceClassification]], - ['camembert', ['CamembertForSequenceClassification', CamembertForSequenceClassification]], - ['deberta', ['DebertaForSequenceClassification', DebertaForSequenceClassification]], - ['deberta-v2', ['DebertaV2ForSequenceClassification', DebertaV2ForSequenceClassification]], - ['mpnet', ['MPNetForSequenceClassification', MPNetForSequenceClassification]], - ['albert', ['AlbertForSequenceClassification', AlbertForSequenceClassification]], - ['distilbert', ['DistilBertForSequenceClassification', DistilBertForSequenceClassification]], - ['roberta', ['RobertaForSequenceClassification', RobertaForSequenceClassification]], - ['xlm', ['XLMForSequenceClassification', XLMForSequenceClassification]], - ['xlm-roberta', ['XLMRobertaForSequenceClassification', XLMRobertaForSequenceClassification]], - ['bart', ['BartForSequenceClassification', BartForSequenceClassification]], - ['mbart', ['MBartForSequenceClassification', MBartForSequenceClassification]], - ['mobilebert', ['MobileBertForSequenceClassification', MobileBertForSequenceClassification]], - ['squeezebert', ['SqueezeBertForSequenceClassification', SqueezeBertForSequenceClassification]], + ["bert", ["BertForSequenceClassification", BertForSequenceClassification]], + ["roformer", ["RoFormerForSequenceClassification", RoFormerForSequenceClassification]], + ["electra", ["ElectraForSequenceClassification", ElectraForSequenceClassification]], + ["esm", ["EsmForSequenceClassification", EsmForSequenceClassification]], + ["convbert", ["ConvBertForSequenceClassification", ConvBertForSequenceClassification]], + ["camembert", ["CamembertForSequenceClassification", CamembertForSequenceClassification]], + ["deberta", ["DebertaForSequenceClassification", DebertaForSequenceClassification]], + ["deberta-v2", ["DebertaV2ForSequenceClassification", DebertaV2ForSequenceClassification]], + ["mpnet", ["MPNetForSequenceClassification", MPNetForSequenceClassification]], + ["albert", ["AlbertForSequenceClassification", AlbertForSequenceClassification]], + ["distilbert", ["DistilBertForSequenceClassification", DistilBertForSequenceClassification]], + ["roberta", ["RobertaForSequenceClassification", RobertaForSequenceClassification]], + ["xlm", ["XLMForSequenceClassification", XLMForSequenceClassification]], + ["xlm-roberta", ["XLMRobertaForSequenceClassification", XLMRobertaForSequenceClassification]], + ["bart", ["BartForSequenceClassification", BartForSequenceClassification]], + ["mbart", ["MBartForSequenceClassification", MBartForSequenceClassification]], + ["mobilebert", ["MobileBertForSequenceClassification", MobileBertForSequenceClassification]], + ["squeezebert", ["SqueezeBertForSequenceClassification", SqueezeBertForSequenceClassification]], ]); const MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES = new Map([ - ['bert', ['BertForTokenClassification', BertForTokenClassification]], - ['roformer', ['RoFormerForTokenClassification', RoFormerForTokenClassification]], - ['electra', ['ElectraForTokenClassification', ElectraForTokenClassification]], - ['esm', ['EsmForTokenClassification', EsmForTokenClassification]], - ['convbert', ['ConvBertForTokenClassification', ConvBertForTokenClassification]], - ['camembert', ['CamembertForTokenClassification', CamembertForTokenClassification]], - ['deberta', ['DebertaForTokenClassification', DebertaForTokenClassification]], - ['deberta-v2', ['DebertaV2ForTokenClassification', DebertaV2ForTokenClassification]], - ['mpnet', ['MPNetForTokenClassification', MPNetForTokenClassification]], - ['distilbert', ['DistilBertForTokenClassification', DistilBertForTokenClassification]], - ['roberta', ['RobertaForTokenClassification', RobertaForTokenClassification]], - ['xlm', ['XLMForTokenClassification', XLMForTokenClassification]], - ['xlm-roberta', ['XLMRobertaForTokenClassification', XLMRobertaForTokenClassification]], + ["bert", ["BertForTokenClassification", BertForTokenClassification]], + ["roformer", ["RoFormerForTokenClassification", RoFormerForTokenClassification]], + ["electra", ["ElectraForTokenClassification", ElectraForTokenClassification]], + ["esm", ["EsmForTokenClassification", EsmForTokenClassification]], + ["convbert", ["ConvBertForTokenClassification", ConvBertForTokenClassification]], + ["camembert", ["CamembertForTokenClassification", CamembertForTokenClassification]], + ["deberta", ["DebertaForTokenClassification", DebertaForTokenClassification]], + ["deberta-v2", ["DebertaV2ForTokenClassification", DebertaV2ForTokenClassification]], + ["mpnet", ["MPNetForTokenClassification", MPNetForTokenClassification]], + ["distilbert", ["DistilBertForTokenClassification", DistilBertForTokenClassification]], + ["roberta", ["RobertaForTokenClassification", RobertaForTokenClassification]], + ["xlm", ["XLMForTokenClassification", XLMForTokenClassification]], + ["xlm-roberta", ["XLMRobertaForTokenClassification", XLMRobertaForTokenClassification]], ]); const MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES = new Map([ - ['t5', ['T5ForConditionalGeneration', T5ForConditionalGeneration]], - ['longt5', ['LongT5ForConditionalGeneration', LongT5ForConditionalGeneration]], - ['mt5', ['MT5ForConditionalGeneration', MT5ForConditionalGeneration]], - ['bart', ['BartForConditionalGeneration', BartForConditionalGeneration]], - ['mbart', ['MBartForConditionalGeneration', MBartForConditionalGeneration]], - ['marian', ['MarianMTModel', MarianMTModel]], - ['m2m_100', ['M2M100ForConditionalGeneration', M2M100ForConditionalGeneration]], - ['blenderbot', ['BlenderbotForConditionalGeneration', BlenderbotForConditionalGeneration]], - ['blenderbot-small', ['BlenderbotSmallForConditionalGeneration', BlenderbotSmallForConditionalGeneration]], + ["t5", ["T5ForConditionalGeneration", T5ForConditionalGeneration]], + ["longt5", ["LongT5ForConditionalGeneration", LongT5ForConditionalGeneration]], + ["mt5", ["MT5ForConditionalGeneration", MT5ForConditionalGeneration]], + ["bart", ["BartForConditionalGeneration", BartForConditionalGeneration]], + ["mbart", ["MBartForConditionalGeneration", MBartForConditionalGeneration]], + ["marian", ["MarianMTModel", MarianMTModel]], + ["m2m_100", ["M2M100ForConditionalGeneration", M2M100ForConditionalGeneration]], + ["blenderbot", ["BlenderbotForConditionalGeneration", BlenderbotForConditionalGeneration]], + ["blenderbot-small", ["BlenderbotSmallForConditionalGeneration", BlenderbotSmallForConditionalGeneration]], ]); const MODEL_FOR_CAUSAL_LM_MAPPING_NAMES = new Map([ - ['bloom', ['BloomForCausalLM', BloomForCausalLM]], - ['gpt2', ['GPT2LMHeadModel', GPT2LMHeadModel]], - ['jais', ['JAISLMHeadModel', JAISLMHeadModel]], - ['gptj', ['GPTJForCausalLM', GPTJForCausalLM]], - ['gpt_bigcode', ['GPTBigCodeForCausalLM', GPTBigCodeForCausalLM]], - ['gpt_neo', ['GPTNeoForCausalLM', GPTNeoForCausalLM]], - ['gpt_neox', ['GPTNeoXForCausalLM', GPTNeoXForCausalLM]], - ['codegen', ['CodeGenForCausalLM', CodeGenForCausalLM]], - ['llama', ['LlamaForCausalLM', LlamaForCausalLM]], - ['olmo', ['OlmoForCausalLM', OlmoForCausalLM]], - ['mobilellm', ['MobileLLMForCausalLM', MobileLLMForCausalLM]], - ['granite', ['GraniteForCausalLM', GraniteForCausalLM]], - ['cohere', ['CohereForCausalLM', CohereForCausalLM]], - ['gemma', ['GemmaForCausalLM', GemmaForCausalLM]], - ['gemma2', ['Gemma2ForCausalLM', Gemma2ForCausalLM]], - ['openelm', ['OpenELMForCausalLM', OpenELMForCausalLM]], - ['qwen2', ['Qwen2ForCausalLM', Qwen2ForCausalLM]], - ['phi', ['PhiForCausalLM', PhiForCausalLM]], - ['phi3', ['Phi3ForCausalLM', Phi3ForCausalLM]], - ['mpt', ['MptForCausalLM', MptForCausalLM]], - ['opt', ['OPTForCausalLM', OPTForCausalLM]], - ['mbart', ['MBartForCausalLM', MBartForCausalLM]], - ['mistral', ['MistralForCausalLM', MistralForCausalLM]], - ['starcoder2', ['Starcoder2ForCausalLM', Starcoder2ForCausalLM]], - ['falcon', ['FalconForCausalLM', FalconForCausalLM]], - ['trocr', ['TrOCRForCausalLM', TrOCRForCausalLM]], - ['stablelm', ['StableLmForCausalLM', StableLmForCausalLM]], + ["bloom", ["BloomForCausalLM", BloomForCausalLM]], + ["gpt2", ["GPT2LMHeadModel", GPT2LMHeadModel]], + ["jais", ["JAISLMHeadModel", JAISLMHeadModel]], + ["gptj", ["GPTJForCausalLM", GPTJForCausalLM]], + ["gpt_bigcode", ["GPTBigCodeForCausalLM", GPTBigCodeForCausalLM]], + ["gpt_neo", ["GPTNeoForCausalLM", GPTNeoForCausalLM]], + ["gpt_neox", ["GPTNeoXForCausalLM", GPTNeoXForCausalLM]], + ["codegen", ["CodeGenForCausalLM", CodeGenForCausalLM]], + ["llama", ["LlamaForCausalLM", LlamaForCausalLM]], + ["olmo", ["OlmoForCausalLM", OlmoForCausalLM]], + ["mobilellm", ["MobileLLMForCausalLM", MobileLLMForCausalLM]], + ["granite", ["GraniteForCausalLM", GraniteForCausalLM]], + ["cohere", ["CohereForCausalLM", CohereForCausalLM]], + ["gemma", ["GemmaForCausalLM", GemmaForCausalLM]], + ["gemma2", ["Gemma2ForCausalLM", Gemma2ForCausalLM]], + ["openelm", ["OpenELMForCausalLM", OpenELMForCausalLM]], + ["qwen2", ["Qwen2ForCausalLM", Qwen2ForCausalLM]], + ["phi", ["PhiForCausalLM", PhiForCausalLM]], + ["phi3", ["Phi3ForCausalLM", Phi3ForCausalLM]], + ["mpt", ["MptForCausalLM", MptForCausalLM]], + ["opt", ["OPTForCausalLM", OPTForCausalLM]], + ["mbart", ["MBartForCausalLM", MBartForCausalLM]], + ["mistral", ["MistralForCausalLM", MistralForCausalLM]], + ["starcoder2", ["Starcoder2ForCausalLM", Starcoder2ForCausalLM]], + ["falcon", ["FalconForCausalLM", FalconForCausalLM]], + ["trocr", ["TrOCRForCausalLM", TrOCRForCausalLM]], + ["stablelm", ["StableLmForCausalLM", StableLmForCausalLM]], ]); const MODEL_FOR_MASKED_LM_MAPPING_NAMES = new Map([ - ['bert', ['BertForMaskedLM', BertForMaskedLM]], - ['roformer', ['RoFormerForMaskedLM', RoFormerForMaskedLM]], - ['electra', ['ElectraForMaskedLM', ElectraForMaskedLM]], - ['esm', ['EsmForMaskedLM', EsmForMaskedLM]], - ['convbert', ['ConvBertForMaskedLM', ConvBertForMaskedLM]], - ['camembert', ['CamembertForMaskedLM', CamembertForMaskedLM]], - ['deberta', ['DebertaForMaskedLM', DebertaForMaskedLM]], - ['deberta-v2', ['DebertaV2ForMaskedLM', DebertaV2ForMaskedLM]], - ['mpnet', ['MPNetForMaskedLM', MPNetForMaskedLM]], - ['albert', ['AlbertForMaskedLM', AlbertForMaskedLM]], - ['distilbert', ['DistilBertForMaskedLM', DistilBertForMaskedLM]], - ['roberta', ['RobertaForMaskedLM', RobertaForMaskedLM]], - ['xlm', ['XLMWithLMHeadModel', XLMWithLMHeadModel]], - ['xlm-roberta', ['XLMRobertaForMaskedLM', XLMRobertaForMaskedLM]], - ['mobilebert', ['MobileBertForMaskedLM', MobileBertForMaskedLM]], - ['squeezebert', ['SqueezeBertForMaskedLM', SqueezeBertForMaskedLM]], + ["bert", ["BertForMaskedLM", BertForMaskedLM]], + ["roformer", ["RoFormerForMaskedLM", RoFormerForMaskedLM]], + ["electra", ["ElectraForMaskedLM", ElectraForMaskedLM]], + ["esm", ["EsmForMaskedLM", EsmForMaskedLM]], + ["convbert", ["ConvBertForMaskedLM", ConvBertForMaskedLM]], + ["camembert", ["CamembertForMaskedLM", CamembertForMaskedLM]], + ["deberta", ["DebertaForMaskedLM", DebertaForMaskedLM]], + ["deberta-v2", ["DebertaV2ForMaskedLM", DebertaV2ForMaskedLM]], + ["mpnet", ["MPNetForMaskedLM", MPNetForMaskedLM]], + ["albert", ["AlbertForMaskedLM", AlbertForMaskedLM]], + ["distilbert", ["DistilBertForMaskedLM", DistilBertForMaskedLM]], + ["roberta", ["RobertaForMaskedLM", RobertaForMaskedLM]], + ["xlm", ["XLMWithLMHeadModel", XLMWithLMHeadModel]], + ["xlm-roberta", ["XLMRobertaForMaskedLM", XLMRobertaForMaskedLM]], + ["mobilebert", ["MobileBertForMaskedLM", MobileBertForMaskedLM]], + ["squeezebert", ["SqueezeBertForMaskedLM", SqueezeBertForMaskedLM]], ]); const MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES = new Map([ - ['bert', ['BertForQuestionAnswering', BertForQuestionAnswering]], - ['roformer', ['RoFormerForQuestionAnswering', RoFormerForQuestionAnswering]], - ['electra', ['ElectraForQuestionAnswering', ElectraForQuestionAnswering]], - ['convbert', ['ConvBertForQuestionAnswering', ConvBertForQuestionAnswering]], - ['camembert', ['CamembertForQuestionAnswering', CamembertForQuestionAnswering]], - ['deberta', ['DebertaForQuestionAnswering', DebertaForQuestionAnswering]], - ['deberta-v2', ['DebertaV2ForQuestionAnswering', DebertaV2ForQuestionAnswering]], - ['mpnet', ['MPNetForQuestionAnswering', MPNetForQuestionAnswering]], - ['albert', ['AlbertForQuestionAnswering', AlbertForQuestionAnswering]], - ['distilbert', ['DistilBertForQuestionAnswering', DistilBertForQuestionAnswering]], - ['roberta', ['RobertaForQuestionAnswering', RobertaForQuestionAnswering]], - ['xlm', ['XLMForQuestionAnswering', XLMForQuestionAnswering]], - ['xlm-roberta', ['XLMRobertaForQuestionAnswering', XLMRobertaForQuestionAnswering]], - ['mobilebert', ['MobileBertForQuestionAnswering', MobileBertForQuestionAnswering]], - ['squeezebert', ['SqueezeBertForQuestionAnswering', SqueezeBertForQuestionAnswering]], + ["bert", ["BertForQuestionAnswering", BertForQuestionAnswering]], + ["roformer", ["RoFormerForQuestionAnswering", RoFormerForQuestionAnswering]], + ["electra", ["ElectraForQuestionAnswering", ElectraForQuestionAnswering]], + ["convbert", ["ConvBertForQuestionAnswering", ConvBertForQuestionAnswering]], + ["camembert", ["CamembertForQuestionAnswering", CamembertForQuestionAnswering]], + ["deberta", ["DebertaForQuestionAnswering", DebertaForQuestionAnswering]], + ["deberta-v2", ["DebertaV2ForQuestionAnswering", DebertaV2ForQuestionAnswering]], + ["mpnet", ["MPNetForQuestionAnswering", MPNetForQuestionAnswering]], + ["albert", ["AlbertForQuestionAnswering", AlbertForQuestionAnswering]], + ["distilbert", ["DistilBertForQuestionAnswering", DistilBertForQuestionAnswering]], + ["roberta", ["RobertaForQuestionAnswering", RobertaForQuestionAnswering]], + ["xlm", ["XLMForQuestionAnswering", XLMForQuestionAnswering]], + ["xlm-roberta", ["XLMRobertaForQuestionAnswering", XLMRobertaForQuestionAnswering]], + ["mobilebert", ["MobileBertForQuestionAnswering", MobileBertForQuestionAnswering]], + ["squeezebert", ["SqueezeBertForQuestionAnswering", SqueezeBertForQuestionAnswering]], ]); const MODEL_FOR_VISION_2_SEQ_MAPPING_NAMES = new Map([ - ['vision-encoder-decoder', ['VisionEncoderDecoderModel', VisionEncoderDecoderModel]], + ["vision-encoder-decoder", ["VisionEncoderDecoderModel", VisionEncoderDecoderModel]], ]); const MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES = new Map([ - ['llava', ['LlavaForConditionalGeneration', LlavaForConditionalGeneration]], - ['moondream1', ['Moondream1ForConditionalGeneration', Moondream1ForConditionalGeneration]], - ['florence2', ['Florence2ForConditionalGeneration', Florence2ForConditionalGeneration]], + ["llava", ["LlavaForConditionalGeneration", LlavaForConditionalGeneration]], + ["moondream1", ["Moondream1ForConditionalGeneration", Moondream1ForConditionalGeneration]], + ["florence2", ["Florence2ForConditionalGeneration", Florence2ForConditionalGeneration]], ]); const MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES = new Map([ - ['vision-encoder-decoder', ['VisionEncoderDecoderModel', VisionEncoderDecoderModel]], + ["vision-encoder-decoder", ["VisionEncoderDecoderModel", VisionEncoderDecoderModel]], ]); const MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES = new Map([ - ['vit', ['ViTForImageClassification', ViTForImageClassification]], - ['pvt', ['PvtForImageClassification', PvtForImageClassification]], - ['vit_msn', ['ViTMSNForImageClassification', ViTMSNForImageClassification]], - ['fastvit', ['FastViTForImageClassification', FastViTForImageClassification]], - ['mobilevit', ['MobileViTForImageClassification', MobileViTForImageClassification]], - ['mobilevitv2', ['MobileViTV2ForImageClassification', MobileViTV2ForImageClassification]], - ['beit', ['BeitForImageClassification', BeitForImageClassification]], - ['deit', ['DeiTForImageClassification', DeiTForImageClassification]], - ['hiera', ['HieraForImageClassification', HieraForImageClassification]], - ['convnext', ['ConvNextForImageClassification', ConvNextForImageClassification]], - ['convnextv2', ['ConvNextV2ForImageClassification', ConvNextV2ForImageClassification]], - ['dinov2', ['Dinov2ForImageClassification', Dinov2ForImageClassification]], - ['resnet', ['ResNetForImageClassification', ResNetForImageClassification]], - ['swin', ['SwinForImageClassification', SwinForImageClassification]], - ['segformer', ['SegformerForImageClassification', SegformerForImageClassification]], - ['efficientnet', ['EfficientNetForImageClassification', EfficientNetForImageClassification]], - ['mobilenet_v1', ['MobileNetV1ForImageClassification', MobileNetV1ForImageClassification]], - ['mobilenet_v2', ['MobileNetV2ForImageClassification', MobileNetV2ForImageClassification]], - ['mobilenet_v3', ['MobileNetV3ForImageClassification', MobileNetV3ForImageClassification]], - ['mobilenet_v4', ['MobileNetV4ForImageClassification', MobileNetV4ForImageClassification]], + ["vit", ["ViTForImageClassification", ViTForImageClassification]], + ["pvt", ["PvtForImageClassification", PvtForImageClassification]], + ["vit_msn", ["ViTMSNForImageClassification", ViTMSNForImageClassification]], + ["fastvit", ["FastViTForImageClassification", FastViTForImageClassification]], + ["mobilevit", ["MobileViTForImageClassification", MobileViTForImageClassification]], + ["mobilevitv2", ["MobileViTV2ForImageClassification", MobileViTV2ForImageClassification]], + ["beit", ["BeitForImageClassification", BeitForImageClassification]], + ["deit", ["DeiTForImageClassification", DeiTForImageClassification]], + ["hiera", ["HieraForImageClassification", HieraForImageClassification]], + ["convnext", ["ConvNextForImageClassification", ConvNextForImageClassification]], + ["convnextv2", ["ConvNextV2ForImageClassification", ConvNextV2ForImageClassification]], + ["dinov2", ["Dinov2ForImageClassification", Dinov2ForImageClassification]], + ["resnet", ["ResNetForImageClassification", ResNetForImageClassification]], + ["swin", ["SwinForImageClassification", SwinForImageClassification]], + ["segformer", ["SegformerForImageClassification", SegformerForImageClassification]], + ["efficientnet", ["EfficientNetForImageClassification", EfficientNetForImageClassification]], + ["mobilenet_v1", ["MobileNetV1ForImageClassification", MobileNetV1ForImageClassification]], + ["mobilenet_v2", ["MobileNetV2ForImageClassification", MobileNetV2ForImageClassification]], + ["mobilenet_v3", ["MobileNetV3ForImageClassification", MobileNetV3ForImageClassification]], + ["mobilenet_v4", ["MobileNetV4ForImageClassification", MobileNetV4ForImageClassification]], ]); const MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES = new Map([ - ['detr', ['DetrForObjectDetection', DetrForObjectDetection]], - ['rt_detr', ['RTDetrForObjectDetection', RTDetrForObjectDetection]], - ['table-transformer', ['TableTransformerForObjectDetection', TableTransformerForObjectDetection]], - ['yolos', ['YolosForObjectDetection', YolosForObjectDetection]], + ["detr", ["DetrForObjectDetection", DetrForObjectDetection]], + ["rt_detr", ["RTDetrForObjectDetection", RTDetrForObjectDetection]], + ["table-transformer", ["TableTransformerForObjectDetection", TableTransformerForObjectDetection]], + ["yolos", ["YolosForObjectDetection", YolosForObjectDetection]], ]); const MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING_NAMES = new Map([ - ['owlvit', ['OwlViTForObjectDetection', OwlViTForObjectDetection]], - ['owlv2', ['Owlv2ForObjectDetection', Owlv2ForObjectDetection]], + ["owlvit", ["OwlViTForObjectDetection", OwlViTForObjectDetection]], + ["owlv2", ["Owlv2ForObjectDetection", Owlv2ForObjectDetection]], ]); const MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES = new Map([ // TODO: Do not add new models here - ['detr', ['DetrForSegmentation', DetrForSegmentation]], - ['clipseg', ['CLIPSegForImageSegmentation', CLIPSegForImageSegmentation]], + ["detr", ["DetrForSegmentation", DetrForSegmentation]], + ["clipseg", ["CLIPSegForImageSegmentation", CLIPSegForImageSegmentation]], ]); const MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING_NAMES = new Map([ - ['segformer', ['SegformerForSemanticSegmentation', SegformerForSemanticSegmentation]], - ['sapiens', ['SapiensForSemanticSegmentation', SapiensForSemanticSegmentation]], + ["segformer", ["SegformerForSemanticSegmentation", SegformerForSemanticSegmentation]], + ["sapiens", ["SapiensForSemanticSegmentation", SapiensForSemanticSegmentation]], ]); const MODEL_FOR_UNIVERSAL_SEGMENTATION_MAPPING_NAMES = new Map([ - ['detr', ['DetrForSegmentation', DetrForSegmentation]], - ['maskformer', ['MaskFormerForInstanceSegmentation', MaskFormerForInstanceSegmentation]], + ["detr", ["DetrForSegmentation", DetrForSegmentation]], + ["maskformer", ["MaskFormerForInstanceSegmentation", MaskFormerForInstanceSegmentation]], ]); -const MODEL_FOR_MASK_GENERATION_MAPPING_NAMES = new Map([ - ['sam', ['SamModel', SamModel]], -]); +const MODEL_FOR_MASK_GENERATION_MAPPING_NAMES = new Map([["sam", ["SamModel", SamModel]]]); const MODEL_FOR_CTC_MAPPING_NAMES = new Map([ - ['wav2vec2', ['Wav2Vec2ForCTC', Wav2Vec2ForCTC]], - ['wav2vec2-bert', ['Wav2Vec2BertForCTC', Wav2Vec2BertForCTC]], - ['unispeech', ['UniSpeechForCTC', UniSpeechForCTC]], - ['unispeech-sat', ['UniSpeechSatForCTC', UniSpeechSatForCTC]], - ['wavlm', ['WavLMForCTC', WavLMForCTC]], - ['hubert', ['HubertForCTC', HubertForCTC]], + ["wav2vec2", ["Wav2Vec2ForCTC", Wav2Vec2ForCTC]], + ["wav2vec2-bert", ["Wav2Vec2BertForCTC", Wav2Vec2BertForCTC]], + ["unispeech", ["UniSpeechForCTC", UniSpeechForCTC]], + ["unispeech-sat", ["UniSpeechSatForCTC", UniSpeechSatForCTC]], + ["wavlm", ["WavLMForCTC", WavLMForCTC]], + ["hubert", ["HubertForCTC", HubertForCTC]], ]); const MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES = new Map([ - ['wav2vec2', ['Wav2Vec2ForSequenceClassification', Wav2Vec2ForSequenceClassification]], - ['wav2vec2-bert', ['Wav2Vec2BertForSequenceClassification', Wav2Vec2BertForSequenceClassification]], - ['unispeech', ['UniSpeechForSequenceClassification', UniSpeechForSequenceClassification]], - ['unispeech-sat', ['UniSpeechSatForSequenceClassification', UniSpeechSatForSequenceClassification]], - ['wavlm', ['WavLMForSequenceClassification', WavLMForSequenceClassification]], - ['hubert', ['HubertForSequenceClassification', HubertForSequenceClassification]], - ['audio-spectrogram-transformer', ['ASTForAudioClassification', ASTForAudioClassification]], + ["wav2vec2", ["Wav2Vec2ForSequenceClassification", Wav2Vec2ForSequenceClassification]], + ["wav2vec2-bert", ["Wav2Vec2BertForSequenceClassification", Wav2Vec2BertForSequenceClassification]], + ["unispeech", ["UniSpeechForSequenceClassification", UniSpeechForSequenceClassification]], + ["unispeech-sat", ["UniSpeechSatForSequenceClassification", UniSpeechSatForSequenceClassification]], + ["wavlm", ["WavLMForSequenceClassification", WavLMForSequenceClassification]], + ["hubert", ["HubertForSequenceClassification", HubertForSequenceClassification]], + ["audio-spectrogram-transformer", ["ASTForAudioClassification", ASTForAudioClassification]], ]); -const MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES = new Map([ - ['wavlm', ['WavLMForXVector', WavLMForXVector]], -]); +const MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES = new Map([["wavlm", ["WavLMForXVector", WavLMForXVector]]]); const MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING_NAMES = new Map([ - ['unispeech-sat', ['UniSpeechSatForAudioFrameClassification', UniSpeechSatForAudioFrameClassification]], - ['wavlm', ['WavLMForAudioFrameClassification', WavLMForAudioFrameClassification]], - ['wav2vec2', ['Wav2Vec2ForAudioFrameClassification', Wav2Vec2ForAudioFrameClassification]], - ['pyannote', ['PyAnnoteForAudioFrameClassification', PyAnnoteForAudioFrameClassification]], + ["unispeech-sat", ["UniSpeechSatForAudioFrameClassification", UniSpeechSatForAudioFrameClassification]], + ["wavlm", ["WavLMForAudioFrameClassification", WavLMForAudioFrameClassification]], + ["wav2vec2", ["Wav2Vec2ForAudioFrameClassification", Wav2Vec2ForAudioFrameClassification]], + ["pyannote", ["PyAnnoteForAudioFrameClassification", PyAnnoteForAudioFrameClassification]], ]); const MODEL_FOR_IMAGE_MATTING_MAPPING_NAMES = new Map([ - ['vitmatte', ['VitMatteForImageMatting', VitMatteForImageMatting]], + ["vitmatte", ["VitMatteForImageMatting", VitMatteForImageMatting]], ]); const MODEL_FOR_IMAGE_TO_IMAGE_MAPPING_NAMES = new Map([ - ['swin2sr', ['Swin2SRForImageSuperResolution', Swin2SRForImageSuperResolution]], -]) + ["swin2sr", ["Swin2SRForImageSuperResolution", Swin2SRForImageSuperResolution]], +]); const MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES = new Map([ - ['dpt', ['DPTForDepthEstimation', DPTForDepthEstimation]], - ['depth_anything', ['DepthAnythingForDepthEstimation', DepthAnythingForDepthEstimation]], - ['glpn', ['GLPNForDepthEstimation', GLPNForDepthEstimation]], - ['sapiens', ['SapiensForDepthEstimation', SapiensForDepthEstimation]], - ['depth_pro', ['DepthProForDepthEstimation', DepthProForDepthEstimation]], -]) + ["dpt", ["DPTForDepthEstimation", DPTForDepthEstimation]], + ["depth_anything", ["DepthAnythingForDepthEstimation", DepthAnythingForDepthEstimation]], + ["glpn", ["GLPNForDepthEstimation", GLPNForDepthEstimation]], + ["sapiens", ["SapiensForDepthEstimation", SapiensForDepthEstimation]], + ["depth_pro", ["DepthProForDepthEstimation", DepthProForDepthEstimation]], +]); const MODEL_FOR_NORMAL_ESTIMATION_MAPPING_NAMES = new Map([ - ['sapiens', ['SapiensForNormalEstimation', SapiensForNormalEstimation]], -]) + ["sapiens", ["SapiensForNormalEstimation", SapiensForNormalEstimation]], +]); // NOTE: This is custom to Transformers.js, and is necessary because certain models // (e.g., CLIP) are split into vision and text components const MODEL_FOR_IMAGE_FEATURE_EXTRACTION_MAPPING_NAMES = new Map([ - ['clip', ['CLIPVisionModelWithProjection', CLIPVisionModelWithProjection]], - ['siglip', ['SiglipVisionModel', SiglipVisionModel]], -]) + ["clip", ["CLIPVisionModelWithProjection", CLIPVisionModelWithProjection]], + ["siglip", ["SiglipVisionModel", SiglipVisionModel]], +]); const MODEL_CLASS_TYPE_MAPPING = [ [MODEL_MAPPING_NAMES_ENCODER_ONLY, MODEL_TYPES.EncoderOnly], @@ -6462,38 +6436,37 @@ for (const [mappings, type] of MODEL_CLASS_TYPE_MAPPING) { const CUSTOM_MAPPING = [ // OVERRIDE: // TODO: Refactor to allow class to specify model - ['MusicgenForConditionalGeneration', MusicgenForConditionalGeneration, MODEL_TYPES.Musicgen], + ["MusicgenForConditionalGeneration", MusicgenForConditionalGeneration, MODEL_TYPES.Musicgen], - ['CLIPTextModelWithProjection', CLIPTextModelWithProjection, MODEL_TYPES.EncoderOnly], - ['SiglipTextModel', SiglipTextModel, MODEL_TYPES.EncoderOnly], - ['ClapTextModelWithProjection', ClapTextModelWithProjection, MODEL_TYPES.EncoderOnly], - ['ClapAudioModelWithProjection', ClapAudioModelWithProjection, MODEL_TYPES.EncoderOnly], -] + ["CLIPTextModelWithProjection", CLIPTextModelWithProjection, MODEL_TYPES.EncoderOnly], + ["SiglipTextModel", SiglipTextModel, MODEL_TYPES.EncoderOnly], + ["ClapTextModelWithProjection", ClapTextModelWithProjection, MODEL_TYPES.EncoderOnly], + ["ClapAudioModelWithProjection", ClapAudioModelWithProjection, MODEL_TYPES.EncoderOnly], +]; for (const [name, model, type] of CUSTOM_MAPPING) { MODEL_TYPE_MAPPING.set(name, type); MODEL_CLASS_TO_NAME_MAPPING.set(model, name); MODEL_NAME_TO_CLASS_MAPPING.set(name, model); } - /** * Helper class which is used to instantiate pretrained models with the `from_pretrained` function. * The chosen model class is determined by the type specified in the model config. - * + * * @example * let model = await AutoModel.from_pretrained('Xenova/bert-base-uncased'); */ export class AutoModel extends PretrainedMixin { /** @type {Map[]} */ // @ts-ignore - static MODEL_CLASS_MAPPINGS = MODEL_CLASS_TYPE_MAPPING.map(x => x[0]); + static MODEL_CLASS_MAPPINGS = MODEL_CLASS_TYPE_MAPPING.map((x) => x[0]); static BASE_IF_FAIL = true; } /** * Helper class which is used to instantiate pretrained sequence classification models with the `from_pretrained` function. * The chosen model class is determined by the type specified in the model config. - * + * * @example * let model = await AutoModelForSequenceClassification.from_pretrained('Xenova/distilbert-base-uncased-finetuned-sst-2-english'); */ @@ -6504,7 +6477,7 @@ export class AutoModelForSequenceClassification extends PretrainedMixin { /** * Helper class which is used to instantiate pretrained token classification models with the `from_pretrained` function. * The chosen model class is determined by the type specified in the model config. - * + * * @example * let model = await AutoModelForTokenClassification.from_pretrained('Xenova/distilbert-base-multilingual-cased-ner-hrl'); */ @@ -6515,7 +6488,7 @@ export class AutoModelForTokenClassification extends PretrainedMixin { /** * Helper class which is used to instantiate pretrained sequence-to-sequence models with the `from_pretrained` function. * The chosen model class is determined by the type specified in the model config. - * + * * @example * let model = await AutoModelForSeq2SeqLM.from_pretrained('Xenova/t5-small'); */ @@ -6526,7 +6499,7 @@ export class AutoModelForSeq2SeqLM extends PretrainedMixin { /** * Helper class which is used to instantiate pretrained sequence-to-sequence speech-to-text models with the `from_pretrained` function. * The chosen model class is determined by the type specified in the model config. - * + * * @example * let model = await AutoModelForSpeechSeq2Seq.from_pretrained('openai/whisper-tiny.en'); */ @@ -6537,7 +6510,7 @@ export class AutoModelForSpeechSeq2Seq extends PretrainedMixin { /** * Helper class which is used to instantiate pretrained sequence-to-sequence text-to-spectrogram models with the `from_pretrained` function. * The chosen model class is determined by the type specified in the model config. - * + * * @example * let model = await AutoModelForTextToSpectrogram.from_pretrained('microsoft/speecht5_tts'); */ @@ -6548,7 +6521,7 @@ export class AutoModelForTextToSpectrogram extends PretrainedMixin { /** * Helper class which is used to instantiate pretrained text-to-waveform models with the `from_pretrained` function. * The chosen model class is determined by the type specified in the model config. - * + * * @example * let model = await AutoModelForTextToSpectrogram.from_pretrained('facebook/mms-tts-eng'); */ @@ -6559,7 +6532,7 @@ export class AutoModelForTextToWaveform extends PretrainedMixin { /** * Helper class which is used to instantiate pretrained causal language models with the `from_pretrained` function. * The chosen model class is determined by the type specified in the model config. - * + * * @example * let model = await AutoModelForCausalLM.from_pretrained('Xenova/gpt2'); */ @@ -6570,7 +6543,7 @@ export class AutoModelForCausalLM extends PretrainedMixin { /** * Helper class which is used to instantiate pretrained masked language models with the `from_pretrained` function. * The chosen model class is determined by the type specified in the model config. - * + * * @example * let model = await AutoModelForMaskedLM.from_pretrained('Xenova/bert-base-uncased'); */ @@ -6581,7 +6554,7 @@ export class AutoModelForMaskedLM extends PretrainedMixin { /** * Helper class which is used to instantiate pretrained question answering models with the `from_pretrained` function. * The chosen model class is determined by the type specified in the model config. - * + * * @example * let model = await AutoModelForQuestionAnswering.from_pretrained('Xenova/distilbert-base-cased-distilled-squad'); */ @@ -6592,7 +6565,7 @@ export class AutoModelForQuestionAnswering extends PretrainedMixin { /** * Helper class which is used to instantiate pretrained vision-to-sequence models with the `from_pretrained` function. * The chosen model class is determined by the type specified in the model config. - * + * * @example * let model = await AutoModelForVision2Seq.from_pretrained('Xenova/vit-gpt2-image-captioning'); */ @@ -6603,7 +6576,7 @@ export class AutoModelForVision2Seq extends PretrainedMixin { /** * Helper class which is used to instantiate pretrained image classification models with the `from_pretrained` function. * The chosen model class is determined by the type specified in the model config. - * + * * @example * let model = await AutoModelForImageClassification.from_pretrained('Xenova/vit-base-patch16-224'); */ @@ -6614,7 +6587,7 @@ export class AutoModelForImageClassification extends PretrainedMixin { /** * Helper class which is used to instantiate pretrained image segmentation models with the `from_pretrained` function. * The chosen model class is determined by the type specified in the model config. - * + * * @example * let model = await AutoModelForImageSegmentation.from_pretrained('Xenova/detr-resnet-50-panoptic'); */ @@ -6625,7 +6598,7 @@ export class AutoModelForImageSegmentation extends PretrainedMixin { /** * Helper class which is used to instantiate pretrained image segmentation models with the `from_pretrained` function. * The chosen model class is determined by the type specified in the model config. - * + * * @example * let model = await AutoModelForSemanticSegmentation.from_pretrained('nvidia/segformer-b3-finetuned-cityscapes-1024-1024'); */ @@ -6636,7 +6609,7 @@ export class AutoModelForSemanticSegmentation extends PretrainedMixin { /** * Helper class which is used to instantiate pretrained universal image segmentation models with the `from_pretrained` function. * The chosen model class is determined by the type specified in the model config. - * + * * @example * let model = await AutoModelForUniversalSegmentation.from_pretrained('hf-internal-testing/tiny-random-MaskFormerForInstanceSegmentation'); */ @@ -6647,7 +6620,7 @@ export class AutoModelForUniversalSegmentation extends PretrainedMixin { /** * Helper class which is used to instantiate pretrained object detection models with the `from_pretrained` function. * The chosen model class is determined by the type specified in the model config. - * + * * @example * let model = await AutoModelForObjectDetection.from_pretrained('Xenova/detr-resnet-50'); */ @@ -6659,11 +6632,10 @@ export class AutoModelForZeroShotObjectDetection extends PretrainedMixin { static MODEL_CLASS_MAPPINGS = [MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING_NAMES]; } - /** * Helper class which is used to instantiate pretrained mask generation models with the `from_pretrained` function. * The chosen model class is determined by the type specified in the model config. - * + * * @example * let model = await AutoModelForMaskGeneration.from_pretrained('Xenova/sam-vit-base'); */ @@ -6807,7 +6779,6 @@ export class QuestionAnsweringModelOutput extends ModelOutput { } } - /** * Base class for causal language model (or autoregressive) outputs. */ diff --git a/src/models/whisper/common_whisper.js b/src/models/whisper/common_whisper.js index df4cce4d5..bdd88df04 100644 --- a/src/models/whisper/common_whisper.js +++ b/src/models/whisper/common_whisper.js @@ -1,5 +1,3 @@ - - const WHISPER_LANGUAGES = [ ["en", "english"], ["zh", "chinese"], @@ -100,7 +98,7 @@ const WHISPER_LANGUAGES = [ ["ba", "bashkir"], ["jw", "javanese"], ["su", "sundanese"], -] +]; // @ts-ignore export const WHISPER_LANGUAGE_MAPPING = new Map(WHISPER_LANGUAGES); @@ -119,7 +117,7 @@ export const WHISPER_TO_LANGUAGE_CODE_MAPPING = new Map([ ["moldovan", "ro"], ["sinhalese", "si"], ["castilian", "es"], - ] + ], ]); /** @@ -138,7 +136,6 @@ export function whisper_language_to_code(language) { if (WHISPER_LANGUAGE_MAPPING.has(language)) { // User provided the language code directly (e.g., "en") language_code = language; - } else { // User provided something that is not a language code or name const is_language_code = language.length === 2; diff --git a/src/models/whisper/generation_whisper.js b/src/models/whisper/generation_whisper.js index 690455ff7..b7a9bddf3 100644 --- a/src/models/whisper/generation_whisper.js +++ b/src/models/whisper/generation_whisper.js @@ -1,7 +1,6 @@ import { GenerationConfig } from "../../generation/configuration_utils.js"; export class WhisperGenerationConfig extends GenerationConfig { - /** * Whether to return the timestamps with the text. This enables the `WhisperTimestampsLogitsProcessor`. * @type {boolean} diff --git a/src/ops/registry.js b/src/ops/registry.js index 9b65fa4a8..020d41a60 100644 --- a/src/ops/registry.js +++ b/src/ops/registry.js @@ -8,25 +8,25 @@ import { Tensor } from "../utils/tensor.js"; * @param {import('onnxruntime-common').InferenceSession.SessionOptions} session_options The options for the ONNX session. * @template {string | [string] | string[]} T * @param {T} names The name(s) of the output tensor(s). - * + * * @returns {Promise): Promise>} * The wrapper function for running the ONNX inference session. */ const wrap = async (session_bytes, session_options, names) => { - const session = await createInferenceSession( - new Uint8Array(session_bytes), session_options, - ); - return /** @type {any} */(async (/** @type {Record} */ inputs) => { - const ortFeed = Object.fromEntries(Object.entries(inputs).map(([k, v]) => [k, v.ort_tensor])); - const outputs = await session.run(ortFeed); + const session = await createInferenceSession(new Uint8Array(session_bytes), session_options); + return /** @type {any} */ ( + async (/** @type {Record} */ inputs) => { + const ortFeed = Object.fromEntries(Object.entries(inputs).map(([k, v]) => [k, v.ort_tensor])); + const outputs = await session.run(ortFeed); - if (Array.isArray(names)) { - return names.map((n) => new Tensor(outputs[n])); - } else { - return new Tensor(outputs[/** @type {string} */(names)]); + if (Array.isArray(names)) { + return names.map((n) => new Tensor(outputs[n])); + } else { + return new Tensor(outputs[/** @type {string} */ (names)]); + } } - }) -} + ); +}; // In-memory registry of initialized ONNX operators export class TensorOpRegistry { @@ -38,9 +38,16 @@ export class TensorOpRegistry { static get bilinear_interpolate_4d() { if (!this._bilinear_interpolate_4d) { this._bilinear_interpolate_4d = wrap( - [8, 9, 18, 0, 58, 128, 1, 10, 40, 10, 1, 120, 10, 0, 10, 0, 10, 1, 115, 18, 1, 121, 34, 6, 82, 101, 115, 105, 122, 101, 42, 17, 10, 4, 109, 111, 100, 101, 34, 6, 108, 105, 110, 101, 97, 114, 160, 1, 3, 18, 1, 114, 90, 31, 10, 1, 120, 18, 26, 10, 24, 8, 1, 18, 20, 10, 3, 18, 1, 98, 10, 3, 18, 1, 99, 10, 3, 18, 1, 104, 10, 3, 18, 1, 119, 90, 15, 10, 1, 115, 18, 10, 10, 8, 8, 7, 18, 4, 10, 2, 8, 4, 98, 31, 10, 1, 121, 18, 26, 10, 24, 8, 1, 18, 20, 10, 3, 18, 1, 98, 10, 3, 18, 1, 99, 10, 3, 18, 1, 104, 10, 3, 18, 1, 119, 66, 2, 16, 20], + [ + 8, 9, 18, 0, 58, 128, 1, 10, 40, 10, 1, 120, 10, 0, 10, 0, 10, 1, 115, 18, 1, 121, 34, 6, 82, 101, + 115, 105, 122, 101, 42, 17, 10, 4, 109, 111, 100, 101, 34, 6, 108, 105, 110, 101, 97, 114, 160, 1, + 3, 18, 1, 114, 90, 31, 10, 1, 120, 18, 26, 10, 24, 8, 1, 18, 20, 10, 3, 18, 1, 98, 10, 3, 18, 1, 99, + 10, 3, 18, 1, 104, 10, 3, 18, 1, 119, 90, 15, 10, 1, 115, 18, 10, 10, 8, 8, 7, 18, 4, 10, 2, 8, 4, + 98, 31, 10, 1, 121, 18, 26, 10, 24, 8, 1, 18, 20, 10, 3, 18, 1, 98, 10, 3, 18, 1, 99, 10, 3, 18, 1, + 104, 10, 3, 18, 1, 119, 66, 2, 16, 20, + ], this.session_options, - 'y', + "y", ); } return this._bilinear_interpolate_4d; @@ -49,9 +56,16 @@ export class TensorOpRegistry { static get bicubic_interpolate_4d() { if (!this._bicubic_interpolate_4d) { this._bicubic_interpolate_4d = wrap( - [8, 9, 18, 0, 58, 127, 10, 39, 10, 1, 120, 10, 0, 10, 0, 10, 1, 115, 18, 1, 121, 34, 6, 82, 101, 115, 105, 122, 101, 42, 16, 10, 4, 109, 111, 100, 101, 34, 5, 99, 117, 98, 105, 99, 160, 1, 3, 18, 1, 114, 90, 31, 10, 1, 120, 18, 26, 10, 24, 8, 1, 18, 20, 10, 3, 18, 1, 98, 10, 3, 18, 1, 99, 10, 3, 18, 1, 104, 10, 3, 18, 1, 119, 90, 15, 10, 1, 115, 18, 10, 10, 8, 8, 7, 18, 4, 10, 2, 8, 4, 98, 31, 10, 1, 121, 18, 26, 10, 24, 8, 1, 18, 20, 10, 3, 18, 1, 98, 10, 3, 18, 1, 99, 10, 3, 18, 1, 104, 10, 3, 18, 1, 119, 66, 2, 16, 20], + [ + 8, 9, 18, 0, 58, 127, 10, 39, 10, 1, 120, 10, 0, 10, 0, 10, 1, 115, 18, 1, 121, 34, 6, 82, 101, 115, + 105, 122, 101, 42, 16, 10, 4, 109, 111, 100, 101, 34, 5, 99, 117, 98, 105, 99, 160, 1, 3, 18, 1, + 114, 90, 31, 10, 1, 120, 18, 26, 10, 24, 8, 1, 18, 20, 10, 3, 18, 1, 98, 10, 3, 18, 1, 99, 10, 3, + 18, 1, 104, 10, 3, 18, 1, 119, 90, 15, 10, 1, 115, 18, 10, 10, 8, 8, 7, 18, 4, 10, 2, 8, 4, 98, 31, + 10, 1, 121, 18, 26, 10, 24, 8, 1, 18, 20, 10, 3, 18, 1, 98, 10, 3, 18, 1, 99, 10, 3, 18, 1, 104, 10, + 3, 18, 1, 119, 66, 2, 16, 20, + ], this.session_options, - 'y', + "y", ); } return this._bicubic_interpolate_4d; @@ -60,9 +74,13 @@ export class TensorOpRegistry { static get matmul() { if (!this._matmul) { this._matmul = wrap( - [8, 9, 18, 0, 58, 55, 10, 17, 10, 1, 97, 10, 1, 98, 18, 1, 99, 34, 6, 77, 97, 116, 77, 117, 108, 18, 1, 114, 90, 9, 10, 1, 97, 18, 4, 10, 2, 8, 1, 90, 9, 10, 1, 98, 18, 4, 10, 2, 8, 1, 98, 9, 10, 1, 99, 18, 4, 10, 2, 8, 1, 66, 2, 16, 20], + [ + 8, 9, 18, 0, 58, 55, 10, 17, 10, 1, 97, 10, 1, 98, 18, 1, 99, 34, 6, 77, 97, 116, 77, 117, 108, 18, + 1, 114, 90, 9, 10, 1, 97, 18, 4, 10, 2, 8, 1, 90, 9, 10, 1, 98, 18, 4, 10, 2, 8, 1, 98, 9, 10, 1, + 99, 18, 4, 10, 2, 8, 1, 66, 2, 16, 20, + ], this.session_options, - 'c', + "c", ); } return this._matmul; @@ -71,10 +89,18 @@ export class TensorOpRegistry { static get stft() { if (!this._stft) { this._stft = wrap( - [8, 7, 18, 0, 58, 148, 1, 10, 38, 10, 1, 115, 10, 1, 106, 10, 1, 119, 10, 1, 108, 18, 1, 111, 34, 4, 83, 84, 70, 84, 42, 15, 10, 8, 111, 110, 101, 115, 105, 100, 101, 100, 24, 1, 160, 1, 2, 18, 1, 115, 90, 26, 10, 1, 115, 18, 21, 10, 19, 8, 1, 18, 15, 10, 3, 18, 1, 98, 10, 3, 18, 1, 115, 10, 3, 18, 1, 99, 90, 11, 10, 1, 106, 18, 6, 10, 4, 8, 7, 18, 0, 90, 16, 10, 1, 119, 18, 11, 10, 9, 8, 1, 18, 5, 10, 3, 18, 1, 119, 90, 11, 10, 1, 108, 18, 6, 10, 4, 8, 7, 18, 0, 98, 31, 10, 1, 111, 18, 26, 10, 24, 8, 1, 18, 20, 10, 3, 18, 1, 98, 10, 3, 18, 1, 102, 10, 3, 18, 1, 100, 10, 3, 18, 1, 99, 66, 2, 16, 17], + [ + 8, 7, 18, 0, 58, 148, 1, 10, 38, 10, 1, 115, 10, 1, 106, 10, 1, 119, 10, 1, 108, 18, 1, 111, 34, 4, + 83, 84, 70, 84, 42, 15, 10, 8, 111, 110, 101, 115, 105, 100, 101, 100, 24, 1, 160, 1, 2, 18, 1, 115, + 90, 26, 10, 1, 115, 18, 21, 10, 19, 8, 1, 18, 15, 10, 3, 18, 1, 98, 10, 3, 18, 1, 115, 10, 3, 18, 1, + 99, 90, 11, 10, 1, 106, 18, 6, 10, 4, 8, 7, 18, 0, 90, 16, 10, 1, 119, 18, 11, 10, 9, 8, 1, 18, 5, + 10, 3, 18, 1, 119, 90, 11, 10, 1, 108, 18, 6, 10, 4, 8, 7, 18, 0, 98, 31, 10, 1, 111, 18, 26, 10, + 24, 8, 1, 18, 20, 10, 3, 18, 1, 98, 10, 3, 18, 1, 102, 10, 3, 18, 1, 100, 10, 3, 18, 1, 99, 66, 2, + 16, 17, + ], this.session_options, - 'o', - ) + "o", + ); } return this._stft; } @@ -82,10 +108,16 @@ export class TensorOpRegistry { static get rfft() { if (!this._rfft) { this._rfft = wrap( - [8, 9, 18, 0, 58, 97, 10, 33, 10, 1, 120, 10, 0, 10, 1, 97, 18, 1, 121, 34, 3, 68, 70, 84, 42, 15, 10, 8, 111, 110, 101, 115, 105, 100, 101, 100, 24, 1, 160, 1, 2, 18, 1, 100, 90, 21, 10, 1, 120, 18, 16, 10, 14, 8, 1, 18, 10, 10, 3, 18, 1, 115, 10, 3, 18, 1, 99, 90, 11, 10, 1, 97, 18, 6, 10, 4, 8, 7, 18, 0, 98, 21, 10, 1, 121, 18, 16, 10, 14, 8, 1, 18, 10, 10, 3, 18, 1, 115, 10, 3, 18, 1, 99, 66, 2, 16, 20], + [ + 8, 9, 18, 0, 58, 97, 10, 33, 10, 1, 120, 10, 0, 10, 1, 97, 18, 1, 121, 34, 3, 68, 70, 84, 42, 15, + 10, 8, 111, 110, 101, 115, 105, 100, 101, 100, 24, 1, 160, 1, 2, 18, 1, 100, 90, 21, 10, 1, 120, 18, + 16, 10, 14, 8, 1, 18, 10, 10, 3, 18, 1, 115, 10, 3, 18, 1, 99, 90, 11, 10, 1, 97, 18, 6, 10, 4, 8, + 7, 18, 0, 98, 21, 10, 1, 121, 18, 16, 10, 14, 8, 1, 18, 10, 10, 3, 18, 1, 115, 10, 3, 18, 1, 99, 66, + 2, 16, 20, + ], this.session_options, - 'y', - ) + "y", + ); } return this._rfft; } @@ -93,10 +125,15 @@ export class TensorOpRegistry { static get top_k() { if (!this._top_k) { this._top_k = wrap( - [8, 10, 18, 0, 58, 73, 10, 18, 10, 1, 120, 10, 1, 107, 18, 1, 118, 18, 1, 105, 34, 4, 84, 111, 112, 75, 18, 1, 116, 90, 9, 10, 1, 120, 18, 4, 10, 2, 8, 1, 90, 15, 10, 1, 107, 18, 10, 10, 8, 8, 7, 18, 4, 10, 2, 8, 1, 98, 9, 10, 1, 118, 18, 4, 10, 2, 8, 1, 98, 9, 10, 1, 105, 18, 4, 10, 2, 8, 7, 66, 2, 16, 21], + [ + 8, 10, 18, 0, 58, 73, 10, 18, 10, 1, 120, 10, 1, 107, 18, 1, 118, 18, 1, 105, 34, 4, 84, 111, 112, + 75, 18, 1, 116, 90, 9, 10, 1, 120, 18, 4, 10, 2, 8, 1, 90, 15, 10, 1, 107, 18, 10, 10, 8, 8, 7, 18, + 4, 10, 2, 8, 1, 98, 9, 10, 1, 118, 18, 4, 10, 2, 8, 1, 98, 9, 10, 1, 105, 18, 4, 10, 2, 8, 7, 66, 2, + 16, 21, + ], this.session_options, - [ /* Values */ 'v', /* Indices */ 'i'] - ) + [/* Values */ "v", /* Indices */ "i"], + ); } return this._top_k; } diff --git a/src/pipelines.js b/src/pipelines.js index 3b7373cf9..69fdb73e6 100644 --- a/src/pipelines.js +++ b/src/pipelines.js @@ -1,22 +1,19 @@ /** * @file Pipelines provide a high-level, easy to use, API for running machine learning models. - * + * * **Example:** Instantiate pipeline using the `pipeline` function. * ```javascript * import { pipeline } from '@huggingface/transformers'; - * + * * const classifier = await pipeline('sentiment-analysis'); * const output = await classifier('I love transformers!'); * // [{'label': 'POSITIVE', 'score': 0.999817686}] * ``` - * + * * @module pipelines */ -import { - AutoTokenizer, - PreTrainedTokenizer, -} from './tokenizers.js'; +import { AutoTokenizer, PreTrainedTokenizer } from "./tokenizers.js"; import { AutoModel, AutoModelForSequenceClassification, @@ -42,38 +39,16 @@ import { AutoModelForDepthEstimation, AutoModelForImageFeatureExtraction, PreTrainedModel, -} from './models.js'; -import { - AutoProcessor, - Processor -} from './processors.js'; - -import { - Callable, -} from './utils/generic.js'; +} from "./models.js"; +import { AutoProcessor, Processor } from "./processors.js"; -import { - dispatchCallback, - pop, - product, -} from './utils/core.js'; -import { - softmax, - max, - round, -} from './utils/maths.js'; -import { - read_audio -} from './utils/audio.js'; -import { - Tensor, - mean_pooling, - interpolate, - quantize_embeddings, - topk, -} from './utils/tensor.js'; -import { RawImage } from './utils/image.js'; +import { Callable } from "./utils/generic.js"; +import { dispatchCallback, pop, product } from "./utils/core.js"; +import { softmax, max, round } from "./utils/maths.js"; +import { read_audio } from "./utils/audio.js"; +import { Tensor, mean_pooling, interpolate, quantize_embeddings, topk } from "./utils/tensor.js"; +import { RawImage } from "./utils/image.js"; /** * @typedef {string | RawImage | URL} ImageInput @@ -92,7 +67,7 @@ async function prepareImages(images) { } // Possibly convert any non-images to images - return await Promise.all(images.map(x => RawImage.read(x))); + return await Promise.all(images.map((x) => RawImage.read(x))); } /** @@ -112,14 +87,16 @@ async function prepareAudios(audios, sampling_rate) { audios = [audios]; } - return await Promise.all(audios.map(x => { - if (typeof x === 'string' || x instanceof URL) { - return read_audio(x, sampling_rate); - } else if (x instanceof Float64Array) { - return new Float32Array(x); - } - return x; - })); + return await Promise.all( + audios.map((x) => { + if (typeof x === "string" || x instanceof URL) { + return read_audio(x, sampling_rate); + } else if (x instanceof Float64Array) { + return new Float32Array(x); + } + return x; + }), + ); } /** @@ -139,18 +116,17 @@ async function prepareAudios(audios, sampling_rate) { */ function get_bounding_box(box, asInteger) { if (asInteger) { - box = box.map(x => x | 0); + box = box.map((x) => x | 0); } const [xmin, ymin, xmax, ymax] = box; return { xmin, ymin, xmax, ymax }; } - /** * @callback DisposeType Disposes the item. * @returns {Promise} A promise that resolves when the item has been disposed. - * + * * @typedef {Object} Disposable * @property {DisposeType} dispose A promise that resolves when the pipeline has been disposed. */ @@ -188,7 +164,7 @@ export class Pipeline extends Callable { * @property {string} task The task of the pipeline. Useful for specifying subtasks. * @property {PreTrainedModel} model The model used by the pipeline. * @property {PreTrainedTokenizer} tokenizer The tokenizer used by the pipeline. - * + * * @typedef {ModelTokenizerConstructorArgs} TextPipelineConstructorArgs An object used to instantiate a text-based pipeline. */ @@ -197,19 +173,18 @@ export class Pipeline extends Callable { * @property {string} task The task of the pipeline. Useful for specifying subtasks. * @property {PreTrainedModel} model The model used by the pipeline. * @property {Processor} processor The processor used by the pipeline. - * + * * @typedef {ModelProcessorConstructorArgs} AudioPipelineConstructorArgs An object used to instantiate an audio-based pipeline. * @typedef {ModelProcessorConstructorArgs} ImagePipelineConstructorArgs An object used to instantiate an image-based pipeline. */ - /** * @typedef {Object} ModelTokenizerProcessorConstructorArgs * @property {string} task The task of the pipeline. Useful for specifying subtasks. * @property {PreTrainedModel} model The model used by the pipeline. * @property {PreTrainedTokenizer} tokenizer The tokenizer used by the pipeline. * @property {Processor} processor The processor used by the pipeline. - * + * * @typedef {ModelTokenizerProcessorConstructorArgs} TextAudioPipelineConstructorArgs An object used to instantiate a text- and audio-based pipeline. * @typedef {ModelTokenizerProcessorConstructorArgs} TextImagePipelineConstructorArgs An object used to instantiate a text- and image-based pipeline. */ @@ -219,15 +194,15 @@ export class Pipeline extends Callable { * @property {string} label The label predicted. * @property {number} score The corresponding probability. * @typedef {TextClassificationSingle[]} TextClassificationOutput - * + * * @typedef {Object} TextClassificationPipelineOptions Parameters specific to text classification pipelines. * @property {number} [top_k=1] The number of top predictions to be returned. - * + * * @callback TextClassificationPipelineCallback Classify the text(s) given as inputs. * @param {string|string[]} texts The input text(s) to be classified. * @param {TextClassificationPipelineOptions} [options] The options to use for text classification. * @returns {Promise} An array or object containing the predicted labels and scores. - * + * * @typedef {TextPipelineConstructorArgs & TextClassificationPipelineCallback & Disposable} TextClassificationPipelineType */ @@ -240,7 +215,7 @@ export class Pipeline extends Callable { * const output = await classifier('I love transformers!'); * // [{ label: 'POSITIVE', score: 0.999788761138916 }] * ``` - * + * * **Example:** Multilingual sentiment-analysis w/ `Xenova/bert-base-multilingual-uncased-sentiment` (and return top 5 classes). * ```javascript * const classifier = await pipeline('sentiment-analysis', 'Xenova/bert-base-multilingual-uncased-sentiment'); @@ -253,7 +228,7 @@ export class Pipeline extends Callable { * // { label: '2 stars', score: 0.0009423971059732139 } * // ] * ``` - * + * * **Example:** Toxic comment classification w/ `Xenova/toxic-bert` (and return all classes). * ```javascript * const classifier = await pipeline('text-classification', 'Xenova/toxic-bert'); @@ -268,8 +243,9 @@ export class Pipeline extends Callable { * // ] * ``` */ -export class TextClassificationPipeline extends (/** @type {new (options: TextPipelineConstructorArgs) => TextClassificationPipelineType} */ (Pipeline)) { - +export class TextClassificationPipeline + extends /** @type {new (options: TextPipelineConstructorArgs) => TextClassificationPipelineType} */ (Pipeline) +{ /** * Create a new TextClassificationPipeline. * @param {TextPipelineConstructorArgs} options An object used to instantiate the pipeline. @@ -279,10 +255,7 @@ export class TextClassificationPipeline extends (/** @type {new (options: TextPi } /** @type {TextClassificationPipelineCallback} */ - async _call(texts, { - top_k = 1 - } = {}) { - + async _call(texts, { top_k = 1 } = {}) { // Run tokenization const model_inputs = this.tokenizer(texts, { padding: true, @@ -290,17 +263,13 @@ export class TextClassificationPipeline extends (/** @type {new (options: TextPi }); // Run model - const outputs = await this.model(model_inputs) + const outputs = await this.model(model_inputs); // TODO: Use softmax tensor function const function_to_apply = - this.model.config.problem_type === 'multi_label_classification' - ? batch => batch.sigmoid() - : batch => new Tensor( - 'float32', - softmax(batch.data), - batch.dims, - ); // single_label_classification (default) + this.model.config.problem_type === "multi_label_classification" + ? (batch) => batch.sigmoid() + : (batch) => new Tensor("float32", softmax(batch.data), batch.dims); // single_label_classification (default) const id2label = this.model.config.id2label; @@ -323,7 +292,9 @@ export class TextClassificationPipeline extends (/** @type {new (options: TextPi } } - return Array.isArray(texts) || top_k === 1 ? /** @type {TextClassificationOutput} */ (toReturn) : /** @type {TextClassificationOutput[]} */ (toReturn)[0]; + return Array.isArray(texts) || top_k === 1 + ? /** @type {TextClassificationOutput} */ (toReturn) + : /** @type {TextClassificationOutput[]} */ (toReturn)[0]; } } @@ -336,21 +307,21 @@ export class TextClassificationPipeline extends (/** @type {new (options: TextPi * @property {number} [start] The index of the start of the corresponding entity in the sentence. * @property {number} [end] The index of the end of the corresponding entity in the sentence. * @typedef {TokenClassificationSingle[]} TokenClassificationOutput - * + * * @typedef {Object} TokenClassificationPipelineOptions Parameters specific to token classification pipelines. * @property {string[]} [ignore_labels] A list of labels to ignore. - * + * * @callback TokenClassificationPipelineCallback Classify each token of the text(s) given as inputs. * @param {string|string[]} texts One or several texts (or one list of texts) for token classification. * @param {TokenClassificationPipelineOptions} [options] The options to use for token classification. * @returns {Promise} The result. - * + * * @typedef {TextPipelineConstructorArgs & TokenClassificationPipelineCallback & Disposable} TokenClassificationPipelineType */ /** * Named Entity Recognition pipeline using any `ModelForTokenClassification`. - * + * * **Example:** Perform named entity recognition with `Xenova/bert-base-NER`. * ```javascript * const classifier = await pipeline('token-classification', 'Xenova/bert-base-NER'); @@ -360,7 +331,7 @@ export class TextClassificationPipeline extends (/** @type {new (options: TextPi * // { entity: 'B-LOC', score: 0.9994474053382874, index: 9, word: 'London' } * // ] * ``` - * + * * **Example:** Perform named entity recognition with `Xenova/bert-base-NER` (and return all labels). * ```javascript * const classifier = await pipeline('token-classification', 'Xenova/bert-base-NER'); @@ -377,8 +348,9 @@ export class TextClassificationPipeline extends (/** @type {new (options: TextPi * // ] * ``` */ -export class TokenClassificationPipeline extends (/** @type {new (options: TextPipelineConstructorArgs) => TokenClassificationPipelineType} */ (Pipeline)) { - +export class TokenClassificationPipeline + extends /** @type {new (options: TextPipelineConstructorArgs) => TokenClassificationPipelineType} */ (Pipeline) +{ /** * Create a new TokenClassificationPipeline. * @param {TextPipelineConstructorArgs} options An object used to instantiate the pipeline. @@ -388,10 +360,7 @@ export class TokenClassificationPipeline extends (/** @type {new (options: TextP } /** @type {TokenClassificationPipelineCallback} */ - async _call(texts, { - ignore_labels = ['O'], - } = {}) { - + async _call(texts, { ignore_labels = ["O"] } = {}) { const isBatched = Array.isArray(texts); // Run tokenization @@ -401,7 +370,7 @@ export class TokenClassificationPipeline extends (/** @type {new (options: TextP }); // Run model - const outputs = await this.model(model_inputs) + const outputs = await this.model(model_inputs); const logits = outputs.logits; const id2label = this.model.config.id2label; @@ -425,7 +394,7 @@ export class TokenClassificationPipeline extends (/** @type {new (options: TextP // TODO add option to keep special tokens? const word = this.tokenizer.decode([ids[j].item()], { skip_special_tokens: true }); - if (word === '') { + if (word === "") { // Was a special token. So, we skip it. continue; } @@ -455,22 +424,22 @@ export class TokenClassificationPipeline extends (/** @type {new (options: TextP * @property {number} [start] The character start index of the answer (in the tokenized version of the input). * @property {number} [end] The character end index of the answer (in the tokenized version of the input). * @property {string} answer The answer to the question. - * + * * @typedef {Object} QuestionAnsweringPipelineOptions Parameters specific to question answering pipelines. * @property {number} [top_k=1] The number of top answer predictions to be returned. - * + * * @callback QuestionAnsweringPipelineCallback Answer the question(s) given as inputs by using the context(s). * @param {string|string[]} question One or several question(s) (must be used in conjunction with the `context` argument). * @param {string|string[]} context One or several context(s) associated with the question(s) (must be used in conjunction with the `question` argument). * @param {QuestionAnsweringPipelineOptions} [options] The options to use for question answering. * @returns {Promise} An array or object containing the predicted answers and scores. - * + * * @typedef {TextPipelineConstructorArgs & QuestionAnsweringPipelineCallback & Disposable} QuestionAnsweringPipelineType */ /** * Question Answering pipeline using any `ModelForQuestionAnswering`. - * + * * **Example:** Run question answering with `Xenova/distilbert-base-uncased-distilled-squad`. * ```javascript * const answerer = await pipeline('question-answering', 'Xenova/distilbert-base-uncased-distilled-squad'); @@ -483,8 +452,9 @@ export class TokenClassificationPipeline extends (/** @type {new (options: TextP * // } * ``` */ -export class QuestionAnsweringPipeline extends (/** @type {new (options: TextPipelineConstructorArgs) => QuestionAnsweringPipelineType} */ (Pipeline)) { - +export class QuestionAnsweringPipeline + extends /** @type {new (options: TextPipelineConstructorArgs) => QuestionAnsweringPipelineType} */ (Pipeline) +{ /** * Create a new QuestionAnsweringPipeline. * @param {TextPipelineConstructorArgs} options An object used to instantiate the pipeline. @@ -494,10 +464,7 @@ export class QuestionAnsweringPipeline extends (/** @type {new (options: TextPip } /** @type {QuestionAnsweringPipelineCallback} */ - async _call(question, context, { - top_k = 1 - } = {}) { - + async _call(question, context, { top_k = 1 } = {}) { // Run tokenization const inputs = this.tokenizer(question, { text_pair: context, @@ -516,23 +483,19 @@ export class QuestionAnsweringPipeline extends (/** @type {new (options: TextPip const toReturn = []; for (let j = 0; j < start_logits.dims[0]; ++j) { const ids = input_ids[j]; - const sepIndex = ids.findIndex(x => - // We use == to match bigint with number - // @ts-ignore - x == this.tokenizer.sep_token_id + const sepIndex = ids.findIndex( + (x) => + // We use == to match bigint with number + // @ts-ignore + x == this.tokenizer.sep_token_id, ); - - const valid_mask = attention_mask[j].map((y, ix) => ( - y == 1 - && ( - ix === 0 // is cls_token - || ( - ix > sepIndex - && special_tokens.findIndex(x => x == ids[ix]) === -1 // token is not a special token (special_tokens_mask == 0) - ) - ) - )); + const valid_mask = attention_mask[j].map( + (y, ix) => + y == 1 && + (ix === 0 || // is cls_token + (ix > sepIndex && special_tokens.findIndex((x) => x == ids[ix]) === -1)), // token is not a special token (special_tokens_mask == 0) + ); const start = start_logits[j].tolist(); const end = end_logits[j].tolist(); @@ -541,9 +504,9 @@ export class QuestionAnsweringPipeline extends (/** @type {new (options: TextPip // NOTE: We keep the cls_token unmasked (some models use it to indicate unanswerable questions) for (let i = 1; i < start.length; ++i) { if ( - attention_mask[j] == 0 // is part of padding - || i <= sepIndex // is before the sep_token - || special_tokens.findIndex(x => x == ids[i]) !== -1 // Is a special token + attention_mask[j] == 0 || // is part of padding + i <= sepIndex || // is before the sep_token + special_tokens.findIndex((x) => x == ids[i]) !== -1 // Is a special token ) { // Make sure non-context indexes in the tensor cannot contribute to the softmax start[i] = -Infinity; @@ -561,14 +524,14 @@ export class QuestionAnsweringPipeline extends (/** @type {new (options: TextPip // Generate all valid spans and select best ones const options = product(start_scores, end_scores) - .filter(x => x[0][1] <= x[1][1]) - .map(x => [x[0][1], x[1][1], x[0][0] * x[1][0]]) + .filter((x) => x[0][1] <= x[1][1]) + .map((x) => [x[0][1], x[1][1], x[0][0] * x[1][0]]) .sort((a, b) => b[2] - a[2]); for (let k = 0; k < Math.min(options.length, top_k); ++k) { const [start, end, score] = options[k]; - const answer_tokens = ids.slice(start, end + 1) + const answer_tokens = ids.slice(start, end + 1); const answer = this.tokenizer.decode(answer_tokens, { skip_special_tokens: true, @@ -577,17 +540,17 @@ export class QuestionAnsweringPipeline extends (/** @type {new (options: TextPip // TODO add start and end? // NOTE: HF returns character index toReturn.push({ - answer, score + answer, + score, }); } } // Mimic HF's return type based on top_k - return (top_k === 1) ? toReturn[0] : toReturn; + return top_k === 1 ? toReturn[0] : toReturn; } } - /** * @typedef {Object} FillMaskSingle * @property {string} sequence The corresponding input with the mask token prediction. @@ -595,10 +558,10 @@ export class QuestionAnsweringPipeline extends (/** @type {new (options: TextPip * @property {number} token The predicted token id (to replace the masked one). * @property {string} token_str The predicted token (to replace the masked one). * @typedef {FillMaskSingle[]} FillMaskOutput - * + * * @typedef {Object} FillMaskPipelineOptions Parameters specific to fill mask pipelines. * @property {number} [top_k=5] When passed, overrides the number of predictions to return. - * + * * @callback FillMaskPipelineCallback Fill the masked token in the text(s) given as inputs. * @param {string|string[]} texts One or several texts (or one list of prompts) with masked tokens. * @param {FillMaskPipelineOptions} [options] The options to use for masked language modelling. @@ -606,13 +569,13 @@ export class QuestionAnsweringPipeline extends (/** @type {new (options: TextPip * and the sequence with the predicted token filled in, or an array of such arrays (one for each input text). * If only one input text is given, the output will be an array of objects. * @throws {Error} When the mask token is not found in the input text. - * + * * @typedef {TextPipelineConstructorArgs & FillMaskPipelineCallback & Disposable} FillMaskPipelineType */ /** * Masked language modeling prediction pipeline using any `ModelWithLMHead`. - * + * * **Example:** Perform masked language modelling (a.k.a. "fill-mask") with `Xenova/bert-base-uncased`. * ```javascript * const unmasker = await pipeline('fill-mask', 'Xenova/bert-base-cased'); @@ -625,7 +588,7 @@ export class QuestionAnsweringPipeline extends (/** @type {new (options: TextPip * // { token_str: 'life', score: 0.01859794743359089, token: 1297, sequence: 'The goal of life is life.' } * // ] * ``` - * + * * **Example:** Perform masked language modelling (a.k.a. "fill-mask") with `Xenova/bert-base-cased` (and return top result). * ```javascript * const unmasker = await pipeline('fill-mask', 'Xenova/bert-base-cased'); @@ -633,8 +596,9 @@ export class QuestionAnsweringPipeline extends (/** @type {new (options: TextPip * // [{ token_str: 'spiral', score: 0.6299987435340881, token: 14061, sequence: 'The Milky Way is a spiral galaxy.' }] * ``` */ -export class FillMaskPipeline extends (/** @type {new (options: TextPipelineConstructorArgs) => FillMaskPipelineType} */ (Pipeline)) { - +export class FillMaskPipeline + extends /** @type {new (options: TextPipelineConstructorArgs) => FillMaskPipelineType} */ (Pipeline) +{ /** * Create a new FillMaskPipeline. * @param {TextPipelineConstructorArgs} options An object used to instantiate the pipeline. @@ -644,10 +608,7 @@ export class FillMaskPipeline extends (/** @type {new (options: TextPipelineCons } /** @type {FillMaskPipelineCallback} */ - async _call(texts, { - top_k = 5 - } = {}) { - + async _call(texts, { top_k = 5 } = {}) { // Run tokenization const model_inputs = this.tokenizer(texts, { padding: true, @@ -655,7 +616,7 @@ export class FillMaskPipeline extends (/** @type {new (options: TextPipelineCons }); // Run model - const { logits } = await this.model(model_inputs) + const { logits } = await this.model(model_inputs); const toReturn = []; @@ -663,57 +624,55 @@ export class FillMaskPipeline extends (/** @type {new (options: TextPipelineCons const input_ids = model_inputs.input_ids.tolist(); for (let i = 0; i < input_ids.length; ++i) { const ids = input_ids[i]; - const mask_token_index = ids.findIndex(x => - // We use == to match bigint with number - // @ts-ignore - x == this.tokenizer.mask_token_id + const mask_token_index = ids.findIndex( + (x) => + // We use == to match bigint with number + // @ts-ignore + x == this.tokenizer.mask_token_id, ); if (mask_token_index === -1) { - throw Error(`Mask token (${this.tokenizer.mask_token}) not found in text.`) + throw Error(`Mask token (${this.tokenizer.mask_token}) not found in text.`); } const itemLogits = logits[i][mask_token_index]; - const scores = await topk(new Tensor( - 'float32', - softmax(itemLogits.data), - itemLogits.dims, - ), top_k); + const scores = await topk(new Tensor("float32", softmax(itemLogits.data), itemLogits.dims), top_k); const values = scores[0].tolist(); const indices = scores[1].tolist(); - toReturn.push(indices.map((x, i) => { - const sequence = ids.slice(); - sequence[mask_token_index] = x; - - return { - score: values[i], - token: Number(x), - token_str: this.tokenizer.model.vocab[x], - sequence: this.tokenizer.decode(sequence, { skip_special_tokens: true }), - } - })); + toReturn.push( + indices.map((x, i) => { + const sequence = ids.slice(); + sequence[mask_token_index] = x; + + return { + score: values[i], + token: Number(x), + token_str: this.tokenizer.model.vocab[x], + sequence: this.tokenizer.decode(sequence, { skip_special_tokens: true }), + }; + }), + ); } return Array.isArray(texts) ? toReturn : toReturn[0]; } } - /** * @typedef {Object} Text2TextGenerationSingle * @property {string} generated_text The generated text. * @typedef {Text2TextGenerationSingle[]} Text2TextGenerationOutput - * + * * @callback Text2TextGenerationPipelineCallback Generate the output text(s) using text(s) given as inputs. * @param {string|string[]} texts Input text for the encoder. * @param {Partial} [options] Additional keyword arguments to pass along to the generate method of the model. * @returns {Promise} - * + * * @typedef {TextPipelineConstructorArgs & Text2TextGenerationPipelineCallback & Disposable} Text2TextGenerationPipelineType */ /** * Text2TextGenerationPipeline class for generating text using a model that performs text-to-text generation tasks. - * + * * **Example:** Text-to-text generation w/ `Xenova/LaMini-Flan-T5-783M`. * ```javascript * const generator = await pipeline('text2text-generation', 'Xenova/LaMini-Flan-T5-783M'); @@ -723,9 +682,11 @@ export class FillMaskPipeline extends (/** @type {new (options: TextPipelineCons * // [{ generated_text: "To become more healthy, you can: 1. Eat a balanced diet with plenty of fruits, vegetables, whole grains, lean proteins, and healthy fats. 2. Stay hydrated by drinking plenty of water. 3. Get enough sleep and manage stress levels. 4. Avoid smoking and excessive alcohol consumption. 5. Regularly exercise and maintain a healthy weight. 6. Practice good hygiene and sanitation. 7. Seek medical attention if you experience any health issues." }] * ``` */ -export class Text2TextGenerationPipeline extends (/** @type {new (options: TextPipelineConstructorArgs) => Text2TextGenerationPipelineType} */ (Pipeline)) { +export class Text2TextGenerationPipeline + extends /** @type {new (options: TextPipelineConstructorArgs) => Text2TextGenerationPipelineType} */ (Pipeline) +{ /** @type {'generated_text'} */ - _key = 'generated_text'; + _key = "generated_text"; /** * Create a new Text2TextGenerationPipeline. @@ -741,18 +702,17 @@ export class Text2TextGenerationPipeline extends (/** @type {new (options: TextP texts = [texts]; } - // Add global prefix, if present if (this.model.config.prefix) { - texts = texts.map(x => this.model.config.prefix + x) + texts = texts.map((x) => this.model.config.prefix + x); } // Handle task specific params: - const task_specific_params = this.model.config.task_specific_params + const task_specific_params = this.model.config.task_specific_params; if (task_specific_params && task_specific_params[this.task]) { // Add prefixes, if present if (task_specific_params[this.task].prefix) { - texts = texts.map(x => task_specific_params[this.task].prefix + x) + texts = texts.map((x) => task_specific_params[this.task].prefix + x); } // TODO update generation config @@ -762,42 +722,42 @@ export class Text2TextGenerationPipeline extends (/** @type {new (options: TextP const tokenizer_options = { padding: true, truncation: true, - } + }; let inputs; - if (this instanceof TranslationPipeline && '_build_translation_inputs' in tokenizer) { + if (this instanceof TranslationPipeline && "_build_translation_inputs" in tokenizer) { // TODO: move to Translation pipeline? // Currently put here to avoid code duplication // @ts-ignore inputs = tokenizer._build_translation_inputs(texts, tokenizer_options, generate_kwargs); - } else { inputs = tokenizer(texts, tokenizer_options); } const outputTokenIds = await this.model.generate({ ...inputs, ...generate_kwargs }); - return tokenizer.batch_decode(/** @type {Tensor} */(outputTokenIds), { - skip_special_tokens: true, - }).map(text => ({ [this._key]: text })); + return tokenizer + .batch_decode(/** @type {Tensor} */ (outputTokenIds), { + skip_special_tokens: true, + }) + .map((text) => ({ [this._key]: text })); } } - /** * @typedef {Object} SummarizationSingle * @property {string} summary_text The summary text. * @typedef {SummarizationSingle[]} SummarizationOutput - * + * * @callback SummarizationPipelineCallback Summarize the text(s) given as inputs. * @param {string|string[]} texts One or several articles (or one list of articles) to summarize. * @param {import('./generation/configuration_utils.js').GenerationConfig} [options] Additional keyword arguments to pass along to the generate method of the model. * @returns {Promise} - * + * * @typedef {TextPipelineConstructorArgs & SummarizationPipelineCallback & Disposable} SummarizationPipelineType */ /** * A pipeline for summarization tasks, inheriting from Text2TextGenerationPipeline. - * + * * **Example:** Summarization w/ `Xenova/distilbart-cnn-6-6`. * ```javascript * const generator = await pipeline('summarization', 'Xenova/distilbart-cnn-6-6'); @@ -815,9 +775,13 @@ export class Text2TextGenerationPipeline extends (/** @type {new (options: TextP * // [{ summary_text: ' The Eiffel Tower is about the same height as an 81-storey building and the tallest structure in Paris. It is the second tallest free-standing structure in France after the Millau Viaduct.' }] * ``` */ -export class SummarizationPipeline extends (/** @type {new (options: TextPipelineConstructorArgs) => SummarizationPipelineType} */ (/** @type {any} */ (Text2TextGenerationPipeline))) { +export class SummarizationPipeline + extends /** @type {new (options: TextPipelineConstructorArgs) => SummarizationPipelineType} */ ( + /** @type {any} */ (Text2TextGenerationPipeline) + ) +{ /** @type {'summary_text'} */ - _key = 'summary_text'; + _key = "summary_text"; /** * Create a new SummarizationPipeline. @@ -828,28 +792,27 @@ export class SummarizationPipeline extends (/** @type {new (options: TextPipelin } } - /** * @typedef {Object} TranslationSingle * @property {string} translation_text The translated text. * @typedef {TranslationSingle[]} TranslationOutput - * + * * @callback TranslationPipelineCallback Translate the text(s) given as inputs. * @param {string|string[]} texts Texts to be translated. * @param {import('./generation/configuration_utils.js').GenerationConfig} [options] Additional keyword arguments to pass along to the generate method of the model. * @returns {Promise} - * + * * @typedef {TextPipelineConstructorArgs & TranslationPipelineCallback & Disposable} TranslationPipelineType */ /** * Translates text from one language to another. - * + * * **Example:** Multilingual translation w/ `Xenova/nllb-200-distilled-600M`. - * + * * See [here](https://github.com/facebookresearch/flores/blob/main/flores200/README.md#languages-in-flores-200) * for the full list of languages and their corresponding codes. - * + * * ```javascript * const translator = await pipeline('translation', 'Xenova/nllb-200-distilled-600M'); * const output = await translator('जीवन एक चॉकलेट बॉक्स की तरह है।', { @@ -858,12 +821,12 @@ export class SummarizationPipeline extends (/** @type {new (options: TextPipelin * }); * // [{ translation_text: 'La vie est comme une boîte à chocolat.' }] * ``` - * + * * **Example:** Multilingual translation w/ `Xenova/m2m100_418M`. - * + * * See [here](https://huggingface.co/facebook/m2m100_418M#languages-covered) * for the full list of languages and their corresponding codes. - * + * * ```javascript * const translator = await pipeline('translation', 'Xenova/m2m100_418M'); * const output = await translator('生活就像一盒巧克力。', { @@ -872,12 +835,12 @@ export class SummarizationPipeline extends (/** @type {new (options: TextPipelin * }); * // [{ translation_text: 'Life is like a box of chocolate.' }] * ``` - * + * * **Example:** Multilingual translation w/ `Xenova/mbart-large-50-many-to-many-mmt`. - * + * * See [here](https://huggingface.co/facebook/mbart-large-50-many-to-many-mmt#languages-covered) * for the full list of languages and their corresponding codes. - * + * * ```javascript * const translator = await pipeline('translation', 'Xenova/mbart-large-50-many-to-many-mmt'); * const output = await translator('संयुक्त राष्ट्र के प्रमुख का कहना है कि सीरिया में कोई सैन्य समाधान नहीं है', { @@ -887,9 +850,13 @@ export class SummarizationPipeline extends (/** @type {new (options: TextPipelin * // [{ translation_text: 'Le chef des Nations affirme qu 'il n 'y a military solution in Syria.' }] * ``` */ -export class TranslationPipeline extends (/** @type {new (options: TextPipelineConstructorArgs) => TranslationPipelineType} */ (/** @type {any} */ (Text2TextGenerationPipeline))) { +export class TranslationPipeline + extends /** @type {new (options: TextPipelineConstructorArgs) => TranslationPipelineType} */ ( + /** @type {any} */ (Text2TextGenerationPipeline) + ) +{ /** @type {'translation_text'} */ - _key = 'translation_text'; + _key = "translation_text"; /** * Create a new TranslationPipeline. @@ -901,26 +868,26 @@ export class TranslationPipeline extends (/** @type {new (options: TextPipelineC } function isChat(x) { - return Array.isArray(x) && x.every(x => 'role' in x && 'content' in x); + return Array.isArray(x) && x.every((x) => "role" in x && "content" in x); } /** * @typedef {import('./tokenizers.js').Message[]} Chat - * + * * @typedef {Object} TextGenerationSingle * @property {string|Chat} generated_text The generated text. * @typedef {TextGenerationSingle[]} TextGenerationOutput - * + * * @typedef {Object} TextGenerationSpecificParams Parameters specific to text-generation pipelines. * @property {boolean} [add_special_tokens] Whether or not to add special tokens when tokenizing the sequences. * @property {boolean} [return_full_text=true] If set to `false` only added text is returned, otherwise the full text is returned. * @typedef {import('./generation/configuration_utils.js').GenerationConfig & TextGenerationSpecificParams} TextGenerationConfig - * + * * @callback TextGenerationPipelineCallback Complete the prompt(s) given as inputs. * @param {string|string[]|Chat|Chat[]} texts One or several prompts (or one list of prompts) to complete. * @param {Partial} [options] Additional keyword arguments to pass along to the generate method of the model. * @returns {Promise} An array or object containing the generated texts. - * + * * @typedef {TextPipelineConstructorArgs & TextGenerationPipelineCallback & Disposable} TextGenerationPipelineType */ @@ -928,7 +895,7 @@ function isChat(x) { * Language generation pipeline using any `ModelWithLMHead` or `ModelForCausalLM`. * This pipeline predicts the words that will follow a specified text prompt. * NOTE: For the full list of generation parameters, see [`GenerationConfig`](./utils/generation#module_utils/generation.GenerationConfig). - * + * * **Example:** Text generation with `Xenova/distilgpt2` (default settings). * ```javascript * const generator = await pipeline('text-generation', 'Xenova/distilgpt2'); @@ -936,7 +903,7 @@ function isChat(x) { * const output = await generator(text); * // [{ generated_text: "I enjoy walking with my cute dog, and I love to play with the other dogs." }] * ``` - * + * * **Example:** Text generation with `Xenova/distilgpt2` (custom settings). * ```javascript * const generator = await pipeline('text-generation', 'Xenova/distilgpt2'); @@ -955,7 +922,7 @@ function isChat(x) { * // "generated_text": "Once upon a time, there was an abundance of information about the most important and influential" * // }] * ``` - * + * * **Example:** Run code generation with `Xenova/codegen-350M-mono`. * ```javascript * const generator = await pipeline('text-generation', 'Xenova/codegen-350M-mono'); @@ -974,8 +941,9 @@ function isChat(x) { * // }] * ``` */ -export class TextGenerationPipeline extends (/** @type {new (options: TextPipelineConstructorArgs) => TextGenerationPipelineType} */ (Pipeline)) { - +export class TextGenerationPipeline + extends /** @type {new (options: TextPipelineConstructorArgs) => TextGenerationPipelineType} */ (Pipeline) +{ /** * Create a new TextGenerationPipeline. * @param {TextPipelineConstructorArgs} options An object used to instantiate the pipeline. @@ -992,49 +960,51 @@ export class TextGenerationPipeline extends (/** @type {new (options: TextPipeli // Normalize inputs /** @type {string[]} */ let inputs; - if (typeof texts === 'string') { + if (typeof texts === "string") { inputs = texts = [texts]; - } else if (Array.isArray(texts) && texts.every(x => typeof x === 'string')) { + } else if (Array.isArray(texts) && texts.every((x) => typeof x === "string")) { isBatched = true; - inputs = /** @type {string[]} */(texts); + inputs = /** @type {string[]} */ (texts); } else { if (isChat(texts)) { - texts = [/** @type {Chat} */(texts)]; + texts = [/** @type {Chat} */ (texts)]; } else if (Array.isArray(texts) && texts.every(isChat)) { isBatched = true; } else { - throw new Error('Input must be a string, an array of strings, a Chat, or an array of Chats'); + throw new Error("Input must be a string, an array of strings, a Chat, or an array of Chats"); } isChatInput = true; // If the input is a chat, we need to apply the chat template - inputs = /** @type {string[]} */(/** @type {Chat[]} */ (texts).map( - x => this.tokenizer.apply_chat_template(x, { - tokenize: false, - add_generation_prompt: true, - }) - )); + inputs = /** @type {string[]} */ ( + /** @type {Chat[]} */ (texts).map((x) => + this.tokenizer.apply_chat_template(x, { + tokenize: false, + add_generation_prompt: true, + }), + ) + ); } // By default, do not add special tokens const add_special_tokens = generate_kwargs.add_special_tokens ?? false; // By default, return full text - const return_full_text = isChatInput - ? false - : generate_kwargs.return_full_text ?? true; + const return_full_text = isChatInput ? false : (generate_kwargs.return_full_text ?? true); - this.tokenizer.padding_side = 'left'; + this.tokenizer.padding_side = "left"; const text_inputs = this.tokenizer(inputs, { add_special_tokens, padding: true, truncation: true, }); - const outputTokenIds = /** @type {Tensor} */(await this.model.generate({ - ...text_inputs, - ...generate_kwargs - })); + const outputTokenIds = /** @type {Tensor} */ ( + await this.model.generate({ + ...text_inputs, + ...generate_kwargs, + }) + ); const decoded = this.tokenizer.batch_decode(outputTokenIds, { skip_special_tokens: true, @@ -1042,15 +1012,17 @@ export class TextGenerationPipeline extends (/** @type {new (options: TextPipeli let promptLengths; if (!return_full_text && text_inputs.input_ids.dims.at(-1) > 0) { - promptLengths = this.tokenizer.batch_decode(text_inputs.input_ids, { - skip_special_tokens: true, - }).map(x => x.length); + promptLengths = this.tokenizer + .batch_decode(text_inputs.input_ids, { + skip_special_tokens: true, + }) + .map((x) => x.length); } /** @type {TextGenerationOutput[]} */ - const toReturn = Array.from({ length: texts.length }, _ => []); + const toReturn = Array.from({ length: texts.length }, (_) => []); for (let i = 0; i < decoded.length; ++i) { - const textIndex = Math.floor(i / outputTokenIds.dims[0] * texts.length); + const textIndex = Math.floor((i / outputTokenIds.dims[0]) * texts.length); if (promptLengths) { // Trim the decoded text to only include the generated part @@ -1058,14 +1030,11 @@ export class TextGenerationPipeline extends (/** @type {new (options: TextPipeli } toReturn[textIndex].push({ generated_text: isChatInput - ? [ - ...((/** @type {Chat[]} */(texts)[textIndex])), - { role: 'assistant', content: decoded[i] }, - ] - : decoded[i] + ? [.../** @type {Chat[]} */ (texts)[textIndex], { role: "assistant", content: decoded[i] }] + : decoded[i], }); } - return (!isBatched && toReturn.length === 1) ? toReturn[0] : toReturn; + return !isBatched && toReturn.length === 1 ? toReturn[0] : toReturn; } } @@ -1074,7 +1043,7 @@ export class TextGenerationPipeline extends (/** @type {new (options: TextPipeli * @property {string} sequence The sequence for which this is the output. * @property {string[]} labels The labels sorted by order of likelihood. * @property {number[]} scores The probabilities for each of the labels. - * + * * @typedef {Object} ZeroShotClassificationPipelineOptions Parameters specific to zero-shot classification pipelines. * @property {string} [hypothesis_template="This example is {}."] The template used to turn each * candidate label into an NLI-style hypothesis. The candidate label will replace the {} placeholder. @@ -1082,14 +1051,14 @@ export class TextGenerationPipeline extends (/** @type {new (options: TextPipeli * If `false`, the scores are normalized such that the sum of the label likelihoods for each sequence * is 1. If `true`, the labels are considered independent and probabilities are normalized for each * candidate by doing a softmax of the entailment score vs. the contradiction score. - * + * * @callback ZeroShotClassificationPipelineCallback Classify the sequence(s) given as inputs. * @param {string|string[]} texts The sequence(s) to classify, will be truncated if the model input is too large. * @param {string|string[]} candidate_labels The set of possible class labels to classify each sequence into. * Can be a single label, a string of comma-separated labels, or a list of labels. * @param {ZeroShotClassificationPipelineOptions} [options] The options to use for zero-shot classification. * @returns {Promise} An array or object containing the predicted labels and scores. - * + * * @typedef {TextPipelineConstructorArgs & ZeroShotClassificationPipelineCallback & Disposable} ZeroShotClassificationPipelineType */ @@ -1098,7 +1067,7 @@ export class TextGenerationPipeline extends (/** @type {new (options: TextPipeli * trained on NLI (natural language inference) tasks. Equivalent of `text-classification` * pipelines, but these models don't require a hardcoded number of potential classes, they * can be chosen at runtime. It usually means it's slower but it is **much** more flexible. - * + * * **Example:** Zero shot classification with `Xenova/mobilebert-uncased-mnli`. * ```javascript * const classifier = await pipeline('zero-shot-classification', 'Xenova/mobilebert-uncased-mnli'); @@ -1111,7 +1080,7 @@ export class TextGenerationPipeline extends (/** @type {new (options: TextPipeli * // scores: [ 0.5562091040482018, 0.1843621307860853, 0.13942646639336376, 0.12000229877234923 ] * // } * ``` - * + * * **Example:** Zero shot classification with `Xenova/nli-deberta-v3-xsmall` (multi-label). * ```javascript * const classifier = await pipeline('zero-shot-classification', 'Xenova/nli-deberta-v3-xsmall'); @@ -1125,7 +1094,9 @@ export class TextGenerationPipeline extends (/** @type {new (options: TextPipeli * // } * ``` */ -export class ZeroShotClassificationPipeline extends (/** @type {new (options: TextPipelineConstructorArgs) => ZeroShotClassificationPipelineType} */ (Pipeline)) { +export class ZeroShotClassificationPipeline + extends /** @type {new (options: TextPipelineConstructorArgs) => ZeroShotClassificationPipelineType} */ (Pipeline) +{ /** * Create a new ZeroShotClassificationPipeline. * @param {TextPipelineConstructorArgs} options An object used to instantiate the pipeline. @@ -1135,18 +1106,16 @@ export class ZeroShotClassificationPipeline extends (/** @type {new (options: Te // Use model config to get label2id mapping this.label2id = Object.fromEntries( - Object.entries((/** @type {any} */(this).model).config.label2id).map( - ([k, v]) => [k.toLowerCase(), v] - ) + Object.entries(/** @type {any} */ (this).model.config.label2id).map(([k, v]) => [k.toLowerCase(), v]), ); - this.entailment_id = this.label2id['entailment']; + this.entailment_id = this.label2id["entailment"]; if (this.entailment_id === undefined) { console.warn("Could not find 'entailment' in label2id mapping. Using 2 as entailment_id."); this.entailment_id = 2; } - this.contradiction_id = this.label2id['contradiction'] ?? this.label2id['not_entailment']; + this.contradiction_id = this.label2id["contradiction"] ?? this.label2id["not_entailment"]; if (this.contradiction_id === undefined) { console.warn("Could not find 'contradiction' in label2id mapping. Using 0 as contradiction_id."); this.contradiction_id = 0; @@ -1154,11 +1123,7 @@ export class ZeroShotClassificationPipeline extends (/** @type {new (options: Te } /** @type {ZeroShotClassificationPipelineCallback} */ - async _call(texts, candidate_labels, { - hypothesis_template = "This example is {}.", - multi_label = false, - } = {}) { - + async _call(texts, candidate_labels, { hypothesis_template = "This example is {}.", multi_label = false } = {}) { const isBatched = Array.isArray(texts); if (!isBatched) { texts = [/** @type {string} */ (texts)]; @@ -1168,9 +1133,7 @@ export class ZeroShotClassificationPipeline extends (/** @type {new (options: Te } // Insert labels into hypothesis template - const hypotheses = candidate_labels.map( - x => hypothesis_template.replace('{}', x) - ); + const hypotheses = candidate_labels.map((x) => hypothesis_template.replace("{}", x)); // How to perform the softmax over the logits: // - true: softmax over the entailment vs. contradiction dim for each label independently @@ -1187,33 +1150,29 @@ export class ZeroShotClassificationPipeline extends (/** @type {new (options: Te text_pair: hypothesis, padding: true, truncation: true, - }) - const outputs = await this.model(inputs) + }); + const outputs = await this.model(inputs); if (softmaxEach) { entails_logits.push([ outputs.logits.data[this.contradiction_id], - outputs.logits.data[this.entailment_id] - ]) + outputs.logits.data[this.entailment_id], + ]); } else { - entails_logits.push(outputs.logits.data[this.entailment_id]) + entails_logits.push(outputs.logits.data[this.entailment_id]); } } /** @type {number[]} */ - const scores = softmaxEach - ? entails_logits.map(x => softmax(x)[1]) - : softmax(entails_logits); + const scores = softmaxEach ? entails_logits.map((x) => softmax(x)[1]) : softmax(entails_logits); // Sort by scores (desc) and return scores with indices - const scores_sorted = scores - .map((x, i) => [x, i]) - .sort((a, b) => (b[0] - a[0])); + const scores_sorted = scores.map((x, i) => [x, i]).sort((a, b) => b[0] - a[0]); toReturn.push({ sequence: premise, - labels: scores_sorted.map(x => candidate_labels[x[1]]), - scores: scores_sorted.map(x => x[0]), + labels: scores_sorted.map((x) => candidate_labels[x[1]]), + scores: scores_sorted.map((x) => x[0]), }); } return isBatched ? toReturn : toReturn[0]; @@ -1225,20 +1184,20 @@ export class ZeroShotClassificationPipeline extends (/** @type {new (options: Te * @property {'none'|'mean'|'cls'} [pooling="none"] The pooling method to use. * @property {boolean} [normalize=false] Whether or not to normalize the embeddings in the last dimension. * @property {boolean} [quantize=false] Whether or not to quantize the embeddings. - * @property {'binary'|'ubinary'} [precision='binary'] The precision to use for quantization. - * + * @property {'binary'|'ubinary'} [precision='binary'] The precision to use for quantization. + * * @callback FeatureExtractionPipelineCallback Extract the features of the input(s). * @param {string|string[]} texts One or several texts (or one list of texts) to get the features of. * @param {FeatureExtractionPipelineOptions} [options] The options to use for feature extraction. * @returns {Promise} The features computed by the model. - * + * * @typedef {TextPipelineConstructorArgs & FeatureExtractionPipelineCallback & Disposable} FeatureExtractionPipelineType */ /** * Feature extraction pipeline using no model head. This pipeline extracts the hidden * states from the base transformer, which can be used as features in downstream tasks. - * + * * **Example:** Run feature extraction with `bert-base-uncased` (without pooling/normalization). * ```javascript * const extractor = await pipeline('feature-extraction', 'Xenova/bert-base-uncased', { revision: 'default' }); @@ -1249,7 +1208,7 @@ export class ZeroShotClassificationPipeline extends (/** @type {new (options: Te * // dims: [1, 8, 768] * // } * ``` - * + * * **Example:** Run feature extraction with `bert-base-uncased` (with pooling/normalization). * ```javascript * const extractor = await pipeline('feature-extraction', 'Xenova/bert-base-uncased', { revision: 'default' }); @@ -1260,7 +1219,7 @@ export class ZeroShotClassificationPipeline extends (/** @type {new (options: Te * // dims: [1, 768] * // } * ``` - * + * * **Example:** Calculating embeddings with `sentence-transformers` models. * ```javascript * const extractor = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2'); @@ -1282,7 +1241,9 @@ export class ZeroShotClassificationPipeline extends (/** @type {new (options: Te * // } * ``` */ -export class FeatureExtractionPipeline extends (/** @type {new (options: TextPipelineConstructorArgs) => FeatureExtractionPipelineType} */ (Pipeline)) { +export class FeatureExtractionPipeline + extends /** @type {new (options: TextPipelineConstructorArgs) => FeatureExtractionPipelineType} */ (Pipeline) +{ /** * Create a new FeatureExtractionPipeline. * @param {TextPipelineConstructorArgs} options An object used to instantiate the pipeline. @@ -1292,13 +1253,15 @@ export class FeatureExtractionPipeline extends (/** @type {new (options: TextPip } /** @type {FeatureExtractionPipelineCallback} */ - async _call(texts, { - pooling = /** @type {'none'} */('none'), - normalize = false, - quantize = false, - precision = /** @type {'binary'} */('binary'), - } = {}) { - + async _call( + texts, + { + pooling = /** @type {'none'} */ ("none"), + normalize = false, + quantize = false, + precision = /** @type {'binary'} */ ("binary"), + } = {}, + ) { // Run tokenization const model_inputs = this.tokenizer(texts, { padding: true, @@ -1306,7 +1269,7 @@ export class FeatureExtractionPipeline extends (/** @type {new (options: TextPip }); // Run model - const outputs = await this.model(model_inputs) + const outputs = await this.model(model_inputs); // TODO: Provide warning to the user that they might be using model which was not exported // specifically for feature extraction @@ -1315,11 +1278,11 @@ export class FeatureExtractionPipeline extends (/** @type {new (options: TextPip /** @type {Tensor} */ let result = outputs.last_hidden_state ?? outputs.logits ?? outputs.token_embeddings; - if (pooling === 'none') { + if (pooling === "none") { // Skip pooling - } else if (pooling === 'mean') { + } else if (pooling === "mean") { result = mean_pooling(result, model_inputs.attention_mask); - } else if (pooling === 'cls') { + } else if (pooling === "cls") { result = result.slice(null, 0); } else { throw Error(`Pooling method '${pooling}' not supported.`); @@ -1337,23 +1300,22 @@ export class FeatureExtractionPipeline extends (/** @type {new (options: TextPip } } - /** * @typedef {Object} ImageFeatureExtractionPipelineOptions Parameters specific to image feature extraction pipelines. * @property {boolean} [pool=null] Whether or not to return the pooled output. If set to `false`, the model will return the raw hidden states. - * + * * @callback ImageFeatureExtractionPipelineCallback Extract the features of the input(s). * @param {ImagePipelineInputs} images One or several images (or one list of images) to get the features of. * @param {ImageFeatureExtractionPipelineOptions} [options] The options to use for image feature extraction. * @returns {Promise} The image features computed by the model. - * + * * @typedef {ImagePipelineConstructorArgs & ImageFeatureExtractionPipelineCallback & Disposable} ImageFeatureExtractionPipelineType */ /** * Image feature extraction pipeline using no model head. This pipeline extracts the hidden * states from the base transformer, which can be used as features in downstream tasks. - * + * * **Example:** Perform image feature extraction with `Xenova/vit-base-patch16-224-in21k`. * ```javascript * const image_feature_extractor = await pipeline('image-feature-extraction', 'Xenova/vit-base-patch16-224-in21k'); @@ -1366,7 +1328,7 @@ export class FeatureExtractionPipeline extends (/** @type {new (options: TextPip * // size: 151296 * // } * ``` - * + * * **Example:** Compute image embeddings with `Xenova/clip-vit-base-patch32`. * ```javascript * const image_feature_extractor = await pipeline('image-feature-extraction', 'Xenova/clip-vit-base-patch32'); @@ -1380,7 +1342,9 @@ export class FeatureExtractionPipeline extends (/** @type {new (options: TextPip * // } * ``` */ -export class ImageFeatureExtractionPipeline extends (/** @type {new (options: ImagePipelineConstructorArgs) => ImageFeatureExtractionPipelineType} */ (Pipeline)) { +export class ImageFeatureExtractionPipeline + extends /** @type {new (options: ImagePipelineConstructorArgs) => ImageFeatureExtractionPipelineType} */ (Pipeline) +{ /** * Create a new ImageFeatureExtractionPipeline. * @param {ImagePipelineConstructorArgs} options An object used to instantiate the pipeline. @@ -1390,10 +1354,7 @@ export class ImageFeatureExtractionPipeline extends (/** @type {new (options: Im } /** @type {ImageFeatureExtractionPipelineCallback} */ - async _call(images, { - pool = null, - } = {}) { - + async _call(images, { pool = null } = {}) { const preparedImages = await prepareImages(images); const { pixel_values } = await this.processor(preparedImages); const outputs = await this.model({ pixel_values }); @@ -1401,11 +1362,12 @@ export class ImageFeatureExtractionPipeline extends (/** @type {new (options: Im /** @type {Tensor} */ let result; if (pool) { - if (!('pooler_output' in outputs)) { - throw Error(`No pooled output was returned. Make sure the model has a 'pooler' layer when using the 'pool' option.`); + if (!("pooler_output" in outputs)) { + throw Error( + `No pooled output was returned. Make sure the model has a 'pooler' layer when using the 'pool' option.`, + ); } result = outputs.pooler_output; - } else { result = outputs.last_hidden_state ?? outputs.logits ?? outputs.image_embeds; } @@ -1422,12 +1384,12 @@ export class ImageFeatureExtractionPipeline extends (/** @type {new (options: Im * @property {string} label The label predicted. * @property {number} score The corresponding probability. * @typedef {AudioClassificationSingle[]} AudioClassificationOutput - * + * * @typedef {Object} AudioClassificationPipelineOptions Parameters specific to audio classification pipelines. * @property {number} [top_k=5] The number of top labels that will be returned by the pipeline. * If the provided number is `null` or higher than the number of labels available in the model configuration, * it will default to the number of labels. - * + * * @callback AudioClassificationPipelineCallback Classify the sequence(s) given as inputs. * @param {AudioPipelineInputs} audio The input audio file(s) to be classified. The input is either: * - `string` or `URL` that is the filename/URL of the audio file, the file will be read at the processor's sampling rate @@ -1436,14 +1398,14 @@ export class ImageFeatureExtractionPipeline extends (/** @type {new (options: Im * - `Float32Array` or `Float64Array` of shape `(n, )`, representing the raw audio at the correct sampling rate (no further check will be done). * @param {AudioClassificationPipelineOptions} [options] The options to use for audio classification. * @returns {Promise} An array or object containing the predicted labels and scores. - * + * * @typedef {AudioPipelineConstructorArgs & AudioClassificationPipelineCallback & Disposable} AudioClassificationPipelineType */ /** * Audio classification pipeline using any `AutoModelForAudioClassification`. * This pipeline predicts the class of a raw waveform or an audio file. - * + * * **Example:** Perform audio classification with `Xenova/wav2vec2-large-xlsr-53-gender-recognition-librispeech`. * ```javascript * const classifier = await pipeline('audio-classification', 'Xenova/wav2vec2-large-xlsr-53-gender-recognition-librispeech'); @@ -1454,7 +1416,7 @@ export class ImageFeatureExtractionPipeline extends (/** @type {new (options: Im * // { label: 'female', score: 0.001845747814513743 } * // ] * ``` - * + * * **Example:** Perform audio classification with `Xenova/ast-finetuned-audioset-10-10-0.4593` and return top 4 results. * ```javascript * const classifier = await pipeline('audio-classification', 'Xenova/ast-finetuned-audioset-10-10-0.4593'); @@ -1468,8 +1430,9 @@ export class ImageFeatureExtractionPipeline extends (/** @type {new (options: Im * // ] * ``` */ -export class AudioClassificationPipeline extends (/** @type {new (options: AudioPipelineConstructorArgs) => AudioClassificationPipelineType} */ (Pipeline)) { - +export class AudioClassificationPipeline + extends /** @type {new (options: AudioPipelineConstructorArgs) => AudioClassificationPipelineType} */ (Pipeline) +{ /** * Create a new AudioClassificationPipeline. * @param {AudioPipelineConstructorArgs} options An object used to instantiate the pipeline. @@ -1479,10 +1442,7 @@ export class AudioClassificationPipeline extends (/** @type {new (options: Audio } /** @type {AudioClassificationPipelineCallback} */ - async _call(audio, { - top_k = 5 - } = {}) { - + async _call(audio, { top_k = 5 } = {}) { const sampling_rate = this.processor.feature_extractor.config.sampling_rate; const preparedAudios = await prepareAudios(audio, sampling_rate); @@ -1494,11 +1454,7 @@ export class AudioClassificationPipeline extends (/** @type {new (options: Audio const output = await this.model(inputs); const logits = output.logits[0]; - const scores = await topk(new Tensor( - 'float32', - softmax(logits.data), - logits.dims, - ), top_k); + const scores = await topk(new Tensor("float32", softmax(logits.data), logits.dims), top_k); const values = scores[0].tolist(); const indices = scores[1].tolist(); @@ -1509,7 +1465,7 @@ export class AudioClassificationPipeline extends (/** @type {new (options: Audio })); toReturn.push(vals); - }; + } return Array.isArray(audio) ? toReturn : toReturn[0]; } } @@ -1518,12 +1474,12 @@ export class AudioClassificationPipeline extends (/** @type {new (options: Audio * @typedef {Object} ZeroShotAudioClassificationOutput * @property {string} label The label identified by the model. It is one of the suggested `candidate_label`. * @property {number} score The score attributed by the model for that label (between 0 and 1). - * + * * @typedef {Object} ZeroShotAudioClassificationPipelineOptions Parameters specific to zero-shot audio classification pipelines. * @property {string} [hypothesis_template="This is a sound of {}."] The sentence used in conjunction with `candidate_labels` * to attempt the audio classification by replacing the placeholder with the candidate_labels. * Then likelihood is estimated by using `logits_per_audio`. - * + * * @callback ZeroShotAudioClassificationPipelineCallback Classify the sequence(s) given as inputs. * @param {AudioPipelineInputs} audio The input audio file(s) to be classified. The input is either: * - `string` or `URL` that is the filename/URL of the audio file, the file will be read at the processor's sampling rate @@ -1533,14 +1489,14 @@ export class AudioClassificationPipeline extends (/** @type {new (options: Audio * @param {string[]} candidate_labels The candidate labels for this audio. * @param {ZeroShotAudioClassificationPipelineOptions} [options] The options to use for zero-shot audio classification. * @returns {Promise} An array of objects containing the predicted labels and scores. - * + * * @typedef {TextAudioPipelineConstructorArgs & ZeroShotAudioClassificationPipelineCallback & Disposable} ZeroShotAudioClassificationPipelineType */ /** * Zero shot audio classification pipeline using `ClapModel`. This pipeline predicts the class of an audio when you * provide an audio and a set of `candidate_labels`. - * + * * **Example**: Perform zero-shot audio classification with `Xenova/clap-htsat-unfused`. * ```javascript * const classifier = await pipeline('zero-shot-audio-classification', 'Xenova/clap-htsat-unfused'); @@ -1553,8 +1509,11 @@ export class AudioClassificationPipeline extends (/** @type {new (options: Audio * // ] * ``` */ -export class ZeroShotAudioClassificationPipeline extends (/** @type {new (options: TextAudioPipelineConstructorArgs) => ZeroShotAudioClassificationPipelineType} */ (Pipeline)) { - +export class ZeroShotAudioClassificationPipeline + extends /** @type {new (options: TextAudioPipelineConstructorArgs) => ZeroShotAudioClassificationPipelineType} */ ( + Pipeline + ) +{ /** * Create a new ZeroShotAudioClassificationPipeline. * @param {TextAudioPipelineConstructorArgs} options An object used to instantiate the pipeline. @@ -1564,19 +1523,14 @@ export class ZeroShotAudioClassificationPipeline extends (/** @type {new (option } /** @type {ZeroShotAudioClassificationPipelineCallback} */ - async _call(audio, candidate_labels, { - hypothesis_template = "This is a sound of {}." - } = {}) { - + async _call(audio, candidate_labels, { hypothesis_template = "This is a sound of {}." } = {}) { const single = !Array.isArray(audio); if (single) { audio = [/** @type {AudioInput} */ (audio)]; } - // Insert label into hypothesis template - const texts = candidate_labels.map( - x => hypothesis_template.replace('{}', x) - ); + // Insert label into hypothesis template + const texts = candidate_labels.map((x) => hypothesis_template.replace("{}", x)); // Run tokenization const text_inputs = this.tokenizer(texts, { @@ -1597,10 +1551,12 @@ export class ZeroShotAudioClassificationPipeline extends (/** @type {new (option // Compute softmax per audio const probs = softmax(output.logits_per_audio.data); - toReturn.push([...probs].map((x, i) => ({ - score: x, - label: candidate_labels[i] - }))); + toReturn.push( + [...probs].map((x, i) => ({ + score: x, + label: candidate_labels[i], + })), + ); } return single ? toReturn[0] : toReturn; } @@ -1617,7 +1573,7 @@ export class ZeroShotAudioClassificationPipeline extends (/** @type {new (option * @property {string} text The recognized text. * @property {Chunk[]} [chunks] When using `return_timestamps`, the `chunks` will become a list * containing all the various text chunks identified by the model. - * + * * @typedef {Object} AutomaticSpeechRecognitionSpecificParams Parameters specific to automatic-speech-recognition pipelines. * @property {boolean|'word'} [return_timestamps] Whether to return timestamps or not. Default is `false`. * @property {number} [chunk_length_s] The length of audio chunks to process in seconds. Default is 0 (no chunking). @@ -1627,7 +1583,7 @@ export class ZeroShotAudioClassificationPipeline extends (/** @type {new (option * @property {string} [task] The task to perform. Default is `null`, meaning it should be auto-detected. * @property {number} [num_frames] The number of frames in the input audio. * @typedef {import('./generation/configuration_utils.js').GenerationConfig & AutomaticSpeechRecognitionSpecificParams} AutomaticSpeechRecognitionConfig - * + * * @callback AutomaticSpeechRecognitionPipelineCallback Transcribe the audio sequence(s) given as inputs to text. * @param {AudioPipelineInputs} audio The input audio file(s) to be transcribed. The input is either: * - `string` or `URL` that is the filename/URL of the audio file, the file will be read at the processor's sampling rate @@ -1636,7 +1592,7 @@ export class ZeroShotAudioClassificationPipeline extends (/** @type {new (option * - `Float32Array` or `Float64Array` of shape `(n, )`, representing the raw audio at the correct sampling rate (no further check will be done). * @param {Partial} [options] Additional keyword arguments to pass along to the generate method of the model. * @returns {Promise} An object containing the transcription text and optionally timestamps if `return_timestamps` is `true`. - * + * * @typedef {TextAudioPipelineConstructorArgs & AutomaticSpeechRecognitionPipelineCallback & Disposable} AutomaticSpeechRecognitionPipelineType */ @@ -1650,7 +1606,7 @@ export class ZeroShotAudioClassificationPipeline extends (/** @type {new (option * const output = await transcriber(url); * // { text: " And so my fellow Americans ask not what your country can do for you, ask what you can do for your country." } * ``` - * + * * **Example:** Transcribe English w/ timestamps. * ```javascript * const transcriber = await pipeline('automatic-speech-recognition', 'Xenova/whisper-tiny.en'); @@ -1664,7 +1620,7 @@ export class ZeroShotAudioClassificationPipeline extends (/** @type {new (option * // ] * // } * ``` - * + * * **Example:** Transcribe English w/ word-level timestamps. * ```javascript * const transcriber = await pipeline('automatic-speech-recognition', 'Xenova/whisper-tiny.en'); @@ -1683,7 +1639,7 @@ export class ZeroShotAudioClassificationPipeline extends (/** @type {new (option * // ] * // } * ``` - * + * * **Example:** Transcribe French. * ```javascript * const transcriber = await pipeline('automatic-speech-recognition', 'Xenova/whisper-small'); @@ -1691,7 +1647,7 @@ export class ZeroShotAudioClassificationPipeline extends (/** @type {new (option * const output = await transcriber(url, { language: 'french', task: 'transcribe' }); * // { text: " J'adore, j'aime, je n'aime pas, je déteste." } * ``` - * + * * **Example:** Translate French to English. * ```javascript * const transcriber = await pipeline('automatic-speech-recognition', 'Xenova/whisper-small'); @@ -1699,7 +1655,7 @@ export class ZeroShotAudioClassificationPipeline extends (/** @type {new (option * const output = await transcriber(url, { language: 'french', task: 'translate' }); * // { text: " I love, I like, I don't like, I hate." } * ``` - * + * * **Example:** Transcribe/translate audio longer than 30 seconds. * ```javascript * const transcriber = await pipeline('automatic-speech-recognition', 'Xenova/whisper-tiny.en'); @@ -1708,8 +1664,11 @@ export class ZeroShotAudioClassificationPipeline extends (/** @type {new (option * // { text: " So in college, I was a government major, which means [...] So I'd start off light and I'd bump it up" } * ``` */ -export class AutomaticSpeechRecognitionPipeline extends (/** @type {new (options: TextAudioPipelineConstructorArgs) => AutomaticSpeechRecognitionPipelineType} */ (Pipeline)) { - +export class AutomaticSpeechRecognitionPipeline + extends /** @type {new (options: TextAudioPipelineConstructorArgs) => AutomaticSpeechRecognitionPipelineType} */ ( + Pipeline + ) +{ /** * Create a new AutomaticSpeechRecognitionPipeline. * @param {TextAudioPipelineConstructorArgs} options An object used to instantiate the pipeline. @@ -1721,16 +1680,18 @@ export class AutomaticSpeechRecognitionPipeline extends (/** @type {new (options /** @type {AutomaticSpeechRecognitionPipelineCallback} */ async _call(audio, kwargs = {}) { switch (this.model.config.model_type) { - case 'whisper': - return this._call_whisper(audio, kwargs) - case 'wav2vec2': - case 'wav2vec2-bert': - case 'unispeech': - case 'unispeech-sat': - case 'hubert': - return this._call_wav2vec2(audio, kwargs) + case "whisper": + return this._call_whisper(audio, kwargs); + case "wav2vec2": + case "wav2vec2-bert": + case "unispeech": + case "unispeech-sat": + case "hubert": + return this._call_wav2vec2(audio, kwargs); default: - throw new Error(`AutomaticSpeechRecognitionPipeline does not support model type '${this.model.config.model_type}'.`) + throw new Error( + `AutomaticSpeechRecognitionPipeline does not support model type '${this.model.config.model_type}'.`, + ); } } @@ -1764,10 +1725,10 @@ export class AutomaticSpeechRecognitionPipeline extends (/** @type {new (options const predicted_ids = []; for (const item of logits) { - predicted_ids.push(max(item.data)[1]) + predicted_ids.push(max(item.data)[1]); } - const predicted_sentences = this.tokenizer.decode(predicted_ids) - toReturn.push({ text: predicted_sentences }) + const predicted_sentences = this.tokenizer.decode(predicted_ids); + toReturn.push({ text: predicted_sentences }); } return single ? toReturn[0] : toReturn; } @@ -1782,11 +1743,11 @@ export class AutomaticSpeechRecognitionPipeline extends (/** @type {new (options const force_full_sequences = kwargs.force_full_sequences ?? false; let stride_length_s = kwargs.stride_length_s ?? null; - const generation_config = { ...kwargs } + const generation_config = { ...kwargs }; - if (return_timestamps === 'word') { - generation_config['return_token_timestamps'] = true; - generation_config['return_timestamps'] = false; // Do not predict timestamp tokens + if (return_timestamps === "word") { + generation_config["return_token_timestamps"] = true; + generation_config["return_timestamps"] = false; // Do not predict timestamp tokens } const single = !Array.isArray(audio); @@ -1794,7 +1755,8 @@ export class AutomaticSpeechRecognitionPipeline extends (/** @type {new (options audio = [/** @type {AudioInput} */ (audio)]; } - const time_precision = this.processor.feature_extractor.config.chunk_length / this.model.config.max_source_positions; + const time_precision = + this.processor.feature_extractor.config.chunk_length / this.model.config.max_source_positions; const hop_length = this.processor.feature_extractor.config.hop_length; const sampling_rate = this.processor.feature_extractor.config.sampling_rate; @@ -1808,7 +1770,7 @@ export class AutomaticSpeechRecognitionPipeline extends (/** @type {new (options if (stride_length_s === null) { stride_length_s = chunk_length_s / 6; } else if (chunk_length_s <= stride_length_s) { - throw Error("`chunk_length_s` must be larger than `stride_length_s`.") + throw Error("`chunk_length_s` must be larger than `stride_length_s`."); } // TODO support different stride_length_s (for left and right) @@ -1827,24 +1789,21 @@ export class AutomaticSpeechRecognitionPipeline extends (/** @type {new (options const is_first = offset === 0; const is_last = offset_end >= aud.length; chunks.push({ - stride: [ - subarr.length, - is_first ? 0 : stride, - is_last ? 0 : stride - ], + stride: [subarr.length, is_first ? 0 : stride, is_last ? 0 : stride], input_features: feature.input_features, is_last, - }) + }); if (is_last) break; offset += jump; } - } else { - chunks = [{ - stride: [aud.length, 0, 0], - input_features: (await this.processor(aud)).input_features, - is_last: true - }] + chunks = [ + { + stride: [aud.length, 0, 0], + input_features: (await this.processor(aud)).input_features, + is_last: true, + }, + ]; } // Generate for each set of input features @@ -1854,31 +1813,32 @@ export class AutomaticSpeechRecognitionPipeline extends (/** @type {new (options // NOTE: doing sequentially for now const data = await this.model.generate({ inputs: chunk.input_features, - ...generation_config + ...generation_config, }); // TODO: Right now we only get top beam - if (return_timestamps === 'word') { + if (return_timestamps === "word") { chunk.tokens = data.sequences.tolist()[0]; - chunk.token_timestamps = data.token_timestamps.tolist()[0].map( - (/** @type {number} */ x) => round(x, 2) - ); - + chunk.token_timestamps = data.token_timestamps + .tolist()[0] + .map((/** @type {number} */ x) => round(x, 2)); } else { - chunk.tokens = (/** @type {Tensor} */(data))[0].tolist(); + chunk.tokens = /** @type {Tensor} */ (data)[0].tolist(); } // convert stride to seconds - chunk.stride = chunk.stride.map(x => x / sampling_rate); + chunk.stride = chunk.stride.map((x) => x / sampling_rate); } // Merge text chunks // @ts-ignore const [full_text, optional] = this.tokenizer._decode_asr(chunks, { - time_precision, return_timestamps, force_full_sequences + time_precision, + return_timestamps, + force_full_sequences, }); - toReturn.push({ text: full_text, ...optional }) + toReturn.push({ text: full_text, ...optional }); } return single ? toReturn[0] : toReturn; } @@ -1888,18 +1848,18 @@ export class AutomaticSpeechRecognitionPipeline extends (/** @type {new (options * @typedef {Object} ImageToTextSingle * @property {string} generated_text The generated text. * @typedef {ImageToTextSingle[]} ImageToTextOutput - * + * * @callback ImageToTextPipelineCallback Assign labels to the image(s) passed as inputs. * @param {ImagePipelineInputs} texts The images to be captioned. * @param {Partial} [options] Additional keyword arguments to pass along to the generate method of the model. * @returns {Promise} An object (or array of objects) containing the generated text(s). - * + * * @typedef {TextImagePipelineConstructorArgs & ImageToTextPipelineCallback & Disposable} ImageToTextPipelineType */ /** * Image To Text pipeline using a `AutoModelForVision2Seq`. This pipeline predicts a caption for a given image. - * + * * **Example:** Generate a caption for an image w/ `Xenova/vit-gpt2-image-captioning`. * ```javascript * const captioner = await pipeline('image-to-text', 'Xenova/vit-gpt2-image-captioning'); @@ -1907,7 +1867,7 @@ export class AutomaticSpeechRecognitionPipeline extends (/** @type {new (options * const output = await captioner(url); * // [{ generated_text: 'a cat laying on a couch with another cat' }] * ``` - * + * * **Example:** Optical Character Recognition (OCR) w/ `Xenova/trocr-small-handwritten`. * ```javascript * const captioner = await pipeline('image-to-text', 'Xenova/trocr-small-handwritten'); @@ -1916,8 +1876,9 @@ export class AutomaticSpeechRecognitionPipeline extends (/** @type {new (options * // [{ generated_text: 'Mr. Brown commented icily.' }] * ``` */ -export class ImageToTextPipeline extends (/** @type {new (options: TextImagePipelineConstructorArgs) => ImageToTextPipelineType} */ (Pipeline)) { - +export class ImageToTextPipeline + extends /** @type {new (options: TextImagePipelineConstructorArgs) => ImageToTextPipelineType} */ (Pipeline) +{ /** * Create a new ImageToTextPipeline. * @param {TextImagePipelineConstructorArgs} options An object used to instantiate the pipeline. @@ -1928,7 +1889,6 @@ export class ImageToTextPipeline extends (/** @type {new (options: TextImagePipe /** @type {ImageToTextPipelineCallback} */ async _call(images, generate_kwargs = {}) { - const isBatched = Array.isArray(images); const preparedImages = await prepareImages(images); @@ -1936,11 +1896,13 @@ export class ImageToTextPipeline extends (/** @type {new (options: TextImagePipe const toReturn = []; for (const batch of pixel_values) { - batch.dims = [1, ...batch.dims] + batch.dims = [1, ...batch.dims]; const output = await this.model.generate({ inputs: batch, ...generate_kwargs }); - const decoded = this.tokenizer.batch_decode(/** @type {Tensor} */(output), { - skip_special_tokens: true, - }).map(x => ({ generated_text: x.trim() })) + const decoded = this.tokenizer + .batch_decode(/** @type {Tensor} */ (output), { + skip_special_tokens: true, + }) + .map((x) => ({ generated_text: x.trim() })); toReturn.push(decoded); } @@ -1953,22 +1915,22 @@ export class ImageToTextPipeline extends (/** @type {new (options: TextImagePipe * @property {string} label The label identified by the model. * @property {number} score The score attributed by the model for that label. * @typedef {ImageClassificationSingle[]} ImageClassificationOutput - * + * * @typedef {Object} ImageClassificationPipelineOptions Parameters specific to image classification pipelines. - * @property {number} [top_k=1] The number of top labels that will be returned by the pipeline. - * + * @property {number} [top_k=1] The number of top labels that will be returned by the pipeline. + * * @callback ImageClassificationPipelineCallback Assign labels to the image(s) passed as inputs. * @param {ImagePipelineInputs} images The input images(s) to be classified. * @param {ImageClassificationPipelineOptions} [options] The options to use for image classification. * @returns {Promise} An array or object containing the predicted labels and scores. - * + * * @typedef {ImagePipelineConstructorArgs & ImageClassificationPipelineCallback & Disposable} ImageClassificationPipelineType */ /** * Image classification pipeline using any `AutoModelForImageClassification`. * This pipeline predicts the class of an image. - * + * * **Example:** Classify an image. * ```javascript * const classifier = await pipeline('image-classification', 'Xenova/vit-base-patch16-224'); @@ -1978,7 +1940,7 @@ export class ImageToTextPipeline extends (/** @type {new (options: TextImagePipe * // { label: 'tiger, Panthera tigris', score: 0.632695734500885 }, * // ] * ``` - * + * * **Example:** Classify an image and return top `n` classes. * ```javascript * const classifier = await pipeline('image-classification', 'Xenova/vit-base-patch16-224'); @@ -1990,7 +1952,7 @@ export class ImageToTextPipeline extends (/** @type {new (options: TextImagePipe * // { label: 'lion, king of beasts, Panthera leo', score: 0.00045060308184474707 }, * // ] * ``` - * + * * **Example:** Classify an image and return all classes. * ```javascript * const classifier = await pipeline('image-classification', 'Xenova/vit-base-patch16-224'); @@ -2005,8 +1967,9 @@ export class ImageToTextPipeline extends (/** @type {new (options: TextImagePipe * // ] * ``` */ -export class ImageClassificationPipeline extends (/** @type {new (options: ImagePipelineConstructorArgs) => ImageClassificationPipelineType} */ (Pipeline)) { - +export class ImageClassificationPipeline + extends /** @type {new (options: ImagePipelineConstructorArgs) => ImageClassificationPipelineType} */ (Pipeline) +{ /** * Create a new ImageClassificationPipeline. * @param {ImagePipelineConstructorArgs} options An object used to instantiate the pipeline. @@ -2016,10 +1979,7 @@ export class ImageClassificationPipeline extends (/** @type {new (options: Image } /** @type {ImageClassificationPipelineCallback} */ - async _call(images, { - top_k = 5 - } = {}) { - + async _call(images, { top_k = 5 } = {}) { const preparedImages = await prepareImages(images); const { pixel_values } = await this.processor(preparedImages); @@ -2030,11 +1990,7 @@ export class ImageClassificationPipeline extends (/** @type {new (options: Image /** @type {ImageClassificationOutput[]} */ const toReturn = []; for (const batch of output.logits) { - const scores = await topk(new Tensor( - 'float32', - softmax(batch.data), - batch.dims, - ), top_k); + const scores = await topk(new Tensor("float32", softmax(batch.data), batch.dims), top_k); const values = scores[0].tolist(); const indices = scores[1].tolist(); @@ -2048,7 +2004,6 @@ export class ImageClassificationPipeline extends (/** @type {new (options: Image return Array.isArray(images) ? toReturn : toReturn[0]; } - } /** @@ -2056,7 +2011,7 @@ export class ImageClassificationPipeline extends (/** @type {new (options: Image * @property {string} label The label of the segment. * @property {number|null} score The score of the segment. * @property {RawImage} mask The mask of the segment. - * + * * @typedef {Object} ImageSegmentationPipelineOptions Parameters specific to image segmentation pipelines. * @property {number} [threshold=0.5] Probability threshold to filter out predicted masks. * @property {number} [mask_threshold=0.5] Threshold to use when turning the predicted masks into binary values. @@ -2065,19 +2020,19 @@ export class ImageClassificationPipeline extends (/** @type {new (options: Image * depending on model capabilities. If not set, the pipeline will attempt to resolve (in that order). * @property {number[]} [label_ids_to_fuse=null] List of label ids to fuse. If not set, do not fuse any labels. * @property {number[][]} [target_sizes=null] List of target sizes for the input images. If not set, use the original image sizes. - * + * * @callback ImageSegmentationPipelineCallback Segment the input images. * @param {ImagePipelineInputs} images The input images. * @param {ImageSegmentationPipelineOptions} [options] The options to use for image segmentation. * @returns {Promise} The annotated segments. - * + * * @typedef {ImagePipelineConstructorArgs & ImageSegmentationPipelineCallback & Disposable} ImageSegmentationPipelineType */ /** * Image segmentation pipeline using any `AutoModelForXXXSegmentation`. * This pipeline predicts masks of objects and their classes. - * + * * **Example:** Perform image segmentation with `Xenova/detr-resnet-50-panoptic`. * ```javascript * const segmenter = await pipeline('image-segmentation', 'Xenova/detr-resnet-50-panoptic'); @@ -2089,7 +2044,9 @@ export class ImageClassificationPipeline extends (/** @type {new (options: Image * // ] * ``` */ -export class ImageSegmentationPipeline extends (/** @type {new (options: ImagePipelineConstructorArgs) => ImageSegmentationPipelineType} */ (Pipeline)) { +export class ImageSegmentationPipeline + extends /** @type {new (options: ImagePipelineConstructorArgs) => ImageSegmentationPipelineType} */ (Pipeline) +{ /** * Create a new ImageSegmentationPipeline. * @param {ImagePipelineConstructorArgs} options An object used to instantiate the pipeline. @@ -2099,21 +2056,24 @@ export class ImageSegmentationPipeline extends (/** @type {new (options: ImagePi this.subtasks_mapping = { // Mapping of subtasks to their corresponding post-processing function names. - panoptic: 'post_process_panoptic_segmentation', - instance: 'post_process_instance_segmentation', - semantic: 'post_process_semantic_segmentation' - } + panoptic: "post_process_panoptic_segmentation", + instance: "post_process_instance_segmentation", + semantic: "post_process_semantic_segmentation", + }; } /** @type {ImageSegmentationPipelineCallback} */ - async _call(images, { - threshold = 0.5, - mask_threshold = 0.5, - overlap_mask_area_threshold = 0.8, - label_ids_to_fuse = null, - target_sizes = null, - subtask = null, - } = {}) { + async _call( + images, + { + threshold = 0.5, + mask_threshold = 0.5, + overlap_mask_area_threshold = 0.8, + label_ids_to_fuse = null, + target_sizes = null, + subtask = null, + } = {}, + ) { const isBatched = Array.isArray(images); if (isBatched && images.length !== 1) { @@ -2121,7 +2081,7 @@ export class ImageSegmentationPipeline extends (/** @type {new (options: ImagePi } const preparedImages = await prepareImages(images); - const imageSizes = preparedImages.map(x => [x.height, x.width]); + const imageSizes = preparedImages.map((x) => [x.height, x.width]); const { pixel_values, pixel_mask } = await this.processor(preparedImages); const output = await this.model({ pixel_values, pixel_mask }); @@ -2143,7 +2103,7 @@ export class ImageSegmentationPipeline extends (/** @type {new (options: ImagePi /** @type {ImageSegmentationPipelineOutput[]} */ const annotation = []; - if (subtask === 'panoptic' || subtask === 'instance') { + if (subtask === "panoptic" || subtask === "instance") { const processed = fn( output, threshold, @@ -2163,16 +2123,15 @@ export class ImageSegmentationPipeline extends (/** @type {new (options: ImagePi } } - const mask = new RawImage(maskData, segmentation.dims[1], segmentation.dims[0], 1) + const mask = new RawImage(maskData, segmentation.dims[1], segmentation.dims[0], 1); annotation.push({ score: segment.score, label: id2label[segment.label_id], - mask: mask - }) + mask: mask, + }); } - - } else if (subtask === 'semantic') { + } else if (subtask === "semantic") { const { segmentation, labels } = fn(output, target_sizes ?? imageSizes)[0]; for (const label of labels) { @@ -2188,7 +2147,7 @@ export class ImageSegmentationPipeline extends (/** @type {new (options: ImagePi annotation.push({ score: null, label: id2label[label], - mask: mask + mask: mask, }); } } else { @@ -2203,25 +2162,25 @@ export class ImageSegmentationPipeline extends (/** @type {new (options: ImagePi * @typedef {Object} ZeroShotImageClassificationOutput * @property {string} label The label identified by the model. It is one of the suggested `candidate_label`. * @property {number} score The score attributed by the model for that label (between 0 and 1). - * + * * @typedef {Object} ZeroShotImageClassificationPipelineOptions Parameters specific to zero-shot image classification pipelines. * @property {string} [hypothesis_template="This is a photo of {}"] The sentence used in conjunction with `candidate_labels` * to attempt the image classification by replacing the placeholder with the candidate_labels. * Then likelihood is estimated by using `logits_per_image`. - * + * * @callback ZeroShotImageClassificationPipelineCallback Assign labels to the image(s) passed as inputs. * @param {ImagePipelineInputs} images The input images. * @param {string[]} candidate_labels The candidate labels for this image. * @param {ZeroShotImageClassificationPipelineOptions} [options] The options to use for zero-shot image classification. * @returns {Promise} An array of objects containing the predicted labels and scores. - * + * * @typedef {TextImagePipelineConstructorArgs & ZeroShotImageClassificationPipelineCallback & Disposable} ZeroShotImageClassificationPipelineType */ /** * Zero shot image classification pipeline. This pipeline predicts the class of * an image when you provide an image and a set of `candidate_labels`. - * + * * **Example:** Zero shot image classification w/ `Xenova/clip-vit-base-patch32`. * ```javascript * const classifier = await pipeline('zero-shot-image-classification', 'Xenova/clip-vit-base-patch32'); @@ -2234,7 +2193,11 @@ export class ImageSegmentationPipeline extends (/** @type {new (options: ImagePi * // ] * ``` */ -export class ZeroShotImageClassificationPipeline extends (/** @type {new (options: TextImagePipelineConstructorArgs) => ZeroShotImageClassificationPipelineType} */ (Pipeline)) { +export class ZeroShotImageClassificationPipeline + extends /** @type {new (options: TextImagePipelineConstructorArgs) => ZeroShotImageClassificationPipelineType} */ ( + Pipeline + ) +{ /** * Create a new ZeroShotImageClassificationPipeline. * @param {TextImagePipelineConstructorArgs} options An object used to instantiate the pipeline. @@ -2244,21 +2207,16 @@ export class ZeroShotImageClassificationPipeline extends (/** @type {new (option } /** @type {ZeroShotImageClassificationPipelineCallback} */ - async _call(images, candidate_labels, { - hypothesis_template = "This is a photo of {}" - } = {}) { - + async _call(images, candidate_labels, { hypothesis_template = "This is a photo of {}" } = {}) { const isBatched = Array.isArray(images); const preparedImages = await prepareImages(images); - // Insert label into hypothesis template - const texts = candidate_labels.map( - x => hypothesis_template.replace('{}', x) - ); + // Insert label into hypothesis template + const texts = candidate_labels.map((x) => hypothesis_template.replace("{}", x)); // Run tokenization const text_inputs = this.tokenizer(texts, { - padding: this.model.config.model_type === 'siglip' ? 'max_length' : true, + padding: this.model.config.model_type === "siglip" ? "max_length" : true, truncation: true, }); @@ -2269,9 +2227,9 @@ export class ZeroShotImageClassificationPipeline extends (/** @type {new (option const output = await this.model({ ...text_inputs, pixel_values }); const function_to_apply = - this.model.config.model_type === 'siglip' - ? batch => batch.sigmoid().data - : batch => softmax(batch.data); + this.model.config.model_type === "siglip" + ? (batch) => batch.sigmoid().data + : (batch) => softmax(batch.data); // Compare each image with each candidate label const toReturn = []; @@ -2281,7 +2239,7 @@ export class ZeroShotImageClassificationPipeline extends (/** @type {new (option const result = [...probs].map((x, i) => ({ score: x, - label: candidate_labels[i] + label: candidate_labels[i], })); result.sort((a, b) => b.score - a.score); // sort by score in descending order toReturn.push(result); @@ -2291,30 +2249,29 @@ export class ZeroShotImageClassificationPipeline extends (/** @type {new (option } } - /** * @typedef {Object} ObjectDetectionPipelineSingle * @property {string} label The class label identified by the model. * @property {number} score The score attributed by the model for that label. * @property {BoundingBox} box The bounding box of detected object in image's original size, or as a percentage if `percentage` is set to true. * @typedef {ObjectDetectionPipelineSingle[]} ObjectDetectionPipelineOutput - * + * * @typedef {Object} ObjectDetectionPipelineOptions Parameters specific to object detection pipelines. * @property {number} [threshold=0.9] The threshold used to filter boxes by score. * @property {boolean} [percentage=false] Whether to return the boxes coordinates in percentage (true) or in pixels (false). - * + * * @callback ObjectDetectionPipelineCallback Detect objects (bounding boxes & classes) in the image(s) passed as inputs. * @param {ImagePipelineInputs} images The input images. * @param {ObjectDetectionPipelineOptions} [options] The options to use for object detection. - * @returns {Promise} A list of objects or a list of list of objects. - * + * @returns {Promise} A list of objects or a list of list of objects. + * * @typedef {ImagePipelineConstructorArgs & ObjectDetectionPipelineCallback & Disposable} ObjectDetectionPipelineType */ /** * Object detection pipeline using any `AutoModelForObjectDetection`. * This pipeline predicts bounding boxes of objects and their classes. - * + * * **Example:** Run object-detection with `Xenova/detr-resnet-50`. * ```javascript * const detector = await pipeline('object-detection', 'Xenova/detr-resnet-50'); @@ -2333,8 +2290,9 @@ export class ZeroShotImageClassificationPipeline extends (/** @type {new (option * // }] * ``` */ -export class ObjectDetectionPipeline extends (/** @type {new (options: ImagePipelineConstructorArgs) => ObjectDetectionPipelineType} */ (Pipeline)) { - +export class ObjectDetectionPipeline + extends /** @type {new (options: ImagePipelineConstructorArgs) => ObjectDetectionPipelineType} */ (Pipeline) +{ /** * Create a new ObjectDetectionPipeline. * @param {ImagePipelineConstructorArgs} options An object used to instantiate the pipeline. @@ -2344,11 +2302,7 @@ export class ObjectDetectionPipeline extends (/** @type {new (options: ImagePipe } /** @type {ObjectDetectionPipelineCallback} */ - async _call(images, { - threshold = 0.9, - percentage = false, - } = {}) { - + async _call(images, { threshold = 0.9, percentage = false } = {}) { const isBatched = Array.isArray(images); if (isBatched && images.length !== 1) { @@ -2356,7 +2310,7 @@ export class ObjectDetectionPipeline extends (/** @type {new (options: ImagePipe } const preparedImages = await prepareImages(images); - const imageSizes = percentage ? null : preparedImages.map(x => [x.height, x.width]); + const imageSizes = percentage ? null : preparedImages.map((x) => [x.height, x.width]); const { pixel_values, pixel_mask } = await this.processor(preparedImages); const output = await this.model({ pixel_values, pixel_mask }); @@ -2369,45 +2323,44 @@ export class ObjectDetectionPipeline extends (/** @type {new (options: ImagePipe // Format output /** @type {ObjectDetectionPipelineOutput[]} */ - const result = processed.map(batch => ( + const result = processed.map((batch) => batch.boxes.map((box, i) => ({ score: batch.scores[i], label: id2label[batch.classes[i]], box: get_bounding_box(box, !percentage), - })) - )) + })), + ); return isBatched ? result : result[0]; } } - /** * @typedef {Object} ZeroShotObjectDetectionOutput * @property {string} label Text query corresponding to the found object. * @property {number} score Score corresponding to the object (between 0 and 1). * @property {BoundingBox} box Bounding box of the detected object in image's original size, or as a percentage if `percentage` is set to true. - * + * * @typedef {Object} ZeroShotObjectDetectionPipelineOptions Parameters specific to zero-shot object detection pipelines. * @property {number} [threshold=0.1] The probability necessary to make a prediction. * @property {number} [top_k=null] The number of top predictions that will be returned by the pipeline. * If the provided number is `null` or higher than the number of predictions available, it will default * to the number of predictions. * @property {boolean} [percentage=false] Whether to return the boxes coordinates in percentage (true) or in pixels (false). - * + * * @callback ZeroShotObjectDetectionPipelineCallback Detect objects (bounding boxes & classes) in the image(s) passed as inputs. * @param {ImagePipelineInputs} images The input images. * @param {string[]} candidate_labels What the model should recognize in the image. * @param {ZeroShotObjectDetectionPipelineOptions} [options] The options to use for zero-shot object detection. * @returns {Promise} An array of objects containing the predicted labels, scores, and bounding boxes. - * + * * @typedef {TextImagePipelineConstructorArgs & ZeroShotObjectDetectionPipelineCallback & Disposable} ZeroShotObjectDetectionPipelineType */ /** * Zero-shot object detection pipeline. This pipeline predicts bounding boxes of * objects when you provide an image and a set of `candidate_labels`. - * + * * **Example:** Zero-shot object detection w/ `Xenova/owlvit-base-patch32`. * ```javascript * const detector = await pipeline('zero-shot-object-detection', 'Xenova/owlvit-base-patch32'); @@ -2437,7 +2390,7 @@ export class ObjectDetectionPipeline extends (/** @type {new (options: ImagePipe * // } * // ] * ``` - * + * * **Example:** Zero-shot object detection w/ `Xenova/owlvit-base-patch32` (returning top 4 matches and setting a threshold). * ```javascript * const detector = await pipeline('zero-shot-object-detection', 'Xenova/owlvit-base-patch32'); @@ -2468,8 +2421,11 @@ export class ObjectDetectionPipeline extends (/** @type {new (options: ImagePipe * // ] * ``` */ -export class ZeroShotObjectDetectionPipeline extends (/** @type {new (options: TextImagePipelineConstructorArgs) => ZeroShotObjectDetectionPipelineType} */ (Pipeline)) { - +export class ZeroShotObjectDetectionPipeline + extends /** @type {new (options: TextImagePipelineConstructorArgs) => ZeroShotObjectDetectionPipelineType} */ ( + Pipeline + ) +{ /** * Create a new ZeroShotObjectDetectionPipeline. * @param {TextImagePipelineConstructorArgs} options An object used to instantiate the pipeline. @@ -2479,12 +2435,7 @@ export class ZeroShotObjectDetectionPipeline extends (/** @type {new (options: T } /** @type {ZeroShotObjectDetectionPipelineCallback} */ - async _call(images, candidate_labels, { - threshold = 0.1, - top_k = null, - percentage = false, - } = {}) { - + async _call(images, candidate_labels, { threshold = 0.1, top_k = null, percentage = false } = {}) { const isBatched = Array.isArray(images); const preparedImages = await prepareImages(images); @@ -2510,16 +2461,23 @@ export class ZeroShotObjectDetectionPipeline extends (/** @type {new (options: T const output = await this.model({ ...text_inputs, pixel_values }); // @ts-ignore - const processed = this.processor.feature_extractor.post_process_object_detection(output, threshold, imageSize, true)[0]; - let result = processed.boxes.map((box, i) => ({ - score: processed.scores[i], - label: candidate_labels[processed.classes[i]], - box: get_bounding_box(box, !percentage), - })).sort((a, b) => b.score - a.score); + const processed = this.processor.feature_extractor.post_process_object_detection( + output, + threshold, + imageSize, + true, + )[0]; + let result = processed.boxes + .map((box, i) => ({ + score: processed.scores[i], + label: candidate_labels[processed.classes[i]], + box: get_bounding_box(box, !percentage), + })) + .sort((a, b) => b.score - a.score); if (top_k !== null) { result = result.slice(0, top_k); } - toReturn.push(result) + toReturn.push(result); } return isBatched ? toReturn : toReturn[0]; @@ -2530,13 +2488,13 @@ export class ZeroShotObjectDetectionPipeline extends (/** @type {new (options: T * @typedef {Object} DocumentQuestionAnsweringSingle * @property {string} answer The generated text. * @typedef {DocumentQuestionAnsweringSingle[]} DocumentQuestionAnsweringOutput - * + * * @callback DocumentQuestionAnsweringPipelineCallback Answer the question given as input by using the document. * @param {ImageInput} image The image of the document to use. * @param {string} question A question to ask of the document. * @param {Partial} [options] Additional keyword arguments to pass along to the generate method of the model. * @returns {Promise} An object (or array of objects) containing the answer(s). - * + * * @typedef {TextImagePipelineConstructorArgs & DocumentQuestionAnsweringPipelineCallback & Disposable} DocumentQuestionAnsweringPipelineType */ @@ -2544,7 +2502,7 @@ export class ZeroShotObjectDetectionPipeline extends (/** @type {new (options: T * Document Question Answering pipeline using any `AutoModelForDocumentQuestionAnswering`. * The inputs/outputs are similar to the (extractive) question answering pipeline; however, * the pipeline takes an image (and optional OCR'd words/boxes) as input instead of text context. - * + * * **Example:** Answer questions about a document with `Xenova/donut-base-finetuned-docvqa`. * ```javascript * const qa_pipeline = await pipeline('document-question-answering', 'Xenova/donut-base-finetuned-docvqa'); @@ -2554,8 +2512,11 @@ export class ZeroShotObjectDetectionPipeline extends (/** @type {new (options: T * // [{ answer: 'us-001' }] * ``` */ -export class DocumentQuestionAnsweringPipeline extends (/** @type {new (options: TextImagePipelineConstructorArgs) => DocumentQuestionAnsweringPipelineType} */ (Pipeline)) { - +export class DocumentQuestionAnsweringPipeline + extends /** @type {new (options: TextImagePipelineConstructorArgs) => DocumentQuestionAnsweringPipelineType} */ ( + Pipeline + ) +{ /** * Create a new DocumentQuestionAnsweringPipeline. * @param {TextImagePipelineConstructorArgs} options An object used to instantiate the pipeline. @@ -2566,7 +2527,6 @@ export class DocumentQuestionAnsweringPipeline extends (/** @type {new (options: /** @type {DocumentQuestionAnsweringPipelineCallback} */ async _call(image, question, generate_kwargs = {}) { - // NOTE: For now, we only support a batch size of 1 // Preprocess image @@ -2590,7 +2550,7 @@ export class DocumentQuestionAnsweringPipeline extends (/** @type {new (options: }); // Decode output - const decoded = this.tokenizer.batch_decode(/** @type {Tensor} */(output))[0]; + const decoded = this.tokenizer.batch_decode(/** @type {Tensor} */ (output))[0]; // Parse answer const match = decoded.match(/(.*?)<\/s_answer>/); @@ -2602,7 +2562,6 @@ export class DocumentQuestionAnsweringPipeline extends (/** @type {new (options: } } - /** * @typedef {Object} VocoderOptions * @property {PreTrainedModel} [vocoder] The vocoder used by the pipeline (if the model uses one). If not provided, use the default HifiGan vocoder. @@ -2613,22 +2572,22 @@ export class DocumentQuestionAnsweringPipeline extends (/** @type {new (options: * @typedef {Object} TextToAudioOutput * @property {Float32Array} audio The generated audio waveform. * @property {number} sampling_rate The sampling rate of the generated audio waveform. - * + * * @typedef {Object} TextToAudioPipelineOptions Parameters specific to text-to-audio pipelines. * @property {Tensor|Float32Array|string|URL} [speaker_embeddings=null] The speaker embeddings (if the model requires it). - * + * * @callback TextToAudioPipelineCallback Generates speech/audio from the inputs. * @param {string|string[]} texts The text(s) to generate. * @param {TextToAudioPipelineOptions} options Parameters passed to the model generation/forward method. * @returns {Promise} An object containing the generated audio and sampling rate. - * + * * @typedef {TextToAudioPipelineConstructorArgs & TextToAudioPipelineCallback & Disposable} TextToAudioPipelineType */ /** * Text-to-audio generation pipeline using any `AutoModelForTextToWaveform` or `AutoModelForTextToSpectrogram`. * This pipeline generates an audio file from an input text and optional other conditional inputs. - * + * * **Example:** Generate audio from text with `Xenova/speecht5_tts`. * ```javascript * const synthesizer = await pipeline('text-to-speech', 'Xenova/speecht5_tts', { quantized: false }); @@ -2639,17 +2598,17 @@ export class DocumentQuestionAnsweringPipeline extends (/** @type {new (options: * // sampling_rate: 16000 * // } * ``` - * + * * You can then save the audio to a .wav file with the `wavefile` package: * ```javascript * import wavefile from 'wavefile'; * import fs from 'fs'; - * + * * const wav = new wavefile.WaveFile(); * wav.fromScratch(1, out.sampling_rate, '32f', out.audio); * fs.writeFileSync('out.wav', wav.toBuffer()); * ``` - * + * * **Example:** Multilingual speech generation with `Xenova/mms-tts-fra`. See [here](https://huggingface.co/models?pipeline_tag=text-to-speech&other=vits&sort=trending) for the full list of available languages (1107). * ```javascript * const synthesizer = await pipeline('text-to-speech', 'Xenova/mms-tts-fra'); @@ -2660,8 +2619,10 @@ export class DocumentQuestionAnsweringPipeline extends (/** @type {new (options: * // } * ``` */ -export class TextToAudioPipeline extends (/** @type {new (options: TextToAudioPipelineConstructorArgs) => TextToAudioPipelineType} */ (Pipeline)) { - DEFAULT_VOCODER_ID = "Xenova/speecht5_hifigan" +export class TextToAudioPipeline + extends /** @type {new (options: TextToAudioPipelineConstructorArgs) => TextToAudioPipelineType} */ (Pipeline) +{ + DEFAULT_VOCODER_ID = "Xenova/speecht5_hifigan"; /** * Create a new TextToAudioPipeline. @@ -2674,12 +2635,8 @@ export class TextToAudioPipeline extends (/** @type {new (options: TextToAudioPi this.vocoder = options.vocoder ?? null; } - /** @type {TextToAudioPipelineCallback} */ - async _call(text_inputs, { - speaker_embeddings = null, - } = {}) { - + async _call(text_inputs, { speaker_embeddings = null } = {}) { // If this.processor is not set, we are using a `AutoModelForTextToWaveform` model if (this.processor) { return this._call_text_to_spectrogram(text_inputs, { speaker_embeddings }); @@ -2689,7 +2646,6 @@ export class TextToAudioPipeline extends (/** @type {new (options: TextToAudioPi } async _call_text_to_waveform(text_inputs) { - // Run tokenization const inputs = this.tokenizer(text_inputs, { padding: true, @@ -2703,33 +2659,26 @@ export class TextToAudioPipeline extends (/** @type {new (options: TextToAudioPi return { audio: waveform.data, sampling_rate, - } + }; } async _call_text_to_spectrogram(text_inputs, { speaker_embeddings }) { - // Load vocoder, if not provided if (!this.vocoder) { - console.log('No vocoder specified, using default HifiGan vocoder.'); - this.vocoder = await AutoModel.from_pretrained(this.DEFAULT_VOCODER_ID, { dtype: 'fp32' }); + console.log("No vocoder specified, using default HifiGan vocoder."); + this.vocoder = await AutoModel.from_pretrained(this.DEFAULT_VOCODER_ID, { dtype: "fp32" }); } // Load speaker embeddings as Float32Array from path/URL - if (typeof speaker_embeddings === 'string' || speaker_embeddings instanceof URL) { + if (typeof speaker_embeddings === "string" || speaker_embeddings instanceof URL) { // Load from URL with fetch - speaker_embeddings = new Float32Array( - await (await fetch(speaker_embeddings)).arrayBuffer() - ); + speaker_embeddings = new Float32Array(await (await fetch(speaker_embeddings)).arrayBuffer()); } if (speaker_embeddings instanceof Float32Array) { - speaker_embeddings = new Tensor( - 'float32', - speaker_embeddings, - [1, speaker_embeddings.length] - ) + speaker_embeddings = new Tensor("float32", speaker_embeddings, [1, speaker_embeddings.length]); } else if (!(speaker_embeddings instanceof Tensor)) { - throw new Error("Speaker embeddings must be a `Tensor`, `Float32Array`, `string`, or `URL`.") + throw new Error("Speaker embeddings must be a `Tensor`, `Float32Array`, `string`, or `URL`."); } // Run tokenization @@ -2746,7 +2695,7 @@ export class TextToAudioPipeline extends (/** @type {new (options: TextToAudioPi return { audio: waveform.data, sampling_rate, - } + }; } } @@ -2754,13 +2703,13 @@ export class TextToAudioPipeline extends (/** @type {new (options: TextToAudioPi * @callback ImageToImagePipelineCallback Transform the image(s) passed as inputs. * @param {ImagePipelineInputs} images The images to transform. * @returns {Promise} The transformed image or list of images. - * + * * @typedef {ImagePipelineConstructorArgs & ImageToImagePipelineCallback & Disposable} ImageToImagePipelineType */ /** * Image to Image pipeline using any `AutoModelForImageToImage`. This pipeline generates an image based on a previous image input. - * + * * **Example:** Super-resolution w/ `Xenova/swin2SR-classical-sr-x2-64` * ```javascript * const upscaler = await pipeline('image-to-image', 'Xenova/swin2SR-classical-sr-x2-64'); @@ -2774,7 +2723,9 @@ export class TextToAudioPipeline extends (/** @type {new (options: TextToAudioPi * // } * ``` */ -export class ImageToImagePipeline extends (/** @type {new (options: ImagePipelineConstructorArgs) => ImageToImagePipelineType} */ (Pipeline)) { +export class ImageToImagePipeline + extends /** @type {new (options: ImagePipelineConstructorArgs) => ImageToImagePipelineType} */ (Pipeline) +{ /** * Create a new ImageToImagePipeline. * @param {ImagePipelineConstructorArgs} options An object used to instantiate the pipeline. @@ -2785,7 +2736,6 @@ export class ImageToImagePipeline extends (/** @type {new (options: ImagePipelin /** @type {ImageToImagePipelineCallback} */ async _call(images) { - const preparedImages = await prepareImages(images); const inputs = await this.processor(preparedImages); const outputs = await this.model(inputs); @@ -2793,7 +2743,7 @@ export class ImageToImagePipeline extends (/** @type {new (options: ImagePipelin /** @type {RawImage[]} */ const toReturn = []; for (const batch of outputs.reconstruction) { - const output = batch.squeeze().clamp_(0, 1).mul_(255).round_().to('uint8'); + const output = batch.squeeze().clamp_(0, 1).mul_(255).round_().to("uint8"); toReturn.push(RawImage.fromTensor(output)); } @@ -2805,17 +2755,17 @@ export class ImageToImagePipeline extends (/** @type {new (options: ImagePipelin * @typedef {Object} DepthEstimationPipelineOutput * @property {Tensor} predicted_depth The raw depth map predicted by the model. * @property {RawImage} depth The processed depth map as an image (with the same size as the input image). - * + * * @callback DepthEstimationPipelineCallback Predicts the depth for the image(s) passed as inputs. * @param {ImagePipelineInputs} images The images to compute depth for. * @returns {Promise} An image or a list of images containing result(s). - * + * * @typedef {ImagePipelineConstructorArgs & DepthEstimationPipelineCallback & Disposable} DepthEstimationPipelineType */ /** * Depth estimation pipeline using any `AutoModelForDepthEstimation`. This pipeline predicts the depth of an image. - * + * * **Example:** Depth estimation w/ `Xenova/dpt-hybrid-midas` * ```javascript * const depth_estimator = await pipeline('depth-estimation', 'Xenova/dpt-hybrid-midas'); @@ -2837,7 +2787,9 @@ export class ImageToImagePipeline extends (/** @type {new (options: ImagePipelin * // } * ``` */ -export class DepthEstimationPipeline extends (/** @type {new (options: ImagePipelineConstructorArgs) => DepthEstimationPipelineType} */ (Pipeline)) { +export class DepthEstimationPipeline + extends /** @type {new (options: ImagePipelineConstructorArgs) => DepthEstimationPipelineType} */ (Pipeline) +{ /** * Create a new DepthEstimationPipeline. * @param {ImagePipelineConstructorArgs} options An object used to instantiate the pipeline. @@ -2848,7 +2800,6 @@ export class DepthEstimationPipeline extends (/** @type {new (options: ImagePipe /** @type {DepthEstimationPipelineCallback} */ async _call(images) { - const preparedImages = await prepareImages(images); const inputs = await this.processor(preparedImages); @@ -2856,8 +2807,8 @@ export class DepthEstimationPipeline extends (/** @type {new (options: ImagePipe const toReturn = []; for (let i = 0; i < preparedImages.length; ++i) { - const prediction = interpolate(predicted_depth[i], preparedImages[i].size.reverse(), 'bilinear', false); - const formatted = prediction.mul_(255 / max(prediction.data)[0]).to('uint8'); + const prediction = interpolate(predicted_depth[i], preparedImages[i].size.reverse(), "bilinear", false); + const formatted = prediction.mul_(255 / max(prediction.data)[0]).to("uint8"); toReturn.push({ predicted_depth: predicted_depth[i], depth: RawImage.fromTensor(formatted), @@ -2870,301 +2821,300 @@ export class DepthEstimationPipeline extends (/** @type {new (options: ImagePipe const SUPPORTED_TASKS = Object.freeze({ "text-classification": { - "tokenizer": AutoTokenizer, - "pipeline": TextClassificationPipeline, - "model": AutoModelForSequenceClassification, - "default": { + tokenizer: AutoTokenizer, + pipeline: TextClassificationPipeline, + model: AutoModelForSequenceClassification, + default: { // TODO: replace with original // "model": "distilbert-base-uncased-finetuned-sst-2-english", - "model": "Xenova/distilbert-base-uncased-finetuned-sst-2-english", + model: "Xenova/distilbert-base-uncased-finetuned-sst-2-english", }, - "type": "text", + type: "text", }, "token-classification": { - "tokenizer": AutoTokenizer, - "pipeline": TokenClassificationPipeline, - "model": AutoModelForTokenClassification, - "default": { + tokenizer: AutoTokenizer, + pipeline: TokenClassificationPipeline, + model: AutoModelForTokenClassification, + default: { // TODO: replace with original // "model": "Davlan/bert-base-multilingual-cased-ner-hrl", - "model": "Xenova/bert-base-multilingual-cased-ner-hrl", + model: "Xenova/bert-base-multilingual-cased-ner-hrl", }, - "type": "text", + type: "text", }, "question-answering": { - "tokenizer": AutoTokenizer, - "pipeline": QuestionAnsweringPipeline, - "model": AutoModelForQuestionAnswering, - "default": { + tokenizer: AutoTokenizer, + pipeline: QuestionAnsweringPipeline, + model: AutoModelForQuestionAnswering, + default: { // TODO: replace with original // "model": "distilbert-base-cased-distilled-squad", - "model": "Xenova/distilbert-base-cased-distilled-squad", + model: "Xenova/distilbert-base-cased-distilled-squad", }, - "type": "text", + type: "text", }, "fill-mask": { - "tokenizer": AutoTokenizer, - "pipeline": FillMaskPipeline, - "model": AutoModelForMaskedLM, - "default": { + tokenizer: AutoTokenizer, + pipeline: FillMaskPipeline, + model: AutoModelForMaskedLM, + default: { // TODO: replace with original // "model": "bert-base-uncased", - "model": "Xenova/bert-base-uncased", + model: "Xenova/bert-base-uncased", }, - "type": "text", + type: "text", }, - "summarization": { - "tokenizer": AutoTokenizer, - "pipeline": SummarizationPipeline, - "model": AutoModelForSeq2SeqLM, - "default": { + summarization: { + tokenizer: AutoTokenizer, + pipeline: SummarizationPipeline, + model: AutoModelForSeq2SeqLM, + default: { // TODO: replace with original // "model": "sshleifer/distilbart-cnn-6-6", - "model": "Xenova/distilbart-cnn-6-6", + model: "Xenova/distilbart-cnn-6-6", }, - "type": "text", + type: "text", }, - "translation": { - "tokenizer": AutoTokenizer, - "pipeline": TranslationPipeline, - "model": AutoModelForSeq2SeqLM, - "default": { + translation: { + tokenizer: AutoTokenizer, + pipeline: TranslationPipeline, + model: AutoModelForSeq2SeqLM, + default: { // TODO: replace with original // "model": "t5-small", - "model": "Xenova/t5-small", + model: "Xenova/t5-small", }, - "type": "text", + type: "text", }, "text2text-generation": { - "tokenizer": AutoTokenizer, - "pipeline": Text2TextGenerationPipeline, - "model": AutoModelForSeq2SeqLM, - "default": { + tokenizer: AutoTokenizer, + pipeline: Text2TextGenerationPipeline, + model: AutoModelForSeq2SeqLM, + default: { // TODO: replace with original // "model": "google/flan-t5-small", - "model": "Xenova/flan-t5-small", + model: "Xenova/flan-t5-small", }, - "type": "text", + type: "text", }, "text-generation": { - "tokenizer": AutoTokenizer, - "pipeline": TextGenerationPipeline, - "model": AutoModelForCausalLM, - "default": { + tokenizer: AutoTokenizer, + pipeline: TextGenerationPipeline, + model: AutoModelForCausalLM, + default: { // TODO: replace with original // "model": "gpt2", - "model": "Xenova/gpt2", + model: "Xenova/gpt2", }, - "type": "text", + type: "text", }, "zero-shot-classification": { - "tokenizer": AutoTokenizer, - "pipeline": ZeroShotClassificationPipeline, - "model": AutoModelForSequenceClassification, - "default": { + tokenizer: AutoTokenizer, + pipeline: ZeroShotClassificationPipeline, + model: AutoModelForSequenceClassification, + default: { // TODO: replace with original // "model": "typeform/distilbert-base-uncased-mnli", - "model": "Xenova/distilbert-base-uncased-mnli", + model: "Xenova/distilbert-base-uncased-mnli", }, - "type": "text", + type: "text", }, "audio-classification": { - "pipeline": AudioClassificationPipeline, - "model": AutoModelForAudioClassification, - "processor": AutoProcessor, - "default": { + pipeline: AudioClassificationPipeline, + model: AutoModelForAudioClassification, + processor: AutoProcessor, + default: { // TODO: replace with original // "model": "superb/wav2vec2-base-superb-ks", - "model": "Xenova/wav2vec2-base-superb-ks", + model: "Xenova/wav2vec2-base-superb-ks", }, - "type": "audio", + type: "audio", }, "zero-shot-audio-classification": { - "tokenizer": AutoTokenizer, - "pipeline": ZeroShotAudioClassificationPipeline, - "model": AutoModel, - "processor": AutoProcessor, - "default": { + tokenizer: AutoTokenizer, + pipeline: ZeroShotAudioClassificationPipeline, + model: AutoModel, + processor: AutoProcessor, + default: { // TODO: replace with original // "model": "laion/clap-htsat-fused", - "model": "Xenova/clap-htsat-unfused", + model: "Xenova/clap-htsat-unfused", }, - "type": "multimodal", + type: "multimodal", }, "automatic-speech-recognition": { - "tokenizer": AutoTokenizer, - "pipeline": AutomaticSpeechRecognitionPipeline, - "model": [AutoModelForSpeechSeq2Seq, AutoModelForCTC], - "processor": AutoProcessor, - "default": { + tokenizer: AutoTokenizer, + pipeline: AutomaticSpeechRecognitionPipeline, + model: [AutoModelForSpeechSeq2Seq, AutoModelForCTC], + processor: AutoProcessor, + default: { // TODO: replace with original // "model": "openai/whisper-tiny.en", - "model": "Xenova/whisper-tiny.en", + model: "Xenova/whisper-tiny.en", }, - "type": "multimodal", + type: "multimodal", }, "text-to-audio": { - "tokenizer": AutoTokenizer, - "pipeline": TextToAudioPipeline, - "model": [AutoModelForTextToWaveform, AutoModelForTextToSpectrogram], - "processor": [AutoProcessor, /* Some don't use a processor */ null], - "default": { + tokenizer: AutoTokenizer, + pipeline: TextToAudioPipeline, + model: [AutoModelForTextToWaveform, AutoModelForTextToSpectrogram], + processor: [AutoProcessor, /* Some don't use a processor */ null], + default: { // TODO: replace with original // "model": "microsoft/speecht5_tts", - "model": "Xenova/speecht5_tts", + model: "Xenova/speecht5_tts", }, - "type": "text", + type: "text", }, "image-to-text": { - "tokenizer": AutoTokenizer, - "pipeline": ImageToTextPipeline, - "model": AutoModelForVision2Seq, - "processor": AutoProcessor, - "default": { + tokenizer: AutoTokenizer, + pipeline: ImageToTextPipeline, + model: AutoModelForVision2Seq, + processor: AutoProcessor, + default: { // TODO: replace with original // "model": "nlpconnect/vit-gpt2-image-captioning", - "model": "Xenova/vit-gpt2-image-captioning", + model: "Xenova/vit-gpt2-image-captioning", }, - "type": "multimodal", + type: "multimodal", }, "image-classification": { // no tokenizer - "pipeline": ImageClassificationPipeline, - "model": AutoModelForImageClassification, - "processor": AutoProcessor, - "default": { + pipeline: ImageClassificationPipeline, + model: AutoModelForImageClassification, + processor: AutoProcessor, + default: { // TODO: replace with original // "model": "google/vit-base-patch16-224", - "model": "Xenova/vit-base-patch16-224", + model: "Xenova/vit-base-patch16-224", }, - "type": "multimodal", + type: "multimodal", }, "image-segmentation": { // no tokenizer - "pipeline": ImageSegmentationPipeline, - "model": [AutoModelForImageSegmentation, AutoModelForSemanticSegmentation, AutoModelForUniversalSegmentation], - "processor": AutoProcessor, - "default": { + pipeline: ImageSegmentationPipeline, + model: [AutoModelForImageSegmentation, AutoModelForSemanticSegmentation, AutoModelForUniversalSegmentation], + processor: AutoProcessor, + default: { // TODO: replace with original // "model": "facebook/detr-resnet-50-panoptic", - "model": "Xenova/detr-resnet-50-panoptic", + model: "Xenova/detr-resnet-50-panoptic", }, - "type": "multimodal", + type: "multimodal", }, "zero-shot-image-classification": { - "tokenizer": AutoTokenizer, - "pipeline": ZeroShotImageClassificationPipeline, - "model": AutoModel, - "processor": AutoProcessor, - "default": { + tokenizer: AutoTokenizer, + pipeline: ZeroShotImageClassificationPipeline, + model: AutoModel, + processor: AutoProcessor, + default: { // TODO: replace with original // "model": "openai/clip-vit-base-patch32", - "model": "Xenova/clip-vit-base-patch32", + model: "Xenova/clip-vit-base-patch32", }, - "type": "multimodal", + type: "multimodal", }, "object-detection": { // no tokenizer - "pipeline": ObjectDetectionPipeline, - "model": AutoModelForObjectDetection, - "processor": AutoProcessor, - "default": { + pipeline: ObjectDetectionPipeline, + model: AutoModelForObjectDetection, + processor: AutoProcessor, + default: { // TODO: replace with original // "model": "facebook/detr-resnet-50", - "model": "Xenova/detr-resnet-50", + model: "Xenova/detr-resnet-50", }, - "type": "multimodal", + type: "multimodal", }, "zero-shot-object-detection": { - "tokenizer": AutoTokenizer, - "pipeline": ZeroShotObjectDetectionPipeline, - "model": AutoModelForZeroShotObjectDetection, - "processor": AutoProcessor, - "default": { + tokenizer: AutoTokenizer, + pipeline: ZeroShotObjectDetectionPipeline, + model: AutoModelForZeroShotObjectDetection, + processor: AutoProcessor, + default: { // TODO: replace with original // "model": "google/owlvit-base-patch32", - "model": "Xenova/owlvit-base-patch32", + model: "Xenova/owlvit-base-patch32", }, - "type": "multimodal", + type: "multimodal", }, "document-question-answering": { - "tokenizer": AutoTokenizer, - "pipeline": DocumentQuestionAnsweringPipeline, - "model": AutoModelForDocumentQuestionAnswering, - "processor": AutoProcessor, - "default": { + tokenizer: AutoTokenizer, + pipeline: DocumentQuestionAnsweringPipeline, + model: AutoModelForDocumentQuestionAnswering, + processor: AutoProcessor, + default: { // TODO: replace with original // "model": "naver-clova-ix/donut-base-finetuned-docvqa", - "model": "Xenova/donut-base-finetuned-docvqa", + model: "Xenova/donut-base-finetuned-docvqa", }, - "type": "multimodal", + type: "multimodal", }, "image-to-image": { // no tokenizer - "pipeline": ImageToImagePipeline, - "model": AutoModelForImageToImage, - "processor": AutoProcessor, - "default": { + pipeline: ImageToImagePipeline, + model: AutoModelForImageToImage, + processor: AutoProcessor, + default: { // TODO: replace with original // "model": "caidas/swin2SR-classical-sr-x2-64", - "model": "Xenova/swin2SR-classical-sr-x2-64", + model: "Xenova/swin2SR-classical-sr-x2-64", }, - "type": "image", + type: "image", }, "depth-estimation": { // no tokenizer - "pipeline": DepthEstimationPipeline, - "model": AutoModelForDepthEstimation, - "processor": AutoProcessor, - "default": { + pipeline: DepthEstimationPipeline, + model: AutoModelForDepthEstimation, + processor: AutoProcessor, + default: { // TODO: replace with original // "model": "Intel/dpt-large", - "model": "Xenova/dpt-large", + model: "Xenova/dpt-large", }, - "type": "image", + type: "image", }, // This task serves as a useful interface for dealing with sentence-transformers (https://huggingface.co/sentence-transformers). "feature-extraction": { - "tokenizer": AutoTokenizer, - "pipeline": FeatureExtractionPipeline, - "model": AutoModel, - "default": { + tokenizer: AutoTokenizer, + pipeline: FeatureExtractionPipeline, + model: AutoModel, + default: { // TODO: replace with original // "model": "sentence-transformers/all-MiniLM-L6-v2", - "model": "Xenova/all-MiniLM-L6-v2", + model: "Xenova/all-MiniLM-L6-v2", }, - "type": "text", + type: "text", }, "image-feature-extraction": { - "processor": AutoProcessor, - "pipeline": ImageFeatureExtractionPipeline, - "model": [AutoModelForImageFeatureExtraction, AutoModel], - "default": { + processor: AutoProcessor, + pipeline: ImageFeatureExtractionPipeline, + model: [AutoModelForImageFeatureExtraction, AutoModel], + default: { // TODO: replace with original // "model": "google/vit-base-patch16-224", - "model": "Xenova/vit-base-patch16-224-in21k", + model: "Xenova/vit-base-patch16-224-in21k", }, - "type": "image", + type: "image", }, -}) - +}); // TODO: Add types for TASK_ALIASES const TASK_ALIASES = Object.freeze({ "sentiment-analysis": "text-classification", - "ner": "token-classification", + ner: "token-classification", // "vqa": "visual-question-answering", // TODO: Add - "asr": "automatic-speech-recognition", + asr: "automatic-speech-recognition", "text-to-speech": "text-to-audio", // Add for backwards compatibility - "embeddings": "feature-extraction", + embeddings: "feature-extraction", }); /** @@ -3178,7 +3128,7 @@ const TASK_ALIASES = Object.freeze({ /** * Utility factory method to build a `Pipeline` object. - * + * * @template {PipelineType} T The type of pipeline to return. * @param {T} task The task defining which pipeline will be returned. Currently accepted tasks are: * - `"audio-classification"`: will return a `AudioClassificationPipeline`. @@ -3216,12 +3166,12 @@ export async function pipeline( config = null, cache_dir = null, local_files_only = false, - revision = 'main', + revision = "main", device = null, dtype = null, model_file_name = null, session_options = {}, - } = {} + } = {}, ) { // Helper method to construct pipeline @@ -3230,14 +3180,14 @@ export async function pipeline( task = TASK_ALIASES[task] ?? task; // Get pipeline info - const pipelineInfo = SUPPORTED_TASKS[task.split('_', 1)[0]]; + const pipelineInfo = SUPPORTED_TASKS[task.split("_", 1)[0]]; if (!pipelineInfo) { - throw Error(`Unsupported pipeline: ${task}. Must be one of [${Object.keys(SUPPORTED_TASKS)}]`) + throw Error(`Unsupported pipeline: ${task}. Must be one of [${Object.keys(SUPPORTED_TASKS)}]`); } // Use model if specified, otherwise, use default if (!model) { - model = pipelineInfo.default.model + model = pipelineInfo.default.model; console.log(`No model specified. Using default model: "${model}".`); } @@ -3251,12 +3201,12 @@ export async function pipeline( dtype, model_file_name, session_options, - } + }; const classes = new Map([ - ['tokenizer', pipelineInfo.tokenizer], - ['model', pipelineInfo.model], - ['processor', pipelineInfo.processor], + ["tokenizer", pipelineInfo.tokenizer], + ["model", pipelineInfo.model], + ["processor", pipelineInfo.processor], ]); // Load model, tokenizer, and processor (if they exist) @@ -3264,16 +3214,15 @@ export async function pipeline( results.task = task; dispatchCallback(progress_callback, { - 'status': 'ready', - 'task': task, - 'model': model, + status: "ready", + task: task, + model: model, }); const pipelineClass = pipelineInfo.pipeline; return new pipelineClass(results); } - /** * Helper function to get applicable model, tokenizer, or processor classes for a given model. * @param {Map} mapping The mapping of names to classes, arrays of classes, or null. @@ -3282,7 +3231,6 @@ export async function pipeline( * @private */ async function loadItems(mapping, model, pretrainedOptions) { - const result = Object.create(null); /**@type {Promise[]} */ @@ -3306,21 +3254,20 @@ async function loadItems(mapping, model, pretrainedOptions) { resolve(await c.from_pretrained(model, pretrainedOptions)); return; } catch (err) { - if (err.message?.includes('Unsupported model type')) { + if (err.message?.includes("Unsupported model type")) { // If the error is due to an unsupported model type, we // save the error and try the next class. e = err; - } else if (err.message?.includes('Could not locate file')) { + } else if (err.message?.includes("Could not locate file")) { e = err; } else { reject(err); return; } - } } reject(e); - }) + }); } else { promise = cls.from_pretrained(model, pretrainedOptions); } @@ -3338,4 +3285,4 @@ async function loadItems(mapping, model, pretrainedOptions) { } return result; -} \ No newline at end of file +} diff --git a/src/processors.js b/src/processors.js index 9af0791be..3c7e511c2 100644 --- a/src/processors.js +++ b/src/processors.js @@ -1,7 +1,6 @@ - /** * @file Processors are used to prepare non-textual inputs (e.g., image or audio) for a model. - * + * * **Example:** Using a `WhisperProcessor` to prepare an audio input for a model. * ```javascript * import { AutoProcessor, read_audio } from '@huggingface/transformers'; @@ -16,55 +15,32 @@ * // size: 240000, * // } * ``` - * + * * @module processors */ -import { - Callable, -} from './utils/generic.js'; - -import { - calculateDimensions, - calculateReflectOffset, -} from './utils/core.js'; +import { Callable } from "./utils/generic.js"; -import { - getModelJSON, -} from './utils/hub.js'; +import { calculateDimensions, calculateReflectOffset } from "./utils/core.js"; -import { - min, - max, - softmax, - bankers_round, -} from './utils/maths.js'; +import { getModelJSON } from "./utils/hub.js"; +import { min, max, softmax, bankers_round } from "./utils/maths.js"; -import { Tensor, cat, interpolate, stack, interpolate_4d, full } from './utils/tensor.js'; - -import { RawImage } from './utils/image.js'; -import { - window_function, - spectrogram, - mel_filter_bank, -} from './utils/audio.js'; +import { Tensor, cat, interpolate, stack, interpolate_4d, full } from "./utils/tensor.js"; +import { RawImage } from "./utils/image.js"; +import { window_function, spectrogram, mel_filter_bank } from "./utils/audio.js"; // Helper functions /** * Converts bounding boxes from center format to corners format. - * + * * @param {number[]} arr The coordinate for the center of the box and its width, height dimensions (center_x, center_y, width, height) * @returns {number[]} The coodinates for the top-left and bottom-right corners of the box (top_left_x, top_left_y, bottom_right_x, bottom_right_y) */ function center_to_corners_format([centerX, centerY, width, height]) { - return [ - centerX - width / 2, - centerY - height / 2, - centerX + width / 2, - centerY + height / 2 - ]; + return [centerX - width / 2, centerY - height / 2, centerX + width / 2, centerY + height / 2]; } /** @@ -84,7 +60,7 @@ function post_process_object_detection(outputs, threshold = 0.5, target_sizes = const [batch_size, num_boxes, num_classes] = out_logits.dims; if (target_sizes !== null && target_sizes.length !== batch_size) { - throw Error("Make sure that you pass in as many target sizes as the batch dimension of the logits") + throw Error("Make sure that you pass in as many target sizes as the batch dimension of the logits"); } let toReturn = []; for (let i = 0; i < batch_size; ++i) { @@ -92,8 +68,8 @@ function post_process_object_detection(outputs, threshold = 0.5, target_sizes = let info = { boxes: [], classes: [], - scores: [] - } + scores: [], + }; let logits = out_logits[i]; let bbox = out_bbox[i]; @@ -110,7 +86,6 @@ function post_process_object_detection(outputs, threshold = 0.5, target_sizes = indices.push(k); } } - } else { // Get most probable class let maxIndex = max(logit.data)[1]; @@ -129,15 +104,14 @@ function post_process_object_detection(outputs, threshold = 0.5, target_sizes = } for (const index of indices) { - // Some class has a high enough probability /** @type {number[]} */ let box = bbox[j].data; // convert to [x0, y0, x1, y1] format - box = center_to_corners_format(box) + box = center_to_corners_format(box); if (target_size !== null) { - box = box.map((x, i) => x * target_size[(i + 1) % 2]) + box = box.map((x, i) => x * target_size[(i + 1) % 2]); } info.boxes.push(box); @@ -150,7 +124,6 @@ function post_process_object_detection(outputs, threshold = 0.5, target_sizes = return toReturn; } - /** * Post-processes the outputs of the model (for semantic segmentation). * @param {*} outputs Raw outputs of the model. @@ -159,12 +132,11 @@ function post_process_object_detection(outputs, threshold = 0.5, target_sizes = * @returns {{segmentation: Tensor; labels: number[]}[]} The semantic segmentation maps. */ function post_process_semantic_segmentation(outputs, target_sizes = null) { - const logits = outputs.logits; const batch_size = logits.dims[0]; if (target_sizes !== null && target_sizes.length !== batch_size) { - throw Error("Make sure that you pass in as many target sizes as the batch dimension of the logits") + throw Error("Make sure that you pass in as many target sizes as the batch dimension of the logits"); } const toReturn = []; @@ -176,15 +148,11 @@ function post_process_semantic_segmentation(outputs, target_sizes = null) { // 1. If target_size is not null, we need to resize the masks to the target size if (target_size !== null) { // resize the masks to the target size - data = interpolate(data, target_size, 'bilinear', false); + data = interpolate(data, target_size, "bilinear", false); } const [height, width] = target_size ?? data.dims.slice(-2); - const segmentation = new Tensor( - 'int32', - new Int32Array(height * width), - [height, width] - ); + const segmentation = new Tensor("int32", new Int32Array(height * width), [height, width]); // Buffer to store current largest value const buffer = data[0].data; @@ -207,14 +175,13 @@ function post_process_semantic_segmentation(outputs, target_sizes = null) { hasLabel[index] = index; } /** @type {number[]} The unique list of labels that were detected */ - const labels = hasLabel.filter(x => x !== undefined); + const labels = hasLabel.filter((x) => x !== undefined); toReturn.push({ segmentation, labels }); } return toReturn; } - /** * Binarize the given masks using `object_mask_threshold`, it returns the associated values of `masks`, `scores` and `labels`. * @param {Tensor} class_logits The class logits. @@ -225,7 +192,6 @@ function post_process_semantic_segmentation(outputs, target_sizes = null) { * @private */ function remove_low_and_no_objects(class_logits, mask_logits, object_mask_threshold, num_labels) { - const mask_probs_item = []; const pred_scores_item = []; const pred_labels_item = []; @@ -262,13 +228,7 @@ function remove_low_and_no_objects(class_logits, mask_logits, object_mask_thresh * @returns {[boolean, number[]]} Whether the segment is valid or not, and the indices of the valid labels. * @private */ -function check_segment_validity( - mask_labels, - mask_probs, - k, - mask_threshold = 0.5, - overlap_mask_area_threshold = 0.8 -) { +function check_segment_validity(mask_labels, mask_probs, k, mask_threshold = 0.5, overlap_mask_area_threshold = 0.8) { // mask_k is a 1D array of indices, indicating where the mask is equal to k const mask_k = []; let mask_k_area = 0; @@ -296,7 +256,7 @@ function check_segment_validity( mask_exists = area_ratio > overlap_mask_area_threshold; } - return [mask_exists, mask_k] + return [mask_exists, mask_k]; } /** @@ -322,24 +282,20 @@ function compute_segments( ) { const [height, width] = target_size ?? mask_probs[0].dims; - const segmentation = new Tensor( - 'int32', - new Int32Array(height * width), - [height, width] - ); + const segmentation = new Tensor("int32", new Int32Array(height * width), [height, width]); const segments = []; // 1. If target_size is not null, we need to resize the masks to the target size if (target_size !== null) { // resize the masks to the target size for (let i = 0; i < mask_probs.length; ++i) { - mask_probs[i] = interpolate(mask_probs[i], target_size, 'bilinear', false); + mask_probs[i] = interpolate(mask_probs[i], target_size, "bilinear", false); } } // 2. Weigh each mask by its prediction score // NOTE: `mask_probs` is updated in-place - // + // // Temporary storage for the best label/scores for each pixel ([height, width]): const mask_labels = new Int32Array(mask_probs[0].data.length); const bestScores = new Float32Array(mask_probs[0].data.length); @@ -350,7 +306,7 @@ function compute_segments( const mask_probs_i_data = mask_probs[i].data; for (let j = 0; j < mask_probs_i_data.length; ++j) { - mask_probs_i_data[j] *= score + mask_probs_i_data[j] *= score; if (mask_probs_i_data[j] > bestScores[j]) { mask_labels[j] = i; bestScores[j] = mask_probs_i_data[j]; @@ -374,8 +330,8 @@ function compute_segments( mask_probs, k, mask_threshold, - overlap_mask_area_threshold - ) + overlap_mask_area_threshold, + ); if (!mask_exists) { // Nothing to see here @@ -390,7 +346,6 @@ function compute_segments( // } ++current_segment_id; - // Add current object segment to final segmentation map for (const index of mask_k) { segmentation_data[index] = current_segment_id; @@ -401,7 +356,7 @@ function compute_segments( label_id: pred_class, // was_fused: should_fuse, TODO score: pred_scores[k], - }) + }); // TODO // if(should_fuse){ @@ -412,7 +367,6 @@ function compute_segments( return [segmentation, segments]; } - /** * Post-process the model output to generate the final panoptic segmentation. * @param {*} outputs The model output to post process @@ -432,20 +386,20 @@ function post_process_panoptic_segmentation( target_sizes = null, ) { if (label_ids_to_fuse === null) { - console.warn("`label_ids_to_fuse` unset. No instance will be fused.") + console.warn("`label_ids_to_fuse` unset. No instance will be fused."); label_ids_to_fuse = new Set(); } const class_queries_logits = outputs.class_queries_logits ?? outputs.logits; // [batch_size, num_queries, num_classes+1] const masks_queries_logits = outputs.masks_queries_logits ?? outputs.pred_masks; // [batch_size, num_queries, height, width] - const mask_probs = masks_queries_logits.sigmoid() // [batch_size, num_queries, height, width] + const mask_probs = masks_queries_logits.sigmoid(); // [batch_size, num_queries, height, width] let [batch_size, num_queries, num_labels] = class_queries_logits.dims; num_labels -= 1; // Remove last class (background) if (target_sizes !== null && target_sizes.length !== batch_size) { - throw Error("Make sure that you pass in as many target sizes as the batch dimension of the logits") + throw Error("Make sure that you pass in as many target sizes as the batch dimension of the logits"); } let toReturn = []; @@ -455,25 +409,25 @@ function post_process_panoptic_segmentation( let class_logits = class_queries_logits[i]; let mask_logits = mask_probs[i]; - let [mask_probs_item, pred_scores_item, pred_labels_item] = remove_low_and_no_objects(class_logits, mask_logits, threshold, num_labels); + let [mask_probs_item, pred_scores_item, pred_labels_item] = remove_low_and_no_objects( + class_logits, + mask_logits, + threshold, + num_labels, + ); if (pred_labels_item.length === 0) { // No mask found let [height, width] = target_size ?? mask_logits.dims.slice(-2); - let segmentation = new Tensor( - 'int32', - new Int32Array(height * width).fill(-1), - [height, width] - ) + let segmentation = new Tensor("int32", new Int32Array(height * width).fill(-1), [height, width]); toReturn.push({ segmentation: segmentation, - segments_info: [] + segments_info: [], }); continue; } - // Get segmentation map and segment information of batch item let [segmentation, segments] = compute_segments( mask_probs_item, @@ -483,18 +437,17 @@ function post_process_panoptic_segmentation( overlap_mask_area_threshold, label_ids_to_fuse, target_size, - ) + ); toReturn.push({ segmentation: segmentation, - segments_info: segments - }) + segments_info: segments, + }); } return toReturn; } - /** * Post-processes the outputs of the model (for instance segmentation). * @param {*} outputs Raw outputs of the model. @@ -504,7 +457,7 @@ function post_process_panoptic_segmentation( * @returns {Array<{ segmentation: Tensor, segments_info: Array<{id: number, label_id: number, score: number}>}>} */ function post_process_instance_segmentation(outputs, threshold = 0.5, target_sizes = null) { - throw new Error('Not implemented yet'); + throw new Error("Not implemented yet"); return []; } @@ -524,8 +477,8 @@ function validate_audio_inputs(audio, feature_extractor) { if (!(audio instanceof Float32Array || audio instanceof Float64Array)) { throw new Error( `${feature_extractor} expects input to be a Float32Array or a Float64Array, but got ${audio?.constructor?.name ?? typeof audio} instead. ` + - `If using the feature extractor directly, remember to use \`read_audio(url, sampling_rate)\` to obtain the raw audio data of the file/url.` - ) + `If using the feature extractor directly, remember to use \`read_audio(url, sampling_rate)\` to obtain the raw audio data of the file/url.`, + ); } } @@ -560,13 +513,9 @@ function constraint_to_multiple_of(val, multiple, minVal = 0, maxVal = null) { * @returns {[number, number]} The rounded size. */ function enforce_size_divisibility([width, height], divisor) { - return [ - Math.max(Math.floor(width / divisor), 1) * divisor, - Math.max(Math.floor(height / divisor), 1) * divisor - ]; + return [Math.max(Math.floor(width / divisor), 1) * divisor, Math.max(Math.floor(height / divisor), 1) * divisor]; } - /** * Base class for feature extractors. * @@ -580,7 +529,7 @@ export class FeatureExtractor extends Callable { */ constructor(config) { super(); - this.config = config + this.config = config; } } @@ -597,7 +546,6 @@ export class FeatureExtractor extends Callable { * @extends FeatureExtractor */ export class ImageFeatureExtractor extends FeatureExtractor { - /** * Constructs a new ImageFeatureExtractor instance. * @@ -621,7 +569,7 @@ export class ImageFeatureExtractor extends FeatureExtractor { this.resample = this.config.resample ?? 2; // 2 => bilinear this.do_rescale = this.config.do_rescale ?? true; - this.rescale_factor = this.config.rescale_factor ?? (1 / 255); + this.rescale_factor = this.config.rescale_factor ?? 1 / 255; this.do_normalize = this.config.do_normalize; this.do_resize = this.config.do_resize; @@ -637,10 +585,16 @@ export class ImageFeatureExtractor extends FeatureExtractor { this.pad_size = this.config.pad_size; this.do_pad = this.config.do_pad; - if (this.do_pad && !this.pad_size && this.size && this.size.width !== undefined && this.size.height !== undefined) { + if ( + this.do_pad && + !this.pad_size && + this.size && + this.size.width !== undefined && + this.size.height !== undefined + ) { // Should pad, but no pad size specified // We infer the pad size from the resize size - this.pad_size = this.size + this.pad_size = this.size; } this.do_flip_channel_order = this.config.do_flip_channel_order ?? false; @@ -662,21 +616,20 @@ export class ImageFeatureExtractor extends FeatureExtractor { const output_width = size.width; // We always resize to the smallest of either the input or output size. - let height = Math.min(input_height, output_height) - let width = Math.min(input_width, output_width) + let height = Math.min(input_height, output_height); + let width = Math.min(input_width, output_width); if (height === input_height && width === input_width) { return image; } if (input_height > input_width) { - width = Math.floor(input_width * height / input_height); + width = Math.floor((input_width * height) / input_height); } else if (input_width > input_height) { - height = Math.floor(input_height * width / input_width); + height = Math.floor((input_height * width) / input_width); } return await image.resize(width, height, { resample }); } - /** * Crops the margin of the image. Gray pixels are considered margin (i.e., pixels with a value below the threshold). * @param {RawImage} image The image to be cropped. @@ -684,7 +637,6 @@ export class ImageFeatureExtractor extends FeatureExtractor { * @returns {Promise} The cropped image. */ async crop_margin(image, gray_threshold = 200) { - const gray_image = image.clone().grayscale(); const minValue = min(gray_image.data)[0]; @@ -697,7 +649,10 @@ export class ImageFeatureExtractor extends FeatureExtractor { const threshold = gray_threshold / 255; - let x_min = gray_image.width, y_min = gray_image.height, x_max = 0, y_max = 0; + let x_min = gray_image.width, + y_min = gray_image.height, + x_max = 0, + y_max = 0; const gray_image_data = gray_image.data; for (let j = 0; j < gray_image.height; ++j) { const row = j * gray_image.width; @@ -727,15 +682,11 @@ export class ImageFeatureExtractor extends FeatureExtractor { * @param {number} [options.constant_values=0] The constant value to use for padding. * @returns {[Float32Array, number[]]} The padded pixel data and image dimensions. */ - pad_image(pixelData, imgDims, padSize, { - mode = 'constant', - center = false, - constant_values = 0, - } = {}) { + pad_image(pixelData, imgDims, padSize, { mode = "constant", center = false, constant_values = 0 } = {}) { const [imageHeight, imageWidth, imageChannels] = imgDims; let paddedImageWidth, paddedImageHeight; - if (typeof padSize === 'number') { + if (typeof padSize === "number") { paddedImageWidth = padSize; paddedImageHeight = padSize; } else { @@ -772,9 +723,9 @@ export class ImageFeatureExtractor extends FeatureExtractor { } } - if (mode === 'symmetric') { + if (mode === "symmetric") { if (center) { - throw new Error('`center` padding is not supported when `mode` is set to `symmetric`.'); + throw new Error("`center` padding is not supported when `mode` is set to `symmetric`."); // TODO: Implement this } const h1 = imageHeight - 1; @@ -796,10 +747,9 @@ export class ImageFeatureExtractor extends FeatureExtractor { } } - // Update pixel data and image dimensions pixelData = paddedPixelData; - imgDims = [paddedImageHeight, paddedImageWidth, imageChannels] + imgDims = [paddedImageHeight, paddedImageWidth, imageChannels]; } return [pixelData, imgDims]; } @@ -819,12 +769,12 @@ export class ImageFeatureExtractor extends FeatureExtractor { * Find the target (width, height) dimension of the output image after * resizing given the input image and the desired size. * @param {RawImage} image The image to resize. - * @param {any} size The size to use for resizing the image. + * @param {any} size The size to use for resizing the image. * @returns {[number, number]} The target (width, height) dimension of the output image after resizing. */ get_resize_output_image_size(image, size) { // `size` comes in many forms, so we need to handle them all here: - // 1. `size` is an integer, in which case we resize the image to be a square + // 1. `size` is an integer, in which case we resize the image to be a square const [srcWidth, srcHeight] = image.size; @@ -834,13 +784,12 @@ export class ImageFeatureExtractor extends FeatureExtractor { if (this.do_thumbnail) { // NOTE: custom logic for `Donut` models const { height, width } = size; - shortest_edge = Math.min(height, width) + shortest_edge = Math.min(height, width); } // Support both formats for backwards compatibility else if (Number.isInteger(size)) { shortest_edge = size; longest_edge = this.config.max_size ?? shortest_edge; - } else if (size !== undefined) { // Extract known properties from `size` shortest_edge = size.shortest_edge; @@ -852,28 +801,32 @@ export class ImageFeatureExtractor extends FeatureExtractor { if (shortest_edge !== undefined || longest_edge !== undefined) { // http://opensourcehacker.com/2011/12/01/calculate-aspect-ratio-conserving-resize-for-images-in-javascript/ // Try resize so that shortest edge is `shortest_edge` (target) - const shortResizeFactor = shortest_edge === undefined - ? 1 // If `shortest_edge` is not set, don't upscale - : Math.max(shortest_edge / srcWidth, shortest_edge / srcHeight); + const shortResizeFactor = + shortest_edge === undefined + ? 1 // If `shortest_edge` is not set, don't upscale + : Math.max(shortest_edge / srcWidth, shortest_edge / srcHeight); const newWidth = srcWidth * shortResizeFactor; const newHeight = srcHeight * shortResizeFactor; // The new width and height might be greater than `longest_edge`, so - // we downscale again to ensure the largest dimension is `longest_edge` - const longResizeFactor = longest_edge === undefined - ? 1 // If `longest_edge` is not set, don't downscale - : Math.min(longest_edge / newWidth, longest_edge / newHeight); + // we downscale again to ensure the largest dimension is `longest_edge` + const longResizeFactor = + longest_edge === undefined + ? 1 // If `longest_edge` is not set, don't downscale + : Math.min(longest_edge / newWidth, longest_edge / newHeight); // To avoid certain floating point precision issues, we round to 2 decimal places let finalWidth = Math.floor(Number((newWidth * longResizeFactor).toFixed(2))); let finalHeight = Math.floor(Number((newHeight * longResizeFactor).toFixed(2))); if (this.size_divisibility !== undefined) { - [finalWidth, finalHeight] = enforce_size_divisibility([finalWidth, finalHeight], this.size_divisibility) + [finalWidth, finalHeight] = enforce_size_divisibility( + [finalWidth, finalHeight], + this.size_divisibility, + ); } return [finalWidth, finalHeight]; - } else if (size !== undefined && size.width !== undefined && size.height !== undefined) { // If `width` and `height` are set, resize to those dimensions @@ -882,7 +835,6 @@ export class ImageFeatureExtractor extends FeatureExtractor { // Custom for DPT models if (this.config.keep_aspect_ratio && this.config.ensure_multiple_of) { - // determine new height and width let scale_height = newHeight / srcHeight; let scale_width = newWidth / srcWidth; @@ -901,11 +853,12 @@ export class ImageFeatureExtractor extends FeatureExtractor { } return [newWidth, newHeight]; - } else if (this.size_divisibility !== undefined) { return enforce_size_divisibility([srcWidth, srcHeight], this.size_divisibility); } else { - throw new Error(`Could not resize image due to unsupported \`this.size\` option in config: ${JSON.stringify(size)}`); + throw new Error( + `Could not resize image due to unsupported \`this.size\` option in config: ${JSON.stringify(size)}`, + ); } } @@ -935,13 +888,16 @@ export class ImageFeatureExtractor extends FeatureExtractor { * @param {Object} overrides The overrides for the preprocessing options. * @returns {Promise} The preprocessed image. */ - async preprocess(image, { - do_normalize = null, - do_pad = null, - do_convert_rgb = null, - do_convert_grayscale = null, - do_flip_channel_order = null, - } = {}) { + async preprocess( + image, + { + do_normalize = null, + do_pad = null, + do_convert_rgb = null, + do_convert_grayscale = null, + do_flip_channel_order = null, + } = {}, + ) { if (this.do_crop_margin) { // NOTE: Specific to nougat processors. This is done before resizing, // and can be interpreted as a pre-preprocessing step. @@ -971,7 +927,6 @@ export class ImageFeatureExtractor extends FeatureExtractor { } if (this.do_center_crop) { - let crop_width; let crop_height; if (Number.isInteger(this.crop_size)) { @@ -989,7 +944,7 @@ export class ImageFeatureExtractor extends FeatureExtractor { const reshaped_input_size = [image.height, image.width]; // NOTE: All pixel-level manipulation (i.e., modifying `pixelData`) - // occurs with data in the hwc format (height, width, channels), + // occurs with data in the hwc format (height, width, channels), // to emulate the behavior of the original Python code (w/ numpy). let pixelData = Float32Array.from(image.data); let imgDims = [image.height, image.width, image.channels]; @@ -1010,7 +965,9 @@ export class ImageFeatureExtractor extends FeatureExtractor { } if (image_mean.length !== image.channels || image_std.length !== image.channels) { - throw new Error(`When set to arrays, the length of \`image_mean\` (${image_mean.length}) and \`image_std\` (${image_std.length}) must match the number of channels in the image (${image.channels}).`); + throw new Error( + `When set to arrays, the length of \`image_mean\` (${image_mean.length}) and \`image_std\` (${image_std.length}) must match the number of channels in the image (${image.channels}).`, + ); } for (let i = 0; i < pixelData.length; i += image.channels) { @@ -1026,14 +983,17 @@ export class ImageFeatureExtractor extends FeatureExtractor { const padded = this.pad_image(pixelData, [image.height, image.width, image.channels], this.pad_size); [pixelData, imgDims] = padded; // Update pixel data and image dimensions } else if (this.size_divisibility) { - const [paddedWidth, paddedHeight] = enforce_size_divisibility([imgDims[1], imgDims[0]], this.size_divisibility); + const [paddedWidth, paddedHeight] = enforce_size_divisibility( + [imgDims[1], imgDims[0]], + this.size_divisibility, + ); [pixelData, imgDims] = this.pad_image(pixelData, imgDims, { width: paddedWidth, height: paddedHeight }); } } if (do_flip_channel_order ?? this.do_flip_channel_order) { if (imgDims[2] !== 3) { - throw new Error('Flipping channel order is only supported for RGB images.'); + throw new Error("Flipping channel order is only supported for RGB images."); } // Convert RGB to BGR for (let i = 0; i < pixelData.length; i += 3) { @@ -1043,14 +1003,13 @@ export class ImageFeatureExtractor extends FeatureExtractor { } } - const pixel_values = new Tensor('float32', pixelData, imgDims) - .permute(2, 0, 1); // convert to channel dimension format (hwc -> chw) + const pixel_values = new Tensor("float32", pixelData, imgDims).permute(2, 0, 1); // convert to channel dimension format (hwc -> chw) return { original_size: [srcHeight, srcWidth], reshaped_input_size: reshaped_input_size, pixel_values, - } + }; } /** @@ -1066,22 +1025,24 @@ export class ImageFeatureExtractor extends FeatureExtractor { images = [images]; } /** @type {PreprocessedImage[]} */ - const imageData = await Promise.all(images.map(x => this.preprocess(x))); + const imageData = await Promise.all(images.map((x) => this.preprocess(x))); // Stack pixel values - const pixel_values = stack(imageData.map(x => x.pixel_values), 0); + const pixel_values = stack( + imageData.map((x) => x.pixel_values), + 0, + ); return { pixel_values, // Original sizes of images - original_sizes: imageData.map(x => x.original_size), + original_sizes: imageData.map((x) => x.original_size), // Reshaped sizes of images, before padding or cropping - reshaped_input_sizes: imageData.map(x => x.reshaped_input_size), - } + reshaped_input_sizes: imageData.map((x) => x.reshaped_input_size), + }; } - } export class SapiensFeatureExtractor extends ImageFeatureExtractor { @@ -1096,15 +1057,15 @@ export class SegformerFeatureExtractor extends ImageFeatureExtractor { return post_process_semantic_segmentation(...args); } } -export class PvtImageProcessor extends ImageFeatureExtractor { } -export class DPTFeatureExtractor extends ImageFeatureExtractor { } -export class DPTImageProcessor extends DPTFeatureExtractor { } // NOTE: extends DPTFeatureExtractor -export class BitImageProcessor extends ImageFeatureExtractor { } -export class GLPNFeatureExtractor extends ImageFeatureExtractor { } -export class CLIPFeatureExtractor extends ImageFeatureExtractor { } -export class CLIPImageProcessor extends CLIPFeatureExtractor { } // NOTE: extends CLIPFeatureExtractor -export class ChineseCLIPFeatureExtractor extends ImageFeatureExtractor { } -export class SiglipImageProcessor extends ImageFeatureExtractor { } +export class PvtImageProcessor extends ImageFeatureExtractor {} +export class DPTFeatureExtractor extends ImageFeatureExtractor {} +export class DPTImageProcessor extends DPTFeatureExtractor {} // NOTE: extends DPTFeatureExtractor +export class BitImageProcessor extends ImageFeatureExtractor {} +export class GLPNFeatureExtractor extends ImageFeatureExtractor {} +export class CLIPFeatureExtractor extends ImageFeatureExtractor {} +export class CLIPImageProcessor extends CLIPFeatureExtractor {} // NOTE: extends CLIPFeatureExtractor +export class ChineseCLIPFeatureExtractor extends ImageFeatureExtractor {} +export class SiglipImageProcessor extends ImageFeatureExtractor {} export class ConvNextFeatureExtractor extends ImageFeatureExtractor { constructor(config) { super(config); @@ -1112,7 +1073,7 @@ export class ConvNextFeatureExtractor extends ImageFeatureExtractor { /** * Percentage of the image to crop. Only has an effect if this.size < 384. */ - this.crop_pct = this.config.crop_pct ?? (224 / 256); + this.crop_pct = this.config.crop_pct ?? 224 / 256; } async resize(image) { @@ -1145,34 +1106,34 @@ export class ConvNextFeatureExtractor extends ImageFeatureExtractor { return image; } } -export class ConvNextImageProcessor extends ConvNextFeatureExtractor { } // NOTE extends ConvNextFeatureExtractor -export class ViTFeatureExtractor extends ImageFeatureExtractor { } -export class ViTImageProcessor extends ImageFeatureExtractor { } +export class ConvNextImageProcessor extends ConvNextFeatureExtractor {} // NOTE extends ConvNextFeatureExtractor +export class ViTFeatureExtractor extends ImageFeatureExtractor {} +export class ViTImageProcessor extends ImageFeatureExtractor {} export class EfficientNetImageProcessor extends ImageFeatureExtractor { constructor(config) { super(config); this.include_top = this.config.include_top ?? true; if (this.include_top) { - this.image_std = this.image_std.map(x => x * x); + this.image_std = this.image_std.map((x) => x * x); } } } -export class MobileNetV1FeatureExtractor extends ImageFeatureExtractor { } -export class MobileNetV2FeatureExtractor extends ImageFeatureExtractor { } -export class MobileNetV3FeatureExtractor extends ImageFeatureExtractor { } -export class MobileNetV4FeatureExtractor extends ImageFeatureExtractor { } +export class MobileNetV1FeatureExtractor extends ImageFeatureExtractor {} +export class MobileNetV2FeatureExtractor extends ImageFeatureExtractor {} +export class MobileNetV3FeatureExtractor extends ImageFeatureExtractor {} +export class MobileNetV4FeatureExtractor extends ImageFeatureExtractor {} -export class MobileViTFeatureExtractor extends ImageFeatureExtractor { } -export class MobileViTImageProcessor extends MobileViTFeatureExtractor { } // NOTE extends MobileViTFeatureExtractor +export class MobileViTFeatureExtractor extends ImageFeatureExtractor {} +export class MobileViTImageProcessor extends MobileViTFeatureExtractor {} // NOTE extends MobileViTFeatureExtractor export class OwlViTFeatureExtractor extends ImageFeatureExtractor { /** @type {typeof post_process_object_detection} */ post_process_object_detection(...args) { return post_process_object_detection(...args); } } -export class Owlv2ImageProcessor extends OwlViTFeatureExtractor { } // NOTE extends OwlViTFeatureExtractor +export class Owlv2ImageProcessor extends OwlViTFeatureExtractor {} // NOTE extends OwlViTFeatureExtractor export class RTDetrImageProcessor extends ImageFeatureExtractor { /** @type {typeof post_process_object_detection} */ @@ -1181,8 +1142,8 @@ export class RTDetrImageProcessor extends ImageFeatureExtractor { } } -export class DeiTFeatureExtractor extends ImageFeatureExtractor { } -export class BeitFeatureExtractor extends ImageFeatureExtractor { } +export class DeiTFeatureExtractor extends ImageFeatureExtractor {} +export class BeitFeatureExtractor extends ImageFeatureExtractor {} export class DonutFeatureExtractor extends ImageFeatureExtractor { pad_image(pixelData, imgDims, padSize, options = {}) { const [imageHeight, imageWidth, imageChannels] = imgDims; @@ -1197,7 +1158,7 @@ export class DonutFeatureExtractor extends ImageFeatureExtractor { image_std = new Array(imageChannels).fill(image_mean); } - const constant_values = image_mean.map((x, i) => - x / image_std[i]); + const constant_values = image_mean.map((x, i) => -x / image_std[i]); return super.pad_image(pixelData, imgDims, padSize, { center: true, @@ -1209,8 +1170,8 @@ export class DonutFeatureExtractor extends ImageFeatureExtractor { }); } } -export class DonutImageProcessor extends DonutFeatureExtractor { } // NOTE extends DonutFeatureExtractor -export class NougatImageProcessor extends DonutFeatureExtractor { } // NOTE extends DonutFeatureExtractor +export class DonutImageProcessor extends DonutFeatureExtractor {} // NOTE extends DonutFeatureExtractor +export class NougatImageProcessor extends DonutFeatureExtractor {} // NOTE extends DonutFeatureExtractor /** * @typedef {object} DetrFeatureExtractorResultProps @@ -1259,7 +1220,6 @@ export class DetrFeatureExtractor extends ImageFeatureExtractor { } export class MaskFormerFeatureExtractor extends ImageFeatureExtractor { - /** @type {typeof post_process_panoptic_segmentation} */ post_process_panoptic_segmentation(...args) { return post_process_panoptic_segmentation(...args); @@ -1271,7 +1231,6 @@ export class MaskFormerFeatureExtractor extends ImageFeatureExtractor { } } - export class YolosFeatureExtractor extends ImageFeatureExtractor { /** @type {typeof post_process_object_detection} */ post_process_object_detection(...args) { @@ -1290,16 +1249,14 @@ export class YolosFeatureExtractor extends ImageFeatureExtractor { */ export class SamImageProcessor extends ImageFeatureExtractor { - /** - * - * @param {any} input_points - * @param {HeightWidth[]} original_sizes - * @param {HeightWidth[]} reshaped_input_sizes + * + * @param {any} input_points + * @param {HeightWidth[]} original_sizes + * @param {HeightWidth[]} reshaped_input_sizes * @returns {Tensor} */ reshape_input_points(input_points, original_sizes, reshaped_input_sizes, is_bounding_box = false) { - // Make deep copy to avoid altering user's input input_points = structuredClone(input_points); let shape = calculateDimensions(input_points); @@ -1312,40 +1269,41 @@ export class SamImageProcessor extends ImageFeatureExtractor { } input_points = [input_points]; } else if (shape.length !== 4) { - throw Error("The input_points must be a 4D tensor of shape `batch_size`, `point_batch_size`, `nb_points_per_image`, `2`.") + throw Error( + "The input_points must be a 4D tensor of shape `batch_size`, `point_batch_size`, `nb_points_per_image`, `2`.", + ); } // Reshape input points - for (let i = 0; i < input_points.length; ++i) { // batch_size + for (let i = 0; i < input_points.length; ++i) { + // batch_size let originalImageSize = original_sizes[i]; let reshapedImageSize = reshaped_input_sizes[i]; let resizeFactors = [ reshapedImageSize[0] / originalImageSize[0], - reshapedImageSize[1] / originalImageSize[1] - ] - - for (let j = 0; j < input_points[i].length; ++j) { // point_batch_size - for (let k = 0; k < input_points[i][j].length; ++k) { // nb_points_per_image - for (let w = 0; w < input_points[i][j][k].length; ++w) { // 2 or 4 + reshapedImageSize[1] / originalImageSize[1], + ]; + + for (let j = 0; j < input_points[i].length; ++j) { + // point_batch_size + for (let k = 0; k < input_points[i][j].length; ++k) { + // nb_points_per_image + for (let w = 0; w < input_points[i][j][k].length; ++w) { + // 2 or 4 input_points[i][j][k][w] *= resizeFactors[w % 2]; } } } } - return new Tensor( - 'float32', - Float32Array.from(input_points.flat(Infinity)), - shape - ) - + return new Tensor("float32", Float32Array.from(input_points.flat(Infinity)), shape); } /** - * - * @param {any} input_labels - * @param {Tensor} input_points + * + * @param {any} input_labels + * @param {Tensor} input_points * @returns {Tensor} */ add_input_labels(input_labels, input_points) { @@ -1355,17 +1313,15 @@ export class SamImageProcessor extends ImageFeatureExtractor { shape = [1, ...shape]; input_labels = [input_labels]; } else if (shape.length !== 3) { - throw Error("The input_points must be a 4D tensor of shape `batch_size`, `point_batch_size`, `nb_points_per_image`, `2`.") + throw Error( + "The input_points must be a 4D tensor of shape `batch_size`, `point_batch_size`, `nb_points_per_image`, `2`.", + ); } if (shape.some((x, i) => x !== input_points.dims[i])) { - throw Error(`The first ${shape.length} dimensions of 'input_points' and 'input_labels' must be the same.`) + throw Error(`The first ${shape.length} dimensions of 'input_points' and 'input_labels' must be the same.`); } - return new Tensor( - 'int64', - input_labels.flat(Infinity).map(BigInt), - shape, - ) + return new Tensor("int64", input_labels.flat(Infinity).map(BigInt), shape); } /** * @param {any[]} images The URL(s) of the image(s) to extract features from. @@ -1387,31 +1343,32 @@ export class SamImageProcessor extends ImageFeatureExtractor { * - `y2`: the y coordinate of the bottom right point of the input box * @returns {Promise} */ - async _call(images, { - input_points = null, - input_labels = null, - input_boxes = null - } = {}) { + async _call(images, { input_points = null, input_labels = null, input_boxes = null } = {}) { // TODO allow user to use preprocessed images /** @type {SamImageProcessorResult} */ const processed = await super._call(images); if (input_points) { processed.input_points = this.reshape_input_points( - input_points, processed.original_sizes, processed.reshaped_input_sizes + input_points, + processed.original_sizes, + processed.reshaped_input_sizes, ); } if (input_labels) { if (!processed.input_points) { - throw Error("`input_points` must be provided if `input_labels` are provided.") + throw Error("`input_points` must be provided if `input_labels` are provided."); } processed.input_labels = this.add_input_labels(input_labels, processed.input_points); } if (input_boxes) { processed.input_boxes = this.reshape_input_points( - input_boxes, processed.original_sizes, processed.reshaped_input_sizes, true, + input_boxes, + processed.original_sizes, + processed.reshaped_input_sizes, + true, ); } @@ -1431,11 +1388,12 @@ export class SamImageProcessor extends ImageFeatureExtractor { * @param {number} [options.pad_size.width] The width the images were padded to. * @returns {Promise} Batched masks in batch_size, num_channels, height, width) format, where (height, width) is given by original_size. */ - async post_process_masks(masks, original_sizes, reshaped_input_sizes, { - mask_threshold = 0.0, - binarize = true, - pad_size = null, - } = {}) { + async post_process_masks( + masks, + original_sizes, + reshaped_input_sizes, + { mask_threshold = 0.0, binarize = true, pad_size = null } = {}, + ) { // masks: [1, 1, 3, 256, 256] const output_masks = []; @@ -1450,19 +1408,18 @@ export class SamImageProcessor extends ImageFeatureExtractor { const reshaped_input_size = reshaped_input_sizes[i]; // Upscale mask to padded size - let interpolated_mask = (await interpolate_4d( - masks[i], - { mode: 'bilinear', size: target_image_size } - )); + let interpolated_mask = await interpolate_4d(masks[i], { mode: "bilinear", size: target_image_size }); // Crop mask - interpolated_mask = interpolated_mask.slice(null, null, [0, reshaped_input_size[0]], [0, reshaped_input_size[1]]); + interpolated_mask = interpolated_mask.slice( + null, + null, + [0, reshaped_input_size[0]], + [0, reshaped_input_size[1]], + ); // Downscale mask - interpolated_mask = (await interpolate_4d( - interpolated_mask, - { mode: 'bilinear', size: original_size } - )); + interpolated_mask = await interpolate_4d(interpolated_mask, { mode: "bilinear", size: original_size }); if (binarize) { const data = interpolated_mask.data; @@ -1472,11 +1429,7 @@ export class SamImageProcessor extends ImageFeatureExtractor { binarizedMaskData[i] = 1; } } - interpolated_mask = new Tensor( - 'bool', - binarizedMaskData, - interpolated_mask.dims - ) + interpolated_mask = new Tensor("bool", binarizedMaskData, interpolated_mask.dims); } output_masks.push(interpolated_mask); @@ -1489,7 +1442,7 @@ export class SamImageProcessor extends ImageFeatureExtractor { * Generates a list of crop boxes of different sizes. Each layer has (2**i)**2 boxes for the ith layer. * @param {RawImage} image Input original image * @param {number} target_size Target size of the resized image - * @param {Object} options Options for generating crop boxes + * @param {Object} options Options for generating crop boxes * @param {number} [options.crop_n_layers] If >0, mask prediction will be run again on crops of the image. * Sets the number of layers to run, where each layer has 2**i_layer number of image crops. * @param {number} [options.overlap_ratio] Sets the degree to which crops overlap. In the first crop layer, @@ -1499,12 +1452,16 @@ export class SamImageProcessor extends ImageFeatureExtractor { * scaled down by crop_n_points_downscale_factor**n. * @returns {Object} An object containing the crop boxes, number of points per crop, cropped images, and input labels. */ - generate_crop_boxes(image, target_size, { - crop_n_layers = 0, - overlap_ratio = 512 / 1500, - points_per_crop = 32, - crop_n_points_downscale_factor = 1, - } = {}) { + generate_crop_boxes( + image, + target_size, + { + crop_n_layers = 0, + overlap_ratio = 512 / 1500, + points_per_crop = 32, + crop_n_points_downscale_factor = 1, + } = {}, + ) { // TODO: Implement // return { crop_boxes, points_per_crop, cropped_images, input_labels } } @@ -1516,18 +1473,23 @@ export class Swin2SRImageProcessor extends ImageFeatureExtractor { // In other words, the image is padded so that its width and height are multiples of `padSize`. const [imageHeight, imageWidth, imageChannels] = imgDims; - return super.pad_image(pixelData, imgDims, { - // NOTE: For Swin2SR models, the original python implementation adds padding even when the image's width/height is already - // a multiple of `pad_size`. However, this is most likely a bug (PR: https://github.com/mv-lab/swin2sr/pull/19). - // For this reason, we only add padding when the image's width/height is not a multiple of `pad_size`. - width: imageWidth + (padSize - imageWidth % padSize) % padSize, - height: imageHeight + (padSize - imageHeight % padSize) % padSize, - }, { - mode: 'symmetric', - center: false, - constant_values: -1, - ...options, - }) + return super.pad_image( + pixelData, + imgDims, + { + // NOTE: For Swin2SR models, the original python implementation adds padding even when the image's width/height is already + // a multiple of `pad_size`. However, this is most likely a bug (PR: https://github.com/mv-lab/swin2sr/pull/19). + // For this reason, we only add padding when the image's width/height is not a multiple of `pad_size`. + width: imageWidth + ((padSize - (imageWidth % padSize)) % padSize), + height: imageHeight + ((padSize - (imageHeight % padSize)) % padSize), + }, + { + mode: "symmetric", + center: false, + constant_values: -1, + ...options, + }, + ); } } @@ -1547,34 +1509,39 @@ export class VitMatteImageProcessor extends ImageFeatureExtractor { trimaps = [trimaps]; } - const imageData = await Promise.all(images.map(x => this.preprocess(x))); - const trimapData = await Promise.all(trimaps.map(x => this.preprocess(x, { - do_normalize: false, - do_convert_rgb: false, - do_convert_grayscale: true, - }))); - + const imageData = await Promise.all(images.map((x) => this.preprocess(x))); + const trimapData = await Promise.all( + trimaps.map((x) => + this.preprocess(x, { + do_normalize: false, + do_convert_rgb: false, + do_convert_grayscale: true, + }), + ), + ); // Stack pixel values - const pixel_values = stack(imageData.map( - // Concatenate images and trimaps - (x, i) => cat([x.pixel_values, trimapData[i].pixel_values], 0) - ), 0); + const pixel_values = stack( + imageData.map( + // Concatenate images and trimaps + (x, i) => cat([x.pixel_values, trimapData[i].pixel_values], 0), + ), + 0, + ); return { pixel_values, // Original sizes of images - original_sizes: imageData.map(x => x.original_size), + original_sizes: imageData.map((x) => x.original_size), // Reshaped sizes of images, before padding or cropping - reshaped_input_sizes: imageData.map(x => x.reshaped_input_size), - } + reshaped_input_sizes: imageData.map((x) => x.reshaped_input_size), + }; } } export class WhisperFeatureExtractor extends FeatureExtractor { - constructor(config) { super(config); @@ -1589,7 +1556,7 @@ export class WhisperFeatureExtractor extends FeatureExtractor { "slaney", // mel_scale ); - this.window = window_function(this.config.n_fft, 'hann'); + this.window = window_function(this.config.n_fft, "hann"); } /** @@ -1606,12 +1573,12 @@ export class WhisperFeatureExtractor extends FeatureExtractor { { power: 2.0, mel_filters: this.config.mel_filters, - log_mel: 'log10', + log_mel: "log10", // Custom max_num_frames: this.config.nb_max_frames, // 3000 - } - ) + }, + ); const data = features.data; const maxValue = max(data)[0]; @@ -1629,14 +1596,14 @@ export class WhisperFeatureExtractor extends FeatureExtractor { * @returns {Promise<{ input_features: Tensor }>} A Promise resolving to an object containing the extracted input features as a Tensor. */ async _call(audio) { - validate_audio_inputs(audio, 'WhisperFeatureExtractor'); + validate_audio_inputs(audio, "WhisperFeatureExtractor"); let waveform; if (audio.length > this.config.n_samples) { console.warn( "Attempting to extract features for audio longer than 30 seconds. " + - "If using a pipeline to extract transcript from a long audio clip, " + - "remember to specify `chunk_length_s` and/or `stride_length_s`." + "If using a pipeline to extract transcript from a long audio clip, " + + "remember to specify `chunk_length_s` and/or `stride_length_s`.", ); waveform = audio.slice(0, this.config.n_samples); } else { @@ -1648,23 +1615,22 @@ export class WhisperFeatureExtractor extends FeatureExtractor { const features = await this._extract_fbank_features(waveform); return { - input_features: features.unsqueeze_(0) + input_features: features.unsqueeze_(0), }; } } export class Wav2Vec2FeatureExtractor extends FeatureExtractor { - /** - * @param {Float32Array} input_values - * @returns {Float32Array} + * @param {Float32Array} input_values + * @returns {Float32Array} */ _zero_mean_unit_var_norm(input_values) { // TODO support batch? const sum = input_values.reduce((a, b) => a + b, 0); const mean = sum / input_values.length; const variance = input_values.reduce((a, b) => a + (b - mean) ** 2, 0) / input_values.length; - return input_values.map(x => (x - mean) / Math.sqrt(variance + 1e-7)); + return input_values.map((x) => (x - mean) / Math.sqrt(variance + 1e-7)); } /** @@ -1673,7 +1639,7 @@ export class Wav2Vec2FeatureExtractor extends FeatureExtractor { * @returns {Promise<{ input_values: Tensor; attention_mask: Tensor }>} A Promise resolving to an object containing the extracted input features and attention mask as Tensors. */ async _call(audio) { - validate_audio_inputs(audio, 'Wav2Vec2FeatureExtractor'); + validate_audio_inputs(audio, "Wav2Vec2FeatureExtractor"); if (audio instanceof Float64Array) { audio = new Float32Array(audio); @@ -1689,14 +1655,13 @@ export class Wav2Vec2FeatureExtractor extends FeatureExtractor { // TODO: allow user to pass in attention mask const shape = [1, input_values.length]; return { - input_values: new Tensor('float32', input_values, shape), - attention_mask: new Tensor('int64', new BigInt64Array(input_values.length).fill(1n), shape) + input_values: new Tensor("float32", input_values, shape), + attention_mask: new Tensor("int64", new BigInt64Array(input_values.length).fill(1n), shape), }; } } export class SeamlessM4TFeatureExtractor extends FeatureExtractor { - constructor(config) { super(config); @@ -1718,9 +1683,9 @@ export class SeamlessM4TFeatureExtractor extends FeatureExtractor { } this.mel_filters = mel_filters; - this.window = window_function(400, 'povey', { + this.window = window_function(400, "povey", { periodic: false, - }) + }); } /** @@ -1734,7 +1699,7 @@ export class SeamlessM4TFeatureExtractor extends FeatureExtractor { // Kaldi compliance: 16-bit signed integers // 32768 == 2 ** 15 - waveform = waveform.map((/** @type {number} */ x) => x * 32768) + waveform = waveform.map((/** @type {number} */ x) => x * 32768); return spectrogram( waveform, @@ -1747,15 +1712,15 @@ export class SeamlessM4TFeatureExtractor extends FeatureExtractor { center: false, preemphasis: 0.97, mel_filters: this.mel_filters, - log_mel: 'log', - mel_floor: 1.192092955078125e-07, + log_mel: "log", + mel_floor: 1.192092955078125e-7, remove_dc_offset: true, // Custom max_num_frames: max_length, transpose: true, - } - ) + }, + ); } /** @@ -1768,13 +1733,11 @@ export class SeamlessM4TFeatureExtractor extends FeatureExtractor { * @param {boolean} [options.return_attention_mask=true] Whether to return the attention mask. * @returns {Promise<{ input_features: Tensor, attention_mask?: Tensor }>} A Promise resolving to an object containing the extracted input features and attention masks as Tensors. */ - async _call(audio, { - padding = true, - pad_to_multiple_of = 2, - do_normalize_per_mel_bins = true, - return_attention_mask = true, - } = {}) { - validate_audio_inputs(audio, 'SeamlessM4TFeatureExtractor'); + async _call( + audio, + { padding = true, pad_to_multiple_of = 2, do_normalize_per_mel_bins = true, return_attention_mask = true } = {}, + ) { + validate_audio_inputs(audio, "SeamlessM4TFeatureExtractor"); let features = await this._extract_fbank_features(audio, this.config.max_length); @@ -1806,27 +1769,22 @@ export class SeamlessM4TFeatureExtractor extends FeatureExtractor { let padded_attention_mask; if (padding) { const [num_frames, num_channels] = features.dims; - const data = /** @type {Float32Array} */(features.data); + const data = /** @type {Float32Array} */ (features.data); const pad_size = num_frames % pad_to_multiple_of; if (pad_size > 0) { const padded_data = new Float32Array(num_channels * (num_frames + pad_size)); - padded_data.set(data) - padded_data.fill(this.config.padding_value, data.length) + padded_data.set(data); + padded_data.fill(this.config.padding_value, data.length); const numPaddedFrames = num_frames + pad_size; - features = new Tensor( - features.type, - padded_data, - [numPaddedFrames, num_channels], - ) + features = new Tensor(features.type, padded_data, [numPaddedFrames, num_channels]); if (return_attention_mask) { - padded_attention_mask = new Tensor( - 'int64', - new BigInt64Array(numPaddedFrames), - [1, numPaddedFrames], - ) + padded_attention_mask = new Tensor("int64", new BigInt64Array(numPaddedFrames), [ + 1, + numPaddedFrames, + ]); padded_attention_mask.data.fill(1n, 0, num_frames); } } @@ -1837,16 +1795,12 @@ export class SeamlessM4TFeatureExtractor extends FeatureExtractor { const stride = this.config.stride; const remainder = num_frames % stride; if (remainder !== 0) { - throw new Error(`The number of frames (${num_frames}) must be a multiple of the stride (${stride}).`) + throw new Error(`The number of frames (${num_frames}) must be a multiple of the stride (${stride}).`); } - const input_features = features.view( - 1, - Math.floor(num_frames / stride), - num_channels * stride, - ); + const input_features = features.view(1, Math.floor(num_frames / stride), num_channels * stride); - const result = { input_features } + const result = { input_features }; if (return_attention_mask) { const reshapedNumFrames = input_features.dims[1]; @@ -1861,11 +1815,7 @@ export class SeamlessM4TFeatureExtractor extends FeatureExtractor { } else { attention_mask_data.fill(1n); } - result.attention_mask = new Tensor( - 'int64', - attention_mask_data, - [1, reshapedNumFrames], - ); + result.attention_mask = new Tensor("int64", attention_mask_data, [1, reshapedNumFrames]); } return result; @@ -1873,8 +1823,6 @@ export class SeamlessM4TFeatureExtractor extends FeatureExtractor { } export class ASTFeatureExtractor extends FeatureExtractor { - - constructor(config) { super(config); @@ -1896,9 +1844,9 @@ export class ASTFeatureExtractor extends FeatureExtractor { } this.mel_filters = mel_filters; - this.window = window_function(400, 'hann', { + this.window = window_function(400, "hann", { periodic: false, - }) + }); this.mean = this.config.mean; this.std = this.config.std; @@ -1923,25 +1871,24 @@ export class ASTFeatureExtractor extends FeatureExtractor { center: false, preemphasis: 0.97, mel_filters: this.mel_filters, - log_mel: 'log', - mel_floor: 1.192092955078125e-07, + log_mel: "log", + mel_floor: 1.192092955078125e-7, remove_dc_offset: true, // Custom max_num_frames: max_length, transpose: true, - } - ) + }, + ); } - /** * Asynchronously extracts features from a given audio using the provided configuration. * @param {Float32Array|Float64Array} audio The audio data as a Float32Array/Float64Array. * @returns {Promise<{ input_values: Tensor }>} A Promise resolving to an object containing the extracted input features as a Tensor. */ async _call(audio) { - validate_audio_inputs(audio, 'ASTFeatureExtractor'); + validate_audio_inputs(audio, "ASTFeatureExtractor"); const features = await this._extract_fbank_features(audio, this.config.max_length); if (this.config.do_normalize) { @@ -1954,13 +1901,12 @@ export class ASTFeatureExtractor extends FeatureExtractor { } return { - input_values: features.unsqueeze_(0) + input_values: features.unsqueeze_(0), }; } } export class ClapFeatureExtractor extends FeatureExtractor { - constructor(config) { super(config); @@ -1984,14 +1930,12 @@ export class ClapFeatureExtractor extends FeatureExtractor { "slaney", // mel_scale ); - this.window = window_function(this.config.fft_window_size, 'hann') - + this.window = window_function(this.config.fft_window_size, "hann"); } - /** * Extracts the mel spectrogram and prepares it for the mode based on the `truncation` and `padding` arguments. - * + * * Four different path are possible: * - `truncation="fusion"` and the length of the waveform is greater than the max length: the mel spectrogram * will be computed on the entire audio. 3 random crops and a dowsampled version of the full mel spectrogram @@ -2002,7 +1946,7 @@ export class ClapFeatureExtractor extends FeatureExtractor { * based on `padding`, and is repeated `4` times. * - `truncation="rand_trunc"` and the length of the waveform is greater than the max length: the mel * spectrogram will be computed on a random crop of the waveform. - * + * * @param {Float32Array|Float64Array} waveform The input waveform. * @param {number} max_length The maximum length of the waveform. * @param {string} truncation The truncation strategy to use. @@ -2011,32 +1955,35 @@ export class ClapFeatureExtractor extends FeatureExtractor { * @private */ async _get_input_mel(waveform, max_length, truncation, padding) { - /** @type {Tensor} */ let input_mel; let longer = false; const diff = waveform.length - max_length; if (diff > 0) { - if (truncation === 'rand_trunc') { + if (truncation === "rand_trunc") { longer = true; const idx = Math.floor(Math.random() * (diff + 1)); waveform = waveform.subarray(idx, idx + max_length); - input_mel = await this._extract_fbank_features(waveform, this.mel_filters_slaney, this.config.nb_max_samples); + input_mel = await this._extract_fbank_features( + waveform, + this.mel_filters_slaney, + this.config.nb_max_samples, + ); } else { // TODO implement fusion strategy - throw new Error(`Truncation strategy "${truncation}" not implemented`) + throw new Error(`Truncation strategy "${truncation}" not implemented`); } } else { if (diff < 0) { let padded = new Float64Array(max_length); // already padded with zeros padded.set(waveform); - if (padding === 'repeat') { + if (padding === "repeat") { for (let i = waveform.length; i < max_length; i += waveform.length) { padded.set(waveform.subarray(0, Math.min(waveform.length, max_length - i)), i); } - } else if (padding === 'repeatpad') { + } else if (padding === "repeatpad") { for (let i = waveform.length; i < -diff; i += waveform.length) { padded.set(waveform, i); } @@ -2044,11 +1991,15 @@ export class ClapFeatureExtractor extends FeatureExtractor { waveform = padded; } - if (truncation === 'fusion') { - throw new Error(`Truncation strategy "${truncation}" not implemented`) + if (truncation === "fusion") { + throw new Error(`Truncation strategy "${truncation}" not implemented`); } - input_mel = await this._extract_fbank_features(waveform, this.mel_filters_slaney, this.config.nb_max_samples); + input_mel = await this._extract_fbank_features( + waveform, + this.mel_filters_slaney, + this.config.nb_max_samples, + ); } return input_mel.unsqueeze_(0); @@ -2063,7 +2014,7 @@ export class ClapFeatureExtractor extends FeatureExtractor { * - `self.mel_filteres_slaney` : they correspond to the default parameters of `librosa` which used * `librosa.filters.mel` when computing the mel spectrogram. These filters were only used in the original * implementation when the truncation mode is not `"fusion"`. - * + * * @param {Float32Array|Float64Array} waveform The audio waveform to process. * @param {number[][]} mel_filters The mel filters to use. * @param {number} [max_length=null] The maximum number of frames to return. @@ -2079,26 +2030,23 @@ export class ClapFeatureExtractor extends FeatureExtractor { { power: 2.0, mel_filters, - log_mel: 'dB', + log_mel: "dB", // Custom max_num_frames: max_length, do_pad: false, transpose: true, - } - ) + }, + ); } - /** * Asynchronously extracts features from a given audio using the provided configuration. * @param {Float32Array|Float64Array} audio The audio data as a Float32Array/Float64Array. * @returns {Promise<{ input_features: Tensor }>} A Promise resolving to an object containing the extracted input features as a Tensor. */ - async _call(audio, { - max_length = null, - } = {}) { - validate_audio_inputs(audio, 'ClapFeatureExtractor'); + async _call(audio, { max_length = null } = {}) { + validate_audio_inputs(audio, "ClapFeatureExtractor"); // convert to mel spectrogram, truncate and pad if needed. const padded_inputs = await this._get_input_mel( @@ -2110,11 +2058,10 @@ export class ClapFeatureExtractor extends FeatureExtractor { return { input_features: padded_inputs.unsqueeze_(0), - } + }; } } - export class PyAnnoteFeatureExtractor extends FeatureExtractor { /** * Asynchronously extracts features from a given audio using the provided configuration. @@ -2122,19 +2069,15 @@ export class PyAnnoteFeatureExtractor extends FeatureExtractor { * @returns {Promise<{ input_values: Tensor; }>} The extracted input features. */ async _call(audio) { - validate_audio_inputs(audio, 'PyAnnoteFeatureExtractor'); + validate_audio_inputs(audio, "PyAnnoteFeatureExtractor"); if (audio instanceof Float64Array) { audio = new Float32Array(audio); } - const shape = [ - 1, /* batch_size */ - 1, /* num_channels */ - audio.length, /* num_samples */ - ]; + const shape = [1 /* batch_size */, 1 /* num_channels */, audio.length /* num_samples */]; return { - input_values: new Tensor('float32', audio, shape), + input_values: new Tensor("float32", audio, shape), }; } @@ -2144,7 +2087,7 @@ export class PyAnnoteFeatureExtractor extends FeatureExtractor { * @returns {number} The number of frames in the audio. */ samples_to_frames(samples) { - return ((samples - this.config.offset) / this.config.step); + return (samples - this.config.offset) / this.config.step; } /** @@ -2154,9 +2097,7 @@ export class PyAnnoteFeatureExtractor extends FeatureExtractor { * @returns {Array>} The post-processed speaker diarization results. */ post_process_speaker_diarization(logits, num_samples) { - const ratio = ( - num_samples / this.samples_to_frames(num_samples) - ) / this.config.sampling_rate; + const ratio = num_samples / this.samples_to_frames(num_samples) / this.config.sampling_rate; const results = []; for (const scores of logits.tolist()) { @@ -2179,24 +2120,24 @@ export class PyAnnoteFeatureExtractor extends FeatureExtractor { } } - results.push(accumulated_segments.map( - // Convert frame-space to time-space - // and compute the confidence - ({ id, start, end, score }) => ({ - id, - start: start * ratio, - end: end * ratio, - confidence: score / (end - start), - }) - )); + results.push( + accumulated_segments.map( + // Convert frame-space to time-space + // and compute the confidence + ({ id, start, end, score }) => ({ + id, + start: start * ratio, + end: end * ratio, + confidence: score / (end - start), + }), + ), + ); } return results; } - } export class WeSpeakerFeatureExtractor extends FeatureExtractor { - constructor(config) { super(config); @@ -2218,9 +2159,9 @@ export class WeSpeakerFeatureExtractor extends FeatureExtractor { } this.mel_filters = mel_filters; - this.window = window_function(400, 'hamming', { + this.window = window_function(400, "hamming", { periodic: false, - }) + }); this.min_num_frames = this.config.min_num_frames; } @@ -2232,7 +2173,7 @@ export class WeSpeakerFeatureExtractor extends FeatureExtractor { async _extract_fbank_features(waveform) { // Kaldi compliance: 16-bit signed integers // 32768 == 2 ** 15 - waveform = waveform.map((/** @type {number} */ x) => x * 32768) + waveform = waveform.map((/** @type {number} */ x) => x * 32768); return spectrogram( waveform, @@ -2245,32 +2186,31 @@ export class WeSpeakerFeatureExtractor extends FeatureExtractor { center: false, preemphasis: 0.97, mel_filters: this.mel_filters, - log_mel: 'log', - mel_floor: 1.192092955078125e-07, + log_mel: "log", + mel_floor: 1.192092955078125e-7, remove_dc_offset: true, // Custom transpose: true, min_num_frames: this.min_num_frames, - } - ) + }, + ); } - /** * Asynchronously extracts features from a given audio using the provided configuration. * @param {Float32Array|Float64Array} audio The audio data as a Float32Array/Float64Array. * @returns {Promise<{ input_features: Tensor }>} A Promise resolving to an object containing the extracted input features as a Tensor. */ async _call(audio) { - validate_audio_inputs(audio, 'WeSpeakerFeatureExtractor'); + validate_audio_inputs(audio, "WeSpeakerFeatureExtractor"); const features = (await this._extract_fbank_features(audio)).unsqueeze_(0); if (this.config.fbank_centering_span === null) { // center features with global average const meanData = /** @type {Float32Array} */ (features.mean(1).data); - const featuresData = /** @type {Float32Array} */(features.data); + const featuresData = /** @type {Float32Array} */ (features.data); const [batch_size, num_frames, feature_size] = features.dims; for (let i = 0; i < batch_size; ++i) { @@ -2286,12 +2226,12 @@ export class WeSpeakerFeatureExtractor extends FeatureExtractor { } return { - input_features: features + input_features: features, }; } } -export class SpeechT5FeatureExtractor extends FeatureExtractor { } +export class SpeechT5FeatureExtractor extends FeatureExtractor {} /** * Represents a Processor that extracts features from an input. @@ -2354,11 +2294,10 @@ export class WhisperProcessor extends Processor { * @returns {Promise} A Promise that resolves with the extracted features. */ async _call(audio) { - return await this.feature_extractor(audio) + return await this.feature_extractor(audio); } } - export class Wav2Vec2ProcessorWithLM extends Processor { /** * Calls the feature_extractor function with the given audio input. @@ -2366,7 +2305,7 @@ export class Wav2Vec2ProcessorWithLM extends Processor { * @returns {Promise} A Promise that resolves with the extracted features. */ async _call(audio) { - return await this.feature_extractor(audio) + return await this.feature_extractor(audio); } } @@ -2377,14 +2316,13 @@ export class PyAnnoteProcessor extends Processor { * @returns {Promise} A Promise that resolves with the extracted features. */ async _call(audio) { - return await this.feature_extractor(audio) + return await this.feature_extractor(audio); } post_process_speaker_diarization(...args) { // @ts-ignore return this.feature_extractor.post_process_speaker_diarization(...args); } - } export class SpeechT5Processor extends Processor { @@ -2394,21 +2332,18 @@ export class SpeechT5Processor extends Processor { * @returns {Promise} A Promise that resolves with the extracted features. */ async _call(input) { - return await this.feature_extractor(input) + return await this.feature_extractor(input); } } -export class OwlViTProcessor extends Processor { } +export class OwlViTProcessor extends Processor {} export class Florence2Processor extends Processor { constructor(feature_extractor) { super(feature_extractor); - const { - tasks_answer_post_processing_type, - task_prompts_without_inputs, - task_prompts_with_input, - } = feature_extractor.config; + const { tasks_answer_post_processing_type, task_prompts_without_inputs, task_prompts_with_input } = + feature_extractor.config; /** @type {Map} */ this.tasks_answer_post_processing_type = new Map(Object.entries(tasks_answer_post_processing_type ?? {})); @@ -2420,9 +2355,10 @@ export class Florence2Processor extends Processor { this.task_prompts_with_input = new Map(Object.entries(task_prompts_with_input ?? {})); this.regexes = { - quad_boxes: /(.+?)/gm, + quad_boxes: + /(.+?)/gm, bboxes: /([^<]+)?/gm, - } + }; this.size_per_bin = 1000; } @@ -2432,7 +2368,7 @@ export class Florence2Processor extends Processor { * @returns {string[]} */ construct_prompts(text) { - if (typeof text === 'string') { + if (typeof text === "string") { text = [text]; } @@ -2442,11 +2378,11 @@ export class Florence2Processor extends Processor { if (this.task_prompts_without_inputs.has(t)) { prompts.push(this.task_prompts_without_inputs.get(t)); } - // 2. task prompts with additional inputs + // 2. task prompts with additional inputs else { for (const [task, prompt] of this.task_prompts_with_input) { if (t.includes(task)) { - prompts.push(prompt.replaceAll('{input}', t).replaceAll(task, '')); + prompts.push(prompt.replaceAll("{input}", t).replaceAll(task, "")); break; } } @@ -2467,31 +2403,34 @@ export class Florence2Processor extends Processor { * @param {[number, number]} image_size The size of the image. height x width. */ post_process_generation(text, task, image_size) { - const task_answer_post_processing_type = this.tasks_answer_post_processing_type.get(task) ?? 'pure_text'; + const task_answer_post_processing_type = this.tasks_answer_post_processing_type.get(task) ?? "pure_text"; // remove the special tokens - text = text.replaceAll('', '').replaceAll('', ''); + text = text.replaceAll("", "").replaceAll("", ""); let final_answer; switch (task_answer_post_processing_type) { - case 'pure_text': + case "pure_text": final_answer = text; break; - case 'description_with_bboxes': - case 'bboxes': - case 'phrase_grounding': - case 'ocr': - const key = task_answer_post_processing_type === 'ocr' ? 'quad_boxes' : 'bboxes'; + case "description_with_bboxes": + case "bboxes": + case "phrase_grounding": + case "ocr": + const key = task_answer_post_processing_type === "ocr" ? "quad_boxes" : "bboxes"; const matches = text.matchAll(this.regexes[key]); const labels = []; const items = []; for (const [_, label, ...locations] of matches) { // Push new label, or duplicate the last label - labels.push(label ? label.trim() : labels.at(-1) ?? ''); - items.push(locations.map((x, i) => - // NOTE: Add 0.5 to use the center position of the bin as the coordinate. - (Number(x) + 0.5) / this.size_per_bin * image_size[i % 2]) + labels.push(label ? label.trim() : (labels.at(-1) ?? "")); + items.push( + locations.map( + (x, i) => + // NOTE: Add 0.5 to use the center position of the bin as the coordinate. + ((Number(x) + 0.5) / this.size_per_bin) * image_size[i % 2], + ), ); } final_answer = { labels, [key]: items }; @@ -2501,7 +2440,7 @@ export class Florence2Processor extends Processor { throw new Error(`Task "${task}" (of type "${task_answer_post_processing_type}") not yet implemented.`); } - return { [task]: final_answer } + return { [task]: final_answer }; } } @@ -2509,12 +2448,12 @@ export class Florence2Processor extends Processor { /** * Helper class which is used to instantiate pretrained processors with the `from_pretrained` function. * The chosen processor class is determined by the type specified in the processor config. - * + * * **Example:** Load a processor using `from_pretrained`. * ```javascript * let processor = await AutoProcessor.from_pretrained('openai/whisper-tiny.en'); * ``` - * + * * **Example:** Run an image through a processor. * ```javascript * let processor = await AutoProcessor.from_pretrained('Xenova/clip-vit-base-patch16'); @@ -2585,7 +2524,7 @@ export class AutoProcessor { ClapFeatureExtractor, PyAnnoteFeatureExtractor, WeSpeakerFeatureExtractor, - } + }; static PROCESSOR_CLASS_MAPPING = { WhisperProcessor, @@ -2595,38 +2534,36 @@ export class AutoProcessor { SpeechT5Processor, OwlViTProcessor, Florence2Processor, - } + }; /** * Instantiate one of the processor classes of the library from a pretrained model. - * + * * The processor class to instantiate is selected based on the `feature_extractor_type` property of the config object * (either passed as an argument or loaded from `pretrained_model_name_or_path` if possible) - * + * * @param {string} pretrained_model_name_or_path The name or path of the pretrained model. Can be either: * - A string, the *model id* of a pretrained processor hosted inside a model repo on huggingface.co. * Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a * user or organization name, like `dbmdz/bert-base-german-cased`. * - A path to a *directory* containing processor files, e.g., `./my_model_directory/`. * @param {import('./utils/hub.js').PretrainedOptions} options Additional options for loading the processor. - * + * * @returns {Promise} A new instance of the Processor class. */ - static async from_pretrained(pretrained_model_name_or_path, { - progress_callback = null, - config = null, - cache_dir = null, - local_files_only = false, - revision = 'main', - } = {}) { - - let preprocessorConfig = config ?? await getModelJSON(pretrained_model_name_or_path, 'preprocessor_config.json', true, { - progress_callback, - config, - cache_dir, - local_files_only, - revision, - }) + static async from_pretrained( + pretrained_model_name_or_path, + { progress_callback = null, config = null, cache_dir = null, local_files_only = false, revision = "main" } = {}, + ) { + let preprocessorConfig = + config ?? + (await getModelJSON(pretrained_model_name_or_path, "preprocessor_config.json", true, { + progress_callback, + config, + cache_dir, + local_files_only, + revision, + })); // Determine feature extractor class // TODO: Ensure backwards compatibility with old configs @@ -2636,7 +2573,9 @@ export class AutoProcessor { if (!feature_extractor_class) { if (preprocessorConfig.size !== undefined) { // Assume ImageFeatureExtractor - console.warn(`Feature extractor type "${key}" not found, assuming ImageFeatureExtractor due to size parameter in config.`); + console.warn( + `Feature extractor type "${key}" not found, assuming ImageFeatureExtractor due to size parameter in config.`, + ); feature_extractor_class = ImageFeatureExtractor; } else { throw new Error(`Unknown Feature Extractor type: ${key}`); @@ -2652,4 +2591,3 @@ export class AutoProcessor { } } ////////////////////////////////////////////////// - diff --git a/src/tokenizers.js b/src/tokenizers.js index 48a26b636..bbff07881 100644 --- a/src/tokenizers.js +++ b/src/tokenizers.js @@ -1,12 +1,11 @@ - /** * @file Tokenizers are used to prepare textual inputs for a model. - * + * * **Example:** Create an `AutoTokenizer` and use it to tokenize a sentence. * This will automatically detect the tokenizer type based on the tokenizer class defined in `tokenizer.json`. * ```javascript * import { AutoTokenizer } from '@huggingface/transformers'; - * + * * const tokenizer = await AutoTokenizer.from_pretrained('Xenova/bert-base-uncased'); * const { input_ids } = await tokenizer('I love transformers!'); * // Tensor { @@ -16,41 +15,24 @@ * // size: 6, * // } * ``` - * + * * @module tokenizers */ -import { - Callable, -} from './utils/generic.js'; - -import { - reverseDictionary, - escapeRegExp, - isIntegralNumber, - mergeArrays, - len, -} from './utils/core.js'; - -import { - getModelJSON, -} from './utils/hub.js'; - -import { max, min, round } from './utils/maths.js'; -import { Tensor } from './utils/tensor.js'; - -import { - PriorityQueue, - TokenLattice, - CharTrie, -} from './utils/data-structures.js'; - -import { Template } from '@huggingface/jinja'; - -import { - WHISPER_LANGUAGE_MAPPING, - whisper_language_to_code, -} from './models/whisper/common_whisper.js'; -import { GITHUB_ISSUE_URL } from './utils/constants.js'; +import { Callable } from "./utils/generic.js"; + +import { reverseDictionary, escapeRegExp, isIntegralNumber, mergeArrays, len } from "./utils/core.js"; + +import { getModelJSON } from "./utils/hub.js"; + +import { max, min, round } from "./utils/maths.js"; +import { Tensor } from "./utils/tensor.js"; + +import { PriorityQueue, TokenLattice, CharTrie } from "./utils/data-structures.js"; + +import { Template } from "@huggingface/jinja"; + +import { WHISPER_LANGUAGE_MAPPING, whisper_language_to_code } from "./models/whisper/common_whisper.js"; +import { GITHUB_ISSUE_URL } from "./utils/constants.js"; /** * @typedef {Object} TokenizerProperties Additional tokenizer-specific properties. @@ -65,11 +47,10 @@ import { GITHUB_ISSUE_URL } from './utils/constants.js'; * @returns {Promise} A promise that resolves with information about the loaded tokenizer. */ async function loadTokenizer(pretrained_model_name_or_path, options) { - const info = await Promise.all([ - getModelJSON(pretrained_model_name_or_path, 'tokenizer.json', true, options), - getModelJSON(pretrained_model_name_or_path, 'tokenizer_config.json', true, options), - ]) + getModelJSON(pretrained_model_name_or_path, "tokenizer.json", true, options), + getModelJSON(pretrained_model_name_or_path, "tokenizer_config.json", true, options), + ]); // Override legacy option if `options.legacy` is not null if (options.legacy !== null) { @@ -78,7 +59,6 @@ async function loadTokenizer(pretrained_model_name_or_path, options) { return info; } - /** * Helper function to split a string on a regex, but keep the delimiters. * This is required, because the JavaScript `.split()` method does not keep the delimiters, @@ -106,7 +86,6 @@ function regexSplit(text, regex) { return result; } - /** * Helper method to construct a pattern from a config object. * @param {Object} pattern The pattern object. @@ -114,29 +93,26 @@ function regexSplit(text, regex) { * @returns {RegExp|null} The compiled pattern. */ function createPattern(pattern, invert = true) { - if (pattern.Regex !== undefined) { // In certain cases, the pattern may contain unnecessary escape sequences (e.g., \# or \& or \~). // i.e., valid in Python (where the patterns are exported from) but invalid in JavaScript (where the patterns are parsed). // This isn't an issue when creating the regex w/o the 'u' flag, but it is when the 'u' flag is used. // For this reason, it is necessary to remove these backslashes before creating the regex. // See https://stackoverflow.com/a/63007777/13989043 for more information - let regex = pattern.Regex.replace(/\\([#&~])/g, '$1'); // TODO: add more characters to this list if necessary + let regex = pattern.Regex.replace(/\\([#&~])/g, "$1"); // TODO: add more characters to this list if necessary // We also handle special cases where the regex contains invalid (non-JS compatible) syntax. for (const [key, value] of PROBLEMATIC_REGEX_MAP) { regex = regex.replaceAll(key, value); } - return new RegExp(regex, 'gu'); - + return new RegExp(regex, "gu"); } else if (pattern.String !== undefined) { const escaped = escapeRegExp(pattern.String); // NOTE: if invert is true, we wrap the pattern in a group so that it is kept when performing .split() - return new RegExp(invert ? escaped : `(${escaped})`, 'gu'); - + return new RegExp(invert ? escaped : `(${escaped})`, "gu"); } else { - console.warn('Unknown pattern type:', pattern) + console.warn("Unknown pattern type:", pattern); return null; } } @@ -162,11 +138,13 @@ function prepareTensorForDecode(tensor) { return tensor.tolist(); case 2: if (dims[0] !== 1) { - throw new Error('Unable to decode tensor with `batch size !== 1`. Use `tokenizer.batch_decode(...)` for batched inputs.'); + throw new Error( + "Unable to decode tensor with `batch size !== 1`. Use `tokenizer.batch_decode(...)` for batched inputs.", + ); } return tensor.tolist()[0]; default: - throw new Error(`Expected tensor to have 1-2 dimensions, got ${dims.length}.`) + throw new Error(`Expected tensor to have 1-2 dimensions, got ${dims.length}.`); } } @@ -178,10 +156,11 @@ function prepareTensorForDecode(tensor) { function clean_up_tokenization(text) { // Clean up a list of simple English tokenization artifacts // like spaces before punctuations and abbreviated forms - return text.replace(/ \./g, '.') - .replace(/ \?/g, '?') - .replace(/ \!/g, '!') - .replace(/ ,/g, ',') + return text + .replace(/ \./g, ".") + .replace(/ \?/g, "?") + .replace(/ \!/g, "!") + .replace(/ ,/g, ",") .replace(/ \' /g, "'") .replace(/ n\'t/g, "n't") .replace(/ \'m/g, "'m") @@ -196,7 +175,7 @@ function clean_up_tokenization(text) { * @returns {string} The text with accents removed. */ function remove_accents(text) { - return text.replace(/\p{M}/gu, ''); + return text.replace(/\p{M}/gu, ""); } /** @@ -208,7 +187,6 @@ function lowercase_and_remove_accent(text) { return remove_accents(text.toLowerCase()); } - /** * Checks whether the given Unicode codepoint represents a CJK (Chinese, Japanese, or Korean) character. * @@ -225,15 +203,15 @@ function lowercase_and_remove_accent(text) { */ export function is_chinese_char(cp) { return ( - (cp >= 0x4E00 && cp <= 0x9FFF) - || (cp >= 0x3400 && cp <= 0x4DBF) - || (cp >= 0x20000 && cp <= 0x2A6DF) - || (cp >= 0x2A700 && cp <= 0x2B73F) - || (cp >= 0x2B740 && cp <= 0x2B81F) - || (cp >= 0x2B820 && cp <= 0x2CEAF) - || (cp >= 0xF900 && cp <= 0xFAFF) - || (cp >= 0x2F800 && cp <= 0x2FA1F) - ) + (cp >= 0x4e00 && cp <= 0x9fff) || + (cp >= 0x3400 && cp <= 0x4dbf) || + (cp >= 0x20000 && cp <= 0x2a6df) || + (cp >= 0x2a700 && cp <= 0x2b73f) || + (cp >= 0x2b740 && cp <= 0x2b81f) || + (cp >= 0x2b820 && cp <= 0x2ceaf) || + (cp >= 0xf900 && cp <= 0xfaff) || + (cp >= 0x2f800 && cp <= 0x2fa1f) + ); } /** @@ -247,7 +225,7 @@ function fuse_unk(arr, tokens_to_ids, unk_token_id) { const fused = []; let i = 0; while (i < arr.length) { - fused.push(arr[i]) + fused.push(arr[i]); if ((tokens_to_ids.get(arr[i]) ?? unk_token_id) !== unk_token_id) { ++i; continue; @@ -272,9 +250,9 @@ function whitespace_split(text) { return text.match(/\S+/g) || []; } -const PUNCTUATION_REGEX = '\\p{P}\\u0021-\\u002F\\u003A-\\u0040\\u005B-\\u0060\\u007B-\\u007E'; -const PUNCTUATION_ONLY_REGEX = new RegExp(`^[${PUNCTUATION_REGEX}]+$`, 'gu'); -const BLOOM_SPLIT_CHARS = '.,!?\u2026\u3002\uff0c\u3001\u0964\u06d4\u060c'; +const PUNCTUATION_REGEX = "\\p{P}\\u0021-\\u002F\\u003A-\\u0040\\u005B-\\u0060\\u007B-\\u007E"; +const PUNCTUATION_ONLY_REGEX = new RegExp(`^[${PUNCTUATION_REGEX}]+$`, "gu"); +const BLOOM_SPLIT_CHARS = ".,!?\u2026\u3002\uff0c\u3001\u0964\u06d4\u060c"; // A mapping of regex patterns to their equivalent (but possibly longer) JS-compatible versions. const PROBLEMATIC_REGEX_MAP = new Map([ @@ -285,8 +263,7 @@ const PROBLEMATIC_REGEX_MAP = new Map([ // Used to override the default (invalid) regex of the bloom pretokenizer. // For more information, see https://github.com/huggingface/transformers.js/issues/94 [` ?[^(\\s|[${BLOOM_SPLIT_CHARS}])]+`, ` ?[^\\s${BLOOM_SPLIT_CHARS}]+`], -]) - +]); /** * Represent a token added by the user on top of the existing Model vocabulary. @@ -357,12 +334,12 @@ export class TokenizerModel extends Callable { */ static fromConfig(config, ...args) { switch (config.type) { - case 'WordPiece': + case "WordPiece": return new WordPieceTokenizer(config); - case 'Unigram': + case "Unigram": // @ts-ignore return new Unigram(config, ...args); - case 'BPE': + case "BPE": return new BPE(config); default: @@ -403,7 +380,7 @@ export class TokenizerModel extends Callable { * @throws Will throw an error if not implemented in a subclass. */ encode(tokens) { - throw Error("encode should be implemented in subclass.") + throw Error("encode should be implemented in subclass."); } /** @@ -412,7 +389,7 @@ export class TokenizerModel extends Callable { * @returns {number[]} The converted token IDs. */ convert_tokens_to_ids(tokens) { - return tokens.map(t => this.tokens_to_ids.get(t) ?? this.unk_token_id); + return tokens.map((t) => this.tokens_to_ids.get(t) ?? this.unk_token_id); } /** @@ -421,7 +398,7 @@ export class TokenizerModel extends Callable { * @returns {string[]} The converted tokens. */ convert_ids_to_tokens(ids) { - return ids.map(i => this.vocab[i] ?? this.unk_token); + return ids.map((i) => this.vocab[i] ?? this.unk_token); } } @@ -495,7 +472,7 @@ class WordPieceTokenizer extends TokenizerModel { let end = chars.length; let currentSubstring = null; while (start < end) { - let substr = chars.slice(start, end).join(''); + let substr = chars.slice(start, end).join(""); if (start > 0) { substr = this.config.continuing_subword_prefix + substr; @@ -523,7 +500,6 @@ class WordPieceTokenizer extends TokenizerModel { return outputTokens; } - } /** @@ -554,7 +530,7 @@ class Unigram extends TokenizerModel { this.unk_token = this.vocab[config.unk_id]; this.tokens_to_ids = new Map(this.vocab.map((x, i) => [x, i])); - this.bos_token = ' '; // beginning of a sentence token + this.bos_token = " "; // beginning of a sentence token this.bos_token_id = this.tokens_to_ids.get(this.bos_token); // NOTE: may be undefined this.eos_token = moreConfig.eos_token; @@ -587,7 +563,7 @@ class Unigram extends TokenizerModel { let hasSingleNode = false; const tokens = []; - const sliced = chars.slice(beginPos).join(''); + const sliced = chars.slice(beginPos).join(""); const prefixedTokens = this.trie.commonPrefixSearch(sliced); for (const token of prefixedTokens) { tokens.push(token); @@ -631,7 +607,6 @@ class Unigram extends TokenizerModel { } return toReturn; } - } /** @@ -658,13 +633,12 @@ const BYTES_TO_UNICODE = (() => { n += 1; } } - const ccs = cs.map(n => String.fromCharCode(n)); + const ccs = cs.map((n) => String.fromCharCode(n)); return Object.fromEntries(bs.map((b, i) => [b, ccs[i]])); })(); const UNICODE_TO_BYTES = reverseDictionary(BYTES_TO_UNICODE); - /** * @typedef {Object} BPENode * @property {string} token The token associated with the node @@ -710,8 +684,8 @@ class BPE extends TokenizerModel { /** @type {[string, string][]} */ this.merges = use_new_merge_format - ? /** @type {[string, string][]} */(config.merges) - : (/** @type {string[]} */(config.merges)).map(x => /** @type {[string, string]} */(x.split(' ', 2))); + ? /** @type {[string, string][]} */ (config.merges) + : /** @type {string[]} */ (config.merges).map((x) => /** @type {[string, string]} */ (x.split(" ", 2))); this.bpe_ranks = new Map(this.merges.map((x, i) => [JSON.stringify(x), i])); this.end_of_word_suffix = config.end_of_word_suffix; @@ -766,19 +740,19 @@ class BPE extends TokenizerModel { bias: 0, prev: null, next: null, - } + }; - let previousNode = startingNode + let previousNode = startingNode; for (let i = 1; i < word.length; ++i) { const currentNode = { bias: i / word.length, // Add fractional component to break ties token: word[i], prev: previousNode, next: null, - } - previousNode.next = currentNode - this._add_node(queue, previousNode) - previousNode = currentNode + }; + previousNode.next = currentNode; + this._add_node(queue, previousNode); + previousNode = currentNode; } while (!queue.isEmpty()) { @@ -795,7 +769,6 @@ class BPE extends TokenizerModel { // Next, we fix the node that comes before the current node (i.e., left side of the merge). if (node.prev) { - // Make a shallow copy of the previous node const newPreviousNode = { ...node.prev }; @@ -820,7 +793,7 @@ class BPE extends TokenizerModel { bias: node.bias, prev: node.prev, next: node.next.next, - } + }; // We now consider where we can add the new merged node to the priority queue: // 1. prev <-> merged @@ -861,10 +834,9 @@ class BPE extends TokenizerModel { return result; } - /** * Helper function to add a node to the priority queue. - * @param {PriorityQueue} queue + * @param {PriorityQueue} queue * @param {BPENode} node * @private */ @@ -898,9 +870,10 @@ class BPE extends TokenizerModel { if (this.tokens_to_ids.has(t)) { outputTokens.push(t); } else if (this.byte_fallback) { - const byteTokens = Array.from(this.text_encoder.encode(t)) - .map(x => `<0x${x.toString(16).toUpperCase().padStart(2, '0')}>`); - if (byteTokens.every(x => this.tokens_to_ids.has(x))) { + const byteTokens = Array.from(this.text_encoder.encode(t)).map( + (x) => `<0x${x.toString(16).toUpperCase().padStart(2, "0")}>`, + ); + if (byteTokens.every((x) => this.tokens_to_ids.has(x))) { // Ensure the byte tokens are actually in the vocabulary, otherwise // we fall back to the unknown token. For more information, see // https://github.com/huggingface/transformers/issues/28096. @@ -916,7 +889,6 @@ class BPE extends TokenizerModel { return outputTokens; } - } /** @@ -933,11 +905,7 @@ class LegacyTokenizerModel extends TokenizerModel { super(config); /**@type {Map} */ - this.tokens_to_ids = objectToMap( - moreConfig.target_lang - ? config.vocab[moreConfig.target_lang] - : config.vocab - ); + this.tokens_to_ids = objectToMap(moreConfig.target_lang ? config.vocab[moreConfig.target_lang] : config.vocab); this.bos_token = moreConfig.bos_token; this.bos_token_id = this.tokens_to_ids.get(this.bos_token); @@ -962,7 +930,6 @@ class LegacyTokenizerModel extends TokenizerModel { } } - /** * A base class for text normalization. * @abstract @@ -986,27 +953,27 @@ class Normalizer extends Callable { static fromConfig(config) { if (config === null) return null; switch (config.type) { - case 'BertNormalizer': + case "BertNormalizer": return new BertNormalizer(config); - case 'Precompiled': + case "Precompiled": return new Precompiled(config); - case 'Sequence': + case "Sequence": return new NormalizerSequence(config); - case 'Replace': + case "Replace": return new Replace(config); - case 'NFC': + case "NFC": return new NFC(config); - case 'NFKC': + case "NFKC": return new NFKC(config); - case 'NFKD': + case "NFKD": return new NFKD(config); - case 'Strip': + case "Strip": return new StripNormalizer(config); - case 'StripAccents': + case "StripAccents": return new StripAccents(config); - case 'Lowercase': + case "Lowercase": return new Lowercase(config); - case 'Prepend': + case "Prepend": return new Prepend(config); default: throw new Error(`Unknown Normalizer type: ${config.type}`); @@ -1021,7 +988,7 @@ class Normalizer extends Callable { * @throws {Error} If this method is not implemented in a subclass. */ normalize(text) { - throw Error("normalize should be implemented in subclass.") + throw Error("normalize should be implemented in subclass."); } /** @@ -1032,7 +999,6 @@ class Normalizer extends Callable { _call(text) { return this.normalize(text); } - } /** @@ -1047,9 +1013,7 @@ class Replace extends Normalizer { */ normalize(text) { const pattern = createPattern(this.config.pattern); - return pattern === null - ? text - : text.replaceAll(pattern, this.config.content); + return pattern === null ? text : text.replaceAll(pattern, this.config.content); } } @@ -1064,7 +1028,7 @@ class NFC extends Normalizer { * @returns {string} The normalized text. */ normalize(text) { - text = text.normalize('NFC') + text = text.normalize("NFC"); return text; } } @@ -1080,7 +1044,7 @@ class NFKC extends Normalizer { * @returns {string} The normalized text. */ normalize(text) { - text = text.normalize('NFKC') + text = text.normalize("NFKC"); return text; } } @@ -1095,7 +1059,7 @@ class NFKD extends Normalizer { * @returns {string} The normalized text. */ normalize(text) { - text = text.normalize('NFKD') + text = text.normalize("NFKD"); return text; } } @@ -1179,19 +1143,19 @@ class Prepend extends Normalizer { */ class NormalizerSequence extends Normalizer { /** - * Create a new instance of NormalizerSequence. - * @param {Object} config The configuration object. - * @param {Object[]} config.normalizers An array of Normalizer configuration objects. - */ + * Create a new instance of NormalizerSequence. + * @param {Object} config The configuration object. + * @param {Object[]} config.normalizers An array of Normalizer configuration objects. + */ constructor(config) { super(config); - this.normalizers = config.normalizers.map(x => Normalizer.fromConfig(x)); + this.normalizers = config.normalizers.map((x) => Normalizer.fromConfig(x)); } /** - * Apply a sequence of Normalizers to the input text. - * @param {string} text The text to normalize. - * @returns {string} The normalized text. - */ + * Apply a sequence of Normalizers to the input text. + * @param {string} text The text to normalize. + * @returns {string} The normalized text. + */ normalize(text) { return this.normalizers.reduce((t, normalizer) => { return normalizer.normalize(t); @@ -1234,10 +1198,9 @@ class BertNormalizer extends Normalizer { */ stripAccents(text) { // "Mark, Nonspacing" (Mn) - return text.normalize('NFD').replace(/\p{Mn}/gu, ''); + return text.normalize("NFD").replace(/\p{Mn}/gu, ""); } - /** * Checks whether `char` is a control character. * @param {string} char The character to check. @@ -1246,9 +1209,9 @@ class BertNormalizer extends Normalizer { */ _is_control(char) { switch (char) { - case '\t': - case '\n': - case '\r': + case "\t": + case "\n": + case "\r": // These are technically control characters but we count them as whitespace characters. return false; @@ -1272,10 +1235,11 @@ class BertNormalizer extends Normalizer { const output = []; for (const char of text) { const cp = char.charCodeAt(0); - if (cp === 0 || cp === 0xFFFD || this._is_control(char)) { + if (cp === 0 || cp === 0xfffd || this._is_control(char)) { continue; } - if (/^\s$/.test(char)) { // is whitespace + if (/^\s$/.test(char)) { + // is whitespace output.push(" "); } else { output.push(char); @@ -1318,37 +1282,37 @@ class BertNormalizer extends Normalizer { */ class PreTokenizer extends Callable { /** - * Factory method that returns an instance of a subclass of `PreTokenizer` based on the provided configuration. - * - * @static - * @param {Object} config A configuration object for the pre-tokenizer. - * @returns {PreTokenizer} An instance of a subclass of `PreTokenizer`. - * @throws {Error} If the provided configuration object does not correspond to any known pre-tokenizer. - */ + * Factory method that returns an instance of a subclass of `PreTokenizer` based on the provided configuration. + * + * @static + * @param {Object} config A configuration object for the pre-tokenizer. + * @returns {PreTokenizer} An instance of a subclass of `PreTokenizer`. + * @throws {Error} If the provided configuration object does not correspond to any known pre-tokenizer. + */ static fromConfig(config) { if (config === null) return null; switch (config.type) { - case 'BertPreTokenizer': + case "BertPreTokenizer": return new BertPreTokenizer(config); - case 'Sequence': + case "Sequence": return new PreTokenizerSequence(config); - case 'Whitespace': + case "Whitespace": return new WhitespacePreTokenizer(config); - case 'WhitespaceSplit': + case "WhitespaceSplit": return new WhitespaceSplit(config); - case 'Metaspace': + case "Metaspace": return new MetaspacePreTokenizer(config); - case 'ByteLevel': + case "ByteLevel": return new ByteLevelPreTokenizer(config); - case 'Split': + case "Split": return new SplitPreTokenizer(config); - case 'Punctuation': + case "Punctuation": return new PunctuationPreTokenizer(config); - case 'Digits': + case "Digits": return new DigitsPreTokenizer(config); - case 'Replace': + case "Replace": return new ReplacePreTokenizer(config); default: throw new Error(`Unknown PreTokenizer type: ${config.type}`); @@ -1365,7 +1329,7 @@ class PreTokenizer extends Callable { * @throws {Error} If the method is not implemented in the subclass. */ pre_tokenize_text(text, options) { - throw Error("pre_tokenize_text should be implemented in subclass.") + throw Error("pre_tokenize_text should be implemented in subclass."); } /** @@ -1375,9 +1339,10 @@ class PreTokenizer extends Callable { * @returns {string[]} An array of pre-tokens. */ pre_tokenize(text, options) { - return (Array.isArray(text) - ? text.map(x => this.pre_tokenize_text(x, options)) - : this.pre_tokenize_text(text, options) + return ( + Array.isArray(text) + ? text.map((x) => this.pre_tokenize_text(x, options)) + : this.pre_tokenize_text(text, options) ).flat(); } @@ -1399,7 +1364,7 @@ class BertPreTokenizer extends PreTokenizer { /** * A PreTokenizer that splits text into wordpieces using a basic tokenization scheme * similar to that used in the original implementation of BERT. - * + * * @param {Object} config The configuration object. */ constructor(config) { @@ -1407,11 +1372,11 @@ class BertPreTokenizer extends PreTokenizer { // Construct a pattern which matches the rust implementation: // https://github.com/huggingface/tokenizers/blob/b4fcc9ce6e4ad5806e82826f816acfdfdc4fcc67/tokenizers/src/pre_tokenizers/bert.rs#L11 // Equivalent to removing whitespace and splitting on punctuation (both \p{P} and other ascii characters) - this.pattern = new RegExp(`[^\\s${PUNCTUATION_REGEX}]+|[${PUNCTUATION_REGEX}]`, 'gu'); + this.pattern = new RegExp(`[^\\s${PUNCTUATION_REGEX}]+|[${PUNCTUATION_REGEX}]`, "gu"); } /** * Tokenizes a single text using the BERT pre-tokenization scheme. - * + * * @param {string} text The text to tokenize. * @param {Object} [options] Additional options for the pre-tokenization logic. * @returns {string[]} An array of tokens. @@ -1466,16 +1431,16 @@ class ByteLevelPreTokenizer extends PreTokenizer { */ pre_tokenize_text(text, options) { // Add a leading space if the option is enabled - if (this.add_prefix_space && !text.startsWith(' ')) { - text = ' ' + text; + if (this.add_prefix_space && !text.startsWith(" ")) { + text = " " + text; } // Split on whitespace and punctuation - const tokens = this.use_regex ? (text.match(this.pattern) || []) : [text]; + const tokens = this.use_regex ? text.match(this.pattern) || [] : [text]; // Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case) - return tokens.map( - token => Array.from(this.text_encoder.encode(token), byte => this.byte_encoder[byte]).join('') + return tokens.map((token) => + Array.from(this.text_encoder.encode(token), (byte) => this.byte_encoder[byte]).join(""), ); } } @@ -1518,8 +1483,8 @@ class SplitPreTokenizer extends PreTokenizer { if (this.config.invert) { return text.match(this.pattern) || []; - } else if (this.config.behavior?.toLowerCase() === 'removed') { - return text.split(this.pattern).filter(x => x); + } else if (this.config.behavior?.toLowerCase() === "removed") { + return text.split(this.pattern).filter((x) => x); } else { return regexSplit(text, this.pattern); } @@ -1538,7 +1503,7 @@ class PunctuationPreTokenizer extends PreTokenizer { constructor(config) { super(); this.config = config; - this.pattern = new RegExp(`[^${PUNCTUATION_REGEX}]+|[${PUNCTUATION_REGEX}]+`, 'gu'); + this.pattern = new RegExp(`[^${PUNCTUATION_REGEX}]+|[${PUNCTUATION_REGEX}]+`, "gu"); } /** @@ -1552,7 +1517,6 @@ class PunctuationPreTokenizer extends PreTokenizer { } } - /** * Splits text based on digits. * @extends PreTokenizer @@ -1567,8 +1531,8 @@ class DigitsPreTokenizer extends PreTokenizer { this.config = config; // Construct a pattern which matches the rust implementation: - const digit_pattern = `[^\\d]+|\\d${this.config.individual_digits ? '' : '+'}`; - this.pattern = new RegExp(digit_pattern, 'gu'); + const digit_pattern = `[^\\d]+|\\d${this.config.individual_digits ? "" : "+"}`; + this.pattern = new RegExp(digit_pattern, "gu"); } /** @@ -1588,7 +1552,6 @@ class DigitsPreTokenizer extends PreTokenizer { * @property {number[]} [token_type_ids] List of token type ids produced by the post-processor. */ - /** * @typedef {Object} EncodingSingle * @property {number[]} input_ids List of token ids to be fed to a model. @@ -1596,12 +1559,10 @@ class DigitsPreTokenizer extends PreTokenizer { * @property {number[]} [token_type_ids] List of indices specifying which tokens should be attended to by the model */ - /** * @extends Callable */ class PostProcessor extends Callable { - /** * @param {Object} config The configuration for the post-processor. */ @@ -1620,18 +1581,18 @@ class PostProcessor extends Callable { static fromConfig(config) { if (config === null) return null; switch (config.type) { - case 'TemplateProcessing': + case "TemplateProcessing": return new TemplateProcessing(config); - case 'ByteLevel': + case "ByteLevel": return new ByteLevelPostProcessor(config); - case 'RobertaProcessing': + case "RobertaProcessing": return new RobertaProcessing(config); - case 'BertProcessing': + case "BertProcessing": return new BertProcessing(config); - case 'Sequence': + case "Sequence": return new PostProcessorSequence(config); default: throw new Error(`Unknown PostProcessor type: ${config.type}`); @@ -1647,7 +1608,7 @@ class PostProcessor extends Callable { * @throws {Error} If the method is not implemented in subclass. */ post_process(tokens, ...args) { - throw Error("post_process should be implemented in subclass.") + throw Error("post_process should be implemented in subclass."); } /** @@ -1684,9 +1645,7 @@ class BertProcessing extends PostProcessor { * @param {string[]} [tokens_pair=null] An optional second set of input tokens. * @returns {PostProcessedOutput} The post-processed tokens with the special tokens added to the beginning and end. */ - post_process(tokens, tokens_pair = null, { - add_special_tokens = true, - } = {}) { + post_process(tokens, tokens_pair = null, { add_special_tokens = true } = {}) { if (add_special_tokens) { tokens = mergeArrays([this.cls], tokens, [this.sep]); } @@ -1695,18 +1654,19 @@ class BertProcessing extends PostProcessor { if (tokens_pair !== null) { // NOTE: It is intended to add 2 EOS tokens after the first set of tokens // https://github.com/huggingface/tokenizers/issues/983 - const middle = (add_special_tokens && this instanceof RobertaProcessing) - ? [this.sep] - : []; + const middle = add_special_tokens && this instanceof RobertaProcessing ? [this.sep] : []; const after = add_special_tokens ? [this.sep] : []; tokens = mergeArrays(tokens, middle, tokens_pair, after); - token_type_ids = mergeArrays(token_type_ids, new Array(tokens_pair.length + middle.length + after.length).fill(1)); + token_type_ids = mergeArrays( + token_type_ids, + new Array(tokens_pair.length + middle.length + after.length).fill(1), + ); } return { tokens, token_type_ids }; } } -class RobertaProcessing extends BertProcessing { } // NOTE: extends BertProcessing +class RobertaProcessing extends BertProcessing {} // NOTE: extends BertProcessing /** * Post processor that replaces special tokens in a template with actual tokens. @@ -1732,25 +1692,22 @@ class TemplateProcessing extends PostProcessor { * @param {string[]} [tokens_pair=null] The list of tokens for the second sequence (optional). * @returns {PostProcessedOutput} An object containing the list of tokens with the special tokens replaced with actual tokens. */ - post_process(tokens, tokens_pair = null, { - add_special_tokens = true, - } = {}) { - const type = tokens_pair === null ? this.single : this.pair + post_process(tokens, tokens_pair = null, { add_special_tokens = true } = {}) { + const type = tokens_pair === null ? this.single : this.pair; let processedTokens = []; let types = []; for (const item of type) { - if ('SpecialToken' in item) { + if ("SpecialToken" in item) { if (add_special_tokens) { processedTokens.push(item.SpecialToken.id); types.push(item.SpecialToken.type_id); } - } else if ('Sequence' in item) { - if (item.Sequence.id === 'A') { + } else if ("Sequence" in item) { + if (item.Sequence.id === "A") { processedTokens = mergeArrays(processedTokens, tokens); types = mergeArrays(types, new Array(tokens.length).fill(item.Sequence.type_id)); - - } else if (item.Sequence.id === 'B') { + } else if (item.Sequence.id === "B") { processedTokens = mergeArrays(processedTokens, tokens_pair); types = mergeArrays(types, new Array(tokens_pair.length).fill(item.Sequence.type_id)); } @@ -1779,12 +1736,10 @@ class ByteLevelPostProcessor extends PostProcessor { } } - /** * A post-processor that applies multiple post-processors in sequence. */ class PostProcessorSequence extends PostProcessor { - /** * Creates a new instance of PostProcessorSequence. * @param {Object} config The configuration object. @@ -1793,7 +1748,7 @@ class PostProcessorSequence extends PostProcessor { constructor(config) { super(config); - this.processors = config.processors.map(x => PostProcessor.fromConfig(x)); + this.processors = config.processors.map((x) => PostProcessor.fromConfig(x)); } /** @@ -1828,12 +1783,11 @@ class PostProcessorSequence extends PostProcessor { * @extends Callable */ class Decoder extends Callable { - /** - * Creates an instance of `Decoder`. - * - * @param {Object} config The configuration object. - */ + * Creates an instance of `Decoder`. + * + * @param {Object} config The configuration object. + */ constructor(config) { super(); this.config = config; @@ -1845,37 +1799,37 @@ class Decoder extends Callable { } /** - * Creates a decoder instance based on the provided configuration. - * - * @param {Object} config The configuration object. - * @returns {Decoder} A decoder instance. - * @throws {Error} If an unknown decoder type is provided. - */ + * Creates a decoder instance based on the provided configuration. + * + * @param {Object} config The configuration object. + * @returns {Decoder} A decoder instance. + * @throws {Error} If an unknown decoder type is provided. + */ static fromConfig(config) { if (config === null) return null; switch (config.type) { - case 'WordPiece': + case "WordPiece": return new WordPieceDecoder(config); - case 'Metaspace': + case "Metaspace": return new MetaspaceDecoder(config); - case 'ByteLevel': + case "ByteLevel": return new ByteLevelDecoder(config); - case 'Replace': + case "Replace": return new ReplaceDecoder(config); - case 'ByteFallback': + case "ByteFallback": return new ByteFallback(config); - case 'Fuse': + case "Fuse": return new FuseDecoder(config); - case 'Strip': + case "Strip": return new StripDecoder(config); - case 'Sequence': + case "Sequence": return new DecoderSequence(config); - case 'CTC': + case "CTC": return new CTCDecoder(config); - case 'BPEDecoder': + case "BPEDecoder": return new BPEDecoder(config); default: throw new Error(`Unknown Decoder type: ${config.type}`); @@ -1883,49 +1837,44 @@ class Decoder extends Callable { } /** - * Calls the `decode` method. - * - * @param {string[]} tokens The list of tokens. - * @returns {string} The decoded string. - */ + * Calls the `decode` method. + * + * @param {string[]} tokens The list of tokens. + * @returns {string} The decoded string. + */ _call(tokens) { return this.decode(tokens); } /** - * Decodes a list of tokens. - * @param {string[]} tokens The list of tokens. - * @returns {string} The decoded string. - */ + * Decodes a list of tokens. + * @param {string[]} tokens The list of tokens. + * @returns {string} The decoded string. + */ decode(tokens) { - return this.decode_chain(tokens).join(''); + return this.decode_chain(tokens).join(""); } /** * Apply the decoder to a list of tokens. - * + * * @param {string[]} tokens The list of tokens. * @returns {string[]} The decoded list of tokens. * @throws {Error} If the `decode_chain` method is not implemented in the subclass. */ decode_chain(tokens) { - throw Error("`decode_chain` should be implemented in subclass.") + throw Error("`decode_chain` should be implemented in subclass."); } - } class ReplaceDecoder extends Decoder { - /** @type {Decoder['decode_chain']} */ decode_chain(tokens) { const pattern = createPattern(this.config.pattern); - return pattern === null - ? tokens - : tokens.map(token => token.replaceAll(pattern, this.config.content)) + return pattern === null ? tokens : tokens.map((token) => token.replaceAll(pattern, this.config.content)); } } - class ByteFallback extends Decoder { constructor(config) { super(config); @@ -1935,13 +1884,12 @@ class ByteFallback extends Decoder { /** @type {Decoder['decode_chain']} */ decode_chain(tokens) { - const new_tokens = []; let previous_byte_tokens = []; for (const token of tokens) { let bytes = null; - if (token.length === 6 && token.startsWith('<0x') && token.endsWith('>')) { + if (token.length === 6 && token.startsWith("<0x") && token.endsWith(">")) { const byte = parseInt(token.slice(3, 5), 16); if (!isNaN(byte)) { bytes = byte; @@ -1974,14 +1922,12 @@ class ByteFallback extends Decoder { * exists incase some decoders need to happen after that step */ class FuseDecoder extends Decoder { - /** @type {Decoder['decode_chain']} */ decode_chain(tokens) { - return [tokens.join('')]; + return [tokens.join("")]; } } - class StripDecoder extends Decoder { constructor(config) { super(config); @@ -1993,7 +1939,7 @@ class StripDecoder extends Decoder { /** @type {Decoder['decode_chain']} */ decode_chain(tokens) { - return tokens.map(token => { + return tokens.map((token) => { let start_cut = 0; for (let i = 0; i < this.start; ++i) { if (token[i] === this.content) { @@ -2015,7 +1961,7 @@ class StripDecoder extends Decoder { } } - return token.slice(start_cut, stop_cut) + return token.slice(start_cut, stop_cut); }); } } @@ -2025,7 +1971,6 @@ class StripDecoder extends Decoder { * @extends Decoder */ class WordPieceDecoder extends Decoder { - /** * Creates a new instance of WordPieceDecoder. * @param {Object} config The configuration object. @@ -2043,13 +1988,13 @@ class WordPieceDecoder extends Decoder { if (i !== 0) { if (token.startsWith(this.config.prefix)) { // NOTE: .replace() is intended; only replace first occurrence - token = token.replace(this.config.prefix, ''); + token = token.replace(this.config.prefix, ""); } else { - token = ' ' + token; + token = " " + token; } } if (this.cleanup) { - token = clean_up_tokenization(token) + token = clean_up_tokenization(token); } return token; @@ -2062,7 +2007,6 @@ class WordPieceDecoder extends Decoder { * @extends Decoder */ class ByteLevelDecoder extends Decoder { - /** * Create a `ByteLevelDecoder` object. * @param {Object} config Configuration object. @@ -2085,8 +2029,8 @@ class ByteLevelDecoder extends Decoder { * @returns {string} The decoded string. */ convert_tokens_to_string(tokens) { - const text = tokens.join(''); - const byteArray = new Uint8Array([...text].map(c => this.byte_decoder[c])); + const text = tokens.join(""); + const byteArray = new Uint8Array([...text].map((c) => this.byte_decoder[c])); const decoded_text = this.text_decoder.decode(byteArray); return decoded_text; } @@ -2107,7 +2051,7 @@ class ByteLevelDecoder extends Decoder { // continue; // } - if (this.added_tokens.find(x => x.content === token) !== undefined) { + if (this.added_tokens.find((x) => x.content === token) !== undefined) { if (current_sub_text.length > 0) { sub_texts.push(this.convert_tokens_to_string(current_sub_text)); current_sub_text = []; @@ -2132,7 +2076,6 @@ class ByteLevelDecoder extends Decoder { * See https://github.com/huggingface/tokenizers/blob/bb38f390a61883fc2f29d659af696f428d1cda6b/tokenizers/src/decoders/ctc.rs */ class CTCDecoder extends Decoder { - constructor(config) { super(config); @@ -2146,7 +2089,7 @@ class CTCDecoder extends Decoder { * @returns {string} The decoded string. */ convert_tokens_to_string(tokens) { - if (tokens.length === 0) return ''; + if (tokens.length === 0) return ""; // group same tokens into non-repeating tokens in CTC style decoding const grouped_tokens = [tokens[0]]; @@ -2157,19 +2100,16 @@ class CTCDecoder extends Decoder { } // filter self.pad_token which is used as CTC-blank token - const filtered_tokens = grouped_tokens.filter(token => token !== this.pad_token); + const filtered_tokens = grouped_tokens.filter((token) => token !== this.pad_token); - let text = filtered_tokens.join(''); + let text = filtered_tokens.join(""); if (this.cleanup) { // cleanup and replace delimiter token - text = clean_up_tokenization(text) - .replaceAll(this.word_delimiter_token, ' ') - .trim(); + text = clean_up_tokenization(text).replaceAll(this.word_delimiter_token, " ").trim(); } return text; } - /** @type {Decoder['decode_chain']} */ decode_chain(tokens) { return [this.convert_tokens_to_string(tokens)]; @@ -2181,7 +2121,6 @@ class CTCDecoder extends Decoder { * @extends Decoder */ class DecoderSequence extends Decoder { - /** * Creates a new instance of DecoderSequence. * @param {Object} config The configuration object. @@ -2189,7 +2128,7 @@ class DecoderSequence extends Decoder { */ constructor(config) { super(config); - this.decoders = config.decoders.map(x => Decoder.fromConfig(x)); + this.decoders = config.decoders.map((x) => Decoder.fromConfig(x)); } /** @type {Decoder['decode_chain']} */ @@ -2199,7 +2138,6 @@ class DecoderSequence extends Decoder { return decoder.decode_chain(toks); }, tokens); } - } class BPEDecoder extends Decoder { @@ -2211,7 +2149,7 @@ class BPEDecoder extends Decoder { /** @type {Decoder['decode_chain']} */ decode_chain(tokens) { return tokens.map((token, i) => { - return token.replaceAll(this.suffix, (i === tokens.length - 1) ? '' : ' ') + return token.replaceAll(this.suffix, i === tokens.length - 1 ? "" : " "); }); } } @@ -2220,7 +2158,7 @@ class BPEDecoder extends Decoder { class VitsDecoder extends Decoder { /** @type {Decoder['decode_chain']} */ decode_chain(tokens) { - let decoded = ''; + let decoded = ""; for (let i = 1; i < tokens.length; i += 2) { decoded += tokens[i]; } @@ -2228,7 +2166,6 @@ class VitsDecoder extends Decoder { } } - /** * This PreTokenizer replaces spaces with the given replacement character, adds a prefix space if requested, * and returns a list of tokens. @@ -2248,7 +2185,7 @@ class MetaspacePreTokenizer extends PreTokenizer { this.addPrefixSpace = config.add_prefix_space; this.replacement = config.replacement; this.strRep = config.str_rep || this.replacement; - this.prepend_scheme = config.prepend_scheme ?? 'always'; + this.prepend_scheme = config.prepend_scheme ?? "always"; } /** @@ -2259,25 +2196,19 @@ class MetaspacePreTokenizer extends PreTokenizer { * @param {number} [options.section_index] The index of the section to pre-tokenize. * @returns {string[]} A new list of pre-tokenized tokens. */ - pre_tokenize_text(text, { - section_index = undefined, - } = {}) { - - let normalized = text.replaceAll(' ', this.strRep); + pre_tokenize_text(text, { section_index = undefined } = {}) { + let normalized = text.replaceAll(" ", this.strRep); if ( // We add a prefix space if: // (1) The addPrefixSpace option is enabled and the normalized // token does not already start with the replacement character. - (this.addPrefixSpace && !normalized.startsWith(this.replacement)) - + this.addPrefixSpace && + !normalized.startsWith(this.replacement) && // and (2) either: // (a) prepend_scheme is 'always' // (b) prepend_scheme is 'first' and this is the first section - && ( - this.prepend_scheme === 'always' || - (this.prepend_scheme === 'first' && section_index === 0) - ) + (this.prepend_scheme === "always" || (this.prepend_scheme === "first" && section_index === 0)) ) { normalized = this.strRep + normalized; } @@ -2307,8 +2238,8 @@ class MetaspaceDecoder extends Decoder { decode_chain(tokens) { const result = []; for (let i = 0; i < tokens.length; ++i) { - let normalized = tokens[i].replaceAll(this.replacement, ' '); - if (this.addPrefixSpace && i == 0 && normalized.startsWith(' ')) { + let normalized = tokens[i].replaceAll(this.replacement, " "); + if (this.addPrefixSpace && i == 0 && normalized.startsWith(" ")) { normalized = normalized.substring(1); } result.push(normalized); @@ -2348,23 +2279,26 @@ class Precompiled extends Normalizer { // 3. nmt_nfkc_cf: nmt_nfkc + Unicode case folding (mostly lower casing) // 4. nfkc_cf: nfkc + Unicode case folding. // 5. identity: no normalization - // + // // For now, we only implement the default (nmt_nfkc). // See https://raw.githubusercontent.com/google/sentencepiece/master/data/nmt_nfkc.tsv for the full list of rules. // TODO: detect when a different `this.charsmap` is used. - text = text.replace(/[\u0001-\u0008\u000B\u000E-\u001F\u007F\u008F\u009F]/gm, ''); // Remove control characters - text = text.replace(/[\u0009\u000A\u000C\u000D\u00A0\u1680\u2000-\u200F\u2028\u2029\u202F\u205F\u2581\u3000\uFEFF\uFFFD]/gm, '\u0020'); // Replace certain characters with a space + text = text.replace(/[\u0001-\u0008\u000B\u000E-\u001F\u007F\u008F\u009F]/gm, ""); // Remove control characters + text = text.replace( + /[\u0009\u000A\u000C\u000D\u00A0\u1680\u2000-\u200F\u2028\u2029\u202F\u205F\u2581\u3000\uFEFF\uFFFD]/gm, + "\u0020", + ); // Replace certain characters with a space - if (text.includes('\uFF5E')) { + if (text.includes("\uFF5E")) { // To match the sentencepiece implementation 100%, we must handle a very strange edge-case. // For some reason, the "Fullwidth Tilde" character (\uFF5E) should not be converted to the standard Tilde character (\u007E). // However, NFKC normalization does do this conversion. As a result, we split the string on the Fullwidth Tilde character, // perform NFKC normalization on each substring, and then join them back together with the Fullwidth Tilde character. - const parts = text.split('\uFF5E'); - text = parts.map(part => part.normalize('NFKC')).join('\uFF5E'); + const parts = text.split("\uFF5E"); + text = parts.map((part) => part.normalize("NFKC")).join("\uFF5E"); } else { - text = text.normalize('NFKC'); + text = text.normalize("NFKC"); } return text; @@ -2383,7 +2317,7 @@ class PreTokenizerSequence extends PreTokenizer { */ constructor(config) { super(); - this.tokenizers = config.pretokenizers.map(x => PreTokenizer.fromConfig(x)); + this.tokenizers = config.pretokenizers.map((x) => PreTokenizer.fromConfig(x)); } /** @@ -2394,9 +2328,12 @@ class PreTokenizerSequence extends PreTokenizer { */ pre_tokenize_text(text, options) { // Use reduce to apply each tokenizer to the text - return this.tokenizers.reduce((preTokenizedText, tokenizer) => { - return tokenizer.pre_tokenize(preTokenizedText, options); - }, [text]); + return this.tokenizers.reduce( + (preTokenizedText, tokenizer) => { + return tokenizer.pre_tokenize(preTokenizedText, options); + }, + [text], + ); } } @@ -2474,18 +2411,18 @@ class ReplacePreTokenizer extends PreTokenizer { } const SPECIAL_TOKEN_ATTRIBUTES = [ - 'bos_token', - 'eos_token', - 'unk_token', - 'sep_token', - 'pad_token', - 'cls_token', - 'mask_token', + "bos_token", + "eos_token", + "unk_token", + "sep_token", + "pad_token", + "cls_token", + "mask_token", // additional_special_tokens (TODO) -] +]; /** - * + * * Helper function for padding values of an object, which are each arrays. * NOTE: No additional checks are made here for validity of arguments. * @param {Record} item The input object. @@ -2500,9 +2437,7 @@ function padHelper(item, length, value_fn, side) { const value = value_fn(key); const padData = new Array(diff).fill(value); - item[key] = side === 'right' - ? mergeArrays(item[key], padData) - : mergeArrays(padData, item[key]); + item[key] = side === "right" ? mergeArrays(item[key], padData) : mergeArrays(padData, item[key]); } } @@ -2521,7 +2456,6 @@ function truncateHelper(item, length) { } } - /** * @typedef {Object} Message * @property {string} role The role of the message (e.g., "user" or "assistant" or "system"). @@ -2531,7 +2465,7 @@ function truncateHelper(item, length) { export class PreTrainedTokenizer extends Callable { return_token_type_ids = false; - padding_side = 'right'; + padding_side = "right"; /** * Create a new PreTrainedTokenizer instance. * @param {Object} tokenizerJSON The JSON of the tokenizer. @@ -2584,25 +2518,29 @@ export class PreTrainedTokenizer extends Callable { this.decoder.end_of_word_suffix = this.model.end_of_word_suffix; } - this.added_tokens_regex = this.added_tokens.length > 0 ? new RegExp( - this.added_tokens.slice() - // Sort by length (desc) to avoid early partial matches - .sort((a, b) => b.content.length - a.content.length) - .map(x => `${x.lstrip ? '\\s*' : ''}(${escapeRegExp(x.content)})${x.rstrip ? '\\s*' : ''}`) - .join('|') - ) : null; + this.added_tokens_regex = + this.added_tokens.length > 0 + ? new RegExp( + this.added_tokens + .slice() + // Sort by length (desc) to avoid early partial matches + .sort((a, b) => b.content.length - a.content.length) + .map((x) => `${x.lstrip ? "\\s*" : ""}(${escapeRegExp(x.content)})${x.rstrip ? "\\s*" : ""}`) + .join("|"), + ) + : null; // Set mask token if present (otherwise will be undefined, which is fine) - this.mask_token = this.getToken('mask_token'); + this.mask_token = this.getToken("mask_token"); this.mask_token_id = this.model.tokens_to_ids.get(this.mask_token); - this.pad_token = this.getToken('pad_token', 'eos_token'); + this.pad_token = this.getToken("pad_token", "eos_token"); this.pad_token_id = this.model.tokens_to_ids.get(this.pad_token); - this.sep_token = this.getToken('sep_token'); + this.sep_token = this.getToken("sep_token"); this.sep_token_id = this.model.tokens_to_ids.get(this.sep_token); - this.unk_token = this.getToken('unk_token'); + this.unk_token = this.getToken("unk_token"); this.unk_token_id = this.model.tokens_to_ids.get(this.unk_token); this.model_max_length = tokenizerConfig.model_max_length; @@ -2625,7 +2563,7 @@ export class PreTrainedTokenizer extends Callable { // we reconstruct that into a single dict while loading them. const chat_template = Object.create(null); for (const { name, template } of this.chat_template) { - if (typeof name !== 'string' || typeof template !== 'string') { + if (typeof name !== "string" || typeof template !== "string") { throw new Error('Chat template must be a list of objects with "name" and "template" properties'); } chat_template[name] = template; @@ -2648,8 +2586,8 @@ export class PreTrainedTokenizer extends Callable { if (!item) continue; - if (typeof item === 'object') { - if (item.__type === 'AddedToken') { + if (typeof item === "object") { + if (item.__type === "AddedToken") { return item.content; } else { throw Error(`Unknown token: ${item}`); @@ -2662,23 +2600,25 @@ export class PreTrainedTokenizer extends Callable { } /** - * Loads a pre-trained tokenizer from the given `pretrained_model_name_or_path`. - * + * Loads a pre-trained tokenizer from the given `pretrained_model_name_or_path`. + * * @param {string} pretrained_model_name_or_path The path to the pre-trained tokenizer. * @param {PretrainedTokenizerOptions} options Additional options for loading the tokenizer. - * + * * @throws {Error} Throws an error if the tokenizer.json or tokenizer_config.json files are not found in the `pretrained_model_name_or_path`. * @returns {Promise} A new instance of the `PreTrainedTokenizer` class. */ - static async from_pretrained(pretrained_model_name_or_path, { - progress_callback = null, - config = null, - cache_dir = null, - local_files_only = false, - revision = 'main', - legacy = null, - } = {}) { - + static async from_pretrained( + pretrained_model_name_or_path, + { + progress_callback = null, + config = null, + cache_dir = null, + local_files_only = false, + revision = "main", + legacy = null, + } = {}, + ) { const info = await loadTokenizer(pretrained_model_name_or_path, { progress_callback, config, @@ -2686,7 +2626,7 @@ export class PreTrainedTokenizer extends Callable { local_files_only, revision, legacy, - }) + }); // @ts-ignore return new this(...info); @@ -2694,7 +2634,7 @@ export class PreTrainedTokenizer extends Callable { /** * @typedef {number[]|number[][]|Tensor} BatchEncodingItem - * + * * @typedef {Object} BatchEncoding Holds the output of the tokenizer's call function. * @property {BatchEncodingItem} input_ids List of token ids to be fed to a model. * @property {BatchEncodingItem} attention_mask List of indices specifying which tokens should be attended to by the model. @@ -2729,7 +2669,6 @@ export class PreTrainedTokenizer extends Callable { return_token_type_ids = null, } = {}, ) { - const isBatched = Array.isArray(text); /** @type {EncodingSingle[]} */ @@ -2737,32 +2676,31 @@ export class PreTrainedTokenizer extends Callable { if (isBatched) { if (text.length === 0) { - throw Error('text array must be non-empty') + throw Error("text array must be non-empty"); } if (text_pair !== null) { if (!Array.isArray(text_pair)) { - throw Error('text_pair must also be an array') - + throw Error("text_pair must also be an array"); } else if (text.length !== text_pair.length) { - throw Error('text and text_pair must have the same length') + throw Error("text and text_pair must have the same length"); } - encodedTokens = text.map( - (t, i) => this._encode_plus(t, { text_pair: text_pair[i], add_special_tokens, return_token_type_ids }) - ) - + encodedTokens = text.map((t, i) => + this._encode_plus(t, { text_pair: text_pair[i], add_special_tokens, return_token_type_ids }), + ); } else { - encodedTokens = text.map(x => this._encode_plus(x, { add_special_tokens, return_token_type_ids })); + encodedTokens = text.map((x) => this._encode_plus(x, { add_special_tokens, return_token_type_ids })); } - } else { if (text === null || text === undefined) { - throw Error('text may not be null or undefined') + throw Error("text may not be null or undefined"); } if (Array.isArray(text_pair)) { - throw Error('When specifying `text_pair`, since `text` is a string, `text_pair` must also be a string (i.e., not an array).') + throw Error( + "When specifying `text_pair`, since `text` is a string, `text_pair` must also be a string (i.e., not an array).", + ); } // For single input, we just wrap in an array, and then unwrap later. @@ -2772,15 +2710,17 @@ export class PreTrainedTokenizer extends Callable { // However, array may be jagged. So, we pad to max_length if (max_length === null) { - if (padding === 'max_length') { + if (padding === "max_length") { max_length = this.model_max_length; } else { // Calculate max length from sequences - max_length = max(encodedTokens.map(x => x.input_ids.length))[0]; + max_length = max(encodedTokens.map((x) => x.input_ids.length))[0]; } } else { if (!truncation) { - console.warn(`Truncation was not explicitly activated but \`max_length\` is provided a specific value, please use \`truncation=true\` to explicitly truncate examples to max length.`) + console.warn( + `Truncation was not explicitly activated but \`max_length\` is provided a specific value, please use \`truncation=true\` to explicitly truncate examples to max length.`, + ); } } @@ -2788,26 +2728,24 @@ export class PreTrainedTokenizer extends Callable { max_length = Math.min(max_length, this.model_max_length ?? Infinity); if (padding || truncation) { - // Perform padding and/or truncation for (let i = 0; i < encodedTokens.length; ++i) { if (encodedTokens[i].input_ids.length === max_length) { continue; - } else if (encodedTokens[i].input_ids.length > max_length) { // possibly truncate if (truncation) { truncateHelper(encodedTokens[i], max_length); } - - } else { // t.length < max_length + } else { + // t.length < max_length // possibly pad if (padding) { padHelper( encodedTokens[i], max_length, - key => key === 'input_ids' ? this.pad_token_id : 0, - this.padding_side + (key) => (key === "input_ids" ? this.pad_token_id : 0), + this.padding_side, ); } } @@ -2822,7 +2760,7 @@ export class PreTrainedTokenizer extends Callable { // we perform additional check if ( - encodedTokens.some(x => { + encodedTokens.some((x) => { for (const key of Object.keys(x)) { if (x[key].length !== encodedTokens[0][key]?.length) { return true; @@ -2833,8 +2771,8 @@ export class PreTrainedTokenizer extends Callable { ) { throw Error( "Unable to create tensor, you should probably activate truncation and/or padding " + - "with 'padding=true' and 'truncation=true' to have batched tensors with the same length." - ) + "with 'padding=true' and 'truncation=true' to have batched tensors with the same length.", + ); } } @@ -2844,15 +2782,15 @@ export class PreTrainedTokenizer extends Callable { const dims = [encodedTokens.length, encodedTokens[0].input_ids.length]; for (const key of Object.keys(encodedTokens[0])) { - result[key] = new Tensor('int64', - BigInt64Array.from(encodedTokens.flatMap(x => x[key]).map(BigInt)), - dims + result[key] = new Tensor( + "int64", + BigInt64Array.from(encodedTokens.flatMap((x) => x[key]).map(BigInt)), + dims, ); } - } else { for (const key of Object.keys(encodedTokens[0])) { - result[key] = encodedTokens.map(x => x[key]); + result[key] = encodedTokens.map((x) => x[key]); } // If not returning a tensor, we match the input type @@ -2864,7 +2802,7 @@ export class PreTrainedTokenizer extends Callable { } } - return /** @type {BatchEncoding} */(result); + return /** @type {BatchEncoding} */ (result); } /** @@ -2879,40 +2817,45 @@ export class PreTrainedTokenizer extends Callable { // Actual function which does encoding, for a single text // First, we take care of special tokens. Needed to avoid issues arising from // normalization and/or pretokenization (which may not preserve special tokens) - const sections = this.added_tokens_regex ? text.split(this.added_tokens_regex).filter(x => x) : [text]; - - const tokens = sections.map((x, section_index) => { - const addedToken = this.added_tokens.find(t => t.content === x); - if (addedToken !== undefined) { - // Ignore added tokens - return x - } else { - if (this.remove_space === true) { - x = x.trim().split(/\s+/).join(' '); - } - if (this.do_lowercase_and_remove_accent) { - x = lowercase_and_remove_accent(x); - } + const sections = this.added_tokens_regex ? text.split(this.added_tokens_regex).filter((x) => x) : [text]; + + const tokens = sections + .map((x, section_index) => { + const addedToken = this.added_tokens.find((t) => t.content === x); + if (addedToken !== undefined) { + // Ignore added tokens + return x; + } else { + if (this.remove_space === true) { + x = x.trim().split(/\s+/).join(" "); + } + if (this.do_lowercase_and_remove_accent) { + x = lowercase_and_remove_accent(x); + } - if (this.normalizer !== null) { - x = this.normalizer(x); - } + if (this.normalizer !== null) { + x = this.normalizer(x); + } - // If, after normalization, this section is empty (e.g., trimming whitespace), - // we return an empty array - if (x.length === 0) { - return []; - } + // If, after normalization, this section is empty (e.g., trimming whitespace), + // we return an empty array + if (x.length === 0) { + return []; + } - const sectionTokens = (this.pre_tokenizer !== null) ? this.pre_tokenizer(x, { - section_index, - }) : [x]; + const sectionTokens = + this.pre_tokenizer !== null + ? this.pre_tokenizer(x, { + section_index, + }) + : [x]; - const tokens = this.model(sectionTokens); + const tokens = this.model(sectionTokens); - return tokens; - } - }).flat(); + return tokens; + } + }) + .flat(); return tokens; } @@ -2928,12 +2871,7 @@ export class PreTrainedTokenizer extends Callable { * @returns {EncodingSingle} An object containing the encoded text. * @private */ - _encode_plus(text, { - text_pair = null, - add_special_tokens = true, - return_token_type_ids = null, - } = {}) { - + _encode_plus(text, { text_pair = null, add_special_tokens = true, return_token_type_ids = null } = {}) { const { tokens, token_type_ids } = this._tokenize_helper(text, { pair: text_pair, add_special_tokens }); const input_ids = this.model.convert_tokens_to_ids(tokens); @@ -2941,7 +2879,7 @@ export class PreTrainedTokenizer extends Callable { const result = { input_ids, attention_mask: new Array(input_ids.length).fill(1), - } + }; if ((return_token_type_ids ?? this.return_token_type_ids) && token_type_ids) { result.token_type_ids = token_type_ids; } @@ -2956,10 +2894,7 @@ export class PreTrainedTokenizer extends Callable { * @param {boolean} [options.add_special_tokens=false] Whether or not to add the special tokens associated with the corresponding model. * @returns {{tokens: string[], token_type_ids?: number[]}} An object containing the tokens and optionally the token type IDs. */ - _tokenize_helper(text, { - pair = null, - add_special_tokens = false, - } = {}) { + _tokenize_helper(text, { pair = null, add_special_tokens = false } = {}) { const tokens = this._encode_text(text); const tokens2 = this._encode_text(pair); @@ -2976,10 +2911,7 @@ export class PreTrainedTokenizer extends Callable { * @param {boolean} [options.add_special_tokens=false] Whether or not to add the special tokens associated with the corresponding model. * @returns {string[]} The list of tokens. */ - tokenize(text, { - pair = null, - add_special_tokens = false, - } = {}) { + tokenize(text, { pair = null, add_special_tokens = false } = {}) { return this._tokenize_helper(text, { pair, add_special_tokens }).tokens; } @@ -2993,11 +2925,7 @@ export class PreTrainedTokenizer extends Callable { * @param {boolean} [options.return_token_type_ids=null] Whether to return token_type_ids. * @returns {number[]} An array of token IDs representing the encoded text(s). */ - encode(text, { - text_pair = null, - add_special_tokens = true, - return_token_type_ids = null, - } = {}) { + encode(text, { text_pair = null, add_special_tokens = true, return_token_type_ids = null } = {}) { return this._encode_plus(text, { text_pair, add_special_tokens, @@ -3015,7 +2943,7 @@ export class PreTrainedTokenizer extends Callable { if (batch instanceof Tensor) { batch = batch.tolist(); } - return batch.map(x => this.decode(x, decode_args)); + return batch.map((x) => this.decode(x, decode_args)); } /** @@ -3029,10 +2957,7 @@ export class PreTrainedTokenizer extends Callable { * @returns {string} The decoded string. * @throws {Error} If `token_ids` is not a non-empty array of integers. */ - decode( - token_ids, - decode_args = {}, - ) { + decode(token_ids, decode_args = {}) { if (token_ids instanceof Tensor) { token_ids = prepareTensorForDecode(token_ids); } @@ -3041,7 +2966,7 @@ export class PreTrainedTokenizer extends Callable { throw Error("token_ids must be a non-empty array of integers."); } - return this.decode_single(token_ids, decode_args) + return this.decode_single(token_ids, decode_args); } /** @@ -3053,27 +2978,21 @@ export class PreTrainedTokenizer extends Callable { * If null, the value is set to `this.decoder.cleanup` if it exists, falling back to `this.clean_up_tokenization_spaces` if it exists, falling back to `true`. * @returns {string} The decoded string */ - decode_single( - token_ids, - { - skip_special_tokens = false, - clean_up_tokenization_spaces = null, - } - ) { + decode_single(token_ids, { skip_special_tokens = false, clean_up_tokenization_spaces = null }) { let tokens = this.model.convert_ids_to_tokens(token_ids); if (skip_special_tokens) { - tokens = tokens.filter(x => !this.special_tokens.includes(x)); + tokens = tokens.filter((x) => !this.special_tokens.includes(x)); } // If `this.decoder` is null, we just join tokens with a space: // https://github.com/huggingface/tokenizers/blob/8edec536a737cb04494b454805be16c020abb14f/tokenizers/src/tokenizer/mod.rs#L835 /** @type {string} */ - let decoded = this.decoder ? this.decoder(tokens) : tokens.join(' '); + let decoded = this.decoder ? this.decoder(tokens) : tokens.join(" "); // Slight hack, but prevents having to pass `skip_special_tokens` to // each call to `decode`, which would lead to code duplication. if (this.decoder && this.decoder.end_of_word_suffix) { - decoded = decoded.replaceAll(this.decoder.end_of_word_suffix, ' '); + decoded = decoded.replaceAll(this.decoder.end_of_word_suffix, " "); if (skip_special_tokens) { decoded = decoded.trim(); } @@ -3090,7 +3009,7 @@ export class PreTrainedTokenizer extends Callable { * Retrieve the chat template string used for tokenizing chat messages. This template is used * internally by the `apply_chat_template` method and can also be used externally to retrieve the model's chat * template for better generation tracking. - * + * * @param {Object} options An optional object containing the following properties: * @param {string} [options.chat_template=null] * A Jinja template or the name of a template to use for this conversion. @@ -3104,29 +3023,25 @@ export class PreTrainedTokenizer extends Callable { * for more information. * @returns {string} The chat template string. */ - get_chat_template({ - chat_template = null, - tools = null, - } = {}) { - + get_chat_template({ chat_template = null, tools = null } = {}) { // First, handle the cases when the model has a dict of multiple templates - if (this.chat_template && typeof this.chat_template === 'object') { + if (this.chat_template && typeof this.chat_template === "object") { const template_dict = this.chat_template; if (chat_template !== null && Object.hasOwn(template_dict, chat_template)) { // The user can pass the name of a template to the chat template argument instead of an entire template chat_template = template_dict[chat_template]; } else if (chat_template === null) { - if (tools !== null && 'tool_use' in template_dict) { - chat_template = template_dict['tool_use']; - } else if ('default' in template_dict) { - chat_template = template_dict['default']; + if (tools !== null && "tool_use" in template_dict) { + chat_template = template_dict["tool_use"]; + } else if ("default" in template_dict) { + chat_template = template_dict["default"]; } else { throw Error( `This model has multiple chat templates with no default specified! Please either pass a chat ` + - `template or the name of the template you wish to use to the 'chat_template' argument. Available ` + - `template names are ${Object.keys(template_dict).sort()}.` - ) + `template or the name of the template you wish to use to the 'chat_template' argument. Available ` + + `template names are ${Object.keys(template_dict).sort()}.`, + ); } } } else if (chat_template === null) { @@ -3137,10 +3052,10 @@ export class PreTrainedTokenizer extends Callable { } else { throw Error( "Cannot use apply_chat_template() because tokenizer.chat_template is not set and no template " + - "argument was passed! For information about writing templates and setting the " + - "tokenizer.chat_template attribute, please see the documentation at " + - "https://huggingface.co/docs/transformers/main/en/chat_templating" - ) + "argument was passed! For information about writing templates and setting the " + + "tokenizer.chat_template attribute, please see the documentation at " + + "https://huggingface.co/docs/transformers/main/en/chat_templating", + ); } } return chat_template; @@ -3150,29 +3065,29 @@ export class PreTrainedTokenizer extends Callable { * Converts a list of message objects with `"role"` and `"content"` keys to a list of token * ids. This method is intended for use with chat models, and will read the tokenizer's chat_template attribute to * determine the format and control tokens to use when converting. - * + * * See [here](https://huggingface.co/docs/transformers/chat_templating) for more information. - * + * * **Example:** Applying a chat template to a conversation. - * + * * ```javascript * import { AutoTokenizer } from "@huggingface/transformers"; - * + * * const tokenizer = await AutoTokenizer.from_pretrained("Xenova/mistral-tokenizer-v1"); - * + * * const chat = [ * { "role": "user", "content": "Hello, how are you?" }, * { "role": "assistant", "content": "I'm doing great. How can I help you today?" }, * { "role": "user", "content": "I'd like to show off how chat templating works!" }, * ] - * + * * const text = tokenizer.apply_chat_template(chat, { tokenize: false }); * // "[INST] Hello, how are you? [/INST]I'm doing great. How can I help you today? [INST] I'd like to show off how chat templating works! [/INST]" - * + * * const input_ids = tokenizer.apply_chat_template(chat, { tokenize: true, return_tensor: false }); * // [1, 733, 16289, 28793, 22557, 28725, 910, 460, 368, 28804, 733, 28748, 16289, 28793, 28737, 28742, 28719, 2548, 1598, 28723, 1602, 541, 315, 1316, 368, 3154, 28804, 2, 28705, 733, 16289, 28793, 315, 28742, 28715, 737, 298, 1347, 805, 910, 10706, 5752, 1077, 3791, 28808, 733, 28748, 16289, 28793] * ``` - * + * * @param {Message[]} conversation A list of message objects with `"role"` and `"content"` keys, * representing the chat history so far. * @param {Object} options An optional object containing the following properties: @@ -3204,24 +3119,26 @@ export class PreTrainedTokenizer extends Callable { * @param {Object} [options.tokenizer_kwargs={}] Additional options to pass to the tokenizer. * @returns {string | Tensor | number[]| number[][]|BatchEncoding} The tokenized output. */ - apply_chat_template(conversation, { - tools = null, - documents = null, - chat_template = null, - add_generation_prompt = false, - tokenize = true, - padding = false, - truncation = false, - max_length = null, - return_tensor = true, - return_dict = false, - tokenizer_kwargs = {}, - ...kwargs - } = {}) { - + apply_chat_template( + conversation, + { + tools = null, + documents = null, + chat_template = null, + add_generation_prompt = false, + tokenize = true, + padding = false, + truncation = false, + max_length = null, + return_tensor = true, + return_dict = false, + tokenizer_kwargs = {}, + ...kwargs + } = {}, + ) { chat_template = this.get_chat_template({ chat_template, tools }); - if (typeof chat_template !== 'string') { + if (typeof chat_template !== "string") { throw Error(`chat_template must be a string, but got ${typeof chat_template}`); } @@ -3300,30 +3217,32 @@ export class ConvBertTokenizer extends PreTrainedTokenizer { export class RoFormerTokenizer extends PreTrainedTokenizer { return_token_type_ids = true; } -export class DistilBertTokenizer extends PreTrainedTokenizer { } -export class CamembertTokenizer extends PreTrainedTokenizer { } +export class DistilBertTokenizer extends PreTrainedTokenizer {} +export class CamembertTokenizer extends PreTrainedTokenizer {} export class XLMTokenizer extends PreTrainedTokenizer { return_token_type_ids = true; constructor(tokenizerJSON, tokenizerConfig) { super(tokenizerJSON, tokenizerConfig); - console.warn('WARNING: `XLMTokenizer` is not yet supported by Hugging Face\'s "fast" tokenizers library. Therefore, you may experience slightly inaccurate results.') + console.warn( + 'WARNING: `XLMTokenizer` is not yet supported by Hugging Face\'s "fast" tokenizers library. Therefore, you may experience slightly inaccurate results.', + ); } } export class ElectraTokenizer extends PreTrainedTokenizer { return_token_type_ids = true; } -export class T5Tokenizer extends PreTrainedTokenizer { } -export class GPT2Tokenizer extends PreTrainedTokenizer { } -export class BartTokenizer extends PreTrainedTokenizer { } +export class T5Tokenizer extends PreTrainedTokenizer {} +export class GPT2Tokenizer extends PreTrainedTokenizer {} +export class BartTokenizer extends PreTrainedTokenizer {} export class MBartTokenizer extends PreTrainedTokenizer { constructor(tokenizerJSON, tokenizerConfig) { super(tokenizerJSON, tokenizerConfig); this.languageRegex = /^[a-z]{2}_[A-Z]{2}$/; - this.language_codes = this.special_tokens.filter(x => this.languageRegex.test(x)); - this.lang_to_token = x => x; // Identity function + this.language_codes = this.special_tokens.filter((x) => this.languageRegex.test(x)); + this.lang_to_token = (x) => x; // Identity function } /** @@ -3337,17 +3256,16 @@ export class MBartTokenizer extends PreTrainedTokenizer { return _build_translation_inputs(this, raw_inputs, tokenizer_options, generate_kwargs); } } -export class MBart50Tokenizer extends MBartTokenizer { } // NOTE: extends MBartTokenizer +export class MBart50Tokenizer extends MBartTokenizer {} // NOTE: extends MBartTokenizer -export class RobertaTokenizer extends PreTrainedTokenizer { } +export class RobertaTokenizer extends PreTrainedTokenizer {} -export class BloomTokenizer extends PreTrainedTokenizer { } +export class BloomTokenizer extends PreTrainedTokenizer {} const SPIECE_UNDERLINE = "▁"; export class LlamaTokenizer extends PreTrainedTokenizer { - - padding_side = 'left'; + padding_side = "left"; constructor(tokenizerJSON, tokenizerConfig) { super(tokenizerJSON, tokenizerConfig); @@ -3384,22 +3302,22 @@ export class LlamaTokenizer extends PreTrainedTokenizer { return tokens; } } -export class CodeLlamaTokenizer extends PreTrainedTokenizer { } +export class CodeLlamaTokenizer extends PreTrainedTokenizer {} -export class XLMRobertaTokenizer extends PreTrainedTokenizer { } -export class MPNetTokenizer extends PreTrainedTokenizer { } +export class XLMRobertaTokenizer extends PreTrainedTokenizer {} +export class MPNetTokenizer extends PreTrainedTokenizer {} -export class FalconTokenizer extends PreTrainedTokenizer { } +export class FalconTokenizer extends PreTrainedTokenizer {} -export class GPTNeoXTokenizer extends PreTrainedTokenizer { } +export class GPTNeoXTokenizer extends PreTrainedTokenizer {} -export class EsmTokenizer extends PreTrainedTokenizer { } +export class EsmTokenizer extends PreTrainedTokenizer {} -export class Qwen2Tokenizer extends PreTrainedTokenizer { } +export class Qwen2Tokenizer extends PreTrainedTokenizer {} -export class GemmaTokenizer extends PreTrainedTokenizer { } +export class GemmaTokenizer extends PreTrainedTokenizer {} -export class Grok1Tokenizer extends PreTrainedTokenizer { } +export class Grok1Tokenizer extends PreTrainedTokenizer {} /** * Helper function to build translation inputs for an `NllbTokenizer` or `M2M100Tokenizer`. @@ -3411,34 +3329,40 @@ export class Grok1Tokenizer extends PreTrainedTokenizer { } * @private */ function _build_translation_inputs(self, raw_inputs, tokenizer_options, generate_kwargs) { - if (!('language_codes' in self) || !Array.isArray(self.language_codes)) { - throw new Error('Tokenizer must have `language_codes` attribute set and it should be an array of language ids.') + if (!("language_codes" in self) || !Array.isArray(self.language_codes)) { + throw new Error( + "Tokenizer must have `language_codes` attribute set and it should be an array of language ids.", + ); } - if (!('languageRegex' in self) || !(self.languageRegex instanceof RegExp)) { - throw new Error('Tokenizer must have `languageRegex` attribute set and it should be a regular expression.') + if (!("languageRegex" in self) || !(self.languageRegex instanceof RegExp)) { + throw new Error("Tokenizer must have `languageRegex` attribute set and it should be a regular expression."); } - if (!('lang_to_token' in self) || typeof self.lang_to_token !== 'function') { - throw new Error('Tokenizer must have `lang_to_token` attribute set and it should be a function.') + if (!("lang_to_token" in self) || typeof self.lang_to_token !== "function") { + throw new Error("Tokenizer must have `lang_to_token` attribute set and it should be a function."); } const src_lang_token = generate_kwargs.src_lang; const tgt_lang_token = generate_kwargs.tgt_lang; // Check that the target language is valid: if (!self.language_codes.includes(tgt_lang_token)) { - throw new Error(`Target language code "${tgt_lang_token}" is not valid. Must be one of: {${self.language_codes.join(', ')}}`); + throw new Error( + `Target language code "${tgt_lang_token}" is not valid. Must be one of: {${self.language_codes.join(", ")}}`, + ); } // Allow `src_lang` to be optional. If not set, we'll use the tokenizer's default. if (src_lang_token !== undefined) { // Check that the source language is valid: if (!self.language_codes.includes(src_lang_token)) { - throw new Error(`Source language code "${src_lang_token}" is not valid. Must be one of: {${self.language_codes.join(', ')}}`); + throw new Error( + `Source language code "${src_lang_token}" is not valid. Must be one of: {${self.language_codes.join(", ")}}`, + ); } // In the same way as the Python library, we override the post-processor // to force the source language to be first: for (const item of self.post_processor.config.single) { - if ('SpecialToken' in item && self.languageRegex.test(item.SpecialToken.id)) { + if ("SpecialToken" in item && self.languageRegex.test(item.SpecialToken.id)) { item.SpecialToken.id = self.lang_to_token(src_lang_token); break; } @@ -3454,25 +3378,24 @@ function _build_translation_inputs(self, raw_inputs, tokenizer_options, generate /** * The NllbTokenizer class is used to tokenize text for NLLB ("No Language Left Behind") models. - * + * * No Language Left Behind (NLLB) is a first-of-its-kind, AI breakthrough project * that open-sources models capable of delivering high-quality translations directly * between any pair of 200+ languages — including low-resource languages like Asturian, * Luganda, Urdu and more. It aims to help people communicate with anyone, anywhere, * regardless of their language preferences. For more information, check out their * [paper](https://arxiv.org/abs/2207.04672). - * + * * For a list of supported languages (along with their language codes), * @see {@link https://github.com/facebookresearch/flores/blob/main/flores200/README.md#languages-in-flores-200} */ export class NllbTokenizer extends PreTrainedTokenizer { - constructor(tokenizerJSON, tokenizerConfig) { super(tokenizerJSON, tokenizerConfig); this.languageRegex = /^[a-z]{3}_[A-Z][a-z]{3}$/; - this.language_codes = this.special_tokens.filter(x => this.languageRegex.test(x)); - this.lang_to_token = x => x; // Identity function + this.language_codes = this.special_tokens.filter((x) => this.languageRegex.test(x)); + this.lang_to_token = (x) => x; // Identity function } /** @@ -3489,11 +3412,11 @@ export class NllbTokenizer extends PreTrainedTokenizer { /** * The M2M100Tokenizer class is used to tokenize text for M2M100 ("Many-to-Many") models. - * + * * M2M100 is a multilingual encoder-decoder (seq-to-seq) model trained for Many-to-Many * multilingual translation. It was introduced in this [paper](https://arxiv.org/abs/2010.11125) * and first released in [this](https://github.com/pytorch/fairseq/tree/master/examples/m2m_100) repository. - * + * * For a list of supported languages (along with their language codes), * @see {@link https://huggingface.co/facebook/m2m100_418M#languages-covered} */ @@ -3502,10 +3425,8 @@ export class M2M100Tokenizer extends PreTrainedTokenizer { super(tokenizerJSON, tokenizerConfig); this.languageRegex = /^__[a-z]{2,3}__$/; - this.language_codes = this.special_tokens - .filter(x => this.languageRegex.test(x)) - .map(x => x.slice(2, -2)); - this.lang_to_token = x => `__${x}__`; + this.language_codes = this.special_tokens.filter((x) => this.languageRegex.test(x)).map((x) => x.slice(2, -2)); + this.lang_to_token = (x) => `__${x}__`; } /** @@ -3525,7 +3446,6 @@ export class M2M100Tokenizer extends PreTrainedTokenizer { * @extends PreTrainedTokenizer */ export class WhisperTokenizer extends PreTrainedTokenizer { - get timestamp_begin() { return this.model.convert_tokens_to_ids(["<|notimestamps|>"])[0] + 1; } @@ -3536,12 +3456,10 @@ export class WhisperTokenizer extends PreTrainedTokenizer { * @param {Object} options The options to use for decoding. * @returns {Array, text: string}>}>} The decoded sequences. */ - _decode_asr(sequences, { - return_timestamps = false, - return_language = false, - time_precision = null, - force_full_sequences = true - } = {}) { + _decode_asr( + sequences, + { return_timestamps = false, return_language = false, time_precision = null, force_full_sequences = true } = {}, + ) { // Set force_full_sequences=false if you want streaming // TODO add support for `return_language` @@ -3562,14 +3480,14 @@ export class WhisperTokenizer extends PreTrainedTokenizer { // - Lots of complexity comes from stride and timestamps if (time_precision === null) { - throw Error("Must specify time_precision") + throw Error("Must specify time_precision"); } let last_language = null; const returnWordTimestamps = return_timestamps === "word"; function new_chunk() { - return { "language": last_language, "timestamp": [null, null], "text": "" }; + return { language: last_language, timestamp: [null, null], text: "" }; } // Welcome to the state machine! @@ -3584,7 +3502,6 @@ export class WhisperTokenizer extends PreTrainedTokenizer { let skip = false; let right_stride_start = null; - const all_special_ids = new Set(this.all_special_ids); for (const output of sequences) { @@ -3618,7 +3535,10 @@ export class WhisperTokenizer extends PreTrainedTokenizer { if (token >= timestamp_begin) { // There can be several token in the right stride // But the last one is ALWAYS going to be skipped - if (last_timestamp !== null && (token - timestamp_begin) * time_precision < right_stride_start) { + if ( + last_timestamp !== null && + (token - timestamp_begin) * time_precision < right_stride_start + ) { break; } last_timestamp = token; @@ -3692,40 +3612,42 @@ export class WhisperTokenizer extends PreTrainedTokenizer { chunk.timestamp[1] = rounded_time; // Handling merges - previous_tokens.push(current_tokens) + previous_tokens.push(current_tokens); if (returnWordTimestamps) { previous_token_timestamps.push(current_token_timestamps); } const [resolved_tokens, resolved_token_timestamps] = this.findLongestCommonSequence( - previous_tokens, previous_token_timestamps - ) + previous_tokens, + previous_token_timestamps, + ); - const resolved_text = this.decode(resolved_tokens) - chunk.text = resolved_text + const resolved_text = this.decode(resolved_tokens); + chunk.text = resolved_text; if (returnWordTimestamps) { chunk.words = this.collateWordTimestamps( - resolved_tokens, resolved_token_timestamps, last_language, - ) + resolved_tokens, + resolved_token_timestamps, + last_language, + ); } - chunks.push(chunk) + chunks.push(chunk); // Flush all our temporary context - previous_tokens = [] - current_tokens = [] - previous_token_timestamps = [] - current_token_timestamps = [] - chunk = new_chunk() + previous_tokens = []; + current_tokens = []; + previous_token_timestamps = []; + current_token_timestamps = []; + chunk = new_chunk(); } } - } else { // 4/ Regular token // We just append to the list of all tokens so we can handle // merges later and decode into text. - current_tokens.push(token) + current_tokens.push(token); if (returnWordTimestamps) { let start_time = round(token_timestamps[i] + time_offset, 2); @@ -3747,30 +3669,28 @@ export class WhisperTokenizer extends PreTrainedTokenizer { } current_token_timestamps.push([start_time, end_time]); } - } } - if ('stride' in output) { + if ("stride" in output) { const [chunk_len, stride_left, stride_right] = output.stride; - time_offset += chunk_len - stride_right + time_offset += chunk_len - stride_right; } // Leftover tokens if (current_tokens.length > 0) { - previous_tokens.push(current_tokens) + previous_tokens.push(current_tokens); if (returnWordTimestamps) { previous_token_timestamps.push(current_token_timestamps); } - } else if (previous_tokens.every(p => p.length === 0)) { + } else if (previous_tokens.every((p) => p.length === 0)) { // Flushing previous tokens (END)" - chunk = new_chunk() - previous_tokens = [] - current_tokens = [] + chunk = new_chunk(); + previous_tokens = []; + current_tokens = []; previous_token_timestamps = []; current_token_timestamps = []; } - } if (previous_tokens.length > 0) { @@ -3779,20 +3699,21 @@ export class WhisperTokenizer extends PreTrainedTokenizer { // leftover throw new Error( "Whisper did not predict an ending timestamp, which can happen if audio is cut off in the middle of a word. " + - "Also make sure WhisperTimeStampLogitsProcessor was used during generation." + "Also make sure WhisperTimeStampLogitsProcessor was used during generation.", ); } // Happens when we don't use timestamps - const [resolved_tokens, resolved_token_timestamps] = this.findLongestCommonSequence(previous_tokens, previous_token_timestamps); + const [resolved_tokens, resolved_token_timestamps] = this.findLongestCommonSequence( + previous_tokens, + previous_token_timestamps, + ); // Flushing previous tokens (FINAL) const resolved_text = this.decode(resolved_tokens); chunk.text = resolved_text; if (returnWordTimestamps) { - chunk.words = this.collateWordTimestamps( - resolved_tokens, resolved_token_timestamps, last_language, - ) + chunk.words = this.collateWordTimestamps(resolved_tokens, resolved_token_timestamps, last_language); } chunks.push(chunk); } @@ -3800,7 +3721,7 @@ export class WhisperTokenizer extends PreTrainedTokenizer { let optional = Object.create(null); // Preparing and cleaning up the pipeline output - const full_text = chunks.map(chunk => chunk.text).join(''); + const full_text = chunks.map((chunk) => chunk.text).join(""); if (return_timestamps || return_language) { for (let i = 0; i < chunks.length; ++i) { const chunk = chunks[i]; @@ -3819,13 +3740,12 @@ export class WhisperTokenizer extends PreTrainedTokenizer { new_chunks.push(word); } } - optional = { "chunks": new_chunks }; + optional = { chunks: new_chunks }; } else { - optional = { "chunks": chunks }; + optional = { chunks: chunks }; } } return [full_text, optional]; - } /** @@ -3845,7 +3765,8 @@ export class WhisperTokenizer extends PreTrainedTokenizer { let leftLength = leftSequence.length; let totalSequence = []; - const use_token_timestamp_sequences = Array.isArray(token_timestamp_sequences) && token_timestamp_sequences.length > 0; + const use_token_timestamp_sequences = + Array.isArray(token_timestamp_sequences) && token_timestamp_sequences.length > 0; let total_token_timestamp_sequence = use_token_timestamp_sequences ? [] : null; let left_token_timestamp_sequence = use_token_timestamp_sequences ? token_timestamp_sequences[0] : null; for (let i = 1; i < sequences.length; ++i) { @@ -3861,7 +3782,6 @@ export class WhisperTokenizer extends PreTrainedTokenizer { // [c, d, f] // = [c, d] == [c, d] - // [a, b, c, d] // [c, d, f] @@ -3894,17 +3814,21 @@ export class WhisperTokenizer extends PreTrainedTokenizer { const rightStop = Math.min(rightLength, j); const right = rightSequence.slice(rightStart, rightStop); if (left.length !== right.length) { - throw new Error("There is a bug within whisper `decode_asr` function, please report it. Dropping to prevent bad inference."); + throw new Error( + "There is a bug within whisper `decode_asr` function, please report it. Dropping to prevent bad inference.", + ); } let matches; if (use_token_timestamp_sequences) { // Get length of longest subsequence of tokens that match // and have timestamps that are in order - matches = left.filter((elem, idx) => ( - elem === right[idx] - && left_token_timestamp_sequence[leftStart + idx] <= token_timestamp_sequences[i][rightStart + idx] - )).length; + matches = left.filter( + (elem, idx) => + elem === right[idx] && + left_token_timestamp_sequence[leftStart + idx] <= + token_timestamp_sequences[i][rightStart + idx], + ).length; } else { matches = left.filter((elem, idx) => elem === right[idx]).length; } @@ -3941,7 +3865,6 @@ export class WhisperTokenizer extends PreTrainedTokenizer { /** @private */ collateWordTimestamps(tokens, token_timestamps, language) { - const [words, _, token_indices] = this.combineTokensIntoWords(tokens, language); const timings = []; @@ -3949,10 +3872,7 @@ export class WhisperTokenizer extends PreTrainedTokenizer { const indices = token_indices[i]; timings.push({ text: words[i], - timestamp: [ - token_timestamps[indices.at(0)][0], - token_timestamps[indices.at(-1)][1], - ], + timestamp: [token_timestamps[indices.at(0)][0], token_timestamps[indices.at(-1)][1]], }); } return timings; @@ -3961,33 +3881,35 @@ export class WhisperTokenizer extends PreTrainedTokenizer { /** * Groups tokens by word. Returns a tuple containing a list of strings with the words, * and a list of `token_id` sequences with the tokens making up each word. - * @param {number[]} tokens - * @param {string} [language] - * @param {string} prepend_punctionations - * @param {string} append_punctuations - * + * @param {number[]} tokens + * @param {string} [language] + * @param {string} prepend_punctionations + * @param {string} append_punctuations + * * @private */ - combineTokensIntoWords(tokens, language, prepend_punctionations = "\"'“¡¿([{-", append_punctuations = "\"'.。,，!！?？:：”)]}、") { - language = language ?? 'english'; + combineTokensIntoWords( + tokens, + language, + prepend_punctionations = "\"'“¡¿([{-", + append_punctuations = "\"'.。,，!！?？:：”)]}、", + ) { + language = language ?? "english"; let words, word_tokens, token_indices; if (["chinese", "japanese", "thai", "lao", "myanmar"].includes(language)) { // These languages don't typically use spaces. - [words, word_tokens, token_indices] = this.splitTokensOnUnicode(tokens) + [words, word_tokens, token_indices] = this.splitTokensOnUnicode(tokens); } else { - [words, word_tokens, token_indices] = this.splitTokensOnSpaces(tokens) + [words, word_tokens, token_indices] = this.splitTokensOnSpaces(tokens); } return this.mergePunctuations(words, word_tokens, token_indices, prepend_punctionations, append_punctuations); } /** @type {PreTrainedTokenizer['decode']} */ - decode( - token_ids, - decode_args, - ) { + decode(token_ids, decode_args) { let text; // @ts-ignore if (decode_args?.decode_with_timestamps) { @@ -4026,16 +3948,14 @@ export class WhisperTokenizer extends PreTrainedTokenizer { outputs[outputs.length - 1].push(token); } } - outputs = outputs.map( - s => typeof s === 'string' ? s : super.decode(s, decode_args) - ) + outputs = outputs.map((s) => (typeof s === "string" ? s : super.decode(s, decode_args))); - return outputs.join(''); + return outputs.join(""); } /** * Combine tokens into words by splitting at any position where the tokens are decoded as valid unicode points. - * @param {number[]} tokens + * @param {number[]} tokens * @returns {*} * @private */ @@ -4044,14 +3964,14 @@ export class WhisperTokenizer extends PreTrainedTokenizer { // @ts-ignore decode_with_timestamps: true, }); - const replacement_char = '\uFFFD'; + const replacement_char = "\uFFFD"; - const words = [] - const word_tokens = [] - const token_indices = [] - let current_tokens = [] - let current_indices = [] - let unicode_offset = 0 + const words = []; + const word_tokens = []; + const token_indices = []; + let current_tokens = []; + let current_indices = []; + let unicode_offset = 0; for (let token_idx = 0; token_idx < tokens.length; ++token_idx) { const token = tokens[token_idx]; @@ -4064,44 +3984,44 @@ export class WhisperTokenizer extends PreTrainedTokenizer { decode_with_timestamps: true, }); - if (!decoded.includes(replacement_char) || decoded_full[unicode_offset + decoded.indexOf(replacement_char)] === replacement_char) { - words.push(decoded) - word_tokens.push(current_tokens) - token_indices.push(current_indices) - current_tokens = [] - current_indices = [] + if ( + !decoded.includes(replacement_char) || + decoded_full[unicode_offset + decoded.indexOf(replacement_char)] === replacement_char + ) { + words.push(decoded); + word_tokens.push(current_tokens); + token_indices.push(current_indices); + current_tokens = []; + current_indices = []; unicode_offset += decoded.length; } - } - return [words, word_tokens, token_indices] + return [words, word_tokens, token_indices]; } /** * Combine tokens into words by splitting at whitespace and punctuation tokens. - * @param {number[]} tokens + * @param {number[]} tokens * @private */ splitTokensOnSpaces(tokens) { - const [subwords, subword_tokens_list, subword_indices_list] = this.splitTokensOnUnicode(tokens); - const words = [] - const word_tokens = [] - const token_indices = [] + const words = []; + const word_tokens = []; + const token_indices = []; - const punctuationRegex = new RegExp(`^[${PUNCTUATION_REGEX}]$`, 'gu'); + const punctuationRegex = new RegExp(`^[${PUNCTUATION_REGEX}]$`, "gu"); for (let i = 0; i < subwords.length; ++i) { - const subword = subwords[i]; const subword_tokens = subword_tokens_list[i]; const subword_indices = subword_indices_list[i]; // @ts-ignore - const special = subword_tokens[0] >= this.model.tokens_to_ids.get('<|endoftext|>'); - const with_space = subword.startsWith(' '); + const special = subword_tokens[0] >= this.model.tokens_to_ids.get("<|endoftext|>"); + const with_space = subword.startsWith(" "); const trimmed = subword.trim(); const punctuation = punctuationRegex.test(trimmed); @@ -4118,35 +4038,32 @@ export class WhisperTokenizer extends PreTrainedTokenizer { } return [words, word_tokens, token_indices]; - } /** * Merges punctuation tokens with neighboring words. - * @param {string[]} words - * @param {number[][]} tokens - * @param {number[][]} indices - * @param {string} prepended - * @param {string} appended + * @param {string[]} words + * @param {number[][]} tokens + * @param {number[][]} indices + * @param {string} prepended + * @param {string} appended * @private */ mergePunctuations(words, tokens, indices, prepended, appended) { - const newWords = structuredClone(words); const newTokens = structuredClone(tokens); const newIndices = structuredClone(indices); - // prepend punctuations let i = newWords.length - 2; let j = newWords.length - 1; while (i >= 0) { - if (newWords[i].startsWith(' ') && prepended.includes(newWords[i].trim())) { + if (newWords[i].startsWith(" ") && prepended.includes(newWords[i].trim())) { newWords[j] = newWords[i] + newWords[j]; newTokens[j] = mergeArrays(newTokens[i], newTokens[j]); newIndices[j] = mergeArrays(newIndices[i], newIndices[j]); - newWords[i] = ''; + newWords[i] = ""; newTokens[i] = []; newIndices[i] = []; } else { @@ -4159,11 +4076,11 @@ export class WhisperTokenizer extends PreTrainedTokenizer { i = 0; j = 1; while (j < newWords.length) { - if (!newWords[i].endsWith(' ') && appended.includes(newWords[j])) { + if (!newWords[i].endsWith(" ") && appended.includes(newWords[j])) { newWords[i] += newWords[j]; newTokens[i] = mergeArrays(newTokens[i], newTokens[j]); newIndices[i] = mergeArrays(newIndices[i], newIndices[j]); - newWords[j] = ''; + newWords[j] = ""; newTokens[j] = []; newIndices[j] = []; } else { @@ -4173,15 +4090,15 @@ export class WhisperTokenizer extends PreTrainedTokenizer { } return [ - newWords.filter(x => x), - newTokens.filter(x => x.length > 0), - newIndices.filter(x => x.length > 0), - ] + newWords.filter((x) => x), + newTokens.filter((x) => x.length > 0), + newIndices.filter((x) => x.length > 0), + ]; } } -export class CodeGenTokenizer extends PreTrainedTokenizer { } -export class CLIPTokenizer extends PreTrainedTokenizer { } -export class SiglipTokenizer extends PreTrainedTokenizer { } +export class CodeGenTokenizer extends PreTrainedTokenizer {} +export class CLIPTokenizer extends PreTrainedTokenizer {} +export class SiglipTokenizer extends PreTrainedTokenizer {} /** * @todo This model is not yet supported by Hugging Face's "fast" tokenizers library (https://github.com/huggingface/tokenizers). @@ -4198,11 +4115,11 @@ export class MarianTokenizer extends PreTrainedTokenizer { this.languageRegex = /^(>>\w+<<)\s*/g; - this.supported_language_codes = this.model.vocab.filter( - x => this.languageRegex.test(x) - ); + this.supported_language_codes = this.model.vocab.filter((x) => this.languageRegex.test(x)); - console.warn('WARNING: `MarianTokenizer` is not yet supported by Hugging Face\'s "fast" tokenizers library. Therefore, you may experience slightly inaccurate results.') + console.warn( + 'WARNING: `MarianTokenizer` is not yet supported by Hugging Face\'s "fast" tokenizers library. Therefore, you may experience slightly inaccurate results.', + ); } /** @@ -4222,31 +4139,30 @@ export class MarianTokenizer extends PreTrainedTokenizer { if (remainder.length === 0) { // No language code, encode normally return super._encode_text(matchInfo); - } else if (remainder.length === 2) { // Text starts with language code, so we do not encode it with sentencepiece. const [language, text] = remainder; if (!this.supported_language_codes.includes(language)) { - console.warn(`Unsupported language code "${language}" detected, which may lead to unexpected behavior. Should be one of: ${JSON.stringify(this.supported_language_codes)}`) + console.warn( + `Unsupported language code "${language}" detected, which may lead to unexpected behavior. Should be one of: ${JSON.stringify(this.supported_language_codes)}`, + ); } return mergeArrays([language], super._encode_text(text)); } } - } -export class Wav2Vec2CTCTokenizer extends PreTrainedTokenizer { } +export class Wav2Vec2CTCTokenizer extends PreTrainedTokenizer {} -export class BlenderbotTokenizer extends PreTrainedTokenizer { } -export class BlenderbotSmallTokenizer extends PreTrainedTokenizer { } +export class BlenderbotTokenizer extends PreTrainedTokenizer {} +export class BlenderbotSmallTokenizer extends PreTrainedTokenizer {} -export class SpeechT5Tokenizer extends PreTrainedTokenizer { } +export class SpeechT5Tokenizer extends PreTrainedTokenizer {} -export class NougatTokenizer extends PreTrainedTokenizer { } +export class NougatTokenizer extends PreTrainedTokenizer {} export class VitsTokenizer extends PreTrainedTokenizer { - constructor(tokenizerJSON, tokenizerConfig) { super(tokenizerJSON, tokenizerConfig); @@ -4255,12 +4171,12 @@ export class VitsTokenizer extends PreTrainedTokenizer { } } -export class CohereTokenizer extends PreTrainedTokenizer { } +export class CohereTokenizer extends PreTrainedTokenizer {} /** * Helper class which is used to instantiate pretrained tokenizers with the `from_pretrained` function. * The chosen tokenizer class is determined by the type specified in the tokenizer config. - * + * * @example * const tokenizer = await AutoTokenizer.from_pretrained('Xenova/bert-base-uncased'); */ @@ -4313,33 +4229,34 @@ export class AutoTokenizer { // Base case: PreTrainedTokenizer, - } - + }; /** * Instantiate one of the tokenizer classes of the library from a pretrained model. - * + * * The tokenizer class to instantiate is selected based on the `tokenizer_class` property of the config object * (either passed as an argument or loaded from `pretrained_model_name_or_path` if possible) - * + * * @param {string} pretrained_model_name_or_path The name or path of the pretrained model. Can be either: * - A string, the *model id* of a pretrained tokenizer hosted inside a model repo on huggingface.co. * Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a * user or organization name, like `dbmdz/bert-base-german-cased`. * - A path to a *directory* containing tokenizer files, e.g., `./my_model_directory/`. * @param {PretrainedTokenizerOptions} options Additional options for loading the tokenizer. - * + * * @returns {Promise} A new instance of the PreTrainedTokenizer class. */ - static async from_pretrained(pretrained_model_name_or_path, { - progress_callback = null, - config = null, - cache_dir = null, - local_files_only = false, - revision = 'main', - legacy = null, - } = {}) { - + static async from_pretrained( + pretrained_model_name_or_path, + { + progress_callback = null, + config = null, + cache_dir = null, + local_files_only = false, + revision = "main", + legacy = null, + } = {}, + ) { const [tokenizerJSON, tokenizerConfig] = await loadTokenizer(pretrained_model_name_or_path, { progress_callback, config, @@ -4347,10 +4264,10 @@ export class AutoTokenizer { local_files_only, revision, legacy, - }) + }); // Some tokenizers are saved with the "Fast" suffix, so we remove that if present. - const tokenizerName = tokenizerConfig.tokenizer_class?.replace(/Fast$/, '') ?? 'PreTrainedTokenizer'; + const tokenizerName = tokenizerConfig.tokenizer_class?.replace(/Fast$/, "") ?? "PreTrainedTokenizer"; let cls = this.TOKENIZER_CLASS_MAPPING[tokenizerName]; if (!cls) { diff --git a/src/transformers.js b/src/transformers.js index be7ad176e..9ace2884f 100644 --- a/src/transformers.js +++ b/src/transformers.js @@ -1,28 +1,27 @@ /** * @file Entry point for the Transformers.js library. Only the exports from this file * are available to the end user, and are grouped as follows: - * + * * 1. [Pipelines](./pipelines) * 2. [Environment variables](./env) * 3. [Models](./models) * 4. [Tokenizers](./tokenizers) * 5. [Processors](./processors) - * + * * @module transformers */ -export { env } from './env.js'; -export * from './pipelines.js'; -export * from './models.js'; -export * from './tokenizers.js'; -export * from './processors.js'; -export * from './configs.js'; +export { env } from "./env.js"; +export * from "./pipelines.js"; +export * from "./models.js"; +export * from "./tokenizers.js"; +export * from "./processors.js"; +export * from "./configs.js"; -export * from './utils/audio.js'; -export * from './utils/image.js'; -export * from './utils/tensor.js'; -export * from './utils/maths.js'; - -export * from './generation/streamers.js'; -export * from './generation/stopping_criteria.js'; +export * from "./utils/audio.js"; +export * from "./utils/image.js"; +export * from "./utils/tensor.js"; +export * from "./utils/maths.js"; +export * from "./generation/streamers.js"; +export * from "./generation/stopping_criteria.js"; diff --git a/src/utils/audio.js b/src/utils/audio.js index a1b1326df..4f3c06f03 100644 --- a/src/utils/audio.js +++ b/src/utils/audio.js @@ -1,21 +1,16 @@ /** - * @file Helper module for audio processing. - * - * These functions and classes are only used internally, + * @file Helper module for audio processing. + * + * These functions and classes are only used internally, * meaning an end-user shouldn't need to access anything here. - * + * * @module utils/audio */ -import { - getFile, -} from './hub.js'; -import { FFT, max } from './maths.js'; -import { - calculateReflectOffset, -} from './core.js'; -import { Tensor, matmul } from './tensor.js'; - +import { getFile } from "./hub.js"; +import { FFT, max } from "./maths.js"; +import { calculateReflectOffset } from "./core.js"; +import { Tensor, matmul } from "./tensor.js"; /** * Helper function to read audio from a path/URL. @@ -24,19 +19,19 @@ import { Tensor, matmul } from './tensor.js'; * @returns {Promise} The decoded audio as a `Float32Array`. */ export async function read_audio(url, sampling_rate) { - if (typeof AudioContext === 'undefined') { + if (typeof AudioContext === "undefined") { // Running in node or an environment without AudioContext throw Error( "Unable to load audio from path/URL since `AudioContext` is not available in your environment. " + - "Instead, audio data should be passed directly to the pipeline/processor. " + - "For more information and some example code, see https://huggingface.co/docs/transformers.js/guides/node-audio-processing." - ) + "Instead, audio data should be passed directly to the pipeline/processor. " + + "For more information and some example code, see https://huggingface.co/docs/transformers.js/guides/node-audio-processing.", + ); } const response = await (await getFile(url)).arrayBuffer(); const audioCTX = new AudioContext({ sampleRate: sampling_rate }); - if (typeof sampling_rate === 'undefined') { - console.warn(`No sampling rate provided, using default of ${audioCTX.sampleRate}Hz.`) + if (typeof sampling_rate === "undefined") { + console.warn(`No sampling rate provided, using default of ${audioCTX.sampleRate}Hz.`); } const decoded = await audioCTX.decodeAudioData(response); @@ -67,9 +62,8 @@ export async function read_audio(url, sampling_rate) { audio = new Float32Array(left.length); for (let i = 0; i < decoded.length; ++i) { - audio[i] = SCALING_FACTOR * (left[i] + right[i]) / 2; + audio[i] = (SCALING_FACTOR * (left[i] + right[i])) / 2; } - } else { // If the audio is not stereo, we can just use the first channel: audio = decoded.getChannelData(0); @@ -94,7 +88,7 @@ function generalized_cosine_window(M, a_0) { } const a_1 = 1 - a_0; - const factor = 2 * Math.PI / (M - 1); + const factor = (2 * Math.PI) / (M - 1); const cos_vals = new Float64Array(M); for (let i = 0; i < M; ++i) { @@ -114,7 +108,6 @@ export function hanning(M) { return generalized_cosine_window(M, 0.5); } - /** * Generates a Hamming window of length M. * See https://numpy.org/doc/stable/reference/generated/numpy.hamming.html for more information. @@ -126,19 +119,16 @@ export function hamming(M) { return generalized_cosine_window(M, 0.54); } - const HERTZ_TO_MEL_MAPPING = { - "htk": (/** @type {number} */ freq) => 2595.0 * Math.log10(1.0 + (freq / 700.0)), - "kaldi": (/** @type {number} */ freq) => 1127.0 * Math.log(1.0 + (freq / 700.0)), - "slaney": (/** @type {number} */ freq, min_log_hertz = 1000.0, min_log_mel = 15.0, logstep = 27.0 / Math.log(6.4)) => - freq >= min_log_hertz - ? min_log_mel + Math.log(freq / min_log_hertz) * logstep - : 3.0 * freq / 200.0, -} + htk: (/** @type {number} */ freq) => 2595.0 * Math.log10(1.0 + freq / 700.0), + kaldi: (/** @type {number} */ freq) => 1127.0 * Math.log(1.0 + freq / 700.0), + slaney: (/** @type {number} */ freq, min_log_hertz = 1000.0, min_log_mel = 15.0, logstep = 27.0 / Math.log(6.4)) => + freq >= min_log_hertz ? min_log_mel + Math.log(freq / min_log_hertz) * logstep : (3.0 * freq) / 200.0, +}; /** - * @template {Float32Array|Float64Array|number} T - * @param {T} freq + * @template {Float32Array|Float64Array|number} T + * @param {T} freq * @param {string} [mel_scale] * @returns {T} */ @@ -148,20 +138,19 @@ function hertz_to_mel(freq, mel_scale = "htk") { throw new Error('mel_scale should be one of "htk", "slaney" or "kaldi".'); } - return typeof freq === 'number' ? fn(freq) : freq.map(x => fn(x)); + return typeof freq === "number" ? fn(freq) : freq.map((x) => fn(x)); } const MEL_TO_HERTZ_MAPPING = { - "htk": (/** @type {number} */ mels) => 700.0 * (10.0 ** (mels / 2595.0) - 1.0), - "kaldi": (/** @type {number} */ mels) => 700.0 * (Math.exp(mels / 1127.0) - 1.0), - "slaney": (/** @type {number} */ mels, min_log_hertz = 1000.0, min_log_mel = 15.0, logstep = Math.log(6.4) / 27.0) => mels >= min_log_mel - ? min_log_hertz * Math.exp(logstep * (mels - min_log_mel)) - : 200.0 * mels / 3.0, -} + htk: (/** @type {number} */ mels) => 700.0 * (10.0 ** (mels / 2595.0) - 1.0), + kaldi: (/** @type {number} */ mels) => 700.0 * (Math.exp(mels / 1127.0) - 1.0), + slaney: (/** @type {number} */ mels, min_log_hertz = 1000.0, min_log_mel = 15.0, logstep = Math.log(6.4) / 27.0) => + mels >= min_log_mel ? min_log_hertz * Math.exp(logstep * (mels - min_log_mel)) : (200.0 * mels) / 3.0, +}; /** - * @template {Float32Array|Float64Array|number} T - * @param {T} mels + * @template {Float32Array|Float64Array|number} T + * @param {T} mels * @param {string} [mel_scale] * @returns {T} */ @@ -171,27 +160,30 @@ function mel_to_hertz(mels, mel_scale = "htk") { throw new Error('mel_scale should be one of "htk", "slaney" or "kaldi".'); } - return typeof mels === 'number' ? fn(mels) : mels.map(x => fn(x)); + return typeof mels === "number" ? fn(mels) : mels.map((x) => fn(x)); } /** -* Creates a triangular filter bank. -* -* Adapted from torchaudio and librosa. -* -* @param {Float64Array} fft_freqs Discrete frequencies of the FFT bins in Hz, of shape `(num_frequency_bins,)`. -* @param {Float64Array} filter_freqs Center frequencies of the triangular filters to create, in Hz, of shape `(num_mel_filters,)`. -* @returns {number[][]} of shape `(num_frequency_bins, num_mel_filters)`. -*/ + * Creates a triangular filter bank. + * + * Adapted from torchaudio and librosa. + * + * @param {Float64Array} fft_freqs Discrete frequencies of the FFT bins in Hz, of shape `(num_frequency_bins,)`. + * @param {Float64Array} filter_freqs Center frequencies of the triangular filters to create, in Hz, of shape `(num_mel_filters,)`. + * @returns {number[][]} of shape `(num_frequency_bins, num_mel_filters)`. + */ function _create_triangular_filter_bank(fft_freqs, filter_freqs) { const filter_diff = Float64Array.from( { length: filter_freqs.length - 1 }, - (_, i) => filter_freqs[i + 1] - filter_freqs[i] + (_, i) => filter_freqs[i + 1] - filter_freqs[i], ); - const slopes = Array.from({ - length: fft_freqs.length - }, () => new Array(filter_freqs.length)); + const slopes = Array.from( + { + length: fft_freqs.length, + }, + () => new Array(filter_freqs.length), + ); for (let j = 0; j < fft_freqs.length; ++j) { const slope = slopes[j]; @@ -203,9 +195,11 @@ function _create_triangular_filter_bank(fft_freqs, filter_freqs) { const numFreqs = filter_freqs.length - 2; const ret = Array.from({ length: numFreqs }, () => new Array(fft_freqs.length)); - for (let j = 0; j < fft_freqs.length; ++j) { // 201 + for (let j = 0; j < fft_freqs.length; ++j) { + // 201 const slope = slopes[j]; - for (let i = 0; i < numFreqs; ++i) { // 80 + for (let i = 0; i < numFreqs; ++i) { + // 80 const down = -slope[i] / filter_diff[i]; const up = slope[i + 2] / filter_diff[i + 1]; ret[i][j] = Math.max(0, Math.min(down, up)); @@ -266,7 +260,10 @@ export function mel_filter_bank( if (triangularize_in_mel_space) { const fft_bin_width = sampling_rate / (num_frequency_bins * 2); - fft_freqs = hertz_to_mel(Float64Array.from({ length: num_frequency_bins }, (_, i) => i * fft_bin_width), mel_scale); + fft_freqs = hertz_to_mel( + Float64Array.from({ length: num_frequency_bins }, (_, i) => i * fft_bin_width), + mel_scale, + ); filter_freqs = mel_freqs; } else { fft_freqs = linspace(0, Math.floor(sampling_rate / 2), num_frequency_bins); @@ -289,7 +286,6 @@ export function mel_filter_bank( // TODO warn if there is a zero row return mel_filters; - } /** @@ -323,32 +319,32 @@ function padReflect(array, left, right) { /** * Helper function to compute `amplitude_to_db` and `power_to_db`. * @template {Float32Array|Float64Array} T - * @param {T} spectrogram - * @param {number} factor - * @param {number} reference - * @param {number} min_value - * @param {number} db_range + * @param {T} spectrogram + * @param {number} factor + * @param {number} reference + * @param {number} min_value + * @param {number} db_range * @returns {T} */ function _db_conversion_helper(spectrogram, factor, reference, min_value, db_range) { if (reference <= 0) { - throw new Error('reference must be greater than zero'); + throw new Error("reference must be greater than zero"); } if (min_value <= 0) { - throw new Error('min_value must be greater than zero'); + throw new Error("min_value must be greater than zero"); } reference = Math.max(min_value, reference); const logReference = Math.log10(reference); for (let i = 0; i < spectrogram.length; ++i) { - spectrogram[i] = factor * Math.log10(Math.max(min_value, spectrogram[i]) - logReference) + spectrogram[i] = factor * Math.log10(Math.max(min_value, spectrogram[i]) - logReference); } if (db_range !== null) { if (db_range <= 0) { - throw new Error('db_range must be greater than zero'); + throw new Error("db_range must be greater than zero"); } const maxValue = max(spectrogram)[0] - db_range; for (let i = 0; i < spectrogram.length; ++i) { @@ -362,12 +358,12 @@ function _db_conversion_helper(spectrogram, factor, reference, min_value, db_ran /** * Converts an amplitude spectrogram to the decibel scale. This computes `20 * log10(spectrogram / reference)`, * using basic logarithm properties for numerical stability. NOTE: Operates in-place. - * + * * The motivation behind applying the log function on the (mel) spectrogram is that humans do not hear loudness on a * linear scale. Generally to double the perceived volume of a sound we need to put 8 times as much energy into it. * This means that large variations in energy may not sound all that different if the sound is loud to begin with. * This compression operation makes the (mel) spectrogram features match more closely what humans actually hear. - * + * * @template {Float32Array|Float64Array} T * @param {T} spectrogram The input amplitude (mel) spectrogram. * @param {number} [reference=1.0] Sets the input spectrogram value that corresponds to 0 dB. @@ -385,14 +381,14 @@ function amplitude_to_db(spectrogram, reference = 1.0, min_value = 1e-5, db_rang /** * Converts a power spectrogram to the decibel scale. This computes `10 * log10(spectrogram / reference)`, * using basic logarithm properties for numerical stability. NOTE: Operates in-place. - * + * * The motivation behind applying the log function on the (mel) spectrogram is that humans do not hear loudness on a * linear scale. Generally to double the perceived volume of a sound we need to put 8 times as much energy into it. * This means that large variations in energy may not sound all that different if the sound is loud to begin with. * This compression operation makes the (mel) spectrogram features match more closely what humans actually hear. - * + * * Based on the implementation of `librosa.power_to_db`. - * + * * @template {Float32Array|Float64Array} T * @param {T} spectrogram The input power (mel) spectrogram. Note that a power spectrogram has the amplitudes squared! * @param {number} [reference=1.0] Sets the input spectrogram value that corresponds to 0 dB. @@ -409,7 +405,7 @@ function power_to_db(spectrogram, reference = 1.0, min_value = 1e-10, db_range = /** * Calculates a spectrogram over one waveform using the Short-Time Fourier Transform. - * + * * This function can create the following kinds of spectrograms: * - amplitude spectrogram (`power = 1.0`) * - power spectrogram (`power = 2.0`) @@ -419,9 +415,9 @@ function power_to_db(spectrogram, reference = 1.0, min_value = 1e-10, db_range = * - log-mel spectrogram (provide `mel_filters` and `log_mel`) * * In this implementation, the window is assumed to be zero-padded to have the same size as the analysis frame. - * A padded window can be obtained from `window_function()`. The FFT input buffer may be larger than the analysis frame, + * A padded window can be obtained from `window_function()`. The FFT input buffer may be larger than the analysis frame, * typically the next power of two. - * + * * @param {Float32Array|Float64Array} waveform The input waveform of shape `(length,)`. This must be a single real-valued, mono waveform. * @param {Float32Array|Float64Array} window The windowing function to apply of shape `(frame_length,)`, including zero-padding if necessary. The actual window length may be * shorter than `frame_length`, but we're assuming the array has already been zero-padded. @@ -484,14 +480,14 @@ export async function spectrogram( max_num_frames = null, do_pad = true, transpose = false, - } = {} + } = {}, ) { const window_length = window.length; if (fft_length === null) { fft_length = frame_length; } if (frame_length > fft_length) { - throw Error(`frame_length (${frame_length}) may not be larger than fft_length (${fft_length})`) + throw Error(`frame_length (${frame_length}) may not be larger than fft_length (${fft_length})`); } if (window_length !== frame_length) { @@ -505,35 +501,37 @@ export async function spectrogram( if (power === null && mel_filters !== null) { throw new Error( "You have provided `mel_filters` but `power` is `None`. Mel spectrogram computation is not yet supported for complex-valued spectrogram. " + - "Specify `power` to fix this issue." + "Specify `power` to fix this issue.", ); } if (center) { - if (pad_mode !== 'reflect') { - throw new Error(`pad_mode="${pad_mode}" not implemented yet.`) + if (pad_mode !== "reflect") { + throw new Error(`pad_mode="${pad_mode}" not implemented yet.`); } const half_window = Math.floor((fft_length - 1) / 2) + 1; waveform = padReflect(waveform, half_window, half_window); } // split waveform into frames of frame_length size - let num_frames = Math.floor(1 + Math.floor((waveform.length - frame_length) / hop_length)) + let num_frames = Math.floor(1 + Math.floor((waveform.length - frame_length) / hop_length)); if (min_num_frames !== null && num_frames < min_num_frames) { - num_frames = min_num_frames + num_frames = min_num_frames; } - const num_frequency_bins = onesided ? Math.floor(fft_length / 2) + 1 : fft_length + const num_frequency_bins = onesided ? Math.floor(fft_length / 2) + 1 : fft_length; let d1 = num_frames; let d1Max = num_frames; // If maximum number of frames is provided, we must either pad or truncate if (max_num_frames !== null) { - if (max_num_frames > num_frames) { // input is too short, so we pad + if (max_num_frames > num_frames) { + // input is too short, so we pad if (do_pad) { d1Max = max_num_frames; } - } else { // input is too long, so we truncate + } else { + // input is too long, so we truncate d1Max = d1 = max_num_frames; } } @@ -612,14 +610,14 @@ export async function spectrogram( // - mel_spec.shape=(80, 3000) let mel_spec = await matmul( // TODO: Make `mel_filters` a Tensor during initialization - new Tensor('float32', mel_filters.flat(), [num_mel_filters, num_frequency_bins]), - new Tensor('float32', transposedMagnitudeData, [num_frequency_bins, d1Max]), + new Tensor("float32", mel_filters.flat(), [num_mel_filters, num_frequency_bins]), + new Tensor("float32", transposedMagnitudeData, [num_frequency_bins, d1Max]), ); if (transpose) { mel_spec = mel_spec.transpose(1, 0); } - const mel_spec_data = /** @type {Float32Array} */(mel_spec.data); + const mel_spec_data = /** @type {Float32Array} */ (mel_spec.data); for (let i = 0; i < mel_spec_data.length; ++i) { mel_spec_data[i] = Math.max(mel_floor, mel_spec_data[i]); } @@ -628,23 +626,23 @@ export async function spectrogram( const o = Math.min(mel_spec_data.length, d1 * num_mel_filters); // NOTE: operates in-place switch (log_mel) { - case 'log': + case "log": for (let i = 0; i < o; ++i) { mel_spec_data[i] = Math.log(mel_spec_data[i]); } break; - case 'log10': + case "log10": for (let i = 0; i < o; ++i) { mel_spec_data[i] = Math.log10(mel_spec_data[i]); } break; - case 'dB': + case "dB": if (power === 1.0) { amplitude_to_db(mel_spec_data, reference, min_value, db_range); } else if (power === 2.0) { power_to_db(mel_spec_data, reference, min_value, db_range); } else { - throw new Error(`Cannot use log_mel option '${log_mel}' with power ${power}`) + throw new Error(`Cannot use log_mel option '${log_mel}' with power ${power}`); } break; default: @@ -666,26 +664,22 @@ export async function spectrogram( * @param {boolean} [options.center=true] Whether to center the window inside the FFT buffer. Only used when `frame_length` is provided. * @returns {Float64Array} The window of shape `(window_length,)` or `(frame_length,)`. */ -export function window_function(window_length, name, { - periodic = true, - frame_length = null, - center = true, -} = {}) { +export function window_function(window_length, name, { periodic = true, frame_length = null, center = true } = {}) { const length = periodic ? window_length + 1 : window_length; let window; switch (name) { - case 'boxcar': + case "boxcar": window = new Float64Array(length).fill(1.0); break; - case 'hann': - case 'hann_window': + case "hann": + case "hann_window": window = hanning(length); break; - case 'hamming': + case "hamming": window = hamming(length); break; - case 'povey': - window = hanning(length).map(x => Math.pow(x, 0.85)); + case "povey": + window = hanning(length).map((x) => Math.pow(x, 0.85)); break; default: throw new Error(`Unknown window type ${name}.`); @@ -697,7 +691,9 @@ export function window_function(window_length, name, { return window; } if (window_length > frame_length) { - throw new Error(`Length of the window (${window_length}) may not be larger than frame_length (${frame_length})`); + throw new Error( + `Length of the window (${window_length}) may not be larger than frame_length (${frame_length})`, + ); } return window; diff --git a/src/utils/constants.js b/src/utils/constants.js index 9d0e9ee42..9947698f0 100644 --- a/src/utils/constants.js +++ b/src/utils/constants.js @@ -1,2 +1 @@ - -export const GITHUB_ISSUE_URL = 'https://github.com/huggingface/transformers.js/issues/new/choose'; \ No newline at end of file +export const GITHUB_ISSUE_URL = "https://github.com/huggingface/transformers.js/issues/new/choose"; diff --git a/src/utils/core.js b/src/utils/core.js index 6a6137dff..1911ed37b 100644 --- a/src/utils/core.js +++ b/src/utils/core.js @@ -1,10 +1,9 @@ - /** * @file Core utility functions/classes for Transformers.js. - * + * * These are only used internally, meaning an end-user shouldn't * need to access anything here. - * + * * @module utils/core */ @@ -39,28 +38,27 @@ export function reverseDictionary(data) { * @returns {string} The escaped string. */ export function escapeRegExp(string) { - return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string + return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); // $& means the whole matched string } /** * Check if a value is a typed array. * @param {*} val The value to check. * @returns {boolean} True if the value is a `TypedArray`, false otherwise. - * + * * Adapted from https://stackoverflow.com/a/71091338/13989043 */ export function isTypedArray(val) { - return val?.prototype?.__proto__?.constructor?.name === 'TypedArray'; + return val?.prototype?.__proto__?.constructor?.name === "TypedArray"; } - /** * Check if a value is an integer. * @param {*} x The value to check. * @returns {boolean} True if the value is a string, false otherwise. */ export function isIntegralNumber(x) { - return Number.isInteger(x) || typeof x === 'bigint' + return Number.isInteger(x) || typeof x === "bigint"; } /** @@ -94,7 +92,7 @@ export function pop(obj, key, defaultValue = undefined) { return value; } if (defaultValue === undefined) { - throw Error(`Key ${key} does not exist in object.`) + throw Error(`Key ${key} does not exist in object.`); } return defaultValue; } @@ -118,7 +116,7 @@ export function mergeArrays(...arrs) { export function product(...a) { // Cartesian product of items // Adapted from https://stackoverflow.com/a/43053803 - return a.reduce((a, b) => a.flatMap(d => b.map(e => [d, e]))); + return a.reduce((a, b) => a.flatMap((d) => b.map((e) => [d, e]))); } /** @@ -128,13 +126,13 @@ export function product(...a) { * @returns {number} The index offset. */ export function calculateReflectOffset(i, w) { - return Math.abs((i + w) % (2 * w) - w); + return Math.abs(((i + w) % (2 * w)) - w); } /** - * - * @param {Object} o - * @param {string[]} props + * + * @param {Object} o + * @param {string[]} props * @returns {Object} */ export function pick(o, props) { @@ -144,14 +142,14 @@ export function pick(o, props) { if (o[prop] !== undefined) { return { [prop]: o[prop] }; } - }) + }), ); } /** * Calculate the length of a string, taking multi-byte characters into account. * This mimics the behavior of Python's `len` function. - * @param {string} s The string to calculate the length of. + * @param {string} s The string to calculate the length of. * @returns {number} The length of the string. */ export function len(s) { diff --git a/src/utils/data-structures.js b/src/utils/data-structures.js index 2340d12c0..81ac7af2f 100644 --- a/src/utils/data-structures.js +++ b/src/utils/data-structures.js @@ -1,25 +1,22 @@ - /** * @file Custom data structures. - * + * * These are only used internally, meaning an end-user shouldn't * need to access anything here. - * + * * @module utils/data-structures */ - /** * Efficient Heap-based Implementation of a Priority Queue. * It uses an array-based binary heap, where the root is at index `0`, and the * children of node `i` are located at indices `2i + 1` and `2i + 2`, respectively. - * + * * Adapted from the following sources: * - https://stackoverflow.com/a/42919752/13989043 (original) * - https://github.com/belladoreai/llama-tokenizer-js (minor improvements) */ export class PriorityQueue { - /** * Create a new PriorityQueue. * @param {function(any, any): boolean} comparator Comparator function to determine priority. Defaults to a MaxHeap. @@ -198,9 +195,10 @@ export class PriorityQueue { (this._left(node) < this.size && this._greater(this._left(node), node)) || (this._right(node) < this.size && this._greater(this._right(node), node)) ) { - const maxChild = (this._right(node) < this.size && this._greater(this._right(node), this._left(node))) - ? this._right(node) - : this._left(node); + const maxChild = + this._right(node) < this.size && this._greater(this._right(node), this._left(node)) + ? this._right(node) + : this._left(node); this._swap(node, maxChild); node = maxChild; } @@ -212,7 +210,7 @@ export class PriorityQueue { * @private */ _smallest() { - return (2 ** (Math.floor(Math.log2(this.size))) - 1); + return 2 ** Math.floor(Math.log2(this.size)) - 1; } } @@ -396,7 +394,7 @@ export class TokenLattice { * @returns {string} The array of nodes representing the most likely sequence of tokens. */ piece(node) { - return this.chars.slice(node.pos, node.pos + node.length).join(''); + return this.chars.slice(node.pos, node.pos + node.length).join(""); } /** @@ -404,7 +402,7 @@ export class TokenLattice { */ tokens() { const nodes = this.viterbi(); - return nodes.map(x => this.piece(x)); + return nodes.map((x) => this.piece(x)); } /** @@ -412,7 +410,7 @@ export class TokenLattice { */ tokenIds() { const nodes = this.viterbi(); - return nodes.map(x => x.tokenId); + return nodes.map((x) => x.tokenId); } } class TokenLatticeNode { diff --git a/src/utils/devices.js b/src/utils/devices.js index 1086b33e4..2102d1aaa 100644 --- a/src/utils/devices.js +++ b/src/utils/devices.js @@ -1,20 +1,19 @@ - /** * The list of devices supported by Transformers.js */ export const DEVICE_TYPES = Object.freeze({ - auto: 'auto', // Auto-detect based on device and environment - gpu: 'gpu', // Auto-detect GPU - cpu: 'cpu', // CPU - wasm: 'wasm', // WebAssembly - webgpu: 'webgpu', // WebGPU - cuda: 'cuda', // CUDA - dml: 'dml', // DirectML + auto: "auto", // Auto-detect based on device and environment + gpu: "gpu", // Auto-detect GPU + cpu: "cpu", // CPU + wasm: "wasm", // WebAssembly + webgpu: "webgpu", // WebGPU + cuda: "cuda", // CUDA + dml: "dml", // DirectML - webnn: 'webnn', // WebNN (default) - 'webnn-npu': 'webnn-npu', // WebNN NPU - 'webnn-gpu': 'webnn-gpu', // WebNN GPU - 'webnn-cpu': 'webnn-cpu', // WebNN CPU + webnn: "webnn", // WebNN (default) + "webnn-npu": "webnn-npu", // WebNN NPU + "webnn-gpu": "webnn-gpu", // WebNN GPU + "webnn-cpu": "webnn-cpu", // WebNN CPU }); /** diff --git a/src/utils/dtypes.js b/src/utils/dtypes.js index fa6d94be5..dfa144ef2 100644 --- a/src/utils/dtypes.js +++ b/src/utils/dtypes.js @@ -20,7 +20,7 @@ export const isWebGpuFp16Supported = (function () { } else { try { const adapter = await navigator.gpu.requestAdapter(); - cachedResult = adapter.features.has('shader-f16'); + cachedResult = adapter.features.has("shader-f16"); } catch (e) { cachedResult = false; } @@ -31,14 +31,14 @@ export const isWebGpuFp16Supported = (function () { })(); export const DATA_TYPES = Object.freeze({ - fp32: 'fp32', - fp16: 'fp16', - q8: 'q8', - int8: 'int8', - uint8: 'uint8', - q4: 'q4', - bnb4: 'bnb4', - q4f16: 'q4f16', // fp16 model with int4 block weight quantization + fp32: "fp32", + fp16: "fp16", + q8: "q8", + int8: "int8", + uint8: "uint8", + q4: "q4", + bnb4: "bnb4", + q4f16: "q4f16", // fp16 model with int4 block weight quantization }); /** @typedef {keyof typeof DATA_TYPES} DataType */ @@ -49,12 +49,12 @@ export const DEFAULT_DEVICE_DTYPE_MAPPING = Object.freeze({ /** @type {Record} */ export const DEFAULT_DTYPE_SUFFIX_MAPPING = Object.freeze({ - [DATA_TYPES.fp32]: '', - [DATA_TYPES.fp16]: '_fp16', - [DATA_TYPES.int8]: '_int8', - [DATA_TYPES.uint8]: '_uint8', - [DATA_TYPES.q8]: '_quantized', - [DATA_TYPES.q4]: '_q4', - [DATA_TYPES.q4f16]: '_q4f16', - [DATA_TYPES.bnb4]: '_bnb4', + [DATA_TYPES.fp32]: "", + [DATA_TYPES.fp16]: "_fp16", + [DATA_TYPES.int8]: "_int8", + [DATA_TYPES.uint8]: "_uint8", + [DATA_TYPES.q8]: "_quantized", + [DATA_TYPES.q4]: "_q4", + [DATA_TYPES.q4f16]: "_q4f16", + [DATA_TYPES.bnb4]: "_bnb4", }); diff --git a/src/utils/generic.js b/src/utils/generic.js index 5ccd467ad..b35a5e4df 100644 --- a/src/utils/generic.js +++ b/src/utils/generic.js @@ -1,35 +1,36 @@ - /** * A base class for creating callable objects. * See [here](https://stackoverflow.com/q/76073890) for more information. - * + * * @type {new () => {(...args: any[]): any, _call(...args: any[]): any}} */ -export const Callable = /** @type {any} */ (class { - /** - * Creates a new instance of the Callable class. - */ - constructor() { +export const Callable = /** @type {any} */ ( + class { /** - * Creates a closure that delegates to a private method '_call' with the given arguments. - * @type {any} - * @param {...any} args Zero or more arguments to pass to the '_call' method. - * @returns {*} The result of calling the '_call' method. + * Creates a new instance of the Callable class. */ - let closure = function (...args) { - return closure._call(...args) + constructor() { + /** + * Creates a closure that delegates to a private method '_call' with the given arguments. + * @type {any} + * @param {...any} args Zero or more arguments to pass to the '_call' method. + * @returns {*} The result of calling the '_call' method. + */ + let closure = function (...args) { + return closure._call(...args); + }; + return Object.setPrototypeOf(closure, new.target.prototype); } - return Object.setPrototypeOf(closure, new.target.prototype) - } - /** - * This method should be implemented in subclasses to provide the - * functionality of the callable object. - * - * @param {any[]} args - * @throws {Error} If the subclass does not implement the `_call` method. - */ - _call(...args) { - throw Error('Must implement _call method in subclass') + /** + * This method should be implemented in subclasses to provide the + * functionality of the callable object. + * + * @param {any[]} args + * @throws {Error} If the subclass does not implement the `_call` method. + */ + _call(...args) { + throw Error("Must implement _call method in subclass"); + } } -}); +); diff --git a/src/utils/hub.js b/src/utils/hub.js index 71c20c861..d05c5ad96 100755 --- a/src/utils/hub.js +++ b/src/utils/hub.js @@ -1,18 +1,17 @@ - /** * @file Utility functions to interact with the Hugging Face Hub (https://huggingface.co/models) - * + * * @module utils/hub */ -import fs from 'fs'; -import path from 'path'; +import fs from "fs"; +import path from "path"; -import { env } from '../env.js'; -import { dispatchCallback } from './core.js'; +import { env } from "../env.js"; +import { dispatchCallback } from "./core.js"; /** - * @typedef {Object} PretrainedOptions Options for loading a pretrained model. + * @typedef {Object} PretrainedOptions Options for loading a pretrained model. * @property {function} [progress_callback=null] If specified, this function will be called during model construction, to provide the user with progress updates. * @property {import('../configs.js').PretrainedConfig} [config=null] Configuration for the model to use instead of an automatically loaded configuration. Configuration can be automatically loaded when: * - The model is a model provided by the library (loaded with the *model id* string of a pretrained model). @@ -43,18 +42,17 @@ import { dispatchCallback } from './core.js'; * Mapping from file extensions to MIME types. */ const CONTENT_TYPE_MAP = { - 'txt': 'text/plain', - 'html': 'text/html', - 'css': 'text/css', - 'js': 'text/javascript', - 'json': 'application/json', - 'png': 'image/png', - 'jpg': 'image/jpeg', - 'jpeg': 'image/jpeg', - 'gif': 'image/gif', -} + txt: "text/plain", + html: "text/html", + css: "text/css", + js: "text/javascript", + json: "application/json", + png: "image/png", + jpg: "image/jpeg", + jpeg: "image/jpeg", + gif: "image/gif", +}; class FileResponse { - /** * Creates a new `FileResponse` object. * @param {string|URL} filePath @@ -66,25 +64,25 @@ class FileResponse { this.exists = fs.existsSync(filePath); if (this.exists) { this.status = 200; - this.statusText = 'OK'; + this.statusText = "OK"; let stats = fs.statSync(filePath); - this.headers.set('content-length', stats.size.toString()); + this.headers.set("content-length", stats.size.toString()); this.updateContentType(); let self = this; this.body = new ReadableStream({ start(controller) { - self.arrayBuffer().then(buffer => { + self.arrayBuffer().then((buffer) => { controller.enqueue(new Uint8Array(buffer)); controller.close(); - }) - } + }); + }, }); } else { this.status = 404; - this.statusText = 'Not Found'; + this.statusText = "Not Found"; this.body = null; } } @@ -96,8 +94,8 @@ class FileResponse { */ updateContentType() { // Set content-type header based on file extension - const extension = this.filePath.toString().split('.').pop().toLowerCase(); - this.headers.set('content-type', CONTENT_TYPE_MAP[extension] ?? 'application/octet-stream'); + const extension = this.filePath.toString().split(".").pop().toLowerCase(); + this.headers.set("content-type", CONTENT_TYPE_MAP[extension] ?? "application/octet-stream"); } /** @@ -132,7 +130,7 @@ class FileResponse { */ async blob() { const data = await fs.promises.readFile(this.filePath); - return new Blob([data], { type: this.headers.get('content-type') }); + return new Blob([data], { type: this.headers.get("content-type") }); } /** @@ -142,14 +140,14 @@ class FileResponse { * @throws {Error} If the file cannot be read. */ async text() { - const data = await fs.promises.readFile(this.filePath, 'utf8'); + const data = await fs.promises.readFile(this.filePath, "utf8"); return data; } /** * Reads the contents of the file specified by the filePath property and returns a Promise that * resolves with a parsed JavaScript object containing the file's contents. - * + * * @returns {Promise} A Promise that resolves with a parsed JavaScript object containing the file's contents. * @throws {Error} If the file cannot be read. */ @@ -188,26 +186,24 @@ function isValidUrl(string, protocols = null, validHosts = null) { * @returns {Promise} A promise that resolves to a FileResponse object (if the file is retrieved using the FileSystem API), or a Response object (if the file is retrieved using the Fetch API). */ export async function getFile(urlOrPath) { - - if (env.useFS && !isValidUrl(urlOrPath, ['http:', 'https:', 'blob:'])) { + if (env.useFS && !isValidUrl(urlOrPath, ["http:", "https:", "blob:"])) { return new FileResponse(urlOrPath); - - } else if (typeof process !== 'undefined' && process?.release?.name === 'node') { + } else if (typeof process !== "undefined" && process?.release?.name === "node") { const IS_CI = !!process.env?.TESTING_REMOTELY; const version = env.version; const headers = new Headers(); - headers.set('User-Agent', `transformers.js/${version}; is_ci/${IS_CI};`); + headers.set("User-Agent", `transformers.js/${version}; is_ci/${IS_CI};`); // Check whether we are making a request to the Hugging Face Hub. - const isHFURL = isValidUrl(urlOrPath, ['http:', 'https:'], ['huggingface.co', 'hf.co']); + const isHFURL = isValidUrl(urlOrPath, ["http:", "https:"], ["huggingface.co", "hf.co"]); if (isHFURL) { // If an access token is present in the environment variables, // we add it to the request headers. // NOTE: We keep `HF_ACCESS_TOKEN` for backwards compatibility (as a fallback). const token = process.env?.HF_TOKEN ?? process.env?.HF_ACCESS_TOKEN; if (token) { - headers.set('Authorization', `Bearer ${token}`); + headers.set("Authorization", `Bearer ${token}`); } } return fetch(urlOrPath, { headers }); @@ -221,18 +217,18 @@ export async function getFile(urlOrPath) { const ERROR_MAPPING = { // 4xx errors (https://developer.mozilla.org/en-US/docs/Web/HTTP/Status#client_error_responses) - 400: 'Bad request error occurred while trying to load file', - 401: 'Unauthorized access to file', - 403: 'Forbidden access to file', - 404: 'Could not locate file', - 408: 'Request timeout error occurred while trying to load file', + 400: "Bad request error occurred while trying to load file", + 401: "Unauthorized access to file", + 403: "Forbidden access to file", + 404: "Could not locate file", + 408: "Request timeout error occurred while trying to load file", // 5xx errors (https://developer.mozilla.org/en-US/docs/Web/HTTP/Status#server_error_responses) - 500: 'Internal server error error occurred while trying to load file', - 502: 'Bad gateway error occurred while trying to load file', - 503: 'Service unavailable error occurred while trying to load file', - 504: 'Gateway timeout error occurred while trying to load file', -} + 500: "Internal server error error occurred while trying to load file", + 502: "Bad gateway error occurred while trying to load file", + 503: "Service unavailable error occurred while trying to load file", + 504: "Gateway timeout error occurred while trying to load file", +}; /** * Helper method to handle fatal errors that occur while trying to load a file from the Hugging Face Hub. * @param {number} status The HTTP status code of the error. @@ -255,7 +251,7 @@ function handleError(status, remoteURL, fatal) { class FileCache { /** * Instantiate a `FileCache` object. - * @param {string} path + * @param {string} path */ constructor(path) { this.path = path; @@ -263,11 +259,10 @@ class FileCache { /** * Checks whether the given request is in the cache. - * @param {string} request + * @param {string} request * @returns {Promise} */ async match(request) { - let filePath = path.join(this.path, request); let file = new FileResponse(filePath); @@ -280,8 +275,8 @@ class FileCache { /** * Adds the given response to the cache. - * @param {string} request - * @param {Response|FileResponse} response + * @param {string} request + * @param {Response|FileResponse} response * @returns {Promise} */ async put(request, response) { @@ -292,9 +287,8 @@ class FileCache { try { await fs.promises.mkdir(path.dirname(outputPath), { recursive: true }); await fs.promises.writeFile(outputPath, buffer); - } catch (err) { - console.warn('An error occurred while writing the file to cache:', err) + console.warn("An error occurred while writing the file to cache:", err); } } @@ -307,7 +301,7 @@ class FileCache { } /** - * + * * @param {FileCache|Cache} cache The cache to search * @param {string[]} names The names of the item to search for * @returns {Promise} The item from the cache, or undefined if not found. @@ -325,45 +319,48 @@ async function tryCache(cache, ...names) { } /** - * + * * Retrieves a file from either a remote URL using the Fetch API or from the local file system using the FileSystem API. * If the filesystem is available and `env.useCache = true`, the file will be downloaded and cached. - * + * * @param {string} path_or_repo_id This can be either: * - a string, the *model id* of a model repo on huggingface.co. * - a path to a *directory* potentially containing the file. * @param {string} filename The name of the file to locate in `path_or_repo`. * @param {boolean} [fatal=true] Whether to throw an error if the file is not found. * @param {PretrainedOptions} [options] An object containing optional parameters. - * + * * @throws Will throw an error if the file is not found and `fatal` is true. * @returns {Promise} A Promise that resolves with the file content as a buffer. */ export async function getModelFile(path_or_repo_id, filename, fatal = true, options = {}) { - if (!env.allowLocalModels) { // User has disabled local models, so we just make sure other settings are correct. if (options.local_files_only) { - throw Error("Invalid configuration detected: local models are disabled (`env.allowLocalModels=false`) but you have requested to only use local models (`local_files_only=true`).") + throw Error( + "Invalid configuration detected: local models are disabled (`env.allowLocalModels=false`) but you have requested to only use local models (`local_files_only=true`).", + ); } else if (!env.allowRemoteModels) { - throw Error("Invalid configuration detected: both local and remote models are disabled. Fix by setting `env.allowLocalModels` or `env.allowRemoteModels` to `true`.") + throw Error( + "Invalid configuration detected: both local and remote models are disabled. Fix by setting `env.allowLocalModels` or `env.allowRemoteModels` to `true`.", + ); } } // Initiate file retrieval dispatchCallback(options.progress_callback, { - status: 'initiate', + status: "initiate", name: path_or_repo_id, - file: filename - }) + file: filename, + }); // First, check if the a caching backend is available // If no caching mechanism available, will download the file every time let cache; if (!cache && env.useBrowserCache) { - if (typeof caches === 'undefined') { - throw Error('Browser cache is not available in this environment.') + if (typeof caches === "undefined") { + throw Error("Browser cache is not available in this environment."); } try { // In some cases, the browser cache may be visible, but not accessible due to security restrictions. @@ -371,9 +368,9 @@ export async function getModelFile(path_or_repo_id, filename, fatal = true, opti // incognito mode, the following error is thrown: `DOMException: Failed to execute 'open' on 'CacheStorage': // An attempt was made to break through the security policy of the user agent.` // So, instead of crashing, we just ignore the error and continue without using the cache. - cache = await caches.open('transformers-cache'); + cache = await caches.open("transformers-cache"); } catch (e) { - console.warn('An error occurred while opening the browser cache:', e); + console.warn("An error occurred while opening the browser cache:", e); } } @@ -387,20 +384,20 @@ export async function getModelFile(path_or_repo_id, filename, fatal = true, opti if (!cache && env.useCustomCache) { // Allow the user to specify a custom cache system. if (!env.customCache) { - throw Error('`env.useCustomCache=true`, but `env.customCache` is not defined.') + throw Error("`env.useCustomCache=true`, but `env.customCache` is not defined."); } // Check that the required methods are defined: if (!env.customCache.match || !env.customCache.put) { throw new Error( "`env.customCache` must be an object which implements the `match` and `put` functions of the Web Cache API. " + - "For more information, see https://developer.mozilla.org/en-US/docs/Web/API/Cache" - ) + "For more information, see https://developer.mozilla.org/en-US/docs/Web/API/Cache", + ); } cache = env.customCache; } - const revision = options.revision ?? 'main'; + const revision = options.revision ?? "main"; let requestURL = pathJoin(path_or_repo_id, filename); let localPath = pathJoin(env.localModelPath, requestURL); @@ -408,15 +405,15 @@ export async function getModelFile(path_or_repo_id, filename, fatal = true, opti let remoteURL = pathJoin( env.remoteHost, env.remotePathTemplate - .replaceAll('{model}', path_or_repo_id) - .replaceAll('{revision}', encodeURIComponent(revision)), - filename + .replaceAll("{model}", path_or_repo_id) + .replaceAll("{revision}", encodeURIComponent(revision)), + filename, ); // Choose cache key for filesystem cache // When using the main revision (default), we use the request URL as the cache key. // If a specific revision is requested, we account for this in the cache key. - let fsCacheKey = revision === 'main' ? requestURL : pathJoin(path_or_repo_id, revision, filename); + let fsCacheKey = revision === "main" ? requestURL : pathJoin(path_or_repo_id, revision, filename); /** @type {string} */ let cacheKey; @@ -444,7 +441,7 @@ export async function getModelFile(path_or_repo_id, filename, fatal = true, opti if (env.allowLocalModels) { // Accessing local models is enabled, so we try to get the file locally. // If request is a valid HTTP URL, we skip the local file check. Otherwise, we try to get the file locally. - const isURL = isValidUrl(requestURL, ['http:', 'https:']); + const isURL = isValidUrl(requestURL, ["http:", "https:"]); if (!isURL) { try { response = await getFile(localPath); @@ -457,7 +454,9 @@ export async function getModelFile(path_or_repo_id, filename, fatal = true, opti } else if (options.local_files_only) { throw new Error(`\`local_files_only=true\`, but attempted to load a remote file from: ${requestURL}.`); } else if (!env.allowRemoteModels) { - throw new Error(`\`env.allowRemoteModels=false\`, but attempted to load a remote file from: ${requestURL}.`); + throw new Error( + `\`env.allowRemoteModels=false\`, but attempted to load a remote file from: ${requestURL}.`, + ); } } @@ -470,7 +469,9 @@ export async function getModelFile(path_or_repo_id, filename, fatal = true, opti if (options.local_files_only || !env.allowRemoteModels) { // User requested local files only, but the file is not found locally. if (fatal) { - throw Error(`\`local_files_only=true\` or \`env.allowRemoteModels=false\` and file was not found locally at "${localPath}".`); + throw Error( + `\`local_files_only=true\` or \`env.allowRemoteModels=false\` and file was not found locally at "${localPath}".`, + ); } else { // File not found, but this file is optional. // TODO in future, cache the response? @@ -491,24 +492,24 @@ export async function getModelFile(path_or_repo_id, filename, fatal = true, opti // Only cache the response if: toCacheResponse = - cache // 1. A caching system is available - && typeof Response !== 'undefined' // 2. `Response` is defined (i.e., we are in a browser-like environment) - && response instanceof Response // 3. result is a `Response` object (i.e., not a `FileResponse`) - && response.status === 200 // 4. request was successful (status code 200) + cache && // 1. A caching system is available + typeof Response !== "undefined" && // 2. `Response` is defined (i.e., we are in a browser-like environment) + response instanceof Response && // 3. result is a `Response` object (i.e., not a `FileResponse`) + response.status === 200; // 4. request was successful (status code 200) } // Start downloading dispatchCallback(options.progress_callback, { - status: 'download', + status: "download", name: path_or_repo_id, - file: filename - }) + file: filename, + }); const progressInfo = { - status: 'progress', + status: "progress", name: path_or_repo_id, - file: filename - } + file: filename, + }; /** @type {Uint8Array} */ let buffer; @@ -517,11 +518,10 @@ export async function getModelFile(path_or_repo_id, filename, fatal = true, opti // If no progress callback is specified, we can use the `.arrayBuffer()` // method to read the response. buffer = new Uint8Array(await response.arrayBuffer()); - } else if ( - cacheHit // The item is being read from the cache - && - typeof navigator !== 'undefined' && /firefox/i.test(navigator.userAgent) // We are in Firefox + cacheHit && // The item is being read from the cache + typeof navigator !== "undefined" && + /firefox/i.test(navigator.userAgent) // We are in Firefox ) { // Due to bug in Firefox, we cannot display progress when loading from cache. // Fortunately, since this should be instantaneous, this should not impact users too much. @@ -533,40 +533,43 @@ export async function getModelFile(path_or_repo_id, filename, fatal = true, opti progress: 100, loaded: buffer.length, total: buffer.length, - }) + }); } else { - buffer = await readResponse(response, data => { + buffer = await readResponse(response, (data) => { dispatchCallback(options.progress_callback, { ...progressInfo, ...data, - }) - }) + }); + }); } if ( // Only cache web responses // i.e., do not cache FileResponses (prevents duplication) - toCacheResponse && cacheKey - && + toCacheResponse && + cacheKey && // Check again whether request is in cache. If not, we add the response to the cache - (await cache.match(cacheKey) === undefined) + (await cache.match(cacheKey)) === undefined ) { // NOTE: We use `new Response(buffer, ...)` instead of `response.clone()` to handle LFS files - await cache.put(cacheKey, new Response(buffer, { - headers: response.headers - })) - .catch(err => { + await cache + .put( + cacheKey, + new Response(buffer, { + headers: response.headers, + }), + ) + .catch((err) => { // Do not crash if unable to add to cache (e.g., QuotaExceededError). // Rather, log a warning and proceed with execution. console.warn(`Unable to add response to browser cache: ${err}.`); }); - } dispatchCallback(options.progress_callback, { - status: 'done', + status: "done", name: path_or_repo_id, - file: filename + file: filename, }); return buffer; @@ -586,10 +589,10 @@ export async function getModelJSON(modelPath, fileName, fatal = true, options = let buffer = await getModelFile(modelPath, fileName, fatal, options); if (buffer === null) { // Return empty object - return {} + return {}; } - let decoder = new TextDecoder('utf-8'); + let decoder = new TextDecoder("utf-8"); let jsonData = decoder.decode(buffer); return JSON.parse(jsonData); @@ -603,12 +606,11 @@ export async function getModelJSON(modelPath, fileName, fatal = true, options = * @returns {Promise} A Promise that resolves with the Uint8Array buffer */ async function readResponse(response, progress_callback) { - - const contentLength = response.headers.get('Content-Length'); + const contentLength = response.headers.get("Content-Length"); if (contentLength === null) { - console.warn('Unable to determine content-length from response headers. Will expand buffer when needed.') + console.warn("Unable to determine content-length from response headers. Will expand buffer when needed."); } - let total = parseInt(contentLength ?? '0'); + let total = parseInt(contentLength ?? "0"); let buffer = new Uint8Array(total); let loaded = 0; @@ -630,7 +632,7 @@ async function readResponse(response, progress_callback) { buffer = newBuffer; } - buffer.set(value, loaded) + buffer.set(value, loaded); loaded = newLoaded; const progress = (loaded / total) * 100; @@ -640,7 +642,7 @@ async function readResponse(response, progress_callback) { progress: progress, loaded: loaded, total: total, - }) + }); return read(); } @@ -661,12 +663,12 @@ function pathJoin(...parts) { // https://stackoverflow.com/a/55142565 parts = parts.map((part, index) => { if (index) { - part = part.replace(new RegExp('^/'), ''); + part = part.replace(new RegExp("^/"), ""); } if (index !== parts.length - 1) { - part = part.replace(new RegExp('/$'), ''); + part = part.replace(new RegExp("/$"), ""); } return part; - }) - return parts.join('/'); + }); + return parts.join("/"); } diff --git a/src/utils/image.js b/src/utils/image.js index 73114b13b..96560dc81 100644 --- a/src/utils/image.js +++ b/src/utils/image.js @@ -1,22 +1,21 @@ - /** - * @file Helper module for image processing. - * - * These functions and classes are only used internally, + * @file Helper module for image processing. + * + * These functions and classes are only used internally, * meaning an end-user shouldn't need to access anything here. - * + * * @module utils/image */ -import { getFile } from './hub.js'; -import { env } from '../env.js'; -import { Tensor } from './tensor.js'; +import { getFile } from "./hub.js"; +import { env } from "../env.js"; +import { Tensor } from "./tensor.js"; // Will be empty (or not used) if running in browser or web-worker -import sharp from 'sharp'; +import sharp from "sharp"; -const BROWSER_ENV = typeof self !== 'undefined'; -const WEBWORKER_ENV = BROWSER_ENV && self.constructor.name === 'DedicatedWorkerGlobalScope'; +const BROWSER_ENV = typeof self !== "undefined"; +const WEBWORKER_ENV = BROWSER_ENV && self.constructor.name === "DedicatedWorkerGlobalScope"; let createCanvasFunction; let ImageDataClass; @@ -25,17 +24,16 @@ if (BROWSER_ENV) { // Running in browser or web-worker createCanvasFunction = (/** @type {number} */ width, /** @type {number} */ height) => { if (!self.OffscreenCanvas) { - throw new Error('OffscreenCanvas not supported by this browser.'); + throw new Error("OffscreenCanvas not supported by this browser."); } - return new self.OffscreenCanvas(width, height) + return new self.OffscreenCanvas(width, height); }; loadImageFunction = self.createImageBitmap; ImageDataClass = self.ImageData; - } else if (sharp) { // Running in Node.js, electron, or other non-browser environment - loadImageFunction = async (/**@type {sharp.Sharp}*/img) => { + loadImageFunction = async (/**@type {sharp.Sharp}*/ img) => { const metadata = await img.metadata(); const rawChannels = metadata.channels; @@ -48,35 +46,32 @@ if (BROWSER_ENV) { newImage.convert(rawChannels); } return newImage; - } - + }; } else { - throw new Error('Unable to load image processing library.'); + throw new Error("Unable to load image processing library."); } - // Defined here: https://github.com/python-pillow/Pillow/blob/a405e8406b83f8bfb8916e93971edc7407b8b1ff/src/libImaging/Imaging.h#L262-L268 const RESAMPLING_MAPPING = { - 0: 'nearest', - 1: 'lanczos', - 2: 'bilinear', - 3: 'bicubic', - 4: 'box', - 5: 'hamming', -} + 0: "nearest", + 1: "lanczos", + 2: "bilinear", + 3: "bicubic", + 4: "box", + 5: "hamming", +}; /** * Mapping from file extensions to MIME types. */ const CONTENT_TYPE_MAP = new Map([ - ['png', 'image/png'], - ['jpg', 'image/jpeg'], - ['jpeg', 'image/jpeg'], - ['gif', 'image/gif'], + ["png", "image/png"], + ["jpg", "image/jpeg"], + ["jpeg", "image/jpeg"], + ["gif", "image/gif"], ]); export class RawImage { - /** * Create a new `RawImage` object. * @param {Uint8ClampedArray|Uint8Array} data The pixel data. @@ -91,7 +86,7 @@ export class RawImage { this.channels = channels; } - /** + /** * Returns the size of the image (width, height). * @returns {[number, number]} The size of the image (width, height). */ @@ -101,9 +96,9 @@ export class RawImage { /** * Helper method for reading an image from a variety of input types. - * @param {RawImage|string|URL} input + * @param {RawImage|string|URL} input * @returns The image object. - * + * * **Example:** Read image from a URL. * ```javascript * let image = await RawImage.read('https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/football-match.jpg'); @@ -118,7 +113,7 @@ export class RawImage { static async read(input) { if (input instanceof RawImage) { return input; - } else if (typeof input === 'string' || input instanceof URL) { + } else if (typeof input === "string" || input instanceof URL) { return await this.fromURL(input); } else { throw new Error(`Unsupported input type: ${typeof input}`); @@ -132,10 +127,10 @@ export class RawImage { */ static fromCanvas(canvas) { if (!BROWSER_ENV) { - throw new Error('fromCanvas() is only supported in browser environments.') + throw new Error("fromCanvas() is only supported in browser environments."); } - const ctx = canvas.getContext('2d'); + const ctx = canvas.getContext("2d"); const data = ctx.getImageData(0, 0, canvas.width, canvas.height).data; return new RawImage(data, canvas.width, canvas.height, 4); } @@ -164,13 +159,12 @@ export class RawImage { // Running in environment with canvas const img = await loadImageFunction(blob); - const ctx = createCanvasFunction(img.width, img.height).getContext('2d'); + const ctx = createCanvasFunction(img.width, img.height).getContext("2d"); // Draw image to context ctx.drawImage(img, 0, 0); return new this(ctx.getImageData(0, 0, img.width, img.height).data, img.width, img.height, 4); - } else { // Use sharp.js to read (and possible resize) the image. const img = sharp(await blob.arrayBuffer()); @@ -181,16 +175,16 @@ export class RawImage { /** * Helper method to create a new Image from a tensor - * @param {Tensor} tensor + * @param {Tensor} tensor */ - static fromTensor(tensor, channel_format = 'CHW') { + static fromTensor(tensor, channel_format = "CHW") { if (tensor.dims.length !== 3) { throw new Error(`Tensor should have 3 dimensions, but has ${tensor.dims.length} dimensions.`); } - if (channel_format === 'CHW') { + if (channel_format === "CHW") { tensor = tensor.transpose(1, 2, 0); - } else if (channel_format === 'HWC') { + } else if (channel_format === "HWC") { // Do nothing } else { throw new Error(`Unsupported channel format: ${channel_format}`); @@ -227,7 +221,7 @@ export class RawImage { const green = this.data[i + 1]; const blue = this.data[i + 2]; - newData[offset++] = Math.round(0.2989 * red + 0.5870 * green + 0.1140 * blue); + newData[offset++] = Math.round(0.2989 * red + 0.587 * green + 0.114 * blue); } break; default: @@ -266,7 +260,6 @@ export class RawImage { throw new Error(`Conversion failed due to unsupported number of channels: ${this.channels}`); } return this._update(newData, this.width, this.height, 3); - } /** @@ -312,10 +305,7 @@ export class RawImage { * @param {0|1|2|3|4|5|string} [options.resample] The resampling method to use. * @returns {Promise} `this` to support chaining. */ - async resize(width, height, { - resample = 2, - } = {}) { - + async resize(width, height, { resample = 2 } = {}) { // Do nothing if the image already has the desired size if (this.width === width && this.height === height) { return this; @@ -334,7 +324,7 @@ export class RawImage { const canvas = this.toCanvas(); // Actually perform resizing using the canvas API - const ctx = createCanvasFunction(width, height).getContext('2d'); + const ctx = createCanvasFunction(width, height).getContext("2d"); // Draw image to context, resizing in the process ctx.drawImage(canvas, 0, 0, width, height); @@ -344,36 +334,38 @@ export class RawImage { // Convert back so that image has the same number of channels as before return resizedImage.convert(numChannels); - } else { // Create sharp image from raw data, and resize let img = this.toSharp(); switch (resampleMethod) { - case 'box': - case 'hamming': - if (resampleMethod === 'box' || resampleMethod === 'hamming') { - console.warn(`Resampling method ${resampleMethod} is not yet supported. Using bilinear instead.`); - resampleMethod = 'bilinear'; + case "box": + case "hamming": + if (resampleMethod === "box" || resampleMethod === "hamming") { + console.warn( + `Resampling method ${resampleMethod} is not yet supported. Using bilinear instead.`, + ); + resampleMethod = "bilinear"; } - case 'nearest': - case 'bilinear': - case 'bicubic': - // Perform resizing using affine transform. + case "nearest": + case "bilinear": + case "bicubic": + // Perform resizing using affine transform. // This matches how the python Pillow library does it. img = img.affine([width / this.width, 0, 0, height / this.height], { - interpolator: resampleMethod + interpolator: resampleMethod, }); break; - case 'lanczos': + case "lanczos": // https://github.com/python-pillow/Pillow/discussions/5519 // https://github.com/lovell/sharp/blob/main/docs/api-resize.md img = img.resize({ - width, height, - fit: 'fill', - kernel: 'lanczos3', // PIL Lanczos uses a kernel size of 3 + width, + height, + fit: "fill", + kernel: "lanczos3", // PIL Lanczos uses a kernel size of 3 }); break; @@ -383,7 +375,6 @@ export class RawImage { return await loadImageFunction(img); } - } async pad([left, right, top, bottom]) { @@ -408,22 +399,16 @@ export class RawImage { const newHeight = this.height + top + bottom; // Create a new canvas of the desired size. - const ctx = createCanvasFunction(newWidth, newHeight).getContext('2d'); + const ctx = createCanvasFunction(newWidth, newHeight).getContext("2d"); // Draw image to context, padding in the process - ctx.drawImage(canvas, - 0, 0, this.width, this.height, - left, top, newWidth, newHeight - ); + ctx.drawImage(canvas, 0, 0, this.width, this.height, left, top, newWidth, newHeight); // Create image from the padded data - const paddedImage = new RawImage( - ctx.getImageData(0, 0, newWidth, newHeight).data, - newWidth, newHeight, 4); + const paddedImage = new RawImage(ctx.getImageData(0, 0, newWidth, newHeight).data, newWidth, newHeight, 4); // Convert back so that image has the same number of channels as before return paddedImage.convert(numChannels); - } else { const img = this.toSharp().extend({ left, right, top, bottom }); return await loadImageFunction(img); @@ -452,22 +437,23 @@ export class RawImage { // Create canvas object for this image const canvas = this.toCanvas(); - // Create a new canvas of the desired size. This is needed since if the + // Create a new canvas of the desired size. This is needed since if the // image is too small, we need to pad it with black pixels. - const ctx = createCanvasFunction(crop_width, crop_height).getContext('2d'); + const ctx = createCanvasFunction(crop_width, crop_height).getContext("2d"); // Draw image to context, cropping in the process - ctx.drawImage(canvas, - x_min, y_min, crop_width, crop_height, - 0, 0, crop_width, crop_height - ); + ctx.drawImage(canvas, x_min, y_min, crop_width, crop_height, 0, 0, crop_width, crop_height); // Create image from the resized data - const resizedImage = new RawImage(ctx.getImageData(0, 0, crop_width, crop_height).data, crop_width, crop_height, 4); + const resizedImage = new RawImage( + ctx.getImageData(0, 0, crop_width, crop_height).data, + crop_width, + crop_height, + 4, + ); // Convert back so that image has the same number of channels as before return resizedImage.convert(numChannels); - } else { // Create sharp image from raw data const img = this.toSharp().extract({ @@ -479,7 +465,6 @@ export class RawImage { return await loadImageFunction(img); } - } async center_crop(crop_width, crop_height) { @@ -492,7 +477,6 @@ export class RawImage { const width_offset = (this.width - crop_width) / 2; const height_offset = (this.height - crop_height) / 2; - if (BROWSER_ENV) { // Store number of channels before resizing const numChannels = this.channels; @@ -500,9 +484,9 @@ export class RawImage { // Create canvas object for this image const canvas = this.toCanvas(); - // Create a new canvas of the desired size. This is needed since if the + // Create a new canvas of the desired size. This is needed since if the // image is too small, we need to pad it with black pixels. - const ctx = createCanvasFunction(crop_width, crop_height).getContext('2d'); + const ctx = createCanvasFunction(crop_width, crop_height).getContext("2d"); let sourceX = 0; let sourceY = 0; @@ -522,17 +506,18 @@ export class RawImage { } // Draw image to context, cropping in the process - ctx.drawImage(canvas, - sourceX, sourceY, crop_width, crop_height, - destX, destY, crop_width, crop_height - ); + ctx.drawImage(canvas, sourceX, sourceY, crop_width, crop_height, destX, destY, crop_width, crop_height); // Create image from the resized data - const resizedImage = new RawImage(ctx.getImageData(0, 0, crop_width, crop_height).data, crop_width, crop_height, 4); + const resizedImage = new RawImage( + ctx.getImageData(0, 0, crop_width, crop_height).data, + crop_width, + crop_height, + 4, + ); // Convert back so that image has the same number of channels as before return resizedImage.convert(numChannels); - } else { // Create sharp image from raw data let img = this.toSharp(); @@ -544,7 +529,7 @@ export class RawImage { top: Math.floor(height_offset), width: crop_width, height: crop_height, - }) + }); } else if (width_offset <= 0 && height_offset <= 0) { // Cropped image lies entirely outside the original image, // so we add padding @@ -580,42 +565,41 @@ export class RawImage { x_extract = Math.floor(width_offset); } - img = img.extend({ - top: y_padding[0], - bottom: y_padding[1], - left: x_padding[0], - right: x_padding[1], - }).extract({ - left: x_extract, - top: y_extract, - width: crop_width, - height: crop_height, - }) + img = img + .extend({ + top: y_padding[0], + bottom: y_padding[1], + left: x_padding[0], + right: x_padding[1], + }) + .extract({ + left: x_extract, + top: y_extract, + width: crop_width, + height: crop_height, + }); } return await loadImageFunction(img); } } - async toBlob(type = 'image/png', quality = 1) { + async toBlob(type = "image/png", quality = 1) { if (!BROWSER_ENV) { - throw new Error('toBlob() is only supported in browser environments.') + throw new Error("toBlob() is only supported in browser environments."); } const canvas = this.toCanvas(); return await canvas.convertToBlob({ type, quality }); } - toTensor(channel_format = 'CHW') { - let tensor = new Tensor( - 'uint8', - new Uint8Array(this.data), - [this.height, this.width, this.channels] - ); + toTensor(channel_format = "CHW") { + let tensor = new Tensor("uint8", new Uint8Array(this.data), [this.height, this.width, this.channels]); - if (channel_format === 'HWC') { + if (channel_format === "HWC") { // Do nothing - } else if (channel_format === 'CHW') { // hwc -> chw + } else if (channel_format === "CHW") { + // hwc -> chw tensor = tensor.permute(2, 0, 1); } else { throw new Error(`Unsupported channel format: ${channel_format}`); @@ -625,7 +609,7 @@ export class RawImage { toCanvas() { if (!BROWSER_ENV) { - throw new Error('toCanvas() is only supported in browser environments.') + throw new Error("toCanvas() is only supported in browser environments."); } // Clone, and convert data to RGBA before drawing to canvas. @@ -637,7 +621,7 @@ export class RawImage { // Draw image to context const data = new ImageDataClass(cloned.data, cloned.width, cloned.height); - clonedCanvas.getContext('2d').putImageData(data, 0, 0); + clonedCanvas.getContext("2d").putImageData(data, 0, 0); return clonedCanvas; } @@ -697,14 +681,13 @@ export class RawImage { * @param {string} path The path to save the image to. */ async save(path) { - if (BROWSER_ENV) { if (WEBWORKER_ENV) { - throw new Error('Unable to save an image from a Web Worker.') + throw new Error("Unable to save an image from a Web Worker."); } - const extension = path.split('.').pop().toLowerCase(); - const mime = CONTENT_TYPE_MAP.get(extension) ?? 'image/png'; + const extension = path.split(".").pop().toLowerCase(); + const mime = CONTENT_TYPE_MAP.get(extension) ?? "image/png"; // Convert image to Blob const blob = await this.toBlob(mime); @@ -713,7 +696,7 @@ export class RawImage { const dataURL = URL.createObjectURL(blob); // Create an anchor element with the data URL as the href attribute - const downloadLink = document.createElement('a'); + const downloadLink = document.createElement("a"); downloadLink.href = dataURL; // Set the download attribute to specify the desired filename for the downloaded image @@ -724,10 +707,8 @@ export class RawImage { // Clean up: remove the anchor element from the DOM downloadLink.remove(); - } else if (!env.useFS) { - throw new Error('Unable to save the image because filesystem is disabled in this environment.') - + throw new Error("Unable to save the image because filesystem is disabled in this environment."); } else { const img = this.toSharp(); return await img.toFile(path); @@ -736,15 +717,15 @@ export class RawImage { toSharp() { if (BROWSER_ENV) { - throw new Error('toSharp() is only supported in server-side environments.') + throw new Error("toSharp() is only supported in server-side environments."); } return sharp(this.data, { raw: { width: this.width, height: this.height, - channels: this.channels - } + channels: this.channels, + }, }); } } diff --git a/src/utils/maths.js b/src/utils/maths.js index e6cb2d6ca..13b5cfefc 100644 --- a/src/utils/maths.js +++ b/src/utils/maths.js @@ -1,10 +1,9 @@ - /** - * @file Helper module for mathematical processing. - * - * These functions and classes are only used internally, + * @file Helper module for mathematical processing. + * + * These functions and classes are only used internally, * meaning an end-user shouldn't need to access anything here. - * + * * @module utils/maths */ @@ -17,7 +16,13 @@ /** * @param {TypedArray} input */ -export function interpolate_data(input, [in_channels, in_height, in_width], [out_height, out_width], mode = 'bilinear', align_corners = false) { +export function interpolate_data( + input, + [in_channels, in_height, in_width], + [out_height, out_width], + mode = "bilinear", + align_corners = false, +) { // TODO use mode and align_corners // Output image dimensions @@ -51,7 +56,6 @@ export function interpolate_data(input, [in_channels, in_height, in_width], [out x1 = Math.max(x1, 0); y1 = Math.max(y1, 0); - // Calculate the fractional distances between the input pixel and the four nearest pixels const s = x - x1; const t = y - y1; @@ -86,13 +90,12 @@ export function interpolate_data(input, [in_channels, in_height, in_width], [out return out_img; } - /** * Helper method to permute a `AnyTypedArray` directly - * @template {AnyTypedArray} T - * @param {T} array - * @param {number[]} dims - * @param {number[]} axes + * @template {AnyTypedArray} T + * @param {T} array + * @param {number[]} dims + * @param {number[]} axes * @returns {[T, number[]]} The permuted array and the new shape. */ export function permute_data(array, dims, axes) { @@ -127,7 +130,6 @@ export function permute_data(array, dims, axes) { return [permutedData, shape]; } - /** * Compute the softmax of an array of numbers. * @template {TypedArray|number[]} T @@ -139,16 +141,16 @@ export function softmax(arr) { const maxVal = max(arr)[0]; // Compute the exponentials of the array values - const exps = arr.map(x => Math.exp(x - maxVal)); + const exps = arr.map((x) => Math.exp(x - maxVal)); // Compute the sum of the exponentials // @ts-ignore const sumExps = exps.reduce((acc, val) => acc + val, 0); // Compute the softmax values - const softmaxArr = exps.map(x => x / sumExps); + const softmaxArr = exps.map((x) => x / sumExps); - return /** @type {T} */(softmaxArr); + return /** @type {T} */ (softmaxArr); } /** @@ -163,7 +165,7 @@ export function log_softmax(arr) { // Compute the sum of the exponentials let sumExps = 0; - for(let i = 0; i < arr.length; ++i) { + for (let i = 0; i < arr.length; ++i) { sumExps += Math.exp(arr[i] - maxVal); } @@ -171,9 +173,9 @@ export function log_softmax(arr) { const logSum = Math.log(sumExps); // Compute the softmax values - const logSoftmaxArr = arr.map(x => x - maxVal - logSum); + const logSoftmaxArr = arr.map((x) => x - maxVal - logSum); - return /** @type {T} */(logSoftmaxArr); + return /** @type {T} */ (logSoftmaxArr); } /** @@ -222,7 +224,6 @@ export function magnitude(arr) { return Math.sqrt(arr.reduce((acc, val) => acc + val * val, 0)); } - /** * Returns the value and index of the minimum element in an array. * @param {number[]|TypedArray} arr array of numbers. @@ -230,7 +231,7 @@ export function magnitude(arr) { * @throws {Error} If array is empty. */ export function min(arr) { - if (arr.length === 0) throw Error('Array must not be empty'); + if (arr.length === 0) throw Error("Array must not be empty"); let min = arr[0]; let indexOfMin = 0; for (let i = 1; i < arr.length; ++i) { @@ -242,7 +243,6 @@ export function min(arr) { return [min, indexOfMin]; } - /** * Returns the value and index of the maximum element in an array. * @param {number[]|AnyTypedArray} arr array of numbers. @@ -250,7 +250,7 @@ export function min(arr) { * @throws {Error} If array is empty. */ export function max(arr) { - if (arr.length === 0) throw Error('Array must not be empty'); + if (arr.length === 0) throw Error("Array must not be empty"); let max = arr[0]; let indexOfMax = 0; for (let i = 1; i < arr.length; ++i) { @@ -264,12 +264,12 @@ export function max(arr) { function isPowerOfTwo(number) { // Check if the number is greater than 0 and has only one bit set to 1 - return (number > 0) && ((number & (number - 1)) === 0); + return number > 0 && (number & (number - 1)) === 0; } /** * Implementation of Radix-4 FFT. - * + * * P2FFT class provides functionality for performing Fast Fourier Transform on arrays * which are a power of two in length. * Code adapted from https://www.npmjs.com/package/fft.js @@ -282,21 +282,20 @@ class P2FFT { constructor(size) { this.size = size | 0; // convert to a 32-bit signed integer if (this.size <= 1 || !isPowerOfTwo(this.size)) - throw new Error('FFT size must be a power of two larger than 1'); + throw new Error("FFT size must be a power of two larger than 1"); this._csize = size << 1; this.table = new Float64Array(this.size * 2); for (let i = 0; i < this.table.length; i += 2) { - const angle = Math.PI * i / this.size; + const angle = (Math.PI * i) / this.size; this.table[i] = Math.cos(angle); this.table[i + 1] = -Math.sin(angle); } // Find size's power of two let power = 0; - for (let t = 1; this.size > t; t <<= 1) - ++power; + for (let t = 1; this.size > t; t <<= 1) ++power; // Calculate initial step's width: // * If we are full radix-4, it is 2x smaller to give inital len=8 @@ -325,15 +324,14 @@ class P2FFT { /** * Converts a complex number representation stored in a Float64Array to an array of real numbers. - * + * * @param {Float64Array} complex The complex number representation to be converted. * @param {number[]} [storage] An optional array to store the result in. * @returns {number[]} An array of real numbers representing the input complex number representation. */ fromComplexArray(complex, storage) { const res = storage || new Array(complex.length >>> 1); - for (let i = 0; i < complex.length; i += 2) - res[i >>> 1] = complex[i]; + for (let i = 0; i < complex.length; i += 2) res[i >>> 1] = complex[i]; return res; } @@ -354,17 +352,16 @@ class P2FFT { /** * Performs a Fast Fourier Transform (FFT) on the given input data and stores the result in the output buffer. - * + * * @param {Float64Array} out The output buffer to store the result. * @param {Float64Array} data The input data to transform. - * + * * @throws {Error} Input and output buffers must be different. - * + * * @returns {void} */ transform(out, data) { - if (out === data) - throw new Error('Input and output buffers must be different'); + if (out === data) throw new Error("Input and output buffers must be different"); this._transform4(out, data, 1 /* DONE */); } @@ -380,8 +377,7 @@ class P2FFT { * @throws {Error} If the input and output buffers are the same. */ realTransform(out, data) { - if (out === data) - throw new Error('Input and output buffers must be different'); + if (out === data) throw new Error("Input and output buffers must be different"); this._realTransform4(out, data, 1 /* DONE */); } @@ -390,19 +386,17 @@ class P2FFT { * Performs an inverse FFT transformation on the given `data` array, and stores the result in `out`. * The `out` array must be a different buffer than the `data` array. The `out` array will contain the * result of the transformation. The `data` array will not be modified. - * + * * @param {Float64Array} out The output buffer for the transformed data. * @param {Float64Array} data The input data to transform. * @throws {Error} If `out` and `data` refer to the same buffer. * @returns {void} */ inverseTransform(out, data) { - if (out === data) - throw new Error('Input and output buffers must be different'); + if (out === data) throw new Error("Input and output buffers must be different"); this._transform4(out, data, -1 /* DONE */); - for (let i = 0; i < out.length; ++i) - out[i] /= this.size; + for (let i = 0; i < out.length; ++i) out[i] /= this.size; } /** @@ -538,7 +532,7 @@ class P2FFT { * @param {number} off Index of input array to start reading from * @param {number} step Step size between elements in input array * @param {number} inv Scaling factor for inverse transform - * + * * @returns {void} */ _singleTransform4(data, out, outOff, off, step, inv) { @@ -678,8 +672,7 @@ class P2FFT { } // Do not overwrite ourselves - if (i === hquarterLen) - continue; + if (i === hquarterLen) continue; const SA = outOff + quarterLen - i; const SB = outOff + halfLen - i; @@ -702,13 +695,13 @@ class P2FFT { /** * Performs a single real input radix-2 transformation on the provided data - * + * * @param {Float64Array} data The input data array * @param {Float64Array} out The output data array * @param {number} outOff The output offset * @param {number} off The input offset * @param {number} step The step - * + * * @returns {void} */ _singleRealTransform2(data, out, outOff, off, step) { @@ -768,11 +761,10 @@ class P2FFT { /** * NP2FFT class provides functionality for performing Fast Fourier Transform on arrays * which are not a power of two in length. In such cases, the chirp-z transform is used. - * + * * For more information, see: https://math.stackexchange.com/questions/77118/non-power-of-2-ffts/77156#77156 */ class NP2FFT { - /** * Constructs a new NP2FFT object. * @param {number} fft_length The length of the FFT @@ -781,7 +773,7 @@ class NP2FFT { // Helper variables const a = 2 * (fft_length - 1); const b = 2 * (2 * fft_length - 1); - const nextP2 = 2 ** (Math.ceil(Math.log2(b))) + const nextP2 = 2 ** Math.ceil(Math.log2(b)); this.bufferSize = nextP2; this._a = a; @@ -796,7 +788,7 @@ class NP2FFT { this._outBuffer2 = new Float64Array(nextP2); // Compute complex exponentiation - const theta = -2 * Math.PI / fft_length; + const theta = (-2 * Math.PI) / fft_length; const baseR = Math.cos(theta); const baseI = Math.sin(theta); @@ -817,7 +809,7 @@ class NP2FFT { // conjugate ichirp[i2] = chirp[i2]; - ichirp[i2 + 1] = - chirp[i2 + 1]; + ichirp[i2 + 1] = -chirp[i2 + 1]; } this._slicedChirpBuffer = chirp.subarray(a, b); @@ -839,7 +831,7 @@ class NP2FFT { if (real) { // Real multiplication for (let j = 0; j < sb.length; j += 2) { - const j2 = j + 1 + const j2 = j + 1; const j3 = j >> 1; const a_real = input[j3]; @@ -849,7 +841,7 @@ class NP2FFT { } else { // Complex multiplication for (let j = 0; j < sb.length; j += 2) { - const j2 = j + 1 + const j2 = j + 1; ib1[j] = input[j] * sb[j] - input[j2] * sb[j2]; ib1[j2] = input[j] * sb[j2] + input[j2] * sb[j]; } @@ -906,16 +898,14 @@ export class FFT { } } - /** * Performs median filter on the provided data. Padding is done by mirroring the data. * @param {AnyTypedArray} data The input array * @param {number} windowSize The window size */ export function medianFilter(data, windowSize) { - if (windowSize % 2 === 0 || windowSize <= 0) { - throw new Error('Window size must be a positive odd number'); + throw new Error("Window size must be a positive odd number"); } // @ts-ignore @@ -962,7 +952,7 @@ export function round(num, decimals) { * Helper function to round a number to the nearest integer, with ties rounded to the nearest even number. * Also known as "bankers' rounding". This is the default rounding mode in python. For example: * 1.5 rounds to 2 and 2.5 rounds to 2. - * + * * @param {number} x The number to round * @returns {number} The rounded number */ @@ -972,11 +962,10 @@ export function bankers_round(x) { return br; } - /** * Measures similarity between two temporal sequences (e.g., input audio and output tokens * to generate token-level timestamps). - * @param {number[][]} matrix + * @param {number[][]} matrix * @returns {number[][]} */ export function dynamic_time_warping(matrix) { @@ -985,16 +974,10 @@ export function dynamic_time_warping(matrix) { const outputShape = [output_length + 1, input_length + 1]; - const cost = Array.from( - { length: outputShape[0] }, - () => Array(outputShape[1]).fill(Infinity) - ); + const cost = Array.from({ length: outputShape[0] }, () => Array(outputShape[1]).fill(Infinity)); cost[0][0] = 0; - const trace = Array.from( - { length: outputShape[0] }, - () => Array(outputShape[1]).fill(-1) - ); + const trace = Array.from({ length: outputShape[0] }, () => Array(outputShape[1]).fill(-1)); for (let j = 1; j < outputShape[1]; ++j) { for (let i = 1; i < outputShape[0]; ++i) { @@ -1018,10 +1001,12 @@ export function dynamic_time_warping(matrix) { } } - for (let i = 0; i < outputShape[1]; ++i) { // trace[0, :] = 2 + for (let i = 0; i < outputShape[1]; ++i) { + // trace[0, :] = 2 trace[0][i] = 2; } - for (let i = 0; i < outputShape[0]; ++i) { // trace[:, 0] = 1 + for (let i = 0; i < outputShape[0]; ++i) { + // trace[:, 0] = 1 trace[i][0] = 1; } @@ -1036,7 +1021,8 @@ export function dynamic_time_warping(matrix) { switch (trace[i][j]) { case 0: - --i; --j; + --i; + --j; break; case 1: --i; @@ -1046,8 +1032,8 @@ export function dynamic_time_warping(matrix) { break; default: throw new Error( - `Internal error in dynamic time warping. Unexpected trace[${i}, ${j}]. Please file a bug report.` - ) + `Internal error in dynamic time warping. Unexpected trace[${i}, ${j}]. Please file a bug report.`, + ); } } @@ -1055,5 +1041,4 @@ export function dynamic_time_warping(matrix) { time_indices.reverse(); return [text_indices, time_indices]; - } diff --git a/src/utils/tensor.js b/src/utils/tensor.js index 536a8c249..82b4f2a8f 100644 --- a/src/utils/tensor.js +++ b/src/utils/tensor.js @@ -7,16 +7,11 @@ * @module utils/tensor */ -import { - interpolate_data, - permute_data -} from './maths.js'; +import { interpolate_data, permute_data } from "./maths.js"; -import { - Tensor as ONNXTensor, isONNXTensor, -} from '../backends/onnx.js'; +import { Tensor as ONNXTensor, isONNXTensor } from "../backends/onnx.js"; -import { TensorOpRegistry } from '../ops/registry.js'; +import { TensorOpRegistry } from "../ops/registry.js"; const DataTypeMap = Object.freeze({ float32: Float32Array, @@ -39,7 +34,6 @@ const DataTypeMap = Object.freeze({ * @typedef {import('./maths.js').AnyTypedArray | any[]} DataArray */ - export class Tensor { /** @type {number[]} Dimensions of the tensor. */ get dims() { @@ -55,7 +49,7 @@ export class Tensor { /** @type {DataType} Type of the tensor. */ get type() { return this.ort_tensor.type; - }; + } /** @type {DataArray} The data stored in the tensor. */ get data() { @@ -65,12 +59,12 @@ export class Tensor { /** @type {number} The number of elements in the tensor. */ get size() { return this.ort_tensor.size; - }; + } /** @type {string} The location of the tensor data. */ get location() { return this.ort_tensor.location; - }; + } ort_tensor; @@ -84,15 +78,15 @@ export class Tensor { } else { // Create new tensor this.ort_tensor = new ONNXTensor( - /** @type {DataType} */(args[0]), - /** @type {Exclude} */(args[1]), - args[2] + /** @type {DataType} */ (args[0]), + /** @type {Exclude} */ (args[1]), + args[2], ); } return new Proxy(this, { get: (obj, key) => { - if (typeof key === 'string') { + if (typeof key === "string") { let index = Number(key); if (Number.isInteger(index)) { // key is an integer (i.e., index) @@ -106,8 +100,8 @@ export class Tensor { // TODO allow setting of data // @ts-ignore - return obj[key] = value; - } + return (obj[key] = value); + }, }); } @@ -130,9 +124,8 @@ export class Tensor { yield this._subarray(i, iterSize, iterDims); } } else { - yield* this.data + yield* this.data; } - } /** @@ -179,10 +172,7 @@ export class Tensor { const o2 = (index + 1) * iterSize; // We use subarray if available (typed array), otherwise we use slice (normal array) - const data = - ('subarray' in this.data) - ? this.data.subarray(o1, o2) - : this.data.slice(o1, o2); + const data = "subarray" in this.data ? this.data.subarray(o1, o2) : this.data.slice(o1, o2); return new Tensor(this.type, data, iterDims); } @@ -205,7 +195,7 @@ export class Tensor { * @returns {Array} */ tolist() { - return reshape(this.data, this.dims) + return reshape(this.data, this.dims); } /** @@ -358,35 +348,25 @@ export class Tensor { // null or undefined means take the whole dimension newOffsets.push([0, this.dims[sliceIndex]]); newTensorDims.push(this.dims[sliceIndex]); - - } else if (typeof slice === 'number') { + } else if (typeof slice === "number") { slice = safeIndex(slice, this.dims[sliceIndex], sliceIndex); // A number means take a single element newOffsets.push([slice, slice + 1]); - } else if (Array.isArray(slice) && slice.length === 2) { // An array of length 2 means take a range of elements let [start, end] = slice; - start = start === null - ? 0 - : safeIndex(start, this.dims[sliceIndex], sliceIndex, false); - end = end === null - ? this.dims[sliceIndex] - : safeIndex(end, this.dims[sliceIndex], sliceIndex, false); + start = start === null ? 0 : safeIndex(start, this.dims[sliceIndex], sliceIndex, false); + end = end === null ? this.dims[sliceIndex] : safeIndex(end, this.dims[sliceIndex], sliceIndex, false); if (start > end) { throw new Error(`Invalid slice: ${slice}`); } - const offsets = [ - Math.max(start, 0), - Math.min(end, this.dims[sliceIndex]) - ]; + const offsets = [Math.max(start, 0), Math.min(end, this.dims[sliceIndex])]; newOffsets.push(offsets); newTensorDims.push(offsets[1] - offsets[0]); - } else { throw new Error(`Invalid slice: ${slice}`); } @@ -413,7 +393,6 @@ export class Tensor { data[i] = this_data[originalIndex]; } return new Tensor(this.type, data, newTensorDims); - } /** @@ -451,11 +430,11 @@ export class Tensor { * @param {boolean} [keepdim=false] Whether the output tensors have dim retained or not. * @returns {Tensor} The norm of the tensor. */ - norm(p = 'fro', dim = null, keepdim = false) { - if (p === 'fro') { + norm(p = "fro", dim = null, keepdim = false) { + if (p === "fro") { // NOTE: Since we only support integer dims, Frobenius norm produces the same result as p=2. p = 2; - } else if (typeof p === 'string') { + } else if (typeof p === "string") { throw Error(`Unsupported norm: ${p}`); } @@ -463,7 +442,7 @@ export class Tensor { if (dim === null) { // @ts-ignore - let val = this_data.reduce((a, b) => a + (b ** p), 0) ** (1 / p); + let val = this_data.reduce((a, b) => a + b ** p, 0) ** (1 / p); return new Tensor(this.type, [val], []); } @@ -480,7 +459,6 @@ export class Tensor { // Iterate over the data array for (let i = 0; i < this_data.length; ++i) { - // Calculate the index in the resulting array let resultIndex = 0; @@ -495,7 +473,7 @@ export class Tensor { } // Accumulate the value at the current index - result[resultIndex] += (this_data[i]) ** p; + result[resultIndex] += this_data[i] ** p; } if (p !== 1) { @@ -525,7 +503,6 @@ export class Tensor { const this_data = this.data; const norm_data = norm.data; for (let i = 0; i < this_data.length; ++i) { - // Calculate the index in the resulting array let resultIndex = 0; @@ -575,11 +552,7 @@ export class Tensor { * @returns {Tensor} The squeezed tensor */ squeeze(dim = null) { - return new Tensor( - this.type, - this.data, - calc_squeeze_dims(this.dims, dim) - ) + return new Tensor(this.type, this.data, calc_squeeze_dims(this.dims, dim)); } /** @@ -599,11 +572,7 @@ export class Tensor { * @returns {Tensor} The unsqueezed tensor */ unsqueeze(dim = null) { - return new Tensor( - this.type, - this.data, - calc_unsqueeze_dims(this.dims, dim) - ); + return new Tensor(this.type, this.data, calc_unsqueeze_dims(this.dims, dim)); } /** @@ -625,7 +594,7 @@ export class Tensor { let dimsToFlatten = this.dims.slice(start_dim, end_dim + 1); let dimsToKeepAfter = this.dims.slice(end_dim + 1); - this.dims = [...dimsToKeepBefore, dimsToFlatten.reduce((a, b) => a * b, 1), ...dimsToKeepAfter] + this.dims = [...dimsToKeepBefore, dimsToFlatten.reduce((a, b) => a * b, 1), ...dimsToKeepAfter]; return this; } @@ -662,7 +631,7 @@ export class Tensor { if (inferredIndex !== -1) { // Some dimension must be inferred const productOther = dims.reduce((product, curr, index) => { - return index !== inferredIndex ? product * curr : product + return index !== inferredIndex ? product * curr : product; }, 1); dims[inferredIndex] = this_data.length / productOther; @@ -773,7 +742,6 @@ export class Tensor { * @returns {NestArray} The reshaped array. */ function reshape(data, dimensions) { - const totalElements = data.length; const dimensionSize = dimensions.reduce((a, b) => a * b); @@ -785,17 +753,20 @@ function reshape(data, dimensions) { let reshapedArray = data; for (let i = dimensions.length - 1; i >= 0; i--) { - reshapedArray = reshapedArray.reduce((acc, val) => { - let lastArray = acc[acc.length - 1]; - - if (lastArray.length < dimensions[i]) { - lastArray.push(val); - } else { - acc.push([val]); - } + reshapedArray = reshapedArray.reduce( + (acc, val) => { + let lastArray = acc[acc.length - 1]; + + if (lastArray.length < dimensions[i]) { + lastArray.push(val); + } else { + acc.push([val]); + } - return acc; - }, [[]]); + return acc; + }, + [[]], + ); } return reshapedArray[0]; @@ -812,7 +783,6 @@ export function permute(tensor, axes) { return new Tensor(tensor.type, permutedData, shape); } - /** * Interpolates an Tensor to the given size. * @param {Tensor} input The input tensor to interpolate. Data must be channel-first (i.e., [c, h, w]) @@ -821,24 +791,22 @@ export function permute(tensor, axes) { * @param {boolean} align_corners Whether to align corners. * @returns {Tensor} The interpolated tensor. */ -export function interpolate(input, [out_height, out_width], mode = 'bilinear', align_corners = false) { - +export function interpolate(input, [out_height, out_width], mode = "bilinear", align_corners = false) { // Input image dimensions const in_channels = input.dims.at(-3) ?? 1; const in_height = input.dims.at(-2); const in_width = input.dims.at(-1); let output = interpolate_data( - /** @type {import('./maths.js').TypedArray}*/(input.data), + /** @type {import('./maths.js').TypedArray}*/ (input.data), [in_channels, in_height, in_width], [out_height, out_width], mode, - align_corners + align_corners, ); return new Tensor(input.type, output, [in_channels, out_height, out_width]); } - /** * Down/up samples the input. * Inspired by https://pytorch.org/docs/stable/generated/torch.nn.functional.interpolate.html. @@ -848,18 +816,14 @@ export function interpolate(input, [out_height, out_width], mode = 'bilinear', a * @param {"bilinear"|"bicubic"} [options.mode='bilinear'] algorithm used for upsampling * @returns {Promise} The interpolated tensor. */ -export async function interpolate_4d(input, { - size = null, - mode = 'bilinear', -} = {}) { - +export async function interpolate_4d(input, { size = null, mode = "bilinear" } = {}) { // Error checking if (input.dims.length !== 4) { - throw new Error('`interpolate_4d` currently only supports 4D input.'); + throw new Error("`interpolate_4d` currently only supports 4D input."); } if (!size) { // TODO: support scale_factor - throw new Error('`interpolate_4d` requires a `size` argument.'); + throw new Error("`interpolate_4d` requires a `size` argument."); } // Fill in missing dimensions @@ -871,19 +835,19 @@ export async function interpolate_4d(input, { } else if (size.length === 4) { targetDims = size; } else { - throw new Error('`size` must be of length 2, 3, or 4.'); + throw new Error("`size` must be of length 2, 3, or 4."); } let op; - if (mode === 'bilinear') { + if (mode === "bilinear") { op = await TensorOpRegistry.bilinear_interpolate_4d; - } else if (mode === 'bicubic') { + } else if (mode === "bicubic") { op = await TensorOpRegistry.bicubic_interpolate_4d; } else { throw new Error(`Unsupported mode: ${mode}`); } - const sizeTensor = new Tensor('int64', new BigInt64Array(targetDims.map(BigInt)), [targetDims.length]); + const sizeTensor = new Tensor("int64", new BigInt64Array(targetDims.map(BigInt)), [targetDims.length]); return await op({ x: input, s: sizeTensor }); } @@ -911,7 +875,6 @@ export async function rfft(x, a) { return await op({ x, a }); } - /** * Returns the k largest elements of the given input tensor. * Inspired by https://pytorch.org/docs/stable/generated/torch.topk.html @@ -929,11 +892,7 @@ export async function topk(x, k) { } return await op({ x, - k: new Tensor( - 'int64', - [BigInt(k)], - [1] - ) + k: new Tensor("int64", [BigInt(k)], [1]), }); } @@ -979,11 +938,7 @@ export function mean_pooling(last_hidden_state, attention_mask) { } } - return new Tensor( - last_hidden_state.type, - returnedData, - shape - ) + return new Tensor(last_hidden_state.type, returnedData, shape); } /** @@ -994,24 +949,22 @@ export function mean_pooling(last_hidden_state, attention_mask) { * @param {number} [options.eps=1e-5] A value added to the denominator for numerical stability. * @returns {Tensor} The normalized tensor. */ -export function layer_norm(input, normalized_shape, { - eps = 1e-5, -} = {}) { +export function layer_norm(input, normalized_shape, { eps = 1e-5 } = {}) { if (input.dims.length !== 2) { - throw new Error('`layer_norm` currently only supports 2D input.'); + throw new Error("`layer_norm` currently only supports 2D input."); } const [batchSize, featureDim] = input.dims; if (normalized_shape.length !== 1 && normalized_shape[0] !== featureDim) { - throw new Error('`normalized_shape` must be a 1D array with shape `[input.dims[1]]`.'); + throw new Error("`normalized_shape` must be a 1D array with shape `[input.dims[1]]`."); } const [std, mean] = std_mean(input, 1, 0, true); - const stdData = /** @type {Float32Array} */(std.data); - const meanData = /** @type {Float32Array} */(mean.data); + const stdData = /** @type {Float32Array} */ (std.data); + const meanData = /** @type {Float32Array} */ (mean.data); - const inputData = /** @type {Float32Array} */(input.data); + const inputData = /** @type {Float32Array} */ (input.data); // @ts-ignore const returnedData = new inputData.constructor(inputData.length); @@ -1037,7 +990,7 @@ function calc_squeeze_dims(dims, dim) { dims = dims.slice(); if (dim === null) { dims = dims.filter((d) => d !== 1); - } else if (typeof dim === 'number') { + } else if (typeof dim === "number") { if (dims[dim] === 1) { dims.splice(dim, 1); } @@ -1078,7 +1031,9 @@ function calc_unsqueeze_dims(dims, dim) { */ function safeIndex(index, size, dimension = null, boundsCheck = true) { if (boundsCheck && (index < -size || index >= size)) { - throw new Error(`IndexError: index ${index} is out of bounds for dimension${dimension === null ? '' : ' ' + dimension} with size ${size}`); + throw new Error( + `IndexError: index ${index} is out of bounds for dimension${dimension === null ? "" : " " + dimension} with size ${size}`, + ); } if (index < 0) { @@ -1119,9 +1074,7 @@ export function cat(tensors, dim = 0) { result.set(tensorData, offset); offset += tensorData.length; } - } else { - let currentDim = 0; for (let t = 0; t < tensors.length; ++t) { @@ -1161,10 +1114,12 @@ export function cat(tensors, dim = 0) { export function stack(tensors, dim = 0) { // TODO do validation of shapes // NOTE: stack expects each tensor to be equal size - return cat(tensors.map(t => t.unsqueeze(dim)), dim); + return cat( + tensors.map((t) => t.unsqueeze(dim)), + dim, + ); } - /** * Calculates the standard deviation and mean over the dimensions specified by dim. dim can be a single dimension or `null` to reduce over all dimensions. * @param {Tensor} input the input tenso @@ -1174,7 +1129,7 @@ export function stack(tensors, dim = 0) { * @returns {Tensor[]} A tuple of (std, mean) tensors. */ export function std_mean(input, dim = null, correction = 1, keepdim = false) { - const inputData = /** @type {Float32Array} */(input.data); + const inputData = /** @type {Float32Array} */ (input.data); const inputDims = input.dims; if (dim === null) { @@ -1183,8 +1138,20 @@ export function std_mean(input, dim = null, correction = 1, keepdim = false) { const mean = sum / inputData.length; const std = Math.sqrt(inputData.reduce((a, b) => a + (b - mean) ** 2, 0) / (inputData.length - correction)); - const meanTensor = new Tensor(input.type, [mean], [/* scalar */]); - const stdTensor = new Tensor(input.type, [std], [/* scalar */]); + const meanTensor = new Tensor( + input.type, + [mean], + [ + /* scalar */ + ], + ); + const stdTensor = new Tensor( + input.type, + [std], + [ + /* scalar */ + ], + ); return [stdTensor, meanTensor]; } @@ -1205,7 +1172,6 @@ export function std_mean(input, dim = null, correction = 1, keepdim = false) { // Iterate over the data array for (let i = 0; i < inputData.length; ++i) { - // Calculate the index in the resulting array let resultIndex = 0; @@ -1236,7 +1202,6 @@ export function std_mean(input, dim = null, correction = 1, keepdim = false) { return [stdTensor, meanTensor]; } - /** * Returns the mean value of each row of the input tensor in the given dimension dim. * @param {Tensor} input the input tensor. @@ -1245,13 +1210,19 @@ export function std_mean(input, dim = null, correction = 1, keepdim = false) { * @returns {Tensor} A new tensor with means taken along the specified dimension. */ export function mean(input, dim = null, keepdim = false) { - const inputData = /** @type {Float32Array} */(input.data); + const inputData = /** @type {Float32Array} */ (input.data); if (dim === null) { // None to reduce over all dimensions. // @ts-ignore const val = inputData.reduce((a, b) => a + b, 0); - return new Tensor(input.type, [val / inputData.length], [/* scalar */]); + return new Tensor( + input.type, + [val / inputData.length], + [ + /* scalar */ + ], + ); } const inputDims = input.dims; @@ -1268,7 +1239,6 @@ export function mean(input, dim = null, keepdim = false) { // Iterate over the data array for (let i = 0; i < inputData.length; ++i) { - // Calculate the index in the resulting array let resultIndex = 0; @@ -1299,7 +1269,6 @@ export function mean(input, dim = null, keepdim = false) { return new Tensor(input.type, result, resultDims); } - function dimsToStride(dims) { const stride = new Array(dims.length); for (let i = dims.length - 1, s2 = 1; i >= 0; --i) { @@ -1311,11 +1280,7 @@ function dimsToStride(dims) { function fullHelper(size, fill_value, dtype, cls) { const numElements = size.reduce((a, b) => a * b, 1); - return new Tensor( - dtype, - new cls(numElements).fill(fill_value), - size - ) + return new Tensor(dtype, new cls(numElements).fill(fill_value), size); } /** @@ -1327,11 +1292,11 @@ function fullHelper(size, fill_value, dtype, cls) { export function full(size, fill_value) { let dtype; let typedArrayCls; - if (typeof fill_value === 'number') { - dtype = 'float32'; + if (typeof fill_value === "number") { + dtype = "float32"; typedArrayCls = Float32Array; - } else if (typeof fill_value === 'bigint') { - dtype = 'int64'; + } else if (typeof fill_value === "bigint") { + dtype = "int64"; typedArrayCls = BigInt64Array; } else { // TODO: support other dtypes @@ -1350,7 +1315,7 @@ export function full_like(tensor, fill_value) { * @returns {Tensor} The ones tensor. */ export function ones(size) { - return fullHelper(size, 1n, 'int64', BigInt64Array); + return fullHelper(size, 1n, "int64", BigInt64Array); } /** @@ -1368,7 +1333,7 @@ export function ones_like(tensor) { * @returns {Tensor} The zeros tensor. */ export function zeros(size) { - return fullHelper(size, 0n, 'int64', BigInt64Array); + return fullHelper(size, 0n, "int64", BigInt64Array); } /** @@ -1393,12 +1358,12 @@ export function quantize_embeddings(tensor, precision) { if (tensor.dims.at(-1) % 8 !== 0) { throw new Error("The last dimension of the tensor must be a multiple of 8"); } - if (!['binary', 'ubinary'].includes(precision)) { + if (!["binary", "ubinary"].includes(precision)) { throw new Error("The precision must be either 'binary' or 'ubinary'"); } - const signed = precision === 'binary'; - const dtype = signed ? 'int8' : 'uint8'; + const signed = precision === "binary"; + const dtype = signed ? "int8" : "uint8"; // Create a typed array to store the packed bits const cls = signed ? Int8Array : Uint8Array; @@ -1419,7 +1384,7 @@ export function quantize_embeddings(tensor, precision) { if (signed && bitPosition === 0) { outputData[arrayIndex] -= 128; } - }; + } return new Tensor(dtype, outputData, [tensor.dims[0], tensor.dims[1] / 8]); } diff --git a/tests/bundles.test.js b/tests/bundles.test.js index 3759e1706..3f381d9e5 100644 --- a/tests/bundles.test.js +++ b/tests/bundles.test.js @@ -14,24 +14,24 @@ const TARGET_OUTPUT = "erdingsAndroid Load"; const wrap_async_iife = (code) => `(async function() { ${code} })();`; const check = (code, module = false) => { - const args = ["-e", code]; - if (module) args.push("--input-type=module"); - const { status, stdout, stderr } = spawnSync("node", args); - expect(stderr.toString()).toBe(""); // No warnings or errors are printed - expect(stdout.toString()).toBe(TARGET_OUTPUT); // The output should match - expect(status).toBe(0); // The process should exit cleanly + const args = ["-e", code]; + if (module) args.push("--input-type=module"); + const { status, stdout, stderr } = spawnSync("node", args); + expect(stderr.toString()).toBe(""); // No warnings or errors are printed + expect(stdout.toString()).toBe(TARGET_OUTPUT); // The output should match + expect(status).toBe(0); // The process should exit cleanly }; describe("Testing the bundle", () => { - it("ECMAScript Module (ESM)", () => { - check(`import { pipeline } from "${MODULE_NAME}";${CODE_BODY}`, true); - }); + it("ECMAScript Module (ESM)", () => { + check(`import { pipeline } from "${MODULE_NAME}";${CODE_BODY}`, true); + }); - it("CommonJS (CJS) with require", () => { - check(`const { pipeline } = require("${MODULE_NAME}");${wrap_async_iife(CODE_BODY)}`); - }); + it("CommonJS (CJS) with require", () => { + check(`const { pipeline } = require("${MODULE_NAME}");${wrap_async_iife(CODE_BODY)}`); + }); - it("CommonJS (CJS) with dynamic import", () => { - check(`${wrap_async_iife(`const { pipeline } = await import("${MODULE_NAME}");${CODE_BODY}`)}`); - }); + it("CommonJS (CJS) with dynamic import", () => { + check(`${wrap_async_iife(`const { pipeline } = await import("${MODULE_NAME}");${CODE_BODY}`)}`); + }); }); diff --git a/tests/configs.test.js b/tests/configs.test.js index f66a8a887..ba21f7df7 100644 --- a/tests/configs.test.js +++ b/tests/configs.test.js @@ -6,18 +6,18 @@ env.allowLocalModels = false; env.useFSCache = false; const TEST_DATA = { - "Xenova/bert-base-uncased": { - model_type: "bert", - }, + "Xenova/bert-base-uncased": { + model_type: "bert", + }, }; describe("Configs", () => { - for (const [model_id, minimal_config] of Object.entries(TEST_DATA)) { - it(model_id, async () => { - const config = await AutoConfig.from_pretrained(model_id); - for (const [key, value] of Object.entries(minimal_config)) { - expect(config[key]).toEqual(value); - } - }); - } + for (const [model_id, minimal_config] of Object.entries(TEST_DATA)) { + it(model_id, async () => { + const config = await AutoConfig.from_pretrained(model_id); + for (const [key, value] of Object.entries(minimal_config)) { + expect(config[key]).toEqual(value); + } + }); + } }); diff --git a/tests/init.js b/tests/init.js index 65f079086..13ad9557f 100644 --- a/tests/init.js +++ b/tests/init.js @@ -13,48 +13,48 @@ import * as ONNX_COMMON from "onnxruntime-common"; * For more information, see: https://github.com/jestjs/jest/issues/11864#issuecomment-1261468011 */ export function init() { - // In rare cases (specifically when running unit tests with GitHub actions), possibly due to - // a large number of concurrent executions, onnxruntime might fallback to use the WASM backend. - // In this case, we set the number of threads to 1 to avoid errors like: - // - `TypeError: The worker script or module filename must be an absolute path or a relative path starting with './' or '../'. Received "blob:nodedata:..."` - ONNX_COMMON.env.wasm.numThreads = 1; + // In rare cases (specifically when running unit tests with GitHub actions), possibly due to + // a large number of concurrent executions, onnxruntime might fallback to use the WASM backend. + // In this case, we set the number of threads to 1 to avoid errors like: + // - `TypeError: The worker script or module filename must be an absolute path or a relative path starting with './' or '../'. Received "blob:nodedata:..."` + ONNX_COMMON.env.wasm.numThreads = 1; - let registerBackend = ONNX_COMMON.registerBackend; + let registerBackend = ONNX_COMMON.registerBackend; - // Define the constructors to monkey-patch - const TYPED_ARRAYS_CONSTRUCTOR_NAMES = ["Int8Array", "Int16Array", "Int32Array", "BigInt64Array", "Uint8Array", "Uint8ClampedArray", "Uint16Array", "Uint32Array", "BigUint64Array", "Float32Array", "Float64Array"]; + // Define the constructors to monkey-patch + const TYPED_ARRAYS_CONSTRUCTOR_NAMES = ["Int8Array", "Int16Array", "Int32Array", "BigInt64Array", "Uint8Array", "Uint8ClampedArray", "Uint16Array", "Uint32Array", "BigUint64Array", "Float32Array", "Float64Array"]; - // Keep a reference to the original initialization method - const originalMethod = onnxruntimeBackend.init; + // Keep a reference to the original initialization method + const originalMethod = onnxruntimeBackend.init; - // Monkey-patch the initialization function - onnxruntimeBackend.init = function (...args) { - // There is probably a better way to do this - Array.isArray = (x) => typeof x === "object" && x !== null && typeof x.length === "number" && x?.constructor.toString() === Array.toString(); + // Monkey-patch the initialization function + onnxruntimeBackend.init = function (...args) { + // There is probably a better way to do this + Array.isArray = (x) => typeof x === "object" && x !== null && typeof x.length === "number" && x?.constructor.toString() === Array.toString(); - // For each typed array constructor - for (const ctorName of TYPED_ARRAYS_CONSTRUCTOR_NAMES) { - // Get the constructor from the current context - const ctor = globalThis[ctorName]; + // For each typed array constructor + for (const ctorName of TYPED_ARRAYS_CONSTRUCTOR_NAMES) { + // Get the constructor from the current context + const ctor = globalThis[ctorName]; - // Get the corresponding test function from the `util` module - const value = types[`is${ctorName}`].bind(types); + // Get the corresponding test function from the `util` module + const value = types[`is${ctorName}`].bind(types); - // Monkey-patch the constructor so "x instanceof ctor" returns "types[`is${ctorName}`](x)" - Object.defineProperty(ctor, Symbol.hasInstance, { - value, - writable: true, // writable=true is necessary to overwrite the default implementation (and allow subsequent overwrites) - configurable: false, - enumerable: false, - }); - } + // Monkey-patch the constructor so "x instanceof ctor" returns "types[`is${ctorName}`](x)" + Object.defineProperty(ctor, Symbol.hasInstance, { + value, + writable: true, // writable=true is necessary to overwrite the default implementation (and allow subsequent overwrites) + configurable: false, + enumerable: false, + }); + } - // Call the original method - return originalMethod.apply(this, args); - }; + // Call the original method + return originalMethod.apply(this, args); + }; - // Register the backend with the highest priority, so it is used instead of the default one - registerBackend("test", onnxruntimeBackend, Number.POSITIVE_INFINITY); + // Register the backend with the highest priority, so it is used instead of the default one + registerBackend("test", onnxruntimeBackend, Number.POSITIVE_INFINITY); } export const MAX_MODEL_LOAD_TIME = 10_000; // 10 seconds diff --git a/tests/models.test.js b/tests/models.test.js index f1bc7961c..2aa7b2e35 100644 --- a/tests/models.test.js +++ b/tests/models.test.js @@ -12,119 +12,126 @@ import { compare } from "./test_utils.js"; init(); describe("Models", () => { - describe("Loading different architecture types", () => { - // List all models which will be tested - const models_to_test = [ - // [name, modelClass, tokenizerClass] - ["hf-internal-testing/tiny-random-BertForMaskedLM", BertModel, BertTokenizer], // Encoder-only - ["hf-internal-testing/tiny-random-GPT2LMHeadModel", GPT2Model, GPT2Tokenizer], // Decoder-only - ["hf-internal-testing/tiny-random-T5ForConditionalGeneration", T5ForConditionalGeneration, T5Tokenizer], // Encoder-decoder - ]; - - const texts = ["Once upon a time", "I like to eat apples"]; - - for (const [model_id, modelClass, tokenizerClass] of models_to_test) { - // Test that both the auto model and the specific model work - const tokenizers = [AutoTokenizer, tokenizerClass]; - const models = [AutoModel, modelClass]; - - for (let i = 0; i < tokenizers.length; ++i) { - const tokenizerClassToTest = tokenizers[i]; - const modelClassToTest = models[i]; + describe("Loading different architecture types", () => { + // List all models which will be tested + const models_to_test = [ + // [name, modelClass, tokenizerClass] + ["hf-internal-testing/tiny-random-BertForMaskedLM", BertModel, BertTokenizer], // Encoder-only + ["hf-internal-testing/tiny-random-GPT2LMHeadModel", GPT2Model, GPT2Tokenizer], // Decoder-only + ["hf-internal-testing/tiny-random-T5ForConditionalGeneration", T5ForConditionalGeneration, T5Tokenizer], // Encoder-decoder + ]; + + const texts = ["Once upon a time", "I like to eat apples"]; + + for (const [model_id, modelClass, tokenizerClass] of models_to_test) { + // Test that both the auto model and the specific model work + const tokenizers = [AutoTokenizer, tokenizerClass]; + const models = [AutoModel, modelClass]; + + for (let i = 0; i < tokenizers.length; ++i) { + const tokenizerClassToTest = tokenizers[i]; + const modelClassToTest = models[i]; + + it( + `${model_id} (${modelClassToTest.name})`, + async () => { + // Load model and tokenizer + const tokenizer = await tokenizerClassToTest.from_pretrained(model_id); + const model = await modelClassToTest.from_pretrained(model_id); + + const tests = [ + texts[0], // single + texts, // batched + ]; + for (const test of tests) { + const inputs = await tokenizer(test, { + truncation: true, + padding: true, + }); + if (model.config.is_encoder_decoder) { + inputs.decoder_input_ids = inputs.input_ids; + } + const output = await model(inputs); + + if (output.logits) { + // Ensure correct shapes + const expected_shape = [...inputs.input_ids.dims, model.config.vocab_size]; + const actual_shape = output.logits.dims; + compare(expected_shape, actual_shape); + } else if (output.last_hidden_state) { + const expected_shape = [...inputs.input_ids.dims, model.config.d_model]; + const actual_shape = output.last_hidden_state.dims; + compare(expected_shape, actual_shape); + } else { + console.warn("Unexpected output", output); + throw new Error("Unexpected output"); + } + } + + await model.dispose(); + }, + MAX_TEST_EXECUTION_TIME, + ); + } + } + }); + describe("Running specific models", () => { + const models_to_test = ["hf-internal-testing/tiny-random-CLIPModel"]; it( - `${model_id} (${modelClassToTest.name})`, - async () => { - // Load model and tokenizer - const tokenizer = await tokenizerClassToTest.from_pretrained(model_id); - const model = await modelClassToTest.from_pretrained(model_id); - - const tests = [ - texts[0], // single - texts, // batched - ]; - for (const test of tests) { - const inputs = await tokenizer(test, { truncation: true, padding: true }); - if (model.config.is_encoder_decoder) { - inputs.decoder_input_ids = inputs.input_ids; - } - const output = await model(inputs); - - if (output.logits) { + `CLIP (text)`, + async () => { + const model_id = models_to_test[0]; + + // Load tokenizer and text model + const tokenizer = await AutoTokenizer.from_pretrained(model_id); + const text_model = await CLIPTextModelWithProjection.from_pretrained(model_id, { + revision: "refs/pr/5", + }); + + // Run tokenization + const texts = ["a photo of a car", "a photo of a football match"]; + const text_inputs = tokenizer(texts, { padding: true, truncation: true }); + + // Compute embeddings + const { text_embeds } = await text_model(text_inputs); + // Ensure correct shapes - const expected_shape = [...inputs.input_ids.dims, model.config.vocab_size]; - const actual_shape = output.logits.dims; + const expected_shape = [texts.length, text_model.config.projection_dim]; + const actual_shape = text_embeds.dims; compare(expected_shape, actual_shape); - } else if (output.last_hidden_state) { - const expected_shape = [...inputs.input_ids.dims, model.config.d_model]; - const actual_shape = output.last_hidden_state.dims; + + await text_model.dispose(); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + `CLIP (vision)`, + async () => { + const model_id = models_to_test[0]; + + // Load processor and vision model + const processor = await AutoProcessor.from_pretrained(model_id); + const vision_model = await CLIPVisionModelWithProjection.from_pretrained(model_id, { + revision: "refs/pr/5", + }); + + // Read image and run processor + const image = await RawImage.read("https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/football-match.jpg"); + const image_inputs = await processor(image); + + // Compute embeddings + const { image_embeds } = await vision_model(image_inputs); + + // Ensure correct shapes + const expected_shape = [1, vision_model.config.projection_dim]; + const actual_shape = image_embeds.dims; compare(expected_shape, actual_shape); - } else { - console.warn("Unexpected output", output); - throw new Error("Unexpected output"); - } - } - await model.dispose(); - }, - MAX_TEST_EXECUTION_TIME, + await vision_model.dispose(); + }, + MAX_TEST_EXECUTION_TIME, ); - } - } - }); - - describe("Running specific models", () => { - const models_to_test = ["hf-internal-testing/tiny-random-CLIPModel"]; - it( - `CLIP (text)`, - async () => { - const model_id = models_to_test[0]; - - // Load tokenizer and text model - const tokenizer = await AutoTokenizer.from_pretrained(model_id); - const text_model = await CLIPTextModelWithProjection.from_pretrained(model_id, { revision: "refs/pr/5" }); - - // Run tokenization - const texts = ["a photo of a car", "a photo of a football match"]; - const text_inputs = tokenizer(texts, { padding: true, truncation: true }); - - // Compute embeddings - const { text_embeds } = await text_model(text_inputs); - - // Ensure correct shapes - const expected_shape = [texts.length, text_model.config.projection_dim]; - const actual_shape = text_embeds.dims; - compare(expected_shape, actual_shape); - - await text_model.dispose(); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - `CLIP (vision)`, - async () => { - const model_id = models_to_test[0]; - - // Load processor and vision model - const processor = await AutoProcessor.from_pretrained(model_id); - const vision_model = await CLIPVisionModelWithProjection.from_pretrained(model_id, { revision: "refs/pr/5" }); - - // Read image and run processor - const image = await RawImage.read("https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/football-match.jpg"); - const image_inputs = await processor(image); - - // Compute embeddings - const { image_embeds } = await vision_model(image_inputs); - - // Ensure correct shapes - const expected_shape = [1, vision_model.config.projection_dim]; - const actual_shape = image_embeds.dims; - compare(expected_shape, actual_shape); - - await vision_model.dispose(); - }, - MAX_TEST_EXECUTION_TIME, - ); - }); + }); }); diff --git a/tests/models/albert/tokenization.js b/tests/models/albert/tokenization.js index 875bc418f..b66bc1938 100644 --- a/tests/models/albert/tokenization.js +++ b/tests/models/albert/tokenization.js @@ -3,181 +3,181 @@ import { BASE_TEST_STRINGS, BERT_TEST_STRINGS } from "../test_strings.js"; export const TOKENIZER_CLASS = AlbertTokenizer; export const TEST_CONFIG = { - // - uses `StripAccents` normalizer - "Xenova/albert-base-v2": { - SIMPLE: { - text: BASE_TEST_STRINGS.SIMPLE, - tokens: ["\u2581how", "\u2581are", "\u2581you", "\u2581doing", "?"], - ids: [2, 184, 50, 42, 845, 60, 3], - decoded: "[CLS] how are you doing?[SEP]", + // - uses `StripAccents` normalizer + "Xenova/albert-base-v2": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["\u2581how", "\u2581are", "\u2581you", "\u2581doing", "?"], + ids: [2, 184, 50, 42, 845, 60, 3], + decoded: "[CLS] how are you doing?[SEP]", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["\u2581you", "\u2581should", "'", "ve", "\u2581done", "\u2581this"], + ids: [2, 42, 378, 22, 195, 677, 48, 3], + decoded: "[CLS] you should've done this[SEP]", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["\u25810", "12", "345", "67", "89", "\u25810", "\u25811", "\u25812", "\u25813", "\u25814", "\u25815", "\u25816", "\u25817", "\u25818", "\u25819", "\u258110", "\u2581100", "\u25811000"], + ids: [2, 713, 918, 21997, 4167, 3877, 713, 137, 172, 203, 268, 331, 400, 453, 469, 561, 332, 808, 6150, 3], + decoded: "[CLS] 0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000[SEP]", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["\u2581the", "\u2581company", "\u2581was", "\u2581founded", "\u2581in", "\u25812016", "."], + ids: [2, 14, 237, 23, 785, 19, 690, 9, 3], + decoded: "[CLS] the company was founded in 2016.[SEP]", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["\u2581a", "\u2581", "'", "ll", "\u2581", "!!", "to", "?'", "d", '"', "d", "\u2581of", ",", "\u2581can", "'", "t", "."], + ids: [2, 21, 13, 22, 211, 13, 19015, 262, 5663, 43, 7, 43, 16, 15, 92, 22, 38, 9, 3], + decoded: "[CLS] a 'll!!to?'d\"d of, can't.[SEP]", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["\u2581def", "\u2581main", "(", ")", ":", "\u2581pass"], + ids: [2, 6312, 407, 5, 6, 45, 1477, 3], + decoded: "[CLS] def main(): pass[SEP]", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["\u2581let", "\u2581a", "\u2581=", "\u2581ob", "j", ".", "to", "string", "(", ")", ";", "\u2581to", "string", "(", ")", ";"], + ids: [2, 408, 21, 800, 5122, 728, 9, 262, 11130, 5, 6, 73, 20, 11130, 5, 6, 73, 3], + decoded: "[CLS] let a = obj.tostring(); tostring();[SEP]", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["\u2581this", "\u2581is", "\u2581a", "\u2581test", "."], + ids: [2, 48, 25, 21, 1289, 9, 3], + decoded: "[CLS] this is a test.[SEP]", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["\u2581unwanted", ",", "running"], + ids: [2, 21095, 15, 11325, 3], + decoded: "[CLS] unwanted,running[SEP]", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["\u25811", "\u0000", "2", "\u25813"], + ids: [2, 137, 1, 135, 203, 3], + decoded: "[CLS] 12 3[SEP]", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["\u2581hello", "\u2581world"], + ids: [2, 10975, 126, 3], + decoded: "[CLS] hello world[SEP]", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["\u2581hello", "\u2581world"], + ids: [2, 10975, 126, 3], + decoded: "[CLS] hello world[SEP]", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u2581", "\u751f\u6d3b\u7684\u771f\u8c1b\u662f"], + ids: [2, 13, 1, 3], + decoded: "[CLS] [SEP]", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u2581leading", "\u2581space"], + ids: [2, 1005, 726, 3], + decoded: "[CLS] leading space[SEP]", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["\u2581trailing", "\u2581space"], + ids: [2, 14323, 726, 3], + decoded: "[CLS] trailing space[SEP]", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["\u2581hi", "\u2581hello"], + ids: [2, 4148, 10975, 3], + decoded: "[CLS] hi hello[SEP]", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["\u2581test", "\u2581$1", "\u2581r", "2", "\u2581#3", "\u2581", "\u20ac", "4", "\u2581", "\u00a3", "5", "\u2581", "\u00a5", "6", "\u2581", "\u20a3", "7", "\u2581", "\u20b9", "8", "\u2581", "\u20b1", "9", "\u2581test"], + ids: [2, 1289, 3742, 761, 135, 11489, 13, 12, 300, 13, 11, 264, 13, 1, 379, 13, 1, 465, 13, 1, 457, 13, 1, 518, 1289, 3], + decoded: "[CLS] test $1 r2 #3 \u20ac4 \u00a35 6 7 8 9 test[SEP]", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["\u2581i", "\u2581bought", "\u2581an", "\u2581apple", "\u2581for", "\u2581$1", ".", "00", "\u2581at", "\u2581the", "\u2581store", "."], + ids: [2, 31, 2448, 40, 4037, 26, 3742, 9, 2032, 35, 14, 1718, 9, 3], + decoded: "[CLS] i bought an apple for $1.00 at the store.[SEP]", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["\u2581you", ".", ".", "."], + ids: [2, 42, 9, 9, 9, 3], + decoded: "[CLS] you...[SEP]", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["\u2581you", ".", ".", "."], + ids: [2, 42, 9, 9, 9, 3], + decoded: "[CLS] you...[SEP]", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["\u2581you", ".", ".", ".", "\u2581you", ".", ".", "."], + ids: [2, 42, 9, 9, 9, 42, 9, 9, 9, 3], + decoded: "[CLS] you... you...[SEP]", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["\u2581weird", "\u2581", "~", "\u2581edge", "\u2581", "~", "\u2581case"], + ids: [2, 5455, 13, 1, 1407, 13, 1, 610, 3], + decoded: "[CLS] weird edge case[SEP]", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u2581this", "\u2581is", "\u2581a", "\u2581test", "\u2581", "."], + ids: [2, 48, 25, 21, 1289, 13, 9, 3], + decoded: "[CLS] this is a test.[SEP]", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u2581", "\ud83d\ude02", "\u2581", "\ud83d\udc4d", "\u2581", "\ud83e\udd23", "\u2581", "\ud83d\ude0d", "\u2581", "\ud83d\ude2d", "\u2581", "\ud83c\udf89", "\u2581", "\ud83d\ude4f", "\u2581", "\ud83d\ude0a", "\u2581", "\ud83d\udd25", "\u2581", "\ud83d\ude01", "\u2581", "\ud83d\ude05", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\ude06", "\u2581", "\ud83d\udc4f", "\u2581", "\u2764", "\u2581", "\ud83d\udc9c", "\u2581", "\ud83d\udc9a", "\u2581", "\ud83d\udc97", "\u2581", "\ud83d\udc99", "\u2581", "\ud83d\udda4", "\u2581", "\ud83d\ude0e", "\u2581", "\ud83d\udc4c", "\u2581", "\ud83e\udd73", "\u2581", "\ud83d\udcaa", "\u2581", "\u2728", "\u2581", "\ud83d\udc49", "\u2581", "\ud83d\udc40", "\u2581", "\ud83d\udcaf", "\u2581", "\ud83c\udf88", "\u2581", "\ud83d\ude48", "\u2581", "\ud83d\ude4c", "\u2581", "\ud83d\udc80", "\u2581", "\ud83d\udc47", "\u2581", "\ud83d\udc4b", "\u2581", "\u2705", "\u2581", "\ud83c\udf81", "\u2581", "\ud83c\udf1e", "\u2581", "\ud83c\udf38", "\u2581", "\ud83d\udcb0"], + ids: [2, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 3], + decoded: "[CLS] [SEP]", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u2581", "\u2728", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\udc41", "\u2581", "\ud83d\udc71\ud83c\udffb", "\u2581", "\ud83d\udd75", "\u2581", "\u2642", "\u2581", "\ud83e\uddd9\ud83c\udffb", "\u2581", "\u2642", "\u2581", "\ud83d\udc68\ud83c\udffb", "\u2581", "\ud83c\udf3e", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83d\udc69", "\u2581", "\u2764", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc67", "\u2581", "\ud83d\udc66", "\u2581", "\ud83e\uddd1\ud83c\udffb", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1\ud83c\udffb", "\u2581", "\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f", "\u2581", "\ud83d\udc68\ud83c\udffb", "\u2581", "\u2764", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68\ud83c\udffc"], + ids: [2, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 3], + decoded: "[CLS] [SEP]", + }, + CHINESE_LATIN_MIXED: { + text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, + tokens: ["\u2581", "ah", "\u535a\u63a8", "zz"], + ids: [2, 13, 1307, 1, 5092, 3], + decoded: "[CLS] ahzz[SEP]", + }, + SIMPLE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, + tokens: ["\u2581hello"], + ids: [2, 10975, 3], + decoded: "[CLS] hello[SEP]", + }, + MIXED_CASE_WITHOUT_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, + tokens: ["\u2581hello", "!", "how", "\u2581are", "\u2581you", "?"], + ids: [2, 10975, 187, 1544, 50, 42, 60, 3], + decoded: "[CLS] hello!how are you?[SEP]", + }, + MIXED_CASE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, + tokens: ["\u2581hall", "o", "!", "how", "\u2581are", "\u2581you", "?"], + ids: [2, 554, 111, 187, 1544, 50, 42, 60, 3], + decoded: "[CLS] hallo!how are you?[SEP]", + }, }, - SIMPLE_WITH_PUNCTUATION: { - text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, - tokens: ["\u2581you", "\u2581should", "'", "ve", "\u2581done", "\u2581this"], - ids: [2, 42, 378, 22, 195, 677, 48, 3], - decoded: "[CLS] you should've done this[SEP]", - }, - NUMBERS: { - text: BASE_TEST_STRINGS.NUMBERS, - tokens: ["\u25810", "12", "345", "67", "89", "\u25810", "\u25811", "\u25812", "\u25813", "\u25814", "\u25815", "\u25816", "\u25817", "\u25818", "\u25819", "\u258110", "\u2581100", "\u25811000"], - ids: [2, 713, 918, 21997, 4167, 3877, 713, 137, 172, 203, 268, 331, 400, 453, 469, 561, 332, 808, 6150, 3], - decoded: "[CLS] 0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000[SEP]", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["\u2581the", "\u2581company", "\u2581was", "\u2581founded", "\u2581in", "\u25812016", "."], - ids: [2, 14, 237, 23, 785, 19, 690, 9, 3], - decoded: "[CLS] the company was founded in 2016.[SEP]", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["\u2581a", "\u2581", "'", "ll", "\u2581", "!!", "to", "?'", "d", '"', "d", "\u2581of", ",", "\u2581can", "'", "t", "."], - ids: [2, 21, 13, 22, 211, 13, 19015, 262, 5663, 43, 7, 43, 16, 15, 92, 22, 38, 9, 3], - decoded: "[CLS] a 'll!!to?'d\"d of, can't.[SEP]", - }, - PYTHON_CODE: { - text: BASE_TEST_STRINGS.PYTHON_CODE, - tokens: ["\u2581def", "\u2581main", "(", ")", ":", "\u2581pass"], - ids: [2, 6312, 407, 5, 6, 45, 1477, 3], - decoded: "[CLS] def main(): pass[SEP]", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["\u2581let", "\u2581a", "\u2581=", "\u2581ob", "j", ".", "to", "string", "(", ")", ";", "\u2581to", "string", "(", ")", ";"], - ids: [2, 408, 21, 800, 5122, 728, 9, 262, 11130, 5, 6, 73, 20, 11130, 5, 6, 73, 3], - decoded: "[CLS] let a = obj.tostring(); tostring();[SEP]", - }, - NEWLINES: { - text: BASE_TEST_STRINGS.NEWLINES, - tokens: ["\u2581this", "\u2581is", "\u2581a", "\u2581test", "."], - ids: [2, 48, 25, 21, 1289, 9, 3], - decoded: "[CLS] this is a test.[SEP]", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["\u2581unwanted", ",", "running"], - ids: [2, 21095, 15, 11325, 3], - decoded: "[CLS] unwanted,running[SEP]", - }, - CONTROL_TOKENS: { - text: BASE_TEST_STRINGS.CONTROL_TOKENS, - tokens: ["\u25811", "\u0000", "2", "\u25813"], - ids: [2, 137, 1, 135, 203, 3], - decoded: "[CLS] 12 3[SEP]", - }, - HELLO_WORLD_TITLECASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, - tokens: ["\u2581hello", "\u2581world"], - ids: [2, 10975, 126, 3], - decoded: "[CLS] hello world[SEP]", - }, - HELLO_WORLD_LOWERCASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, - tokens: ["\u2581hello", "\u2581world"], - ids: [2, 10975, 126, 3], - decoded: "[CLS] hello world[SEP]", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u2581", "\u751f\u6d3b\u7684\u771f\u8c1b\u662f"], - ids: [2, 13, 1, 3], - decoded: "[CLS] [SEP]", - }, - LEADING_SPACE: { - text: BASE_TEST_STRINGS.LEADING_SPACE, - tokens: ["\u2581leading", "\u2581space"], - ids: [2, 1005, 726, 3], - decoded: "[CLS] leading space[SEP]", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["\u2581trailing", "\u2581space"], - ids: [2, 14323, 726, 3], - decoded: "[CLS] trailing space[SEP]", - }, - DOUBLE_SPACE: { - text: BASE_TEST_STRINGS.DOUBLE_SPACE, - tokens: ["\u2581hi", "\u2581hello"], - ids: [2, 4148, 10975, 3], - decoded: "[CLS] hi hello[SEP]", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["\u2581test", "\u2581$1", "\u2581r", "2", "\u2581#3", "\u2581", "\u20ac", "4", "\u2581", "\u00a3", "5", "\u2581", "\u00a5", "6", "\u2581", "\u20a3", "7", "\u2581", "\u20b9", "8", "\u2581", "\u20b1", "9", "\u2581test"], - ids: [2, 1289, 3742, 761, 135, 11489, 13, 12, 300, 13, 11, 264, 13, 1, 379, 13, 1, 465, 13, 1, 457, 13, 1, 518, 1289, 3], - decoded: "[CLS] test $1 r2 #3 \u20ac4 \u00a35 6 7 8 9 test[SEP]", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["\u2581i", "\u2581bought", "\u2581an", "\u2581apple", "\u2581for", "\u2581$1", ".", "00", "\u2581at", "\u2581the", "\u2581store", "."], - ids: [2, 31, 2448, 40, 4037, 26, 3742, 9, 2032, 35, 14, 1718, 9, 3], - decoded: "[CLS] i bought an apple for $1.00 at the store.[SEP]", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - tokens: ["\u2581you", ".", ".", "."], - ids: [2, 42, 9, 9, 9, 3], - decoded: "[CLS] you...[SEP]", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - tokens: ["\u2581you", ".", ".", "."], - ids: [2, 42, 9, 9, 9, 3], - decoded: "[CLS] you...[SEP]", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - tokens: ["\u2581you", ".", ".", ".", "\u2581you", ".", ".", "."], - ids: [2, 42, 9, 9, 9, 42, 9, 9, 9, 3], - decoded: "[CLS] you... you...[SEP]", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["\u2581weird", "\u2581", "~", "\u2581edge", "\u2581", "~", "\u2581case"], - ids: [2, 5455, 13, 1, 1407, 13, 1, 610, 3], - decoded: "[CLS] weird edge case[SEP]", - }, - SPIECE_UNDERSCORE: { - text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, - tokens: ["\u2581this", "\u2581is", "\u2581a", "\u2581test", "\u2581", "."], - ids: [2, 48, 25, 21, 1289, 13, 9, 3], - decoded: "[CLS] this is a test.[SEP]", - }, - POPULAR_EMOJIS: { - text: BASE_TEST_STRINGS.POPULAR_EMOJIS, - tokens: ["\u2581", "\ud83d\ude02", "\u2581", "\ud83d\udc4d", "\u2581", "\ud83e\udd23", "\u2581", "\ud83d\ude0d", "\u2581", "\ud83d\ude2d", "\u2581", "\ud83c\udf89", "\u2581", "\ud83d\ude4f", "\u2581", "\ud83d\ude0a", "\u2581", "\ud83d\udd25", "\u2581", "\ud83d\ude01", "\u2581", "\ud83d\ude05", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\ude06", "\u2581", "\ud83d\udc4f", "\u2581", "\u2764", "\u2581", "\ud83d\udc9c", "\u2581", "\ud83d\udc9a", "\u2581", "\ud83d\udc97", "\u2581", "\ud83d\udc99", "\u2581", "\ud83d\udda4", "\u2581", "\ud83d\ude0e", "\u2581", "\ud83d\udc4c", "\u2581", "\ud83e\udd73", "\u2581", "\ud83d\udcaa", "\u2581", "\u2728", "\u2581", "\ud83d\udc49", "\u2581", "\ud83d\udc40", "\u2581", "\ud83d\udcaf", "\u2581", "\ud83c\udf88", "\u2581", "\ud83d\ude48", "\u2581", "\ud83d\ude4c", "\u2581", "\ud83d\udc80", "\u2581", "\ud83d\udc47", "\u2581", "\ud83d\udc4b", "\u2581", "\u2705", "\u2581", "\ud83c\udf81", "\u2581", "\ud83c\udf1e", "\u2581", "\ud83c\udf38", "\u2581", "\ud83d\udcb0"], - ids: [2, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 3], - decoded: "[CLS] [SEP]", - }, - MULTIBYTE_EMOJIS: { - text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, - tokens: ["\u2581", "\u2728", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\udc41", "\u2581", "\ud83d\udc71\ud83c\udffb", "\u2581", "\ud83d\udd75", "\u2581", "\u2642", "\u2581", "\ud83e\uddd9\ud83c\udffb", "\u2581", "\u2642", "\u2581", "\ud83d\udc68\ud83c\udffb", "\u2581", "\ud83c\udf3e", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83d\udc69", "\u2581", "\u2764", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc67", "\u2581", "\ud83d\udc66", "\u2581", "\ud83e\uddd1\ud83c\udffb", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1\ud83c\udffb", "\u2581", "\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f", "\u2581", "\ud83d\udc68\ud83c\udffb", "\u2581", "\u2764", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68\ud83c\udffc"], - ids: [2, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 3], - decoded: "[CLS] [SEP]", - }, - CHINESE_LATIN_MIXED: { - text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, - tokens: ["\u2581", "ah", "\u535a\u63a8", "zz"], - ids: [2, 13, 1307, 1, 5092, 3], - decoded: "[CLS] ahzz[SEP]", - }, - SIMPLE_WITH_ACCENTS: { - text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, - tokens: ["\u2581hello"], - ids: [2, 10975, 3], - decoded: "[CLS] hello[SEP]", - }, - MIXED_CASE_WITHOUT_ACCENTS: { - text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, - tokens: ["\u2581hello", "!", "how", "\u2581are", "\u2581you", "?"], - ids: [2, 10975, 187, 1544, 50, 42, 60, 3], - decoded: "[CLS] hello!how are you?[SEP]", - }, - MIXED_CASE_WITH_ACCENTS: { - text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, - tokens: ["\u2581hall", "o", "!", "how", "\u2581are", "\u2581you", "?"], - ids: [2, 554, 111, 187, 1544, 50, 42, 60, 3], - decoded: "[CLS] hallo!how are you?[SEP]", - }, - }, }; diff --git a/tests/models/bert/tokenization.js b/tests/models/bert/tokenization.js index 54b253260..5e8f1fe4c 100644 --- a/tests/models/bert/tokenization.js +++ b/tests/models/bert/tokenization.js @@ -3,1333 +3,1333 @@ import { BASE_TEST_STRINGS, BERT_TEST_STRINGS } from "../test_strings.js"; export const TOKENIZER_CLASS = BertTokenizer; export const TEST_CONFIG = { - "Xenova/bert-base-uncased": { - SIMPLE: { - text: BASE_TEST_STRINGS.SIMPLE, - tokens: ["how", "are", "you", "doing", "?"], - ids: [101, 2129, 2024, 2017, 2725, 1029, 102], - decoded: "[CLS] how are you doing? [SEP]", - }, - SIMPLE_WITH_PUNCTUATION: { - text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, - tokens: ["you", "should", "'", "ve", "done", "this"], - ids: [101, 2017, 2323, 1005, 2310, 2589, 2023, 102], - decoded: "[CLS] you should've done this [SEP]", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["the", "company", "was", "founded", "in", "2016", "."], - ids: [101, 1996, 2194, 2001, 2631, 1999, 2355, 1012, 102], - decoded: "[CLS] the company was founded in 2016. [SEP]", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["a", "'", "ll", "!", "!", "to", "?", "'", "d", "'", "'", "d", "of", ",", "can", "'", "t", "."], - ids: [101, 1037, 1005, 2222, 999, 999, 2000, 1029, 1005, 1040, 1005, 1005, 1040, 1997, 1010, 2064, 1005, 1056, 1012, 102], - decoded: "[CLS] a'll!! to?'d'' d of, can't. [SEP]", - }, - PYTHON_CODE: { - text: BASE_TEST_STRINGS.PYTHON_CODE, - tokens: ["def", "main", "(", ")", ":", "pass"], - ids: [101, 13366, 2364, 1006, 1007, 1024, 3413, 102], - decoded: "[CLS] def main ( ) : pass [SEP]", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["let", "a", "=", "ob", "##j", ".", "to", "##st", "##ring", "(", ")", ";", "to", "##st", "##ring", "(", ")", ";"], - ids: [101, 2292, 1037, 1027, 27885, 3501, 1012, 2000, 3367, 4892, 1006, 1007, 1025, 2000, 3367, 4892, 1006, 1007, 1025, 102], - decoded: "[CLS] let a = obj. tostring ( ) ; tostring ( ) ; [SEP]", - }, - NEWLINES: { - text: BASE_TEST_STRINGS.NEWLINES, - tokens: ["this", "is", "a", "test", "."], - ids: [101, 2023, 2003, 1037, 3231, 1012, 102], - decoded: "[CLS] this is a test. [SEP]", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["unwanted", ",", "running"], - ids: [101, 18162, 1010, 2770, 102], - decoded: "[CLS] unwanted, running [SEP]", - }, - CONTROL_TOKENS: { - text: BASE_TEST_STRINGS.CONTROL_TOKENS, - tokens: ["123"], - ids: [101, 13138, 102], - decoded: "[CLS] 123 [SEP]", - }, - HELLO_WORLD_TITLECASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, - tokens: ["hello", "world"], - ids: [101, 7592, 2088, 102], - decoded: "[CLS] hello world [SEP]", - }, - HELLO_WORLD_LOWERCASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, - tokens: ["hello", "world"], - ids: [101, 7592, 2088, 102], - decoded: "[CLS] hello world [SEP]", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u751f", "[UNK]", "\u7684", "\u771f", "[UNK]", "[UNK]"], - ids: [101, 1910, 100, 1916, 1921, 100, 100, 102], - decoded: "[CLS] \u751f [UNK] \u7684 \u771f [UNK] [UNK] [SEP]", - }, - LEADING_SPACE: { - text: BASE_TEST_STRINGS.LEADING_SPACE, - tokens: ["leading", "space"], - ids: [101, 2877, 2686, 102], - decoded: "[CLS] leading space [SEP]", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["trailing", "space"], - ids: [101, 12542, 2686, 102], - decoded: "[CLS] trailing space [SEP]", - }, - DOUBLE_SPACE: { - text: BASE_TEST_STRINGS.DOUBLE_SPACE, - tokens: ["hi", "hello"], - ids: [101, 7632, 7592, 102], - decoded: "[CLS] hi hello [SEP]", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["test", "$", "1", "r", "##2", "#", "3", "\u20ac", "##4", "\u00a35", "\u00a5", "##6", "[UNK]", "\u20b9", "##8", "\u20b1", "##9", "test"], - ids: [101, 3231, 1002, 1015, 1054, 2475, 1001, 1017, 1574, 2549, 27813, 1071, 2575, 100, 1576, 2620, 1575, 2683, 3231, 102], - decoded: "[CLS] test $ 1 r2 # 3 \u20ac4 \u00a35 \u00a56 [UNK] \u20b98 \u20b19 test [SEP]", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["i", "bought", "an", "apple", "for", "$", "1", ".", "00", "at", "the", "store", "."], - ids: [101, 1045, 4149, 2019, 6207, 2005, 1002, 1015, 1012, 4002, 2012, 1996, 3573, 1012, 102], - decoded: "[CLS] i bought an apple for $ 1. 00 at the store. [SEP]", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - tokens: ["you", "\u2026"], - ids: [101, 2017, 1529, 102], - decoded: "[CLS] you \u2026 [SEP]", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - tokens: ["you", "\u2026"], - ids: [101, 2017, 1529, 102], - decoded: "[CLS] you \u2026 [SEP]", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - tokens: ["you", "\u2026", "you", "\u2026"], - ids: [101, 2017, 1529, 2017, 1529, 102], - decoded: "[CLS] you \u2026 you \u2026 [SEP]", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["weird", "\uff5e", "edge", "\uff5e", "case"], - ids: [101, 6881, 1995, 3341, 1995, 2553, 102], - decoded: "[CLS] weird \uff5e edge \uff5e case [SEP]", - }, - SPIECE_UNDERSCORE: { - text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, - tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "."], - ids: [101, 100, 100, 100, 100, 100, 1012, 102], - decoded: "[CLS] [UNK] [UNK] [UNK] [UNK] [UNK]. [SEP]", - }, - POPULAR_EMOJIS: { - text: BASE_TEST_STRINGS.POPULAR_EMOJIS, - tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]"], - ids: [101, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 102], - decoded: "[CLS] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [SEP]", - }, - MULTIBYTE_EMOJIS: { - text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, - tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]"], - ids: [101, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 102], - decoded: "[CLS] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [SEP]", - }, - CHINESE_LATIN_MIXED: { - text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, - tokens: ["ah", "\u535a", "[UNK]", "z", "##z"], - ids: [101, 6289, 1786, 100, 1062, 2480, 102], - decoded: "[CLS] ah \u535a [UNK] zz [SEP]", - }, - SIMPLE_WITH_ACCENTS: { - text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, - tokens: ["hello"], - ids: [101, 7592, 102], - decoded: "[CLS] hello [SEP]", - }, - MIXED_CASE_WITHOUT_ACCENTS: { - text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, - tokens: ["hello", "!", "how", "are", "you", "?"], - ids: [101, 7592, 999, 2129, 2024, 2017, 1029, 102], - decoded: "[CLS] hello! how are you? [SEP]", - }, - MIXED_CASE_WITH_ACCENTS: { - text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, - tokens: ["hall", "##o", "!", "how", "are", "you", "?"], - ids: [101, 2534, 2080, 999, 2129, 2024, 2017, 1029, 102], - decoded: "[CLS] hallo! how are you? [SEP]", - }, - ONLY_WHITESPACE: { - text: BASE_TEST_STRINGS.ONLY_WHITESPACE, - tokens: [], - ids: [101, 102], - decoded: "[CLS] [SEP]", - }, + "Xenova/bert-base-uncased": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["how", "are", "you", "doing", "?"], + ids: [101, 2129, 2024, 2017, 2725, 1029, 102], + decoded: "[CLS] how are you doing? [SEP]", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["you", "should", "'", "ve", "done", "this"], + ids: [101, 2017, 2323, 1005, 2310, 2589, 2023, 102], + decoded: "[CLS] you should've done this [SEP]", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["the", "company", "was", "founded", "in", "2016", "."], + ids: [101, 1996, 2194, 2001, 2631, 1999, 2355, 1012, 102], + decoded: "[CLS] the company was founded in 2016. [SEP]", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["a", "'", "ll", "!", "!", "to", "?", "'", "d", "'", "'", "d", "of", ",", "can", "'", "t", "."], + ids: [101, 1037, 1005, 2222, 999, 999, 2000, 1029, 1005, 1040, 1005, 1005, 1040, 1997, 1010, 2064, 1005, 1056, 1012, 102], + decoded: "[CLS] a'll!! to?'d'' d of, can't. [SEP]", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["def", "main", "(", ")", ":", "pass"], + ids: [101, 13366, 2364, 1006, 1007, 1024, 3413, 102], + decoded: "[CLS] def main ( ) : pass [SEP]", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "a", "=", "ob", "##j", ".", "to", "##st", "##ring", "(", ")", ";", "to", "##st", "##ring", "(", ")", ";"], + ids: [101, 2292, 1037, 1027, 27885, 3501, 1012, 2000, 3367, 4892, 1006, 1007, 1025, 2000, 3367, 4892, 1006, 1007, 1025, 102], + decoded: "[CLS] let a = obj. tostring ( ) ; tostring ( ) ; [SEP]", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["this", "is", "a", "test", "."], + ids: [101, 2023, 2003, 1037, 3231, 1012, 102], + decoded: "[CLS] this is a test. [SEP]", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["unwanted", ",", "running"], + ids: [101, 18162, 1010, 2770, 102], + decoded: "[CLS] unwanted, running [SEP]", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["123"], + ids: [101, 13138, 102], + decoded: "[CLS] 123 [SEP]", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["hello", "world"], + ids: [101, 7592, 2088, 102], + decoded: "[CLS] hello world [SEP]", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["hello", "world"], + ids: [101, 7592, 2088, 102], + decoded: "[CLS] hello world [SEP]", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u751f", "[UNK]", "\u7684", "\u771f", "[UNK]", "[UNK]"], + ids: [101, 1910, 100, 1916, 1921, 100, 100, 102], + decoded: "[CLS] \u751f [UNK] \u7684 \u771f [UNK] [UNK] [SEP]", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["leading", "space"], + ids: [101, 2877, 2686, 102], + decoded: "[CLS] leading space [SEP]", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["trailing", "space"], + ids: [101, 12542, 2686, 102], + decoded: "[CLS] trailing space [SEP]", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["hi", "hello"], + ids: [101, 7632, 7592, 102], + decoded: "[CLS] hi hello [SEP]", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "$", "1", "r", "##2", "#", "3", "\u20ac", "##4", "\u00a35", "\u00a5", "##6", "[UNK]", "\u20b9", "##8", "\u20b1", "##9", "test"], + ids: [101, 3231, 1002, 1015, 1054, 2475, 1001, 1017, 1574, 2549, 27813, 1071, 2575, 100, 1576, 2620, 1575, 2683, 3231, 102], + decoded: "[CLS] test $ 1 r2 # 3 \u20ac4 \u00a35 \u00a56 [UNK] \u20b98 \u20b19 test [SEP]", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["i", "bought", "an", "apple", "for", "$", "1", ".", "00", "at", "the", "store", "."], + ids: [101, 1045, 4149, 2019, 6207, 2005, 1002, 1015, 1012, 4002, 2012, 1996, 3573, 1012, 102], + decoded: "[CLS] i bought an apple for $ 1. 00 at the store. [SEP]", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "\u2026"], + ids: [101, 2017, 1529, 102], + decoded: "[CLS] you \u2026 [SEP]", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "\u2026"], + ids: [101, 2017, 1529, 102], + decoded: "[CLS] you \u2026 [SEP]", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "\u2026", "you", "\u2026"], + ids: [101, 2017, 1529, 2017, 1529, 102], + decoded: "[CLS] you \u2026 you \u2026 [SEP]", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["weird", "\uff5e", "edge", "\uff5e", "case"], + ids: [101, 6881, 1995, 3341, 1995, 2553, 102], + decoded: "[CLS] weird \uff5e edge \uff5e case [SEP]", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "."], + ids: [101, 100, 100, 100, 100, 100, 1012, 102], + decoded: "[CLS] [UNK] [UNK] [UNK] [UNK] [UNK]. [SEP]", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]"], + ids: [101, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 102], + decoded: "[CLS] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [SEP]", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]"], + ids: [101, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 102], + decoded: "[CLS] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [SEP]", + }, + CHINESE_LATIN_MIXED: { + text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, + tokens: ["ah", "\u535a", "[UNK]", "z", "##z"], + ids: [101, 6289, 1786, 100, 1062, 2480, 102], + decoded: "[CLS] ah \u535a [UNK] zz [SEP]", + }, + SIMPLE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, + tokens: ["hello"], + ids: [101, 7592, 102], + decoded: "[CLS] hello [SEP]", + }, + MIXED_CASE_WITHOUT_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, + tokens: ["hello", "!", "how", "are", "you", "?"], + ids: [101, 7592, 999, 2129, 2024, 2017, 1029, 102], + decoded: "[CLS] hello! how are you? [SEP]", + }, + MIXED_CASE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, + tokens: ["hall", "##o", "!", "how", "are", "you", "?"], + ids: [101, 2534, 2080, 999, 2129, 2024, 2017, 1029, 102], + decoded: "[CLS] hallo! how are you? [SEP]", + }, + ONLY_WHITESPACE: { + text: BASE_TEST_STRINGS.ONLY_WHITESPACE, + tokens: [], + ids: [101, 102], + decoded: "[CLS] [SEP]", + }, - TEXT_PAIR: { - text: "hello", - text_pair: "world", - tokens: ["hello", "world"], - ids: [101, 7592, 102, 2088, 102], - decoded: "[CLS] hello [SEP] world [SEP]", - }, - }, - "Xenova/bert-base-cased": { - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["let", "a", "=", "o", "##b", "##j", ".", "to", "##S", "##tring", "(", ")", ";", "to", "##S", "##tring", "(", ")", ";"], - ids: [101, 1519, 170, 134, 184, 1830, 3361, 119, 1106, 1708, 28108, 113, 114, 132, 1106, 1708, 28108, 113, 114, 132, 102], - decoded: "[CLS] let a = obj. toString ( ) ; toString ( ) ; [SEP]", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["UN", "##wan", "##t\u00e9", "##d", ",", "running"], - ids: [101, 7414, 5491, 14608, 1181, 117, 1919, 102], - decoded: "[CLS] UNwant\u00e9d, running [SEP]", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u751f", "[UNK]", "[UNK]", "\u771f", "[UNK]", "[UNK]"], - ids: [101, 1056, 100, 100, 1061, 100, 100, 102], - decoded: "[CLS] \u751f [UNK] [UNK] \u771f [UNK] [UNK] [SEP]", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["test", "$", "1", "R", "##2", "#", "3", "\u20ac", "##4", "\u00a3", "##5", "\u00a5", "##6", "[UNK]", "\u20b9", "##8", "\u20b1", "##9", "test"], - ids: [101, 2774, 109, 122, 155, 1477, 108, 124, 836, 1527, 202, 1571, 203, 1545, 100, 838, 1604, 837, 1580, 2774, 102], - decoded: "[CLS] test $ 1 R2 # 3 \u20ac4 \u00a35 \u00a56 [UNK] \u20b98 \u20b19 test [SEP]", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["I", "bought", "an", "apple", "for", "$", "1", ".", "00", "at", "the", "store", "."], - ids: [101, 146, 3306, 1126, 12075, 1111, 109, 122, 119, 3135, 1120, 1103, 2984, 119, 102], - decoded: "[CLS] I bought an apple for $ 1. 00 at the store. [SEP]", + TEXT_PAIR: { + text: "hello", + text_pair: "world", + tokens: ["hello", "world"], + ids: [101, 7592, 102, 2088, 102], + decoded: "[CLS] hello [SEP] world [SEP]", + }, + }, + "Xenova/bert-base-cased": { + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "a", "=", "o", "##b", "##j", ".", "to", "##S", "##tring", "(", ")", ";", "to", "##S", "##tring", "(", ")", ";"], + ids: [101, 1519, 170, 134, 184, 1830, 3361, 119, 1106, 1708, 28108, 113, 114, 132, 1106, 1708, 28108, 113, 114, 132, 102], + decoded: "[CLS] let a = obj. toString ( ) ; toString ( ) ; [SEP]", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "##wan", "##t\u00e9", "##d", ",", "running"], + ids: [101, 7414, 5491, 14608, 1181, 117, 1919, 102], + decoded: "[CLS] UNwant\u00e9d, running [SEP]", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u751f", "[UNK]", "[UNK]", "\u771f", "[UNK]", "[UNK]"], + ids: [101, 1056, 100, 100, 1061, 100, 100, 102], + decoded: "[CLS] \u751f [UNK] [UNK] \u771f [UNK] [UNK] [SEP]", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "$", "1", "R", "##2", "#", "3", "\u20ac", "##4", "\u00a3", "##5", "\u00a5", "##6", "[UNK]", "\u20b9", "##8", "\u20b1", "##9", "test"], + ids: [101, 2774, 109, 122, 155, 1477, 108, 124, 836, 1527, 202, 1571, 203, 1545, 100, 838, 1604, 837, 1580, 2774, 102], + decoded: "[CLS] test $ 1 R2 # 3 \u20ac4 \u00a35 \u00a56 [UNK] \u20b98 \u20b19 test [SEP]", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "bought", "an", "apple", "for", "$", "1", ".", "00", "at", "the", "store", "."], + ids: [101, 146, 3306, 1126, 12075, 1111, 109, 122, 119, 3135, 1120, 1103, 2984, 119, 102], + decoded: "[CLS] I bought an apple for $ 1. 00 at the store. [SEP]", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["weird", "[UNK]", "edge", "[UNK]", "case"], + ids: [101, 6994, 100, 2652, 100, 1692, 102], + decoded: "[CLS] weird [UNK] edge [UNK] case [SEP]", + }, + CHINESE_LATIN_MIXED: { + text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, + tokens: ["ah", "[UNK]", "[UNK]", "z", "##z"], + ids: [101, 18257, 100, 100, 195, 1584, 102], + decoded: "[CLS] ah [UNK] [UNK] zz [SEP]", + }, + SIMPLE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, + tokens: ["H", "##\u00e9", "##llo"], + ids: [101, 145, 2744, 6643, 102], + decoded: "[CLS] H\u00e9llo [SEP]", + }, }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["weird", "[UNK]", "edge", "[UNK]", "case"], - ids: [101, 6994, 100, 2652, 100, 1692, 102], - decoded: "[CLS] weird [UNK] edge [UNK] case [SEP]", - }, - CHINESE_LATIN_MIXED: { - text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, - tokens: ["ah", "[UNK]", "[UNK]", "z", "##z"], - ids: [101, 18257, 100, 100, 195, 1584, 102], - decoded: "[CLS] ah [UNK] [UNK] zz [SEP]", - }, - SIMPLE_WITH_ACCENTS: { - text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, - tokens: ["H", "##\u00e9", "##llo"], - ids: [101, 145, 2744, 6643, 102], - decoded: "[CLS] H\u00e9llo [SEP]", - }, - }, - "Xenova/bert-base-multilingual-cased-ner-hrl": { - SIMPLE: { - text: BASE_TEST_STRINGS.SIMPLE, - tokens: ["How", "are", "you", "doing", "?"], - ids: [101, 14962, 10301, 13028, 30918, 136, 102], - decoded: "[CLS] How are you doing? [SEP]", - }, - SIMPLE_WITH_PUNCTUATION: { - text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, - tokens: ["You", "should", "'", "ve", "done", "this"], - ids: [101, 11065, 14819, 112, 10323, 20378, 10531, 102], - decoded: "[CLS] You should've done this [SEP]", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["The", "company", "was", "founded", "in", "2016", "."], - ids: [101, 10117, 12100, 10134, 14078, 10106, 10255, 119, 102], - decoded: "[CLS] The company was founded in 2016. [SEP]", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["A", "'", "ll", "!", "!", "to", "?", "'", "d", "'", "'", "d", "of", ",", "can", "'", "t", "."], - ids: [101, 138, 112, 22469, 106, 106, 10114, 136, 112, 172, 112, 112, 172, 10108, 117, 10944, 112, 188, 119, 102], - decoded: "[CLS] A'll!! to?'d'' d of, can't. [SEP]", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["let", "a", "=", "ob", "##j", ".", "to", "##S", "##trin", "##g", "(", ")", ";", "to", "##S", "##trin", "##g", "(", ")", ";"], - ids: [101, 13595, 169, 134, 17339, 10418, 119, 10114, 10731, 109163, 10240, 113, 114, 132, 10114, 10731, 109163, 10240, 113, 114, 132, 102], - decoded: "[CLS] let a = obj. toString ( ) ; toString ( ) ; [SEP]", - }, - NEWLINES: { - text: BASE_TEST_STRINGS.NEWLINES, - tokens: ["This", "is", "a", "test", "."], - ids: [101, 10747, 10124, 169, 15839, 119, 102], - decoded: "[CLS] This is a test. [SEP]", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["UN", "##want", "##\u00e9d", ",", "running"], - ids: [101, 26578, 104216, 84193, 117, 18020, 102], - decoded: "[CLS] UNwant\u00e9d, running [SEP]", - }, - HELLO_WORLD_TITLECASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, - tokens: ["Hello", "World"], - ids: [101, 31178, 10315, 102], - decoded: "[CLS] Hello World [SEP]", - }, - HELLO_WORLD_LOWERCASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, - tokens: ["hell", "##o", "world"], - ids: [101, 61694, 10133, 11356, 102], - decoded: "[CLS] hello world [SEP]", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u751f", "\u6d3b", "\u7684", "\u771f", "\u8c1b", "\u662f"], - ids: [101, 5600, 4978, 5718, 5769, 7378, 4380, 102], - decoded: "[CLS] \u751f \u6d3b \u7684 \u771f \u8c1b \u662f [SEP]", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["trail", "##ing", "space"], - ids: [101, 56559, 10230, 16199, 102], - decoded: "[CLS] trailing space [SEP]", - }, - DOUBLE_SPACE: { - text: BASE_TEST_STRINGS.DOUBLE_SPACE, - tokens: ["Hi", "Hello"], - ids: [101, 20065, 31178, 102], - decoded: "[CLS] Hi Hello [SEP]", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["test", "$", "1", "R2", "#", "3", "\u20ac", "##4", "\u00a3", "##5", "\u00a5", "##6", "[UNK]", "\u20b9", "##8", "[UNK]", "test"], - ids: [101, 15839, 109, 122, 94000, 108, 124, 1775, 11011, 201, 11166, 202, 11211, 100, 1776, 11396, 100, 15839, 102], - decoded: "[CLS] test $ 1 R2 # 3 \u20ac4 \u00a35 \u00a56 [UNK] \u20b98 [UNK] test [SEP]", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["I", "bought", "an", "app", "##le", "for", "$", "1", ".", "00", "at", "the", "store", "."], - ids: [101, 146, 28870, 10151, 72894, 10284, 10142, 109, 122, 119, 11025, 10160, 10105, 13708, 119, 102], - decoded: "[CLS] I bought an apple for $ 1. 00 at the store. [SEP]", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - tokens: ["you", "[UNK]"], - ids: [101, 13028, 100, 102], - decoded: "[CLS] you [UNK] [SEP]", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - tokens: ["you", "[UNK]"], - ids: [101, 13028, 100, 102], - decoded: "[CLS] you [UNK] [SEP]", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - tokens: ["you", "[UNK]", "you", "[UNK]"], - ids: [101, 13028, 100, 13028, 100, 102], - decoded: "[CLS] you [UNK] you [UNK] [SEP]", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["wei", "##rd", "\uff5e", "edge", "\uff5e", "case"], - ids: [101, 86981, 12023, 10096, 30599, 10096, 13474, 102], - decoded: "[CLS] weird \uff5e edge \uff5e case [SEP]", - }, - CHINESE_LATIN_MIXED: { - text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, - tokens: ["ah", "\u535a", "\u63a8", "z", "##z"], - ids: [101, 69863, 2684, 4163, 194, 10305, 102], - decoded: "[CLS] ah \u535a \u63a8 zz [SEP]", - }, - SIMPLE_WITH_ACCENTS: { - text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, - tokens: ["H", "##\u00e9l", "##lo"], - ids: [101, 145, 24817, 10715, 102], - decoded: "[CLS] H\u00e9llo [SEP]", - }, - MIXED_CASE_WITHOUT_ACCENTS: { - text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, - tokens: ["He", "##LL", "##o", "!", "how", "Are", "yo", "##U", "?"], - ids: [101, 10357, 82834, 10133, 106, 14796, 13491, 13672, 12022, 136, 102], - decoded: "[CLS] HeLLo! how Are yoU? [SEP]", - }, - MIXED_CASE_WITH_ACCENTS: { - text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, - tokens: ["H", "##\u00e4", "##LL", "##o", "!", "how", "Are", "yo", "##U", "?"], - ids: [101, 145, 11013, 82834, 10133, 106, 14796, 13491, 13672, 12022, 136, 102], - decoded: "[CLS] H\u00e4LLo! how Are yoU? [SEP]", - }, - }, - "Xenova/paraphrase-multilingual-MiniLM-L12-v2": { - SIMPLE: { - text: BASE_TEST_STRINGS.SIMPLE, - tokens: ["\u2581How", "\u2581are", "\u2581you", "\u2581doing", "?"], - ids: [0, 11249, 621, 398, 20594, 32, 2], - decoded: " How are you doing?", - }, - SIMPLE_WITH_PUNCTUATION: { - text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, - tokens: ["\u2581You", "\u2581should", "'", "ve", "\u2581done", "\u2581this"], - ids: [0, 2583, 5608, 25, 272, 16940, 903, 2], - decoded: " You should've done this", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["\u2581The", "\u2581company", "\u2581was", "\u2581found", "ed", "\u2581in", "\u25812016."], - ids: [0, 581, 14380, 509, 14037, 297, 23, 6360, 2], - decoded: " The company was founded in 2016.", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["\u2581A", "\u2581'", "ll", "\u2581!!", "to", "?", "'", "d", "''", "d", "\u2581of", ",", "\u2581can", "'", "t", "."], - ids: [0, 62, 242, 1181, 6506, 188, 32, 25, 71, 4765, 71, 111, 4, 831, 25, 18, 5, 2], - decoded: " A 'll!!to?'d''d of, can't.", - }, - PYTHON_CODE: { - text: BASE_TEST_STRINGS.PYTHON_CODE, - tokens: ["\u2581de", "f", "\u2581main", "(", "):", "\u2581pass"], - ids: [0, 8, 420, 5201, 132, 2077, 27875, 2], - decoded: " def main(): pass", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["\u2581let", "\u2581a", "\u2581=", "\u2581ob", "j", ".", "to", "Str", "ing", "(", ");", "\u2581to", "Str", "ing", "(", ");"], - ids: [0, 2633, 10, 2203, 995, 170, 5, 188, 71713, 214, 132, 3142, 47, 71713, 214, 132, 3142, 2], - decoded: " let a = obj.toString(); toString();", - }, - NEWLINES: { - text: BASE_TEST_STRINGS.NEWLINES, - tokens: ["\u2581This", "\u2581is", "\u2581a", "\u2581test", "."], - ids: [0, 3293, 83, 10, 3034, 5, 2], - decoded: " This is a test.", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["\u2581UN", "wan", "t\u00e9", "d", ",", "run", "ning"], - ids: [0, 8274, 3206, 2312, 71, 4, 16428, 592, 2], - decoded: " UNwant\u00e9d,running", - }, - CONTROL_TOKENS: { - text: BASE_TEST_STRINGS.CONTROL_TOKENS, - tokens: ["\u25811", "\u0000", "2", "\u25813"], - ids: [0, 106, 3, 304, 138, 2], - decoded: " 12 3", - }, - HELLO_WORLD_TITLECASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, - tokens: ["\u2581Hello", "\u2581World"], - ids: [0, 35378, 6661, 2], - decoded: " Hello World", - }, - HELLO_WORLD_LOWERCASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, - tokens: ["\u2581hell", "o", "\u2581world"], - ids: [0, 33600, 31, 8999, 2], - decoded: " hello world", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u2581", "\u751f\u6d3b\u7684", "\u771f", "\u8c1b", "\u662f"], - ids: [0, 6, 62668, 5364, 245875, 354, 2], - decoded: " \u751f\u6d3b\u7684\u771f\u8c1b\u662f", - }, - LEADING_SPACE: { - text: BASE_TEST_STRINGS.LEADING_SPACE, - tokens: ["\u2581leading", "\u2581space"], - ids: [0, 105207, 32628, 2], - decoded: " leading space", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["\u2581trail", "ing", "\u2581space"], - ids: [0, 141037, 214, 32628, 2], - decoded: " trailing space", - }, - DOUBLE_SPACE: { - text: BASE_TEST_STRINGS.DOUBLE_SPACE, - tokens: ["\u2581Hi", "\u2581Hello"], - ids: [0, 2673, 35378, 2], - decoded: " Hi Hello", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["\u2581test", "\u2581$1", "\u2581R", "2", "\u2581#3", "\u2581\u20ac", "4", "\u2581\u00a3", "5", "\u2581", "\u00a5", "6", "\u2581", "\u20a3", "7", "\u2581\u20b9", "8", "\u2581", "\u20b1", "9", "\u2581test"], - ids: [0, 3034, 38629, 627, 304, 111378, 2505, 617, 11762, 758, 6, 32389, 910, 6, 3, 966, 87316, 1019, 6, 247425, 1126, 3034, 2], - decoded: " test $1 R2 #3 \u20ac4 \u00a35 \u00a56 7 \u20b98 \u20b19 test", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["\u2581I", "\u2581bought", "\u2581an", "\u2581apple", "\u2581for", "\u2581$", "1.00", "\u2581at", "\u2581the", "\u2581store", "."], - ids: [0, 87, 123997, 142, 108787, 100, 3650, 146533, 99, 70, 4343, 5, 2], - decoded: " I bought an apple for $1.00 at the store.", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - tokens: ["\u2581you", "..."], - ids: [0, 398, 27, 2], - decoded: " you...", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - tokens: ["\u2581you", "..."], - ids: [0, 398, 27, 2], - decoded: " you...", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - tokens: ["\u2581you", "...", "\u2581you", "..."], - ids: [0, 398, 27, 398, 27, 2], - decoded: " you... you...", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["\u2581weird", "\u2581", "\uff5e", "\u2581edge", "\u2581", "\uff5e", "\u2581case"], - ids: [0, 179459, 6, 6087, 121303, 6, 6087, 7225, 2], - decoded: " weird \uff5e edge \uff5e case", - }, - SPIECE_UNDERSCORE: { - text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, - tokens: ["\u2581This", "\u2581is", "\u2581a", "\u2581test", "\u2581", "."], - ids: [0, 3293, 83, 10, 3034, 6, 5, 2], - decoded: " This is a test.", - }, - POPULAR_EMOJIS: { - text: BASE_TEST_STRINGS.POPULAR_EMOJIS, - tokens: ["\u2581", "\ud83d\ude02", "\u2581", "\ud83d\udc4d", "\u2581", "\ud83e\udd23", "\u2581", "\ud83d\ude0d", "\u2581", "\ud83d\ude2d", "\u2581", "\ud83c\udf89", "\u2581", "\ud83d\ude4f", "\u2581", "\ud83d\ude0a", "\u2581", "\ud83d\udd25", "\u2581", "\ud83d\ude01", "\u2581", "\ud83d\ude05", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\ude06", "\u2581", "\ud83d\udc4f", "\u2581\u2764", "\ufe0f", "\u2581", "\ud83d\udc9c", "\u2581", "\ud83d\udc9a", "\u2581", "\ud83d\udc97", "\u2581", "\ud83d\udc99", "\u2581", "\ud83d\udda4", "\u2581", "\ud83d\ude0e", "\u2581", "\ud83d\udc4c", "\u2581", "\ud83e\udd73", "\u2581", "\ud83d\udcaa", "\u2581", "\u2728", "\u2581", "\ud83d\udc49", "\u2581", "\ud83d\udc40", "\u2581", "\ud83d\udcaf", "\u2581", "\ud83c\udf88", "\u2581", "\ud83d\ude48", "\u2581", "\ud83d\ude4c", "\u2581", "\ud83d\udc80", "\u2581", "\ud83d\udc47", "\u2581", "\ud83d\udc4b", "\u2581", "\u2705", "\u2581", "\ud83c\udf81", "\u2581", "\ud83c\udf1e", "\u2581", "\ud83c\udf38", "\u2581", "\ud83d\udcb0"], - ids: [0, 6, 115114, 6, 118280, 6, 243385, 6, 84464, 6, 232773, 6, 243816, 6, 113612, 6, 82803, 6, 222326, 6, 201344, 6, 239569, 6, 243544, 6, 191876, 6, 243404, 49933, 15755, 6, 244233, 6, 244162, 6, 244181, 6, 243892, 6, 245820, 6, 161546, 6, 204811, 6, 3, 6, 238992, 6, 167474, 6, 120242, 6, 245561, 6, 244864, 6, 246144, 6, 244459, 6, 244703, 6, 246887, 6, 144400, 6, 246511, 6, 142325, 6, 244230, 6, 245559, 6, 243374, 6, 245200, 2], - decoded: " \ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0", - }, - MULTIBYTE_EMOJIS: { - text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, - tokens: ["\u2581", "\u2728", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\udc41", "\ufe0f", "\u2581", "\ud83d\udc71", "\ud83c\udffb", "\u2581", "\ud83d\udd75", "\u2581", "\u2642", "\ufe0f", "\u2581", "\ud83e\uddd9", "\ud83c\udffb", "\u2581", "\u2642", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u2581", "\ud83c\udf3e", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83d\udc69", "\u2581\u2764", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc67", "\u2581", "\ud83d\udc66", "\u2581", "\ud83e\uddd1", "\ud83c\udffb", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1", "\ud83c\udffb", "\u2581", "\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u2581\u2764", "\ufe0f", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\ud83c\udffc"], - ids: [0, 6, 167474, 6, 243544, 6, 246984, 15755, 6, 247201, 79500, 6, 248325, 6, 228250, 15755, 6, 3, 79500, 6, 228250, 6, 244314, 79500, 6, 246529, 6, 3, 6, 247443, 6, 3, 6, 244785, 49933, 6, 244960, 6, 244314, 6, 244785, 6, 244785, 6, 245719, 6, 246167, 6, 3, 79500, 6, 247443, 6, 3, 79500, 6, 3, 6, 244314, 79500, 49933, 15755, 6, 244960, 6, 244314, 239719, 2], - decoded: " \u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75 \u2642\ufe0f \ud83c\udffb \u2642 \ud83d\udc68\ud83c\udffb \ud83c\udf3e \ud83e\udd1d \ud83d\udc69 \u2764 \ud83d\udc8b \ud83d\udc68 \ud83d\udc69 \ud83d\udc69 \ud83d\udc67 \ud83d\udc66 \ud83c\udffb \ud83e\udd1d \ud83c\udffb \ud83d\udc68\ud83c\udffb \u2764\ufe0f \ud83d\udc8b \ud83d\udc68\ud83c\udffc", - }, - CHINESE_LATIN_MIXED: { - text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, - tokens: ["\u2581ah", "\u535a", "\u63a8", "zz"], - ids: [0, 1263, 11173, 10238, 13894, 2], - decoded: " ah\u535a\u63a8zz", - }, - SIMPLE_WITH_ACCENTS: { - text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, - tokens: ["\u2581H\u00e9", "llo"], - ids: [0, 88064, 9284, 2], - decoded: " H\u00e9llo", - }, - MIXED_CASE_WITHOUT_ACCENTS: { - text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, - tokens: ["\u2581He", "LL", "o", "!", "how", "\u2581Are", "\u2581yo", "U", "?"], - ids: [0, 1529, 23708, 31, 38, 47251, 15901, 3005, 1062, 32, 2], - decoded: " HeLLo!how Are yoU?", - }, - MIXED_CASE_WITH_ACCENTS: { - text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, - tokens: ["\u2581H\u00e4", "LL", "o", "!", "how", "\u2581Are", "\u2581yo", "U", "?"], - ids: [0, 28863, 23708, 31, 38, 47251, 15901, 3005, 1062, 32, 2], - decoded: " H\u00e4LLo!how Are yoU?", - }, - }, - "Xenova/bert-base-multilingual-uncased-sentiment": { - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["let", "a", "=", "ob", "##j", ".", "tos", "##tri", "##ng", "(", ")", ";", "tos", "##tri", "##ng", "(", ")", ";"], - ids: [101, 12421, 143, 134, 15547, 10428, 119, 53564, 27711, 10422, 113, 114, 132, 53564, 27711, 10422, 113, 114, 132, 102], - decoded: "[CLS] let a = obj. tostring ( ) ; tostring ( ) ; [SEP]", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["un", "##wan", "##ted", ",", "running"], - ids: [101, 10119, 15134, 11894, 117, 16484, 102], - decoded: "[CLS] unwanted, running [SEP]", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["test", "$", "1", "r2", "#", "3", "\u20ac", "##4", "\u00a3", "##5", "\u00a5", "##6", "[UNK]", "\u20b9", "##8", "\u20b1", "##9", "test"], - ids: [101, 14084, 109, 122, 85583, 108, 124, 1329, 11124, 175, 11301, 177, 11325, 100, 1332, 11544, 1330, 11518, 14084, 102], - decoded: "[CLS] test $ 1 r2 # 3 \u20ac4 \u00a35 \u00a56 [UNK] \u20b98 \u20b19 test [SEP]", - }, - }, - "Xenova/multilingual-e5-small": { - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["\u2581trail", "ing", "\u2581space", "\u2581"], - ids: [0, 141037, 214, 32628, 6, 2], - decoded: " trailing space ", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - tokens: ["\u2581you", "...", "\u2581"], - ids: [0, 398, 27, 6, 2], - decoded: " you... ", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - tokens: ["\u2581you", "...", "\u2581"], - ids: [0, 398, 27, 6, 2], - decoded: " you... ", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - tokens: ["\u2581you", "...", "\u2581you", "...", "\u2581"], - ids: [0, 398, 27, 398, 27, 6, 2], - decoded: " you... you... ", - }, - MIXED_CASE_WITHOUT_ACCENTS: { - text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, - tokens: ["\u2581He", "LL", "o", "!", "how", "\u2581Are", "\u2581yo", "U", "?", "\u2581"], - ids: [0, 1529, 23708, 31, 38, 47251, 15901, 3005, 1062, 32, 6, 2], - decoded: " HeLLo!how Are yoU? ", - }, - MIXED_CASE_WITH_ACCENTS: { - text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, - tokens: ["\u2581H\u00e4", "LL", "o", "!", "how", "\u2581Are", "\u2581yo", "U", "?", "\u2581"], - ids: [0, 28863, 23708, 31, 38, 47251, 15901, 3005, 1062, 32, 6, 2], - decoded: " H\u00e4LLo!how Are yoU? ", - }, - }, - "Xenova/bge-small-zh-v1.5": { - SIMPLE: { - text: BASE_TEST_STRINGS.SIMPLE, - tokens: ["[UNK]", "are", "you", "doi", "##ng", "?"], - ids: [101, 100, 8995, 8357, 9962, 8291, 136, 102], - decoded: "[CLS] [UNK] are you doing? [SEP]", - }, - SIMPLE_WITH_PUNCTUATION: { - text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, - tokens: ["[UNK]", "sh", "##ould", "'", "ve", "don", "##e", "this"], - ids: [101, 100, 11167, 11734, 112, 12810, 9524, 8154, 8554, 102], - decoded: "[CLS] [UNK] should've done this [SEP]", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["[UNK]", "company", "was", "f", "##ound", "##ed", "in", "2016", "."], - ids: [101, 100, 10007, 9947, 148, 11477, 8303, 8217, 8112, 119, 102], - decoded: "[CLS] [UNK] company was founded in 2016. [SEP]", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["[UNK]", "'", "ll", "!", "!", "to", "?", "'", "d", "'", "'", "d", "of", ",", "can", "'", "t", "."], - ids: [101, 100, 112, 10856, 106, 106, 8228, 136, 112, 146, 112, 112, 146, 8205, 117, 9109, 112, 162, 119, 102], - decoded: "[CLS] [UNK]'ll!! to?'d'' d of, can't. [SEP]", - }, - PYTHON_CODE: { - text: BASE_TEST_STRINGS.PYTHON_CODE, - tokens: ["de", "##f", "main", "(", ")", ":", "pass"], - ids: [101, 8363, 8189, 9139, 113, 114, 131, 9703, 102], - decoded: "[CLS] def main ( ) : pass [SEP]", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["let", "a", "=", "ob", "##j", ".", "[UNK]", "(", ")", ";", "[UNK]", "(", ")", ";"], - ids: [101, 9946, 143, 134, 12639, 8334, 119, 100, 113, 114, 132, 100, 113, 114, 132, 102], - decoded: "[CLS] let a = obj. [UNK] ( ) ; [UNK] ( ) ; [SEP]", - }, - NEWLINES: { - text: BASE_TEST_STRINGS.NEWLINES, - tokens: ["[UNK]", "is", "a", "test", "."], - ids: [101, 100, 8310, 143, 10060, 119, 102], - decoded: "[CLS] [UNK] is a test. [SEP]", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["[UNK]", ",", "running"], - ids: [101, 100, 117, 11620, 102], - decoded: "[CLS] [UNK], running [SEP]", - }, - HELLO_WORLD_TITLECASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, - tokens: ["[UNK]", "[UNK]"], - ids: [101, 100, 100, 102], - decoded: "[CLS] [UNK] [UNK] [SEP]", - }, - LEADING_SPACE: { - text: BASE_TEST_STRINGS.LEADING_SPACE, - tokens: ["le", "##ad", "##ing", "space"], - ids: [101, 8983, 8695, 8221, 9634, 102], - decoded: "[CLS] leading space [SEP]", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["t", "##rail", "##ing", "space"], - ids: [101, 162, 12783, 8221, 9634, 102], - decoded: "[CLS] trailing space [SEP]", - }, - DOUBLE_SPACE: { - text: BASE_TEST_STRINGS.DOUBLE_SPACE, - tokens: ["[UNK]", "[UNK]"], - ids: [101, 100, 100, 102], - decoded: "[CLS] [UNK] [UNK] [SEP]", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["test", "$", "1", "[UNK]", "#", "3", "\u20ac", "##4", "\u00a3", "##5", "\u00a5", "##6", "[UNK]", "[UNK]", "[UNK]", "test"], - ids: [101, 10060, 109, 122, 100, 108, 124, 359, 8159, 173, 8157, 175, 8158, 100, 100, 100, 10060, 102], - decoded: "[CLS] test $ 1 [UNK] # 3 \u20ac4 \u00a35 \u00a56 [UNK] [UNK] [UNK] test [SEP]", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["[UNK]", "bo", "##ugh", "##t", "an", "apple", "for", "$", "1", ".", "00", "at", "the", "store", "."], - ids: [101, 100, 11059, 12667, 8165, 9064, 8350, 8330, 109, 122, 119, 8136, 8243, 8174, 8719, 119, 102], - decoded: "[CLS] [UNK] bought an apple for $ 1. 00 at the store. [SEP]", - }, - POPULAR_EMOJIS: { - text: BASE_TEST_STRINGS.POPULAR_EMOJIS, - tokens: ["\ud83d\ude02", "\ud83d\udc4d", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "\ud83d\udd25", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "\ud83d\ude0e", "[UNK]", "[UNK]", "[UNK]", "\u2728", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]"], - ids: [101, 8104, 8102, 100, 100, 100, 100, 100, 100, 8103, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 8105, 100, 100, 100, 501, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 102], - decoded: "[CLS] \ud83d\ude02 \ud83d\udc4d [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] \ud83d\udd25 [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] \ud83d\ude0e [UNK] [UNK] [UNK] \u2728 [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [SEP]", - }, - MULTIBYTE_EMOJIS: { - text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, - tokens: ["\u2728", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]"], - ids: [101, 501, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 102], - decoded: "[CLS] \u2728 [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [SEP]", - }, - SIMPLE_WITH_ACCENTS: { - text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, - tokens: ["[UNK]"], - ids: [101, 100, 102], - decoded: "[CLS] [UNK] [SEP]", - }, - MIXED_CASE_WITHOUT_ACCENTS: { - text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, - tokens: ["[UNK]", "!", "how", "[UNK]", "[UNK]", "?"], - ids: [101, 100, 106, 9510, 100, 100, 136, 102], - decoded: "[CLS] [UNK]! how [UNK] [UNK]? [SEP]", - }, - MIXED_CASE_WITH_ACCENTS: { - text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, - tokens: ["[UNK]", "!", "how", "[UNK]", "[UNK]", "?"], - ids: [101, 100, 106, 9510, 100, 100, 136, 102], - decoded: "[CLS] [UNK]! how [UNK] [UNK]? [SEP]", - }, - }, - "Xenova/bge-base-zh-v1.5": { - SIMPLE: { - text: BASE_TEST_STRINGS.SIMPLE, - tokens: ["how", "are", "you", "doi", "##ng", "?"], - ids: [101, 9510, 8995, 8357, 9962, 8291, 136, 102], - decoded: "[CLS] how are you doing? [SEP]", - }, - SIMPLE_WITH_PUNCTUATION: { - text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, - tokens: ["you", "sh", "##ould", "'", "ve", "don", "##e", "this"], - ids: [101, 8357, 11167, 11734, 112, 12810, 9524, 8154, 8554, 102], - decoded: "[CLS] you should've done this [SEP]", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["the", "company", "was", "f", "##ound", "##ed", "in", "2016", "."], - ids: [101, 8174, 10007, 9947, 148, 11477, 8303, 8217, 8112, 119, 102], - decoded: "[CLS] the company was founded in 2016. [SEP]", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["u", "##n", "##wan", "##ted", ",", "running"], - ids: [101, 163, 8171, 9951, 9255, 117, 11620, 102], - decoded: "[CLS] unwanted, running [SEP]", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["test", "$", "1", "r2", "#", "3", "\u20ac", "##4", "\u00a3", "##5", "\u00a5", "##6", "[UNK]", "[UNK]", "[UNK]", "test"], - ids: [101, 10060, 109, 122, 11345, 108, 124, 359, 8159, 173, 8157, 175, 8158, 100, 100, 100, 10060, 102], - decoded: "[CLS] test $ 1 r2 # 3 \u20ac4 \u00a35 \u00a56 [UNK] [UNK] [UNK] test [SEP]", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["i", "bo", "##ugh", "##t", "an", "apple", "for", "$", "1", ".", "00", "at", "the", "store", "."], - ids: [101, 151, 11059, 12667, 8165, 9064, 8350, 8330, 109, 122, 119, 8136, 8243, 8174, 8719, 119, 102], - decoded: "[CLS] i bought an apple for $ 1. 00 at the store. [SEP]", - }, - POPULAR_EMOJIS: { - text: BASE_TEST_STRINGS.POPULAR_EMOJIS, - tokens: ["\ud83d\ude02", "\ud83d\udc4d", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "\ud83d\udd25", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "\u2764", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "\ud83d\ude0e", "[UNK]", "[UNK]", "[UNK]", "\u2728", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]"], - ids: [101, 8104, 8102, 100, 100, 100, 100, 100, 100, 8103, 100, 100, 100, 100, 100, 506, 100, 100, 100, 100, 100, 8105, 100, 100, 100, 501, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 102], - decoded: "[CLS] \ud83d\ude02 \ud83d\udc4d [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] \ud83d\udd25 [UNK] [UNK] [UNK] [UNK] [UNK] \u2764 [UNK] [UNK] [UNK] [UNK] [UNK] \ud83d\ude0e [UNK] [UNK] [UNK] \u2728 [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [SEP]", - }, - }, - "Xenova/indobert-base-p1": { - SIMPLE_WITH_PUNCTUATION: { - text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, - tokens: ["you", "sho", "##uld", "'", "ve", "don", "##e", "this"], - ids: [2, 3299, 9596, 15370, 30463, 28239, 4081, 30357, 5379, 3], - decoded: "[CLS] you should've done this [SEP]", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["the", "company", "was", "found", "##ed", "in", "2016", "."], - ids: [2, 1002, 9105, 2738, 11009, 133, 48, 1538, 30470, 3], - decoded: "[CLS] the company was founded in 2016. [SEP]", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["let", "a", "=", "ob", "##j", ".", "tos", "##trin", "##g", "(", ")", ";", "tos", "##trin", "##g", "(", ")", ";"], - ids: [2, 4734, 253, 30475, 559, 30372, 30470, 20498, 12448, 30365, 30464, 30465, 30473, 20498, 12448, 30365, 30464, 30465, 30473, 3], - decoded: "[CLS] let a = obj. tostring ( ) ; tostring ( ) ; [SEP]", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["un", "##wan", "##te", "##d", ",", "running"], - ids: [2, 78, 1322, 3298, 30364, 30468, 22715, 3], - decoded: "[CLS] unwanted, running [SEP]", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]"], - ids: [2, 1, 1, 1, 1, 1, 1, 3], - decoded: "[CLS] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [SEP]", - }, - LEADING_SPACE: { - text: BASE_TEST_STRINGS.LEADING_SPACE, - tokens: ["lead", "##ing", "space"], - ids: [2, 9196, 55, 14561, 3], - decoded: "[CLS] leading space [SEP]", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["test", "$", "1", "r", "##2", "#", "3", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "test"], - ids: [2, 4243, 30460, 111, 56, 30378, 30459, 283, 1, 1, 1, 1, 1, 1, 4243, 3], - decoded: "[CLS] test $ 1 r2 # 3 [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] test [SEP]", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["i", "bo", "##ught", "an", "apple", "for", "$", "1", ".", "00", "at", "the", "store", "."], - ids: [2, 89, 1880, 25009, 223, 7761, 1548, 30460, 111, 30470, 4230, 117, 1002, 8052, 30470, 3], - decoded: "[CLS] i bought an apple for $ 1. 00 at the store. [SEP]", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["wei", "##rd", "[UNK]", "edge", "[UNK]", "case"], - ids: [2, 27753, 12548, 1, 21418, 1, 13687, 3], - decoded: "[CLS] weird [UNK] edge [UNK] case [SEP]", - }, - MIXED_CASE_WITH_ACCENTS: { - text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, - tokens: ["hallo", "!", "how", "are", "you", "?"], - ids: [2, 19598, 30457, 11088, 5811, 3299, 30477, 3], - decoded: "[CLS] hallo! how are you? [SEP]", - }, - }, - "Xenova/spanbert-large-cased": { - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["let", "a", "=", "o", "##b", "##j", ".", "to", "##st", "##ring", "(", ")", ";", "to", "##st", "##ring", "(", ")", ";"], - ids: [101, 1519, 170, 134, 184, 1830, 3361, 119, 1106, 2050, 3384, 113, 114, 132, 1106, 2050, 3384, 113, 114, 132, 102], - decoded: "[CLS] let a = obj. tostring ( ) ; tostring ( ) ; [SEP]", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["test", "$", "1", "r", "##2", "#", "3", "\u20ac", "##4", "\u00a3", "##5", "\u00a5", "##6", "[UNK]", "\u20b9", "##8", "\u20b1", "##9", "test"], - ids: [101, 2774, 109, 122, 187, 1477, 108, 124, 836, 1527, 202, 1571, 203, 1545, 100, 838, 1604, 837, 1580, 2774, 102], - decoded: "[CLS] test $ 1 r2 # 3 \u20ac4 \u00a35 \u00a56 [UNK] \u20b98 \u20b19 test [SEP]", - }, - }, - "Xenova/UMLSBert_ENG": { - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["let", "a", "=", "obj", ".", "tos", "##tring", "(", ")", ";", "tos", "##tring", "(", ")", ";"], - ids: [2, 8894, 42, 32, 2473, 17, 22660, 23640, 11, 12, 30, 22660, 23640, 11, 12, 30, 3], - decoded: "[CLS] let a = obj. tostring ( ) ; tostring ( ) ; [SEP]", - }, - HELLO_WORLD_TITLECASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, - tokens: ["hel", "##lo", "world"], - ids: [2, 3018, 5368, 4517, 3], - decoded: "[CLS] hello world [SEP]", - }, - HELLO_WORLD_LOWERCASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, - tokens: ["hel", "##lo", "world"], - ids: [2, 3018, 5368, 4517, 3], - decoded: "[CLS] hello world [SEP]", - }, - DOUBLE_SPACE: { - text: BASE_TEST_STRINGS.DOUBLE_SPACE, - tokens: ["hi", "hel", "##lo"], - ids: [2, 11245, 3018, 5368, 3], - decoded: "[CLS] hi hello [SEP]", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["test", "$", "1", "r2", "#", "3", "\u20ac", "##4", "\u00a3", "##5", "\u00a5", "##6", "\u20a3", "##7", "\u20b9", "##8", "\u20b1", "##9", "test"], - ids: [2, 2313, 7, 20, 9663, 6, 22, 528, 1017, 74, 1009, 76, 1018, 524, 1019, 531, 1011, 529, 1038, 2313, 3], - decoded: "[CLS] test $ 1 r2 # 3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test [SEP]", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["we", "##ir", "##d", "\uff5e", "edge", "\uff5e", "case"], - ids: [2, 1802, 1753, 1022, 943, 9676, 943, 2632, 3], - decoded: "[CLS] weird \uff5e edge \uff5e case [SEP]", - }, - SIMPLE_WITH_ACCENTS: { - text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, - tokens: ["hel", "##lo"], - ids: [2, 3018, 5368, 3], - decoded: "[CLS] hello [SEP]", - }, - MIXED_CASE_WITHOUT_ACCENTS: { - text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, - tokens: ["hel", "##lo", "!", "how", "are", "you", "?"], - ids: [2, 3018, 5368, 5, 2135, 1810, 17915, 34, 3], - decoded: "[CLS] hello! how are you? [SEP]", - }, - }, - "Xenova/SapBERT-from-PubMedBERT-fulltext": { - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u751f", "\u6d3b", "\u7684", "[UNK]", "[UNK]", "\u662f"], - ids: [2, 799, 776, 811, 1, 1, 731, 3], - decoded: "[CLS] \u751f \u6d3b \u7684 [UNK] [UNK] \u662f [SEP]", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["test", "$", "1", "r2", "#", "3", "\u20ac", "##4", "\u00a3", "##5", "\u00a5", "##6", "[UNK]", "\u20b9", "##8", "[UNK]", "test"], - ids: [2, 2648, 8, 21, 7261, 7, 23, 281, 1006, 76, 1015, 78, 1016, 1, 282, 1025, 1, 2648, 3], - decoded: "[CLS] test $ 1 r2 # 3 \u20ac4 \u00a35 \u00a56 [UNK] \u20b98 [UNK] test [SEP]", - }, - }, - "Xenova/rubert-base-cased": { - SIMPLE: { - text: BASE_TEST_STRINGS.SIMPLE, - tokens: ["How", "are", "you", "do", "##ing", "?"], - ids: [101, 15474, 10813, 13540, 10661, 7729, 166, 102], - decoded: "[CLS] How are you doing? [SEP]", - }, - SIMPLE_WITH_PUNCTUATION: { - text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, - tokens: ["You", "sh", "##oul", "##d", "'", "ve", "don", "##e", "this"], - ids: [101, 11577, 45942, 76143, 239, 118, 10835, 17450, 241, 11043, 102], - decoded: "[CLS] You should've done this [SEP]", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["The", "comp", "##any", "was", "f", "##ound", "##ed", "in", "2016", "."], - ids: [101, 6821, 71382, 17927, 10646, 242, 71129, 7491, 10618, 8273, 132, 102], - decoded: "[CLS] The company was founded in 2016. [SEP]", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["let", "a", "=", "ob", "##j", ".", "to", "##St", "##ring", "(", ")", ";", "to", "##St", "##ring", "(", ")", ";"], - ids: [101, 14107, 232, 162, 17851, 251, 132, 10626, 21568, 13647, 120, 122, 158, 10626, 21568, 13647, 120, 122, 158, 102], - decoded: "[CLS] let a = obj. toString ( ) ; toString ( ) ; [SEP]", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["UN", "##wan", "##t", "##\u00e9d", ",", "run", "##ning"], - ids: [101, 27090, 14906, 271, 84705, 128, 14607, 11781, 102], - decoded: "[CLS] UNwant\u00e9d, running [SEP]", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u751f", "\u6d3b", "\u7684", "\u771f", "[UNK]", "\u662f"], - ids: [101, 6104, 5480, 6222, 6273, 100, 4877, 102], - decoded: "[CLS] \u751f \u6d3b \u7684 \u771f [UNK] \u662f [SEP]", - }, - LEADING_SPACE: { - text: BASE_TEST_STRINGS.LEADING_SPACE, - tokens: ["le", "##ading", "sp", "##ace"], - ids: [101, 10653, 73130, 33162, 13967, 102], - decoded: "[CLS] leading space [SEP]", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["tra", "##ili", "##ng", "sp", "##ace"], - ids: [101, 11776, 14296, 10888, 33162, 13967, 102], - decoded: "[CLS] trailing space [SEP]", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["I", "bo", "##ught", "an", "app", "##le", "for", "$", "1", ".", "00", "at", "the", "st", "##ore", "."], - ids: [101, 186, 21018, 53718, 10663, 73406, 7159, 10654, 112, 138, 132, 11537, 10672, 10617, 28668, 13536, 132, 102], - decoded: "[CLS] I bought an apple for $ 1. 00 at the store. [SEP]", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["we", "##ird", "\uff5e", "ed", "##ge", "\uff5e", "cas", "##e"], - ids: [101, 12463, 36865, 10608, 11051, 11037, 10608, 15501, 241, 102], - decoded: "[CLS] weird \uff5e edge \uff5e case [SEP]", - }, - CHINESE_LATIN_MIXED: { - text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, - tokens: ["a", "##h", "\u535a", "\u63a8", "z", "##z"], - ids: [101, 232, 247, 3166, 4657, 282, 283, 102], - decoded: "[CLS] ah \u535a \u63a8 zz [SEP]", - }, - MIXED_CASE_WITHOUT_ACCENTS: { - text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, - tokens: ["He", "##LL", "##o", "!", "ho", "##w", "Are", "yo", "##U", "?"], - ids: [101, 10869, 83346, 261, 106, 13685, 277, 14003, 14184, 211, 166, 102], - decoded: "[CLS] HeLLo! how Are yoU? [SEP]", - }, - MIXED_CASE_WITH_ACCENTS: { - text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, - tokens: ["H", "##\u00e4", "##LL", "##o", "!", "ho", "##w", "Are", "yo", "##U", "?"], - ids: [101, 184, 384, 83346, 261, 106, 13685, 277, 14003, 14184, 211, 166, 102], - decoded: "[CLS] H\u00e4LLo! how Are yoU? [SEP]", - }, - }, - "Xenova/kobert": { - SIMPLE: { - text: BASE_TEST_STRINGS.SIMPLE, - tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]", "?"], - ids: [2, 0, 0, 0, 0, 258, 3], - decoded: "[CLS] [UNK] [UNK] [UNK] [UNK]? [SEP]", - }, - SIMPLE_WITH_PUNCTUATION: { - text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, - tokens: ["[UNK]", "[UNK]", "'", "[UNK]", "[UNK]", "[UNK]"], - ids: [2, 0, 0, 15, 0, 0, 0, 3], - decoded: "[CLS] [UNK] [UNK]'[UNK] [UNK] [UNK] [SEP]", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["The", "[UNK]", "[UNK]", "[UNK]", "in", "[UNK]", "."], - ids: [2, 355, 0, 0, 0, 409, 0, 54, 3], - decoded: "[CLS] The [UNK] [UNK] [UNK] in [UNK]. [SEP]", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["A", "'", "[UNK]", "!", "!", "[UNK]", "?", "'", "d", "'", "'", "d", "[UNK]", ",", "[UNK]", "'", "t", "."], - ids: [2, 264, 15, 0, 5, 5, 0, 258, 15, 388, 15, 15, 388, 0, 46, 0, 15, 442, 54, 3], - decoded: "[CLS] A'[UNK]!! [UNK]?'d'' d [UNK], [UNK]'t. [SEP]", - }, - PYTHON_CODE: { - text: BASE_TEST_STRINGS.PYTHON_CODE, - tokens: ["[UNK]", "[UNK]", "(", ")", ":", "[UNK]"], - ids: [2, 0, 0, 18, 40, 249, 0, 3], - decoded: "[CLS] [UNK] [UNK] ( ) : [UNK] [SEP]", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["[UNK]", "a", "=", "[UNK]", ".", "[UNK]", "(", ")", ";", "[UNK]", "(", ")", ";"], - ids: [2, 0, 367, 254, 0, 54, 0, 18, 40, 252, 0, 18, 40, 252, 3], - decoded: "[CLS] [UNK] a = [UNK]. [UNK] ( ) ; [UNK] ( ) ; [SEP]", - }, - NEWLINES: { - text: BASE_TEST_STRINGS.NEWLINES, - tokens: ["[UNK]", "is", "a", "[UNK]", "."], - ids: [2, 0, 412, 367, 0, 54, 3], - decoded: "[CLS] [UNK] is a [UNK]. [SEP]", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["[UNK]", ",", "[UNK]"], - ids: [2, 0, 46, 0, 3], - decoded: "[CLS] [UNK], [UNK] [SEP]", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u751f", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]"], - ids: [2, 5298, 0, 0, 0, 0, 0, 3], - decoded: "[CLS] \u751f [UNK] [UNK] [UNK] [UNK] [UNK] [SEP]", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["[UNK]", "$", "1", "[UNK]", "#", "3", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]"], - ids: [2, 0, 10, 93, 0, 9, 142, 0, 0, 0, 0, 0, 0, 0, 3], - decoded: "[CLS] [UNK] $ 1 [UNK] # 3 [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [SEP]", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["I", "[UNK]", "an", "[UNK]", "[UNK]", "$", "1", ".", "00", "at", "[UNK]", "[UNK]", "."], - ids: [2, 296, 0, 374, 0, 0, 10, 93, 54, 79, 377, 0, 0, 54, 3], - decoded: "[CLS] I [UNK] an [UNK] [UNK] $ 1. 00 at [UNK] [UNK]. [SEP]", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]"], - ids: [2, 0, 0, 0, 0, 3], - decoded: "[CLS] [UNK] [UNK] [UNK] [UNK] [SEP]", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]"], - ids: [2, 0, 0, 0, 0, 0, 3], - decoded: "[CLS] [UNK] [UNK] [UNK] [UNK] [UNK] [SEP]", - }, - SPIECE_UNDERSCORE: { - text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, - tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]", "\u2581", "."], - ids: [2, 0, 0, 0, 0, 517, 54, 3], - decoded: "[CLS] [UNK] [UNK] [UNK] [UNK] \u2581. [SEP]", - }, - CHINESE_LATIN_MIXED: { - text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, - tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]"], - ids: [2, 0, 0, 0, 0, 3], - decoded: "[CLS] [UNK] [UNK] [UNK] [UNK] [SEP]", - }, - MIXED_CASE_WITHOUT_ACCENTS: { - text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, - tokens: ["[UNK]", "!", "[UNK]", "[UNK]", "[UNK]", "?"], - ids: [2, 0, 5, 0, 0, 0, 258, 3], - decoded: "[CLS] [UNK]! [UNK] [UNK] [UNK]? [SEP]", - }, - MIXED_CASE_WITH_ACCENTS: { - text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, - tokens: ["[UNK]", "!", "[UNK]", "[UNK]", "[UNK]", "?"], - ids: [2, 0, 5, 0, 0, 0, 258, 3], - decoded: "[CLS] [UNK]! [UNK] [UNK] [UNK]? [SEP]", - }, - }, - "Xenova/scibert_scivocab_uncased": { - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["let", "a", "=", "obj", ".", "to", "##string", "(", ")", ";", "to", "##string", "(", ")", ";"], - ids: [102, 1293, 106, 275, 2324, 205, 147, 20301, 145, 546, 1814, 147, 20301, 145, 546, 1814, 103], - decoded: "[CLS] let a = obj. tostring ( ) ; tostring ( ) ; [SEP]", - }, - DOUBLE_SPACE: { - text: BASE_TEST_STRINGS.DOUBLE_SPACE, - tokens: ["hi", "hell", "##o"], - ids: [102, 5305, 29423, 30112, 103], - decoded: "[CLS] hi hello [SEP]", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["test", "$", "1", "r", "##2", "#", "3", "\u20ac", "##4", "\u00a3", "##5", "\u00a5", "##6", "[UNK]", "[UNK]", "[UNK]", "test"], - ids: [102, 856, 3250, 158, 182, 30132, 3000, 239, 20801, 30140, 11221, 30139, 20704, 30142, 101, 101, 101, 856, 103], - decoded: "[CLS] test $ 1 r2 # 3 \u20ac4 \u00a35 \u00a56 [UNK] [UNK] [UNK] test [SEP]", - }, - CHINESE_LATIN_MIXED: { - text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, - tokens: ["ah", "[UNK]", "[UNK]", "zz"], - ids: [102, 7839, 101, 101, 23591, 103], - decoded: "[CLS] ah [UNK] [UNK] zz [SEP]", - }, - SIMPLE_WITH_ACCENTS: { - text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, - tokens: ["hell", "##o"], - ids: [102, 29423, 30112, 103], - decoded: "[CLS] hello [SEP]", - }, - MIXED_CASE_WITHOUT_ACCENTS: { - text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, - tokens: ["hell", "##o", "!", "how", "are", "you", "?"], - ids: [102, 29423, 30112, 3190, 539, 220, 3034, 3912, 103], - decoded: "[CLS] hello! how are you? [SEP]", - }, - }, - "Xenova/LaBSE": { - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["let", "a", "=", "obj", ".", "to", "##String", "(", ")", ";", "to", "##String", "(", ")", ";"], - ids: [101, 17214, 170, 134, 228877, 119, 14986, 368304, 113, 114, 132, 14986, 368304, 113, 114, 132, 102], - decoded: "[CLS] let a = obj. toString ( ) ; toString ( ) ; [SEP]", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["test", "$", "1", "R2", "#", "3", "\u20ac", "##4", "\u00a35", "\u00a5", "##6", "\u20a3", "##7", "\u20b9", "##8", "\u20b1", "##9", "test"], - ids: [101, 17678, 109, 122, 51222, 108, 124, 3030, 16006, 279082, 205, 16151, 3023, 16187, 3037, 16175, 3033, 16236, 17678, 102], - decoded: "[CLS] test $ 1 R2 # 3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test [SEP]", - }, - SPIECE_UNDERSCORE: { - text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, - tokens: ["\u2581", "##This", "\u2581", "##is", "\u2581", "##a", "\u2581", "##test", "\u2581", "."], - ids: [101, 3283, 342068, 3283, 15319, 3283, 14983, 3283, 50149, 3283, 119, 102], - decoded: "[CLS] \u2581This \u2581is \u2581a \u2581test \u2581. [SEP]", - }, - POPULAR_EMOJIS: { - text: BASE_TEST_STRINGS.POPULAR_EMOJIS, - tokens: ["\ud83d\ude02", "\ud83d\udc4d", "\ud83e\udd23", "\ud83d\ude0d", "\ud83d\ude2d", "\ud83c\udf89", "\ud83d\ude4f", "\ud83d\ude0a", "\ud83d\udd25", "\ud83d\ude01", "\ud83d\ude05", "\ud83e\udd17", "\ud83d\ude06", "\ud83d\udc4f", "\u2764\ufe0f", "\ud83d\udc9c", "\ud83d\udc9a", "\ud83d\udc97", "\ud83d\udc99", "\ud83d\udda4", "\ud83d\ude0e", "\ud83d\udc4c", "\ud83e\udd73", "\ud83d\udcaa", "\u2728", "\ud83d\udc49", "\ud83d\udc40", "\ud83d\udcaf", "\ud83c\udf88", "\ud83d\ude48", "\ud83d\ude4c", "\ud83d\udc80", "\ud83d\udc47", "\ud83d\udc4b", "\u2705", "\ud83c\udf81", "\ud83c\udf1e", "\ud83c\udf38", "\ud83d\udcb0"], - ids: [101, 14820, 14617, 14933, 14831, 14863, 14496, 14893, 14828, 14775, 14819, 14823, 14926, 14824, 14619, 91822, 14687, 14685, 14682, 14684, 14810, 14832, 14616, 14956, 14701, 3496, 14613, 14606, 14706, 14495, 14887, 14891, 14660, 14611, 14615, 3465, 14488, 14416, 14430, 14707, 102], - decoded: "[CLS] \ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0 [SEP]", - }, - MULTIBYTE_EMOJIS: { - text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, - tokens: ["\u2728", "\ud83e\udd17", "\ud83d\udc41\ufe0f", "\ud83d\udc71", "##\ud83c\udffb", "[UNK]", "[UNK]", "\ud83d\udc68", "##\ud83c\udffb", "##\ud83c\udf3e", "[UNK]", "\ud83d\udc69", "##\u2764", "##\ud83d\udc8b", "##\ud83d\udc68", "\ud83d\udc69", "##\ud83d\udc69", "##\ud83d\udc67", "##\ud83d\udc66", "[UNK]", "\ud83c\udff4", "\ud83d\udc68", "##\ud83c\udffb", "##\u2764", "##\ufe0f", "##\ud83d\udc8b", "##\ud83d\udc68", "##\ud83c\udffc"], - ids: [101, 3496, 14926, 350545, 14648, 130826, 100, 100, 14639, 130826, 498832, 100, 14640, 488649, 499065, 499034, 14640, 499035, 499033, 499032, 100, 14555, 14639, 130826, 488649, 44450, 499065, 499034, 421916, 102], - decoded: "[CLS] \u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb [UNK] [UNK] \ud83d\udc68\ud83c\udffb\ud83c\udf3e [UNK] \ud83d\udc69\u2764\ud83d\udc8b\ud83d\udc68 \ud83d\udc69\ud83d\udc69\ud83d\udc67\ud83d\udc66 [UNK] \ud83c\udff4 \ud83d\udc68\ud83c\udffb\u2764\ufe0f\ud83d\udc8b\ud83d\udc68\ud83c\udffc [SEP]", - }, - CHINESE_LATIN_MIXED: { - text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, - tokens: ["ah", "\u535a", "\u63a8", "zz"], - ids: [101, 15524, 4573, 6405, 441764, 102], - decoded: "[CLS] ah \u535a \u63a8 zz [SEP]", - }, - SIMPLE_WITH_ACCENTS: { - text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, - tokens: ["H\u00e9", "##llo"], - ids: [101, 220855, 23025, 102], - decoded: "[CLS] H\u00e9llo [SEP]", - }, - }, - "Xenova/herbert-large-cased": { - SIMPLE: { - text: BASE_TEST_STRINGS.SIMPLE, - tokens: ["Ho", "w", "are", "you", "do", "ing", "?"], - ids: [0, 5213, 1019, 25720, 20254, 2065, 5129, 1550, 2], - decoded: "How are you doing? ", - }, - SIMPLE_WITH_PUNCTUATION: { - text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, - tokens: ["You", "sho", "uld", "'", "ve", "d", "one", "this"], - ids: [0, 32795, 14924, 48273, 1571, 6647, 72, 2290, 48846, 2], - decoded: "You should've done this ", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["The", "co", "mpany", "was", "fo", "un", "de", "d", "in", "20", "16", "."], - ids: [0, 7117, 2406, 41449, 9873, 3435, 2195, 2101, 1038, 2651, 5646, 2555, 1899, 2], - decoded: "The company was founded in 2016. ", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["A", "'", "ll", "!", "!", "to", "?", "'", "d", "'", "'", "d", "of", ",", "can", "'", "t", "."], - ids: [0, 1012, 1571, 9396, 1725, 1725, 2063, 1550, 1571, 1038, 1571, 1571, 1038, 6595, 1947, 26794, 1571, 1026, 1899, 2], - decoded: "A'll!! to?'d'' d of, can't. ", - }, - PYTHON_CODE: { - text: BASE_TEST_STRINGS.PYTHON_CODE, - tokens: ["de", "f", "main", "(", ")", ":", "pa", "ss"], - ids: [0, 2101, 1050, 41851, 1341, 1940, 1335, 2083, 5357, 2], - decoded: "def main ( ) : pass ", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["let", "a", "=", "ob", "j", ".", "to", "S", "tr", "ing", "(", ")", ";", "to", "S", "tr", "ing", "(", ")", ";"], - ids: [0, 11324, 1011, 1789, 2033, 1013, 1899, 2146, 55, 2518, 5129, 1341, 1940, 1195, 2146, 55, 2518, 5129, 1341, 1940, 1195, 2], - decoded: "let a = obj. toString ( ) ; toString ( ) ; ", - }, - NEWLINES: { - text: BASE_TEST_STRINGS.NEWLINES, - tokens: ["T", "his", "is", "a", "test", "."], - ids: [0, 56, 22855, 6869, 1011, 14825, 1899, 2], - decoded: "This is a test. ", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["UN", "wan", "t", "\u00e9", "d", ",", "run", "ning"], - ids: [0, 23029, 2688, 88, 163, 1038, 1947, 4980, 17843, 2], - decoded: "UNwant\u00e9d, running ", - }, - CONTROL_TOKENS: { - text: BASE_TEST_STRINGS.CONTROL_TOKENS, - tokens: ["123"], - ids: [0, 19049, 2], - decoded: "123 ", - }, - HELLO_WORLD_TITLECASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, - tokens: ["Hel", "lo", "World"], - ids: [0, 12156, 6170, 21207, 2], - decoded: "Hello World ", - }, - HELLO_WORLD_LOWERCASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, - tokens: ["hel", "lo", "world"], - ids: [0, 11526, 6170, 38188, 2], - decoded: "hello world ", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["", "", "", "", "", "\u662f"], - ids: [0, 3, 3, 3, 3, 3, 1776, 2], - decoded: "\u662f ", - }, - LEADING_SPACE: { - text: BASE_TEST_STRINGS.LEADING_SPACE, - tokens: ["le", "ad", "ing", "space"], - ids: [0, 2018, 2035, 5129, 46489, 2], - decoded: "leading space ", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["tra", "i", "ling", "space"], - ids: [0, 2201, 77, 16342, 46489, 2], - decoded: "trailing space ", - }, - DOUBLE_SPACE: { - text: BASE_TEST_STRINGS.DOUBLE_SPACE, - tokens: ["H", "i", "Hel", "lo"], - ids: [0, 44, 1009, 12156, 6170, 2], - decoded: "Hi Hello ", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["test", "$", "1", "R", "2", "#", "3", "\u20ac", "4", "\u00a3", "5", "", "6", "", "7", "", "8", "", "9", "test"], - ids: [0, 14825, 1927, 1029, 54, 1025, 1393, 1034, 706, 1018, 100, 1008, 3, 1036, 3, 1030, 3, 1064, 3, 1017, 14825, 2], - decoded: "test $ 1 R2 # 3 \u20ac4 \u00a35 6 7 8 9 test ", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["I", "bou", "ght", "an", "ap", "ple", "for", "$", "1", ".", "00", "at", "the", "st", "ore", "."], - ids: [0, 1056, 13016, 15272, 2879, 10309, 20861, 15181, 1927, 1029, 1899, 2291, 4772, 6854, 1989, 24005, 1899, 2], - decoded: "I bought an apple for $ 1. 00 at the store. ", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - tokens: ["you", "\u2026"], - ids: [0, 20254, 1826, 2], - decoded: "you \u2026 ", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - tokens: ["you", "\u2026"], - ids: [0, 20254, 1826, 2], - decoded: "you \u2026 ", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - tokens: ["you", "\u2026", "you", "\u2026"], - ids: [0, 20254, 1826, 20254, 1826, 2], - decoded: "you \u2026 you \u2026 ", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["we", "ir", "d", "", "e", "dge", "", "ca", "se"], - ids: [0, 2149, 17435, 1038, 3, 73, 25801, 3, 3833, 4417, 2], - decoded: "weird edge case ", - }, - SPIECE_UNDERSCORE: { - text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, - tokens: ["", "T", "his", "", "is", "", "a", "", "test", "", "."], - ids: [0, 3, 56, 22855, 3, 6869, 3, 1011, 3, 14825, 3, 1899, 2], - decoded: "This is a test . ", - }, - CHINESE_LATIN_MIXED: { - text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, - tokens: ["a", "h", "", "", "zz"], - ids: [0, 69, 1021, 3, 3, 49185, 2], - decoded: "ah zz ", - }, - SIMPLE_WITH_ACCENTS: { - text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, - tokens: ["H", "\u00e9", "l", "lo"], - ids: [0, 44, 163, 80, 6170, 2], - decoded: "H\u00e9llo ", - }, - MIXED_CASE_WITHOUT_ACCENTS: { - text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, - tokens: ["He", "L", "L", "o", "!", "ho", "w", "Ar", "e", "yo", "U", "?"], - ids: [0, 4596, 48, 48, 1007, 1725, 3145, 1019, 2921, 1015, 13908, 1041, 1550, 2], - decoded: "HeLLo! how Are yoU? ", - }, - MIXED_CASE_WITH_ACCENTS: { - text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, - tokens: ["H", "\u00e4", "L", "L", "o", "!", "ho", "w", "Ar", "e", "yo", "U", "?"], - ids: [0, 44, 158, 48, 48, 1007, 1725, 3145, 1019, 2921, 1015, 13908, 1041, 1550, 2], - decoded: "H\u00e4LLo! how Are yoU? ", - }, - }, - "Xenova/ernie-gram-zh": { - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["test", "$", "1", "r2", "#", "3", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "test"], - ids: [1, 6943, 18005, 208, 6847, 9474, 284, 18017, 18017, 18017, 18017, 18017, 18017, 6943, 2], - decoded: "[CLS] test $ 1 r2 # 3 [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] test [SEP]", + "Xenova/bert-base-multilingual-cased-ner-hrl": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["How", "are", "you", "doing", "?"], + ids: [101, 14962, 10301, 13028, 30918, 136, 102], + decoded: "[CLS] How are you doing? [SEP]", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["You", "should", "'", "ve", "done", "this"], + ids: [101, 11065, 14819, 112, 10323, 20378, 10531, 102], + decoded: "[CLS] You should've done this [SEP]", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "company", "was", "founded", "in", "2016", "."], + ids: [101, 10117, 12100, 10134, 14078, 10106, 10255, 119, 102], + decoded: "[CLS] The company was founded in 2016. [SEP]", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "'", "ll", "!", "!", "to", "?", "'", "d", "'", "'", "d", "of", ",", "can", "'", "t", "."], + ids: [101, 138, 112, 22469, 106, 106, 10114, 136, 112, 172, 112, 112, 172, 10108, 117, 10944, 112, 188, 119, 102], + decoded: "[CLS] A'll!! to?'d'' d of, can't. [SEP]", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "a", "=", "ob", "##j", ".", "to", "##S", "##trin", "##g", "(", ")", ";", "to", "##S", "##trin", "##g", "(", ")", ";"], + ids: [101, 13595, 169, 134, 17339, 10418, 119, 10114, 10731, 109163, 10240, 113, 114, 132, 10114, 10731, 109163, 10240, 113, 114, 132, 102], + decoded: "[CLS] let a = obj. toString ( ) ; toString ( ) ; [SEP]", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["This", "is", "a", "test", "."], + ids: [101, 10747, 10124, 169, 15839, 119, 102], + decoded: "[CLS] This is a test. [SEP]", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "##want", "##\u00e9d", ",", "running"], + ids: [101, 26578, 104216, 84193, 117, 18020, 102], + decoded: "[CLS] UNwant\u00e9d, running [SEP]", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["Hello", "World"], + ids: [101, 31178, 10315, 102], + decoded: "[CLS] Hello World [SEP]", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["hell", "##o", "world"], + ids: [101, 61694, 10133, 11356, 102], + decoded: "[CLS] hello world [SEP]", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u751f", "\u6d3b", "\u7684", "\u771f", "\u8c1b", "\u662f"], + ids: [101, 5600, 4978, 5718, 5769, 7378, 4380, 102], + decoded: "[CLS] \u751f \u6d3b \u7684 \u771f \u8c1b \u662f [SEP]", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["trail", "##ing", "space"], + ids: [101, 56559, 10230, 16199, 102], + decoded: "[CLS] trailing space [SEP]", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["Hi", "Hello"], + ids: [101, 20065, 31178, 102], + decoded: "[CLS] Hi Hello [SEP]", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "$", "1", "R2", "#", "3", "\u20ac", "##4", "\u00a3", "##5", "\u00a5", "##6", "[UNK]", "\u20b9", "##8", "[UNK]", "test"], + ids: [101, 15839, 109, 122, 94000, 108, 124, 1775, 11011, 201, 11166, 202, 11211, 100, 1776, 11396, 100, 15839, 102], + decoded: "[CLS] test $ 1 R2 # 3 \u20ac4 \u00a35 \u00a56 [UNK] \u20b98 [UNK] test [SEP]", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "bought", "an", "app", "##le", "for", "$", "1", ".", "00", "at", "the", "store", "."], + ids: [101, 146, 28870, 10151, 72894, 10284, 10142, 109, 122, 119, 11025, 10160, 10105, 13708, 119, 102], + decoded: "[CLS] I bought an apple for $ 1. 00 at the store. [SEP]", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "[UNK]"], + ids: [101, 13028, 100, 102], + decoded: "[CLS] you [UNK] [SEP]", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "[UNK]"], + ids: [101, 13028, 100, 102], + decoded: "[CLS] you [UNK] [SEP]", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "[UNK]", "you", "[UNK]"], + ids: [101, 13028, 100, 13028, 100, 102], + decoded: "[CLS] you [UNK] you [UNK] [SEP]", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["wei", "##rd", "\uff5e", "edge", "\uff5e", "case"], + ids: [101, 86981, 12023, 10096, 30599, 10096, 13474, 102], + decoded: "[CLS] weird \uff5e edge \uff5e case [SEP]", + }, + CHINESE_LATIN_MIXED: { + text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, + tokens: ["ah", "\u535a", "\u63a8", "z", "##z"], + ids: [101, 69863, 2684, 4163, 194, 10305, 102], + decoded: "[CLS] ah \u535a \u63a8 zz [SEP]", + }, + SIMPLE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, + tokens: ["H", "##\u00e9l", "##lo"], + ids: [101, 145, 24817, 10715, 102], + decoded: "[CLS] H\u00e9llo [SEP]", + }, + MIXED_CASE_WITHOUT_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, + tokens: ["He", "##LL", "##o", "!", "how", "Are", "yo", "##U", "?"], + ids: [101, 10357, 82834, 10133, 106, 14796, 13491, 13672, 12022, 136, 102], + decoded: "[CLS] HeLLo! how Are yoU? [SEP]", + }, + MIXED_CASE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, + tokens: ["H", "##\u00e4", "##LL", "##o", "!", "how", "Are", "yo", "##U", "?"], + ids: [101, 145, 11013, 82834, 10133, 106, 14796, 13491, 13672, 12022, 136, 102], + decoded: "[CLS] H\u00e4LLo! how Are yoU? [SEP]", + }, + }, + "Xenova/paraphrase-multilingual-MiniLM-L12-v2": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["\u2581How", "\u2581are", "\u2581you", "\u2581doing", "?"], + ids: [0, 11249, 621, 398, 20594, 32, 2], + decoded: " How are you doing?", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["\u2581You", "\u2581should", "'", "ve", "\u2581done", "\u2581this"], + ids: [0, 2583, 5608, 25, 272, 16940, 903, 2], + decoded: " You should've done this", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["\u2581The", "\u2581company", "\u2581was", "\u2581found", "ed", "\u2581in", "\u25812016."], + ids: [0, 581, 14380, 509, 14037, 297, 23, 6360, 2], + decoded: " The company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["\u2581A", "\u2581'", "ll", "\u2581!!", "to", "?", "'", "d", "''", "d", "\u2581of", ",", "\u2581can", "'", "t", "."], + ids: [0, 62, 242, 1181, 6506, 188, 32, 25, 71, 4765, 71, 111, 4, 831, 25, 18, 5, 2], + decoded: " A 'll!!to?'d''d of, can't.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["\u2581de", "f", "\u2581main", "(", "):", "\u2581pass"], + ids: [0, 8, 420, 5201, 132, 2077, 27875, 2], + decoded: " def main(): pass", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["\u2581let", "\u2581a", "\u2581=", "\u2581ob", "j", ".", "to", "Str", "ing", "(", ");", "\u2581to", "Str", "ing", "(", ");"], + ids: [0, 2633, 10, 2203, 995, 170, 5, 188, 71713, 214, 132, 3142, 47, 71713, 214, 132, 3142, 2], + decoded: " let a = obj.toString(); toString();", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["\u2581This", "\u2581is", "\u2581a", "\u2581test", "."], + ids: [0, 3293, 83, 10, 3034, 5, 2], + decoded: " This is a test.", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["\u2581UN", "wan", "t\u00e9", "d", ",", "run", "ning"], + ids: [0, 8274, 3206, 2312, 71, 4, 16428, 592, 2], + decoded: " UNwant\u00e9d,running", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["\u25811", "\u0000", "2", "\u25813"], + ids: [0, 106, 3, 304, 138, 2], + decoded: " 12 3", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["\u2581Hello", "\u2581World"], + ids: [0, 35378, 6661, 2], + decoded: " Hello World", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["\u2581hell", "o", "\u2581world"], + ids: [0, 33600, 31, 8999, 2], + decoded: " hello world", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u2581", "\u751f\u6d3b\u7684", "\u771f", "\u8c1b", "\u662f"], + ids: [0, 6, 62668, 5364, 245875, 354, 2], + decoded: " \u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u2581leading", "\u2581space"], + ids: [0, 105207, 32628, 2], + decoded: " leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["\u2581trail", "ing", "\u2581space"], + ids: [0, 141037, 214, 32628, 2], + decoded: " trailing space", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["\u2581Hi", "\u2581Hello"], + ids: [0, 2673, 35378, 2], + decoded: " Hi Hello", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["\u2581test", "\u2581$1", "\u2581R", "2", "\u2581#3", "\u2581\u20ac", "4", "\u2581\u00a3", "5", "\u2581", "\u00a5", "6", "\u2581", "\u20a3", "7", "\u2581\u20b9", "8", "\u2581", "\u20b1", "9", "\u2581test"], + ids: [0, 3034, 38629, 627, 304, 111378, 2505, 617, 11762, 758, 6, 32389, 910, 6, 3, 966, 87316, 1019, 6, 247425, 1126, 3034, 2], + decoded: " test $1 R2 #3 \u20ac4 \u00a35 \u00a56 7 \u20b98 \u20b19 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["\u2581I", "\u2581bought", "\u2581an", "\u2581apple", "\u2581for", "\u2581$", "1.00", "\u2581at", "\u2581the", "\u2581store", "."], + ids: [0, 87, 123997, 142, 108787, 100, 3650, 146533, 99, 70, 4343, 5, 2], + decoded: " I bought an apple for $1.00 at the store.", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["\u2581you", "..."], + ids: [0, 398, 27, 2], + decoded: " you...", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["\u2581you", "..."], + ids: [0, 398, 27, 2], + decoded: " you...", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["\u2581you", "...", "\u2581you", "..."], + ids: [0, 398, 27, 398, 27, 2], + decoded: " you... you...", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["\u2581weird", "\u2581", "\uff5e", "\u2581edge", "\u2581", "\uff5e", "\u2581case"], + ids: [0, 179459, 6, 6087, 121303, 6, 6087, 7225, 2], + decoded: " weird \uff5e edge \uff5e case", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u2581This", "\u2581is", "\u2581a", "\u2581test", "\u2581", "."], + ids: [0, 3293, 83, 10, 3034, 6, 5, 2], + decoded: " This is a test.", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u2581", "\ud83d\ude02", "\u2581", "\ud83d\udc4d", "\u2581", "\ud83e\udd23", "\u2581", "\ud83d\ude0d", "\u2581", "\ud83d\ude2d", "\u2581", "\ud83c\udf89", "\u2581", "\ud83d\ude4f", "\u2581", "\ud83d\ude0a", "\u2581", "\ud83d\udd25", "\u2581", "\ud83d\ude01", "\u2581", "\ud83d\ude05", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\ude06", "\u2581", "\ud83d\udc4f", "\u2581\u2764", "\ufe0f", "\u2581", "\ud83d\udc9c", "\u2581", "\ud83d\udc9a", "\u2581", "\ud83d\udc97", "\u2581", "\ud83d\udc99", "\u2581", "\ud83d\udda4", "\u2581", "\ud83d\ude0e", "\u2581", "\ud83d\udc4c", "\u2581", "\ud83e\udd73", "\u2581", "\ud83d\udcaa", "\u2581", "\u2728", "\u2581", "\ud83d\udc49", "\u2581", "\ud83d\udc40", "\u2581", "\ud83d\udcaf", "\u2581", "\ud83c\udf88", "\u2581", "\ud83d\ude48", "\u2581", "\ud83d\ude4c", "\u2581", "\ud83d\udc80", "\u2581", "\ud83d\udc47", "\u2581", "\ud83d\udc4b", "\u2581", "\u2705", "\u2581", "\ud83c\udf81", "\u2581", "\ud83c\udf1e", "\u2581", "\ud83c\udf38", "\u2581", "\ud83d\udcb0"], + ids: [0, 6, 115114, 6, 118280, 6, 243385, 6, 84464, 6, 232773, 6, 243816, 6, 113612, 6, 82803, 6, 222326, 6, 201344, 6, 239569, 6, 243544, 6, 191876, 6, 243404, 49933, 15755, 6, 244233, 6, 244162, 6, 244181, 6, 243892, 6, 245820, 6, 161546, 6, 204811, 6, 3, 6, 238992, 6, 167474, 6, 120242, 6, 245561, 6, 244864, 6, 246144, 6, 244459, 6, 244703, 6, 246887, 6, 144400, 6, 246511, 6, 142325, 6, 244230, 6, 245559, 6, 243374, 6, 245200, 2], + decoded: " \ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u2581", "\u2728", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\udc41", "\ufe0f", "\u2581", "\ud83d\udc71", "\ud83c\udffb", "\u2581", "\ud83d\udd75", "\u2581", "\u2642", "\ufe0f", "\u2581", "\ud83e\uddd9", "\ud83c\udffb", "\u2581", "\u2642", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u2581", "\ud83c\udf3e", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83d\udc69", "\u2581\u2764", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc67", "\u2581", "\ud83d\udc66", "\u2581", "\ud83e\uddd1", "\ud83c\udffb", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1", "\ud83c\udffb", "\u2581", "\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u2581\u2764", "\ufe0f", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\ud83c\udffc"], + ids: [0, 6, 167474, 6, 243544, 6, 246984, 15755, 6, 247201, 79500, 6, 248325, 6, 228250, 15755, 6, 3, 79500, 6, 228250, 6, 244314, 79500, 6, 246529, 6, 3, 6, 247443, 6, 3, 6, 244785, 49933, 6, 244960, 6, 244314, 6, 244785, 6, 244785, 6, 245719, 6, 246167, 6, 3, 79500, 6, 247443, 6, 3, 79500, 6, 3, 6, 244314, 79500, 49933, 15755, 6, 244960, 6, 244314, 239719, 2], + decoded: " \u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75 \u2642\ufe0f \ud83c\udffb \u2642 \ud83d\udc68\ud83c\udffb \ud83c\udf3e \ud83e\udd1d \ud83d\udc69 \u2764 \ud83d\udc8b \ud83d\udc68 \ud83d\udc69 \ud83d\udc69 \ud83d\udc67 \ud83d\udc66 \ud83c\udffb \ud83e\udd1d \ud83c\udffb \ud83d\udc68\ud83c\udffb \u2764\ufe0f \ud83d\udc8b \ud83d\udc68\ud83c\udffc", + }, + CHINESE_LATIN_MIXED: { + text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, + tokens: ["\u2581ah", "\u535a", "\u63a8", "zz"], + ids: [0, 1263, 11173, 10238, 13894, 2], + decoded: " ah\u535a\u63a8zz", + }, + SIMPLE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, + tokens: ["\u2581H\u00e9", "llo"], + ids: [0, 88064, 9284, 2], + decoded: " H\u00e9llo", + }, + MIXED_CASE_WITHOUT_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, + tokens: ["\u2581He", "LL", "o", "!", "how", "\u2581Are", "\u2581yo", "U", "?"], + ids: [0, 1529, 23708, 31, 38, 47251, 15901, 3005, 1062, 32, 2], + decoded: " HeLLo!how Are yoU?", + }, + MIXED_CASE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, + tokens: ["\u2581H\u00e4", "LL", "o", "!", "how", "\u2581Are", "\u2581yo", "U", "?"], + ids: [0, 28863, 23708, 31, 38, 47251, 15901, 3005, 1062, 32, 2], + decoded: " H\u00e4LLo!how Are yoU?", + }, + }, + "Xenova/bert-base-multilingual-uncased-sentiment": { + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "a", "=", "ob", "##j", ".", "tos", "##tri", "##ng", "(", ")", ";", "tos", "##tri", "##ng", "(", ")", ";"], + ids: [101, 12421, 143, 134, 15547, 10428, 119, 53564, 27711, 10422, 113, 114, 132, 53564, 27711, 10422, 113, 114, 132, 102], + decoded: "[CLS] let a = obj. tostring ( ) ; tostring ( ) ; [SEP]", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["un", "##wan", "##ted", ",", "running"], + ids: [101, 10119, 15134, 11894, 117, 16484, 102], + decoded: "[CLS] unwanted, running [SEP]", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "$", "1", "r2", "#", "3", "\u20ac", "##4", "\u00a3", "##5", "\u00a5", "##6", "[UNK]", "\u20b9", "##8", "\u20b1", "##9", "test"], + ids: [101, 14084, 109, 122, 85583, 108, 124, 1329, 11124, 175, 11301, 177, 11325, 100, 1332, 11544, 1330, 11518, 14084, 102], + decoded: "[CLS] test $ 1 r2 # 3 \u20ac4 \u00a35 \u00a56 [UNK] \u20b98 \u20b19 test [SEP]", + }, + }, + "Xenova/multilingual-e5-small": { + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["\u2581trail", "ing", "\u2581space", "\u2581"], + ids: [0, 141037, 214, 32628, 6, 2], + decoded: " trailing space ", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["\u2581you", "...", "\u2581"], + ids: [0, 398, 27, 6, 2], + decoded: " you... ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["\u2581you", "...", "\u2581"], + ids: [0, 398, 27, 6, 2], + decoded: " you... ", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["\u2581you", "...", "\u2581you", "...", "\u2581"], + ids: [0, 398, 27, 398, 27, 6, 2], + decoded: " you... you... ", + }, + MIXED_CASE_WITHOUT_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, + tokens: ["\u2581He", "LL", "o", "!", "how", "\u2581Are", "\u2581yo", "U", "?", "\u2581"], + ids: [0, 1529, 23708, 31, 38, 47251, 15901, 3005, 1062, 32, 6, 2], + decoded: " HeLLo!how Are yoU? ", + }, + MIXED_CASE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, + tokens: ["\u2581H\u00e4", "LL", "o", "!", "how", "\u2581Are", "\u2581yo", "U", "?", "\u2581"], + ids: [0, 28863, 23708, 31, 38, 47251, 15901, 3005, 1062, 32, 6, 2], + decoded: " H\u00e4LLo!how Are yoU? ", + }, + }, + "Xenova/bge-small-zh-v1.5": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["[UNK]", "are", "you", "doi", "##ng", "?"], + ids: [101, 100, 8995, 8357, 9962, 8291, 136, 102], + decoded: "[CLS] [UNK] are you doing? [SEP]", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["[UNK]", "sh", "##ould", "'", "ve", "don", "##e", "this"], + ids: [101, 100, 11167, 11734, 112, 12810, 9524, 8154, 8554, 102], + decoded: "[CLS] [UNK] should've done this [SEP]", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["[UNK]", "company", "was", "f", "##ound", "##ed", "in", "2016", "."], + ids: [101, 100, 10007, 9947, 148, 11477, 8303, 8217, 8112, 119, 102], + decoded: "[CLS] [UNK] company was founded in 2016. [SEP]", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["[UNK]", "'", "ll", "!", "!", "to", "?", "'", "d", "'", "'", "d", "of", ",", "can", "'", "t", "."], + ids: [101, 100, 112, 10856, 106, 106, 8228, 136, 112, 146, 112, 112, 146, 8205, 117, 9109, 112, 162, 119, 102], + decoded: "[CLS] [UNK]'ll!! to?'d'' d of, can't. [SEP]", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["de", "##f", "main", "(", ")", ":", "pass"], + ids: [101, 8363, 8189, 9139, 113, 114, 131, 9703, 102], + decoded: "[CLS] def main ( ) : pass [SEP]", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "a", "=", "ob", "##j", ".", "[UNK]", "(", ")", ";", "[UNK]", "(", ")", ";"], + ids: [101, 9946, 143, 134, 12639, 8334, 119, 100, 113, 114, 132, 100, 113, 114, 132, 102], + decoded: "[CLS] let a = obj. [UNK] ( ) ; [UNK] ( ) ; [SEP]", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["[UNK]", "is", "a", "test", "."], + ids: [101, 100, 8310, 143, 10060, 119, 102], + decoded: "[CLS] [UNK] is a test. [SEP]", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["[UNK]", ",", "running"], + ids: [101, 100, 117, 11620, 102], + decoded: "[CLS] [UNK], running [SEP]", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["[UNK]", "[UNK]"], + ids: [101, 100, 100, 102], + decoded: "[CLS] [UNK] [UNK] [SEP]", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["le", "##ad", "##ing", "space"], + ids: [101, 8983, 8695, 8221, 9634, 102], + decoded: "[CLS] leading space [SEP]", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["t", "##rail", "##ing", "space"], + ids: [101, 162, 12783, 8221, 9634, 102], + decoded: "[CLS] trailing space [SEP]", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["[UNK]", "[UNK]"], + ids: [101, 100, 100, 102], + decoded: "[CLS] [UNK] [UNK] [SEP]", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "$", "1", "[UNK]", "#", "3", "\u20ac", "##4", "\u00a3", "##5", "\u00a5", "##6", "[UNK]", "[UNK]", "[UNK]", "test"], + ids: [101, 10060, 109, 122, 100, 108, 124, 359, 8159, 173, 8157, 175, 8158, 100, 100, 100, 10060, 102], + decoded: "[CLS] test $ 1 [UNK] # 3 \u20ac4 \u00a35 \u00a56 [UNK] [UNK] [UNK] test [SEP]", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["[UNK]", "bo", "##ugh", "##t", "an", "apple", "for", "$", "1", ".", "00", "at", "the", "store", "."], + ids: [101, 100, 11059, 12667, 8165, 9064, 8350, 8330, 109, 122, 119, 8136, 8243, 8174, 8719, 119, 102], + decoded: "[CLS] [UNK] bought an apple for $ 1. 00 at the store. [SEP]", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\ud83d\ude02", "\ud83d\udc4d", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "\ud83d\udd25", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "\ud83d\ude0e", "[UNK]", "[UNK]", "[UNK]", "\u2728", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]"], + ids: [101, 8104, 8102, 100, 100, 100, 100, 100, 100, 8103, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 8105, 100, 100, 100, 501, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 102], + decoded: "[CLS] \ud83d\ude02 \ud83d\udc4d [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] \ud83d\udd25 [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] \ud83d\ude0e [UNK] [UNK] [UNK] \u2728 [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [SEP]", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u2728", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]"], + ids: [101, 501, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 102], + decoded: "[CLS] \u2728 [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [SEP]", + }, + SIMPLE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, + tokens: ["[UNK]"], + ids: [101, 100, 102], + decoded: "[CLS] [UNK] [SEP]", + }, + MIXED_CASE_WITHOUT_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, + tokens: ["[UNK]", "!", "how", "[UNK]", "[UNK]", "?"], + ids: [101, 100, 106, 9510, 100, 100, 136, 102], + decoded: "[CLS] [UNK]! how [UNK] [UNK]? [SEP]", + }, + MIXED_CASE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, + tokens: ["[UNK]", "!", "how", "[UNK]", "[UNK]", "?"], + ids: [101, 100, 106, 9510, 100, 100, 136, 102], + decoded: "[CLS] [UNK]! how [UNK] [UNK]? [SEP]", + }, + }, + "Xenova/bge-base-zh-v1.5": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["how", "are", "you", "doi", "##ng", "?"], + ids: [101, 9510, 8995, 8357, 9962, 8291, 136, 102], + decoded: "[CLS] how are you doing? [SEP]", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["you", "sh", "##ould", "'", "ve", "don", "##e", "this"], + ids: [101, 8357, 11167, 11734, 112, 12810, 9524, 8154, 8554, 102], + decoded: "[CLS] you should've done this [SEP]", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["the", "company", "was", "f", "##ound", "##ed", "in", "2016", "."], + ids: [101, 8174, 10007, 9947, 148, 11477, 8303, 8217, 8112, 119, 102], + decoded: "[CLS] the company was founded in 2016. [SEP]", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["u", "##n", "##wan", "##ted", ",", "running"], + ids: [101, 163, 8171, 9951, 9255, 117, 11620, 102], + decoded: "[CLS] unwanted, running [SEP]", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "$", "1", "r2", "#", "3", "\u20ac", "##4", "\u00a3", "##5", "\u00a5", "##6", "[UNK]", "[UNK]", "[UNK]", "test"], + ids: [101, 10060, 109, 122, 11345, 108, 124, 359, 8159, 173, 8157, 175, 8158, 100, 100, 100, 10060, 102], + decoded: "[CLS] test $ 1 r2 # 3 \u20ac4 \u00a35 \u00a56 [UNK] [UNK] [UNK] test [SEP]", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["i", "bo", "##ugh", "##t", "an", "apple", "for", "$", "1", ".", "00", "at", "the", "store", "."], + ids: [101, 151, 11059, 12667, 8165, 9064, 8350, 8330, 109, 122, 119, 8136, 8243, 8174, 8719, 119, 102], + decoded: "[CLS] i bought an apple for $ 1. 00 at the store. [SEP]", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\ud83d\ude02", "\ud83d\udc4d", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "\ud83d\udd25", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "\u2764", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "\ud83d\ude0e", "[UNK]", "[UNK]", "[UNK]", "\u2728", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]"], + ids: [101, 8104, 8102, 100, 100, 100, 100, 100, 100, 8103, 100, 100, 100, 100, 100, 506, 100, 100, 100, 100, 100, 8105, 100, 100, 100, 501, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 102], + decoded: "[CLS] \ud83d\ude02 \ud83d\udc4d [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] \ud83d\udd25 [UNK] [UNK] [UNK] [UNK] [UNK] \u2764 [UNK] [UNK] [UNK] [UNK] [UNK] \ud83d\ude0e [UNK] [UNK] [UNK] \u2728 [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [SEP]", + }, + }, + "Xenova/indobert-base-p1": { + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["you", "sho", "##uld", "'", "ve", "don", "##e", "this"], + ids: [2, 3299, 9596, 15370, 30463, 28239, 4081, 30357, 5379, 3], + decoded: "[CLS] you should've done this [SEP]", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["the", "company", "was", "found", "##ed", "in", "2016", "."], + ids: [2, 1002, 9105, 2738, 11009, 133, 48, 1538, 30470, 3], + decoded: "[CLS] the company was founded in 2016. [SEP]", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "a", "=", "ob", "##j", ".", "tos", "##trin", "##g", "(", ")", ";", "tos", "##trin", "##g", "(", ")", ";"], + ids: [2, 4734, 253, 30475, 559, 30372, 30470, 20498, 12448, 30365, 30464, 30465, 30473, 20498, 12448, 30365, 30464, 30465, 30473, 3], + decoded: "[CLS] let a = obj. tostring ( ) ; tostring ( ) ; [SEP]", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["un", "##wan", "##te", "##d", ",", "running"], + ids: [2, 78, 1322, 3298, 30364, 30468, 22715, 3], + decoded: "[CLS] unwanted, running [SEP]", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]"], + ids: [2, 1, 1, 1, 1, 1, 1, 3], + decoded: "[CLS] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [SEP]", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["lead", "##ing", "space"], + ids: [2, 9196, 55, 14561, 3], + decoded: "[CLS] leading space [SEP]", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "$", "1", "r", "##2", "#", "3", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "test"], + ids: [2, 4243, 30460, 111, 56, 30378, 30459, 283, 1, 1, 1, 1, 1, 1, 4243, 3], + decoded: "[CLS] test $ 1 r2 # 3 [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] test [SEP]", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["i", "bo", "##ught", "an", "apple", "for", "$", "1", ".", "00", "at", "the", "store", "."], + ids: [2, 89, 1880, 25009, 223, 7761, 1548, 30460, 111, 30470, 4230, 117, 1002, 8052, 30470, 3], + decoded: "[CLS] i bought an apple for $ 1. 00 at the store. [SEP]", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["wei", "##rd", "[UNK]", "edge", "[UNK]", "case"], + ids: [2, 27753, 12548, 1, 21418, 1, 13687, 3], + decoded: "[CLS] weird [UNK] edge [UNK] case [SEP]", + }, + MIXED_CASE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, + tokens: ["hallo", "!", "how", "are", "you", "?"], + ids: [2, 19598, 30457, 11088, 5811, 3299, 30477, 3], + decoded: "[CLS] hallo! how are you? [SEP]", + }, + }, + "Xenova/spanbert-large-cased": { + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "a", "=", "o", "##b", "##j", ".", "to", "##st", "##ring", "(", ")", ";", "to", "##st", "##ring", "(", ")", ";"], + ids: [101, 1519, 170, 134, 184, 1830, 3361, 119, 1106, 2050, 3384, 113, 114, 132, 1106, 2050, 3384, 113, 114, 132, 102], + decoded: "[CLS] let a = obj. tostring ( ) ; tostring ( ) ; [SEP]", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "$", "1", "r", "##2", "#", "3", "\u20ac", "##4", "\u00a3", "##5", "\u00a5", "##6", "[UNK]", "\u20b9", "##8", "\u20b1", "##9", "test"], + ids: [101, 2774, 109, 122, 187, 1477, 108, 124, 836, 1527, 202, 1571, 203, 1545, 100, 838, 1604, 837, 1580, 2774, 102], + decoded: "[CLS] test $ 1 r2 # 3 \u20ac4 \u00a35 \u00a56 [UNK] \u20b98 \u20b19 test [SEP]", + }, + }, + "Xenova/UMLSBert_ENG": { + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "a", "=", "obj", ".", "tos", "##tring", "(", ")", ";", "tos", "##tring", "(", ")", ";"], + ids: [2, 8894, 42, 32, 2473, 17, 22660, 23640, 11, 12, 30, 22660, 23640, 11, 12, 30, 3], + decoded: "[CLS] let a = obj. tostring ( ) ; tostring ( ) ; [SEP]", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["hel", "##lo", "world"], + ids: [2, 3018, 5368, 4517, 3], + decoded: "[CLS] hello world [SEP]", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["hel", "##lo", "world"], + ids: [2, 3018, 5368, 4517, 3], + decoded: "[CLS] hello world [SEP]", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["hi", "hel", "##lo"], + ids: [2, 11245, 3018, 5368, 3], + decoded: "[CLS] hi hello [SEP]", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "$", "1", "r2", "#", "3", "\u20ac", "##4", "\u00a3", "##5", "\u00a5", "##6", "\u20a3", "##7", "\u20b9", "##8", "\u20b1", "##9", "test"], + ids: [2, 2313, 7, 20, 9663, 6, 22, 528, 1017, 74, 1009, 76, 1018, 524, 1019, 531, 1011, 529, 1038, 2313, 3], + decoded: "[CLS] test $ 1 r2 # 3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test [SEP]", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["we", "##ir", "##d", "\uff5e", "edge", "\uff5e", "case"], + ids: [2, 1802, 1753, 1022, 943, 9676, 943, 2632, 3], + decoded: "[CLS] weird \uff5e edge \uff5e case [SEP]", + }, + SIMPLE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, + tokens: ["hel", "##lo"], + ids: [2, 3018, 5368, 3], + decoded: "[CLS] hello [SEP]", + }, + MIXED_CASE_WITHOUT_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, + tokens: ["hel", "##lo", "!", "how", "are", "you", "?"], + ids: [2, 3018, 5368, 5, 2135, 1810, 17915, 34, 3], + decoded: "[CLS] hello! how are you? [SEP]", + }, + }, + "Xenova/SapBERT-from-PubMedBERT-fulltext": { + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u751f", "\u6d3b", "\u7684", "[UNK]", "[UNK]", "\u662f"], + ids: [2, 799, 776, 811, 1, 1, 731, 3], + decoded: "[CLS] \u751f \u6d3b \u7684 [UNK] [UNK] \u662f [SEP]", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "$", "1", "r2", "#", "3", "\u20ac", "##4", "\u00a3", "##5", "\u00a5", "##6", "[UNK]", "\u20b9", "##8", "[UNK]", "test"], + ids: [2, 2648, 8, 21, 7261, 7, 23, 281, 1006, 76, 1015, 78, 1016, 1, 282, 1025, 1, 2648, 3], + decoded: "[CLS] test $ 1 r2 # 3 \u20ac4 \u00a35 \u00a56 [UNK] \u20b98 [UNK] test [SEP]", + }, + }, + "Xenova/rubert-base-cased": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["How", "are", "you", "do", "##ing", "?"], + ids: [101, 15474, 10813, 13540, 10661, 7729, 166, 102], + decoded: "[CLS] How are you doing? [SEP]", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["You", "sh", "##oul", "##d", "'", "ve", "don", "##e", "this"], + ids: [101, 11577, 45942, 76143, 239, 118, 10835, 17450, 241, 11043, 102], + decoded: "[CLS] You should've done this [SEP]", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "comp", "##any", "was", "f", "##ound", "##ed", "in", "2016", "."], + ids: [101, 6821, 71382, 17927, 10646, 242, 71129, 7491, 10618, 8273, 132, 102], + decoded: "[CLS] The company was founded in 2016. [SEP]", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "a", "=", "ob", "##j", ".", "to", "##St", "##ring", "(", ")", ";", "to", "##St", "##ring", "(", ")", ";"], + ids: [101, 14107, 232, 162, 17851, 251, 132, 10626, 21568, 13647, 120, 122, 158, 10626, 21568, 13647, 120, 122, 158, 102], + decoded: "[CLS] let a = obj. toString ( ) ; toString ( ) ; [SEP]", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "##wan", "##t", "##\u00e9d", ",", "run", "##ning"], + ids: [101, 27090, 14906, 271, 84705, 128, 14607, 11781, 102], + decoded: "[CLS] UNwant\u00e9d, running [SEP]", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u751f", "\u6d3b", "\u7684", "\u771f", "[UNK]", "\u662f"], + ids: [101, 6104, 5480, 6222, 6273, 100, 4877, 102], + decoded: "[CLS] \u751f \u6d3b \u7684 \u771f [UNK] \u662f [SEP]", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["le", "##ading", "sp", "##ace"], + ids: [101, 10653, 73130, 33162, 13967, 102], + decoded: "[CLS] leading space [SEP]", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["tra", "##ili", "##ng", "sp", "##ace"], + ids: [101, 11776, 14296, 10888, 33162, 13967, 102], + decoded: "[CLS] trailing space [SEP]", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "bo", "##ught", "an", "app", "##le", "for", "$", "1", ".", "00", "at", "the", "st", "##ore", "."], + ids: [101, 186, 21018, 53718, 10663, 73406, 7159, 10654, 112, 138, 132, 11537, 10672, 10617, 28668, 13536, 132, 102], + decoded: "[CLS] I bought an apple for $ 1. 00 at the store. [SEP]", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["we", "##ird", "\uff5e", "ed", "##ge", "\uff5e", "cas", "##e"], + ids: [101, 12463, 36865, 10608, 11051, 11037, 10608, 15501, 241, 102], + decoded: "[CLS] weird \uff5e edge \uff5e case [SEP]", + }, + CHINESE_LATIN_MIXED: { + text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, + tokens: ["a", "##h", "\u535a", "\u63a8", "z", "##z"], + ids: [101, 232, 247, 3166, 4657, 282, 283, 102], + decoded: "[CLS] ah \u535a \u63a8 zz [SEP]", + }, + MIXED_CASE_WITHOUT_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, + tokens: ["He", "##LL", "##o", "!", "ho", "##w", "Are", "yo", "##U", "?"], + ids: [101, 10869, 83346, 261, 106, 13685, 277, 14003, 14184, 211, 166, 102], + decoded: "[CLS] HeLLo! how Are yoU? [SEP]", + }, + MIXED_CASE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, + tokens: ["H", "##\u00e4", "##LL", "##o", "!", "ho", "##w", "Are", "yo", "##U", "?"], + ids: [101, 184, 384, 83346, 261, 106, 13685, 277, 14003, 14184, 211, 166, 102], + decoded: "[CLS] H\u00e4LLo! how Are yoU? [SEP]", + }, + }, + "Xenova/kobert": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]", "?"], + ids: [2, 0, 0, 0, 0, 258, 3], + decoded: "[CLS] [UNK] [UNK] [UNK] [UNK]? [SEP]", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["[UNK]", "[UNK]", "'", "[UNK]", "[UNK]", "[UNK]"], + ids: [2, 0, 0, 15, 0, 0, 0, 3], + decoded: "[CLS] [UNK] [UNK]'[UNK] [UNK] [UNK] [SEP]", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "[UNK]", "[UNK]", "[UNK]", "in", "[UNK]", "."], + ids: [2, 355, 0, 0, 0, 409, 0, 54, 3], + decoded: "[CLS] The [UNK] [UNK] [UNK] in [UNK]. [SEP]", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "'", "[UNK]", "!", "!", "[UNK]", "?", "'", "d", "'", "'", "d", "[UNK]", ",", "[UNK]", "'", "t", "."], + ids: [2, 264, 15, 0, 5, 5, 0, 258, 15, 388, 15, 15, 388, 0, 46, 0, 15, 442, 54, 3], + decoded: "[CLS] A'[UNK]!! [UNK]?'d'' d [UNK], [UNK]'t. [SEP]", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["[UNK]", "[UNK]", "(", ")", ":", "[UNK]"], + ids: [2, 0, 0, 18, 40, 249, 0, 3], + decoded: "[CLS] [UNK] [UNK] ( ) : [UNK] [SEP]", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["[UNK]", "a", "=", "[UNK]", ".", "[UNK]", "(", ")", ";", "[UNK]", "(", ")", ";"], + ids: [2, 0, 367, 254, 0, 54, 0, 18, 40, 252, 0, 18, 40, 252, 3], + decoded: "[CLS] [UNK] a = [UNK]. [UNK] ( ) ; [UNK] ( ) ; [SEP]", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["[UNK]", "is", "a", "[UNK]", "."], + ids: [2, 0, 412, 367, 0, 54, 3], + decoded: "[CLS] [UNK] is a [UNK]. [SEP]", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["[UNK]", ",", "[UNK]"], + ids: [2, 0, 46, 0, 3], + decoded: "[CLS] [UNK], [UNK] [SEP]", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u751f", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]"], + ids: [2, 5298, 0, 0, 0, 0, 0, 3], + decoded: "[CLS] \u751f [UNK] [UNK] [UNK] [UNK] [UNK] [SEP]", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["[UNK]", "$", "1", "[UNK]", "#", "3", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]"], + ids: [2, 0, 10, 93, 0, 9, 142, 0, 0, 0, 0, 0, 0, 0, 3], + decoded: "[CLS] [UNK] $ 1 [UNK] # 3 [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [SEP]", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "[UNK]", "an", "[UNK]", "[UNK]", "$", "1", ".", "00", "at", "[UNK]", "[UNK]", "."], + ids: [2, 296, 0, 374, 0, 0, 10, 93, 54, 79, 377, 0, 0, 54, 3], + decoded: "[CLS] I [UNK] an [UNK] [UNK] $ 1. 00 at [UNK] [UNK]. [SEP]", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]"], + ids: [2, 0, 0, 0, 0, 3], + decoded: "[CLS] [UNK] [UNK] [UNK] [UNK] [SEP]", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]"], + ids: [2, 0, 0, 0, 0, 0, 3], + decoded: "[CLS] [UNK] [UNK] [UNK] [UNK] [UNK] [SEP]", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]", "\u2581", "."], + ids: [2, 0, 0, 0, 0, 517, 54, 3], + decoded: "[CLS] [UNK] [UNK] [UNK] [UNK] \u2581. [SEP]", + }, + CHINESE_LATIN_MIXED: { + text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, + tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]"], + ids: [2, 0, 0, 0, 0, 3], + decoded: "[CLS] [UNK] [UNK] [UNK] [UNK] [SEP]", + }, + MIXED_CASE_WITHOUT_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, + tokens: ["[UNK]", "!", "[UNK]", "[UNK]", "[UNK]", "?"], + ids: [2, 0, 5, 0, 0, 0, 258, 3], + decoded: "[CLS] [UNK]! [UNK] [UNK] [UNK]? [SEP]", + }, + MIXED_CASE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, + tokens: ["[UNK]", "!", "[UNK]", "[UNK]", "[UNK]", "?"], + ids: [2, 0, 5, 0, 0, 0, 258, 3], + decoded: "[CLS] [UNK]! [UNK] [UNK] [UNK]? [SEP]", + }, + }, + "Xenova/scibert_scivocab_uncased": { + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "a", "=", "obj", ".", "to", "##string", "(", ")", ";", "to", "##string", "(", ")", ";"], + ids: [102, 1293, 106, 275, 2324, 205, 147, 20301, 145, 546, 1814, 147, 20301, 145, 546, 1814, 103], + decoded: "[CLS] let a = obj. tostring ( ) ; tostring ( ) ; [SEP]", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["hi", "hell", "##o"], + ids: [102, 5305, 29423, 30112, 103], + decoded: "[CLS] hi hello [SEP]", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "$", "1", "r", "##2", "#", "3", "\u20ac", "##4", "\u00a3", "##5", "\u00a5", "##6", "[UNK]", "[UNK]", "[UNK]", "test"], + ids: [102, 856, 3250, 158, 182, 30132, 3000, 239, 20801, 30140, 11221, 30139, 20704, 30142, 101, 101, 101, 856, 103], + decoded: "[CLS] test $ 1 r2 # 3 \u20ac4 \u00a35 \u00a56 [UNK] [UNK] [UNK] test [SEP]", + }, + CHINESE_LATIN_MIXED: { + text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, + tokens: ["ah", "[UNK]", "[UNK]", "zz"], + ids: [102, 7839, 101, 101, 23591, 103], + decoded: "[CLS] ah [UNK] [UNK] zz [SEP]", + }, + SIMPLE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, + tokens: ["hell", "##o"], + ids: [102, 29423, 30112, 103], + decoded: "[CLS] hello [SEP]", + }, + MIXED_CASE_WITHOUT_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, + tokens: ["hell", "##o", "!", "how", "are", "you", "?"], + ids: [102, 29423, 30112, 3190, 539, 220, 3034, 3912, 103], + decoded: "[CLS] hello! how are you? [SEP]", + }, + }, + "Xenova/LaBSE": { + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "a", "=", "obj", ".", "to", "##String", "(", ")", ";", "to", "##String", "(", ")", ";"], + ids: [101, 17214, 170, 134, 228877, 119, 14986, 368304, 113, 114, 132, 14986, 368304, 113, 114, 132, 102], + decoded: "[CLS] let a = obj. toString ( ) ; toString ( ) ; [SEP]", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "$", "1", "R2", "#", "3", "\u20ac", "##4", "\u00a35", "\u00a5", "##6", "\u20a3", "##7", "\u20b9", "##8", "\u20b1", "##9", "test"], + ids: [101, 17678, 109, 122, 51222, 108, 124, 3030, 16006, 279082, 205, 16151, 3023, 16187, 3037, 16175, 3033, 16236, 17678, 102], + decoded: "[CLS] test $ 1 R2 # 3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test [SEP]", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u2581", "##This", "\u2581", "##is", "\u2581", "##a", "\u2581", "##test", "\u2581", "."], + ids: [101, 3283, 342068, 3283, 15319, 3283, 14983, 3283, 50149, 3283, 119, 102], + decoded: "[CLS] \u2581This \u2581is \u2581a \u2581test \u2581. [SEP]", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\ud83d\ude02", "\ud83d\udc4d", "\ud83e\udd23", "\ud83d\ude0d", "\ud83d\ude2d", "\ud83c\udf89", "\ud83d\ude4f", "\ud83d\ude0a", "\ud83d\udd25", "\ud83d\ude01", "\ud83d\ude05", "\ud83e\udd17", "\ud83d\ude06", "\ud83d\udc4f", "\u2764\ufe0f", "\ud83d\udc9c", "\ud83d\udc9a", "\ud83d\udc97", "\ud83d\udc99", "\ud83d\udda4", "\ud83d\ude0e", "\ud83d\udc4c", "\ud83e\udd73", "\ud83d\udcaa", "\u2728", "\ud83d\udc49", "\ud83d\udc40", "\ud83d\udcaf", "\ud83c\udf88", "\ud83d\ude48", "\ud83d\ude4c", "\ud83d\udc80", "\ud83d\udc47", "\ud83d\udc4b", "\u2705", "\ud83c\udf81", "\ud83c\udf1e", "\ud83c\udf38", "\ud83d\udcb0"], + ids: [101, 14820, 14617, 14933, 14831, 14863, 14496, 14893, 14828, 14775, 14819, 14823, 14926, 14824, 14619, 91822, 14687, 14685, 14682, 14684, 14810, 14832, 14616, 14956, 14701, 3496, 14613, 14606, 14706, 14495, 14887, 14891, 14660, 14611, 14615, 3465, 14488, 14416, 14430, 14707, 102], + decoded: "[CLS] \ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0 [SEP]", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u2728", "\ud83e\udd17", "\ud83d\udc41\ufe0f", "\ud83d\udc71", "##\ud83c\udffb", "[UNK]", "[UNK]", "\ud83d\udc68", "##\ud83c\udffb", "##\ud83c\udf3e", "[UNK]", "\ud83d\udc69", "##\u2764", "##\ud83d\udc8b", "##\ud83d\udc68", "\ud83d\udc69", "##\ud83d\udc69", "##\ud83d\udc67", "##\ud83d\udc66", "[UNK]", "\ud83c\udff4", "\ud83d\udc68", "##\ud83c\udffb", "##\u2764", "##\ufe0f", "##\ud83d\udc8b", "##\ud83d\udc68", "##\ud83c\udffc"], + ids: [101, 3496, 14926, 350545, 14648, 130826, 100, 100, 14639, 130826, 498832, 100, 14640, 488649, 499065, 499034, 14640, 499035, 499033, 499032, 100, 14555, 14639, 130826, 488649, 44450, 499065, 499034, 421916, 102], + decoded: "[CLS] \u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb [UNK] [UNK] \ud83d\udc68\ud83c\udffb\ud83c\udf3e [UNK] \ud83d\udc69\u2764\ud83d\udc8b\ud83d\udc68 \ud83d\udc69\ud83d\udc69\ud83d\udc67\ud83d\udc66 [UNK] \ud83c\udff4 \ud83d\udc68\ud83c\udffb\u2764\ufe0f\ud83d\udc8b\ud83d\udc68\ud83c\udffc [SEP]", + }, + CHINESE_LATIN_MIXED: { + text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, + tokens: ["ah", "\u535a", "\u63a8", "zz"], + ids: [101, 15524, 4573, 6405, 441764, 102], + decoded: "[CLS] ah \u535a \u63a8 zz [SEP]", + }, + SIMPLE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, + tokens: ["H\u00e9", "##llo"], + ids: [101, 220855, 23025, 102], + decoded: "[CLS] H\u00e9llo [SEP]", + }, + }, + "Xenova/herbert-large-cased": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["Ho", "w", "are", "you", "do", "ing", "?"], + ids: [0, 5213, 1019, 25720, 20254, 2065, 5129, 1550, 2], + decoded: "How are you doing? ", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["You", "sho", "uld", "'", "ve", "d", "one", "this"], + ids: [0, 32795, 14924, 48273, 1571, 6647, 72, 2290, 48846, 2], + decoded: "You should've done this ", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "co", "mpany", "was", "fo", "un", "de", "d", "in", "20", "16", "."], + ids: [0, 7117, 2406, 41449, 9873, 3435, 2195, 2101, 1038, 2651, 5646, 2555, 1899, 2], + decoded: "The company was founded in 2016. ", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "'", "ll", "!", "!", "to", "?", "'", "d", "'", "'", "d", "of", ",", "can", "'", "t", "."], + ids: [0, 1012, 1571, 9396, 1725, 1725, 2063, 1550, 1571, 1038, 1571, 1571, 1038, 6595, 1947, 26794, 1571, 1026, 1899, 2], + decoded: "A'll!! to?'d'' d of, can't. ", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["de", "f", "main", "(", ")", ":", "pa", "ss"], + ids: [0, 2101, 1050, 41851, 1341, 1940, 1335, 2083, 5357, 2], + decoded: "def main ( ) : pass ", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "a", "=", "ob", "j", ".", "to", "S", "tr", "ing", "(", ")", ";", "to", "S", "tr", "ing", "(", ")", ";"], + ids: [0, 11324, 1011, 1789, 2033, 1013, 1899, 2146, 55, 2518, 5129, 1341, 1940, 1195, 2146, 55, 2518, 5129, 1341, 1940, 1195, 2], + decoded: "let a = obj. toString ( ) ; toString ( ) ; ", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["T", "his", "is", "a", "test", "."], + ids: [0, 56, 22855, 6869, 1011, 14825, 1899, 2], + decoded: "This is a test. ", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "wan", "t", "\u00e9", "d", ",", "run", "ning"], + ids: [0, 23029, 2688, 88, 163, 1038, 1947, 4980, 17843, 2], + decoded: "UNwant\u00e9d, running ", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["123"], + ids: [0, 19049, 2], + decoded: "123 ", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["Hel", "lo", "World"], + ids: [0, 12156, 6170, 21207, 2], + decoded: "Hello World ", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["hel", "lo", "world"], + ids: [0, 11526, 6170, 38188, 2], + decoded: "hello world ", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["", "", "", "", "", "\u662f"], + ids: [0, 3, 3, 3, 3, 3, 1776, 2], + decoded: "\u662f ", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["le", "ad", "ing", "space"], + ids: [0, 2018, 2035, 5129, 46489, 2], + decoded: "leading space ", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["tra", "i", "ling", "space"], + ids: [0, 2201, 77, 16342, 46489, 2], + decoded: "trailing space ", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["H", "i", "Hel", "lo"], + ids: [0, 44, 1009, 12156, 6170, 2], + decoded: "Hi Hello ", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "$", "1", "R", "2", "#", "3", "\u20ac", "4", "\u00a3", "5", "", "6", "", "7", "", "8", "", "9", "test"], + ids: [0, 14825, 1927, 1029, 54, 1025, 1393, 1034, 706, 1018, 100, 1008, 3, 1036, 3, 1030, 3, 1064, 3, 1017, 14825, 2], + decoded: "test $ 1 R2 # 3 \u20ac4 \u00a35 6 7 8 9 test ", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "bou", "ght", "an", "ap", "ple", "for", "$", "1", ".", "00", "at", "the", "st", "ore", "."], + ids: [0, 1056, 13016, 15272, 2879, 10309, 20861, 15181, 1927, 1029, 1899, 2291, 4772, 6854, 1989, 24005, 1899, 2], + decoded: "I bought an apple for $ 1. 00 at the store. ", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "\u2026"], + ids: [0, 20254, 1826, 2], + decoded: "you \u2026 ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "\u2026"], + ids: [0, 20254, 1826, 2], + decoded: "you \u2026 ", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "\u2026", "you", "\u2026"], + ids: [0, 20254, 1826, 20254, 1826, 2], + decoded: "you \u2026 you \u2026 ", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["we", "ir", "d", "", "e", "dge", "", "ca", "se"], + ids: [0, 2149, 17435, 1038, 3, 73, 25801, 3, 3833, 4417, 2], + decoded: "weird edge case ", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["", "T", "his", "", "is", "", "a", "", "test", "", "."], + ids: [0, 3, 56, 22855, 3, 6869, 3, 1011, 3, 14825, 3, 1899, 2], + decoded: "This is a test . ", + }, + CHINESE_LATIN_MIXED: { + text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, + tokens: ["a", "h", "", "", "zz"], + ids: [0, 69, 1021, 3, 3, 49185, 2], + decoded: "ah zz ", + }, + SIMPLE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, + tokens: ["H", "\u00e9", "l", "lo"], + ids: [0, 44, 163, 80, 6170, 2], + decoded: "H\u00e9llo ", + }, + MIXED_CASE_WITHOUT_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, + tokens: ["He", "L", "L", "o", "!", "ho", "w", "Ar", "e", "yo", "U", "?"], + ids: [0, 4596, 48, 48, 1007, 1725, 3145, 1019, 2921, 1015, 13908, 1041, 1550, 2], + decoded: "HeLLo! how Are yoU? ", + }, + MIXED_CASE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, + tokens: ["H", "\u00e4", "L", "L", "o", "!", "ho", "w", "Ar", "e", "yo", "U", "?"], + ids: [0, 44, 158, 48, 48, 1007, 1725, 3145, 1019, 2921, 1015, 13908, 1041, 1550, 2], + decoded: "H\u00e4LLo! how Are yoU? ", + }, + }, + "Xenova/ernie-gram-zh": { + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "$", "1", "r2", "#", "3", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "test"], + ids: [1, 6943, 18005, 208, 6847, 9474, 284, 18017, 18017, 18017, 18017, 18017, 18017, 6943, 2], + decoded: "[CLS] test $ 1 r2 # 3 [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] test [SEP]", + }, }, - }, }; diff --git a/tests/models/blenderbot_small/tokenization.js b/tests/models/blenderbot_small/tokenization.js index 6bf4bbb93..d390ae610 100644 --- a/tests/models/blenderbot_small/tokenization.js +++ b/tests/models/blenderbot_small/tokenization.js @@ -5,162 +5,162 @@ export const TOKENIZER_CLASS = BlenderbotSmallTokenizer; // NOTE: `.tokenize()` is disabled for BlenderbotSmallTokenizer export const TEST_CONFIG = { - "Xenova/blenderbot_small-90M": { - SIMPLE: { - text: BASE_TEST_STRINGS.SIMPLE, - // "tokens": ["how", "are", "you", "doing", "?"], - ids: [102, 46, 15, 267, 20], - decoded: "how are you doing?", + "Xenova/blenderbot_small-90M": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + // "tokens": ["how", "are", "you", "doing", "?"], + ids: [102, 46, 15, 267, 20], + decoded: "how are you doing?", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + // "tokens": ["you", "should", "'", "ve", "done", "this"], + ids: [15, 197, 8, 117, 369, 36], + decoded: "you should've done this", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + // "tokens": ["0@@", "1@@", "2@@", "3@@", "4@@", "5@@", "6@@", "7@@", "89", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "100", "1000"], + ids: [1988, 2388, 735, 801, 827, 948, 981, 1110, 4814, 520, 143, 176, 216, 260, 253, 345, 374, 420, 475, 316, 773, 6217], + decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + // "tokens": ["the", "company", "was", "founded", "in", "2016", "."], + ids: [7, 293, 18, 912, 13, 845, 5], + decoded: "the company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + // "tokens": ["a", "__newln__", "'", "ll", "!", "!@@", "to", "?", "'", "d", "'", "'", "d", "of", ",", "can", "'", "t", "."], + ids: [12, 4, 8, 97, 37, 3, 11, 20, 8, 85, 8, 8, 85, 10, 6, 62, 8, 30, 5], + decoded: "a __newln__'ll! __unk__ to?'d'' d of, can't.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + // "tokens": ["def", "main", "(", ")@@", ":", "__newln__", "pass"], + ids: [21996, 550, 40, 3, 106, 4, 1314], + decoded: "def main ( __unk__ : __newln__ pass", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + // "tokens": ["let", "a", "=", "ob@@", "j", ".@@", "to@@", "string", "(", ")@@", ";", "__newln__", "to@@", "string", "(", ")@@", ";"], + ids: [131, 12, 1381, 2808, 755, 3, 752, 4529, 40, 3, 118, 4, 752, 4529, 40, 3, 118], + decoded: "let a = obj __unk__ tostring ( __unk__ ; __newln__ tostring ( __unk__ ;", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + // "tokens": ["this", "__newln__", "is", "__newln__", "a", "__newln__", "test", "."], + ids: [36, 4, 24, 4, 12, 4, 1248, 5], + decoded: "this __newln__ is __newln__ a __newln__ test.", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + // "tokens": ["un@@", "wan@@", "t@@", "\u00e9@@", "d", ",@@", "running"], + ids: [204, 4151, 291, 1677, 85, 3, 785], + decoded: "unwant\u00e9d __unk__ running", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + // "tokens": ["1@@", "\u0000@@", "2@@", "\ufffd@@", "3"], + ids: [2388, 3, 735, 3, 216], + decoded: "1__unk__ 2__unk__ 3", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + // "tokens": ["hello", "world"], + ids: [880, 159], + decoded: "hello world", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + // "tokens": ["hello", "world"], + ids: [880, 159], + decoded: "hello world", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + // "tokens": ["\u751f@@", "\u6d3b@@", "\u7684@@", "\u771f@@", "\u8c1b@@", "\u662f"], + ids: [30488, 32756, 29891, 30813, 3, 34037], + decoded: "\u751f\u6d3b\u7684\u771f__unk__ \u662f", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + // "tokens": ["leading", "space"], + ids: [1164, 833], + decoded: "leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + // "tokens": ["trailing", "space"], + ids: [12499, 833], + decoded: "trailing space", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + // "tokens": ["hi", "hello"], + ids: [792, 880], + decoded: "hi hello", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + // "tokens": ["test", "$@@", "1", "r@@", "2", "#@@", "3", "\u20ac@@", "4", "\u00a3@@", "5", "\u00a5@@", "6", "\u20a3@@", "7", "\u20b9@@", "8", "\u20b1@@", "9", "test"], + ids: [1248, 3, 143, 510, 176, 3, 216, 3, 260, 3, 253, 3, 345, 3, 374, 3, 420, 3, 475, 1248], + decoded: "test __unk__ 1 r2 __unk__ 3 __unk__ 4 __unk__ 5 __unk__ 6 __unk__ 7 __unk__ 8 __unk__ 9 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + // "tokens": ["i", "bought", "an", "apple", "for", "$@@", "1", ".@@", "00", "at", "the", "store", "."], + ids: [14, 1890, 50, 4758, 26, 3, 143, 3, 1966, 32, 7, 1640, 5], + decoded: "i bought an apple for __unk__ 1 __unk__ 00 at the store.", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + // "tokens": ["you@@", "\u2026"], + ids: [7984, 1244], + decoded: "you\u2026", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + // "tokens": ["you@@", "\u2026"], + ids: [7984, 1244], + decoded: "you\u2026", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + // "tokens": ["you@@", "\u2026", "you@@", "\u2026"], + ids: [7984, 1244, 7984, 1244], + decoded: "you\u2026 you\u2026", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + // "tokens": ["weird", "\uff5e", "edge", "\uff5e", "case"], + ids: [2614, 30831, 1649, 30831, 543], + decoded: "weird \uff5e edge \uff5e case", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + // "tokens": ["\u2581@@", "this", "\u2581@@", "is", "\u2581@@", "a", "\u2581@@", "test", "\u2581", "."], + ids: [3, 36, 3, 24, 3, 12, 3, 1248, 50106, 5], + decoded: "__unk__ this __unk__ is __unk__ a __unk__ test \u2581.", + }, + SPECIAL_TOKENS: { + text: BLENDERBOT_SMALL_TEST_STRINGS.SPECIAL_TOKENS, + // "tokens": ["__start__", "hello", "world", "__end__"], + ids: [1, 880, 159, 2], + decoded: "__start__ hello world __end__", + }, + WHITESPACE_1: { + text: BLENDERBOT_SMALL_TEST_STRINGS.WHITESPACE_1, + // "tokens": ["__start__", "hey", "__end__"], + ids: [1, 226, 2], + decoded: "__start__ hey __end__", + }, + WHITESPACE_2: { + text: BLENDERBOT_SMALL_TEST_STRINGS.WHITESPACE_2, + // "tokens": ["__start__", "hey", "__end__"], + ids: [1, 226, 2], + decoded: "__start__ hey __end__", + }, }, - SIMPLE_WITH_PUNCTUATION: { - text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, - // "tokens": ["you", "should", "'", "ve", "done", "this"], - ids: [15, 197, 8, 117, 369, 36], - decoded: "you should've done this", - }, - NUMBERS: { - text: BASE_TEST_STRINGS.NUMBERS, - // "tokens": ["0@@", "1@@", "2@@", "3@@", "4@@", "5@@", "6@@", "7@@", "89", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "100", "1000"], - ids: [1988, 2388, 735, 801, 827, 948, 981, 1110, 4814, 520, 143, 176, 216, 260, 253, 345, 374, 420, 475, 316, 773, 6217], - decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - // "tokens": ["the", "company", "was", "founded", "in", "2016", "."], - ids: [7, 293, 18, 912, 13, 845, 5], - decoded: "the company was founded in 2016.", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - // "tokens": ["a", "__newln__", "'", "ll", "!", "!@@", "to", "?", "'", "d", "'", "'", "d", "of", ",", "can", "'", "t", "."], - ids: [12, 4, 8, 97, 37, 3, 11, 20, 8, 85, 8, 8, 85, 10, 6, 62, 8, 30, 5], - decoded: "a __newln__'ll! __unk__ to?'d'' d of, can't.", - }, - PYTHON_CODE: { - text: BASE_TEST_STRINGS.PYTHON_CODE, - // "tokens": ["def", "main", "(", ")@@", ":", "__newln__", "pass"], - ids: [21996, 550, 40, 3, 106, 4, 1314], - decoded: "def main ( __unk__ : __newln__ pass", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - // "tokens": ["let", "a", "=", "ob@@", "j", ".@@", "to@@", "string", "(", ")@@", ";", "__newln__", "to@@", "string", "(", ")@@", ";"], - ids: [131, 12, 1381, 2808, 755, 3, 752, 4529, 40, 3, 118, 4, 752, 4529, 40, 3, 118], - decoded: "let a = obj __unk__ tostring ( __unk__ ; __newln__ tostring ( __unk__ ;", - }, - NEWLINES: { - text: BASE_TEST_STRINGS.NEWLINES, - // "tokens": ["this", "__newln__", "is", "__newln__", "a", "__newln__", "test", "."], - ids: [36, 4, 24, 4, 12, 4, 1248, 5], - decoded: "this __newln__ is __newln__ a __newln__ test.", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - // "tokens": ["un@@", "wan@@", "t@@", "\u00e9@@", "d", ",@@", "running"], - ids: [204, 4151, 291, 1677, 85, 3, 785], - decoded: "unwant\u00e9d __unk__ running", - }, - CONTROL_TOKENS: { - text: BASE_TEST_STRINGS.CONTROL_TOKENS, - // "tokens": ["1@@", "\u0000@@", "2@@", "\ufffd@@", "3"], - ids: [2388, 3, 735, 3, 216], - decoded: "1__unk__ 2__unk__ 3", - }, - HELLO_WORLD_TITLECASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, - // "tokens": ["hello", "world"], - ids: [880, 159], - decoded: "hello world", - }, - HELLO_WORLD_LOWERCASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, - // "tokens": ["hello", "world"], - ids: [880, 159], - decoded: "hello world", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - // "tokens": ["\u751f@@", "\u6d3b@@", "\u7684@@", "\u771f@@", "\u8c1b@@", "\u662f"], - ids: [30488, 32756, 29891, 30813, 3, 34037], - decoded: "\u751f\u6d3b\u7684\u771f__unk__ \u662f", - }, - LEADING_SPACE: { - text: BASE_TEST_STRINGS.LEADING_SPACE, - // "tokens": ["leading", "space"], - ids: [1164, 833], - decoded: "leading space", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - // "tokens": ["trailing", "space"], - ids: [12499, 833], - decoded: "trailing space", - }, - DOUBLE_SPACE: { - text: BASE_TEST_STRINGS.DOUBLE_SPACE, - // "tokens": ["hi", "hello"], - ids: [792, 880], - decoded: "hi hello", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - // "tokens": ["test", "$@@", "1", "r@@", "2", "#@@", "3", "\u20ac@@", "4", "\u00a3@@", "5", "\u00a5@@", "6", "\u20a3@@", "7", "\u20b9@@", "8", "\u20b1@@", "9", "test"], - ids: [1248, 3, 143, 510, 176, 3, 216, 3, 260, 3, 253, 3, 345, 3, 374, 3, 420, 3, 475, 1248], - decoded: "test __unk__ 1 r2 __unk__ 3 __unk__ 4 __unk__ 5 __unk__ 6 __unk__ 7 __unk__ 8 __unk__ 9 test", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - // "tokens": ["i", "bought", "an", "apple", "for", "$@@", "1", ".@@", "00", "at", "the", "store", "."], - ids: [14, 1890, 50, 4758, 26, 3, 143, 3, 1966, 32, 7, 1640, 5], - decoded: "i bought an apple for __unk__ 1 __unk__ 00 at the store.", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - // "tokens": ["you@@", "\u2026"], - ids: [7984, 1244], - decoded: "you\u2026", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - // "tokens": ["you@@", "\u2026"], - ids: [7984, 1244], - decoded: "you\u2026", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - // "tokens": ["you@@", "\u2026", "you@@", "\u2026"], - ids: [7984, 1244, 7984, 1244], - decoded: "you\u2026 you\u2026", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - // "tokens": ["weird", "\uff5e", "edge", "\uff5e", "case"], - ids: [2614, 30831, 1649, 30831, 543], - decoded: "weird \uff5e edge \uff5e case", - }, - SPIECE_UNDERSCORE: { - text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, - // "tokens": ["\u2581@@", "this", "\u2581@@", "is", "\u2581@@", "a", "\u2581@@", "test", "\u2581", "."], - ids: [3, 36, 3, 24, 3, 12, 3, 1248, 50106, 5], - decoded: "__unk__ this __unk__ is __unk__ a __unk__ test \u2581.", - }, - SPECIAL_TOKENS: { - text: BLENDERBOT_SMALL_TEST_STRINGS.SPECIAL_TOKENS, - // "tokens": ["__start__", "hello", "world", "__end__"], - ids: [1, 880, 159, 2], - decoded: "__start__ hello world __end__", - }, - WHITESPACE_1: { - text: BLENDERBOT_SMALL_TEST_STRINGS.WHITESPACE_1, - // "tokens": ["__start__", "hey", "__end__"], - ids: [1, 226, 2], - decoded: "__start__ hey __end__", - }, - WHITESPACE_2: { - text: BLENDERBOT_SMALL_TEST_STRINGS.WHITESPACE_2, - // "tokens": ["__start__", "hey", "__end__"], - ids: [1, 226, 2], - decoded: "__start__ hey __end__", - }, - }, }; diff --git a/tests/models/bloom/tokenization.js b/tests/models/bloom/tokenization.js index 03b95d63a..b04d7456a 100644 --- a/tests/models/bloom/tokenization.js +++ b/tests/models/bloom/tokenization.js @@ -3,192 +3,192 @@ import { BASE_TEST_STRINGS, BLOOM_TEST_STRINGS, SENTENCEPIECE_TEST_STRINGS } fro export const TOKENIZER_CLASS = BloomTokenizer; export const TEST_CONFIG = { - "Xenova/bloom-560m": { - SIMPLE: { - text: BASE_TEST_STRINGS.SIMPLE, - tokens: ["How", "\u0120are", "\u0120you", "\u0120doing", "?"], - ids: [7572, 1306, 1152, 12491, 34], - decoded: "How are you doing?", + "Xenova/bloom-560m": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["How", "\u0120are", "\u0120you", "\u0120doing", "?"], + ids: [7572, 1306, 1152, 12491, 34], + decoded: "How are you doing?", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["You", "\u0120should", "'ve", "\u0120done", "\u0120this"], + ids: [5448, 3403, 7300, 11541, 1119], + decoded: "You should've done this", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["0123", "456789", "\u01200", "\u01201", "\u01202", "\u01203", "\u01204", "\u01205", "\u01206", "\u01207", "\u01208", "\u01209", "\u012010", "\u0120100", "\u01201000"], + ids: [166660, 145647, 931, 404, 415, 735, 934, 973, 1231, 1392, 1445, 1575, 1581, 4334, 19526], + decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "\u0120company", "\u0120was", "\u0120founded", "\u0120in", "\u01202016", "."], + ids: [2175, 16333, 1620, 88289, 361, 5854, 17], + decoded: "The company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "\u010a", "'ll", "\u0120!!", "to", "?", "'d", "''", "d", "\u0120of", ",", "\u0120can't", "."], + ids: [36, 189, 8722, 49825, 1025, 34, 10628, 2328, 71, 461, 15, 11229, 17], + decoded: "A\n'll !!to?'d''d of, can't.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["def", "\u0120main", "()", ":", "\u010a\u0109", "pass"], + ids: [7564, 4291, 883, 29, 1582, 12608], + decoded: "def main():\n\tpass", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".", "toString", "()", ";", "\u010a", "toString", "()", ";"], + ids: [2963, 267, 564, 17949, 17, 27392, 883, 30, 189, 27392, 883, 30], + decoded: "let a = obj.toString();\ntoString();", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["This", "\u010a\u010a", "is", "\u010a", "a", "\u010a", "test", "."], + ids: [6168, 603, 290, 189, 68, 189, 9234, 17], + decoded: "This\n\nis\na\ntest.", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "want", "\u00c3\u00a9d", ",", "running"], + ids: [5777, 75642, 2454, 15, 101897], + decoded: "UNwant\u00e9d,running", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["1", "\u0100", "2", "\u00ef\u00bf\u00bd", "3"], + ids: [20, 179, 21, 23181, 22], + decoded: "1\u00002\ufffd3", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["Hello", "\u0120World"], + ids: [59414, 12155], + decoded: "Hello World", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["hello", "\u0120world"], + ids: [101579, 8876], + decoded: "hello world", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u00e7\u0136\u0141\u00e6\u00b4\u00bb\u00e7\u013c\u0126", "\u00e7\u013e\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"], + ids: [71167, 4137, 1927, 239, 644], + decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u0120\u0120", "\u0120leading", "\u0120space"], + ids: [250, 36128, 12978], + decoded: " leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["tra", "iling", "\u0120space", "\u0120\u0120\u0120"], + ids: [1900, 17022, 12978, 416], + decoded: "trailing space ", + }, + SURROUNDING_SPACE: { + text: BASE_TEST_STRINGS.SURROUNDING_SPACE, + tokens: ["\u0120\u0120", "\u0120surrounding", "\u0120space", "\u0120\u0120\u0120"], + ids: [250, 66599, 12978, 416], + decoded: " surrounding space ", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["Hi", "\u0120", "\u0120Hello"], + ids: [30050, 210, 86153], + decoded: "Hi Hello", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "\u0120$1", "\u0120R2", "\u0120#3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2\u00a5", "6", "\u0120\u00e2\u0124", "\u00a3", "7", "\u0120\u00e2\u0124\u00b9", "8", "\u0120\u00e2\u0124", "\u00b1", "9", "\u0120test"], + ids: [9234, 41448, 80774, 201642, 20117, 23, 40300, 24, 62153, 25, 72279, 100, 26, 120434, 27, 72279, 113, 28, 4006], + decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "\u0120bought", "\u0120an", "\u0120apple", "\u0120for", "\u0120$1", ".", "00", "\u0120at", "\u0120the", "\u0120store", "."], + ids: [44, 87926, 660, 101091, 613, 41448, 17, 462, 919, 368, 18706, 17], + decoded: "I bought an apple for $1.00 at the store.", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u0120\u0120"], + ids: [23438, 4346, 250], + decoded: "you\u2026 ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], + ids: [23438, 4346, 12361], + decoded: "you\u2026\u00a0\u00a0", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], + ids: [23438, 4346, 12361, 23438, 4346, 12361], + decoded: "you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["we", "ird", "\u0120\u00ef\u00bd", "\u0140", "\u0120edge", "\u0120\u00ef\u00bd", "\u0140", "\u0120case"], + ids: [2136, 7589, 122354, 242, 29655, 122354, 242, 4462], + decoded: "weird \uff5e edge \uff5e case", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u00e2\u0138", "\u0123", "This", "\u0120\u00e2\u0138", "\u0123", "is", "\u0120\u00e2\u0138", "\u0123", "a", "\u0120\u00e2\u0138", "\u0123", "test", "\u0120\u00e2\u0138", "\u0123", "."], + ids: [26127, 213, 6168, 15299, 213, 290, 15299, 213, 68, 15299, 213, 9234, 15299, 213, 17], + decoded: "\u2581This \u2581is \u2581a \u2581test \u2581.", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u00f0\u0141\u013a", "\u0124", "\u0120\u00f0\u0141", "\u0133", "\u012f", "\u0120\u00f0\u0141", "\u00a4", "\u00a3", "\u0120\u00f0\u0141\u013a", "\u012f", "\u0120\u00f0\u0141\u013a", "\u0143", "\u0120\u00f0\u0141", "\u0130", "\u012b", "\u0120\u00f0\u0141", "\u013b", "\u0131", "\u0120\u00f0\u0141\u013a", "\u012c", "\u0120\u00f0\u0141", "\u0136", "\u00a5", "\u0120\u00f0\u0141\u013a", "\u0123", "\u0120\u00f0\u0141\u013a", "\u0127", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141\u013a", "\u0128", "\u0120\u00f0\u0141", "\u0133", "\u0131", "\u0120\u00e2\u013f", "\u00a4", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141", "\u0134", "\u013e", "\u0120\u00f0\u0141", "\u0134", "\u013c", "\u0120\u00f0\u0141", "\u0134", "\u0139", "\u0120\u00f0\u0141", "\u0134", "\u013b", "\u0120\u00f0\u0141", "\u0138", "\u00a4", "\u0120\u00f0\u0141\u013a", "\u0130", "\u0120\u00f0\u0141", "\u0133", "\u012e", "\u0120\u00f0\u0141", "\u00a5", "\u00b3", "\u0120\u00f0\u0141", "\u0134", "\u00aa", "\u0120\u00e2\u013e", "\u00a8", "\u0120\u00f0\u0141", "\u0133", "\u012b", "\u0120\u00f0\u0141", "\u0133", "\u0122", "\u0120\u00f0\u0141", "\u0134", "\u00af", "\u0120\u00f0\u0141", "\u0130", "\u012a", "\u0120\u00f0\u0141", "\u013b", "\u012a", "\u0120\u00f0\u0141", "\u013b", "\u012e", "\u0120\u00f0\u0141", "\u0134", "\u0122", "\u0120\u00f0\u0141", "\u0133", "\u0129", "\u0120\u00f0\u0141", "\u0133", "\u012d", "\u0120\u00e2\u013e", "\u0127", "\u0120\u00f0\u0141", "\u0130", "\u0123", "\u0120\u00f0\u0141", "\u012e", "\u0140", "\u0120\u00f0\u0141", "\u012e", "\u00b8", "\u0120\u00f0\u0141", "\u0134", "\u00b0"], + ids: [127322, 214, 41234, 229, 225, 41234, 101, 100, 126342, 225, 126342, 245, 41234, 226, 221, 41234, 237, 227, 126342, 222, 41234, 232, 102, 126342, 213, 126342, 217, 41234, 101, 235, 126342, 218, 41234, 229, 227, 189367, 101, 116057, 41234, 230, 240, 41234, 230, 238, 41234, 230, 235, 41234, 230, 237, 41234, 234, 101, 126342, 226, 41234, 229, 224, 41234, 102, 115, 41234, 230, 107, 76758, 105, 41234, 229, 221, 41234, 229, 212, 41234, 230, 111, 41234, 226, 220, 41234, 237, 220, 41234, 237, 224, 41234, 230, 212, 41234, 229, 219, 41234, 229, 223, 76758, 217, 41234, 226, 213, 41234, 224, 242, 41234, 224, 120, 41234, 230, 112], + decoded: "\ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u00e2\u013e", "\u00a8", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141", "\u0133", "\u0123", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141", "\u0133", "\u00b1", "\u00f0\u0141\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0137", "\u00b5", "\u00e2\u0122\u012f", "\u00e2\u013b", "\u0124", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141", "\u00a7", "\u013b", "\u00f0\u0141\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00e2\u013b", "\u0124", "\u0120\u00f0\u0141", "\u0133", "\u00a8", "\u00f0\u0141\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u012e", "\u00be", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u0120\u00f0\u0141", "\u0133", "\u00a9", "\u00e2\u0122\u012f", "\u00e2\u013f", "\u00a4", "\u00e2\u0122\u012f", "\u00f0\u0141\u0134", "\u012d", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a8", "\u0120\u00f0\u0141", "\u0133", "\u00a9", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a9", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a7", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a6", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0131", "\u00b4", "\u00f3", "\u0142", "\u0123", "\u00a7", "\u00f3", "\u0142", "\u0123", "\u00a2", "\u00f3", "\u0142", "\u0123", "\u00a5", "\u00f3", "\u0142", "\u0123", "\u00ae", "\u00f3", "\u0142", "\u0123", "\u00a7", "\u00f3", "\u0142", "\u0123", "\u00bf", "\u0120\u00f0\u0141", "\u0133", "\u00a8", "\u00f0\u0141\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00e2\u013f", "\u00a4", "\u00ef\u00b8\u0131", "\u00e2\u0122\u012f", "\u00f0\u0141\u0134", "\u012d", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a8", "\u00f0\u0141\u0131", "\u00bc"], + ids: [120709, 105, 41234, 101, 235, 41234, 229, 213, 116057, 41234, 229, 113, 244635, 123, 41234, 233, 117, 1553, 15596, 214, 116057, 41234, 104, 237, 244635, 123, 1553, 15596, 214, 41234, 229, 105, 244635, 123, 1553, 22618, 224, 126, 41234, 104, 229, 1553, 22618, 101, 241, 1553, 22618, 104, 229, 41234, 229, 106, 1553, 157147, 101, 1553, 139500, 223, 1553, 22618, 229, 105, 41234, 229, 106, 1553, 22618, 229, 106, 1553, 22618, 229, 104, 1553, 22618, 229, 103, 41234, 104, 229, 244635, 123, 1553, 22618, 101, 241, 1553, 22618, 104, 229, 244635, 123, 41234, 227, 116, 177, 244, 213, 104, 177, 244, 213, 99, 177, 244, 213, 102, 177, 244, 213, 110, 177, 244, 213, 104, 177, 244, 213, 127, 41234, 229, 105, 244635, 123, 1553, 157147, 101, 116057, 1553, 139500, 223, 1553, 22618, 229, 105, 244635, 124], + decoded: "\u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75\u200d\u2642\ufe0f \ud83e\uddd9\ud83c\udffb\u200d\u2642 \ud83d\udc68\ud83c\udffb\u200d\ud83c\udf3e \ud83e\uddd1\u200d\ud83e\udd1d\u200d\ud83e\uddd1 \ud83d\udc69\u200d\u2764\u200d\ud83d\udc8b\u200d\ud83d\udc68 \ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66 \ud83e\uddd1\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c\udffb \ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f \ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c\udffc", + }, + ONLY_WHITESPACE: { + text: BASE_TEST_STRINGS.ONLY_WHITESPACE, + tokens: ["\u0120\u0109", "\u010a"], + ids: [33651, 189], + decoded: " \t\n", + }, + END_OF_SENTENCE_PUNCTUATION: { + text: BLOOM_TEST_STRINGS.END_OF_SENTENCE_PUNCTUATION, + tokens: ["test", ".", "\u0120test", ",", "\u0120test", "!", "\u0120test", "?", "\u0120test", "\u00e2\u0122\u00a6", "\u0120test", "\u00e3\u0122\u0124", "\u0120test", "\u00ef\u00bc\u012e", "\u0120test", "\u00e3\u0122\u0123", "\u0120test", "\u00e0\u00a5\u00a4", "\u0120test", "\u00db\u0136", "\u0120test", "\u00d8\u012e", "\u0120test"], + ids: [9234, 17, 4006, 15, 4006, 4, 4006, 34, 4006, 4346, 4006, 420, 4006, 355, 4006, 594, 4006, 527, 4006, 1174, 4006, 687, 4006], + decoded: "test. test, test! test? test\u2026 test\u3002 test\uff0c test\u3001 test\u0964 test\u06d4 test\u060c test", + }, + SPECIAL_WITH_TRAILING_WHITESPACE: { + text: SENTENCEPIECE_TEST_STRINGS.SPECIAL_WITH_TRAILING_WHITESPACE, + tokens: ["", "\u010a"], + ids: [1, 189], + decoded: "\n", + }, + SPECIAL_SURROUNDED_BY_WHITESPACE: { + text: SENTENCEPIECE_TEST_STRINGS.SPECIAL_SURROUNDED_BY_WHITESPACE, + tokens: ["\u0120", "", "\u0120test", "\u0120", "", "\u0120"], + ids: [210, 2, 4006, 210, 2, 210], + decoded: " test ", + }, + SPECIAL_NO_WHITESPACE: { + text: SENTENCEPIECE_TEST_STRINGS.SPECIAL_NO_WHITESPACE, + tokens: ["", "test", ""], + ids: [2, 9234, 2], + decoded: "test", + }, }, - SIMPLE_WITH_PUNCTUATION: { - text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, - tokens: ["You", "\u0120should", "'ve", "\u0120done", "\u0120this"], - ids: [5448, 3403, 7300, 11541, 1119], - decoded: "You should've done this", - }, - NUMBERS: { - text: BASE_TEST_STRINGS.NUMBERS, - tokens: ["0123", "456789", "\u01200", "\u01201", "\u01202", "\u01203", "\u01204", "\u01205", "\u01206", "\u01207", "\u01208", "\u01209", "\u012010", "\u0120100", "\u01201000"], - ids: [166660, 145647, 931, 404, 415, 735, 934, 973, 1231, 1392, 1445, 1575, 1581, 4334, 19526], - decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["The", "\u0120company", "\u0120was", "\u0120founded", "\u0120in", "\u01202016", "."], - ids: [2175, 16333, 1620, 88289, 361, 5854, 17], - decoded: "The company was founded in 2016.", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["A", "\u010a", "'ll", "\u0120!!", "to", "?", "'d", "''", "d", "\u0120of", ",", "\u0120can't", "."], - ids: [36, 189, 8722, 49825, 1025, 34, 10628, 2328, 71, 461, 15, 11229, 17], - decoded: "A\n'll !!to?'d''d of, can't.", - }, - PYTHON_CODE: { - text: BASE_TEST_STRINGS.PYTHON_CODE, - tokens: ["def", "\u0120main", "()", ":", "\u010a\u0109", "pass"], - ids: [7564, 4291, 883, 29, 1582, 12608], - decoded: "def main():\n\tpass", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".", "toString", "()", ";", "\u010a", "toString", "()", ";"], - ids: [2963, 267, 564, 17949, 17, 27392, 883, 30, 189, 27392, 883, 30], - decoded: "let a = obj.toString();\ntoString();", - }, - NEWLINES: { - text: BASE_TEST_STRINGS.NEWLINES, - tokens: ["This", "\u010a\u010a", "is", "\u010a", "a", "\u010a", "test", "."], - ids: [6168, 603, 290, 189, 68, 189, 9234, 17], - decoded: "This\n\nis\na\ntest.", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["UN", "want", "\u00c3\u00a9d", ",", "running"], - ids: [5777, 75642, 2454, 15, 101897], - decoded: "UNwant\u00e9d,running", - }, - CONTROL_TOKENS: { - text: BASE_TEST_STRINGS.CONTROL_TOKENS, - tokens: ["1", "\u0100", "2", "\u00ef\u00bf\u00bd", "3"], - ids: [20, 179, 21, 23181, 22], - decoded: "1\u00002\ufffd3", - }, - HELLO_WORLD_TITLECASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, - tokens: ["Hello", "\u0120World"], - ids: [59414, 12155], - decoded: "Hello World", - }, - HELLO_WORLD_LOWERCASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, - tokens: ["hello", "\u0120world"], - ids: [101579, 8876], - decoded: "hello world", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u00e7\u0136\u0141\u00e6\u00b4\u00bb\u00e7\u013c\u0126", "\u00e7\u013e\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"], - ids: [71167, 4137, 1927, 239, 644], - decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", - }, - LEADING_SPACE: { - text: BASE_TEST_STRINGS.LEADING_SPACE, - tokens: ["\u0120\u0120", "\u0120leading", "\u0120space"], - ids: [250, 36128, 12978], - decoded: " leading space", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["tra", "iling", "\u0120space", "\u0120\u0120\u0120"], - ids: [1900, 17022, 12978, 416], - decoded: "trailing space ", - }, - SURROUNDING_SPACE: { - text: BASE_TEST_STRINGS.SURROUNDING_SPACE, - tokens: ["\u0120\u0120", "\u0120surrounding", "\u0120space", "\u0120\u0120\u0120"], - ids: [250, 66599, 12978, 416], - decoded: " surrounding space ", - }, - DOUBLE_SPACE: { - text: BASE_TEST_STRINGS.DOUBLE_SPACE, - tokens: ["Hi", "\u0120", "\u0120Hello"], - ids: [30050, 210, 86153], - decoded: "Hi Hello", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["test", "\u0120$1", "\u0120R2", "\u0120#3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2\u00a5", "6", "\u0120\u00e2\u0124", "\u00a3", "7", "\u0120\u00e2\u0124\u00b9", "8", "\u0120\u00e2\u0124", "\u00b1", "9", "\u0120test"], - ids: [9234, 41448, 80774, 201642, 20117, 23, 40300, 24, 62153, 25, 72279, 100, 26, 120434, 27, 72279, 113, 28, 4006], - decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["I", "\u0120bought", "\u0120an", "\u0120apple", "\u0120for", "\u0120$1", ".", "00", "\u0120at", "\u0120the", "\u0120store", "."], - ids: [44, 87926, 660, 101091, 613, 41448, 17, 462, 919, 368, 18706, 17], - decoded: "I bought an apple for $1.00 at the store.", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - tokens: ["you", "\u00e2\u0122\u00a6", "\u0120\u0120"], - ids: [23438, 4346, 250], - decoded: "you\u2026 ", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], - ids: [23438, 4346, 12361], - decoded: "you\u2026\u00a0\u00a0", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], - ids: [23438, 4346, 12361, 23438, 4346, 12361], - decoded: "you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["we", "ird", "\u0120\u00ef\u00bd", "\u0140", "\u0120edge", "\u0120\u00ef\u00bd", "\u0140", "\u0120case"], - ids: [2136, 7589, 122354, 242, 29655, 122354, 242, 4462], - decoded: "weird \uff5e edge \uff5e case", - }, - SPIECE_UNDERSCORE: { - text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, - tokens: ["\u00e2\u0138", "\u0123", "This", "\u0120\u00e2\u0138", "\u0123", "is", "\u0120\u00e2\u0138", "\u0123", "a", "\u0120\u00e2\u0138", "\u0123", "test", "\u0120\u00e2\u0138", "\u0123", "."], - ids: [26127, 213, 6168, 15299, 213, 290, 15299, 213, 68, 15299, 213, 9234, 15299, 213, 17], - decoded: "\u2581This \u2581is \u2581a \u2581test \u2581.", - }, - POPULAR_EMOJIS: { - text: BASE_TEST_STRINGS.POPULAR_EMOJIS, - tokens: ["\u00f0\u0141\u013a", "\u0124", "\u0120\u00f0\u0141", "\u0133", "\u012f", "\u0120\u00f0\u0141", "\u00a4", "\u00a3", "\u0120\u00f0\u0141\u013a", "\u012f", "\u0120\u00f0\u0141\u013a", "\u0143", "\u0120\u00f0\u0141", "\u0130", "\u012b", "\u0120\u00f0\u0141", "\u013b", "\u0131", "\u0120\u00f0\u0141\u013a", "\u012c", "\u0120\u00f0\u0141", "\u0136", "\u00a5", "\u0120\u00f0\u0141\u013a", "\u0123", "\u0120\u00f0\u0141\u013a", "\u0127", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141\u013a", "\u0128", "\u0120\u00f0\u0141", "\u0133", "\u0131", "\u0120\u00e2\u013f", "\u00a4", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141", "\u0134", "\u013e", "\u0120\u00f0\u0141", "\u0134", "\u013c", "\u0120\u00f0\u0141", "\u0134", "\u0139", "\u0120\u00f0\u0141", "\u0134", "\u013b", "\u0120\u00f0\u0141", "\u0138", "\u00a4", "\u0120\u00f0\u0141\u013a", "\u0130", "\u0120\u00f0\u0141", "\u0133", "\u012e", "\u0120\u00f0\u0141", "\u00a5", "\u00b3", "\u0120\u00f0\u0141", "\u0134", "\u00aa", "\u0120\u00e2\u013e", "\u00a8", "\u0120\u00f0\u0141", "\u0133", "\u012b", "\u0120\u00f0\u0141", "\u0133", "\u0122", "\u0120\u00f0\u0141", "\u0134", "\u00af", "\u0120\u00f0\u0141", "\u0130", "\u012a", "\u0120\u00f0\u0141", "\u013b", "\u012a", "\u0120\u00f0\u0141", "\u013b", "\u012e", "\u0120\u00f0\u0141", "\u0134", "\u0122", "\u0120\u00f0\u0141", "\u0133", "\u0129", "\u0120\u00f0\u0141", "\u0133", "\u012d", "\u0120\u00e2\u013e", "\u0127", "\u0120\u00f0\u0141", "\u0130", "\u0123", "\u0120\u00f0\u0141", "\u012e", "\u0140", "\u0120\u00f0\u0141", "\u012e", "\u00b8", "\u0120\u00f0\u0141", "\u0134", "\u00b0"], - ids: [127322, 214, 41234, 229, 225, 41234, 101, 100, 126342, 225, 126342, 245, 41234, 226, 221, 41234, 237, 227, 126342, 222, 41234, 232, 102, 126342, 213, 126342, 217, 41234, 101, 235, 126342, 218, 41234, 229, 227, 189367, 101, 116057, 41234, 230, 240, 41234, 230, 238, 41234, 230, 235, 41234, 230, 237, 41234, 234, 101, 126342, 226, 41234, 229, 224, 41234, 102, 115, 41234, 230, 107, 76758, 105, 41234, 229, 221, 41234, 229, 212, 41234, 230, 111, 41234, 226, 220, 41234, 237, 220, 41234, 237, 224, 41234, 230, 212, 41234, 229, 219, 41234, 229, 223, 76758, 217, 41234, 226, 213, 41234, 224, 242, 41234, 224, 120, 41234, 230, 112], - decoded: "\ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0", - }, - MULTIBYTE_EMOJIS: { - text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, - tokens: ["\u00e2\u013e", "\u00a8", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141", "\u0133", "\u0123", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141", "\u0133", "\u00b1", "\u00f0\u0141\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0137", "\u00b5", "\u00e2\u0122\u012f", "\u00e2\u013b", "\u0124", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141", "\u00a7", "\u013b", "\u00f0\u0141\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00e2\u013b", "\u0124", "\u0120\u00f0\u0141", "\u0133", "\u00a8", "\u00f0\u0141\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u012e", "\u00be", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u0120\u00f0\u0141", "\u0133", "\u00a9", "\u00e2\u0122\u012f", "\u00e2\u013f", "\u00a4", "\u00e2\u0122\u012f", "\u00f0\u0141\u0134", "\u012d", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a8", "\u0120\u00f0\u0141", "\u0133", "\u00a9", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a9", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a7", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a6", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0131", "\u00b4", "\u00f3", "\u0142", "\u0123", "\u00a7", "\u00f3", "\u0142", "\u0123", "\u00a2", "\u00f3", "\u0142", "\u0123", "\u00a5", "\u00f3", "\u0142", "\u0123", "\u00ae", "\u00f3", "\u0142", "\u0123", "\u00a7", "\u00f3", "\u0142", "\u0123", "\u00bf", "\u0120\u00f0\u0141", "\u0133", "\u00a8", "\u00f0\u0141\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00e2\u013f", "\u00a4", "\u00ef\u00b8\u0131", "\u00e2\u0122\u012f", "\u00f0\u0141\u0134", "\u012d", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a8", "\u00f0\u0141\u0131", "\u00bc"], - ids: [120709, 105, 41234, 101, 235, 41234, 229, 213, 116057, 41234, 229, 113, 244635, 123, 41234, 233, 117, 1553, 15596, 214, 116057, 41234, 104, 237, 244635, 123, 1553, 15596, 214, 41234, 229, 105, 244635, 123, 1553, 22618, 224, 126, 41234, 104, 229, 1553, 22618, 101, 241, 1553, 22618, 104, 229, 41234, 229, 106, 1553, 157147, 101, 1553, 139500, 223, 1553, 22618, 229, 105, 41234, 229, 106, 1553, 22618, 229, 106, 1553, 22618, 229, 104, 1553, 22618, 229, 103, 41234, 104, 229, 244635, 123, 1553, 22618, 101, 241, 1553, 22618, 104, 229, 244635, 123, 41234, 227, 116, 177, 244, 213, 104, 177, 244, 213, 99, 177, 244, 213, 102, 177, 244, 213, 110, 177, 244, 213, 104, 177, 244, 213, 127, 41234, 229, 105, 244635, 123, 1553, 157147, 101, 116057, 1553, 139500, 223, 1553, 22618, 229, 105, 244635, 124], - decoded: "\u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75\u200d\u2642\ufe0f \ud83e\uddd9\ud83c\udffb\u200d\u2642 \ud83d\udc68\ud83c\udffb\u200d\ud83c\udf3e \ud83e\uddd1\u200d\ud83e\udd1d\u200d\ud83e\uddd1 \ud83d\udc69\u200d\u2764\u200d\ud83d\udc8b\u200d\ud83d\udc68 \ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66 \ud83e\uddd1\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c\udffb \ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f \ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c\udffc", - }, - ONLY_WHITESPACE: { - text: BASE_TEST_STRINGS.ONLY_WHITESPACE, - tokens: ["\u0120\u0109", "\u010a"], - ids: [33651, 189], - decoded: " \t\n", - }, - END_OF_SENTENCE_PUNCTUATION: { - text: BLOOM_TEST_STRINGS.END_OF_SENTENCE_PUNCTUATION, - tokens: ["test", ".", "\u0120test", ",", "\u0120test", "!", "\u0120test", "?", "\u0120test", "\u00e2\u0122\u00a6", "\u0120test", "\u00e3\u0122\u0124", "\u0120test", "\u00ef\u00bc\u012e", "\u0120test", "\u00e3\u0122\u0123", "\u0120test", "\u00e0\u00a5\u00a4", "\u0120test", "\u00db\u0136", "\u0120test", "\u00d8\u012e", "\u0120test"], - ids: [9234, 17, 4006, 15, 4006, 4, 4006, 34, 4006, 4346, 4006, 420, 4006, 355, 4006, 594, 4006, 527, 4006, 1174, 4006, 687, 4006], - decoded: "test. test, test! test? test\u2026 test\u3002 test\uff0c test\u3001 test\u0964 test\u06d4 test\u060c test", - }, - SPECIAL_WITH_TRAILING_WHITESPACE: { - text: SENTENCEPIECE_TEST_STRINGS.SPECIAL_WITH_TRAILING_WHITESPACE, - tokens: ["", "\u010a"], - ids: [1, 189], - decoded: "\n", - }, - SPECIAL_SURROUNDED_BY_WHITESPACE: { - text: SENTENCEPIECE_TEST_STRINGS.SPECIAL_SURROUNDED_BY_WHITESPACE, - tokens: ["\u0120", "", "\u0120test", "\u0120", "", "\u0120"], - ids: [210, 2, 4006, 210, 2, 210], - decoded: " test ", - }, - SPECIAL_NO_WHITESPACE: { - text: SENTENCEPIECE_TEST_STRINGS.SPECIAL_NO_WHITESPACE, - tokens: ["", "test", ""], - ids: [2, 9234, 2], - decoded: "test", - }, - }, }; diff --git a/tests/models/clip/tokenization.js b/tests/models/clip/tokenization.js index 73cacda3c..dfcf1ee68 100644 --- a/tests/models/clip/tokenization.js +++ b/tests/models/clip/tokenization.js @@ -3,164 +3,164 @@ import { BASE_TEST_STRINGS } from "../test_strings.js"; export const TOKENIZER_CLASS = CLIPTokenizer; export const TEST_CONFIG = { - "Xenova/clip-vit-base-patch16": { - SIMPLE: { - text: BASE_TEST_STRINGS.SIMPLE, - tokens: ["how", "are", "you", "doing", "?"], - ids: [49406, 829, 631, 592, 1960, 286, 49407], - decoded: "<|startoftext|>how are you doing? <|endoftext|>", + "Xenova/clip-vit-base-patch16": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["how", "are", "you", "doing", "?"], + ids: [49406, 829, 631, 592, 1960, 286, 49407], + decoded: "<|startoftext|>how are you doing? <|endoftext|>", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["you", "should", "'ve", "done", "this"], + ids: [49406, 592, 1535, 1200, 1700, 589, 49407], + decoded: "<|startoftext|>you should've done this <|endoftext|>", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "1", "0", "1", "0", "0", "1", "0", "0", "0"], + ids: [49406, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 272, 271, 272, 271, 271, 272, 271, 271, 271, 49407], + decoded: "<|startoftext|>0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 1 0 1 0 0 1 0 0 0 <|endoftext|>", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["the", "company", "was", "founded", "in", "2", "0", "1", "6", "."], + ids: [49406, 518, 2634, 739, 12240, 530, 273, 271, 272, 277, 269, 49407], + decoded: "<|startoftext|>the company was founded in 2 0 1 6. <|endoftext|>", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["a", "'ll", "!!", "to", "?'", "d", "''", "d", "of", ",", "can", "'t", "."], + ids: [49406, 320, 1342, 748, 531, 13610, 323, 8445, 323, 539, 267, 753, 713, 269, 49407], + decoded: "<|startoftext|>a 'll!! to?' d '' d of, can 't. <|endoftext|>", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["def", "main", "(", "):", "pass"], + ids: [49406, 11649, 2623, 7, 4143, 3511, 49407], + decoded: "<|startoftext|>def main (): pass <|endoftext|>", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "a", "=", "ob", "j", ".", "to", "string", "(", ");", "to", "string", "(", ");"], + ids: [49406, 1094, 320, 284, 1411, 329, 269, 580, 9696, 7, 19686, 580, 9696, 7, 19686, 49407], + decoded: "<|startoftext|>let a = obj. tostring (); tostring (); <|endoftext|>", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["this", "is", "a", "test", "."], + ids: [49406, 589, 533, 320, 1628, 269, 49407], + decoded: "<|startoftext|>this is a test. <|endoftext|>", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["un", "want", "\u00c3\u00a9", "d", ",", "running"], + ids: [49406, 569, 18356, 3459, 323, 267, 2761, 49407], + decoded: "<|startoftext|>unwant\u00e9d, running <|endoftext|>", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["1", "\u0100", "2", "\u00ef\u00bf\u00bd", "3"], + ids: [49406, 272, 444, 273, 39802, 274, 49407], + decoded: "<|startoftext|>1 \u0000 2 \ufffd 3 <|endoftext|>", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["hello", "world"], + ids: [49406, 3306, 1002, 49407], + decoded: "<|startoftext|>hello world <|endoftext|>", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["hello", "world"], + ids: [49406, 3306, 1002, 49407], + decoded: "<|startoftext|>hello world <|endoftext|>", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u00e7\u0136\u0141", "\u00e6", "\u00b4", "\u00bb", "\u00e7", "\u013c", "\u0126", "\u00e7\u013e\u0141", "\u00e8", "\u00b0", "\u013d", "\u00e6\u013a", "\u00af"], + ids: [49406, 33375, 162, 112, 119, 163, 248, 226, 41570, 164, 108, 249, 42891, 363, 49407], + decoded: "<|startoftext|>\u751f\u6d3b\u7684\u771f\u8c1b\u662f <|endoftext|>", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["leading", "space"], + ids: [49406, 3833, 2138, 49407], + decoded: "<|startoftext|>leading space <|endoftext|>", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["trailing", "space"], + ids: [49406, 37427, 2138, 49407], + decoded: "<|startoftext|>trailing space <|endoftext|>", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["hi", "hello"], + ids: [49406, 1883, 3306, 49407], + decoded: "<|startoftext|>hi hello <|endoftext|>", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "$", "1", "r", "2", "#", "3", "\u00e2\u0124\u00ac", "4", "\u00c2\u00a3", "5", "\u00c2\u00a5", "6", "\u00e2\u0124", "\u00a3", "7", "\u00e2\u0124\u00b9", "8", "\u00e2\u0124", "\u00b1", "9", "test"], + ids: [49406, 1628, 259, 272, 337, 273, 258, 274, 6309, 275, 1950, 276, 20199, 277, 5227, 352, 278, 21777, 279, 5227, 365, 280, 1628, 49407], + decoded: "<|startoftext|>test $ 1 r 2 # 3 \u20ac 4 \u00a3 5 \u00a5 6 \u20a3 7 \u20b9 8 \u20b1 9 test <|endoftext|>", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["i", "bought", "an", "apple", "for", "$", "1", ".", "0", "0", "at", "the", "store", "."], + ids: [49406, 328, 4142, 550, 3055, 556, 259, 272, 269, 271, 271, 536, 518, 2183, 269, 49407], + decoded: "<|startoftext|>i bought an apple for $ 1. 0 0 at the store. <|endoftext|>", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "\u00e2\u0122\u00a6"], + ids: [49406, 592, 959, 49407], + decoded: "<|startoftext|>you \u2026 <|endoftext|>", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "\u00e2\u0122\u00a6"], + ids: [49406, 592, 959, 49407], + decoded: "<|startoftext|>you \u2026 <|endoftext|>", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "\u00e2\u0122\u00a6", "you", "\u00e2\u0122\u00a6"], + ids: [49406, 592, 959, 592, 959, 49407], + decoded: "<|startoftext|>you \u2026 you \u2026 <|endoftext|>", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["weird", "\u00ef", "\u00bd", "\u0140", "edge", "\u00ef", "\u00bd", "\u0140", "case"], + ids: [49406, 5613, 171, 121, 508, 5461, 171, 121, 508, 2068, 49407], + decoded: "<|startoftext|>weird \uff5e edge \uff5e case <|endoftext|>", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u00e2\u0138", "\u0123", "this", "\u00e2\u0138", "\u0123", "is", "\u00e2\u0138", "\u0123", "a", "\u00e2\u0138", "\u0123", "test", "\u00e2\u0138", "\u0123", "."], + ids: [49406, 4168, 479, 589, 4168, 479, 533, 4168, 479, 320, 4168, 479, 1628, 4168, 223, 269, 49407], + decoded: "<|startoftext|>\u2581 this \u2581 is \u2581 a \u2581 test \u2581. <|endoftext|>", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u00f0\u0141\u013a\u0124", "\u00f0\u0141\u0133\u012f", "\u00f0\u0141\u00a4\u00a3", "\u00f0\u0141\u013a\u012f", "\u00f0\u0141\u013a\u0143", "\u00f0\u0141\u0130\u012b", "\u00f0\u0141\u013b\u0131", "\u00f0\u0141\u013a\u012c", "\u00f0\u0141\u0136\u00a5", "\u00f0\u0141\u013a\u0123", "\u00f0\u0141\u013a\u0127", "\u00f0\u0141\u00a4\u0139", "\u00f0\u0141\u013a\u0128", "\u00f0\u0141\u0133\u0131", "\u00e2\u013f\u00a4\u00ef\u00b8\u0131", "\u00f0\u0141\u0134\u013e", "\u00f0\u0141\u0134\u013c", "\u00f0\u0141\u0134\u0139", "\u00f0\u0141\u0134\u013b", "\u00f0\u0141\u0138\u00a4", "\u00f0\u0141\u013a\u0130", "\u00f0\u0141\u0133\u012e", "\u00f0\u0141\u00a5\u00b3", "\u00f0\u0141\u0134\u00aa", "\u00e2\u013e\u00a8", "\u00f0\u0141\u0133\u012b", "\u00f0\u0141\u0133\u0122", "\u00f0\u0141\u0134\u00af", "\u00f0\u0141\u0130\u012a", "\u00f0\u0141\u013b\u012a", "\u00f0\u0141\u013b\u012e", "\u00f0\u0141\u0134\u0122", "\u00f0\u0141\u0133\u0129", "\u00f0\u0141\u0133\u012d", "\u00e2\u013e\u0127", "\u00f0\u0141\u0130\u0123", "\u00f0\u0141\u012e\u0140", "\u00f0\u0141\u012e\u00b8", "\u00f0\u0141\u0134\u00b0"], + ids: [49406, 1558, 4201, 9909, 1754, 3915, 3986, 5503, 3020, 3016, 4821, 9188, 10465, 10943, 4829, 1752, 4882, 6521, 6690, 4074, 10860, 4345, 4494, 28055, 6440, 3531, 3988, 5908, 7018, 14448, 9516, 4855, 12158, 7475, 17686, 5564, 13462, 12980, 10980, 14078, 49407], + decoded: "<|startoftext|>\ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0 <|endoftext|>", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u00e2\u013e\u00a8", "\u00f0\u0141\u00a4\u0139", "\u00f0\u0141\u0133\u0123", "\u00ef\u00b8\u0131", "\u00f0\u0141\u0133", "\u00b1", "\u00f0\u0141\u0131\u00bb", "\u00f0\u0141\u0137", "\u00b5", "\u00e2\u0122\u012f\u00e2\u013b\u0124\u00ef\u00b8\u0131", "\u00f0\u0141\u00a7", "\u013b", "\u00f0\u0141\u0131\u00bb", "\u00e2\u0122\u012f\u00e2\u013b", "\u0124", "\u00f0\u0141\u0133\u00a8", "\u00f0\u0141\u0131\u00bb\u00e2\u0122\u012f", "\u00f0\u0141\u012e\u00be", "\u00f0\u0141\u00a7", "\u0133", "\u00e2\u0122\u012f", "\u00f0\u0141\u00a4", "\u013f", "\u00e2\u0122\u012f", "\u00f0\u0141\u00a7", "\u0133", "\u00f0\u0141\u0133\u00a9\u00e2\u0122\u012f", "\u00e2\u013f\u00a4", "\u00e2\u0122\u012f", "\u00f0\u0141\u0134\u012d", "\u00e2\u0122\u012f", "\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141\u0133\u00a9\u00e2\u0122\u012f", "\u00f0\u0141\u0133\u00a9\u00e2\u0122\u012f", "\u00f0\u0141\u0133\u00a7", "\u00e2\u0122\u012f", "\u00f0\u0141\u0133", "\u00a6", "\u00f0\u0141\u00a7", "\u0133", "\u00f0\u0141\u0131\u00bb\u00e2\u0122\u012f", "\u00f0\u0141\u00a4", "\u013f", "\u00e2\u0122\u012f", "\u00f0\u0141\u00a7", "\u0133", "\u00f0\u0141\u0131\u00bb", "\u00f0\u0141\u0131\u00b4", "\u00f3", "\u0142", "\u0123", "\u00a7", "\u00f3", "\u0142", "\u0123", "\u00a2", "\u00f3", "\u0142", "\u0123", "\u00a5", "\u00f3", "\u0142", "\u0123", "\u00ae", "\u00f3", "\u0142", "\u0123", "\u00a7", "\u00f3", "\u0142", "\u0123", "\u00bf", "\u00f0\u0141\u0133\u00a8", "\u00f0\u0141\u0131\u00bb\u00e2\u0122\u012f", "\u00e2\u013f\u00a4\u00ef\u00b8\u0131", "\u00e2\u0122\u012f", "\u00f0\u0141\u0134\u012d", "\u00e2\u0122\u012f", "\u00f0\u0141\u0133\u00a8", "\u00f0\u0141\u0131\u00bc"], + ids: [49406, 3531, 10465, 47796, 1001, 964, 109, 3702, 7692, 113, 10613, 8792, 247, 5042, 5177, 480, 18966, 46250, 39796, 8792, 239, 4244, 1793, 251, 4244, 8792, 495, 26304, 1266, 4244, 12217, 4244, 964, 357, 26304, 26304, 48938, 4244, 964, 355, 8792, 239, 46250, 1793, 251, 4244, 8792, 239, 3702, 39690, 175, 254, 223, 100, 175, 254, 223, 95, 175, 254, 223, 98, 175, 254, 223, 106, 175, 254, 223, 100, 175, 254, 223, 379, 18966, 46250, 2626, 4244, 12217, 4244, 18966, 4027, 49407], + decoded: "<|startoftext|>\u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75\u200d\u2642\ufe0f \ud83e\uddd9\ud83c\udffb\u200d\u2642 \ud83d\udc68\ud83c\udffb\u200d\ud83c\udf3e \ud83e\uddd1\u200d\ud83e\udd1d\u200d\ud83e\uddd1 \ud83d\udc69\u200d\u2764\u200d\ud83d\udc8b\u200d\ud83d\udc68 \ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66 \ud83e\uddd1\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c\udffb \ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f \ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c\udffc <|endoftext|>", + }, + }, + "Xenova/owlvit-base-patch32": { + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["a", "'ll", "!", "!", "to", "?'", "d", "''", "d", "of", ",", "can", "'t", "."], + ids: [49406, 320, 1342, 0, 0, 531, 13610, 323, 8445, 323, 539, 267, 753, 713, 269, 49407], + decoded: "<|startoftext|>a 'll!!to?' d '' d of, can 't. <|endoftext|>", + }, }, - SIMPLE_WITH_PUNCTUATION: { - text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, - tokens: ["you", "should", "'ve", "done", "this"], - ids: [49406, 592, 1535, 1200, 1700, 589, 49407], - decoded: "<|startoftext|>you should've done this <|endoftext|>", - }, - NUMBERS: { - text: BASE_TEST_STRINGS.NUMBERS, - tokens: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "1", "0", "1", "0", "0", "1", "0", "0", "0"], - ids: [49406, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 272, 271, 272, 271, 271, 272, 271, 271, 271, 49407], - decoded: "<|startoftext|>0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 1 0 1 0 0 1 0 0 0 <|endoftext|>", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["the", "company", "was", "founded", "in", "2", "0", "1", "6", "."], - ids: [49406, 518, 2634, 739, 12240, 530, 273, 271, 272, 277, 269, 49407], - decoded: "<|startoftext|>the company was founded in 2 0 1 6. <|endoftext|>", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["a", "'ll", "!!", "to", "?'", "d", "''", "d", "of", ",", "can", "'t", "."], - ids: [49406, 320, 1342, 748, 531, 13610, 323, 8445, 323, 539, 267, 753, 713, 269, 49407], - decoded: "<|startoftext|>a 'll!! to?' d '' d of, can 't. <|endoftext|>", - }, - PYTHON_CODE: { - text: BASE_TEST_STRINGS.PYTHON_CODE, - tokens: ["def", "main", "(", "):", "pass"], - ids: [49406, 11649, 2623, 7, 4143, 3511, 49407], - decoded: "<|startoftext|>def main (): pass <|endoftext|>", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["let", "a", "=", "ob", "j", ".", "to", "string", "(", ");", "to", "string", "(", ");"], - ids: [49406, 1094, 320, 284, 1411, 329, 269, 580, 9696, 7, 19686, 580, 9696, 7, 19686, 49407], - decoded: "<|startoftext|>let a = obj. tostring (); tostring (); <|endoftext|>", - }, - NEWLINES: { - text: BASE_TEST_STRINGS.NEWLINES, - tokens: ["this", "is", "a", "test", "."], - ids: [49406, 589, 533, 320, 1628, 269, 49407], - decoded: "<|startoftext|>this is a test. <|endoftext|>", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["un", "want", "\u00c3\u00a9", "d", ",", "running"], - ids: [49406, 569, 18356, 3459, 323, 267, 2761, 49407], - decoded: "<|startoftext|>unwant\u00e9d, running <|endoftext|>", - }, - CONTROL_TOKENS: { - text: BASE_TEST_STRINGS.CONTROL_TOKENS, - tokens: ["1", "\u0100", "2", "\u00ef\u00bf\u00bd", "3"], - ids: [49406, 272, 444, 273, 39802, 274, 49407], - decoded: "<|startoftext|>1 \u0000 2 \ufffd 3 <|endoftext|>", - }, - HELLO_WORLD_TITLECASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, - tokens: ["hello", "world"], - ids: [49406, 3306, 1002, 49407], - decoded: "<|startoftext|>hello world <|endoftext|>", - }, - HELLO_WORLD_LOWERCASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, - tokens: ["hello", "world"], - ids: [49406, 3306, 1002, 49407], - decoded: "<|startoftext|>hello world <|endoftext|>", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u00e7\u0136\u0141", "\u00e6", "\u00b4", "\u00bb", "\u00e7", "\u013c", "\u0126", "\u00e7\u013e\u0141", "\u00e8", "\u00b0", "\u013d", "\u00e6\u013a", "\u00af"], - ids: [49406, 33375, 162, 112, 119, 163, 248, 226, 41570, 164, 108, 249, 42891, 363, 49407], - decoded: "<|startoftext|>\u751f\u6d3b\u7684\u771f\u8c1b\u662f <|endoftext|>", - }, - LEADING_SPACE: { - text: BASE_TEST_STRINGS.LEADING_SPACE, - tokens: ["leading", "space"], - ids: [49406, 3833, 2138, 49407], - decoded: "<|startoftext|>leading space <|endoftext|>", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["trailing", "space"], - ids: [49406, 37427, 2138, 49407], - decoded: "<|startoftext|>trailing space <|endoftext|>", - }, - DOUBLE_SPACE: { - text: BASE_TEST_STRINGS.DOUBLE_SPACE, - tokens: ["hi", "hello"], - ids: [49406, 1883, 3306, 49407], - decoded: "<|startoftext|>hi hello <|endoftext|>", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["test", "$", "1", "r", "2", "#", "3", "\u00e2\u0124\u00ac", "4", "\u00c2\u00a3", "5", "\u00c2\u00a5", "6", "\u00e2\u0124", "\u00a3", "7", "\u00e2\u0124\u00b9", "8", "\u00e2\u0124", "\u00b1", "9", "test"], - ids: [49406, 1628, 259, 272, 337, 273, 258, 274, 6309, 275, 1950, 276, 20199, 277, 5227, 352, 278, 21777, 279, 5227, 365, 280, 1628, 49407], - decoded: "<|startoftext|>test $ 1 r 2 # 3 \u20ac 4 \u00a3 5 \u00a5 6 \u20a3 7 \u20b9 8 \u20b1 9 test <|endoftext|>", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["i", "bought", "an", "apple", "for", "$", "1", ".", "0", "0", "at", "the", "store", "."], - ids: [49406, 328, 4142, 550, 3055, 556, 259, 272, 269, 271, 271, 536, 518, 2183, 269, 49407], - decoded: "<|startoftext|>i bought an apple for $ 1. 0 0 at the store. <|endoftext|>", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - tokens: ["you", "\u00e2\u0122\u00a6"], - ids: [49406, 592, 959, 49407], - decoded: "<|startoftext|>you \u2026 <|endoftext|>", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - tokens: ["you", "\u00e2\u0122\u00a6"], - ids: [49406, 592, 959, 49407], - decoded: "<|startoftext|>you \u2026 <|endoftext|>", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - tokens: ["you", "\u00e2\u0122\u00a6", "you", "\u00e2\u0122\u00a6"], - ids: [49406, 592, 959, 592, 959, 49407], - decoded: "<|startoftext|>you \u2026 you \u2026 <|endoftext|>", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["weird", "\u00ef", "\u00bd", "\u0140", "edge", "\u00ef", "\u00bd", "\u0140", "case"], - ids: [49406, 5613, 171, 121, 508, 5461, 171, 121, 508, 2068, 49407], - decoded: "<|startoftext|>weird \uff5e edge \uff5e case <|endoftext|>", - }, - SPIECE_UNDERSCORE: { - text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, - tokens: ["\u00e2\u0138", "\u0123", "this", "\u00e2\u0138", "\u0123", "is", "\u00e2\u0138", "\u0123", "a", "\u00e2\u0138", "\u0123", "test", "\u00e2\u0138", "\u0123", "."], - ids: [49406, 4168, 479, 589, 4168, 479, 533, 4168, 479, 320, 4168, 479, 1628, 4168, 223, 269, 49407], - decoded: "<|startoftext|>\u2581 this \u2581 is \u2581 a \u2581 test \u2581. <|endoftext|>", - }, - POPULAR_EMOJIS: { - text: BASE_TEST_STRINGS.POPULAR_EMOJIS, - tokens: ["\u00f0\u0141\u013a\u0124", "\u00f0\u0141\u0133\u012f", "\u00f0\u0141\u00a4\u00a3", "\u00f0\u0141\u013a\u012f", "\u00f0\u0141\u013a\u0143", "\u00f0\u0141\u0130\u012b", "\u00f0\u0141\u013b\u0131", "\u00f0\u0141\u013a\u012c", "\u00f0\u0141\u0136\u00a5", "\u00f0\u0141\u013a\u0123", "\u00f0\u0141\u013a\u0127", "\u00f0\u0141\u00a4\u0139", "\u00f0\u0141\u013a\u0128", "\u00f0\u0141\u0133\u0131", "\u00e2\u013f\u00a4\u00ef\u00b8\u0131", "\u00f0\u0141\u0134\u013e", "\u00f0\u0141\u0134\u013c", "\u00f0\u0141\u0134\u0139", "\u00f0\u0141\u0134\u013b", "\u00f0\u0141\u0138\u00a4", "\u00f0\u0141\u013a\u0130", "\u00f0\u0141\u0133\u012e", "\u00f0\u0141\u00a5\u00b3", "\u00f0\u0141\u0134\u00aa", "\u00e2\u013e\u00a8", "\u00f0\u0141\u0133\u012b", "\u00f0\u0141\u0133\u0122", "\u00f0\u0141\u0134\u00af", "\u00f0\u0141\u0130\u012a", "\u00f0\u0141\u013b\u012a", "\u00f0\u0141\u013b\u012e", "\u00f0\u0141\u0134\u0122", "\u00f0\u0141\u0133\u0129", "\u00f0\u0141\u0133\u012d", "\u00e2\u013e\u0127", "\u00f0\u0141\u0130\u0123", "\u00f0\u0141\u012e\u0140", "\u00f0\u0141\u012e\u00b8", "\u00f0\u0141\u0134\u00b0"], - ids: [49406, 1558, 4201, 9909, 1754, 3915, 3986, 5503, 3020, 3016, 4821, 9188, 10465, 10943, 4829, 1752, 4882, 6521, 6690, 4074, 10860, 4345, 4494, 28055, 6440, 3531, 3988, 5908, 7018, 14448, 9516, 4855, 12158, 7475, 17686, 5564, 13462, 12980, 10980, 14078, 49407], - decoded: "<|startoftext|>\ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0 <|endoftext|>", - }, - MULTIBYTE_EMOJIS: { - text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, - tokens: ["\u00e2\u013e\u00a8", "\u00f0\u0141\u00a4\u0139", "\u00f0\u0141\u0133\u0123", "\u00ef\u00b8\u0131", "\u00f0\u0141\u0133", "\u00b1", "\u00f0\u0141\u0131\u00bb", "\u00f0\u0141\u0137", "\u00b5", "\u00e2\u0122\u012f\u00e2\u013b\u0124\u00ef\u00b8\u0131", "\u00f0\u0141\u00a7", "\u013b", "\u00f0\u0141\u0131\u00bb", "\u00e2\u0122\u012f\u00e2\u013b", "\u0124", "\u00f0\u0141\u0133\u00a8", "\u00f0\u0141\u0131\u00bb\u00e2\u0122\u012f", "\u00f0\u0141\u012e\u00be", "\u00f0\u0141\u00a7", "\u0133", "\u00e2\u0122\u012f", "\u00f0\u0141\u00a4", "\u013f", "\u00e2\u0122\u012f", "\u00f0\u0141\u00a7", "\u0133", "\u00f0\u0141\u0133\u00a9\u00e2\u0122\u012f", "\u00e2\u013f\u00a4", "\u00e2\u0122\u012f", "\u00f0\u0141\u0134\u012d", "\u00e2\u0122\u012f", "\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141\u0133\u00a9\u00e2\u0122\u012f", "\u00f0\u0141\u0133\u00a9\u00e2\u0122\u012f", "\u00f0\u0141\u0133\u00a7", "\u00e2\u0122\u012f", "\u00f0\u0141\u0133", "\u00a6", "\u00f0\u0141\u00a7", "\u0133", "\u00f0\u0141\u0131\u00bb\u00e2\u0122\u012f", "\u00f0\u0141\u00a4", "\u013f", "\u00e2\u0122\u012f", "\u00f0\u0141\u00a7", "\u0133", "\u00f0\u0141\u0131\u00bb", "\u00f0\u0141\u0131\u00b4", "\u00f3", "\u0142", "\u0123", "\u00a7", "\u00f3", "\u0142", "\u0123", "\u00a2", "\u00f3", "\u0142", "\u0123", "\u00a5", "\u00f3", "\u0142", "\u0123", "\u00ae", "\u00f3", "\u0142", "\u0123", "\u00a7", "\u00f3", "\u0142", "\u0123", "\u00bf", "\u00f0\u0141\u0133\u00a8", "\u00f0\u0141\u0131\u00bb\u00e2\u0122\u012f", "\u00e2\u013f\u00a4\u00ef\u00b8\u0131", "\u00e2\u0122\u012f", "\u00f0\u0141\u0134\u012d", "\u00e2\u0122\u012f", "\u00f0\u0141\u0133\u00a8", "\u00f0\u0141\u0131\u00bc"], - ids: [49406, 3531, 10465, 47796, 1001, 964, 109, 3702, 7692, 113, 10613, 8792, 247, 5042, 5177, 480, 18966, 46250, 39796, 8792, 239, 4244, 1793, 251, 4244, 8792, 495, 26304, 1266, 4244, 12217, 4244, 964, 357, 26304, 26304, 48938, 4244, 964, 355, 8792, 239, 46250, 1793, 251, 4244, 8792, 239, 3702, 39690, 175, 254, 223, 100, 175, 254, 223, 95, 175, 254, 223, 98, 175, 254, 223, 106, 175, 254, 223, 100, 175, 254, 223, 379, 18966, 46250, 2626, 4244, 12217, 4244, 18966, 4027, 49407], - decoded: "<|startoftext|>\u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75\u200d\u2642\ufe0f \ud83e\uddd9\ud83c\udffb\u200d\u2642 \ud83d\udc68\ud83c\udffb\u200d\ud83c\udf3e \ud83e\uddd1\u200d\ud83e\udd1d\u200d\ud83e\uddd1 \ud83d\udc69\u200d\u2764\u200d\ud83d\udc8b\u200d\ud83d\udc68 \ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66 \ud83e\uddd1\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c\udffb \ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f \ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c\udffc <|endoftext|>", - }, - }, - "Xenova/owlvit-base-patch32": { - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["a", "'ll", "!", "!", "to", "?'", "d", "''", "d", "of", ",", "can", "'t", "."], - ids: [49406, 320, 1342, 0, 0, 531, 13610, 323, 8445, 323, 539, 267, 753, 713, 269, 49407], - decoded: "<|startoftext|>a 'll!!to?' d '' d of, can 't. <|endoftext|>", - }, - }, }; diff --git a/tests/models/deberta-v2/tokenization.js b/tests/models/deberta-v2/tokenization.js index 177502340..6046e3e1f 100644 --- a/tests/models/deberta-v2/tokenization.js +++ b/tests/models/deberta-v2/tokenization.js @@ -3,302 +3,302 @@ import { BASE_TEST_STRINGS, BERT_TEST_STRINGS } from "../test_strings.js"; export const TOKENIZER_CLASS = DebertaV2Tokenizer; export const TEST_CONFIG = { - "Xenova/nli-deberta-v3-small": { - SIMPLE: { - text: BASE_TEST_STRINGS.SIMPLE, - tokens: ["\u2581How", "\u2581are", "\u2581you", "\u2581doing", "?"], - ids: [1, 577, 281, 274, 653, 302, 2], - decoded: "[CLS] How are you doing?[SEP]", + "Xenova/nli-deberta-v3-small": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["\u2581How", "\u2581are", "\u2581you", "\u2581doing", "?"], + ids: [1, 577, 281, 274, 653, 302, 2], + decoded: "[CLS] How are you doing?[SEP]", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["\u2581You", "\u2581should", "'", "ve", "\u2581done", "\u2581this"], + ids: [1, 367, 403, 280, 415, 619, 291, 2], + decoded: "[CLS] You should've done this[SEP]", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["\u25810", "123456", "789", "\u25810", "\u25811", "\u25812", "\u25813", "\u25814", "\u25815", "\u25816", "\u25817", "\u25818", "\u25819", "\u258110", "\u2581100", "\u25811000"], + ids: [1, 767, 120304, 51535, 767, 376, 392, 404, 453, 456, 525, 574, 578, 712, 466, 803, 4985, 2], + decoded: "[CLS] 0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000[SEP]", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["\u2581The", "\u2581company", "\u2581was", "\u2581founded", "\u2581in", "\u25812016", "."], + ids: [1, 279, 483, 284, 3679, 267, 892, 260, 2], + decoded: "[CLS] The company was founded in 2016.[SEP]", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["\u2581A", "\u2581'", "ll", "\u2581!", "!", "to", "?", "'", "d", "'", "'", "d", "\u2581of", ",", "\u2581can", "'", "t", "."], + ids: [1, 336, 382, 436, 1084, 300, 725, 302, 280, 407, 280, 280, 407, 265, 261, 295, 280, 297, 260, 2], + decoded: "[CLS] A 'll!!to?'d''d of, can't.[SEP]", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["\u2581def", "\u2581main", "(", ")", ":", "\u2581pass"], + ids: [1, 23097, 872, 555, 285, 294, 1633, 2], + decoded: "[CLS] def main(): pass[SEP]", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["\u2581let", "\u2581a", "\u2581=", "\u2581obj", ".", "to", "String", "(", ")", ";", "\u2581to", "String", "(", ")", ";"], + ids: [1, 678, 266, 1842, 68215, 260, 725, 29867, 555, 285, 346, 264, 29867, 555, 285, 346, 2], + decoded: "[CLS] let a = obj.toString(); toString();[SEP]", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["\u2581This", "\u2581is", "\u2581a", "\u2581test", "."], + ids: [1, 329, 269, 266, 1010, 260, 2], + decoded: "[CLS] This is a test.[SEP]", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["\u2581UN", "want", "\u00e9", "d", ",", "running"], + ids: [1, 4647, 27364, 5858, 407, 261, 15243, 2], + decoded: "[CLS] UNwant\u00e9d,running[SEP]", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["\u25811", "\u0000", "2", "\u25813"], + ids: [1, 376, 3, 445, 404, 2], + decoded: "[CLS] 1[UNK]2 3[SEP]", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["\u2581Hello", "\u2581World"], + ids: [1, 5365, 964, 2], + decoded: "[CLS] Hello World[SEP]", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["\u2581hello", "\u2581world"], + ids: [1, 12018, 447, 2], + decoded: "[CLS] hello world[SEP]", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u2581", "\u751f", "\u6d3b", "\u7684", "\u771f", "\u8c1b", "\u662f"], + ids: [1, 507, 41065, 101952, 9301, 98186, 3, 30060, 2], + decoded: "[CLS] \u751f\u6d3b\u7684\u771f[UNK]\u662f[SEP]", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u2581leading", "\u2581space"], + ids: [1, 1249, 754, 2], + decoded: "[CLS] leading space[SEP]", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["\u2581trailing", "\u2581space"], + ids: [1, 18347, 754, 2], + decoded: "[CLS] trailing space[SEP]", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["\u2581Hi", "\u2581Hello"], + ids: [1, 2684, 5365, 2], + decoded: "[CLS] Hi Hello[SEP]", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["\u2581test", "\u2581$", "1", "\u2581R", "2", "\u2581#", "3", "\u2581\u20ac4", "\u2581\u00a35", "\u2581\u00a5", "6", "\u2581", "\u20a3", "7", "\u2581\u20b9", "8", "\u2581\u20b1", "9", "\u2581test"], + ids: [1, 1010, 419, 435, 909, 445, 953, 508, 56238, 14636, 56478, 765, 507, 3, 819, 34880, 804, 121499, 1088, 1010, 2], + decoded: "[CLS] test $1 R2 #3 \u20ac4 \u00a35 \u00a56 [UNK]7 \u20b98 \u20b19 test[SEP]", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["\u2581I", "\u2581bought", "\u2581an", "\u2581apple", "\u2581for", "\u2581$", "1", ".", "00", "\u2581at", "\u2581the", "\u2581store", "."], + ids: [1, 273, 2031, 299, 6038, 270, 419, 435, 260, 962, 288, 262, 1106, 260, 2], + decoded: "[CLS] I bought an apple for $1.00 at the store.[SEP]", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["\u2581you", ".", ".", "."], + ids: [1, 274, 260, 260, 260, 2], + decoded: "[CLS] you...[SEP]", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["\u2581you", ".", ".", "."], + ids: [1, 274, 260, 260, 260, 2], + decoded: "[CLS] you...[SEP]", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["\u2581you", ".", ".", ".", "\u2581you", ".", ".", "."], + ids: [1, 274, 260, 260, 260, 274, 260, 260, 260, 2], + decoded: "[CLS] you... you...[SEP]", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["\u2581weird", "\u2581", "\uff5e", "\u2581edge", "\u2581", "\uff5e", "\u2581case"], + ids: [1, 4926, 507, 96622, 2363, 507, 96622, 571, 2], + decoded: "[CLS] weird \uff5e edge \uff5e case[SEP]", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u2581This", "\u2581is", "\u2581a", "\u2581test", "\u2581."], + ids: [1, 329, 269, 266, 1010, 323, 2], + decoded: "[CLS] This is a test.[SEP]", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u2581\ud83d\ude02", "\u2581", "\ud83d\udc4d", "\u2581", "\ud83e\udd23", "\u2581", "\ud83d\ude0d", "\u2581", "\ud83d\ude2d", "\u2581", "\ud83c\udf89", "\u2581", "\ud83d\ude4f", "\u2581\ud83d\ude0a", "\u2581\ud83d\udd25", "\u2581", "\ud83d\ude01", "\u2581", "\ud83d\ude05", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\ude06", "\u2581", "\ud83d\udc4f", "\u2581\u2764", "\ufe0f", "\u2581", "\ud83d\udc9c", "\u2581", "\ud83d\udc9a", "\u2581", "\ud83d\udc97", "\u2581", "\ud83d\udc99", "\u2581", "\ud83d\udda4", "\u2581", "\ud83d\ude0e", "\u2581", "\ud83d\udc4c", "\u2581", "\ud83e\udd73", "\u2581", "\ud83d\udcaa", "\u2581", "\u2728", "\u2581", "\ud83d\udc49", "\u2581", "\ud83d\udc40", "\u2581", "\ud83d\udcaf", "\u2581", "\ud83c\udf88", "\u2581", "\ud83d\ude48", "\u2581", "\ud83d\ude4c", "\u2581", "\ud83d\udc80", "\u2581", "\ud83d\udc47", "\u2581", "\ud83d\udc4b", "\u2581\u2705", "\u2581", "\ud83c\udf81", "\u2581", "\ud83c\udf1e", "\u2581", "\ud83c\udf38", "\u2581", "\ud83d\udcb0"], + ids: [1, 97504, 507, 117545, 507, 123057, 507, 96353, 507, 123058, 507, 123169, 507, 121772, 109976, 115475, 507, 122874, 507, 124017, 507, 123983, 507, 123571, 507, 122632, 49509, 25377, 507, 123614, 507, 124105, 507, 124077, 507, 123384, 507, 124382, 507, 123340, 507, 123492, 507, 3, 507, 123306, 507, 110119, 507, 122633, 507, 123659, 507, 123765, 507, 125799, 507, 124322, 507, 122878, 507, 125843, 507, 124011, 507, 125021, 88523, 507, 124698, 507, 125612, 507, 123887, 507, 123979, 2], + decoded: "[CLS] \ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c [UNK] \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0[SEP]", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u2581", "\u2728", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\udc41", "\ufe0f", "\u2581", "\ud83d\udc71", "\ud83c\udffb", "\u2581", "\ud83d\udd75", "\u2581", "\u2642", "\ufe0f", "\u2581", "\ud83e\uddd9", "\ud83c\udffb", "\u2581", "\u2642", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u2581", "\ud83c\udf3e", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83d\udc69", "\u2581\u2764", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc67", "\u2581", "\ud83d\udc66", "\u2581", "\ud83e\uddd1", "\ud83c\udffb", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1", "\ud83c\udffb", "\u2581", "\ud83c\udff4", "\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u2581\u2764", "\ufe0f", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\ud83c\udffc"], + ids: [1, 507, 110119, 507, 123983, 507, 127294, 25377, 507, 3, 108391, 507, 3, 507, 117868, 25377, 507, 3, 108391, 507, 117868, 507, 125199, 108391, 507, 3, 507, 3, 507, 3, 507, 3, 507, 124709, 49509, 507, 124327, 507, 125199, 507, 124709, 507, 124709, 507, 126640, 507, 126853, 507, 3, 108391, 507, 3, 507, 3, 108391, 507, 126132, 3, 507, 125199, 108391, 49509, 25377, 507, 124327, 507, 125199, 118155, 2], + decoded: "[CLS] \u2728 \ud83e\udd17 \ud83d\udc41\ufe0f [UNK]\ud83c\udffb [UNK] \u2642\ufe0f [UNK]\ud83c\udffb \u2642 \ud83d\udc68\ud83c\udffb [UNK] [UNK] [UNK] [UNK] \ud83d\udc69 \u2764 \ud83d\udc8b \ud83d\udc68 \ud83d\udc69 \ud83d\udc69 \ud83d\udc67 \ud83d\udc66 [UNK]\ud83c\udffb [UNK] [UNK]\ud83c\udffb \ud83c\udff4[UNK] \ud83d\udc68\ud83c\udffb \u2764\ufe0f \ud83d\udc8b \ud83d\udc68\ud83c\udffc[SEP]", + }, + CHINESE_LATIN_MIXED: { + text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, + tokens: ["\u2581a", "h", "\u535a", "\u63a8", "zz"], + ids: [1, 266, 1537, 122598, 111743, 23260, 2], + decoded: "[CLS] ah\u535a\u63a8zz[SEP]", + }, + SIMPLE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, + tokens: ["\u2581H\u00e9", "llo"], + ids: [1, 93519, 25341, 2], + decoded: "[CLS] H\u00e9llo[SEP]", + }, + MIXED_CASE_WITHOUT_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, + tokens: ["\u2581He", "LL", "o", "!", "how", "\u2581Are", "\u2581yo", "U", "?"], + ids: [1, 383, 17145, 795, 300, 5608, 1396, 14469, 2628, 302, 2], + decoded: "[CLS] HeLLo!how Are yoU?[SEP]", + }, + MIXED_CASE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, + tokens: ["\u2581H\u00e4", "LL", "o", "!", "how", "\u2581Are", "\u2581yo", "U", "?"], + ids: [1, 62693, 17145, 795, 300, 5608, 1396, 14469, 2628, 302, 2], + decoded: "[CLS] H\u00e4LLo!how Are yoU?[SEP]", + }, + }, + "Xenova/mDeBERTa-v3-base-xnli-multilingual-nli-2mil7": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["\u2581How", "\u2581are", "\u2581you", "\u2581do", "ing", "?"], + ids: [1, 5101, 419, 522, 343, 348, 292, 2], + decoded: "[CLS] How are you doing?[SEP]", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["\u2581", "0123456789", "\u25810", "\u25811", "\u25812", "\u25813", "\u25814", "\u25815", "\u25816", "\u25817", "\u25818", "\u25819", "\u258110", "\u2581100", "\u25811000"], + ids: [1, 260, 170160, 498, 334, 357, 382, 420, 431, 571, 618, 631, 775, 476, 967, 3884, 2], + decoded: "[CLS] 0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000[SEP]", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["\u2581The", "\u2581company", "\u2581was", "\u2581found", "ed", "\u2581in", "\u25812016."], + ids: [1, 487, 5836, 640, 5898, 346, 282, 13792, 2], + decoded: "[CLS] The company was founded in 2016.[SEP]", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["\u2581A", "\u2581", "'", "ll", "\u2581", "!!", "to", "?", "'", "d", "''", "d", "\u2581of", ",", "\u2581can", "'", "t", "."], + ids: [1, 299, 260, 278, 1579, 260, 1524, 477, 292, 278, 286, 4461, 286, 305, 262, 739, 278, 271, 261, 2], + decoded: "[CLS] A 'll!!to?'d''d of, can't.[SEP]", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["\u2581de", "f", "\u2581main", "():", "\u2581pass"], + ids: [1, 270, 368, 4398, 78612, 4748, 2], + decoded: "[CLS] def main(): pass[SEP]", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["\u2581let", "\u2581", "a", "\u2581", "=", "\u2581obj", ".", "toString", "();", "\u2581", "toString", "();"], + ids: [1, 3257, 260, 263, 260, 350, 50670, 261, 64577, 1994, 260, 64577, 1994, 2], + decoded: "[CLS] let a = obj.toString(); toString();[SEP]", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["\u2581This", "\u2581is", "\u2581", "a", "\u2581test", "."], + ids: [1, 1495, 340, 260, 263, 2979, 261, 2], + decoded: "[CLS] This is a test.[SEP]", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["\u2581UN", "wan", "t\u00e9", "d", ",", "running"], + ids: [1, 10970, 3016, 3986, 286, 262, 170565, 2], + decoded: "[CLS] UNwant\u00e9d,running[SEP]", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u2581", "\u751f\u6d3b\u7684", "\u771f", "\u8c1b", "\u662f"], + ids: [1, 260, 197263, 7275, 241962, 1544, 2], + decoded: "[CLS] \u751f\u6d3b\u7684\u771f\u8c1b\u662f[SEP]", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u2581", "leading", "\u2581space"], + ids: [1, 260, 22120, 11496, 2], + decoded: "[CLS] leading space[SEP]", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["\u2581trail", "ing", "\u2581space"], + ids: [1, 66699, 348, 11496, 2], + decoded: "[CLS] trailing space[SEP]", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["\u2581test", "\u2581$1", "\u2581R", "2", "\u2581#3", "\u2581\u20ac4", "\u2581\u00a35", "\u2581\u00a5", "6", "\u2581", "\u20a3", "7", "\u2581\u20b9", "8", "\u2581", "\u20b1", "9", "\u2581test"], + ids: [1, 2979, 21793, 532, 339, 19403, 157186, 156260, 33481, 452, 260, 242687, 488, 39568, 450, 260, 211232, 496, 2979, 2], + decoded: "[CLS] test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test[SEP]", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["\u2581I", "\u2581b", "ought", "\u2581an", "\u2581apple", "\u2581for", "\u2581$", "1.00", "\u2581at", "\u2581the", "\u2581store", "."], + ids: [1, 337, 331, 22280, 462, 44791, 333, 1161, 42645, 345, 288, 5318, 261, 2], + decoded: "[CLS] I bought an apple for $1.00 at the store.[SEP]", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["\u2581you", "..."], + ids: [1, 522, 303, 2], + decoded: "[CLS] you...[SEP]", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["\u2581you", "..."], + ids: [1, 522, 303, 2], + decoded: "[CLS] you...[SEP]", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["\u2581you", "...", "\u2581you", "..."], + ids: [1, 522, 303, 522, 303, 2], + decoded: "[CLS] you... you...[SEP]", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["\u2581w", "eird", "\u2581", "\uff5e", "\u2581edge", "\u2581", "\uff5e", "\u2581case"], + ids: [1, 415, 116640, 260, 2790, 53876, 260, 2790, 4073, 2], + decoded: "[CLS] weird \uff5e edge \uff5e case[SEP]", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u2581This", "\u2581is", "\u2581", "a", "\u2581test", "\u2581", "."], + ids: [1, 1495, 340, 260, 263, 2979, 260, 261, 2], + decoded: "[CLS] This is a test.[SEP]", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u2581", "\ud83d\ude02", "\u2581", "\ud83d\udc4d", "\u2581", "\ud83e\udd23", "\u2581", "\ud83d\ude0d", "\u2581", "\ud83d\ude2d", "\u2581", "\ud83c\udf89", "\u2581", "\ud83d\ude4f", "\u2581", "\ud83d\ude0a", "\u2581", "\ud83d\udd25", "\u2581", "\ud83d\ude01", "\u2581", "\ud83d\ude05", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\ude06", "\u2581", "\ud83d\udc4f", "\u2581\u2764", "\ufe0f", "\u2581", "\ud83d\udc9c", "\u2581", "\ud83d\udc9a", "\u2581", "\ud83d\udc97", "\u2581", "\ud83d\udc99", "\u2581", "\ud83d\udda4", "\u2581", "\ud83d\ude0e", "\u2581", "\ud83d\udc4c", "\u2581", "\ud83e\udd73", "\u2581", "\ud83d\udcaa", "\u2581", "\u2728", "\u2581\ud83d\udc49", "\u2581", "\ud83d\udc40", "\u2581", "\ud83d\udcaf", "\u2581", "\ud83c\udf88", "\u2581", "\ud83d\ude48", "\u2581", "\ud83d\ude4c", "\u2581", "\ud83d\udc80", "\u2581", "\ud83d\udc47", "\u2581", "\ud83d\udc4b", "\u2581\u2705", "\u2581", "\ud83c\udf81", "\u2581", "\ud83c\udf1e", "\u2581", "\ud83c\udf38", "\u2581", "\ud83d\udcb0"], + ids: [1, 260, 116844, 260, 72330, 260, 160951, 260, 78796, 260, 180546, 260, 212774, 260, 102930, 260, 71509, 260, 96089, 260, 137652, 260, 194608, 260, 182033, 260, 164467, 260, 149267, 56787, 4668, 260, 210251, 260, 195202, 260, 178523, 260, 167604, 260, 236081, 260, 157800, 260, 162843, 260, 242580, 260, 174590, 260, 65271, 113700, 260, 239652, 260, 237474, 260, 240937, 260, 239131, 260, 216701, 260, 242618, 260, 133395, 260, 240645, 82147, 260, 49599, 260, 239888, 260, 152102, 260, 239168, 2], + decoded: "[CLS] \ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0[SEP]", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u2581", "\u2728", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\udc41", "\ufe0f", "\u2581", "\ud83d\udc71", "\ud83c\udffb", "\u2581", "\ud83d\udd75", "\u2581", "\u2642", "\ufe0f", "\u2581", "\ud83e\uddd9", "\ud83c\udffb", "\u2581", "\u2642", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u2581", "\ud83c\udf3e", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83d\udc69", "\u2581\u2764", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc67", "\u2581", "\ud83d\udc66", "\u2581", "\ud83e\uddd1", "\ud83c\udffb", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1", "\ud83c\udffb", "\u2581", "\ud83c\udff4", "\udb40\udc67", "\udb40\udc62", "\udb40\udc65", "\udb40\udc6e", "\udb40\udc67", "\udb40\udc7f", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u2581\u2764", "\ufe0f", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\ud83c\udffc"], + ids: [1, 260, 65271, 260, 182033, 260, 16307, 4668, 260, 244774, 75846, 260, 247133, 260, 50622, 4668, 260, 3, 75846, 260, 50622, 260, 239432, 75846, 260, 243052, 260, 244250, 260, 243394, 260, 244250, 260, 239098, 56787, 260, 223802, 260, 239432, 260, 239098, 260, 239098, 260, 241727, 260, 242446, 260, 244250, 75846, 260, 243394, 260, 244250, 75846, 260, 244177, 245994, 247023, 248837, 248531, 245994, 245953, 260, 239432, 75846, 56787, 4668, 260, 223802, 260, 239432, 159667, 2], + decoded: "[CLS] \u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75 \u2642\ufe0f [UNK]\ud83c\udffb \u2642 \ud83d\udc68\ud83c\udffb \ud83c\udf3e \ud83e\uddd1 \ud83e\udd1d \ud83e\uddd1 \ud83d\udc69 \u2764 \ud83d\udc8b \ud83d\udc68 \ud83d\udc69 \ud83d\udc69 \ud83d\udc67 \ud83d\udc66 \ud83e\uddd1\ud83c\udffb \ud83e\udd1d \ud83e\uddd1\ud83c\udffb \ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f \ud83d\udc68\ud83c\udffb \u2764\ufe0f \ud83d\udc8b \ud83d\udc68\ud83c\udffc[SEP]", + }, }, - SIMPLE_WITH_PUNCTUATION: { - text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, - tokens: ["\u2581You", "\u2581should", "'", "ve", "\u2581done", "\u2581this"], - ids: [1, 367, 403, 280, 415, 619, 291, 2], - decoded: "[CLS] You should've done this[SEP]", - }, - NUMBERS: { - text: BASE_TEST_STRINGS.NUMBERS, - tokens: ["\u25810", "123456", "789", "\u25810", "\u25811", "\u25812", "\u25813", "\u25814", "\u25815", "\u25816", "\u25817", "\u25818", "\u25819", "\u258110", "\u2581100", "\u25811000"], - ids: [1, 767, 120304, 51535, 767, 376, 392, 404, 453, 456, 525, 574, 578, 712, 466, 803, 4985, 2], - decoded: "[CLS] 0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000[SEP]", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["\u2581The", "\u2581company", "\u2581was", "\u2581founded", "\u2581in", "\u25812016", "."], - ids: [1, 279, 483, 284, 3679, 267, 892, 260, 2], - decoded: "[CLS] The company was founded in 2016.[SEP]", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["\u2581A", "\u2581'", "ll", "\u2581!", "!", "to", "?", "'", "d", "'", "'", "d", "\u2581of", ",", "\u2581can", "'", "t", "."], - ids: [1, 336, 382, 436, 1084, 300, 725, 302, 280, 407, 280, 280, 407, 265, 261, 295, 280, 297, 260, 2], - decoded: "[CLS] A 'll!!to?'d''d of, can't.[SEP]", - }, - PYTHON_CODE: { - text: BASE_TEST_STRINGS.PYTHON_CODE, - tokens: ["\u2581def", "\u2581main", "(", ")", ":", "\u2581pass"], - ids: [1, 23097, 872, 555, 285, 294, 1633, 2], - decoded: "[CLS] def main(): pass[SEP]", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["\u2581let", "\u2581a", "\u2581=", "\u2581obj", ".", "to", "String", "(", ")", ";", "\u2581to", "String", "(", ")", ";"], - ids: [1, 678, 266, 1842, 68215, 260, 725, 29867, 555, 285, 346, 264, 29867, 555, 285, 346, 2], - decoded: "[CLS] let a = obj.toString(); toString();[SEP]", - }, - NEWLINES: { - text: BASE_TEST_STRINGS.NEWLINES, - tokens: ["\u2581This", "\u2581is", "\u2581a", "\u2581test", "."], - ids: [1, 329, 269, 266, 1010, 260, 2], - decoded: "[CLS] This is a test.[SEP]", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["\u2581UN", "want", "\u00e9", "d", ",", "running"], - ids: [1, 4647, 27364, 5858, 407, 261, 15243, 2], - decoded: "[CLS] UNwant\u00e9d,running[SEP]", - }, - CONTROL_TOKENS: { - text: BASE_TEST_STRINGS.CONTROL_TOKENS, - tokens: ["\u25811", "\u0000", "2", "\u25813"], - ids: [1, 376, 3, 445, 404, 2], - decoded: "[CLS] 1[UNK]2 3[SEP]", - }, - HELLO_WORLD_TITLECASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, - tokens: ["\u2581Hello", "\u2581World"], - ids: [1, 5365, 964, 2], - decoded: "[CLS] Hello World[SEP]", - }, - HELLO_WORLD_LOWERCASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, - tokens: ["\u2581hello", "\u2581world"], - ids: [1, 12018, 447, 2], - decoded: "[CLS] hello world[SEP]", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u2581", "\u751f", "\u6d3b", "\u7684", "\u771f", "\u8c1b", "\u662f"], - ids: [1, 507, 41065, 101952, 9301, 98186, 3, 30060, 2], - decoded: "[CLS] \u751f\u6d3b\u7684\u771f[UNK]\u662f[SEP]", - }, - LEADING_SPACE: { - text: BASE_TEST_STRINGS.LEADING_SPACE, - tokens: ["\u2581leading", "\u2581space"], - ids: [1, 1249, 754, 2], - decoded: "[CLS] leading space[SEP]", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["\u2581trailing", "\u2581space"], - ids: [1, 18347, 754, 2], - decoded: "[CLS] trailing space[SEP]", - }, - DOUBLE_SPACE: { - text: BASE_TEST_STRINGS.DOUBLE_SPACE, - tokens: ["\u2581Hi", "\u2581Hello"], - ids: [1, 2684, 5365, 2], - decoded: "[CLS] Hi Hello[SEP]", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["\u2581test", "\u2581$", "1", "\u2581R", "2", "\u2581#", "3", "\u2581\u20ac4", "\u2581\u00a35", "\u2581\u00a5", "6", "\u2581", "\u20a3", "7", "\u2581\u20b9", "8", "\u2581\u20b1", "9", "\u2581test"], - ids: [1, 1010, 419, 435, 909, 445, 953, 508, 56238, 14636, 56478, 765, 507, 3, 819, 34880, 804, 121499, 1088, 1010, 2], - decoded: "[CLS] test $1 R2 #3 \u20ac4 \u00a35 \u00a56 [UNK]7 \u20b98 \u20b19 test[SEP]", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["\u2581I", "\u2581bought", "\u2581an", "\u2581apple", "\u2581for", "\u2581$", "1", ".", "00", "\u2581at", "\u2581the", "\u2581store", "."], - ids: [1, 273, 2031, 299, 6038, 270, 419, 435, 260, 962, 288, 262, 1106, 260, 2], - decoded: "[CLS] I bought an apple for $1.00 at the store.[SEP]", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - tokens: ["\u2581you", ".", ".", "."], - ids: [1, 274, 260, 260, 260, 2], - decoded: "[CLS] you...[SEP]", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - tokens: ["\u2581you", ".", ".", "."], - ids: [1, 274, 260, 260, 260, 2], - decoded: "[CLS] you...[SEP]", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - tokens: ["\u2581you", ".", ".", ".", "\u2581you", ".", ".", "."], - ids: [1, 274, 260, 260, 260, 274, 260, 260, 260, 2], - decoded: "[CLS] you... you...[SEP]", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["\u2581weird", "\u2581", "\uff5e", "\u2581edge", "\u2581", "\uff5e", "\u2581case"], - ids: [1, 4926, 507, 96622, 2363, 507, 96622, 571, 2], - decoded: "[CLS] weird \uff5e edge \uff5e case[SEP]", - }, - SPIECE_UNDERSCORE: { - text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, - tokens: ["\u2581This", "\u2581is", "\u2581a", "\u2581test", "\u2581."], - ids: [1, 329, 269, 266, 1010, 323, 2], - decoded: "[CLS] This is a test.[SEP]", - }, - POPULAR_EMOJIS: { - text: BASE_TEST_STRINGS.POPULAR_EMOJIS, - tokens: ["\u2581\ud83d\ude02", "\u2581", "\ud83d\udc4d", "\u2581", "\ud83e\udd23", "\u2581", "\ud83d\ude0d", "\u2581", "\ud83d\ude2d", "\u2581", "\ud83c\udf89", "\u2581", "\ud83d\ude4f", "\u2581\ud83d\ude0a", "\u2581\ud83d\udd25", "\u2581", "\ud83d\ude01", "\u2581", "\ud83d\ude05", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\ude06", "\u2581", "\ud83d\udc4f", "\u2581\u2764", "\ufe0f", "\u2581", "\ud83d\udc9c", "\u2581", "\ud83d\udc9a", "\u2581", "\ud83d\udc97", "\u2581", "\ud83d\udc99", "\u2581", "\ud83d\udda4", "\u2581", "\ud83d\ude0e", "\u2581", "\ud83d\udc4c", "\u2581", "\ud83e\udd73", "\u2581", "\ud83d\udcaa", "\u2581", "\u2728", "\u2581", "\ud83d\udc49", "\u2581", "\ud83d\udc40", "\u2581", "\ud83d\udcaf", "\u2581", "\ud83c\udf88", "\u2581", "\ud83d\ude48", "\u2581", "\ud83d\ude4c", "\u2581", "\ud83d\udc80", "\u2581", "\ud83d\udc47", "\u2581", "\ud83d\udc4b", "\u2581\u2705", "\u2581", "\ud83c\udf81", "\u2581", "\ud83c\udf1e", "\u2581", "\ud83c\udf38", "\u2581", "\ud83d\udcb0"], - ids: [1, 97504, 507, 117545, 507, 123057, 507, 96353, 507, 123058, 507, 123169, 507, 121772, 109976, 115475, 507, 122874, 507, 124017, 507, 123983, 507, 123571, 507, 122632, 49509, 25377, 507, 123614, 507, 124105, 507, 124077, 507, 123384, 507, 124382, 507, 123340, 507, 123492, 507, 3, 507, 123306, 507, 110119, 507, 122633, 507, 123659, 507, 123765, 507, 125799, 507, 124322, 507, 122878, 507, 125843, 507, 124011, 507, 125021, 88523, 507, 124698, 507, 125612, 507, 123887, 507, 123979, 2], - decoded: "[CLS] \ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c [UNK] \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0[SEP]", - }, - MULTIBYTE_EMOJIS: { - text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, - tokens: ["\u2581", "\u2728", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\udc41", "\ufe0f", "\u2581", "\ud83d\udc71", "\ud83c\udffb", "\u2581", "\ud83d\udd75", "\u2581", "\u2642", "\ufe0f", "\u2581", "\ud83e\uddd9", "\ud83c\udffb", "\u2581", "\u2642", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u2581", "\ud83c\udf3e", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83d\udc69", "\u2581\u2764", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc67", "\u2581", "\ud83d\udc66", "\u2581", "\ud83e\uddd1", "\ud83c\udffb", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1", "\ud83c\udffb", "\u2581", "\ud83c\udff4", "\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u2581\u2764", "\ufe0f", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\ud83c\udffc"], - ids: [1, 507, 110119, 507, 123983, 507, 127294, 25377, 507, 3, 108391, 507, 3, 507, 117868, 25377, 507, 3, 108391, 507, 117868, 507, 125199, 108391, 507, 3, 507, 3, 507, 3, 507, 3, 507, 124709, 49509, 507, 124327, 507, 125199, 507, 124709, 507, 124709, 507, 126640, 507, 126853, 507, 3, 108391, 507, 3, 507, 3, 108391, 507, 126132, 3, 507, 125199, 108391, 49509, 25377, 507, 124327, 507, 125199, 118155, 2], - decoded: "[CLS] \u2728 \ud83e\udd17 \ud83d\udc41\ufe0f [UNK]\ud83c\udffb [UNK] \u2642\ufe0f [UNK]\ud83c\udffb \u2642 \ud83d\udc68\ud83c\udffb [UNK] [UNK] [UNK] [UNK] \ud83d\udc69 \u2764 \ud83d\udc8b \ud83d\udc68 \ud83d\udc69 \ud83d\udc69 \ud83d\udc67 \ud83d\udc66 [UNK]\ud83c\udffb [UNK] [UNK]\ud83c\udffb \ud83c\udff4[UNK] \ud83d\udc68\ud83c\udffb \u2764\ufe0f \ud83d\udc8b \ud83d\udc68\ud83c\udffc[SEP]", - }, - CHINESE_LATIN_MIXED: { - text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, - tokens: ["\u2581a", "h", "\u535a", "\u63a8", "zz"], - ids: [1, 266, 1537, 122598, 111743, 23260, 2], - decoded: "[CLS] ah\u535a\u63a8zz[SEP]", - }, - SIMPLE_WITH_ACCENTS: { - text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, - tokens: ["\u2581H\u00e9", "llo"], - ids: [1, 93519, 25341, 2], - decoded: "[CLS] H\u00e9llo[SEP]", - }, - MIXED_CASE_WITHOUT_ACCENTS: { - text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, - tokens: ["\u2581He", "LL", "o", "!", "how", "\u2581Are", "\u2581yo", "U", "?"], - ids: [1, 383, 17145, 795, 300, 5608, 1396, 14469, 2628, 302, 2], - decoded: "[CLS] HeLLo!how Are yoU?[SEP]", - }, - MIXED_CASE_WITH_ACCENTS: { - text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, - tokens: ["\u2581H\u00e4", "LL", "o", "!", "how", "\u2581Are", "\u2581yo", "U", "?"], - ids: [1, 62693, 17145, 795, 300, 5608, 1396, 14469, 2628, 302, 2], - decoded: "[CLS] H\u00e4LLo!how Are yoU?[SEP]", - }, - }, - "Xenova/mDeBERTa-v3-base-xnli-multilingual-nli-2mil7": { - SIMPLE: { - text: BASE_TEST_STRINGS.SIMPLE, - tokens: ["\u2581How", "\u2581are", "\u2581you", "\u2581do", "ing", "?"], - ids: [1, 5101, 419, 522, 343, 348, 292, 2], - decoded: "[CLS] How are you doing?[SEP]", - }, - NUMBERS: { - text: BASE_TEST_STRINGS.NUMBERS, - tokens: ["\u2581", "0123456789", "\u25810", "\u25811", "\u25812", "\u25813", "\u25814", "\u25815", "\u25816", "\u25817", "\u25818", "\u25819", "\u258110", "\u2581100", "\u25811000"], - ids: [1, 260, 170160, 498, 334, 357, 382, 420, 431, 571, 618, 631, 775, 476, 967, 3884, 2], - decoded: "[CLS] 0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000[SEP]", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["\u2581The", "\u2581company", "\u2581was", "\u2581found", "ed", "\u2581in", "\u25812016."], - ids: [1, 487, 5836, 640, 5898, 346, 282, 13792, 2], - decoded: "[CLS] The company was founded in 2016.[SEP]", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["\u2581A", "\u2581", "'", "ll", "\u2581", "!!", "to", "?", "'", "d", "''", "d", "\u2581of", ",", "\u2581can", "'", "t", "."], - ids: [1, 299, 260, 278, 1579, 260, 1524, 477, 292, 278, 286, 4461, 286, 305, 262, 739, 278, 271, 261, 2], - decoded: "[CLS] A 'll!!to?'d''d of, can't.[SEP]", - }, - PYTHON_CODE: { - text: BASE_TEST_STRINGS.PYTHON_CODE, - tokens: ["\u2581de", "f", "\u2581main", "():", "\u2581pass"], - ids: [1, 270, 368, 4398, 78612, 4748, 2], - decoded: "[CLS] def main(): pass[SEP]", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["\u2581let", "\u2581", "a", "\u2581", "=", "\u2581obj", ".", "toString", "();", "\u2581", "toString", "();"], - ids: [1, 3257, 260, 263, 260, 350, 50670, 261, 64577, 1994, 260, 64577, 1994, 2], - decoded: "[CLS] let a = obj.toString(); toString();[SEP]", - }, - NEWLINES: { - text: BASE_TEST_STRINGS.NEWLINES, - tokens: ["\u2581This", "\u2581is", "\u2581", "a", "\u2581test", "."], - ids: [1, 1495, 340, 260, 263, 2979, 261, 2], - decoded: "[CLS] This is a test.[SEP]", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["\u2581UN", "wan", "t\u00e9", "d", ",", "running"], - ids: [1, 10970, 3016, 3986, 286, 262, 170565, 2], - decoded: "[CLS] UNwant\u00e9d,running[SEP]", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u2581", "\u751f\u6d3b\u7684", "\u771f", "\u8c1b", "\u662f"], - ids: [1, 260, 197263, 7275, 241962, 1544, 2], - decoded: "[CLS] \u751f\u6d3b\u7684\u771f\u8c1b\u662f[SEP]", - }, - LEADING_SPACE: { - text: BASE_TEST_STRINGS.LEADING_SPACE, - tokens: ["\u2581", "leading", "\u2581space"], - ids: [1, 260, 22120, 11496, 2], - decoded: "[CLS] leading space[SEP]", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["\u2581trail", "ing", "\u2581space"], - ids: [1, 66699, 348, 11496, 2], - decoded: "[CLS] trailing space[SEP]", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["\u2581test", "\u2581$1", "\u2581R", "2", "\u2581#3", "\u2581\u20ac4", "\u2581\u00a35", "\u2581\u00a5", "6", "\u2581", "\u20a3", "7", "\u2581\u20b9", "8", "\u2581", "\u20b1", "9", "\u2581test"], - ids: [1, 2979, 21793, 532, 339, 19403, 157186, 156260, 33481, 452, 260, 242687, 488, 39568, 450, 260, 211232, 496, 2979, 2], - decoded: "[CLS] test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test[SEP]", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["\u2581I", "\u2581b", "ought", "\u2581an", "\u2581apple", "\u2581for", "\u2581$", "1.00", "\u2581at", "\u2581the", "\u2581store", "."], - ids: [1, 337, 331, 22280, 462, 44791, 333, 1161, 42645, 345, 288, 5318, 261, 2], - decoded: "[CLS] I bought an apple for $1.00 at the store.[SEP]", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - tokens: ["\u2581you", "..."], - ids: [1, 522, 303, 2], - decoded: "[CLS] you...[SEP]", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - tokens: ["\u2581you", "..."], - ids: [1, 522, 303, 2], - decoded: "[CLS] you...[SEP]", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - tokens: ["\u2581you", "...", "\u2581you", "..."], - ids: [1, 522, 303, 522, 303, 2], - decoded: "[CLS] you... you...[SEP]", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["\u2581w", "eird", "\u2581", "\uff5e", "\u2581edge", "\u2581", "\uff5e", "\u2581case"], - ids: [1, 415, 116640, 260, 2790, 53876, 260, 2790, 4073, 2], - decoded: "[CLS] weird \uff5e edge \uff5e case[SEP]", - }, - SPIECE_UNDERSCORE: { - text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, - tokens: ["\u2581This", "\u2581is", "\u2581", "a", "\u2581test", "\u2581", "."], - ids: [1, 1495, 340, 260, 263, 2979, 260, 261, 2], - decoded: "[CLS] This is a test.[SEP]", - }, - POPULAR_EMOJIS: { - text: BASE_TEST_STRINGS.POPULAR_EMOJIS, - tokens: ["\u2581", "\ud83d\ude02", "\u2581", "\ud83d\udc4d", "\u2581", "\ud83e\udd23", "\u2581", "\ud83d\ude0d", "\u2581", "\ud83d\ude2d", "\u2581", "\ud83c\udf89", "\u2581", "\ud83d\ude4f", "\u2581", "\ud83d\ude0a", "\u2581", "\ud83d\udd25", "\u2581", "\ud83d\ude01", "\u2581", "\ud83d\ude05", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\ude06", "\u2581", "\ud83d\udc4f", "\u2581\u2764", "\ufe0f", "\u2581", "\ud83d\udc9c", "\u2581", "\ud83d\udc9a", "\u2581", "\ud83d\udc97", "\u2581", "\ud83d\udc99", "\u2581", "\ud83d\udda4", "\u2581", "\ud83d\ude0e", "\u2581", "\ud83d\udc4c", "\u2581", "\ud83e\udd73", "\u2581", "\ud83d\udcaa", "\u2581", "\u2728", "\u2581\ud83d\udc49", "\u2581", "\ud83d\udc40", "\u2581", "\ud83d\udcaf", "\u2581", "\ud83c\udf88", "\u2581", "\ud83d\ude48", "\u2581", "\ud83d\ude4c", "\u2581", "\ud83d\udc80", "\u2581", "\ud83d\udc47", "\u2581", "\ud83d\udc4b", "\u2581\u2705", "\u2581", "\ud83c\udf81", "\u2581", "\ud83c\udf1e", "\u2581", "\ud83c\udf38", "\u2581", "\ud83d\udcb0"], - ids: [1, 260, 116844, 260, 72330, 260, 160951, 260, 78796, 260, 180546, 260, 212774, 260, 102930, 260, 71509, 260, 96089, 260, 137652, 260, 194608, 260, 182033, 260, 164467, 260, 149267, 56787, 4668, 260, 210251, 260, 195202, 260, 178523, 260, 167604, 260, 236081, 260, 157800, 260, 162843, 260, 242580, 260, 174590, 260, 65271, 113700, 260, 239652, 260, 237474, 260, 240937, 260, 239131, 260, 216701, 260, 242618, 260, 133395, 260, 240645, 82147, 260, 49599, 260, 239888, 260, 152102, 260, 239168, 2], - decoded: "[CLS] \ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0[SEP]", - }, - MULTIBYTE_EMOJIS: { - text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, - tokens: ["\u2581", "\u2728", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\udc41", "\ufe0f", "\u2581", "\ud83d\udc71", "\ud83c\udffb", "\u2581", "\ud83d\udd75", "\u2581", "\u2642", "\ufe0f", "\u2581", "\ud83e\uddd9", "\ud83c\udffb", "\u2581", "\u2642", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u2581", "\ud83c\udf3e", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83d\udc69", "\u2581\u2764", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc67", "\u2581", "\ud83d\udc66", "\u2581", "\ud83e\uddd1", "\ud83c\udffb", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1", "\ud83c\udffb", "\u2581", "\ud83c\udff4", "\udb40\udc67", "\udb40\udc62", "\udb40\udc65", "\udb40\udc6e", "\udb40\udc67", "\udb40\udc7f", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u2581\u2764", "\ufe0f", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\ud83c\udffc"], - ids: [1, 260, 65271, 260, 182033, 260, 16307, 4668, 260, 244774, 75846, 260, 247133, 260, 50622, 4668, 260, 3, 75846, 260, 50622, 260, 239432, 75846, 260, 243052, 260, 244250, 260, 243394, 260, 244250, 260, 239098, 56787, 260, 223802, 260, 239432, 260, 239098, 260, 239098, 260, 241727, 260, 242446, 260, 244250, 75846, 260, 243394, 260, 244250, 75846, 260, 244177, 245994, 247023, 248837, 248531, 245994, 245953, 260, 239432, 75846, 56787, 4668, 260, 223802, 260, 239432, 159667, 2], - decoded: "[CLS] \u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75 \u2642\ufe0f [UNK]\ud83c\udffb \u2642 \ud83d\udc68\ud83c\udffb \ud83c\udf3e \ud83e\uddd1 \ud83e\udd1d \ud83e\uddd1 \ud83d\udc69 \u2764 \ud83d\udc8b \ud83d\udc68 \ud83d\udc69 \ud83d\udc69 \ud83d\udc67 \ud83d\udc66 \ud83e\uddd1\ud83c\udffb \ud83e\udd1d \ud83e\uddd1\ud83c\udffb \ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f \ud83d\udc68\ud83c\udffb \u2764\ufe0f \ud83d\udc8b \ud83d\udc68\ud83c\udffc[SEP]", - }, - }, }; diff --git a/tests/models/distilbert/tokenization.js b/tests/models/distilbert/tokenization.js index 5fc1f3b93..7500e61ca 100644 --- a/tests/models/distilbert/tokenization.js +++ b/tests/models/distilbert/tokenization.js @@ -3,304 +3,304 @@ import { BASE_TEST_STRINGS } from "../test_strings.js"; export const TOKENIZER_CLASS = DistilBertTokenizer; export const TEST_CONFIG = { - "Xenova/distilbert-base-cased-distilled-squad": { - SIMPLE: { - text: BASE_TEST_STRINGS.SIMPLE, - tokens: ["How", "are", "you", "doing", "?"], - ids: [101, 1731, 1132, 1128, 1833, 136, 102], - decoded: "[CLS] How are you doing? [SEP]", + "Xenova/distilbert-base-cased-distilled-squad": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["How", "are", "you", "doing", "?"], + ids: [101, 1731, 1132, 1128, 1833, 136, 102], + decoded: "[CLS] How are you doing? [SEP]", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["You", "should", "'", "ve", "done", "this"], + ids: [101, 1192, 1431, 112, 1396, 1694, 1142, 102], + decoded: "[CLS] You should've done this [SEP]", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["01", "##23", "##45", "##6", "##7", "##8", "##9", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "100", "1000"], + ids: [101, 5187, 22737, 21336, 1545, 1559, 1604, 1580, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 1275, 1620, 6087, 102], + decoded: "[CLS] 0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000 [SEP]", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "company", "was", "founded", "in", "2016", "."], + ids: [101, 1109, 1419, 1108, 1771, 1107, 1446, 119, 102], + decoded: "[CLS] The company was founded in 2016. [SEP]", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "'", "ll", "!", "!", "to", "?", "'", "d", "'", "'", "d", "of", ",", "can", "'", "t", "."], + ids: [101, 138, 112, 1325, 106, 106, 1106, 136, 112, 173, 112, 112, 173, 1104, 117, 1169, 112, 189, 119, 102], + decoded: "[CLS] A'll!! to?'d'' d of, can't. [SEP]", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["def", "main", "(", ")", ":", "pass"], + ids: [101, 19353, 1514, 113, 114, 131, 2789, 102], + decoded: "[CLS] def main ( ) : pass [SEP]", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "a", "=", "o", "##b", "##j", ".", "to", "##S", "##tring", "(", ")", ";", "to", "##S", "##tring", "(", ")", ";"], + ids: [101, 1519, 170, 134, 184, 1830, 3361, 119, 1106, 1708, 28108, 113, 114, 132, 1106, 1708, 28108, 113, 114, 132, 102], + decoded: "[CLS] let a = obj. toString ( ) ; toString ( ) ; [SEP]", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["This", "is", "a", "test", "."], + ids: [101, 1188, 1110, 170, 2774, 119, 102], + decoded: "[CLS] This is a test. [SEP]", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "##wan", "##t\u00e9", "##d", ",", "running"], + ids: [101, 7414, 5491, 14608, 1181, 117, 1919, 102], + decoded: "[CLS] UNwant\u00e9d, running [SEP]", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["123"], + ids: [101, 13414, 102], + decoded: "[CLS] 123 [SEP]", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["Hello", "World"], + ids: [101, 8667, 1291, 102], + decoded: "[CLS] Hello World [SEP]", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["hello", "world"], + ids: [101, 19082, 1362, 102], + decoded: "[CLS] hello world [SEP]", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u751f", "[UNK]", "[UNK]", "\u771f", "[UNK]", "[UNK]"], + ids: [101, 1056, 100, 100, 1061, 100, 100, 102], + decoded: "[CLS] \u751f [UNK] [UNK] \u771f [UNK] [UNK] [SEP]", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["leading", "space"], + ids: [101, 2020, 2000, 102], + decoded: "[CLS] leading space [SEP]", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["trailing", "space"], + ids: [101, 13161, 2000, 102], + decoded: "[CLS] trailing space [SEP]", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["Hi", "Hello"], + ids: [101, 8790, 8667, 102], + decoded: "[CLS] Hi Hello [SEP]", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "$", "1", "R", "##2", "#", "3", "\u20ac", "##4", "\u00a3", "##5", "\u00a5", "##6", "[UNK]", "\u20b9", "##8", "\u20b1", "##9", "test"], + ids: [101, 2774, 109, 122, 155, 1477, 108, 124, 836, 1527, 202, 1571, 203, 1545, 100, 838, 1604, 837, 1580, 2774, 102], + decoded: "[CLS] test $ 1 R2 # 3 \u20ac4 \u00a35 \u00a56 [UNK] \u20b98 \u20b19 test [SEP]", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "bought", "an", "apple", "for", "$", "1", ".", "00", "at", "the", "store", "."], + ids: [101, 146, 3306, 1126, 12075, 1111, 109, 122, 119, 3135, 1120, 1103, 2984, 119, 102], + decoded: "[CLS] I bought an apple for $ 1. 00 at the store. [SEP]", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "\u2026"], + ids: [101, 1128, 795, 102], + decoded: "[CLS] you \u2026 [SEP]", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "\u2026"], + ids: [101, 1128, 795, 102], + decoded: "[CLS] you \u2026 [SEP]", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "\u2026", "you", "\u2026"], + ids: [101, 1128, 795, 1128, 795, 102], + decoded: "[CLS] you \u2026 you \u2026 [SEP]", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["weird", "[UNK]", "edge", "[UNK]", "case"], + ids: [101, 6994, 100, 2652, 100, 1692, 102], + decoded: "[CLS] weird [UNK] edge [UNK] case [SEP]", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "."], + ids: [101, 100, 100, 100, 100, 100, 119, 102], + decoded: "[CLS] [UNK] [UNK] [UNK] [UNK] [UNK]. [SEP]", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]"], + ids: [101, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 102], + decoded: "[CLS] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [SEP]", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]"], + ids: [101, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 102], + decoded: "[CLS] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [SEP]", + }, + }, + "Xenova/distilbert-base-uncased-finetuned-sst-2-english": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["how", "are", "you", "doing", "?"], + ids: [101, 2129, 2024, 2017, 2725, 1029, 102], + decoded: "[CLS] how are you doing? [SEP]", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["you", "should", "'", "ve", "done", "this"], + ids: [101, 2017, 2323, 1005, 2310, 2589, 2023, 102], + decoded: "[CLS] you should've done this [SEP]", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["the", "company", "was", "founded", "in", "2016", "."], + ids: [101, 1996, 2194, 2001, 2631, 1999, 2355, 1012, 102], + decoded: "[CLS] the company was founded in 2016. [SEP]", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["a", "'", "ll", "!", "!", "to", "?", "'", "d", "'", "'", "d", "of", ",", "can", "'", "t", "."], + ids: [101, 1037, 1005, 2222, 999, 999, 2000, 1029, 1005, 1040, 1005, 1005, 1040, 1997, 1010, 2064, 1005, 1056, 1012, 102], + decoded: "[CLS] a'll!! to?'d'' d of, can't. [SEP]", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "a", "=", "ob", "##j", ".", "to", "##st", "##ring", "(", ")", ";", "to", "##st", "##ring", "(", ")", ";"], + ids: [101, 2292, 1037, 1027, 27885, 3501, 1012, 2000, 3367, 4892, 1006, 1007, 1025, 2000, 3367, 4892, 1006, 1007, 1025, 102], + decoded: "[CLS] let a = obj. tostring ( ) ; tostring ( ) ; [SEP]", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["this", "is", "a", "test", "."], + ids: [101, 2023, 2003, 1037, 3231, 1012, 102], + decoded: "[CLS] this is a test. [SEP]", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["unwanted", ",", "running"], + ids: [101, 18162, 1010, 2770, 102], + decoded: "[CLS] unwanted, running [SEP]", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u751f", "[UNK]", "\u7684", "\u771f", "[UNK]", "[UNK]"], + ids: [101, 1910, 100, 1916, 1921, 100, 100, 102], + decoded: "[CLS] \u751f [UNK] \u7684 \u771f [UNK] [UNK] [SEP]", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["hi", "hello"], + ids: [101, 7632, 7592, 102], + decoded: "[CLS] hi hello [SEP]", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "$", "1", "r", "##2", "#", "3", "\u20ac", "##4", "\u00a35", "\u00a5", "##6", "[UNK]", "\u20b9", "##8", "\u20b1", "##9", "test"], + ids: [101, 3231, 1002, 1015, 1054, 2475, 1001, 1017, 1574, 2549, 27813, 1071, 2575, 100, 1576, 2620, 1575, 2683, 3231, 102], + decoded: "[CLS] test $ 1 r2 # 3 \u20ac4 \u00a35 \u00a56 [UNK] \u20b98 \u20b19 test [SEP]", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["i", "bought", "an", "apple", "for", "$", "1", ".", "00", "at", "the", "store", "."], + ids: [101, 1045, 4149, 2019, 6207, 2005, 1002, 1015, 1012, 4002, 2012, 1996, 3573, 1012, 102], + decoded: "[CLS] i bought an apple for $ 1. 00 at the store. [SEP]", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["weird", "\uff5e", "edge", "\uff5e", "case"], + ids: [101, 6881, 1995, 3341, 1995, 2553, 102], + decoded: "[CLS] weird \uff5e edge \uff5e case [SEP]", + }, + }, + "Xenova/distiluse-base-multilingual-cased-v2": { + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["012", "##34", "##5", "##6", "##7", "##8", "##9", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "100", "1000"], + ids: [101, 69878, 78301, 11166, 11211, 11305, 11396, 11373, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 10150, 10407, 12186, 102], + decoded: "[CLS] 0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000 [SEP]", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "a", "=", "ob", "##j", ".", "to", "##S", "##trin", "##g", "(", ")", ";", "to", "##S", "##trin", "##g", "(", ")", ";"], + ids: [101, 13595, 169, 134, 17339, 10418, 119, 10114, 10731, 109163, 10240, 113, 114, 132, 10114, 10731, 109163, 10240, 113, 114, 132, 102], + decoded: "[CLS] let a = obj. toString ( ) ; toString ( ) ; [SEP]", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "##want", "##\u00e9d", ",", "running"], + ids: [101, 26578, 104216, 84193, 117, 18020, 102], + decoded: "[CLS] UNwant\u00e9d, running [SEP]", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["hell", "##o", "world"], + ids: [101, 61694, 10133, 11356, 102], + decoded: "[CLS] hello world [SEP]", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u751f", "\u6d3b", "\u7684", "\u771f", "\u8c1b", "\u662f"], + ids: [101, 5600, 4978, 5718, 5769, 7378, 4380, 102], + decoded: "[CLS] \u751f \u6d3b \u7684 \u771f \u8c1b \u662f [SEP]", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["trail", "##ing", "space"], + ids: [101, 56559, 10230, 16199, 102], + decoded: "[CLS] trailing space [SEP]", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "$", "1", "R2", "#", "3", "\u20ac", "##4", "\u00a3", "##5", "\u00a5", "##6", "[UNK]", "\u20b9", "##8", "[UNK]", "test"], + ids: [101, 15839, 109, 122, 94000, 108, 124, 1775, 11011, 201, 11166, 202, 11211, 100, 1776, 11396, 100, 15839, 102], + decoded: "[CLS] test $ 1 R2 # 3 \u20ac4 \u00a35 \u00a56 [UNK] \u20b98 [UNK] test [SEP]", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "bought", "an", "app", "##le", "for", "$", "1", ".", "00", "at", "the", "store", "."], + ids: [101, 146, 28870, 10151, 72894, 10284, 10142, 109, 122, 119, 11025, 10160, 10105, 13708, 119, 102], + decoded: "[CLS] I bought an apple for $ 1. 00 at the store. [SEP]", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "[UNK]"], + ids: [101, 13028, 100, 102], + decoded: "[CLS] you [UNK] [SEP]", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "[UNK]"], + ids: [101, 13028, 100, 102], + decoded: "[CLS] you [UNK] [SEP]", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "[UNK]", "you", "[UNK]"], + ids: [101, 13028, 100, 13028, 100, 102], + decoded: "[CLS] you [UNK] you [UNK] [SEP]", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["wei", "##rd", "\uff5e", "edge", "\uff5e", "case"], + ids: [101, 86981, 12023, 10096, 30599, 10096, 13474, 102], + decoded: "[CLS] weird \uff5e edge \uff5e case [SEP]", + }, }, - SIMPLE_WITH_PUNCTUATION: { - text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, - tokens: ["You", "should", "'", "ve", "done", "this"], - ids: [101, 1192, 1431, 112, 1396, 1694, 1142, 102], - decoded: "[CLS] You should've done this [SEP]", - }, - NUMBERS: { - text: BASE_TEST_STRINGS.NUMBERS, - tokens: ["01", "##23", "##45", "##6", "##7", "##8", "##9", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "100", "1000"], - ids: [101, 5187, 22737, 21336, 1545, 1559, 1604, 1580, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 1275, 1620, 6087, 102], - decoded: "[CLS] 0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000 [SEP]", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["The", "company", "was", "founded", "in", "2016", "."], - ids: [101, 1109, 1419, 1108, 1771, 1107, 1446, 119, 102], - decoded: "[CLS] The company was founded in 2016. [SEP]", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["A", "'", "ll", "!", "!", "to", "?", "'", "d", "'", "'", "d", "of", ",", "can", "'", "t", "."], - ids: [101, 138, 112, 1325, 106, 106, 1106, 136, 112, 173, 112, 112, 173, 1104, 117, 1169, 112, 189, 119, 102], - decoded: "[CLS] A'll!! to?'d'' d of, can't. [SEP]", - }, - PYTHON_CODE: { - text: BASE_TEST_STRINGS.PYTHON_CODE, - tokens: ["def", "main", "(", ")", ":", "pass"], - ids: [101, 19353, 1514, 113, 114, 131, 2789, 102], - decoded: "[CLS] def main ( ) : pass [SEP]", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["let", "a", "=", "o", "##b", "##j", ".", "to", "##S", "##tring", "(", ")", ";", "to", "##S", "##tring", "(", ")", ";"], - ids: [101, 1519, 170, 134, 184, 1830, 3361, 119, 1106, 1708, 28108, 113, 114, 132, 1106, 1708, 28108, 113, 114, 132, 102], - decoded: "[CLS] let a = obj. toString ( ) ; toString ( ) ; [SEP]", - }, - NEWLINES: { - text: BASE_TEST_STRINGS.NEWLINES, - tokens: ["This", "is", "a", "test", "."], - ids: [101, 1188, 1110, 170, 2774, 119, 102], - decoded: "[CLS] This is a test. [SEP]", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["UN", "##wan", "##t\u00e9", "##d", ",", "running"], - ids: [101, 7414, 5491, 14608, 1181, 117, 1919, 102], - decoded: "[CLS] UNwant\u00e9d, running [SEP]", - }, - CONTROL_TOKENS: { - text: BASE_TEST_STRINGS.CONTROL_TOKENS, - tokens: ["123"], - ids: [101, 13414, 102], - decoded: "[CLS] 123 [SEP]", - }, - HELLO_WORLD_TITLECASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, - tokens: ["Hello", "World"], - ids: [101, 8667, 1291, 102], - decoded: "[CLS] Hello World [SEP]", - }, - HELLO_WORLD_LOWERCASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, - tokens: ["hello", "world"], - ids: [101, 19082, 1362, 102], - decoded: "[CLS] hello world [SEP]", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u751f", "[UNK]", "[UNK]", "\u771f", "[UNK]", "[UNK]"], - ids: [101, 1056, 100, 100, 1061, 100, 100, 102], - decoded: "[CLS] \u751f [UNK] [UNK] \u771f [UNK] [UNK] [SEP]", - }, - LEADING_SPACE: { - text: BASE_TEST_STRINGS.LEADING_SPACE, - tokens: ["leading", "space"], - ids: [101, 2020, 2000, 102], - decoded: "[CLS] leading space [SEP]", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["trailing", "space"], - ids: [101, 13161, 2000, 102], - decoded: "[CLS] trailing space [SEP]", - }, - DOUBLE_SPACE: { - text: BASE_TEST_STRINGS.DOUBLE_SPACE, - tokens: ["Hi", "Hello"], - ids: [101, 8790, 8667, 102], - decoded: "[CLS] Hi Hello [SEP]", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["test", "$", "1", "R", "##2", "#", "3", "\u20ac", "##4", "\u00a3", "##5", "\u00a5", "##6", "[UNK]", "\u20b9", "##8", "\u20b1", "##9", "test"], - ids: [101, 2774, 109, 122, 155, 1477, 108, 124, 836, 1527, 202, 1571, 203, 1545, 100, 838, 1604, 837, 1580, 2774, 102], - decoded: "[CLS] test $ 1 R2 # 3 \u20ac4 \u00a35 \u00a56 [UNK] \u20b98 \u20b19 test [SEP]", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["I", "bought", "an", "apple", "for", "$", "1", ".", "00", "at", "the", "store", "."], - ids: [101, 146, 3306, 1126, 12075, 1111, 109, 122, 119, 3135, 1120, 1103, 2984, 119, 102], - decoded: "[CLS] I bought an apple for $ 1. 00 at the store. [SEP]", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - tokens: ["you", "\u2026"], - ids: [101, 1128, 795, 102], - decoded: "[CLS] you \u2026 [SEP]", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - tokens: ["you", "\u2026"], - ids: [101, 1128, 795, 102], - decoded: "[CLS] you \u2026 [SEP]", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - tokens: ["you", "\u2026", "you", "\u2026"], - ids: [101, 1128, 795, 1128, 795, 102], - decoded: "[CLS] you \u2026 you \u2026 [SEP]", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["weird", "[UNK]", "edge", "[UNK]", "case"], - ids: [101, 6994, 100, 2652, 100, 1692, 102], - decoded: "[CLS] weird [UNK] edge [UNK] case [SEP]", - }, - SPIECE_UNDERSCORE: { - text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, - tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "."], - ids: [101, 100, 100, 100, 100, 100, 119, 102], - decoded: "[CLS] [UNK] [UNK] [UNK] [UNK] [UNK]. [SEP]", - }, - POPULAR_EMOJIS: { - text: BASE_TEST_STRINGS.POPULAR_EMOJIS, - tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]"], - ids: [101, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 102], - decoded: "[CLS] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [SEP]", - }, - MULTIBYTE_EMOJIS: { - text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, - tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]"], - ids: [101, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 102], - decoded: "[CLS] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [SEP]", - }, - }, - "Xenova/distilbert-base-uncased-finetuned-sst-2-english": { - SIMPLE: { - text: BASE_TEST_STRINGS.SIMPLE, - tokens: ["how", "are", "you", "doing", "?"], - ids: [101, 2129, 2024, 2017, 2725, 1029, 102], - decoded: "[CLS] how are you doing? [SEP]", - }, - SIMPLE_WITH_PUNCTUATION: { - text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, - tokens: ["you", "should", "'", "ve", "done", "this"], - ids: [101, 2017, 2323, 1005, 2310, 2589, 2023, 102], - decoded: "[CLS] you should've done this [SEP]", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["the", "company", "was", "founded", "in", "2016", "."], - ids: [101, 1996, 2194, 2001, 2631, 1999, 2355, 1012, 102], - decoded: "[CLS] the company was founded in 2016. [SEP]", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["a", "'", "ll", "!", "!", "to", "?", "'", "d", "'", "'", "d", "of", ",", "can", "'", "t", "."], - ids: [101, 1037, 1005, 2222, 999, 999, 2000, 1029, 1005, 1040, 1005, 1005, 1040, 1997, 1010, 2064, 1005, 1056, 1012, 102], - decoded: "[CLS] a'll!! to?'d'' d of, can't. [SEP]", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["let", "a", "=", "ob", "##j", ".", "to", "##st", "##ring", "(", ")", ";", "to", "##st", "##ring", "(", ")", ";"], - ids: [101, 2292, 1037, 1027, 27885, 3501, 1012, 2000, 3367, 4892, 1006, 1007, 1025, 2000, 3367, 4892, 1006, 1007, 1025, 102], - decoded: "[CLS] let a = obj. tostring ( ) ; tostring ( ) ; [SEP]", - }, - NEWLINES: { - text: BASE_TEST_STRINGS.NEWLINES, - tokens: ["this", "is", "a", "test", "."], - ids: [101, 2023, 2003, 1037, 3231, 1012, 102], - decoded: "[CLS] this is a test. [SEP]", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["unwanted", ",", "running"], - ids: [101, 18162, 1010, 2770, 102], - decoded: "[CLS] unwanted, running [SEP]", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u751f", "[UNK]", "\u7684", "\u771f", "[UNK]", "[UNK]"], - ids: [101, 1910, 100, 1916, 1921, 100, 100, 102], - decoded: "[CLS] \u751f [UNK] \u7684 \u771f [UNK] [UNK] [SEP]", - }, - DOUBLE_SPACE: { - text: BASE_TEST_STRINGS.DOUBLE_SPACE, - tokens: ["hi", "hello"], - ids: [101, 7632, 7592, 102], - decoded: "[CLS] hi hello [SEP]", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["test", "$", "1", "r", "##2", "#", "3", "\u20ac", "##4", "\u00a35", "\u00a5", "##6", "[UNK]", "\u20b9", "##8", "\u20b1", "##9", "test"], - ids: [101, 3231, 1002, 1015, 1054, 2475, 1001, 1017, 1574, 2549, 27813, 1071, 2575, 100, 1576, 2620, 1575, 2683, 3231, 102], - decoded: "[CLS] test $ 1 r2 # 3 \u20ac4 \u00a35 \u00a56 [UNK] \u20b98 \u20b19 test [SEP]", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["i", "bought", "an", "apple", "for", "$", "1", ".", "00", "at", "the", "store", "."], - ids: [101, 1045, 4149, 2019, 6207, 2005, 1002, 1015, 1012, 4002, 2012, 1996, 3573, 1012, 102], - decoded: "[CLS] i bought an apple for $ 1. 00 at the store. [SEP]", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["weird", "\uff5e", "edge", "\uff5e", "case"], - ids: [101, 6881, 1995, 3341, 1995, 2553, 102], - decoded: "[CLS] weird \uff5e edge \uff5e case [SEP]", - }, - }, - "Xenova/distiluse-base-multilingual-cased-v2": { - NUMBERS: { - text: BASE_TEST_STRINGS.NUMBERS, - tokens: ["012", "##34", "##5", "##6", "##7", "##8", "##9", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "100", "1000"], - ids: [101, 69878, 78301, 11166, 11211, 11305, 11396, 11373, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 10150, 10407, 12186, 102], - decoded: "[CLS] 0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000 [SEP]", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["let", "a", "=", "ob", "##j", ".", "to", "##S", "##trin", "##g", "(", ")", ";", "to", "##S", "##trin", "##g", "(", ")", ";"], - ids: [101, 13595, 169, 134, 17339, 10418, 119, 10114, 10731, 109163, 10240, 113, 114, 132, 10114, 10731, 109163, 10240, 113, 114, 132, 102], - decoded: "[CLS] let a = obj. toString ( ) ; toString ( ) ; [SEP]", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["UN", "##want", "##\u00e9d", ",", "running"], - ids: [101, 26578, 104216, 84193, 117, 18020, 102], - decoded: "[CLS] UNwant\u00e9d, running [SEP]", - }, - HELLO_WORLD_LOWERCASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, - tokens: ["hell", "##o", "world"], - ids: [101, 61694, 10133, 11356, 102], - decoded: "[CLS] hello world [SEP]", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u751f", "\u6d3b", "\u7684", "\u771f", "\u8c1b", "\u662f"], - ids: [101, 5600, 4978, 5718, 5769, 7378, 4380, 102], - decoded: "[CLS] \u751f \u6d3b \u7684 \u771f \u8c1b \u662f [SEP]", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["trail", "##ing", "space"], - ids: [101, 56559, 10230, 16199, 102], - decoded: "[CLS] trailing space [SEP]", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["test", "$", "1", "R2", "#", "3", "\u20ac", "##4", "\u00a3", "##5", "\u00a5", "##6", "[UNK]", "\u20b9", "##8", "[UNK]", "test"], - ids: [101, 15839, 109, 122, 94000, 108, 124, 1775, 11011, 201, 11166, 202, 11211, 100, 1776, 11396, 100, 15839, 102], - decoded: "[CLS] test $ 1 R2 # 3 \u20ac4 \u00a35 \u00a56 [UNK] \u20b98 [UNK] test [SEP]", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["I", "bought", "an", "app", "##le", "for", "$", "1", ".", "00", "at", "the", "store", "."], - ids: [101, 146, 28870, 10151, 72894, 10284, 10142, 109, 122, 119, 11025, 10160, 10105, 13708, 119, 102], - decoded: "[CLS] I bought an apple for $ 1. 00 at the store. [SEP]", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - tokens: ["you", "[UNK]"], - ids: [101, 13028, 100, 102], - decoded: "[CLS] you [UNK] [SEP]", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - tokens: ["you", "[UNK]"], - ids: [101, 13028, 100, 102], - decoded: "[CLS] you [UNK] [SEP]", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - tokens: ["you", "[UNK]", "you", "[UNK]"], - ids: [101, 13028, 100, 13028, 100, 102], - decoded: "[CLS] you [UNK] you [UNK] [SEP]", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["wei", "##rd", "\uff5e", "edge", "\uff5e", "case"], - ids: [101, 86981, 12023, 10096, 30599, 10096, 13474, 102], - decoded: "[CLS] weird \uff5e edge \uff5e case [SEP]", - }, - }, }; diff --git a/tests/models/esm/tokenization.js b/tests/models/esm/tokenization.js index c072d5251..fb0a8aa1b 100644 --- a/tests/models/esm/tokenization.js +++ b/tests/models/esm/tokenization.js @@ -3,320 +3,320 @@ import { BASE_TEST_STRINGS, ESM_TEST_STRINGS } from "../test_strings.js"; export const TOKENIZER_CLASS = EsmTokenizer; export const TEST_CONFIG = { - "Xenova/nucleotide-transformer-500m-human-ref": { - SIMPLE: { - text: BASE_TEST_STRINGS.SIMPLE, - // "tokens": ["How", "are", "you", "doing?"], - ids: [3, 0, 0, 0, 0], - decoded: " ", + "Xenova/nucleotide-transformer-500m-human-ref": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + // "tokens": ["How", "are", "you", "doing?"], + ids: [3, 0, 0, 0, 0], + decoded: " ", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + // "tokens": ["You", "should've", "done", "this"], + ids: [3, 0, 0, 0, 0], + decoded: " ", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + // "tokens": ["0123456789", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "100", "1000"], + ids: [3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + decoded: " ", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + // "tokens": ["T", "he", "company", "was", "founded", "in", "2016."], + ids: [3, 4101, 0, 0, 0, 0, 0, 0], + decoded: " T ", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + // "tokens": ["A", "'ll", "!!to?'d''d", "of,", "can't."], + ids: [3, 4100, 0, 0, 0, 0], + decoded: " A ", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + // "tokens": ["def", "main():", "pass"], + ids: [3, 0, 0, 0], + decoded: " ", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + // "tokens": ["let", "a", "=", "obj.toString();", "toString();"], + ids: [3, 0, 0, 0, 0, 0], + decoded: " ", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + // "tokens": ["T", "his", "is", "a", "test."], + ids: [3, 4101, 0, 0, 0, 0], + decoded: " T ", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + // "tokens": ["U", "N", "want\u00e9d,running"], + ids: [3, 0, 4104, 0], + decoded: " N ", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + // "tokens": ["1\u00002\ufffd3"], + ids: [3, 0], + decoded: " ", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + // "tokens": ["Hello", "World"], + ids: [3, 0, 0], + decoded: " ", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + // "tokens": ["hello", "world"], + ids: [3, 0, 0], + decoded: " ", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + // "tokens": ["\u751f\u6d3b\u7684\u771f\u8c1b\u662f"], + ids: [3, 0], + decoded: " ", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + // "tokens": ["leading", "space"], + ids: [3, 0, 0], + decoded: " ", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + // "tokens": ["trailing", "space"], + ids: [3, 0, 0], + decoded: " ", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + // "tokens": ["Hi", "Hello"], + ids: [3, 0, 0], + decoded: " ", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + // "tokens": ["test", "$1", "R2", "#3", "\u20ac4", "\u00a35", "\u00a56", "\u20a37", "\u20b98", "\u20b19", "test"], + ids: [3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + decoded: " ", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + // "tokens": ["I", "bought", "an", "apple", "for", "$1.00", "at", "the", "store."], + ids: [3, 0, 0, 0, 0, 0, 0, 0, 0, 0], + decoded: " ", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + // "tokens": ["you\u2026"], + ids: [3, 0], + decoded: " ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + // "tokens": ["you\u2026"], + ids: [3, 0], + decoded: " ", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + // "tokens": ["you\u2026", "you\u2026"], + ids: [3, 0, 0], + decoded: " ", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + // "tokens": ["weird", "\uff5e", "edge", "\uff5e", "case"], + ids: [3, 0, 0, 0, 0, 0], + decoded: " ", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + // "tokens": ["\u2581", "T", "his", "\u2581is", "\u2581a", "\u2581test", "\u2581."], + ids: [3, 0, 4101, 0, 0, 0, 0, 0], + decoded: " T ", + }, + SPECIAL_TOKENS: { + text: ESM_TEST_STRINGS.SPECIAL_TOKENS, + tokens: ["", "", "", "", "", ""], + ids: [3, 0, 1, 2, 3, 4105, 4106], + decoded: " ", + }, + PROTEIN_SEQUENCES_1: { + text: ESM_TEST_STRINGS.PROTEIN_SEQUENCES_1, + tokens: ["ATTCCG", "ATTCCG", "ATTCCG"], + ids: [3, 367, 367, 367], + decoded: " ATTCCG ATTCCG ATTCCG", + }, + PROTEIN_SEQUENCES_2: { + text: ESM_TEST_STRINGS.PROTEIN_SEQUENCES_2, + tokens: ["ATTTCT", "CTCTCT", "CTCTGA", "GATCGA", "TCGATC", "G", "A", "T"], + ids: [3, 349, 2461, 2464, 3184, 1738, 4103, 4100, 4101], + decoded: " ATTTCT CTCTCT CTCTGA GATCGA TCGATC G A T", + }, + }, + "Xenova/esm2_t12_35M_UR50D": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + // "tokens": ["H", "ow", "are", "you", "doing?"], + ids: [0, 21, 3, 3, 3, 3, 2], + decoded: " H ", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + // "tokens": ["Y", "ou", "should've", "done", "this"], + ids: [0, 19, 3, 3, 3, 3, 2], + decoded: " Y ", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + // "tokens": ["0123456789", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "100", "1000"], + ids: [0, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2], + decoded: " ", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + // "tokens": ["T", "he", "company", "was", "founded", "in", "2016", "."], + ids: [0, 11, 3, 3, 3, 3, 3, 3, 29, 2], + decoded: " T . ", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + // "tokens": ["A", "'ll", "!!to?'d''d", "of,", "can't", "."], + ids: [0, 5, 3, 3, 3, 3, 29, 2], + decoded: " A . ", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + // "tokens": ["def", "main():", "pass"], + ids: [0, 3, 3, 3, 2], + decoded: " ", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + // "tokens": ["let", "a", "=", "obj", ".", "to", "S", "tring();", "to", "S", "tring();"], + ids: [0, 3, 3, 3, 3, 29, 3, 8, 3, 3, 8, 3, 2], + decoded: " . S S ", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + // "tokens": ["T", "his", "is", "a", "test", "."], + ids: [0, 11, 3, 3, 3, 3, 29, 2], + decoded: " T . ", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + // "tokens": ["U", "N", "want\u00e9d,running"], + ids: [0, 26, 17, 3, 2], + decoded: " U N ", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + // "tokens": ["1\u00002\ufffd3"], + ids: [0, 3, 2], + decoded: " ", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + // "tokens": ["H", "ello", "W", "orld"], + ids: [0, 21, 3, 22, 3, 2], + decoded: " H W ", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + // "tokens": ["hello", "world"], + ids: [0, 3, 3, 2], + decoded: " ", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + // "tokens": ["\u751f\u6d3b\u7684\u771f\u8c1b\u662f"], + ids: [0, 3, 2], + decoded: " ", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + // "tokens": ["leading", "space"], + ids: [0, 3, 3, 2], + decoded: " ", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + // "tokens": ["trailing", "space"], + ids: [0, 3, 3, 2], + decoded: " ", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + // "tokens": ["H", "i", "H", "ello"], + ids: [0, 21, 3, 21, 3, 2], + decoded: " H H ", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + // "tokens": ["test", "$1", "R", "2", "#3", "\u20ac4", "\u00a35", "\u00a56", "\u20a37", "\u20b98", "\u20b19", "test"], + ids: [0, 3, 3, 10, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2], + decoded: " R ", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + // "tokens": ["I", "bought", "an", "apple", "for", "$1", ".", "00", "at", "the", "store", "."], + ids: [0, 12, 3, 3, 3, 3, 3, 29, 3, 3, 3, 3, 29, 2], + decoded: " I . . ", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + // "tokens": ["you\u2026"], + ids: [0, 3, 2], + decoded: " ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + // "tokens": ["you\u2026"], + ids: [0, 3, 2], + decoded: " ", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + // "tokens": ["you\u2026", "you\u2026"], + ids: [0, 3, 3, 2], + decoded: " ", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + // "tokens": ["weird", "\uff5e", "edge", "\uff5e", "case"], + ids: [0, 3, 3, 3, 3, 3, 2], + decoded: " ", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + // "tokens": ["\u2581", "T", "his", "\u2581is", "\u2581a", "\u2581test", "\u2581", "."], + ids: [0, 3, 11, 3, 3, 3, 3, 3, 29, 2], + decoded: " T . ", + }, + SPECIAL_TOKENS: { + text: ESM_TEST_STRINGS.SPECIAL_TOKENS, + // "tokens": ["", "", "", "", "", ""], + ids: [0, 3, 1, 32, 0, 2, 3, 2], + decoded: " ", + }, + PROTEIN_SEQUENCES_1: { + text: ESM_TEST_STRINGS.PROTEIN_SEQUENCES_1, + tokens: ["A", "T", "T", "C", "C", "G", "A", "T", "T", "C", "C", "G", "A", "T", "T", "C", "C", "G"], + ids: [0, 5, 11, 11, 23, 23, 6, 5, 11, 11, 23, 23, 6, 5, 11, 11, 23, 23, 6, 2], + decoded: " A T T C C G A T T C C G A T T C C G ", + }, + PROTEIN_SEQUENCES_2: { + text: ESM_TEST_STRINGS.PROTEIN_SEQUENCES_2, + tokens: ["A", "T", "T", "T", "C", "T", "C", "T", "C", "T", "C", "T", "C", "T", "C", "T", "G", "A", "G", "A", "T", "C", "G", "A", "T", "C", "G", "A", "T", "C", "G", "A", "T"], + ids: [0, 5, 11, 11, 11, 23, 11, 23, 11, 23, 11, 23, 11, 23, 11, 23, 11, 6, 5, 6, 5, 11, 23, 6, 5, 11, 23, 6, 5, 11, 23, 6, 5, 11, 2], + decoded: " A T T T C T C T C T C T C T C T G A G A T C G A T C G A T C G A T ", + }, }, - SIMPLE_WITH_PUNCTUATION: { - text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, - // "tokens": ["You", "should've", "done", "this"], - ids: [3, 0, 0, 0, 0], - decoded: " ", - }, - NUMBERS: { - text: BASE_TEST_STRINGS.NUMBERS, - // "tokens": ["0123456789", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "100", "1000"], - ids: [3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - decoded: " ", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - // "tokens": ["T", "he", "company", "was", "founded", "in", "2016."], - ids: [3, 4101, 0, 0, 0, 0, 0, 0], - decoded: " T ", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - // "tokens": ["A", "'ll", "!!to?'d''d", "of,", "can't."], - ids: [3, 4100, 0, 0, 0, 0], - decoded: " A ", - }, - PYTHON_CODE: { - text: BASE_TEST_STRINGS.PYTHON_CODE, - // "tokens": ["def", "main():", "pass"], - ids: [3, 0, 0, 0], - decoded: " ", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - // "tokens": ["let", "a", "=", "obj.toString();", "toString();"], - ids: [3, 0, 0, 0, 0, 0], - decoded: " ", - }, - NEWLINES: { - text: BASE_TEST_STRINGS.NEWLINES, - // "tokens": ["T", "his", "is", "a", "test."], - ids: [3, 4101, 0, 0, 0, 0], - decoded: " T ", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - // "tokens": ["U", "N", "want\u00e9d,running"], - ids: [3, 0, 4104, 0], - decoded: " N ", - }, - CONTROL_TOKENS: { - text: BASE_TEST_STRINGS.CONTROL_TOKENS, - // "tokens": ["1\u00002\ufffd3"], - ids: [3, 0], - decoded: " ", - }, - HELLO_WORLD_TITLECASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, - // "tokens": ["Hello", "World"], - ids: [3, 0, 0], - decoded: " ", - }, - HELLO_WORLD_LOWERCASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, - // "tokens": ["hello", "world"], - ids: [3, 0, 0], - decoded: " ", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - // "tokens": ["\u751f\u6d3b\u7684\u771f\u8c1b\u662f"], - ids: [3, 0], - decoded: " ", - }, - LEADING_SPACE: { - text: BASE_TEST_STRINGS.LEADING_SPACE, - // "tokens": ["leading", "space"], - ids: [3, 0, 0], - decoded: " ", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - // "tokens": ["trailing", "space"], - ids: [3, 0, 0], - decoded: " ", - }, - DOUBLE_SPACE: { - text: BASE_TEST_STRINGS.DOUBLE_SPACE, - // "tokens": ["Hi", "Hello"], - ids: [3, 0, 0], - decoded: " ", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - // "tokens": ["test", "$1", "R2", "#3", "\u20ac4", "\u00a35", "\u00a56", "\u20a37", "\u20b98", "\u20b19", "test"], - ids: [3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - decoded: " ", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - // "tokens": ["I", "bought", "an", "apple", "for", "$1.00", "at", "the", "store."], - ids: [3, 0, 0, 0, 0, 0, 0, 0, 0, 0], - decoded: " ", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - // "tokens": ["you\u2026"], - ids: [3, 0], - decoded: " ", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - // "tokens": ["you\u2026"], - ids: [3, 0], - decoded: " ", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - // "tokens": ["you\u2026", "you\u2026"], - ids: [3, 0, 0], - decoded: " ", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - // "tokens": ["weird", "\uff5e", "edge", "\uff5e", "case"], - ids: [3, 0, 0, 0, 0, 0], - decoded: " ", - }, - SPIECE_UNDERSCORE: { - text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, - // "tokens": ["\u2581", "T", "his", "\u2581is", "\u2581a", "\u2581test", "\u2581."], - ids: [3, 0, 4101, 0, 0, 0, 0, 0], - decoded: " T ", - }, - SPECIAL_TOKENS: { - text: ESM_TEST_STRINGS.SPECIAL_TOKENS, - tokens: ["", "", "", "", "", ""], - ids: [3, 0, 1, 2, 3, 4105, 4106], - decoded: " ", - }, - PROTEIN_SEQUENCES_1: { - text: ESM_TEST_STRINGS.PROTEIN_SEQUENCES_1, - tokens: ["ATTCCG", "ATTCCG", "ATTCCG"], - ids: [3, 367, 367, 367], - decoded: " ATTCCG ATTCCG ATTCCG", - }, - PROTEIN_SEQUENCES_2: { - text: ESM_TEST_STRINGS.PROTEIN_SEQUENCES_2, - tokens: ["ATTTCT", "CTCTCT", "CTCTGA", "GATCGA", "TCGATC", "G", "A", "T"], - ids: [3, 349, 2461, 2464, 3184, 1738, 4103, 4100, 4101], - decoded: " ATTTCT CTCTCT CTCTGA GATCGA TCGATC G A T", - }, - }, - "Xenova/esm2_t12_35M_UR50D": { - SIMPLE: { - text: BASE_TEST_STRINGS.SIMPLE, - // "tokens": ["H", "ow", "are", "you", "doing?"], - ids: [0, 21, 3, 3, 3, 3, 2], - decoded: " H ", - }, - SIMPLE_WITH_PUNCTUATION: { - text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, - // "tokens": ["Y", "ou", "should've", "done", "this"], - ids: [0, 19, 3, 3, 3, 3, 2], - decoded: " Y ", - }, - NUMBERS: { - text: BASE_TEST_STRINGS.NUMBERS, - // "tokens": ["0123456789", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "100", "1000"], - ids: [0, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2], - decoded: " ", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - // "tokens": ["T", "he", "company", "was", "founded", "in", "2016", "."], - ids: [0, 11, 3, 3, 3, 3, 3, 3, 29, 2], - decoded: " T . ", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - // "tokens": ["A", "'ll", "!!to?'d''d", "of,", "can't", "."], - ids: [0, 5, 3, 3, 3, 3, 29, 2], - decoded: " A . ", - }, - PYTHON_CODE: { - text: BASE_TEST_STRINGS.PYTHON_CODE, - // "tokens": ["def", "main():", "pass"], - ids: [0, 3, 3, 3, 2], - decoded: " ", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - // "tokens": ["let", "a", "=", "obj", ".", "to", "S", "tring();", "to", "S", "tring();"], - ids: [0, 3, 3, 3, 3, 29, 3, 8, 3, 3, 8, 3, 2], - decoded: " . S S ", - }, - NEWLINES: { - text: BASE_TEST_STRINGS.NEWLINES, - // "tokens": ["T", "his", "is", "a", "test", "."], - ids: [0, 11, 3, 3, 3, 3, 29, 2], - decoded: " T . ", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - // "tokens": ["U", "N", "want\u00e9d,running"], - ids: [0, 26, 17, 3, 2], - decoded: " U N ", - }, - CONTROL_TOKENS: { - text: BASE_TEST_STRINGS.CONTROL_TOKENS, - // "tokens": ["1\u00002\ufffd3"], - ids: [0, 3, 2], - decoded: " ", - }, - HELLO_WORLD_TITLECASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, - // "tokens": ["H", "ello", "W", "orld"], - ids: [0, 21, 3, 22, 3, 2], - decoded: " H W ", - }, - HELLO_WORLD_LOWERCASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, - // "tokens": ["hello", "world"], - ids: [0, 3, 3, 2], - decoded: " ", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - // "tokens": ["\u751f\u6d3b\u7684\u771f\u8c1b\u662f"], - ids: [0, 3, 2], - decoded: " ", - }, - LEADING_SPACE: { - text: BASE_TEST_STRINGS.LEADING_SPACE, - // "tokens": ["leading", "space"], - ids: [0, 3, 3, 2], - decoded: " ", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - // "tokens": ["trailing", "space"], - ids: [0, 3, 3, 2], - decoded: " ", - }, - DOUBLE_SPACE: { - text: BASE_TEST_STRINGS.DOUBLE_SPACE, - // "tokens": ["H", "i", "H", "ello"], - ids: [0, 21, 3, 21, 3, 2], - decoded: " H H ", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - // "tokens": ["test", "$1", "R", "2", "#3", "\u20ac4", "\u00a35", "\u00a56", "\u20a37", "\u20b98", "\u20b19", "test"], - ids: [0, 3, 3, 10, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2], - decoded: " R ", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - // "tokens": ["I", "bought", "an", "apple", "for", "$1", ".", "00", "at", "the", "store", "."], - ids: [0, 12, 3, 3, 3, 3, 3, 29, 3, 3, 3, 3, 29, 2], - decoded: " I . . ", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - // "tokens": ["you\u2026"], - ids: [0, 3, 2], - decoded: " ", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - // "tokens": ["you\u2026"], - ids: [0, 3, 2], - decoded: " ", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - // "tokens": ["you\u2026", "you\u2026"], - ids: [0, 3, 3, 2], - decoded: " ", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - // "tokens": ["weird", "\uff5e", "edge", "\uff5e", "case"], - ids: [0, 3, 3, 3, 3, 3, 2], - decoded: " ", - }, - SPIECE_UNDERSCORE: { - text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, - // "tokens": ["\u2581", "T", "his", "\u2581is", "\u2581a", "\u2581test", "\u2581", "."], - ids: [0, 3, 11, 3, 3, 3, 3, 3, 29, 2], - decoded: " T . ", - }, - SPECIAL_TOKENS: { - text: ESM_TEST_STRINGS.SPECIAL_TOKENS, - // "tokens": ["", "", "", "", "", ""], - ids: [0, 3, 1, 32, 0, 2, 3, 2], - decoded: " ", - }, - PROTEIN_SEQUENCES_1: { - text: ESM_TEST_STRINGS.PROTEIN_SEQUENCES_1, - tokens: ["A", "T", "T", "C", "C", "G", "A", "T", "T", "C", "C", "G", "A", "T", "T", "C", "C", "G"], - ids: [0, 5, 11, 11, 23, 23, 6, 5, 11, 11, 23, 23, 6, 5, 11, 11, 23, 23, 6, 2], - decoded: " A T T C C G A T T C C G A T T C C G ", - }, - PROTEIN_SEQUENCES_2: { - text: ESM_TEST_STRINGS.PROTEIN_SEQUENCES_2, - tokens: ["A", "T", "T", "T", "C", "T", "C", "T", "C", "T", "C", "T", "C", "T", "C", "T", "G", "A", "G", "A", "T", "C", "G", "A", "T", "C", "G", "A", "T", "C", "G", "A", "T"], - ids: [0, 5, 11, 11, 11, 23, 11, 23, 11, 23, 11, 23, 11, 23, 11, 23, 11, 6, 5, 6, 5, 11, 23, 6, 5, 11, 23, 6, 5, 11, 23, 6, 5, 11, 2], - decoded: " A T T T C T C T C T C T C T C T G A G A T C G A T C G A T C G A T ", - }, - }, }; diff --git a/tests/models/falcon/tokenization.js b/tests/models/falcon/tokenization.js index 9dc5827ce..024671e60 100644 --- a/tests/models/falcon/tokenization.js +++ b/tests/models/falcon/tokenization.js @@ -3,242 +3,242 @@ import { BASE_TEST_STRINGS, FALCON_TEST_STRINGS } from "../test_strings.js"; export const TOKENIZER_CLASS = FalconTokenizer; export const TEST_CONFIG = { - "tiiuae/falcon-7b": { - SIMPLE: { - text: BASE_TEST_STRINGS.SIMPLE, - tokens: ["How", "\u0120are", "\u0120you", "\u0120doing", "?"], - ids: [1830, 362, 299, 1836, 42], - decoded: "How are you doing?", + "tiiuae/falcon-7b": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["How", "\u0120are", "\u0120you", "\u0120doing", "?"], + ids: [1830, 362, 299, 1836, 42], + decoded: "How are you doing?", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["You", "\u0120should", "'", "ve", "\u0120done", "\u0120this"], + ids: [1357, 808, 18, 298, 1782, 414], + decoded: "You should've done this", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["012", "345", "678", "9", "\u0120", "0", "\u0120", "1", "\u0120", "2", "\u0120", "3", "\u0120", "4", "\u0120", "5", "\u0120", "6", "\u0120", "7", "\u0120", "8", "\u0120", "9", "\u0120", "10", "\u0120", "100", "\u0120", "100", "0"], + ids: [24445, 29094, 41583, 36, 204, 27, 204, 28, 204, 29, 204, 30, 204, 31, 204, 32, 204, 33, 204, 34, 204, 35, 204, 36, 204, 696, 204, 1425, 204, 1425, 27], + decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "\u0120company", "\u0120was", "\u0120founded", "\u0120in", "\u0120", "201", "6", "."], + ids: [487, 1438, 398, 9923, 272, 204, 626, 33, 25], + decoded: "The company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "\u010a", "'", "ll", "\u0120", "!!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'", "t", "."], + ids: [44, 193, 18, 567, 204, 1409, 534, 12493, 79, 7544, 79, 275, 23, 418, 18, 95, 25], + decoded: "A\n'll!!to?'d''d of, can't.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["def", "\u0120main", "():", "\u010a", "\u0109", "pass"], + ids: [3071, 1316, 13160, 193, 192, 5412], + decoded: "def main():\n\tpass", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "\u0120a", "\u0120", "=", "\u0120obj", ".", "toString", "();", "\u010a", "toString", "();"], + ids: [1025, 241, 204, 40, 13756, 25, 19409, 2032, 193, 19409, 2032], + decoded: "let a = obj.toString();\ntoString();", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["This", "\u010a", "\u010a", "is", "\u010a", "a", "\u010a", "test", "."], + ids: [1182, 193, 193, 259, 193, 76, 193, 4780, 25], + decoded: "This\n\nis\na\ntest.", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "want", "\u00c3\u00a9d", ",", "running"], + ids: [4000, 32108, 5706, 23, 27386], + decoded: "UNwant\u00e9d,running", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["1", "\u0100", "2", "\u00ef\u00bf", "\u00bd", "3"], + ids: [28, 186, 29, 13112, 133, 30], + decoded: "1\u00002\ufffd3", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["Hello", "\u0120World"], + ids: [9856, 2889], + decoded: "Hello World", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["hello", "\u0120world"], + ids: [30835, 1079], + decoded: "hello world", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u00e7\u0136\u0141\u00e6\u00b4\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"], + ids: [32725, 1105, 15498, 8061, 233, 2364], + decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u0120\u0120", "\u0120leading", "\u0120space"], + ids: [258, 3736, 2151], + decoded: " leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["tra", "iling", "\u0120space", "\u0120\u0120\u0120"], + ids: [9172, 4447, 2151, 466], + decoded: "trailing space ", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["Hi", "\u0120", "\u0120Hello"], + ids: [5516, 204, 23090], + decoded: "Hi Hello", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "\u0120", "$", "1", "\u0120R", "2", "\u0120", "#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2", "\u00a5", "6", "\u0120\u00e2\u0124", "\u00a3", "7", "\u0120\u00e2\u0124", "\u00b9", "8", "\u0120\u00e2\u0124", "\u00b1", "9", "\u0120test"], + ids: [4780, 204, 15, 28, 382, 29, 204, 14, 30, 6471, 31, 5131, 32, 3068, 110, 33, 25631, 108, 34, 25631, 129, 35, 25631, 121, 36, 1318], + decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "\u0120bought", "\u0120an", "\u0120apple", "\u0120for", "\u0120", "$", "1", ".", "00", "\u0120at", "\u0120the", "\u0120store", "."], + ids: [52, 5659, 267, 12381, 312, 204, 15, 28, 25, 527, 388, 248, 2946, 25], + decoded: "I bought an apple for $1.00 at the store.", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u0120\u0120"], + ids: [5667, 898, 258], + decoded: "you\u2026 ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], + ids: [5667, 898, 60482], + decoded: "you\u2026\u00a0\u00a0", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142", "\u00c2\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], + ids: [5667, 898, 4381, 4381, 5667, 898, 60482], + decoded: "you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["we", "ird", "\u0120", "\u00ef", "\u00bd", "\u0140", "\u0120edge", "\u0120", "\u00ef", "\u00bd", "\u0140", "\u0120case"], + ids: [698, 1505, 204, 181, 133, 236, 5753, 204, 181, 133, 236, 1494], + decoded: "weird \uff5e edge \uff5e case", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u00e2\u0138", "\u0123", "This", "\u0120\u00e2\u0138", "\u0123", "is", "\u0120\u00e2\u0138", "\u0123", "a", "\u0120\u00e2\u0138", "\u0123", "test", "\u0120\u00e2\u0138", "\u0123", "."], + ids: [13856, 207, 1182, 26607, 207, 259, 26607, 207, 76, 26607, 207, 4780, 26607, 207, 25], + decoded: "\u2581This \u2581is \u2581a \u2581test \u2581.", + }, + NUMBERS_SPLIT: { + text: FALCON_TEST_STRINGS.NUMBERS_SPLIT, + tokens: ["12", "\u0120and", "\u0120", "123", "\u0120and", "\u0120", "123", "4"], + ids: [928, 273, 204, 10963, 273, 204, 10963, 31], + decoded: "12 and 123 and 1234", + }, + }, + "tiiuae/falcon-rw-1b": { + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["You", "\u0120should", "'ve", "\u0120done", "\u0120this"], + ids: [1639, 815, 1053, 1760, 428], + decoded: "You should've done this", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["01", "23", "45", "67", "89", "\u01200", "\u01201", "\u01202", "\u01203", "\u01204", "\u01205", "\u01206", "\u01207", "\u01208", "\u01209", "\u012010", "\u0120100", "\u01201000"], + ids: [486, 1954, 2231, 3134, 4531, 657, 352, 362, 513, 604, 642, 718, 767, 807, 860, 838, 1802, 8576], + decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "\u0120company", "\u0120was", "\u0120founded", "\u0120in", "\u01202016", "."], + ids: [464, 1664, 373, 9393, 287, 1584, 13], + decoded: "The company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "\u010a", "'ll", "\u0120!!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'t", "."], + ids: [32, 198, 1183, 37867, 1462, 8348, 67, 7061, 67, 286, 11, 460, 470, 13], + decoded: "A\n'll!!to?'d''d of, can't.", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".", "to", "String", "();", "\u010a", "to", "String", "();"], + ids: [1616, 257, 796, 26181, 13, 1462, 10100, 9783, 198, 1462, 10100, 9783], + decoded: "let a = obj.toString();\ntoString();", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "want", "\u00c3\u00a9", "d", ",", "running"], + ids: [4944, 42949, 2634, 67, 11, 20270], + decoded: "UNwant\u00e9d,running", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["1", "\u0100", "2", "\u00ef\u00bf\u00bd", "3"], + ids: [16, 188, 17, 4210, 18], + decoded: "1\u00002\ufffd3", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u00e7\u0136\u0141", "\u00e6", "\u00b4", "\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e", "\u0141", "\u00e8", "\u00b0", "\u013d", "\u00e6\u013a\u00af"], + ids: [37955, 162, 112, 119, 21410, 40367, 253, 164, 108, 249, 42468], + decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u0120", "\u0120", "\u0120leading", "\u0120space"], + ids: [220, 220, 3756, 2272], + decoded: " leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["tra", "iling", "\u0120space", "\u0120", "\u0120", "\u0120"], + ids: [9535, 4386, 2272, 220, 220, 220], + decoded: "trailing space ", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2", "\u0124", "\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120test"], + ids: [9288, 720, 16, 371, 17, 1303, 18, 10432, 19, 4248, 20, 38221, 21, 2343, 224, 96, 22, 2343, 224, 117, 23, 2343, 224, 109, 24, 1332], + decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "\u0120bought", "\u0120an", "\u0120apple", "\u0120for", "\u0120$", "1", ".", "00", "\u0120at", "\u0120the", "\u0120store", "."], + ids: [40, 5839, 281, 17180, 329, 720, 16, 13, 405, 379, 262, 3650, 13], + decoded: "I bought an apple for $1.00 at the store.", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u0120", "\u0120"], + ids: [5832, 1399, 220, 220], + decoded: "you\u2026 ", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["we", "ird", "\u0120\u00ef", "\u00bd", "\u0140", "\u0120edge", "\u0120\u00ef", "\u00bd", "\u0140", "\u0120case"], + ids: [732, 1447, 27332, 121, 252, 5743, 27332, 121, 252, 1339], + decoded: "weird \uff5e edge \uff5e case", + }, + NUMBERS_SPLIT: { + text: FALCON_TEST_STRINGS.NUMBERS_SPLIT, + tokens: ["12", "\u0120and", "\u0120123", "\u0120and", "\u012012", "34"], + ids: [1065, 290, 17031, 290, 1105, 2682], + decoded: "12 and 123 and 1234", + }, }, - SIMPLE_WITH_PUNCTUATION: { - text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, - tokens: ["You", "\u0120should", "'", "ve", "\u0120done", "\u0120this"], - ids: [1357, 808, 18, 298, 1782, 414], - decoded: "You should've done this", - }, - NUMBERS: { - text: BASE_TEST_STRINGS.NUMBERS, - tokens: ["012", "345", "678", "9", "\u0120", "0", "\u0120", "1", "\u0120", "2", "\u0120", "3", "\u0120", "4", "\u0120", "5", "\u0120", "6", "\u0120", "7", "\u0120", "8", "\u0120", "9", "\u0120", "10", "\u0120", "100", "\u0120", "100", "0"], - ids: [24445, 29094, 41583, 36, 204, 27, 204, 28, 204, 29, 204, 30, 204, 31, 204, 32, 204, 33, 204, 34, 204, 35, 204, 36, 204, 696, 204, 1425, 204, 1425, 27], - decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["The", "\u0120company", "\u0120was", "\u0120founded", "\u0120in", "\u0120", "201", "6", "."], - ids: [487, 1438, 398, 9923, 272, 204, 626, 33, 25], - decoded: "The company was founded in 2016.", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["A", "\u010a", "'", "ll", "\u0120", "!!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'", "t", "."], - ids: [44, 193, 18, 567, 204, 1409, 534, 12493, 79, 7544, 79, 275, 23, 418, 18, 95, 25], - decoded: "A\n'll!!to?'d''d of, can't.", - }, - PYTHON_CODE: { - text: BASE_TEST_STRINGS.PYTHON_CODE, - tokens: ["def", "\u0120main", "():", "\u010a", "\u0109", "pass"], - ids: [3071, 1316, 13160, 193, 192, 5412], - decoded: "def main():\n\tpass", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["let", "\u0120a", "\u0120", "=", "\u0120obj", ".", "toString", "();", "\u010a", "toString", "();"], - ids: [1025, 241, 204, 40, 13756, 25, 19409, 2032, 193, 19409, 2032], - decoded: "let a = obj.toString();\ntoString();", - }, - NEWLINES: { - text: BASE_TEST_STRINGS.NEWLINES, - tokens: ["This", "\u010a", "\u010a", "is", "\u010a", "a", "\u010a", "test", "."], - ids: [1182, 193, 193, 259, 193, 76, 193, 4780, 25], - decoded: "This\n\nis\na\ntest.", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["UN", "want", "\u00c3\u00a9d", ",", "running"], - ids: [4000, 32108, 5706, 23, 27386], - decoded: "UNwant\u00e9d,running", - }, - CONTROL_TOKENS: { - text: BASE_TEST_STRINGS.CONTROL_TOKENS, - tokens: ["1", "\u0100", "2", "\u00ef\u00bf", "\u00bd", "3"], - ids: [28, 186, 29, 13112, 133, 30], - decoded: "1\u00002\ufffd3", - }, - HELLO_WORLD_TITLECASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, - tokens: ["Hello", "\u0120World"], - ids: [9856, 2889], - decoded: "Hello World", - }, - HELLO_WORLD_LOWERCASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, - tokens: ["hello", "\u0120world"], - ids: [30835, 1079], - decoded: "hello world", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u00e7\u0136\u0141\u00e6\u00b4\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"], - ids: [32725, 1105, 15498, 8061, 233, 2364], - decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", - }, - LEADING_SPACE: { - text: BASE_TEST_STRINGS.LEADING_SPACE, - tokens: ["\u0120\u0120", "\u0120leading", "\u0120space"], - ids: [258, 3736, 2151], - decoded: " leading space", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["tra", "iling", "\u0120space", "\u0120\u0120\u0120"], - ids: [9172, 4447, 2151, 466], - decoded: "trailing space ", - }, - DOUBLE_SPACE: { - text: BASE_TEST_STRINGS.DOUBLE_SPACE, - tokens: ["Hi", "\u0120", "\u0120Hello"], - ids: [5516, 204, 23090], - decoded: "Hi Hello", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["test", "\u0120", "$", "1", "\u0120R", "2", "\u0120", "#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2", "\u00a5", "6", "\u0120\u00e2\u0124", "\u00a3", "7", "\u0120\u00e2\u0124", "\u00b9", "8", "\u0120\u00e2\u0124", "\u00b1", "9", "\u0120test"], - ids: [4780, 204, 15, 28, 382, 29, 204, 14, 30, 6471, 31, 5131, 32, 3068, 110, 33, 25631, 108, 34, 25631, 129, 35, 25631, 121, 36, 1318], - decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["I", "\u0120bought", "\u0120an", "\u0120apple", "\u0120for", "\u0120", "$", "1", ".", "00", "\u0120at", "\u0120the", "\u0120store", "."], - ids: [52, 5659, 267, 12381, 312, 204, 15, 28, 25, 527, 388, 248, 2946, 25], - decoded: "I bought an apple for $1.00 at the store.", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - tokens: ["you", "\u00e2\u0122\u00a6", "\u0120\u0120"], - ids: [5667, 898, 258], - decoded: "you\u2026 ", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], - ids: [5667, 898, 60482], - decoded: "you\u2026\u00a0\u00a0", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142", "\u00c2\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], - ids: [5667, 898, 4381, 4381, 5667, 898, 60482], - decoded: "you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["we", "ird", "\u0120", "\u00ef", "\u00bd", "\u0140", "\u0120edge", "\u0120", "\u00ef", "\u00bd", "\u0140", "\u0120case"], - ids: [698, 1505, 204, 181, 133, 236, 5753, 204, 181, 133, 236, 1494], - decoded: "weird \uff5e edge \uff5e case", - }, - SPIECE_UNDERSCORE: { - text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, - tokens: ["\u00e2\u0138", "\u0123", "This", "\u0120\u00e2\u0138", "\u0123", "is", "\u0120\u00e2\u0138", "\u0123", "a", "\u0120\u00e2\u0138", "\u0123", "test", "\u0120\u00e2\u0138", "\u0123", "."], - ids: [13856, 207, 1182, 26607, 207, 259, 26607, 207, 76, 26607, 207, 4780, 26607, 207, 25], - decoded: "\u2581This \u2581is \u2581a \u2581test \u2581.", - }, - NUMBERS_SPLIT: { - text: FALCON_TEST_STRINGS.NUMBERS_SPLIT, - tokens: ["12", "\u0120and", "\u0120", "123", "\u0120and", "\u0120", "123", "4"], - ids: [928, 273, 204, 10963, 273, 204, 10963, 31], - decoded: "12 and 123 and 1234", - }, - }, - "tiiuae/falcon-rw-1b": { - SIMPLE_WITH_PUNCTUATION: { - text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, - tokens: ["You", "\u0120should", "'ve", "\u0120done", "\u0120this"], - ids: [1639, 815, 1053, 1760, 428], - decoded: "You should've done this", - }, - NUMBERS: { - text: BASE_TEST_STRINGS.NUMBERS, - tokens: ["01", "23", "45", "67", "89", "\u01200", "\u01201", "\u01202", "\u01203", "\u01204", "\u01205", "\u01206", "\u01207", "\u01208", "\u01209", "\u012010", "\u0120100", "\u01201000"], - ids: [486, 1954, 2231, 3134, 4531, 657, 352, 362, 513, 604, 642, 718, 767, 807, 860, 838, 1802, 8576], - decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["The", "\u0120company", "\u0120was", "\u0120founded", "\u0120in", "\u01202016", "."], - ids: [464, 1664, 373, 9393, 287, 1584, 13], - decoded: "The company was founded in 2016.", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["A", "\u010a", "'ll", "\u0120!!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'t", "."], - ids: [32, 198, 1183, 37867, 1462, 8348, 67, 7061, 67, 286, 11, 460, 470, 13], - decoded: "A\n'll!!to?'d''d of, can't.", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".", "to", "String", "();", "\u010a", "to", "String", "();"], - ids: [1616, 257, 796, 26181, 13, 1462, 10100, 9783, 198, 1462, 10100, 9783], - decoded: "let a = obj.toString();\ntoString();", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["UN", "want", "\u00c3\u00a9", "d", ",", "running"], - ids: [4944, 42949, 2634, 67, 11, 20270], - decoded: "UNwant\u00e9d,running", - }, - CONTROL_TOKENS: { - text: BASE_TEST_STRINGS.CONTROL_TOKENS, - tokens: ["1", "\u0100", "2", "\u00ef\u00bf\u00bd", "3"], - ids: [16, 188, 17, 4210, 18], - decoded: "1\u00002\ufffd3", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u00e7\u0136\u0141", "\u00e6", "\u00b4", "\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e", "\u0141", "\u00e8", "\u00b0", "\u013d", "\u00e6\u013a\u00af"], - ids: [37955, 162, 112, 119, 21410, 40367, 253, 164, 108, 249, 42468], - decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", - }, - LEADING_SPACE: { - text: BASE_TEST_STRINGS.LEADING_SPACE, - tokens: ["\u0120", "\u0120", "\u0120leading", "\u0120space"], - ids: [220, 220, 3756, 2272], - decoded: " leading space", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["tra", "iling", "\u0120space", "\u0120", "\u0120", "\u0120"], - ids: [9535, 4386, 2272, 220, 220, 220], - decoded: "trailing space ", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2", "\u0124", "\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120test"], - ids: [9288, 720, 16, 371, 17, 1303, 18, 10432, 19, 4248, 20, 38221, 21, 2343, 224, 96, 22, 2343, 224, 117, 23, 2343, 224, 109, 24, 1332], - decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["I", "\u0120bought", "\u0120an", "\u0120apple", "\u0120for", "\u0120$", "1", ".", "00", "\u0120at", "\u0120the", "\u0120store", "."], - ids: [40, 5839, 281, 17180, 329, 720, 16, 13, 405, 379, 262, 3650, 13], - decoded: "I bought an apple for $1.00 at the store.", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - tokens: ["you", "\u00e2\u0122\u00a6", "\u0120", "\u0120"], - ids: [5832, 1399, 220, 220], - decoded: "you\u2026 ", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["we", "ird", "\u0120\u00ef", "\u00bd", "\u0140", "\u0120edge", "\u0120\u00ef", "\u00bd", "\u0140", "\u0120case"], - ids: [732, 1447, 27332, 121, 252, 5743, 27332, 121, 252, 1339], - decoded: "weird \uff5e edge \uff5e case", - }, - NUMBERS_SPLIT: { - text: FALCON_TEST_STRINGS.NUMBERS_SPLIT, - tokens: ["12", "\u0120and", "\u0120123", "\u0120and", "\u012012", "34"], - ids: [1065, 290, 17031, 290, 1105, 2682], - decoded: "12 and 123 and 1234", - }, - }, }; diff --git a/tests/models/gemma/tokenization.js b/tests/models/gemma/tokenization.js index 2e6c4b3c0..82eb8ac60 100644 --- a/tests/models/gemma/tokenization.js +++ b/tests/models/gemma/tokenization.js @@ -3,229 +3,229 @@ import { BASE_TEST_STRINGS, LLAMA_TEST_STRINGS } from "../test_strings.js"; export const TOKENIZER_CLASS = GemmaTokenizer; export const TEST_CONFIG = { - // Xenova/gemma-tokenizer - "Xenova/gemma2-tokenizer": { - SIMPLE: { - text: BASE_TEST_STRINGS.SIMPLE, - tokens: ["How", "\u2581are", "\u2581you", "\u2581doing", "?"], - ids: [2, 2299, 708, 692, 3900, 235336], - decoded: "How are you doing?", + // Xenova/gemma-tokenizer + "Xenova/gemma2-tokenizer": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["How", "\u2581are", "\u2581you", "\u2581doing", "?"], + ids: [2, 2299, 708, 692, 3900, 235336], + decoded: "How are you doing?", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["You", "\u2581should", "'", "ve", "\u2581done", "\u2581this"], + ids: [2, 2045, 1412, 235303, 524, 3015, 736], + decoded: "You should've done this", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "\u2581", "0", "\u2581", "1", "\u2581", "2", "\u2581", "3", "\u2581", "4", "\u2581", "5", "\u2581", "6", "\u2581", "7", "\u2581", "8", "\u2581", "9", "\u2581", "1", "0", "\u2581", "1", "0", "0", "\u2581", "1", "0", "0", "0"], + ids: [2, 235276, 235274, 235284, 235304, 235310, 235308, 235318, 235324, 235321, 235315, 235248, 235276, 235248, 235274, 235248, 235284, 235248, 235304, 235248, 235310, 235248, 235308, 235248, 235318, 235248, 235324, 235248, 235321, 235248, 235315, 235248, 235274, 235276, 235248, 235274, 235276, 235276, 235248, 235274, 235276, 235276, 235276], + decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "\u2581company", "\u2581was", "\u2581founded", "\u2581in", "\u2581", "2", "0", "1", "6", "."], + ids: [2, 651, 3277, 729, 18942, 575, 235248, 235284, 235276, 235274, 235318, 235265], + decoded: "The company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "\n", "'", "ll", "\u2581!!", "to", "?'", "d", "''", "d", "\u2581of", ",", "\u2581can", "'", "t", "."], + ids: [2, 235280, 108, 235303, 529, 9063, 511, 18016, 235258, 3404, 235258, 576, 235269, 798, 235303, 235251, 235265], + decoded: "A\n'll !!to?'d''d of, can't.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["def", "\u2581main", "():", "\n", "\t", "pass"], + ids: [2, 1293, 1872, 4409, 108, 226, 3095], + decoded: "def main():\n\tpass", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "\u2581a", "\u2581=", "\u2581obj", ".", "toString", "();", "\n", "toString", "();"], + ids: [2, 1243, 476, 589, 6555, 235265, 7114, 821, 108, 7114, 821], + decoded: "let a = obj.toString();\ntoString();", + }, + NEWLINES: { + text: LLAMA_TEST_STRINGS.NEWLINES, + tokens: ["ax", "\n", "####", "\n", "boo"], + ids: [2, 1247, 108, 3308, 108, 31931], + decoded: "ax\n####\nboo", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "want", "\u00e9d", ",", "running"], + ids: [2, 2019, 29007, 45346, 235269, 23655], + decoded: "UNwant\u00e9d,running", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["1", "<0x00>", "2", "\ufffd", "3"], + ids: [2, 235274, 217, 235284, 236193, 235304], + decoded: "1\u00002\ufffd3", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["Hello", "\u2581World"], + ids: [2, 4521, 3855], + decoded: "Hello World", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["hello", "\u2581world"], + ids: [2, 17534, 2134], + decoded: "hello world", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u751f\u6d3b\u7684", "\u771f", "\u8c1b", "\u662f"], + ids: [2, 122182, 235710, 245467, 235427], + decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u2581\u2581\u2581", "leading", "\u2581space"], + ids: [2, 140, 26650, 3641], + decoded: " leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["trailing", "\u2581space", "\u2581\u2581\u2581"], + ids: [2, 100504, 3641, 140], + decoded: "trailing space ", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["Hi", "\u2581\u2581", "Hello"], + ids: [2, 2151, 139, 4521], + decoded: "Hi Hello", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "\u2581$", "1", "\u2581R", "2", "\u2581#", "3", "\u2581\u20ac", "4", "\u2581\u00a3", "5", "\u2581\u00a5", "6", "\u2581", "\u20a3", "7", "\u2581\u20b9", "8", "\u2581", "\u20b1", "9", "\u2581test"], + ids: [2, 2195, 697, 235274, 625, 235284, 1700, 235304, 8296, 235310, 5955, 235308, 74393, 235318, 235248, 252058, 235324, 56712, 235321, 235248, 243132, 235315, 2121], + decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "\u2581bought", "\u2581an", "\u2581apple", "\u2581for", "\u2581$", "1", ".", "0", "0", "\u2581at", "\u2581the", "\u2581store", "."], + ids: [2, 235285, 8989, 671, 15491, 604, 697, 235274, 235265, 235276, 235276, 696, 573, 4659, 235265], + decoded: "I bought an apple for $1.00 at the store.", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "\u2026", "\u2581\u2581"], + ids: [2, 4747, 235417, 139], + decoded: "you\u2026 ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "\u2026", "\u00a0\u00a0"], + ids: [2, 4747, 235417, 25445], + decoded: "you\u2026\u00a0\u00a0", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "\u2026", "\u00a0\u00a0", "you", "\u2026", "\u00a0\u00a0"], + ids: [2, 4747, 235417, 25445, 4747, 235417, 25445], + decoded: "you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["weird", "\u2581\uff5e", "\u2581edge", "\u2581\uff5e", "\u2581case"], + ids: [2, 102422, 134012, 8541, 134012, 2270], + decoded: "weird \uff5e edge \uff5e case", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u2581This", "\u2581\u2581", "is", "\u2581\u2581", "a", "\u2581\u2581", "test", "\u2581\u2581", "."], + ids: [2, 1417, 139, 502, 139, 235250, 139, 2195, 139, 235265], + decoded: " This is a test .", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\ud83d\ude02", "\u2581\ud83d\udc4d", "\u2581\ud83e\udd23", "\u2581\ud83d\ude0d", "\u2581\ud83d\ude2d", "\u2581\ud83c\udf89", "\u2581\ud83d\ude4f", "\u2581\ud83d\ude0a", "\u2581\ud83d\udd25", "\u2581\ud83d\ude01", "\u2581\ud83d\ude05", "\u2581\ud83e\udd17", "\u2581\ud83d\ude06", "\u2581\ud83d\udc4f", "\u2581\u2764\ufe0f", "\u2581\ud83d\udc9c", "\u2581\ud83d\udc9a", "\u2581\ud83d\udc97", "\u2581\ud83d\udc99", "\u2581\ud83d\udda4", "\u2581\ud83d\ude0e", "\u2581\ud83d\udc4c", "\u2581\ud83e\udd73", "\u2581\ud83d\udcaa", "\u2581\u2728", "\u2581\ud83d\udc49", "\u2581\ud83d\udc40", "\u2581\ud83d\udcaf", "\u2581", "\ud83c\udf88", "\u2581", "\ud83d\ude48", "\u2581\ud83d\ude4c", "\u2581\ud83d\udc80", "\u2581\ud83d\udc47", "\u2581\ud83d\udc4b", "\u2581\u2705", "\u2581", "\ud83c\udf81", "\u2581", "\ud83c\udf1e", "\u2581\ud83c\udf38", "\u2581", "\ud83d\udcb0"], + ids: [2, 236471, 38104, 55937, 46434, 55605, 160588, 68226, 44416, 72373, 70636, 75298, 156808, 120433, 104492, 35373, 131674, 191384, 204903, 146773, 166620, 87949, 83860, 211978, 142816, 64726, 166368, 108892, 174882, 235248, 242431, 235248, 241259, 134540, 106918, 154601, 169692, 92641, 235248, 241227, 235248, 241971, 233958, 235248, 241034], + decoded: "\ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u2728", "\u2581\ud83e\udd17", "\u2581", "\ud83d\udc41", "\ufe0f", "\u2581", "\ud83d\udc71", "\ud83c\udffb", "\u2581", "\ud83d\udd75", "\u200d\u2642\ufe0f", "\u2581", "\ud83e\uddd9", "\ud83c\udffb", "\u200d\u2642", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u200d", "\ud83c\udf3e", "\u2581", "\ud83e\uddd1", "\u200d", "\ud83e\udd1d", "\u200d", "\ud83e\uddd1", "\u2581", "\ud83d\udc69", "\u200d", "\u2764", "\u200d", "\ud83d\udc8b", "\u200d", "\ud83d\udc68", "\u2581", "\ud83d\udc69", "\u200d", "\ud83d\udc69", "\u200d", "\ud83d\udc67", "\u200d", "\ud83d\udc66", "\u2581", "\ud83e\uddd1", "\ud83c\udffb", "\u200d", "\ud83e\udd1d", "\u200d", "\ud83e\uddd1", "\ud83c\udffb", "\u2581", "\ud83c\udff4", "\udb40\udc67", "\udb40\udc62", "\udb40\udc65", "\udb40\udc6e", "\udb40\udc67", "\udb40\udc7f", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u200d", "\u2764\ufe0f", "\u200d", "\ud83d\udc8b", "\u200d", "\ud83d\udc68", "\ud83c\udffc"], + ids: [2, 236309, 156808, 235248, 241666, 235969, 235248, 247216, 237933, 235248, 246522, 68399, 235248, 246422, 237933, 63233, 235248, 241568, 237933, 235879, 244448, 235248, 243634, 235879, 241668, 235879, 243634, 235248, 241355, 235879, 236457, 235879, 240887, 235879, 241568, 235248, 241355, 235879, 241355, 235879, 244355, 235879, 244670, 235248, 243634, 237933, 235879, 241668, 235879, 243634, 237933, 235248, 244443, 246738, 247704, 250142, 250123, 246738, 247662, 235248, 241568, 237933, 235879, 16176, 235879, 240887, 235879, 241568, 238683], + decoded: "\u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75\u200d\u2642\ufe0f \ud83e\uddd9\ud83c\udffb\u200d\u2642 \ud83d\udc68\ud83c\udffb\u200d\ud83c\udf3e \ud83e\uddd1\u200d\ud83e\udd1d\u200d\ud83e\uddd1 \ud83d\udc69\u200d\u2764\u200d\ud83d\udc8b\u200d\ud83d\udc68 \ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66 \ud83e\uddd1\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c\udffb \ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f \ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c\udffc", + }, + BPE_SCORES_PRIORITY_1: { + text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_1, + tokens: ["grab", "bed"], + ids: [2, 59031, 2907], + decoded: "grabbed", + }, + BPE_SCORES_PRIORITY_2: { + text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_2, + tokens: ["\u2581grabbed"], + ids: [2, 41939], + decoded: " grabbed", + }, + BPE_SCORES_PRIORITY_3: { + text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_3, + tokens: ["\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581", "grab", "bed"], + ids: [2, 148, 59031, 2907], + decoded: " grabbed", + }, + NEWLINE: { + text: LLAMA_TEST_STRINGS.NEWLINE, + tokens: ["\n"], + ids: [2, 108], + decoded: "\n", + }, + NEWLINE_WITH_LEADING_SPACE: { + text: LLAMA_TEST_STRINGS.NEWLINE_WITH_LEADING_SPACE, + tokens: ["\u2581", "\n"], + ids: [2, 235248, 108], + decoded: " \n", + }, + TABS: { + text: LLAMA_TEST_STRINGS.TABS, + tokens: ["\t", "tabs", "\t\t\t\t", "out", "\u2581here"], + ids: [2, 226, 31973, 255971, 745, 1517], + decoded: "\ttabs\t\t\t\tout here", + }, + NEWLINE_AND_TAB: { + text: LLAMA_TEST_STRINGS.NEWLINE_AND_TAB, + tokens: ["\n", "\t", "\n"], + ids: [2, 108, 226, 108], + decoded: "\n\t\n", + }, + CHINESE_LETTER: { + text: LLAMA_TEST_STRINGS.CHINESE_LETTER, + tokens: ["\u9547"], + ids: [2, 237796], + decoded: "\u9547", + }, + EMOJIS_1: { + text: LLAMA_TEST_STRINGS.EMOJIS_1, + tokens: ["\ud83e\udd99"], + ids: [2, 250645], + decoded: "\ud83e\udd99", + }, + EMOJIS_2: { + text: LLAMA_TEST_STRINGS.EMOJIS_2, + tokens: ["\ud83e\udd99", "<0xEA>", "<0x99>", "<0x8A>"], + ids: [2, 250645, 451, 370, 355], + decoded: "\ud83e\udd99\ua64a", + }, + EMOJIS_3: { + text: LLAMA_TEST_STRINGS.EMOJIS_3, + tokens: ["<0xEA>", "<0x99>", "<0x8A>", "\ud83e\udd99"], + ids: [2, 451, 370, 355, 250645], + decoded: "\ua64a\ud83e\udd99", + }, + PARAGRAPH: { + text: LLAMA_TEST_STRINGS.PARAGRAPH, + tokens: ["The", "\u2581llama", "\u2581(/", "\u02c8", "l", "\u0251", "\u02d0", "m\u0259", "/;", "\u2581", "\ud83e\udd99", "Spanish", "\u2581pronunciation", ":", "\u2581[", "\u02c8", "\u028e", "ama", "])", "\u2581(", "Lama", "\u2581g", "lama", ")", "\u2581is", "\u2581a", "\u2581domesticated", "\u2581South", "\u2581American", "\u2581came", "lid", ",", "\u2581widely", "\u2581used", "\u2581as", "\u2581a", "\u2581meat", "\u2581and", "\u2581pack", "\u2581animal", "\u2581by", "\u2581Andean", "\u2581cultures", "\u2581since", "\u2581the", "\u2581Pre", "-", "Columb", "ian", "\u2581era", ".", "\u2581Lla", "mas", "\u2581are", "\u2581social", "\u2581animals", "\u2581and", "\u2581live", "\u2581with", "\u2581others", "\u2581as", "\u2581a", "\u2581herd", ".", "\u2581Their", "\u2581wool", "\u2581is", "\u2581soft", "\u2581and", "\u2581contains", "\u2581only", "\u2581a", "\u2581small", "\u2581amount", "\u2581of", "\u2581lan", "olin", ".[", "2", "]", "\u2581Lla", "mas", "\u2581can", "\u2581learn", "\u2581simple", "\u2581tasks", "\u2581after", "\u2581a", "\u2581few", "\u2581repetitions", ".", "\u2581When", "\u2581using", "\u2581a", "\u2581pack", ",", "\u2581they", "\u2581can", "\u2581carry", "\u2581about", "\u2581", "2", "5", "\u2581to", "\u2581", "3", "0", "%", "\u2581of", "\u2581their", "\u2581body", "\u2581weight", "\u2581for", "\u2581", "8", "\u2581to", "\u2581", "1", "3", "\u2581km", "\u2581(", "5", "\u2013", "8", "\u2581miles", ").[", "3", "]", "\u2581The", "\u2581name", "\u2581llama", "\u2581(", "in", "\u2581the", "\u2581past", "\u2581also", "\u2581spelled", '\u2581"', "lama", '"', "\u2581or", '\u2581"', "g", "lama", '")', "\u2581was", "\u2581adopted", "\u2581by", "\u2581European", "\u2581settlers", "\u2581from", "\u2581native", "\u2581Peru", "vi", "ans", ".[", "4", "]", "\u2581The", "\u2581ancestors", "\u2581of", "\u2581llamas", "\u2581are", "\u2581thought", "\u2581to", "\u2581have", "\u2581originated", "\u2581from", "\u2581the", "\u2581Great", "\u2581Plains", "\u2581of", "\u2581North", "\u2581America", "\u2581about", "\u2581", "4", "0", "\u2581million", "\u2581years", "\u2581ago", ",", "\u2581and", "\u2581subsequently", "\u2581migrated", "\u2581to", "\u2581South", "\u2581America", "\u2581about", "\u2581three", "\u2581million", "\u2581years", "\u2581ago", "\u2581during", "\u2581the", "\u2581Great", "\u2581American", "\u2581Interchange", ".", "\u2581By", "\u2581the", "\u2581end", "\u2581of", "\u2581the", "\u2581last", "\u2581ice", "\u2581age", "\u2581(", "1", "0", ",", "0", "0", "0", "\u2013", "1", "2", ",", "0", "0", "0", "\u2581years", "\u2581ago", "),", "\u2581came", "lids", "\u2581were", "\u2581extinct", "\u2581in", "\u2581North", "\u2581America", ".[", "3", "]", "\u2581As", "\u2581of", "\u2581", "2", "0", "0", "7", ",", "\u2581there", "\u2581were", "\u2581over", "\u2581seven", "\u2581million", "\u2581llamas", "\u2581and", "\u2581al", "pac", "as", "\u2581in", "\u2581South", "\u2581America", "\u2581and", "\u2581over", "\u2581", "1", "5", "8", ",", "0", "0", "0", "\u2581llamas", "\u2581and", "\u2581", "1", "0", "0", ",", "0", "0", "0", "<0xEA>", "<0x99>", "<0x8A>", "\ud83e\udd99", "\u2581al", "pac", "as", ",", "\u2581descended", "\u2581from", "\u2581progen", "itors", "\u2581imported", "\u2581late", "\u2581in", "\u2581the", "\u2581", "2", "0", "th", "\u2581century", ",", "\u2581in", "\u2581the", "\u2581United", "\u2581States", "\u2581and", "\u2581Canada", ".[", "5", "]", "\u2581In", "\u2581A", "ym", "ara", "\u2581mythology", ",", "\u2581llamas", "\u2581are", "\u2581important", "\u2581beings", ".", "\u2581The", "\u2581Heavenly", "\u2581Llama", "\u2581is", "\u2581said", "\u2581to", "\u2581drink", "\u2581water", "\u2581from", "\u2581the", "\u2581ocean", "\u2581and", "\u2581urin", "ates", "\u2581as", "\u2581it", "\u2581rains", ".[", "6", "]", "\u2581According", "\u2581to", "\u2581A", "ym", "ara", "\u2581es", "ch", "atology", ",", "\u2581llamas", "\u2581will", "\u2581return", "\u2581to", "\u2581the", "\u2581water", "\u2581springs", "\u2581and", "\u2581lagoons", "\u2581where", "\u2581they", "\u2581come", "\u2581from", "\u2581at", "\u2581the", "\u2581end", "\u2581of", "\u2581time", ".[", "6", "]"], + ids: [2, 651, 19001, 101949, 239229, 235257, 240527, 240342, 128631, 102430, 235248, 250645, 51590, 74569, 235292, 892, 239229, 246752, 2867, 3013, 591, 221520, 583, 10450, 235275, 603, 476, 183304, 4316, 3725, 3392, 3353, 235269, 16714, 1671, 685, 476, 11827, 578, 3386, 8205, 731, 207552, 24541, 2754, 573, 2769, 235290, 222963, 1282, 6063, 235265, 172809, 2616, 708, 3127, 8398, 578, 3685, 675, 3588, 685, 476, 48010, 235265, 10368, 23834, 603, 4072, 578, 7744, 1297, 476, 2301, 3619, 576, 7607, 28424, 19047, 235284, 235307, 172809, 2616, 798, 3918, 3890, 13333, 1452, 476, 2619, 126286, 235265, 3194, 2177, 476, 3386, 235269, 984, 798, 6383, 1105, 235248, 235284, 235308, 577, 235248, 235304, 235276, 235358, 576, 1024, 2971, 5171, 604, 235248, 235321, 577, 235248, 235274, 235304, 5821, 591, 235308, 235389, 235321, 7112, 232524, 235304, 235307, 714, 1503, 19001, 591, 473, 573, 3433, 1170, 73003, 664, 10450, 235281, 689, 664, 235264, 10450, 1388, 729, 13861, 731, 7737, 57710, 774, 11634, 30160, 893, 779, 19047, 235310, 235307, 714, 44106, 576, 129953, 708, 3421, 577, 791, 52102, 774, 573, 6553, 55118, 576, 4612, 5783, 1105, 235248, 235310, 235276, 4416, 1658, 3958, 235269, 578, 27956, 106398, 577, 4316, 5783, 1105, 2149, 4416, 1658, 3958, 2290, 573, 6553, 3725, 193879, 235265, 3339, 573, 1580, 576, 573, 2001, 8357, 3911, 591, 235274, 235276, 235269, 235276, 235276, 235276, 235389, 235274, 235284, 235269, 235276, 235276, 235276, 1658, 3958, 823, 3392, 41253, 1049, 78561, 575, 4612, 5783, 19047, 235304, 235307, 1877, 576, 235248, 235284, 235276, 235276, 235324, 235269, 1104, 1049, 1163, 6861, 4416, 129953, 578, 717, 23337, 508, 575, 4316, 5783, 578, 1163, 235248, 235274, 235308, 235321, 235269, 235276, 235276, 235276, 129953, 578, 235248, 235274, 235276, 235276, 235269, 235276, 235276, 235276, 451, 370, 355, 250645, 717, 23337, 508, 235269, 64700, 774, 66279, 15517, 29271, 5245, 575, 573, 235248, 235284, 235276, 489, 7861, 235269, 575, 573, 3520, 3858, 578, 6591, 19047, 235308, 235307, 878, 586, 3985, 1610, 76701, 235269, 129953, 708, 2845, 27290, 235265, 714, 89830, 170669, 603, 1180, 577, 7182, 2003, 774, 573, 13940, 578, 111204, 1204, 685, 665, 50852, 19047, 235318, 235307, 11926, 577, 586, 3985, 1610, 875, 530, 92764, 235269, 129953, 877, 2203, 577, 573, 2003, 31104, 578, 221493, 1570, 984, 2063, 774, 696, 573, 1580, 576, 1069, 19047, 235318, 235307], + decoded: 'The llama (/\u02c8l\u0251\u02d0m\u0259/; \ud83e\udd99Spanish pronunciation: [\u02c8\u028eama]) (Lama glama) is a domesticated South American camelid, widely used as a meat and pack animal by Andean cultures since the Pre-Columbian era. Llamas are social animals and live with others as a herd. Their wool is soft and contains only a small amount of lanolin.[2] Llamas can learn simple tasks after a few repetitions. When using a pack, they can carry about 25 to 30% of their body weight for 8 to 13 km (5\u20138 miles).[3] The name llama (in the past also spelled "lama" or "glama") was adopted by European settlers from native Peruvians.[4] The ancestors of llamas are thought to have originated from the Great Plains of North America about 40 million years ago, and subsequently migrated to South America about three million years ago during the Great American Interchange. By the end of the last ice age (10,000\u201312,000 years ago), camelids were extinct in North America.[3] As of 2007, there were over seven million llamas and alpacas in South America and over 158,000 llamas and 100,000\ua64a\ud83e\udd99 alpacas, descended from progenitors imported late in the 20th century, in the United States and Canada.[5] In Aymara mythology, llamas are important beings. The Heavenly Llama is said to drink water from the ocean and urinates as it rains.[6] According to Aymara eschatology, llamas will return to the water springs and lagoons where they come from at the end of time.[6]', + }, }, - SIMPLE_WITH_PUNCTUATION: { - text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, - tokens: ["You", "\u2581should", "'", "ve", "\u2581done", "\u2581this"], - ids: [2, 2045, 1412, 235303, 524, 3015, 736], - decoded: "You should've done this", - }, - NUMBERS: { - text: BASE_TEST_STRINGS.NUMBERS, - tokens: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "\u2581", "0", "\u2581", "1", "\u2581", "2", "\u2581", "3", "\u2581", "4", "\u2581", "5", "\u2581", "6", "\u2581", "7", "\u2581", "8", "\u2581", "9", "\u2581", "1", "0", "\u2581", "1", "0", "0", "\u2581", "1", "0", "0", "0"], - ids: [2, 235276, 235274, 235284, 235304, 235310, 235308, 235318, 235324, 235321, 235315, 235248, 235276, 235248, 235274, 235248, 235284, 235248, 235304, 235248, 235310, 235248, 235308, 235248, 235318, 235248, 235324, 235248, 235321, 235248, 235315, 235248, 235274, 235276, 235248, 235274, 235276, 235276, 235248, 235274, 235276, 235276, 235276], - decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["The", "\u2581company", "\u2581was", "\u2581founded", "\u2581in", "\u2581", "2", "0", "1", "6", "."], - ids: [2, 651, 3277, 729, 18942, 575, 235248, 235284, 235276, 235274, 235318, 235265], - decoded: "The company was founded in 2016.", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["A", "\n", "'", "ll", "\u2581!!", "to", "?'", "d", "''", "d", "\u2581of", ",", "\u2581can", "'", "t", "."], - ids: [2, 235280, 108, 235303, 529, 9063, 511, 18016, 235258, 3404, 235258, 576, 235269, 798, 235303, 235251, 235265], - decoded: "A\n'll !!to?'d''d of, can't.", - }, - PYTHON_CODE: { - text: BASE_TEST_STRINGS.PYTHON_CODE, - tokens: ["def", "\u2581main", "():", "\n", "\t", "pass"], - ids: [2, 1293, 1872, 4409, 108, 226, 3095], - decoded: "def main():\n\tpass", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["let", "\u2581a", "\u2581=", "\u2581obj", ".", "toString", "();", "\n", "toString", "();"], - ids: [2, 1243, 476, 589, 6555, 235265, 7114, 821, 108, 7114, 821], - decoded: "let a = obj.toString();\ntoString();", - }, - NEWLINES: { - text: LLAMA_TEST_STRINGS.NEWLINES, - tokens: ["ax", "\n", "####", "\n", "boo"], - ids: [2, 1247, 108, 3308, 108, 31931], - decoded: "ax\n####\nboo", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["UN", "want", "\u00e9d", ",", "running"], - ids: [2, 2019, 29007, 45346, 235269, 23655], - decoded: "UNwant\u00e9d,running", - }, - CONTROL_TOKENS: { - text: BASE_TEST_STRINGS.CONTROL_TOKENS, - tokens: ["1", "<0x00>", "2", "\ufffd", "3"], - ids: [2, 235274, 217, 235284, 236193, 235304], - decoded: "1\u00002\ufffd3", - }, - HELLO_WORLD_TITLECASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, - tokens: ["Hello", "\u2581World"], - ids: [2, 4521, 3855], - decoded: "Hello World", - }, - HELLO_WORLD_LOWERCASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, - tokens: ["hello", "\u2581world"], - ids: [2, 17534, 2134], - decoded: "hello world", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u751f\u6d3b\u7684", "\u771f", "\u8c1b", "\u662f"], - ids: [2, 122182, 235710, 245467, 235427], - decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", - }, - LEADING_SPACE: { - text: BASE_TEST_STRINGS.LEADING_SPACE, - tokens: ["\u2581\u2581\u2581", "leading", "\u2581space"], - ids: [2, 140, 26650, 3641], - decoded: " leading space", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["trailing", "\u2581space", "\u2581\u2581\u2581"], - ids: [2, 100504, 3641, 140], - decoded: "trailing space ", - }, - DOUBLE_SPACE: { - text: BASE_TEST_STRINGS.DOUBLE_SPACE, - tokens: ["Hi", "\u2581\u2581", "Hello"], - ids: [2, 2151, 139, 4521], - decoded: "Hi Hello", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["test", "\u2581$", "1", "\u2581R", "2", "\u2581#", "3", "\u2581\u20ac", "4", "\u2581\u00a3", "5", "\u2581\u00a5", "6", "\u2581", "\u20a3", "7", "\u2581\u20b9", "8", "\u2581", "\u20b1", "9", "\u2581test"], - ids: [2, 2195, 697, 235274, 625, 235284, 1700, 235304, 8296, 235310, 5955, 235308, 74393, 235318, 235248, 252058, 235324, 56712, 235321, 235248, 243132, 235315, 2121], - decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["I", "\u2581bought", "\u2581an", "\u2581apple", "\u2581for", "\u2581$", "1", ".", "0", "0", "\u2581at", "\u2581the", "\u2581store", "."], - ids: [2, 235285, 8989, 671, 15491, 604, 697, 235274, 235265, 235276, 235276, 696, 573, 4659, 235265], - decoded: "I bought an apple for $1.00 at the store.", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - tokens: ["you", "\u2026", "\u2581\u2581"], - ids: [2, 4747, 235417, 139], - decoded: "you\u2026 ", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - tokens: ["you", "\u2026", "\u00a0\u00a0"], - ids: [2, 4747, 235417, 25445], - decoded: "you\u2026\u00a0\u00a0", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - tokens: ["you", "\u2026", "\u00a0\u00a0", "you", "\u2026", "\u00a0\u00a0"], - ids: [2, 4747, 235417, 25445, 4747, 235417, 25445], - decoded: "you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["weird", "\u2581\uff5e", "\u2581edge", "\u2581\uff5e", "\u2581case"], - ids: [2, 102422, 134012, 8541, 134012, 2270], - decoded: "weird \uff5e edge \uff5e case", - }, - SPIECE_UNDERSCORE: { - text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, - tokens: ["\u2581This", "\u2581\u2581", "is", "\u2581\u2581", "a", "\u2581\u2581", "test", "\u2581\u2581", "."], - ids: [2, 1417, 139, 502, 139, 235250, 139, 2195, 139, 235265], - decoded: " This is a test .", - }, - POPULAR_EMOJIS: { - text: BASE_TEST_STRINGS.POPULAR_EMOJIS, - tokens: ["\ud83d\ude02", "\u2581\ud83d\udc4d", "\u2581\ud83e\udd23", "\u2581\ud83d\ude0d", "\u2581\ud83d\ude2d", "\u2581\ud83c\udf89", "\u2581\ud83d\ude4f", "\u2581\ud83d\ude0a", "\u2581\ud83d\udd25", "\u2581\ud83d\ude01", "\u2581\ud83d\ude05", "\u2581\ud83e\udd17", "\u2581\ud83d\ude06", "\u2581\ud83d\udc4f", "\u2581\u2764\ufe0f", "\u2581\ud83d\udc9c", "\u2581\ud83d\udc9a", "\u2581\ud83d\udc97", "\u2581\ud83d\udc99", "\u2581\ud83d\udda4", "\u2581\ud83d\ude0e", "\u2581\ud83d\udc4c", "\u2581\ud83e\udd73", "\u2581\ud83d\udcaa", "\u2581\u2728", "\u2581\ud83d\udc49", "\u2581\ud83d\udc40", "\u2581\ud83d\udcaf", "\u2581", "\ud83c\udf88", "\u2581", "\ud83d\ude48", "\u2581\ud83d\ude4c", "\u2581\ud83d\udc80", "\u2581\ud83d\udc47", "\u2581\ud83d\udc4b", "\u2581\u2705", "\u2581", "\ud83c\udf81", "\u2581", "\ud83c\udf1e", "\u2581\ud83c\udf38", "\u2581", "\ud83d\udcb0"], - ids: [2, 236471, 38104, 55937, 46434, 55605, 160588, 68226, 44416, 72373, 70636, 75298, 156808, 120433, 104492, 35373, 131674, 191384, 204903, 146773, 166620, 87949, 83860, 211978, 142816, 64726, 166368, 108892, 174882, 235248, 242431, 235248, 241259, 134540, 106918, 154601, 169692, 92641, 235248, 241227, 235248, 241971, 233958, 235248, 241034], - decoded: "\ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0", - }, - MULTIBYTE_EMOJIS: { - text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, - tokens: ["\u2728", "\u2581\ud83e\udd17", "\u2581", "\ud83d\udc41", "\ufe0f", "\u2581", "\ud83d\udc71", "\ud83c\udffb", "\u2581", "\ud83d\udd75", "\u200d\u2642\ufe0f", "\u2581", "\ud83e\uddd9", "\ud83c\udffb", "\u200d\u2642", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u200d", "\ud83c\udf3e", "\u2581", "\ud83e\uddd1", "\u200d", "\ud83e\udd1d", "\u200d", "\ud83e\uddd1", "\u2581", "\ud83d\udc69", "\u200d", "\u2764", "\u200d", "\ud83d\udc8b", "\u200d", "\ud83d\udc68", "\u2581", "\ud83d\udc69", "\u200d", "\ud83d\udc69", "\u200d", "\ud83d\udc67", "\u200d", "\ud83d\udc66", "\u2581", "\ud83e\uddd1", "\ud83c\udffb", "\u200d", "\ud83e\udd1d", "\u200d", "\ud83e\uddd1", "\ud83c\udffb", "\u2581", "\ud83c\udff4", "\udb40\udc67", "\udb40\udc62", "\udb40\udc65", "\udb40\udc6e", "\udb40\udc67", "\udb40\udc7f", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u200d", "\u2764\ufe0f", "\u200d", "\ud83d\udc8b", "\u200d", "\ud83d\udc68", "\ud83c\udffc"], - ids: [2, 236309, 156808, 235248, 241666, 235969, 235248, 247216, 237933, 235248, 246522, 68399, 235248, 246422, 237933, 63233, 235248, 241568, 237933, 235879, 244448, 235248, 243634, 235879, 241668, 235879, 243634, 235248, 241355, 235879, 236457, 235879, 240887, 235879, 241568, 235248, 241355, 235879, 241355, 235879, 244355, 235879, 244670, 235248, 243634, 237933, 235879, 241668, 235879, 243634, 237933, 235248, 244443, 246738, 247704, 250142, 250123, 246738, 247662, 235248, 241568, 237933, 235879, 16176, 235879, 240887, 235879, 241568, 238683], - decoded: "\u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75\u200d\u2642\ufe0f \ud83e\uddd9\ud83c\udffb\u200d\u2642 \ud83d\udc68\ud83c\udffb\u200d\ud83c\udf3e \ud83e\uddd1\u200d\ud83e\udd1d\u200d\ud83e\uddd1 \ud83d\udc69\u200d\u2764\u200d\ud83d\udc8b\u200d\ud83d\udc68 \ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66 \ud83e\uddd1\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c\udffb \ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f \ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c\udffc", - }, - BPE_SCORES_PRIORITY_1: { - text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_1, - tokens: ["grab", "bed"], - ids: [2, 59031, 2907], - decoded: "grabbed", - }, - BPE_SCORES_PRIORITY_2: { - text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_2, - tokens: ["\u2581grabbed"], - ids: [2, 41939], - decoded: " grabbed", - }, - BPE_SCORES_PRIORITY_3: { - text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_3, - tokens: ["\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581", "grab", "bed"], - ids: [2, 148, 59031, 2907], - decoded: " grabbed", - }, - NEWLINE: { - text: LLAMA_TEST_STRINGS.NEWLINE, - tokens: ["\n"], - ids: [2, 108], - decoded: "\n", - }, - NEWLINE_WITH_LEADING_SPACE: { - text: LLAMA_TEST_STRINGS.NEWLINE_WITH_LEADING_SPACE, - tokens: ["\u2581", "\n"], - ids: [2, 235248, 108], - decoded: " \n", - }, - TABS: { - text: LLAMA_TEST_STRINGS.TABS, - tokens: ["\t", "tabs", "\t\t\t\t", "out", "\u2581here"], - ids: [2, 226, 31973, 255971, 745, 1517], - decoded: "\ttabs\t\t\t\tout here", - }, - NEWLINE_AND_TAB: { - text: LLAMA_TEST_STRINGS.NEWLINE_AND_TAB, - tokens: ["\n", "\t", "\n"], - ids: [2, 108, 226, 108], - decoded: "\n\t\n", - }, - CHINESE_LETTER: { - text: LLAMA_TEST_STRINGS.CHINESE_LETTER, - tokens: ["\u9547"], - ids: [2, 237796], - decoded: "\u9547", - }, - EMOJIS_1: { - text: LLAMA_TEST_STRINGS.EMOJIS_1, - tokens: ["\ud83e\udd99"], - ids: [2, 250645], - decoded: "\ud83e\udd99", - }, - EMOJIS_2: { - text: LLAMA_TEST_STRINGS.EMOJIS_2, - tokens: ["\ud83e\udd99", "<0xEA>", "<0x99>", "<0x8A>"], - ids: [2, 250645, 451, 370, 355], - decoded: "\ud83e\udd99\ua64a", - }, - EMOJIS_3: { - text: LLAMA_TEST_STRINGS.EMOJIS_3, - tokens: ["<0xEA>", "<0x99>", "<0x8A>", "\ud83e\udd99"], - ids: [2, 451, 370, 355, 250645], - decoded: "\ua64a\ud83e\udd99", - }, - PARAGRAPH: { - text: LLAMA_TEST_STRINGS.PARAGRAPH, - tokens: ["The", "\u2581llama", "\u2581(/", "\u02c8", "l", "\u0251", "\u02d0", "m\u0259", "/;", "\u2581", "\ud83e\udd99", "Spanish", "\u2581pronunciation", ":", "\u2581[", "\u02c8", "\u028e", "ama", "])", "\u2581(", "Lama", "\u2581g", "lama", ")", "\u2581is", "\u2581a", "\u2581domesticated", "\u2581South", "\u2581American", "\u2581came", "lid", ",", "\u2581widely", "\u2581used", "\u2581as", "\u2581a", "\u2581meat", "\u2581and", "\u2581pack", "\u2581animal", "\u2581by", "\u2581Andean", "\u2581cultures", "\u2581since", "\u2581the", "\u2581Pre", "-", "Columb", "ian", "\u2581era", ".", "\u2581Lla", "mas", "\u2581are", "\u2581social", "\u2581animals", "\u2581and", "\u2581live", "\u2581with", "\u2581others", "\u2581as", "\u2581a", "\u2581herd", ".", "\u2581Their", "\u2581wool", "\u2581is", "\u2581soft", "\u2581and", "\u2581contains", "\u2581only", "\u2581a", "\u2581small", "\u2581amount", "\u2581of", "\u2581lan", "olin", ".[", "2", "]", "\u2581Lla", "mas", "\u2581can", "\u2581learn", "\u2581simple", "\u2581tasks", "\u2581after", "\u2581a", "\u2581few", "\u2581repetitions", ".", "\u2581When", "\u2581using", "\u2581a", "\u2581pack", ",", "\u2581they", "\u2581can", "\u2581carry", "\u2581about", "\u2581", "2", "5", "\u2581to", "\u2581", "3", "0", "%", "\u2581of", "\u2581their", "\u2581body", "\u2581weight", "\u2581for", "\u2581", "8", "\u2581to", "\u2581", "1", "3", "\u2581km", "\u2581(", "5", "\u2013", "8", "\u2581miles", ").[", "3", "]", "\u2581The", "\u2581name", "\u2581llama", "\u2581(", "in", "\u2581the", "\u2581past", "\u2581also", "\u2581spelled", '\u2581"', "lama", '"', "\u2581or", '\u2581"', "g", "lama", '")', "\u2581was", "\u2581adopted", "\u2581by", "\u2581European", "\u2581settlers", "\u2581from", "\u2581native", "\u2581Peru", "vi", "ans", ".[", "4", "]", "\u2581The", "\u2581ancestors", "\u2581of", "\u2581llamas", "\u2581are", "\u2581thought", "\u2581to", "\u2581have", "\u2581originated", "\u2581from", "\u2581the", "\u2581Great", "\u2581Plains", "\u2581of", "\u2581North", "\u2581America", "\u2581about", "\u2581", "4", "0", "\u2581million", "\u2581years", "\u2581ago", ",", "\u2581and", "\u2581subsequently", "\u2581migrated", "\u2581to", "\u2581South", "\u2581America", "\u2581about", "\u2581three", "\u2581million", "\u2581years", "\u2581ago", "\u2581during", "\u2581the", "\u2581Great", "\u2581American", "\u2581Interchange", ".", "\u2581By", "\u2581the", "\u2581end", "\u2581of", "\u2581the", "\u2581last", "\u2581ice", "\u2581age", "\u2581(", "1", "0", ",", "0", "0", "0", "\u2013", "1", "2", ",", "0", "0", "0", "\u2581years", "\u2581ago", "),", "\u2581came", "lids", "\u2581were", "\u2581extinct", "\u2581in", "\u2581North", "\u2581America", ".[", "3", "]", "\u2581As", "\u2581of", "\u2581", "2", "0", "0", "7", ",", "\u2581there", "\u2581were", "\u2581over", "\u2581seven", "\u2581million", "\u2581llamas", "\u2581and", "\u2581al", "pac", "as", "\u2581in", "\u2581South", "\u2581America", "\u2581and", "\u2581over", "\u2581", "1", "5", "8", ",", "0", "0", "0", "\u2581llamas", "\u2581and", "\u2581", "1", "0", "0", ",", "0", "0", "0", "<0xEA>", "<0x99>", "<0x8A>", "\ud83e\udd99", "\u2581al", "pac", "as", ",", "\u2581descended", "\u2581from", "\u2581progen", "itors", "\u2581imported", "\u2581late", "\u2581in", "\u2581the", "\u2581", "2", "0", "th", "\u2581century", ",", "\u2581in", "\u2581the", "\u2581United", "\u2581States", "\u2581and", "\u2581Canada", ".[", "5", "]", "\u2581In", "\u2581A", "ym", "ara", "\u2581mythology", ",", "\u2581llamas", "\u2581are", "\u2581important", "\u2581beings", ".", "\u2581The", "\u2581Heavenly", "\u2581Llama", "\u2581is", "\u2581said", "\u2581to", "\u2581drink", "\u2581water", "\u2581from", "\u2581the", "\u2581ocean", "\u2581and", "\u2581urin", "ates", "\u2581as", "\u2581it", "\u2581rains", ".[", "6", "]", "\u2581According", "\u2581to", "\u2581A", "ym", "ara", "\u2581es", "ch", "atology", ",", "\u2581llamas", "\u2581will", "\u2581return", "\u2581to", "\u2581the", "\u2581water", "\u2581springs", "\u2581and", "\u2581lagoons", "\u2581where", "\u2581they", "\u2581come", "\u2581from", "\u2581at", "\u2581the", "\u2581end", "\u2581of", "\u2581time", ".[", "6", "]"], - ids: [2, 651, 19001, 101949, 239229, 235257, 240527, 240342, 128631, 102430, 235248, 250645, 51590, 74569, 235292, 892, 239229, 246752, 2867, 3013, 591, 221520, 583, 10450, 235275, 603, 476, 183304, 4316, 3725, 3392, 3353, 235269, 16714, 1671, 685, 476, 11827, 578, 3386, 8205, 731, 207552, 24541, 2754, 573, 2769, 235290, 222963, 1282, 6063, 235265, 172809, 2616, 708, 3127, 8398, 578, 3685, 675, 3588, 685, 476, 48010, 235265, 10368, 23834, 603, 4072, 578, 7744, 1297, 476, 2301, 3619, 576, 7607, 28424, 19047, 235284, 235307, 172809, 2616, 798, 3918, 3890, 13333, 1452, 476, 2619, 126286, 235265, 3194, 2177, 476, 3386, 235269, 984, 798, 6383, 1105, 235248, 235284, 235308, 577, 235248, 235304, 235276, 235358, 576, 1024, 2971, 5171, 604, 235248, 235321, 577, 235248, 235274, 235304, 5821, 591, 235308, 235389, 235321, 7112, 232524, 235304, 235307, 714, 1503, 19001, 591, 473, 573, 3433, 1170, 73003, 664, 10450, 235281, 689, 664, 235264, 10450, 1388, 729, 13861, 731, 7737, 57710, 774, 11634, 30160, 893, 779, 19047, 235310, 235307, 714, 44106, 576, 129953, 708, 3421, 577, 791, 52102, 774, 573, 6553, 55118, 576, 4612, 5783, 1105, 235248, 235310, 235276, 4416, 1658, 3958, 235269, 578, 27956, 106398, 577, 4316, 5783, 1105, 2149, 4416, 1658, 3958, 2290, 573, 6553, 3725, 193879, 235265, 3339, 573, 1580, 576, 573, 2001, 8357, 3911, 591, 235274, 235276, 235269, 235276, 235276, 235276, 235389, 235274, 235284, 235269, 235276, 235276, 235276, 1658, 3958, 823, 3392, 41253, 1049, 78561, 575, 4612, 5783, 19047, 235304, 235307, 1877, 576, 235248, 235284, 235276, 235276, 235324, 235269, 1104, 1049, 1163, 6861, 4416, 129953, 578, 717, 23337, 508, 575, 4316, 5783, 578, 1163, 235248, 235274, 235308, 235321, 235269, 235276, 235276, 235276, 129953, 578, 235248, 235274, 235276, 235276, 235269, 235276, 235276, 235276, 451, 370, 355, 250645, 717, 23337, 508, 235269, 64700, 774, 66279, 15517, 29271, 5245, 575, 573, 235248, 235284, 235276, 489, 7861, 235269, 575, 573, 3520, 3858, 578, 6591, 19047, 235308, 235307, 878, 586, 3985, 1610, 76701, 235269, 129953, 708, 2845, 27290, 235265, 714, 89830, 170669, 603, 1180, 577, 7182, 2003, 774, 573, 13940, 578, 111204, 1204, 685, 665, 50852, 19047, 235318, 235307, 11926, 577, 586, 3985, 1610, 875, 530, 92764, 235269, 129953, 877, 2203, 577, 573, 2003, 31104, 578, 221493, 1570, 984, 2063, 774, 696, 573, 1580, 576, 1069, 19047, 235318, 235307], - decoded: 'The llama (/\u02c8l\u0251\u02d0m\u0259/; \ud83e\udd99Spanish pronunciation: [\u02c8\u028eama]) (Lama glama) is a domesticated South American camelid, widely used as a meat and pack animal by Andean cultures since the Pre-Columbian era. Llamas are social animals and live with others as a herd. Their wool is soft and contains only a small amount of lanolin.[2] Llamas can learn simple tasks after a few repetitions. When using a pack, they can carry about 25 to 30% of their body weight for 8 to 13 km (5\u20138 miles).[3] The name llama (in the past also spelled "lama" or "glama") was adopted by European settlers from native Peruvians.[4] The ancestors of llamas are thought to have originated from the Great Plains of North America about 40 million years ago, and subsequently migrated to South America about three million years ago during the Great American Interchange. By the end of the last ice age (10,000\u201312,000 years ago), camelids were extinct in North America.[3] As of 2007, there were over seven million llamas and alpacas in South America and over 158,000 llamas and 100,000\ua64a\ud83e\udd99 alpacas, descended from progenitors imported late in the 20th century, in the United States and Canada.[5] In Aymara mythology, llamas are important beings. The Heavenly Llama is said to drink water from the ocean and urinates as it rains.[6] According to Aymara eschatology, llamas will return to the water springs and lagoons where they come from at the end of time.[6]', - }, - }, }; diff --git a/tests/models/gpt2/tokenization.js b/tests/models/gpt2/tokenization.js index c573e7b5c..534a700f3 100644 --- a/tests/models/gpt2/tokenization.js +++ b/tests/models/gpt2/tokenization.js @@ -3,460 +3,460 @@ import { BASE_TEST_STRINGS, SENTENCEPIECE_TEST_STRINGS } from "../test_strings.j export const TOKENIZER_CLASS = GPT2Tokenizer; export const TEST_CONFIG = { - // - clean_up_tokenization_spaces=true - // - default pretokenization regex - "Xenova/gpt2": { - SIMPLE: { - text: BASE_TEST_STRINGS.SIMPLE, - tokens: ["How", "\u0120are", "\u0120you", "\u0120doing", "?"], - ids: [2437, 389, 345, 1804, 30], - decoded: "How are you doing?", + // - clean_up_tokenization_spaces=true + // - default pretokenization regex + "Xenova/gpt2": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["How", "\u0120are", "\u0120you", "\u0120doing", "?"], + ids: [2437, 389, 345, 1804, 30], + decoded: "How are you doing?", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["You", "\u0120should", "'ve", "\u0120done", "\u0120this"], + ids: [1639, 815, 1053, 1760, 428], + decoded: "You should've done this", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["01", "23", "45", "67", "89", "Ġ0", "Ġ1", "Ġ2", "Ġ3", "Ġ4", "Ġ5", "Ġ6", "Ġ7", "Ġ8", "Ġ9", "Ġ10", "Ġ100", "Ġ1000"], + ids: [486, 1954, 2231, 3134, 4531, 657, 352, 362, 513, 604, 642, 718, 767, 807, 860, 838, 1802, 8576], + decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "\u0120company", "\u0120was", "\u0120founded", "\u0120in", "\u01202016", "."], + ids: [464, 1664, 373, 9393, 287, 1584, 13], + decoded: "The company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "\u010a", "'ll", "\u0120!!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'t", "."], + ids: [32, 198, 1183, 37867, 1462, 8348, 67, 7061, 67, 286, 11, 460, 470, 13], + decoded: "A\n'll!!to?'d''d of, can't.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["def", "\u0120main", "():", "\u010a", "\u0109", "pass"], + ids: [4299, 1388, 33529, 198, 197, 6603], + decoded: "def main():\n\tpass", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".", "to", "String", "();", "\u010a", "to", "String", "();"], + ids: [1616, 257, 796, 26181, 13, 1462, 10100, 9783, 198, 1462, 10100, 9783], + decoded: "let a = obj.toString();\ntoString();", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["This", "\u010a", "\u010a", "is", "\u010a", "a", "\u010a", "test", "."], + ids: [1212, 198, 198, 271, 198, 64, 198, 9288, 13], + decoded: "This\n\nis\na\ntest.", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "want", "\u00c3\u00a9", "d", ",", "running"], + ids: [4944, 42949, 2634, 67, 11, 20270], + decoded: "UNwant\u00e9d,running", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["1", "\u0100", "2", "\u00ef\u00bf\u00bd", "3"], + ids: [16, 188, 17, 4210, 18], + decoded: "1\u00002\ufffd3", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["Hello", "\u0120World"], + ids: [15496, 2159], + decoded: "Hello World", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["hello", "\u0120world"], + ids: [31373, 995], + decoded: "hello world", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u00e7\u0136\u0141", "\u00e6", "\u00b4", "\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e", "\u0141", "\u00e8", "\u00b0", "\u013d", "\u00e6\u013a\u00af"], + ids: [37955, 162, 112, 119, 21410, 40367, 253, 164, 108, 249, 42468], + decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u0120", "\u0120", "\u0120leading", "\u0120space"], + ids: [220, 220, 3756, 2272], + decoded: " leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["tra", "iling", "\u0120space", "\u0120", "\u0120", "\u0120"], + ids: [9535, 4386, 2272, 220, 220, 220], + decoded: "trailing space ", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["Hi", "\u0120", "\u0120Hello"], + ids: [17250, 220, 18435], + decoded: "Hi Hello", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2", "\u0124", "\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120test"], + ids: [9288, 720, 16, 371, 17, 1303, 18, 10432, 19, 4248, 20, 38221, 21, 2343, 224, 96, 22, 2343, 224, 117, 23, 2343, 224, 109, 24, 1332], + decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "\u0120bought", "\u0120an", "\u0120apple", "\u0120for", "\u0120$", "1", ".", "00", "\u0120at", "\u0120the", "\u0120store", "."], + ids: [40, 5839, 281, 17180, 329, 720, 16, 13, 405, 379, 262, 3650, 13], + decoded: "I bought an apple for $1.00 at the store.", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u0120", "\u0120"], + ids: [5832, 1399, 220, 220], + decoded: "you\u2026 ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], + ids: [5832, 1399, 4603], + decoded: "you\u2026\u00a0\u00a0", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142", "\u00c2\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], + ids: [5832, 1399, 1849, 1849, 5832, 1399, 4603], + decoded: "you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["we", "ird", "\u0120\u00ef", "\u00bd", "\u0140", "\u0120edge", "\u0120\u00ef", "\u00bd", "\u0140", "\u0120case"], + ids: [732, 1447, 27332, 121, 252, 5743, 27332, 121, 252, 1339], + decoded: "weird \uff5e edge \uff5e case", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u00e2\u0138", "\u0123", "This", "\u0120\u00e2\u0138", "\u0123", "is", "\u0120\u00e2\u0138", "\u0123", "a", "\u0120\u00e2\u0138", "\u0123", "test", "\u0120\u00e2\u0138", "\u0123", "."], + ids: [5008, 223, 1212, 11019, 223, 271, 11019, 223, 64, 11019, 223, 9288, 11019, 223, 13], + decoded: "\u2581This \u2581is \u2581a \u2581test \u2581.", + }, + SPECIAL_WITH_TRAILING_WHITESPACE: { + text: SENTENCEPIECE_TEST_STRINGS.SPECIAL_WITH_TRAILING_WHITESPACE, + tokens: ["<", "s", ">", "\u010a"], + ids: [27, 82, 29, 198], + decoded: "\n", + }, + SPECIAL_SURROUNDED_BY_WHITESPACE: { + text: SENTENCEPIECE_TEST_STRINGS.SPECIAL_SURROUNDED_BY_WHITESPACE, + tokens: ["\u0120", "\u0120test", "\u0120", "\u0120"], + ids: [7359, 82, 29, 1332, 7359, 82, 29, 220], + decoded: " test ", + }, + SPECIAL_NO_WHITESPACE: { + text: SENTENCEPIECE_TEST_STRINGS.SPECIAL_NO_WHITESPACE, + tokens: ["", "test", ""], + ids: [3556, 82, 29, 9288, 3556, 82, 29], + decoded: "test", + }, + }, + // - clean_up_tokenization_spaces=false + // - custom pretokenization regex + "Xenova/gpt-4": { + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "\u010a", "'ll", "\u0120!!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'t", "."], + ids: [32, 198, 3358, 11261, 998, 20837, 67, 4708, 67, 315, 11, 649, 956, 13], + decoded: "A\n'll !!to?'d''d of, can't.", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".toString", "();\u010a", "toString", "();"], + ids: [1169, 264, 284, 2909, 5180, 545, 6712, 2178], + decoded: "let a = obj.toString();\ntoString();", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2\u0124\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120test"], + ids: [1985, 400, 16, 432, 17, 674, 18, 13281, 19, 7083, 20, 72588, 21, 2928, 224, 96, 22, 90891, 23, 2928, 224, 109, 24, 1296], + decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["we", "ird", "\u0120", "\u00ef\u00bd\u0140", "\u0120edge", "\u0120", "\u00ef\u00bd\u0140", "\u0120case"], + ids: [906, 2668, 220, 21909, 6964, 220, 21909, 1162], + decoded: "weird \uff5e edge \uff5e case", + }, + }, + "Xenova/gpt-4o": { + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["012", "345", "678", "9", "Ġ", "0", "Ġ", "1", "Ġ", "2", "Ġ", "3", "Ġ", "4", "Ġ", "5", "Ġ", "6", "Ġ", "7", "Ġ", "8", "Ġ", "9", "Ġ", "10", "Ġ", "100", "Ġ", "100", "0"], + ids: [19267, 22901, 30833, 24, 220, 15, 220, 16, 220, 17, 220, 18, 220, 19, 220, 20, 220, 21, 220, 22, 220, 23, 220, 24, 220, 702, 220, 1353, 220, 1353, 15], + decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "\u0120company", "\u0120was", "\u0120founded", "\u0120in", "\u0120", "201", "6", "."], + ids: [976, 3175, 673, 24303, 306, 220, 667, 21, 13], + decoded: "The company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "\u010a", "'ll", "\u0120!!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can't", "."], + ids: [32, 198, 6090, 17131, 935, 48511, 67, 5830, 67, 328, 11, 8535, 13], + decoded: "A\n'll !!to?'d''d of, can't.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["def", "\u0120main", "():\u010a", "\u0109pass"], + ids: [1314, 2758, 8595, 100653], + decoded: "def main():\n\tpass", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".to", "String", "();\u010a", "to", "String", "();"], + ids: [1347, 261, 314, 4099, 3552, 916, 740, 935, 916, 4177], + decoded: "let a = obj.toString();\ntoString();", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["This", "\u010a\u010a", "is", "\u010a", "a", "\u010a", "test", "."], + ids: [2500, 279, 276, 198, 64, 198, 3190, 13], + decoded: "This\n\nis\na\ntest.", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "want", "\u00c3\u00a9d", ",r", "unning"], + ids: [2926, 72517, 6383, 33654, 11244], + decoded: "UNwant\u00e9d,running", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u00e7\u0136\u0141\u00e6\u00b4\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"], + ids: [32479, 1616, 7910, 7856, 249, 3221], + decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u0120\u0120", "\u0120leading", "\u0120space"], + ids: [256, 8117, 4918], + decoded: " leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["tr", "ailing", "\u0120space", "\u0120\u0120\u0120"], + ids: [371, 24408, 4918, 271], + decoded: "trailing space ", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2\u00a5", "6", "\u0120\u00e2\u0124", "\u00a3", "7", "\u0120\u00e2\u0124\u00b9", "8", "\u0120\u00e2\u0124", "\u00b1", "9", "\u0120test"], + ids: [3190, 548, 16, 460, 17, 1069, 18, 7950, 19, 8989, 20, 123814, 21, 59790, 96, 22, 73406, 23, 59790, 109, 24, 1746], + decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u0120\u0120"], + ids: [13320, 1131, 256], + decoded: "you\u2026 ", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["we", "ird", "\u0120\u00ef\u00bd\u0140", "\u0120edge", "\u0120\u00ef\u00bd\u0140", "\u0120case"], + ids: [854, 2716, 105665, 11165, 105665, 1890], + decoded: "weird \uff5e edge \uff5e case", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u00e2\u0138", "\u0123", "This", "\u0120\u00e2\u0138\u0123", "is", "\u0120\u00e2\u0138\u0123", "a", "\u0120\u00e2\u0138\u0123", "test", "\u0120\u00e2\u0138\u0123", "."], + ids: [6762, 223, 2500, 39960, 276, 39960, 64, 39960, 3190, 39960, 13], + decoded: "\u2581This \u2581is \u2581a \u2581test \u2581.", + }, + SPECIAL_WITH_TRAILING_WHITESPACE: { + text: SENTENCEPIECE_TEST_STRINGS.SPECIAL_WITH_TRAILING_WHITESPACE, + tokens: ["\u010a"], + ids: [101950, 523], + decoded: "\n", + }, + }, + "Xenova/claude-tokenizer": { + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".", "toString", "();", "\u010a", "toString", "();"], + ids: [1785, 269, 284, 2652, 18, 26492, 4370, 203, 26492, 4370], + decoded: "let a = obj.toString();\ntoString();", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "want", "\u00c3\u00a9d", ",", "running"], + ids: [2359, 17571, 37911, 16, 7889], + decoded: "UNwant\u00e9d,running", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u00e7\u0136\u0141", "\u00e6\u00b4\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"], + ids: [14706, 37675, 2471, 56904, 15959, 254, 5977], + decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["trailing", "\u0120space", "\u0120\u0120\u0120"], + ids: [40110, 3384, 264], + decoded: "trailing space ", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2", "\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2", "\u0124", "\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120test"], + ids: [765, 734, 21, 487, 22, 379, 23, 36714, 24, 13206, 25, 2455, 103, 26, 4937, 229, 101, 27, 4937, 229, 122, 28, 4937, 229, 114, 29, 722], + decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "...", "\u0120\u0120"], + ids: [6773, 1174, 261], + decoded: "you... ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "...", "\u0120\u0120"], + ids: [6773, 1174, 261], + decoded: "you... ", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "...", "\u0120", "\u0120you", "...", "\u0120\u0120"], + ids: [6773, 1174, 225, 583, 1174, 261], + decoded: "you... you... ", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["we", "ird", "\u0120~", "\u0120edge", "\u0120~", "\u0120case"], + ids: [798, 2650, 6217, 4915, 6217, 1544], + decoded: "weird ~ edge ~ case", + }, + }, + "bigcode/santacoder": { + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "Ġ", "0", "Ġ", "1", "Ġ", "2", "Ġ", "3", "Ġ", "4", "Ġ", "5", "Ġ", "6", "Ġ", "7", "Ġ", "8", "Ġ", "9", "Ġ", "1", "0", "Ġ", "1", "0", "0", "Ġ", "1", "0", "0", "0"], + ids: [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 207, 15, 207, 16, 207, 17, 207, 18, 207, 19, 207, 20, 207, 21, 207, 22, 207, 23, 207, 24, 207, 16, 15, 207, 16, 15, 15, 207, 16, 15, 15, 15], + decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "\u0120company", "\u0120was", "\u0120fo", "unded", "\u0120in", "\u0120", "2", "0", "1", "6", "."], + ids: [2111, 10107, 2501, 17436, 7584, 319, 207, 17, 15, 16, 21, 13], + decoded: "The company was founded in 2016.", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u00e7\u0136\u0141", "\u00e6\u00b4\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"], + ids: [8715, 24543, 1825, 34717, 37452, 236, 4343], + decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "\u0120bo", "ught", "\u0120an", "\u0120apple", "\u0120for", "\u0120$", "1", ".", "0", "0", "\u0120at", "\u0120the", "\u0120store", "."], + ids: [40, 12307, 10310, 743, 29806, 408, 763, 16, 13, 15, 15, 869, 331, 2823, 13], + decoded: "I bought an apple for $1.00 at the store.", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["we", "ird", "\u0120", "\u00ef\u00bd", "\u0140", "\u0120edge", "\u0120", "\u00ef\u00bd", "\u0140", "\u0120case"], + ids: [1850, 4427, 207, 29217, 239, 4959, 207, 29217, 239, 1210], + decoded: "weird \uff5e edge \uff5e case", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u00e2\u0138", "\u0123", "This", "\u0120", "\u00e2\u0138", "\u0123", "is", "\u0120", "\u00e2\u0138", "\u0123", "a", "\u0120", "\u00e2\u0138", "\u0123", "test", "\u0120", "\u00e2\u0138", "\u0123", "."], + ids: [3718, 210, 3456, 207, 3718, 210, 280, 207, 3718, 210, 64, 207, 3718, 210, 706, 207, 3718, 210, 13], + decoded: "\u2581This \u2581is \u2581a \u2581test \u2581.", + }, + }, + "Xenova/CodeGPT-tokenizer": { + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u00e7\u0136\u0141", "\u00e6", "\u00b4", "\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e", "\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"], + ids: [25506, 165, 115, 122, 5137, 43415, 256, 20679, 252, 13283], + decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["trailing", "\u0120space", "\u0120", "\u0120", "\u0120"], + ids: [15584, 3497, 223, 223, 223], + decoded: "trailing space ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142"], + ids: [13953, 29502, 129, 257, 129, 257], + decoded: "you\u2026\u00a0\u00a0", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142"], + ids: [13953, 29502, 129, 257, 129, 257, 13953, 29502, 129, 257, 129, 257], + decoded: "you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0", + }, + }, + "huggingface-course/codeparrot-ds": { + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["0123456789", "Ġ0", "Ġ1", "Ġ2", "Ġ3", "Ġ4", "Ġ5", "Ġ6", "Ġ7", "Ġ8", "Ġ9", "Ġ10", "Ġ100", "Ġ1000"], + ids: [25218, 443, 396, 554, 869, 1163, 1462, 1911, 2624, 2070, 2837, 2009, 3038, 4764], + decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "\u0120company", "\u0120was", "\u0120fo", "unded", "\u0120in", "\u01202016", "."], + ids: [2096, 16502, 1442, 11689, 7865, 253, 8780, 14], + decoded: "The company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "\u010a", "'ll", "\u0120!", "!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'t", "."], + ids: [33, 173, 6402, 905, 1, 403, 15227, 68, 589, 68, 311, 12, 796, 1059, 14], + decoded: "A\n'll!!to?'d''d of, can't.", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".", "toString", "();", "\u010a", "toString", "();"], + ids: [2047, 231, 233, 1300, 14, 30494, 16248, 173, 30494, 16248], + decoded: "let a = obj.toString();\ntoString();", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u00e7\u0136\u0141", "\u00e6\u00b4", "\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e", "\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"], + ids: [20185, 43799, 120, 3994, 37782, 211, 15933, 207, 11130], + decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["trailing", "\u0120space", "\u0120\u0120\u0120"], + ids: [17031, 3000, 216], + decoded: "trailing space ", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2", "\u0124\u00ac", "4", "\u0120\u00c2", "\u00a3", "5", "\u0120\u00c2", "\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2", "\u0124", "\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120test"], + ids: [1824, 3748, 17, 683, 18, 294, 19, 5161, 28898, 20, 23446, 97, 21, 23446, 99, 22, 5161, 182, 97, 23, 5161, 182, 118, 24, 5161, 182, 110, 25, 1737], + decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "\u0120bo", "ught", "\u0120an", "\u0120app", "le", "\u0120for", "\u0120$", "1", ".", "00", "\u0120at", "\u0120the", "\u0120store", "."], + ids: [41, 772, 8272, 309, 870, 239, 296, 3748, 17, 14, 543, 815, 256, 2689, 14], + decoded: "I bought an apple for $1.00 at the store.", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["we", "ird", "\u0120", "\u00ef", "\u00bd", "\u0140", "\u0120edge", "\u0120", "\u00ef", "\u00bd", "\u0140", "\u0120case"], + ids: [955, 6075, 179, 166, 122, 210, 2703, 179, 166, 122, 210, 1539], + decoded: "weird \uff5e edge \uff5e case", + }, }, - SIMPLE_WITH_PUNCTUATION: { - text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, - tokens: ["You", "\u0120should", "'ve", "\u0120done", "\u0120this"], - ids: [1639, 815, 1053, 1760, 428], - decoded: "You should've done this", - }, - NUMBERS: { - text: BASE_TEST_STRINGS.NUMBERS, - tokens: ["01", "23", "45", "67", "89", "Ġ0", "Ġ1", "Ġ2", "Ġ3", "Ġ4", "Ġ5", "Ġ6", "Ġ7", "Ġ8", "Ġ9", "Ġ10", "Ġ100", "Ġ1000"], - ids: [486, 1954, 2231, 3134, 4531, 657, 352, 362, 513, 604, 642, 718, 767, 807, 860, 838, 1802, 8576], - decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["The", "\u0120company", "\u0120was", "\u0120founded", "\u0120in", "\u01202016", "."], - ids: [464, 1664, 373, 9393, 287, 1584, 13], - decoded: "The company was founded in 2016.", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["A", "\u010a", "'ll", "\u0120!!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'t", "."], - ids: [32, 198, 1183, 37867, 1462, 8348, 67, 7061, 67, 286, 11, 460, 470, 13], - decoded: "A\n'll!!to?'d''d of, can't.", - }, - PYTHON_CODE: { - text: BASE_TEST_STRINGS.PYTHON_CODE, - tokens: ["def", "\u0120main", "():", "\u010a", "\u0109", "pass"], - ids: [4299, 1388, 33529, 198, 197, 6603], - decoded: "def main():\n\tpass", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".", "to", "String", "();", "\u010a", "to", "String", "();"], - ids: [1616, 257, 796, 26181, 13, 1462, 10100, 9783, 198, 1462, 10100, 9783], - decoded: "let a = obj.toString();\ntoString();", - }, - NEWLINES: { - text: BASE_TEST_STRINGS.NEWLINES, - tokens: ["This", "\u010a", "\u010a", "is", "\u010a", "a", "\u010a", "test", "."], - ids: [1212, 198, 198, 271, 198, 64, 198, 9288, 13], - decoded: "This\n\nis\na\ntest.", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["UN", "want", "\u00c3\u00a9", "d", ",", "running"], - ids: [4944, 42949, 2634, 67, 11, 20270], - decoded: "UNwant\u00e9d,running", - }, - CONTROL_TOKENS: { - text: BASE_TEST_STRINGS.CONTROL_TOKENS, - tokens: ["1", "\u0100", "2", "\u00ef\u00bf\u00bd", "3"], - ids: [16, 188, 17, 4210, 18], - decoded: "1\u00002\ufffd3", - }, - HELLO_WORLD_TITLECASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, - tokens: ["Hello", "\u0120World"], - ids: [15496, 2159], - decoded: "Hello World", - }, - HELLO_WORLD_LOWERCASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, - tokens: ["hello", "\u0120world"], - ids: [31373, 995], - decoded: "hello world", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u00e7\u0136\u0141", "\u00e6", "\u00b4", "\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e", "\u0141", "\u00e8", "\u00b0", "\u013d", "\u00e6\u013a\u00af"], - ids: [37955, 162, 112, 119, 21410, 40367, 253, 164, 108, 249, 42468], - decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", - }, - LEADING_SPACE: { - text: BASE_TEST_STRINGS.LEADING_SPACE, - tokens: ["\u0120", "\u0120", "\u0120leading", "\u0120space"], - ids: [220, 220, 3756, 2272], - decoded: " leading space", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["tra", "iling", "\u0120space", "\u0120", "\u0120", "\u0120"], - ids: [9535, 4386, 2272, 220, 220, 220], - decoded: "trailing space ", - }, - DOUBLE_SPACE: { - text: BASE_TEST_STRINGS.DOUBLE_SPACE, - tokens: ["Hi", "\u0120", "\u0120Hello"], - ids: [17250, 220, 18435], - decoded: "Hi Hello", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2", "\u0124", "\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120test"], - ids: [9288, 720, 16, 371, 17, 1303, 18, 10432, 19, 4248, 20, 38221, 21, 2343, 224, 96, 22, 2343, 224, 117, 23, 2343, 224, 109, 24, 1332], - decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["I", "\u0120bought", "\u0120an", "\u0120apple", "\u0120for", "\u0120$", "1", ".", "00", "\u0120at", "\u0120the", "\u0120store", "."], - ids: [40, 5839, 281, 17180, 329, 720, 16, 13, 405, 379, 262, 3650, 13], - decoded: "I bought an apple for $1.00 at the store.", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - tokens: ["you", "\u00e2\u0122\u00a6", "\u0120", "\u0120"], - ids: [5832, 1399, 220, 220], - decoded: "you\u2026 ", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], - ids: [5832, 1399, 4603], - decoded: "you\u2026\u00a0\u00a0", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142", "\u00c2\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], - ids: [5832, 1399, 1849, 1849, 5832, 1399, 4603], - decoded: "you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["we", "ird", "\u0120\u00ef", "\u00bd", "\u0140", "\u0120edge", "\u0120\u00ef", "\u00bd", "\u0140", "\u0120case"], - ids: [732, 1447, 27332, 121, 252, 5743, 27332, 121, 252, 1339], - decoded: "weird \uff5e edge \uff5e case", - }, - SPIECE_UNDERSCORE: { - text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, - tokens: ["\u00e2\u0138", "\u0123", "This", "\u0120\u00e2\u0138", "\u0123", "is", "\u0120\u00e2\u0138", "\u0123", "a", "\u0120\u00e2\u0138", "\u0123", "test", "\u0120\u00e2\u0138", "\u0123", "."], - ids: [5008, 223, 1212, 11019, 223, 271, 11019, 223, 64, 11019, 223, 9288, 11019, 223, 13], - decoded: "\u2581This \u2581is \u2581a \u2581test \u2581.", - }, - SPECIAL_WITH_TRAILING_WHITESPACE: { - text: SENTENCEPIECE_TEST_STRINGS.SPECIAL_WITH_TRAILING_WHITESPACE, - tokens: ["<", "s", ">", "\u010a"], - ids: [27, 82, 29, 198], - decoded: "\n", - }, - SPECIAL_SURROUNDED_BY_WHITESPACE: { - text: SENTENCEPIECE_TEST_STRINGS.SPECIAL_SURROUNDED_BY_WHITESPACE, - tokens: ["\u0120", "\u0120test", "\u0120", "\u0120"], - ids: [7359, 82, 29, 1332, 7359, 82, 29, 220], - decoded: " test ", - }, - SPECIAL_NO_WHITESPACE: { - text: SENTENCEPIECE_TEST_STRINGS.SPECIAL_NO_WHITESPACE, - tokens: ["", "test", ""], - ids: [3556, 82, 29, 9288, 3556, 82, 29], - decoded: "test", - }, - }, - // - clean_up_tokenization_spaces=false - // - custom pretokenization regex - "Xenova/gpt-4": { - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["A", "\u010a", "'ll", "\u0120!!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'t", "."], - ids: [32, 198, 3358, 11261, 998, 20837, 67, 4708, 67, 315, 11, 649, 956, 13], - decoded: "A\n'll !!to?'d''d of, can't.", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".toString", "();\u010a", "toString", "();"], - ids: [1169, 264, 284, 2909, 5180, 545, 6712, 2178], - decoded: "let a = obj.toString();\ntoString();", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2\u0124\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120test"], - ids: [1985, 400, 16, 432, 17, 674, 18, 13281, 19, 7083, 20, 72588, 21, 2928, 224, 96, 22, 90891, 23, 2928, 224, 109, 24, 1296], - decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["we", "ird", "\u0120", "\u00ef\u00bd\u0140", "\u0120edge", "\u0120", "\u00ef\u00bd\u0140", "\u0120case"], - ids: [906, 2668, 220, 21909, 6964, 220, 21909, 1162], - decoded: "weird \uff5e edge \uff5e case", - }, - }, - "Xenova/gpt-4o": { - NUMBERS: { - text: BASE_TEST_STRINGS.NUMBERS, - tokens: ["012", "345", "678", "9", "Ġ", "0", "Ġ", "1", "Ġ", "2", "Ġ", "3", "Ġ", "4", "Ġ", "5", "Ġ", "6", "Ġ", "7", "Ġ", "8", "Ġ", "9", "Ġ", "10", "Ġ", "100", "Ġ", "100", "0"], - ids: [19267, 22901, 30833, 24, 220, 15, 220, 16, 220, 17, 220, 18, 220, 19, 220, 20, 220, 21, 220, 22, 220, 23, 220, 24, 220, 702, 220, 1353, 220, 1353, 15], - decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["The", "\u0120company", "\u0120was", "\u0120founded", "\u0120in", "\u0120", "201", "6", "."], - ids: [976, 3175, 673, 24303, 306, 220, 667, 21, 13], - decoded: "The company was founded in 2016.", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["A", "\u010a", "'ll", "\u0120!!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can't", "."], - ids: [32, 198, 6090, 17131, 935, 48511, 67, 5830, 67, 328, 11, 8535, 13], - decoded: "A\n'll !!to?'d''d of, can't.", - }, - PYTHON_CODE: { - text: BASE_TEST_STRINGS.PYTHON_CODE, - tokens: ["def", "\u0120main", "():\u010a", "\u0109pass"], - ids: [1314, 2758, 8595, 100653], - decoded: "def main():\n\tpass", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".to", "String", "();\u010a", "to", "String", "();"], - ids: [1347, 261, 314, 4099, 3552, 916, 740, 935, 916, 4177], - decoded: "let a = obj.toString();\ntoString();", - }, - NEWLINES: { - text: BASE_TEST_STRINGS.NEWLINES, - tokens: ["This", "\u010a\u010a", "is", "\u010a", "a", "\u010a", "test", "."], - ids: [2500, 279, 276, 198, 64, 198, 3190, 13], - decoded: "This\n\nis\na\ntest.", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["UN", "want", "\u00c3\u00a9d", ",r", "unning"], - ids: [2926, 72517, 6383, 33654, 11244], - decoded: "UNwant\u00e9d,running", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u00e7\u0136\u0141\u00e6\u00b4\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"], - ids: [32479, 1616, 7910, 7856, 249, 3221], - decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", - }, - LEADING_SPACE: { - text: BASE_TEST_STRINGS.LEADING_SPACE, - tokens: ["\u0120\u0120", "\u0120leading", "\u0120space"], - ids: [256, 8117, 4918], - decoded: " leading space", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["tr", "ailing", "\u0120space", "\u0120\u0120\u0120"], - ids: [371, 24408, 4918, 271], - decoded: "trailing space ", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2\u00a5", "6", "\u0120\u00e2\u0124", "\u00a3", "7", "\u0120\u00e2\u0124\u00b9", "8", "\u0120\u00e2\u0124", "\u00b1", "9", "\u0120test"], - ids: [3190, 548, 16, 460, 17, 1069, 18, 7950, 19, 8989, 20, 123814, 21, 59790, 96, 22, 73406, 23, 59790, 109, 24, 1746], - decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - tokens: ["you", "\u00e2\u0122\u00a6", "\u0120\u0120"], - ids: [13320, 1131, 256], - decoded: "you\u2026 ", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["we", "ird", "\u0120\u00ef\u00bd\u0140", "\u0120edge", "\u0120\u00ef\u00bd\u0140", "\u0120case"], - ids: [854, 2716, 105665, 11165, 105665, 1890], - decoded: "weird \uff5e edge \uff5e case", - }, - SPIECE_UNDERSCORE: { - text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, - tokens: ["\u00e2\u0138", "\u0123", "This", "\u0120\u00e2\u0138\u0123", "is", "\u0120\u00e2\u0138\u0123", "a", "\u0120\u00e2\u0138\u0123", "test", "\u0120\u00e2\u0138\u0123", "."], - ids: [6762, 223, 2500, 39960, 276, 39960, 64, 39960, 3190, 39960, 13], - decoded: "\u2581This \u2581is \u2581a \u2581test \u2581.", - }, - SPECIAL_WITH_TRAILING_WHITESPACE: { - text: SENTENCEPIECE_TEST_STRINGS.SPECIAL_WITH_TRAILING_WHITESPACE, - tokens: ["\u010a"], - ids: [101950, 523], - decoded: "\n", - }, - }, - "Xenova/claude-tokenizer": { - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".", "toString", "();", "\u010a", "toString", "();"], - ids: [1785, 269, 284, 2652, 18, 26492, 4370, 203, 26492, 4370], - decoded: "let a = obj.toString();\ntoString();", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["UN", "want", "\u00c3\u00a9d", ",", "running"], - ids: [2359, 17571, 37911, 16, 7889], - decoded: "UNwant\u00e9d,running", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u00e7\u0136\u0141", "\u00e6\u00b4\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"], - ids: [14706, 37675, 2471, 56904, 15959, 254, 5977], - decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["trailing", "\u0120space", "\u0120\u0120\u0120"], - ids: [40110, 3384, 264], - decoded: "trailing space ", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2", "\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2", "\u0124", "\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120test"], - ids: [765, 734, 21, 487, 22, 379, 23, 36714, 24, 13206, 25, 2455, 103, 26, 4937, 229, 101, 27, 4937, 229, 122, 28, 4937, 229, 114, 29, 722], - decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - tokens: ["you", "...", "\u0120\u0120"], - ids: [6773, 1174, 261], - decoded: "you... ", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - tokens: ["you", "...", "\u0120\u0120"], - ids: [6773, 1174, 261], - decoded: "you... ", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - tokens: ["you", "...", "\u0120", "\u0120you", "...", "\u0120\u0120"], - ids: [6773, 1174, 225, 583, 1174, 261], - decoded: "you... you... ", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["we", "ird", "\u0120~", "\u0120edge", "\u0120~", "\u0120case"], - ids: [798, 2650, 6217, 4915, 6217, 1544], - decoded: "weird ~ edge ~ case", - }, - }, - "bigcode/santacoder": { - NUMBERS: { - text: BASE_TEST_STRINGS.NUMBERS, - tokens: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "Ġ", "0", "Ġ", "1", "Ġ", "2", "Ġ", "3", "Ġ", "4", "Ġ", "5", "Ġ", "6", "Ġ", "7", "Ġ", "8", "Ġ", "9", "Ġ", "1", "0", "Ġ", "1", "0", "0", "Ġ", "1", "0", "0", "0"], - ids: [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 207, 15, 207, 16, 207, 17, 207, 18, 207, 19, 207, 20, 207, 21, 207, 22, 207, 23, 207, 24, 207, 16, 15, 207, 16, 15, 15, 207, 16, 15, 15, 15], - decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["The", "\u0120company", "\u0120was", "\u0120fo", "unded", "\u0120in", "\u0120", "2", "0", "1", "6", "."], - ids: [2111, 10107, 2501, 17436, 7584, 319, 207, 17, 15, 16, 21, 13], - decoded: "The company was founded in 2016.", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u00e7\u0136\u0141", "\u00e6\u00b4\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"], - ids: [8715, 24543, 1825, 34717, 37452, 236, 4343], - decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["I", "\u0120bo", "ught", "\u0120an", "\u0120apple", "\u0120for", "\u0120$", "1", ".", "0", "0", "\u0120at", "\u0120the", "\u0120store", "."], - ids: [40, 12307, 10310, 743, 29806, 408, 763, 16, 13, 15, 15, 869, 331, 2823, 13], - decoded: "I bought an apple for $1.00 at the store.", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["we", "ird", "\u0120", "\u00ef\u00bd", "\u0140", "\u0120edge", "\u0120", "\u00ef\u00bd", "\u0140", "\u0120case"], - ids: [1850, 4427, 207, 29217, 239, 4959, 207, 29217, 239, 1210], - decoded: "weird \uff5e edge \uff5e case", - }, - SPIECE_UNDERSCORE: { - text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, - tokens: ["\u00e2\u0138", "\u0123", "This", "\u0120", "\u00e2\u0138", "\u0123", "is", "\u0120", "\u00e2\u0138", "\u0123", "a", "\u0120", "\u00e2\u0138", "\u0123", "test", "\u0120", "\u00e2\u0138", "\u0123", "."], - ids: [3718, 210, 3456, 207, 3718, 210, 280, 207, 3718, 210, 64, 207, 3718, 210, 706, 207, 3718, 210, 13], - decoded: "\u2581This \u2581is \u2581a \u2581test \u2581.", - }, - }, - "Xenova/CodeGPT-tokenizer": { - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u00e7\u0136\u0141", "\u00e6", "\u00b4", "\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e", "\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"], - ids: [25506, 165, 115, 122, 5137, 43415, 256, 20679, 252, 13283], - decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["trailing", "\u0120space", "\u0120", "\u0120", "\u0120"], - ids: [15584, 3497, 223, 223, 223], - decoded: "trailing space ", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142"], - ids: [13953, 29502, 129, 257, 129, 257], - decoded: "you\u2026\u00a0\u00a0", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142"], - ids: [13953, 29502, 129, 257, 129, 257, 13953, 29502, 129, 257, 129, 257], - decoded: "you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0", - }, - }, - "huggingface-course/codeparrot-ds": { - NUMBERS: { - text: BASE_TEST_STRINGS.NUMBERS, - tokens: ["0123456789", "Ġ0", "Ġ1", "Ġ2", "Ġ3", "Ġ4", "Ġ5", "Ġ6", "Ġ7", "Ġ8", "Ġ9", "Ġ10", "Ġ100", "Ġ1000"], - ids: [25218, 443, 396, 554, 869, 1163, 1462, 1911, 2624, 2070, 2837, 2009, 3038, 4764], - decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["The", "\u0120company", "\u0120was", "\u0120fo", "unded", "\u0120in", "\u01202016", "."], - ids: [2096, 16502, 1442, 11689, 7865, 253, 8780, 14], - decoded: "The company was founded in 2016.", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["A", "\u010a", "'ll", "\u0120!", "!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'t", "."], - ids: [33, 173, 6402, 905, 1, 403, 15227, 68, 589, 68, 311, 12, 796, 1059, 14], - decoded: "A\n'll!!to?'d''d of, can't.", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".", "toString", "();", "\u010a", "toString", "();"], - ids: [2047, 231, 233, 1300, 14, 30494, 16248, 173, 30494, 16248], - decoded: "let a = obj.toString();\ntoString();", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u00e7\u0136\u0141", "\u00e6\u00b4", "\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e", "\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"], - ids: [20185, 43799, 120, 3994, 37782, 211, 15933, 207, 11130], - decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["trailing", "\u0120space", "\u0120\u0120\u0120"], - ids: [17031, 3000, 216], - decoded: "trailing space ", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2", "\u0124\u00ac", "4", "\u0120\u00c2", "\u00a3", "5", "\u0120\u00c2", "\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2", "\u0124", "\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120test"], - ids: [1824, 3748, 17, 683, 18, 294, 19, 5161, 28898, 20, 23446, 97, 21, 23446, 99, 22, 5161, 182, 97, 23, 5161, 182, 118, 24, 5161, 182, 110, 25, 1737], - decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["I", "\u0120bo", "ught", "\u0120an", "\u0120app", "le", "\u0120for", "\u0120$", "1", ".", "00", "\u0120at", "\u0120the", "\u0120store", "."], - ids: [41, 772, 8272, 309, 870, 239, 296, 3748, 17, 14, 543, 815, 256, 2689, 14], - decoded: "I bought an apple for $1.00 at the store.", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["we", "ird", "\u0120", "\u00ef", "\u00bd", "\u0140", "\u0120edge", "\u0120", "\u00ef", "\u00bd", "\u0140", "\u0120case"], - ids: [955, 6075, 179, 166, 122, 210, 2703, 179, 166, 122, 210, 1539], - decoded: "weird \uff5e edge \uff5e case", - }, - }, }; diff --git a/tests/models/llama/tokenization.js b/tests/models/llama/tokenization.js index dcbbe9770..dfcd8c15c 100644 --- a/tests/models/llama/tokenization.js +++ b/tests/models/llama/tokenization.js @@ -3,1294 +3,1296 @@ import { BASE_TEST_STRINGS, LLAMA_TEST_STRINGS } from "../test_strings.js"; export const TOKENIZER_CLASS = LlamaTokenizer; export const TEST_CONFIG = { - "Xenova/llama-tokenizer": { - SIMPLE: { - text: BASE_TEST_STRINGS.SIMPLE, - tokens: ["\u2581How", "\u2581are", "\u2581you", "\u2581doing", "?"], - ids: [1, 1128, 526, 366, 2599, 29973], - decoded: " How are you doing?", - }, - SIMPLE_WITH_PUNCTUATION: { - text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, - tokens: ["\u2581You", "\u2581should", "'", "ve", "\u2581done", "\u2581this"], - ids: [1, 887, 881, 29915, 345, 2309, 445], - decoded: " You should've done this", - }, - NUMBERS: { - text: BASE_TEST_STRINGS.NUMBERS, - tokens: ["\u2581", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "\u2581", "0", "\u2581", "1", "\u2581", "2", "\u2581", "3", "\u2581", "4", "\u2581", "5", "\u2581", "6", "\u2581", "7", "\u2581", "8", "\u2581", "9", "\u2581", "1", "0", "\u2581", "1", "0", "0", "\u2581", "1", "0", "0", "0"], - ids: [1, 29871, 29900, 29896, 29906, 29941, 29946, 29945, 29953, 29955, 29947, 29929, 29871, 29900, 29871, 29896, 29871, 29906, 29871, 29941, 29871, 29946, 29871, 29945, 29871, 29953, 29871, 29955, 29871, 29947, 29871, 29929, 29871, 29896, 29900, 29871, 29896, 29900, 29900, 29871, 29896, 29900, 29900, 29900], - decoded: " 0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["\u2581The", "\u2581company", "\u2581was", "\u2581founded", "\u2581in", "\u2581", "2", "0", "1", "6", "."], - ids: [1, 450, 5001, 471, 11091, 297, 29871, 29906, 29900, 29896, 29953, 29889], - decoded: " The company was founded in 2016.", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["\u2581A", "<0x0A>", "'", "ll", "\u2581!!", "to", "?'", "d", "''", "d", "\u2581of", ",", "\u2581can", "'", "t", "."], - ids: [1, 319, 13, 29915, 645, 21443, 517, 17901, 29881, 4907, 29881, 310, 29892, 508, 29915, 29873, 29889], - decoded: " A\n'll !!to?'d''d of, can't.", - }, - PYTHON_CODE: { - text: BASE_TEST_STRINGS.PYTHON_CODE, - tokens: ["\u2581def", "\u2581main", "():", "<0x0A>", "<0x09>", "pass"], - ids: [1, 822, 1667, 7295, 13, 12, 3364], - decoded: " def main():\n\tpass", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["\u2581let", "\u2581a", "\u2581=", "\u2581obj", ".", "toString", "();", "<0x0A>", "toString", "();"], - ids: [1, 1235, 263, 353, 5446, 29889, 7711, 890, 13, 7711, 890], - decoded: " let a = obj.toString();\ntoString();", - }, - NEWLINES: { - text: LLAMA_TEST_STRINGS.NEWLINES, - tokens: ["\u2581ax", "<0x0A>", "####", "<0x0A>", "bo", "o"], - ids: [1, 4853, 13, 4136, 13, 833, 29877], - decoded: " ax\n####\nboo", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["\u2581UN", "w", "ant", "\u00e9d", ",", "running"], - ids: [1, 8291, 29893, 424, 2487, 29892, 21094], - decoded: " UNwant\u00e9d,running", - }, - CONTROL_TOKENS: { - text: BASE_TEST_STRINGS.CONTROL_TOKENS, - tokens: ["\u2581", "1", "<0x00>", "2", "\ufffd", "3"], - ids: [1, 29871, 29896, 3, 29906, 30140, 29941], - decoded: " 1\u00002\ufffd3", - }, - HELLO_WORLD_TITLECASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, - tokens: ["\u2581Hello", "\u2581World"], - ids: [1, 15043, 2787], - decoded: " Hello World", - }, - HELLO_WORLD_LOWERCASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, - tokens: ["\u2581hello", "\u2581world"], - ids: [1, 22172, 3186], - decoded: " hello world", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u2581", "\u751f", "\u6d3b", "\u7684", "\u771f", "<0xE8>", "<0xB0>", "<0x9B>", "\u662f"], - ids: [1, 29871, 30486, 31704, 30210, 30848, 235, 179, 158, 30392], - decoded: " \u751f\u6d3b\u7684\u771f\u8c1b\u662f", - }, - LEADING_SPACE: { - text: BASE_TEST_STRINGS.LEADING_SPACE, - tokens: ["\u2581\u2581\u2581", "\u2581leading", "\u2581space"], - ids: [1, 1678, 8236, 2913], - decoded: " leading space", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["\u2581trailing", "\u2581space", "\u2581\u2581\u2581"], - ids: [1, 25053, 2913, 1678], - decoded: " trailing space ", - }, - DOUBLE_SPACE: { - text: BASE_TEST_STRINGS.DOUBLE_SPACE, - tokens: ["\u2581Hi", "\u2581", "\u2581Hello"], - ids: [1, 6324, 29871, 15043], - decoded: " Hi Hello", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["\u2581test", "\u2581$", "1", "\u2581R", "2", "\u2581#", "3", "\u2581\u20ac", "4", "\u2581\u00a3", "5", "\u2581", "\u00a5", "6", "\u2581", "<0xE2>", "<0x82>", "<0xA3>", "7", "\u2581", "\u20b9", "8", "\u2581", "<0xE2>", "<0x82>", "<0xB1>", "9", "\u2581test"], - ids: [1, 1243, 395, 29896, 390, 29906, 396, 29941, 25540, 29946, 15151, 29945, 29871, 30563, 29953, 29871, 229, 133, 166, 29955, 29871, 30620, 29947, 29871, 229, 133, 180, 29929, 1243], - decoded: " test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["\u2581I", "\u2581bought", "\u2581an", "\u2581apple", "\u2581for", "\u2581$", "1", ".", "0", "0", "\u2581at", "\u2581the", "\u2581store", "."], - ids: [1, 306, 18093, 385, 26163, 363, 395, 29896, 29889, 29900, 29900, 472, 278, 3787, 29889], - decoded: " I bought an apple for $1.00 at the store.", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - tokens: ["\u2581you", "\u2026", "\u2581\u2581"], - ids: [1, 366, 30098, 259], - decoded: " you\u2026 ", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - tokens: ["\u2581you", "\u2026", "\u00a0\u00a0"], - ids: [1, 366, 30098, 8655], - decoded: " you\u2026\u00a0\u00a0", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - tokens: ["\u2581you", "\u2026", "\u00a0\u00a0", "you", "\u2026", "\u00a0\u00a0"], - ids: [1, 366, 30098, 8655, 6293, 30098, 8655], - decoded: " you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["\u2581weird", "\u2581", "\uff5e", "\u2581edge", "\u2581", "\uff5e", "\u2581case"], - ids: [1, 13543, 29871, 30739, 7636, 29871, 30739, 1206], - decoded: " weird \uff5e edge \uff5e case", - }, - SPIECE_UNDERSCORE: { - text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, - tokens: ["\u2581", "\u2581This", "\u2581", "\u2581is", "\u2581", "\u2581a", "\u2581", "\u2581test", "\u2581", "\u2581."], - ids: [1, 29871, 910, 29871, 338, 29871, 263, 29871, 1243, 29871, 869], - decoded: " This is a test .", - }, - POPULAR_EMOJIS: { - text: BASE_TEST_STRINGS.POPULAR_EMOJIS, - tokens: ["\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0x82>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x8D>", "\u2581", "<0xF0>", "<0x9F>", "<0xA4>", "<0xA3>", "\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0x8D>", "\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0xAD>", "\u2581", "<0xF0>", "<0x9F>", "<0x8E>", "<0x89>", "\u2581", "<0xF0>", "<0x9F>", "<0x99>", "<0x8F>", "\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0x8A>", "\u2581", "<0xF0>", "<0x9F>", "<0x94>", "<0xA5>", "\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0x81>", "\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0x85>", "\u2581", "<0xF0>", "<0x9F>", "<0xA4>", "<0x97>", "\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0x86>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x8F>", "\u2581", "<0xE2>", "<0x9D>", "<0xA4>", "\ufe0f", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0x9C>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0x9A>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0x97>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0x99>", "\u2581", "<0xF0>", "<0x9F>", "<0x96>", "<0xA4>", "\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0x8E>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x8C>", "\u2581", "<0xF0>", "<0x9F>", "<0xA5>", "<0xB3>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0xAA>", "\u2581", "<0xE2>", "<0x9C>", "<0xA8>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x89>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x80>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0xAF>", "\u2581", "<0xF0>", "<0x9F>", "<0x8E>", "<0x88>", "\u2581", "<0xF0>", "<0x9F>", "<0x99>", "<0x88>", "\u2581", "<0xF0>", "<0x9F>", "<0x99>", "<0x8C>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0x80>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x87>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x8B>", "\u2581", "\u2705", "\u2581", "<0xF0>", "<0x9F>", "<0x8E>", "<0x81>", "\u2581", "<0xF0>", "<0x9F>", "<0x8C>", "<0x9E>", "\u2581", "<0xF0>", "<0x9F>", "<0x8C>", "<0xB8>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0xB0>"], - ids: [1, 29871, 243, 162, 155, 133, 29871, 243, 162, 148, 144, 29871, 243, 162, 167, 166, 29871, 243, 162, 155, 144, 29871, 243, 162, 155, 176, 29871, 243, 162, 145, 140, 29871, 243, 162, 156, 146, 29871, 243, 162, 155, 141, 29871, 243, 162, 151, 168, 29871, 243, 162, 155, 132, 29871, 243, 162, 155, 136, 29871, 243, 162, 167, 154, 29871, 243, 162, 155, 137, 29871, 243, 162, 148, 146, 29871, 229, 160, 167, 30598, 29871, 243, 162, 149, 159, 29871, 243, 162, 149, 157, 29871, 243, 162, 149, 154, 29871, 243, 162, 149, 156, 29871, 243, 162, 153, 167, 29871, 243, 162, 155, 145, 29871, 243, 162, 148, 143, 29871, 243, 162, 168, 182, 29871, 243, 162, 149, 173, 29871, 229, 159, 171, 29871, 243, 162, 148, 140, 29871, 243, 162, 148, 131, 29871, 243, 162, 149, 178, 29871, 243, 162, 145, 139, 29871, 243, 162, 156, 139, 29871, 243, 162, 156, 143, 29871, 243, 162, 149, 131, 29871, 243, 162, 148, 138, 29871, 243, 162, 148, 142, 29871, 31681, 29871, 243, 162, 145, 132, 29871, 243, 162, 143, 161, 29871, 243, 162, 143, 187, 29871, 243, 162, 149, 179], - decoded: " \ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0", - }, - MULTIBYTE_EMOJIS: { - text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, - tokens: ["\u2581", "<0xE2>", "<0x9C>", "<0xA8>", "\u2581", "<0xF0>", "<0x9F>", "<0xA4>", "<0x97>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x81>", "\ufe0f", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0xB1>", "<0xF0>", "<0x9F>", "<0x8F>", "<0xBB>", "\u2581", "<0xF0>", "<0x9F>", "<0x95>", "<0xB5>", "\u200d", "\u2642", "\ufe0f", "\u2581", "<0xF0>", "<0x9F>", "<0xA7>", "<0x99>", "<0xF0>", "<0x9F>", "<0x8F>", "<0xBB>", "\u200d", "\u2642", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0xA8>", "<0xF0>", "<0x9F>", "<0x8F>", "<0xBB>", "\u200d", "<0xF0>", "<0x9F>", "<0x8C>", "<0xBE>", "\u2581", "<0xF0>", "<0x9F>", "<0xA7>", "<0x91>", "\u200d", "<0xF0>", "<0x9F>", "<0xA4>", "<0x9D>", "\u200d", "<0xF0>", "<0x9F>", "<0xA7>", "<0x91>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0xA9>", "\u200d", "<0xE2>", "<0x9D>", "<0xA4>", "\u200d", "<0xF0>", "<0x9F>", "<0x92>", "<0x8B>", "\u200d", "<0xF0>", "<0x9F>", "<0x91>", "<0xA8>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0xA9>", "\u200d", "<0xF0>", "<0x9F>", "<0x91>", "<0xA9>", "\u200d", "<0xF0>", "<0x9F>", "<0x91>", "<0xA7>", "\u200d", "<0xF0>", "<0x9F>", "<0x91>", "<0xA6>", "\u2581", "<0xF0>", "<0x9F>", "<0xA7>", "<0x91>", "<0xF0>", "<0x9F>", "<0x8F>", "<0xBB>", "\u200d", "<0xF0>", "<0x9F>", "<0xA4>", "<0x9D>", "\u200d", "<0xF0>", "<0x9F>", "<0xA7>", "<0x91>", "<0xF0>", "<0x9F>", "<0x8F>", "<0xBB>", "\u2581", "<0xF0>", "<0x9F>", "<0x8F>", "<0xB4>", "<0xF3>", "<0xA0>", "<0x81>", "<0xA7>", "<0xF3>", "<0xA0>", "<0x81>", "<0xA2>", "<0xF3>", "<0xA0>", "<0x81>", "<0xA5>", "<0xF3>", "<0xA0>", "<0x81>", "<0xAE>", "<0xF3>", "<0xA0>", "<0x81>", "<0xA7>", "<0xF3>", "<0xA0>", "<0x81>", "<0xBF>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0xA8>", "<0xF0>", "<0x9F>", "<0x8F>", "<0xBB>", "\u200d", "<0xE2>", "<0x9D>", "<0xA4>", "\ufe0f", "\u200d", "<0xF0>", "<0x9F>", "<0x92>", "<0x8B>", "\u200d", "<0xF0>", "<0x9F>", "<0x91>", "<0xA8>", "<0xF0>", "<0x9F>", "<0x8F>", "<0xBC>"], - ids: [1, 29871, 229, 159, 171, 29871, 243, 162, 167, 154, 29871, 243, 162, 148, 132, 30598, 29871, 243, 162, 148, 180, 243, 162, 146, 190, 29871, 243, 162, 152, 184, 30722, 31135, 30598, 29871, 243, 162, 170, 156, 243, 162, 146, 190, 30722, 31135, 29871, 243, 162, 148, 171, 243, 162, 146, 190, 30722, 243, 162, 143, 193, 29871, 243, 162, 170, 148, 30722, 243, 162, 167, 160, 30722, 243, 162, 170, 148, 29871, 243, 162, 148, 172, 30722, 229, 160, 167, 30722, 243, 162, 149, 142, 30722, 243, 162, 148, 171, 29871, 243, 162, 148, 172, 30722, 243, 162, 148, 172, 30722, 243, 162, 148, 170, 30722, 243, 162, 148, 169, 29871, 243, 162, 170, 148, 243, 162, 146, 190, 30722, 243, 162, 167, 160, 30722, 243, 162, 170, 148, 243, 162, 146, 190, 29871, 243, 162, 146, 183, 246, 163, 132, 170, 246, 163, 132, 165, 246, 163, 132, 168, 246, 163, 132, 177, 246, 163, 132, 170, 246, 163, 132, 194, 29871, 243, 162, 148, 171, 243, 162, 146, 190, 30722, 229, 160, 167, 30598, 30722, 243, 162, 149, 142, 30722, 243, 162, 148, 171, 243, 162, 146, 191], - decoded: " \u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75\u200d\u2642\ufe0f \ud83e\uddd9\ud83c\udffb\u200d\u2642 \ud83d\udc68\ud83c\udffb\u200d\ud83c\udf3e \ud83e\uddd1\u200d\ud83e\udd1d\u200d\ud83e\uddd1 \ud83d\udc69\u200d\u2764\u200d\ud83d\udc8b\u200d\ud83d\udc68 \ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66 \ud83e\uddd1\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c\udffb \ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f \ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c\udffc", - }, - BPE_SCORES_PRIORITY_1: { - text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_1, - tokens: ["\u2581gra", "bb", "ed"], - ids: [1, 2646, 1327, 287], - decoded: " grabbed", - }, - BPE_SCORES_PRIORITY_2: { - text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_2, - tokens: ["\u2581", "\u2581gra", "bb", "ed"], - ids: [1, 29871, 2646, 1327, 287], - decoded: " grabbed", - }, - BPE_SCORES_PRIORITY_3: { - text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_3, - tokens: ["\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581", "\u2581gra", "bb", "ed"], - ids: [1, 9651, 2646, 1327, 287], - decoded: " grabbed", - }, - NEWLINE: { - text: LLAMA_TEST_STRINGS.NEWLINE, - tokens: ["\u2581", "<0x0A>"], - ids: [1, 29871, 13], - decoded: " \n", - }, - NEWLINE_WITH_LEADING_SPACE: { - text: LLAMA_TEST_STRINGS.NEWLINE_WITH_LEADING_SPACE, - tokens: ["\u2581\u2581", "<0x0A>"], - ids: [1, 259, 13], - decoded: " \n", - }, - TABS: { - text: LLAMA_TEST_STRINGS.TABS, - tokens: ["\u2581", "<0x09>", "tabs", "<0x09>", "<0x09>", "<0x09>", "<0x09>", "out", "\u2581here"], - ids: [1, 29871, 12, 21175, 12, 12, 12, 12, 449, 1244], - decoded: " \ttabs\t\t\t\tout here", - }, - NEWLINE_AND_TAB: { - text: LLAMA_TEST_STRINGS.NEWLINE_AND_TAB, - tokens: ["\u2581", "<0x0A>", "<0x09>", "<0x0A>"], - ids: [1, 29871, 13, 12, 13], - decoded: " \n\t\n", - }, - CHINESE_LETTER: { - text: LLAMA_TEST_STRINGS.CHINESE_LETTER, - tokens: ["\u2581", "\u9547"], - ids: [1, 29871, 30411], - decoded: " \u9547", - }, - EMOJIS_1: { - text: LLAMA_TEST_STRINGS.EMOJIS_1, - tokens: ["\u2581", "<0xF0>", "<0x9F>", "<0xA6>", "<0x99>"], - ids: [1, 29871, 243, 162, 169, 156], - decoded: " \ud83e\udd99", - }, - EMOJIS_2: { - text: LLAMA_TEST_STRINGS.EMOJIS_2, - tokens: ["\u2581", "<0xF0>", "<0x9F>", "<0xA6>", "<0x99>", "<0xEA>", "<0x99>", "<0x8A>"], - ids: [1, 29871, 243, 162, 169, 156, 237, 156, 141], - decoded: " \ud83e\udd99\ua64a", - }, - EMOJIS_3: { - text: LLAMA_TEST_STRINGS.EMOJIS_3, - tokens: ["\u2581", "<0xEA>", "<0x99>", "<0x8A>", "<0xF0>", "<0x9F>", "<0xA6>", "<0x99>"], - ids: [1, 29871, 237, 156, 141, 243, 162, 169, 156], - decoded: " \ua64a\ud83e\udd99", - }, - PARAGRAPH: { - text: LLAMA_TEST_STRINGS.PARAGRAPH, - tokens: ["\u2581The", "\u2581ll", "ama", "\u2581(/", "\u02c8", "l", "\u0251", "\u02d0", "m", "\u0259", "/", ";", "\u2581", "<0xF0>", "<0x9F>", "<0xA6>", "<0x99>", "Span", "ish", "\u2581pron", "unci", "ation", ":", "\u2581[", "\u02c8", "\u028e", "ama", "])", "\u2581(", "L", "ama", "\u2581gl", "ama", ")", "\u2581is", "\u2581a", "\u2581domestic", "ated", "\u2581South", "\u2581American", "\u2581cam", "el", "id", ",", "\u2581widely", "\u2581used", "\u2581as", "\u2581a", "\u2581meat", "\u2581and", "\u2581pack", "\u2581animal", "\u2581by", "\u2581And", "e", "an", "\u2581cult", "ures", "\u2581since", "\u2581the", "\u2581Pre", "-", "Col", "umb", "ian", "\u2581era", ".", "\u2581L", "lam", "as", "\u2581are", "\u2581social", "\u2581animals", "\u2581and", "\u2581live", "\u2581with", "\u2581others", "\u2581as", "\u2581a", "\u2581her", "d", ".", "\u2581Their", "\u2581w", "ool", "\u2581is", "\u2581soft", "\u2581and", "\u2581contains", "\u2581only", "\u2581a", "\u2581small", "\u2581amount", "\u2581of", "\u2581lan", "olin", ".[", "2", "]", "\u2581L", "lam", "as", "\u2581can", "\u2581learn", "\u2581simple", "\u2581tasks", "\u2581after", "\u2581a", "\u2581few", "\u2581repet", "itions", ".", "\u2581When", "\u2581using", "\u2581a", "\u2581pack", ",", "\u2581they", "\u2581can", "\u2581carry", "\u2581about", "\u2581", "2", "5", "\u2581to", "\u2581", "3", "0", "%", "\u2581of", "\u2581their", "\u2581body", "\u2581weight", "\u2581for", "\u2581", "8", "\u2581to", "\u2581", "1", "3", "\u2581km", "\u2581(", "5", "\u2013", "8", "\u2581miles", ").", "[", "3", "]", "\u2581The", "\u2581name", "\u2581ll", "ama", "\u2581(", "in", "\u2581the", "\u2581past", "\u2581also", "\u2581sp", "elled", '\u2581"', "l", "ama", '"', "\u2581or", '\u2581"', "gl", "ama", '")', "\u2581was", "\u2581adopted", "\u2581by", "\u2581European", "\u2581sett", "lers", "\u2581from", "\u2581native", "\u2581Peru", "vi", "ans", ".[", "4", "]", "\u2581The", "\u2581ancest", "ors", "\u2581of", "\u2581llam", "as", "\u2581are", "\u2581thought", "\u2581to", "\u2581have", "\u2581origin", "ated", "\u2581from", "\u2581the", "\u2581Great", "\u2581Pla", "ins", "\u2581of", "\u2581North", "\u2581America", "\u2581about", "\u2581", "4", "0", "\u2581million", "\u2581years", "\u2581ago", ",", "\u2581and", "\u2581subsequently", "\u2581migr", "ated", "\u2581to", "\u2581South", "\u2581America", "\u2581about", "\u2581three", "\u2581million", "\u2581years", "\u2581ago", "\u2581during", "\u2581the", "\u2581Great", "\u2581American", "\u2581Inter", "change", ".", "\u2581By", "\u2581the", "\u2581end", "\u2581of", "\u2581the", "\u2581last", "\u2581ice", "\u2581age", "\u2581(", "1", "0", ",", "0", "0", "0", "\u2013", "1", "2", ",", "0", "0", "0", "\u2581years", "\u2581ago", "),", "\u2581cam", "el", "ids", "\u2581were", "\u2581ext", "inct", "\u2581in", "\u2581North", "\u2581America", ".[", "3", "]", "\u2581As", "\u2581of", "\u2581", "2", "0", "0", "7", ",", "\u2581there", "\u2581were", "\u2581over", "\u2581seven", "\u2581million", "\u2581llam", "as", "\u2581and", "\u2581al", "p", "ac", "as", "\u2581in", "\u2581South", "\u2581America", "\u2581and", "\u2581over", "\u2581", "1", "5", "8", ",", "0", "0", "0", "\u2581llam", "as", "\u2581and", "\u2581", "1", "0", "0", ",", "0", "0", "0", "<0xEA>", "<0x99>", "<0x8A>", "<0xF0>", "<0x9F>", "<0xA6>", "<0x99>", "\u2581al", "p", "ac", "as", ",", "\u2581desc", "ended", "\u2581from", "\u2581pro", "gen", "itors", "\u2581imported", "\u2581late", "\u2581in", "\u2581the", "\u2581", "2", "0", "th", "\u2581century", ",", "\u2581in", "\u2581the", "\u2581United", "\u2581States", "\u2581and", "\u2581Canada", ".[", "5", "]", "\u2581In", "\u2581A", "ym", "ara", "\u2581myth", "ology", ",", "\u2581llam", "as", "\u2581are", "\u2581important", "\u2581be", "ings", ".", "\u2581The", "\u2581Heaven", "ly", "\u2581L", "l", "ama", "\u2581is", "\u2581said", "\u2581to", "\u2581drink", "\u2581water", "\u2581from", "\u2581the", "\u2581ocean", "\u2581and", "\u2581ur", "in", "ates", "\u2581as", "\u2581it", "\u2581ra", "ins", ".[", "6", "]", "\u2581According", "\u2581to", "\u2581A", "ym", "ara", "\u2581es", "chat", "ology", ",", "\u2581llam", "as", "\u2581will", "\u2581return", "\u2581to", "\u2581the", "\u2581water", "\u2581spr", "ings", "\u2581and", "\u2581l", "ago", "ons", "\u2581where", "\u2581they", "\u2581come", "\u2581from", "\u2581at", "\u2581the", "\u2581end", "\u2581of", "\u2581time", ".[", "6", "]"], - ids: [1, 450, 11148, 3304, 20374, 30176, 29880, 30426, 30215, 29885, 30184, 29914, 29936, 29871, 243, 162, 169, 156, 15495, 728, 11504, 11173, 362, 29901, 518, 30176, 31743, 3304, 2314, 313, 29931, 3304, 3144, 3304, 29897, 338, 263, 21849, 630, 4275, 3082, 3949, 295, 333, 29892, 17644, 1304, 408, 263, 27654, 322, 4870, 13019, 491, 1126, 29872, 273, 4185, 1973, 1951, 278, 4721, 29899, 1625, 3774, 713, 3152, 29889, 365, 5288, 294, 526, 5264, 15006, 322, 5735, 411, 4045, 408, 263, 902, 29881, 29889, 11275, 281, 1507, 338, 4964, 322, 3743, 871, 263, 2319, 5253, 310, 10906, 22878, 7226, 29906, 29962, 365, 5288, 294, 508, 5110, 2560, 9595, 1156, 263, 2846, 21159, 2187, 29889, 1932, 773, 263, 4870, 29892, 896, 508, 8677, 1048, 29871, 29906, 29945, 304, 29871, 29941, 29900, 29995, 310, 1009, 3573, 7688, 363, 29871, 29947, 304, 29871, 29896, 29941, 2383, 313, 29945, 29994, 29947, 7800, 467, 29961, 29941, 29962, 450, 1024, 11148, 3304, 313, 262, 278, 4940, 884, 805, 14356, 376, 29880, 3304, 29908, 470, 376, 3820, 3304, 1159, 471, 16356, 491, 7824, 3604, 9306, 515, 7531, 25493, 1403, 550, 7226, 29946, 29962, 450, 19525, 943, 310, 11829, 294, 526, 2714, 304, 505, 3978, 630, 515, 278, 7027, 13494, 1144, 310, 4644, 6813, 1048, 29871, 29946, 29900, 7284, 2440, 8020, 29892, 322, 17602, 9725, 630, 304, 4275, 6813, 1048, 2211, 7284, 2440, 8020, 2645, 278, 7027, 3082, 4124, 3167, 29889, 2648, 278, 1095, 310, 278, 1833, 14890, 5046, 313, 29896, 29900, 29892, 29900, 29900, 29900, 29994, 29896, 29906, 29892, 29900, 29900, 29900, 2440, 8020, 511, 3949, 295, 4841, 892, 1294, 5562, 297, 4644, 6813, 7226, 29941, 29962, 1094, 310, 29871, 29906, 29900, 29900, 29955, 29892, 727, 892, 975, 9881, 7284, 11829, 294, 322, 394, 29886, 562, 294, 297, 4275, 6813, 322, 975, 29871, 29896, 29945, 29947, 29892, 29900, 29900, 29900, 11829, 294, 322, 29871, 29896, 29900, 29900, 29892, 29900, 29900, 29900, 237, 156, 141, 243, 162, 169, 156, 394, 29886, 562, 294, 29892, 5153, 2760, 515, 410, 1885, 17259, 19673, 5683, 297, 278, 29871, 29906, 29900, 386, 6462, 29892, 297, 278, 3303, 3900, 322, 7400, 7226, 29945, 29962, 512, 319, 962, 2518, 22082, 3002, 29892, 11829, 294, 526, 4100, 367, 886, 29889, 450, 22977, 368, 365, 29880, 3304, 338, 1497, 304, 13748, 4094, 515, 278, 23474, 322, 5065, 262, 1078, 408, 372, 1153, 1144, 7226, 29953, 29962, 7579, 304, 319, 962, 2518, 831, 13496, 3002, 29892, 11829, 294, 674, 736, 304, 278, 4094, 7689, 886, 322, 301, 4425, 787, 988, 896, 2041, 515, 472, 278, 1095, 310, 931, 7226, 29953, 29962], - decoded: ' The llama (/\u02c8l\u0251\u02d0m\u0259/; \ud83e\udd99Spanish pronunciation: [\u02c8\u028eama]) (Lama glama) is a domesticated South American camelid, widely used as a meat and pack animal by Andean cultures since the Pre-Columbian era. Llamas are social animals and live with others as a herd. Their wool is soft and contains only a small amount of lanolin.[2] Llamas can learn simple tasks after a few repetitions. When using a pack, they can carry about 25 to 30% of their body weight for 8 to 13 km (5\u20138 miles).[3] The name llama (in the past also spelled "lama" or "glama") was adopted by European settlers from native Peruvians.[4] The ancestors of llamas are thought to have originated from the Great Plains of North America about 40 million years ago, and subsequently migrated to South America about three million years ago during the Great American Interchange. By the end of the last ice age (10,000\u201312,000 years ago), camelids were extinct in North America.[3] As of 2007, there were over seven million llamas and alpacas in South America and over 158,000 llamas and 100,000\ua64a\ud83e\udd99 alpacas, descended from progenitors imported late in the 20th century, in the United States and Canada.[5] In Aymara mythology, llamas are important beings. The Heavenly Llama is said to drink water from the ocean and urinates as it rains.[6] According to Aymara eschatology, llamas will return to the water springs and lagoons where they come from at the end of time.[6]', - }, - }, - "Xenova/llama3-tokenizer": { - SIMPLE: { - text: BASE_TEST_STRINGS.SIMPLE, - tokens: ["How", "\u0120are", "\u0120you", "\u0120doing", "?"], - ids: [4438, 527, 499, 3815, 30], - decoded: "How are you doing?", - }, - SIMPLE_WITH_PUNCTUATION: { - text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, - tokens: ["You", "\u0120should", "'ve", "\u0120done", "\u0120this"], - ids: [2675, 1288, 3077, 2884, 420], - decoded: "You should've done this", - }, - NUMBERS: { - text: BASE_TEST_STRINGS.NUMBERS, - tokens: ["012", "345", "678", "9", "\u0120", "0", "\u0120", "1", "\u0120", "2", "\u0120", "3", "\u0120", "4", "\u0120", "5", "\u0120", "6", "\u0120", "7", "\u0120", "8", "\u0120", "9", "\u0120", "10", "\u0120", "100", "\u0120", "100", "0"], - ids: [11531, 12901, 17458, 24, 220, 15, 220, 16, 220, 17, 220, 18, 220, 19, 220, 20, 220, 21, 220, 22, 220, 23, 220, 24, 220, 605, 220, 1041, 220, 1041, 15], - decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["The", "\u0120company", "\u0120was", "\u0120founded", "\u0120in", "\u0120", "201", "6", "."], - ids: [791, 2883, 574, 18538, 304, 220, 679, 21, 13], - decoded: "The company was founded in 2016.", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["A", "\u010a", "'ll", "\u0120!!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'t", "."], - ids: [32, 198, 3358, 11261, 998, 20837, 67, 4708, 67, 315, 11, 649, 956, 13], - decoded: "A\n'll!!to?'d''d of, can't.", - }, - PYTHON_CODE: { - text: BASE_TEST_STRINGS.PYTHON_CODE, - tokens: ["def", "\u0120main", "():\u010a", "\u0109pass"], - ids: [755, 1925, 4019, 42531], - decoded: "def main():\n\tpass", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".toString", "();\u010a", "toString", "();"], - ids: [1169, 264, 284, 2909, 5180, 545, 6712, 2178], - decoded: "let a = obj.toString();\ntoString();", - }, - NEWLINES: { - text: LLAMA_TEST_STRINGS.NEWLINES, - tokens: ["ax", "\u010a", "####\u010a", "boo"], - ids: [710, 198, 71050, 34093], - decoded: "ax\n####\nboo", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["UN", "want", "\u00c3\u00a9d", ",", "running"], - ids: [1899, 53757, 15433, 11, 28272], - decoded: "UNwant\u00e9d,running", - }, - CONTROL_TOKENS: { - text: BASE_TEST_STRINGS.CONTROL_TOKENS, - tokens: ["1", "\u0100", "2", "\u00ef\u00bf\u00bd", "3"], - ids: [16, 188, 17, 5809, 18], - decoded: "1\u00002\ufffd3", - }, - HELLO_WORLD_TITLECASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, - tokens: ["Hello", "\u0120World"], - ids: [9906, 4435], - decoded: "Hello World", - }, - HELLO_WORLD_LOWERCASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, - tokens: ["hello", "\u0120world"], - ids: [15339, 1917], - decoded: "hello world", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u00e7\u0136\u0141\u00e6\u00b4\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"], - ids: [104654, 9554, 89151, 39013, 249, 21043], - decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", - }, - LEADING_SPACE: { - text: BASE_TEST_STRINGS.LEADING_SPACE, - tokens: ["\u0120\u0120", "\u0120leading", "\u0120space"], - ids: [256, 6522, 3634], - decoded: " leading space", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["tr", "ailing", "\u0120space", "\u0120\u0120\u0120"], - ids: [376, 14612, 3634, 262], - decoded: "trailing space ", - }, - DOUBLE_SPACE: { - text: BASE_TEST_STRINGS.DOUBLE_SPACE, - tokens: ["Hi", "\u0120", "\u0120Hello"], - ids: [13347, 220, 22691], - decoded: "Hi Hello", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2\u00a5", "6", "\u0120\u00e2\u0124", "\u00a3", "7", "\u0120\u00e2\u0124\u00b9", "8", "\u0120\u00e2\u0124", "\u00b1", "9", "\u0120test"], - ids: [1985, 400, 16, 432, 17, 674, 18, 13281, 19, 7083, 20, 72588, 21, 113384, 96, 22, 90891, 23, 113384, 109, 24, 1296], - decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["I", "\u0120bought", "\u0120an", "\u0120apple", "\u0120for", "\u0120$", "1", ".", "00", "\u0120at", "\u0120the", "\u0120store", "."], - ids: [40, 11021, 459, 24149, 369, 400, 16, 13, 410, 520, 279, 3637, 13], - decoded: "I bought an apple for $1.00 at the store.", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - tokens: ["you", "\u00e2\u0122\u00a6", "\u0120\u0120"], - ids: [9514, 1981, 256], - decoded: "you\u2026 ", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], - ids: [9514, 1981, 9421], - decoded: "you\u2026\u00a0\u00a0", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142", "\u00c2\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], - ids: [9514, 1981, 4194, 4194, 9514, 1981, 9421], - decoded: "you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["we", "ird", "\u0120\u00ef\u00bd\u0140", "\u0120edge", "\u0120\u00ef\u00bd\u0140", "\u0120case"], - ids: [906, 2668, 111942, 6964, 111942, 1162], - decoded: "weird \uff5e edge \uff5e case", - }, - SPIECE_UNDERSCORE: { - text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, - tokens: ["\u00e2\u0138", "\u0123", "This", "\u0120\u00e2\u0138", "\u0123", "is", "\u0120\u00e2\u0138", "\u0123", "a", "\u0120\u00e2\u0138", "\u0123", "test", "\u0120\u00e2\u0138", "\u0123", "."], - ids: [10634, 223, 2028, 14860, 223, 285, 14860, 223, 64, 14860, 223, 1985, 14860, 223, 13], - decoded: "\u2581This \u2581is \u2581a \u2581test \u2581.", - }, - POPULAR_EMOJIS: { - text: BASE_TEST_STRINGS.POPULAR_EMOJIS, - tokens: ["\u00f0\u0141\u013a", "\u0124", "\u0120\u00f0\u0141\u0133", "\u012f", "\u0120\u00f0\u0141", "\u00a4", "\u00a3", "\u0120\u00f0\u0141\u013a", "\u012f", "\u0120\u00f0\u0141\u013a", "\u0143", "\u0120\u00f0\u0141", "\u0130", "\u012b", "\u0120\u00f0\u0141", "\u013b", "\u0131", "\u0120\u00f0\u0141\u013a", "\u012c", "\u0120\u00f0\u0141\u0136", "\u00a5", "\u0120\u00f0\u0141\u013a", "\u0123", "\u0120\u00f0\u0141\u013a", "\u0127", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141\u013a", "\u0128", "\u0120\u00f0\u0141\u0133", "\u0131", "\u0120\u00e2\u013f\u00a4", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141\u0134", "\u013e", "\u0120\u00f0\u0141\u0134", "\u013c", "\u0120\u00f0\u0141\u0134", "\u0139", "\u0120\u00f0\u0141\u0134", "\u013b", "\u0120\u00f0\u0141", "\u0138", "\u00a4", "\u0120\u00f0\u0141\u013a", "\u0130", "\u0120\u00f0\u0141\u0133", "\u012e", "\u0120\u00f0\u0141", "\u00a5", "\u00b3", "\u0120\u00f0\u0141\u0134", "\u00aa", "\u0120\u00e2\u013e", "\u00a8", "\u0120\u00f0\u0141\u0133", "\u012b", "\u0120\u00f0\u0141\u0133", "\u0122", "\u0120\u00f0\u0141\u0134", "\u00af", "\u0120\u00f0\u0141", "\u0130", "\u012a", "\u0120\u00f0\u0141", "\u013b", "\u012a", "\u0120\u00f0\u0141", "\u013b", "\u012e", "\u0120\u00f0\u0141\u0134", "\u0122", "\u0120\u00f0\u0141\u0133", "\u0129", "\u0120\u00f0\u0141\u0133", "\u012d", "\u0120\u00e2\u013e", "\u0127", "\u0120\u00f0\u0141", "\u0130", "\u0123", "\u0120\u00f0\u0141", "\u012e", "\u0140", "\u0120\u00f0\u0141", "\u012e", "\u00b8", "\u0120\u00f0\u0141\u0134", "\u00b0"], - ids: [76460, 224, 62904, 235, 11410, 97, 96, 27623, 235, 27623, 255, 11410, 236, 231, 11410, 247, 237, 27623, 232, 96169, 98, 27623, 223, 27623, 227, 11410, 97, 245, 27623, 228, 62904, 237, 71570, 31643, 64139, 250, 64139, 248, 64139, 245, 64139, 247, 11410, 244, 97, 27623, 236, 62904, 234, 11410, 98, 111, 64139, 103, 26602, 101, 62904, 231, 62904, 222, 64139, 107, 11410, 236, 230, 11410, 247, 230, 11410, 247, 234, 64139, 222, 62904, 229, 62904, 233, 26602, 227, 11410, 236, 223, 11410, 234, 252, 11410, 234, 116, 64139, 108], - decoded: "\ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0", - }, - MULTIBYTE_EMOJIS: { - text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, - tokens: ["\u00e2\u013e", "\u00a8", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141\u0133", "\u0123", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141\u0133", "\u00b1", "\u00f0\u0141", "\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0137", "\u00b5", "\u00e2\u0122\u012f", "\u00e2\u013b", "\u0124", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141", "\u00a7", "\u013b", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00e2\u013b", "\u0124", "\u0120\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u012e", "\u00be", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u0120\u00f0\u0141\u0133", "\u00a9", "\u00e2\u0122\u012f", "\u00e2\u013f\u00a4", "\u00e2\u0122\u012f", "\u00f0\u0141\u0134", "\u012d", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a8", "\u0120\u00f0\u0141\u0133", "\u00a9", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a9", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a7", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a6", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141", "\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0131", "\u00b4", "\u00f3", "\u0142\u0123", "\u00a7", "\u00f3", "\u0142\u0123", "\u00a2", "\u00f3", "\u0142\u0123", "\u00a5", "\u00f3", "\u0142\u0123", "\u00ae", "\u00f3", "\u0142\u0123", "\u00a7", "\u00f3", "\u0142\u0123", "\u00bf", "\u0120\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00e2\u013f\u00a4", "\u00ef\u00b8\u0131", "\u00e2\u0122\u012f", "\u00f0\u0141\u0134", "\u012d", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bc"], - ids: [38798, 101, 11410, 97, 245, 62904, 223, 31643, 62904, 109, 9468, 237, 119, 11410, 243, 113, 102470, 17245, 224, 31643, 11410, 100, 247, 9468, 237, 119, 102470, 17245, 224, 62904, 101, 9468, 237, 119, 102470, 9468, 234, 122, 11410, 100, 239, 102470, 9468, 97, 251, 102470, 9468, 100, 239, 62904, 102, 102470, 121643, 102470, 93273, 233, 102470, 9468, 239, 101, 62904, 102, 102470, 9468, 239, 102, 102470, 9468, 239, 100, 102470, 9468, 239, 99, 11410, 100, 239, 9468, 237, 119, 102470, 9468, 97, 251, 102470, 9468, 100, 239, 9468, 237, 119, 11410, 237, 112, 175, 16050, 100, 175, 16050, 95, 175, 16050, 98, 175, 16050, 106, 175, 16050, 100, 175, 16050, 123, 62904, 101, 9468, 237, 119, 102470, 121643, 31643, 102470, 93273, 233, 102470, 9468, 239, 101, 9468, 237, 120], - decoded: "\u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75\u200d\u2642\ufe0f \ud83e\uddd9\ud83c\udffb\u200d\u2642 \ud83d\udc68\ud83c\udffb\u200d\ud83c\udf3e \ud83e\uddd1\u200d\ud83e\udd1d\u200d\ud83e\uddd1 \ud83d\udc69\u200d\u2764\u200d\ud83d\udc8b\u200d\ud83d\udc68 \ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66 \ud83e\uddd1\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c\udffb \ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f \ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c\udffc", - }, - BPE_SCORES_PRIORITY_1: { - text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_1, - tokens: ["grab", "bed"], - ids: [59312, 2788], - decoded: "grabbed", - }, - BPE_SCORES_PRIORITY_2: { - text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_2, - tokens: ["\u0120grabbed"], - ids: [30418], - decoded: " grabbed", - }, - BPE_SCORES_PRIORITY_3: { - text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_3, - tokens: ["\u0120\u0120\u0120\u0120\u0120\u0120\u0120\u0120\u0120\u0120", "\u0120grabbed"], - ids: [1881, 30418], - decoded: " grabbed", - }, - NEWLINE: { - text: LLAMA_TEST_STRINGS.NEWLINE, - tokens: ["\u010a"], - ids: [198], - decoded: "\n", - }, - NEWLINE_WITH_LEADING_SPACE: { - text: LLAMA_TEST_STRINGS.NEWLINE_WITH_LEADING_SPACE, - tokens: ["\u0120\u010a"], - ids: [720], - decoded: " \n", - }, - TABS: { - text: LLAMA_TEST_STRINGS.TABS, - tokens: ["\u0109t", "abs", "\u0109\u0109\u0109", "\u0109out", "\u0120here"], - ids: [3324, 3518, 573, 14294, 1618], - decoded: "\ttabs\t\t\t\tout here", - }, - NEWLINE_AND_TAB: { - text: LLAMA_TEST_STRINGS.NEWLINE_AND_TAB, - tokens: ["\u010a\u0109\u010a"], - ids: [18108], - decoded: "\n\t\n", - }, - CHINESE_LETTER: { - text: LLAMA_TEST_STRINGS.CHINESE_LETTER, - tokens: ["\u00e9\u0137\u0129"], - ids: [104643], - decoded: "\u9547", - }, - EMOJIS_1: { - text: LLAMA_TEST_STRINGS.EMOJIS_1, - tokens: ["\u00f0\u0141", "\u00a6", "\u013b"], - ids: [9468, 99, 247], - decoded: "\ud83e\udd99", - }, - EMOJIS_2: { - text: LLAMA_TEST_STRINGS.EMOJIS_2, - tokens: ["\u00f0\u0141", "\u00a6", "\u013b", "\u00ea", "\u013b", "\u012c"], - ids: [9468, 99, 247, 166, 247, 232], - decoded: "\ud83e\udd99\ua64a", - }, - EMOJIS_3: { - text: LLAMA_TEST_STRINGS.EMOJIS_3, - tokens: ["\u00ea", "\u013b", "\u012c", "\u00f0\u0141", "\u00a6", "\u013b"], - ids: [166, 247, 232, 9468, 99, 247], - decoded: "\ua64a\ud83e\udd99", + "Xenova/llama-tokenizer": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["\u2581How", "\u2581are", "\u2581you", "\u2581doing", "?"], + ids: [1, 1128, 526, 366, 2599, 29973], + decoded: " How are you doing?", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["\u2581You", "\u2581should", "'", "ve", "\u2581done", "\u2581this"], + ids: [1, 887, 881, 29915, 345, 2309, 445], + decoded: " You should've done this", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["\u2581", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "\u2581", "0", "\u2581", "1", "\u2581", "2", "\u2581", "3", "\u2581", "4", "\u2581", "5", "\u2581", "6", "\u2581", "7", "\u2581", "8", "\u2581", "9", "\u2581", "1", "0", "\u2581", "1", "0", "0", "\u2581", "1", "0", "0", "0"], + ids: [1, 29871, 29900, 29896, 29906, 29941, 29946, 29945, 29953, 29955, 29947, 29929, 29871, 29900, 29871, 29896, 29871, 29906, 29871, 29941, 29871, 29946, 29871, 29945, 29871, 29953, 29871, 29955, 29871, 29947, 29871, 29929, 29871, 29896, 29900, 29871, 29896, 29900, 29900, 29871, 29896, 29900, 29900, 29900], + decoded: " 0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["\u2581The", "\u2581company", "\u2581was", "\u2581founded", "\u2581in", "\u2581", "2", "0", "1", "6", "."], + ids: [1, 450, 5001, 471, 11091, 297, 29871, 29906, 29900, 29896, 29953, 29889], + decoded: " The company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["\u2581A", "<0x0A>", "'", "ll", "\u2581!!", "to", "?'", "d", "''", "d", "\u2581of", ",", "\u2581can", "'", "t", "."], + ids: [1, 319, 13, 29915, 645, 21443, 517, 17901, 29881, 4907, 29881, 310, 29892, 508, 29915, 29873, 29889], + decoded: " A\n'll !!to?'d''d of, can't.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["\u2581def", "\u2581main", "():", "<0x0A>", "<0x09>", "pass"], + ids: [1, 822, 1667, 7295, 13, 12, 3364], + decoded: " def main():\n\tpass", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["\u2581let", "\u2581a", "\u2581=", "\u2581obj", ".", "toString", "();", "<0x0A>", "toString", "();"], + ids: [1, 1235, 263, 353, 5446, 29889, 7711, 890, 13, 7711, 890], + decoded: " let a = obj.toString();\ntoString();", + }, + NEWLINES: { + text: LLAMA_TEST_STRINGS.NEWLINES, + tokens: ["\u2581ax", "<0x0A>", "####", "<0x0A>", "bo", "o"], + ids: [1, 4853, 13, 4136, 13, 833, 29877], + decoded: " ax\n####\nboo", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["\u2581UN", "w", "ant", "\u00e9d", ",", "running"], + ids: [1, 8291, 29893, 424, 2487, 29892, 21094], + decoded: " UNwant\u00e9d,running", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["\u2581", "1", "<0x00>", "2", "\ufffd", "3"], + ids: [1, 29871, 29896, 3, 29906, 30140, 29941], + decoded: " 1\u00002\ufffd3", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["\u2581Hello", "\u2581World"], + ids: [1, 15043, 2787], + decoded: " Hello World", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["\u2581hello", "\u2581world"], + ids: [1, 22172, 3186], + decoded: " hello world", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u2581", "\u751f", "\u6d3b", "\u7684", "\u771f", "<0xE8>", "<0xB0>", "<0x9B>", "\u662f"], + ids: [1, 29871, 30486, 31704, 30210, 30848, 235, 179, 158, 30392], + decoded: " \u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u2581\u2581\u2581", "\u2581leading", "\u2581space"], + ids: [1, 1678, 8236, 2913], + decoded: " leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["\u2581trailing", "\u2581space", "\u2581\u2581\u2581"], + ids: [1, 25053, 2913, 1678], + decoded: " trailing space ", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["\u2581Hi", "\u2581", "\u2581Hello"], + ids: [1, 6324, 29871, 15043], + decoded: " Hi Hello", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["\u2581test", "\u2581$", "1", "\u2581R", "2", "\u2581#", "3", "\u2581\u20ac", "4", "\u2581\u00a3", "5", "\u2581", "\u00a5", "6", "\u2581", "<0xE2>", "<0x82>", "<0xA3>", "7", "\u2581", "\u20b9", "8", "\u2581", "<0xE2>", "<0x82>", "<0xB1>", "9", "\u2581test"], + ids: [1, 1243, 395, 29896, 390, 29906, 396, 29941, 25540, 29946, 15151, 29945, 29871, 30563, 29953, 29871, 229, 133, 166, 29955, 29871, 30620, 29947, 29871, 229, 133, 180, 29929, 1243], + decoded: " test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["\u2581I", "\u2581bought", "\u2581an", "\u2581apple", "\u2581for", "\u2581$", "1", ".", "0", "0", "\u2581at", "\u2581the", "\u2581store", "."], + ids: [1, 306, 18093, 385, 26163, 363, 395, 29896, 29889, 29900, 29900, 472, 278, 3787, 29889], + decoded: " I bought an apple for $1.00 at the store.", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["\u2581you", "\u2026", "\u2581\u2581"], + ids: [1, 366, 30098, 259], + decoded: " you\u2026 ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["\u2581you", "\u2026", "\u00a0\u00a0"], + ids: [1, 366, 30098, 8655], + decoded: " you\u2026\u00a0\u00a0", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["\u2581you", "\u2026", "\u00a0\u00a0", "you", "\u2026", "\u00a0\u00a0"], + ids: [1, 366, 30098, 8655, 6293, 30098, 8655], + decoded: " you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["\u2581weird", "\u2581", "\uff5e", "\u2581edge", "\u2581", "\uff5e", "\u2581case"], + ids: [1, 13543, 29871, 30739, 7636, 29871, 30739, 1206], + decoded: " weird \uff5e edge \uff5e case", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u2581", "\u2581This", "\u2581", "\u2581is", "\u2581", "\u2581a", "\u2581", "\u2581test", "\u2581", "\u2581."], + ids: [1, 29871, 910, 29871, 338, 29871, 263, 29871, 1243, 29871, 869], + decoded: " This is a test .", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0x82>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x8D>", "\u2581", "<0xF0>", "<0x9F>", "<0xA4>", "<0xA3>", "\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0x8D>", "\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0xAD>", "\u2581", "<0xF0>", "<0x9F>", "<0x8E>", "<0x89>", "\u2581", "<0xF0>", "<0x9F>", "<0x99>", "<0x8F>", "\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0x8A>", "\u2581", "<0xF0>", "<0x9F>", "<0x94>", "<0xA5>", "\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0x81>", "\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0x85>", "\u2581", "<0xF0>", "<0x9F>", "<0xA4>", "<0x97>", "\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0x86>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x8F>", "\u2581", "<0xE2>", "<0x9D>", "<0xA4>", "\ufe0f", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0x9C>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0x9A>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0x97>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0x99>", "\u2581", "<0xF0>", "<0x9F>", "<0x96>", "<0xA4>", "\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0x8E>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x8C>", "\u2581", "<0xF0>", "<0x9F>", "<0xA5>", "<0xB3>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0xAA>", "\u2581", "<0xE2>", "<0x9C>", "<0xA8>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x89>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x80>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0xAF>", "\u2581", "<0xF0>", "<0x9F>", "<0x8E>", "<0x88>", "\u2581", "<0xF0>", "<0x9F>", "<0x99>", "<0x88>", "\u2581", "<0xF0>", "<0x9F>", "<0x99>", "<0x8C>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0x80>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x87>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x8B>", "\u2581", "\u2705", "\u2581", "<0xF0>", "<0x9F>", "<0x8E>", "<0x81>", "\u2581", "<0xF0>", "<0x9F>", "<0x8C>", "<0x9E>", "\u2581", "<0xF0>", "<0x9F>", "<0x8C>", "<0xB8>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0xB0>"], + ids: [1, 29871, 243, 162, 155, 133, 29871, 243, 162, 148, 144, 29871, 243, 162, 167, 166, 29871, 243, 162, 155, 144, 29871, 243, 162, 155, 176, 29871, 243, 162, 145, 140, 29871, 243, 162, 156, 146, 29871, 243, 162, 155, 141, 29871, 243, 162, 151, 168, 29871, 243, 162, 155, 132, 29871, 243, 162, 155, 136, 29871, 243, 162, 167, 154, 29871, 243, 162, 155, 137, 29871, 243, 162, 148, 146, 29871, 229, 160, 167, 30598, 29871, 243, 162, 149, 159, 29871, 243, 162, 149, 157, 29871, 243, 162, 149, 154, 29871, 243, 162, 149, 156, 29871, 243, 162, 153, 167, 29871, 243, 162, 155, 145, 29871, 243, 162, 148, 143, 29871, 243, 162, 168, 182, 29871, 243, 162, 149, 173, 29871, 229, 159, 171, 29871, 243, 162, 148, 140, 29871, 243, 162, 148, 131, 29871, 243, 162, 149, 178, 29871, 243, 162, 145, 139, 29871, 243, 162, 156, 139, 29871, 243, 162, 156, 143, 29871, 243, 162, 149, 131, 29871, 243, 162, 148, 138, 29871, 243, 162, 148, 142, 29871, 31681, 29871, 243, 162, 145, 132, 29871, 243, 162, 143, 161, 29871, 243, 162, 143, 187, 29871, 243, 162, 149, 179], + decoded: " \ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u2581", "<0xE2>", "<0x9C>", "<0xA8>", "\u2581", "<0xF0>", "<0x9F>", "<0xA4>", "<0x97>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x81>", "\ufe0f", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0xB1>", "<0xF0>", "<0x9F>", "<0x8F>", "<0xBB>", "\u2581", "<0xF0>", "<0x9F>", "<0x95>", "<0xB5>", "\u200d", "\u2642", "\ufe0f", "\u2581", "<0xF0>", "<0x9F>", "<0xA7>", "<0x99>", "<0xF0>", "<0x9F>", "<0x8F>", "<0xBB>", "\u200d", "\u2642", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0xA8>", "<0xF0>", "<0x9F>", "<0x8F>", "<0xBB>", "\u200d", "<0xF0>", "<0x9F>", "<0x8C>", "<0xBE>", "\u2581", "<0xF0>", "<0x9F>", "<0xA7>", "<0x91>", "\u200d", "<0xF0>", "<0x9F>", "<0xA4>", "<0x9D>", "\u200d", "<0xF0>", "<0x9F>", "<0xA7>", "<0x91>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0xA9>", "\u200d", "<0xE2>", "<0x9D>", "<0xA4>", "\u200d", "<0xF0>", "<0x9F>", "<0x92>", "<0x8B>", "\u200d", "<0xF0>", "<0x9F>", "<0x91>", "<0xA8>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0xA9>", "\u200d", "<0xF0>", "<0x9F>", "<0x91>", "<0xA9>", "\u200d", "<0xF0>", "<0x9F>", "<0x91>", "<0xA7>", "\u200d", "<0xF0>", "<0x9F>", "<0x91>", "<0xA6>", "\u2581", "<0xF0>", "<0x9F>", "<0xA7>", "<0x91>", "<0xF0>", "<0x9F>", "<0x8F>", "<0xBB>", "\u200d", "<0xF0>", "<0x9F>", "<0xA4>", "<0x9D>", "\u200d", "<0xF0>", "<0x9F>", "<0xA7>", "<0x91>", "<0xF0>", "<0x9F>", "<0x8F>", "<0xBB>", "\u2581", "<0xF0>", "<0x9F>", "<0x8F>", "<0xB4>", "<0xF3>", "<0xA0>", "<0x81>", "<0xA7>", "<0xF3>", "<0xA0>", "<0x81>", "<0xA2>", "<0xF3>", "<0xA0>", "<0x81>", "<0xA5>", "<0xF3>", "<0xA0>", "<0x81>", "<0xAE>", "<0xF3>", "<0xA0>", "<0x81>", "<0xA7>", "<0xF3>", "<0xA0>", "<0x81>", "<0xBF>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0xA8>", "<0xF0>", "<0x9F>", "<0x8F>", "<0xBB>", "\u200d", "<0xE2>", "<0x9D>", "<0xA4>", "\ufe0f", "\u200d", "<0xF0>", "<0x9F>", "<0x92>", "<0x8B>", "\u200d", "<0xF0>", "<0x9F>", "<0x91>", "<0xA8>", "<0xF0>", "<0x9F>", "<0x8F>", "<0xBC>"], + ids: [1, 29871, 229, 159, 171, 29871, 243, 162, 167, 154, 29871, 243, 162, 148, 132, 30598, 29871, 243, 162, 148, 180, 243, 162, 146, 190, 29871, 243, 162, 152, 184, 30722, 31135, 30598, 29871, 243, 162, 170, 156, 243, 162, 146, 190, 30722, 31135, 29871, 243, 162, 148, 171, 243, 162, 146, 190, 30722, 243, 162, 143, 193, 29871, 243, 162, 170, 148, 30722, 243, 162, 167, 160, 30722, 243, 162, 170, 148, 29871, 243, 162, 148, 172, 30722, 229, 160, 167, 30722, 243, 162, 149, 142, 30722, 243, 162, 148, 171, 29871, 243, 162, 148, 172, 30722, 243, 162, 148, 172, 30722, 243, 162, 148, 170, 30722, 243, 162, 148, 169, 29871, 243, 162, 170, 148, 243, 162, 146, 190, 30722, 243, 162, 167, 160, 30722, 243, 162, 170, 148, 243, 162, 146, 190, 29871, 243, 162, 146, 183, 246, 163, 132, 170, 246, 163, 132, 165, 246, 163, 132, 168, 246, 163, 132, 177, 246, 163, 132, 170, 246, 163, 132, 194, 29871, 243, 162, 148, 171, 243, 162, 146, 190, 30722, 229, 160, 167, 30598, 30722, 243, 162, 149, 142, 30722, 243, 162, 148, 171, 243, 162, 146, 191], + decoded: " \u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75\u200d\u2642\ufe0f \ud83e\uddd9\ud83c\udffb\u200d\u2642 \ud83d\udc68\ud83c\udffb\u200d\ud83c\udf3e \ud83e\uddd1\u200d\ud83e\udd1d\u200d\ud83e\uddd1 \ud83d\udc69\u200d\u2764\u200d\ud83d\udc8b\u200d\ud83d\udc68 \ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66 \ud83e\uddd1\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c\udffb \ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f \ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c\udffc", + }, + BPE_SCORES_PRIORITY_1: { + text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_1, + tokens: ["\u2581gra", "bb", "ed"], + ids: [1, 2646, 1327, 287], + decoded: " grabbed", + }, + BPE_SCORES_PRIORITY_2: { + text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_2, + tokens: ["\u2581", "\u2581gra", "bb", "ed"], + ids: [1, 29871, 2646, 1327, 287], + decoded: " grabbed", + }, + BPE_SCORES_PRIORITY_3: { + text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_3, + tokens: ["\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581", "\u2581gra", "bb", "ed"], + ids: [1, 9651, 2646, 1327, 287], + decoded: " grabbed", + }, + NEWLINE: { + text: LLAMA_TEST_STRINGS.NEWLINE, + tokens: ["\u2581", "<0x0A>"], + ids: [1, 29871, 13], + decoded: " \n", + }, + NEWLINE_WITH_LEADING_SPACE: { + text: LLAMA_TEST_STRINGS.NEWLINE_WITH_LEADING_SPACE, + tokens: ["\u2581\u2581", "<0x0A>"], + ids: [1, 259, 13], + decoded: " \n", + }, + TABS: { + text: LLAMA_TEST_STRINGS.TABS, + tokens: ["\u2581", "<0x09>", "tabs", "<0x09>", "<0x09>", "<0x09>", "<0x09>", "out", "\u2581here"], + ids: [1, 29871, 12, 21175, 12, 12, 12, 12, 449, 1244], + decoded: " \ttabs\t\t\t\tout here", + }, + NEWLINE_AND_TAB: { + text: LLAMA_TEST_STRINGS.NEWLINE_AND_TAB, + tokens: ["\u2581", "<0x0A>", "<0x09>", "<0x0A>"], + ids: [1, 29871, 13, 12, 13], + decoded: " \n\t\n", + }, + CHINESE_LETTER: { + text: LLAMA_TEST_STRINGS.CHINESE_LETTER, + tokens: ["\u2581", "\u9547"], + ids: [1, 29871, 30411], + decoded: " \u9547", + }, + EMOJIS_1: { + text: LLAMA_TEST_STRINGS.EMOJIS_1, + tokens: ["\u2581", "<0xF0>", "<0x9F>", "<0xA6>", "<0x99>"], + ids: [1, 29871, 243, 162, 169, 156], + decoded: " \ud83e\udd99", + }, + EMOJIS_2: { + text: LLAMA_TEST_STRINGS.EMOJIS_2, + tokens: ["\u2581", "<0xF0>", "<0x9F>", "<0xA6>", "<0x99>", "<0xEA>", "<0x99>", "<0x8A>"], + ids: [1, 29871, 243, 162, 169, 156, 237, 156, 141], + decoded: " \ud83e\udd99\ua64a", + }, + EMOJIS_3: { + text: LLAMA_TEST_STRINGS.EMOJIS_3, + tokens: ["\u2581", "<0xEA>", "<0x99>", "<0x8A>", "<0xF0>", "<0x9F>", "<0xA6>", "<0x99>"], + ids: [1, 29871, 237, 156, 141, 243, 162, 169, 156], + decoded: " \ua64a\ud83e\udd99", + }, + PARAGRAPH: { + text: LLAMA_TEST_STRINGS.PARAGRAPH, + tokens: ["\u2581The", "\u2581ll", "ama", "\u2581(/", "\u02c8", "l", "\u0251", "\u02d0", "m", "\u0259", "/", ";", "\u2581", "<0xF0>", "<0x9F>", "<0xA6>", "<0x99>", "Span", "ish", "\u2581pron", "unci", "ation", ":", "\u2581[", "\u02c8", "\u028e", "ama", "])", "\u2581(", "L", "ama", "\u2581gl", "ama", ")", "\u2581is", "\u2581a", "\u2581domestic", "ated", "\u2581South", "\u2581American", "\u2581cam", "el", "id", ",", "\u2581widely", "\u2581used", "\u2581as", "\u2581a", "\u2581meat", "\u2581and", "\u2581pack", "\u2581animal", "\u2581by", "\u2581And", "e", "an", "\u2581cult", "ures", "\u2581since", "\u2581the", "\u2581Pre", "-", "Col", "umb", "ian", "\u2581era", ".", "\u2581L", "lam", "as", "\u2581are", "\u2581social", "\u2581animals", "\u2581and", "\u2581live", "\u2581with", "\u2581others", "\u2581as", "\u2581a", "\u2581her", "d", ".", "\u2581Their", "\u2581w", "ool", "\u2581is", "\u2581soft", "\u2581and", "\u2581contains", "\u2581only", "\u2581a", "\u2581small", "\u2581amount", "\u2581of", "\u2581lan", "olin", ".[", "2", "]", "\u2581L", "lam", "as", "\u2581can", "\u2581learn", "\u2581simple", "\u2581tasks", "\u2581after", "\u2581a", "\u2581few", "\u2581repet", "itions", ".", "\u2581When", "\u2581using", "\u2581a", "\u2581pack", ",", "\u2581they", "\u2581can", "\u2581carry", "\u2581about", "\u2581", "2", "5", "\u2581to", "\u2581", "3", "0", "%", "\u2581of", "\u2581their", "\u2581body", "\u2581weight", "\u2581for", "\u2581", "8", "\u2581to", "\u2581", "1", "3", "\u2581km", "\u2581(", "5", "\u2013", "8", "\u2581miles", ").", "[", "3", "]", "\u2581The", "\u2581name", "\u2581ll", "ama", "\u2581(", "in", "\u2581the", "\u2581past", "\u2581also", "\u2581sp", "elled", '\u2581"', "l", "ama", '"', "\u2581or", '\u2581"', "gl", "ama", '")', "\u2581was", "\u2581adopted", "\u2581by", "\u2581European", "\u2581sett", "lers", "\u2581from", "\u2581native", "\u2581Peru", "vi", "ans", ".[", "4", "]", "\u2581The", "\u2581ancest", "ors", "\u2581of", "\u2581llam", "as", "\u2581are", "\u2581thought", "\u2581to", "\u2581have", "\u2581origin", "ated", "\u2581from", "\u2581the", "\u2581Great", "\u2581Pla", "ins", "\u2581of", "\u2581North", "\u2581America", "\u2581about", "\u2581", "4", "0", "\u2581million", "\u2581years", "\u2581ago", ",", "\u2581and", "\u2581subsequently", "\u2581migr", "ated", "\u2581to", "\u2581South", "\u2581America", "\u2581about", "\u2581three", "\u2581million", "\u2581years", "\u2581ago", "\u2581during", "\u2581the", "\u2581Great", "\u2581American", "\u2581Inter", "change", ".", "\u2581By", "\u2581the", "\u2581end", "\u2581of", "\u2581the", "\u2581last", "\u2581ice", "\u2581age", "\u2581(", "1", "0", ",", "0", "0", "0", "\u2013", "1", "2", ",", "0", "0", "0", "\u2581years", "\u2581ago", "),", "\u2581cam", "el", "ids", "\u2581were", "\u2581ext", "inct", "\u2581in", "\u2581North", "\u2581America", ".[", "3", "]", "\u2581As", "\u2581of", "\u2581", "2", "0", "0", "7", ",", "\u2581there", "\u2581were", "\u2581over", "\u2581seven", "\u2581million", "\u2581llam", "as", "\u2581and", "\u2581al", "p", "ac", "as", "\u2581in", "\u2581South", "\u2581America", "\u2581and", "\u2581over", "\u2581", "1", "5", "8", ",", "0", "0", "0", "\u2581llam", "as", "\u2581and", "\u2581", "1", "0", "0", ",", "0", "0", "0", "<0xEA>", "<0x99>", "<0x8A>", "<0xF0>", "<0x9F>", "<0xA6>", "<0x99>", "\u2581al", "p", "ac", "as", ",", "\u2581desc", "ended", "\u2581from", "\u2581pro", "gen", "itors", "\u2581imported", "\u2581late", "\u2581in", "\u2581the", "\u2581", "2", "0", "th", "\u2581century", ",", "\u2581in", "\u2581the", "\u2581United", "\u2581States", "\u2581and", "\u2581Canada", ".[", "5", "]", "\u2581In", "\u2581A", "ym", "ara", "\u2581myth", "ology", ",", "\u2581llam", "as", "\u2581are", "\u2581important", "\u2581be", "ings", ".", "\u2581The", "\u2581Heaven", "ly", "\u2581L", "l", "ama", "\u2581is", "\u2581said", "\u2581to", "\u2581drink", "\u2581water", "\u2581from", "\u2581the", "\u2581ocean", "\u2581and", "\u2581ur", "in", "ates", "\u2581as", "\u2581it", "\u2581ra", "ins", ".[", "6", "]", "\u2581According", "\u2581to", "\u2581A", "ym", "ara", "\u2581es", "chat", "ology", ",", "\u2581llam", "as", "\u2581will", "\u2581return", "\u2581to", "\u2581the", "\u2581water", "\u2581spr", "ings", "\u2581and", "\u2581l", "ago", "ons", "\u2581where", "\u2581they", "\u2581come", "\u2581from", "\u2581at", "\u2581the", "\u2581end", "\u2581of", "\u2581time", ".[", "6", "]"], + ids: [1, 450, 11148, 3304, 20374, 30176, 29880, 30426, 30215, 29885, 30184, 29914, 29936, 29871, 243, 162, 169, 156, 15495, 728, 11504, 11173, 362, 29901, 518, 30176, 31743, 3304, 2314, 313, 29931, 3304, 3144, 3304, 29897, 338, 263, 21849, 630, 4275, 3082, 3949, 295, 333, 29892, 17644, 1304, 408, 263, 27654, 322, 4870, 13019, 491, 1126, 29872, 273, 4185, 1973, 1951, 278, 4721, 29899, 1625, 3774, 713, 3152, 29889, 365, 5288, 294, 526, 5264, 15006, 322, 5735, 411, 4045, 408, 263, 902, 29881, 29889, 11275, 281, 1507, 338, 4964, 322, 3743, 871, 263, 2319, 5253, 310, 10906, 22878, 7226, 29906, 29962, 365, 5288, 294, 508, 5110, 2560, 9595, 1156, 263, 2846, 21159, 2187, 29889, 1932, 773, 263, 4870, 29892, 896, 508, 8677, 1048, 29871, 29906, 29945, 304, 29871, 29941, 29900, 29995, 310, 1009, 3573, 7688, 363, 29871, 29947, 304, 29871, 29896, 29941, 2383, 313, 29945, 29994, 29947, 7800, 467, 29961, 29941, 29962, 450, 1024, 11148, 3304, 313, 262, 278, 4940, 884, 805, 14356, 376, 29880, 3304, 29908, 470, 376, 3820, 3304, 1159, 471, 16356, 491, 7824, 3604, 9306, 515, 7531, 25493, 1403, 550, 7226, 29946, 29962, 450, 19525, 943, 310, 11829, 294, 526, 2714, 304, 505, 3978, 630, 515, 278, 7027, 13494, 1144, 310, 4644, 6813, 1048, 29871, 29946, 29900, 7284, 2440, 8020, 29892, 322, 17602, 9725, 630, 304, 4275, 6813, 1048, 2211, 7284, 2440, 8020, 2645, 278, 7027, 3082, 4124, 3167, 29889, 2648, 278, 1095, 310, 278, 1833, 14890, 5046, 313, 29896, 29900, 29892, 29900, 29900, 29900, 29994, 29896, 29906, 29892, 29900, 29900, 29900, 2440, 8020, 511, 3949, 295, 4841, 892, 1294, 5562, 297, 4644, 6813, 7226, 29941, 29962, 1094, 310, 29871, 29906, 29900, 29900, 29955, 29892, 727, 892, 975, 9881, 7284, 11829, 294, 322, 394, 29886, 562, 294, 297, 4275, 6813, 322, 975, 29871, 29896, 29945, 29947, 29892, 29900, 29900, 29900, 11829, 294, 322, 29871, 29896, 29900, 29900, 29892, 29900, 29900, 29900, 237, 156, 141, 243, 162, 169, 156, 394, 29886, 562, 294, 29892, 5153, 2760, 515, 410, 1885, 17259, 19673, 5683, 297, 278, 29871, 29906, 29900, 386, 6462, 29892, 297, 278, 3303, 3900, 322, 7400, 7226, 29945, 29962, 512, 319, 962, 2518, 22082, 3002, 29892, 11829, 294, 526, 4100, 367, 886, 29889, 450, 22977, 368, 365, 29880, 3304, 338, 1497, 304, 13748, 4094, 515, 278, 23474, 322, 5065, 262, 1078, 408, 372, 1153, 1144, 7226, 29953, 29962, 7579, 304, 319, 962, 2518, 831, 13496, 3002, 29892, 11829, 294, 674, 736, 304, 278, 4094, 7689, 886, 322, 301, 4425, 787, 988, 896, 2041, 515, 472, 278, 1095, 310, 931, 7226, 29953, 29962], + decoded: ' The llama (/\u02c8l\u0251\u02d0m\u0259/; \ud83e\udd99Spanish pronunciation: [\u02c8\u028eama]) (Lama glama) is a domesticated South American camelid, widely used as a meat and pack animal by Andean cultures since the Pre-Columbian era. Llamas are social animals and live with others as a herd. Their wool is soft and contains only a small amount of lanolin.[2] Llamas can learn simple tasks after a few repetitions. When using a pack, they can carry about 25 to 30% of their body weight for 8 to 13 km (5\u20138 miles).[3] The name llama (in the past also spelled "lama" or "glama") was adopted by European settlers from native Peruvians.[4] The ancestors of llamas are thought to have originated from the Great Plains of North America about 40 million years ago, and subsequently migrated to South America about three million years ago during the Great American Interchange. By the end of the last ice age (10,000\u201312,000 years ago), camelids were extinct in North America.[3] As of 2007, there were over seven million llamas and alpacas in South America and over 158,000 llamas and 100,000\ua64a\ud83e\udd99 alpacas, descended from progenitors imported late in the 20th century, in the United States and Canada.[5] In Aymara mythology, llamas are important beings. The Heavenly Llama is said to drink water from the ocean and urinates as it rains.[6] According to Aymara eschatology, llamas will return to the water springs and lagoons where they come from at the end of time.[6]', + }, }, - PARAGRAPH: { - text: LLAMA_TEST_STRINGS.PARAGRAPH, - tokens: ["The", "\u0120llama", "\u0120(/", "\u00cb", "\u012a", "l", "\u00c9", "\u0133", "\u00cb", "\u0132", "m", "\u00c9\u013b", "/", ";", "\u0120\u00f0\u0141", "\u00a6", "\u013b", "Spanish", "\u0120pronunciation", ":", "\u0120[", "\u00cb", "\u012a", "\u00ca", "\u0130", "ama", "])", "\u0120(", "L", "ama", "\u0120gl", "ama", ")", "\u0120is", "\u0120a", "\u0120domestic", "ated", "\u0120South", "\u0120American", "\u0120camel", "id", ",", "\u0120widely", "\u0120used", "\u0120as", "\u0120a", "\u0120meat", "\u0120and", "\u0120pack", "\u0120animal", "\u0120by", "\u0120And", "ean", "\u0120cultures", "\u0120since", "\u0120the", "\u0120Pre", "-C", "olum", "bian", "\u0120era", ".", "\u0120L", "lam", "as", "\u0120are", "\u0120social", "\u0120animals", "\u0120and", "\u0120live", "\u0120with", "\u0120others", "\u0120as", "\u0120a", "\u0120herd", ".", "\u0120Their", "\u0120wool", "\u0120is", "\u0120soft", "\u0120and", "\u0120contains", "\u0120only", "\u0120a", "\u0120small", "\u0120amount", "\u0120of", "\u0120lan", "olin", ".[", "2", "]", "\u0120L", "lam", "as", "\u0120can", "\u0120learn", "\u0120simple", "\u0120tasks", "\u0120after", "\u0120a", "\u0120few", "\u0120repetitions", ".", "\u0120When", "\u0120using", "\u0120a", "\u0120pack", ",", "\u0120they", "\u0120can", "\u0120carry", "\u0120about", "\u0120", "25", "\u0120to", "\u0120", "30", "%", "\u0120of", "\u0120their", "\u0120body", "\u0120weight", "\u0120for", "\u0120", "8", "\u0120to", "\u0120", "13", "\u0120km", "\u0120(", "5", "\u00e2\u0122\u0135", "8", "\u0120miles", ").[", "3", "]", "\u0120The", "\u0120name", "\u0120llama", "\u0120(", "in", "\u0120the", "\u0120past", "\u0120also", "\u0120spelled", '\u0120"', "lama", '"', "\u0120or", '\u0120"', "gl", "ama", '")', "\u0120was", "\u0120adopted", "\u0120by", "\u0120European", "\u0120settlers", "\u0120from", "\u0120native", "\u0120Per", "uv", "ians", ".[", "4", "]", "\u0120The", "\u0120ancestors", "\u0120of", "\u0120ll", "amas", "\u0120are", "\u0120thought", "\u0120to", "\u0120have", "\u0120originated", "\u0120from", "\u0120the", "\u0120Great", "\u0120Plains", "\u0120of", "\u0120North", "\u0120America", "\u0120about", "\u0120", "40", "\u0120million", "\u0120years", "\u0120ago", ",", "\u0120and", "\u0120subsequently", "\u0120migrated", "\u0120to", "\u0120South", "\u0120America", "\u0120about", "\u0120three", "\u0120million", "\u0120years", "\u0120ago", "\u0120during", "\u0120the", "\u0120Great", "\u0120American", "\u0120Inter", "change", ".", "\u0120By", "\u0120the", "\u0120end", "\u0120of", "\u0120the", "\u0120last", "\u0120ice", "\u0120age", "\u0120(", "10", ",", "000", "\u00e2\u0122\u0135", "12", ",", "000", "\u0120years", "\u0120ago", "),", "\u0120camel", "ids", "\u0120were", "\u0120extinct", "\u0120in", "\u0120North", "\u0120America", ".[", "3", "]", "\u0120As", "\u0120of", "\u0120", "200", "7", ",", "\u0120there", "\u0120were", "\u0120over", "\u0120seven", "\u0120million", "\u0120ll", "amas", "\u0120and", "\u0120al", "pac", "as", "\u0120in", "\u0120South", "\u0120America", "\u0120and", "\u0120over", "\u0120", "158", ",", "000", "\u0120ll", "amas", "\u0120and", "\u0120", "100", ",", "000", "\u00ea", "\u013b", "\u012c", "\u00f0\u0141", "\u00a6", "\u013b", "\u0120al", "pac", "as", ",", "\u0120descended", "\u0120from", "\u0120progen", "itors", "\u0120imported", "\u0120late", "\u0120in", "\u0120the", "\u0120", "20", "th", "\u0120century", ",", "\u0120in", "\u0120the", "\u0120United", "\u0120States", "\u0120and", "\u0120Canada", ".[", "5", "]", "\u0120In", "\u0120A", "ym", "ara", "\u0120mythology", ",", "\u0120ll", "amas", "\u0120are", "\u0120important", "\u0120beings", ".", "\u0120The", "\u0120Heavenly", "\u0120L", "lama", "\u0120is", "\u0120said", "\u0120to", "\u0120drink", "\u0120water", "\u0120from", "\u0120the", "\u0120ocean", "\u0120and", "\u0120ur", "in", "ates", "\u0120as", "\u0120it", "\u0120rains", ".[", "6", "]", "\u0120According", "\u0120to", "\u0120A", "ym", "ara", "\u0120es", "chat", "ology", ",", "\u0120ll", "amas", "\u0120will", "\u0120return", "\u0120to", "\u0120the", "\u0120water", "\u0120springs", "\u0120and", "\u0120l", "ago", "ons", "\u0120where", "\u0120they", "\u0120come", "\u0120from", "\u0120at", "\u0120the", "\u0120end", "\u0120of", "\u0120time", ".[", "6", "]"], - ids: [791, 94776, 47325, 135, 230, 75, 133, 239, 135, 238, 76, 99638, 14, 26, 11410, 99, 247, 62897, 71722, 25, 510, 135, 230, 134, 236, 3105, 2526, 320, 43, 3105, 2840, 3105, 8, 374, 264, 13018, 660, 4987, 3778, 50252, 307, 11, 13882, 1511, 439, 264, 13339, 323, 3854, 10065, 555, 1628, 5420, 27833, 2533, 279, 5075, 7813, 1152, 13464, 11639, 13, 445, 24705, 300, 527, 3674, 10099, 323, 3974, 449, 3885, 439, 264, 59213, 13, 11205, 39640, 374, 8579, 323, 5727, 1193, 264, 2678, 3392, 315, 31791, 37737, 8032, 17, 60, 445, 24705, 300, 649, 4048, 4382, 9256, 1306, 264, 2478, 86066, 13, 3277, 1701, 264, 3854, 11, 814, 649, 6920, 922, 220, 914, 311, 220, 966, 4, 315, 872, 2547, 4785, 369, 220, 23, 311, 220, 1032, 13437, 320, 20, 4235, 23, 8931, 94638, 18, 60, 578, 836, 94776, 320, 258, 279, 3347, 1101, 68918, 330, 81101, 1, 477, 330, 6200, 3105, 909, 574, 18306, 555, 7665, 61107, 505, 10068, 3700, 12328, 5493, 8032, 19, 60, 578, 38618, 315, 9507, 29189, 527, 3463, 311, 617, 44853, 505, 279, 8681, 63911, 315, 4892, 5270, 922, 220, 1272, 3610, 1667, 4227, 11, 323, 28520, 73691, 311, 4987, 5270, 922, 2380, 3610, 1667, 4227, 2391, 279, 8681, 3778, 5783, 3455, 13, 3296, 279, 842, 315, 279, 1566, 10054, 4325, 320, 605, 11, 931, 4235, 717, 11, 931, 1667, 4227, 705, 50252, 3447, 1051, 69918, 304, 4892, 5270, 8032, 18, 60, 1666, 315, 220, 1049, 22, 11, 1070, 1051, 927, 8254, 3610, 9507, 29189, 323, 453, 46051, 300, 304, 4987, 5270, 323, 927, 220, 11286, 11, 931, 9507, 29189, 323, 220, 1041, 11, 931, 166, 247, 232, 9468, 99, 247, 453, 46051, 300, 11, 58842, 505, 84360, 12170, 25973, 3389, 304, 279, 220, 508, 339, 9478, 11, 304, 279, 3723, 4273, 323, 7008, 8032, 20, 60, 763, 362, 1631, 5169, 59492, 11, 9507, 29189, 527, 3062, 23837, 13, 578, 88150, 445, 81101, 374, 1071, 311, 7172, 3090, 505, 279, 18435, 323, 4433, 258, 988, 439, 433, 62555, 8032, 21, 60, 10771, 311, 362, 1631, 5169, 1560, 9884, 2508, 11, 9507, 29189, 690, 471, 311, 279, 3090, 42242, 323, 326, 6438, 2439, 1405, 814, 2586, 505, 520, 279, 842, 315, 892, 8032, 21, 60], - decoded: 'The llama (/\u02c8l\u0251\u02d0m\u0259/; \ud83e\udd99Spanish pronunciation: [\u02c8\u028eama]) (Lama glama) is a domesticated South American camelid, widely used as a meat and pack animal by Andean cultures since the Pre-Columbian era. Llamas are social animals and live with others as a herd. Their wool is soft and contains only a small amount of lanolin.[2] Llamas can learn simple tasks after a few repetitions. When using a pack, they can carry about 25 to 30% of their body weight for 8 to 13 km (5\u20138 miles).[3] The name llama (in the past also spelled "lama" or "glama") was adopted by European settlers from native Peruvians.[4] The ancestors of llamas are thought to have originated from the Great Plains of North America about 40 million years ago, and subsequently migrated to South America about three million years ago during the Great American Interchange. By the end of the last ice age (10,000\u201312,000 years ago), camelids were extinct in North America.[3] As of 2007, there were over seven million llamas and alpacas in South America and over 158,000 llamas and 100,000\ua64a\ud83e\udd99 alpacas, descended from progenitors imported late in the 20th century, in the United States and Canada.[5] In Aymara mythology, llamas are important beings. The Heavenly Llama is said to drink water from the ocean and urinates as it rains.[6] According to Aymara eschatology, llamas will return to the water springs and lagoons where they come from at the end of time.[6]', + "Xenova/llama3-tokenizer": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["How", "\u0120are", "\u0120you", "\u0120doing", "?"], + ids: [4438, 527, 499, 3815, 30], + decoded: "How are you doing?", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["You", "\u0120should", "'ve", "\u0120done", "\u0120this"], + ids: [2675, 1288, 3077, 2884, 420], + decoded: "You should've done this", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["012", "345", "678", "9", "\u0120", "0", "\u0120", "1", "\u0120", "2", "\u0120", "3", "\u0120", "4", "\u0120", "5", "\u0120", "6", "\u0120", "7", "\u0120", "8", "\u0120", "9", "\u0120", "10", "\u0120", "100", "\u0120", "100", "0"], + ids: [11531, 12901, 17458, 24, 220, 15, 220, 16, 220, 17, 220, 18, 220, 19, 220, 20, 220, 21, 220, 22, 220, 23, 220, 24, 220, 605, 220, 1041, 220, 1041, 15], + decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "\u0120company", "\u0120was", "\u0120founded", "\u0120in", "\u0120", "201", "6", "."], + ids: [791, 2883, 574, 18538, 304, 220, 679, 21, 13], + decoded: "The company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "\u010a", "'ll", "\u0120!!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'t", "."], + ids: [32, 198, 3358, 11261, 998, 20837, 67, 4708, 67, 315, 11, 649, 956, 13], + decoded: "A\n'll!!to?'d''d of, can't.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["def", "\u0120main", "():\u010a", "\u0109pass"], + ids: [755, 1925, 4019, 42531], + decoded: "def main():\n\tpass", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".toString", "();\u010a", "toString", "();"], + ids: [1169, 264, 284, 2909, 5180, 545, 6712, 2178], + decoded: "let a = obj.toString();\ntoString();", + }, + NEWLINES: { + text: LLAMA_TEST_STRINGS.NEWLINES, + tokens: ["ax", "\u010a", "####\u010a", "boo"], + ids: [710, 198, 71050, 34093], + decoded: "ax\n####\nboo", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "want", "\u00c3\u00a9d", ",", "running"], + ids: [1899, 53757, 15433, 11, 28272], + decoded: "UNwant\u00e9d,running", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["1", "\u0100", "2", "\u00ef\u00bf\u00bd", "3"], + ids: [16, 188, 17, 5809, 18], + decoded: "1\u00002\ufffd3", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["Hello", "\u0120World"], + ids: [9906, 4435], + decoded: "Hello World", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["hello", "\u0120world"], + ids: [15339, 1917], + decoded: "hello world", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u00e7\u0136\u0141\u00e6\u00b4\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"], + ids: [104654, 9554, 89151, 39013, 249, 21043], + decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u0120\u0120", "\u0120leading", "\u0120space"], + ids: [256, 6522, 3634], + decoded: " leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["tr", "ailing", "\u0120space", "\u0120\u0120\u0120"], + ids: [376, 14612, 3634, 262], + decoded: "trailing space ", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["Hi", "\u0120", "\u0120Hello"], + ids: [13347, 220, 22691], + decoded: "Hi Hello", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2\u00a5", "6", "\u0120\u00e2\u0124", "\u00a3", "7", "\u0120\u00e2\u0124\u00b9", "8", "\u0120\u00e2\u0124", "\u00b1", "9", "\u0120test"], + ids: [1985, 400, 16, 432, 17, 674, 18, 13281, 19, 7083, 20, 72588, 21, 113384, 96, 22, 90891, 23, 113384, 109, 24, 1296], + decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "\u0120bought", "\u0120an", "\u0120apple", "\u0120for", "\u0120$", "1", ".", "00", "\u0120at", "\u0120the", "\u0120store", "."], + ids: [40, 11021, 459, 24149, 369, 400, 16, 13, 410, 520, 279, 3637, 13], + decoded: "I bought an apple for $1.00 at the store.", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u0120\u0120"], + ids: [9514, 1981, 256], + decoded: "you\u2026 ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], + ids: [9514, 1981, 9421], + decoded: "you\u2026\u00a0\u00a0", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142", "\u00c2\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], + ids: [9514, 1981, 4194, 4194, 9514, 1981, 9421], + decoded: "you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["we", "ird", "\u0120\u00ef\u00bd\u0140", "\u0120edge", "\u0120\u00ef\u00bd\u0140", "\u0120case"], + ids: [906, 2668, 111942, 6964, 111942, 1162], + decoded: "weird \uff5e edge \uff5e case", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u00e2\u0138", "\u0123", "This", "\u0120\u00e2\u0138", "\u0123", "is", "\u0120\u00e2\u0138", "\u0123", "a", "\u0120\u00e2\u0138", "\u0123", "test", "\u0120\u00e2\u0138", "\u0123", "."], + ids: [10634, 223, 2028, 14860, 223, 285, 14860, 223, 64, 14860, 223, 1985, 14860, 223, 13], + decoded: "\u2581This \u2581is \u2581a \u2581test \u2581.", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u00f0\u0141\u013a", "\u0124", "\u0120\u00f0\u0141\u0133", "\u012f", "\u0120\u00f0\u0141", "\u00a4", "\u00a3", "\u0120\u00f0\u0141\u013a", "\u012f", "\u0120\u00f0\u0141\u013a", "\u0143", "\u0120\u00f0\u0141", "\u0130", "\u012b", "\u0120\u00f0\u0141", "\u013b", "\u0131", "\u0120\u00f0\u0141\u013a", "\u012c", "\u0120\u00f0\u0141\u0136", "\u00a5", "\u0120\u00f0\u0141\u013a", "\u0123", "\u0120\u00f0\u0141\u013a", "\u0127", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141\u013a", "\u0128", "\u0120\u00f0\u0141\u0133", "\u0131", "\u0120\u00e2\u013f\u00a4", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141\u0134", "\u013e", "\u0120\u00f0\u0141\u0134", "\u013c", "\u0120\u00f0\u0141\u0134", "\u0139", "\u0120\u00f0\u0141\u0134", "\u013b", "\u0120\u00f0\u0141", "\u0138", "\u00a4", "\u0120\u00f0\u0141\u013a", "\u0130", "\u0120\u00f0\u0141\u0133", "\u012e", "\u0120\u00f0\u0141", "\u00a5", "\u00b3", "\u0120\u00f0\u0141\u0134", "\u00aa", "\u0120\u00e2\u013e", "\u00a8", "\u0120\u00f0\u0141\u0133", "\u012b", "\u0120\u00f0\u0141\u0133", "\u0122", "\u0120\u00f0\u0141\u0134", "\u00af", "\u0120\u00f0\u0141", "\u0130", "\u012a", "\u0120\u00f0\u0141", "\u013b", "\u012a", "\u0120\u00f0\u0141", "\u013b", "\u012e", "\u0120\u00f0\u0141\u0134", "\u0122", "\u0120\u00f0\u0141\u0133", "\u0129", "\u0120\u00f0\u0141\u0133", "\u012d", "\u0120\u00e2\u013e", "\u0127", "\u0120\u00f0\u0141", "\u0130", "\u0123", "\u0120\u00f0\u0141", "\u012e", "\u0140", "\u0120\u00f0\u0141", "\u012e", "\u00b8", "\u0120\u00f0\u0141\u0134", "\u00b0"], + ids: [76460, 224, 62904, 235, 11410, 97, 96, 27623, 235, 27623, 255, 11410, 236, 231, 11410, 247, 237, 27623, 232, 96169, 98, 27623, 223, 27623, 227, 11410, 97, 245, 27623, 228, 62904, 237, 71570, 31643, 64139, 250, 64139, 248, 64139, 245, 64139, 247, 11410, 244, 97, 27623, 236, 62904, 234, 11410, 98, 111, 64139, 103, 26602, 101, 62904, 231, 62904, 222, 64139, 107, 11410, 236, 230, 11410, 247, 230, 11410, 247, 234, 64139, 222, 62904, 229, 62904, 233, 26602, 227, 11410, 236, 223, 11410, 234, 252, 11410, 234, 116, 64139, 108], + decoded: "\ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u00e2\u013e", "\u00a8", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141\u0133", "\u0123", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141\u0133", "\u00b1", "\u00f0\u0141", "\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0137", "\u00b5", "\u00e2\u0122\u012f", "\u00e2\u013b", "\u0124", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141", "\u00a7", "\u013b", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00e2\u013b", "\u0124", "\u0120\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u012e", "\u00be", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u0120\u00f0\u0141\u0133", "\u00a9", "\u00e2\u0122\u012f", "\u00e2\u013f\u00a4", "\u00e2\u0122\u012f", "\u00f0\u0141\u0134", "\u012d", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a8", "\u0120\u00f0\u0141\u0133", "\u00a9", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a9", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a7", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a6", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141", "\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0131", "\u00b4", "\u00f3", "\u0142\u0123", "\u00a7", "\u00f3", "\u0142\u0123", "\u00a2", "\u00f3", "\u0142\u0123", "\u00a5", "\u00f3", "\u0142\u0123", "\u00ae", "\u00f3", "\u0142\u0123", "\u00a7", "\u00f3", "\u0142\u0123", "\u00bf", "\u0120\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00e2\u013f\u00a4", "\u00ef\u00b8\u0131", "\u00e2\u0122\u012f", "\u00f0\u0141\u0134", "\u012d", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bc"], + ids: [38798, 101, 11410, 97, 245, 62904, 223, 31643, 62904, 109, 9468, 237, 119, 11410, 243, 113, 102470, 17245, 224, 31643, 11410, 100, 247, 9468, 237, 119, 102470, 17245, 224, 62904, 101, 9468, 237, 119, 102470, 9468, 234, 122, 11410, 100, 239, 102470, 9468, 97, 251, 102470, 9468, 100, 239, 62904, 102, 102470, 121643, 102470, 93273, 233, 102470, 9468, 239, 101, 62904, 102, 102470, 9468, 239, 102, 102470, 9468, 239, 100, 102470, 9468, 239, 99, 11410, 100, 239, 9468, 237, 119, 102470, 9468, 97, 251, 102470, 9468, 100, 239, 9468, 237, 119, 11410, 237, 112, 175, 16050, 100, 175, 16050, 95, 175, 16050, 98, 175, 16050, 106, 175, 16050, 100, 175, 16050, 123, 62904, 101, 9468, 237, 119, 102470, 121643, 31643, 102470, 93273, 233, 102470, 9468, 239, 101, 9468, 237, 120], + decoded: "\u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75\u200d\u2642\ufe0f \ud83e\uddd9\ud83c\udffb\u200d\u2642 \ud83d\udc68\ud83c\udffb\u200d\ud83c\udf3e \ud83e\uddd1\u200d\ud83e\udd1d\u200d\ud83e\uddd1 \ud83d\udc69\u200d\u2764\u200d\ud83d\udc8b\u200d\ud83d\udc68 \ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66 \ud83e\uddd1\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c\udffb \ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f \ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c\udffc", + }, + BPE_SCORES_PRIORITY_1: { + text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_1, + tokens: ["grab", "bed"], + ids: [59312, 2788], + decoded: "grabbed", + }, + BPE_SCORES_PRIORITY_2: { + text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_2, + tokens: ["\u0120grabbed"], + ids: [30418], + decoded: " grabbed", + }, + BPE_SCORES_PRIORITY_3: { + text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_3, + tokens: ["\u0120\u0120\u0120\u0120\u0120\u0120\u0120\u0120\u0120\u0120", "\u0120grabbed"], + ids: [1881, 30418], + decoded: " grabbed", + }, + NEWLINE: { + text: LLAMA_TEST_STRINGS.NEWLINE, + tokens: ["\u010a"], + ids: [198], + decoded: "\n", + }, + NEWLINE_WITH_LEADING_SPACE: { + text: LLAMA_TEST_STRINGS.NEWLINE_WITH_LEADING_SPACE, + tokens: ["\u0120\u010a"], + ids: [720], + decoded: " \n", + }, + TABS: { + text: LLAMA_TEST_STRINGS.TABS, + tokens: ["\u0109t", "abs", "\u0109\u0109\u0109", "\u0109out", "\u0120here"], + ids: [3324, 3518, 573, 14294, 1618], + decoded: "\ttabs\t\t\t\tout here", + }, + NEWLINE_AND_TAB: { + text: LLAMA_TEST_STRINGS.NEWLINE_AND_TAB, + tokens: ["\u010a\u0109\u010a"], + ids: [18108], + decoded: "\n\t\n", + }, + CHINESE_LETTER: { + text: LLAMA_TEST_STRINGS.CHINESE_LETTER, + tokens: ["\u00e9\u0137\u0129"], + ids: [104643], + decoded: "\u9547", + }, + EMOJIS_1: { + text: LLAMA_TEST_STRINGS.EMOJIS_1, + tokens: ["\u00f0\u0141", "\u00a6", "\u013b"], + ids: [9468, 99, 247], + decoded: "\ud83e\udd99", + }, + EMOJIS_2: { + text: LLAMA_TEST_STRINGS.EMOJIS_2, + tokens: ["\u00f0\u0141", "\u00a6", "\u013b", "\u00ea", "\u013b", "\u012c"], + ids: [9468, 99, 247, 166, 247, 232], + decoded: "\ud83e\udd99\ua64a", + }, + EMOJIS_3: { + text: LLAMA_TEST_STRINGS.EMOJIS_3, + tokens: ["\u00ea", "\u013b", "\u012c", "\u00f0\u0141", "\u00a6", "\u013b"], + ids: [166, 247, 232, 9468, 99, 247], + decoded: "\ua64a\ud83e\udd99", + }, + PARAGRAPH: { + text: LLAMA_TEST_STRINGS.PARAGRAPH, + tokens: ["The", "\u0120llama", "\u0120(/", "\u00cb", "\u012a", "l", "\u00c9", "\u0133", "\u00cb", "\u0132", "m", "\u00c9\u013b", "/", ";", "\u0120\u00f0\u0141", "\u00a6", "\u013b", "Spanish", "\u0120pronunciation", ":", "\u0120[", "\u00cb", "\u012a", "\u00ca", "\u0130", "ama", "])", "\u0120(", "L", "ama", "\u0120gl", "ama", ")", "\u0120is", "\u0120a", "\u0120domestic", "ated", "\u0120South", "\u0120American", "\u0120camel", "id", ",", "\u0120widely", "\u0120used", "\u0120as", "\u0120a", "\u0120meat", "\u0120and", "\u0120pack", "\u0120animal", "\u0120by", "\u0120And", "ean", "\u0120cultures", "\u0120since", "\u0120the", "\u0120Pre", "-C", "olum", "bian", "\u0120era", ".", "\u0120L", "lam", "as", "\u0120are", "\u0120social", "\u0120animals", "\u0120and", "\u0120live", "\u0120with", "\u0120others", "\u0120as", "\u0120a", "\u0120herd", ".", "\u0120Their", "\u0120wool", "\u0120is", "\u0120soft", "\u0120and", "\u0120contains", "\u0120only", "\u0120a", "\u0120small", "\u0120amount", "\u0120of", "\u0120lan", "olin", ".[", "2", "]", "\u0120L", "lam", "as", "\u0120can", "\u0120learn", "\u0120simple", "\u0120tasks", "\u0120after", "\u0120a", "\u0120few", "\u0120repetitions", ".", "\u0120When", "\u0120using", "\u0120a", "\u0120pack", ",", "\u0120they", "\u0120can", "\u0120carry", "\u0120about", "\u0120", "25", "\u0120to", "\u0120", "30", "%", "\u0120of", "\u0120their", "\u0120body", "\u0120weight", "\u0120for", "\u0120", "8", "\u0120to", "\u0120", "13", "\u0120km", "\u0120(", "5", "\u00e2\u0122\u0135", "8", "\u0120miles", ").[", "3", "]", "\u0120The", "\u0120name", "\u0120llama", "\u0120(", "in", "\u0120the", "\u0120past", "\u0120also", "\u0120spelled", '\u0120"', "lama", '"', "\u0120or", '\u0120"', "gl", "ama", '")', "\u0120was", "\u0120adopted", "\u0120by", "\u0120European", "\u0120settlers", "\u0120from", "\u0120native", "\u0120Per", "uv", "ians", ".[", "4", "]", "\u0120The", "\u0120ancestors", "\u0120of", "\u0120ll", "amas", "\u0120are", "\u0120thought", "\u0120to", "\u0120have", "\u0120originated", "\u0120from", "\u0120the", "\u0120Great", "\u0120Plains", "\u0120of", "\u0120North", "\u0120America", "\u0120about", "\u0120", "40", "\u0120million", "\u0120years", "\u0120ago", ",", "\u0120and", "\u0120subsequently", "\u0120migrated", "\u0120to", "\u0120South", "\u0120America", "\u0120about", "\u0120three", "\u0120million", "\u0120years", "\u0120ago", "\u0120during", "\u0120the", "\u0120Great", "\u0120American", "\u0120Inter", "change", ".", "\u0120By", "\u0120the", "\u0120end", "\u0120of", "\u0120the", "\u0120last", "\u0120ice", "\u0120age", "\u0120(", "10", ",", "000", "\u00e2\u0122\u0135", "12", ",", "000", "\u0120years", "\u0120ago", "),", "\u0120camel", "ids", "\u0120were", "\u0120extinct", "\u0120in", "\u0120North", "\u0120America", ".[", "3", "]", "\u0120As", "\u0120of", "\u0120", "200", "7", ",", "\u0120there", "\u0120were", "\u0120over", "\u0120seven", "\u0120million", "\u0120ll", "amas", "\u0120and", "\u0120al", "pac", "as", "\u0120in", "\u0120South", "\u0120America", "\u0120and", "\u0120over", "\u0120", "158", ",", "000", "\u0120ll", "amas", "\u0120and", "\u0120", "100", ",", "000", "\u00ea", "\u013b", "\u012c", "\u00f0\u0141", "\u00a6", "\u013b", "\u0120al", "pac", "as", ",", "\u0120descended", "\u0120from", "\u0120progen", "itors", "\u0120imported", "\u0120late", "\u0120in", "\u0120the", "\u0120", "20", "th", "\u0120century", ",", "\u0120in", "\u0120the", "\u0120United", "\u0120States", "\u0120and", "\u0120Canada", ".[", "5", "]", "\u0120In", "\u0120A", "ym", "ara", "\u0120mythology", ",", "\u0120ll", "amas", "\u0120are", "\u0120important", "\u0120beings", ".", "\u0120The", "\u0120Heavenly", "\u0120L", "lama", "\u0120is", "\u0120said", "\u0120to", "\u0120drink", "\u0120water", "\u0120from", "\u0120the", "\u0120ocean", "\u0120and", "\u0120ur", "in", "ates", "\u0120as", "\u0120it", "\u0120rains", ".[", "6", "]", "\u0120According", "\u0120to", "\u0120A", "ym", "ara", "\u0120es", "chat", "ology", ",", "\u0120ll", "amas", "\u0120will", "\u0120return", "\u0120to", "\u0120the", "\u0120water", "\u0120springs", "\u0120and", "\u0120l", "ago", "ons", "\u0120where", "\u0120they", "\u0120come", "\u0120from", "\u0120at", "\u0120the", "\u0120end", "\u0120of", "\u0120time", ".[", "6", "]"], + ids: [791, 94776, 47325, 135, 230, 75, 133, 239, 135, 238, 76, 99638, 14, 26, 11410, 99, 247, 62897, 71722, 25, 510, 135, 230, 134, 236, 3105, 2526, 320, 43, 3105, 2840, 3105, 8, 374, 264, 13018, 660, 4987, 3778, 50252, 307, 11, 13882, 1511, 439, 264, 13339, 323, 3854, 10065, 555, 1628, 5420, 27833, 2533, 279, 5075, 7813, 1152, 13464, 11639, 13, 445, 24705, 300, 527, 3674, 10099, 323, 3974, 449, 3885, 439, 264, 59213, 13, 11205, 39640, 374, 8579, 323, 5727, 1193, 264, 2678, 3392, 315, 31791, 37737, 8032, 17, 60, 445, 24705, 300, 649, 4048, 4382, 9256, 1306, 264, 2478, 86066, 13, 3277, 1701, 264, 3854, 11, 814, 649, 6920, 922, 220, 914, 311, 220, 966, 4, 315, 872, 2547, 4785, 369, 220, 23, 311, 220, 1032, 13437, 320, 20, 4235, 23, 8931, 94638, 18, 60, 578, 836, 94776, 320, 258, 279, 3347, 1101, 68918, 330, 81101, 1, 477, 330, 6200, 3105, 909, 574, 18306, 555, 7665, 61107, 505, 10068, 3700, 12328, 5493, 8032, 19, 60, 578, 38618, 315, 9507, 29189, 527, 3463, 311, 617, 44853, 505, 279, 8681, 63911, 315, 4892, 5270, 922, 220, 1272, 3610, 1667, 4227, 11, 323, 28520, 73691, 311, 4987, 5270, 922, 2380, 3610, 1667, 4227, 2391, 279, 8681, 3778, 5783, 3455, 13, 3296, 279, 842, 315, 279, 1566, 10054, 4325, 320, 605, 11, 931, 4235, 717, 11, 931, 1667, 4227, 705, 50252, 3447, 1051, 69918, 304, 4892, 5270, 8032, 18, 60, 1666, 315, 220, 1049, 22, 11, 1070, 1051, 927, 8254, 3610, 9507, 29189, 323, 453, 46051, 300, 304, 4987, 5270, 323, 927, 220, 11286, 11, 931, 9507, 29189, 323, 220, 1041, 11, 931, 166, 247, 232, 9468, 99, 247, 453, 46051, 300, 11, 58842, 505, 84360, 12170, 25973, 3389, 304, 279, 220, 508, 339, 9478, 11, 304, 279, 3723, 4273, 323, 7008, 8032, 20, 60, 763, 362, 1631, 5169, 59492, 11, 9507, 29189, 527, 3062, 23837, 13, 578, 88150, 445, 81101, 374, 1071, 311, 7172, 3090, 505, 279, 18435, 323, 4433, 258, 988, 439, 433, 62555, 8032, 21, 60, 10771, 311, 362, 1631, 5169, 1560, 9884, 2508, 11, 9507, 29189, 690, 471, 311, 279, 3090, 42242, 323, 326, 6438, 2439, 1405, 814, 2586, 505, 520, 279, 842, 315, 892, 8032, 21, 60], + decoded: 'The llama (/\u02c8l\u0251\u02d0m\u0259/; \ud83e\udd99Spanish pronunciation: [\u02c8\u028eama]) (Lama glama) is a domesticated South American camelid, widely used as a meat and pack animal by Andean cultures since the Pre-Columbian era. Llamas are social animals and live with others as a herd. Their wool is soft and contains only a small amount of lanolin.[2] Llamas can learn simple tasks after a few repetitions. When using a pack, they can carry about 25 to 30% of their body weight for 8 to 13 km (5\u20138 miles).[3] The name llama (in the past also spelled "lama" or "glama") was adopted by European settlers from native Peruvians.[4] The ancestors of llamas are thought to have originated from the Great Plains of North America about 40 million years ago, and subsequently migrated to South America about three million years ago during the Great American Interchange. By the end of the last ice age (10,000\u201312,000 years ago), camelids were extinct in North America.[3] As of 2007, there were over seven million llamas and alpacas in South America and over 158,000 llamas and 100,000\ua64a\ud83e\udd99 alpacas, descended from progenitors imported late in the 20th century, in the United States and Canada.[5] In Aymara mythology, llamas are important beings. The Heavenly Llama is said to drink water from the ocean and urinates as it rains.[6] According to Aymara eschatology, llamas will return to the water springs and lagoons where they come from at the end of time.[6]', + }, }, - }, - // - Sequence PostProcessor - // - "ignore_merges": true - "Xenova/llama3-tokenizer-new": { - SIMPLE: { - text: BASE_TEST_STRINGS.SIMPLE, - tokens: ["How", "\u0120are", "\u0120you", "\u0120doing", "?"], - ids: [128000, 4438, 527, 499, 3815, 30], - decoded: "<|begin_of_text|>How are you doing?", - }, - SIMPLE_WITH_PUNCTUATION: { - text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, - tokens: ["You", "\u0120should", "'ve", "\u0120done", "\u0120this"], - ids: [128000, 2675, 1288, 3077, 2884, 420], - decoded: "<|begin_of_text|>You should've done this", - }, - NUMBERS: { - text: BASE_TEST_STRINGS.NUMBERS, - tokens: ["012", "345", "678", "9", "\u0120", "0", "\u0120", "1", "\u0120", "2", "\u0120", "3", "\u0120", "4", "\u0120", "5", "\u0120", "6", "\u0120", "7", "\u0120", "8", "\u0120", "9", "\u0120", "10", "\u0120", "100", "\u0120", "100", "0"], - ids: [128000, 11531, 12901, 17458, 24, 220, 15, 220, 16, 220, 17, 220, 18, 220, 19, 220, 20, 220, 21, 220, 22, 220, 23, 220, 24, 220, 605, 220, 1041, 220, 1041, 15], - decoded: "<|begin_of_text|>0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["The", "\u0120company", "\u0120was", "\u0120founded", "\u0120in", "\u0120", "201", "6", "."], - ids: [128000, 791, 2883, 574, 18538, 304, 220, 679, 21, 13], - decoded: "<|begin_of_text|>The company was founded in 2016.", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["A", "\u010a", "'ll", "\u0120!!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'t", "."], - ids: [128000, 32, 198, 3358, 11261, 998, 20837, 67, 4708, 67, 315, 11, 649, 956, 13], - decoded: "<|begin_of_text|>A\n'll!!to?'d''d of, can't.", - }, - PYTHON_CODE: { - text: BASE_TEST_STRINGS.PYTHON_CODE, - tokens: ["def", "\u0120main", "():\u010a", "\u0109pass"], - ids: [128000, 755, 1925, 4019, 42531], - decoded: "<|begin_of_text|>def main():\n\tpass", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".toString", "();\u010a", "toString", "();"], - ids: [128000, 1169, 264, 284, 2909, 5180, 545, 6712, 2178], - decoded: "<|begin_of_text|>let a = obj.toString();\ntoString();", - }, - NEWLINES: { - text: LLAMA_TEST_STRINGS.NEWLINES, - tokens: ["ax", "\u010a", "####\u010a", "boo"], - ids: [128000, 710, 198, 71050, 34093], - decoded: "<|begin_of_text|>ax\n####\nboo", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["UN", "want", "\u00c3\u00a9d", ",", "running"], - ids: [128000, 1899, 53757, 15433, 11, 28272], - decoded: "<|begin_of_text|>UNwant\u00e9d,running", - }, - CONTROL_TOKENS: { - text: BASE_TEST_STRINGS.CONTROL_TOKENS, - tokens: ["1", "\u0100", "2", "\u00ef\u00bf\u00bd", "3"], - ids: [128000, 16, 188, 17, 5809, 18], - decoded: "<|begin_of_text|>1\u00002\ufffd3", - }, - HELLO_WORLD_TITLECASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, - tokens: ["Hello", "\u0120World"], - ids: [128000, 9906, 4435], - decoded: "<|begin_of_text|>Hello World", - }, - HELLO_WORLD_LOWERCASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, - tokens: ["hello", "\u0120world"], - ids: [128000, 15339, 1917], - decoded: "<|begin_of_text|>hello world", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u00e7\u0136\u0141\u00e6\u00b4\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"], - ids: [128000, 104654, 9554, 89151, 39013, 249, 21043], - decoded: "<|begin_of_text|>\u751f\u6d3b\u7684\u771f\u8c1b\u662f", - }, - LEADING_SPACE: { - text: BASE_TEST_STRINGS.LEADING_SPACE, - tokens: ["\u0120\u0120", "\u0120leading", "\u0120space"], - ids: [128000, 256, 6522, 3634], - decoded: "<|begin_of_text|> leading space", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["tr", "ailing", "\u0120space", "\u0120\u0120\u0120"], - ids: [128000, 376, 14612, 3634, 262], - decoded: "<|begin_of_text|>trailing space ", - }, - DOUBLE_SPACE: { - text: BASE_TEST_STRINGS.DOUBLE_SPACE, - tokens: ["Hi", "\u0120", "\u0120Hello"], - ids: [128000, 13347, 220, 22691], - decoded: "<|begin_of_text|>Hi Hello", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2\u00a5", "6", "\u0120\u00e2\u0124", "\u00a3", "7", "\u0120\u00e2\u0124\u00b9", "8", "\u0120\u00e2\u0124", "\u00b1", "9", "\u0120test"], - ids: [128000, 1985, 400, 16, 432, 17, 674, 18, 13281, 19, 7083, 20, 72588, 21, 113384, 96, 22, 90891, 23, 113384, 109, 24, 1296], - decoded: "<|begin_of_text|>test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["I", "\u0120bought", "\u0120an", "\u0120apple", "\u0120for", "\u0120$", "1", ".", "00", "\u0120at", "\u0120the", "\u0120store", "."], - ids: [128000, 40, 11021, 459, 24149, 369, 400, 16, 13, 410, 520, 279, 3637, 13], - decoded: "<|begin_of_text|>I bought an apple for $1.00 at the store.", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - tokens: ["you", "\u00e2\u0122\u00a6", "\u0120\u0120"], - ids: [128000, 9514, 1981, 256], - decoded: "<|begin_of_text|>you\u2026 ", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], - ids: [128000, 9514, 1981, 9421], - decoded: "<|begin_of_text|>you\u2026\u00a0\u00a0", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142", "\u00c2\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], - ids: [128000, 9514, 1981, 4194, 4194, 9514, 1981, 9421], - decoded: "<|begin_of_text|>you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["we", "ird", "\u0120\u00ef\u00bd\u0140", "\u0120edge", "\u0120\u00ef\u00bd\u0140", "\u0120case"], - ids: [128000, 906, 2668, 111942, 6964, 111942, 1162], - decoded: "<|begin_of_text|>weird \uff5e edge \uff5e case", - }, - SPIECE_UNDERSCORE: { - text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, - tokens: ["\u00e2\u0138", "\u0123", "This", "\u0120\u00e2\u0138", "\u0123", "is", "\u0120\u00e2\u0138", "\u0123", "a", "\u0120\u00e2\u0138", "\u0123", "test", "\u0120\u00e2\u0138", "\u0123", "."], - ids: [128000, 10634, 223, 2028, 14860, 223, 285, 14860, 223, 64, 14860, 223, 1985, 14860, 223, 13], - decoded: "<|begin_of_text|>\u2581This \u2581is \u2581a \u2581test \u2581.", - }, - POPULAR_EMOJIS: { - text: BASE_TEST_STRINGS.POPULAR_EMOJIS, - tokens: ["\u00f0\u0141\u013a", "\u0124", "\u0120\u00f0\u0141\u0133", "\u012f", "\u0120\u00f0\u0141", "\u00a4", "\u00a3", "\u0120\u00f0\u0141\u013a", "\u012f", "\u0120\u00f0\u0141\u013a", "\u0143", "\u0120\u00f0\u0141", "\u0130", "\u012b", "\u0120\u00f0\u0141", "\u013b", "\u0131", "\u0120\u00f0\u0141\u013a", "\u012c", "\u0120\u00f0\u0141\u0136", "\u00a5", "\u0120\u00f0\u0141\u013a", "\u0123", "\u0120\u00f0\u0141\u013a", "\u0127", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141\u013a", "\u0128", "\u0120\u00f0\u0141\u0133", "\u0131", "\u0120\u00e2\u013f\u00a4", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141\u0134", "\u013e", "\u0120\u00f0\u0141\u0134", "\u013c", "\u0120\u00f0\u0141\u0134", "\u0139", "\u0120\u00f0\u0141\u0134", "\u013b", "\u0120\u00f0\u0141", "\u0138", "\u00a4", "\u0120\u00f0\u0141\u013a", "\u0130", "\u0120\u00f0\u0141\u0133", "\u012e", "\u0120\u00f0\u0141", "\u00a5", "\u00b3", "\u0120\u00f0\u0141\u0134", "\u00aa", "\u0120\u00e2\u013e", "\u00a8", "\u0120\u00f0\u0141\u0133", "\u012b", "\u0120\u00f0\u0141\u0133", "\u0122", "\u0120\u00f0\u0141\u0134", "\u00af", "\u0120\u00f0\u0141", "\u0130", "\u012a", "\u0120\u00f0\u0141", "\u013b", "\u012a", "\u0120\u00f0\u0141", "\u013b", "\u012e", "\u0120\u00f0\u0141\u0134", "\u0122", "\u0120\u00f0\u0141\u0133", "\u0129", "\u0120\u00f0\u0141\u0133", "\u012d", "\u0120\u00e2\u013e", "\u0127", "\u0120\u00f0\u0141", "\u0130", "\u0123", "\u0120\u00f0\u0141", "\u012e", "\u0140", "\u0120\u00f0\u0141", "\u012e", "\u00b8", "\u0120\u00f0\u0141\u0134", "\u00b0"], - ids: [128000, 76460, 224, 62904, 235, 11410, 97, 96, 27623, 235, 27623, 255, 11410, 236, 231, 11410, 247, 237, 27623, 232, 96169, 98, 27623, 223, 27623, 227, 11410, 97, 245, 27623, 228, 62904, 237, 71570, 31643, 64139, 250, 64139, 248, 64139, 245, 64139, 247, 11410, 244, 97, 27623, 236, 62904, 234, 11410, 98, 111, 64139, 103, 26602, 101, 62904, 231, 62904, 222, 64139, 107, 11410, 236, 230, 11410, 247, 230, 11410, 247, 234, 64139, 222, 62904, 229, 62904, 233, 26602, 227, 11410, 236, 223, 11410, 234, 252, 11410, 234, 116, 64139, 108], - decoded: "<|begin_of_text|>\ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0", - }, - MULTIBYTE_EMOJIS: { - text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, - tokens: ["\u00e2\u013e", "\u00a8", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141\u0133", "\u0123", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141\u0133", "\u00b1", "\u00f0\u0141", "\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0137", "\u00b5", "\u00e2\u0122\u012f", "\u00e2\u013b", "\u0124", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141", "\u00a7", "\u013b", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00e2\u013b", "\u0124", "\u0120\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u012e", "\u00be", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u0120\u00f0\u0141\u0133", "\u00a9", "\u00e2\u0122\u012f", "\u00e2\u013f\u00a4", "\u00e2\u0122\u012f", "\u00f0\u0141\u0134", "\u012d", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a8", "\u0120\u00f0\u0141\u0133", "\u00a9", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a9", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a7", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a6", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141", "\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0131", "\u00b4", "\u00f3", "\u0142\u0123", "\u00a7", "\u00f3", "\u0142\u0123", "\u00a2", "\u00f3", "\u0142\u0123", "\u00a5", "\u00f3", "\u0142\u0123", "\u00ae", "\u00f3", "\u0142\u0123", "\u00a7", "\u00f3", "\u0142\u0123", "\u00bf", "\u0120\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00e2\u013f\u00a4", "\u00ef\u00b8\u0131", "\u00e2\u0122\u012f", "\u00f0\u0141\u0134", "\u012d", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bc"], - ids: [128000, 38798, 101, 11410, 97, 245, 62904, 223, 31643, 62904, 109, 9468, 237, 119, 11410, 243, 113, 102470, 17245, 224, 31643, 11410, 100, 247, 9468, 237, 119, 102470, 17245, 224, 62904, 101, 9468, 237, 119, 102470, 9468, 234, 122, 11410, 100, 239, 102470, 9468, 97, 251, 102470, 9468, 100, 239, 62904, 102, 102470, 121643, 102470, 93273, 233, 102470, 9468, 239, 101, 62904, 102, 102470, 9468, 239, 102, 102470, 9468, 239, 100, 102470, 9468, 239, 99, 11410, 100, 239, 9468, 237, 119, 102470, 9468, 97, 251, 102470, 9468, 100, 239, 9468, 237, 119, 11410, 237, 112, 175, 16050, 100, 175, 16050, 95, 175, 16050, 98, 175, 16050, 106, 175, 16050, 100, 175, 16050, 123, 62904, 101, 9468, 237, 119, 102470, 121643, 31643, 102470, 93273, 233, 102470, 9468, 239, 101, 9468, 237, 120], - decoded: "<|begin_of_text|>\u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75\u200d\u2642\ufe0f \ud83e\uddd9\ud83c\udffb\u200d\u2642 \ud83d\udc68\ud83c\udffb\u200d\ud83c\udf3e \ud83e\uddd1\u200d\ud83e\udd1d\u200d\ud83e\uddd1 \ud83d\udc69\u200d\u2764\u200d\ud83d\udc8b\u200d\ud83d\udc68 \ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66 \ud83e\uddd1\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c\udffb \ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f \ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c\udffc", - }, - BPE_SCORES_PRIORITY_1: { - text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_1, - tokens: ["grab", "bed"], - ids: [128000, 59312, 2788], - decoded: "<|begin_of_text|>grabbed", - }, - BPE_SCORES_PRIORITY_2: { - text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_2, - tokens: ["\u0120grabbed"], - ids: [128000, 30418], - decoded: "<|begin_of_text|> grabbed", - }, - BPE_SCORES_PRIORITY_3: { - text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_3, - tokens: ["\u0120\u0120\u0120\u0120\u0120\u0120\u0120\u0120\u0120\u0120", "\u0120grabbed"], - ids: [128000, 1881, 30418], - decoded: "<|begin_of_text|> grabbed", - }, - NEWLINE: { - text: LLAMA_TEST_STRINGS.NEWLINE, - tokens: ["\u010a"], - ids: [128000, 198], - decoded: "<|begin_of_text|>\n", - }, - NEWLINE_WITH_LEADING_SPACE: { - text: LLAMA_TEST_STRINGS.NEWLINE_WITH_LEADING_SPACE, - tokens: ["\u0120\u010a"], - ids: [128000, 720], - decoded: "<|begin_of_text|> \n", - }, - TABS: { - text: LLAMA_TEST_STRINGS.TABS, - tokens: ["\u0109t", "abs", "\u0109\u0109\u0109", "\u0109out", "\u0120here"], - ids: [128000, 3324, 3518, 573, 14294, 1618], - decoded: "<|begin_of_text|>\ttabs\t\t\t\tout here", - }, - NEWLINE_AND_TAB: { - text: LLAMA_TEST_STRINGS.NEWLINE_AND_TAB, - tokens: ["\u010a\u0109\u010a"], - ids: [128000, 18108], - decoded: "<|begin_of_text|>\n\t\n", - }, - CHINESE_LETTER: { - text: LLAMA_TEST_STRINGS.CHINESE_LETTER, - tokens: ["\u00e9\u0137\u0129"], - ids: [128000, 104643], - decoded: "<|begin_of_text|>\u9547", - }, - EMOJIS_1: { - text: LLAMA_TEST_STRINGS.EMOJIS_1, - tokens: ["\u00f0\u0141", "\u00a6", "\u013b"], - ids: [128000, 9468, 99, 247], - decoded: "<|begin_of_text|>\ud83e\udd99", - }, - EMOJIS_2: { - text: LLAMA_TEST_STRINGS.EMOJIS_2, - tokens: ["\u00f0\u0141", "\u00a6", "\u013b", "\u00ea", "\u013b", "\u012c"], - ids: [128000, 9468, 99, 247, 166, 247, 232], - decoded: "<|begin_of_text|>\ud83e\udd99\ua64a", - }, - EMOJIS_3: { - text: LLAMA_TEST_STRINGS.EMOJIS_3, - tokens: ["\u00ea", "\u013b", "\u012c", "\u00f0\u0141", "\u00a6", "\u013b"], - ids: [128000, 166, 247, 232, 9468, 99, 247], - decoded: "<|begin_of_text|>\ua64a\ud83e\udd99", - }, - PARAGRAPH: { - text: LLAMA_TEST_STRINGS.PARAGRAPH, - tokens: ["The", "\u0120llama", "\u0120(/", "\u00cb", "\u012a", "l", "\u00c9", "\u0133", "\u00cb", "\u0132", "m", "\u00c9\u013b", "/", ";", "\u0120\u00f0\u0141", "\u00a6", "\u013b", "Spanish", "\u0120pronunciation", ":", "\u0120[", "\u00cb", "\u012a", "\u00ca", "\u0130", "ama", "])", "\u0120(", "L", "ama", "\u0120gl", "ama", ")", "\u0120is", "\u0120a", "\u0120domestic", "ated", "\u0120South", "\u0120American", "\u0120camel", "id", ",", "\u0120widely", "\u0120used", "\u0120as", "\u0120a", "\u0120meat", "\u0120and", "\u0120pack", "\u0120animal", "\u0120by", "\u0120And", "ean", "\u0120cultures", "\u0120since", "\u0120the", "\u0120Pre", "-C", "olum", "bian", "\u0120era", ".", "\u0120L", "lam", "as", "\u0120are", "\u0120social", "\u0120animals", "\u0120and", "\u0120live", "\u0120with", "\u0120others", "\u0120as", "\u0120a", "\u0120herd", ".", "\u0120Their", "\u0120wool", "\u0120is", "\u0120soft", "\u0120and", "\u0120contains", "\u0120only", "\u0120a", "\u0120small", "\u0120amount", "\u0120of", "\u0120lan", "olin", ".[", "2", "]", "\u0120L", "lam", "as", "\u0120can", "\u0120learn", "\u0120simple", "\u0120tasks", "\u0120after", "\u0120a", "\u0120few", "\u0120repetitions", ".", "\u0120When", "\u0120using", "\u0120a", "\u0120pack", ",", "\u0120they", "\u0120can", "\u0120carry", "\u0120about", "\u0120", "25", "\u0120to", "\u0120", "30", "%", "\u0120of", "\u0120their", "\u0120body", "\u0120weight", "\u0120for", "\u0120", "8", "\u0120to", "\u0120", "13", "\u0120km", "\u0120(", "5", "\u00e2\u0122\u0135", "8", "\u0120miles", ").[", "3", "]", "\u0120The", "\u0120name", "\u0120llama", "\u0120(", "in", "\u0120the", "\u0120past", "\u0120also", "\u0120spelled", '\u0120"', "lama", '"', "\u0120or", '\u0120"', "gl", "ama", '")', "\u0120was", "\u0120adopted", "\u0120by", "\u0120European", "\u0120settlers", "\u0120from", "\u0120native", "\u0120Per", "uv", "ians", ".[", "4", "]", "\u0120The", "\u0120ancestors", "\u0120of", "\u0120ll", "amas", "\u0120are", "\u0120thought", "\u0120to", "\u0120have", "\u0120originated", "\u0120from", "\u0120the", "\u0120Great", "\u0120Plains", "\u0120of", "\u0120North", "\u0120America", "\u0120about", "\u0120", "40", "\u0120million", "\u0120years", "\u0120ago", ",", "\u0120and", "\u0120subsequently", "\u0120migrated", "\u0120to", "\u0120South", "\u0120America", "\u0120about", "\u0120three", "\u0120million", "\u0120years", "\u0120ago", "\u0120during", "\u0120the", "\u0120Great", "\u0120American", "\u0120Inter", "change", ".", "\u0120By", "\u0120the", "\u0120end", "\u0120of", "\u0120the", "\u0120last", "\u0120ice", "\u0120age", "\u0120(", "10", ",", "000", "\u00e2\u0122\u0135", "12", ",", "000", "\u0120years", "\u0120ago", "),", "\u0120camel", "ids", "\u0120were", "\u0120extinct", "\u0120in", "\u0120North", "\u0120America", ".[", "3", "]", "\u0120As", "\u0120of", "\u0120", "200", "7", ",", "\u0120there", "\u0120were", "\u0120over", "\u0120seven", "\u0120million", "\u0120ll", "amas", "\u0120and", "\u0120al", "pac", "as", "\u0120in", "\u0120South", "\u0120America", "\u0120and", "\u0120over", "\u0120", "158", ",", "000", "\u0120ll", "amas", "\u0120and", "\u0120", "100", ",", "000", "\u00ea", "\u013b", "\u012c", "\u00f0\u0141", "\u00a6", "\u013b", "\u0120al", "pac", "as", ",", "\u0120descended", "\u0120from", "\u0120progen", "itors", "\u0120imported", "\u0120late", "\u0120in", "\u0120the", "\u0120", "20", "th", "\u0120century", ",", "\u0120in", "\u0120the", "\u0120United", "\u0120States", "\u0120and", "\u0120Canada", ".[", "5", "]", "\u0120In", "\u0120A", "ym", "ara", "\u0120mythology", ",", "\u0120ll", "amas", "\u0120are", "\u0120important", "\u0120beings", ".", "\u0120The", "\u0120Heavenly", "\u0120L", "lama", "\u0120is", "\u0120said", "\u0120to", "\u0120drink", "\u0120water", "\u0120from", "\u0120the", "\u0120ocean", "\u0120and", "\u0120ur", "in", "ates", "\u0120as", "\u0120it", "\u0120rains", ".[", "6", "]", "\u0120According", "\u0120to", "\u0120A", "ym", "ara", "\u0120es", "chat", "ology", ",", "\u0120ll", "amas", "\u0120will", "\u0120return", "\u0120to", "\u0120the", "\u0120water", "\u0120springs", "\u0120and", "\u0120l", "ago", "ons", "\u0120where", "\u0120they", "\u0120come", "\u0120from", "\u0120at", "\u0120the", "\u0120end", "\u0120of", "\u0120time", ".[", "6", "]"], - ids: [128000, 791, 94776, 47325, 135, 230, 75, 133, 239, 135, 238, 76, 99638, 14, 26, 11410, 99, 247, 62897, 71722, 25, 510, 135, 230, 134, 236, 3105, 2526, 320, 43, 3105, 2840, 3105, 8, 374, 264, 13018, 660, 4987, 3778, 50252, 307, 11, 13882, 1511, 439, 264, 13339, 323, 3854, 10065, 555, 1628, 5420, 27833, 2533, 279, 5075, 7813, 1152, 13464, 11639, 13, 445, 24705, 300, 527, 3674, 10099, 323, 3974, 449, 3885, 439, 264, 59213, 13, 11205, 39640, 374, 8579, 323, 5727, 1193, 264, 2678, 3392, 315, 31791, 37737, 8032, 17, 60, 445, 24705, 300, 649, 4048, 4382, 9256, 1306, 264, 2478, 86066, 13, 3277, 1701, 264, 3854, 11, 814, 649, 6920, 922, 220, 914, 311, 220, 966, 4, 315, 872, 2547, 4785, 369, 220, 23, 311, 220, 1032, 13437, 320, 20, 4235, 23, 8931, 94638, 18, 60, 578, 836, 94776, 320, 258, 279, 3347, 1101, 68918, 330, 81101, 1, 477, 330, 6200, 3105, 909, 574, 18306, 555, 7665, 61107, 505, 10068, 3700, 12328, 5493, 8032, 19, 60, 578, 38618, 315, 9507, 29189, 527, 3463, 311, 617, 44853, 505, 279, 8681, 63911, 315, 4892, 5270, 922, 220, 1272, 3610, 1667, 4227, 11, 323, 28520, 73691, 311, 4987, 5270, 922, 2380, 3610, 1667, 4227, 2391, 279, 8681, 3778, 5783, 3455, 13, 3296, 279, 842, 315, 279, 1566, 10054, 4325, 320, 605, 11, 931, 4235, 717, 11, 931, 1667, 4227, 705, 50252, 3447, 1051, 69918, 304, 4892, 5270, 8032, 18, 60, 1666, 315, 220, 1049, 22, 11, 1070, 1051, 927, 8254, 3610, 9507, 29189, 323, 453, 46051, 300, 304, 4987, 5270, 323, 927, 220, 11286, 11, 931, 9507, 29189, 323, 220, 1041, 11, 931, 166, 247, 232, 9468, 99, 247, 453, 46051, 300, 11, 58842, 505, 84360, 12170, 25973, 3389, 304, 279, 220, 508, 339, 9478, 11, 304, 279, 3723, 4273, 323, 7008, 8032, 20, 60, 763, 362, 1631, 5169, 59492, 11, 9507, 29189, 527, 3062, 23837, 13, 578, 88150, 445, 81101, 374, 1071, 311, 7172, 3090, 505, 279, 18435, 323, 4433, 258, 988, 439, 433, 62555, 8032, 21, 60, 10771, 311, 362, 1631, 5169, 1560, 9884, 2508, 11, 9507, 29189, 690, 471, 311, 279, 3090, 42242, 323, 326, 6438, 2439, 1405, 814, 2586, 505, 520, 279, 842, 315, 892, 8032, 21, 60], - decoded: '<|begin_of_text|>The llama (/\u02c8l\u0251\u02d0m\u0259/; \ud83e\udd99Spanish pronunciation: [\u02c8\u028eama]) (Lama glama) is a domesticated South American camelid, widely used as a meat and pack animal by Andean cultures since the Pre-Columbian era. Llamas are social animals and live with others as a herd. Their wool is soft and contains only a small amount of lanolin.[2] Llamas can learn simple tasks after a few repetitions. When using a pack, they can carry about 25 to 30% of their body weight for 8 to 13 km (5\u20138 miles).[3] The name llama (in the past also spelled "lama" or "glama") was adopted by European settlers from native Peruvians.[4] The ancestors of llamas are thought to have originated from the Great Plains of North America about 40 million years ago, and subsequently migrated to South America about three million years ago during the Great American Interchange. By the end of the last ice age (10,000\u201312,000 years ago), camelids were extinct in North America.[3] As of 2007, there were over seven million llamas and alpacas in South America and over 158,000 llamas and 100,000\ua64a\ud83e\udd99 alpacas, descended from progenitors imported late in the 20th century, in the United States and Canada.[5] In Aymara mythology, llamas are important beings. The Heavenly Llama is said to drink water from the ocean and urinates as it rains.[6] According to Aymara eschatology, llamas will return to the water springs and lagoons where they come from at the end of time.[6]', - }, - }, - "Xenova/TinyLLama-v0": { - NEWLINES: { - text: LLAMA_TEST_STRINGS.NEWLINES, - tokens: ["\u2581ax", "<0x0A>", "####", "<0x0A>", "b", "oo"], - ids: [1, 9013, 13, 20411, 13, 31842, 2742], - decoded: " ax\n####\nboo", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u2581", "<0xE7>", "<0x94>", "<0x9F>", "<0xE6>", "<0xB4>", "<0xBB>", "<0xE7>", "<0x9A>", "<0x84>", "<0xE7>", "<0x9C>", "<0x9F>", "<0xE8>", "<0xB0>", "<0x9B>", "<0xE6>", "<0x98>", "<0xAF>"], - ids: [1, 31822, 234, 151, 162, 233, 183, 190, 234, 157, 135, 234, 159, 162, 235, 179, 158, 233, 155, 178], - decoded: " \u751f\u6d3b\u7684\u771f\u8c1b\u662f", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["\u2581trailing", "\u2581space", "\u2581", "\u2581", "\u2581"], - ids: [1, 30174, 2138, 31822, 31822, 31822], - decoded: " trailing space ", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["\u2581test", "\u2581$", "1", "\u2581R", "2", "\u2581#", "3", "\u2581\u20ac", "4", "\u2581\u00a3", "5", "\u2581", "<0xC2>", "<0xA5>", "6", "\u2581", "<0xE2>", "<0x82>", "<0xA3>", "7", "\u2581", "<0xE2>", "<0x82>", "<0xB9>", "8", "\u2581", "<0xE2>", "<0x82>", "<0xB1>", "9", "\u2581test"], - ids: [1, 1397, 569, 31853, 360, 31855, 1257, 31878, 9390, 31882, 3922, 31880, 31822, 197, 168, 31887, 31822, 229, 133, 166, 31888, 31822, 229, 133, 188, 31886, 31822, 229, 133, 180, 31877, 1397], - decoded: " test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - tokens: ["\u2581you", "\u2026", "\u2581", "\u2581"], - ids: [1, 365, 31925, 31822, 31822], - decoded: " you\u2026 ", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - tokens: ["\u2581you", "\u2026", "\u00a0", "\u00a0"], - ids: [1, 365, 31925, 31963, 31963], - decoded: " you\u2026\u00a0\u00a0", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - tokens: ["\u2581you", "\u2026", "\u00a0", "\u00a0", "you", "\u2026", "\u00a0", "\u00a0"], - ids: [1, 365, 31925, 31963, 31963, 7936, 31925, 31963, 31963], - decoded: " you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["\u2581weird", "\u2581", "<0xEF>", "<0xBD>", "<0x9E>", "\u2581edge", "\u2581", "<0xEF>", "<0xBD>", "<0x9E>", "\u2581case"], - ids: [1, 9907, 31822, 242, 192, 161, 5991, 31822, 242, 192, 161, 1372], - decoded: " weird \uff5e edge \uff5e case", - }, - POPULAR_EMOJIS: { - text: BASE_TEST_STRINGS.POPULAR_EMOJIS, - tokens: ["\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0x82>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x8D>", "\u2581", "<0xF0>", "<0x9F>", "<0xA4>", "<0xA3>", "\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0x8D>", "\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0xAD>", "\u2581", "<0xF0>", "<0x9F>", "<0x8E>", "<0x89>", "\u2581", "<0xF0>", "<0x9F>", "<0x99>", "<0x8F>", "\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0x8A>", "\u2581", "<0xF0>", "<0x9F>", "<0x94>", "<0xA5>", "\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0x81>", "\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0x85>", "\u2581", "<0xF0>", "<0x9F>", "<0xA4>", "<0x97>", "\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0x86>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x8F>", "\u2581", "<0xE2>", "<0x9D>", "<0xA4>", "<0xEF>", "<0xB8>", "<0x8F>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0x9C>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0x9A>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0x97>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0x99>", "\u2581", "<0xF0>", "<0x9F>", "<0x96>", "<0xA4>", "\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0x8E>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x8C>", "\u2581", "<0xF0>", "<0x9F>", "<0xA5>", "<0xB3>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0xAA>", "\u2581", "<0xE2>", "<0x9C>", "<0xA8>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x89>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x80>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0xAF>", "\u2581", "<0xF0>", "<0x9F>", "<0x8E>", "<0x88>", "\u2581", "<0xF0>", "<0x9F>", "<0x99>", "<0x88>", "\u2581", "<0xF0>", "<0x9F>", "<0x99>", "<0x8C>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0x80>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x87>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x8B>", "\u2581", "<0xE2>", "<0x9C>", "<0x85>", "\u2581", "<0xF0>", "<0x9F>", "<0x8E>", "<0x81>", "\u2581", "<0xF0>", "<0x9F>", "<0x8C>", "<0x9E>", "\u2581", "<0xF0>", "<0x9F>", "<0x8C>", "<0xB8>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0xB0>"], - ids: [1, 31822, 243, 162, 155, 133, 31822, 243, 162, 148, 144, 31822, 243, 162, 167, 166, 31822, 243, 162, 155, 144, 31822, 243, 162, 155, 176, 31822, 243, 162, 145, 140, 31822, 243, 162, 156, 146, 31822, 243, 162, 155, 141, 31822, 243, 162, 151, 168, 31822, 243, 162, 155, 132, 31822, 243, 162, 155, 136, 31822, 243, 162, 167, 154, 31822, 243, 162, 155, 137, 31822, 243, 162, 148, 146, 31822, 229, 160, 167, 242, 187, 146, 31822, 243, 162, 149, 159, 31822, 243, 162, 149, 157, 31822, 243, 162, 149, 154, 31822, 243, 162, 149, 156, 31822, 243, 162, 153, 167, 31822, 243, 162, 155, 145, 31822, 243, 162, 148, 143, 31822, 243, 162, 168, 182, 31822, 243, 162, 149, 173, 31822, 229, 159, 171, 31822, 243, 162, 148, 140, 31822, 243, 162, 148, 131, 31822, 243, 162, 149, 178, 31822, 243, 162, 145, 139, 31822, 243, 162, 156, 139, 31822, 243, 162, 156, 143, 31822, 243, 162, 149, 131, 31822, 243, 162, 148, 138, 31822, 243, 162, 148, 142, 31822, 229, 159, 136, 31822, 243, 162, 145, 132, 31822, 243, 162, 143, 161, 31822, 243, 162, 143, 187, 31822, 243, 162, 149, 179], - decoded: " \ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0", - }, - MULTIBYTE_EMOJIS: { - text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, - tokens: ["\u2581", "<0xE2>", "<0x9C>", "<0xA8>", "\u2581", "<0xF0>", "<0x9F>", "<0xA4>", "<0x97>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x81>", "<0xEF>", "<0xB8>", "<0x8F>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0xB1>", "<0xF0>", "<0x9F>", "<0x8F>", "<0xBB>", "\u2581", "<0xF0>", "<0x9F>", "<0x95>", "<0xB5>", "<0xE2>", "<0x80>", "<0x8D>", "<0xE2>", "<0x99>", "<0x82>", "<0xEF>", "<0xB8>", "<0x8F>", "\u2581", "<0xF0>", "<0x9F>", "<0xA7>", "<0x99>", "<0xF0>", "<0x9F>", "<0x8F>", "<0xBB>", "<0xE2>", "<0x80>", "<0x8D>", "<0xE2>", "<0x99>", "<0x82>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0xA8>", "<0xF0>", "<0x9F>", "<0x8F>", "<0xBB>", "<0xE2>", "<0x80>", "<0x8D>", "<0xF0>", "<0x9F>", "<0x8C>", "<0xBE>", "\u2581", "<0xF0>", "<0x9F>", "<0xA7>", "<0x91>", "<0xE2>", "<0x80>", "<0x8D>", "<0xF0>", "<0x9F>", "<0xA4>", "<0x9D>", "<0xE2>", "<0x80>", "<0x8D>", "<0xF0>", "<0x9F>", "<0xA7>", "<0x91>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0xA9>", "<0xE2>", "<0x80>", "<0x8D>", "<0xE2>", "<0x9D>", "<0xA4>", "<0xE2>", "<0x80>", "<0x8D>", "<0xF0>", "<0x9F>", "<0x92>", "<0x8B>", "<0xE2>", "<0x80>", "<0x8D>", "<0xF0>", "<0x9F>", "<0x91>", "<0xA8>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0xA9>", "<0xE2>", "<0x80>", "<0x8D>", "<0xF0>", "<0x9F>", "<0x91>", "<0xA9>", "<0xE2>", "<0x80>", "<0x8D>", "<0xF0>", "<0x9F>", "<0x91>", "<0xA7>", "<0xE2>", "<0x80>", "<0x8D>", "<0xF0>", "<0x9F>", "<0x91>", "<0xA6>", "\u2581", "<0xF0>", "<0x9F>", "<0xA7>", "<0x91>", "<0xF0>", "<0x9F>", "<0x8F>", "<0xBB>", "<0xE2>", "<0x80>", "<0x8D>", "<0xF0>", "<0x9F>", "<0xA4>", "<0x9D>", "<0xE2>", "<0x80>", "<0x8D>", "<0xF0>", "<0x9F>", "<0xA7>", "<0x91>", "<0xF0>", "<0x9F>", "<0x8F>", "<0xBB>", "\u2581", "<0xF0>", "<0x9F>", "<0x8F>", "<0xB4>", "<0xF3>", "<0xA0>", "<0x81>", "<0xA7>", "<0xF3>", "<0xA0>", "<0x81>", "<0xA2>", "<0xF3>", "<0xA0>", "<0x81>", "<0xA5>", "<0xF3>", "<0xA0>", "<0x81>", "<0xAE>", "<0xF3>", "<0xA0>", "<0x81>", "<0xA7>", "<0xF3>", "<0xA0>", "<0x81>", "<0xBF>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0xA8>", "<0xF0>", "<0x9F>", "<0x8F>", "<0xBB>", "<0xE2>", "<0x80>", "<0x8D>", "<0xE2>", "<0x9D>", "<0xA4>", "<0xEF>", "<0xB8>", "<0x8F>", "<0xE2>", "<0x80>", "<0x8D>", "<0xF0>", "<0x9F>", "<0x92>", "<0x8B>", "<0xE2>", "<0x80>", "<0x8D>", "<0xF0>", "<0x9F>", "<0x91>", "<0xA8>", "<0xF0>", "<0x9F>", "<0x8F>", "<0xBC>"], - ids: [1, 31822, 229, 159, 171, 31822, 243, 162, 167, 154, 31822, 243, 162, 148, 132, 242, 187, 146, 31822, 243, 162, 148, 180, 243, 162, 146, 190, 31822, 243, 162, 152, 184, 229, 131, 144, 229, 156, 133, 242, 187, 146, 31822, 243, 162, 170, 156, 243, 162, 146, 190, 229, 131, 144, 229, 156, 133, 31822, 243, 162, 148, 171, 243, 162, 146, 190, 229, 131, 144, 243, 162, 143, 193, 31822, 243, 162, 170, 148, 229, 131, 144, 243, 162, 167, 160, 229, 131, 144, 243, 162, 170, 148, 31822, 243, 162, 148, 172, 229, 131, 144, 229, 160, 167, 229, 131, 144, 243, 162, 149, 142, 229, 131, 144, 243, 162, 148, 171, 31822, 243, 162, 148, 172, 229, 131, 144, 243, 162, 148, 172, 229, 131, 144, 243, 162, 148, 170, 229, 131, 144, 243, 162, 148, 169, 31822, 243, 162, 170, 148, 243, 162, 146, 190, 229, 131, 144, 243, 162, 167, 160, 229, 131, 144, 243, 162, 170, 148, 243, 162, 146, 190, 31822, 243, 162, 146, 183, 246, 163, 132, 170, 246, 163, 132, 165, 246, 163, 132, 168, 246, 163, 132, 177, 246, 163, 132, 170, 246, 163, 132, 194, 31822, 243, 162, 148, 171, 243, 162, 146, 190, 229, 131, 144, 229, 160, 167, 242, 187, 146, 229, 131, 144, 243, 162, 149, 142, 229, 131, 144, 243, 162, 148, 171, 243, 162, 146, 191], - decoded: " \u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75\u200d\u2642\ufe0f \ud83e\uddd9\ud83c\udffb\u200d\u2642 \ud83d\udc68\ud83c\udffb\u200d\ud83c\udf3e \ud83e\uddd1\u200d\ud83e\udd1d\u200d\ud83e\uddd1 \ud83d\udc69\u200d\u2764\u200d\ud83d\udc8b\u200d\ud83d\udc68 \ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66 \ud83e\uddd1\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c\udffb \ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f \ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c\udffc", - }, - NEWLINE_WITH_LEADING_SPACE: { - text: LLAMA_TEST_STRINGS.NEWLINE_WITH_LEADING_SPACE, - tokens: ["\u2581", "\u2581", "<0x0A>"], - ids: [1, 31822, 31822, 13], - decoded: " \n", - }, - CHINESE_LETTER: { - text: LLAMA_TEST_STRINGS.CHINESE_LETTER, - tokens: ["\u2581", "<0xE9>", "<0x95>", "<0x87>"], - ids: [1, 31822, 236, 152, 138], - decoded: " \u9547", - }, - PARAGRAPH: { - text: LLAMA_TEST_STRINGS.PARAGRAPH, - tokens: ["\u2581The", "\u2581ll", "ama", "\u2581(", "/", "<0xCB>", "<0x88>", "l", "<0xC9>", "<0x91>", "<0xCB>", "<0x90>", "m", "<0xC9>", "<0x99>", "/", ";", "\u2581", "<0xF0>", "<0x9F>", "<0xA6>", "<0x99>", "Sp", "anish", "\u2581pron", "unciation", ":", "\u2581[", "<0xCB>", "<0x88>", "<0xCA>", "<0x8E>", "ama", "])", "\u2581(", "L", "ama", "\u2581gl", "ama", ")", "\u2581is", "\u2581a", "\u2581domest", "icated", "\u2581South", "\u2581American", "\u2581cam", "el", "id", ",", "\u2581widely", "\u2581used", "\u2581as", "\u2581a", "\u2581meat", "\u2581and", "\u2581pack", "\u2581animal", "\u2581by", "\u2581And", "ean", "\u2581cultures", "\u2581since", "\u2581the", "\u2581Pre", "-", "Col", "umb", "ian", "\u2581era", ".", "\u2581L", "lam", "as", "\u2581are", "\u2581social", "\u2581animals", "\u2581and", "\u2581live", "\u2581with", "\u2581others", "\u2581as", "\u2581a", "\u2581herd", ".", "\u2581Their", "\u2581wool", "\u2581is", "\u2581soft", "\u2581and", "\u2581contains", "\u2581only", "\u2581a", "\u2581small", "\u2581amount", "\u2581of", "\u2581l", "anol", "in", ".[", "2", "]", "\u2581L", "lam", "as", "\u2581can", "\u2581learn", "\u2581simple", "\u2581", "t", "asks", "\u2581after", "\u2581a", "\u2581few", "\u2581repet", "itions", ".", "\u2581When", "\u2581using", "\u2581a", "\u2581pack", ",", "\u2581they", "\u2581can", "\u2581carry", "\u2581about", "\u2581", "2", "5", "\u2581to", "\u2581", "3", "0", "%", "\u2581of", "\u2581their", "\u2581body", "\u2581weight", "\u2581for", "\u2581", "8", "\u2581to", "\u2581", "1", "3", "\u2581km", "\u2581(", "5", "\u2013", "8", "\u2581miles", ").", "[", "3", "]", "\u2581The", "\u2581name", "\u2581ll", "ama", "\u2581(", "in", "\u2581the", "\u2581past", "\u2581also", "\u2581sp", "elled", '\u2581"', "l", "ama", '"', "\u2581or", '\u2581"', "gl", "ama", '")', "\u2581was", "\u2581adopted", "\u2581by", "\u2581European", "\u2581settlers", "\u2581from", "\u2581native", "\u2581Per", "uv", "ians", ".[", "4", "]", "\u2581The", "\u2581ancestors", "\u2581of", "\u2581l", "lam", "as", "\u2581are", "\u2581thought", "\u2581to", "\u2581have", "\u2581originated", "\u2581from", "\u2581the", "\u2581Great", "\u2581Plains", "\u2581of", "\u2581North", "\u2581America", "\u2581about", "\u2581", "4", "0", "\u2581million", "\u2581years", "\u2581ago", ",", "\u2581and", "\u2581subsequently", "\u2581mig", "rated", "\u2581to", "\u2581South", "\u2581America", "\u2581about", "\u2581three", "\u2581million", "\u2581years", "\u2581ago", "\u2581during", "\u2581the", "\u2581Great", "\u2581American", "\u2581Inter", "change", ".", "\u2581By", "\u2581the", "\u2581end", "\u2581of", "\u2581the", "\u2581last", "\u2581ice", "\u2581age", "\u2581(", "1", "0", ",", "0", "0", "0", "\u2013", "1", "2", ",", "0", "0", "0", "\u2581years", "\u2581ago", "),", "\u2581cam", "el", "ids", "\u2581were", "\u2581extinct", "\u2581in", "\u2581North", "\u2581America", ".[", "3", "]", "\u2581As", "\u2581of", "\u2581", "2", "0", "0", "7", ",", "\u2581there", "\u2581were", "\u2581over", "\u2581seven", "\u2581million", "\u2581l", "lam", "as", "\u2581and", "\u2581al", "p", "ac", "as", "\u2581in", "\u2581South", "\u2581America", "\u2581and", "\u2581over", "\u2581", "1", "5", "8", ",", "0", "0", "0", "\u2581l", "lam", "as", "\u2581and", "\u2581", "1", "0", "0", ",", "0", "0", "0", "<0xEA>", "<0x99>", "<0x8A>", "<0xF0>", "<0x9F>", "<0xA6>", "<0x99>", "\u2581al", "p", "ac", "as", ",", "\u2581descended", "\u2581from", "\u2581pro", "gen", "itors", "\u2581imported", "\u2581late", "\u2581in", "\u2581the", "\u2581", "2", "0", "th", "\u2581century", ",", "\u2581in", "\u2581the", "\u2581United", "\u2581States", "\u2581and", "\u2581Canada", ".[", "5", "]", "\u2581In", "\u2581A", "ym", "ara", "\u2581mythology", ",", "\u2581l", "lam", "as", "\u2581are", "\u2581important", "\u2581beings", ".", "\u2581The", "\u2581Heaven", "ly", "\u2581Ll", "ama", "\u2581is", "\u2581said", "\u2581to", "\u2581drink", "\u2581water", "\u2581from", "\u2581the", "\u2581ocean", "\u2581and", "\u2581ur", "inates", "\u2581as", "\u2581it", "\u2581rains", ".[", "6", "]", "\u2581According", "\u2581to", "\u2581A", "ym", "ara", "\u2581es", "chat", "ology", ",", "\u2581l", "lam", "as", "\u2581will", "\u2581return", "\u2581to", "\u2581the", "\u2581water", "\u2581springs", "\u2581and", "\u2581l", "ago", "ons", "\u2581where", "\u2581they", "\u2581come", "\u2581from", "\u2581at", "\u2581the", "\u2581end", "\u2581of", "\u2581time", ".[", "6", "]"], - ids: [1, 347, 31763, 2269, 352, 31873, 206, 139, 31832, 204, 148, 206, 147, 31836, 204, 156, 31873, 31891, 31822, 243, 162, 169, 156, 8889, 5817, 11155, 26128, 31871, 836, 206, 139, 205, 145, 2269, 9772, 352, 31867, 2269, 1192, 2269, 31861, 322, 260, 27940, 2672, 1897, 1454, 3764, 307, 317, 31844, 7055, 1065, 362, 260, 8659, 291, 2667, 6075, 417, 787, 14083, 10775, 1314, 266, 2345, 31854, 4848, 2234, 620, 5998, 31843, 372, 3082, 295, 397, 1619, 5220, 291, 1983, 351, 1892, 362, 260, 27172, 31843, 4585, 22729, 322, 2647, 291, 5140, 744, 260, 1435, 2399, 287, 309, 18426, 261, 3564, 31855, 31908, 372, 3082, 295, 473, 1977, 3102, 31822, 31824, 5577, 768, 260, 1346, 17042, 1479, 31843, 1408, 1340, 260, 2667, 31844, 526, 473, 3875, 562, 31822, 31855, 31880, 289, 31822, 31878, 31852, 31914, 287, 518, 2108, 4182, 329, 31822, 31886, 289, 31822, 31853, 31878, 6512, 352, 31880, 31906, 31886, 4465, 656, 31907, 31878, 31908, 347, 1382, 31763, 2269, 352, 261, 266, 1646, 615, 612, 5902, 495, 31832, 2269, 31875, 405, 495, 4261, 2269, 4290, 393, 7574, 417, 2821, 23343, 427, 6412, 2083, 10099, 1580, 3564, 31882, 31908, 347, 18294, 287, 309, 3082, 295, 397, 1991, 289, 435, 20355, 427, 266, 3172, 26744, 287, 1975, 2139, 562, 31822, 31882, 31852, 1577, 778, 2236, 31844, 291, 11786, 21052, 3397, 289, 1897, 2139, 562, 1166, 1577, 778, 2236, 1177, 266, 3172, 1454, 3029, 3604, 31843, 1433, 266, 928, 287, 266, 1060, 5707, 2253, 352, 31853, 31852, 31844, 31852, 31852, 31852, 31906, 31853, 31855, 31844, 31852, 31852, 31852, 778, 2236, 698, 3764, 307, 1982, 577, 30610, 288, 1975, 2139, 3564, 31878, 31908, 717, 287, 31822, 31855, 31852, 31852, 31888, 31844, 635, 577, 648, 3931, 1577, 309, 3082, 295, 291, 366, 31837, 380, 295, 288, 1897, 2139, 291, 648, 31822, 31853, 31880, 31886, 31844, 31852, 31852, 31852, 309, 3082, 295, 291, 31822, 31853, 31852, 31852, 31844, 31852, 31852, 31852, 237, 156, 141, 243, 162, 169, 156, 366, 31837, 380, 295, 31844, 27627, 427, 375, 3353, 4705, 17798, 2732, 288, 266, 31822, 31855, 31852, 388, 3373, 31844, 288, 266, 1494, 1769, 291, 3008, 3564, 31880, 31908, 455, 308, 1276, 2776, 24143, 31844, 309, 3082, 295, 397, 1480, 11844, 31843, 347, 15836, 326, 11321, 2269, 322, 664, 289, 5065, 1579, 427, 266, 8622, 291, 4328, 11466, 362, 357, 28738, 3564, 31887, 31908, 3252, 289, 308, 1276, 2776, 1582, 20068, 1058, 31844, 309, 3082, 295, 482, 1199, 289, 266, 1579, 24250, 291, 309, 3405, 680, 804, 526, 1412, 427, 389, 266, 928, 287, 647, 3564, 31887, 31908], - decoded: ' The llama (/\u02c8l\u0251\u02d0m\u0259/; \ud83e\udd99Spanish pronunciation: [\u02c8\u028eama]) (Lama glama) is a domesticated South American camelid, widely used as a meat and pack animal by Andean cultures since the Pre-Columbian era. Llamas are social animals and live with others as a herd. Their wool is soft and contains only a small amount of lanolin.[2] Llamas can learn simple tasks after a few repetitions. When using a pack, they can carry about 25 to 30% of their body weight for 8 to 13 km (5\u20138 miles).[3] The name llama (in the past also spelled "lama" or "glama") was adopted by European settlers from native Peruvians.[4] The ancestors of llamas are thought to have originated from the Great Plains of North America about 40 million years ago, and subsequently migrated to South America about three million years ago during the Great American Interchange. By the end of the last ice age (10,000\u201312,000 years ago), camelids were extinct in North America.[3] As of 2007, there were over seven million llamas and alpacas in South America and over 158,000 llamas and 100,000\ua64a\ud83e\udd99 alpacas, descended from progenitors imported late in the 20th century, in the United States and Canada.[5] In Aymara mythology, llamas are important beings. The Heavenly Llama is said to drink water from the ocean and urinates as it rains.[6] According to Aymara eschatology, llamas will return to the water springs and lagoons where they come from at the end of time.[6]', - }, - }, - "Xenova/deepseek-coder-1.3b-instruct": { - SIMPLE: { - text: BASE_TEST_STRINGS.SIMPLE, - tokens: ["How", "\u0120are", "\u0120you", "\u0120doing", "?"], - ids: [32013, 2808, 417, 340, 3207, 30], - decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>How are you doing?", - }, - SIMPLE_WITH_PUNCTUATION: { - text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, - tokens: ["You", "\u0120should", "'", "ve", "\u0120done", "\u0120this"], - ids: [32013, 2042, 1020, 6, 312, 2359, 437], - decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>You should've done this", - }, - NUMBERS: { - text: BASE_TEST_STRINGS.NUMBERS, - tokens: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "\u0120", "0", "\u0120", "1", "\u0120", "2", "\u0120", "3", "\u0120", "4", "\u0120", "5", "\u0120", "6", "\u0120", "7", "\u0120", "8", "\u0120", "9", "\u0120", "1", "0", "\u0120", "1", "0", "0", "\u0120", "1", "0", "0", "0"], - ids: [32013, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 207, 15, 207, 16, 207, 17, 207, 18, 207, 19, 207, 20, 207, 21, 207, 22, 207, 23, 207, 24, 207, 16, 15, 207, 16, 15, 15, 207, 16, 15, 15, 15], - decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["The", "\u0120company", "\u0120was", "\u0120founded", "\u0120in", "\u0120", "2", "0", "1", "6", "."], - ids: [32013, 546, 2595, 438, 16316, 279, 207, 17, 15, 16, 21, 13], - decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>The company was founded in 2016.", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["A", "\u010a", "'", "ll", "\u0120!!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'", "t", "."], - ids: [32013, 32, 185, 6, 642, 24466, 577, 11665, 67, 4191, 67, 280, 11, 482, 6, 83, 13], - decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>A\n'll !!to?'d''d of, can't.", - }, - PYTHON_CODE: { - text: BASE_TEST_STRINGS.PYTHON_CODE, - tokens: ["def", "\u0120main", "():", "\u010a", "\u0109", "pass"], - ids: [32013, 1551, 1959, 10942, 185, 184, 4805], - decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>def main():\n\tpass", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".", "toString", "();", "\u010a", "toString", "();"], - ids: [32013, 1160, 245, 405, 6528, 13, 12617, 1293, 185, 12617, 1293], - decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>let a = obj.toString();\ntoString();", - }, - NEWLINES: { - text: LLAMA_TEST_STRINGS.NEWLINES, - tokens: ["ax", "\u010a", "####", "\u010a", "bo", "o"], - ids: [32013, 1099, 185, 3576, 185, 952, 78], - decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>ax\n####\nboo", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["UN", "want", "\u00c3\u00a9d", ",", "running"], - ids: [32013, 4348, 28626, 31898, 11, 22785], - decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>UNwant\u00e9d,running", - }, - CONTROL_TOKENS: { - text: BASE_TEST_STRINGS.CONTROL_TOKENS, - tokens: ["1", "\u0100", "2", "\u00ef\u00bf\u00bd", "3"], - ids: [32013, 16, 175, 17, 10006, 18], - decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>1\u00002\ufffd3", - }, - HELLO_WORLD_TITLECASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, - tokens: ["Hello", "\u0120World"], - ids: [32013, 17535, 5414], - decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>Hello World", - }, - HELLO_WORLD_LOWERCASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, - tokens: ["hello", "\u0120world"], - ids: [32013, 31702, 1835], - decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>hello world", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u00e7\u0136\u0141\u00e6\u00b4\u00bb\u00e7\u013c\u0126", "\u00e7\u013e\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"], - ids: [32013, 23393, 2651, 1534, 236, 502], - decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>\u751f\u6d3b\u7684\u771f\u8c1b\u662f", - }, - LEADING_SPACE: { - text: BASE_TEST_STRINGS.LEADING_SPACE, - tokens: ["\u0120\u0120", "\u0120leading", "\u0120space"], - ids: [32013, 243, 5877, 2507], - decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c> leading space", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["tra", "iling", "\u0120space", "\u0120\u0120\u0120"], - ids: [32013, 7246, 5964, 2507, 315], - decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>trailing space ", - }, - DOUBLE_SPACE: { - text: BASE_TEST_STRINGS.DOUBLE_SPACE, - tokens: ["Hi", "\u0120", "\u0120H", "ello"], - ids: [32013, 11041, 207, 414, 9489], - decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>Hi Hello", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120", "\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2", "\u00a5", "6", "\u0120", "\u00e2\u0124", "\u00a3", "7", "\u0120", "\u00e2\u0124", "\u00b9", "8", "\u0120", "\u00e2\u0124", "\u00b1", "9", "\u0120test"], - ids: [32013, 2806, 371, 16, 432, 17, 1494, 18, 207, 11010, 19, 8761, 20, 2688, 98, 21, 207, 7935, 96, 22, 207, 7935, 117, 23, 207, 7935, 109, 24, 1719], - decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["I", "\u0120bought", "\u0120an", "\u0120apple", "\u0120for", "\u0120$", "1", ".", "0", "0", "\u0120at", "\u0120the", "\u0120store", "."], - ids: [32013, 40, 8942, 274, 15902, 327, 371, 16, 13, 15, 15, 429, 254, 4730, 13], - decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>I bought an apple for $1.00 at the store.", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - tokens: ["you", "\u00e2\u0122\u00a6", "\u0120\u0120"], - ids: [32013, 4209, 2484, 243], - decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>you\u2026 ", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], - ids: [32013, 4209, 2484, 10447], - decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>you\u2026\u00a0\u00a0", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142", "\u00c2\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], - ids: [32013, 4209, 2484, 1200, 1200, 4209, 2484, 10447], - decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["we", "ird", "\u0120", "\u00ef", "\u00bd", "\u0140", "\u0120edge", "\u0120", "\u00ef", "\u00bd", "\u0140", "\u0120case"], - ids: [32013, 828, 2369, 207, 169, 121, 239, 5935, 207, 169, 121, 239, 1452], - decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>weird \uff5e edge \uff5e case", - }, - POPULAR_EMOJIS: { - text: BASE_TEST_STRINGS.POPULAR_EMOJIS, - tokens: ["\u00f0\u0141", "\u013a", "\u0124", "\u0120\u00f0\u0141", "\u0133", "\u012f", "\u0120\u00f0\u0141", "\u00a4", "\u00a3", "\u0120\u00f0\u0141", "\u013a", "\u012f", "\u0120\u00f0\u0141", "\u013a", "\u0143", "\u0120\u00f0\u0141", "\u0130", "\u012b", "\u0120\u00f0\u0141\u013b", "\u0131", "\u0120\u00f0\u0141", "\u013a", "\u012c", "\u0120\u00f0\u0141", "\u0136", "\u00a5", "\u0120\u00f0\u0141", "\u013a", "\u0123", "\u0120\u00f0\u0141", "\u013a", "\u0127", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141", "\u013a", "\u0128", "\u0120\u00f0\u0141", "\u0133", "\u0131", "\u0120", "\u00e2", "\u013f", "\u00a4", "\u00ef", "\u00b8", "\u0131", "\u0120\u00f0\u0141", "\u0134", "\u013e", "\u0120\u00f0\u0141", "\u0134", "\u013c", "\u0120\u00f0\u0141", "\u0134", "\u0139", "\u0120\u00f0\u0141", "\u0134", "\u013b", "\u0120\u00f0\u0141", "\u0138", "\u00a4", "\u0120\u00f0\u0141", "\u013a", "\u0130", "\u0120\u00f0\u0141", "\u0133", "\u012e", "\u0120\u00f0\u0141", "\u00a5", "\u00b3", "\u0120\u00f0\u0141", "\u0134", "\u00aa", "\u0120", "\u00e2", "\u013e", "\u00a8", "\u0120\u00f0\u0141", "\u0133", "\u012b", "\u0120\u00f0\u0141", "\u0133", "\u0122", "\u0120\u00f0\u0141", "\u0134", "\u00af", "\u0120\u00f0\u0141", "\u0130", "\u012a", "\u0120\u00f0\u0141\u013b", "\u012a", "\u0120\u00f0\u0141\u013b", "\u012e", "\u0120\u00f0\u0141", "\u0134", "\u0122", "\u0120\u00f0\u0141", "\u0133", "\u0129", "\u0120\u00f0\u0141", "\u0133", "\u012d", "\u0120", "\u00e2", "\u013e", "\u0127", "\u0120\u00f0\u0141", "\u0130", "\u0123", "\u0120\u00f0\u0141", "\u012e", "\u0140", "\u0120\u00f0\u0141", "\u012e", "\u00b8", "\u0120\u00f0\u0141", "\u0134", "\u00b0"], - ids: [32013, 10047, 233, 211, 12394, 226, 222, 12394, 97, 96, 12394, 233, 222, 12394, 233, 242, 12394, 223, 218, 22709, 224, 12394, 233, 219, 12394, 229, 98, 12394, 233, 210, 12394, 233, 214, 12394, 97, 232, 12394, 233, 215, 12394, 226, 224, 207, 156, 238, 97, 169, 116, 224, 12394, 227, 237, 12394, 227, 235, 12394, 227, 232, 12394, 227, 234, 12394, 231, 97, 12394, 233, 223, 12394, 226, 221, 12394, 98, 111, 12394, 227, 103, 207, 156, 237, 101, 12394, 226, 218, 12394, 226, 209, 12394, 227, 107, 12394, 223, 217, 22709, 217, 22709, 221, 12394, 227, 209, 12394, 226, 216, 12394, 226, 220, 207, 156, 237, 214, 12394, 223, 210, 12394, 221, 239, 12394, 221, 116, 12394, 227, 108], - decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>\ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0", - }, - MULTIBYTE_EMOJIS: { - text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, - tokens: ["\u00e2", "\u013e", "\u00a8", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141", "\u0133", "\u0123", "\u00ef", "\u00b8", "\u0131", "\u0120\u00f0\u0141", "\u0133", "\u00b1", "\u00f0\u0141", "\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0137", "\u00b5", "\u00e2\u0122", "\u012f", "\u00e2", "\u013b", "\u0124", "\u00ef", "\u00b8", "\u0131", "\u0120\u00f0\u0141", "\u00a7", "\u013b", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00e2", "\u013b", "\u0124", "\u0120\u00f0\u0141", "\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u012e", "\u00be", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u0120\u00f0\u0141", "\u0133", "\u00a9", "\u00e2\u0122", "\u012f", "\u00e2", "\u013f", "\u00a4", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0134", "\u012d", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a8", "\u0120\u00f0\u0141", "\u0133", "\u00a9", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a9", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a7", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a6", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141", "\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0131", "\u00b4", "\u00f3", "\u0142", "\u0123", "\u00a7", "\u00f3", "\u0142", "\u0123", "\u00a2", "\u00f3", "\u0142", "\u0123", "\u00a5", "\u00f3", "\u0142", "\u0123", "\u00ae", "\u00f3", "\u0142", "\u0123", "\u00a7", "\u00f3", "\u0142", "\u0123", "\u00bf", "\u0120\u00f0\u0141", "\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00e2", "\u013f", "\u00a4", "\u00ef", "\u00b8", "\u0131", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0134", "\u012d", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bc"], - ids: [32013, 156, 237, 101, 12394, 97, 232, 12394, 226, 210, 169, 116, 224, 12394, 226, 109, 10047, 224, 119, 12394, 230, 113, 350, 222, 156, 234, 211, 169, 116, 224, 12394, 100, 234, 10047, 224, 119, 350, 222, 156, 234, 211, 12394, 226, 101, 10047, 224, 119, 350, 222, 10047, 221, 122, 12394, 100, 226, 350, 222, 10047, 97, 238, 350, 222, 10047, 100, 226, 12394, 226, 102, 350, 222, 156, 238, 97, 350, 222, 10047, 227, 220, 350, 222, 10047, 226, 101, 12394, 226, 102, 350, 222, 10047, 226, 102, 350, 222, 10047, 226, 100, 350, 222, 10047, 226, 99, 12394, 100, 226, 10047, 224, 119, 350, 222, 10047, 97, 238, 350, 222, 10047, 100, 226, 10047, 224, 119, 12394, 224, 112, 173, 241, 210, 100, 173, 241, 210, 95, 173, 241, 210, 98, 173, 241, 210, 106, 173, 241, 210, 100, 173, 241, 210, 123, 12394, 226, 101, 10047, 224, 119, 350, 222, 156, 238, 97, 169, 116, 224, 350, 222, 10047, 227, 220, 350, 222, 10047, 226, 101, 10047, 224, 120], - decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>\u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75\u200d\u2642\ufe0f \ud83e\uddd9\ud83c\udffb\u200d\u2642 \ud83d\udc68\ud83c\udffb\u200d\ud83c\udf3e \ud83e\uddd1\u200d\ud83e\udd1d\u200d\ud83e\uddd1 \ud83d\udc69\u200d\u2764\u200d\ud83d\udc8b\u200d\ud83d\udc68 \ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66 \ud83e\uddd1\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c\udffb \ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f \ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c\udffc", - }, - SPIECE_UNDERSCORE: { - text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, - tokens: ["\u00e2\u0138", "\u0123", "This", "\u0120", "\u00e2\u0138", "\u0123", "is", "\u0120", "\u00e2\u0138", "\u0123", "a", "\u0120", "\u00e2\u0138", "\u0123", "test", "\u0120", "\u00e2\u0138", "\u0123", "."], - ids: [32013, 11028, 210, 1559, 207, 11028, 210, 262, 207, 11028, 210, 64, 207, 11028, 210, 2806, 207, 11028, 210, 13], - decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>\u2581This \u2581is \u2581a \u2581test \u2581.", - }, - BPE_SCORES_PRIORITY_1: { - text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_1, - tokens: ["gr", "ab", "bed"], - ids: [32013, 877, 356, 3861], - decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>grabbed", - }, - BPE_SCORES_PRIORITY_2: { - text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_2, - tokens: ["\u0120grab", "bed"], - ids: [32013, 14596, 3861], - decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c> grabbed", - }, - BPE_SCORES_PRIORITY_3: { - text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_3, - tokens: ["\u0120\u0120\u0120\u0120\u0120\u0120\u0120\u0120\u0120\u0120", "\u0120grab", "bed"], - ids: [32013, 3137, 14596, 3861], - decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c> grabbed", - }, - NEWLINE: { - text: LLAMA_TEST_STRINGS.NEWLINE, - tokens: ["\u010a"], - ids: [32013, 185], - decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>\n", - }, - NEWLINE_WITH_LEADING_SPACE: { - text: LLAMA_TEST_STRINGS.NEWLINE_WITH_LEADING_SPACE, - tokens: ["\u0120", "\u010a"], - ids: [32013, 207, 185], - decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c> \n", - }, - TABS: { - text: LLAMA_TEST_STRINGS.TABS, - tokens: ["\u0109", "tabs", "\u0109\u0109\u0109", "\u0109", "out", "\u0120here"], - ids: [32013, 184, 20611, 1749, 184, 406, 1283], - decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>\ttabs\t\t\t\tout here", - }, - NEWLINE_AND_TAB: { - text: LLAMA_TEST_STRINGS.NEWLINE_AND_TAB, - tokens: ["\u010a", "\u0109", "\u010a"], - ids: [32013, 185, 184, 185], - decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>\n\t\n", - }, - CHINESE_LETTER: { - text: LLAMA_TEST_STRINGS.CHINESE_LETTER, - tokens: ["\u00e9\u0137\u0129"], - ids: [32013, 6759], - decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>\u9547", - }, - EMOJIS_1: { - text: LLAMA_TEST_STRINGS.EMOJIS_1, - tokens: ["\u00f0\u0141", "\u00a6", "\u013b"], - ids: [32013, 10047, 99, 234], - decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>\ud83e\udd99", - }, - EMOJIS_2: { - text: LLAMA_TEST_STRINGS.EMOJIS_2, - tokens: ["\u00f0\u0141", "\u00a6", "\u013b", "\u00ea", "\u013b", "\u012c"], - ids: [32013, 10047, 99, 234, 164, 234, 219], - decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>\ud83e\udd99\ua64a", - }, - EMOJIS_3: { - text: LLAMA_TEST_STRINGS.EMOJIS_3, - tokens: ["\u00ea", "\u013b", "\u012c", "\u00f0\u0141", "\u00a6", "\u013b"], - ids: [32013, 164, 234, 219, 10047, 99, 234], - decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>\ua64a\ud83e\udd99", - }, - PARAGRAPH: { - text: LLAMA_TEST_STRINGS.PARAGRAPH, - tokens: ["The", "\u0120ll", "ama", "\u0120(/", "\u00cb\u012a", "l", "\u00c9", "\u0133", "\u00cb", "\u0132", "m", "\u00c9\u013b", "/", ";", "\u0120\u00f0\u0141", "\u00a6", "\u013b", "Span", "ish", "\u0120pron", "unciation", ":", "\u0120[", "\u00cb\u012a", "\u00ca", "\u0130", "ama", "])", "\u0120(", "L", "ama", "\u0120gl", "ama", ")", "\u0120is", "\u0120a", "\u0120domestic", "ated", "\u0120South", "\u0120American", "\u0120cam", "el", "id", ",", "\u0120widely", "\u0120used", "\u0120as", "\u0120a", "\u0120meat", "\u0120and", "\u0120pack", "\u0120animal", "\u0120by", "\u0120And", "ean", "\u0120cultures", "\u0120since", "\u0120the", "\u0120Pre", "-", "Col", "umb", "ian", "\u0120era", ".", "\u0120L", "lam", "as", "\u0120are", "\u0120social", "\u0120animals", "\u0120and", "\u0120live", "\u0120with", "\u0120others", "\u0120as", "\u0120a", "\u0120her", "d", ".", "\u0120Their", "\u0120wool", "\u0120is", "\u0120soft", "\u0120and", "\u0120contains", "\u0120only", "\u0120a", "\u0120small", "\u0120amount", "\u0120of", "\u0120lan", "ol", "in", ".[", "2", "]", "\u0120L", "lam", "as", "\u0120can", "\u0120learn", "\u0120simple", "\u0120tasks", "\u0120after", "\u0120a", "\u0120few", "\u0120repet", "itions", ".", "\u0120When", "\u0120using", "\u0120a", "\u0120pack", ",", "\u0120they", "\u0120can", "\u0120carry", "\u0120about", "\u0120", "2", "5", "\u0120to", "\u0120", "3", "0", "%", "\u0120of", "\u0120their", "\u0120body", "\u0120weight", "\u0120for", "\u0120", "8", "\u0120to", "\u0120", "1", "3", "\u0120km", "\u0120(", "5", "\u00e2\u0122\u0135", "8", "\u0120miles", ").", "[", "3", "]", "\u0120The", "\u0120name", "\u0120ll", "ama", "\u0120(", "in", "\u0120the", "\u0120past", "\u0120also", "\u0120sp", "elled", '\u0120"', "l", "ama", '"', "\u0120or", '\u0120"', "gl", "ama", '")', "\u0120was", "\u0120adopted", "\u0120by", "\u0120European", "\u0120sett", "lers", "\u0120from", "\u0120native", "\u0120Per", "uv", "ians", ".[", "4", "]", "\u0120The", "\u0120ancest", "ors", "\u0120of", "\u0120llam", "as", "\u0120are", "\u0120thought", "\u0120to", "\u0120have", "\u0120origin", "ated", "\u0120from", "\u0120the", "\u0120Great", "\u0120Pl", "ains", "\u0120of", "\u0120North", "\u0120America", "\u0120about", "\u0120", "4", "0", "\u0120million", "\u0120years", "\u0120ago", ",", "\u0120and", "\u0120subsequently", "\u0120mig", "rated", "\u0120to", "\u0120South", "\u0120America", "\u0120about", "\u0120three", "\u0120million", "\u0120years", "\u0120ago", "\u0120during", "\u0120the", "\u0120Great", "\u0120American", "\u0120Inter", "change", ".", "\u0120By", "\u0120the", "\u0120end", "\u0120of", "\u0120the", "\u0120last", "\u0120ice", "\u0120age", "\u0120(", "1", "0", ",", "0", "0", "0", "\u00e2\u0122\u0135", "1", "2", ",", "0", "0", "0", "\u0120years", "\u0120ago", "),", "\u0120cam", "el", "ids", "\u0120were", "\u0120ext", "inct", "\u0120in", "\u0120North", "\u0120America", ".[", "3", "]", "\u0120As", "\u0120of", "\u0120", "2", "0", "0", "7", ",", "\u0120there", "\u0120were", "\u0120over", "\u0120seven", "\u0120million", "\u0120llam", "as", "\u0120and", "\u0120al", "p", "ac", "as", "\u0120in", "\u0120South", "\u0120America", "\u0120and", "\u0120over", "\u0120", "1", "5", "8", ",", "0", "0", "0", "\u0120llam", "as", "\u0120and", "\u0120", "1", "0", "0", ",", "0", "0", "0", "\u00ea", "\u013b", "\u012c", "\u00f0\u0141", "\u00a6", "\u013b", "\u0120al", "p", "ac", "as", ",", "\u0120desc", "ended", "\u0120from", "\u0120pro", "gen", "itors", "\u0120imported", "\u0120late", "\u0120in", "\u0120the", "\u0120", "2", "0", "th", "\u0120century", ",", "\u0120in", "\u0120the", "\u0120United", "\u0120States", "\u0120and", "\u0120Canada", ".[", "5", "]", "\u0120In", "\u0120A", "ym", "ara", "\u0120myth", "ology", ",", "\u0120llam", "as", "\u0120are", "\u0120important", "\u0120beings", ".", "\u0120The", "\u0120Heaven", "ly", "\u0120Ll", "ama", "\u0120is", "\u0120said", "\u0120to", "\u0120drink", "\u0120water", "\u0120from", "\u0120the", "\u0120ocean", "\u0120and", "\u0120ur", "in", "ates", "\u0120as", "\u0120it", "\u0120ra", "ins", ".[", "6", "]", "\u0120According", "\u0120to", "\u0120A", "ym", "ara", "\u0120es", "chat", "ology", ",", "\u0120llam", "as", "\u0120will", "\u0120return", "\u0120to", "\u0120the", "\u0120water", "\u0120springs", "\u0120and", "\u0120l", "ago", "ons", "\u0120where", "\u0120they", "\u0120come", "\u0120from", "\u0120at", "\u0120the", "\u0120end", "\u0120of", "\u0120time", ".[", "6", "]"], - ids: [32013, 546, 1703, 4204, 31905, 31459, 75, 131, 226, 133, 225, 76, 28747, 14, 26, 12394, 99, 234, 20786, 840, 9119, 25307, 25, 821, 31459, 132, 223, 4204, 5589, 334, 43, 4204, 1649, 4204, 8, 317, 245, 13569, 612, 5168, 4115, 4370, 282, 304, 11, 13620, 1219, 372, 245, 12342, 285, 2379, 9542, 457, 1306, 24391, 24783, 1952, 254, 7606, 12, 2608, 4313, 987, 2895, 13, 412, 8265, 281, 417, 3601, 8469, 285, 3516, 365, 3060, 372, 245, 706, 67, 13, 9195, 24547, 317, 2829, 285, 5396, 885, 245, 1752, 3733, 280, 27264, 313, 246, 9469, 17, 60, 412, 8265, 281, 482, 3059, 2966, 9227, 1164, 245, 1853, 15747, 2160, 13, 2463, 1242, 245, 2379, 11, 653, 482, 5642, 782, 207, 17, 20, 276, 207, 18, 15, 4, 280, 699, 3110, 4285, 327, 207, 23, 276, 207, 16, 18, 9004, 334, 20, 887, 23, 6595, 628, 58, 18, 60, 428, 1208, 1703, 4204, 334, 246, 254, 2872, 835, 731, 6679, 440, 75, 4204, 1, 409, 440, 2521, 4204, 2456, 438, 13509, 457, 8717, 6762, 12104, 473, 8118, 3043, 12466, 3091, 9469, 19, 60, 428, 18901, 710, 280, 15410, 281, 417, 2207, 276, 463, 6948, 612, 473, 254, 6984, 2284, 2200, 280, 5216, 6092, 782, 207, 19, 15, 4866, 1547, 4074, 11, 285, 23909, 8290, 9831, 276, 5168, 6092, 782, 1846, 4866, 1547, 4074, 2310, 254, 6984, 4115, 6660, 4865, 13, 3550, 254, 1223, 280, 254, 1554, 9405, 4489, 334, 16, 15, 11, 15, 15, 15, 887, 16, 17, 11, 15, 15, 15, 1547, 4074, 650, 4370, 282, 2929, 773, 1309, 5729, 279, 5216, 6092, 9469, 18, 60, 1725, 280, 207, 17, 15, 15, 22, 11, 741, 773, 851, 7970, 4866, 15410, 281, 285, 360, 79, 305, 281, 279, 5168, 6092, 285, 851, 207, 16, 20, 23, 11, 15, 15, 15, 15410, 281, 285, 207, 16, 15, 15, 11, 15, 15, 15, 164, 234, 219, 10047, 99, 234, 360, 79, 305, 281, 11, 1774, 2611, 473, 381, 4920, 6041, 26357, 5179, 279, 254, 207, 17, 15, 392, 8299, 11, 279, 254, 4783, 5098, 285, 8905, 9469, 20, 60, 680, 338, 1254, 3367, 25157, 2333, 11, 15410, 281, 417, 2364, 22792, 13, 428, 18933, 326, 9140, 4204, 317, 989, 276, 7371, 2345, 473, 254, 15439, 285, 8580, 246, 980, 372, 359, 1809, 1231, 9469, 21, 60, 10068, 276, 338, 1254, 3367, 707, 24570, 2333, 11, 15410, 281, 540, 967, 276, 254, 2345, 30851, 285, 284, 5980, 875, 1064, 653, 1857, 473, 429, 254, 1223, 280, 761, 9469, 21, 60], - decoded: '<\uff5cbegin\u2581of\u2581sentence\uff5c>The llama (/\u02c8l\u0251\u02d0m\u0259/; \ud83e\udd99Spanish pronunciation: [\u02c8\u028eama]) (Lama glama) is a domesticated South American camelid, widely used as a meat and pack animal by Andean cultures since the Pre-Columbian era. Llamas are social animals and live with others as a herd. Their wool is soft and contains only a small amount of lanolin.[2] Llamas can learn simple tasks after a few repetitions. When using a pack, they can carry about 25 to 30% of their body weight for 8 to 13 km (5\u20138 miles).[3] The name llama (in the past also spelled "lama" or "glama") was adopted by European settlers from native Peruvians.[4] The ancestors of llamas are thought to have originated from the Great Plains of North America about 40 million years ago, and subsequently migrated to South America about three million years ago during the Great American Interchange. By the end of the last ice age (10,000\u201312,000 years ago), camelids were extinct in North America.[3] As of 2007, there were over seven million llamas and alpacas in South America and over 158,000 llamas and 100,000\ua64a\ud83e\udd99 alpacas, descended from progenitors imported late in the 20th century, in the United States and Canada.[5] In Aymara mythology, llamas are important beings. The Heavenly Llama is said to drink water from the ocean and urinates as it rains.[6] According to Aymara eschatology, llamas will return to the water springs and lagoons where they come from at the end of time.[6]', - }, - }, - "Xenova/tamillama_tiny_30m": { - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["\u2581The", "\u2581company", "\u2581was", "\u2581found", "ed", "\u2581in", "\u2581", "2", "0", "1", "6", "."], - ids: [1, 147, 10984, 139, 949, 78, 198, 31654, 13, 21, 12, 17, 34], - decoded: " The company was founded in 2016.", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["\u2581A", "\n", "'", "ll", "\u2581", "!", "!", "to", "?", "'", "d", "'", "'", "d", "\u2581of", ",", "\u2581can", "'", "t", "."], - ids: [1, 231, 5, 31, 370, 31654, 31715, 31715, 5140, 31725, 31, 31679, 31, 31, 31679, 251, 35, 645, 31, 31665, 34], - decoded: " A\n'll !!to?'d''d of, can't.", - }, - PYTHON_CODE: { - text: BASE_TEST_STRINGS.PYTHON_CODE, - tokens: ["\u2581def", "\u2581main", "(", ")", ":", "\n", "", "p", "ass"], - ids: [1, 12849, 17375, 32, 33, 29, 5, 0, 31694, 1917], - decoded: " def main():\npass", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["\u2581let", "\u2581a", "\u2581", "=", "\u2581ob", "j", ".", "to", "St", "ring", "(", ")", ";", "\n", "to", "St", "ring", "(", ")", ";"], - ids: [1, 1996, 48, 31654, 25, 4083, 31733, 34, 5140, 23417, 6631, 32, 33, 30, 5, 5140, 23417, 6631, 32, 33, 30], - decoded: " let a = obj.toString();\ntoString();", - }, - NEWLINES: { - text: LLAMA_TEST_STRINGS.NEWLINES, - tokens: ["\u2581ax", "\n", "#", "#", "#", "#", "\n", "boo"], - ids: [1, 11441, 5, 22, 22, 22, 22, 5, 21260], - decoded: " ax\n####\nboo", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["\u2581U", "N", "w", "ant", "\u00e9", "d", ",", "r", "un", "ning"], - ids: [1, 5841, 31748, 31689, 1027, 31771, 31679, 35, 31678, 367, 1855], - decoded: " UNwant\u00e9d,running", - }, - CONTROL_TOKENS: { - text: BASE_TEST_STRINGS.CONTROL_TOKENS, - tokens: ["\u2581", "1", "", "2", "", "3"], - ids: [1, 31654, 12, 0, 13, 0, 14], - decoded: " 123", - }, - HELLO_WORLD_TITLECASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, - tokens: ["\u2581H", "ello", "\u2581World"], - ids: [1, 207, 3589, 25544], - decoded: " Hello World", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u2581", ""], - ids: [1, 31654, 0], - decoded: " ", - }, - LEADING_SPACE: { - text: BASE_TEST_STRINGS.LEADING_SPACE, - tokens: ["\u2581", "\u2581", "\u2581", "\u2581leading", "\u2581space"], - ids: [1, 31654, 31654, 31654, 7951, 7259], - decoded: " leading space", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["\u2581tra", "iling", "\u2581space", "\u2581", "\u2581", "\u2581"], - ids: [1, 2036, 9850, 7259, 31654, 31654, 31654], - decoded: " trailing space ", - }, - DOUBLE_SPACE: { - text: BASE_TEST_STRINGS.DOUBLE_SPACE, - tokens: ["\u2581H", "i", "\u2581", "\u2581H", "ello"], - ids: [1, 207, 31673, 31654, 207, 3589], - decoded: " Hi Hello", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["\u2581test", "\u2581", "$", "1", "\u2581R", "2", "\u2581", "#", "3", "\u2581", "\u20ac", "4", "\u2581", "\u00a3", "5", "\u2581", "", "6", "\u2581", "", "7", "\u2581", "\u20b9", "8", "\u2581", "", "9", "\u2581test"], - ids: [1, 6370, 31654, 9, 12, 947, 13, 31654, 22, 14, 31654, 31746, 15, 31654, 31792, 16, 31654, 0, 17, 31654, 0, 18, 31654, 31999, 19, 31654, 0, 20, 6370], - decoded: " test $1 R2 #3 \u20ac4 \u00a35 6 7 \u20b98 9 test", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["\u2581I", "\u2581bought", "\u2581an", "\u2581apple", "\u2581for", "\u2581", "$", "1", ".", "0", "0", "\u2581at", "\u2581the", "\u2581store", "."], - ids: [1, 320, 4685, 446, 4223, 347, 31654, 9, 12, 34, 21, 21, 586, 70, 2023, 34], - decoded: " I bought an apple for $1.00 at the store.", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - tokens: ["\u2581you", "", "\u2581", "\u2581"], - ids: [1, 356, 0, 31654, 31654], - decoded: " you ", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - tokens: ["\u2581you", ""], - ids: [1, 356, 0], - decoded: " you", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - tokens: ["\u2581you", "", "you", ""], - ids: [1, 356, 0, 21984, 0], - decoded: " youyou", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["\u2581weird", "\u2581", "", "\u2581edge", "\u2581", "", "\u2581case"], - ids: [1, 7865, 31654, 0, 11148, 31654, 0, 10143], - decoded: " weird edge case", - }, - SPIECE_UNDERSCORE: { - text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, - tokens: ["\u2581", "\u2581This", "\u2581", "\u2581is", "\u2581", "\u2581a", "\u2581", "\u2581test", "\u2581", "\u2581", "."], - ids: [1, 31654, 3827, 31654, 344, 31654, 48, 31654, 6370, 31654, 31654, 34], - decoded: " This is a test .", - }, - POPULAR_EMOJIS: { - text: BASE_TEST_STRINGS.POPULAR_EMOJIS, - tokens: ["\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", ""], - ids: [1, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0], - decoded: " ", - }, - MULTIBYTE_EMOJIS: { - text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, - tokens: ["\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u200d", "", "\u2581", "", "\u200d", "", "\u2581", "", "\u200d", "", "\u2581", "", "\u200d", "", "\u200d", "", "\u2581", "", "\u200d", "", "\u200d", "", "\u200d", "", "\u2581", "", "\u200d", "", "\u200d", "", "\u200d", "", "\u2581", "", "\u200d", "", "\u200d", "", "\u2581", "", "\u2581", "", "\u200d", "", "\u200d", "", "\u200d", ""], - ids: [1, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31928, 0, 31654, 0, 31928, 0, 31654, 0, 31928, 0, 31654, 0, 31928, 0, 31928, 0, 31654, 0, 31928, 0, 31928, 0, 31928, 0, 31654, 0, 31928, 0, 31928, 0, 31928, 0, 31654, 0, 31928, 0, 31928, 0, 31654, 0, 31654, 0, 31928, 0, 31928, 0, 31928, 0], - decoded: " \u200d \u200d \u200d \u200d\u200d \u200d\u200d\u200d \u200d\u200d\u200d \u200d\u200d \u200d\u200d\u200d", - }, - BPE_SCORES_PRIORITY_1: { - text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_1, - tokens: ["\u2581grabbed"], - ids: [1, 3618], - decoded: " grabbed", - }, - BPE_SCORES_PRIORITY_2: { - text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_2, - tokens: ["\u2581", "\u2581grabbed"], - ids: [1, 31654, 3618], - decoded: " grabbed", - }, - BPE_SCORES_PRIORITY_3: { - text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_3, - tokens: ["\u2581", "\u2581", "\u2581", "\u2581", "\u2581", "\u2581", "\u2581", "\u2581", "\u2581", "\u2581", "\u2581", "\u2581grabbed"], - ids: [1, 31654, 31654, 31654, 31654, 31654, 31654, 31654, 31654, 31654, 31654, 31654, 3618], - decoded: " grabbed", - }, - NEWLINE: { - text: LLAMA_TEST_STRINGS.NEWLINE, - tokens: ["\u2581", "\n"], - ids: [1, 31654, 5], - decoded: " \n", - }, - NEWLINE_WITH_LEADING_SPACE: { - text: LLAMA_TEST_STRINGS.NEWLINE_WITH_LEADING_SPACE, - tokens: ["\u2581", "\u2581", "\n"], - ids: [1, 31654, 31654, 5], - decoded: " \n", - }, - TABS: { - text: LLAMA_TEST_STRINGS.TABS, - tokens: ["\u2581", "", "t", "ab", "s", "", "out", "\u2581here"], - ids: [1, 31654, 0, 31665, 878, 31675, 0, 415, 3278], - decoded: " tabsout here", - }, - NEWLINE_AND_TAB: { - text: LLAMA_TEST_STRINGS.NEWLINE_AND_TAB, - tokens: ["\u2581", "\n", "", "\n"], - ids: [1, 31654, 5, 0, 5], - decoded: " \n\n", - }, - CHINESE_LETTER: { - text: LLAMA_TEST_STRINGS.CHINESE_LETTER, - tokens: ["\u2581", ""], - ids: [1, 31654, 0], - decoded: " ", - }, - EMOJIS_1: { - text: LLAMA_TEST_STRINGS.EMOJIS_1, - tokens: ["\u2581", ""], - ids: [1, 31654, 0], - decoded: " ", + // - Sequence PostProcessor + // - "ignore_merges": true + "Xenova/llama3-tokenizer-new": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["How", "\u0120are", "\u0120you", "\u0120doing", "?"], + ids: [128000, 4438, 527, 499, 3815, 30], + decoded: "<|begin_of_text|>How are you doing?", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["You", "\u0120should", "'ve", "\u0120done", "\u0120this"], + ids: [128000, 2675, 1288, 3077, 2884, 420], + decoded: "<|begin_of_text|>You should've done this", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["012", "345", "678", "9", "\u0120", "0", "\u0120", "1", "\u0120", "2", "\u0120", "3", "\u0120", "4", "\u0120", "5", "\u0120", "6", "\u0120", "7", "\u0120", "8", "\u0120", "9", "\u0120", "10", "\u0120", "100", "\u0120", "100", "0"], + ids: [128000, 11531, 12901, 17458, 24, 220, 15, 220, 16, 220, 17, 220, 18, 220, 19, 220, 20, 220, 21, 220, 22, 220, 23, 220, 24, 220, 605, 220, 1041, 220, 1041, 15], + decoded: "<|begin_of_text|>0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "\u0120company", "\u0120was", "\u0120founded", "\u0120in", "\u0120", "201", "6", "."], + ids: [128000, 791, 2883, 574, 18538, 304, 220, 679, 21, 13], + decoded: "<|begin_of_text|>The company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "\u010a", "'ll", "\u0120!!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'t", "."], + ids: [128000, 32, 198, 3358, 11261, 998, 20837, 67, 4708, 67, 315, 11, 649, 956, 13], + decoded: "<|begin_of_text|>A\n'll!!to?'d''d of, can't.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["def", "\u0120main", "():\u010a", "\u0109pass"], + ids: [128000, 755, 1925, 4019, 42531], + decoded: "<|begin_of_text|>def main():\n\tpass", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".toString", "();\u010a", "toString", "();"], + ids: [128000, 1169, 264, 284, 2909, 5180, 545, 6712, 2178], + decoded: "<|begin_of_text|>let a = obj.toString();\ntoString();", + }, + NEWLINES: { + text: LLAMA_TEST_STRINGS.NEWLINES, + tokens: ["ax", "\u010a", "####\u010a", "boo"], + ids: [128000, 710, 198, 71050, 34093], + decoded: "<|begin_of_text|>ax\n####\nboo", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "want", "\u00c3\u00a9d", ",", "running"], + ids: [128000, 1899, 53757, 15433, 11, 28272], + decoded: "<|begin_of_text|>UNwant\u00e9d,running", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["1", "\u0100", "2", "\u00ef\u00bf\u00bd", "3"], + ids: [128000, 16, 188, 17, 5809, 18], + decoded: "<|begin_of_text|>1\u00002\ufffd3", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["Hello", "\u0120World"], + ids: [128000, 9906, 4435], + decoded: "<|begin_of_text|>Hello World", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["hello", "\u0120world"], + ids: [128000, 15339, 1917], + decoded: "<|begin_of_text|>hello world", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u00e7\u0136\u0141\u00e6\u00b4\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"], + ids: [128000, 104654, 9554, 89151, 39013, 249, 21043], + decoded: "<|begin_of_text|>\u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u0120\u0120", "\u0120leading", "\u0120space"], + ids: [128000, 256, 6522, 3634], + decoded: "<|begin_of_text|> leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["tr", "ailing", "\u0120space", "\u0120\u0120\u0120"], + ids: [128000, 376, 14612, 3634, 262], + decoded: "<|begin_of_text|>trailing space ", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["Hi", "\u0120", "\u0120Hello"], + ids: [128000, 13347, 220, 22691], + decoded: "<|begin_of_text|>Hi Hello", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2\u00a5", "6", "\u0120\u00e2\u0124", "\u00a3", "7", "\u0120\u00e2\u0124\u00b9", "8", "\u0120\u00e2\u0124", "\u00b1", "9", "\u0120test"], + ids: [128000, 1985, 400, 16, 432, 17, 674, 18, 13281, 19, 7083, 20, 72588, 21, 113384, 96, 22, 90891, 23, 113384, 109, 24, 1296], + decoded: "<|begin_of_text|>test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "\u0120bought", "\u0120an", "\u0120apple", "\u0120for", "\u0120$", "1", ".", "00", "\u0120at", "\u0120the", "\u0120store", "."], + ids: [128000, 40, 11021, 459, 24149, 369, 400, 16, 13, 410, 520, 279, 3637, 13], + decoded: "<|begin_of_text|>I bought an apple for $1.00 at the store.", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u0120\u0120"], + ids: [128000, 9514, 1981, 256], + decoded: "<|begin_of_text|>you\u2026 ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], + ids: [128000, 9514, 1981, 9421], + decoded: "<|begin_of_text|>you\u2026\u00a0\u00a0", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142", "\u00c2\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], + ids: [128000, 9514, 1981, 4194, 4194, 9514, 1981, 9421], + decoded: "<|begin_of_text|>you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["we", "ird", "\u0120\u00ef\u00bd\u0140", "\u0120edge", "\u0120\u00ef\u00bd\u0140", "\u0120case"], + ids: [128000, 906, 2668, 111942, 6964, 111942, 1162], + decoded: "<|begin_of_text|>weird \uff5e edge \uff5e case", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u00e2\u0138", "\u0123", "This", "\u0120\u00e2\u0138", "\u0123", "is", "\u0120\u00e2\u0138", "\u0123", "a", "\u0120\u00e2\u0138", "\u0123", "test", "\u0120\u00e2\u0138", "\u0123", "."], + ids: [128000, 10634, 223, 2028, 14860, 223, 285, 14860, 223, 64, 14860, 223, 1985, 14860, 223, 13], + decoded: "<|begin_of_text|>\u2581This \u2581is \u2581a \u2581test \u2581.", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u00f0\u0141\u013a", "\u0124", "\u0120\u00f0\u0141\u0133", "\u012f", "\u0120\u00f0\u0141", "\u00a4", "\u00a3", "\u0120\u00f0\u0141\u013a", "\u012f", "\u0120\u00f0\u0141\u013a", "\u0143", "\u0120\u00f0\u0141", "\u0130", "\u012b", "\u0120\u00f0\u0141", "\u013b", "\u0131", "\u0120\u00f0\u0141\u013a", "\u012c", "\u0120\u00f0\u0141\u0136", "\u00a5", "\u0120\u00f0\u0141\u013a", "\u0123", "\u0120\u00f0\u0141\u013a", "\u0127", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141\u013a", "\u0128", "\u0120\u00f0\u0141\u0133", "\u0131", "\u0120\u00e2\u013f\u00a4", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141\u0134", "\u013e", "\u0120\u00f0\u0141\u0134", "\u013c", "\u0120\u00f0\u0141\u0134", "\u0139", "\u0120\u00f0\u0141\u0134", "\u013b", "\u0120\u00f0\u0141", "\u0138", "\u00a4", "\u0120\u00f0\u0141\u013a", "\u0130", "\u0120\u00f0\u0141\u0133", "\u012e", "\u0120\u00f0\u0141", "\u00a5", "\u00b3", "\u0120\u00f0\u0141\u0134", "\u00aa", "\u0120\u00e2\u013e", "\u00a8", "\u0120\u00f0\u0141\u0133", "\u012b", "\u0120\u00f0\u0141\u0133", "\u0122", "\u0120\u00f0\u0141\u0134", "\u00af", "\u0120\u00f0\u0141", "\u0130", "\u012a", "\u0120\u00f0\u0141", "\u013b", "\u012a", "\u0120\u00f0\u0141", "\u013b", "\u012e", "\u0120\u00f0\u0141\u0134", "\u0122", "\u0120\u00f0\u0141\u0133", "\u0129", "\u0120\u00f0\u0141\u0133", "\u012d", "\u0120\u00e2\u013e", "\u0127", "\u0120\u00f0\u0141", "\u0130", "\u0123", "\u0120\u00f0\u0141", "\u012e", "\u0140", "\u0120\u00f0\u0141", "\u012e", "\u00b8", "\u0120\u00f0\u0141\u0134", "\u00b0"], + ids: [128000, 76460, 224, 62904, 235, 11410, 97, 96, 27623, 235, 27623, 255, 11410, 236, 231, 11410, 247, 237, 27623, 232, 96169, 98, 27623, 223, 27623, 227, 11410, 97, 245, 27623, 228, 62904, 237, 71570, 31643, 64139, 250, 64139, 248, 64139, 245, 64139, 247, 11410, 244, 97, 27623, 236, 62904, 234, 11410, 98, 111, 64139, 103, 26602, 101, 62904, 231, 62904, 222, 64139, 107, 11410, 236, 230, 11410, 247, 230, 11410, 247, 234, 64139, 222, 62904, 229, 62904, 233, 26602, 227, 11410, 236, 223, 11410, 234, 252, 11410, 234, 116, 64139, 108], + decoded: "<|begin_of_text|>\ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u00e2\u013e", "\u00a8", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141\u0133", "\u0123", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141\u0133", "\u00b1", "\u00f0\u0141", "\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0137", "\u00b5", "\u00e2\u0122\u012f", "\u00e2\u013b", "\u0124", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141", "\u00a7", "\u013b", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00e2\u013b", "\u0124", "\u0120\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u012e", "\u00be", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u0120\u00f0\u0141\u0133", "\u00a9", "\u00e2\u0122\u012f", "\u00e2\u013f\u00a4", "\u00e2\u0122\u012f", "\u00f0\u0141\u0134", "\u012d", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a8", "\u0120\u00f0\u0141\u0133", "\u00a9", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a9", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a7", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a6", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141", "\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0131", "\u00b4", "\u00f3", "\u0142\u0123", "\u00a7", "\u00f3", "\u0142\u0123", "\u00a2", "\u00f3", "\u0142\u0123", "\u00a5", "\u00f3", "\u0142\u0123", "\u00ae", "\u00f3", "\u0142\u0123", "\u00a7", "\u00f3", "\u0142\u0123", "\u00bf", "\u0120\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00e2\u013f\u00a4", "\u00ef\u00b8\u0131", "\u00e2\u0122\u012f", "\u00f0\u0141\u0134", "\u012d", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bc"], + ids: [128000, 38798, 101, 11410, 97, 245, 62904, 223, 31643, 62904, 109, 9468, 237, 119, 11410, 243, 113, 102470, 17245, 224, 31643, 11410, 100, 247, 9468, 237, 119, 102470, 17245, 224, 62904, 101, 9468, 237, 119, 102470, 9468, 234, 122, 11410, 100, 239, 102470, 9468, 97, 251, 102470, 9468, 100, 239, 62904, 102, 102470, 121643, 102470, 93273, 233, 102470, 9468, 239, 101, 62904, 102, 102470, 9468, 239, 102, 102470, 9468, 239, 100, 102470, 9468, 239, 99, 11410, 100, 239, 9468, 237, 119, 102470, 9468, 97, 251, 102470, 9468, 100, 239, 9468, 237, 119, 11410, 237, 112, 175, 16050, 100, 175, 16050, 95, 175, 16050, 98, 175, 16050, 106, 175, 16050, 100, 175, 16050, 123, 62904, 101, 9468, 237, 119, 102470, 121643, 31643, 102470, 93273, 233, 102470, 9468, 239, 101, 9468, 237, 120], + decoded: "<|begin_of_text|>\u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75\u200d\u2642\ufe0f \ud83e\uddd9\ud83c\udffb\u200d\u2642 \ud83d\udc68\ud83c\udffb\u200d\ud83c\udf3e \ud83e\uddd1\u200d\ud83e\udd1d\u200d\ud83e\uddd1 \ud83d\udc69\u200d\u2764\u200d\ud83d\udc8b\u200d\ud83d\udc68 \ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66 \ud83e\uddd1\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c\udffb \ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f \ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c\udffc", + }, + BPE_SCORES_PRIORITY_1: { + text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_1, + tokens: ["grab", "bed"], + ids: [128000, 59312, 2788], + decoded: "<|begin_of_text|>grabbed", + }, + BPE_SCORES_PRIORITY_2: { + text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_2, + tokens: ["\u0120grabbed"], + ids: [128000, 30418], + decoded: "<|begin_of_text|> grabbed", + }, + BPE_SCORES_PRIORITY_3: { + text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_3, + tokens: ["\u0120\u0120\u0120\u0120\u0120\u0120\u0120\u0120\u0120\u0120", "\u0120grabbed"], + ids: [128000, 1881, 30418], + decoded: "<|begin_of_text|> grabbed", + }, + NEWLINE: { + text: LLAMA_TEST_STRINGS.NEWLINE, + tokens: ["\u010a"], + ids: [128000, 198], + decoded: "<|begin_of_text|>\n", + }, + NEWLINE_WITH_LEADING_SPACE: { + text: LLAMA_TEST_STRINGS.NEWLINE_WITH_LEADING_SPACE, + tokens: ["\u0120\u010a"], + ids: [128000, 720], + decoded: "<|begin_of_text|> \n", + }, + TABS: { + text: LLAMA_TEST_STRINGS.TABS, + tokens: ["\u0109t", "abs", "\u0109\u0109\u0109", "\u0109out", "\u0120here"], + ids: [128000, 3324, 3518, 573, 14294, 1618], + decoded: "<|begin_of_text|>\ttabs\t\t\t\tout here", + }, + NEWLINE_AND_TAB: { + text: LLAMA_TEST_STRINGS.NEWLINE_AND_TAB, + tokens: ["\u010a\u0109\u010a"], + ids: [128000, 18108], + decoded: "<|begin_of_text|>\n\t\n", + }, + CHINESE_LETTER: { + text: LLAMA_TEST_STRINGS.CHINESE_LETTER, + tokens: ["\u00e9\u0137\u0129"], + ids: [128000, 104643], + decoded: "<|begin_of_text|>\u9547", + }, + EMOJIS_1: { + text: LLAMA_TEST_STRINGS.EMOJIS_1, + tokens: ["\u00f0\u0141", "\u00a6", "\u013b"], + ids: [128000, 9468, 99, 247], + decoded: "<|begin_of_text|>\ud83e\udd99", + }, + EMOJIS_2: { + text: LLAMA_TEST_STRINGS.EMOJIS_2, + tokens: ["\u00f0\u0141", "\u00a6", "\u013b", "\u00ea", "\u013b", "\u012c"], + ids: [128000, 9468, 99, 247, 166, 247, 232], + decoded: "<|begin_of_text|>\ud83e\udd99\ua64a", + }, + EMOJIS_3: { + text: LLAMA_TEST_STRINGS.EMOJIS_3, + tokens: ["\u00ea", "\u013b", "\u012c", "\u00f0\u0141", "\u00a6", "\u013b"], + ids: [128000, 166, 247, 232, 9468, 99, 247], + decoded: "<|begin_of_text|>\ua64a\ud83e\udd99", + }, + PARAGRAPH: { + text: LLAMA_TEST_STRINGS.PARAGRAPH, + tokens: ["The", "\u0120llama", "\u0120(/", "\u00cb", "\u012a", "l", "\u00c9", "\u0133", "\u00cb", "\u0132", "m", "\u00c9\u013b", "/", ";", "\u0120\u00f0\u0141", "\u00a6", "\u013b", "Spanish", "\u0120pronunciation", ":", "\u0120[", "\u00cb", "\u012a", "\u00ca", "\u0130", "ama", "])", "\u0120(", "L", "ama", "\u0120gl", "ama", ")", "\u0120is", "\u0120a", "\u0120domestic", "ated", "\u0120South", "\u0120American", "\u0120camel", "id", ",", "\u0120widely", "\u0120used", "\u0120as", "\u0120a", "\u0120meat", "\u0120and", "\u0120pack", "\u0120animal", "\u0120by", "\u0120And", "ean", "\u0120cultures", "\u0120since", "\u0120the", "\u0120Pre", "-C", "olum", "bian", "\u0120era", ".", "\u0120L", "lam", "as", "\u0120are", "\u0120social", "\u0120animals", "\u0120and", "\u0120live", "\u0120with", "\u0120others", "\u0120as", "\u0120a", "\u0120herd", ".", "\u0120Their", "\u0120wool", "\u0120is", "\u0120soft", "\u0120and", "\u0120contains", "\u0120only", "\u0120a", "\u0120small", "\u0120amount", "\u0120of", "\u0120lan", "olin", ".[", "2", "]", "\u0120L", "lam", "as", "\u0120can", "\u0120learn", "\u0120simple", "\u0120tasks", "\u0120after", "\u0120a", "\u0120few", "\u0120repetitions", ".", "\u0120When", "\u0120using", "\u0120a", "\u0120pack", ",", "\u0120they", "\u0120can", "\u0120carry", "\u0120about", "\u0120", "25", "\u0120to", "\u0120", "30", "%", "\u0120of", "\u0120their", "\u0120body", "\u0120weight", "\u0120for", "\u0120", "8", "\u0120to", "\u0120", "13", "\u0120km", "\u0120(", "5", "\u00e2\u0122\u0135", "8", "\u0120miles", ").[", "3", "]", "\u0120The", "\u0120name", "\u0120llama", "\u0120(", "in", "\u0120the", "\u0120past", "\u0120also", "\u0120spelled", '\u0120"', "lama", '"', "\u0120or", '\u0120"', "gl", "ama", '")', "\u0120was", "\u0120adopted", "\u0120by", "\u0120European", "\u0120settlers", "\u0120from", "\u0120native", "\u0120Per", "uv", "ians", ".[", "4", "]", "\u0120The", "\u0120ancestors", "\u0120of", "\u0120ll", "amas", "\u0120are", "\u0120thought", "\u0120to", "\u0120have", "\u0120originated", "\u0120from", "\u0120the", "\u0120Great", "\u0120Plains", "\u0120of", "\u0120North", "\u0120America", "\u0120about", "\u0120", "40", "\u0120million", "\u0120years", "\u0120ago", ",", "\u0120and", "\u0120subsequently", "\u0120migrated", "\u0120to", "\u0120South", "\u0120America", "\u0120about", "\u0120three", "\u0120million", "\u0120years", "\u0120ago", "\u0120during", "\u0120the", "\u0120Great", "\u0120American", "\u0120Inter", "change", ".", "\u0120By", "\u0120the", "\u0120end", "\u0120of", "\u0120the", "\u0120last", "\u0120ice", "\u0120age", "\u0120(", "10", ",", "000", "\u00e2\u0122\u0135", "12", ",", "000", "\u0120years", "\u0120ago", "),", "\u0120camel", "ids", "\u0120were", "\u0120extinct", "\u0120in", "\u0120North", "\u0120America", ".[", "3", "]", "\u0120As", "\u0120of", "\u0120", "200", "7", ",", "\u0120there", "\u0120were", "\u0120over", "\u0120seven", "\u0120million", "\u0120ll", "amas", "\u0120and", "\u0120al", "pac", "as", "\u0120in", "\u0120South", "\u0120America", "\u0120and", "\u0120over", "\u0120", "158", ",", "000", "\u0120ll", "amas", "\u0120and", "\u0120", "100", ",", "000", "\u00ea", "\u013b", "\u012c", "\u00f0\u0141", "\u00a6", "\u013b", "\u0120al", "pac", "as", ",", "\u0120descended", "\u0120from", "\u0120progen", "itors", "\u0120imported", "\u0120late", "\u0120in", "\u0120the", "\u0120", "20", "th", "\u0120century", ",", "\u0120in", "\u0120the", "\u0120United", "\u0120States", "\u0120and", "\u0120Canada", ".[", "5", "]", "\u0120In", "\u0120A", "ym", "ara", "\u0120mythology", ",", "\u0120ll", "amas", "\u0120are", "\u0120important", "\u0120beings", ".", "\u0120The", "\u0120Heavenly", "\u0120L", "lama", "\u0120is", "\u0120said", "\u0120to", "\u0120drink", "\u0120water", "\u0120from", "\u0120the", "\u0120ocean", "\u0120and", "\u0120ur", "in", "ates", "\u0120as", "\u0120it", "\u0120rains", ".[", "6", "]", "\u0120According", "\u0120to", "\u0120A", "ym", "ara", "\u0120es", "chat", "ology", ",", "\u0120ll", "amas", "\u0120will", "\u0120return", "\u0120to", "\u0120the", "\u0120water", "\u0120springs", "\u0120and", "\u0120l", "ago", "ons", "\u0120where", "\u0120they", "\u0120come", "\u0120from", "\u0120at", "\u0120the", "\u0120end", "\u0120of", "\u0120time", ".[", "6", "]"], + ids: [128000, 791, 94776, 47325, 135, 230, 75, 133, 239, 135, 238, 76, 99638, 14, 26, 11410, 99, 247, 62897, 71722, 25, 510, 135, 230, 134, 236, 3105, 2526, 320, 43, 3105, 2840, 3105, 8, 374, 264, 13018, 660, 4987, 3778, 50252, 307, 11, 13882, 1511, 439, 264, 13339, 323, 3854, 10065, 555, 1628, 5420, 27833, 2533, 279, 5075, 7813, 1152, 13464, 11639, 13, 445, 24705, 300, 527, 3674, 10099, 323, 3974, 449, 3885, 439, 264, 59213, 13, 11205, 39640, 374, 8579, 323, 5727, 1193, 264, 2678, 3392, 315, 31791, 37737, 8032, 17, 60, 445, 24705, 300, 649, 4048, 4382, 9256, 1306, 264, 2478, 86066, 13, 3277, 1701, 264, 3854, 11, 814, 649, 6920, 922, 220, 914, 311, 220, 966, 4, 315, 872, 2547, 4785, 369, 220, 23, 311, 220, 1032, 13437, 320, 20, 4235, 23, 8931, 94638, 18, 60, 578, 836, 94776, 320, 258, 279, 3347, 1101, 68918, 330, 81101, 1, 477, 330, 6200, 3105, 909, 574, 18306, 555, 7665, 61107, 505, 10068, 3700, 12328, 5493, 8032, 19, 60, 578, 38618, 315, 9507, 29189, 527, 3463, 311, 617, 44853, 505, 279, 8681, 63911, 315, 4892, 5270, 922, 220, 1272, 3610, 1667, 4227, 11, 323, 28520, 73691, 311, 4987, 5270, 922, 2380, 3610, 1667, 4227, 2391, 279, 8681, 3778, 5783, 3455, 13, 3296, 279, 842, 315, 279, 1566, 10054, 4325, 320, 605, 11, 931, 4235, 717, 11, 931, 1667, 4227, 705, 50252, 3447, 1051, 69918, 304, 4892, 5270, 8032, 18, 60, 1666, 315, 220, 1049, 22, 11, 1070, 1051, 927, 8254, 3610, 9507, 29189, 323, 453, 46051, 300, 304, 4987, 5270, 323, 927, 220, 11286, 11, 931, 9507, 29189, 323, 220, 1041, 11, 931, 166, 247, 232, 9468, 99, 247, 453, 46051, 300, 11, 58842, 505, 84360, 12170, 25973, 3389, 304, 279, 220, 508, 339, 9478, 11, 304, 279, 3723, 4273, 323, 7008, 8032, 20, 60, 763, 362, 1631, 5169, 59492, 11, 9507, 29189, 527, 3062, 23837, 13, 578, 88150, 445, 81101, 374, 1071, 311, 7172, 3090, 505, 279, 18435, 323, 4433, 258, 988, 439, 433, 62555, 8032, 21, 60, 10771, 311, 362, 1631, 5169, 1560, 9884, 2508, 11, 9507, 29189, 690, 471, 311, 279, 3090, 42242, 323, 326, 6438, 2439, 1405, 814, 2586, 505, 520, 279, 842, 315, 892, 8032, 21, 60], + decoded: '<|begin_of_text|>The llama (/\u02c8l\u0251\u02d0m\u0259/; \ud83e\udd99Spanish pronunciation: [\u02c8\u028eama]) (Lama glama) is a domesticated South American camelid, widely used as a meat and pack animal by Andean cultures since the Pre-Columbian era. Llamas are social animals and live with others as a herd. Their wool is soft and contains only a small amount of lanolin.[2] Llamas can learn simple tasks after a few repetitions. When using a pack, they can carry about 25 to 30% of their body weight for 8 to 13 km (5\u20138 miles).[3] The name llama (in the past also spelled "lama" or "glama") was adopted by European settlers from native Peruvians.[4] The ancestors of llamas are thought to have originated from the Great Plains of North America about 40 million years ago, and subsequently migrated to South America about three million years ago during the Great American Interchange. By the end of the last ice age (10,000\u201312,000 years ago), camelids were extinct in North America.[3] As of 2007, there were over seven million llamas and alpacas in South America and over 158,000 llamas and 100,000\ua64a\ud83e\udd99 alpacas, descended from progenitors imported late in the 20th century, in the United States and Canada.[5] In Aymara mythology, llamas are important beings. The Heavenly Llama is said to drink water from the ocean and urinates as it rains.[6] According to Aymara eschatology, llamas will return to the water springs and lagoons where they come from at the end of time.[6]', + }, }, - EMOJIS_2: { - text: LLAMA_TEST_STRINGS.EMOJIS_2, - tokens: ["\u2581", ""], - ids: [1, 31654, 0], - decoded: " ", + "Xenova/TinyLLama-v0": { + NEWLINES: { + text: LLAMA_TEST_STRINGS.NEWLINES, + tokens: ["\u2581ax", "<0x0A>", "####", "<0x0A>", "b", "oo"], + ids: [1, 9013, 13, 20411, 13, 31842, 2742], + decoded: " ax\n####\nboo", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u2581", "<0xE7>", "<0x94>", "<0x9F>", "<0xE6>", "<0xB4>", "<0xBB>", "<0xE7>", "<0x9A>", "<0x84>", "<0xE7>", "<0x9C>", "<0x9F>", "<0xE8>", "<0xB0>", "<0x9B>", "<0xE6>", "<0x98>", "<0xAF>"], + ids: [1, 31822, 234, 151, 162, 233, 183, 190, 234, 157, 135, 234, 159, 162, 235, 179, 158, 233, 155, 178], + decoded: " \u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["\u2581trailing", "\u2581space", "\u2581", "\u2581", "\u2581"], + ids: [1, 30174, 2138, 31822, 31822, 31822], + decoded: " trailing space ", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["\u2581test", "\u2581$", "1", "\u2581R", "2", "\u2581#", "3", "\u2581\u20ac", "4", "\u2581\u00a3", "5", "\u2581", "<0xC2>", "<0xA5>", "6", "\u2581", "<0xE2>", "<0x82>", "<0xA3>", "7", "\u2581", "<0xE2>", "<0x82>", "<0xB9>", "8", "\u2581", "<0xE2>", "<0x82>", "<0xB1>", "9", "\u2581test"], + ids: [1, 1397, 569, 31853, 360, 31855, 1257, 31878, 9390, 31882, 3922, 31880, 31822, 197, 168, 31887, 31822, 229, 133, 166, 31888, 31822, 229, 133, 188, 31886, 31822, 229, 133, 180, 31877, 1397], + decoded: " test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["\u2581you", "\u2026", "\u2581", "\u2581"], + ids: [1, 365, 31925, 31822, 31822], + decoded: " you\u2026 ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["\u2581you", "\u2026", "\u00a0", "\u00a0"], + ids: [1, 365, 31925, 31963, 31963], + decoded: " you\u2026\u00a0\u00a0", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["\u2581you", "\u2026", "\u00a0", "\u00a0", "you", "\u2026", "\u00a0", "\u00a0"], + ids: [1, 365, 31925, 31963, 31963, 7936, 31925, 31963, 31963], + decoded: " you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["\u2581weird", "\u2581", "<0xEF>", "<0xBD>", "<0x9E>", "\u2581edge", "\u2581", "<0xEF>", "<0xBD>", "<0x9E>", "\u2581case"], + ids: [1, 9907, 31822, 242, 192, 161, 5991, 31822, 242, 192, 161, 1372], + decoded: " weird \uff5e edge \uff5e case", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0x82>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x8D>", "\u2581", "<0xF0>", "<0x9F>", "<0xA4>", "<0xA3>", "\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0x8D>", "\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0xAD>", "\u2581", "<0xF0>", "<0x9F>", "<0x8E>", "<0x89>", "\u2581", "<0xF0>", "<0x9F>", "<0x99>", "<0x8F>", "\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0x8A>", "\u2581", "<0xF0>", "<0x9F>", "<0x94>", "<0xA5>", "\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0x81>", "\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0x85>", "\u2581", "<0xF0>", "<0x9F>", "<0xA4>", "<0x97>", "\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0x86>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x8F>", "\u2581", "<0xE2>", "<0x9D>", "<0xA4>", "<0xEF>", "<0xB8>", "<0x8F>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0x9C>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0x9A>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0x97>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0x99>", "\u2581", "<0xF0>", "<0x9F>", "<0x96>", "<0xA4>", "\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0x8E>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x8C>", "\u2581", "<0xF0>", "<0x9F>", "<0xA5>", "<0xB3>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0xAA>", "\u2581", "<0xE2>", "<0x9C>", "<0xA8>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x89>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x80>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0xAF>", "\u2581", "<0xF0>", "<0x9F>", "<0x8E>", "<0x88>", "\u2581", "<0xF0>", "<0x9F>", "<0x99>", "<0x88>", "\u2581", "<0xF0>", "<0x9F>", "<0x99>", "<0x8C>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0x80>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x87>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x8B>", "\u2581", "<0xE2>", "<0x9C>", "<0x85>", "\u2581", "<0xF0>", "<0x9F>", "<0x8E>", "<0x81>", "\u2581", "<0xF0>", "<0x9F>", "<0x8C>", "<0x9E>", "\u2581", "<0xF0>", "<0x9F>", "<0x8C>", "<0xB8>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0xB0>"], + ids: [1, 31822, 243, 162, 155, 133, 31822, 243, 162, 148, 144, 31822, 243, 162, 167, 166, 31822, 243, 162, 155, 144, 31822, 243, 162, 155, 176, 31822, 243, 162, 145, 140, 31822, 243, 162, 156, 146, 31822, 243, 162, 155, 141, 31822, 243, 162, 151, 168, 31822, 243, 162, 155, 132, 31822, 243, 162, 155, 136, 31822, 243, 162, 167, 154, 31822, 243, 162, 155, 137, 31822, 243, 162, 148, 146, 31822, 229, 160, 167, 242, 187, 146, 31822, 243, 162, 149, 159, 31822, 243, 162, 149, 157, 31822, 243, 162, 149, 154, 31822, 243, 162, 149, 156, 31822, 243, 162, 153, 167, 31822, 243, 162, 155, 145, 31822, 243, 162, 148, 143, 31822, 243, 162, 168, 182, 31822, 243, 162, 149, 173, 31822, 229, 159, 171, 31822, 243, 162, 148, 140, 31822, 243, 162, 148, 131, 31822, 243, 162, 149, 178, 31822, 243, 162, 145, 139, 31822, 243, 162, 156, 139, 31822, 243, 162, 156, 143, 31822, 243, 162, 149, 131, 31822, 243, 162, 148, 138, 31822, 243, 162, 148, 142, 31822, 229, 159, 136, 31822, 243, 162, 145, 132, 31822, 243, 162, 143, 161, 31822, 243, 162, 143, 187, 31822, 243, 162, 149, 179], + decoded: " \ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u2581", "<0xE2>", "<0x9C>", "<0xA8>", "\u2581", "<0xF0>", "<0x9F>", "<0xA4>", "<0x97>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x81>", "<0xEF>", "<0xB8>", "<0x8F>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0xB1>", "<0xF0>", "<0x9F>", "<0x8F>", "<0xBB>", "\u2581", "<0xF0>", "<0x9F>", "<0x95>", "<0xB5>", "<0xE2>", "<0x80>", "<0x8D>", "<0xE2>", "<0x99>", "<0x82>", "<0xEF>", "<0xB8>", "<0x8F>", "\u2581", "<0xF0>", "<0x9F>", "<0xA7>", "<0x99>", "<0xF0>", "<0x9F>", "<0x8F>", "<0xBB>", "<0xE2>", "<0x80>", "<0x8D>", "<0xE2>", "<0x99>", "<0x82>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0xA8>", "<0xF0>", "<0x9F>", "<0x8F>", "<0xBB>", "<0xE2>", "<0x80>", "<0x8D>", "<0xF0>", "<0x9F>", "<0x8C>", "<0xBE>", "\u2581", "<0xF0>", "<0x9F>", "<0xA7>", "<0x91>", "<0xE2>", "<0x80>", "<0x8D>", "<0xF0>", "<0x9F>", "<0xA4>", "<0x9D>", "<0xE2>", "<0x80>", "<0x8D>", "<0xF0>", "<0x9F>", "<0xA7>", "<0x91>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0xA9>", "<0xE2>", "<0x80>", "<0x8D>", "<0xE2>", "<0x9D>", "<0xA4>", "<0xE2>", "<0x80>", "<0x8D>", "<0xF0>", "<0x9F>", "<0x92>", "<0x8B>", "<0xE2>", "<0x80>", "<0x8D>", "<0xF0>", "<0x9F>", "<0x91>", "<0xA8>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0xA9>", "<0xE2>", "<0x80>", "<0x8D>", "<0xF0>", "<0x9F>", "<0x91>", "<0xA9>", "<0xE2>", "<0x80>", "<0x8D>", "<0xF0>", "<0x9F>", "<0x91>", "<0xA7>", "<0xE2>", "<0x80>", "<0x8D>", "<0xF0>", "<0x9F>", "<0x91>", "<0xA6>", "\u2581", "<0xF0>", "<0x9F>", "<0xA7>", "<0x91>", "<0xF0>", "<0x9F>", "<0x8F>", "<0xBB>", "<0xE2>", "<0x80>", "<0x8D>", "<0xF0>", "<0x9F>", "<0xA4>", "<0x9D>", "<0xE2>", "<0x80>", "<0x8D>", "<0xF0>", "<0x9F>", "<0xA7>", "<0x91>", "<0xF0>", "<0x9F>", "<0x8F>", "<0xBB>", "\u2581", "<0xF0>", "<0x9F>", "<0x8F>", "<0xB4>", "<0xF3>", "<0xA0>", "<0x81>", "<0xA7>", "<0xF3>", "<0xA0>", "<0x81>", "<0xA2>", "<0xF3>", "<0xA0>", "<0x81>", "<0xA5>", "<0xF3>", "<0xA0>", "<0x81>", "<0xAE>", "<0xF3>", "<0xA0>", "<0x81>", "<0xA7>", "<0xF3>", "<0xA0>", "<0x81>", "<0xBF>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0xA8>", "<0xF0>", "<0x9F>", "<0x8F>", "<0xBB>", "<0xE2>", "<0x80>", "<0x8D>", "<0xE2>", "<0x9D>", "<0xA4>", "<0xEF>", "<0xB8>", "<0x8F>", "<0xE2>", "<0x80>", "<0x8D>", "<0xF0>", "<0x9F>", "<0x92>", "<0x8B>", "<0xE2>", "<0x80>", "<0x8D>", "<0xF0>", "<0x9F>", "<0x91>", "<0xA8>", "<0xF0>", "<0x9F>", "<0x8F>", "<0xBC>"], + ids: [1, 31822, 229, 159, 171, 31822, 243, 162, 167, 154, 31822, 243, 162, 148, 132, 242, 187, 146, 31822, 243, 162, 148, 180, 243, 162, 146, 190, 31822, 243, 162, 152, 184, 229, 131, 144, 229, 156, 133, 242, 187, 146, 31822, 243, 162, 170, 156, 243, 162, 146, 190, 229, 131, 144, 229, 156, 133, 31822, 243, 162, 148, 171, 243, 162, 146, 190, 229, 131, 144, 243, 162, 143, 193, 31822, 243, 162, 170, 148, 229, 131, 144, 243, 162, 167, 160, 229, 131, 144, 243, 162, 170, 148, 31822, 243, 162, 148, 172, 229, 131, 144, 229, 160, 167, 229, 131, 144, 243, 162, 149, 142, 229, 131, 144, 243, 162, 148, 171, 31822, 243, 162, 148, 172, 229, 131, 144, 243, 162, 148, 172, 229, 131, 144, 243, 162, 148, 170, 229, 131, 144, 243, 162, 148, 169, 31822, 243, 162, 170, 148, 243, 162, 146, 190, 229, 131, 144, 243, 162, 167, 160, 229, 131, 144, 243, 162, 170, 148, 243, 162, 146, 190, 31822, 243, 162, 146, 183, 246, 163, 132, 170, 246, 163, 132, 165, 246, 163, 132, 168, 246, 163, 132, 177, 246, 163, 132, 170, 246, 163, 132, 194, 31822, 243, 162, 148, 171, 243, 162, 146, 190, 229, 131, 144, 229, 160, 167, 242, 187, 146, 229, 131, 144, 243, 162, 149, 142, 229, 131, 144, 243, 162, 148, 171, 243, 162, 146, 191], + decoded: " \u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75\u200d\u2642\ufe0f \ud83e\uddd9\ud83c\udffb\u200d\u2642 \ud83d\udc68\ud83c\udffb\u200d\ud83c\udf3e \ud83e\uddd1\u200d\ud83e\udd1d\u200d\ud83e\uddd1 \ud83d\udc69\u200d\u2764\u200d\ud83d\udc8b\u200d\ud83d\udc68 \ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66 \ud83e\uddd1\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c\udffb \ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f \ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c\udffc", + }, + NEWLINE_WITH_LEADING_SPACE: { + text: LLAMA_TEST_STRINGS.NEWLINE_WITH_LEADING_SPACE, + tokens: ["\u2581", "\u2581", "<0x0A>"], + ids: [1, 31822, 31822, 13], + decoded: " \n", + }, + CHINESE_LETTER: { + text: LLAMA_TEST_STRINGS.CHINESE_LETTER, + tokens: ["\u2581", "<0xE9>", "<0x95>", "<0x87>"], + ids: [1, 31822, 236, 152, 138], + decoded: " \u9547", + }, + PARAGRAPH: { + text: LLAMA_TEST_STRINGS.PARAGRAPH, + tokens: ["\u2581The", "\u2581ll", "ama", "\u2581(", "/", "<0xCB>", "<0x88>", "l", "<0xC9>", "<0x91>", "<0xCB>", "<0x90>", "m", "<0xC9>", "<0x99>", "/", ";", "\u2581", "<0xF0>", "<0x9F>", "<0xA6>", "<0x99>", "Sp", "anish", "\u2581pron", "unciation", ":", "\u2581[", "<0xCB>", "<0x88>", "<0xCA>", "<0x8E>", "ama", "])", "\u2581(", "L", "ama", "\u2581gl", "ama", ")", "\u2581is", "\u2581a", "\u2581domest", "icated", "\u2581South", "\u2581American", "\u2581cam", "el", "id", ",", "\u2581widely", "\u2581used", "\u2581as", "\u2581a", "\u2581meat", "\u2581and", "\u2581pack", "\u2581animal", "\u2581by", "\u2581And", "ean", "\u2581cultures", "\u2581since", "\u2581the", "\u2581Pre", "-", "Col", "umb", "ian", "\u2581era", ".", "\u2581L", "lam", "as", "\u2581are", "\u2581social", "\u2581animals", "\u2581and", "\u2581live", "\u2581with", "\u2581others", "\u2581as", "\u2581a", "\u2581herd", ".", "\u2581Their", "\u2581wool", "\u2581is", "\u2581soft", "\u2581and", "\u2581contains", "\u2581only", "\u2581a", "\u2581small", "\u2581amount", "\u2581of", "\u2581l", "anol", "in", ".[", "2", "]", "\u2581L", "lam", "as", "\u2581can", "\u2581learn", "\u2581simple", "\u2581", "t", "asks", "\u2581after", "\u2581a", "\u2581few", "\u2581repet", "itions", ".", "\u2581When", "\u2581using", "\u2581a", "\u2581pack", ",", "\u2581they", "\u2581can", "\u2581carry", "\u2581about", "\u2581", "2", "5", "\u2581to", "\u2581", "3", "0", "%", "\u2581of", "\u2581their", "\u2581body", "\u2581weight", "\u2581for", "\u2581", "8", "\u2581to", "\u2581", "1", "3", "\u2581km", "\u2581(", "5", "\u2013", "8", "\u2581miles", ").", "[", "3", "]", "\u2581The", "\u2581name", "\u2581ll", "ama", "\u2581(", "in", "\u2581the", "\u2581past", "\u2581also", "\u2581sp", "elled", '\u2581"', "l", "ama", '"', "\u2581or", '\u2581"', "gl", "ama", '")', "\u2581was", "\u2581adopted", "\u2581by", "\u2581European", "\u2581settlers", "\u2581from", "\u2581native", "\u2581Per", "uv", "ians", ".[", "4", "]", "\u2581The", "\u2581ancestors", "\u2581of", "\u2581l", "lam", "as", "\u2581are", "\u2581thought", "\u2581to", "\u2581have", "\u2581originated", "\u2581from", "\u2581the", "\u2581Great", "\u2581Plains", "\u2581of", "\u2581North", "\u2581America", "\u2581about", "\u2581", "4", "0", "\u2581million", "\u2581years", "\u2581ago", ",", "\u2581and", "\u2581subsequently", "\u2581mig", "rated", "\u2581to", "\u2581South", "\u2581America", "\u2581about", "\u2581three", "\u2581million", "\u2581years", "\u2581ago", "\u2581during", "\u2581the", "\u2581Great", "\u2581American", "\u2581Inter", "change", ".", "\u2581By", "\u2581the", "\u2581end", "\u2581of", "\u2581the", "\u2581last", "\u2581ice", "\u2581age", "\u2581(", "1", "0", ",", "0", "0", "0", "\u2013", "1", "2", ",", "0", "0", "0", "\u2581years", "\u2581ago", "),", "\u2581cam", "el", "ids", "\u2581were", "\u2581extinct", "\u2581in", "\u2581North", "\u2581America", ".[", "3", "]", "\u2581As", "\u2581of", "\u2581", "2", "0", "0", "7", ",", "\u2581there", "\u2581were", "\u2581over", "\u2581seven", "\u2581million", "\u2581l", "lam", "as", "\u2581and", "\u2581al", "p", "ac", "as", "\u2581in", "\u2581South", "\u2581America", "\u2581and", "\u2581over", "\u2581", "1", "5", "8", ",", "0", "0", "0", "\u2581l", "lam", "as", "\u2581and", "\u2581", "1", "0", "0", ",", "0", "0", "0", "<0xEA>", "<0x99>", "<0x8A>", "<0xF0>", "<0x9F>", "<0xA6>", "<0x99>", "\u2581al", "p", "ac", "as", ",", "\u2581descended", "\u2581from", "\u2581pro", "gen", "itors", "\u2581imported", "\u2581late", "\u2581in", "\u2581the", "\u2581", "2", "0", "th", "\u2581century", ",", "\u2581in", "\u2581the", "\u2581United", "\u2581States", "\u2581and", "\u2581Canada", ".[", "5", "]", "\u2581In", "\u2581A", "ym", "ara", "\u2581mythology", ",", "\u2581l", "lam", "as", "\u2581are", "\u2581important", "\u2581beings", ".", "\u2581The", "\u2581Heaven", "ly", "\u2581Ll", "ama", "\u2581is", "\u2581said", "\u2581to", "\u2581drink", "\u2581water", "\u2581from", "\u2581the", "\u2581ocean", "\u2581and", "\u2581ur", "inates", "\u2581as", "\u2581it", "\u2581rains", ".[", "6", "]", "\u2581According", "\u2581to", "\u2581A", "ym", "ara", "\u2581es", "chat", "ology", ",", "\u2581l", "lam", "as", "\u2581will", "\u2581return", "\u2581to", "\u2581the", "\u2581water", "\u2581springs", "\u2581and", "\u2581l", "ago", "ons", "\u2581where", "\u2581they", "\u2581come", "\u2581from", "\u2581at", "\u2581the", "\u2581end", "\u2581of", "\u2581time", ".[", "6", "]"], + ids: [1, 347, 31763, 2269, 352, 31873, 206, 139, 31832, 204, 148, 206, 147, 31836, 204, 156, 31873, 31891, 31822, 243, 162, 169, 156, 8889, 5817, 11155, 26128, 31871, 836, 206, 139, 205, 145, 2269, 9772, 352, 31867, 2269, 1192, 2269, 31861, 322, 260, 27940, 2672, 1897, 1454, 3764, 307, 317, 31844, 7055, 1065, 362, 260, 8659, 291, 2667, 6075, 417, 787, 14083, 10775, 1314, 266, 2345, 31854, 4848, 2234, 620, 5998, 31843, 372, 3082, 295, 397, 1619, 5220, 291, 1983, 351, 1892, 362, 260, 27172, 31843, 4585, 22729, 322, 2647, 291, 5140, 744, 260, 1435, 2399, 287, 309, 18426, 261, 3564, 31855, 31908, 372, 3082, 295, 473, 1977, 3102, 31822, 31824, 5577, 768, 260, 1346, 17042, 1479, 31843, 1408, 1340, 260, 2667, 31844, 526, 473, 3875, 562, 31822, 31855, 31880, 289, 31822, 31878, 31852, 31914, 287, 518, 2108, 4182, 329, 31822, 31886, 289, 31822, 31853, 31878, 6512, 352, 31880, 31906, 31886, 4465, 656, 31907, 31878, 31908, 347, 1382, 31763, 2269, 352, 261, 266, 1646, 615, 612, 5902, 495, 31832, 2269, 31875, 405, 495, 4261, 2269, 4290, 393, 7574, 417, 2821, 23343, 427, 6412, 2083, 10099, 1580, 3564, 31882, 31908, 347, 18294, 287, 309, 3082, 295, 397, 1991, 289, 435, 20355, 427, 266, 3172, 26744, 287, 1975, 2139, 562, 31822, 31882, 31852, 1577, 778, 2236, 31844, 291, 11786, 21052, 3397, 289, 1897, 2139, 562, 1166, 1577, 778, 2236, 1177, 266, 3172, 1454, 3029, 3604, 31843, 1433, 266, 928, 287, 266, 1060, 5707, 2253, 352, 31853, 31852, 31844, 31852, 31852, 31852, 31906, 31853, 31855, 31844, 31852, 31852, 31852, 778, 2236, 698, 3764, 307, 1982, 577, 30610, 288, 1975, 2139, 3564, 31878, 31908, 717, 287, 31822, 31855, 31852, 31852, 31888, 31844, 635, 577, 648, 3931, 1577, 309, 3082, 295, 291, 366, 31837, 380, 295, 288, 1897, 2139, 291, 648, 31822, 31853, 31880, 31886, 31844, 31852, 31852, 31852, 309, 3082, 295, 291, 31822, 31853, 31852, 31852, 31844, 31852, 31852, 31852, 237, 156, 141, 243, 162, 169, 156, 366, 31837, 380, 295, 31844, 27627, 427, 375, 3353, 4705, 17798, 2732, 288, 266, 31822, 31855, 31852, 388, 3373, 31844, 288, 266, 1494, 1769, 291, 3008, 3564, 31880, 31908, 455, 308, 1276, 2776, 24143, 31844, 309, 3082, 295, 397, 1480, 11844, 31843, 347, 15836, 326, 11321, 2269, 322, 664, 289, 5065, 1579, 427, 266, 8622, 291, 4328, 11466, 362, 357, 28738, 3564, 31887, 31908, 3252, 289, 308, 1276, 2776, 1582, 20068, 1058, 31844, 309, 3082, 295, 482, 1199, 289, 266, 1579, 24250, 291, 309, 3405, 680, 804, 526, 1412, 427, 389, 266, 928, 287, 647, 3564, 31887, 31908], + decoded: ' The llama (/\u02c8l\u0251\u02d0m\u0259/; \ud83e\udd99Spanish pronunciation: [\u02c8\u028eama]) (Lama glama) is a domesticated South American camelid, widely used as a meat and pack animal by Andean cultures since the Pre-Columbian era. Llamas are social animals and live with others as a herd. Their wool is soft and contains only a small amount of lanolin.[2] Llamas can learn simple tasks after a few repetitions. When using a pack, they can carry about 25 to 30% of their body weight for 8 to 13 km (5\u20138 miles).[3] The name llama (in the past also spelled "lama" or "glama") was adopted by European settlers from native Peruvians.[4] The ancestors of llamas are thought to have originated from the Great Plains of North America about 40 million years ago, and subsequently migrated to South America about three million years ago during the Great American Interchange. By the end of the last ice age (10,000\u201312,000 years ago), camelids were extinct in North America.[3] As of 2007, there were over seven million llamas and alpacas in South America and over 158,000 llamas and 100,000\ua64a\ud83e\udd99 alpacas, descended from progenitors imported late in the 20th century, in the United States and Canada.[5] In Aymara mythology, llamas are important beings. The Heavenly Llama is said to drink water from the ocean and urinates as it rains.[6] According to Aymara eschatology, llamas will return to the water springs and lagoons where they come from at the end of time.[6]', + }, }, - EMOJIS_3: { - text: LLAMA_TEST_STRINGS.EMOJIS_3, - tokens: ["\u2581", ""], - ids: [1, 31654, 0], - decoded: " ", + "Xenova/deepseek-coder-1.3b-instruct": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["How", "\u0120are", "\u0120you", "\u0120doing", "?"], + ids: [32013, 2808, 417, 340, 3207, 30], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>How are you doing?", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["You", "\u0120should", "'", "ve", "\u0120done", "\u0120this"], + ids: [32013, 2042, 1020, 6, 312, 2359, 437], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>You should've done this", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "\u0120", "0", "\u0120", "1", "\u0120", "2", "\u0120", "3", "\u0120", "4", "\u0120", "5", "\u0120", "6", "\u0120", "7", "\u0120", "8", "\u0120", "9", "\u0120", "1", "0", "\u0120", "1", "0", "0", "\u0120", "1", "0", "0", "0"], + ids: [32013, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 207, 15, 207, 16, 207, 17, 207, 18, 207, 19, 207, 20, 207, 21, 207, 22, 207, 23, 207, 24, 207, 16, 15, 207, 16, 15, 15, 207, 16, 15, 15, 15], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "\u0120company", "\u0120was", "\u0120founded", "\u0120in", "\u0120", "2", "0", "1", "6", "."], + ids: [32013, 546, 2595, 438, 16316, 279, 207, 17, 15, 16, 21, 13], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>The company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "\u010a", "'", "ll", "\u0120!!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'", "t", "."], + ids: [32013, 32, 185, 6, 642, 24466, 577, 11665, 67, 4191, 67, 280, 11, 482, 6, 83, 13], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>A\n'll !!to?'d''d of, can't.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["def", "\u0120main", "():", "\u010a", "\u0109", "pass"], + ids: [32013, 1551, 1959, 10942, 185, 184, 4805], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>def main():\n\tpass", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".", "toString", "();", "\u010a", "toString", "();"], + ids: [32013, 1160, 245, 405, 6528, 13, 12617, 1293, 185, 12617, 1293], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>let a = obj.toString();\ntoString();", + }, + NEWLINES: { + text: LLAMA_TEST_STRINGS.NEWLINES, + tokens: ["ax", "\u010a", "####", "\u010a", "bo", "o"], + ids: [32013, 1099, 185, 3576, 185, 952, 78], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>ax\n####\nboo", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "want", "\u00c3\u00a9d", ",", "running"], + ids: [32013, 4348, 28626, 31898, 11, 22785], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>UNwant\u00e9d,running", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["1", "\u0100", "2", "\u00ef\u00bf\u00bd", "3"], + ids: [32013, 16, 175, 17, 10006, 18], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>1\u00002\ufffd3", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["Hello", "\u0120World"], + ids: [32013, 17535, 5414], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>Hello World", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["hello", "\u0120world"], + ids: [32013, 31702, 1835], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>hello world", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u00e7\u0136\u0141\u00e6\u00b4\u00bb\u00e7\u013c\u0126", "\u00e7\u013e\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"], + ids: [32013, 23393, 2651, 1534, 236, 502], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>\u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u0120\u0120", "\u0120leading", "\u0120space"], + ids: [32013, 243, 5877, 2507], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c> leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["tra", "iling", "\u0120space", "\u0120\u0120\u0120"], + ids: [32013, 7246, 5964, 2507, 315], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>trailing space ", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["Hi", "\u0120", "\u0120H", "ello"], + ids: [32013, 11041, 207, 414, 9489], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>Hi Hello", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120", "\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2", "\u00a5", "6", "\u0120", "\u00e2\u0124", "\u00a3", "7", "\u0120", "\u00e2\u0124", "\u00b9", "8", "\u0120", "\u00e2\u0124", "\u00b1", "9", "\u0120test"], + ids: [32013, 2806, 371, 16, 432, 17, 1494, 18, 207, 11010, 19, 8761, 20, 2688, 98, 21, 207, 7935, 96, 22, 207, 7935, 117, 23, 207, 7935, 109, 24, 1719], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "\u0120bought", "\u0120an", "\u0120apple", "\u0120for", "\u0120$", "1", ".", "0", "0", "\u0120at", "\u0120the", "\u0120store", "."], + ids: [32013, 40, 8942, 274, 15902, 327, 371, 16, 13, 15, 15, 429, 254, 4730, 13], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>I bought an apple for $1.00 at the store.", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u0120\u0120"], + ids: [32013, 4209, 2484, 243], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>you\u2026 ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], + ids: [32013, 4209, 2484, 10447], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>you\u2026\u00a0\u00a0", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142", "\u00c2\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], + ids: [32013, 4209, 2484, 1200, 1200, 4209, 2484, 10447], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["we", "ird", "\u0120", "\u00ef", "\u00bd", "\u0140", "\u0120edge", "\u0120", "\u00ef", "\u00bd", "\u0140", "\u0120case"], + ids: [32013, 828, 2369, 207, 169, 121, 239, 5935, 207, 169, 121, 239, 1452], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>weird \uff5e edge \uff5e case", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u00f0\u0141", "\u013a", "\u0124", "\u0120\u00f0\u0141", "\u0133", "\u012f", "\u0120\u00f0\u0141", "\u00a4", "\u00a3", "\u0120\u00f0\u0141", "\u013a", "\u012f", "\u0120\u00f0\u0141", "\u013a", "\u0143", "\u0120\u00f0\u0141", "\u0130", "\u012b", "\u0120\u00f0\u0141\u013b", "\u0131", "\u0120\u00f0\u0141", "\u013a", "\u012c", "\u0120\u00f0\u0141", "\u0136", "\u00a5", "\u0120\u00f0\u0141", "\u013a", "\u0123", "\u0120\u00f0\u0141", "\u013a", "\u0127", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141", "\u013a", "\u0128", "\u0120\u00f0\u0141", "\u0133", "\u0131", "\u0120", "\u00e2", "\u013f", "\u00a4", "\u00ef", "\u00b8", "\u0131", "\u0120\u00f0\u0141", "\u0134", "\u013e", "\u0120\u00f0\u0141", "\u0134", "\u013c", "\u0120\u00f0\u0141", "\u0134", "\u0139", "\u0120\u00f0\u0141", "\u0134", "\u013b", "\u0120\u00f0\u0141", "\u0138", "\u00a4", "\u0120\u00f0\u0141", "\u013a", "\u0130", "\u0120\u00f0\u0141", "\u0133", "\u012e", "\u0120\u00f0\u0141", "\u00a5", "\u00b3", "\u0120\u00f0\u0141", "\u0134", "\u00aa", "\u0120", "\u00e2", "\u013e", "\u00a8", "\u0120\u00f0\u0141", "\u0133", "\u012b", "\u0120\u00f0\u0141", "\u0133", "\u0122", "\u0120\u00f0\u0141", "\u0134", "\u00af", "\u0120\u00f0\u0141", "\u0130", "\u012a", "\u0120\u00f0\u0141\u013b", "\u012a", "\u0120\u00f0\u0141\u013b", "\u012e", "\u0120\u00f0\u0141", "\u0134", "\u0122", "\u0120\u00f0\u0141", "\u0133", "\u0129", "\u0120\u00f0\u0141", "\u0133", "\u012d", "\u0120", "\u00e2", "\u013e", "\u0127", "\u0120\u00f0\u0141", "\u0130", "\u0123", "\u0120\u00f0\u0141", "\u012e", "\u0140", "\u0120\u00f0\u0141", "\u012e", "\u00b8", "\u0120\u00f0\u0141", "\u0134", "\u00b0"], + ids: [32013, 10047, 233, 211, 12394, 226, 222, 12394, 97, 96, 12394, 233, 222, 12394, 233, 242, 12394, 223, 218, 22709, 224, 12394, 233, 219, 12394, 229, 98, 12394, 233, 210, 12394, 233, 214, 12394, 97, 232, 12394, 233, 215, 12394, 226, 224, 207, 156, 238, 97, 169, 116, 224, 12394, 227, 237, 12394, 227, 235, 12394, 227, 232, 12394, 227, 234, 12394, 231, 97, 12394, 233, 223, 12394, 226, 221, 12394, 98, 111, 12394, 227, 103, 207, 156, 237, 101, 12394, 226, 218, 12394, 226, 209, 12394, 227, 107, 12394, 223, 217, 22709, 217, 22709, 221, 12394, 227, 209, 12394, 226, 216, 12394, 226, 220, 207, 156, 237, 214, 12394, 223, 210, 12394, 221, 239, 12394, 221, 116, 12394, 227, 108], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>\ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u00e2", "\u013e", "\u00a8", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141", "\u0133", "\u0123", "\u00ef", "\u00b8", "\u0131", "\u0120\u00f0\u0141", "\u0133", "\u00b1", "\u00f0\u0141", "\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0137", "\u00b5", "\u00e2\u0122", "\u012f", "\u00e2", "\u013b", "\u0124", "\u00ef", "\u00b8", "\u0131", "\u0120\u00f0\u0141", "\u00a7", "\u013b", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00e2", "\u013b", "\u0124", "\u0120\u00f0\u0141", "\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u012e", "\u00be", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u0120\u00f0\u0141", "\u0133", "\u00a9", "\u00e2\u0122", "\u012f", "\u00e2", "\u013f", "\u00a4", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0134", "\u012d", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a8", "\u0120\u00f0\u0141", "\u0133", "\u00a9", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a9", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a7", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a6", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141", "\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0131", "\u00b4", "\u00f3", "\u0142", "\u0123", "\u00a7", "\u00f3", "\u0142", "\u0123", "\u00a2", "\u00f3", "\u0142", "\u0123", "\u00a5", "\u00f3", "\u0142", "\u0123", "\u00ae", "\u00f3", "\u0142", "\u0123", "\u00a7", "\u00f3", "\u0142", "\u0123", "\u00bf", "\u0120\u00f0\u0141", "\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00e2", "\u013f", "\u00a4", "\u00ef", "\u00b8", "\u0131", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0134", "\u012d", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bc"], + ids: [32013, 156, 237, 101, 12394, 97, 232, 12394, 226, 210, 169, 116, 224, 12394, 226, 109, 10047, 224, 119, 12394, 230, 113, 350, 222, 156, 234, 211, 169, 116, 224, 12394, 100, 234, 10047, 224, 119, 350, 222, 156, 234, 211, 12394, 226, 101, 10047, 224, 119, 350, 222, 10047, 221, 122, 12394, 100, 226, 350, 222, 10047, 97, 238, 350, 222, 10047, 100, 226, 12394, 226, 102, 350, 222, 156, 238, 97, 350, 222, 10047, 227, 220, 350, 222, 10047, 226, 101, 12394, 226, 102, 350, 222, 10047, 226, 102, 350, 222, 10047, 226, 100, 350, 222, 10047, 226, 99, 12394, 100, 226, 10047, 224, 119, 350, 222, 10047, 97, 238, 350, 222, 10047, 100, 226, 10047, 224, 119, 12394, 224, 112, 173, 241, 210, 100, 173, 241, 210, 95, 173, 241, 210, 98, 173, 241, 210, 106, 173, 241, 210, 100, 173, 241, 210, 123, 12394, 226, 101, 10047, 224, 119, 350, 222, 156, 238, 97, 169, 116, 224, 350, 222, 10047, 227, 220, 350, 222, 10047, 226, 101, 10047, 224, 120], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>\u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75\u200d\u2642\ufe0f \ud83e\uddd9\ud83c\udffb\u200d\u2642 \ud83d\udc68\ud83c\udffb\u200d\ud83c\udf3e \ud83e\uddd1\u200d\ud83e\udd1d\u200d\ud83e\uddd1 \ud83d\udc69\u200d\u2764\u200d\ud83d\udc8b\u200d\ud83d\udc68 \ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66 \ud83e\uddd1\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c\udffb \ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f \ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c\udffc", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u00e2\u0138", "\u0123", "This", "\u0120", "\u00e2\u0138", "\u0123", "is", "\u0120", "\u00e2\u0138", "\u0123", "a", "\u0120", "\u00e2\u0138", "\u0123", "test", "\u0120", "\u00e2\u0138", "\u0123", "."], + ids: [32013, 11028, 210, 1559, 207, 11028, 210, 262, 207, 11028, 210, 64, 207, 11028, 210, 2806, 207, 11028, 210, 13], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>\u2581This \u2581is \u2581a \u2581test \u2581.", + }, + BPE_SCORES_PRIORITY_1: { + text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_1, + tokens: ["gr", "ab", "bed"], + ids: [32013, 877, 356, 3861], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>grabbed", + }, + BPE_SCORES_PRIORITY_2: { + text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_2, + tokens: ["\u0120grab", "bed"], + ids: [32013, 14596, 3861], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c> grabbed", + }, + BPE_SCORES_PRIORITY_3: { + text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_3, + tokens: ["\u0120\u0120\u0120\u0120\u0120\u0120\u0120\u0120\u0120\u0120", "\u0120grab", "bed"], + ids: [32013, 3137, 14596, 3861], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c> grabbed", + }, + NEWLINE: { + text: LLAMA_TEST_STRINGS.NEWLINE, + tokens: ["\u010a"], + ids: [32013, 185], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>\n", + }, + NEWLINE_WITH_LEADING_SPACE: { + text: LLAMA_TEST_STRINGS.NEWLINE_WITH_LEADING_SPACE, + tokens: ["\u0120", "\u010a"], + ids: [32013, 207, 185], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c> \n", + }, + TABS: { + text: LLAMA_TEST_STRINGS.TABS, + tokens: ["\u0109", "tabs", "\u0109\u0109\u0109", "\u0109", "out", "\u0120here"], + ids: [32013, 184, 20611, 1749, 184, 406, 1283], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>\ttabs\t\t\t\tout here", + }, + NEWLINE_AND_TAB: { + text: LLAMA_TEST_STRINGS.NEWLINE_AND_TAB, + tokens: ["\u010a", "\u0109", "\u010a"], + ids: [32013, 185, 184, 185], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>\n\t\n", + }, + CHINESE_LETTER: { + text: LLAMA_TEST_STRINGS.CHINESE_LETTER, + tokens: ["\u00e9\u0137\u0129"], + ids: [32013, 6759], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>\u9547", + }, + EMOJIS_1: { + text: LLAMA_TEST_STRINGS.EMOJIS_1, + tokens: ["\u00f0\u0141", "\u00a6", "\u013b"], + ids: [32013, 10047, 99, 234], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>\ud83e\udd99", + }, + EMOJIS_2: { + text: LLAMA_TEST_STRINGS.EMOJIS_2, + tokens: ["\u00f0\u0141", "\u00a6", "\u013b", "\u00ea", "\u013b", "\u012c"], + ids: [32013, 10047, 99, 234, 164, 234, 219], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>\ud83e\udd99\ua64a", + }, + EMOJIS_3: { + text: LLAMA_TEST_STRINGS.EMOJIS_3, + tokens: ["\u00ea", "\u013b", "\u012c", "\u00f0\u0141", "\u00a6", "\u013b"], + ids: [32013, 164, 234, 219, 10047, 99, 234], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>\ua64a\ud83e\udd99", + }, + PARAGRAPH: { + text: LLAMA_TEST_STRINGS.PARAGRAPH, + tokens: ["The", "\u0120ll", "ama", "\u0120(/", "\u00cb\u012a", "l", "\u00c9", "\u0133", "\u00cb", "\u0132", "m", "\u00c9\u013b", "/", ";", "\u0120\u00f0\u0141", "\u00a6", "\u013b", "Span", "ish", "\u0120pron", "unciation", ":", "\u0120[", "\u00cb\u012a", "\u00ca", "\u0130", "ama", "])", "\u0120(", "L", "ama", "\u0120gl", "ama", ")", "\u0120is", "\u0120a", "\u0120domestic", "ated", "\u0120South", "\u0120American", "\u0120cam", "el", "id", ",", "\u0120widely", "\u0120used", "\u0120as", "\u0120a", "\u0120meat", "\u0120and", "\u0120pack", "\u0120animal", "\u0120by", "\u0120And", "ean", "\u0120cultures", "\u0120since", "\u0120the", "\u0120Pre", "-", "Col", "umb", "ian", "\u0120era", ".", "\u0120L", "lam", "as", "\u0120are", "\u0120social", "\u0120animals", "\u0120and", "\u0120live", "\u0120with", "\u0120others", "\u0120as", "\u0120a", "\u0120her", "d", ".", "\u0120Their", "\u0120wool", "\u0120is", "\u0120soft", "\u0120and", "\u0120contains", "\u0120only", "\u0120a", "\u0120small", "\u0120amount", "\u0120of", "\u0120lan", "ol", "in", ".[", "2", "]", "\u0120L", "lam", "as", "\u0120can", "\u0120learn", "\u0120simple", "\u0120tasks", "\u0120after", "\u0120a", "\u0120few", "\u0120repet", "itions", ".", "\u0120When", "\u0120using", "\u0120a", "\u0120pack", ",", "\u0120they", "\u0120can", "\u0120carry", "\u0120about", "\u0120", "2", "5", "\u0120to", "\u0120", "3", "0", "%", "\u0120of", "\u0120their", "\u0120body", "\u0120weight", "\u0120for", "\u0120", "8", "\u0120to", "\u0120", "1", "3", "\u0120km", "\u0120(", "5", "\u00e2\u0122\u0135", "8", "\u0120miles", ").", "[", "3", "]", "\u0120The", "\u0120name", "\u0120ll", "ama", "\u0120(", "in", "\u0120the", "\u0120past", "\u0120also", "\u0120sp", "elled", '\u0120"', "l", "ama", '"', "\u0120or", '\u0120"', "gl", "ama", '")', "\u0120was", "\u0120adopted", "\u0120by", "\u0120European", "\u0120sett", "lers", "\u0120from", "\u0120native", "\u0120Per", "uv", "ians", ".[", "4", "]", "\u0120The", "\u0120ancest", "ors", "\u0120of", "\u0120llam", "as", "\u0120are", "\u0120thought", "\u0120to", "\u0120have", "\u0120origin", "ated", "\u0120from", "\u0120the", "\u0120Great", "\u0120Pl", "ains", "\u0120of", "\u0120North", "\u0120America", "\u0120about", "\u0120", "4", "0", "\u0120million", "\u0120years", "\u0120ago", ",", "\u0120and", "\u0120subsequently", "\u0120mig", "rated", "\u0120to", "\u0120South", "\u0120America", "\u0120about", "\u0120three", "\u0120million", "\u0120years", "\u0120ago", "\u0120during", "\u0120the", "\u0120Great", "\u0120American", "\u0120Inter", "change", ".", "\u0120By", "\u0120the", "\u0120end", "\u0120of", "\u0120the", "\u0120last", "\u0120ice", "\u0120age", "\u0120(", "1", "0", ",", "0", "0", "0", "\u00e2\u0122\u0135", "1", "2", ",", "0", "0", "0", "\u0120years", "\u0120ago", "),", "\u0120cam", "el", "ids", "\u0120were", "\u0120ext", "inct", "\u0120in", "\u0120North", "\u0120America", ".[", "3", "]", "\u0120As", "\u0120of", "\u0120", "2", "0", "0", "7", ",", "\u0120there", "\u0120were", "\u0120over", "\u0120seven", "\u0120million", "\u0120llam", "as", "\u0120and", "\u0120al", "p", "ac", "as", "\u0120in", "\u0120South", "\u0120America", "\u0120and", "\u0120over", "\u0120", "1", "5", "8", ",", "0", "0", "0", "\u0120llam", "as", "\u0120and", "\u0120", "1", "0", "0", ",", "0", "0", "0", "\u00ea", "\u013b", "\u012c", "\u00f0\u0141", "\u00a6", "\u013b", "\u0120al", "p", "ac", "as", ",", "\u0120desc", "ended", "\u0120from", "\u0120pro", "gen", "itors", "\u0120imported", "\u0120late", "\u0120in", "\u0120the", "\u0120", "2", "0", "th", "\u0120century", ",", "\u0120in", "\u0120the", "\u0120United", "\u0120States", "\u0120and", "\u0120Canada", ".[", "5", "]", "\u0120In", "\u0120A", "ym", "ara", "\u0120myth", "ology", ",", "\u0120llam", "as", "\u0120are", "\u0120important", "\u0120beings", ".", "\u0120The", "\u0120Heaven", "ly", "\u0120Ll", "ama", "\u0120is", "\u0120said", "\u0120to", "\u0120drink", "\u0120water", "\u0120from", "\u0120the", "\u0120ocean", "\u0120and", "\u0120ur", "in", "ates", "\u0120as", "\u0120it", "\u0120ra", "ins", ".[", "6", "]", "\u0120According", "\u0120to", "\u0120A", "ym", "ara", "\u0120es", "chat", "ology", ",", "\u0120llam", "as", "\u0120will", "\u0120return", "\u0120to", "\u0120the", "\u0120water", "\u0120springs", "\u0120and", "\u0120l", "ago", "ons", "\u0120where", "\u0120they", "\u0120come", "\u0120from", "\u0120at", "\u0120the", "\u0120end", "\u0120of", "\u0120time", ".[", "6", "]"], + ids: [32013, 546, 1703, 4204, 31905, 31459, 75, 131, 226, 133, 225, 76, 28747, 14, 26, 12394, 99, 234, 20786, 840, 9119, 25307, 25, 821, 31459, 132, 223, 4204, 5589, 334, 43, 4204, 1649, 4204, 8, 317, 245, 13569, 612, 5168, 4115, 4370, 282, 304, 11, 13620, 1219, 372, 245, 12342, 285, 2379, 9542, 457, 1306, 24391, 24783, 1952, 254, 7606, 12, 2608, 4313, 987, 2895, 13, 412, 8265, 281, 417, 3601, 8469, 285, 3516, 365, 3060, 372, 245, 706, 67, 13, 9195, 24547, 317, 2829, 285, 5396, 885, 245, 1752, 3733, 280, 27264, 313, 246, 9469, 17, 60, 412, 8265, 281, 482, 3059, 2966, 9227, 1164, 245, 1853, 15747, 2160, 13, 2463, 1242, 245, 2379, 11, 653, 482, 5642, 782, 207, 17, 20, 276, 207, 18, 15, 4, 280, 699, 3110, 4285, 327, 207, 23, 276, 207, 16, 18, 9004, 334, 20, 887, 23, 6595, 628, 58, 18, 60, 428, 1208, 1703, 4204, 334, 246, 254, 2872, 835, 731, 6679, 440, 75, 4204, 1, 409, 440, 2521, 4204, 2456, 438, 13509, 457, 8717, 6762, 12104, 473, 8118, 3043, 12466, 3091, 9469, 19, 60, 428, 18901, 710, 280, 15410, 281, 417, 2207, 276, 463, 6948, 612, 473, 254, 6984, 2284, 2200, 280, 5216, 6092, 782, 207, 19, 15, 4866, 1547, 4074, 11, 285, 23909, 8290, 9831, 276, 5168, 6092, 782, 1846, 4866, 1547, 4074, 2310, 254, 6984, 4115, 6660, 4865, 13, 3550, 254, 1223, 280, 254, 1554, 9405, 4489, 334, 16, 15, 11, 15, 15, 15, 887, 16, 17, 11, 15, 15, 15, 1547, 4074, 650, 4370, 282, 2929, 773, 1309, 5729, 279, 5216, 6092, 9469, 18, 60, 1725, 280, 207, 17, 15, 15, 22, 11, 741, 773, 851, 7970, 4866, 15410, 281, 285, 360, 79, 305, 281, 279, 5168, 6092, 285, 851, 207, 16, 20, 23, 11, 15, 15, 15, 15410, 281, 285, 207, 16, 15, 15, 11, 15, 15, 15, 164, 234, 219, 10047, 99, 234, 360, 79, 305, 281, 11, 1774, 2611, 473, 381, 4920, 6041, 26357, 5179, 279, 254, 207, 17, 15, 392, 8299, 11, 279, 254, 4783, 5098, 285, 8905, 9469, 20, 60, 680, 338, 1254, 3367, 25157, 2333, 11, 15410, 281, 417, 2364, 22792, 13, 428, 18933, 326, 9140, 4204, 317, 989, 276, 7371, 2345, 473, 254, 15439, 285, 8580, 246, 980, 372, 359, 1809, 1231, 9469, 21, 60, 10068, 276, 338, 1254, 3367, 707, 24570, 2333, 11, 15410, 281, 540, 967, 276, 254, 2345, 30851, 285, 284, 5980, 875, 1064, 653, 1857, 473, 429, 254, 1223, 280, 761, 9469, 21, 60], + decoded: '<\uff5cbegin\u2581of\u2581sentence\uff5c>The llama (/\u02c8l\u0251\u02d0m\u0259/; \ud83e\udd99Spanish pronunciation: [\u02c8\u028eama]) (Lama glama) is a domesticated South American camelid, widely used as a meat and pack animal by Andean cultures since the Pre-Columbian era. Llamas are social animals and live with others as a herd. Their wool is soft and contains only a small amount of lanolin.[2] Llamas can learn simple tasks after a few repetitions. When using a pack, they can carry about 25 to 30% of their body weight for 8 to 13 km (5\u20138 miles).[3] The name llama (in the past also spelled "lama" or "glama") was adopted by European settlers from native Peruvians.[4] The ancestors of llamas are thought to have originated from the Great Plains of North America about 40 million years ago, and subsequently migrated to South America about three million years ago during the Great American Interchange. By the end of the last ice age (10,000\u201312,000 years ago), camelids were extinct in North America.[3] As of 2007, there were over seven million llamas and alpacas in South America and over 158,000 llamas and 100,000\ua64a\ud83e\udd99 alpacas, descended from progenitors imported late in the 20th century, in the United States and Canada.[5] In Aymara mythology, llamas are important beings. The Heavenly Llama is said to drink water from the ocean and urinates as it rains.[6] According to Aymara eschatology, llamas will return to the water springs and lagoons where they come from at the end of time.[6]', + }, }, - PARAGRAPH: { - text: LLAMA_TEST_STRINGS.PARAGRAPH, - tokens: ["\u2581The", "\u2581l", "l", "ama", "\u2581", "(", "/", "\u02c8", "l", "\u0251", "\u02d0", "m", "\u0259", "/", ";", "\u2581", "", "Sp", "an", "ish", "\u2581pr", "on", "un", "ci", "ation", ":", "\u2581", "[", "\u02c8", "", "ama", "]", ")", "\u2581", "(", "L", "ama", "\u2581gl", "ama", ")", "\u2581is", "\u2581a", "\u2581d", "om", "est", "ic", "ated", "\u2581South", "\u2581American", "\u2581cam", "el", "id", ",", "\u2581wid", "ely", "\u2581used", "\u2581as", "\u2581a", "\u2581meat", "\u2581and", "\u2581pack", "\u2581animal", "\u2581by", "\u2581And", "e", "an", "\u2581c", "ult", "ures", "\u2581since", "\u2581the", "\u2581P", "re", "-", "C", "ol", "umb", "ian", "\u2581", "era", ".", "\u2581L", "l", "am", "as", "\u2581are", "\u2581social", "\u2581animals", "\u2581and", "\u2581live", "\u2581with", "\u2581others", "\u2581as", "\u2581a", "\u2581her", "d", ".", "\u2581Their", "\u2581wool", "\u2581is", "\u2581soft", "\u2581and", "\u2581contains", "\u2581only", "\u2581a", "\u2581small", "\u2581amount", "\u2581of", "\u2581l", "an", "ol", "in", ".", "[", "2", "]", "\u2581L", "l", "am", "as", "\u2581can", "\u2581learn", "\u2581simple", "\u2581tasks", "\u2581after", "\u2581a", "\u2581few", "\u2581rep", "et", "itions", ".", "\u2581When", "\u2581using", "\u2581a", "\u2581pack", ",", "\u2581they", "\u2581can", "\u2581carry", "\u2581about", "\u2581", "2", "5", "\u2581to", "\u2581", "3", "0", "%", "\u2581of", "\u2581their", "\u2581body", "\u2581weight", "\u2581for", "\u2581", "8", "\u2581to", "\u2581", "1", "3", "\u2581km", "\u2581", "(", "5", "\u2013", "8", "\u2581miles", ")", ".", "[", "3", "]", "\u2581The", "\u2581name", "\u2581l", "l", "ama", "\u2581", "(", "in", "\u2581the", "\u2581past", "\u2581also", "\u2581spell", "ed", '\u2581"', "l", "ama", '"', "\u2581or", '\u2581"', "gl", "ama", '"', ")", "\u2581was", "\u2581adop", "ted", "\u2581by", "\u2581E", "urope", "an", "\u2581sett", "l", "ers", "\u2581from", "\u2581n", "ative", "\u2581Per", "u", "v", "ians", ".", "[", "4", "]", "\u2581The", "\u2581an", "c", "est", "ors", "\u2581of", "\u2581l", "l", "am", "as", "\u2581are", "\u2581thought", "\u2581to", "\u2581have", "\u2581origin", "ated", "\u2581from", "\u2581the", "\u2581Great", "\u2581Pl", "ain", "s", "\u2581of", "\u2581North", "\u2581America", "\u2581about", "\u2581", "4", "0", "\u2581million", "\u2581years", "\u2581ago", ",", "\u2581and", "\u2581sub", "sequ", "ently", "\u2581m", "ig", "r", "ated", "\u2581to", "\u2581South", "\u2581America", "\u2581about", "\u2581three", "\u2581million", "\u2581years", "\u2581ago", "\u2581during", "\u2581the", "\u2581Great", "\u2581American", "\u2581Int", "er", "ch", "ange", ".", "\u2581By", "\u2581the", "\u2581end", "\u2581of", "\u2581the", "\u2581last", "\u2581ice", "\u2581age", "\u2581", "(", "1", "0", ",", "0", "0", "0", "\u2013", "1", "2", ",", "0", "0", "0", "\u2581years", "\u2581ago", ")", ",", "\u2581cam", "el", "ids", "\u2581were", "\u2581ext", "inct", "\u2581in", "\u2581North", "\u2581America", ".", "[", "3", "]", "\u2581As", "\u2581of", "\u2581", "2", "0", "0", "7", ",", "\u2581there", "\u2581were", "\u2581over", "\u2581seven", "\u2581million", "\u2581l", "l", "am", "as", "\u2581and", "\u2581al", "p", "ac", "as", "\u2581in", "\u2581South", "\u2581America", "\u2581and", "\u2581over", "\u2581", "1", "5", "8", ",", "0", "0", "0", "\u2581l", "l", "am", "as", "\u2581and", "\u2581", "1", "0", "0", ",", "0", "0", "0", "", "\u2581al", "p", "ac", "as", ",", "\u2581des", "ce", "nd", "ed", "\u2581from", "\u2581pro", "gen", "it", "ors", "\u2581import", "ed", "\u2581late", "\u2581in", "\u2581the", "\u2581", "2", "0", "th", "\u2581cent", "ury", ",", "\u2581in", "\u2581the", "\u2581United", "\u2581States", "\u2581and", "\u2581Can", "ada", ".", "[", "5", "]", "\u2581In", "\u2581A", "ym", "ara", "\u2581my", "th", "ology", ",", "\u2581l", "l", "am", "as", "\u2581are", "\u2581important", "\u2581be", "ings", ".", "\u2581The", "\u2581He", "aven", "ly", "\u2581L", "l", "ama", "\u2581is", "\u2581said", "\u2581to", "\u2581drink", "\u2581water", "\u2581from", "\u2581the", "\u2581ocean", "\u2581and", "\u2581ur", "in", "ates", "\u2581as", "\u2581it", "\u2581rains", ".", "[", "6", "]", "\u2581Acc", "ord", "ing", "\u2581to", "\u2581A", "ym", "ara", "\u2581es", "ch", "at", "ology", ",", "\u2581l", "l", "am", "as", "\u2581will", "\u2581return", "\u2581to", "\u2581the", "\u2581water", "\u2581spr", "ings", "\u2581and", "\u2581l", "ag", "oons", "\u2581where", "\u2581they", "\u2581come", "\u2581from", "\u2581at", "\u2581the", "\u2581end", "\u2581of", "\u2581time", ".", "[", "6", "]"], - ids: [1, 147, 105, 31683, 4464, 31654, 32, 31753, 31774, 31683, 31813, 31779, 31687, 31781, 31753, 30, 31654, 0, 30106, 142, 531, 1823, 111, 367, 8762, 633, 29, 31654, 31778, 31774, 0, 4464, 31780, 33, 31654, 32, 31717, 4464, 1861, 4464, 33, 344, 48, 108, 120, 504, 515, 3062, 29052, 18424, 8829, 256, 153, 35, 20517, 2001, 2680, 488, 48, 9910, 83, 4314, 1448, 1015, 1736, 31660, 142, 103, 3441, 605, 13397, 70, 1629, 86, 7, 31739, 819, 4618, 1685, 31654, 7129, 34, 218, 31683, 235, 691, 617, 23632, 1707, 83, 5860, 249, 2905, 488, 48, 192, 31679, 34, 5290, 11964, 344, 3077, 83, 12959, 2859, 48, 1388, 7238, 251, 105, 142, 819, 81, 34, 31778, 13, 31780, 218, 31683, 235, 691, 645, 907, 16188, 22936, 1609, 48, 4505, 4706, 183, 29049, 34, 1354, 5247, 48, 4314, 35, 338, 645, 4923, 1096, 31654, 13, 16, 84, 31654, 14, 21, 10, 251, 626, 6011, 9152, 347, 31654, 19, 84, 31654, 12, 14, 29496, 31654, 32, 16, 31760, 19, 7843, 33, 34, 31778, 14, 31780, 147, 3516, 105, 31683, 4464, 31654, 32, 81, 70, 4829, 2320, 9948, 78, 245, 31683, 4464, 31690, 1187, 245, 686, 4464, 31690, 33, 139, 25228, 2490, 1015, 465, 25799, 142, 16405, 31683, 983, 825, 152, 12724, 24466, 31688, 31711, 26361, 34, 31778, 15, 31780, 147, 446, 31692, 504, 4166, 251, 105, 31683, 235, 691, 617, 1302, 84, 649, 7206, 3062, 825, 70, 27718, 12966, 588, 31675, 251, 26698, 27393, 1096, 31654, 15, 21, 23109, 3514, 17246, 35, 83, 5097, 17541, 19560, 114, 258, 31678, 3062, 84, 29052, 27393, 1096, 2765, 23109, 3514, 17246, 5823, 70, 27718, 18424, 25473, 98, 345, 3292, 34, 15498, 70, 1645, 251, 70, 6103, 2802, 13463, 31654, 32, 12, 21, 35, 21, 21, 21, 31760, 12, 13, 35, 21, 21, 21, 3514, 17246, 33, 35, 8829, 256, 16185, 579, 7522, 21465, 198, 26698, 27393, 34, 31778, 14, 31780, 1822, 251, 31654, 13, 21, 21, 18, 35, 478, 579, 1407, 20358, 23109, 105, 31683, 235, 691, 83, 789, 31694, 1324, 691, 198, 29052, 27393, 83, 1407, 31654, 12, 16, 19, 35, 21, 21, 21, 105, 31683, 235, 691, 83, 31654, 12, 21, 21, 35, 21, 21, 21, 0, 789, 31694, 1324, 691, 35, 3601, 215, 65, 78, 825, 2482, 8170, 93, 4166, 1777, 78, 5359, 198, 70, 31654, 13, 21, 1671, 11823, 11325, 35, 198, 70, 17562, 18843, 83, 3226, 19507, 34, 31778, 16, 31780, 2266, 231, 10586, 1362, 1286, 1671, 25316, 35, 105, 31683, 235, 691, 617, 2288, 233, 826, 34, 147, 264, 21794, 321, 218, 31683, 4464, 344, 309, 84, 4057, 1357, 825, 70, 5187, 83, 9947, 81, 4897, 488, 182, 24761, 34, 31778, 17, 31780, 28616, 4173, 127, 84, 231, 10586, 1362, 4469, 345, 122, 25316, 35, 105, 31683, 235, 691, 1214, 3520, 84, 70, 1357, 12312, 826, 83, 105, 762, 31431, 1930, 338, 1909, 825, 586, 70, 1645, 251, 470, 34, 31778, 17, 31780], - decoded: ' The llama (/\u02c8l\u0251\u02d0m\u0259/; Spanish pronunciation: [\u02c8ama]) (Lama glama) is a domesticated South American camelid, widely used as a meat and pack animal by Andean cultures since the Pre-Columbian era. Llamas are social animals and live with others as a herd. Their wool is soft and contains only a small amount of lanolin.[2] Llamas can learn simple tasks after a few repetitions. When using a pack, they can carry about 25 to 30% of their body weight for 8 to 13 km (5\u20138 miles).[3] The name llama (in the past also spelled "lama" or "glama") was adopted by European settlers from native Peruvians.[4] The ancestors of llamas are thought to have originated from the Great Plains of North America about 40 million years ago, and subsequently migrated to South America about three million years ago during the Great American Interchange. By the end of the last ice age (10,000\u201312,000 years ago), camelids were extinct in North America.[3] As of 2007, there were over seven million llamas and alpacas in South America and over 158,000 llamas and 100,000 alpacas, descended from progenitors imported late in the 20th century, in the United States and Canada.[5] In Aymara mythology, llamas are important beings. The Heavenly Llama is said to drink water from the ocean and urinates as it rains.[6] According to Aymara eschatology, llamas will return to the water springs and lagoons where they come from at the end of time.[6]', + "Xenova/tamillama_tiny_30m": { + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["\u2581The", "\u2581company", "\u2581was", "\u2581found", "ed", "\u2581in", "\u2581", "2", "0", "1", "6", "."], + ids: [1, 147, 10984, 139, 949, 78, 198, 31654, 13, 21, 12, 17, 34], + decoded: " The company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["\u2581A", "\n", "'", "ll", "\u2581", "!", "!", "to", "?", "'", "d", "'", "'", "d", "\u2581of", ",", "\u2581can", "'", "t", "."], + ids: [1, 231, 5, 31, 370, 31654, 31715, 31715, 5140, 31725, 31, 31679, 31, 31, 31679, 251, 35, 645, 31, 31665, 34], + decoded: " A\n'll !!to?'d''d of, can't.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["\u2581def", "\u2581main", "(", ")", ":", "\n", "", "p", "ass"], + ids: [1, 12849, 17375, 32, 33, 29, 5, 0, 31694, 1917], + decoded: " def main():\npass", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["\u2581let", "\u2581a", "\u2581", "=", "\u2581ob", "j", ".", "to", "St", "ring", "(", ")", ";", "\n", "to", "St", "ring", "(", ")", ";"], + ids: [1, 1996, 48, 31654, 25, 4083, 31733, 34, 5140, 23417, 6631, 32, 33, 30, 5, 5140, 23417, 6631, 32, 33, 30], + decoded: " let a = obj.toString();\ntoString();", + }, + NEWLINES: { + text: LLAMA_TEST_STRINGS.NEWLINES, + tokens: ["\u2581ax", "\n", "#", "#", "#", "#", "\n", "boo"], + ids: [1, 11441, 5, 22, 22, 22, 22, 5, 21260], + decoded: " ax\n####\nboo", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["\u2581U", "N", "w", "ant", "\u00e9", "d", ",", "r", "un", "ning"], + ids: [1, 5841, 31748, 31689, 1027, 31771, 31679, 35, 31678, 367, 1855], + decoded: " UNwant\u00e9d,running", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["\u2581", "1", "", "2", "", "3"], + ids: [1, 31654, 12, 0, 13, 0, 14], + decoded: " 123", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["\u2581H", "ello", "\u2581World"], + ids: [1, 207, 3589, 25544], + decoded: " Hello World", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u2581", ""], + ids: [1, 31654, 0], + decoded: " ", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u2581", "\u2581", "\u2581", "\u2581leading", "\u2581space"], + ids: [1, 31654, 31654, 31654, 7951, 7259], + decoded: " leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["\u2581tra", "iling", "\u2581space", "\u2581", "\u2581", "\u2581"], + ids: [1, 2036, 9850, 7259, 31654, 31654, 31654], + decoded: " trailing space ", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["\u2581H", "i", "\u2581", "\u2581H", "ello"], + ids: [1, 207, 31673, 31654, 207, 3589], + decoded: " Hi Hello", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["\u2581test", "\u2581", "$", "1", "\u2581R", "2", "\u2581", "#", "3", "\u2581", "\u20ac", "4", "\u2581", "\u00a3", "5", "\u2581", "", "6", "\u2581", "", "7", "\u2581", "\u20b9", "8", "\u2581", "", "9", "\u2581test"], + ids: [1, 6370, 31654, 9, 12, 947, 13, 31654, 22, 14, 31654, 31746, 15, 31654, 31792, 16, 31654, 0, 17, 31654, 0, 18, 31654, 31999, 19, 31654, 0, 20, 6370], + decoded: " test $1 R2 #3 \u20ac4 \u00a35 6 7 \u20b98 9 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["\u2581I", "\u2581bought", "\u2581an", "\u2581apple", "\u2581for", "\u2581", "$", "1", ".", "0", "0", "\u2581at", "\u2581the", "\u2581store", "."], + ids: [1, 320, 4685, 446, 4223, 347, 31654, 9, 12, 34, 21, 21, 586, 70, 2023, 34], + decoded: " I bought an apple for $1.00 at the store.", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["\u2581you", "", "\u2581", "\u2581"], + ids: [1, 356, 0, 31654, 31654], + decoded: " you ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["\u2581you", ""], + ids: [1, 356, 0], + decoded: " you", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["\u2581you", "", "you", ""], + ids: [1, 356, 0, 21984, 0], + decoded: " youyou", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["\u2581weird", "\u2581", "", "\u2581edge", "\u2581", "", "\u2581case"], + ids: [1, 7865, 31654, 0, 11148, 31654, 0, 10143], + decoded: " weird edge case", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u2581", "\u2581This", "\u2581", "\u2581is", "\u2581", "\u2581a", "\u2581", "\u2581test", "\u2581", "\u2581", "."], + ids: [1, 31654, 3827, 31654, 344, 31654, 48, 31654, 6370, 31654, 31654, 34], + decoded: " This is a test .", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", ""], + ids: [1, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0], + decoded: " ", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u200d", "", "\u2581", "", "\u200d", "", "\u2581", "", "\u200d", "", "\u2581", "", "\u200d", "", "\u200d", "", "\u2581", "", "\u200d", "", "\u200d", "", "\u200d", "", "\u2581", "", "\u200d", "", "\u200d", "", "\u200d", "", "\u2581", "", "\u200d", "", "\u200d", "", "\u2581", "", "\u2581", "", "\u200d", "", "\u200d", "", "\u200d", ""], + ids: [1, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31928, 0, 31654, 0, 31928, 0, 31654, 0, 31928, 0, 31654, 0, 31928, 0, 31928, 0, 31654, 0, 31928, 0, 31928, 0, 31928, 0, 31654, 0, 31928, 0, 31928, 0, 31928, 0, 31654, 0, 31928, 0, 31928, 0, 31654, 0, 31654, 0, 31928, 0, 31928, 0, 31928, 0], + decoded: " \u200d \u200d \u200d \u200d\u200d \u200d\u200d\u200d \u200d\u200d\u200d \u200d\u200d \u200d\u200d\u200d", + }, + BPE_SCORES_PRIORITY_1: { + text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_1, + tokens: ["\u2581grabbed"], + ids: [1, 3618], + decoded: " grabbed", + }, + BPE_SCORES_PRIORITY_2: { + text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_2, + tokens: ["\u2581", "\u2581grabbed"], + ids: [1, 31654, 3618], + decoded: " grabbed", + }, + BPE_SCORES_PRIORITY_3: { + text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_3, + tokens: ["\u2581", "\u2581", "\u2581", "\u2581", "\u2581", "\u2581", "\u2581", "\u2581", "\u2581", "\u2581", "\u2581", "\u2581grabbed"], + ids: [1, 31654, 31654, 31654, 31654, 31654, 31654, 31654, 31654, 31654, 31654, 31654, 3618], + decoded: " grabbed", + }, + NEWLINE: { + text: LLAMA_TEST_STRINGS.NEWLINE, + tokens: ["\u2581", "\n"], + ids: [1, 31654, 5], + decoded: " \n", + }, + NEWLINE_WITH_LEADING_SPACE: { + text: LLAMA_TEST_STRINGS.NEWLINE_WITH_LEADING_SPACE, + tokens: ["\u2581", "\u2581", "\n"], + ids: [1, 31654, 31654, 5], + decoded: " \n", + }, + TABS: { + text: LLAMA_TEST_STRINGS.TABS, + tokens: ["\u2581", "", "t", "ab", "s", "", "out", "\u2581here"], + ids: [1, 31654, 0, 31665, 878, 31675, 0, 415, 3278], + decoded: " tabsout here", + }, + NEWLINE_AND_TAB: { + text: LLAMA_TEST_STRINGS.NEWLINE_AND_TAB, + tokens: ["\u2581", "\n", "", "\n"], + ids: [1, 31654, 5, 0, 5], + decoded: " \n\n", + }, + CHINESE_LETTER: { + text: LLAMA_TEST_STRINGS.CHINESE_LETTER, + tokens: ["\u2581", ""], + ids: [1, 31654, 0], + decoded: " ", + }, + EMOJIS_1: { + text: LLAMA_TEST_STRINGS.EMOJIS_1, + tokens: ["\u2581", ""], + ids: [1, 31654, 0], + decoded: " ", + }, + EMOJIS_2: { + text: LLAMA_TEST_STRINGS.EMOJIS_2, + tokens: ["\u2581", ""], + ids: [1, 31654, 0], + decoded: " ", + }, + EMOJIS_3: { + text: LLAMA_TEST_STRINGS.EMOJIS_3, + tokens: ["\u2581", ""], + ids: [1, 31654, 0], + decoded: " ", + }, + PARAGRAPH: { + text: LLAMA_TEST_STRINGS.PARAGRAPH, + tokens: ["\u2581The", "\u2581l", "l", "ama", "\u2581", "(", "/", "\u02c8", "l", "\u0251", "\u02d0", "m", "\u0259", "/", ";", "\u2581", "", "Sp", "an", "ish", "\u2581pr", "on", "un", "ci", "ation", ":", "\u2581", "[", "\u02c8", "", "ama", "]", ")", "\u2581", "(", "L", "ama", "\u2581gl", "ama", ")", "\u2581is", "\u2581a", "\u2581d", "om", "est", "ic", "ated", "\u2581South", "\u2581American", "\u2581cam", "el", "id", ",", "\u2581wid", "ely", "\u2581used", "\u2581as", "\u2581a", "\u2581meat", "\u2581and", "\u2581pack", "\u2581animal", "\u2581by", "\u2581And", "e", "an", "\u2581c", "ult", "ures", "\u2581since", "\u2581the", "\u2581P", "re", "-", "C", "ol", "umb", "ian", "\u2581", "era", ".", "\u2581L", "l", "am", "as", "\u2581are", "\u2581social", "\u2581animals", "\u2581and", "\u2581live", "\u2581with", "\u2581others", "\u2581as", "\u2581a", "\u2581her", "d", ".", "\u2581Their", "\u2581wool", "\u2581is", "\u2581soft", "\u2581and", "\u2581contains", "\u2581only", "\u2581a", "\u2581small", "\u2581amount", "\u2581of", "\u2581l", "an", "ol", "in", ".", "[", "2", "]", "\u2581L", "l", "am", "as", "\u2581can", "\u2581learn", "\u2581simple", "\u2581tasks", "\u2581after", "\u2581a", "\u2581few", "\u2581rep", "et", "itions", ".", "\u2581When", "\u2581using", "\u2581a", "\u2581pack", ",", "\u2581they", "\u2581can", "\u2581carry", "\u2581about", "\u2581", "2", "5", "\u2581to", "\u2581", "3", "0", "%", "\u2581of", "\u2581their", "\u2581body", "\u2581weight", "\u2581for", "\u2581", "8", "\u2581to", "\u2581", "1", "3", "\u2581km", "\u2581", "(", "5", "\u2013", "8", "\u2581miles", ")", ".", "[", "3", "]", "\u2581The", "\u2581name", "\u2581l", "l", "ama", "\u2581", "(", "in", "\u2581the", "\u2581past", "\u2581also", "\u2581spell", "ed", '\u2581"', "l", "ama", '"', "\u2581or", '\u2581"', "gl", "ama", '"', ")", "\u2581was", "\u2581adop", "ted", "\u2581by", "\u2581E", "urope", "an", "\u2581sett", "l", "ers", "\u2581from", "\u2581n", "ative", "\u2581Per", "u", "v", "ians", ".", "[", "4", "]", "\u2581The", "\u2581an", "c", "est", "ors", "\u2581of", "\u2581l", "l", "am", "as", "\u2581are", "\u2581thought", "\u2581to", "\u2581have", "\u2581origin", "ated", "\u2581from", "\u2581the", "\u2581Great", "\u2581Pl", "ain", "s", "\u2581of", "\u2581North", "\u2581America", "\u2581about", "\u2581", "4", "0", "\u2581million", "\u2581years", "\u2581ago", ",", "\u2581and", "\u2581sub", "sequ", "ently", "\u2581m", "ig", "r", "ated", "\u2581to", "\u2581South", "\u2581America", "\u2581about", "\u2581three", "\u2581million", "\u2581years", "\u2581ago", "\u2581during", "\u2581the", "\u2581Great", "\u2581American", "\u2581Int", "er", "ch", "ange", ".", "\u2581By", "\u2581the", "\u2581end", "\u2581of", "\u2581the", "\u2581last", "\u2581ice", "\u2581age", "\u2581", "(", "1", "0", ",", "0", "0", "0", "\u2013", "1", "2", ",", "0", "0", "0", "\u2581years", "\u2581ago", ")", ",", "\u2581cam", "el", "ids", "\u2581were", "\u2581ext", "inct", "\u2581in", "\u2581North", "\u2581America", ".", "[", "3", "]", "\u2581As", "\u2581of", "\u2581", "2", "0", "0", "7", ",", "\u2581there", "\u2581were", "\u2581over", "\u2581seven", "\u2581million", "\u2581l", "l", "am", "as", "\u2581and", "\u2581al", "p", "ac", "as", "\u2581in", "\u2581South", "\u2581America", "\u2581and", "\u2581over", "\u2581", "1", "5", "8", ",", "0", "0", "0", "\u2581l", "l", "am", "as", "\u2581and", "\u2581", "1", "0", "0", ",", "0", "0", "0", "", "\u2581al", "p", "ac", "as", ",", "\u2581des", "ce", "nd", "ed", "\u2581from", "\u2581pro", "gen", "it", "ors", "\u2581import", "ed", "\u2581late", "\u2581in", "\u2581the", "\u2581", "2", "0", "th", "\u2581cent", "ury", ",", "\u2581in", "\u2581the", "\u2581United", "\u2581States", "\u2581and", "\u2581Can", "ada", ".", "[", "5", "]", "\u2581In", "\u2581A", "ym", "ara", "\u2581my", "th", "ology", ",", "\u2581l", "l", "am", "as", "\u2581are", "\u2581important", "\u2581be", "ings", ".", "\u2581The", "\u2581He", "aven", "ly", "\u2581L", "l", "ama", "\u2581is", "\u2581said", "\u2581to", "\u2581drink", "\u2581water", "\u2581from", "\u2581the", "\u2581ocean", "\u2581and", "\u2581ur", "in", "ates", "\u2581as", "\u2581it", "\u2581rains", ".", "[", "6", "]", "\u2581Acc", "ord", "ing", "\u2581to", "\u2581A", "ym", "ara", "\u2581es", "ch", "at", "ology", ",", "\u2581l", "l", "am", "as", "\u2581will", "\u2581return", "\u2581to", "\u2581the", "\u2581water", "\u2581spr", "ings", "\u2581and", "\u2581l", "ag", "oons", "\u2581where", "\u2581they", "\u2581come", "\u2581from", "\u2581at", "\u2581the", "\u2581end", "\u2581of", "\u2581time", ".", "[", "6", "]"], + ids: [1, 147, 105, 31683, 4464, 31654, 32, 31753, 31774, 31683, 31813, 31779, 31687, 31781, 31753, 30, 31654, 0, 30106, 142, 531, 1823, 111, 367, 8762, 633, 29, 31654, 31778, 31774, 0, 4464, 31780, 33, 31654, 32, 31717, 4464, 1861, 4464, 33, 344, 48, 108, 120, 504, 515, 3062, 29052, 18424, 8829, 256, 153, 35, 20517, 2001, 2680, 488, 48, 9910, 83, 4314, 1448, 1015, 1736, 31660, 142, 103, 3441, 605, 13397, 70, 1629, 86, 7, 31739, 819, 4618, 1685, 31654, 7129, 34, 218, 31683, 235, 691, 617, 23632, 1707, 83, 5860, 249, 2905, 488, 48, 192, 31679, 34, 5290, 11964, 344, 3077, 83, 12959, 2859, 48, 1388, 7238, 251, 105, 142, 819, 81, 34, 31778, 13, 31780, 218, 31683, 235, 691, 645, 907, 16188, 22936, 1609, 48, 4505, 4706, 183, 29049, 34, 1354, 5247, 48, 4314, 35, 338, 645, 4923, 1096, 31654, 13, 16, 84, 31654, 14, 21, 10, 251, 626, 6011, 9152, 347, 31654, 19, 84, 31654, 12, 14, 29496, 31654, 32, 16, 31760, 19, 7843, 33, 34, 31778, 14, 31780, 147, 3516, 105, 31683, 4464, 31654, 32, 81, 70, 4829, 2320, 9948, 78, 245, 31683, 4464, 31690, 1187, 245, 686, 4464, 31690, 33, 139, 25228, 2490, 1015, 465, 25799, 142, 16405, 31683, 983, 825, 152, 12724, 24466, 31688, 31711, 26361, 34, 31778, 15, 31780, 147, 446, 31692, 504, 4166, 251, 105, 31683, 235, 691, 617, 1302, 84, 649, 7206, 3062, 825, 70, 27718, 12966, 588, 31675, 251, 26698, 27393, 1096, 31654, 15, 21, 23109, 3514, 17246, 35, 83, 5097, 17541, 19560, 114, 258, 31678, 3062, 84, 29052, 27393, 1096, 2765, 23109, 3514, 17246, 5823, 70, 27718, 18424, 25473, 98, 345, 3292, 34, 15498, 70, 1645, 251, 70, 6103, 2802, 13463, 31654, 32, 12, 21, 35, 21, 21, 21, 31760, 12, 13, 35, 21, 21, 21, 3514, 17246, 33, 35, 8829, 256, 16185, 579, 7522, 21465, 198, 26698, 27393, 34, 31778, 14, 31780, 1822, 251, 31654, 13, 21, 21, 18, 35, 478, 579, 1407, 20358, 23109, 105, 31683, 235, 691, 83, 789, 31694, 1324, 691, 198, 29052, 27393, 83, 1407, 31654, 12, 16, 19, 35, 21, 21, 21, 105, 31683, 235, 691, 83, 31654, 12, 21, 21, 35, 21, 21, 21, 0, 789, 31694, 1324, 691, 35, 3601, 215, 65, 78, 825, 2482, 8170, 93, 4166, 1777, 78, 5359, 198, 70, 31654, 13, 21, 1671, 11823, 11325, 35, 198, 70, 17562, 18843, 83, 3226, 19507, 34, 31778, 16, 31780, 2266, 231, 10586, 1362, 1286, 1671, 25316, 35, 105, 31683, 235, 691, 617, 2288, 233, 826, 34, 147, 264, 21794, 321, 218, 31683, 4464, 344, 309, 84, 4057, 1357, 825, 70, 5187, 83, 9947, 81, 4897, 488, 182, 24761, 34, 31778, 17, 31780, 28616, 4173, 127, 84, 231, 10586, 1362, 4469, 345, 122, 25316, 35, 105, 31683, 235, 691, 1214, 3520, 84, 70, 1357, 12312, 826, 83, 105, 762, 31431, 1930, 338, 1909, 825, 586, 70, 1645, 251, 470, 34, 31778, 17, 31780], + decoded: ' The llama (/\u02c8l\u0251\u02d0m\u0259/; Spanish pronunciation: [\u02c8ama]) (Lama glama) is a domesticated South American camelid, widely used as a meat and pack animal by Andean cultures since the Pre-Columbian era. Llamas are social animals and live with others as a herd. Their wool is soft and contains only a small amount of lanolin.[2] Llamas can learn simple tasks after a few repetitions. When using a pack, they can carry about 25 to 30% of their body weight for 8 to 13 km (5\u20138 miles).[3] The name llama (in the past also spelled "lama" or "glama") was adopted by European settlers from native Peruvians.[4] The ancestors of llamas are thought to have originated from the Great Plains of North America about 40 million years ago, and subsequently migrated to South America about three million years ago during the Great American Interchange. By the end of the last ice age (10,000\u201312,000 years ago), camelids were extinct in North America.[3] As of 2007, there were over seven million llamas and alpacas in South America and over 158,000 llamas and 100,000 alpacas, descended from progenitors imported late in the 20th century, in the United States and Canada.[5] In Aymara mythology, llamas are important beings. The Heavenly Llama is said to drink water from the ocean and urinates as it rains.[6] According to Aymara eschatology, llamas will return to the water springs and lagoons where they come from at the end of time.[6]', + }, }, - }, }; const MAX_EXECUTION_TIME = 10_000; export const CUSTOM_TESTS = () => { - // Tests to ensure that no matter what, the correct tokenization is returned. - // This is necessary since there are sometimes bugs in the transformers library. - describe("hard-coded", () => { - const TESTS = { - "Xenova/llama-tokenizer": [ - // Test legacy compatibility - { - // legacy unset => legacy=true - // NOTE: While incorrect, it is necessary to match legacy behaviour - data: { - "\n": [1, 29871, 13], - }, - legacy: null, - }, - { - // override legacy=true (same results as above) - data: { - "\n": [1, 29871, 13], - }, - legacy: true, - }, - { - // override legacy=false (fixed results) - data: { - "\n": [1, 13], - }, - legacy: false, - }, - ], + // Tests to ensure that no matter what, the correct tokenization is returned. + // This is necessary since there are sometimes bugs in the transformers library. + describe("hard-coded", () => { + const TESTS = { + "Xenova/llama-tokenizer": [ + // Test legacy compatibility + { + // legacy unset => legacy=true + // NOTE: While incorrect, it is necessary to match legacy behaviour + data: { + "\n": [1, 29871, 13], + }, + legacy: null, + }, + { + // override legacy=true (same results as above) + data: { + "\n": [1, 29871, 13], + }, + legacy: true, + }, + { + // override legacy=false (fixed results) + data: { + "\n": [1, 13], + }, + legacy: false, + }, + ], - "Xenova/llama-tokenizer_new": [ - // legacy=false - { - data: { - " 1 2 3 4 ": [259, 2, 29871, 29896, 259, 29906, 1678, 29941, 268, 29946, 1678], - "\n": [1, 13], - "test": [2, 1688, 2], - " test ": [259, 2, 1243, 29871, 2, 29871], - "A\n'll": [319, 13, 29915, 645], - "Hey . how are you": [18637, 29871, 2, 29889, 920, 526, 366], - " Hi Hello ": [259, 6324, 29871, 15043, 259], - }, - reversible: true, - legacy: null, - }, - { - // override legacy=true (incorrect results, but necessary to match legacy behaviour) - data: { - "\n": [1, 29871, 13], - }, - legacy: true, - }, - ], + "Xenova/llama-tokenizer_new": [ + // legacy=false + { + data: { + " 1 2 3 4 ": [259, 2, 29871, 29896, 259, 29906, 1678, 29941, 268, 29946, 1678], + "\n": [1, 13], + "test": [2, 1688, 2], + " test ": [259, 2, 1243, 29871, 2, 29871], + "A\n'll": [319, 13, 29915, 645], + "Hey . how are you": [18637, 29871, 2, 29889, 920, 526, 366], + " Hi Hello ": [259, 6324, 29871, 15043, 259], + }, + reversible: true, + legacy: null, + }, + { + // override legacy=true (incorrect results, but necessary to match legacy behaviour) + data: { + "\n": [1, 29871, 13], + }, + legacy: true, + }, + ], - // new serialization format (tokenizers >= 0.20.0) - // BPE merges are now [string, string][] instead of string[] - "Xenova/Llama-3.2-Tokenizer": [ - { - data: { - "hello world": [15339, 1917], - " belirtilen": [120909], - }, - reversible: true, - }, + // new serialization format (tokenizers >= 0.20.0) + // BPE merges are now [string, string][] instead of string[] + "Xenova/Llama-3.2-Tokenizer": [ + { + data: { + "hello world": [15339, 1917], + " belirtilen": [120909], + }, + reversible: true, + }, - // Test ignore_merges=false - { - data: { - "hello world": [15339, 1917], - " belirtilen": [101664, 1678, 268], - }, - reversible: true, - override: (tokenizer) => { - tokenizer.model.ignore_merges = false; - }, - }, - ], - }; + // Test ignore_merges=false + { + data: { + "hello world": [15339, 1917], + " belirtilen": [101664, 1678, 268], + }, + reversible: true, + override: (tokenizer) => { + tokenizer.model.ignore_merges = false; + }, + }, + ], + }; - // Re-use the same tests for the llama2 tokenizer - TESTS["Xenova/llama2-tokenizer"] = TESTS["Xenova/llama-tokenizer_new"]; + // Re-use the same tests for the llama2 tokenizer + TESTS["Xenova/llama2-tokenizer"] = TESTS["Xenova/llama-tokenizer_new"]; - for (const [tokenizerName, test_data] of Object.entries(TESTS)) { - it( - tokenizerName, - async () => { - for (const { data, reversible, legacy, override } of test_data) { - const tokenizer = await LlamaTokenizer.from_pretrained(tokenizerName, { legacy }); - if (override) { - override(tokenizer); - } - for (const [text, expected] of Object.entries(data)) { - const token_ids = tokenizer.encode(text, { add_special_tokens: false }); - expect(token_ids).toEqual(expected); + for (const [tokenizerName, test_data] of Object.entries(TESTS)) { + it( + tokenizerName, + async () => { + for (const { data, reversible, legacy, override } of test_data) { + const tokenizer = await LlamaTokenizer.from_pretrained(tokenizerName, { + legacy, + }); + if (override) { + override(tokenizer); + } + for (const [text, expected] of Object.entries(data)) { + const token_ids = tokenizer.encode(text, { add_special_tokens: false }); + expect(token_ids).toEqual(expected); - // If reversible, test that decoding produces the original text - if (reversible) { - const decoded = tokenizer.decode(token_ids); - expect(decoded).toEqual(text); - } - } - } - }, - MAX_EXECUTION_TIME, - ); - } - }); + // If reversible, test that decoding produces the original text + if (reversible) { + const decoded = tokenizer.decode(token_ids); + expect(decoded).toEqual(text); + } + } + } + }, + MAX_EXECUTION_TIME, + ); + } + }); }; diff --git a/tests/models/m2m_100/tokenization.js b/tests/models/m2m_100/tokenization.js index 6af51f44f..b976b0596 100644 --- a/tests/models/m2m_100/tokenization.js +++ b/tests/models/m2m_100/tokenization.js @@ -6,174 +6,174 @@ export const TOKENIZER_CLASS = M2M100Tokenizer; // NOTE: The slow tokenizer (used by transformers) has minor inconsistencies against the fast tokenizer. // For this reason, we may override the expected results for certain tests. export const TEST_CONFIG = { - "Xenova/m2m100_418M": { - SIMPLE: { - text: BASE_TEST_STRINGS.SIMPLE, - tokens: ["\u2581How", "\u2581are", "\u2581you", "\u2581doing", "?"], - ids: [128022, 34226, 4234, 8251, 123047, 24, 2], - decoded: "__en__ How are you doing?", + "Xenova/m2m100_418M": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["\u2581How", "\u2581are", "\u2581you", "\u2581doing", "?"], + ids: [128022, 34226, 4234, 8251, 123047, 24, 2], + decoded: "__en__ How are you doing?", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["\u2581You", "\u2581should", "'", "ve", "\u2581done", "\u2581this"], + ids: [128022, 14921, 119092, 12, 470, 111108, 15911, 2], + decoded: "__en__ You should've done this", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["\u25810", "123", "45", "6", "78", "9", "\u25810", "\u25811", "\u25812", "\u25813", "\u25814", "\u25815", "\u25816", "\u25817", "\u25818", "\u25819", "\u258110", "\u2581100", "\u25811000"], + ids: [128022, 847, 78596, 3834, 435, 7049, 718, 847, 161, 168, 205, 273, 265, 376, 442, 455, 572, 301, 1245, 7336, 2], + decoded: "__en__ 0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["\u2581The", "\u2581company", "\u2581was", "\u2581found", "ed", "\u2581in", "\u25812016."], + ids: [128022, 1658, 66486, 1513, 118728, 241, 28, 8860, 2], + decoded: "__en__ The company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["\u2581A", "\u2581'", "ll", "\u2581!!", "to", "?'", "d", "''", "d", "\u2581of", ",", "\u2581can", "'", "t", "."], + ids: [128022, 58, 244, 2279, 9403, 428, 72956, 173, 8471, 173, 432, 4, 3154, 12, 88, 5, 2], + decoded: "__en__ A 'll!!to?'d''d of, can't.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["\u2581def", "\u2581main", "(", "):", "\u2581pass"], + ids: [128022, 8268, 9359, 249, 2825, 4799, 2], + decoded: "__en__ def main(): pass", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["\u2581let", "\u2581a", "\u2581=", "\u2581ob", "j", ".", "to", "Str", "ing", "(", ");", "\u2581to", "Str", "ing", "(", ");"], + ids: [128022, 2507, 8, 3255, 607, 189, 5, 428, 41549, 150, 249, 5294, 128, 41549, 150, 249, 5294, 2], + decoded: "__en__ let a = obj.toString(); toString();", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["\u2581This", "\u2581is", "\u2581a", "\u2581test", "."], + ids: [128022, 36606, 117, 8, 4183, 5, 2], + decoded: "__en__ This is a test.", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["\u2581UN", "want", "\u00e9d", ",", "run", "ning"], + ids: [128022, 6984, 108054, 7151, 4, 18634, 656, 2], + decoded: "__en__ UNwant\u00e9d,running", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["\u25811", "\u0000", "2", "\u25813"], + ids: [128022, 161, 4163, 339, 205, 2], + decoded: "__en__ 1\u00002 3", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["\u2581Hello", "\u2581World"], + ids: [128022, 65761, 10581, 2], + decoded: "__en__ Hello World", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["\u2581hello", "\u2581world"], + ids: [128022, 110013, 55185, 2], + decoded: "__en__ hello world", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u2581", "\u751f\u6d3b", "\u7684", "\u771f", /* "\u8c1b" */ "", "\u662f"], + ids: [128022, 22, 8523, 80, 10418, 3, 775, 2], + decoded: "__en__ \u751f\u6d3b\u7684\u771f\u662f", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u2581leading", "\u2581space"], + ids: [128022, 124476, 118561, 2], + decoded: "__en__ leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["\u2581tra", "iling", "\u2581space"], + ids: [128022, 1368, 19217, 118561, 2], + decoded: "__en__ trailing space", + }, + SURROUNDING_SPACE: { + text: BASE_TEST_STRINGS.SURROUNDING_SPACE, + tokens: ["\u2581surround", "ing", "\u2581space"], + ids: [128022, 124728, 150, 118561, 2], + decoded: "__en__ surrounding space", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["\u2581Hi", "\u2581Hello"], + ids: [128022, 7676, 65761, 2], + decoded: "__en__ Hi Hello", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["\u2581test", "\u2581$", "1", "\u2581R", "2", "\u2581#", "3", "\u2581\u20ac", "4", "\u2581\u00a3", "5", "\u2581", "\u00a5", "6", "\u2581", /* "\u20a3" */ "", "7", "\u2581", "\u20b9", "8", "\u2581", /* "\u20b1" */ "", "9", "\u2581test"], + ids: [128022, 4183, 4352, 451, 180, 339, 584, 425, 4257, 465, 13506, 679, 22, 43832, 435, 22, 3, 622, 22, 115056, 677, 22, 3, 718, 4183, 2], + decoded: "__en__ test $1 R2 #3 \u20ac4 \u00a35 \u00a56 7 \u20b98 9 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["\u2581I", "\u2581bought", "\u2581an", "\u2581ap", "ple", "\u2581for", "\u2581$", "1.", "00", "\u2581at", "\u2581the", "\u2581store", "."], + ids: [128022, 203, 127797, 48, 722, 6857, 193, 4352, 2023, 1365, 120, 1197, 9160, 5, 2], + decoded: "__en__ I bought an apple for $1.00 at the store.", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["\u2581you", "..."], + ids: [128022, 8251, 26, 2], + decoded: "__en__ you...", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["\u2581you", "..."], + ids: [128022, 8251, 26, 2], + decoded: "__en__ you...", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["\u2581you", "...", "\u2581you", "..."], + ids: [128022, 8251, 26, 8251, 26, 2], + decoded: "__en__ you... you...", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["\u2581we", "ird", "\u2581", "\uff5e", "\u2581ed", "ge", "\u2581", "\uff5e", "\u2581case"], + ids: [128022, 1710, 13067, 22, 14691, 1500, 568, 22, 14691, 24306, 2], + decoded: "__en__ weird \uff5e edge \uff5e case", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u2581This", "\u2581is", "\u2581a", "\u2581test", "\u2581."], + ids: [128022, 36606, 117, 8, 4183, 237, 2], + decoded: "__en__ This is a test.", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u2581", "\ud83d\ude02", "\u2581", "\ud83d\udc4d", "\u2581", "\ud83e\udd23", "\u2581", "\ud83d\ude0d", "\u2581", "\ud83d\ude2d", "\u2581", "\ud83c\udf89", "\u2581", "\ud83d\ude4f", "\u2581", "\ud83d\ude0a", "\u2581", "\ud83d\udd25", "\u2581", "\ud83d\ude01", "\u2581", "\ud83d\ude05", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\ude06", "\u2581", "\ud83d\udc4f", "\u2581\u2764", "\ufe0f", "\u2581", "\ud83d\udc9c", "\u2581", "\ud83d\udc9a", "\u2581", "\ud83d\udc97", "\u2581", "\ud83d\udc99", "\u2581", "\ud83d\udda4", "\u2581", "\ud83d\ude0e", "\u2581", "\ud83d\udc4c", "\u2581", /* "\ud83e\udd73" */ "", "\u2581", "\ud83d\udcaa", "\u2581", "\u2728", "\u2581", "\ud83d\udc49", "\u2581", "\ud83d\udc40", "\u2581", "\ud83d\udcaf", "\u2581", "\ud83c\udf88", "\u2581", "\ud83d\ude48", "\u2581", "\ud83d\ude4c", "\u2581", /* "\ud83d\udc80" */ "", "\u2581", "\ud83d\udc47", "\u2581", "\ud83d\udc4b", "\u2581", "\u2705", "\u2581", "\ud83c\udf81", "\u2581", /* "\ud83c\udf1e" */ "", "\u2581", "\ud83c\udf38", "\u2581", "\ud83d\udcb0"], + ids: [128022, 22, 74222, 22, 118514, 22, 124385, 22, 99683, 22, 123842, 22, 124821, 22, 117689, 22, 103111, 22, 121924, 22, 121088, 22, 124207, 22, 123955, 22, 120137, 22, 123534, 66038, 18905, 22, 125385, 22, 125317, 22, 126071, 22, 124787, 22, 127396, 22, 120119, 22, 122813, 22, 3, 22, 123482, 22, 120563, 22, 117995, 22, 127978, 22, 126507, 22, 127269, 22, 126179, 22, 125300, 22, 3, 22, 120807, 22, 127143, 22, 118682, 22, 125350, 22, 3, 22, 123790, 22, 126948, 2], + decoded: /* "__en__ \ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf38 \ud83d\udcb0" */ "__en__ \ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf38 \ud83d\udcb0", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u2581", "\u2728", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\udc41", "\ufe0f", "\u2581", /* "\ud83d\udc71" */ "", "\ud83c\udffb", "\u2581", /* "\ud83d\udd75" */ "", "\u2581", "\u2642", "\ufe0f", "\u2581", /* "\ud83e\uddd9" */ "", "\ud83c\udffb", "\u2581", "\u2642", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u2581", /* "\ud83c\udf3e" */ "", "\u2581", /* "\ud83e\uddd1" */ "", "\u2581", /* "\ud83e\udd1d" */ "", "\u2581", /* "\ud83e\uddd1" */ "", "\u2581", "\ud83d\udc69", "\u2581\u2764", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc69", "\u2581", /* "\ud83d\udc67" */ "", "\u2581", /* "\ud83d\udc66" */ "", "\u2581", /* "\ud83e\uddd1" */ "", "\ud83c\udffb", "\u2581", /* "\ud83e\udd1d" */ "", "\u2581", /* "\ud83e\uddd1" */ "", "\ud83c\udffb", "\u2581", /* "\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f"*/ "", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u2581\u2764", "\ufe0f", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\ud83c\udffc"], + ids: [128022, 22, 120563, 22, 123955, 22, 121442, 18905, 22, 3, 116617, 22, 3, 22, 122517, 18905, 22, 3, 116617, 22, 122517, 22, 127603, 116617, 22, 3, 22, 3, 22, 3, 22, 3, 22, 126739, 66038, 22, 126237, 22, 127603, 22, 126739, 22, 126739, 22, 3, 22, 3, 22, 3, 116617, 22, 3, 22, 3, 116617, 22, 3, 22, 127603, 116617, 66038, 18905, 22, 126237, 22, 127603, 123285, 2], + decoded: /* "__en__ \u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83c\udffb \u2642\ufe0f \ud83c\udffb \u2642 \ud83d\udc68\ud83c\udffb \ud83d\udc69 \u2764 \ud83d\udc8b \ud83d\udc68 \ud83d\udc69 \ud83d\udc69 \ud83c\udffb \ud83c\udffb \ud83d\udc68\ud83c\udffb \u2764\ufe0f \ud83d\udc8b \ud83d\udc68\ud83c\udffc" */ "__en__ \u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83c\udffb \u2642\ufe0f \ud83c\udffb \u2642 \ud83d\udc68\ud83c\udffb \ud83d\udc69 \u2764 \ud83d\udc8b \ud83d\udc68 \ud83d\udc69 \ud83d\udc69 \ud83c\udffb \ud83c\udffb \ud83d\udc68\ud83c\udffb \u2764\ufe0f \ud83d\udc8b \ud83d\udc68\ud83c\udffc", + }, + ONLY_WHITESPACE: { + text: BASE_TEST_STRINGS.ONLY_WHITESPACE, + tokens: [], + ids: [128022, 2], + decoded: /* "__en__ " */ "__en__", + }, + TRANSLATION_INPUTS: { + text: M2M_100_TEST_STRINGS.TRANSLATION_INPUTS, + tokens: ["__en__", "\u2581hello", "\u2581world", ""], + ids: [128022, 128022, 110013, 55185, 2, 2], + decoded: /* "__en__ __en__ hello world" */ "__en____en__ hello world", + }, }, - SIMPLE_WITH_PUNCTUATION: { - text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, - tokens: ["\u2581You", "\u2581should", "'", "ve", "\u2581done", "\u2581this"], - ids: [128022, 14921, 119092, 12, 470, 111108, 15911, 2], - decoded: "__en__ You should've done this", - }, - NUMBERS: { - text: BASE_TEST_STRINGS.NUMBERS, - tokens: ["\u25810", "123", "45", "6", "78", "9", "\u25810", "\u25811", "\u25812", "\u25813", "\u25814", "\u25815", "\u25816", "\u25817", "\u25818", "\u25819", "\u258110", "\u2581100", "\u25811000"], - ids: [128022, 847, 78596, 3834, 435, 7049, 718, 847, 161, 168, 205, 273, 265, 376, 442, 455, 572, 301, 1245, 7336, 2], - decoded: "__en__ 0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["\u2581The", "\u2581company", "\u2581was", "\u2581found", "ed", "\u2581in", "\u25812016."], - ids: [128022, 1658, 66486, 1513, 118728, 241, 28, 8860, 2], - decoded: "__en__ The company was founded in 2016.", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["\u2581A", "\u2581'", "ll", "\u2581!!", "to", "?'", "d", "''", "d", "\u2581of", ",", "\u2581can", "'", "t", "."], - ids: [128022, 58, 244, 2279, 9403, 428, 72956, 173, 8471, 173, 432, 4, 3154, 12, 88, 5, 2], - decoded: "__en__ A 'll!!to?'d''d of, can't.", - }, - PYTHON_CODE: { - text: BASE_TEST_STRINGS.PYTHON_CODE, - tokens: ["\u2581def", "\u2581main", "(", "):", "\u2581pass"], - ids: [128022, 8268, 9359, 249, 2825, 4799, 2], - decoded: "__en__ def main(): pass", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["\u2581let", "\u2581a", "\u2581=", "\u2581ob", "j", ".", "to", "Str", "ing", "(", ");", "\u2581to", "Str", "ing", "(", ");"], - ids: [128022, 2507, 8, 3255, 607, 189, 5, 428, 41549, 150, 249, 5294, 128, 41549, 150, 249, 5294, 2], - decoded: "__en__ let a = obj.toString(); toString();", - }, - NEWLINES: { - text: BASE_TEST_STRINGS.NEWLINES, - tokens: ["\u2581This", "\u2581is", "\u2581a", "\u2581test", "."], - ids: [128022, 36606, 117, 8, 4183, 5, 2], - decoded: "__en__ This is a test.", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["\u2581UN", "want", "\u00e9d", ",", "run", "ning"], - ids: [128022, 6984, 108054, 7151, 4, 18634, 656, 2], - decoded: "__en__ UNwant\u00e9d,running", - }, - CONTROL_TOKENS: { - text: BASE_TEST_STRINGS.CONTROL_TOKENS, - tokens: ["\u25811", "\u0000", "2", "\u25813"], - ids: [128022, 161, 4163, 339, 205, 2], - decoded: "__en__ 1\u00002 3", - }, - HELLO_WORLD_TITLECASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, - tokens: ["\u2581Hello", "\u2581World"], - ids: [128022, 65761, 10581, 2], - decoded: "__en__ Hello World", - }, - HELLO_WORLD_LOWERCASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, - tokens: ["\u2581hello", "\u2581world"], - ids: [128022, 110013, 55185, 2], - decoded: "__en__ hello world", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u2581", "\u751f\u6d3b", "\u7684", "\u771f", /* "\u8c1b" */ "", "\u662f"], - ids: [128022, 22, 8523, 80, 10418, 3, 775, 2], - decoded: "__en__ \u751f\u6d3b\u7684\u771f\u662f", - }, - LEADING_SPACE: { - text: BASE_TEST_STRINGS.LEADING_SPACE, - tokens: ["\u2581leading", "\u2581space"], - ids: [128022, 124476, 118561, 2], - decoded: "__en__ leading space", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["\u2581tra", "iling", "\u2581space"], - ids: [128022, 1368, 19217, 118561, 2], - decoded: "__en__ trailing space", - }, - SURROUNDING_SPACE: { - text: BASE_TEST_STRINGS.SURROUNDING_SPACE, - tokens: ["\u2581surround", "ing", "\u2581space"], - ids: [128022, 124728, 150, 118561, 2], - decoded: "__en__ surrounding space", - }, - DOUBLE_SPACE: { - text: BASE_TEST_STRINGS.DOUBLE_SPACE, - tokens: ["\u2581Hi", "\u2581Hello"], - ids: [128022, 7676, 65761, 2], - decoded: "__en__ Hi Hello", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["\u2581test", "\u2581$", "1", "\u2581R", "2", "\u2581#", "3", "\u2581\u20ac", "4", "\u2581\u00a3", "5", "\u2581", "\u00a5", "6", "\u2581", /* "\u20a3" */ "", "7", "\u2581", "\u20b9", "8", "\u2581", /* "\u20b1" */ "", "9", "\u2581test"], - ids: [128022, 4183, 4352, 451, 180, 339, 584, 425, 4257, 465, 13506, 679, 22, 43832, 435, 22, 3, 622, 22, 115056, 677, 22, 3, 718, 4183, 2], - decoded: "__en__ test $1 R2 #3 \u20ac4 \u00a35 \u00a56 7 \u20b98 9 test", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["\u2581I", "\u2581bought", "\u2581an", "\u2581ap", "ple", "\u2581for", "\u2581$", "1.", "00", "\u2581at", "\u2581the", "\u2581store", "."], - ids: [128022, 203, 127797, 48, 722, 6857, 193, 4352, 2023, 1365, 120, 1197, 9160, 5, 2], - decoded: "__en__ I bought an apple for $1.00 at the store.", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - tokens: ["\u2581you", "..."], - ids: [128022, 8251, 26, 2], - decoded: "__en__ you...", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - tokens: ["\u2581you", "..."], - ids: [128022, 8251, 26, 2], - decoded: "__en__ you...", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - tokens: ["\u2581you", "...", "\u2581you", "..."], - ids: [128022, 8251, 26, 8251, 26, 2], - decoded: "__en__ you... you...", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["\u2581we", "ird", "\u2581", "\uff5e", "\u2581ed", "ge", "\u2581", "\uff5e", "\u2581case"], - ids: [128022, 1710, 13067, 22, 14691, 1500, 568, 22, 14691, 24306, 2], - decoded: "__en__ weird \uff5e edge \uff5e case", - }, - SPIECE_UNDERSCORE: { - text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, - tokens: ["\u2581This", "\u2581is", "\u2581a", "\u2581test", "\u2581."], - ids: [128022, 36606, 117, 8, 4183, 237, 2], - decoded: "__en__ This is a test.", - }, - POPULAR_EMOJIS: { - text: BASE_TEST_STRINGS.POPULAR_EMOJIS, - tokens: ["\u2581", "\ud83d\ude02", "\u2581", "\ud83d\udc4d", "\u2581", "\ud83e\udd23", "\u2581", "\ud83d\ude0d", "\u2581", "\ud83d\ude2d", "\u2581", "\ud83c\udf89", "\u2581", "\ud83d\ude4f", "\u2581", "\ud83d\ude0a", "\u2581", "\ud83d\udd25", "\u2581", "\ud83d\ude01", "\u2581", "\ud83d\ude05", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\ude06", "\u2581", "\ud83d\udc4f", "\u2581\u2764", "\ufe0f", "\u2581", "\ud83d\udc9c", "\u2581", "\ud83d\udc9a", "\u2581", "\ud83d\udc97", "\u2581", "\ud83d\udc99", "\u2581", "\ud83d\udda4", "\u2581", "\ud83d\ude0e", "\u2581", "\ud83d\udc4c", "\u2581", /* "\ud83e\udd73" */ "", "\u2581", "\ud83d\udcaa", "\u2581", "\u2728", "\u2581", "\ud83d\udc49", "\u2581", "\ud83d\udc40", "\u2581", "\ud83d\udcaf", "\u2581", "\ud83c\udf88", "\u2581", "\ud83d\ude48", "\u2581", "\ud83d\ude4c", "\u2581", /* "\ud83d\udc80" */ "", "\u2581", "\ud83d\udc47", "\u2581", "\ud83d\udc4b", "\u2581", "\u2705", "\u2581", "\ud83c\udf81", "\u2581", /* "\ud83c\udf1e" */ "", "\u2581", "\ud83c\udf38", "\u2581", "\ud83d\udcb0"], - ids: [128022, 22, 74222, 22, 118514, 22, 124385, 22, 99683, 22, 123842, 22, 124821, 22, 117689, 22, 103111, 22, 121924, 22, 121088, 22, 124207, 22, 123955, 22, 120137, 22, 123534, 66038, 18905, 22, 125385, 22, 125317, 22, 126071, 22, 124787, 22, 127396, 22, 120119, 22, 122813, 22, 3, 22, 123482, 22, 120563, 22, 117995, 22, 127978, 22, 126507, 22, 127269, 22, 126179, 22, 125300, 22, 3, 22, 120807, 22, 127143, 22, 118682, 22, 125350, 22, 3, 22, 123790, 22, 126948, 2], - decoded: /* "__en__ \ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf38 \ud83d\udcb0" */ "__en__ \ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf38 \ud83d\udcb0", - }, - MULTIBYTE_EMOJIS: { - text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, - tokens: ["\u2581", "\u2728", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\udc41", "\ufe0f", "\u2581", /* "\ud83d\udc71" */ "", "\ud83c\udffb", "\u2581", /* "\ud83d\udd75" */ "", "\u2581", "\u2642", "\ufe0f", "\u2581", /* "\ud83e\uddd9" */ "", "\ud83c\udffb", "\u2581", "\u2642", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u2581", /* "\ud83c\udf3e" */ "", "\u2581", /* "\ud83e\uddd1" */ "", "\u2581", /* "\ud83e\udd1d" */ "", "\u2581", /* "\ud83e\uddd1" */ "", "\u2581", "\ud83d\udc69", "\u2581\u2764", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc69", "\u2581", /* "\ud83d\udc67" */ "", "\u2581", /* "\ud83d\udc66" */ "", "\u2581", /* "\ud83e\uddd1" */ "", "\ud83c\udffb", "\u2581", /* "\ud83e\udd1d" */ "", "\u2581", /* "\ud83e\uddd1" */ "", "\ud83c\udffb", "\u2581", /* "\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f"*/ "", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u2581\u2764", "\ufe0f", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\ud83c\udffc"], - ids: [128022, 22, 120563, 22, 123955, 22, 121442, 18905, 22, 3, 116617, 22, 3, 22, 122517, 18905, 22, 3, 116617, 22, 122517, 22, 127603, 116617, 22, 3, 22, 3, 22, 3, 22, 3, 22, 126739, 66038, 22, 126237, 22, 127603, 22, 126739, 22, 126739, 22, 3, 22, 3, 22, 3, 116617, 22, 3, 22, 3, 116617, 22, 3, 22, 127603, 116617, 66038, 18905, 22, 126237, 22, 127603, 123285, 2], - decoded: /* "__en__ \u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83c\udffb \u2642\ufe0f \ud83c\udffb \u2642 \ud83d\udc68\ud83c\udffb \ud83d\udc69 \u2764 \ud83d\udc8b \ud83d\udc68 \ud83d\udc69 \ud83d\udc69 \ud83c\udffb \ud83c\udffb \ud83d\udc68\ud83c\udffb \u2764\ufe0f \ud83d\udc8b \ud83d\udc68\ud83c\udffc" */ "__en__ \u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83c\udffb \u2642\ufe0f \ud83c\udffb \u2642 \ud83d\udc68\ud83c\udffb \ud83d\udc69 \u2764 \ud83d\udc8b \ud83d\udc68 \ud83d\udc69 \ud83d\udc69 \ud83c\udffb \ud83c\udffb \ud83d\udc68\ud83c\udffb \u2764\ufe0f \ud83d\udc8b \ud83d\udc68\ud83c\udffc", - }, - ONLY_WHITESPACE: { - text: BASE_TEST_STRINGS.ONLY_WHITESPACE, - tokens: [], - ids: [128022, 2], - decoded: /* "__en__ " */ "__en__", - }, - TRANSLATION_INPUTS: { - text: M2M_100_TEST_STRINGS.TRANSLATION_INPUTS, - tokens: ["__en__", "\u2581hello", "\u2581world", ""], - ids: [128022, 128022, 110013, 55185, 2, 2], - decoded: /* "__en__ __en__ hello world" */ "__en____en__ hello world", - }, - }, }; diff --git a/tests/models/mpnet/tokenization.js b/tests/models/mpnet/tokenization.js index 13f688d73..57d198a2e 100644 --- a/tests/models/mpnet/tokenization.js +++ b/tests/models/mpnet/tokenization.js @@ -3,156 +3,156 @@ import { BASE_TEST_STRINGS } from "../test_strings.js"; export const TOKENIZER_CLASS = MPNetTokenizer; export const TEST_CONFIG = { - "Xenova/all-mpnet-base-v2": { - SIMPLE: { - text: BASE_TEST_STRINGS.SIMPLE, - tokens: ["how", "are", "you", "doing", "?"], - ids: [0, 2133, 2028, 2021, 2729, 1033, 2], - decoded: " how are you doing? ", + "Xenova/all-mpnet-base-v2": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["how", "are", "you", "doing", "?"], + ids: [0, 2133, 2028, 2021, 2729, 1033, 2], + decoded: " how are you doing? ", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["you", "should", "'", "ve", "done", "this"], + ids: [0, 2021, 2327, 1009, 2314, 2593, 2027, 2], + decoded: " you should've done this ", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["01", "##23", "##45", "##6", "##7", "##8", "##9", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "100", "1000"], + ids: [0, 5894, 21930, 19965, 2579, 2585, 2624, 2687, 1018, 1019, 1020, 1021, 1022, 1023, 1024, 1025, 1026, 1027, 2188, 2535, 6698, 2], + decoded: " 0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000 ", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["the", "company", "was", "founded", "in", "2016", "."], + ids: [0, 2000, 2198, 2005, 2635, 2003, 2359, 1016, 2], + decoded: " the company was founded in 2016. ", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["a", "'", "ll", "!", "!", "to", "?", "'", "d", "'", "'", "d", "of", ",", "can", "'", "t", "."], + ids: [0, 1041, 1009, 2226, 1003, 1003, 2004, 1033, 1009, 1044, 1009, 1009, 1044, 2001, 1014, 2068, 1009, 1060, 1016, 2], + decoded: " a'll!! to?'d'' d of, can't. ", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["def", "main", "(", ")", ":", "pass"], + ids: [0, 13370, 2368, 1010, 1011, 1028, 3417, 2], + decoded: " def main ( ) : pass ", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "a", "=", "ob", "##j", ".", "to", "##st", "##ring", "(", ")", ";", "to", "##st", "##ring", "(", ")", ";"], + ids: [0, 2296, 1041, 1031, 27889, 3505, 1016, 2004, 3371, 4896, 1010, 1011, 1029, 2004, 3371, 4896, 1010, 1011, 1029, 2], + decoded: " let a = obj. tostring ( ) ; tostring ( ) ; ", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["this", "is", "a", "test", "."], + ids: [0, 2027, 2007, 1041, 3235, 1016, 2], + decoded: " this is a test. ", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["unwanted", ",", "running"], + ids: [0, 18166, 1014, 2774, 2], + decoded: " unwanted, running ", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["123"], + ids: [0, 13142, 2], + decoded: " 123 ", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["hello", "world"], + ids: [0, 7596, 2092, 2], + decoded: " hello world ", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["hello", "world"], + ids: [0, 7596, 2092, 2], + decoded: " hello world ", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u751f", "[UNK]", "\u7684", "\u771f", "[UNK]", "[UNK]"], + ids: [0, 1914, 104, 1920, 1925, 104, 104, 2], + decoded: " \u751f [UNK] \u7684 \u771f [UNK] [UNK] ", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["leading", "space"], + ids: [0, 2881, 2690, 2], + decoded: " leading space ", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["trailing", "space"], + ids: [0, 12546, 2690, 2], + decoded: " trailing space ", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["hi", "hello"], + ids: [0, 7636, 7596, 2], + decoded: " hi hello ", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "$", "1", "r", "##2", "#", "3", "\u20ac", "##4", "\u00a35", "\u00a5", "##6", "[UNK]", "\u20b9", "##8", "\u20b1", "##9", "test"], + ids: [0, 3235, 1006, 1019, 1058, 2479, 1005, 1021, 1578, 2553, 27817, 1075, 2579, 104, 1580, 2624, 1579, 2687, 3235, 2], + decoded: " test $ 1 r2 # 3 \u20ac4 \u00a35 \u00a56 [UNK] \u20b98 \u20b19 test ", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["i", "bought", "an", "apple", "for", "$", "1", ".", "00", "at", "the", "store", "."], + ids: [0, 1049, 4153, 2023, 6211, 2009, 1006, 1019, 1016, 4006, 2016, 2000, 3577, 1016, 2], + decoded: " i bought an apple for $ 1. 00 at the store. ", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "\u2026"], + ids: [0, 2021, 1533, 2], + decoded: " you \u2026 ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "\u2026"], + ids: [0, 2021, 1533, 2], + decoded: " you \u2026 ", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "\u2026", "you", "\u2026"], + ids: [0, 2021, 1533, 2021, 1533, 2], + decoded: " you \u2026 you \u2026 ", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["weird", "\uff5e", "edge", "\uff5e", "case"], + ids: [0, 6885, 1999, 3345, 1999, 2557, 2], + decoded: " weird \uff5e edge \uff5e case ", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "."], + ids: [0, 104, 104, 104, 104, 104, 1016, 2], + decoded: " [UNK] [UNK] [UNK] [UNK] [UNK]. ", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]"], + ids: [0, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 2], + decoded: " [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] ", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]"], + ids: [0, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 2], + decoded: " [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] ", + }, }, - SIMPLE_WITH_PUNCTUATION: { - text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, - tokens: ["you", "should", "'", "ve", "done", "this"], - ids: [0, 2021, 2327, 1009, 2314, 2593, 2027, 2], - decoded: " you should've done this ", - }, - NUMBERS: { - text: BASE_TEST_STRINGS.NUMBERS, - tokens: ["01", "##23", "##45", "##6", "##7", "##8", "##9", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "100", "1000"], - ids: [0, 5894, 21930, 19965, 2579, 2585, 2624, 2687, 1018, 1019, 1020, 1021, 1022, 1023, 1024, 1025, 1026, 1027, 2188, 2535, 6698, 2], - decoded: " 0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000 ", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["the", "company", "was", "founded", "in", "2016", "."], - ids: [0, 2000, 2198, 2005, 2635, 2003, 2359, 1016, 2], - decoded: " the company was founded in 2016. ", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["a", "'", "ll", "!", "!", "to", "?", "'", "d", "'", "'", "d", "of", ",", "can", "'", "t", "."], - ids: [0, 1041, 1009, 2226, 1003, 1003, 2004, 1033, 1009, 1044, 1009, 1009, 1044, 2001, 1014, 2068, 1009, 1060, 1016, 2], - decoded: " a'll!! to?'d'' d of, can't. ", - }, - PYTHON_CODE: { - text: BASE_TEST_STRINGS.PYTHON_CODE, - tokens: ["def", "main", "(", ")", ":", "pass"], - ids: [0, 13370, 2368, 1010, 1011, 1028, 3417, 2], - decoded: " def main ( ) : pass ", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["let", "a", "=", "ob", "##j", ".", "to", "##st", "##ring", "(", ")", ";", "to", "##st", "##ring", "(", ")", ";"], - ids: [0, 2296, 1041, 1031, 27889, 3505, 1016, 2004, 3371, 4896, 1010, 1011, 1029, 2004, 3371, 4896, 1010, 1011, 1029, 2], - decoded: " let a = obj. tostring ( ) ; tostring ( ) ; ", - }, - NEWLINES: { - text: BASE_TEST_STRINGS.NEWLINES, - tokens: ["this", "is", "a", "test", "."], - ids: [0, 2027, 2007, 1041, 3235, 1016, 2], - decoded: " this is a test. ", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["unwanted", ",", "running"], - ids: [0, 18166, 1014, 2774, 2], - decoded: " unwanted, running ", - }, - CONTROL_TOKENS: { - text: BASE_TEST_STRINGS.CONTROL_TOKENS, - tokens: ["123"], - ids: [0, 13142, 2], - decoded: " 123 ", - }, - HELLO_WORLD_TITLECASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, - tokens: ["hello", "world"], - ids: [0, 7596, 2092, 2], - decoded: " hello world ", - }, - HELLO_WORLD_LOWERCASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, - tokens: ["hello", "world"], - ids: [0, 7596, 2092, 2], - decoded: " hello world ", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u751f", "[UNK]", "\u7684", "\u771f", "[UNK]", "[UNK]"], - ids: [0, 1914, 104, 1920, 1925, 104, 104, 2], - decoded: " \u751f [UNK] \u7684 \u771f [UNK] [UNK] ", - }, - LEADING_SPACE: { - text: BASE_TEST_STRINGS.LEADING_SPACE, - tokens: ["leading", "space"], - ids: [0, 2881, 2690, 2], - decoded: " leading space ", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["trailing", "space"], - ids: [0, 12546, 2690, 2], - decoded: " trailing space ", - }, - DOUBLE_SPACE: { - text: BASE_TEST_STRINGS.DOUBLE_SPACE, - tokens: ["hi", "hello"], - ids: [0, 7636, 7596, 2], - decoded: " hi hello ", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["test", "$", "1", "r", "##2", "#", "3", "\u20ac", "##4", "\u00a35", "\u00a5", "##6", "[UNK]", "\u20b9", "##8", "\u20b1", "##9", "test"], - ids: [0, 3235, 1006, 1019, 1058, 2479, 1005, 1021, 1578, 2553, 27817, 1075, 2579, 104, 1580, 2624, 1579, 2687, 3235, 2], - decoded: " test $ 1 r2 # 3 \u20ac4 \u00a35 \u00a56 [UNK] \u20b98 \u20b19 test ", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["i", "bought", "an", "apple", "for", "$", "1", ".", "00", "at", "the", "store", "."], - ids: [0, 1049, 4153, 2023, 6211, 2009, 1006, 1019, 1016, 4006, 2016, 2000, 3577, 1016, 2], - decoded: " i bought an apple for $ 1. 00 at the store. ", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - tokens: ["you", "\u2026"], - ids: [0, 2021, 1533, 2], - decoded: " you \u2026 ", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - tokens: ["you", "\u2026"], - ids: [0, 2021, 1533, 2], - decoded: " you \u2026 ", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - tokens: ["you", "\u2026", "you", "\u2026"], - ids: [0, 2021, 1533, 2021, 1533, 2], - decoded: " you \u2026 you \u2026 ", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["weird", "\uff5e", "edge", "\uff5e", "case"], - ids: [0, 6885, 1999, 3345, 1999, 2557, 2], - decoded: " weird \uff5e edge \uff5e case ", - }, - SPIECE_UNDERSCORE: { - text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, - tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "."], - ids: [0, 104, 104, 104, 104, 104, 1016, 2], - decoded: " [UNK] [UNK] [UNK] [UNK] [UNK]. ", - }, - POPULAR_EMOJIS: { - text: BASE_TEST_STRINGS.POPULAR_EMOJIS, - tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]"], - ids: [0, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 2], - decoded: " [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] ", - }, - MULTIBYTE_EMOJIS: { - text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, - tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]"], - ids: [0, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 2], - decoded: " [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] ", - }, - }, }; diff --git a/tests/models/nllb/tokenization.js b/tests/models/nllb/tokenization.js index 4cfc6592f..c0eeb5538 100644 --- a/tests/models/nllb/tokenization.js +++ b/tests/models/nllb/tokenization.js @@ -3,156 +3,156 @@ import { BASE_TEST_STRINGS } from "../test_strings.js"; export const TOKENIZER_CLASS = NllbTokenizer; export const TEST_CONFIG = { - "Xenova/nllb-200-distilled-600M": { - SIMPLE: { - text: BASE_TEST_STRINGS.SIMPLE, - tokens: ["\u2581How", "\u2581are", "\u2581you", "\u2581doing", "?"], - ids: [256047, 13374, 2442, 1259, 34512, 248130, 2], - decoded: "eng_Latn How are you doing?", + "Xenova/nllb-200-distilled-600M": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["\u2581How", "\u2581are", "\u2581you", "\u2581doing", "?"], + ids: [256047, 13374, 2442, 1259, 34512, 248130, 2], + decoded: "eng_Latn How are you doing?", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["\u2581You", "\u2581should", "'", "ve", "\u2581done", "\u2581this"], + ids: [256047, 3555, 12516, 248116, 279, 27236, 3423, 2], + decoded: "eng_Latn You should've done this", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["\u25810", "123", "45", "67", "89", "\u25810", "\u25811", "\u25812", "\u25813", "\u25814", "\u25815", "\u25816", "\u25817", "\u25818", "\u25819", "\u258110", "\u2581100", "\u25811000"], + ids: [256047, 4097, 232903, 25497, 37462, 42763, 4097, 94, 140, 315, 436, 481, 617, 757, 799, 855, 772, 3037, 18041, 2], + decoded: "eng_Latn 0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["\u2581The", "\u2581company", "\u2581was", "\u2581found", "ed", "\u2581in", "\u25812016."], + ids: [256047, 1617, 32796, 1398, 26710, 76, 108, 31889, 2], + decoded: "eng_Latn The company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["\u2581A", "\u2581'", "ll", "\u2581!!", "to", "?'", "d", "'", "'", "d", "\u2581of", ",", "\u2581can", "'", "t", "."], + ids: [256047, 70, 238, 1015, 12434, 208, 7358, 248072, 248116, 248116, 248072, 452, 248079, 2125, 248116, 248065, 248075, 2], + decoded: "eng_Latn A 'll!!to?'d''d of, can't.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["\u2581def", "\u2581main", "(", "):", "\u2581pass"], + ids: [256047, 9274, 8385, 248168, 9481, 5800, 2], + decoded: "eng_Latn def main(): pass", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["\u2581let", "\u2581a", "\u2581=", "\u2581ob", "j", ".", "to", "Str", "ing", "(", ");", "\u2581to", "Str", "ing", "(", ");"], + ids: [256047, 3190, 9, 5636, 859, 248086, 248075, 208, 134293, 87, 248168, 12387, 202, 134293, 87, 248168, 12387, 2], + decoded: "eng_Latn let a = obj.toString(); toString();", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["\u2581This", "\u2581is", "\u2581a", "\u2581test", "."], + ids: [256047, 9680, 248, 9, 7356, 248075, 2], + decoded: "eng_Latn This is a test.", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["\u2581UN", "want", "\u00e9d", ",", "run", "ning"], + ids: [256047, 16297, 41691, 11317, 248079, 8464, 888, 2], + decoded: "eng_Latn UNwant\u00e9d,running", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["\u25811", "", "2", "\u25813"], + ids: [256047, 94, 3, 248147, 315, 2], + decoded: "eng_Latn 12 3", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["\u2581Hello", "\u2581World"], + ids: [256047, 94124, 13855, 2], + decoded: "eng_Latn Hello World", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["\u2581hello", "\u2581world"], + ids: [256047, 133863, 15697, 2], + decoded: "eng_Latn hello world", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u2581\u751f\u6d3b", "\u7684", "\u771f", "", "\u662f"], + ids: [256047, 182892, 248506, 249573, 3, 249221, 2], + decoded: "eng_Latn \u751f\u6d3b\u7684\u771f\u662f", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u2581leading", "\u2581space"], + ids: [256047, 151175, 72147, 2], + decoded: "eng_Latn leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["\u2581tra", "iling", "\u2581space", "\u2581"], + ids: [256047, 1372, 21263, 72147, 248059, 2], + decoded: "eng_Latn trailing space ", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["\u2581Hi", "\u2581Hello"], + ids: [256047, 2867, 94124, 2], + decoded: "eng_Latn Hi Hello", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["\u2581test", "\u2581$1", "\u2581R", "2", "\u2581#3", "\u2581\u20ac", "4", "\u2581\u00a3", "5", "\u2581", "\u00a5", "6", "\u2581", "", "7", "\u2581", "\u20b9", "8", "\u2581", "", "9", "\u2581test"], + ids: [256047, 7356, 68462, 250, 248147, 186447, 22935, 248215, 25400, 248210, 248059, 252351, 248262, 248059, 3, 248283, 248059, 254867, 248268, 248059, 3, 248212, 7356, 2], + decoded: "eng_Latn test $1 R2 #3 \u20ac4 \u00a35 \u00a56 7 \u20b98 9 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["\u2581I", "\u2581bought", "\u2581an", "\u2581apple", "\u2581for", "\u2581$", "1.", "00", "\u2581at", "\u2581the", "\u2581store", "."], + ids: [256047, 117, 177233, 111, 203152, 351, 4589, 3044, 460, 230, 349, 21087, 248075, 2], + decoded: "eng_Latn I bought an apple for $1.00 at the store.", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["\u2581you", "...", "\u2581"], + ids: [256047, 1259, 284, 248059, 2], + decoded: "eng_Latn you... ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["\u2581you", "...", "\u2581"], + ids: [256047, 1259, 284, 248059, 2], + decoded: "eng_Latn you... ", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["\u2581you", "...", "\u2581you", "...", "\u2581"], + ids: [256047, 1259, 284, 1259, 284, 248059, 2], + decoded: "eng_Latn you... you... ", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["\u2581weird", "\u2581", "", "\u2581ed", "ge", "\u2581", "", "\u2581case"], + ids: [256047, 197348, 248059, 3, 1074, 479, 248059, 3, 23555, 2], + decoded: "eng_Latn weird edge case", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u2581This", "\u2581is", "\u2581a", "\u2581test", "\u2581."], + ids: [256047, 9680, 248, 9, 7356, 81, 2], + decoded: "eng_Latn This is a test.", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u2581\ud83d\ude02", "\u2581", "\ud83d\udc4d", "\u2581", "\ud83e\udd23", "\u2581\ud83d\ude0d", "\u2581", "\ud83d\ude2d", "\u2581", "", "\u2581", "\ud83d\ude4f", "\u2581", "\ud83d\ude0a", "\u2581", "\ud83d\udd25", "\u2581", "\ud83d\ude01", "\u2581", "\ud83d\ude05", "\u2581", "\ud83e\udd17", "\u2581\ud83d\ude06", "\u2581", "\ud83d\udc4f", "\u2581\u2764", "\ufe0f", "\u2581", "\ud83d\udc9c", "\u2581", "\ud83d\udc9a", "\u2581", "", "\u2581", "\ud83d\udc99", "\u2581", "", "\u2581", "\ud83d\ude0e", "\u2581", "\ud83d\udc4c", "\u2581", "", "\u2581", "\ud83d\udcaa", "\u2581", "\u2728", "\u2581", "\ud83d\udc49", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "\ud83d\ude48", "\u2581", "", "\u2581", "", "\u2581", "\ud83d\udc47", "\u2581", "", "\u2581", "\u2705", "\u2581", "\ud83c\udf81", "\u2581", "", "\u2581", "\ud83c\udf38", "\u2581", ""], + ids: [256047, 104709, 248059, 253416, 248059, 253516, 241830, 248059, 253476, 248059, 3, 248059, 253443, 248059, 253515, 248059, 254402, 248059, 253288, 248059, 253776, 248059, 255232, 147677, 248059, 255420, 82495, 251759, 248059, 255742, 248059, 255949, 248059, 3, 248059, 255649, 248059, 3, 248059, 254297, 248059, 254723, 248059, 3, 248059, 255515, 248059, 254957, 248059, 253985, 248059, 3, 248059, 3, 248059, 3, 248059, 255855, 248059, 3, 248059, 3, 248059, 255354, 248059, 3, 248059, 254268, 248059, 255879, 248059, 3, 248059, 255952, 248059, 3, 2], + decoded: "eng_Latn \ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc99 \ud83d\ude0e \ud83d\udc4c \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\ude48 \ud83d\udc47 \u2705 \ud83c\udf81 \ud83c\udf38 ", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u2581", "\u2728", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\udc41", "\ufe0f", "\u2581", "", "\ud83c\udffb", "\u2581", "", "\u2581", "\u2642", "\ufe0f", "\u2581", "", "\ud83c\udffb", "\u2581", "\u2642", "\u2581", "", "\ud83c\udffb", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581\u2764", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\ud83c\udffb", "\u2581", "", "\u2581", "", "\ud83c\udffb", "\u2581", "", "\u2581", "", "\ud83c\udffb", "\u2581\u2764", "\ufe0f", "\u2581", "", "\u2581", "", "\ud83c\udffc"], + ids: [256047, 248059, 254957, 248059, 255232, 248059, 255123, 251759, 248059, 3, 254422, 248059, 3, 248059, 255331, 251759, 248059, 3, 254422, 248059, 255331, 248059, 3, 254422, 248059, 3, 248059, 3, 248059, 3, 248059, 3, 248059, 3, 82495, 248059, 3, 248059, 3, 248059, 3, 248059, 3, 248059, 3, 248059, 3, 248059, 3, 254422, 248059, 3, 248059, 3, 254422, 248059, 3, 248059, 3, 254422, 82495, 251759, 248059, 3, 248059, 3, 255832, 2], + decoded: "eng_Latn \u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83c\udffb \u2642\ufe0f \ud83c\udffb \u2642 \ud83c\udffb \u2764 \ud83c\udffb \ud83c\udffb \ud83c\udffb \u2764\ufe0f \ud83c\udffc", + }, }, - SIMPLE_WITH_PUNCTUATION: { - text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, - tokens: ["\u2581You", "\u2581should", "'", "ve", "\u2581done", "\u2581this"], - ids: [256047, 3555, 12516, 248116, 279, 27236, 3423, 2], - decoded: "eng_Latn You should've done this", - }, - NUMBERS: { - text: BASE_TEST_STRINGS.NUMBERS, - tokens: ["\u25810", "123", "45", "67", "89", "\u25810", "\u25811", "\u25812", "\u25813", "\u25814", "\u25815", "\u25816", "\u25817", "\u25818", "\u25819", "\u258110", "\u2581100", "\u25811000"], - ids: [256047, 4097, 232903, 25497, 37462, 42763, 4097, 94, 140, 315, 436, 481, 617, 757, 799, 855, 772, 3037, 18041, 2], - decoded: "eng_Latn 0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["\u2581The", "\u2581company", "\u2581was", "\u2581found", "ed", "\u2581in", "\u25812016."], - ids: [256047, 1617, 32796, 1398, 26710, 76, 108, 31889, 2], - decoded: "eng_Latn The company was founded in 2016.", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["\u2581A", "\u2581'", "ll", "\u2581!!", "to", "?'", "d", "'", "'", "d", "\u2581of", ",", "\u2581can", "'", "t", "."], - ids: [256047, 70, 238, 1015, 12434, 208, 7358, 248072, 248116, 248116, 248072, 452, 248079, 2125, 248116, 248065, 248075, 2], - decoded: "eng_Latn A 'll!!to?'d''d of, can't.", - }, - PYTHON_CODE: { - text: BASE_TEST_STRINGS.PYTHON_CODE, - tokens: ["\u2581def", "\u2581main", "(", "):", "\u2581pass"], - ids: [256047, 9274, 8385, 248168, 9481, 5800, 2], - decoded: "eng_Latn def main(): pass", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["\u2581let", "\u2581a", "\u2581=", "\u2581ob", "j", ".", "to", "Str", "ing", "(", ");", "\u2581to", "Str", "ing", "(", ");"], - ids: [256047, 3190, 9, 5636, 859, 248086, 248075, 208, 134293, 87, 248168, 12387, 202, 134293, 87, 248168, 12387, 2], - decoded: "eng_Latn let a = obj.toString(); toString();", - }, - NEWLINES: { - text: BASE_TEST_STRINGS.NEWLINES, - tokens: ["\u2581This", "\u2581is", "\u2581a", "\u2581test", "."], - ids: [256047, 9680, 248, 9, 7356, 248075, 2], - decoded: "eng_Latn This is a test.", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["\u2581UN", "want", "\u00e9d", ",", "run", "ning"], - ids: [256047, 16297, 41691, 11317, 248079, 8464, 888, 2], - decoded: "eng_Latn UNwant\u00e9d,running", - }, - CONTROL_TOKENS: { - text: BASE_TEST_STRINGS.CONTROL_TOKENS, - tokens: ["\u25811", "", "2", "\u25813"], - ids: [256047, 94, 3, 248147, 315, 2], - decoded: "eng_Latn 12 3", - }, - HELLO_WORLD_TITLECASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, - tokens: ["\u2581Hello", "\u2581World"], - ids: [256047, 94124, 13855, 2], - decoded: "eng_Latn Hello World", - }, - HELLO_WORLD_LOWERCASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, - tokens: ["\u2581hello", "\u2581world"], - ids: [256047, 133863, 15697, 2], - decoded: "eng_Latn hello world", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u2581\u751f\u6d3b", "\u7684", "\u771f", "", "\u662f"], - ids: [256047, 182892, 248506, 249573, 3, 249221, 2], - decoded: "eng_Latn \u751f\u6d3b\u7684\u771f\u662f", - }, - LEADING_SPACE: { - text: BASE_TEST_STRINGS.LEADING_SPACE, - tokens: ["\u2581leading", "\u2581space"], - ids: [256047, 151175, 72147, 2], - decoded: "eng_Latn leading space", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["\u2581tra", "iling", "\u2581space", "\u2581"], - ids: [256047, 1372, 21263, 72147, 248059, 2], - decoded: "eng_Latn trailing space ", - }, - DOUBLE_SPACE: { - text: BASE_TEST_STRINGS.DOUBLE_SPACE, - tokens: ["\u2581Hi", "\u2581Hello"], - ids: [256047, 2867, 94124, 2], - decoded: "eng_Latn Hi Hello", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["\u2581test", "\u2581$1", "\u2581R", "2", "\u2581#3", "\u2581\u20ac", "4", "\u2581\u00a3", "5", "\u2581", "\u00a5", "6", "\u2581", "", "7", "\u2581", "\u20b9", "8", "\u2581", "", "9", "\u2581test"], - ids: [256047, 7356, 68462, 250, 248147, 186447, 22935, 248215, 25400, 248210, 248059, 252351, 248262, 248059, 3, 248283, 248059, 254867, 248268, 248059, 3, 248212, 7356, 2], - decoded: "eng_Latn test $1 R2 #3 \u20ac4 \u00a35 \u00a56 7 \u20b98 9 test", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["\u2581I", "\u2581bought", "\u2581an", "\u2581apple", "\u2581for", "\u2581$", "1.", "00", "\u2581at", "\u2581the", "\u2581store", "."], - ids: [256047, 117, 177233, 111, 203152, 351, 4589, 3044, 460, 230, 349, 21087, 248075, 2], - decoded: "eng_Latn I bought an apple for $1.00 at the store.", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - tokens: ["\u2581you", "...", "\u2581"], - ids: [256047, 1259, 284, 248059, 2], - decoded: "eng_Latn you... ", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - tokens: ["\u2581you", "...", "\u2581"], - ids: [256047, 1259, 284, 248059, 2], - decoded: "eng_Latn you... ", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - tokens: ["\u2581you", "...", "\u2581you", "...", "\u2581"], - ids: [256047, 1259, 284, 1259, 284, 248059, 2], - decoded: "eng_Latn you... you... ", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["\u2581weird", "\u2581", "", "\u2581ed", "ge", "\u2581", "", "\u2581case"], - ids: [256047, 197348, 248059, 3, 1074, 479, 248059, 3, 23555, 2], - decoded: "eng_Latn weird edge case", - }, - SPIECE_UNDERSCORE: { - text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, - tokens: ["\u2581This", "\u2581is", "\u2581a", "\u2581test", "\u2581."], - ids: [256047, 9680, 248, 9, 7356, 81, 2], - decoded: "eng_Latn This is a test.", - }, - POPULAR_EMOJIS: { - text: BASE_TEST_STRINGS.POPULAR_EMOJIS, - tokens: ["\u2581\ud83d\ude02", "\u2581", "\ud83d\udc4d", "\u2581", "\ud83e\udd23", "\u2581\ud83d\ude0d", "\u2581", "\ud83d\ude2d", "\u2581", "", "\u2581", "\ud83d\ude4f", "\u2581", "\ud83d\ude0a", "\u2581", "\ud83d\udd25", "\u2581", "\ud83d\ude01", "\u2581", "\ud83d\ude05", "\u2581", "\ud83e\udd17", "\u2581\ud83d\ude06", "\u2581", "\ud83d\udc4f", "\u2581\u2764", "\ufe0f", "\u2581", "\ud83d\udc9c", "\u2581", "\ud83d\udc9a", "\u2581", "", "\u2581", "\ud83d\udc99", "\u2581", "", "\u2581", "\ud83d\ude0e", "\u2581", "\ud83d\udc4c", "\u2581", "", "\u2581", "\ud83d\udcaa", "\u2581", "\u2728", "\u2581", "\ud83d\udc49", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "\ud83d\ude48", "\u2581", "", "\u2581", "", "\u2581", "\ud83d\udc47", "\u2581", "", "\u2581", "\u2705", "\u2581", "\ud83c\udf81", "\u2581", "", "\u2581", "\ud83c\udf38", "\u2581", ""], - ids: [256047, 104709, 248059, 253416, 248059, 253516, 241830, 248059, 253476, 248059, 3, 248059, 253443, 248059, 253515, 248059, 254402, 248059, 253288, 248059, 253776, 248059, 255232, 147677, 248059, 255420, 82495, 251759, 248059, 255742, 248059, 255949, 248059, 3, 248059, 255649, 248059, 3, 248059, 254297, 248059, 254723, 248059, 3, 248059, 255515, 248059, 254957, 248059, 253985, 248059, 3, 248059, 3, 248059, 3, 248059, 255855, 248059, 3, 248059, 3, 248059, 255354, 248059, 3, 248059, 254268, 248059, 255879, 248059, 3, 248059, 255952, 248059, 3, 2], - decoded: "eng_Latn \ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc99 \ud83d\ude0e \ud83d\udc4c \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\ude48 \ud83d\udc47 \u2705 \ud83c\udf81 \ud83c\udf38 ", - }, - MULTIBYTE_EMOJIS: { - text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, - tokens: ["\u2581", "\u2728", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\udc41", "\ufe0f", "\u2581", "", "\ud83c\udffb", "\u2581", "", "\u2581", "\u2642", "\ufe0f", "\u2581", "", "\ud83c\udffb", "\u2581", "\u2642", "\u2581", "", "\ud83c\udffb", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581\u2764", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\ud83c\udffb", "\u2581", "", "\u2581", "", "\ud83c\udffb", "\u2581", "", "\u2581", "", "\ud83c\udffb", "\u2581\u2764", "\ufe0f", "\u2581", "", "\u2581", "", "\ud83c\udffc"], - ids: [256047, 248059, 254957, 248059, 255232, 248059, 255123, 251759, 248059, 3, 254422, 248059, 3, 248059, 255331, 251759, 248059, 3, 254422, 248059, 255331, 248059, 3, 254422, 248059, 3, 248059, 3, 248059, 3, 248059, 3, 248059, 3, 82495, 248059, 3, 248059, 3, 248059, 3, 248059, 3, 248059, 3, 248059, 3, 248059, 3, 254422, 248059, 3, 248059, 3, 254422, 248059, 3, 248059, 3, 254422, 82495, 251759, 248059, 3, 248059, 3, 255832, 2], - decoded: "eng_Latn \u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83c\udffb \u2642\ufe0f \ud83c\udffb \u2642 \ud83c\udffb \u2764 \ud83c\udffb \ud83c\udffb \ud83c\udffb \u2764\ufe0f \ud83c\udffc", - }, - }, }; diff --git a/tests/models/qwen2/tokenization.js b/tests/models/qwen2/tokenization.js index 4bcdeaeed..ddf23f0ae 100644 --- a/tests/models/qwen2/tokenization.js +++ b/tests/models/qwen2/tokenization.js @@ -3,156 +3,156 @@ import { BASE_TEST_STRINGS } from "../test_strings.js"; export const TOKENIZER_CLASS = Qwen2Tokenizer; export const TEST_CONFIG = { - "Xenova/Qwen1.5-0.5B-Chat": { - SIMPLE: { - text: BASE_TEST_STRINGS.SIMPLE, - tokens: ["How", "\u0120are", "\u0120you", "\u0120doing", "?"], - ids: [4340, 525, 498, 3730, 30], - decoded: "How are you doing?", + "Xenova/Qwen1.5-0.5B-Chat": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["How", "\u0120are", "\u0120you", "\u0120doing", "?"], + ids: [4340, 525, 498, 3730, 30], + decoded: "How are you doing?", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["You", "\u0120should", "'ve", "\u0120done", "\u0120this"], + ids: [2610, 1265, 3003, 2814, 419], + decoded: "You should've done this", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "\u0120", "0", "\u0120", "1", "\u0120", "2", "\u0120", "3", "\u0120", "4", "\u0120", "5", "\u0120", "6", "\u0120", "7", "\u0120", "8", "\u0120", "9", "\u0120", "1", "0", "\u0120", "1", "0", "0", "\u0120", "1", "0", "0", "0"], + ids: [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 220, 15, 220, 16, 220, 17, 220, 18, 220, 19, 220, 20, 220, 21, 220, 22, 220, 23, 220, 24, 220, 16, 15, 220, 16, 15, 15, 220, 16, 15, 15, 15], + decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "\u0120company", "\u0120was", "\u0120founded", "\u0120in", "\u0120", "2", "0", "1", "6", "."], + ids: [785, 2813, 572, 18047, 304, 220, 17, 15, 16, 21, 13], + decoded: "The company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "\u010a", "'ll", "\u0120!!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'t", "."], + ids: [32, 198, 3278, 11015, 983, 20224, 67, 4605, 67, 315, 11, 646, 944, 13], + decoded: "A\n'll !!to?'d''d of, can't.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["def", "\u0120main", "():\u010a", "\u0109pass"], + ids: [750, 1887, 3932, 41431], + decoded: "def main():\n\tpass", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".toString", "();\u010a", "toString", "();"], + ids: [1149, 264, 284, 2839, 5070, 543, 6575, 2129], + decoded: "let a = obj.toString();\ntoString();", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["This", "\u010a\u010a", "is", "\u010a", "a", "\u010a", "test", "."], + ids: [1986, 271, 285, 198, 64, 198, 1944, 13], + decoded: "This\n\nis\na\ntest.", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "want", "\u00c3\u00a9d", ",", "running"], + ids: [1861, 52657, 15083, 11, 27173], + decoded: "UNwant\u00e9d,running", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["1", "\u0100", "2", "\u00ef\u00bf\u00bd", "3"], + ids: [16, 188, 17, 5691, 18], + decoded: "1\u00002\ufffd3", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["Hello", "\u0120World"], + ids: [9707, 4337], + decoded: "Hello World", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["hello", "\u0120world"], + ids: [14990, 1879], + decoded: "hello world", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u00e7\u0136\u0141\u00e6\u00b4\u00bb\u00e7\u013c\u0126", "\u00e7\u013e\u0141", "\u00e8\u00b0\u013d", "\u00e6\u013a\u00af"], + ids: [105301, 88051, 116109, 20412], + decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u0120\u0120", "\u0120leading", "\u0120space"], + ids: [256, 6388, 3550], + decoded: " leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["tr", "ailing", "\u0120space", "\u0120\u0120\u0120"], + ids: [376, 14277, 3550, 262], + decoded: "trailing space ", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["Hi", "\u0120", "\u0120Hello"], + ids: [13048, 220, 21927], + decoded: "Hi Hello", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2\u0124\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120test"], + ids: [1944, 400, 16, 431, 17, 671, 18, 12984, 19, 6938, 20, 71488, 21, 2858, 224, 96, 22, 89791, 23, 2858, 224, 109, 24, 1273], + decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "\u0120bought", "\u0120an", "\u0120apple", "\u0120for", "\u0120$", "1", ".", "0", "0", "\u0120at", "\u0120the", "\u0120store", "."], + ids: [40, 10788, 458, 23268, 369, 400, 16, 13, 15, 15, 518, 279, 3553, 13], + decoded: "I bought an apple for $1.00 at the store.", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u0120\u0120"], + ids: [9330, 1940, 256], + decoded: "you\u2026 ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], + ids: [9330, 1940, 9238], + decoded: "you\u2026\u00a0\u00a0", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142", "\u00c2\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], + ids: [9330, 1940, 4102, 4102, 9330, 1940, 9238], + decoded: "you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["we", "ird", "\u0120", "\u00ef\u00bd\u0140", "\u0120edge", "\u0120", "\u00ef\u00bd\u0140", "\u0120case"], + ids: [896, 2603, 220, 21216, 6821, 220, 21216, 1142], + decoded: "weird \uff5e edge \uff5e case", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u00e2\u0138", "\u0123", "This", "\u0120\u00e2\u0138", "\u0123", "is", "\u0120\u00e2\u0138", "\u0123", "a", "\u0120\u00e2\u0138", "\u0123", "test", "\u0120\u00e2\u0138", "\u0123", "."], + ids: [10417, 223, 1986, 14520, 223, 285, 14520, 223, 64, 14520, 223, 1944, 14520, 223, 13], + decoded: "\u2581This \u2581is \u2581a \u2581test \u2581.", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u00f0\u0141\u013a\u0124", "\u0120\u00f0\u0141\u0133", "\u012f", "\u0120\u00f0\u0141", "\u00a4", "\u00a3", "\u0120\u00f0\u0141\u013a", "\u012f", "\u0120\u00f0\u0141\u013a", "\u0143", "\u0120\u00f0\u0141", "\u0130", "\u012b", "\u0120\u00f0\u0141", "\u013b", "\u0131", "\u0120\u00f0\u0141\u013a", "\u012c", "\u0120\u00f0\u0141\u0136", "\u00a5", "\u0120\u00f0\u0141\u013a", "\u0123", "\u0120\u00f0\u0141\u013a", "\u0127", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141\u013a", "\u0128", "\u0120\u00f0\u0141\u0133", "\u0131", "\u0120\u00e2\u013f\u00a4", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141\u0134", "\u013e", "\u0120\u00f0\u0141\u0134", "\u013c", "\u0120\u00f0\u0141\u0134", "\u0139", "\u0120\u00f0\u0141\u0134", "\u013b", "\u0120\u00f0\u0141", "\u0138", "\u00a4", "\u0120\u00f0\u0141\u013a", "\u0130", "\u0120\u00f0\u0141\u0133", "\u012e", "\u0120\u00f0\u0141", "\u00a5", "\u00b3", "\u0120\u00f0\u0141\u0134", "\u00aa", "\u0120\u00e2\u013e", "\u00a8", "\u0120\u00f0\u0141\u0133", "\u012b", "\u0120\u00f0\u0141\u0133", "\u0122", "\u0120\u00f0\u0141\u0134", "\u00af", "\u0120\u00f0\u0141", "\u0130", "\u012a", "\u0120\u00f0\u0141", "\u013b", "\u012a", "\u0120\u00f0\u0141", "\u013b", "\u012e", "\u0120\u00f0\u0141\u0134", "\u0122", "\u0120\u00f0\u0141\u0133", "\u0129", "\u0120\u00f0\u0141\u0133", "\u012d", "\u0120\u00e2\u013e", "\u0127", "\u0120\u00f0\u0141", "\u0130", "\u0123", "\u0120\u00f0\u0141", "\u012e", "\u0140", "\u0120\u00f0\u0141", "\u012e", "\u00b8", "\u0120\u00f0\u0141\u0134", "\u00b0"], + ids: [144185, 61804, 235, 11162, 97, 96, 26525, 235, 26525, 255, 11162, 236, 231, 11162, 247, 237, 26525, 232, 95069, 98, 26525, 223, 26525, 227, 11162, 97, 245, 26525, 228, 61804, 237, 70470, 30543, 63039, 250, 63039, 248, 63039, 245, 63039, 247, 11162, 244, 97, 26525, 236, 61804, 234, 11162, 98, 111, 63039, 103, 25521, 101, 61804, 231, 61804, 222, 63039, 107, 11162, 236, 230, 11162, 247, 230, 11162, 247, 234, 63039, 222, 61804, 229, 61804, 233, 25521, 227, 11162, 236, 223, 11162, 234, 252, 11162, 234, 116, 63039, 108], + decoded: "\ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u00e2\u013e\u00a8", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141\u0133", "\u0123", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141\u0133", "\u00b1", "\u00f0\u0141\u0131\u00bb", "\u0120\u00f0\u0141", "\u0137", "\u00b5", "\u00e2\u0122", "\u012f", "\u00e2\u013b\u0124", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141", "\u00a7", "\u013b", "\u00f0\u0141\u0131\u00bb", "\u00e2\u0122", "\u012f", "\u00e2\u013b\u0124", "\u0120\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141\u0131\u00bb", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u012e\u00be", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u00a4\u013f", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u00a7\u0133", "\u0120\u00f0\u0141\u0133", "\u00a9", "\u00e2\u0122", "\u012f", "\u00e2\u013f\u00a4", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u0134\u012d", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u0133\u00a8", "\u0120\u00f0\u0141\u0133", "\u00a9", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u0133\u00a9", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u0133\u00a7", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u0133\u00a6", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141\u0131\u00bb", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u00a4\u013f", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u00a7\u0133", "\u00f0\u0141\u0131\u00bb", "\u0120\u00f0\u0141", "\u0131", "\u00b4", "\u00f3", "\u0142\u0123", "\u00a7", "\u00f3", "\u0142\u0123", "\u00a2", "\u00f3", "\u0142\u0123", "\u00a5", "\u00f3", "\u0142\u0123", "\u00ae", "\u00f3", "\u0142\u0123", "\u00a7", "\u00f3", "\u0142\u0123", "\u00bf", "\u0120\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141\u0131\u00bb", "\u00e2\u0122", "\u012f", "\u00e2\u013f\u00a4", "\u00ef\u00b8\u0131", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u0134\u012d", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u0133\u00a8", "\u00f0\u0141\u0131\u00bc"], + ids: [144232, 11162, 97, 245, 61804, 223, 30543, 61804, 109, 144321, 11162, 243, 113, 378, 235, 144693, 30543, 11162, 100, 247, 144321, 378, 235, 144693, 61804, 101, 144321, 378, 235, 146467, 11162, 100, 239, 378, 235, 146392, 378, 235, 148738, 61804, 102, 378, 235, 141390, 378, 235, 145002, 378, 235, 145367, 61804, 102, 378, 235, 145233, 378, 235, 145665, 378, 235, 145988, 11162, 100, 239, 144321, 378, 235, 146392, 378, 235, 148738, 144321, 11162, 237, 112, 175, 15675, 100, 175, 15675, 95, 175, 15675, 98, 175, 15675, 106, 175, 15675, 100, 175, 15675, 123, 61804, 101, 144321, 378, 235, 141390, 30543, 378, 235, 145002, 378, 235, 145367, 144784], + decoded: "\u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75\u200d\u2642\ufe0f \ud83e\uddd9\ud83c\udffb\u200d\u2642 \ud83d\udc68\ud83c\udffb\u200d\ud83c\udf3e \ud83e\uddd1\u200d\ud83e\udd1d\u200d\ud83e\uddd1 \ud83d\udc69\u200d\u2764\u200d\ud83d\udc8b\u200d\ud83d\udc68 \ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66 \ud83e\uddd1\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c\udffb \ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f \ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c\udffc", + }, }, - SIMPLE_WITH_PUNCTUATION: { - text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, - tokens: ["You", "\u0120should", "'ve", "\u0120done", "\u0120this"], - ids: [2610, 1265, 3003, 2814, 419], - decoded: "You should've done this", - }, - NUMBERS: { - text: BASE_TEST_STRINGS.NUMBERS, - tokens: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "\u0120", "0", "\u0120", "1", "\u0120", "2", "\u0120", "3", "\u0120", "4", "\u0120", "5", "\u0120", "6", "\u0120", "7", "\u0120", "8", "\u0120", "9", "\u0120", "1", "0", "\u0120", "1", "0", "0", "\u0120", "1", "0", "0", "0"], - ids: [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 220, 15, 220, 16, 220, 17, 220, 18, 220, 19, 220, 20, 220, 21, 220, 22, 220, 23, 220, 24, 220, 16, 15, 220, 16, 15, 15, 220, 16, 15, 15, 15], - decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["The", "\u0120company", "\u0120was", "\u0120founded", "\u0120in", "\u0120", "2", "0", "1", "6", "."], - ids: [785, 2813, 572, 18047, 304, 220, 17, 15, 16, 21, 13], - decoded: "The company was founded in 2016.", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["A", "\u010a", "'ll", "\u0120!!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'t", "."], - ids: [32, 198, 3278, 11015, 983, 20224, 67, 4605, 67, 315, 11, 646, 944, 13], - decoded: "A\n'll !!to?'d''d of, can't.", - }, - PYTHON_CODE: { - text: BASE_TEST_STRINGS.PYTHON_CODE, - tokens: ["def", "\u0120main", "():\u010a", "\u0109pass"], - ids: [750, 1887, 3932, 41431], - decoded: "def main():\n\tpass", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".toString", "();\u010a", "toString", "();"], - ids: [1149, 264, 284, 2839, 5070, 543, 6575, 2129], - decoded: "let a = obj.toString();\ntoString();", - }, - NEWLINES: { - text: BASE_TEST_STRINGS.NEWLINES, - tokens: ["This", "\u010a\u010a", "is", "\u010a", "a", "\u010a", "test", "."], - ids: [1986, 271, 285, 198, 64, 198, 1944, 13], - decoded: "This\n\nis\na\ntest.", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["UN", "want", "\u00c3\u00a9d", ",", "running"], - ids: [1861, 52657, 15083, 11, 27173], - decoded: "UNwant\u00e9d,running", - }, - CONTROL_TOKENS: { - text: BASE_TEST_STRINGS.CONTROL_TOKENS, - tokens: ["1", "\u0100", "2", "\u00ef\u00bf\u00bd", "3"], - ids: [16, 188, 17, 5691, 18], - decoded: "1\u00002\ufffd3", - }, - HELLO_WORLD_TITLECASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, - tokens: ["Hello", "\u0120World"], - ids: [9707, 4337], - decoded: "Hello World", - }, - HELLO_WORLD_LOWERCASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, - tokens: ["hello", "\u0120world"], - ids: [14990, 1879], - decoded: "hello world", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u00e7\u0136\u0141\u00e6\u00b4\u00bb\u00e7\u013c\u0126", "\u00e7\u013e\u0141", "\u00e8\u00b0\u013d", "\u00e6\u013a\u00af"], - ids: [105301, 88051, 116109, 20412], - decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", - }, - LEADING_SPACE: { - text: BASE_TEST_STRINGS.LEADING_SPACE, - tokens: ["\u0120\u0120", "\u0120leading", "\u0120space"], - ids: [256, 6388, 3550], - decoded: " leading space", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["tr", "ailing", "\u0120space", "\u0120\u0120\u0120"], - ids: [376, 14277, 3550, 262], - decoded: "trailing space ", - }, - DOUBLE_SPACE: { - text: BASE_TEST_STRINGS.DOUBLE_SPACE, - tokens: ["Hi", "\u0120", "\u0120Hello"], - ids: [13048, 220, 21927], - decoded: "Hi Hello", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2\u0124\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120test"], - ids: [1944, 400, 16, 431, 17, 671, 18, 12984, 19, 6938, 20, 71488, 21, 2858, 224, 96, 22, 89791, 23, 2858, 224, 109, 24, 1273], - decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["I", "\u0120bought", "\u0120an", "\u0120apple", "\u0120for", "\u0120$", "1", ".", "0", "0", "\u0120at", "\u0120the", "\u0120store", "."], - ids: [40, 10788, 458, 23268, 369, 400, 16, 13, 15, 15, 518, 279, 3553, 13], - decoded: "I bought an apple for $1.00 at the store.", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - tokens: ["you", "\u00e2\u0122\u00a6", "\u0120\u0120"], - ids: [9330, 1940, 256], - decoded: "you\u2026 ", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], - ids: [9330, 1940, 9238], - decoded: "you\u2026\u00a0\u00a0", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142", "\u00c2\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], - ids: [9330, 1940, 4102, 4102, 9330, 1940, 9238], - decoded: "you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["we", "ird", "\u0120", "\u00ef\u00bd\u0140", "\u0120edge", "\u0120", "\u00ef\u00bd\u0140", "\u0120case"], - ids: [896, 2603, 220, 21216, 6821, 220, 21216, 1142], - decoded: "weird \uff5e edge \uff5e case", - }, - SPIECE_UNDERSCORE: { - text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, - tokens: ["\u00e2\u0138", "\u0123", "This", "\u0120\u00e2\u0138", "\u0123", "is", "\u0120\u00e2\u0138", "\u0123", "a", "\u0120\u00e2\u0138", "\u0123", "test", "\u0120\u00e2\u0138", "\u0123", "."], - ids: [10417, 223, 1986, 14520, 223, 285, 14520, 223, 64, 14520, 223, 1944, 14520, 223, 13], - decoded: "\u2581This \u2581is \u2581a \u2581test \u2581.", - }, - POPULAR_EMOJIS: { - text: BASE_TEST_STRINGS.POPULAR_EMOJIS, - tokens: ["\u00f0\u0141\u013a\u0124", "\u0120\u00f0\u0141\u0133", "\u012f", "\u0120\u00f0\u0141", "\u00a4", "\u00a3", "\u0120\u00f0\u0141\u013a", "\u012f", "\u0120\u00f0\u0141\u013a", "\u0143", "\u0120\u00f0\u0141", "\u0130", "\u012b", "\u0120\u00f0\u0141", "\u013b", "\u0131", "\u0120\u00f0\u0141\u013a", "\u012c", "\u0120\u00f0\u0141\u0136", "\u00a5", "\u0120\u00f0\u0141\u013a", "\u0123", "\u0120\u00f0\u0141\u013a", "\u0127", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141\u013a", "\u0128", "\u0120\u00f0\u0141\u0133", "\u0131", "\u0120\u00e2\u013f\u00a4", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141\u0134", "\u013e", "\u0120\u00f0\u0141\u0134", "\u013c", "\u0120\u00f0\u0141\u0134", "\u0139", "\u0120\u00f0\u0141\u0134", "\u013b", "\u0120\u00f0\u0141", "\u0138", "\u00a4", "\u0120\u00f0\u0141\u013a", "\u0130", "\u0120\u00f0\u0141\u0133", "\u012e", "\u0120\u00f0\u0141", "\u00a5", "\u00b3", "\u0120\u00f0\u0141\u0134", "\u00aa", "\u0120\u00e2\u013e", "\u00a8", "\u0120\u00f0\u0141\u0133", "\u012b", "\u0120\u00f0\u0141\u0133", "\u0122", "\u0120\u00f0\u0141\u0134", "\u00af", "\u0120\u00f0\u0141", "\u0130", "\u012a", "\u0120\u00f0\u0141", "\u013b", "\u012a", "\u0120\u00f0\u0141", "\u013b", "\u012e", "\u0120\u00f0\u0141\u0134", "\u0122", "\u0120\u00f0\u0141\u0133", "\u0129", "\u0120\u00f0\u0141\u0133", "\u012d", "\u0120\u00e2\u013e", "\u0127", "\u0120\u00f0\u0141", "\u0130", "\u0123", "\u0120\u00f0\u0141", "\u012e", "\u0140", "\u0120\u00f0\u0141", "\u012e", "\u00b8", "\u0120\u00f0\u0141\u0134", "\u00b0"], - ids: [144185, 61804, 235, 11162, 97, 96, 26525, 235, 26525, 255, 11162, 236, 231, 11162, 247, 237, 26525, 232, 95069, 98, 26525, 223, 26525, 227, 11162, 97, 245, 26525, 228, 61804, 237, 70470, 30543, 63039, 250, 63039, 248, 63039, 245, 63039, 247, 11162, 244, 97, 26525, 236, 61804, 234, 11162, 98, 111, 63039, 103, 25521, 101, 61804, 231, 61804, 222, 63039, 107, 11162, 236, 230, 11162, 247, 230, 11162, 247, 234, 63039, 222, 61804, 229, 61804, 233, 25521, 227, 11162, 236, 223, 11162, 234, 252, 11162, 234, 116, 63039, 108], - decoded: "\ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0", - }, - MULTIBYTE_EMOJIS: { - text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, - tokens: ["\u00e2\u013e\u00a8", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141\u0133", "\u0123", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141\u0133", "\u00b1", "\u00f0\u0141\u0131\u00bb", "\u0120\u00f0\u0141", "\u0137", "\u00b5", "\u00e2\u0122", "\u012f", "\u00e2\u013b\u0124", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141", "\u00a7", "\u013b", "\u00f0\u0141\u0131\u00bb", "\u00e2\u0122", "\u012f", "\u00e2\u013b\u0124", "\u0120\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141\u0131\u00bb", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u012e\u00be", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u00a4\u013f", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u00a7\u0133", "\u0120\u00f0\u0141\u0133", "\u00a9", "\u00e2\u0122", "\u012f", "\u00e2\u013f\u00a4", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u0134\u012d", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u0133\u00a8", "\u0120\u00f0\u0141\u0133", "\u00a9", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u0133\u00a9", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u0133\u00a7", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u0133\u00a6", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141\u0131\u00bb", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u00a4\u013f", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u00a7\u0133", "\u00f0\u0141\u0131\u00bb", "\u0120\u00f0\u0141", "\u0131", "\u00b4", "\u00f3", "\u0142\u0123", "\u00a7", "\u00f3", "\u0142\u0123", "\u00a2", "\u00f3", "\u0142\u0123", "\u00a5", "\u00f3", "\u0142\u0123", "\u00ae", "\u00f3", "\u0142\u0123", "\u00a7", "\u00f3", "\u0142\u0123", "\u00bf", "\u0120\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141\u0131\u00bb", "\u00e2\u0122", "\u012f", "\u00e2\u013f\u00a4", "\u00ef\u00b8\u0131", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u0134\u012d", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u0133\u00a8", "\u00f0\u0141\u0131\u00bc"], - ids: [144232, 11162, 97, 245, 61804, 223, 30543, 61804, 109, 144321, 11162, 243, 113, 378, 235, 144693, 30543, 11162, 100, 247, 144321, 378, 235, 144693, 61804, 101, 144321, 378, 235, 146467, 11162, 100, 239, 378, 235, 146392, 378, 235, 148738, 61804, 102, 378, 235, 141390, 378, 235, 145002, 378, 235, 145367, 61804, 102, 378, 235, 145233, 378, 235, 145665, 378, 235, 145988, 11162, 100, 239, 144321, 378, 235, 146392, 378, 235, 148738, 144321, 11162, 237, 112, 175, 15675, 100, 175, 15675, 95, 175, 15675, 98, 175, 15675, 106, 175, 15675, 100, 175, 15675, 123, 61804, 101, 144321, 378, 235, 141390, 30543, 378, 235, 145002, 378, 235, 145367, 144784], - decoded: "\u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75\u200d\u2642\ufe0f \ud83e\uddd9\ud83c\udffb\u200d\u2642 \ud83d\udc68\ud83c\udffb\u200d\ud83c\udf3e \ud83e\uddd1\u200d\ud83e\udd1d\u200d\ud83e\uddd1 \ud83d\udc69\u200d\u2764\u200d\ud83d\udc8b\u200d\ud83d\udc68 \ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66 \ud83e\uddd1\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c\udffb \ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f \ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c\udffc", - }, - }, }; diff --git a/tests/models/roberta/tokenization.js b/tests/models/roberta/tokenization.js index 458430878..f6cb0a1d8 100644 --- a/tests/models/roberta/tokenization.js +++ b/tests/models/roberta/tokenization.js @@ -3,702 +3,702 @@ import { BASE_TEST_STRINGS, BERT_TEST_STRINGS } from "../test_strings.js"; export const TOKENIZER_CLASS = RobertaTokenizer; export const TEST_CONFIG = { - "jinaai/jina-embeddings-v2-base-de": { - SIMPLE: { - text: BASE_TEST_STRINGS.SIMPLE, - tokens: ["How", "\u0120are", "\u0120you", "\u0120doing", "?"], - ids: [0, 3267, 459, 426, 3174, 35, 2], - decoded: "How are you doing?", - }, - SIMPLE_WITH_PUNCTUATION: { - text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, - tokens: ["You", "\u0120should", "'ve", "\u0120done", "\u0120this"], - ids: [0, 2606, 1303, 1990, 3022, 555, 2], - decoded: "You should've done this", - }, - NUMBERS: { - text: BASE_TEST_STRINGS.NUMBERS, - tokens: ["0", "123", "456", "789", "\u01200", "\u01201", "\u01202", "\u01203", "\u01204", "\u01205", "\u01206", "\u01207", "\u01208", "\u01209", "\u012010", "\u0120100", "\u01201000"], - ids: [0, 20, 21911, 40271, 51355, 885, 387, 381, 589, 699, 703, 866, 964, 991, 1045, 949, 1873, 8611, 2], - decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["The", "\u0120company", "\u0120was", "\u0120founded", "\u0120in", "\u01202016", "."], - ids: [0, 710, 1891, 503, 15604, 295, 2262, 18, 2], - decoded: "The company was founded in 2016.", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["A", "\u010a", "'ll", "\u0120!!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'t", "."], - ids: [0, 37, 203, 2202, 26143, 764, 30080, 72, 12228, 72, 314, 16, 571, 797, 18, 2], - decoded: "A\n'll!!to?'d''d of, can't.", - }, - PYTHON_CODE: { - text: BASE_TEST_STRINGS.PYTHON_CODE, - tokens: ["def", "\u0120main", "(", "):", "\u010a", "\u0109", "pass"], - ids: [0, 28273, 1911, 12, 4025, 203, 202, 5517, 2], - decoded: "def main():\n\tpass", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".", "to", "String", "();", "\u010a", "to", "String", "();"], - ids: [0, 1642, 264, 3887, 8273, 18, 764, 53889, 54181, 203, 764, 53889, 54181, 2], - decoded: "let a = obj.toString();\ntoString();", - }, - NEWLINES: { - text: BASE_TEST_STRINGS.NEWLINES, - tokens: ["This", "\u010a", "\u010a", "is", "\u010a", "a", "\u010a", "test", "."], - ids: [0, 1803, 203, 203, 276, 203, 69, 203, 4451, 18, 2], - decoded: "This\n\nis\na\ntest.", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["UN", "w", "ant", "\u00c3\u00a9d", ",", "running"], - ids: [0, 3854, 91, 526, 46298, 16, 47232, 2], - decoded: "UNwant\u00e9d,running", - }, - CONTROL_TOKENS: { - text: BASE_TEST_STRINGS.CONTROL_TOKENS, - tokens: ["1", "\u0100", "2", "\u00ef\u00bf\u00bd", "3"], - ids: [0, 21, 193, 22, 1998, 23, 2], - decoded: "1\u00002\ufffd3", - }, - HELLO_WORLD_TITLECASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, - tokens: ["Hello", "\u0120World"], - ids: [0, 17964, 3519, 2], - decoded: "Hello World", - }, - HELLO_WORLD_LOWERCASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, - tokens: ["hell", "o", "\u0120world"], - ids: [0, 17067, 83, 1568, 2], - decoded: "hello world", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u00e7\u0136", "\u0141", "\u00e6", "\u00b4", "\u00bb", "\u00e7\u013c\u0126", "\u00e7", "\u013e", "\u0141", "\u00e8", "\u00b0", "\u013d", "\u00e6", "\u013a", "\u00af"], - ids: [0, 55225, 258, 167, 117, 124, 44574, 168, 255, 258, 169, 113, 254, 167, 251, 112, 2], - decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", - }, - LEADING_SPACE: { - text: BASE_TEST_STRINGS.LEADING_SPACE, - tokens: ["\u0120\u0120", "\u0120leading", "\u0120space"], - ids: [0, 6733, 5344, 3435, 2], - decoded: " leading space", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["tra", "iling", "\u0120space", "\u0120\u0120\u0120"], - ids: [0, 766, 7462, 3435, 53448, 2], - decoded: "trailing space ", - }, - DOUBLE_SPACE: { - text: BASE_TEST_STRINGS.DOUBLE_SPACE, - tokens: ["Hi", "\u0120", "\u0120Hello"], - ids: [0, 10103, 225, 29546, 2], - decoded: "Hi Hello", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2", "\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2\u0124\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120test"], - ids: [0, 4451, 1350, 21, 366, 22, 1805, 23, 2712, 24, 6339, 25, 960, 103, 26, 1581, 229, 101, 27, 58720, 28, 1581, 229, 114, 29, 2839, 2], - decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["I", "\u0120bought", "\u0120an", "\u0120apple", "\u0120for", "\u0120$", "1", ".", "00", "\u0120at", "\u0120the", "\u0120store", "."], - ids: [0, 45, 8928, 371, 19798, 382, 1350, 21, 18, 505, 495, 285, 4569, 18, 2], - decoded: "I bought an apple for $1.00 at the store.", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - tokens: ["you", "\u00e2\u0122\u00a6", "\u0120\u0120"], - ids: [0, 10695, 1179, 6733, 2], - decoded: "you\u2026 ", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], - ids: [0, 10695, 1179, 44105, 2], - decoded: "you\u2026\u00a0\u00a0", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142", "\u00c2\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], - ids: [0, 10695, 1179, 15529, 15529, 10695, 1179, 44105, 2], - decoded: "you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["we", "ird", "\u0120\u00ef", "\u00bd", "\u0140", "\u0120edge", "\u0120\u00ef", "\u00bd", "\u0140", "\u0120case"], - ids: [0, 487, 2394, 17740, 126, 257, 9911, 17740, 126, 257, 2600, 2], - decoded: "weird \uff5e edge \uff5e case", - }, - SPIECE_UNDERSCORE: { - text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, - tokens: ["\u00e2\u0138", "\u0123", "This", "\u0120\u00e2\u0138", "\u0123", "is", "\u0120\u00e2\u0138", "\u0123", "a", "\u0120\u00e2\u0138", "\u0123", "test", "\u0120\u00e2\u0138", "\u0123", "."], - ids: [0, 12790, 228, 1803, 20068, 228, 276, 20068, 228, 69, 20068, 228, 4451, 20068, 228, 18, 2], - decoded: "\u2581This \u2581is \u2581a \u2581test \u2581.", - }, - POPULAR_EMOJIS: { - text: BASE_TEST_STRINGS.POPULAR_EMOJIS, - tokens: ["\u00f0\u0141\u013a", "\u0124", "\u0120\u00f0\u0141\u0133", "\u012f", "\u0120\u00f0\u0141", "\u00a4", "\u00a3", "\u0120\u00f0\u0141\u013a", "\u012f", "\u0120\u00f0\u0141\u013a", "\u0143", "\u0120\u00f0\u0141", "\u0130", "\u012b", "\u0120\u00f0\u0141\u013b", "\u0131", "\u0120\u00f0\u0141\u013a", "\u012c", "\u0120\u00f0\u0141", "\u0136", "\u00a5", "\u0120\u00f0\u0141\u013a", "\u0123", "\u0120\u00f0\u0141\u013a", "\u0127", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141\u013a", "\u0128", "\u0120\u00f0\u0141\u0133", "\u0131", "\u0120\u00e2\u013f\u00a4", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141\u0134", "\u013e", "\u0120\u00f0\u0141\u0134", "\u013c", "\u0120\u00f0\u0141\u0134", "\u0139", "\u0120\u00f0\u0141\u0134", "\u013b", "\u0120\u00f0\u0141", "\u0138", "\u00a4", "\u0120\u00f0\u0141\u013a", "\u0130", "\u0120\u00f0\u0141\u0133", "\u012e", "\u0120\u00f0\u0141", "\u00a5", "\u00b3", "\u0120\u00f0\u0141\u0134", "\u00aa", "\u0120\u00e2\u013e", "\u00a8", "\u0120\u00f0\u0141\u0133", "\u012b", "\u0120\u00f0\u0141\u0133", "\u0122", "\u0120\u00f0\u0141\u0134", "\u00af", "\u0120\u00f0\u0141", "\u0130", "\u012a", "\u0120\u00f0\u0141\u013b", "\u012a", "\u0120\u00f0\u0141\u013b", "\u012e", "\u0120\u00f0\u0141\u0134", "\u0122", "\u0120\u00f0\u0141\u0133", "\u0129", "\u0120\u00f0\u0141\u0133", "\u012d", "\u0120\u00e2\u013e\u0127", "\u0120\u00f0\u0141", "\u0130", "\u0123", "\u0120\u00f0\u0141", "\u012e", "\u0140", "\u0120\u00f0\u0141", "\u012e", "\u00b8", "\u0120\u00f0\u0141\u0134", "\u00b0"], - ids: [0, 32164, 229, 49904, 240, 5060, 102, 101, 10278, 240, 10278, 260, 5060, 241, 236, 10319, 242, 10278, 237, 5060, 247, 103, 10278, 228, 10278, 232, 5060, 102, 250, 10278, 233, 49904, 242, 42009, 16598, 52278, 255, 52278, 253, 52278, 250, 52278, 252, 5060, 249, 102, 10278, 241, 49904, 239, 5060, 103, 116, 52278, 108, 10792, 106, 49904, 236, 49904, 227, 52278, 112, 5060, 241, 235, 10319, 235, 10319, 239, 52278, 227, 49904, 234, 49904, 238, 38607, 5060, 241, 228, 5060, 239, 257, 5060, 239, 121, 52278, 113, 2], - decoded: "\ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0", - }, - MULTIBYTE_EMOJIS: { - text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, - tokens: ["\u00e2\u013e", "\u00a8", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141\u0133", "\u0123", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141\u0133", "\u00b1", "\u00f0\u0141\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0137", "\u00b5", "\u00e2\u0122\u012f", "\u00e2\u013b", "\u0124", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141", "\u00a7", "\u013b", "\u00f0\u0141\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00e2\u013b", "\u0124", "\u0120\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u012e", "\u00be", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u0120\u00f0\u0141\u0133", "\u00a9", "\u00e2\u0122\u012f", "\u00e2\u013f\u00a4", "\u00e2\u0122\u012f", "\u00f0\u0141\u0134", "\u012d", "\u00e2\u0122\u012f", "\u00f0\u0141\u0133", "\u00a8", "\u0120\u00f0\u0141\u0133", "\u00a9", "\u00e2\u0122\u012f", "\u00f0\u0141\u0133", "\u00a9", "\u00e2\u0122\u012f", "\u00f0\u0141\u0133", "\u00a7", "\u00e2\u0122\u012f", "\u00f0\u0141\u0133", "\u00a6", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0131", "\u00b4", "\u00f3", "\u0142", "\u0123", "\u00a7", "\u00f3", "\u0142", "\u0123", "\u00a2", "\u00f3", "\u0142", "\u0123", "\u00a5", "\u00f3", "\u0142", "\u0123", "\u00ae", "\u00f3", "\u0142", "\u0123", "\u00a7", "\u00f3", "\u0142", "\u0123", "\u00bf", "\u0120\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00e2\u013f\u00a4", "\u00ef\u00b8\u0131", "\u00e2\u0122\u012f", "\u00f0\u0141\u0134", "\u012d", "\u00e2\u0122\u012f", "\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141\u0131", "\u00bc"], - ids: [0, 20675, 106, 5060, 102, 250, 49904, 228, 16598, 49904, 114, 49365, 124, 5060, 248, 118, 54678, 26323, 229, 16598, 5060, 105, 252, 49365, 124, 54678, 26323, 229, 49904, 106, 49365, 124, 54678, 3753, 239, 127, 5060, 105, 244, 54678, 3753, 102, 256, 54678, 3753, 105, 244, 49904, 107, 54678, 49144, 54678, 41347, 238, 54678, 43307, 106, 49904, 107, 54678, 43307, 107, 54678, 43307, 105, 54678, 43307, 104, 5060, 105, 244, 49365, 124, 54678, 3753, 102, 256, 54678, 3753, 105, 244, 49365, 124, 5060, 242, 117, 180, 259, 228, 105, 180, 259, 228, 100, 180, 259, 228, 103, 180, 259, 228, 111, 180, 259, 228, 105, 180, 259, 228, 128, 49904, 106, 49365, 124, 54678, 49144, 16598, 54678, 41347, 238, 54678, 43307, 106, 49365, 125, 2], - decoded: "\u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75\u200d\u2642\ufe0f \ud83e\uddd9\ud83c\udffb\u200d\u2642 \ud83d\udc68\ud83c\udffb\u200d\ud83c\udf3e \ud83e\uddd1\u200d\ud83e\udd1d\u200d\ud83e\uddd1 \ud83d\udc69\u200d\u2764\u200d\ud83d\udc8b\u200d\ud83d\udc68 \ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66 \ud83e\uddd1\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c\udffb \ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f \ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c\udffc", - }, - CHINESE_LATIN_MIXED: { - text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, - tokens: ["ah", "\u00e5", "\u012f", "\u013c", "\u00e6", "\u0130", "\u00a8", "zz"], - ids: [0, 500, 166, 240, 253, 167, 241, 106, 9326, 2], - decoded: "ah\u535a\u63a8zz", - }, - SIMPLE_WITH_ACCENTS: { - text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, - tokens: ["H", "\u00c3\u00a9", "llo"], - ids: [0, 44, 2277, 31053, 2], - decoded: "H\u00e9llo", - }, - MIXED_CASE_WITHOUT_ACCENTS: { - text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, - tokens: ["\u0120", "\u0109", "He", "L", "Lo", "!", "how", "\u0120\u0120", "\u010a", "\u0120Are", "\u0120y", "o", "U", "?", "\u0120\u0120"], - ids: [0, 225, 202, 2523, 48, 17901, 5, 7253, 6733, 203, 5175, 361, 83, 57, 35, 6733, 2], - decoded: " \tHeLLo!how \n Are yoU? ", - }, - MIXED_CASE_WITH_ACCENTS: { - text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, - tokens: ["\u0120", "\u0109", "H", "\u00c3\u00a4", "L", "Lo", "!", "how", "\u0120\u0120", "\u010a", "\u0120Are", "\u0120y", "o", "U", "?", "\u0120\u0120"], - ids: [0, 225, 202, 44, 325, 48, 17901, 5, 7253, 6733, 203, 5175, 361, 83, 57, 35, 6733, 2], - decoded: " \tH\u00e4LLo!how \n Are yoU? ", - }, + "jinaai/jina-embeddings-v2-base-de": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["How", "\u0120are", "\u0120you", "\u0120doing", "?"], + ids: [0, 3267, 459, 426, 3174, 35, 2], + decoded: "How are you doing?", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["You", "\u0120should", "'ve", "\u0120done", "\u0120this"], + ids: [0, 2606, 1303, 1990, 3022, 555, 2], + decoded: "You should've done this", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["0", "123", "456", "789", "\u01200", "\u01201", "\u01202", "\u01203", "\u01204", "\u01205", "\u01206", "\u01207", "\u01208", "\u01209", "\u012010", "\u0120100", "\u01201000"], + ids: [0, 20, 21911, 40271, 51355, 885, 387, 381, 589, 699, 703, 866, 964, 991, 1045, 949, 1873, 8611, 2], + decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "\u0120company", "\u0120was", "\u0120founded", "\u0120in", "\u01202016", "."], + ids: [0, 710, 1891, 503, 15604, 295, 2262, 18, 2], + decoded: "The company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "\u010a", "'ll", "\u0120!!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'t", "."], + ids: [0, 37, 203, 2202, 26143, 764, 30080, 72, 12228, 72, 314, 16, 571, 797, 18, 2], + decoded: "A\n'll!!to?'d''d of, can't.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["def", "\u0120main", "(", "):", "\u010a", "\u0109", "pass"], + ids: [0, 28273, 1911, 12, 4025, 203, 202, 5517, 2], + decoded: "def main():\n\tpass", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".", "to", "String", "();", "\u010a", "to", "String", "();"], + ids: [0, 1642, 264, 3887, 8273, 18, 764, 53889, 54181, 203, 764, 53889, 54181, 2], + decoded: "let a = obj.toString();\ntoString();", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["This", "\u010a", "\u010a", "is", "\u010a", "a", "\u010a", "test", "."], + ids: [0, 1803, 203, 203, 276, 203, 69, 203, 4451, 18, 2], + decoded: "This\n\nis\na\ntest.", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "w", "ant", "\u00c3\u00a9d", ",", "running"], + ids: [0, 3854, 91, 526, 46298, 16, 47232, 2], + decoded: "UNwant\u00e9d,running", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["1", "\u0100", "2", "\u00ef\u00bf\u00bd", "3"], + ids: [0, 21, 193, 22, 1998, 23, 2], + decoded: "1\u00002\ufffd3", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["Hello", "\u0120World"], + ids: [0, 17964, 3519, 2], + decoded: "Hello World", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["hell", "o", "\u0120world"], + ids: [0, 17067, 83, 1568, 2], + decoded: "hello world", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u00e7\u0136", "\u0141", "\u00e6", "\u00b4", "\u00bb", "\u00e7\u013c\u0126", "\u00e7", "\u013e", "\u0141", "\u00e8", "\u00b0", "\u013d", "\u00e6", "\u013a", "\u00af"], + ids: [0, 55225, 258, 167, 117, 124, 44574, 168, 255, 258, 169, 113, 254, 167, 251, 112, 2], + decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u0120\u0120", "\u0120leading", "\u0120space"], + ids: [0, 6733, 5344, 3435, 2], + decoded: " leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["tra", "iling", "\u0120space", "\u0120\u0120\u0120"], + ids: [0, 766, 7462, 3435, 53448, 2], + decoded: "trailing space ", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["Hi", "\u0120", "\u0120Hello"], + ids: [0, 10103, 225, 29546, 2], + decoded: "Hi Hello", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2", "\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2\u0124\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120test"], + ids: [0, 4451, 1350, 21, 366, 22, 1805, 23, 2712, 24, 6339, 25, 960, 103, 26, 1581, 229, 101, 27, 58720, 28, 1581, 229, 114, 29, 2839, 2], + decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "\u0120bought", "\u0120an", "\u0120apple", "\u0120for", "\u0120$", "1", ".", "00", "\u0120at", "\u0120the", "\u0120store", "."], + ids: [0, 45, 8928, 371, 19798, 382, 1350, 21, 18, 505, 495, 285, 4569, 18, 2], + decoded: "I bought an apple for $1.00 at the store.", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u0120\u0120"], + ids: [0, 10695, 1179, 6733, 2], + decoded: "you\u2026 ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], + ids: [0, 10695, 1179, 44105, 2], + decoded: "you\u2026\u00a0\u00a0", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142", "\u00c2\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], + ids: [0, 10695, 1179, 15529, 15529, 10695, 1179, 44105, 2], + decoded: "you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["we", "ird", "\u0120\u00ef", "\u00bd", "\u0140", "\u0120edge", "\u0120\u00ef", "\u00bd", "\u0140", "\u0120case"], + ids: [0, 487, 2394, 17740, 126, 257, 9911, 17740, 126, 257, 2600, 2], + decoded: "weird \uff5e edge \uff5e case", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u00e2\u0138", "\u0123", "This", "\u0120\u00e2\u0138", "\u0123", "is", "\u0120\u00e2\u0138", "\u0123", "a", "\u0120\u00e2\u0138", "\u0123", "test", "\u0120\u00e2\u0138", "\u0123", "."], + ids: [0, 12790, 228, 1803, 20068, 228, 276, 20068, 228, 69, 20068, 228, 4451, 20068, 228, 18, 2], + decoded: "\u2581This \u2581is \u2581a \u2581test \u2581.", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u00f0\u0141\u013a", "\u0124", "\u0120\u00f0\u0141\u0133", "\u012f", "\u0120\u00f0\u0141", "\u00a4", "\u00a3", "\u0120\u00f0\u0141\u013a", "\u012f", "\u0120\u00f0\u0141\u013a", "\u0143", "\u0120\u00f0\u0141", "\u0130", "\u012b", "\u0120\u00f0\u0141\u013b", "\u0131", "\u0120\u00f0\u0141\u013a", "\u012c", "\u0120\u00f0\u0141", "\u0136", "\u00a5", "\u0120\u00f0\u0141\u013a", "\u0123", "\u0120\u00f0\u0141\u013a", "\u0127", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141\u013a", "\u0128", "\u0120\u00f0\u0141\u0133", "\u0131", "\u0120\u00e2\u013f\u00a4", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141\u0134", "\u013e", "\u0120\u00f0\u0141\u0134", "\u013c", "\u0120\u00f0\u0141\u0134", "\u0139", "\u0120\u00f0\u0141\u0134", "\u013b", "\u0120\u00f0\u0141", "\u0138", "\u00a4", "\u0120\u00f0\u0141\u013a", "\u0130", "\u0120\u00f0\u0141\u0133", "\u012e", "\u0120\u00f0\u0141", "\u00a5", "\u00b3", "\u0120\u00f0\u0141\u0134", "\u00aa", "\u0120\u00e2\u013e", "\u00a8", "\u0120\u00f0\u0141\u0133", "\u012b", "\u0120\u00f0\u0141\u0133", "\u0122", "\u0120\u00f0\u0141\u0134", "\u00af", "\u0120\u00f0\u0141", "\u0130", "\u012a", "\u0120\u00f0\u0141\u013b", "\u012a", "\u0120\u00f0\u0141\u013b", "\u012e", "\u0120\u00f0\u0141\u0134", "\u0122", "\u0120\u00f0\u0141\u0133", "\u0129", "\u0120\u00f0\u0141\u0133", "\u012d", "\u0120\u00e2\u013e\u0127", "\u0120\u00f0\u0141", "\u0130", "\u0123", "\u0120\u00f0\u0141", "\u012e", "\u0140", "\u0120\u00f0\u0141", "\u012e", "\u00b8", "\u0120\u00f0\u0141\u0134", "\u00b0"], + ids: [0, 32164, 229, 49904, 240, 5060, 102, 101, 10278, 240, 10278, 260, 5060, 241, 236, 10319, 242, 10278, 237, 5060, 247, 103, 10278, 228, 10278, 232, 5060, 102, 250, 10278, 233, 49904, 242, 42009, 16598, 52278, 255, 52278, 253, 52278, 250, 52278, 252, 5060, 249, 102, 10278, 241, 49904, 239, 5060, 103, 116, 52278, 108, 10792, 106, 49904, 236, 49904, 227, 52278, 112, 5060, 241, 235, 10319, 235, 10319, 239, 52278, 227, 49904, 234, 49904, 238, 38607, 5060, 241, 228, 5060, 239, 257, 5060, 239, 121, 52278, 113, 2], + decoded: "\ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u00e2\u013e", "\u00a8", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141\u0133", "\u0123", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141\u0133", "\u00b1", "\u00f0\u0141\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0137", "\u00b5", "\u00e2\u0122\u012f", "\u00e2\u013b", "\u0124", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141", "\u00a7", "\u013b", "\u00f0\u0141\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00e2\u013b", "\u0124", "\u0120\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u012e", "\u00be", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u0120\u00f0\u0141\u0133", "\u00a9", "\u00e2\u0122\u012f", "\u00e2\u013f\u00a4", "\u00e2\u0122\u012f", "\u00f0\u0141\u0134", "\u012d", "\u00e2\u0122\u012f", "\u00f0\u0141\u0133", "\u00a8", "\u0120\u00f0\u0141\u0133", "\u00a9", "\u00e2\u0122\u012f", "\u00f0\u0141\u0133", "\u00a9", "\u00e2\u0122\u012f", "\u00f0\u0141\u0133", "\u00a7", "\u00e2\u0122\u012f", "\u00f0\u0141\u0133", "\u00a6", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0131", "\u00b4", "\u00f3", "\u0142", "\u0123", "\u00a7", "\u00f3", "\u0142", "\u0123", "\u00a2", "\u00f3", "\u0142", "\u0123", "\u00a5", "\u00f3", "\u0142", "\u0123", "\u00ae", "\u00f3", "\u0142", "\u0123", "\u00a7", "\u00f3", "\u0142", "\u0123", "\u00bf", "\u0120\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00e2\u013f\u00a4", "\u00ef\u00b8\u0131", "\u00e2\u0122\u012f", "\u00f0\u0141\u0134", "\u012d", "\u00e2\u0122\u012f", "\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141\u0131", "\u00bc"], + ids: [0, 20675, 106, 5060, 102, 250, 49904, 228, 16598, 49904, 114, 49365, 124, 5060, 248, 118, 54678, 26323, 229, 16598, 5060, 105, 252, 49365, 124, 54678, 26323, 229, 49904, 106, 49365, 124, 54678, 3753, 239, 127, 5060, 105, 244, 54678, 3753, 102, 256, 54678, 3753, 105, 244, 49904, 107, 54678, 49144, 54678, 41347, 238, 54678, 43307, 106, 49904, 107, 54678, 43307, 107, 54678, 43307, 105, 54678, 43307, 104, 5060, 105, 244, 49365, 124, 54678, 3753, 102, 256, 54678, 3753, 105, 244, 49365, 124, 5060, 242, 117, 180, 259, 228, 105, 180, 259, 228, 100, 180, 259, 228, 103, 180, 259, 228, 111, 180, 259, 228, 105, 180, 259, 228, 128, 49904, 106, 49365, 124, 54678, 49144, 16598, 54678, 41347, 238, 54678, 43307, 106, 49365, 125, 2], + decoded: "\u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75\u200d\u2642\ufe0f \ud83e\uddd9\ud83c\udffb\u200d\u2642 \ud83d\udc68\ud83c\udffb\u200d\ud83c\udf3e \ud83e\uddd1\u200d\ud83e\udd1d\u200d\ud83e\uddd1 \ud83d\udc69\u200d\u2764\u200d\ud83d\udc8b\u200d\ud83d\udc68 \ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66 \ud83e\uddd1\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c\udffb \ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f \ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c\udffc", + }, + CHINESE_LATIN_MIXED: { + text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, + tokens: ["ah", "\u00e5", "\u012f", "\u013c", "\u00e6", "\u0130", "\u00a8", "zz"], + ids: [0, 500, 166, 240, 253, 167, 241, 106, 9326, 2], + decoded: "ah\u535a\u63a8zz", + }, + SIMPLE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, + tokens: ["H", "\u00c3\u00a9", "llo"], + ids: [0, 44, 2277, 31053, 2], + decoded: "H\u00e9llo", + }, + MIXED_CASE_WITHOUT_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, + tokens: ["\u0120", "\u0109", "He", "L", "Lo", "!", "how", "\u0120\u0120", "\u010a", "\u0120Are", "\u0120y", "o", "U", "?", "\u0120\u0120"], + ids: [0, 225, 202, 2523, 48, 17901, 5, 7253, 6733, 203, 5175, 361, 83, 57, 35, 6733, 2], + decoded: " \tHeLLo!how \n Are yoU? ", + }, + MIXED_CASE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, + tokens: ["\u0120", "\u0109", "H", "\u00c3\u00a4", "L", "Lo", "!", "how", "\u0120\u0120", "\u010a", "\u0120Are", "\u0120y", "o", "U", "?", "\u0120\u0120"], + ids: [0, 225, 202, 44, 325, 48, 17901, 5, 7253, 6733, 203, 5175, 361, 83, 57, 35, 6733, 2], + decoded: " \tH\u00e4LLo!how \n Are yoU? ", + }, - TEXT_PAIR: { - text: "hello", - text_pair: "world", - tokens: ["hell", "o", "world"], - ids: [0, 17067, 83, 2, 2, 13639, 2], - decoded: "helloworld", - }, - }, - "jinaai/jina-embeddings-v2-base-code": { - NUMBERS: { - text: BASE_TEST_STRINGS.NUMBERS, - tokens: ["0123456789", "\u01200", "\u01201", "\u01202", "\u01203", "\u01204", "\u01205", "\u01206", "\u01207", "\u01208", "\u01209", "\u012010", "\u0120100", "\u01201000"], - ids: [0, 22133, 325, 397, 491, 795, 879, 997, 1434, 1577, 1240, 1926, 1528, 2069, 5216, 2], - decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["The", "\u0120company", "\u0120was", "\u0120f", "ounded", "\u0120in", "\u01202016", "."], - ids: [0, 1664, 18100, 2146, 304, 12402, 338, 7541, 18, 2], - decoded: "The company was founded in 2016.", - }, - PYTHON_CODE: { - text: BASE_TEST_STRINGS.PYTHON_CODE, - tokens: ["def", "\u0120main", "():", "\u010a", "\u0109", "pass"], - ids: [0, 406, 3578, 3281, 203, 202, 4557, 2], - decoded: "def main():\n\tpass", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".", "toString", "();", "\u010a", "toString", "();"], - ids: [0, 953, 323, 278, 2666, 18, 3411, 467, 203, 3411, 467, 2], - decoded: "let a = obj.toString();\ntoString();", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["UN", "want", "\u00c3\u00a9", "d", ",", "running"], - ids: [0, 1129, 13944, 2521, 72, 16, 8423, 2], - decoded: "UNwant\u00e9d,running", - }, - HELLO_WORLD_LOWERCASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, - tokens: ["hello", "\u0120world"], - ids: [0, 9522, 7550, 2], - decoded: "hello world", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u00e7\u0136\u0141", "\u00e6\u00b4\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"], - ids: [0, 12173, 28408, 2149, 36264, 12338, 254, 4988, 2], - decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["trailing", "\u0120space", "\u0120\u0120\u0120"], - ids: [0, 29801, 4113, 264, 2], - decoded: "trailing space ", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2", "\u0124\u00ac", "4", "\u0120\u00c2", "\u00a3", "5", "\u0120\u00c2", "\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2", "\u0124", "\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120test"], - ids: [0, 1052, 393, 21, 741, 22, 592, 23, 12284, 16181, 24, 5519, 101, 25, 5519, 103, 26, 12284, 229, 101, 27, 12284, 229, 122, 28, 12284, 229, 114, 29, 1089, 2], - decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["I", "\u0120b", "ought", "\u0120an", "\u0120apple", "\u0120for", "\u0120$", "1", ".", "00", "\u0120at", "\u0120the", "\u0120store", "."], - ids: [0, 45, 328, 17412, 692, 33091, 455, 393, 21, 18, 337, 913, 329, 3205, 18, 2], - decoded: "I bought an apple for $1.00 at the store.", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["we", "ird", "\u0120", "\u00ef\u00bd", "\u0140", "\u0120edge", "\u0120", "\u00ef\u00bd", "\u0140", "\u0120case"], - ids: [0, 1643, 6005, 225, 44634, 257, 7158, 225, 44634, 257, 1007, 2], - decoded: "weird \uff5e edge \uff5e case", - }, - SPIECE_UNDERSCORE: { - text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, - tokens: ["\u00e2\u0138", "\u0123", "This", "\u0120", "\u00e2\u0138", "\u0123", "is", "\u0120", "\u00e2\u0138", "\u0123", "a", "\u0120", "\u00e2\u0138", "\u0123", "test", "\u0120", "\u00e2\u0138", "\u0123", "."], - ids: [0, 8550, 228, 2744, 225, 8550, 228, 302, 225, 8550, 228, 69, 225, 8550, 228, 1052, 225, 8550, 228, 18, 2], - decoded: "\u2581This \u2581is \u2581a \u2581test \u2581.", - }, - POPULAR_EMOJIS: { - text: BASE_TEST_STRINGS.POPULAR_EMOJIS, - tokens: ["\u00f0\u0141", "\u013a", "\u0124", "\u0120\u00f0\u0141\u0133", "\u012f", "\u0120\u00f0\u0141", "\u00a4", "\u00a3", "\u0120\u00f0\u0141", "\u013a", "\u012f", "\u0120\u00f0\u0141", "\u013a", "\u0143", "\u0120\u00f0\u0141", "\u0130", "\u012b", "\u0120\u00f0\u0141", "\u013b", "\u0131", "\u0120\u00f0\u0141", "\u013a", "\u012c", "\u0120\u00f0\u0141", "\u0136", "\u00a5", "\u0120\u00f0\u0141", "\u013a", "\u0123", "\u0120\u00f0\u0141", "\u013a\u0127", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141", "\u013a", "\u0128", "\u0120\u00f0\u0141\u0133", "\u0131", "\u0120\u00e2", "\u013f", "\u00a4", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141", "\u0134", "\u013e", "\u0120\u00f0\u0141", "\u0134", "\u013c", "\u0120\u00f0\u0141", "\u0134", "\u0139", "\u0120\u00f0\u0141", "\u0134", "\u013b", "\u0120\u00f0\u0141", "\u0138", "\u00a4", "\u0120\u00f0\u0141", "\u013a", "\u0130", "\u0120\u00f0\u0141\u0133", "\u012e", "\u0120\u00f0\u0141", "\u00a5", "\u00b3", "\u0120\u00f0\u0141", "\u0134", "\u00aa", "\u0120\u00e2", "\u013e", "\u00a8", "\u0120\u00f0\u0141\u0133", "\u012b", "\u0120\u00f0\u0141\u0133", "\u0122", "\u0120\u00f0\u0141", "\u0134", "\u00af", "\u0120\u00f0\u0141", "\u0130", "\u012a", "\u0120\u00f0\u0141", "\u013b", "\u012a", "\u0120\u00f0\u0141", "\u013b", "\u012e", "\u0120\u00f0\u0141", "\u0134", "\u0122", "\u0120\u00f0\u0141\u0133", "\u0129", "\u0120\u00f0\u0141\u0133", "\u012d", "\u0120\u00e2", "\u013e", "\u0127", "\u0120\u00f0\u0141", "\u0130", "\u0123", "\u0120\u00f0\u0141", "\u012e", "\u0140", "\u0120\u00f0\u0141", "\u012e", "\u00b8", "\u0120\u00f0\u0141", "\u0134", "\u00b0"], - ids: [0, 8000, 251, 229, 22730, 240, 9919, 102, 101, 9919, 251, 240, 9919, 251, 260, 9919, 241, 236, 9919, 252, 242, 9919, 251, 237, 9919, 247, 103, 9919, 251, 228, 9919, 38879, 9919, 102, 250, 9919, 251, 233, 22730, 242, 12284, 256, 102, 26726, 9919, 245, 255, 9919, 245, 253, 9919, 245, 250, 9919, 245, 252, 9919, 249, 102, 9919, 251, 241, 22730, 239, 9919, 103, 116, 9919, 245, 108, 12284, 255, 106, 22730, 236, 22730, 227, 9919, 245, 112, 9919, 241, 235, 9919, 252, 235, 9919, 252, 239, 9919, 245, 227, 22730, 234, 22730, 238, 12284, 255, 232, 9919, 241, 228, 9919, 239, 257, 9919, 239, 121, 9919, 245, 113, 2], - decoded: "\ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0", - }, - MULTIBYTE_EMOJIS: { - text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, - tokens: ["\u00e2\u013e", "\u00a8", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141\u0133", "\u0123", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141\u0133", "\u00b1", "\u00f0\u0141", "\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0137", "\u00b5", "\u00e2\u0122", "\u012f", "\u00e2\u013b", "\u0124", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141\u00a7", "\u013b", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00e2\u013b", "\u0124", "\u0120\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u012e", "\u00be", "\u0120\u00f0\u0141\u00a7", "\u0133", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u0120\u00f0\u0141\u0133", "\u00a9", "\u00e2\u0122", "\u012f", "\u00e2", "\u013f", "\u00a4", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0134", "\u012d", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a8", "\u0120\u00f0\u0141\u0133", "\u00a9", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a9", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a7", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a6", "\u0120\u00f0\u0141\u00a7", "\u0133", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141", "\u0131", "\u00bb", "\u0120\u00f0\u0141\u0131", "\u00b4", "\u00f3", "\u0142\u0123", "\u00a7", "\u00f3", "\u0142\u0123", "\u00a2", "\u00f3", "\u0142\u0123", "\u00a5", "\u00f3", "\u0142\u0123", "\u00ae", "\u00f3", "\u0142\u0123", "\u00a7", "\u00f3", "\u0142\u0123", "\u00bf", "\u0120\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00e2", "\u013f", "\u00a4", "\u00ef\u00b8\u0131", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0134", "\u012d", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bc"], - ids: [0, 60553, 106, 9919, 102, 250, 22730, 228, 26726, 22730, 114, 8000, 242, 124, 9919, 248, 118, 2965, 240, 30370, 229, 26726, 31249, 252, 8000, 242, 124, 2965, 240, 30370, 229, 22730, 106, 8000, 242, 124, 2965, 240, 8000, 239, 127, 31249, 244, 2965, 240, 8000, 102, 256, 2965, 240, 8000, 105, 244, 22730, 107, 2965, 240, 163, 256, 102, 2965, 240, 8000, 245, 238, 2965, 240, 8000, 244, 106, 22730, 107, 2965, 240, 8000, 244, 107, 2965, 240, 8000, 244, 105, 2965, 240, 8000, 244, 104, 31249, 244, 8000, 242, 124, 2965, 240, 8000, 102, 256, 2965, 240, 8000, 105, 244, 8000, 242, 124, 58646, 117, 180, 9752, 105, 180, 9752, 100, 180, 9752, 103, 180, 9752, 111, 180, 9752, 105, 180, 9752, 128, 22730, 106, 8000, 242, 124, 2965, 240, 163, 256, 102, 26726, 2965, 240, 8000, 245, 238, 2965, 240, 8000, 244, 106, 8000, 242, 125, 2], - decoded: "\u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75\u200d\u2642\ufe0f \ud83e\uddd9\ud83c\udffb\u200d\u2642 \ud83d\udc68\ud83c\udffb\u200d\ud83c\udf3e \ud83e\uddd1\u200d\ud83e\udd1d\u200d\ud83e\uddd1 \ud83d\udc69\u200d\u2764\u200d\ud83d\udc8b\u200d\ud83d\udc68 \ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66 \ud83e\uddd1\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c\udffb \ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f \ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c\udffc", - }, - CHINESE_LATIN_MIXED: { - text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, - tokens: ["ah", "\u00e5\u012f\u013c", "\u00e6\u0130\u00a8", "zz"], - ids: [0, 3885, 33588, 28002, 4881, 2], - decoded: "ah\u535a\u63a8zz", - }, - SIMPLE_WITH_ACCENTS: { - text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, - tokens: ["H", "\u00c3\u00a9", "l", "lo"], - ids: [0, 44, 2521, 80, 324, 2], - decoded: "H\u00e9llo", - }, - MIXED_CASE_WITHOUT_ACCENTS: { - text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, - tokens: ["\u0120", "\u0109", "He", "L", "Lo", "!", "how", "\u0120\u0120\u010a", "\u0120Are", "\u0120y", "o", "U", "?", "\u0120\u0120"], - ids: [0, 225, 202, 1397, 48, 1898, 5, 11452, 11092, 14877, 711, 83, 57, 35, 261, 2], - decoded: " \tHeLLo!how \n Are yoU? ", - }, - MIXED_CASE_WITH_ACCENTS: { - text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, - tokens: ["\u0120", "\u0109", "H", "\u00c3\u00a4", "L", "Lo", "!", "how", "\u0120\u0120\u010a", "\u0120Are", "\u0120y", "o", "U", "?", "\u0120\u0120"], - ids: [0, 225, 202, 44, 4319, 48, 1898, 5, 11452, 11092, 14877, 711, 83, 57, 35, 261, 2], - decoded: " \tH\u00e4LLo!how \n Are yoU? ", - }, - }, - "jinaai/jina-reranker-v1-tiny-en": { - SIMPLE: { - text: BASE_TEST_STRINGS.SIMPLE, - tokens: ["how", "are", "you", "doing", "?"], - ids: [0, 21431, 21182, 21166, 22540, 61, 2], - decoded: " how are you doing? ", - }, - SIMPLE_WITH_PUNCTUATION: { - text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, - tokens: ["you", "should", "'", "ve", "done", "this"], - ids: [0, 21166, 21602, 37, 21165, 22366, 21225, 2], - decoded: " you should've done this ", - }, - NUMBERS: { - text: BASE_TEST_STRINGS.NUMBERS, - tokens: ["0", "123", "456", "78", "9", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "100", "1000"], - ids: [0, 46, 32683, 56678, 25106, 55, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 21449, 22057, 25148, 2], - decoded: " 0 123 456 78 9 0 1 2 3 4 5 6 7 8 9 10 100 1000 ", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["the", "company", "was", "founded", "in", "2016", "."], - ids: [0, 21138, 21781, 21257, 26707, 21135, 22672, 44, 2], - decoded: " the company was founded in 2016. ", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["a", "'", "ll", "!!", "to", "?'", "d", "''", "d", "of", ",", "can", "'", "t", "."], - ids: [0, 69, 37, 21264, 22236, 21148, 58125, 72, 31803, 72, 21155, 42, 21243, 37, 88, 44, 2], - decoded: " a'll!! to?' d '' d of, can't. ", - }, - PYTHON_CODE: { - text: BASE_TEST_STRINGS.PYTHON_CODE, - tokens: ["def", "main", "(", "):", "pass"], - ids: [0, 22747, 21810, 38, 25247, 21924, 2], - decoded: " def main ( ): pass ", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["let", "a", "=", "ob", "j", ".", "to", "string", "(", ");", "to", "string", "(", ");"], - ids: [0, 21621, 69, 59, 21706, 78, 44, 21148, 26762, 38, 25750, 21148, 26762, 38, 25750, 2], - decoded: " let a = ob j. to string ( ); to string ( ); ", - }, - NEWLINES: { - text: BASE_TEST_STRINGS.NEWLINES, - tokens: ["this", "is", "a", "test", "."], - ids: [0, 21225, 21152, 69, 21828, 44, 2], - decoded: " this is a test. ", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["un", "want", "\u00e9", "d", ",", "running"], - ids: [0, 21193, 21569, 173, 72, 42, 23225, 2], - decoded: " un want \u00e9 d, running ", + TEXT_PAIR: { + text: "hello", + text_pair: "world", + tokens: ["hell", "o", "world"], + ids: [0, 17067, 83, 2, 2, 13639, 2], + decoded: "helloworld", + }, + }, + "jinaai/jina-embeddings-v2-base-code": { + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["0123456789", "\u01200", "\u01201", "\u01202", "\u01203", "\u01204", "\u01205", "\u01206", "\u01207", "\u01208", "\u01209", "\u012010", "\u0120100", "\u01201000"], + ids: [0, 22133, 325, 397, 491, 795, 879, 997, 1434, 1577, 1240, 1926, 1528, 2069, 5216, 2], + decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "\u0120company", "\u0120was", "\u0120f", "ounded", "\u0120in", "\u01202016", "."], + ids: [0, 1664, 18100, 2146, 304, 12402, 338, 7541, 18, 2], + decoded: "The company was founded in 2016.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["def", "\u0120main", "():", "\u010a", "\u0109", "pass"], + ids: [0, 406, 3578, 3281, 203, 202, 4557, 2], + decoded: "def main():\n\tpass", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".", "toString", "();", "\u010a", "toString", "();"], + ids: [0, 953, 323, 278, 2666, 18, 3411, 467, 203, 3411, 467, 2], + decoded: "let a = obj.toString();\ntoString();", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "want", "\u00c3\u00a9", "d", ",", "running"], + ids: [0, 1129, 13944, 2521, 72, 16, 8423, 2], + decoded: "UNwant\u00e9d,running", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["hello", "\u0120world"], + ids: [0, 9522, 7550, 2], + decoded: "hello world", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u00e7\u0136\u0141", "\u00e6\u00b4\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"], + ids: [0, 12173, 28408, 2149, 36264, 12338, 254, 4988, 2], + decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["trailing", "\u0120space", "\u0120\u0120\u0120"], + ids: [0, 29801, 4113, 264, 2], + decoded: "trailing space ", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2", "\u0124\u00ac", "4", "\u0120\u00c2", "\u00a3", "5", "\u0120\u00c2", "\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2", "\u0124", "\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120test"], + ids: [0, 1052, 393, 21, 741, 22, 592, 23, 12284, 16181, 24, 5519, 101, 25, 5519, 103, 26, 12284, 229, 101, 27, 12284, 229, 122, 28, 12284, 229, 114, 29, 1089, 2], + decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "\u0120b", "ought", "\u0120an", "\u0120apple", "\u0120for", "\u0120$", "1", ".", "00", "\u0120at", "\u0120the", "\u0120store", "."], + ids: [0, 45, 328, 17412, 692, 33091, 455, 393, 21, 18, 337, 913, 329, 3205, 18, 2], + decoded: "I bought an apple for $1.00 at the store.", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["we", "ird", "\u0120", "\u00ef\u00bd", "\u0140", "\u0120edge", "\u0120", "\u00ef\u00bd", "\u0140", "\u0120case"], + ids: [0, 1643, 6005, 225, 44634, 257, 7158, 225, 44634, 257, 1007, 2], + decoded: "weird \uff5e edge \uff5e case", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u00e2\u0138", "\u0123", "This", "\u0120", "\u00e2\u0138", "\u0123", "is", "\u0120", "\u00e2\u0138", "\u0123", "a", "\u0120", "\u00e2\u0138", "\u0123", "test", "\u0120", "\u00e2\u0138", "\u0123", "."], + ids: [0, 8550, 228, 2744, 225, 8550, 228, 302, 225, 8550, 228, 69, 225, 8550, 228, 1052, 225, 8550, 228, 18, 2], + decoded: "\u2581This \u2581is \u2581a \u2581test \u2581.", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u00f0\u0141", "\u013a", "\u0124", "\u0120\u00f0\u0141\u0133", "\u012f", "\u0120\u00f0\u0141", "\u00a4", "\u00a3", "\u0120\u00f0\u0141", "\u013a", "\u012f", "\u0120\u00f0\u0141", "\u013a", "\u0143", "\u0120\u00f0\u0141", "\u0130", "\u012b", "\u0120\u00f0\u0141", "\u013b", "\u0131", "\u0120\u00f0\u0141", "\u013a", "\u012c", "\u0120\u00f0\u0141", "\u0136", "\u00a5", "\u0120\u00f0\u0141", "\u013a", "\u0123", "\u0120\u00f0\u0141", "\u013a\u0127", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141", "\u013a", "\u0128", "\u0120\u00f0\u0141\u0133", "\u0131", "\u0120\u00e2", "\u013f", "\u00a4", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141", "\u0134", "\u013e", "\u0120\u00f0\u0141", "\u0134", "\u013c", "\u0120\u00f0\u0141", "\u0134", "\u0139", "\u0120\u00f0\u0141", "\u0134", "\u013b", "\u0120\u00f0\u0141", "\u0138", "\u00a4", "\u0120\u00f0\u0141", "\u013a", "\u0130", "\u0120\u00f0\u0141\u0133", "\u012e", "\u0120\u00f0\u0141", "\u00a5", "\u00b3", "\u0120\u00f0\u0141", "\u0134", "\u00aa", "\u0120\u00e2", "\u013e", "\u00a8", "\u0120\u00f0\u0141\u0133", "\u012b", "\u0120\u00f0\u0141\u0133", "\u0122", "\u0120\u00f0\u0141", "\u0134", "\u00af", "\u0120\u00f0\u0141", "\u0130", "\u012a", "\u0120\u00f0\u0141", "\u013b", "\u012a", "\u0120\u00f0\u0141", "\u013b", "\u012e", "\u0120\u00f0\u0141", "\u0134", "\u0122", "\u0120\u00f0\u0141\u0133", "\u0129", "\u0120\u00f0\u0141\u0133", "\u012d", "\u0120\u00e2", "\u013e", "\u0127", "\u0120\u00f0\u0141", "\u0130", "\u0123", "\u0120\u00f0\u0141", "\u012e", "\u0140", "\u0120\u00f0\u0141", "\u012e", "\u00b8", "\u0120\u00f0\u0141", "\u0134", "\u00b0"], + ids: [0, 8000, 251, 229, 22730, 240, 9919, 102, 101, 9919, 251, 240, 9919, 251, 260, 9919, 241, 236, 9919, 252, 242, 9919, 251, 237, 9919, 247, 103, 9919, 251, 228, 9919, 38879, 9919, 102, 250, 9919, 251, 233, 22730, 242, 12284, 256, 102, 26726, 9919, 245, 255, 9919, 245, 253, 9919, 245, 250, 9919, 245, 252, 9919, 249, 102, 9919, 251, 241, 22730, 239, 9919, 103, 116, 9919, 245, 108, 12284, 255, 106, 22730, 236, 22730, 227, 9919, 245, 112, 9919, 241, 235, 9919, 252, 235, 9919, 252, 239, 9919, 245, 227, 22730, 234, 22730, 238, 12284, 255, 232, 9919, 241, 228, 9919, 239, 257, 9919, 239, 121, 9919, 245, 113, 2], + decoded: "\ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u00e2\u013e", "\u00a8", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141\u0133", "\u0123", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141\u0133", "\u00b1", "\u00f0\u0141", "\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0137", "\u00b5", "\u00e2\u0122", "\u012f", "\u00e2\u013b", "\u0124", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141\u00a7", "\u013b", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00e2\u013b", "\u0124", "\u0120\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u012e", "\u00be", "\u0120\u00f0\u0141\u00a7", "\u0133", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u0120\u00f0\u0141\u0133", "\u00a9", "\u00e2\u0122", "\u012f", "\u00e2", "\u013f", "\u00a4", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0134", "\u012d", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a8", "\u0120\u00f0\u0141\u0133", "\u00a9", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a9", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a7", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a6", "\u0120\u00f0\u0141\u00a7", "\u0133", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141", "\u0131", "\u00bb", "\u0120\u00f0\u0141\u0131", "\u00b4", "\u00f3", "\u0142\u0123", "\u00a7", "\u00f3", "\u0142\u0123", "\u00a2", "\u00f3", "\u0142\u0123", "\u00a5", "\u00f3", "\u0142\u0123", "\u00ae", "\u00f3", "\u0142\u0123", "\u00a7", "\u00f3", "\u0142\u0123", "\u00bf", "\u0120\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00e2", "\u013f", "\u00a4", "\u00ef\u00b8\u0131", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0134", "\u012d", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bc"], + ids: [0, 60553, 106, 9919, 102, 250, 22730, 228, 26726, 22730, 114, 8000, 242, 124, 9919, 248, 118, 2965, 240, 30370, 229, 26726, 31249, 252, 8000, 242, 124, 2965, 240, 30370, 229, 22730, 106, 8000, 242, 124, 2965, 240, 8000, 239, 127, 31249, 244, 2965, 240, 8000, 102, 256, 2965, 240, 8000, 105, 244, 22730, 107, 2965, 240, 163, 256, 102, 2965, 240, 8000, 245, 238, 2965, 240, 8000, 244, 106, 22730, 107, 2965, 240, 8000, 244, 107, 2965, 240, 8000, 244, 105, 2965, 240, 8000, 244, 104, 31249, 244, 8000, 242, 124, 2965, 240, 8000, 102, 256, 2965, 240, 8000, 105, 244, 8000, 242, 124, 58646, 117, 180, 9752, 105, 180, 9752, 100, 180, 9752, 103, 180, 9752, 111, 180, 9752, 105, 180, 9752, 128, 22730, 106, 8000, 242, 124, 2965, 240, 163, 256, 102, 26726, 2965, 240, 8000, 245, 238, 2965, 240, 8000, 244, 106, 8000, 242, 125, 2], + decoded: "\u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75\u200d\u2642\ufe0f \ud83e\uddd9\ud83c\udffb\u200d\u2642 \ud83d\udc68\ud83c\udffb\u200d\ud83c\udf3e \ud83e\uddd1\u200d\ud83e\udd1d\u200d\ud83e\uddd1 \ud83d\udc69\u200d\u2764\u200d\ud83d\udc8b\u200d\ud83d\udc68 \ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66 \ud83e\uddd1\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c\udffb \ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f \ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c\udffc", + }, + CHINESE_LATIN_MIXED: { + text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, + tokens: ["ah", "\u00e5\u012f\u013c", "\u00e6\u0130\u00a8", "zz"], + ids: [0, 3885, 33588, 28002, 4881, 2], + decoded: "ah\u535a\u63a8zz", + }, + SIMPLE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, + tokens: ["H", "\u00c3\u00a9", "l", "lo"], + ids: [0, 44, 2521, 80, 324, 2], + decoded: "H\u00e9llo", + }, + MIXED_CASE_WITHOUT_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, + tokens: ["\u0120", "\u0109", "He", "L", "Lo", "!", "how", "\u0120\u0120\u010a", "\u0120Are", "\u0120y", "o", "U", "?", "\u0120\u0120"], + ids: [0, 225, 202, 1397, 48, 1898, 5, 11452, 11092, 14877, 711, 83, 57, 35, 261, 2], + decoded: " \tHeLLo!how \n Are yoU? ", + }, + MIXED_CASE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, + tokens: ["\u0120", "\u0109", "H", "\u00c3\u00a4", "L", "Lo", "!", "how", "\u0120\u0120\u010a", "\u0120Are", "\u0120y", "o", "U", "?", "\u0120\u0120"], + ids: [0, 225, 202, 44, 4319, 48, 1898, 5, 11452, 11092, 14877, 711, 83, 57, 35, 261, 2], + decoded: " \tH\u00e4LLo!how \n Are yoU? ", + }, + }, + "jinaai/jina-reranker-v1-tiny-en": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["how", "are", "you", "doing", "?"], + ids: [0, 21431, 21182, 21166, 22540, 61, 2], + decoded: " how are you doing? ", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["you", "should", "'", "ve", "done", "this"], + ids: [0, 21166, 21602, 37, 21165, 22366, 21225, 2], + decoded: " you should've done this ", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["0", "123", "456", "78", "9", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "100", "1000"], + ids: [0, 46, 32683, 56678, 25106, 55, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 21449, 22057, 25148, 2], + decoded: " 0 123 456 78 9 0 1 2 3 4 5 6 7 8 9 10 100 1000 ", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["the", "company", "was", "founded", "in", "2016", "."], + ids: [0, 21138, 21781, 21257, 26707, 21135, 22672, 44, 2], + decoded: " the company was founded in 2016. ", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["a", "'", "ll", "!!", "to", "?'", "d", "''", "d", "of", ",", "can", "'", "t", "."], + ids: [0, 69, 37, 21264, 22236, 21148, 58125, 72, 31803, 72, 21155, 42, 21243, 37, 88, 44, 2], + decoded: " a'll!! to?' d '' d of, can't. ", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["def", "main", "(", "):", "pass"], + ids: [0, 22747, 21810, 38, 25247, 21924, 2], + decoded: " def main ( ): pass ", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "a", "=", "ob", "j", ".", "to", "string", "(", ");", "to", "string", "(", ");"], + ids: [0, 21621, 69, 59, 21706, 78, 44, 21148, 26762, 38, 25750, 21148, 26762, 38, 25750, 2], + decoded: " let a = ob j. to string ( ); to string ( ); ", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["this", "is", "a", "test", "."], + ids: [0, 21225, 21152, 69, 21828, 44, 2], + decoded: " this is a test. ", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["un", "want", "\u00e9", "d", ",", "running"], + ids: [0, 21193, 21569, 173, 72, 42, 23225, 2], + decoded: " un want \u00e9 d, running ", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["1", "\u0000", "2", "\ufffd", "3"], + ids: [0, 47, 5, 48, 20321, 49, 2], + decoded: " 1 \u0000 2 \ufffd 3 ", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["hello", "world"], + ids: [0, 28687, 21628, 2], + decoded: " hello world ", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["hello", "world"], + ids: [0, 28687, 21628, 2], + decoded: " hello world ", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u751f\u6d3b\u7684", "\u771f", "\u8c1b", "\u662f"], + ids: [0, 31805, 11140, 14597, 8097, 2], + decoded: " \u751f\u6d3b\u7684 \u771f \u8c1b \u662f ", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["leading", "space"], + ids: [0, 23462, 22283, 2], + decoded: " leading space ", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["trailing", "space"], + ids: [0, 52572, 22283, 2], + decoded: " trailing space ", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["hi", "hello"], + ids: [0, 23233, 28687, 2], + decoded: " hi hello ", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "$", "1", "r2", "#", "3", "\u20ac", "4", "\u00a3", "5", "\u00a5", "6", "", "7", "\u20b9", "8", "\u20b1", "9", "test"], + ids: [0, 21828, 34, 47, 46925, 33, 49, 2155, 50, 133, 51, 135, 52, 3, 53, 2159, 54, 2157, 55, 21828, 2], + decoded: " test $ 1 r2 # 3 \u20ac 4 \u00a3 5 \u00a5 6 7 \u20b9 8 \u20b1 9 test ", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["i", "bought", "an", "apple", "for", "$", "1", ".", "00", "at", "the", "store", "."], + ids: [0, 77, 25474, 21136, 24208, 21169, 34, 47, 44, 21298, 21141, 21138, 22657, 44, 2], + decoded: " i bought an apple for $ 1. 00 at the store. ", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "\u2026"], + ids: [0, 21166, 2091, 2], + decoded: " you \u2026 ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "\u2026"], + ids: [0, 21166, 2091, 2], + decoded: " you \u2026 ", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "\u2026", "you", "\u2026"], + ids: [0, 21166, 2091, 21166, 2091, 2], + decoded: " you \u2026 you \u2026 ", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["weird", "\uff5e", "edge", "\uff5e", "case"], + ids: [0, 31376, 20249, 24273, 20249, 22111, 2], + decoded: " weird \uff5e edge \uff5e case ", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u2581", "this", "\u2581", "is", "\u2581", "a", "\u2581", "test", "\u2581", "."], + ids: [0, 2541, 21225, 2541, 21152, 2541, 69, 2541, 21828, 2541, 44, 2], + decoded: " \u2581 this \u2581 is \u2581 a \u2581 test \u2581. ", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\ud83d\ude02", "\ud83d\udc4d", "\ud83e\udd23", "\ud83d\ude0d", "\ud83d\ude2d", "\ud83c\udf89", "\ud83d\ude4f", "\ud83d\ude0a", "\ud83d\udd25", "\ud83d\ude01", "\ud83d\ude05", "\ud83e\udd17", "\ud83d\ude06", "\ud83d\udc4f", "\u2764", "\ufe0f", "\ud83d\udc9c", "\ud83d\udc9a", "\ud83d\udc97", "\ud83d\udc99", "\ud83d\udda4", "\ud83d\ude0e", "\ud83d\udc4c", "\ud83e\udd73", "\ud83d\udcaa", "\u2728", "\ud83d\udc49", "\ud83d\udc40", "\ud83d\udcaf", "\ud83c\udf88", "\ud83d\ude48", "\ud83d\ude4c", "\ud83d\udc80", "\ud83d\udc47", "\ud83d\udc4b", "\u2705", "\ud83c\udf81", "\ud83c\udf1e", "\ud83c\udf38", "\ud83d\udcb0"], + ids: [0, 20904, 20749, 21000, 20915, 20943, 20645, 20964, 20912, 20879, 20903, 20907, 20992, 20908, 20751, 2781, 20133, 20807, 20805, 20802, 20804, 20898, 20916, 20748, 21031, 20817, 2742, 20745, 20738, 20821, 20644, 20961, 20963, 20784, 20743, 20747, 2720, 20637, 20574, 20588, 20822, 2], + decoded: " \ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764 \ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0 ", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u2728", "\ud83e\udd17", "", "\ufe0f", "\ud83d\udc71", "\ud83c\udffb", "\ud83d\udd75", "\u200d", "\u2642", "\ufe0f", "\ud83e\uddd9", "\ud83c\udffb", "\u200d", "\u2642", "\ud83d\udc68", "\ud83c\udffb", "\u200d", "\ud83c\udf3e", "\ud83e\uddd1", "\u200d", "\ud83e\udd1d", "\u200d", "\ud83e\uddd1", "\ud83d\udc69", "\u200d", "\u2764", "\u200d", "\ud83d\udc8b", "\u200d", "\ud83d\udc68", "\ud83d\udc69", "\u200d", "\ud83d\udc69", "\u200d", "\ud83d\udc67", "\u200d", "\ud83d\udc66", "\ud83e\uddd1", "\ud83c\udffb", "\u200d", "\ud83e\udd1d", "\u200d", "\ud83e\uddd1", "\ud83c\udffb", "\ud83c\udff4", "", "\udb40\udc62", "", "", "", "\udb40\udc7f", "\ud83d\udc68", "\ud83c\udffb", "\u200d", "\u2764", "\ufe0f", "\u200d", "\ud83d\udc8b", "\u200d", "\ud83d\udc68", "\ud83c\udffc"], + ids: [0, 2742, 20992, 3, 20133, 20775, 20700, 20894, 2067, 2662, 20133, 21050, 20700, 2067, 2662, 20768, 20700, 2067, 20593, 21049, 2067, 20995, 2067, 21049, 20769, 2067, 2781, 2067, 20792, 2067, 20768, 20769, 2067, 20769, 2067, 20767, 2067, 20766, 21049, 20700, 2067, 20995, 2067, 21049, 20700, 20697, 3, 21126, 3, 3, 3, 21130, 20768, 20700, 2067, 2781, 20133, 2067, 20792, 2067, 20768, 20701, 2], + decoded: " \u2728 \ud83e\udd17 \ufe0f \ud83d\udc71 \ud83c\udffb \ud83d\udd75 \u200d \u2642 \ufe0f \ud83e\uddd9 \ud83c\udffb \u200d \u2642 \ud83d\udc68 \ud83c\udffb \u200d \ud83c\udf3e \ud83e\uddd1 \u200d \ud83e\udd1d \u200d \ud83e\uddd1 \ud83d\udc69 \u200d \u2764 \u200d \ud83d\udc8b \u200d \ud83d\udc68 \ud83d\udc69 \u200d \ud83d\udc69 \u200d \ud83d\udc67 \u200d \ud83d\udc66 \ud83e\uddd1 \ud83c\udffb \u200d \ud83e\udd1d \u200d \ud83e\uddd1 \ud83c\udffb \ud83c\udff4 \udb40\udc62 \udb40\udc7f \ud83d\udc68 \ud83c\udffb \u200d \u2764 \ufe0f \u200d \ud83d\udc8b \u200d \ud83d\udc68 \ud83c\udffc ", + }, + CHINESE_LATIN_MIXED: { + text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, + tokens: ["ah", "\u535a", "\u63a8", "zz"], + ids: [0, 22311, 4352, 7628, 24387, 2], + decoded: " ah \u535a \u63a8 zz ", + }, + SIMPLE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, + tokens: ["h", "\u00e9", "llo"], + ids: [0, 76, 173, 48932, 2], + decoded: " h \u00e9 llo ", + }, + MIXED_CASE_WITHOUT_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, + tokens: ["hello", "!", "how", "are", "you", "?"], + ids: [0, 28687, 31, 21431, 21182, 21166, 61, 2], + decoded: " hello! how are you? ", + }, + MIXED_CASE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, + tokens: ["h", "\u00e4", "llo", "!", "how", "are", "you", "?"], + ids: [0, 76, 168, 48932, 31, 21431, 21182, 21166, 61, 2], + decoded: " h \u00e4 llo! how are you? ", + }, + }, + "jinaai/jina-embeddings-v2-base-zh": { + // https://huggingface.co/jinaai/jina-embeddings-v2-base-zh/discussions/16 + // Slow vs. fast tokenizer mismatch + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["1", "\u0000", "2", "\ufffd", "3"], + ids: [0, 47, 5, 48, 20321, 49, 2], + decoded: " 1 \u0000 2 \ufffd 3 ", + }, + }, + "Xenova/all-distilroberta-v1": { + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["01", "23", "45", "67", "89", "\u01200", "\u01201", "\u01202", "\u01203", "\u01204", "\u01205", "\u01206", "\u01207", "\u01208", "\u01209", "\u012010", "\u0120100", "\u01201000"], + ids: [0, 2663, 1922, 1898, 4111, 5046, 321, 112, 132, 155, 204, 195, 231, 262, 290, 361, 158, 727, 10775, 2], + decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u00e7\u0136\u0141", "\u00e6", "\u00b4", "\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e", "\u0141", "\u00e8", "\u00b0", "\u013d", "\u00e6\u013a\u00af"], + ids: [0, 48998, 37127, 20024, 2023, 44574, 49122, 4333, 36484, 7487, 3726, 48569, 2], + decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u0120", "\u0120", "\u0120leading", "\u0120space"], + ids: [0, 1437, 1437, 981, 980, 2], + decoded: " leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["tra", "iling", "\u0120space", "\u0120", "\u0120", "\u0120"], + ids: [0, 9738, 7022, 980, 1437, 1437, 1437, 2], + decoded: "trailing space ", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2", "\u0124", "\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120test"], + ids: [0, 21959, 68, 134, 248, 176, 849, 246, 4480, 306, 984, 245, 30844, 401, 14333, 9264, 2469, 406, 14333, 9264, 9253, 398, 14333, 9264, 15389, 466, 1296, 2], + decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u0120", "\u0120"], + ids: [0, 6968, 1174, 1437, 1437, 2], + decoded: "you\u2026 ", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u00f0\u0141\u013a", "\u0124", "\u0120\u00f0\u0141\u0133", "\u012f", "\u0120\u00f0\u0141", "\u00a4", "\u00a3", "\u0120\u00f0\u0141\u013a", "\u012f", "\u0120\u00f0\u0141\u013a", "\u0143", "\u0120\u00f0\u0141", "\u0130", "\u012b", "\u0120\u00f0\u0141", "\u013b", "\u0131", "\u0120\u00f0\u0141\u013a", "\u012c", "\u0120\u00f0\u0141", "\u0136", "\u00a5", "\u0120\u00f0\u0141\u013a", "\u0123", "\u0120\u00f0\u0141", "\u013a\u0127", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141\u013a", "\u0128", "\u0120\u00f0\u0141\u0133", "\u0131", "\u0120\u00e2\u013f", "\u00a4", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141", "\u0134", "\u013e", "\u0120\u00f0\u0141", "\u0134", "\u013c", "\u0120\u00f0\u0141", "\u0134", "\u0139", "\u0120\u00f0\u0141", "\u0134", "\u013b", "\u0120\u00f0\u0141", "\u0138", "\u00a4", "\u0120\u00f0\u0141\u013a", "\u0130", "\u0120\u00f0\u0141\u0133", "\u012e", "\u0120\u00f0\u0141", "\u00a5", "\u00b3", "\u0120\u00f0\u0141", "\u0134", "\u00aa", "\u0120\u00e2\u013e", "\u00a8", "\u0120\u00f0\u0141\u0133", "\u012b", "\u0120\u00f0\u0141\u0133", "\u0122", "\u0120\u00f0\u0141", "\u0134", "\u00af", "\u0120\u00f0\u0141", "\u0130", "\u012a", "\u0120\u00f0\u0141", "\u013b", "\u012a", "\u0120\u00f0\u0141", "\u013b", "\u012e", "\u0120\u00f0\u0141", "\u0134", "\u0122", "\u0120\u00f0\u0141\u0133", "\u0129", "\u0120\u00f0\u0141\u0133", "\u012d", "\u0120\u00e2\u013e", "\u0127", "\u0120\u00f0\u0141", "\u0130", "\u0123", "\u0120\u00f0\u0141", "\u012e", "\u0140", "\u0120\u00f0\u0141", "\u012e", "\u00b8", "\u0120\u00f0\u0141", "\u0134", "\u00b0"], + ids: [0, 18636, 9264, 26964, 8384, 8103, 10470, 2469, 17841, 8384, 17841, 12410, 8103, 12736, 23171, 8103, 27, 9357, 17841, 27969, 8103, 10674, 8210, 17841, 10172, 8103, 48278, 8103, 10470, 6800, 17841, 27819, 26964, 9357, 28775, 10470, 12605, 8103, 10659, 48, 8103, 10659, 15113, 8103, 10659, 6800, 8103, 10659, 27, 8103, 25448, 10470, 17841, 12736, 26964, 14285, 8103, 8210, 15264, 8103, 10659, 10278, 36174, 11423, 26964, 23171, 26964, 7471, 8103, 10659, 10965, 8103, 12736, 23133, 8103, 27, 23133, 8103, 27, 14285, 8103, 10659, 7471, 26964, 6382, 26964, 13859, 36174, 5782, 8103, 12736, 10172, 8103, 14285, 17772, 8103, 14285, 18537, 8103, 10659, 7487, 2], + decoded: "\ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u00e2\u013e", "\u00a8", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141\u0133", "\u0123", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141\u0133", "\u00b1", "\u00f0\u0141", "\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0137", "\u00b5", "\u00e2\u0122", "\u012f", "\u00e2\u013b", "\u0124", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141", "\u00a7", "\u013b", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00e2\u013b", "\u0124", "\u0120\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u012e", "\u00be", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u0120\u00f0\u0141\u0133", "\u00a9", "\u00e2\u0122", "\u012f", "\u00e2\u013f", "\u00a4", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0134", "\u012d", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u0133", "\u00a8", "\u0120\u00f0\u0141\u0133", "\u00a9", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u0133", "\u00a9", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u0133", "\u00a7", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u0133", "\u00a6", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141", "\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0131", "\u00b4", "\u00f3", "\u0142", "\u0123", "\u00a7", "\u00f3", "\u0142", "\u0123", "\u00a2", "\u00f3", "\u0142", "\u0123", "\u00a5", "\u00f3", "\u0142", "\u0123", "\u00ae", "\u00f3", "\u0142", "\u0123", "\u00a7", "\u00f3", "\u0142", "\u0123", "\u00bf", "\u0120\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00e2\u013f", "\u00a4", "\u00ef\u00b8\u0131", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0134", "\u012d", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bc"], + ids: [0, 39817, 11423, 8103, 10470, 6800, 26964, 10172, 12605, 26964, 15389, 6569, 9357, 2023, 8103, 15722, 8906, 17, 8384, 38718, 9264, 12605, 8103, 6248, 27, 6569, 9357, 2023, 17, 8384, 38718, 9264, 26964, 11423, 6569, 9357, 2023, 17, 8384, 6569, 14285, 4726, 8103, 6248, 3602, 17, 8384, 6569, 10470, 46, 17, 8384, 6569, 6248, 3602, 26964, 15375, 17, 8384, 30151, 10470, 17, 8384, 6569, 10659, 13859, 17, 8384, 31193, 11423, 26964, 15375, 17, 8384, 31193, 15375, 17, 8384, 31193, 6248, 17, 8384, 31193, 18164, 8103, 6248, 3602, 6569, 9357, 2023, 17, 8384, 6569, 10470, 46, 17, 8384, 6569, 6248, 3602, 6569, 9357, 2023, 8103, 9357, 20024, 49078, 21402, 10172, 6248, 49078, 21402, 10172, 7258, 49078, 21402, 10172, 8210, 49078, 21402, 10172, 2840, 49078, 21402, 10172, 6248, 49078, 21402, 10172, 9470, 26964, 11423, 6569, 9357, 2023, 17, 8384, 30151, 10470, 12605, 17, 8384, 6569, 10659, 13859, 17, 8384, 31193, 11423, 6569, 9357, 4394, 2], + decoded: "\u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75\u200d\u2642\ufe0f \ud83e\uddd9\ud83c\udffb\u200d\u2642 \ud83d\udc68\ud83c\udffb\u200d\ud83c\udf3e \ud83e\uddd1\u200d\ud83e\udd1d\u200d\ud83e\uddd1 \ud83d\udc69\u200d\u2764\u200d\ud83d\udc8b\u200d\ud83d\udc68 \ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66 \ud83e\uddd1\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c\udffb \ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f \ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c\udffc", + }, + CHINESE_LATIN_MIXED: { + text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, + tokens: ["ah", "\u00e5\u012f", "\u013c", "\u00e6", "\u0130", "\u00a8", "zz"], + ids: [0, 895, 47658, 15113, 37127, 12736, 11423, 7399, 2], + decoded: "ah\u535a\u63a8zz", + }, + MIXED_CASE_WITHOUT_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, + tokens: ["\u0120", "\u0109", "He", "LL", "o", "!", "how", "\u0120", "\u0120", "\u010a", "\u0120Are", "\u0120yo", "U", "?", "\u0120", "\u0120"], + ids: [0, 1437, 50117, 894, 6006, 139, 328, 9178, 1437, 1437, 50118, 3945, 25610, 791, 116, 1437, 1437, 2], + decoded: " \tHeLLo!how \n Are yoU? ", + }, + MIXED_CASE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, + tokens: ["\u0120", "\u0109", "H", "\u00c3\u00a4", "LL", "o", "!", "how", "\u0120", "\u0120", "\u010a", "\u0120Are", "\u0120yo", "U", "?", "\u0120", "\u0120"], + ids: [0, 1437, 50117, 725, 1561, 6006, 139, 328, 9178, 1437, 1437, 50118, 3945, 25610, 791, 116, 1437, 1437, 2], + decoded: " \tH\u00e4LLo!how \n Are yoU? ", + }, + }, + "Xenova/EsperBERTo-small-pos": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["How", "\u0120are", "\u0120you", "\u0120do", "ing", "?"], + ids: [0, 50702, 1694, 12426, 661, 948, 35, 2], + decoded: "How are you doing?", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["You", "\u0120sho", "uld", "'ve", "\u0120don", "e", "\u0120this"], + ids: [0, 36894, 21906, 8512, 6091, 851, 73, 18955, 2], + decoded: "You should've done this", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["0", "123456", "78", "9", "\u01200", "\u01201", "\u01202", "\u01203", "\u01204", "\u01205", "\u01206", "\u01207", "\u01208", "\u01209", "\u012010", "\u0120100", "\u01201000"], + ids: [0, 20, 11816, 6229, 29, 2042, 355, 411, 620, 818, 839, 1031, 1142, 1166, 1274, 1312, 2450, 8403, 2], + decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "\u0120c", "ompan", "y", "\u0120was", "\u0120fo", "und", "ed", "\u0120in", "\u01202016", "."], + ids: [0, 5490, 467, 2833, 93, 30687, 1204, 3936, 347, 327, 3653, 18, 2], + decoded: "The company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "\u010a", "'ll", "\u0120!", "!", "to", "?", "'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'t", "."], + ids: [0, 37, 203, 17792, 4883, 5, 288, 35, 11, 72, 15271, 72, 682, 16, 15597, 3761, 18, 2], + decoded: "A\n'll!!to?'d''d of, can't.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["de", "f", "\u0120ma", "in", "(", "):", "\u010a", "\u0109", "pas", "s"], + ids: [0, 387, 74, 633, 282, 12, 3914, 203, 202, 1208, 87, 2], + decoded: "def main():\n\tpass", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "\u0120a", "\u0120=", "\u0120ob", "j", ".", "to", "S", "tr", "ing", "(", ");", "\u010a", "to", "S", "tr", "ing", "(", ");"], + ids: [0, 5745, 278, 3945, 1080, 78, 18, 288, 55, 497, 948, 12, 3429, 203, 288, 55, 497, 948, 12, 3429, 2], + decoded: "let a = obj.toString();\ntoString();", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "w", "ant", "\u00c3\u00a9", "d", ",", "run", "ning"], + ids: [0, 7390, 91, 1799, 1174, 72, 16, 1307, 13715, 2], + decoded: "UNwant\u00e9d,running", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["Hel", "lo", "\u0120World"], + ids: [0, 4152, 310, 7717, 2], + decoded: "Hello World", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["hel", "lo", "\u0120wor", "ld"], + ids: [0, 1686, 310, 39013, 3580, 2], + decoded: "hello world", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u00e7\u0136", "\u0141", "\u00e6", "\u00b4", "\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e", "\u0141", "\u00e8", "\u00b0", "\u013d", "\u00e6\u013a\u00af"], + ids: [0, 25856, 258, 167, 117, 124, 19584, 42803, 258, 169, 113, 254, 34946, 2], + decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u0120\u0120", "\u0120le", "ading", "\u0120space"], + ids: [0, 2399, 591, 30214, 51965, 2], + decoded: " leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["tra", "i", "ling", "\u0120space", "\u0120\u0120", "\u0120"], + ids: [0, 440, 77, 879, 51965, 2399, 225, 2], + decoded: "trailing space ", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["Hi", "\u0120", "\u0120Hel", "lo"], + ids: [0, 15893, 225, 3558, 310, 2], + decoded: "Hi Hello", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2", "\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2", "\u0124", "\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120test"], + ids: [0, 30747, 9416, 21, 462, 22, 4668, 23, 7537, 24, 12407, 25, 790, 103, 26, 2097, 229, 101, 27, 2097, 229, 122, 28, 2097, 229, 114, 29, 18885, 2], + decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "\u0120bo", "u", "ght", "\u0120an", "\u0120ap", "ple", "\u0120for", "\u0120$", "1", ".", "00", "\u0120at", "\u0120the", "\u0120s", "tore", "."], + ids: [0, 45, 1716, 89, 8840, 353, 560, 720, 434, 9416, 21, 18, 455, 3993, 2814, 275, 14003, 18, 2], + decoded: "I bought an apple for $1.00 at the store.", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142"], + ids: [0, 9642, 1322, 131, 259, 131, 259, 2], + decoded: "you\u2026\u00a0\u00a0", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142"], + ids: [0, 9642, 1322, 131, 259, 131, 259, 9642, 1322, 131, 259, 131, 259, 2], + decoded: "you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["we", "ir", "d", "\u0120", "\u00ef\u00bd", "\u0140", "\u0120ed", "ge", "\u0120", "\u00ef\u00bd", "\u0140", "\u0120c", "ase"], + ids: [0, 4983, 861, 72, 225, 30624, 257, 1263, 587, 225, 30624, 257, 467, 14285, 2], + decoded: "weird \uff5e edge \uff5e case", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u00f0\u0141\u013a", "\u0124", "\u0120\u00f0\u0141", "\u0133", "\u012f", "\u0120\u00f0\u0141", "\u00a4", "\u00a3", "\u0120\u00f0\u0141\u013a", "\u012f", "\u0120\u00f0\u0141\u013a", "\u0143", "\u0120\u00f0\u0141", "\u0130", "\u012b", "\u0120\u00f0\u0141", "\u013b", "\u0131", "\u0120\u00f0\u0141\u013a", "\u012c", "\u0120\u00f0\u0141", "\u0136", "\u00a5", "\u0120\u00f0\u0141\u013a", "\u0123", "\u0120\u00f0\u0141\u013a", "\u0127", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141\u013a", "\u0128", "\u0120\u00f0\u0141", "\u0133", "\u0131", "\u0120\u00e2", "\u013f", "\u00a4", "\u00ef\u00b8", "\u0131", "\u0120\u00f0\u0141", "\u0134", "\u013e", "\u0120\u00f0\u0141", "\u0134", "\u013c", "\u0120\u00f0\u0141", "\u0134", "\u0139", "\u0120\u00f0\u0141", "\u0134", "\u013b", "\u0120\u00f0\u0141", "\u0138", "\u00a4", "\u0120\u00f0\u0141\u013a", "\u0130", "\u0120\u00f0\u0141", "\u0133", "\u012e", "\u0120\u00f0\u0141", "\u00a5", "\u00b3", "\u0120\u00f0\u0141", "\u0134", "\u00aa", "\u0120\u00e2", "\u013e", "\u00a8", "\u0120\u00f0\u0141", "\u0133", "\u012b", "\u0120\u00f0\u0141", "\u0133", "\u0122", "\u0120\u00f0\u0141", "\u0134", "\u00af", "\u0120\u00f0\u0141", "\u0130", "\u012a", "\u0120\u00f0\u0141", "\u013b", "\u012a", "\u0120\u00f0\u0141", "\u013b", "\u012e", "\u0120\u00f0\u0141", "\u0134", "\u0122", "\u0120\u00f0\u0141", "\u0133", "\u0129", "\u0120\u00f0\u0141", "\u0133", "\u012d", "\u0120\u00e2", "\u013e", "\u0127", "\u0120\u00f0\u0141", "\u0130", "\u0123", "\u0120\u00f0\u0141", "\u012e", "\u0140", "\u0120\u00f0\u0141", "\u012e", "\u00b8", "\u0120\u00f0\u0141", "\u0134", "\u00b0"], + ids: [0, 10626, 229, 32340, 244, 240, 32340, 102, 101, 30199, 240, 30199, 260, 32340, 241, 236, 32340, 252, 242, 30199, 237, 32340, 247, 103, 30199, 228, 30199, 232, 32340, 102, 250, 30199, 233, 32340, 244, 242, 2097, 256, 102, 27027, 242, 32340, 245, 255, 32340, 245, 253, 32340, 245, 250, 32340, 245, 252, 32340, 249, 102, 30199, 241, 32340, 244, 239, 32340, 103, 116, 32340, 245, 108, 2097, 255, 106, 32340, 244, 236, 32340, 244, 227, 32340, 245, 112, 32340, 241, 235, 32340, 252, 235, 32340, 252, 239, 32340, 245, 227, 32340, 244, 234, 32340, 244, 238, 2097, 255, 232, 32340, 241, 228, 32340, 239, 257, 32340, 239, 121, 32340, 245, 113, 2], + decoded: "\ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u00e2", "\u013e", "\u00a8", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141", "\u0133", "\u0123", "\u00ef\u00b8", "\u0131", "\u0120\u00f0\u0141", "\u0133", "\u00b1", "\u00f0\u0141", "\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0137", "\u00b5", "\u00e2\u0122", "\u012f", "\u00e2\u013b", "\u0124", "\u00ef\u00b8", "\u0131", "\u0120\u00f0\u0141", "\u00a7", "\u013b", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00e2\u013b", "\u0124", "\u0120\u00f0\u0141", "\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u012e", "\u00be", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u0120\u00f0\u0141", "\u0133", "\u00a9", "\u00e2\u0122", "\u012f", "\u00e2", "\u013f", "\u00a4", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0134", "\u012d", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a8", "\u0120\u00f0\u0141", "\u0133", "\u00a9", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a9", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a7", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a6", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141", "\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0131", "\u00b4", "\u00f3", "\u0142", "\u0123", "\u00a7", "\u00f3", "\u0142", "\u0123", "\u00a2", "\u00f3", "\u0142", "\u0123", "\u00a5", "\u00f3", "\u0142", "\u0123", "\u00ae", "\u00f3", "\u0142", "\u0123", "\u00a7", "\u00f3", "\u0142", "\u0123", "\u00bf", "\u0120\u00f0\u0141", "\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00e2", "\u013f", "\u00a4", "\u00ef\u00b8", "\u0131", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0134", "\u012d", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bc"], + ids: [0, 163, 255, 106, 32340, 102, 250, 32340, 244, 228, 27027, 242, 32340, 244, 114, 9132, 242, 124, 32340, 248, 118, 348, 240, 20419, 229, 27027, 242, 32340, 105, 252, 9132, 242, 124, 348, 240, 20419, 229, 32340, 244, 106, 9132, 242, 124, 348, 240, 9132, 239, 127, 32340, 105, 244, 348, 240, 9132, 102, 256, 348, 240, 9132, 105, 244, 32340, 244, 107, 348, 240, 163, 256, 102, 348, 240, 9132, 245, 238, 348, 240, 9132, 244, 106, 32340, 244, 107, 348, 240, 9132, 244, 107, 348, 240, 9132, 244, 105, 348, 240, 9132, 244, 104, 32340, 105, 244, 9132, 242, 124, 348, 240, 9132, 102, 256, 348, 240, 9132, 105, 244, 9132, 242, 124, 32340, 242, 117, 180, 259, 228, 105, 180, 259, 228, 100, 180, 259, 228, 103, 180, 259, 228, 111, 180, 259, 228, 105, 180, 259, 228, 128, 32340, 244, 106, 9132, 242, 124, 348, 240, 163, 256, 102, 27027, 242, 348, 240, 9132, 245, 238, 348, 240, 9132, 244, 106, 9132, 242, 125, 2], + decoded: "\u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75\u200d\u2642\ufe0f \ud83e\uddd9\ud83c\udffb\u200d\u2642 \ud83d\udc68\ud83c\udffb\u200d\ud83c\udf3e \ud83e\uddd1\u200d\ud83e\udd1d\u200d\ud83e\uddd1 \ud83d\udc69\u200d\u2764\u200d\ud83d\udc8b\u200d\ud83d\udc68 \ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66 \ud83e\uddd1\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c\udffb \ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f \ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c\udffc", + }, + MIXED_CASE_WITHOUT_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, + tokens: ["\u0120", "\u0109", "He", "L", "Lo", "!", "ho", "w", "\u0120\u0120", "\u010a", "\u0120Are", "\u0120yo", "U", "?", "\u0120\u0120"], + ids: [0, 225, 202, 13029, 48, 4876, 5, 882, 91, 2399, 203, 31676, 27961, 57, 35, 2399, 2], + decoded: " \tHeLLo!how \n Are yoU? ", + }, + MIXED_CASE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, + tokens: ["\u0120", "\u0109", "H", "\u00c3\u00a4", "L", "Lo", "!", "ho", "w", "\u0120\u0120", "\u010a", "\u0120Are", "\u0120yo", "U", "?", "\u0120\u0120"], + ids: [0, 225, 202, 44, 3203, 48, 4876, 5, 882, 91, 2399, 203, 31676, 27961, 57, 35, 2399, 2], + decoded: " \tH\u00e4LLo!how \n Are yoU? ", + }, }, - CONTROL_TOKENS: { - text: BASE_TEST_STRINGS.CONTROL_TOKENS, - tokens: ["1", "\u0000", "2", "\ufffd", "3"], - ids: [0, 47, 5, 48, 20321, 49, 2], - decoded: " 1 \u0000 2 \ufffd 3 ", - }, - HELLO_WORLD_TITLECASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, - tokens: ["hello", "world"], - ids: [0, 28687, 21628, 2], - decoded: " hello world ", - }, - HELLO_WORLD_LOWERCASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, - tokens: ["hello", "world"], - ids: [0, 28687, 21628, 2], - decoded: " hello world ", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u751f\u6d3b\u7684", "\u771f", "\u8c1b", "\u662f"], - ids: [0, 31805, 11140, 14597, 8097, 2], - decoded: " \u751f\u6d3b\u7684 \u771f \u8c1b \u662f ", - }, - LEADING_SPACE: { - text: BASE_TEST_STRINGS.LEADING_SPACE, - tokens: ["leading", "space"], - ids: [0, 23462, 22283, 2], - decoded: " leading space ", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["trailing", "space"], - ids: [0, 52572, 22283, 2], - decoded: " trailing space ", - }, - DOUBLE_SPACE: { - text: BASE_TEST_STRINGS.DOUBLE_SPACE, - tokens: ["hi", "hello"], - ids: [0, 23233, 28687, 2], - decoded: " hi hello ", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["test", "$", "1", "r2", "#", "3", "\u20ac", "4", "\u00a3", "5", "\u00a5", "6", "", "7", "\u20b9", "8", "\u20b1", "9", "test"], - ids: [0, 21828, 34, 47, 46925, 33, 49, 2155, 50, 133, 51, 135, 52, 3, 53, 2159, 54, 2157, 55, 21828, 2], - decoded: " test $ 1 r2 # 3 \u20ac 4 \u00a3 5 \u00a5 6 7 \u20b9 8 \u20b1 9 test ", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["i", "bought", "an", "apple", "for", "$", "1", ".", "00", "at", "the", "store", "."], - ids: [0, 77, 25474, 21136, 24208, 21169, 34, 47, 44, 21298, 21141, 21138, 22657, 44, 2], - decoded: " i bought an apple for $ 1. 00 at the store. ", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - tokens: ["you", "\u2026"], - ids: [0, 21166, 2091, 2], - decoded: " you \u2026 ", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - tokens: ["you", "\u2026"], - ids: [0, 21166, 2091, 2], - decoded: " you \u2026 ", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - tokens: ["you", "\u2026", "you", "\u2026"], - ids: [0, 21166, 2091, 21166, 2091, 2], - decoded: " you \u2026 you \u2026 ", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["weird", "\uff5e", "edge", "\uff5e", "case"], - ids: [0, 31376, 20249, 24273, 20249, 22111, 2], - decoded: " weird \uff5e edge \uff5e case ", - }, - SPIECE_UNDERSCORE: { - text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, - tokens: ["\u2581", "this", "\u2581", "is", "\u2581", "a", "\u2581", "test", "\u2581", "."], - ids: [0, 2541, 21225, 2541, 21152, 2541, 69, 2541, 21828, 2541, 44, 2], - decoded: " \u2581 this \u2581 is \u2581 a \u2581 test \u2581. ", - }, - POPULAR_EMOJIS: { - text: BASE_TEST_STRINGS.POPULAR_EMOJIS, - tokens: ["\ud83d\ude02", "\ud83d\udc4d", "\ud83e\udd23", "\ud83d\ude0d", "\ud83d\ude2d", "\ud83c\udf89", "\ud83d\ude4f", "\ud83d\ude0a", "\ud83d\udd25", "\ud83d\ude01", "\ud83d\ude05", "\ud83e\udd17", "\ud83d\ude06", "\ud83d\udc4f", "\u2764", "\ufe0f", "\ud83d\udc9c", "\ud83d\udc9a", "\ud83d\udc97", "\ud83d\udc99", "\ud83d\udda4", "\ud83d\ude0e", "\ud83d\udc4c", "\ud83e\udd73", "\ud83d\udcaa", "\u2728", "\ud83d\udc49", "\ud83d\udc40", "\ud83d\udcaf", "\ud83c\udf88", "\ud83d\ude48", "\ud83d\ude4c", "\ud83d\udc80", "\ud83d\udc47", "\ud83d\udc4b", "\u2705", "\ud83c\udf81", "\ud83c\udf1e", "\ud83c\udf38", "\ud83d\udcb0"], - ids: [0, 20904, 20749, 21000, 20915, 20943, 20645, 20964, 20912, 20879, 20903, 20907, 20992, 20908, 20751, 2781, 20133, 20807, 20805, 20802, 20804, 20898, 20916, 20748, 21031, 20817, 2742, 20745, 20738, 20821, 20644, 20961, 20963, 20784, 20743, 20747, 2720, 20637, 20574, 20588, 20822, 2], - decoded: " \ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764 \ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0 ", - }, - MULTIBYTE_EMOJIS: { - text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, - tokens: ["\u2728", "\ud83e\udd17", "", "\ufe0f", "\ud83d\udc71", "\ud83c\udffb", "\ud83d\udd75", "\u200d", "\u2642", "\ufe0f", "\ud83e\uddd9", "\ud83c\udffb", "\u200d", "\u2642", "\ud83d\udc68", "\ud83c\udffb", "\u200d", "\ud83c\udf3e", "\ud83e\uddd1", "\u200d", "\ud83e\udd1d", "\u200d", "\ud83e\uddd1", "\ud83d\udc69", "\u200d", "\u2764", "\u200d", "\ud83d\udc8b", "\u200d", "\ud83d\udc68", "\ud83d\udc69", "\u200d", "\ud83d\udc69", "\u200d", "\ud83d\udc67", "\u200d", "\ud83d\udc66", "\ud83e\uddd1", "\ud83c\udffb", "\u200d", "\ud83e\udd1d", "\u200d", "\ud83e\uddd1", "\ud83c\udffb", "\ud83c\udff4", "", "\udb40\udc62", "", "", "", "\udb40\udc7f", "\ud83d\udc68", "\ud83c\udffb", "\u200d", "\u2764", "\ufe0f", "\u200d", "\ud83d\udc8b", "\u200d", "\ud83d\udc68", "\ud83c\udffc"], - ids: [0, 2742, 20992, 3, 20133, 20775, 20700, 20894, 2067, 2662, 20133, 21050, 20700, 2067, 2662, 20768, 20700, 2067, 20593, 21049, 2067, 20995, 2067, 21049, 20769, 2067, 2781, 2067, 20792, 2067, 20768, 20769, 2067, 20769, 2067, 20767, 2067, 20766, 21049, 20700, 2067, 20995, 2067, 21049, 20700, 20697, 3, 21126, 3, 3, 3, 21130, 20768, 20700, 2067, 2781, 20133, 2067, 20792, 2067, 20768, 20701, 2], - decoded: " \u2728 \ud83e\udd17 \ufe0f \ud83d\udc71 \ud83c\udffb \ud83d\udd75 \u200d \u2642 \ufe0f \ud83e\uddd9 \ud83c\udffb \u200d \u2642 \ud83d\udc68 \ud83c\udffb \u200d \ud83c\udf3e \ud83e\uddd1 \u200d \ud83e\udd1d \u200d \ud83e\uddd1 \ud83d\udc69 \u200d \u2764 \u200d \ud83d\udc8b \u200d \ud83d\udc68 \ud83d\udc69 \u200d \ud83d\udc69 \u200d \ud83d\udc67 \u200d \ud83d\udc66 \ud83e\uddd1 \ud83c\udffb \u200d \ud83e\udd1d \u200d \ud83e\uddd1 \ud83c\udffb \ud83c\udff4 \udb40\udc62 \udb40\udc7f \ud83d\udc68 \ud83c\udffb \u200d \u2764 \ufe0f \u200d \ud83d\udc8b \u200d \ud83d\udc68 \ud83c\udffc ", - }, - CHINESE_LATIN_MIXED: { - text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, - tokens: ["ah", "\u535a", "\u63a8", "zz"], - ids: [0, 22311, 4352, 7628, 24387, 2], - decoded: " ah \u535a \u63a8 zz ", - }, - SIMPLE_WITH_ACCENTS: { - text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, - tokens: ["h", "\u00e9", "llo"], - ids: [0, 76, 173, 48932, 2], - decoded: " h \u00e9 llo ", - }, - MIXED_CASE_WITHOUT_ACCENTS: { - text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, - tokens: ["hello", "!", "how", "are", "you", "?"], - ids: [0, 28687, 31, 21431, 21182, 21166, 61, 2], - decoded: " hello! how are you? ", - }, - MIXED_CASE_WITH_ACCENTS: { - text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, - tokens: ["h", "\u00e4", "llo", "!", "how", "are", "you", "?"], - ids: [0, 76, 168, 48932, 31, 21431, 21182, 21166, 61, 2], - decoded: " h \u00e4 llo! how are you? ", - }, - }, - "jinaai/jina-embeddings-v2-base-zh": { - // https://huggingface.co/jinaai/jina-embeddings-v2-base-zh/discussions/16 - // Slow vs. fast tokenizer mismatch - CONTROL_TOKENS: { - text: BASE_TEST_STRINGS.CONTROL_TOKENS, - tokens: ["1", "\u0000", "2", "\ufffd", "3"], - ids: [0, 47, 5, 48, 20321, 49, 2], - decoded: " 1 \u0000 2 \ufffd 3 ", - }, - }, - "Xenova/all-distilroberta-v1": { - NUMBERS: { - text: BASE_TEST_STRINGS.NUMBERS, - tokens: ["01", "23", "45", "67", "89", "\u01200", "\u01201", "\u01202", "\u01203", "\u01204", "\u01205", "\u01206", "\u01207", "\u01208", "\u01209", "\u012010", "\u0120100", "\u01201000"], - ids: [0, 2663, 1922, 1898, 4111, 5046, 321, 112, 132, 155, 204, 195, 231, 262, 290, 361, 158, 727, 10775, 2], - decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u00e7\u0136\u0141", "\u00e6", "\u00b4", "\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e", "\u0141", "\u00e8", "\u00b0", "\u013d", "\u00e6\u013a\u00af"], - ids: [0, 48998, 37127, 20024, 2023, 44574, 49122, 4333, 36484, 7487, 3726, 48569, 2], - decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", - }, - LEADING_SPACE: { - text: BASE_TEST_STRINGS.LEADING_SPACE, - tokens: ["\u0120", "\u0120", "\u0120leading", "\u0120space"], - ids: [0, 1437, 1437, 981, 980, 2], - decoded: " leading space", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["tra", "iling", "\u0120space", "\u0120", "\u0120", "\u0120"], - ids: [0, 9738, 7022, 980, 1437, 1437, 1437, 2], - decoded: "trailing space ", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2", "\u0124", "\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120test"], - ids: [0, 21959, 68, 134, 248, 176, 849, 246, 4480, 306, 984, 245, 30844, 401, 14333, 9264, 2469, 406, 14333, 9264, 9253, 398, 14333, 9264, 15389, 466, 1296, 2], - decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - tokens: ["you", "\u00e2\u0122\u00a6", "\u0120", "\u0120"], - ids: [0, 6968, 1174, 1437, 1437, 2], - decoded: "you\u2026 ", - }, - POPULAR_EMOJIS: { - text: BASE_TEST_STRINGS.POPULAR_EMOJIS, - tokens: ["\u00f0\u0141\u013a", "\u0124", "\u0120\u00f0\u0141\u0133", "\u012f", "\u0120\u00f0\u0141", "\u00a4", "\u00a3", "\u0120\u00f0\u0141\u013a", "\u012f", "\u0120\u00f0\u0141\u013a", "\u0143", "\u0120\u00f0\u0141", "\u0130", "\u012b", "\u0120\u00f0\u0141", "\u013b", "\u0131", "\u0120\u00f0\u0141\u013a", "\u012c", "\u0120\u00f0\u0141", "\u0136", "\u00a5", "\u0120\u00f0\u0141\u013a", "\u0123", "\u0120\u00f0\u0141", "\u013a\u0127", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141\u013a", "\u0128", "\u0120\u00f0\u0141\u0133", "\u0131", "\u0120\u00e2\u013f", "\u00a4", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141", "\u0134", "\u013e", "\u0120\u00f0\u0141", "\u0134", "\u013c", "\u0120\u00f0\u0141", "\u0134", "\u0139", "\u0120\u00f0\u0141", "\u0134", "\u013b", "\u0120\u00f0\u0141", "\u0138", "\u00a4", "\u0120\u00f0\u0141\u013a", "\u0130", "\u0120\u00f0\u0141\u0133", "\u012e", "\u0120\u00f0\u0141", "\u00a5", "\u00b3", "\u0120\u00f0\u0141", "\u0134", "\u00aa", "\u0120\u00e2\u013e", "\u00a8", "\u0120\u00f0\u0141\u0133", "\u012b", "\u0120\u00f0\u0141\u0133", "\u0122", "\u0120\u00f0\u0141", "\u0134", "\u00af", "\u0120\u00f0\u0141", "\u0130", "\u012a", "\u0120\u00f0\u0141", "\u013b", "\u012a", "\u0120\u00f0\u0141", "\u013b", "\u012e", "\u0120\u00f0\u0141", "\u0134", "\u0122", "\u0120\u00f0\u0141\u0133", "\u0129", "\u0120\u00f0\u0141\u0133", "\u012d", "\u0120\u00e2\u013e", "\u0127", "\u0120\u00f0\u0141", "\u0130", "\u0123", "\u0120\u00f0\u0141", "\u012e", "\u0140", "\u0120\u00f0\u0141", "\u012e", "\u00b8", "\u0120\u00f0\u0141", "\u0134", "\u00b0"], - ids: [0, 18636, 9264, 26964, 8384, 8103, 10470, 2469, 17841, 8384, 17841, 12410, 8103, 12736, 23171, 8103, 27, 9357, 17841, 27969, 8103, 10674, 8210, 17841, 10172, 8103, 48278, 8103, 10470, 6800, 17841, 27819, 26964, 9357, 28775, 10470, 12605, 8103, 10659, 48, 8103, 10659, 15113, 8103, 10659, 6800, 8103, 10659, 27, 8103, 25448, 10470, 17841, 12736, 26964, 14285, 8103, 8210, 15264, 8103, 10659, 10278, 36174, 11423, 26964, 23171, 26964, 7471, 8103, 10659, 10965, 8103, 12736, 23133, 8103, 27, 23133, 8103, 27, 14285, 8103, 10659, 7471, 26964, 6382, 26964, 13859, 36174, 5782, 8103, 12736, 10172, 8103, 14285, 17772, 8103, 14285, 18537, 8103, 10659, 7487, 2], - decoded: "\ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0", - }, - MULTIBYTE_EMOJIS: { - text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, - tokens: ["\u00e2\u013e", "\u00a8", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141\u0133", "\u0123", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141\u0133", "\u00b1", "\u00f0\u0141", "\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0137", "\u00b5", "\u00e2\u0122", "\u012f", "\u00e2\u013b", "\u0124", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141", "\u00a7", "\u013b", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00e2\u013b", "\u0124", "\u0120\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u012e", "\u00be", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u0120\u00f0\u0141\u0133", "\u00a9", "\u00e2\u0122", "\u012f", "\u00e2\u013f", "\u00a4", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0134", "\u012d", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u0133", "\u00a8", "\u0120\u00f0\u0141\u0133", "\u00a9", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u0133", "\u00a9", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u0133", "\u00a7", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u0133", "\u00a6", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141", "\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0131", "\u00b4", "\u00f3", "\u0142", "\u0123", "\u00a7", "\u00f3", "\u0142", "\u0123", "\u00a2", "\u00f3", "\u0142", "\u0123", "\u00a5", "\u00f3", "\u0142", "\u0123", "\u00ae", "\u00f3", "\u0142", "\u0123", "\u00a7", "\u00f3", "\u0142", "\u0123", "\u00bf", "\u0120\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00e2\u013f", "\u00a4", "\u00ef\u00b8\u0131", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0134", "\u012d", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bc"], - ids: [0, 39817, 11423, 8103, 10470, 6800, 26964, 10172, 12605, 26964, 15389, 6569, 9357, 2023, 8103, 15722, 8906, 17, 8384, 38718, 9264, 12605, 8103, 6248, 27, 6569, 9357, 2023, 17, 8384, 38718, 9264, 26964, 11423, 6569, 9357, 2023, 17, 8384, 6569, 14285, 4726, 8103, 6248, 3602, 17, 8384, 6569, 10470, 46, 17, 8384, 6569, 6248, 3602, 26964, 15375, 17, 8384, 30151, 10470, 17, 8384, 6569, 10659, 13859, 17, 8384, 31193, 11423, 26964, 15375, 17, 8384, 31193, 15375, 17, 8384, 31193, 6248, 17, 8384, 31193, 18164, 8103, 6248, 3602, 6569, 9357, 2023, 17, 8384, 6569, 10470, 46, 17, 8384, 6569, 6248, 3602, 6569, 9357, 2023, 8103, 9357, 20024, 49078, 21402, 10172, 6248, 49078, 21402, 10172, 7258, 49078, 21402, 10172, 8210, 49078, 21402, 10172, 2840, 49078, 21402, 10172, 6248, 49078, 21402, 10172, 9470, 26964, 11423, 6569, 9357, 2023, 17, 8384, 30151, 10470, 12605, 17, 8384, 6569, 10659, 13859, 17, 8384, 31193, 11423, 6569, 9357, 4394, 2], - decoded: "\u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75\u200d\u2642\ufe0f \ud83e\uddd9\ud83c\udffb\u200d\u2642 \ud83d\udc68\ud83c\udffb\u200d\ud83c\udf3e \ud83e\uddd1\u200d\ud83e\udd1d\u200d\ud83e\uddd1 \ud83d\udc69\u200d\u2764\u200d\ud83d\udc8b\u200d\ud83d\udc68 \ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66 \ud83e\uddd1\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c\udffb \ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f \ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c\udffc", - }, - CHINESE_LATIN_MIXED: { - text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, - tokens: ["ah", "\u00e5\u012f", "\u013c", "\u00e6", "\u0130", "\u00a8", "zz"], - ids: [0, 895, 47658, 15113, 37127, 12736, 11423, 7399, 2], - decoded: "ah\u535a\u63a8zz", - }, - MIXED_CASE_WITHOUT_ACCENTS: { - text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, - tokens: ["\u0120", "\u0109", "He", "LL", "o", "!", "how", "\u0120", "\u0120", "\u010a", "\u0120Are", "\u0120yo", "U", "?", "\u0120", "\u0120"], - ids: [0, 1437, 50117, 894, 6006, 139, 328, 9178, 1437, 1437, 50118, 3945, 25610, 791, 116, 1437, 1437, 2], - decoded: " \tHeLLo!how \n Are yoU? ", - }, - MIXED_CASE_WITH_ACCENTS: { - text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, - tokens: ["\u0120", "\u0109", "H", "\u00c3\u00a4", "LL", "o", "!", "how", "\u0120", "\u0120", "\u010a", "\u0120Are", "\u0120yo", "U", "?", "\u0120", "\u0120"], - ids: [0, 1437, 50117, 725, 1561, 6006, 139, 328, 9178, 1437, 1437, 50118, 3945, 25610, 791, 116, 1437, 1437, 2], - decoded: " \tH\u00e4LLo!how \n Are yoU? ", - }, - }, - "Xenova/EsperBERTo-small-pos": { - SIMPLE: { - text: BASE_TEST_STRINGS.SIMPLE, - tokens: ["How", "\u0120are", "\u0120you", "\u0120do", "ing", "?"], - ids: [0, 50702, 1694, 12426, 661, 948, 35, 2], - decoded: "How are you doing?", - }, - SIMPLE_WITH_PUNCTUATION: { - text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, - tokens: ["You", "\u0120sho", "uld", "'ve", "\u0120don", "e", "\u0120this"], - ids: [0, 36894, 21906, 8512, 6091, 851, 73, 18955, 2], - decoded: "You should've done this", - }, - NUMBERS: { - text: BASE_TEST_STRINGS.NUMBERS, - tokens: ["0", "123456", "78", "9", "\u01200", "\u01201", "\u01202", "\u01203", "\u01204", "\u01205", "\u01206", "\u01207", "\u01208", "\u01209", "\u012010", "\u0120100", "\u01201000"], - ids: [0, 20, 11816, 6229, 29, 2042, 355, 411, 620, 818, 839, 1031, 1142, 1166, 1274, 1312, 2450, 8403, 2], - decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["The", "\u0120c", "ompan", "y", "\u0120was", "\u0120fo", "und", "ed", "\u0120in", "\u01202016", "."], - ids: [0, 5490, 467, 2833, 93, 30687, 1204, 3936, 347, 327, 3653, 18, 2], - decoded: "The company was founded in 2016.", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["A", "\u010a", "'ll", "\u0120!", "!", "to", "?", "'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'t", "."], - ids: [0, 37, 203, 17792, 4883, 5, 288, 35, 11, 72, 15271, 72, 682, 16, 15597, 3761, 18, 2], - decoded: "A\n'll!!to?'d''d of, can't.", - }, - PYTHON_CODE: { - text: BASE_TEST_STRINGS.PYTHON_CODE, - tokens: ["de", "f", "\u0120ma", "in", "(", "):", "\u010a", "\u0109", "pas", "s"], - ids: [0, 387, 74, 633, 282, 12, 3914, 203, 202, 1208, 87, 2], - decoded: "def main():\n\tpass", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["let", "\u0120a", "\u0120=", "\u0120ob", "j", ".", "to", "S", "tr", "ing", "(", ");", "\u010a", "to", "S", "tr", "ing", "(", ");"], - ids: [0, 5745, 278, 3945, 1080, 78, 18, 288, 55, 497, 948, 12, 3429, 203, 288, 55, 497, 948, 12, 3429, 2], - decoded: "let a = obj.toString();\ntoString();", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["UN", "w", "ant", "\u00c3\u00a9", "d", ",", "run", "ning"], - ids: [0, 7390, 91, 1799, 1174, 72, 16, 1307, 13715, 2], - decoded: "UNwant\u00e9d,running", - }, - HELLO_WORLD_TITLECASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, - tokens: ["Hel", "lo", "\u0120World"], - ids: [0, 4152, 310, 7717, 2], - decoded: "Hello World", - }, - HELLO_WORLD_LOWERCASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, - tokens: ["hel", "lo", "\u0120wor", "ld"], - ids: [0, 1686, 310, 39013, 3580, 2], - decoded: "hello world", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u00e7\u0136", "\u0141", "\u00e6", "\u00b4", "\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e", "\u0141", "\u00e8", "\u00b0", "\u013d", "\u00e6\u013a\u00af"], - ids: [0, 25856, 258, 167, 117, 124, 19584, 42803, 258, 169, 113, 254, 34946, 2], - decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", - }, - LEADING_SPACE: { - text: BASE_TEST_STRINGS.LEADING_SPACE, - tokens: ["\u0120\u0120", "\u0120le", "ading", "\u0120space"], - ids: [0, 2399, 591, 30214, 51965, 2], - decoded: " leading space", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["tra", "i", "ling", "\u0120space", "\u0120\u0120", "\u0120"], - ids: [0, 440, 77, 879, 51965, 2399, 225, 2], - decoded: "trailing space ", - }, - DOUBLE_SPACE: { - text: BASE_TEST_STRINGS.DOUBLE_SPACE, - tokens: ["Hi", "\u0120", "\u0120Hel", "lo"], - ids: [0, 15893, 225, 3558, 310, 2], - decoded: "Hi Hello", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2", "\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2", "\u0124", "\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120test"], - ids: [0, 30747, 9416, 21, 462, 22, 4668, 23, 7537, 24, 12407, 25, 790, 103, 26, 2097, 229, 101, 27, 2097, 229, 122, 28, 2097, 229, 114, 29, 18885, 2], - decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["I", "\u0120bo", "u", "ght", "\u0120an", "\u0120ap", "ple", "\u0120for", "\u0120$", "1", ".", "00", "\u0120at", "\u0120the", "\u0120s", "tore", "."], - ids: [0, 45, 1716, 89, 8840, 353, 560, 720, 434, 9416, 21, 18, 455, 3993, 2814, 275, 14003, 18, 2], - decoded: "I bought an apple for $1.00 at the store.", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142"], - ids: [0, 9642, 1322, 131, 259, 131, 259, 2], - decoded: "you\u2026\u00a0\u00a0", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142"], - ids: [0, 9642, 1322, 131, 259, 131, 259, 9642, 1322, 131, 259, 131, 259, 2], - decoded: "you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["we", "ir", "d", "\u0120", "\u00ef\u00bd", "\u0140", "\u0120ed", "ge", "\u0120", "\u00ef\u00bd", "\u0140", "\u0120c", "ase"], - ids: [0, 4983, 861, 72, 225, 30624, 257, 1263, 587, 225, 30624, 257, 467, 14285, 2], - decoded: "weird \uff5e edge \uff5e case", - }, - POPULAR_EMOJIS: { - text: BASE_TEST_STRINGS.POPULAR_EMOJIS, - tokens: ["\u00f0\u0141\u013a", "\u0124", "\u0120\u00f0\u0141", "\u0133", "\u012f", "\u0120\u00f0\u0141", "\u00a4", "\u00a3", "\u0120\u00f0\u0141\u013a", "\u012f", "\u0120\u00f0\u0141\u013a", "\u0143", "\u0120\u00f0\u0141", "\u0130", "\u012b", "\u0120\u00f0\u0141", "\u013b", "\u0131", "\u0120\u00f0\u0141\u013a", "\u012c", "\u0120\u00f0\u0141", "\u0136", "\u00a5", "\u0120\u00f0\u0141\u013a", "\u0123", "\u0120\u00f0\u0141\u013a", "\u0127", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141\u013a", "\u0128", "\u0120\u00f0\u0141", "\u0133", "\u0131", "\u0120\u00e2", "\u013f", "\u00a4", "\u00ef\u00b8", "\u0131", "\u0120\u00f0\u0141", "\u0134", "\u013e", "\u0120\u00f0\u0141", "\u0134", "\u013c", "\u0120\u00f0\u0141", "\u0134", "\u0139", "\u0120\u00f0\u0141", "\u0134", "\u013b", "\u0120\u00f0\u0141", "\u0138", "\u00a4", "\u0120\u00f0\u0141\u013a", "\u0130", "\u0120\u00f0\u0141", "\u0133", "\u012e", "\u0120\u00f0\u0141", "\u00a5", "\u00b3", "\u0120\u00f0\u0141", "\u0134", "\u00aa", "\u0120\u00e2", "\u013e", "\u00a8", "\u0120\u00f0\u0141", "\u0133", "\u012b", "\u0120\u00f0\u0141", "\u0133", "\u0122", "\u0120\u00f0\u0141", "\u0134", "\u00af", "\u0120\u00f0\u0141", "\u0130", "\u012a", "\u0120\u00f0\u0141", "\u013b", "\u012a", "\u0120\u00f0\u0141", "\u013b", "\u012e", "\u0120\u00f0\u0141", "\u0134", "\u0122", "\u0120\u00f0\u0141", "\u0133", "\u0129", "\u0120\u00f0\u0141", "\u0133", "\u012d", "\u0120\u00e2", "\u013e", "\u0127", "\u0120\u00f0\u0141", "\u0130", "\u0123", "\u0120\u00f0\u0141", "\u012e", "\u0140", "\u0120\u00f0\u0141", "\u012e", "\u00b8", "\u0120\u00f0\u0141", "\u0134", "\u00b0"], - ids: [0, 10626, 229, 32340, 244, 240, 32340, 102, 101, 30199, 240, 30199, 260, 32340, 241, 236, 32340, 252, 242, 30199, 237, 32340, 247, 103, 30199, 228, 30199, 232, 32340, 102, 250, 30199, 233, 32340, 244, 242, 2097, 256, 102, 27027, 242, 32340, 245, 255, 32340, 245, 253, 32340, 245, 250, 32340, 245, 252, 32340, 249, 102, 30199, 241, 32340, 244, 239, 32340, 103, 116, 32340, 245, 108, 2097, 255, 106, 32340, 244, 236, 32340, 244, 227, 32340, 245, 112, 32340, 241, 235, 32340, 252, 235, 32340, 252, 239, 32340, 245, 227, 32340, 244, 234, 32340, 244, 238, 2097, 255, 232, 32340, 241, 228, 32340, 239, 257, 32340, 239, 121, 32340, 245, 113, 2], - decoded: "\ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0", - }, - MULTIBYTE_EMOJIS: { - text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, - tokens: ["\u00e2", "\u013e", "\u00a8", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141", "\u0133", "\u0123", "\u00ef\u00b8", "\u0131", "\u0120\u00f0\u0141", "\u0133", "\u00b1", "\u00f0\u0141", "\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0137", "\u00b5", "\u00e2\u0122", "\u012f", "\u00e2\u013b", "\u0124", "\u00ef\u00b8", "\u0131", "\u0120\u00f0\u0141", "\u00a7", "\u013b", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00e2\u013b", "\u0124", "\u0120\u00f0\u0141", "\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u012e", "\u00be", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u0120\u00f0\u0141", "\u0133", "\u00a9", "\u00e2\u0122", "\u012f", "\u00e2", "\u013f", "\u00a4", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0134", "\u012d", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a8", "\u0120\u00f0\u0141", "\u0133", "\u00a9", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a9", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a7", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a6", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141", "\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0131", "\u00b4", "\u00f3", "\u0142", "\u0123", "\u00a7", "\u00f3", "\u0142", "\u0123", "\u00a2", "\u00f3", "\u0142", "\u0123", "\u00a5", "\u00f3", "\u0142", "\u0123", "\u00ae", "\u00f3", "\u0142", "\u0123", "\u00a7", "\u00f3", "\u0142", "\u0123", "\u00bf", "\u0120\u00f0\u0141", "\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00e2", "\u013f", "\u00a4", "\u00ef\u00b8", "\u0131", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0134", "\u012d", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bc"], - ids: [0, 163, 255, 106, 32340, 102, 250, 32340, 244, 228, 27027, 242, 32340, 244, 114, 9132, 242, 124, 32340, 248, 118, 348, 240, 20419, 229, 27027, 242, 32340, 105, 252, 9132, 242, 124, 348, 240, 20419, 229, 32340, 244, 106, 9132, 242, 124, 348, 240, 9132, 239, 127, 32340, 105, 244, 348, 240, 9132, 102, 256, 348, 240, 9132, 105, 244, 32340, 244, 107, 348, 240, 163, 256, 102, 348, 240, 9132, 245, 238, 348, 240, 9132, 244, 106, 32340, 244, 107, 348, 240, 9132, 244, 107, 348, 240, 9132, 244, 105, 348, 240, 9132, 244, 104, 32340, 105, 244, 9132, 242, 124, 348, 240, 9132, 102, 256, 348, 240, 9132, 105, 244, 9132, 242, 124, 32340, 242, 117, 180, 259, 228, 105, 180, 259, 228, 100, 180, 259, 228, 103, 180, 259, 228, 111, 180, 259, 228, 105, 180, 259, 228, 128, 32340, 244, 106, 9132, 242, 124, 348, 240, 163, 256, 102, 27027, 242, 348, 240, 9132, 245, 238, 348, 240, 9132, 244, 106, 9132, 242, 125, 2], - decoded: "\u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75\u200d\u2642\ufe0f \ud83e\uddd9\ud83c\udffb\u200d\u2642 \ud83d\udc68\ud83c\udffb\u200d\ud83c\udf3e \ud83e\uddd1\u200d\ud83e\udd1d\u200d\ud83e\uddd1 \ud83d\udc69\u200d\u2764\u200d\ud83d\udc8b\u200d\ud83d\udc68 \ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66 \ud83e\uddd1\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c\udffb \ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f \ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c\udffc", - }, - MIXED_CASE_WITHOUT_ACCENTS: { - text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, - tokens: ["\u0120", "\u0109", "He", "L", "Lo", "!", "ho", "w", "\u0120\u0120", "\u010a", "\u0120Are", "\u0120yo", "U", "?", "\u0120\u0120"], - ids: [0, 225, 202, 13029, 48, 4876, 5, 882, 91, 2399, 203, 31676, 27961, 57, 35, 2399, 2], - decoded: " \tHeLLo!how \n Are yoU? ", - }, - MIXED_CASE_WITH_ACCENTS: { - text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, - tokens: ["\u0120", "\u0109", "H", "\u00c3\u00a4", "L", "Lo", "!", "ho", "w", "\u0120\u0120", "\u010a", "\u0120Are", "\u0120yo", "U", "?", "\u0120\u0120"], - ids: [0, 225, 202, 44, 3203, 48, 4876, 5, 882, 91, 2399, 203, 31676, 27961, 57, 35, 2399, 2], - decoded: " \tH\u00e4LLo!how \n Are yoU? ", - }, - }, - // Split tokenizer with behavior="Removed" and invert=false - "onnx-community/camembertv2-base": { - SIMPLE: { - text: BASE_TEST_STRINGS.SIMPLE, - tokens: ['How', 'are', 'you', 'doi', '##ng', '?'], - ids: [1, 14473, 9556, 10577, 6471, 9274, 38, 2], - decoded: "[CLS] How are you doing? [SEP]", - } - }, + // Split tokenizer with behavior="Removed" and invert=false + "onnx-community/camembertv2-base": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["How", "are", "you", "doi", "##ng", "?"], + ids: [1, 14473, 9556, 10577, 6471, 9274, 38, 2], + decoded: "[CLS] How are you doing? [SEP]", + }, + }, }; diff --git a/tests/models/t5/tokenization.js b/tests/models/t5/tokenization.js index 43e7a0fd9..fdbf3ddc3 100644 --- a/tests/models/t5/tokenization.js +++ b/tests/models/t5/tokenization.js @@ -3,240 +3,240 @@ import { BASE_TEST_STRINGS, SENTENCEPIECE_TEST_STRINGS, T5_TEST_STRINGS } from " export const TOKENIZER_CLASS = T5Tokenizer; export const TEST_CONFIG = { - "Xenova/t5-small": { - SIMPLE: { - text: BASE_TEST_STRINGS.SIMPLE, - tokens: ["\u2581How", "\u2581are", "\u2581you", "\u2581doing", "?"], - ids: [571, 33, 25, 692, 58, 1], - decoded: "How are you doing?", - }, - SIMPLE_WITH_PUNCTUATION: { - text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, - tokens: ["\u2581You", "\u2581should", "'", "ve", "\u2581done", "\u2581this"], - ids: [148, 225, 31, 162, 612, 48, 1], - decoded: "You should've done this", - }, - NUMBERS: { - text: BASE_TEST_STRINGS.NUMBERS, - tokens: ["\u258101", "23", "45", "67", "89", "\u2581", "0", "\u25811", "\u25812", "\u25813", "\u25814", "\u25815", "\u25816", "\u25817", "\u25818", "\u25819", "\u258110", "\u2581100", "\u25811000"], - ids: [7088, 2773, 2128, 3708, 3914, 3, 632, 209, 204, 220, 314, 305, 431, 489, 505, 668, 335, 910, 5580, 1], - decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["\u2581The", "\u2581company", "\u2581was", "\u2581founded", "\u2581in", "\u25812016."], - ids: [37, 349, 47, 5710, 16, 4619, 1], - decoded: "The company was founded in 2016.", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["\u2581A", "\u2581", "'", "ll", "\u2581", "!!", "to", "?", "'", "d", "'", "'", "d", "\u2581of", ",", "\u2581can", "'", "t", "."], - ids: [71, 3, 31, 195, 3, 1603, 235, 58, 31, 26, 31, 31, 26, 13, 6, 54, 31, 17, 5, 1], - decoded: "A 'll!!to?'d''d of, can't.", - }, - PYTHON_CODE: { - text: BASE_TEST_STRINGS.PYTHON_CODE, - tokens: ["\u2581de", "f", "\u2581main", "()", ":", "\u2581pass"], - ids: [20, 89, 711, 9960, 10, 1903, 1], - decoded: "def main(): pass", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["\u2581let", "\u2581", "a", "\u2581=", "\u2581", "o", "b", "j", ".", "to", "Str", "ing", "()", ";", "\u2581to", "Str", "ing", "()", ";"], - ids: [752, 3, 9, 3274, 3, 32, 115, 354, 5, 235, 11500, 53, 9960, 117, 12, 11500, 53, 9960, 117, 1], - decoded: "let a = obj.toString(); toString();", - }, - NEWLINES: { - text: BASE_TEST_STRINGS.NEWLINES, - tokens: ["\u2581This", "\u2581is", "\u2581", "a", "\u2581test", "."], - ids: [100, 19, 3, 9, 794, 5, 1], - decoded: "This is a test.", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["\u2581UN", "wan", "t\u00e9", "d", ",", "running"], - ids: [4417, 3877, 2229, 26, 6, 24549, 1], - decoded: "UNwant\u00e9d,running", - }, - CONTROL_TOKENS: { - text: BASE_TEST_STRINGS.CONTROL_TOKENS, - tokens: ["\u25811", "\u0000", "2", "\u25813"], - ids: [209, 2, 357, 220, 1], - decoded: "12 3", - }, - HELLO_WORLD_TITLECASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, - tokens: ["\u2581Hello", "\u2581World"], - ids: [8774, 1150, 1], - decoded: "Hello World", - }, - HELLO_WORLD_LOWERCASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, - tokens: ["\u2581hello", "\u2581world"], - ids: [21820, 296, 1], - decoded: "hello world", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u2581", "\u751f\u6d3b\u7684\u771f\u8c1b\u662f"], - ids: [3, 2, 1], - decoded: "", - }, - LEADING_SPACE: { - text: BASE_TEST_STRINGS.LEADING_SPACE, - tokens: ["\u2581leading", "\u2581space"], - ids: [1374, 628, 1], - decoded: "leading space", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["\u2581trail", "ing", "\u2581space"], - ids: [5032, 53, 628, 1], - decoded: "trailing space", - }, - DOUBLE_SPACE: { - text: BASE_TEST_STRINGS.DOUBLE_SPACE, - tokens: ["\u2581Hi", "\u2581Hello"], - ids: [2018, 8774, 1], - decoded: "Hi Hello", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["\u2581test", "\u2581$1", "\u2581R", "2", "\u2581#3", "\u2581\u20ac", "4", "\u2581\u00a35", "\u2581", "\u00a5", "6", "\u2581", "\u20a3", "7", "\u2581", "\u20b9", "8", "\u2581", "\u20b1", "9", "\u2581test"], - ids: [794, 1970, 391, 357, 20206, 3416, 591, 23978, 3, 2, 948, 3, 2, 940, 3, 2, 927, 3, 2, 1298, 794, 1], - decoded: "test $1 R2 #3 \u20ac4 \u00a35 6 7 8 9 test", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["\u2581I", "\u2581bought", "\u2581an", "\u2581apple", "\u2581for", "\u2581$1", ".00", "\u2581at", "\u2581the", "\u2581store", "."], - ids: [27, 2944, 46, 8947, 21, 1970, 4200, 44, 8, 1078, 5, 1], - decoded: "I bought an apple for $1.00 at the store.", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - tokens: ["\u2581you", "..."], - ids: [25, 233, 1], - decoded: "you...", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - tokens: ["\u2581you", "..."], - ids: [25, 233, 1], - decoded: "you...", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - tokens: ["\u2581you", "...", "\u2581you", "..."], - ids: [25, 233, 25, 233, 1], - decoded: "you... you...", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["\u2581weird", "\u2581", "\uff5e", "\u2581edge", "\u2581", "\uff5e", "\u2581case"], - ids: [10088, 3, 2, 3023, 3, 2, 495, 1], - decoded: "weird edge case", - }, - SPIECE_UNDERSCORE: { - text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, - tokens: ["\u2581This", "\u2581is", "\u2581", "a", "\u2581test", "\u2581", "."], - ids: [100, 19, 3, 9, 794, 3, 5, 1], - decoded: "This is a test.", - }, - POPULAR_EMOJIS: { - text: BASE_TEST_STRINGS.POPULAR_EMOJIS, - tokens: ["\u2581", "\ud83d\ude02", "\u2581", "\ud83d\udc4d", "\u2581", "\ud83e\udd23", "\u2581", "\ud83d\ude0d", "\u2581", "\ud83d\ude2d", "\u2581", "\ud83c\udf89", "\u2581", "\ud83d\ude4f", "\u2581", "\ud83d\ude0a", "\u2581", "\ud83d\udd25", "\u2581", "\ud83d\ude01", "\u2581", "\ud83d\ude05", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\ude06", "\u2581", "\ud83d\udc4f", "\u2581", "\u2764\ufe0f", "\u2581", "\ud83d\udc9c", "\u2581", "\ud83d\udc9a", "\u2581", "\ud83d\udc97", "\u2581", "\ud83d\udc99", "\u2581", "\ud83d\udda4", "\u2581", "\ud83d\ude0e", "\u2581", "\ud83d\udc4c", "\u2581", "\ud83e\udd73", "\u2581", "\ud83d\udcaa", "\u2581", "\u2728", "\u2581", "\ud83d\udc49", "\u2581", "\ud83d\udc40", "\u2581", "\ud83d\udcaf", "\u2581", "\ud83c\udf88", "\u2581", "\ud83d\ude48", "\u2581", "\ud83d\ude4c", "\u2581", "\ud83d\udc80", "\u2581", "\ud83d\udc47", "\u2581", "\ud83d\udc4b", "\u2581", "\u2705", "\u2581", "\ud83c\udf81", "\u2581", "\ud83c\udf1e", "\u2581", "\ud83c\udf38", "\u2581", "\ud83d\udcb0"], - ids: [3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 1], - decoded: " ", - }, - MULTIBYTE_EMOJIS: { - text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, - tokens: ["\u2581", "\u2728", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\udc41\ufe0f", "\u2581", "\ud83d\udc71\ud83c\udffb", "\u2581", "\ud83d\udd75", "\u2581", "\u2642\ufe0f", "\u2581", "\ud83e\uddd9\ud83c\udffb", "\u2581", "\u2642", "\u2581", "\ud83d\udc68\ud83c\udffb", "\u2581", "\ud83c\udf3e", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83d\udc69", "\u2581", "\u2764", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc67", "\u2581", "\ud83d\udc66", "\u2581", "\ud83e\uddd1\ud83c\udffb", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1\ud83c\udffb", "\u2581", "\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f", "\u2581", "\ud83d\udc68\ud83c\udffb", "\u2581", "\u2764\ufe0f", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68\ud83c\udffc"], - ids: [3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 1], - decoded: " ", - }, - SPECIAL_WITH_TRAILING_WHITESPACE: { - text: SENTENCEPIECE_TEST_STRINGS.SPECIAL_WITH_TRAILING_WHITESPACE, - tokens: ["\u2581", "<", "s", ">"], - ids: [3, 2, 7, 3155, 1], - decoded: "s>", - }, - SPECIAL_SURROUNDED_BY_WHITESPACE: { - text: SENTENCEPIECE_TEST_STRINGS.SPECIAL_SURROUNDED_BY_WHITESPACE, - tokens: ["", "\u2581test", ""], - ids: [1, 794, 1, 1], - decoded: " test", - }, - SPECIAL_NO_WHITESPACE: { - text: SENTENCEPIECE_TEST_STRINGS.SPECIAL_NO_WHITESPACE, - tokens: ["", "\u2581test", ""], - ids: [1, 794, 1, 1], - decoded: " test", - }, - PREPEND_SCHEME: { - text: T5_TEST_STRINGS.PREPEND_SCHEME, - tokens: ["\u2581Hey", "", "\u2581", ".", "\u2581how", "\u2581are", "\u2581you"], - ids: [9459, 1, 3, 5, 149, 33, 25, 1], - decoded: "Hey. how are you", - }, - }, - "Xenova/t5-tokenizer-new": { - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["\u2581trail", "ing", "\u2581space", "\u2581"], - ids: [5032, 53, 628, 3, 1], - decoded: "trailing space ", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - tokens: ["\u2581you", "...", "\u2581"], - ids: [25, 233, 3, 1], - decoded: "you... ", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - tokens: ["\u2581you", "...", "\u2581"], - ids: [25, 233, 3, 1], - decoded: "you... ", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - tokens: ["\u2581you", "...", "\u2581you", "...", "\u2581"], - ids: [25, 233, 25, 233, 3, 1], - decoded: "you... you... ", - }, - SPECIAL_WITH_TRAILING_WHITESPACE: { - text: SENTENCEPIECE_TEST_STRINGS.SPECIAL_WITH_TRAILING_WHITESPACE, - tokens: ["\u2581", "<", "s", ">", "\u2581"], - ids: [3, 2, 7, 3155, 3, 1], - decoded: "s> ", - }, - SPECIAL_SURROUNDED_BY_WHITESPACE: { - text: SENTENCEPIECE_TEST_STRINGS.SPECIAL_SURROUNDED_BY_WHITESPACE, - tokens: ["\u2581", "", "\u2581test", "\u2581", "", "\u2581"], - ids: [3, 1, 794, 3, 1, 3, 1], - decoded: " test ", - }, - SPECIAL_NO_WHITESPACE: { - text: SENTENCEPIECE_TEST_STRINGS.SPECIAL_NO_WHITESPACE, - tokens: ["", "test", ""], - ids: [1, 4377, 1, 1], - decoded: "test", + "Xenova/t5-small": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["\u2581How", "\u2581are", "\u2581you", "\u2581doing", "?"], + ids: [571, 33, 25, 692, 58, 1], + decoded: "How are you doing?", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["\u2581You", "\u2581should", "'", "ve", "\u2581done", "\u2581this"], + ids: [148, 225, 31, 162, 612, 48, 1], + decoded: "You should've done this", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["\u258101", "23", "45", "67", "89", "\u2581", "0", "\u25811", "\u25812", "\u25813", "\u25814", "\u25815", "\u25816", "\u25817", "\u25818", "\u25819", "\u258110", "\u2581100", "\u25811000"], + ids: [7088, 2773, 2128, 3708, 3914, 3, 632, 209, 204, 220, 314, 305, 431, 489, 505, 668, 335, 910, 5580, 1], + decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["\u2581The", "\u2581company", "\u2581was", "\u2581founded", "\u2581in", "\u25812016."], + ids: [37, 349, 47, 5710, 16, 4619, 1], + decoded: "The company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["\u2581A", "\u2581", "'", "ll", "\u2581", "!!", "to", "?", "'", "d", "'", "'", "d", "\u2581of", ",", "\u2581can", "'", "t", "."], + ids: [71, 3, 31, 195, 3, 1603, 235, 58, 31, 26, 31, 31, 26, 13, 6, 54, 31, 17, 5, 1], + decoded: "A 'll!!to?'d''d of, can't.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["\u2581de", "f", "\u2581main", "()", ":", "\u2581pass"], + ids: [20, 89, 711, 9960, 10, 1903, 1], + decoded: "def main(): pass", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["\u2581let", "\u2581", "a", "\u2581=", "\u2581", "o", "b", "j", ".", "to", "Str", "ing", "()", ";", "\u2581to", "Str", "ing", "()", ";"], + ids: [752, 3, 9, 3274, 3, 32, 115, 354, 5, 235, 11500, 53, 9960, 117, 12, 11500, 53, 9960, 117, 1], + decoded: "let a = obj.toString(); toString();", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["\u2581This", "\u2581is", "\u2581", "a", "\u2581test", "."], + ids: [100, 19, 3, 9, 794, 5, 1], + decoded: "This is a test.", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["\u2581UN", "wan", "t\u00e9", "d", ",", "running"], + ids: [4417, 3877, 2229, 26, 6, 24549, 1], + decoded: "UNwant\u00e9d,running", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["\u25811", "\u0000", "2", "\u25813"], + ids: [209, 2, 357, 220, 1], + decoded: "12 3", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["\u2581Hello", "\u2581World"], + ids: [8774, 1150, 1], + decoded: "Hello World", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["\u2581hello", "\u2581world"], + ids: [21820, 296, 1], + decoded: "hello world", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u2581", "\u751f\u6d3b\u7684\u771f\u8c1b\u662f"], + ids: [3, 2, 1], + decoded: "", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u2581leading", "\u2581space"], + ids: [1374, 628, 1], + decoded: "leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["\u2581trail", "ing", "\u2581space"], + ids: [5032, 53, 628, 1], + decoded: "trailing space", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["\u2581Hi", "\u2581Hello"], + ids: [2018, 8774, 1], + decoded: "Hi Hello", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["\u2581test", "\u2581$1", "\u2581R", "2", "\u2581#3", "\u2581\u20ac", "4", "\u2581\u00a35", "\u2581", "\u00a5", "6", "\u2581", "\u20a3", "7", "\u2581", "\u20b9", "8", "\u2581", "\u20b1", "9", "\u2581test"], + ids: [794, 1970, 391, 357, 20206, 3416, 591, 23978, 3, 2, 948, 3, 2, 940, 3, 2, 927, 3, 2, 1298, 794, 1], + decoded: "test $1 R2 #3 \u20ac4 \u00a35 6 7 8 9 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["\u2581I", "\u2581bought", "\u2581an", "\u2581apple", "\u2581for", "\u2581$1", ".00", "\u2581at", "\u2581the", "\u2581store", "."], + ids: [27, 2944, 46, 8947, 21, 1970, 4200, 44, 8, 1078, 5, 1], + decoded: "I bought an apple for $1.00 at the store.", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["\u2581you", "..."], + ids: [25, 233, 1], + decoded: "you...", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["\u2581you", "..."], + ids: [25, 233, 1], + decoded: "you...", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["\u2581you", "...", "\u2581you", "..."], + ids: [25, 233, 25, 233, 1], + decoded: "you... you...", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["\u2581weird", "\u2581", "\uff5e", "\u2581edge", "\u2581", "\uff5e", "\u2581case"], + ids: [10088, 3, 2, 3023, 3, 2, 495, 1], + decoded: "weird edge case", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u2581This", "\u2581is", "\u2581", "a", "\u2581test", "\u2581", "."], + ids: [100, 19, 3, 9, 794, 3, 5, 1], + decoded: "This is a test.", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u2581", "\ud83d\ude02", "\u2581", "\ud83d\udc4d", "\u2581", "\ud83e\udd23", "\u2581", "\ud83d\ude0d", "\u2581", "\ud83d\ude2d", "\u2581", "\ud83c\udf89", "\u2581", "\ud83d\ude4f", "\u2581", "\ud83d\ude0a", "\u2581", "\ud83d\udd25", "\u2581", "\ud83d\ude01", "\u2581", "\ud83d\ude05", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\ude06", "\u2581", "\ud83d\udc4f", "\u2581", "\u2764\ufe0f", "\u2581", "\ud83d\udc9c", "\u2581", "\ud83d\udc9a", "\u2581", "\ud83d\udc97", "\u2581", "\ud83d\udc99", "\u2581", "\ud83d\udda4", "\u2581", "\ud83d\ude0e", "\u2581", "\ud83d\udc4c", "\u2581", "\ud83e\udd73", "\u2581", "\ud83d\udcaa", "\u2581", "\u2728", "\u2581", "\ud83d\udc49", "\u2581", "\ud83d\udc40", "\u2581", "\ud83d\udcaf", "\u2581", "\ud83c\udf88", "\u2581", "\ud83d\ude48", "\u2581", "\ud83d\ude4c", "\u2581", "\ud83d\udc80", "\u2581", "\ud83d\udc47", "\u2581", "\ud83d\udc4b", "\u2581", "\u2705", "\u2581", "\ud83c\udf81", "\u2581", "\ud83c\udf1e", "\u2581", "\ud83c\udf38", "\u2581", "\ud83d\udcb0"], + ids: [3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 1], + decoded: " ", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u2581", "\u2728", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\udc41\ufe0f", "\u2581", "\ud83d\udc71\ud83c\udffb", "\u2581", "\ud83d\udd75", "\u2581", "\u2642\ufe0f", "\u2581", "\ud83e\uddd9\ud83c\udffb", "\u2581", "\u2642", "\u2581", "\ud83d\udc68\ud83c\udffb", "\u2581", "\ud83c\udf3e", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83d\udc69", "\u2581", "\u2764", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc67", "\u2581", "\ud83d\udc66", "\u2581", "\ud83e\uddd1\ud83c\udffb", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1\ud83c\udffb", "\u2581", "\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f", "\u2581", "\ud83d\udc68\ud83c\udffb", "\u2581", "\u2764\ufe0f", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68\ud83c\udffc"], + ids: [3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 1], + decoded: " ", + }, + SPECIAL_WITH_TRAILING_WHITESPACE: { + text: SENTENCEPIECE_TEST_STRINGS.SPECIAL_WITH_TRAILING_WHITESPACE, + tokens: ["\u2581", "<", "s", ">"], + ids: [3, 2, 7, 3155, 1], + decoded: "s>", + }, + SPECIAL_SURROUNDED_BY_WHITESPACE: { + text: SENTENCEPIECE_TEST_STRINGS.SPECIAL_SURROUNDED_BY_WHITESPACE, + tokens: ["", "\u2581test", ""], + ids: [1, 794, 1, 1], + decoded: " test", + }, + SPECIAL_NO_WHITESPACE: { + text: SENTENCEPIECE_TEST_STRINGS.SPECIAL_NO_WHITESPACE, + tokens: ["", "\u2581test", ""], + ids: [1, 794, 1, 1], + decoded: " test", + }, + PREPEND_SCHEME: { + text: T5_TEST_STRINGS.PREPEND_SCHEME, + tokens: ["\u2581Hey", "", "\u2581", ".", "\u2581how", "\u2581are", "\u2581you"], + ids: [9459, 1, 3, 5, 149, 33, 25, 1], + decoded: "Hey. how are you", + }, }, - PREPEND_SCHEME: { - text: T5_TEST_STRINGS.PREPEND_SCHEME, - tokens: ["\u2581Hey", "\u2581", "", ".", "\u2581how", "\u2581are", "\u2581you"], - ids: [9459, 3, 1, 5, 149, 33, 25, 1], - decoded: "Hey . how are you", + "Xenova/t5-tokenizer-new": { + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["\u2581trail", "ing", "\u2581space", "\u2581"], + ids: [5032, 53, 628, 3, 1], + decoded: "trailing space ", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["\u2581you", "...", "\u2581"], + ids: [25, 233, 3, 1], + decoded: "you... ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["\u2581you", "...", "\u2581"], + ids: [25, 233, 3, 1], + decoded: "you... ", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["\u2581you", "...", "\u2581you", "...", "\u2581"], + ids: [25, 233, 25, 233, 3, 1], + decoded: "you... you... ", + }, + SPECIAL_WITH_TRAILING_WHITESPACE: { + text: SENTENCEPIECE_TEST_STRINGS.SPECIAL_WITH_TRAILING_WHITESPACE, + tokens: ["\u2581", "<", "s", ">", "\u2581"], + ids: [3, 2, 7, 3155, 3, 1], + decoded: "s> ", + }, + SPECIAL_SURROUNDED_BY_WHITESPACE: { + text: SENTENCEPIECE_TEST_STRINGS.SPECIAL_SURROUNDED_BY_WHITESPACE, + tokens: ["\u2581", "", "\u2581test", "\u2581", "", "\u2581"], + ids: [3, 1, 794, 3, 1, 3, 1], + decoded: " test ", + }, + SPECIAL_NO_WHITESPACE: { + text: SENTENCEPIECE_TEST_STRINGS.SPECIAL_NO_WHITESPACE, + tokens: ["", "test", ""], + ids: [1, 4377, 1, 1], + decoded: "test", + }, + PREPEND_SCHEME: { + text: T5_TEST_STRINGS.PREPEND_SCHEME, + tokens: ["\u2581Hey", "\u2581", "", ".", "\u2581how", "\u2581are", "\u2581you"], + ids: [9459, 3, 1, 5, 149, 33, 25, 1], + decoded: "Hey . how are you", + }, }, - }, - "Xenova/LaMini-Flan-T5-783M": { - PREPEND_SCHEME: { - text: T5_TEST_STRINGS.PREPEND_SCHEME, - tokens: ["\u2581Hey", "\u2581", "", "\u2581", ".", "\u2581how", "\u2581are", "\u2581you"], - ids: [9459, 3, 1, 3, 5, 149, 33, 25, 1], - decoded: "Hey . how are you", + "Xenova/LaMini-Flan-T5-783M": { + PREPEND_SCHEME: { + text: T5_TEST_STRINGS.PREPEND_SCHEME, + tokens: ["\u2581Hey", "\u2581", "", "\u2581", ".", "\u2581how", "\u2581are", "\u2581you"], + ids: [9459, 3, 1, 3, 5, 149, 33, 25, 1], + decoded: "Hey . how are you", + }, }, - }, }; // Test that tokenizer type can be inferred (`type: "Unigram"` is missing) @@ -244,53 +244,55 @@ TEST_CONFIG["google-t5/t5-small"] = TEST_CONFIG["Xenova/t5-small"]; const MAX_EXECUTION_TIME = 10_000; export const CUSTOM_TESTS = () => { - // Tests to ensure that no matter what, the correct tokenization is returned. - // This is necessary since there are sometimes bugs in the transformers library. - describe("hard-coded", () => { - const TESTS = { - // legacy=false - "Xenova/t5-tokenizer-new": [ - { - data: { - // https://github.com/huggingface/transformers/pull/26678 - // ['▁Hey', '▁', '', '.', '▁how', '▁are', '▁you'] - "Hey . how are you": [9459, 3, 1, 5, 149, 33, 25], - }, - reversible: true, - legacy: null, - }, - { - data: { - "\n": [1, 3], - "A\n'll": [71, 3, 31, 195], - }, - reversible: false, - legacy: null, - }, - ], - }; + // Tests to ensure that no matter what, the correct tokenization is returned. + // This is necessary since there are sometimes bugs in the transformers library. + describe("hard-coded", () => { + const TESTS = { + // legacy=false + "Xenova/t5-tokenizer-new": [ + { + data: { + // https://github.com/huggingface/transformers/pull/26678 + // ['▁Hey', '▁', '', '.', '▁how', '▁are', '▁you'] + "Hey . how are you": [9459, 3, 1, 5, 149, 33, 25], + }, + reversible: true, + legacy: null, + }, + { + data: { + "\n": [1, 3], + "A\n'll": [71, 3, 31, 195], + }, + reversible: false, + legacy: null, + }, + ], + }; - for (const [tokenizerName, test_data] of Object.entries(TESTS)) { - it( - tokenizerName, - async () => { - for (const { data, reversible, legacy } of test_data) { - const tokenizer = await T5Tokenizer.from_pretrained(tokenizerName, { legacy }); + for (const [tokenizerName, test_data] of Object.entries(TESTS)) { + it( + tokenizerName, + async () => { + for (const { data, reversible, legacy } of test_data) { + const tokenizer = await T5Tokenizer.from_pretrained(tokenizerName, { + legacy, + }); - for (const [text, expected] of Object.entries(data)) { - const token_ids = tokenizer.encode(text, { add_special_tokens: false }); - expect(token_ids).toEqual(expected); + for (const [text, expected] of Object.entries(data)) { + const token_ids = tokenizer.encode(text, { add_special_tokens: false }); + expect(token_ids).toEqual(expected); - // If reversible, test that decoding produces the original text - if (reversible) { - const decoded = tokenizer.decode(token_ids); - expect(decoded).toEqual(text); - } - } - } - }, - MAX_EXECUTION_TIME, - ); - } - }); + // If reversible, test that decoding produces the original text + if (reversible) { + const decoded = tokenizer.decode(token_ids); + expect(decoded).toEqual(text); + } + } + } + }, + MAX_EXECUTION_TIME, + ); + } + }); }; diff --git a/tests/models/test_strings.js b/tests/models/test_strings.js index 80793c9ad..cf915db3d 100644 --- a/tests/models/test_strings.js +++ b/tests/models/test_strings.js @@ -1,115 +1,115 @@ export const BASE_TEST_STRINGS = { - SIMPLE: "How are you doing?", - SIMPLE_WITH_PUNCTUATION: "You should've done this", - NUMBERS: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", - TEXT_WITH_NUMBERS: "The company was founded in 2016.", - PUNCTUATION: "A\n'll !!to?'d''d of, can't.", - PYTHON_CODE: "def main():\n\tpass", - JAVASCRIPT_CODE: "let a = obj.toString();\ntoString();", - NEWLINES: "This\n\nis\na\ntest.", - BASIC: "UNwant\u00e9d,running", - CONTROL_TOKENS: "1\u00002\uFFFD3", - HELLO_WORLD_TITLECASE: "Hello World", - HELLO_WORLD_LOWERCASE: "hello world", - CHINESE_ONLY: "生活的真谛是", - LEADING_SPACE: " leading space", - TRAILING_SPACE: "trailing space ", - SURROUNDING_SPACE: " surrounding space ", - DOUBLE_SPACE: "Hi Hello", - CURRENCY: "test $1 R2 #3 €4 £5 ¥6 ₣7 ₹8 ₱9 test", - CURRENCY_WITH_DECIMALS: "I bought an apple for $1.00 at the store.", - ELLIPSIS: "you… ", - TEXT_WITH_ESCAPE_CHARACTERS: "\u0079\u006F\u0075\u2026\u00A0\u00A0", - TEXT_WITH_ESCAPE_CHARACTERS_2: "\u0079\u006F\u0075\u2026\u00A0\u00A0\u0079\u006F\u0075\u2026\u00A0\u00A0", - TILDE_NORMALIZATION: "weird \uFF5E edge \uFF5E case", - SPIECE_UNDERSCORE: "▁This ▁is ▁a ▁test ▁.", - POPULAR_EMOJIS: "😂 👍 🤣 😍 😭 🎉 🙏 😊 🔥 😁 😅 🤗 😆 👏 ❤️ 💜 💚 💗 💙 🖤 😎 👌 🥳 💪 ✨ 👉 👀 💯 🎈 🙈 🙌 💀 👇 👋 ✅ 🎁 🌞 🌸 💰", - MULTIBYTE_EMOJIS: "✨ 🤗 👁️ 👱🏻 🕵‍♂️ 🧙🏻‍♂ 👨🏻‍🌾 🧑‍🤝‍🧑 👩‍❤‍💋‍👨 👩‍👩‍👧‍👦 🧑🏻‍🤝‍🧑🏻 🏴󠁧󠁢󠁥󠁮󠁧󠁿 👨🏻‍❤️‍💋‍👨🏼", // 1 2 3 4 5 6 7 8 10 11 12 14 15 - ONLY_WHITESPACE: " \t\n", + SIMPLE: "How are you doing?", + SIMPLE_WITH_PUNCTUATION: "You should've done this", + NUMBERS: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + TEXT_WITH_NUMBERS: "The company was founded in 2016.", + PUNCTUATION: "A\n'll !!to?'d''d of, can't.", + PYTHON_CODE: "def main():\n\tpass", + JAVASCRIPT_CODE: "let a = obj.toString();\ntoString();", + NEWLINES: "This\n\nis\na\ntest.", + BASIC: "UNwant\u00e9d,running", + CONTROL_TOKENS: "1\u00002\uFFFD3", + HELLO_WORLD_TITLECASE: "Hello World", + HELLO_WORLD_LOWERCASE: "hello world", + CHINESE_ONLY: "生活的真谛是", + LEADING_SPACE: " leading space", + TRAILING_SPACE: "trailing space ", + SURROUNDING_SPACE: " surrounding space ", + DOUBLE_SPACE: "Hi Hello", + CURRENCY: "test $1 R2 #3 €4 £5 ¥6 ₣7 ₹8 ₱9 test", + CURRENCY_WITH_DECIMALS: "I bought an apple for $1.00 at the store.", + ELLIPSIS: "you… ", + TEXT_WITH_ESCAPE_CHARACTERS: "\u0079\u006F\u0075\u2026\u00A0\u00A0", + TEXT_WITH_ESCAPE_CHARACTERS_2: "\u0079\u006F\u0075\u2026\u00A0\u00A0\u0079\u006F\u0075\u2026\u00A0\u00A0", + TILDE_NORMALIZATION: "weird \uFF5E edge \uFF5E case", + SPIECE_UNDERSCORE: "▁This ▁is ▁a ▁test ▁.", + POPULAR_EMOJIS: "😂 👍 🤣 😍 😭 🎉 🙏 😊 🔥 😁 😅 🤗 😆 👏 ❤️ 💜 💚 💗 💙 🖤 😎 👌 🥳 💪 ✨ 👉 👀 💯 🎈 🙈 🙌 💀 👇 👋 ✅ 🎁 🌞 🌸 💰", + MULTIBYTE_EMOJIS: "✨ 🤗 👁️ 👱🏻 🕵‍♂️ 🧙🏻‍♂ 👨🏻‍🌾 🧑‍🤝‍🧑 👩‍❤‍💋‍👨 👩‍👩‍👧‍👦 🧑🏻‍🤝‍🧑🏻 🏴󠁧󠁢󠁥󠁮󠁧󠁿 👨🏻‍❤️‍💋‍👨🏼", // 1 2 3 4 5 6 7 8 10 11 12 14 15 + ONLY_WHITESPACE: " \t\n", }; export const BERT_TEST_STRINGS = { - CHINESE_LATIN_MIXED: "ah\u535a\u63a8zz", - SIMPLE_WITH_ACCENTS: "H\u00e9llo", - MIXED_CASE_WITHOUT_ACCENTS: " \tHeLLo!how \n Are yoU? ", - MIXED_CASE_WITH_ACCENTS: " \tHäLLo!how \n Are yoU? ", + CHINESE_LATIN_MIXED: "ah\u535a\u63a8zz", + SIMPLE_WITH_ACCENTS: "H\u00e9llo", + MIXED_CASE_WITHOUT_ACCENTS: " \tHeLLo!how \n Are yoU? ", + MIXED_CASE_WITH_ACCENTS: " \tHäLLo!how \n Are yoU? ", }; // SentencePiece-specific test cases export const SENTENCEPIECE_TEST_STRINGS = { - SPECIAL_WITH_TRAILING_WHITESPACE: "\n", - SPECIAL_SURROUNDED_BY_WHITESPACE: " test ", - SPECIAL_NO_WHITESPACE: "test", + SPECIAL_WITH_TRAILING_WHITESPACE: "\n", + SPECIAL_SURROUNDED_BY_WHITESPACE: " test ", + SPECIAL_NO_WHITESPACE: "test", }; // Additional test-cases for the Llama tokenizer, adapted from // https://github.com/belladoreai/llama-tokenizer-js/blob/master/llama-tokenizer.js#L381-L452 export const LLAMA_TEST_STRINGS = { - BPE_SCORES_PRIORITY_1: "grabbed", - BPE_SCORES_PRIORITY_2: " grabbed", - BPE_SCORES_PRIORITY_3: " grabbed", - NEWLINE: "\n", - NEWLINES: "ax\n####\nboo", - NEWLINE_WITH_LEADING_SPACE: " \n", - TABS: " tabs out here", - NEWLINE_AND_TAB: "\n\t\n", - CHINESE_LETTER: "镇", - EMOJIS_1: "🦙", - EMOJIS_2: "🦙Ꙋ", - EMOJIS_3: "Ꙋ🦙", - PARAGRAPH: 'The llama (/ˈlɑːmə/; 🦙Spanish pronunciation: [ˈʎama]) (Lama glama) is a domesticated South American camelid, widely used as a meat and pack animal by Andean cultures since the Pre-Columbian era. Llamas are social animals and live with others as a herd. Their wool is soft and contains only a small amount of lanolin.[2] Llamas can learn simple tasks after a few repetitions. When using a pack, they can carry about 25 to 30% of their body weight for 8 to 13 km (5–8 miles).[3] The name llama (in the past also spelled "lama" or "glama") was adopted by European settlers from native Peruvians.[4] The ancestors of llamas are thought to have originated from the Great Plains of North America about 40 million years ago, and subsequently migrated to South America about three million years ago during the Great American Interchange. By the end of the last ice age (10,000–12,000 years ago), camelids were extinct in North America.[3] As of 2007, there were over seven million llamas and alpacas in South America and over 158,000 llamas and 100,000Ꙋ🦙 alpacas, descended from progenitors imported late in the 20th century, in the United States and Canada.[5] In Aymara mythology, llamas are important beings. The Heavenly Llama is said to drink water from the ocean and urinates as it rains.[6] According to Aymara eschatology, llamas will return to the water springs and lagoons where they come from at the end of time.[6]', - IGNORE_MERGES: "Ne için gittiğimi falan bilmiyordum, Washington'da belirtilen bir yere rapor vermem gerekiyordu.", + BPE_SCORES_PRIORITY_1: "grabbed", + BPE_SCORES_PRIORITY_2: " grabbed", + BPE_SCORES_PRIORITY_3: " grabbed", + NEWLINE: "\n", + NEWLINES: "ax\n####\nboo", + NEWLINE_WITH_LEADING_SPACE: " \n", + TABS: " tabs out here", + NEWLINE_AND_TAB: "\n\t\n", + CHINESE_LETTER: "镇", + EMOJIS_1: "🦙", + EMOJIS_2: "🦙Ꙋ", + EMOJIS_3: "Ꙋ🦙", + PARAGRAPH: 'The llama (/ˈlɑːmə/; 🦙Spanish pronunciation: [ˈʎama]) (Lama glama) is a domesticated South American camelid, widely used as a meat and pack animal by Andean cultures since the Pre-Columbian era. Llamas are social animals and live with others as a herd. Their wool is soft and contains only a small amount of lanolin.[2] Llamas can learn simple tasks after a few repetitions. When using a pack, they can carry about 25 to 30% of their body weight for 8 to 13 km (5–8 miles).[3] The name llama (in the past also spelled "lama" or "glama") was adopted by European settlers from native Peruvians.[4] The ancestors of llamas are thought to have originated from the Great Plains of North America about 40 million years ago, and subsequently migrated to South America about three million years ago during the Great American Interchange. By the end of the last ice age (10,000–12,000 years ago), camelids were extinct in North America.[3] As of 2007, there were over seven million llamas and alpacas in South America and over 158,000 llamas and 100,000Ꙋ🦙 alpacas, descended from progenitors imported late in the 20th century, in the United States and Canada.[5] In Aymara mythology, llamas are important beings. The Heavenly Llama is said to drink water from the ocean and urinates as it rains.[6] According to Aymara eschatology, llamas will return to the water springs and lagoons where they come from at the end of time.[6]', + IGNORE_MERGES: "Ne için gittiğimi falan bilmiyordum, Washington'da belirtilen bir yere rapor vermem gerekiyordu.", }; export const VITS_TEST_STRINGS = { - BASIC: "abcdefghijklmnopqrstuvwxyz01234567890", - // Special treatment of characters in certain language - SPECIAL_CHARACTERS: "ț ţ", + BASIC: "abcdefghijklmnopqrstuvwxyz01234567890", + // Special treatment of characters in certain language + SPECIAL_CHARACTERS: "ț ţ", }; export const QWEN_TEST_STRINGS = { - PUNCTUATION_SPLIT: "i'm i'M i've i've i'Ve i'vE i'VE", + PUNCTUATION_SPLIT: "i'm i'M i've i've i'Ve i'vE i'VE", }; export const WHISPER_TEST_STRINGS = { - SPECIAL_TOKENS: " <|startoftranscript|> <|en|> ", // Tests lstrip+rstrip + SPECIAL_TOKENS: " <|startoftranscript|> <|en|> ", // Tests lstrip+rstrip }; export const BLENDERBOT_SMALL_TEST_STRINGS = { - SPECIAL_TOKENS: "__start__hello world__end__", - // The original (python) tokenizer simply joins by spaces (regardless of special tokens or not) - WHITESPACE_1: "__start__ hey __end__", // --> ... --> "__start__ hey __end__" - WHITESPACE_2: "__start__hey __end__", // --> ... --> "__start__ hey __end__" + SPECIAL_TOKENS: "__start__hello world__end__", + // The original (python) tokenizer simply joins by spaces (regardless of special tokens or not) + WHITESPACE_1: "__start__ hey __end__", // --> ... --> "__start__ hey __end__" + WHITESPACE_2: "__start__hey __end__", // --> ... --> "__start__ hey __end__" }; export const T5_TEST_STRINGS = { - // Tests the new T5 tokenizer, which uses a different prepend_scheme for its pre_tokenizer: - // tokenizer._tokenizer.pre_tokenizer = Metaspace(add_prefix_space = True, replacement = "▁", prepend_scheme = "first") - // See https://github.com/huggingface/transformers/pull/26678 for more information. - // - Old (incorrect): ['▁Hey', '▁', '', '▁', '.', '▁how', '▁are', '▁you'] - // - New (correct): ['▁Hey', '▁', '', '.', '▁how', '▁are', '▁you'] - PREPEND_SCHEME: "Hey . how are you", + // Tests the new T5 tokenizer, which uses a different prepend_scheme for its pre_tokenizer: + // tokenizer._tokenizer.pre_tokenizer = Metaspace(add_prefix_space = True, replacement = "▁", prepend_scheme = "first") + // See https://github.com/huggingface/transformers/pull/26678 for more information. + // - Old (incorrect): ['▁Hey', '▁', '', '▁', '.', '▁how', '▁are', '▁you'] + // - New (correct): ['▁Hey', '▁', '', '.', '▁how', '▁are', '▁you'] + PREPEND_SCHEME: "Hey . how are you", }; export const FALCON_TEST_STRINGS = { - // Special case for splitting on 3 numbers - NUMBERS_SPLIT: "12 and 123 and 1234", + // Special case for splitting on 3 numbers + NUMBERS_SPLIT: "12 and 123 and 1234", }; export const ESM_TEST_STRINGS = { - // Special tokens - SPECIAL_TOKENS: "", - // Actual protein sequences - PROTEIN_SEQUENCES_1: "ATTCCGATTCCGATTCCG", - PROTEIN_SEQUENCES_2: "ATTTCTCTCTCTCTCTGAGATCGATCGATCGAT", + // Special tokens + SPECIAL_TOKENS: "", + // Actual protein sequences + PROTEIN_SEQUENCES_1: "ATTCCGATTCCGATTCCG", + PROTEIN_SEQUENCES_2: "ATTTCTCTCTCTCTCTGAGATCGATCGATCGAT", }; export const BLOOM_TEST_STRINGS = { - END_OF_SENTENCE_PUNCTUATION: "test. test, test! test? test… test。 test， test、 test। test۔ test، test", + END_OF_SENTENCE_PUNCTUATION: "test. test, test! test? test… test。 test， test、 test। test۔ test، test", }; export const M2M_100_TEST_STRINGS = { - TRANSLATION_INPUTS: "__en__ hello world", - HIDNI_TEXT: "जीवन एक चॉकलेट बॉक्स की तरह है।", - CHINESE_TEXT: "生活就像一盒巧克力。", + TRANSLATION_INPUTS: "__en__ hello world", + HIDNI_TEXT: "जीवन एक चॉकलेट बॉक्स की तरह है।", + CHINESE_TEXT: "生活就像一盒巧克力。", }; diff --git a/tests/models/vits/tokenization.js b/tests/models/vits/tokenization.js index e051d6bf7..64aad406c 100644 --- a/tests/models/vits/tokenization.js +++ b/tests/models/vits/tokenization.js @@ -3,74 +3,74 @@ import { BASE_TEST_STRINGS, VITS_TEST_STRINGS } from "../test_strings.js"; export const TOKENIZER_CLASS = VitsTokenizer; export const TEST_CONFIG = { - "Xenova/mms-tts-eng": { - SIMPLE: { - text: BASE_TEST_STRINGS.SIMPLE, - tokens: ["k", "h", "k", "o", "k", "w", "k", " ", "k", "a", "k", "r", "k", "e", "k", " ", "k", "y", "k", "o", "k", "u", "k", " ", "k", "d", "k", "o", "k", "i", "k", "n", "k", "g", "k"], - ids: [0, 6, 0, 22, 0, 9, 0, 19, 0, 26, 0, 25, 0, 7, 0, 19, 0, 3, 0, 22, 0, 4, 0, 19, 0, 5, 0, 22, 0, 18, 0, 29, 0, 37, 0], - decoded: "how are you doing", + "Xenova/mms-tts-eng": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["k", "h", "k", "o", "k", "w", "k", " ", "k", "a", "k", "r", "k", "e", "k", " ", "k", "y", "k", "o", "k", "u", "k", " ", "k", "d", "k", "o", "k", "i", "k", "n", "k", "g", "k"], + ids: [0, 6, 0, 22, 0, 9, 0, 19, 0, 26, 0, 25, 0, 7, 0, 19, 0, 3, 0, 22, 0, 4, 0, 19, 0, 5, 0, 22, 0, 18, 0, 29, 0, 37, 0], + decoded: "how are you doing", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["k", "y", "k", "o", "k", "u", "k", " ", "k", "s", "k", "h", "k", "o", "k", "u", "k", "l", "k", "d", "k", "'", "k", "v", "k", "e", "k", " ", "k", "d", "k", "o", "k", "n", "k", "e", "k", " ", "k", "t", "k", "h", "k", "i", "k", "s", "k"], + ids: [0, 3, 0, 22, 0, 4, 0, 19, 0, 8, 0, 6, 0, 22, 0, 4, 0, 21, 0, 5, 0, 1, 0, 32, 0, 7, 0, 19, 0, 5, 0, 22, 0, 29, 0, 7, 0, 19, 0, 33, 0, 6, 0, 18, 0, 8, 0], + decoded: "you should've done this", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["k", "0", "k", "1", "k", "2", "k", "3", "k", "4", "k", "5", "k", "6", "k", " ", "k", "0", "k", " ", "k", "1", "k", " ", "k", "2", "k", " ", "k", "3", "k", " ", "k", "4", "k", " ", "k", "5", "k", " ", "k", "6", "k", " ", "k", " ", "k", " ", "k", " ", "k", "1", "k", "0", "k", " ", "k", "1", "k", "0", "k", "0", "k", " ", "k", "1", "k", "0", "k", "0", "k", "0", "k"], + ids: [0, 23, 0, 15, 0, 28, 0, 11, 0, 27, 0, 35, 0, 36, 0, 19, 0, 23, 0, 19, 0, 15, 0, 19, 0, 28, 0, 19, 0, 11, 0, 19, 0, 27, 0, 19, 0, 35, 0, 19, 0, 36, 0, 19, 0, 19, 0, 19, 0, 19, 0, 15, 0, 23, 0, 19, 0, 15, 0, 23, 0, 23, 0, 19, 0, 15, 0, 23, 0, 23, 0, 23, 0], + decoded: "0123456 0 1 2 3 4 5 6 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["k", "t", "k", "h", "k", "e", "k", " ", "k", "c", "k", "o", "k", "m", "k", "p", "k", "a", "k", "n", "k", "y", "k", " ", "k", "w", "k", "a", "k", "s", "k", " ", "k", "f", "k", "o", "k", "u", "k", "n", "k", "d", "k", "e", "k", "d", "k", " ", "k", "i", "k", "n", "k", " ", "k", "2", "k", "0", "k", "1", "k", "6", "k"], + ids: [0, 33, 0, 6, 0, 7, 0, 19, 0, 12, 0, 22, 0, 17, 0, 13, 0, 26, 0, 29, 0, 3, 0, 19, 0, 9, 0, 26, 0, 8, 0, 19, 0, 20, 0, 22, 0, 4, 0, 29, 0, 5, 0, 7, 0, 5, 0, 19, 0, 18, 0, 29, 0, 19, 0, 28, 0, 23, 0, 15, 0, 36, 0], + decoded: "the company was founded in 2016", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["k", "t", "k", "h", "k", "i", "k", "s", "k", "i", "k", "s", "k", "a", "k", "t", "k", "e", "k", "s", "k", "t", "k"], + ids: [0, 33, 0, 6, 0, 18, 0, 8, 0, 18, 0, 8, 0, 26, 0, 33, 0, 7, 0, 8, 0, 33, 0], + decoded: "thisisatest", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: [], + ids: [], + decoded: "", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["k", "l", "k", "e", "k", "a", "k", "d", "k", "i", "k", "n", "k", "g", "k", " ", "k", "s", "k", "p", "k", "a", "k", "c", "k", "e", "k"], + ids: [0, 21, 0, 7, 0, 26, 0, 5, 0, 18, 0, 29, 0, 37, 0, 19, 0, 8, 0, 13, 0, 26, 0, 12, 0, 7, 0], + decoded: "leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["k", "t", "k", "r", "k", "a", "k", "i", "k", "l", "k", "i", "k", "n", "k", "g", "k", " ", "k", "s", "k", "p", "k", "a", "k", "c", "k", "e", "k"], + ids: [0, 33, 0, 25, 0, 26, 0, 18, 0, 21, 0, 18, 0, 29, 0, 37, 0, 19, 0, 8, 0, 13, 0, 26, 0, 12, 0, 7, 0], + decoded: "trailing space", + }, + SURROUNDING_SPACE: { + text: BASE_TEST_STRINGS.SURROUNDING_SPACE, + tokens: ["k", "s", "k", "u", "k", "r", "k", "r", "k", "o", "k", "u", "k", "n", "k", "d", "k", "i", "k", "n", "k", "g", "k", " ", "k", "s", "k", "p", "k", "a", "k", "c", "k", "e", "k"], + ids: [0, 8, 0, 4, 0, 25, 0, 25, 0, 22, 0, 4, 0, 29, 0, 5, 0, 18, 0, 29, 0, 37, 0, 19, 0, 8, 0, 13, 0, 26, 0, 12, 0, 7, 0], + decoded: "surrounding space", + }, + SPECIAL_CHARACTERS: { + text: VITS_TEST_STRINGS.SPECIAL_CHARACTERS, + tokens: [], + ids: [], + decoded: "", + }, }, - SIMPLE_WITH_PUNCTUATION: { - text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, - tokens: ["k", "y", "k", "o", "k", "u", "k", " ", "k", "s", "k", "h", "k", "o", "k", "u", "k", "l", "k", "d", "k", "'", "k", "v", "k", "e", "k", " ", "k", "d", "k", "o", "k", "n", "k", "e", "k", " ", "k", "t", "k", "h", "k", "i", "k", "s", "k"], - ids: [0, 3, 0, 22, 0, 4, 0, 19, 0, 8, 0, 6, 0, 22, 0, 4, 0, 21, 0, 5, 0, 1, 0, 32, 0, 7, 0, 19, 0, 5, 0, 22, 0, 29, 0, 7, 0, 19, 0, 33, 0, 6, 0, 18, 0, 8, 0], - decoded: "you should've done this", + "Xenova/mms-tts-ron": { + SPECIAL_CHARACTERS: { + text: VITS_TEST_STRINGS.SPECIAL_CHARACTERS, + tokens: ["c", "\u0163", "c", " ", "c", "\u0163", "c"], + ids: [0, 32, 0, 28, 0, 32, 0], + decoded: "\u0163 \u0163", + }, }, - NUMBERS: { - text: BASE_TEST_STRINGS.NUMBERS, - tokens: ["k", "0", "k", "1", "k", "2", "k", "3", "k", "4", "k", "5", "k", "6", "k", " ", "k", "0", "k", " ", "k", "1", "k", " ", "k", "2", "k", " ", "k", "3", "k", " ", "k", "4", "k", " ", "k", "5", "k", " ", "k", "6", "k", " ", "k", " ", "k", " ", "k", " ", "k", "1", "k", "0", "k", " ", "k", "1", "k", "0", "k", "0", "k", " ", "k", "1", "k", "0", "k", "0", "k", "0", "k"], - ids: [0, 23, 0, 15, 0, 28, 0, 11, 0, 27, 0, 35, 0, 36, 0, 19, 0, 23, 0, 19, 0, 15, 0, 19, 0, 28, 0, 19, 0, 11, 0, 19, 0, 27, 0, 19, 0, 35, 0, 19, 0, 36, 0, 19, 0, 19, 0, 19, 0, 19, 0, 15, 0, 23, 0, 19, 0, 15, 0, 23, 0, 23, 0, 19, 0, 15, 0, 23, 0, 23, 0, 23, 0], - decoded: "0123456 0 1 2 3 4 5 6 10 100 1000", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["k", "t", "k", "h", "k", "e", "k", " ", "k", "c", "k", "o", "k", "m", "k", "p", "k", "a", "k", "n", "k", "y", "k", " ", "k", "w", "k", "a", "k", "s", "k", " ", "k", "f", "k", "o", "k", "u", "k", "n", "k", "d", "k", "e", "k", "d", "k", " ", "k", "i", "k", "n", "k", " ", "k", "2", "k", "0", "k", "1", "k", "6", "k"], - ids: [0, 33, 0, 6, 0, 7, 0, 19, 0, 12, 0, 22, 0, 17, 0, 13, 0, 26, 0, 29, 0, 3, 0, 19, 0, 9, 0, 26, 0, 8, 0, 19, 0, 20, 0, 22, 0, 4, 0, 29, 0, 5, 0, 7, 0, 5, 0, 19, 0, 18, 0, 29, 0, 19, 0, 28, 0, 23, 0, 15, 0, 36, 0], - decoded: "the company was founded in 2016", - }, - NEWLINES: { - text: BASE_TEST_STRINGS.NEWLINES, - tokens: ["k", "t", "k", "h", "k", "i", "k", "s", "k", "i", "k", "s", "k", "a", "k", "t", "k", "e", "k", "s", "k", "t", "k"], - ids: [0, 33, 0, 6, 0, 18, 0, 8, 0, 18, 0, 8, 0, 26, 0, 33, 0, 7, 0, 8, 0, 33, 0], - decoded: "thisisatest", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: [], - ids: [], - decoded: "", - }, - LEADING_SPACE: { - text: BASE_TEST_STRINGS.LEADING_SPACE, - tokens: ["k", "l", "k", "e", "k", "a", "k", "d", "k", "i", "k", "n", "k", "g", "k", " ", "k", "s", "k", "p", "k", "a", "k", "c", "k", "e", "k"], - ids: [0, 21, 0, 7, 0, 26, 0, 5, 0, 18, 0, 29, 0, 37, 0, 19, 0, 8, 0, 13, 0, 26, 0, 12, 0, 7, 0], - decoded: "leading space", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["k", "t", "k", "r", "k", "a", "k", "i", "k", "l", "k", "i", "k", "n", "k", "g", "k", " ", "k", "s", "k", "p", "k", "a", "k", "c", "k", "e", "k"], - ids: [0, 33, 0, 25, 0, 26, 0, 18, 0, 21, 0, 18, 0, 29, 0, 37, 0, 19, 0, 8, 0, 13, 0, 26, 0, 12, 0, 7, 0], - decoded: "trailing space", - }, - SURROUNDING_SPACE: { - text: BASE_TEST_STRINGS.SURROUNDING_SPACE, - tokens: ["k", "s", "k", "u", "k", "r", "k", "r", "k", "o", "k", "u", "k", "n", "k", "d", "k", "i", "k", "n", "k", "g", "k", " ", "k", "s", "k", "p", "k", "a", "k", "c", "k", "e", "k"], - ids: [0, 8, 0, 4, 0, 25, 0, 25, 0, 22, 0, 4, 0, 29, 0, 5, 0, 18, 0, 29, 0, 37, 0, 19, 0, 8, 0, 13, 0, 26, 0, 12, 0, 7, 0], - decoded: "surrounding space", - }, - SPECIAL_CHARACTERS: { - text: VITS_TEST_STRINGS.SPECIAL_CHARACTERS, - tokens: [], - ids: [], - decoded: "", - }, - }, - "Xenova/mms-tts-ron": { - SPECIAL_CHARACTERS: { - text: VITS_TEST_STRINGS.SPECIAL_CHARACTERS, - tokens: ["c", "\u0163", "c", " ", "c", "\u0163", "c"], - ids: [0, 32, 0, 28, 0, 32, 0], - decoded: "\u0163 \u0163", - }, - }, }; diff --git a/tests/models/wav2vec2/tokenization.js b/tests/models/wav2vec2/tokenization.js index e6798f0b0..eff804b38 100644 --- a/tests/models/wav2vec2/tokenization.js +++ b/tests/models/wav2vec2/tokenization.js @@ -3,470 +3,470 @@ import { BASE_TEST_STRINGS, BERT_TEST_STRINGS } from "../test_strings.js"; export const TOKENIZER_CLASS = Wav2Vec2CTCTokenizer; export const TEST_CONFIG = { - "Xenova/wav2vec2-base-960h": { - SIMPLE: { - text: BASE_TEST_STRINGS.SIMPLE, - tokens: ["H", "o", "w", "|", "a", "r", "e", "|", "y", "o", "u", "|", "d", "o", "i", "n", "g", "?"], - ids: [11, 3, 3, 4, 3, 3, 3, 4, 3, 3, 3, 4, 3, 3, 3, 3, 3, 3], - decoded: "H ", + "Xenova/wav2vec2-base-960h": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["H", "o", "w", "|", "a", "r", "e", "|", "y", "o", "u", "|", "d", "o", "i", "n", "g", "?"], + ids: [11, 3, 3, 4, 3, 3, 3, 4, 3, 3, 3, 4, 3, 3, 3, 3, 3, 3], + decoded: "H ", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["Y", "o", "u", "|", "s", "h", "o", "u", "l", "d", "'", "v", "e", "|", "d", "o", "n", "e", "|", "t", "h", "i", "s"], + ids: [22, 3, 3, 4, 3, 3, 3, 3, 3, 3, 27, 3, 3, 4, 3, 3, 3, 3, 4, 3, 3, 3, 3], + decoded: "Y ' ", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "|", "0", "|", "1", "|", "2", "|", "3", "|", "4", "|", "5", "|", "6", "|", "7", "|", "8", "|", "9", "|", "1", "0", "|", "1", "0", "0", "|", "1", "0", "0", "0"], + ids: [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 3, 4, 3, 3, 3, 4, 3, 3, 3, 3], + decoded: " ", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["T", "h", "e", "|", "c", "o", "m", "p", "a", "n", "y", "|", "w", "a", "s", "|", "f", "o", "u", "n", "d", "e", "d", "|", "i", "n", "|", "2", "0", "1", "6", "."], + ids: [6, 3, 3, 4, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 3, 4, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 4, 3, 3, 3, 3, 3], + decoded: "T ", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "\n", "'", "l", "l", "|", "!", "!", "t", "o", "?", "'", "d", "'", "'", "d", "|", "o", "f", ",", "|", "c", "a", "n", "'", "t", "."], + ids: [7, 3, 27, 3, 3, 4, 3, 3, 3, 3, 3, 27, 3, 27, 27, 3, 4, 3, 3, 3, 4, 3, 3, 3, 27, 3, 3], + decoded: "A' '' '", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["d", "e", "f", "|", "m", "a", "i", "n", "(", ")", ":", "\n", "\t", "p", "a", "s", "s"], + ids: [3, 3, 3, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3], + decoded: " ", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["l", "e", "t", "|", "a", "|", "=", "|", "o", "b", "j", ".", "t", "o", "S", "t", "r", "i", "n", "g", "(", ")", ";", "\n", "t", "o", "S", "t", "r", "i", "n", "g", "(", ")", ";"], + ids: [3, 3, 3, 4, 3, 4, 3, 4, 3, 3, 3, 3, 3, 3, 12, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 12, 3, 3, 3, 3, 3, 3, 3, 3], + decoded: " SS", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["T", "h", "i", "s", "\n", "\n", "i", "s", "\n", "a", "\n", "t", "e", "s", "t", "."], + ids: [6, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3], + decoded: "T", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["U", "N", "w", "a", "n", "t", "\u00e9", "d", ",", "r", "u", "n", "n", "i", "n", "g"], + ids: [16, 9, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3], + decoded: "UN", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["1", "\u0000", "2", "\ufffd", "3"], + ids: [3, 3, 3, 3, 3], + decoded: "", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["H", "e", "l", "l", "o", "|", "W", "o", "r", "l", "d"], + ids: [11, 3, 3, 3, 3, 4, 18, 3, 3, 3, 3], + decoded: "H W", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["h", "e", "l", "l", "o", "|", "w", "o", "r", "l", "d"], + ids: [3, 3, 3, 3, 3, 4, 3, 3, 3, 3, 3], + decoded: " ", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u751f", "\u6d3b", "\u7684", "\u771f", "\u8c1b", "\u662f"], + ids: [3, 3, 3, 3, 3, 3], + decoded: "", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["|", "|", "|", "l", "e", "a", "d", "i", "n", "g", "|", "s", "p", "a", "c", "e"], + ids: [4, 4, 4, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 3, 3, 3], + decoded: " ", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["t", "r", "a", "i", "l", "i", "n", "g", "|", "s", "p", "a", "c", "e", "|", "|", "|"], + ids: [3, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 3, 3, 3, 4, 4, 4], + decoded: " ", + }, + SURROUNDING_SPACE: { + text: BASE_TEST_STRINGS.SURROUNDING_SPACE, + tokens: ["|", "|", "|", "s", "u", "r", "r", "o", "u", "n", "d", "i", "n", "g", "|", "s", "p", "a", "c", "e", "|", "|", "|"], + ids: [4, 4, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 3, 3, 3, 4, 4, 4], + decoded: " ", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["H", "i", "|", "|", "H", "e", "l", "l", "o"], + ids: [11, 3, 4, 4, 11, 3, 3, 3, 3], + decoded: "H H", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["t", "e", "s", "t", "|", "$", "1", "|", "R", "2", "|", "#", "3", "|", "\u20ac", "4", "|", "\u00a3", "5", "|", "\u00a5", "6", "|", "\u20a3", "7", "|", "\u20b9", "8", "|", "\u20b1", "9", "|", "t", "e", "s", "t"], + ids: [3, 3, 3, 3, 4, 3, 3, 4, 13, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 3, 3], + decoded: " R ", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "|", "b", "o", "u", "g", "h", "t", "|", "a", "n", "|", "a", "p", "p", "l", "e", "|", "f", "o", "r", "|", "$", "1", ".", "0", "0", "|", "a", "t", "|", "t", "h", "e", "|", "s", "t", "o", "r", "e", "."], + ids: [10, 4, 3, 3, 3, 3, 3, 3, 4, 3, 3, 4, 3, 3, 3, 3, 3, 4, 3, 3, 3, 4, 3, 3, 3, 3, 3, 4, 3, 3, 4, 3, 3, 3, 4, 3, 3, 3, 3, 3, 3], + decoded: "I ", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["y", "o", "u", "\u2026", "|", "|"], + ids: [3, 3, 3, 3, 4, 4], + decoded: "", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["y", "o", "u", "\u2026", "\u00a0", "\u00a0"], + ids: [3, 3, 3, 3, 3, 3], + decoded: "", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["y", "o", "u", "\u2026", "\u00a0", "\u00a0", "y", "o", "u", "\u2026", "\u00a0", "\u00a0"], + ids: [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3], + decoded: "", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["w", "e", "i", "r", "d", "|", "\uff5e", "|", "e", "d", "g", "e", "|", "\uff5e", "|", "c", "a", "s", "e"], + ids: [3, 3, 3, 3, 3, 4, 3, 4, 3, 3, 3, 3, 4, 3, 4, 3, 3, 3, 3], + decoded: " ", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u2581", "T", "h", "i", "s", "|", "\u2581", "i", "s", "|", "\u2581", "a", "|", "\u2581", "t", "e", "s", "t", "|", "\u2581", "."], + ids: [3, 6, 3, 3, 3, 4, 3, 3, 3, 4, 3, 3, 4, 3, 3, 3, 3, 3, 4, 3, 3], + decoded: "T ", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\ud83d\ude02", "|", "\ud83d\udc4d", "|", "\ud83e\udd23", "|", "\ud83d\ude0d", "|", "\ud83d\ude2d", "|", "\ud83c\udf89", "|", "\ud83d\ude4f", "|", "\ud83d\ude0a", "|", "\ud83d\udd25", "|", "\ud83d\ude01", "|", "\ud83d\ude05", "|", "\ud83e\udd17", "|", "\ud83d\ude06", "|", "\ud83d\udc4f", "|", "\u2764", "\ufe0f", "|", "\ud83d\udc9c", "|", "\ud83d\udc9a", "|", "\ud83d\udc97", "|", "\ud83d\udc99", "|", "\ud83d\udda4", "|", "\ud83d\ude0e", "|", "\ud83d\udc4c", "|", "\ud83e\udd73", "|", "\ud83d\udcaa", "|", "\u2728", "|", "\ud83d\udc49", "|", "\ud83d\udc40", "|", "\ud83d\udcaf", "|", "\ud83c\udf88", "|", "\ud83d\ude48", "|", "\ud83d\ude4c", "|", "\ud83d\udc80", "|", "\ud83d\udc47", "|", "\ud83d\udc4b", "|", "\u2705", "|", "\ud83c\udf81", "|", "\ud83c\udf1e", "|", "\ud83c\udf38", "|", "\ud83d\udcb0"], + ids: [3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3], + decoded: " ", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u2728", "|", "\ud83e\udd17", "|", "\ud83d\udc41", "\ufe0f", "|", "\ud83d\udc71", "\ud83c\udffb", "|", "\ud83d\udd75", "\u200d", "\u2642", "\ufe0f", "|", "\ud83e\uddd9", "\ud83c\udffb", "\u200d", "\u2642", "|", "\ud83d\udc68", "\ud83c\udffb", "\u200d", "\ud83c\udf3e", "|", "\ud83e\uddd1", "\u200d", "\ud83e\udd1d", "\u200d", "\ud83e\uddd1", "|", "\ud83d\udc69", "\u200d", "\u2764", "\u200d", "\ud83d\udc8b", "\u200d", "\ud83d\udc68", "|", "\ud83d\udc69", "\u200d", "\ud83d\udc69", "\u200d", "\ud83d\udc67", "\u200d", "\ud83d\udc66", "|", "\ud83e\uddd1", "\ud83c\udffb", "\u200d", "\ud83e\udd1d", "\u200d", "\ud83e\uddd1", "\ud83c\udffb", "|", "\ud83c\udff4", "\udb40\udc67", "\udb40\udc62", "\udb40\udc65", "\udb40\udc6e", "\udb40\udc67", "\udb40\udc7f", "|", "\ud83d\udc68", "\ud83c\udffb", "\u200d", "\u2764", "\ufe0f", "\u200d", "\ud83d\udc8b", "\u200d", "\ud83d\udc68", "\ud83c\udffc"], + ids: [3, 4, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 3, 3, 4, 3, 3, 3, 3, 4, 3, 3, 3, 3, 4, 3, 3, 3, 3, 3, 4, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3], + decoded: " ", + }, + ONLY_WHITESPACE: { + text: BASE_TEST_STRINGS.ONLY_WHITESPACE, + tokens: ["|", "\t", "\n"], + ids: [4, 3, 3], + decoded: "", + }, + CHINESE_LATIN_MIXED: { + text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, + tokens: ["a", "h", "\u535a", "\u63a8", "z", "z"], + ids: [3, 3, 3, 3, 3, 3], + decoded: "", + }, + SIMPLE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, + tokens: ["H", "\u00e9", "l", "l", "o"], + ids: [11, 3, 3, 3, 3], + decoded: "H", + }, + MIXED_CASE_WITHOUT_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, + tokens: ["|", "\t", "H", "e", "L", "L", "o", "!", "h", "o", "w", "|", "|", "\n", "|", "A", "r", "e", "|", "y", "o", "U", "?", "|", "|"], + ids: [4, 3, 11, 3, 15, 15, 3, 3, 3, 3, 3, 4, 4, 3, 4, 7, 3, 3, 4, 3, 3, 16, 3, 4, 4], + decoded: "HL A U", + }, + MIXED_CASE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, + tokens: ["|", "\t", "H", "\u00e4", "L", "L", "o", "!", "h", "o", "w", "|", "|", "\n", "|", "A", "r", "e", "|", "y", "o", "U", "?", "|", "|"], + ids: [4, 3, 11, 3, 15, 15, 3, 3, 3, 3, 3, 4, 4, 3, 4, 7, 3, 3, 4, 3, 3, 16, 3, 4, 4], + decoded: "HL A U", + }, + }, + "Xenova/wav2vec2-large-xlsr-53-english": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["H", "o", "w", "|", "a", "r", "e", "|", "y", "o", "u", "|", "d", "o", "i", "n", "g", "?"], + ids: [3, 21, 29, 4, 7, 24, 11, 4, 31, 21, 27, 4, 10, 21, 15, 20, 13, 3], + decoded: "ow are you doing", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["Y", "o", "u", "|", "s", "h", "o", "u", "l", "d", "'", "v", "e", "|", "d", "o", "n", "e", "|", "t", "h", "i", "s"], + ids: [3, 21, 27, 4, 25, 14, 21, 27, 18, 10, 5, 28, 11, 4, 10, 21, 20, 11, 4, 26, 14, 15, 25], + decoded: "ou should've done this", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["T", "h", "e", "|", "c", "o", "m", "p", "a", "n", "y", "|", "w", "a", "s", "|", "f", "o", "u", "n", "d", "e", "d", "|", "i", "n", "|", "2", "0", "1", "6", "."], + ids: [3, 14, 11, 4, 9, 21, 19, 22, 7, 20, 31, 4, 29, 7, 25, 4, 12, 21, 27, 20, 10, 11, 10, 4, 15, 20, 4, 3, 3, 3, 3, 3], + decoded: "he company was founded in ", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "\n", "'", "l", "l", "|", "!", "!", "t", "o", "?", "'", "d", "'", "'", "d", "|", "o", "f", ",", "|", "c", "a", "n", "'", "t", "."], + ids: [3, 3, 5, 18, 18, 4, 3, 3, 26, 21, 3, 5, 10, 5, 5, 10, 4, 21, 12, 3, 4, 9, 7, 20, 5, 26, 3], + decoded: "'l to'd'd of can't", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["d", "e", "f", "|", "m", "a", "i", "n", "(", ")", ":", "\n", "\t", "p", "a", "s", "s"], + ids: [10, 11, 12, 4, 19, 7, 15, 20, 3, 3, 3, 3, 3, 22, 7, 25, 25], + decoded: "def mainpas", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["l", "e", "t", "|", "a", "|", "=", "|", "o", "b", "j", ".", "t", "o", "S", "t", "r", "i", "n", "g", "(", ")", ";", "\n", "t", "o", "S", "t", "r", "i", "n", "g", "(", ")", ";"], + ids: [18, 11, 26, 4, 7, 4, 3, 4, 21, 8, 16, 3, 26, 21, 3, 26, 24, 15, 20, 13, 3, 3, 3, 3, 26, 21, 3, 26, 24, 15, 20, 13, 3, 3, 3], + decoded: "let a objtotringtotring", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["T", "h", "i", "s", "\n", "\n", "i", "s", "\n", "a", "\n", "t", "e", "s", "t", "."], + ids: [3, 14, 15, 25, 3, 3, 15, 25, 3, 7, 3, 26, 11, 25, 26, 3], + decoded: "hisisatest", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["U", "N", "w", "a", "n", "t", "\u00e9", "d", ",", "r", "u", "n", "n", "i", "n", "g"], + ids: [3, 3, 29, 7, 20, 26, 3, 10, 3, 24, 27, 20, 20, 15, 20, 13], + decoded: "wantdruning", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["H", "e", "l", "l", "o", "|", "W", "o", "r", "l", "d"], + ids: [3, 11, 18, 18, 21, 4, 3, 21, 24, 18, 10], + decoded: "elo orld", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["h", "e", "l", "l", "o", "|", "w", "o", "r", "l", "d"], + ids: [14, 11, 18, 18, 21, 4, 29, 21, 24, 18, 10], + decoded: "helo world", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["|", "|", "|", "l", "e", "a", "d", "i", "n", "g", "|", "s", "p", "a", "c", "e"], + ids: [4, 4, 4, 18, 11, 7, 10, 15, 20, 13, 4, 25, 22, 7, 9, 11], + decoded: "leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["t", "r", "a", "i", "l", "i", "n", "g", "|", "s", "p", "a", "c", "e", "|", "|", "|"], + ids: [26, 24, 7, 15, 18, 15, 20, 13, 4, 25, 22, 7, 9, 11, 4, 4, 4], + decoded: "trailing space", + }, + SURROUNDING_SPACE: { + text: BASE_TEST_STRINGS.SURROUNDING_SPACE, + tokens: ["|", "|", "|", "s", "u", "r", "r", "o", "u", "n", "d", "i", "n", "g", "|", "s", "p", "a", "c", "e", "|", "|", "|"], + ids: [4, 4, 4, 25, 27, 24, 24, 21, 27, 20, 10, 15, 20, 13, 4, 25, 22, 7, 9, 11, 4, 4, 4], + decoded: "surounding space", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["H", "i", "|", "|", "H", "e", "l", "l", "o"], + ids: [3, 15, 4, 4, 3, 11, 18, 18, 21], + decoded: "i elo", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["t", "e", "s", "t", "|", "$", "1", "|", "R", "2", "|", "#", "3", "|", "\u20ac", "4", "|", "\u00a3", "5", "|", "\u00a5", "6", "|", "\u20a3", "7", "|", "\u20b9", "8", "|", "\u20b1", "9", "|", "t", "e", "s", "t"], + ids: [26, 11, 25, 26, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 26, 11, 25, 26], + decoded: "test test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "|", "b", "o", "u", "g", "h", "t", "|", "a", "n", "|", "a", "p", "p", "l", "e", "|", "f", "o", "r", "|", "$", "1", ".", "0", "0", "|", "a", "t", "|", "t", "h", "e", "|", "s", "t", "o", "r", "e", "."], + ids: [3, 4, 8, 21, 27, 13, 14, 26, 4, 7, 20, 4, 7, 22, 22, 18, 11, 4, 12, 21, 24, 4, 3, 3, 3, 3, 3, 4, 7, 26, 4, 26, 14, 11, 4, 25, 26, 21, 24, 11, 3], + decoded: " bought an aple for at the store", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["y", "o", "u", "\u2026", "|", "|"], + ids: [31, 21, 27, 3, 4, 4], + decoded: "you", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["y", "o", "u", "\u2026", "\u00a0", "\u00a0"], + ids: [31, 21, 27, 3, 3, 3], + decoded: "you", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["y", "o", "u", "\u2026", "\u00a0", "\u00a0", "y", "o", "u", "\u2026", "\u00a0", "\u00a0"], + ids: [31, 21, 27, 3, 3, 3, 31, 21, 27, 3, 3, 3], + decoded: "youyou", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["w", "e", "i", "r", "d", "|", "\uff5e", "|", "e", "d", "g", "e", "|", "\uff5e", "|", "c", "a", "s", "e"], + ids: [29, 11, 15, 24, 10, 4, 3, 4, 11, 10, 13, 11, 4, 3, 4, 9, 7, 25, 11], + decoded: "weird edge case", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u2581", "T", "h", "i", "s", "|", "\u2581", "i", "s", "|", "\u2581", "a", "|", "\u2581", "t", "e", "s", "t", "|", "\u2581", "."], + ids: [3, 3, 14, 15, 25, 4, 3, 15, 25, 4, 3, 7, 4, 3, 26, 11, 25, 26, 4, 3, 3], + decoded: "his is a test ", + }, + CHINESE_LATIN_MIXED: { + text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, + tokens: ["a", "h", "\u535a", "\u63a8", "z", "z"], + ids: [7, 14, 3, 3, 32, 32], + decoded: "ahz", + }, + SIMPLE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, + tokens: ["H", "\u00e9", "l", "l", "o"], + ids: [3, 3, 18, 18, 21], + decoded: "lo", + }, + MIXED_CASE_WITHOUT_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, + tokens: ["|", "\t", "H", "e", "L", "L", "o", "!", "h", "o", "w", "|", "|", "\n", "|", "A", "r", "e", "|", "y", "o", "U", "?", "|", "|"], + ids: [4, 3, 3, 11, 3, 3, 21, 3, 14, 21, 29, 4, 4, 3, 4, 3, 24, 11, 4, 31, 21, 3, 3, 4, 4], + decoded: "eohow re yo", + }, + MIXED_CASE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, + tokens: ["|", "\t", "H", "\u00e4", "L", "L", "o", "!", "h", "o", "w", "|", "|", "\n", "|", "A", "r", "e", "|", "y", "o", "U", "?", "|", "|"], + ids: [4, 3, 3, 3, 3, 3, 21, 3, 14, 21, 29, 4, 4, 3, 4, 3, 24, 11, 4, 31, 21, 3, 3, 4, 4], + decoded: "ohow re yo", + }, + }, + "Xenova/mms-1b-all": { + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "|", "0", "|", "1", "|", "2", "|", "3", "|", "4", "|", "5", "|", "6", "|", "7", "|", "8", "|", "9", "|", "1", "0", "|", "1", "0", "0", "|", "1", "0", "0", "0"], + ids: [27, 30, 35, 41, 39, 38, 40, 43, 42, 36, 4, 27, 4, 30, 4, 35, 4, 41, 4, 39, 4, 38, 4, 40, 4, 43, 4, 42, 4, 36, 4, 30, 27, 4, 30, 27, 27, 4, 30, 27, 27, 27], + decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 10 10", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["T", "h", "e", "|", "c", "o", "m", "p", "a", "n", "y", "|", "w", "a", "s", "|", "f", "o", "u", "n", "d", "e", "d", "|", "i", "n", "|", "2", "0", "1", "6", "."], + ids: [3, 13, 5, 4, 16, 8, 18, 20, 7, 10, 22, 4, 23, 7, 11, 4, 19, 8, 17, 10, 15, 5, 15, 4, 9, 10, 4, 35, 27, 30, 40, 37], + decoded: "he company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "\n", "'", "l", "l", "|", "!", "!", "t", "o", "?", "'", "d", "'", "'", "d", "|", "o", "f", ",", "|", "c", "a", "n", "'", "t", "."], + ids: [3, 3, 31, 14, 14, 4, 75, 75, 6, 8, 3, 31, 15, 31, 31, 15, 4, 8, 19, 44, 4, 16, 7, 10, 31, 6, 37], + decoded: "'l!to'd'd of, can't.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["d", "e", "f", "|", "m", "a", "i", "n", "(", ")", ":", "\n", "\t", "p", "a", "s", "s"], + ids: [15, 5, 19, 4, 18, 7, 9, 10, 3, 3, 46, 3, 3, 20, 7, 11, 11], + decoded: "def main:pas", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["l", "e", "t", "|", "a", "|", "=", "|", "o", "b", "j", ".", "t", "o", "S", "t", "r", "i", "n", "g", "(", ")", ";", "\n", "t", "o", "S", "t", "r", "i", "n", "g", "(", ")", ";"], + ids: [14, 5, 6, 4, 7, 4, 3, 4, 8, 24, 29, 37, 6, 8, 3, 6, 12, 9, 10, 21, 3, 3, 52, 3, 6, 8, 3, 6, 12, 9, 10, 21, 3, 3, 52], + decoded: "let a obj.totring;totring;", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["T", "h", "i", "s", "\n", "\n", "i", "s", "\n", "a", "\n", "t", "e", "s", "t", "."], + ids: [3, 13, 9, 11, 3, 3, 9, 11, 3, 7, 3, 6, 5, 11, 6, 37], + decoded: "hisisatest.", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["U", "N", "w", "a", "n", "t", "\u00e9", "d", ",", "r", "u", "n", "n", "i", "n", "g"], + ids: [3, 3, 23, 7, 10, 6, 55, 15, 44, 12, 17, 10, 10, 9, 10, 21], + decoded: "want\u00e9d,runing", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["1", "\u0000", "2", "\ufffd", "3"], + ids: [30, 3, 35, 3, 41], + decoded: "123", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u751f", "\u6d3b", "\u7684", "\u771f", "\u8c1b", "\u662f"], + ids: [136, 3, 3, 3, 3, 3], + decoded: "\u751f", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["t", "e", "s", "t", "|", "$", "1", "|", "R", "2", "|", "#", "3", "|", "\u20ac", "4", "|", "\u00a3", "5", "|", "\u00a5", "6", "|", "\u20a3", "7", "|", "\u20b9", "8", "|", "\u20b1", "9", "|", "t", "e", "s", "t"], + ids: [6, 5, 11, 6, 4, 48, 30, 4, 3, 35, 4, 3, 41, 4, 3, 39, 4, 68, 38, 4, 53, 40, 4, 3, 43, 4, 3, 42, 4, 3, 36, 4, 6, 5, 11, 6], + decoded: "test $1 2 3 4 \u00a35 \u00a56 7 8 9 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "|", "b", "o", "u", "g", "h", "t", "|", "a", "n", "|", "a", "p", "p", "l", "e", "|", "f", "o", "r", "|", "$", "1", ".", "0", "0", "|", "a", "t", "|", "t", "h", "e", "|", "s", "t", "o", "r", "e", "."], + ids: [3, 4, 24, 8, 17, 21, 13, 6, 4, 7, 10, 4, 7, 20, 20, 14, 5, 4, 19, 8, 12, 4, 48, 30, 37, 27, 27, 4, 7, 6, 4, 6, 13, 5, 4, 11, 6, 8, 12, 5, 37], + decoded: " bought an aple for $1.0 at the store.", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u2581", "T", "h", "i", "s", "|", "\u2581", "i", "s", "|", "\u2581", "a", "|", "\u2581", "t", "e", "s", "t", "|", "\u2581", "."], + ids: [3, 3, 13, 9, 11, 4, 3, 9, 11, 4, 3, 7, 4, 3, 6, 5, 11, 6, 4, 3, 37], + decoded: "his is a test .", + }, + SIMPLE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, + tokens: ["H", "\u00e9", "l", "l", "o"], + ids: [3, 55, 14, 14, 8], + decoded: "\u00e9lo", + }, + MIXED_CASE_WITHOUT_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, + tokens: ["|", "\t", "H", "e", "L", "L", "o", "!", "h", "o", "w", "|", "|", "\n", "|", "A", "r", "e", "|", "y", "o", "U", "?", "|", "|"], + ids: [4, 3, 3, 5, 3, 3, 8, 75, 13, 8, 23, 4, 4, 3, 4, 3, 12, 5, 4, 22, 8, 3, 3, 4, 4], + decoded: "eo!how re yo", + }, + MIXED_CASE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, + tokens: ["|", "\t", "H", "\u00e4", "L", "L", "o", "!", "h", "o", "w", "|", "|", "\n", "|", "A", "r", "e", "|", "y", "o", "U", "?", "|", "|"], + ids: [4, 3, 3, 78, 3, 3, 8, 75, 13, 8, 23, 4, 4, 3, 4, 3, 12, 5, 4, 22, 8, 3, 3, 4, 4], + decoded: "\u00e4o!how re yo", + }, + }, + "Xenova/mms-1b-fl102": { + MIXED_CASE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, + tokens: ["|", "\t", "H", "\u00e4", "L", "L", "o", "!", "h", "o", "w", "|", "|", "\n", "|", "A", "r", "e", "|", "y", "o", "U", "?", "|", "|"], + ids: [4, 3, 3, 3, 3, 3, 8, 75, 13, 8, 23, 4, 4, 3, 4, 3, 12, 5, 4, 22, 8, 3, 3, 4, 4], + decoded: "o!how re yo", + }, + }, + "Xenova/mms-1b-l1107": { + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "|", "0", "|", "1", "|", "2", "|", "3", "|", "4", "|", "5", "|", "6", "|", "7", "|", "8", "|", "9", "|", "1", "0", "|", "1", "0", "0", "|", "1", "0", "0", "0"], + ids: [34, 36, 37, 42, 38, 41, 39, 3, 3, 3, 4, 34, 4, 36, 4, 37, 4, 42, 4, 38, 4, 41, 4, 39, 4, 3, 4, 3, 4, 3, 4, 36, 34, 4, 36, 34, 34, 4, 36, 34, 34, 34], + decoded: "0123456 0 1 2 3 4 5 6 10 10 10", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["T", "h", "e", "|", "c", "o", "m", "p", "a", "n", "y", "|", "w", "a", "s", "|", "f", "o", "u", "n", "d", "e", "d", "|", "i", "n", "|", "2", "0", "1", "6", "."], + ids: [3, 9, 5, 4, 21, 7, 18, 24, 8, 10, 20, 4, 17, 8, 12, 4, 19, 7, 16, 10, 14, 5, 14, 4, 11, 10, 4, 37, 34, 36, 39, 3], + decoded: "he company was founded in 2016", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["t", "e", "s", "t", "|", "$", "1", "|", "R", "2", "|", "#", "3", "|", "\u20ac", "4", "|", "\u00a3", "5", "|", "\u00a5", "6", "|", "\u20a3", "7", "|", "\u20b9", "8", "|", "\u20b1", "9", "|", "t", "e", "s", "t"], + ids: [6, 5, 12, 6, 4, 3, 36, 4, 3, 37, 4, 3, 42, 4, 3, 38, 4, 3, 41, 4, 3, 39, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 6, 5, 12, 6], + decoded: "test 1 2 3 4 5 6 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "|", "b", "o", "u", "g", "h", "t", "|", "a", "n", "|", "a", "p", "p", "l", "e", "|", "f", "o", "r", "|", "$", "1", ".", "0", "0", "|", "a", "t", "|", "t", "h", "e", "|", "s", "t", "o", "r", "e", "."], + ids: [3, 4, 23, 7, 16, 22, 9, 6, 4, 8, 10, 4, 8, 24, 24, 15, 5, 4, 19, 7, 13, 4, 3, 36, 3, 34, 34, 4, 8, 6, 4, 6, 9, 5, 4, 12, 6, 7, 13, 5, 3], + decoded: " bought an aple for 10 at the store", + }, }, - SIMPLE_WITH_PUNCTUATION: { - text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, - tokens: ["Y", "o", "u", "|", "s", "h", "o", "u", "l", "d", "'", "v", "e", "|", "d", "o", "n", "e", "|", "t", "h", "i", "s"], - ids: [22, 3, 3, 4, 3, 3, 3, 3, 3, 3, 27, 3, 3, 4, 3, 3, 3, 3, 4, 3, 3, 3, 3], - decoded: "Y ' ", - }, - NUMBERS: { - text: BASE_TEST_STRINGS.NUMBERS, - tokens: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "|", "0", "|", "1", "|", "2", "|", "3", "|", "4", "|", "5", "|", "6", "|", "7", "|", "8", "|", "9", "|", "1", "0", "|", "1", "0", "0", "|", "1", "0", "0", "0"], - ids: [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 3, 4, 3, 3, 3, 4, 3, 3, 3, 3], - decoded: " ", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["T", "h", "e", "|", "c", "o", "m", "p", "a", "n", "y", "|", "w", "a", "s", "|", "f", "o", "u", "n", "d", "e", "d", "|", "i", "n", "|", "2", "0", "1", "6", "."], - ids: [6, 3, 3, 4, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 3, 4, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 4, 3, 3, 3, 3, 3], - decoded: "T ", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["A", "\n", "'", "l", "l", "|", "!", "!", "t", "o", "?", "'", "d", "'", "'", "d", "|", "o", "f", ",", "|", "c", "a", "n", "'", "t", "."], - ids: [7, 3, 27, 3, 3, 4, 3, 3, 3, 3, 3, 27, 3, 27, 27, 3, 4, 3, 3, 3, 4, 3, 3, 3, 27, 3, 3], - decoded: "A' '' '", - }, - PYTHON_CODE: { - text: BASE_TEST_STRINGS.PYTHON_CODE, - tokens: ["d", "e", "f", "|", "m", "a", "i", "n", "(", ")", ":", "\n", "\t", "p", "a", "s", "s"], - ids: [3, 3, 3, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3], - decoded: " ", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["l", "e", "t", "|", "a", "|", "=", "|", "o", "b", "j", ".", "t", "o", "S", "t", "r", "i", "n", "g", "(", ")", ";", "\n", "t", "o", "S", "t", "r", "i", "n", "g", "(", ")", ";"], - ids: [3, 3, 3, 4, 3, 4, 3, 4, 3, 3, 3, 3, 3, 3, 12, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 12, 3, 3, 3, 3, 3, 3, 3, 3], - decoded: " SS", - }, - NEWLINES: { - text: BASE_TEST_STRINGS.NEWLINES, - tokens: ["T", "h", "i", "s", "\n", "\n", "i", "s", "\n", "a", "\n", "t", "e", "s", "t", "."], - ids: [6, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3], - decoded: "T", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["U", "N", "w", "a", "n", "t", "\u00e9", "d", ",", "r", "u", "n", "n", "i", "n", "g"], - ids: [16, 9, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3], - decoded: "UN", - }, - CONTROL_TOKENS: { - text: BASE_TEST_STRINGS.CONTROL_TOKENS, - tokens: ["1", "\u0000", "2", "\ufffd", "3"], - ids: [3, 3, 3, 3, 3], - decoded: "", - }, - HELLO_WORLD_TITLECASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, - tokens: ["H", "e", "l", "l", "o", "|", "W", "o", "r", "l", "d"], - ids: [11, 3, 3, 3, 3, 4, 18, 3, 3, 3, 3], - decoded: "H W", - }, - HELLO_WORLD_LOWERCASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, - tokens: ["h", "e", "l", "l", "o", "|", "w", "o", "r", "l", "d"], - ids: [3, 3, 3, 3, 3, 4, 3, 3, 3, 3, 3], - decoded: " ", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u751f", "\u6d3b", "\u7684", "\u771f", "\u8c1b", "\u662f"], - ids: [3, 3, 3, 3, 3, 3], - decoded: "", - }, - LEADING_SPACE: { - text: BASE_TEST_STRINGS.LEADING_SPACE, - tokens: ["|", "|", "|", "l", "e", "a", "d", "i", "n", "g", "|", "s", "p", "a", "c", "e"], - ids: [4, 4, 4, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 3, 3, 3], - decoded: " ", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["t", "r", "a", "i", "l", "i", "n", "g", "|", "s", "p", "a", "c", "e", "|", "|", "|"], - ids: [3, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 3, 3, 3, 4, 4, 4], - decoded: " ", - }, - SURROUNDING_SPACE: { - text: BASE_TEST_STRINGS.SURROUNDING_SPACE, - tokens: ["|", "|", "|", "s", "u", "r", "r", "o", "u", "n", "d", "i", "n", "g", "|", "s", "p", "a", "c", "e", "|", "|", "|"], - ids: [4, 4, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 3, 3, 3, 4, 4, 4], - decoded: " ", - }, - DOUBLE_SPACE: { - text: BASE_TEST_STRINGS.DOUBLE_SPACE, - tokens: ["H", "i", "|", "|", "H", "e", "l", "l", "o"], - ids: [11, 3, 4, 4, 11, 3, 3, 3, 3], - decoded: "H H", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["t", "e", "s", "t", "|", "$", "1", "|", "R", "2", "|", "#", "3", "|", "\u20ac", "4", "|", "\u00a3", "5", "|", "\u00a5", "6", "|", "\u20a3", "7", "|", "\u20b9", "8", "|", "\u20b1", "9", "|", "t", "e", "s", "t"], - ids: [3, 3, 3, 3, 4, 3, 3, 4, 13, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 3, 3], - decoded: " R ", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["I", "|", "b", "o", "u", "g", "h", "t", "|", "a", "n", "|", "a", "p", "p", "l", "e", "|", "f", "o", "r", "|", "$", "1", ".", "0", "0", "|", "a", "t", "|", "t", "h", "e", "|", "s", "t", "o", "r", "e", "."], - ids: [10, 4, 3, 3, 3, 3, 3, 3, 4, 3, 3, 4, 3, 3, 3, 3, 3, 4, 3, 3, 3, 4, 3, 3, 3, 3, 3, 4, 3, 3, 4, 3, 3, 3, 4, 3, 3, 3, 3, 3, 3], - decoded: "I ", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - tokens: ["y", "o", "u", "\u2026", "|", "|"], - ids: [3, 3, 3, 3, 4, 4], - decoded: "", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - tokens: ["y", "o", "u", "\u2026", "\u00a0", "\u00a0"], - ids: [3, 3, 3, 3, 3, 3], - decoded: "", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - tokens: ["y", "o", "u", "\u2026", "\u00a0", "\u00a0", "y", "o", "u", "\u2026", "\u00a0", "\u00a0"], - ids: [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3], - decoded: "", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["w", "e", "i", "r", "d", "|", "\uff5e", "|", "e", "d", "g", "e", "|", "\uff5e", "|", "c", "a", "s", "e"], - ids: [3, 3, 3, 3, 3, 4, 3, 4, 3, 3, 3, 3, 4, 3, 4, 3, 3, 3, 3], - decoded: " ", - }, - SPIECE_UNDERSCORE: { - text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, - tokens: ["\u2581", "T", "h", "i", "s", "|", "\u2581", "i", "s", "|", "\u2581", "a", "|", "\u2581", "t", "e", "s", "t", "|", "\u2581", "."], - ids: [3, 6, 3, 3, 3, 4, 3, 3, 3, 4, 3, 3, 4, 3, 3, 3, 3, 3, 4, 3, 3], - decoded: "T ", - }, - POPULAR_EMOJIS: { - text: BASE_TEST_STRINGS.POPULAR_EMOJIS, - tokens: ["\ud83d\ude02", "|", "\ud83d\udc4d", "|", "\ud83e\udd23", "|", "\ud83d\ude0d", "|", "\ud83d\ude2d", "|", "\ud83c\udf89", "|", "\ud83d\ude4f", "|", "\ud83d\ude0a", "|", "\ud83d\udd25", "|", "\ud83d\ude01", "|", "\ud83d\ude05", "|", "\ud83e\udd17", "|", "\ud83d\ude06", "|", "\ud83d\udc4f", "|", "\u2764", "\ufe0f", "|", "\ud83d\udc9c", "|", "\ud83d\udc9a", "|", "\ud83d\udc97", "|", "\ud83d\udc99", "|", "\ud83d\udda4", "|", "\ud83d\ude0e", "|", "\ud83d\udc4c", "|", "\ud83e\udd73", "|", "\ud83d\udcaa", "|", "\u2728", "|", "\ud83d\udc49", "|", "\ud83d\udc40", "|", "\ud83d\udcaf", "|", "\ud83c\udf88", "|", "\ud83d\ude48", "|", "\ud83d\ude4c", "|", "\ud83d\udc80", "|", "\ud83d\udc47", "|", "\ud83d\udc4b", "|", "\u2705", "|", "\ud83c\udf81", "|", "\ud83c\udf1e", "|", "\ud83c\udf38", "|", "\ud83d\udcb0"], - ids: [3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3], - decoded: " ", - }, - MULTIBYTE_EMOJIS: { - text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, - tokens: ["\u2728", "|", "\ud83e\udd17", "|", "\ud83d\udc41", "\ufe0f", "|", "\ud83d\udc71", "\ud83c\udffb", "|", "\ud83d\udd75", "\u200d", "\u2642", "\ufe0f", "|", "\ud83e\uddd9", "\ud83c\udffb", "\u200d", "\u2642", "|", "\ud83d\udc68", "\ud83c\udffb", "\u200d", "\ud83c\udf3e", "|", "\ud83e\uddd1", "\u200d", "\ud83e\udd1d", "\u200d", "\ud83e\uddd1", "|", "\ud83d\udc69", "\u200d", "\u2764", "\u200d", "\ud83d\udc8b", "\u200d", "\ud83d\udc68", "|", "\ud83d\udc69", "\u200d", "\ud83d\udc69", "\u200d", "\ud83d\udc67", "\u200d", "\ud83d\udc66", "|", "\ud83e\uddd1", "\ud83c\udffb", "\u200d", "\ud83e\udd1d", "\u200d", "\ud83e\uddd1", "\ud83c\udffb", "|", "\ud83c\udff4", "\udb40\udc67", "\udb40\udc62", "\udb40\udc65", "\udb40\udc6e", "\udb40\udc67", "\udb40\udc7f", "|", "\ud83d\udc68", "\ud83c\udffb", "\u200d", "\u2764", "\ufe0f", "\u200d", "\ud83d\udc8b", "\u200d", "\ud83d\udc68", "\ud83c\udffc"], - ids: [3, 4, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 3, 3, 4, 3, 3, 3, 3, 4, 3, 3, 3, 3, 4, 3, 3, 3, 3, 3, 4, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3], - decoded: " ", - }, - ONLY_WHITESPACE: { - text: BASE_TEST_STRINGS.ONLY_WHITESPACE, - tokens: ["|", "\t", "\n"], - ids: [4, 3, 3], - decoded: "", - }, - CHINESE_LATIN_MIXED: { - text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, - tokens: ["a", "h", "\u535a", "\u63a8", "z", "z"], - ids: [3, 3, 3, 3, 3, 3], - decoded: "", - }, - SIMPLE_WITH_ACCENTS: { - text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, - tokens: ["H", "\u00e9", "l", "l", "o"], - ids: [11, 3, 3, 3, 3], - decoded: "H", - }, - MIXED_CASE_WITHOUT_ACCENTS: { - text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, - tokens: ["|", "\t", "H", "e", "L", "L", "o", "!", "h", "o", "w", "|", "|", "\n", "|", "A", "r", "e", "|", "y", "o", "U", "?", "|", "|"], - ids: [4, 3, 11, 3, 15, 15, 3, 3, 3, 3, 3, 4, 4, 3, 4, 7, 3, 3, 4, 3, 3, 16, 3, 4, 4], - decoded: "HL A U", - }, - MIXED_CASE_WITH_ACCENTS: { - text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, - tokens: ["|", "\t", "H", "\u00e4", "L", "L", "o", "!", "h", "o", "w", "|", "|", "\n", "|", "A", "r", "e", "|", "y", "o", "U", "?", "|", "|"], - ids: [4, 3, 11, 3, 15, 15, 3, 3, 3, 3, 3, 4, 4, 3, 4, 7, 3, 3, 4, 3, 3, 16, 3, 4, 4], - decoded: "HL A U", - }, - }, - "Xenova/wav2vec2-large-xlsr-53-english": { - SIMPLE: { - text: BASE_TEST_STRINGS.SIMPLE, - tokens: ["H", "o", "w", "|", "a", "r", "e", "|", "y", "o", "u", "|", "d", "o", "i", "n", "g", "?"], - ids: [3, 21, 29, 4, 7, 24, 11, 4, 31, 21, 27, 4, 10, 21, 15, 20, 13, 3], - decoded: "ow are you doing", - }, - SIMPLE_WITH_PUNCTUATION: { - text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, - tokens: ["Y", "o", "u", "|", "s", "h", "o", "u", "l", "d", "'", "v", "e", "|", "d", "o", "n", "e", "|", "t", "h", "i", "s"], - ids: [3, 21, 27, 4, 25, 14, 21, 27, 18, 10, 5, 28, 11, 4, 10, 21, 20, 11, 4, 26, 14, 15, 25], - decoded: "ou should've done this", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["T", "h", "e", "|", "c", "o", "m", "p", "a", "n", "y", "|", "w", "a", "s", "|", "f", "o", "u", "n", "d", "e", "d", "|", "i", "n", "|", "2", "0", "1", "6", "."], - ids: [3, 14, 11, 4, 9, 21, 19, 22, 7, 20, 31, 4, 29, 7, 25, 4, 12, 21, 27, 20, 10, 11, 10, 4, 15, 20, 4, 3, 3, 3, 3, 3], - decoded: "he company was founded in ", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["A", "\n", "'", "l", "l", "|", "!", "!", "t", "o", "?", "'", "d", "'", "'", "d", "|", "o", "f", ",", "|", "c", "a", "n", "'", "t", "."], - ids: [3, 3, 5, 18, 18, 4, 3, 3, 26, 21, 3, 5, 10, 5, 5, 10, 4, 21, 12, 3, 4, 9, 7, 20, 5, 26, 3], - decoded: "'l to'd'd of can't", - }, - PYTHON_CODE: { - text: BASE_TEST_STRINGS.PYTHON_CODE, - tokens: ["d", "e", "f", "|", "m", "a", "i", "n", "(", ")", ":", "\n", "\t", "p", "a", "s", "s"], - ids: [10, 11, 12, 4, 19, 7, 15, 20, 3, 3, 3, 3, 3, 22, 7, 25, 25], - decoded: "def mainpas", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["l", "e", "t", "|", "a", "|", "=", "|", "o", "b", "j", ".", "t", "o", "S", "t", "r", "i", "n", "g", "(", ")", ";", "\n", "t", "o", "S", "t", "r", "i", "n", "g", "(", ")", ";"], - ids: [18, 11, 26, 4, 7, 4, 3, 4, 21, 8, 16, 3, 26, 21, 3, 26, 24, 15, 20, 13, 3, 3, 3, 3, 26, 21, 3, 26, 24, 15, 20, 13, 3, 3, 3], - decoded: "let a objtotringtotring", - }, - NEWLINES: { - text: BASE_TEST_STRINGS.NEWLINES, - tokens: ["T", "h", "i", "s", "\n", "\n", "i", "s", "\n", "a", "\n", "t", "e", "s", "t", "."], - ids: [3, 14, 15, 25, 3, 3, 15, 25, 3, 7, 3, 26, 11, 25, 26, 3], - decoded: "hisisatest", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["U", "N", "w", "a", "n", "t", "\u00e9", "d", ",", "r", "u", "n", "n", "i", "n", "g"], - ids: [3, 3, 29, 7, 20, 26, 3, 10, 3, 24, 27, 20, 20, 15, 20, 13], - decoded: "wantdruning", - }, - HELLO_WORLD_TITLECASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, - tokens: ["H", "e", "l", "l", "o", "|", "W", "o", "r", "l", "d"], - ids: [3, 11, 18, 18, 21, 4, 3, 21, 24, 18, 10], - decoded: "elo orld", - }, - HELLO_WORLD_LOWERCASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, - tokens: ["h", "e", "l", "l", "o", "|", "w", "o", "r", "l", "d"], - ids: [14, 11, 18, 18, 21, 4, 29, 21, 24, 18, 10], - decoded: "helo world", - }, - LEADING_SPACE: { - text: BASE_TEST_STRINGS.LEADING_SPACE, - tokens: ["|", "|", "|", "l", "e", "a", "d", "i", "n", "g", "|", "s", "p", "a", "c", "e"], - ids: [4, 4, 4, 18, 11, 7, 10, 15, 20, 13, 4, 25, 22, 7, 9, 11], - decoded: "leading space", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["t", "r", "a", "i", "l", "i", "n", "g", "|", "s", "p", "a", "c", "e", "|", "|", "|"], - ids: [26, 24, 7, 15, 18, 15, 20, 13, 4, 25, 22, 7, 9, 11, 4, 4, 4], - decoded: "trailing space", - }, - SURROUNDING_SPACE: { - text: BASE_TEST_STRINGS.SURROUNDING_SPACE, - tokens: ["|", "|", "|", "s", "u", "r", "r", "o", "u", "n", "d", "i", "n", "g", "|", "s", "p", "a", "c", "e", "|", "|", "|"], - ids: [4, 4, 4, 25, 27, 24, 24, 21, 27, 20, 10, 15, 20, 13, 4, 25, 22, 7, 9, 11, 4, 4, 4], - decoded: "surounding space", - }, - DOUBLE_SPACE: { - text: BASE_TEST_STRINGS.DOUBLE_SPACE, - tokens: ["H", "i", "|", "|", "H", "e", "l", "l", "o"], - ids: [3, 15, 4, 4, 3, 11, 18, 18, 21], - decoded: "i elo", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["t", "e", "s", "t", "|", "$", "1", "|", "R", "2", "|", "#", "3", "|", "\u20ac", "4", "|", "\u00a3", "5", "|", "\u00a5", "6", "|", "\u20a3", "7", "|", "\u20b9", "8", "|", "\u20b1", "9", "|", "t", "e", "s", "t"], - ids: [26, 11, 25, 26, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 26, 11, 25, 26], - decoded: "test test", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["I", "|", "b", "o", "u", "g", "h", "t", "|", "a", "n", "|", "a", "p", "p", "l", "e", "|", "f", "o", "r", "|", "$", "1", ".", "0", "0", "|", "a", "t", "|", "t", "h", "e", "|", "s", "t", "o", "r", "e", "."], - ids: [3, 4, 8, 21, 27, 13, 14, 26, 4, 7, 20, 4, 7, 22, 22, 18, 11, 4, 12, 21, 24, 4, 3, 3, 3, 3, 3, 4, 7, 26, 4, 26, 14, 11, 4, 25, 26, 21, 24, 11, 3], - decoded: " bought an aple for at the store", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - tokens: ["y", "o", "u", "\u2026", "|", "|"], - ids: [31, 21, 27, 3, 4, 4], - decoded: "you", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - tokens: ["y", "o", "u", "\u2026", "\u00a0", "\u00a0"], - ids: [31, 21, 27, 3, 3, 3], - decoded: "you", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - tokens: ["y", "o", "u", "\u2026", "\u00a0", "\u00a0", "y", "o", "u", "\u2026", "\u00a0", "\u00a0"], - ids: [31, 21, 27, 3, 3, 3, 31, 21, 27, 3, 3, 3], - decoded: "youyou", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["w", "e", "i", "r", "d", "|", "\uff5e", "|", "e", "d", "g", "e", "|", "\uff5e", "|", "c", "a", "s", "e"], - ids: [29, 11, 15, 24, 10, 4, 3, 4, 11, 10, 13, 11, 4, 3, 4, 9, 7, 25, 11], - decoded: "weird edge case", - }, - SPIECE_UNDERSCORE: { - text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, - tokens: ["\u2581", "T", "h", "i", "s", "|", "\u2581", "i", "s", "|", "\u2581", "a", "|", "\u2581", "t", "e", "s", "t", "|", "\u2581", "."], - ids: [3, 3, 14, 15, 25, 4, 3, 15, 25, 4, 3, 7, 4, 3, 26, 11, 25, 26, 4, 3, 3], - decoded: "his is a test ", - }, - CHINESE_LATIN_MIXED: { - text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, - tokens: ["a", "h", "\u535a", "\u63a8", "z", "z"], - ids: [7, 14, 3, 3, 32, 32], - decoded: "ahz", - }, - SIMPLE_WITH_ACCENTS: { - text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, - tokens: ["H", "\u00e9", "l", "l", "o"], - ids: [3, 3, 18, 18, 21], - decoded: "lo", - }, - MIXED_CASE_WITHOUT_ACCENTS: { - text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, - tokens: ["|", "\t", "H", "e", "L", "L", "o", "!", "h", "o", "w", "|", "|", "\n", "|", "A", "r", "e", "|", "y", "o", "U", "?", "|", "|"], - ids: [4, 3, 3, 11, 3, 3, 21, 3, 14, 21, 29, 4, 4, 3, 4, 3, 24, 11, 4, 31, 21, 3, 3, 4, 4], - decoded: "eohow re yo", - }, - MIXED_CASE_WITH_ACCENTS: { - text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, - tokens: ["|", "\t", "H", "\u00e4", "L", "L", "o", "!", "h", "o", "w", "|", "|", "\n", "|", "A", "r", "e", "|", "y", "o", "U", "?", "|", "|"], - ids: [4, 3, 3, 3, 3, 3, 21, 3, 14, 21, 29, 4, 4, 3, 4, 3, 24, 11, 4, 31, 21, 3, 3, 4, 4], - decoded: "ohow re yo", - }, - }, - "Xenova/mms-1b-all": { - NUMBERS: { - text: BASE_TEST_STRINGS.NUMBERS, - tokens: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "|", "0", "|", "1", "|", "2", "|", "3", "|", "4", "|", "5", "|", "6", "|", "7", "|", "8", "|", "9", "|", "1", "0", "|", "1", "0", "0", "|", "1", "0", "0", "0"], - ids: [27, 30, 35, 41, 39, 38, 40, 43, 42, 36, 4, 27, 4, 30, 4, 35, 4, 41, 4, 39, 4, 38, 4, 40, 4, 43, 4, 42, 4, 36, 4, 30, 27, 4, 30, 27, 27, 4, 30, 27, 27, 27], - decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 10 10", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["T", "h", "e", "|", "c", "o", "m", "p", "a", "n", "y", "|", "w", "a", "s", "|", "f", "o", "u", "n", "d", "e", "d", "|", "i", "n", "|", "2", "0", "1", "6", "."], - ids: [3, 13, 5, 4, 16, 8, 18, 20, 7, 10, 22, 4, 23, 7, 11, 4, 19, 8, 17, 10, 15, 5, 15, 4, 9, 10, 4, 35, 27, 30, 40, 37], - decoded: "he company was founded in 2016.", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["A", "\n", "'", "l", "l", "|", "!", "!", "t", "o", "?", "'", "d", "'", "'", "d", "|", "o", "f", ",", "|", "c", "a", "n", "'", "t", "."], - ids: [3, 3, 31, 14, 14, 4, 75, 75, 6, 8, 3, 31, 15, 31, 31, 15, 4, 8, 19, 44, 4, 16, 7, 10, 31, 6, 37], - decoded: "'l!to'd'd of, can't.", - }, - PYTHON_CODE: { - text: BASE_TEST_STRINGS.PYTHON_CODE, - tokens: ["d", "e", "f", "|", "m", "a", "i", "n", "(", ")", ":", "\n", "\t", "p", "a", "s", "s"], - ids: [15, 5, 19, 4, 18, 7, 9, 10, 3, 3, 46, 3, 3, 20, 7, 11, 11], - decoded: "def main:pas", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["l", "e", "t", "|", "a", "|", "=", "|", "o", "b", "j", ".", "t", "o", "S", "t", "r", "i", "n", "g", "(", ")", ";", "\n", "t", "o", "S", "t", "r", "i", "n", "g", "(", ")", ";"], - ids: [14, 5, 6, 4, 7, 4, 3, 4, 8, 24, 29, 37, 6, 8, 3, 6, 12, 9, 10, 21, 3, 3, 52, 3, 6, 8, 3, 6, 12, 9, 10, 21, 3, 3, 52], - decoded: "let a obj.totring;totring;", - }, - NEWLINES: { - text: BASE_TEST_STRINGS.NEWLINES, - tokens: ["T", "h", "i", "s", "\n", "\n", "i", "s", "\n", "a", "\n", "t", "e", "s", "t", "."], - ids: [3, 13, 9, 11, 3, 3, 9, 11, 3, 7, 3, 6, 5, 11, 6, 37], - decoded: "hisisatest.", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["U", "N", "w", "a", "n", "t", "\u00e9", "d", ",", "r", "u", "n", "n", "i", "n", "g"], - ids: [3, 3, 23, 7, 10, 6, 55, 15, 44, 12, 17, 10, 10, 9, 10, 21], - decoded: "want\u00e9d,runing", - }, - CONTROL_TOKENS: { - text: BASE_TEST_STRINGS.CONTROL_TOKENS, - tokens: ["1", "\u0000", "2", "\ufffd", "3"], - ids: [30, 3, 35, 3, 41], - decoded: "123", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u751f", "\u6d3b", "\u7684", "\u771f", "\u8c1b", "\u662f"], - ids: [136, 3, 3, 3, 3, 3], - decoded: "\u751f", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["t", "e", "s", "t", "|", "$", "1", "|", "R", "2", "|", "#", "3", "|", "\u20ac", "4", "|", "\u00a3", "5", "|", "\u00a5", "6", "|", "\u20a3", "7", "|", "\u20b9", "8", "|", "\u20b1", "9", "|", "t", "e", "s", "t"], - ids: [6, 5, 11, 6, 4, 48, 30, 4, 3, 35, 4, 3, 41, 4, 3, 39, 4, 68, 38, 4, 53, 40, 4, 3, 43, 4, 3, 42, 4, 3, 36, 4, 6, 5, 11, 6], - decoded: "test $1 2 3 4 \u00a35 \u00a56 7 8 9 test", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["I", "|", "b", "o", "u", "g", "h", "t", "|", "a", "n", "|", "a", "p", "p", "l", "e", "|", "f", "o", "r", "|", "$", "1", ".", "0", "0", "|", "a", "t", "|", "t", "h", "e", "|", "s", "t", "o", "r", "e", "."], - ids: [3, 4, 24, 8, 17, 21, 13, 6, 4, 7, 10, 4, 7, 20, 20, 14, 5, 4, 19, 8, 12, 4, 48, 30, 37, 27, 27, 4, 7, 6, 4, 6, 13, 5, 4, 11, 6, 8, 12, 5, 37], - decoded: " bought an aple for $1.0 at the store.", - }, - SPIECE_UNDERSCORE: { - text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, - tokens: ["\u2581", "T", "h", "i", "s", "|", "\u2581", "i", "s", "|", "\u2581", "a", "|", "\u2581", "t", "e", "s", "t", "|", "\u2581", "."], - ids: [3, 3, 13, 9, 11, 4, 3, 9, 11, 4, 3, 7, 4, 3, 6, 5, 11, 6, 4, 3, 37], - decoded: "his is a test .", - }, - SIMPLE_WITH_ACCENTS: { - text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, - tokens: ["H", "\u00e9", "l", "l", "o"], - ids: [3, 55, 14, 14, 8], - decoded: "\u00e9lo", - }, - MIXED_CASE_WITHOUT_ACCENTS: { - text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, - tokens: ["|", "\t", "H", "e", "L", "L", "o", "!", "h", "o", "w", "|", "|", "\n", "|", "A", "r", "e", "|", "y", "o", "U", "?", "|", "|"], - ids: [4, 3, 3, 5, 3, 3, 8, 75, 13, 8, 23, 4, 4, 3, 4, 3, 12, 5, 4, 22, 8, 3, 3, 4, 4], - decoded: "eo!how re yo", - }, - MIXED_CASE_WITH_ACCENTS: { - text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, - tokens: ["|", "\t", "H", "\u00e4", "L", "L", "o", "!", "h", "o", "w", "|", "|", "\n", "|", "A", "r", "e", "|", "y", "o", "U", "?", "|", "|"], - ids: [4, 3, 3, 78, 3, 3, 8, 75, 13, 8, 23, 4, 4, 3, 4, 3, 12, 5, 4, 22, 8, 3, 3, 4, 4], - decoded: "\u00e4o!how re yo", - }, - }, - "Xenova/mms-1b-fl102": { - MIXED_CASE_WITH_ACCENTS: { - text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, - tokens: ["|", "\t", "H", "\u00e4", "L", "L", "o", "!", "h", "o", "w", "|", "|", "\n", "|", "A", "r", "e", "|", "y", "o", "U", "?", "|", "|"], - ids: [4, 3, 3, 3, 3, 3, 8, 75, 13, 8, 23, 4, 4, 3, 4, 3, 12, 5, 4, 22, 8, 3, 3, 4, 4], - decoded: "o!how re yo", - }, - }, - "Xenova/mms-1b-l1107": { - NUMBERS: { - text: BASE_TEST_STRINGS.NUMBERS, - tokens: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "|", "0", "|", "1", "|", "2", "|", "3", "|", "4", "|", "5", "|", "6", "|", "7", "|", "8", "|", "9", "|", "1", "0", "|", "1", "0", "0", "|", "1", "0", "0", "0"], - ids: [34, 36, 37, 42, 38, 41, 39, 3, 3, 3, 4, 34, 4, 36, 4, 37, 4, 42, 4, 38, 4, 41, 4, 39, 4, 3, 4, 3, 4, 3, 4, 36, 34, 4, 36, 34, 34, 4, 36, 34, 34, 34], - decoded: "0123456 0 1 2 3 4 5 6 10 10 10", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["T", "h", "e", "|", "c", "o", "m", "p", "a", "n", "y", "|", "w", "a", "s", "|", "f", "o", "u", "n", "d", "e", "d", "|", "i", "n", "|", "2", "0", "1", "6", "."], - ids: [3, 9, 5, 4, 21, 7, 18, 24, 8, 10, 20, 4, 17, 8, 12, 4, 19, 7, 16, 10, 14, 5, 14, 4, 11, 10, 4, 37, 34, 36, 39, 3], - decoded: "he company was founded in 2016", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["t", "e", "s", "t", "|", "$", "1", "|", "R", "2", "|", "#", "3", "|", "\u20ac", "4", "|", "\u00a3", "5", "|", "\u00a5", "6", "|", "\u20a3", "7", "|", "\u20b9", "8", "|", "\u20b1", "9", "|", "t", "e", "s", "t"], - ids: [6, 5, 12, 6, 4, 3, 36, 4, 3, 37, 4, 3, 42, 4, 3, 38, 4, 3, 41, 4, 3, 39, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 6, 5, 12, 6], - decoded: "test 1 2 3 4 5 6 test", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["I", "|", "b", "o", "u", "g", "h", "t", "|", "a", "n", "|", "a", "p", "p", "l", "e", "|", "f", "o", "r", "|", "$", "1", ".", "0", "0", "|", "a", "t", "|", "t", "h", "e", "|", "s", "t", "o", "r", "e", "."], - ids: [3, 4, 23, 7, 16, 22, 9, 6, 4, 8, 10, 4, 8, 24, 24, 15, 5, 4, 19, 7, 13, 4, 3, 36, 3, 34, 34, 4, 8, 6, 4, 6, 9, 5, 4, 12, 6, 7, 13, 5, 3], - decoded: " bought an aple for 10 at the store", - }, - }, }; diff --git a/tests/models/whisper/tokenization.js b/tests/models/whisper/tokenization.js index bb3b1c685..74d1b4250 100644 --- a/tests/models/whisper/tokenization.js +++ b/tests/models/whisper/tokenization.js @@ -4,775 +4,775 @@ import { compare } from "../../test_utils.js"; export const TOKENIZER_CLASS = WhisperTokenizer; export const TEST_CONFIG = { - "onnx-community/whisper-tiny.en": { - SIMPLE: { - text: BASE_TEST_STRINGS.SIMPLE, - tokens: ["How", "\u0120are", "\u0120you", "\u0120doing", "?"], - ids: [50257, 50362, 2437, 389, 345, 1804, 30, 50256], - decoded: "<|startoftranscript|><|notimestamps|>How are you doing?<|endoftext|>", - }, - SIMPLE_WITH_PUNCTUATION: { - text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, - tokens: ["You", "\u0120should", "'ve", "\u0120done", "\u0120this"], - ids: [50257, 50362, 1639, 815, 1053, 1760, 428, 50256], - decoded: "<|startoftranscript|><|notimestamps|>You should've done this<|endoftext|>", - }, - NUMBERS: { - text: BASE_TEST_STRINGS.NUMBERS, - tokens: ["01", "23", "45", "67", "89", "\u01200", "\u01201", "\u01202", "\u01203", "\u01204", "\u01205", "\u01206", "\u01207", "\u01208", "\u01209", "\u012010", "\u0120100", "\u01201000"], - ids: [50257, 50362, 486, 1954, 2231, 3134, 4531, 657, 352, 362, 513, 604, 642, 718, 767, 807, 860, 838, 1802, 8576, 50256], - decoded: "<|startoftranscript|><|notimestamps|>0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000<|endoftext|>", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["The", "\u0120company", "\u0120was", "\u0120founded", "\u0120in", "\u01202016", "."], - ids: [50257, 50362, 464, 1664, 373, 9393, 287, 1584, 13, 50256], - decoded: "<|startoftranscript|><|notimestamps|>The company was founded in 2016.<|endoftext|>", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["A", "\u010a", "'ll", "\u0120!!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'t", "."], - ids: [50257, 50362, 32, 198, 1183, 37867, 1462, 8348, 67, 7061, 67, 286, 11, 460, 470, 13, 50256], - decoded: "<|startoftranscript|><|notimestamps|>A\n'll!!to?'d''d of, can't.<|endoftext|>", - }, - PYTHON_CODE: { - text: BASE_TEST_STRINGS.PYTHON_CODE, - tokens: ["def", "\u0120main", "():", "\u010a", "\u0109", "pass"], - ids: [50257, 50362, 4299, 1388, 33529, 198, 197, 6603, 50256], - decoded: "<|startoftranscript|><|notimestamps|>def main():\n\tpass<|endoftext|>", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".", "to", "String", "();", "\u010a", "to", "String", "();"], - ids: [50257, 50362, 1616, 257, 796, 26181, 13, 1462, 10100, 9783, 198, 1462, 10100, 9783, 50256], - decoded: "<|startoftranscript|><|notimestamps|>let a = obj.toString();\ntoString();<|endoftext|>", - }, - NEWLINES: { - text: BASE_TEST_STRINGS.NEWLINES, - tokens: ["This", "\u010a", "\u010a", "is", "\u010a", "a", "\u010a", "test", "."], - ids: [50257, 50362, 1212, 198, 198, 271, 198, 64, 198, 9288, 13, 50256], - decoded: "<|startoftranscript|><|notimestamps|>This\n\nis\na\ntest.<|endoftext|>", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["UN", "want", "\u00c3\u00a9", "d", ",", "running"], - ids: [50257, 50362, 4944, 42949, 2634, 67, 11, 20270, 50256], - decoded: "<|startoftranscript|><|notimestamps|>UNwant\u00e9d,running<|endoftext|>", - }, - CONTROL_TOKENS: { - text: BASE_TEST_STRINGS.CONTROL_TOKENS, - tokens: ["1", "\u0100", "2", "\u00ef\u00bf\u00bd", "3"], - ids: [50257, 50362, 16, 188, 17, 4210, 18, 50256], - decoded: "<|startoftranscript|><|notimestamps|>1\u00002\ufffd3<|endoftext|>", - }, - HELLO_WORLD_TITLECASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, - tokens: ["Hello", "\u0120World"], - ids: [50257, 50362, 15496, 2159, 50256], - decoded: "<|startoftranscript|><|notimestamps|>Hello World<|endoftext|>", - }, - HELLO_WORLD_LOWERCASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, - tokens: ["hello", "\u0120world"], - ids: [50257, 50362, 31373, 995, 50256], - decoded: "<|startoftranscript|><|notimestamps|>hello world<|endoftext|>", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u00e7\u0136\u0141", "\u00e6", "\u00b4", "\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e", "\u0141", "\u00e8", "\u00b0", "\u013d", "\u00e6\u013a\u00af"], - ids: [50257, 50362, 37955, 162, 112, 119, 21410, 40367, 253, 164, 108, 249, 42468, 50256], - decoded: "<|startoftranscript|><|notimestamps|>\u751f\u6d3b\u7684\u771f\u8c1b\u662f<|endoftext|>", - }, - LEADING_SPACE: { - text: BASE_TEST_STRINGS.LEADING_SPACE, - tokens: ["\u0120", "\u0120", "\u0120leading", "\u0120space"], - ids: [50257, 50362, 220, 220, 3756, 2272, 50256], - decoded: "<|startoftranscript|><|notimestamps|> leading space<|endoftext|>", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["tra", "iling", "\u0120space", "\u0120", "\u0120", "\u0120"], - ids: [50257, 50362, 9535, 4386, 2272, 220, 220, 220, 50256], - decoded: "<|startoftranscript|><|notimestamps|>trailing space <|endoftext|>", - }, - DOUBLE_SPACE: { - text: BASE_TEST_STRINGS.DOUBLE_SPACE, - tokens: ["Hi", "\u0120", "\u0120Hello"], - ids: [50257, 50362, 17250, 220, 18435, 50256], - decoded: "<|startoftranscript|><|notimestamps|>Hi Hello<|endoftext|>", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2", "\u0124", "\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120test"], - ids: [50257, 50362, 9288, 720, 16, 371, 17, 1303, 18, 10432, 19, 4248, 20, 38221, 21, 2343, 224, 96, 22, 2343, 224, 117, 23, 2343, 224, 109, 24, 1332, 50256], - decoded: "<|startoftranscript|><|notimestamps|>test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test<|endoftext|>", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["I", "\u0120bought", "\u0120an", "\u0120apple", "\u0120for", "\u0120$", "1", ".", "00", "\u0120at", "\u0120the", "\u0120store", "."], - ids: [50257, 50362, 40, 5839, 281, 17180, 329, 720, 16, 13, 405, 379, 262, 3650, 13, 50256], - decoded: "<|startoftranscript|><|notimestamps|>I bought an apple for $1.00 at the store.<|endoftext|>", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - tokens: ["you", "\u00e2\u0122\u00a6", "\u0120", "\u0120"], - ids: [50257, 50362, 5832, 1399, 220, 220, 50256], - decoded: "<|startoftranscript|><|notimestamps|>you\u2026 <|endoftext|>", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], - ids: [50257, 50362, 5832, 1399, 4603, 50256], - decoded: "<|startoftranscript|><|notimestamps|>you\u2026\u00a0\u00a0<|endoftext|>", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142", "\u00c2\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], - ids: [50257, 50362, 5832, 1399, 1849, 1849, 5832, 1399, 4603, 50256], - decoded: "<|startoftranscript|><|notimestamps|>you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0<|endoftext|>", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["we", "ird", "\u0120\u00ef", "\u00bd", "\u0140", "\u0120edge", "\u0120\u00ef", "\u00bd", "\u0140", "\u0120case"], - ids: [50257, 50362, 732, 1447, 27332, 121, 252, 5743, 27332, 121, 252, 1339, 50256], - decoded: "<|startoftranscript|><|notimestamps|>weird \uff5e edge \uff5e case<|endoftext|>", - }, - SPIECE_UNDERSCORE: { - text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, - tokens: ["\u00e2\u0138", "\u0123", "This", "\u0120\u00e2\u0138", "\u0123", "is", "\u0120\u00e2\u0138", "\u0123", "a", "\u0120\u00e2\u0138", "\u0123", "test", "\u0120\u00e2\u0138", "\u0123", "."], - ids: [50257, 50362, 5008, 223, 1212, 11019, 223, 271, 11019, 223, 64, 11019, 223, 9288, 11019, 223, 13, 50256], - decoded: "<|startoftranscript|><|notimestamps|>\u2581This \u2581is \u2581a \u2581test \u2581.<|endoftext|>", - }, - SPECIAL_TOKENS: { - text: WHISPER_TEST_STRINGS.SPECIAL_TOKENS, - tokens: ["\u0120", "\u0120", "\u0120", "<|startoftranscript|>", "\u0120", "<|en|>", "\u0120", "\u0120", "\u0120"], - ids: [50257, 50362, 220, 220, 220, 50257, 220, 50258, 220, 220, 220, 50256], - decoded: "<|startoftranscript|><|notimestamps|> <|startoftranscript|> <|en|> <|endoftext|>", - }, - }, - "distil-whisper/distil-large-v3": { - NUMBERS: { - text: BASE_TEST_STRINGS.NUMBERS, - tokens: ["0", "12", "3", "45", "67", "89", "\u01200", "\u01201", "\u01202", "\u01203", "\u01204", "\u01205", "\u01206", "\u01207", "\u01208", "\u01209", "\u012010", "\u0120100", "\u01201000"], - ids: [50258, 50364, 15, 4762, 18, 8465, 22452, 21115, 1958, 502, 568, 805, 1017, 1025, 1386, 1614, 1649, 1722, 1266, 2319, 9714, 50257], - decoded: "<|startoftranscript|><|notimestamps|>0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000<|endoftext|>", - }, - PYTHON_CODE: { - text: BASE_TEST_STRINGS.PYTHON_CODE, - tokens: ["def", "\u0120main", "(", "):", "\u010a", "\u0109", "pass"], - ids: [50258, 50364, 20595, 2135, 7, 4507, 198, 197, 9216, 50257], - decoded: "<|startoftranscript|><|notimestamps|>def main():\n\tpass<|endoftext|>", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["let", "\u0120a", "\u0120=", "\u0120ob", "j", ".", "to", "St", "ring", "(", ");", "\u010a", "to", "St", "ring", "(", ");"], - ids: [50258, 50364, 2631, 257, 6585, 1111, 73, 13, 1353, 4520, 2937, 7, 34446, 198, 1353, 4520, 2937, 7, 34446, 50257], - decoded: "<|startoftranscript|><|notimestamps|>let a = obj.toString();\ntoString();<|endoftext|>", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["UN", "w", "ant", "\u00c3\u00a9d", ",", "running"], - ids: [50258, 50364, 3979, 86, 394, 7811, 11, 45482, 50257], - decoded: "<|startoftranscript|><|notimestamps|>UNwant\u00e9d,running<|endoftext|>", - }, - HELLO_WORLD_LOWERCASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, - tokens: ["he", "llo", "\u0120world"], - ids: [50258, 50364, 675, 1913, 1002, 50257], - decoded: "<|startoftranscript|><|notimestamps|>hello world<|endoftext|>", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u00e7\u0136\u0141\u00e6\u00b4\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"], - ids: [50258, 50364, 49958, 1546, 6303, 8897, 249, 1541, 50257], - decoded: "<|startoftranscript|><|notimestamps|>\u751f\u6d3b\u7684\u771f\u8c1b\u662f<|endoftext|>", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2", "\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2", "\u0124", "\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120test"], - ids: [50258, 50364, 31636, 1848, 16, 497, 17, 3536, 18, 17450, 19, 14378, 20, 1815, 98, 21, 672, 224, 96, 22, 672, 224, 117, 23, 672, 224, 109, 24, 1500, 50257], - decoded: "<|startoftranscript|><|notimestamps|>test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test<|endoftext|>", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142"], - ids: [50258, 50364, 5616, 1260, 126, 254, 126, 254, 50257], - decoded: "<|startoftranscript|><|notimestamps|>you\u2026\u00a0\u00a0<|endoftext|>", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142"], - ids: [50258, 50364, 5616, 1260, 126, 254, 126, 254, 5616, 1260, 126, 254, 126, 254, 50257], - decoded: "<|startoftranscript|><|notimestamps|>you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0<|endoftext|>", - }, - }, - "distil-whisper/distil-large-v2": { - SIMPLE: { - text: BASE_TEST_STRINGS.SIMPLE, - tokens: ["How", "\u0120are", "\u0120you", "\u0120doing", "?"], - ids: [50258, 50259, 50359, 50363, 6462, 366, 291, 884, 30, 50257], - decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>How are you doing?<|endoftext|>", - }, - SIMPLE_WITH_PUNCTUATION: { - text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, - tokens: ["You", "\u0120should", "'ve", "\u0120done", "\u0120", "this"], - ids: [50258, 50259, 50359, 50363, 3223, 820, 600, 1096, 220, 11176, 50257], - decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>You should've done this<|endoftext|>", - }, - NUMBERS: { - text: BASE_TEST_STRINGS.NUMBERS, - tokens: ["0", "12", "3", "45", "67", "89", "\u01200", "\u01201", "\u01202", "\u01203", "\u01204", "\u01205", "\u01206", "\u01207", "\u01208", "\u01209", "\u012010", "\u0120100", "\u01201000"], - ids: [50258, 50259, 50359, 50363, 15, 4762, 18, 8465, 22452, 21115, 1958, 502, 568, 805, 1017, 1025, 1386, 1614, 1649, 1722, 1266, 2319, 9714, 50257], - decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000<|endoftext|>", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["The", "\u0120company", "\u0120was", "\u0120founded", "\u0120in", "\u01202016", "."], - ids: [50258, 50259, 50359, 50363, 2278, 2237, 390, 13234, 294, 6549, 13, 50257], - decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>The company was founded in 2016.<|endoftext|>", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["A", "\u010a", "'ll", "\u0120!!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'t", "."], - ids: [50258, 50259, 50359, 50363, 32, 198, 603, 15138, 1353, 8569, 67, 15025, 67, 295, 11, 393, 380, 13, 50257], - decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>A\n'll!!to?'d''d of, can't.<|endoftext|>", - }, - PYTHON_CODE: { - text: BASE_TEST_STRINGS.PYTHON_CODE, - tokens: ["def", "\u0120main", "(", "):", "\u010a", "\u0109", "pass"], - ids: [50258, 50259, 50359, 50363, 20595, 2135, 7, 4507, 198, 197, 9216, 50257], - decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>def main():\n\tpass<|endoftext|>", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["let", "\u0120a", "\u0120=", "\u0120ob", "j", ".", "to", "St", "ring", "(", ");", "\u010a", "to", "St", "ring", "(", ");"], - ids: [50258, 50259, 50359, 50363, 2631, 257, 6585, 1111, 73, 13, 1353, 4520, 2937, 7, 34446, 198, 1353, 4520, 2937, 7, 34446, 50257], - decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>let a = obj.toString();\ntoString();<|endoftext|>", - }, - NEWLINES: { - text: BASE_TEST_STRINGS.NEWLINES, - tokens: ["This", "\u010a", "\u010a", "is", "\u010a", "a", "\u010a", "test", "."], - ids: [50258, 50259, 50359, 50363, 5723, 198, 198, 271, 198, 64, 198, 31636, 13, 50257], - decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>This\n\nis\na\ntest.<|endoftext|>", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["UN", "w", "ant", "\u00c3\u00a9d", ",", "running"], - ids: [50258, 50259, 50359, 50363, 3979, 86, 394, 7811, 11, 45482, 50257], - decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>UNwant\u00e9d,running<|endoftext|>", - }, - CONTROL_TOKENS: { - text: BASE_TEST_STRINGS.CONTROL_TOKENS, - tokens: ["1", "\u0100", "2", "\u00ef\u00bf\u00bd", "3"], - ids: [50258, 50259, 50359, 50363, 16, 188, 17, 5342, 18, 50257], - decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>1\u00002\ufffd3<|endoftext|>", - }, - HELLO_WORLD_TITLECASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, - tokens: ["Hello", "\u0120World"], - ids: [50258, 50259, 50359, 50363, 15947, 3937, 50257], - decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>Hello World<|endoftext|>", - }, - HELLO_WORLD_LOWERCASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, - tokens: ["he", "llo", "\u0120world"], - ids: [50258, 50259, 50359, 50363, 675, 1913, 1002, 50257], - decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>hello world<|endoftext|>", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u00e7\u0136\u0141\u00e6\u00b4\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"], - ids: [50258, 50259, 50359, 50363, 49958, 1546, 6303, 8897, 249, 1541, 50257], - decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>\u751f\u6d3b\u7684\u771f\u8c1b\u662f<|endoftext|>", - }, - LEADING_SPACE: { - text: BASE_TEST_STRINGS.LEADING_SPACE, - tokens: ["\u0120", "\u0120", "\u0120leading", "\u0120space"], - ids: [50258, 50259, 50359, 50363, 220, 220, 5775, 1901, 50257], - decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|> leading space<|endoftext|>", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["tra", "iling", "\u0120space", "\u0120", "\u0120", "\u0120"], - ids: [50258, 50259, 50359, 50363, 17227, 4883, 1901, 220, 220, 220, 50257], - decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>trailing space <|endoftext|>", - }, - DOUBLE_SPACE: { - text: BASE_TEST_STRINGS.DOUBLE_SPACE, - tokens: ["Hi", "\u0120", "\u0120Hello"], - ids: [50258, 50259, 50359, 50363, 17155, 220, 2425, 50257], - decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>Hi Hello<|endoftext|>", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2", "\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2", "\u0124", "\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120", "test"], - ids: [50258, 50259, 50359, 50363, 31636, 1848, 16, 497, 17, 3536, 18, 17450, 19, 14378, 20, 1815, 98, 21, 672, 224, 96, 22, 672, 224, 117, 23, 672, 224, 109, 24, 220, 31636, 50257], - decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test<|endoftext|>", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["I", "\u0120bought", "\u0120an", "\u0120apple", "\u0120for", "\u0120$", "1", ".", "00", "\u0120at", "\u0120", "the", "\u0120store", "."], - ids: [50258, 50259, 50359, 50363, 40, 4243, 364, 10606, 337, 1848, 16, 13, 628, 412, 220, 3322, 3531, 13, 50257], - decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>I bought an apple for $1.00 at the store.<|endoftext|>", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - tokens: ["you", "\u00e2\u0122\u00a6", "\u0120", "\u0120"], - ids: [50258, 50259, 50359, 50363, 5616, 1260, 220, 220, 50257], - decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>you\u2026 <|endoftext|>", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142"], - ids: [50258, 50259, 50359, 50363, 5616, 1260, 126, 254, 126, 254, 50257], - decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>you\u2026\u00a0\u00a0<|endoftext|>", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142"], - ids: [50258, 50259, 50359, 50363, 5616, 1260, 126, 254, 126, 254, 5616, 1260, 126, 254, 126, 254, 50257], - decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0<|endoftext|>", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["we", "ird", "\u0120\u00ef", "\u00bd", "\u0140", "\u0120edge", "\u0120\u00ef", "\u00bd", "\u0140", "\u0120case"], - ids: [50258, 50259, 50359, 50363, 826, 1271, 25072, 121, 252, 4691, 25072, 121, 252, 1389, 50257], - decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>weird \uff5e edge \uff5e case<|endoftext|>", - }, - SPIECE_UNDERSCORE: { - text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, - tokens: ["\u00e2\u0138", "\u0123", "This", "\u0120\u00e2\u0138", "\u0123", "is", "\u0120\u00e2\u0138", "\u0123", "a", "\u0120\u00e2\u0138", "\u0123", "test", "\u0120\u00e2\u0138", "\u0123", "."], - ids: [50258, 50259, 50359, 50363, 39984, 223, 5723, 29405, 223, 271, 29405, 223, 64, 29405, 223, 31636, 29405, 223, 13, 50257], - decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>\u2581This \u2581is \u2581a \u2581test \u2581.<|endoftext|>", - }, - SPECIAL_TOKENS: { - text: WHISPER_TEST_STRINGS.SPECIAL_TOKENS, - tokens: ["\u0120", "\u0120", "\u0120", "<|startoftranscript|>", "\u0120", "<|en|>", "\u0120", "\u0120", "\u0120"], - ids: [50258, 50259, 50359, 50363, 220, 220, 220, 50258, 220, 50259, 220, 220, 220, 50257], - decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|> <|startoftranscript|> <|en|> <|endoftext|>", - }, - }, - "distil-whisper/distil-small.en": { - SPECIAL_TOKENS: { - text: WHISPER_TEST_STRINGS.SPECIAL_TOKENS, + "onnx-community/whisper-tiny.en": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["How", "\u0120are", "\u0120you", "\u0120doing", "?"], + ids: [50257, 50362, 2437, 389, 345, 1804, 30, 50256], + decoded: "<|startoftranscript|><|notimestamps|>How are you doing?<|endoftext|>", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["You", "\u0120should", "'ve", "\u0120done", "\u0120this"], + ids: [50257, 50362, 1639, 815, 1053, 1760, 428, 50256], + decoded: "<|startoftranscript|><|notimestamps|>You should've done this<|endoftext|>", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["01", "23", "45", "67", "89", "\u01200", "\u01201", "\u01202", "\u01203", "\u01204", "\u01205", "\u01206", "\u01207", "\u01208", "\u01209", "\u012010", "\u0120100", "\u01201000"], + ids: [50257, 50362, 486, 1954, 2231, 3134, 4531, 657, 352, 362, 513, 604, 642, 718, 767, 807, 860, 838, 1802, 8576, 50256], + decoded: "<|startoftranscript|><|notimestamps|>0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000<|endoftext|>", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "\u0120company", "\u0120was", "\u0120founded", "\u0120in", "\u01202016", "."], + ids: [50257, 50362, 464, 1664, 373, 9393, 287, 1584, 13, 50256], + decoded: "<|startoftranscript|><|notimestamps|>The company was founded in 2016.<|endoftext|>", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "\u010a", "'ll", "\u0120!!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'t", "."], + ids: [50257, 50362, 32, 198, 1183, 37867, 1462, 8348, 67, 7061, 67, 286, 11, 460, 470, 13, 50256], + decoded: "<|startoftranscript|><|notimestamps|>A\n'll!!to?'d''d of, can't.<|endoftext|>", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["def", "\u0120main", "():", "\u010a", "\u0109", "pass"], + ids: [50257, 50362, 4299, 1388, 33529, 198, 197, 6603, 50256], + decoded: "<|startoftranscript|><|notimestamps|>def main():\n\tpass<|endoftext|>", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".", "to", "String", "();", "\u010a", "to", "String", "();"], + ids: [50257, 50362, 1616, 257, 796, 26181, 13, 1462, 10100, 9783, 198, 1462, 10100, 9783, 50256], + decoded: "<|startoftranscript|><|notimestamps|>let a = obj.toString();\ntoString();<|endoftext|>", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["This", "\u010a", "\u010a", "is", "\u010a", "a", "\u010a", "test", "."], + ids: [50257, 50362, 1212, 198, 198, 271, 198, 64, 198, 9288, 13, 50256], + decoded: "<|startoftranscript|><|notimestamps|>This\n\nis\na\ntest.<|endoftext|>", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "want", "\u00c3\u00a9", "d", ",", "running"], + ids: [50257, 50362, 4944, 42949, 2634, 67, 11, 20270, 50256], + decoded: "<|startoftranscript|><|notimestamps|>UNwant\u00e9d,running<|endoftext|>", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["1", "\u0100", "2", "\u00ef\u00bf\u00bd", "3"], + ids: [50257, 50362, 16, 188, 17, 4210, 18, 50256], + decoded: "<|startoftranscript|><|notimestamps|>1\u00002\ufffd3<|endoftext|>", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["Hello", "\u0120World"], + ids: [50257, 50362, 15496, 2159, 50256], + decoded: "<|startoftranscript|><|notimestamps|>Hello World<|endoftext|>", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["hello", "\u0120world"], + ids: [50257, 50362, 31373, 995, 50256], + decoded: "<|startoftranscript|><|notimestamps|>hello world<|endoftext|>", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u00e7\u0136\u0141", "\u00e6", "\u00b4", "\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e", "\u0141", "\u00e8", "\u00b0", "\u013d", "\u00e6\u013a\u00af"], + ids: [50257, 50362, 37955, 162, 112, 119, 21410, 40367, 253, 164, 108, 249, 42468, 50256], + decoded: "<|startoftranscript|><|notimestamps|>\u751f\u6d3b\u7684\u771f\u8c1b\u662f<|endoftext|>", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u0120", "\u0120", "\u0120leading", "\u0120space"], + ids: [50257, 50362, 220, 220, 3756, 2272, 50256], + decoded: "<|startoftranscript|><|notimestamps|> leading space<|endoftext|>", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["tra", "iling", "\u0120space", "\u0120", "\u0120", "\u0120"], + ids: [50257, 50362, 9535, 4386, 2272, 220, 220, 220, 50256], + decoded: "<|startoftranscript|><|notimestamps|>trailing space <|endoftext|>", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["Hi", "\u0120", "\u0120Hello"], + ids: [50257, 50362, 17250, 220, 18435, 50256], + decoded: "<|startoftranscript|><|notimestamps|>Hi Hello<|endoftext|>", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2", "\u0124", "\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120test"], + ids: [50257, 50362, 9288, 720, 16, 371, 17, 1303, 18, 10432, 19, 4248, 20, 38221, 21, 2343, 224, 96, 22, 2343, 224, 117, 23, 2343, 224, 109, 24, 1332, 50256], + decoded: "<|startoftranscript|><|notimestamps|>test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test<|endoftext|>", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "\u0120bought", "\u0120an", "\u0120apple", "\u0120for", "\u0120$", "1", ".", "00", "\u0120at", "\u0120the", "\u0120store", "."], + ids: [50257, 50362, 40, 5839, 281, 17180, 329, 720, 16, 13, 405, 379, 262, 3650, 13, 50256], + decoded: "<|startoftranscript|><|notimestamps|>I bought an apple for $1.00 at the store.<|endoftext|>", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u0120", "\u0120"], + ids: [50257, 50362, 5832, 1399, 220, 220, 50256], + decoded: "<|startoftranscript|><|notimestamps|>you\u2026 <|endoftext|>", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], + ids: [50257, 50362, 5832, 1399, 4603, 50256], + decoded: "<|startoftranscript|><|notimestamps|>you\u2026\u00a0\u00a0<|endoftext|>", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142", "\u00c2\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], + ids: [50257, 50362, 5832, 1399, 1849, 1849, 5832, 1399, 4603, 50256], + decoded: "<|startoftranscript|><|notimestamps|>you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0<|endoftext|>", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["we", "ird", "\u0120\u00ef", "\u00bd", "\u0140", "\u0120edge", "\u0120\u00ef", "\u00bd", "\u0140", "\u0120case"], + ids: [50257, 50362, 732, 1447, 27332, 121, 252, 5743, 27332, 121, 252, 1339, 50256], + decoded: "<|startoftranscript|><|notimestamps|>weird \uff5e edge \uff5e case<|endoftext|>", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u00e2\u0138", "\u0123", "This", "\u0120\u00e2\u0138", "\u0123", "is", "\u0120\u00e2\u0138", "\u0123", "a", "\u0120\u00e2\u0138", "\u0123", "test", "\u0120\u00e2\u0138", "\u0123", "."], + ids: [50257, 50362, 5008, 223, 1212, 11019, 223, 271, 11019, 223, 64, 11019, 223, 9288, 11019, 223, 13, 50256], + decoded: "<|startoftranscript|><|notimestamps|>\u2581This \u2581is \u2581a \u2581test \u2581.<|endoftext|>", + }, + SPECIAL_TOKENS: { + text: WHISPER_TEST_STRINGS.SPECIAL_TOKENS, + tokens: ["\u0120", "\u0120", "\u0120", "<|startoftranscript|>", "\u0120", "<|en|>", "\u0120", "\u0120", "\u0120"], + ids: [50257, 50362, 220, 220, 220, 50257, 220, 50258, 220, 220, 220, 50256], + decoded: "<|startoftranscript|><|notimestamps|> <|startoftranscript|> <|en|> <|endoftext|>", + }, + }, + "distil-whisper/distil-large-v3": { + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["0", "12", "3", "45", "67", "89", "\u01200", "\u01201", "\u01202", "\u01203", "\u01204", "\u01205", "\u01206", "\u01207", "\u01208", "\u01209", "\u012010", "\u0120100", "\u01201000"], + ids: [50258, 50364, 15, 4762, 18, 8465, 22452, 21115, 1958, 502, 568, 805, 1017, 1025, 1386, 1614, 1649, 1722, 1266, 2319, 9714, 50257], + decoded: "<|startoftranscript|><|notimestamps|>0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000<|endoftext|>", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["def", "\u0120main", "(", "):", "\u010a", "\u0109", "pass"], + ids: [50258, 50364, 20595, 2135, 7, 4507, 198, 197, 9216, 50257], + decoded: "<|startoftranscript|><|notimestamps|>def main():\n\tpass<|endoftext|>", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "\u0120a", "\u0120=", "\u0120ob", "j", ".", "to", "St", "ring", "(", ");", "\u010a", "to", "St", "ring", "(", ");"], + ids: [50258, 50364, 2631, 257, 6585, 1111, 73, 13, 1353, 4520, 2937, 7, 34446, 198, 1353, 4520, 2937, 7, 34446, 50257], + decoded: "<|startoftranscript|><|notimestamps|>let a = obj.toString();\ntoString();<|endoftext|>", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "w", "ant", "\u00c3\u00a9d", ",", "running"], + ids: [50258, 50364, 3979, 86, 394, 7811, 11, 45482, 50257], + decoded: "<|startoftranscript|><|notimestamps|>UNwant\u00e9d,running<|endoftext|>", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["he", "llo", "\u0120world"], + ids: [50258, 50364, 675, 1913, 1002, 50257], + decoded: "<|startoftranscript|><|notimestamps|>hello world<|endoftext|>", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u00e7\u0136\u0141\u00e6\u00b4\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"], + ids: [50258, 50364, 49958, 1546, 6303, 8897, 249, 1541, 50257], + decoded: "<|startoftranscript|><|notimestamps|>\u751f\u6d3b\u7684\u771f\u8c1b\u662f<|endoftext|>", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2", "\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2", "\u0124", "\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120test"], + ids: [50258, 50364, 31636, 1848, 16, 497, 17, 3536, 18, 17450, 19, 14378, 20, 1815, 98, 21, 672, 224, 96, 22, 672, 224, 117, 23, 672, 224, 109, 24, 1500, 50257], + decoded: "<|startoftranscript|><|notimestamps|>test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test<|endoftext|>", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142"], + ids: [50258, 50364, 5616, 1260, 126, 254, 126, 254, 50257], + decoded: "<|startoftranscript|><|notimestamps|>you\u2026\u00a0\u00a0<|endoftext|>", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142"], + ids: [50258, 50364, 5616, 1260, 126, 254, 126, 254, 5616, 1260, 126, 254, 126, 254, 50257], + decoded: "<|startoftranscript|><|notimestamps|>you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0<|endoftext|>", + }, + }, + "distil-whisper/distil-large-v2": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["How", "\u0120are", "\u0120you", "\u0120doing", "?"], + ids: [50258, 50259, 50359, 50363, 6462, 366, 291, 884, 30, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>How are you doing?<|endoftext|>", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["You", "\u0120should", "'ve", "\u0120done", "\u0120", "this"], + ids: [50258, 50259, 50359, 50363, 3223, 820, 600, 1096, 220, 11176, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>You should've done this<|endoftext|>", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["0", "12", "3", "45", "67", "89", "\u01200", "\u01201", "\u01202", "\u01203", "\u01204", "\u01205", "\u01206", "\u01207", "\u01208", "\u01209", "\u012010", "\u0120100", "\u01201000"], + ids: [50258, 50259, 50359, 50363, 15, 4762, 18, 8465, 22452, 21115, 1958, 502, 568, 805, 1017, 1025, 1386, 1614, 1649, 1722, 1266, 2319, 9714, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000<|endoftext|>", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "\u0120company", "\u0120was", "\u0120founded", "\u0120in", "\u01202016", "."], + ids: [50258, 50259, 50359, 50363, 2278, 2237, 390, 13234, 294, 6549, 13, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>The company was founded in 2016.<|endoftext|>", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "\u010a", "'ll", "\u0120!!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'t", "."], + ids: [50258, 50259, 50359, 50363, 32, 198, 603, 15138, 1353, 8569, 67, 15025, 67, 295, 11, 393, 380, 13, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>A\n'll!!to?'d''d of, can't.<|endoftext|>", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["def", "\u0120main", "(", "):", "\u010a", "\u0109", "pass"], + ids: [50258, 50259, 50359, 50363, 20595, 2135, 7, 4507, 198, 197, 9216, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>def main():\n\tpass<|endoftext|>", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "\u0120a", "\u0120=", "\u0120ob", "j", ".", "to", "St", "ring", "(", ");", "\u010a", "to", "St", "ring", "(", ");"], + ids: [50258, 50259, 50359, 50363, 2631, 257, 6585, 1111, 73, 13, 1353, 4520, 2937, 7, 34446, 198, 1353, 4520, 2937, 7, 34446, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>let a = obj.toString();\ntoString();<|endoftext|>", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["This", "\u010a", "\u010a", "is", "\u010a", "a", "\u010a", "test", "."], + ids: [50258, 50259, 50359, 50363, 5723, 198, 198, 271, 198, 64, 198, 31636, 13, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>This\n\nis\na\ntest.<|endoftext|>", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "w", "ant", "\u00c3\u00a9d", ",", "running"], + ids: [50258, 50259, 50359, 50363, 3979, 86, 394, 7811, 11, 45482, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>UNwant\u00e9d,running<|endoftext|>", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["1", "\u0100", "2", "\u00ef\u00bf\u00bd", "3"], + ids: [50258, 50259, 50359, 50363, 16, 188, 17, 5342, 18, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>1\u00002\ufffd3<|endoftext|>", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["Hello", "\u0120World"], + ids: [50258, 50259, 50359, 50363, 15947, 3937, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>Hello World<|endoftext|>", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["he", "llo", "\u0120world"], + ids: [50258, 50259, 50359, 50363, 675, 1913, 1002, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>hello world<|endoftext|>", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u00e7\u0136\u0141\u00e6\u00b4\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"], + ids: [50258, 50259, 50359, 50363, 49958, 1546, 6303, 8897, 249, 1541, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>\u751f\u6d3b\u7684\u771f\u8c1b\u662f<|endoftext|>", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u0120", "\u0120", "\u0120leading", "\u0120space"], + ids: [50258, 50259, 50359, 50363, 220, 220, 5775, 1901, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|> leading space<|endoftext|>", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["tra", "iling", "\u0120space", "\u0120", "\u0120", "\u0120"], + ids: [50258, 50259, 50359, 50363, 17227, 4883, 1901, 220, 220, 220, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>trailing space <|endoftext|>", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["Hi", "\u0120", "\u0120Hello"], + ids: [50258, 50259, 50359, 50363, 17155, 220, 2425, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>Hi Hello<|endoftext|>", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2", "\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2", "\u0124", "\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120", "test"], + ids: [50258, 50259, 50359, 50363, 31636, 1848, 16, 497, 17, 3536, 18, 17450, 19, 14378, 20, 1815, 98, 21, 672, 224, 96, 22, 672, 224, 117, 23, 672, 224, 109, 24, 220, 31636, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test<|endoftext|>", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "\u0120bought", "\u0120an", "\u0120apple", "\u0120for", "\u0120$", "1", ".", "00", "\u0120at", "\u0120", "the", "\u0120store", "."], + ids: [50258, 50259, 50359, 50363, 40, 4243, 364, 10606, 337, 1848, 16, 13, 628, 412, 220, 3322, 3531, 13, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>I bought an apple for $1.00 at the store.<|endoftext|>", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u0120", "\u0120"], + ids: [50258, 50259, 50359, 50363, 5616, 1260, 220, 220, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>you\u2026 <|endoftext|>", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142"], + ids: [50258, 50259, 50359, 50363, 5616, 1260, 126, 254, 126, 254, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>you\u2026\u00a0\u00a0<|endoftext|>", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142"], + ids: [50258, 50259, 50359, 50363, 5616, 1260, 126, 254, 126, 254, 5616, 1260, 126, 254, 126, 254, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0<|endoftext|>", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["we", "ird", "\u0120\u00ef", "\u00bd", "\u0140", "\u0120edge", "\u0120\u00ef", "\u00bd", "\u0140", "\u0120case"], + ids: [50258, 50259, 50359, 50363, 826, 1271, 25072, 121, 252, 4691, 25072, 121, 252, 1389, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>weird \uff5e edge \uff5e case<|endoftext|>", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u00e2\u0138", "\u0123", "This", "\u0120\u00e2\u0138", "\u0123", "is", "\u0120\u00e2\u0138", "\u0123", "a", "\u0120\u00e2\u0138", "\u0123", "test", "\u0120\u00e2\u0138", "\u0123", "."], + ids: [50258, 50259, 50359, 50363, 39984, 223, 5723, 29405, 223, 271, 29405, 223, 64, 29405, 223, 31636, 29405, 223, 13, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>\u2581This \u2581is \u2581a \u2581test \u2581.<|endoftext|>", + }, + SPECIAL_TOKENS: { + text: WHISPER_TEST_STRINGS.SPECIAL_TOKENS, + tokens: ["\u0120", "\u0120", "\u0120", "<|startoftranscript|>", "\u0120", "<|en|>", "\u0120", "\u0120", "\u0120"], + ids: [50258, 50259, 50359, 50363, 220, 220, 220, 50258, 220, 50259, 220, 220, 220, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|> <|startoftranscript|> <|en|> <|endoftext|>", + }, + }, + "distil-whisper/distil-small.en": { + SPECIAL_TOKENS: { + text: WHISPER_TEST_STRINGS.SPECIAL_TOKENS, - // https://github.com/huggingface/transformers/issues/33371 - // tokens: [" <|startoftranscript|> ", "<|en|> "], - tokens: ["<|startoftranscript|>", "<|en|>"], - ids: [50257, 50362, 50257, 50258, 50256], - decoded: "<|startoftranscript|><|notimestamps|><|startoftranscript|><|en|><|endoftext|>", - }, - }, - "Xenova/nb-whisper-tiny-beta": { - SIMPLE: { - text: BASE_TEST_STRINGS.SIMPLE, - tokens: ["\u0120How", "\u0120are", "\u0120you", "\u0120doing", "?"], - ids: [50258, 50288, 50359, 50363, 1012, 366, 291, 884, 30, 50257], - decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> How are you doing?<|endoftext|>", - }, - SIMPLE_WITH_PUNCTUATION: { - text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, - tokens: ["\u0120You", "\u0120should", "'ve", "\u0120done", "\u0120", "this"], - ids: [50258, 50288, 50359, 50363, 509, 820, 600, 1096, 220, 11176, 50257], - decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> You should've done this<|endoftext|>", - }, - NUMBERS: { - text: BASE_TEST_STRINGS.NUMBERS, - tokens: ["\u01200", "12", "3", "45", "67", "89", "\u01200", "\u01201", "\u01202", "\u01203", "\u01204", "\u01205", "\u01206", "\u01207", "\u01208", "\u01209", "\u012010", "\u0120100", "\u01201000"], - ids: [50258, 50288, 50359, 50363, 1958, 4762, 18, 8465, 22452, 21115, 1958, 502, 568, 805, 1017, 1025, 1386, 1614, 1649, 1722, 1266, 2319, 9714, 50257], - decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> 0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000<|endoftext|>", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["\u0120The", "\u0120company", "\u0120was", "\u0120founded", "\u0120in", "\u01202016", "."], - ids: [50258, 50288, 50359, 50363, 440, 2237, 390, 13234, 294, 6549, 13, 50257], - decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> The company was founded in 2016.<|endoftext|>", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["\u0120A", "\u010a", "'ll", "\u0120!!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'t", "."], - ids: [50258, 50288, 50359, 50363, 316, 198, 603, 15138, 1353, 8569, 67, 15025, 67, 295, 11, 393, 380, 13, 50257], - decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> A\n'll!!to?'d''d of, can't.<|endoftext|>", - }, - PYTHON_CODE: { - text: BASE_TEST_STRINGS.PYTHON_CODE, - tokens: ["\u0120def", "\u0120main", "(", "):", "\u010a", "\u0109", "pass"], - ids: [50258, 50288, 50359, 50363, 1060, 2135, 7, 4507, 198, 197, 9216, 50257], - decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> def main():\n\tpass<|endoftext|>", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["\u0120let", "\u0120a", "\u0120=", "\u0120ob", "j", ".", "to", "St", "ring", "(", ");", "\u010a", "to", "St", "ring", "(", ");"], - ids: [50258, 50288, 50359, 50363, 718, 257, 6585, 1111, 73, 13, 1353, 4520, 2937, 7, 34446, 198, 1353, 4520, 2937, 7, 34446, 50257], - decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> let a = obj.toString();\ntoString();<|endoftext|>", - }, - NEWLINES: { - text: BASE_TEST_STRINGS.NEWLINES, - tokens: ["\u0120This", "\u010a", "\u010a", "is", "\u010a", "a", "\u010a", "test", "."], - ids: [50258, 50288, 50359, 50363, 639, 198, 198, 271, 198, 64, 198, 31636, 13, 50257], - decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> This\n\nis\na\ntest.<|endoftext|>", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["\u0120UN", "w", "ant", "\u00c3\u00a9d", ",", "running"], - ids: [50258, 50288, 50359, 50363, 8229, 86, 394, 7811, 11, 45482, 50257], - decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> UNwant\u00e9d,running<|endoftext|>", - }, - CONTROL_TOKENS: { - text: BASE_TEST_STRINGS.CONTROL_TOKENS, - tokens: ["\u01201", "\u0100", "2", "\u00ef\u00bf\u00bd", "3"], - ids: [50258, 50288, 50359, 50363, 502, 188, 17, 5342, 18, 50257], - decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> 1\u00002\ufffd3<|endoftext|>", - }, - HELLO_WORLD_TITLECASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, - tokens: ["\u0120Hello", "\u0120World"], - ids: [50258, 50288, 50359, 50363, 2425, 3937, 50257], - decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> Hello World<|endoftext|>", - }, - HELLO_WORLD_LOWERCASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, - tokens: ["\u0120hello", "\u0120world"], - ids: [50258, 50288, 50359, 50363, 7751, 1002, 50257], - decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> hello world<|endoftext|>", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u0120", "\u00e7\u0136\u0141\u00e6\u00b4\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"], - ids: [50258, 50288, 50359, 50363, 220, 49958, 1546, 6303, 8897, 249, 1541, 50257], - decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> \u751f\u6d3b\u7684\u771f\u8c1b\u662f<|endoftext|>", - }, - LEADING_SPACE: { - text: BASE_TEST_STRINGS.LEADING_SPACE, - tokens: ["\u0120", "\u0120", "\u0120leading", "\u0120space"], - ids: [50258, 50288, 50359, 50363, 220, 220, 5775, 1901, 50257], - decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> leading space<|endoftext|>", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["\u0120", "tra", "iling", "\u0120space", "\u0120", "\u0120", "\u0120"], - ids: [50258, 50288, 50359, 50363, 220, 17227, 4883, 1901, 220, 220, 220, 50257], - decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> trailing space <|endoftext|>", - }, - DOUBLE_SPACE: { - text: BASE_TEST_STRINGS.DOUBLE_SPACE, - tokens: ["\u0120Hi", "\u0120", "\u0120Hello"], - ids: [50258, 50288, 50359, 50363, 2421, 220, 2425, 50257], - decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> Hi Hello<|endoftext|>", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["\u0120", "test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2", "\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2", "\u0124", "\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120", "test"], - ids: [50258, 50288, 50359, 50363, 220, 31636, 1848, 16, 497, 17, 3536, 18, 17450, 19, 14378, 20, 1815, 98, 21, 672, 224, 96, 22, 672, 224, 117, 23, 672, 224, 109, 24, 220, 31636, 50257], - decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test<|endoftext|>", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["\u0120I", "\u0120bought", "\u0120an", "\u0120apple", "\u0120for", "\u0120$", "1", ".", "00", "\u0120at", "\u0120", "the", "\u0120store", "."], - ids: [50258, 50288, 50359, 50363, 286, 4243, 364, 10606, 337, 1848, 16, 13, 628, 412, 220, 3322, 3531, 13, 50257], - decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> I bought an apple for $1.00 at the store.<|endoftext|>", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - tokens: ["\u0120you", "\u00e2\u0122\u00a6", "\u0120", "\u0120"], - ids: [50258, 50288, 50359, 50363, 291, 1260, 220, 220, 50257], - decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> you\u2026 <|endoftext|>", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - tokens: ["\u0120you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142"], - ids: [50258, 50288, 50359, 50363, 291, 1260, 126, 254, 126, 254, 50257], - decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> you\u2026\u00a0\u00a0<|endoftext|>", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - tokens: ["\u0120you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142"], - ids: [50258, 50288, 50359, 50363, 291, 1260, 126, 254, 126, 254, 5616, 1260, 126, 254, 126, 254, 50257], - decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0<|endoftext|>", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["\u0120weird", "\u0120\u00ef", "\u00bd", "\u0140", "\u0120edge", "\u0120\u00ef", "\u00bd", "\u0140", "\u0120case"], - ids: [50258, 50288, 50359, 50363, 3657, 25072, 121, 252, 4691, 25072, 121, 252, 1389, 50257], - decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> weird \uff5e edge \uff5e case<|endoftext|>", - }, - SPIECE_UNDERSCORE: { - text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, - tokens: ["\u0120\u00e2\u0138", "\u0123", "This", "\u0120\u00e2\u0138", "\u0123", "is", "\u0120\u00e2\u0138", "\u0123", "a", "\u0120\u00e2\u0138", "\u0123", "test", "\u0120\u00e2\u0138", "\u0123", "."], - ids: [50258, 50288, 50359, 50363, 29405, 223, 5723, 29405, 223, 271, 29405, 223, 64, 29405, 223, 31636, 29405, 223, 13, 50257], - decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> \u2581This \u2581is \u2581a \u2581test \u2581.<|endoftext|>", - }, - SPECIAL_TOKENS: { - text: WHISPER_TEST_STRINGS.SPECIAL_TOKENS, - tokens: ["\u0120", "\u0120", "\u0120", "<|startoftranscript|>", "\u0120", "<|en|>", "\u0120", "\u0120", "\u0120"], - ids: [50258, 50288, 50359, 50363, 220, 220, 220, 50258, 220, 50259, 220, 220, 220, 50257], - decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> <|startoftranscript|> <|en|> <|endoftext|>", + // https://github.com/huggingface/transformers/issues/33371 + // tokens: [" <|startoftranscript|> ", "<|en|> "], + tokens: ["<|startoftranscript|>", "<|en|>"], + ids: [50257, 50362, 50257, 50258, 50256], + decoded: "<|startoftranscript|><|notimestamps|><|startoftranscript|><|en|><|endoftext|>", + }, + }, + "Xenova/nb-whisper-tiny-beta": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["\u0120How", "\u0120are", "\u0120you", "\u0120doing", "?"], + ids: [50258, 50288, 50359, 50363, 1012, 366, 291, 884, 30, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> How are you doing?<|endoftext|>", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["\u0120You", "\u0120should", "'ve", "\u0120done", "\u0120", "this"], + ids: [50258, 50288, 50359, 50363, 509, 820, 600, 1096, 220, 11176, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> You should've done this<|endoftext|>", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["\u01200", "12", "3", "45", "67", "89", "\u01200", "\u01201", "\u01202", "\u01203", "\u01204", "\u01205", "\u01206", "\u01207", "\u01208", "\u01209", "\u012010", "\u0120100", "\u01201000"], + ids: [50258, 50288, 50359, 50363, 1958, 4762, 18, 8465, 22452, 21115, 1958, 502, 568, 805, 1017, 1025, 1386, 1614, 1649, 1722, 1266, 2319, 9714, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> 0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000<|endoftext|>", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["\u0120The", "\u0120company", "\u0120was", "\u0120founded", "\u0120in", "\u01202016", "."], + ids: [50258, 50288, 50359, 50363, 440, 2237, 390, 13234, 294, 6549, 13, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> The company was founded in 2016.<|endoftext|>", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["\u0120A", "\u010a", "'ll", "\u0120!!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'t", "."], + ids: [50258, 50288, 50359, 50363, 316, 198, 603, 15138, 1353, 8569, 67, 15025, 67, 295, 11, 393, 380, 13, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> A\n'll!!to?'d''d of, can't.<|endoftext|>", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["\u0120def", "\u0120main", "(", "):", "\u010a", "\u0109", "pass"], + ids: [50258, 50288, 50359, 50363, 1060, 2135, 7, 4507, 198, 197, 9216, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> def main():\n\tpass<|endoftext|>", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["\u0120let", "\u0120a", "\u0120=", "\u0120ob", "j", ".", "to", "St", "ring", "(", ");", "\u010a", "to", "St", "ring", "(", ");"], + ids: [50258, 50288, 50359, 50363, 718, 257, 6585, 1111, 73, 13, 1353, 4520, 2937, 7, 34446, 198, 1353, 4520, 2937, 7, 34446, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> let a = obj.toString();\ntoString();<|endoftext|>", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["\u0120This", "\u010a", "\u010a", "is", "\u010a", "a", "\u010a", "test", "."], + ids: [50258, 50288, 50359, 50363, 639, 198, 198, 271, 198, 64, 198, 31636, 13, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> This\n\nis\na\ntest.<|endoftext|>", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["\u0120UN", "w", "ant", "\u00c3\u00a9d", ",", "running"], + ids: [50258, 50288, 50359, 50363, 8229, 86, 394, 7811, 11, 45482, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> UNwant\u00e9d,running<|endoftext|>", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["\u01201", "\u0100", "2", "\u00ef\u00bf\u00bd", "3"], + ids: [50258, 50288, 50359, 50363, 502, 188, 17, 5342, 18, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> 1\u00002\ufffd3<|endoftext|>", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["\u0120Hello", "\u0120World"], + ids: [50258, 50288, 50359, 50363, 2425, 3937, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> Hello World<|endoftext|>", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["\u0120hello", "\u0120world"], + ids: [50258, 50288, 50359, 50363, 7751, 1002, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> hello world<|endoftext|>", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u0120", "\u00e7\u0136\u0141\u00e6\u00b4\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"], + ids: [50258, 50288, 50359, 50363, 220, 49958, 1546, 6303, 8897, 249, 1541, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> \u751f\u6d3b\u7684\u771f\u8c1b\u662f<|endoftext|>", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u0120", "\u0120", "\u0120leading", "\u0120space"], + ids: [50258, 50288, 50359, 50363, 220, 220, 5775, 1901, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> leading space<|endoftext|>", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["\u0120", "tra", "iling", "\u0120space", "\u0120", "\u0120", "\u0120"], + ids: [50258, 50288, 50359, 50363, 220, 17227, 4883, 1901, 220, 220, 220, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> trailing space <|endoftext|>", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["\u0120Hi", "\u0120", "\u0120Hello"], + ids: [50258, 50288, 50359, 50363, 2421, 220, 2425, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> Hi Hello<|endoftext|>", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["\u0120", "test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2", "\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2", "\u0124", "\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120", "test"], + ids: [50258, 50288, 50359, 50363, 220, 31636, 1848, 16, 497, 17, 3536, 18, 17450, 19, 14378, 20, 1815, 98, 21, 672, 224, 96, 22, 672, 224, 117, 23, 672, 224, 109, 24, 220, 31636, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test<|endoftext|>", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["\u0120I", "\u0120bought", "\u0120an", "\u0120apple", "\u0120for", "\u0120$", "1", ".", "00", "\u0120at", "\u0120", "the", "\u0120store", "."], + ids: [50258, 50288, 50359, 50363, 286, 4243, 364, 10606, 337, 1848, 16, 13, 628, 412, 220, 3322, 3531, 13, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> I bought an apple for $1.00 at the store.<|endoftext|>", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["\u0120you", "\u00e2\u0122\u00a6", "\u0120", "\u0120"], + ids: [50258, 50288, 50359, 50363, 291, 1260, 220, 220, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> you\u2026 <|endoftext|>", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["\u0120you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142"], + ids: [50258, 50288, 50359, 50363, 291, 1260, 126, 254, 126, 254, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> you\u2026\u00a0\u00a0<|endoftext|>", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["\u0120you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142"], + ids: [50258, 50288, 50359, 50363, 291, 1260, 126, 254, 126, 254, 5616, 1260, 126, 254, 126, 254, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0<|endoftext|>", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["\u0120weird", "\u0120\u00ef", "\u00bd", "\u0140", "\u0120edge", "\u0120\u00ef", "\u00bd", "\u0140", "\u0120case"], + ids: [50258, 50288, 50359, 50363, 3657, 25072, 121, 252, 4691, 25072, 121, 252, 1389, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> weird \uff5e edge \uff5e case<|endoftext|>", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u0120\u00e2\u0138", "\u0123", "This", "\u0120\u00e2\u0138", "\u0123", "is", "\u0120\u00e2\u0138", "\u0123", "a", "\u0120\u00e2\u0138", "\u0123", "test", "\u0120\u00e2\u0138", "\u0123", "."], + ids: [50258, 50288, 50359, 50363, 29405, 223, 5723, 29405, 223, 271, 29405, 223, 64, 29405, 223, 31636, 29405, 223, 13, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> \u2581This \u2581is \u2581a \u2581test \u2581.<|endoftext|>", + }, + SPECIAL_TOKENS: { + text: WHISPER_TEST_STRINGS.SPECIAL_TOKENS, + tokens: ["\u0120", "\u0120", "\u0120", "<|startoftranscript|>", "\u0120", "<|en|>", "\u0120", "\u0120", "\u0120"], + ids: [50258, 50288, 50359, 50363, 220, 220, 220, 50258, 220, 50259, 220, 220, 220, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> <|startoftranscript|> <|en|> <|endoftext|>", + }, }, - }, }; const MAX_EXECUTION_TIME = 10_000; export const CUSTOM_TESTS = () => { - describe("Decode ASR", () => { - it( - "should decode ASR outputs", - async () => { - const tokenizer = await WhisperTokenizer.from_pretrained("onnx-community/whisper-tiny.en_timestamped"); + describe("Decode ASR", () => { + it( + "should decode ASR outputs", + async () => { + const tokenizer = await WhisperTokenizer.from_pretrained("onnx-community/whisper-tiny.en_timestamped"); - const model_outputs = [ - { - stride: [30, 0, 5], - tokens: [50257n, 50362n, 8410n, 7283n, 0n, 2329n, 8410n, 7283n, 0n, 2094n, 470n, 1309n, 534n, 10625n, 307n, 10625n, 13n, 34668n, 345n, 531n, 9439n, 11n, 523n, 655n, 8410n, 7283n, 0n, 39134n, 16592n, 10625n, 0n, 9440n, 36n, 26751n, 0n, 25848n, 8410n, 7283n, 0n, 2773n, 661n, 4320n, 1943n, 981n, 345n, 821n, 8066n, 7765n, 510n, 290n, 670n, 1327n, 379n, 340n, 13n, 10528n, 318n, 5340n, 0n, 50256n], - token_timestamps: [0, 0, 0, 3.78, 4.22, 5.34, 6.04, 6.56, 7, 7.92, 8.58, 8.58, 8.88, 9.14, 9.54, 9.94, 10.58, 11.38, 11.88, 12.42, 12.62, 13, 13.36, 13.64, 14.26, 14.76, 15.12, 15.4, 15.74, 16.12, 16.66, 17.14, 17.24, 17.24, 17.72, 18.38, 18.6, 19.38, 19.92, 22.66, 22.9, 23.24, 23.5, 24.14, 24.56, 24.7, 24.72, 24.94, 25.18, 25.54, 25.72, 26.02, 26.34, 26.44, 26.84, 27.04, 27.16, 27.54, 28.06, 29.92], - }, - { - stride: [30, 5, 5], - tokens: [50257n, 50362n, 2773n, 661n, 4320n, 1943n, 981n, 345n, 821n, 8066n, 7765n, 510n, 290n, 670n, 1327n, 379n, 340n, 13n, 10528n, 318n, 5340n, 13n, 921n, 815n, 651n, 284n, 262n, 966n, 810n, 2687n, 2073n, 561n, 11238n, 290n, 345n, 821n, 407n, 8066n, 2245n, 612n, 13n, 1400n, 11n, 644n, 389n, 345n, 4953n, 329n, 30n, 2141n, 340n, 0n, 2329n, 466n, 340n, 0n, 3363n, 345n, 460n, 0n, 2329n, 466n, 340n, 0n, 50256n], - token_timestamps: [0, 0, 0, 2.92, 3.24, 3.48, 4.14, 4.56, 4.7, 4.74, 4.92, 5.18, 5.54, 5.72, 6.04, 6.34, 6.46, 6.84, 7.04, 7.16, 7.54, 8.12, 10.16, 10.7, 10.9, 11.12, 11.24, 11.48, 11.84, 12.44, 12.82, 13.2, 13.46, 13.72, 14.06, 14.28, 14.34, 14.56, 14.8, 15.16, 15.9, 16.42, 16.82, 16.86, 17.02, 17.1, 17.22, 17.56, 18.06, 19.28, 19.62, 20.26, 21.96, 22.64, 24.28, 24.76, 25.18, 25.56, 25.78, 26.28, 27.12, 27.54, 27.82, 28.22, 29.48], - }, - { - stride: [23.7728125, 5, 0], - tokens: [50257n, 50362n, 2329n, 466n, 340n, 0n, 3363n, 345n, 460n, 0n, 2329n, 466n, 340n, 0n, 1002n, 345n, 821n, 10032n, 286n, 3599n, 625n, 11n, 2245n, 3501n, 510n, 13n, 50256n], - token_timestamps: [0, 0, 0, 2.44, 4.3, 5.04, 5.06, 5.56, 5.8, 6.32, 7.12, 7.56, 7.8, 8.9, 10.92, 12.96, 13.28, 13.28, 13.44, 13.72, 13.96, 14.84, 15.5, 16.06, 16.86, 17.88, 20.92], - }, - ]; + const model_outputs = [ + { + stride: [30, 0, 5], + tokens: [50257n, 50362n, 8410n, 7283n, 0n, 2329n, 8410n, 7283n, 0n, 2094n, 470n, 1309n, 534n, 10625n, 307n, 10625n, 13n, 34668n, 345n, 531n, 9439n, 11n, 523n, 655n, 8410n, 7283n, 0n, 39134n, 16592n, 10625n, 0n, 9440n, 36n, 26751n, 0n, 25848n, 8410n, 7283n, 0n, 2773n, 661n, 4320n, 1943n, 981n, 345n, 821n, 8066n, 7765n, 510n, 290n, 670n, 1327n, 379n, 340n, 13n, 10528n, 318n, 5340n, 0n, 50256n], + token_timestamps: [0, 0, 0, 3.78, 4.22, 5.34, 6.04, 6.56, 7, 7.92, 8.58, 8.58, 8.88, 9.14, 9.54, 9.94, 10.58, 11.38, 11.88, 12.42, 12.62, 13, 13.36, 13.64, 14.26, 14.76, 15.12, 15.4, 15.74, 16.12, 16.66, 17.14, 17.24, 17.24, 17.72, 18.38, 18.6, 19.38, 19.92, 22.66, 22.9, 23.24, 23.5, 24.14, 24.56, 24.7, 24.72, 24.94, 25.18, 25.54, 25.72, 26.02, 26.34, 26.44, 26.84, 27.04, 27.16, 27.54, 28.06, 29.92], + }, + { + stride: [30, 5, 5], + tokens: [50257n, 50362n, 2773n, 661n, 4320n, 1943n, 981n, 345n, 821n, 8066n, 7765n, 510n, 290n, 670n, 1327n, 379n, 340n, 13n, 10528n, 318n, 5340n, 13n, 921n, 815n, 651n, 284n, 262n, 966n, 810n, 2687n, 2073n, 561n, 11238n, 290n, 345n, 821n, 407n, 8066n, 2245n, 612n, 13n, 1400n, 11n, 644n, 389n, 345n, 4953n, 329n, 30n, 2141n, 340n, 0n, 2329n, 466n, 340n, 0n, 3363n, 345n, 460n, 0n, 2329n, 466n, 340n, 0n, 50256n], + token_timestamps: [0, 0, 0, 2.92, 3.24, 3.48, 4.14, 4.56, 4.7, 4.74, 4.92, 5.18, 5.54, 5.72, 6.04, 6.34, 6.46, 6.84, 7.04, 7.16, 7.54, 8.12, 10.16, 10.7, 10.9, 11.12, 11.24, 11.48, 11.84, 12.44, 12.82, 13.2, 13.46, 13.72, 14.06, 14.28, 14.34, 14.56, 14.8, 15.16, 15.9, 16.42, 16.82, 16.86, 17.02, 17.1, 17.22, 17.56, 18.06, 19.28, 19.62, 20.26, 21.96, 22.64, 24.28, 24.76, 25.18, 25.56, 25.78, 26.28, 27.12, 27.54, 27.82, 28.22, 29.48], + }, + { + stride: [23.7728125, 5, 0], + tokens: [50257n, 50362n, 2329n, 466n, 340n, 0n, 3363n, 345n, 460n, 0n, 2329n, 466n, 340n, 0n, 1002n, 345n, 821n, 10032n, 286n, 3599n, 625n, 11n, 2245n, 3501n, 510n, 13n, 50256n], + token_timestamps: [0, 0, 0, 2.44, 4.3, 5.04, 5.06, 5.56, 5.8, 6.32, 7.12, 7.56, 7.8, 8.9, 10.92, 12.96, 13.28, 13.28, 13.44, 13.72, 13.96, 14.84, 15.5, 16.06, 16.86, 17.88, 20.92], + }, + ]; - const target = [ - " DO IT! Just DO IT! Don't let your dreams be dreams. Yesterday you said tomorrow, so just DO IT! MAKE YOUR dreams! COME TRUE! JUST DO IT! Some people dream success while you're gonna wake up and work hard at it. Nothing is impossible. You should get to the point where anyone else would quit and you're not gonna stop there. No, what are you waiting for? Do it! Just do it! Yes you can! Just do it! If you're tired of starting over, stop giving up.", - { - chunks: [ - { text: " DO", timestamp: [0.0, 3.78] }, - { text: " IT!", timestamp: [3.78, 4.24 /* 5.34 */] }, - { text: " Just", timestamp: [5.34, 6.04] }, - { text: " DO", timestamp: [6.04, 6.56] }, - { text: " IT!", timestamp: [6.56, 7.02 /* 7.92 */] }, - { text: " Don't", timestamp: [7.92, 8.58] }, - { text: " let", timestamp: [8.58, 8.88] }, - { text: " your", timestamp: [8.88, 9.14] }, - { text: " dreams", timestamp: [9.14, 9.54] }, - { text: " be", timestamp: [9.54, 9.94] }, - { text: " dreams.", timestamp: [9.94, 10.6 /* 11.38 */] }, - { text: " Yesterday", timestamp: [11.38, 11.88] }, - { text: " you", timestamp: [11.88, 12.42] }, - { text: " said", timestamp: [12.42, 12.62] }, - { text: " tomorrow,", timestamp: [12.62, 13.02 /* 13.36 */] }, - { text: " so", timestamp: [13.36, 13.64] }, - { text: " just", timestamp: [13.64, 14.26] }, - { text: " DO", timestamp: [14.26, 14.76] }, - { text: " IT!", timestamp: [14.76, 15.14 /* 15.4 */] }, - { text: " MAKE", timestamp: [15.4, 15.74] }, - { text: " YOUR", timestamp: [15.74, 16.12] }, - { text: " dreams!", timestamp: [16.12, 16.68 /* 17.14 */] }, - { text: " COME", timestamp: [17.14, 17.24] }, - { text: " TRUE!", timestamp: [17.24, 17.74 /* 18.38 */] }, - { text: " JUST", timestamp: [18.38, 18.6] }, - { text: " DO", timestamp: [18.6, 19.38] }, - { text: " IT!", timestamp: [19.38, 19.94 /* 22.66 */] }, - { text: " Some", timestamp: [22.66, 22.9] }, - { text: " people", timestamp: [22.9, 23.24] }, - { text: " dream", timestamp: [23.24, 23.5] }, - { text: " success", timestamp: [23.5, 24.14] }, - { text: " while", timestamp: [24.14, 24.56] }, - { text: " you're", timestamp: [24.56, 24.72] }, - { text: " gonna", timestamp: [24.72, 24.94] }, - { text: " wake", timestamp: [24.94, 25.18] }, - { text: " up", timestamp: [25.18, 25.54] }, - { text: " and", timestamp: [25.54, 25.72] }, - { text: " work", timestamp: [25.72, 26.04] }, - { text: " hard", timestamp: [26.04, 26.34] }, - { text: " at", timestamp: [26.34, 26.46] }, - { text: " it.", timestamp: [26.46, 26.86 /* 27.04 */] }, - { text: " Nothing", timestamp: [27.04, 27.16] }, - { text: " is", timestamp: [27.16, 27.54] }, - { text: " impossible.", timestamp: [27.54, 28.14 /* 30.16 */] }, - { text: " You", timestamp: [30.16, 30.7] }, - { text: " should", timestamp: [30.7, 30.9] }, - { text: " get", timestamp: [30.9, 31.12] }, - { text: " to", timestamp: [31.12, 31.24] }, - { text: " the", timestamp: [31.24, 31.48] }, - { text: " point", timestamp: [31.48, 31.84] }, - { text: " where", timestamp: [31.84, 32.44] }, - { text: " anyone", timestamp: [32.44, 32.82] }, - { text: " else", timestamp: [32.82, 33.2] }, - { text: " would", timestamp: [33.2, 33.46] }, - { text: " quit", timestamp: [33.46, 33.72] }, - { text: " and", timestamp: [33.72, 34.06] }, - { text: " you're", timestamp: [34.06, 34.34] }, - { text: " not", timestamp: [34.34, 34.56] }, - { text: " gonna", timestamp: [34.56, 34.8] }, - { text: " stop", timestamp: [34.8, 35.16] }, - { text: " there.", timestamp: [35.16, 35.92 /* 36.42 */] }, - { text: " No,", timestamp: [36.42, 36.84 /* 36.86 */] }, - { text: " what", timestamp: [36.86, 37.02] }, - { text: " are", timestamp: [37.02, 37.1] }, - { text: " you", timestamp: [37.1, 37.22] }, - { text: " waiting", timestamp: [37.22, 37.56] }, - { text: " for?", timestamp: [37.56, 38.08 /* 39.28 */] }, - { text: " Do", timestamp: [39.28, 39.62] }, - { text: " it!", timestamp: [39.62, 40.28 /* 41.96 */] }, - { text: " Just", timestamp: [41.96, 42.64] }, - { text: " do", timestamp: [42.64, 44.28] }, - { text: " it!", timestamp: [44.28, 44.78 /* 45.18 */] }, - { text: " Yes", timestamp: [45.18, 45.56] }, - { text: " you", timestamp: [45.56, 45.78] }, - { text: " can!", timestamp: [45.8, 46.34 /* 47.12 */] }, - { text: " Just", timestamp: [47.12, 47.56] }, - { text: " do", timestamp: [47.56, 47.8] }, - { text: " it!", timestamp: [47.8, 48.92 /* 50.92 */] }, - { text: " If", timestamp: [50.92, 52.96] }, - { text: " you're", timestamp: [52.96, 53.28] }, - { text: " tired", timestamp: [53.28, 53.44] }, - { text: " of", timestamp: [53.44, 53.72] }, - { text: " starting", timestamp: [53.72, 53.96] }, - { text: " over,", timestamp: [53.96, 54.86 /* 55.5 */] }, - { text: " stop", timestamp: [55.5, 56.06] }, - { text: " giving", timestamp: [56.06, 56.86] }, - { text: " up.", timestamp: [56.86, 57.9 /* 60.92 */] }, - ], - }, - ]; + const target = [ + " DO IT! Just DO IT! Don't let your dreams be dreams. Yesterday you said tomorrow, so just DO IT! MAKE YOUR dreams! COME TRUE! JUST DO IT! Some people dream success while you're gonna wake up and work hard at it. Nothing is impossible. You should get to the point where anyone else would quit and you're not gonna stop there. No, what are you waiting for? Do it! Just do it! Yes you can! Just do it! If you're tired of starting over, stop giving up.", + { + chunks: [ + { text: " DO", timestamp: [0.0, 3.78] }, + { text: " IT!", timestamp: [3.78, 4.24 /* 5.34 */] }, + { text: " Just", timestamp: [5.34, 6.04] }, + { text: " DO", timestamp: [6.04, 6.56] }, + { text: " IT!", timestamp: [6.56, 7.02 /* 7.92 */] }, + { text: " Don't", timestamp: [7.92, 8.58] }, + { text: " let", timestamp: [8.58, 8.88] }, + { text: " your", timestamp: [8.88, 9.14] }, + { text: " dreams", timestamp: [9.14, 9.54] }, + { text: " be", timestamp: [9.54, 9.94] }, + { text: " dreams.", timestamp: [9.94, 10.6 /* 11.38 */] }, + { text: " Yesterday", timestamp: [11.38, 11.88] }, + { text: " you", timestamp: [11.88, 12.42] }, + { text: " said", timestamp: [12.42, 12.62] }, + { text: " tomorrow,", timestamp: [12.62, 13.02 /* 13.36 */] }, + { text: " so", timestamp: [13.36, 13.64] }, + { text: " just", timestamp: [13.64, 14.26] }, + { text: " DO", timestamp: [14.26, 14.76] }, + { text: " IT!", timestamp: [14.76, 15.14 /* 15.4 */] }, + { text: " MAKE", timestamp: [15.4, 15.74] }, + { text: " YOUR", timestamp: [15.74, 16.12] }, + { text: " dreams!", timestamp: [16.12, 16.68 /* 17.14 */] }, + { text: " COME", timestamp: [17.14, 17.24] }, + { text: " TRUE!", timestamp: [17.24, 17.74 /* 18.38 */] }, + { text: " JUST", timestamp: [18.38, 18.6] }, + { text: " DO", timestamp: [18.6, 19.38] }, + { text: " IT!", timestamp: [19.38, 19.94 /* 22.66 */] }, + { text: " Some", timestamp: [22.66, 22.9] }, + { text: " people", timestamp: [22.9, 23.24] }, + { text: " dream", timestamp: [23.24, 23.5] }, + { text: " success", timestamp: [23.5, 24.14] }, + { text: " while", timestamp: [24.14, 24.56] }, + { text: " you're", timestamp: [24.56, 24.72] }, + { text: " gonna", timestamp: [24.72, 24.94] }, + { text: " wake", timestamp: [24.94, 25.18] }, + { text: " up", timestamp: [25.18, 25.54] }, + { text: " and", timestamp: [25.54, 25.72] }, + { text: " work", timestamp: [25.72, 26.04] }, + { text: " hard", timestamp: [26.04, 26.34] }, + { text: " at", timestamp: [26.34, 26.46] }, + { text: " it.", timestamp: [26.46, 26.86 /* 27.04 */] }, + { text: " Nothing", timestamp: [27.04, 27.16] }, + { text: " is", timestamp: [27.16, 27.54] }, + { text: " impossible.", timestamp: [27.54, 28.14 /* 30.16 */] }, + { text: " You", timestamp: [30.16, 30.7] }, + { text: " should", timestamp: [30.7, 30.9] }, + { text: " get", timestamp: [30.9, 31.12] }, + { text: " to", timestamp: [31.12, 31.24] }, + { text: " the", timestamp: [31.24, 31.48] }, + { text: " point", timestamp: [31.48, 31.84] }, + { text: " where", timestamp: [31.84, 32.44] }, + { text: " anyone", timestamp: [32.44, 32.82] }, + { text: " else", timestamp: [32.82, 33.2] }, + { text: " would", timestamp: [33.2, 33.46] }, + { text: " quit", timestamp: [33.46, 33.72] }, + { text: " and", timestamp: [33.72, 34.06] }, + { text: " you're", timestamp: [34.06, 34.34] }, + { text: " not", timestamp: [34.34, 34.56] }, + { text: " gonna", timestamp: [34.56, 34.8] }, + { text: " stop", timestamp: [34.8, 35.16] }, + { text: " there.", timestamp: [35.16, 35.92 /* 36.42 */] }, + { text: " No,", timestamp: [36.42, 36.84 /* 36.86 */] }, + { text: " what", timestamp: [36.86, 37.02] }, + { text: " are", timestamp: [37.02, 37.1] }, + { text: " you", timestamp: [37.1, 37.22] }, + { text: " waiting", timestamp: [37.22, 37.56] }, + { text: " for?", timestamp: [37.56, 38.08 /* 39.28 */] }, + { text: " Do", timestamp: [39.28, 39.62] }, + { text: " it!", timestamp: [39.62, 40.28 /* 41.96 */] }, + { text: " Just", timestamp: [41.96, 42.64] }, + { text: " do", timestamp: [42.64, 44.28] }, + { text: " it!", timestamp: [44.28, 44.78 /* 45.18 */] }, + { text: " Yes", timestamp: [45.18, 45.56] }, + { text: " you", timestamp: [45.56, 45.78] }, + { text: " can!", timestamp: [45.8, 46.34 /* 47.12 */] }, + { text: " Just", timestamp: [47.12, 47.56] }, + { text: " do", timestamp: [47.56, 47.8] }, + { text: " it!", timestamp: [47.8, 48.92 /* 50.92 */] }, + { text: " If", timestamp: [50.92, 52.96] }, + { text: " you're", timestamp: [52.96, 53.28] }, + { text: " tired", timestamp: [53.28, 53.44] }, + { text: " of", timestamp: [53.44, 53.72] }, + { text: " starting", timestamp: [53.72, 53.96] }, + { text: " over,", timestamp: [53.96, 54.86 /* 55.5 */] }, + { text: " stop", timestamp: [55.5, 56.06] }, + { text: " giving", timestamp: [56.06, 56.86] }, + { text: " up.", timestamp: [56.86, 57.9 /* 60.92 */] }, + ], + }, + ]; - compare( - tokenizer._decode_asr(model_outputs, { - return_timestamps: "word", - time_precision: 0.02, - force_full_sequences: false, - }), - target, - 1e-2, + compare( + tokenizer._decode_asr(model_outputs, { + return_timestamps: "word", + time_precision: 0.02, + force_full_sequences: false, + }), + target, + 1e-2, + ); + }, + MAX_EXECUTION_TIME, ); - }, - MAX_EXECUTION_TIME, - ); - it( - "should handle overlapping edge case", - async () => { - const tokenizer = await WhisperTokenizer.from_pretrained("onnx-community/whisper-tiny.en_timestamped"); + it( + "should handle overlapping edge case", + async () => { + const tokenizer = await WhisperTokenizer.from_pretrained("onnx-community/whisper-tiny.en_timestamped"); - const model_outputs = [ - { - stride: [30, 0, 5], - tokens: [50257n, 50362n, 8410n, 7283n, 0n, 2329n, 8410n, 7283n, 0n, 2094n, 470n, 1309n, 534n, 10625n, 307n, 10625n, 13n, 34668n, 11n, 345n, 531n, 9439n, 11n, 523n, 655n, 8410n, 7283n, 0n, 39134n, 16592n, 10560n, 3955n, 50n, 0n, 7102n, 5446n, 46n, 0n, 25848n, 8410n, 7283n, 0n, 2773n, 661n, 4320n, 1943n, 981n, 345n, 821n, 8066n, 7765n, 510n, 290n, 670n, 1327n, 379n, 340n, 13n, 10528n, 318n, 5340n, 13n, 50256n], - token_timestamps: [0, 0, 0, 3.78, 4.22, 5.26, 6.04, 6.54, 7, 7.94, 8.58, 8.58, 8.88, 9.16, 9.54, 9.94, 10.6, 11.38, 11.88, 12.38, 12.44, 12.62, 13, 13.36, 13.64, 14.24, 14.74, 15.12, 15.4, 15.74, 16.1, 16.54, 16.54, 16.78, 17.08, 17.2, 17.36, 17.56, 18.08, 18.58, 19.38, 19.88, 22.54, 22.9, 23.24, 23.5, 24.14, 24.56, 24.7, 24.94, 24.94, 25.18, 25.54, 25.72, 26.04, 26.34, 26.46, 26.84, 27.04, 27.14, 27.54, 28.06, 29.92], - }, - { - stride: [30, 5, 5], - tokens: [50257n, 50362n, 2773n, 661n, 4320n, 1943n, 981n, 345n, 821n, 8066n, 7765n, 510n, 290n, 670n, 1327n, 379n, 340n, 13n, 10528n, 318n, 5340n, 13n, 921n, 815n, 651n, 284n, 262n, 966n, 810n, 2687n, 2073n, 561n, 11238n, 290n, 345n, 821n, 407n, 8066n, 2245n, 612n, 13n, 1400n, 11n, 644n, 389n, 345n, 4953n, 329n, 30n, 2141n, 340n, 0n, 2329n, 466n, 340n, 0n, 3363n, 11n, 345n, 460n, 0n, 2329n, 466n, 340n, 0n, 50256n], - token_timestamps: [0, 0, 0, 2.92, 3.24, 3.5, 4.14, 4.56, 4.7, 4.74, 4.92, 5.18, 5.54, 5.74, 6.04, 6.34, 6.46, 6.84, 7.04, 7.18, 7.56, 8.12, 9.68, 10.7, 10.88, 11.1, 11.24, 11.48, 11.82, 12.46, 12.82, 13.2, 13.46, 13.72, 14.08, 14.28, 14.34, 14.56, 14.82, 15.16, 15.72, 16.42, 16.82, 16.86, 17, 17.1, 17.2, 17.56, 18.06, 19.28, 19.6, 20.28, 21.96, 22.64, 24.28, 24.76, 25.18, 25.56, 25.56, 25.84, 26.36, 27.12, 27.54, 27.82, 28.16, 29.48], - }, - { - stride: [23.7728125, 5, 0], - tokens: [50257n, 50362n, 2329n, 466n, 340n, 0n, 3363n, 345n, 460n, 0n, 2329n, 466n, 340n, 0n, 1002n, 534n, 15867n, 318n, 3599n, 625n, 11n, 2245n, 3501n, 510n, 13n, 50256n], - token_timestamps: [0, 0, 0, 2.44, 4.3, 5.04, 5.06, 5.56, 5.8, 6.32, 7.12, 7.56, 7.8, 8.72, 10.04, 12.96, 13.3, 13.44, 13.72, 13.98, 14.86, 15.5, 16, 16.88, 17.76, 20.9], - }, - ]; + const model_outputs = [ + { + stride: [30, 0, 5], + tokens: [50257n, 50362n, 8410n, 7283n, 0n, 2329n, 8410n, 7283n, 0n, 2094n, 470n, 1309n, 534n, 10625n, 307n, 10625n, 13n, 34668n, 11n, 345n, 531n, 9439n, 11n, 523n, 655n, 8410n, 7283n, 0n, 39134n, 16592n, 10560n, 3955n, 50n, 0n, 7102n, 5446n, 46n, 0n, 25848n, 8410n, 7283n, 0n, 2773n, 661n, 4320n, 1943n, 981n, 345n, 821n, 8066n, 7765n, 510n, 290n, 670n, 1327n, 379n, 340n, 13n, 10528n, 318n, 5340n, 13n, 50256n], + token_timestamps: [0, 0, 0, 3.78, 4.22, 5.26, 6.04, 6.54, 7, 7.94, 8.58, 8.58, 8.88, 9.16, 9.54, 9.94, 10.6, 11.38, 11.88, 12.38, 12.44, 12.62, 13, 13.36, 13.64, 14.24, 14.74, 15.12, 15.4, 15.74, 16.1, 16.54, 16.54, 16.78, 17.08, 17.2, 17.36, 17.56, 18.08, 18.58, 19.38, 19.88, 22.54, 22.9, 23.24, 23.5, 24.14, 24.56, 24.7, 24.94, 24.94, 25.18, 25.54, 25.72, 26.04, 26.34, 26.46, 26.84, 27.04, 27.14, 27.54, 28.06, 29.92], + }, + { + stride: [30, 5, 5], + tokens: [50257n, 50362n, 2773n, 661n, 4320n, 1943n, 981n, 345n, 821n, 8066n, 7765n, 510n, 290n, 670n, 1327n, 379n, 340n, 13n, 10528n, 318n, 5340n, 13n, 921n, 815n, 651n, 284n, 262n, 966n, 810n, 2687n, 2073n, 561n, 11238n, 290n, 345n, 821n, 407n, 8066n, 2245n, 612n, 13n, 1400n, 11n, 644n, 389n, 345n, 4953n, 329n, 30n, 2141n, 340n, 0n, 2329n, 466n, 340n, 0n, 3363n, 11n, 345n, 460n, 0n, 2329n, 466n, 340n, 0n, 50256n], + token_timestamps: [0, 0, 0, 2.92, 3.24, 3.5, 4.14, 4.56, 4.7, 4.74, 4.92, 5.18, 5.54, 5.74, 6.04, 6.34, 6.46, 6.84, 7.04, 7.18, 7.56, 8.12, 9.68, 10.7, 10.88, 11.1, 11.24, 11.48, 11.82, 12.46, 12.82, 13.2, 13.46, 13.72, 14.08, 14.28, 14.34, 14.56, 14.82, 15.16, 15.72, 16.42, 16.82, 16.86, 17, 17.1, 17.2, 17.56, 18.06, 19.28, 19.6, 20.28, 21.96, 22.64, 24.28, 24.76, 25.18, 25.56, 25.56, 25.84, 26.36, 27.12, 27.54, 27.82, 28.16, 29.48], + }, + { + stride: [23.7728125, 5, 0], + tokens: [50257n, 50362n, 2329n, 466n, 340n, 0n, 3363n, 345n, 460n, 0n, 2329n, 466n, 340n, 0n, 1002n, 534n, 15867n, 318n, 3599n, 625n, 11n, 2245n, 3501n, 510n, 13n, 50256n], + token_timestamps: [0, 0, 0, 2.44, 4.3, 5.04, 5.06, 5.56, 5.8, 6.32, 7.12, 7.56, 7.8, 8.72, 10.04, 12.96, 13.3, 13.44, 13.72, 13.98, 14.86, 15.5, 16, 16.88, 17.76, 20.9], + }, + ]; - const target = [ - " DO IT! Just DO IT! Don't let your dreams be dreams. Yesterday, you said tomorrow, so just DO IT! MAKE YOUR DRIMS! CONTRO! JUST DO IT! Some people dream success while you're gonna wake up and work hard at it. Nothing is impossible. You should get to the point where anyone else would quit and you're not gonna stop there. No, what are you waiting for? Do it! Just do it! Yes, you can! Just do it! If your tire is starting over, stop giving up.", - { - chunks: [ - { text: " DO", timestamp: [0, 3.78] }, - { text: " IT!", timestamp: [3.78, 4.24] }, - { text: " Just", timestamp: [5.26, 6.04] }, - { text: " DO", timestamp: [6.04, 6.54] }, - { text: " IT!", timestamp: [6.54, 7.02] }, - { text: " Don't", timestamp: [7.94, 8.58] }, - { text: " let", timestamp: [8.58, 8.88] }, - { text: " your", timestamp: [8.88, 9.16] }, - { text: " dreams", timestamp: [9.16, 9.54] }, - { text: " be", timestamp: [9.54, 9.94] }, - { text: " dreams.", timestamp: [9.94, 10.62] }, - { text: " Yesterday,", timestamp: [11.38, 11.9] }, - { text: " you", timestamp: [12.38, 12.44] }, - { text: " said", timestamp: [12.44, 12.62] }, - { text: " tomorrow,", timestamp: [12.62, 13.02] }, - { text: " so", timestamp: [13.36, 13.64] }, - { text: " just", timestamp: [13.64, 14.24] }, - { text: " DO", timestamp: [14.24, 14.74] }, - { text: " IT!", timestamp: [14.74, 15.14] }, - { text: " MAKE", timestamp: [15.4, 15.74] }, - { text: " YOUR", timestamp: [15.74, 16.1] }, - { text: " DRIMS!", timestamp: [16.1, 16.8] }, - { text: " CONTRO!", timestamp: [17.08, 17.58] }, - { text: " JUST", timestamp: [18.08, 18.58] }, - { text: " DO", timestamp: [18.58, 19.38] }, - { text: " IT!", timestamp: [19.38, 19.9] }, - { text: " Some", timestamp: [22.54, 22.9] }, - { text: " people", timestamp: [22.9, 23.24] }, - { text: " dream", timestamp: [23.24, 23.5] }, - { text: " success", timestamp: [23.5, 24.14] }, - { text: " while", timestamp: [24.14, 24.56] }, - { text: " you're", timestamp: [24.56, 24.94] }, - { text: " gonna", timestamp: [24.94, 24.94] }, - { text: " wake", timestamp: [24.94, 25.18] }, - { text: " up", timestamp: [25.18, 25.54] }, - { text: " and", timestamp: [25.54, 25.74] }, - { text: " work", timestamp: [25.74, 26.04] }, - { text: " hard", timestamp: [26.04, 26.34] }, - { text: " at", timestamp: [26.34, 26.46] }, - { text: " it.", timestamp: [26.46, 26.86] }, - { text: " Nothing", timestamp: [27.04, 27.18] }, - { text: " is", timestamp: [27.18, 27.56] }, - { text: " impossible.", timestamp: [27.56, 28.14] }, - { text: " You", timestamp: [29.68, 30.7] }, - { text: " should", timestamp: [30.7, 30.88] }, - { text: " get", timestamp: [30.88, 31.1] }, - { text: " to", timestamp: [31.1, 31.24] }, - { text: " the", timestamp: [31.24, 31.48] }, - { text: " point", timestamp: [31.48, 31.82] }, - { text: " where", timestamp: [31.82, 32.46] }, - { text: " anyone", timestamp: [32.46, 32.82] }, - { text: " else", timestamp: [32.82, 33.2] }, - { text: " would", timestamp: [33.2, 33.46] }, - { text: " quit", timestamp: [33.46, 33.72] }, - { text: " and", timestamp: [33.72, 34.08] }, - { text: " you're", timestamp: [34.08, 34.34] }, - { text: " not", timestamp: [34.34, 34.56] }, - { text: " gonna", timestamp: [34.56, 34.82] }, - { text: " stop", timestamp: [34.82, 35.16] }, - { text: " there.", timestamp: [35.16, 35.74] }, - { text: " No,", timestamp: [36.42, 36.84] }, - { text: " what", timestamp: [36.86, 37] }, - { text: " are", timestamp: [37, 37.1] }, - { text: " you", timestamp: [37.1, 37.2] }, - { text: " waiting", timestamp: [37.2, 37.56] }, - { text: " for?", timestamp: [37.56, 38.08] }, - { text: " Do", timestamp: [39.28, 39.6] }, - { text: " it!", timestamp: [39.6, 40.3] }, - { text: " Just", timestamp: [41.96, 42.64] }, - { text: " do", timestamp: [42.64, 44.28] }, - { text: " it!", timestamp: [44.28, 44.78] }, - { text: " Yes,", timestamp: [45.18, 45.56] }, - { text: " you", timestamp: [45.56, 45.84] }, - { text: " can!", timestamp: [45.8, 46.34] }, - { text: " Just", timestamp: [47.12, 47.56] }, - { text: " do", timestamp: [47.56, 47.8] }, - { text: " it!", timestamp: [47.8, 48.74] }, - { text: " If", timestamp: [50.04, 52.96] }, - { text: " your", timestamp: [52.96, 53.3] }, - { text: " tire", timestamp: [53.3, 53.44] }, - { text: " is", timestamp: [53.44, 53.72] }, - { text: " starting", timestamp: [53.72, 53.98] }, - { text: " over,", timestamp: [53.98, 54.88] }, - { text: " stop", timestamp: [55.5, 56] }, - { text: " giving", timestamp: [56, 56.88] }, - { text: " up.", timestamp: [56.88, 57.78] }, - ], - }, - ]; + const target = [ + " DO IT! Just DO IT! Don't let your dreams be dreams. Yesterday, you said tomorrow, so just DO IT! MAKE YOUR DRIMS! CONTRO! JUST DO IT! Some people dream success while you're gonna wake up and work hard at it. Nothing is impossible. You should get to the point where anyone else would quit and you're not gonna stop there. No, what are you waiting for? Do it! Just do it! Yes, you can! Just do it! If your tire is starting over, stop giving up.", + { + chunks: [ + { text: " DO", timestamp: [0, 3.78] }, + { text: " IT!", timestamp: [3.78, 4.24] }, + { text: " Just", timestamp: [5.26, 6.04] }, + { text: " DO", timestamp: [6.04, 6.54] }, + { text: " IT!", timestamp: [6.54, 7.02] }, + { text: " Don't", timestamp: [7.94, 8.58] }, + { text: " let", timestamp: [8.58, 8.88] }, + { text: " your", timestamp: [8.88, 9.16] }, + { text: " dreams", timestamp: [9.16, 9.54] }, + { text: " be", timestamp: [9.54, 9.94] }, + { text: " dreams.", timestamp: [9.94, 10.62] }, + { text: " Yesterday,", timestamp: [11.38, 11.9] }, + { text: " you", timestamp: [12.38, 12.44] }, + { text: " said", timestamp: [12.44, 12.62] }, + { text: " tomorrow,", timestamp: [12.62, 13.02] }, + { text: " so", timestamp: [13.36, 13.64] }, + { text: " just", timestamp: [13.64, 14.24] }, + { text: " DO", timestamp: [14.24, 14.74] }, + { text: " IT!", timestamp: [14.74, 15.14] }, + { text: " MAKE", timestamp: [15.4, 15.74] }, + { text: " YOUR", timestamp: [15.74, 16.1] }, + { text: " DRIMS!", timestamp: [16.1, 16.8] }, + { text: " CONTRO!", timestamp: [17.08, 17.58] }, + { text: " JUST", timestamp: [18.08, 18.58] }, + { text: " DO", timestamp: [18.58, 19.38] }, + { text: " IT!", timestamp: [19.38, 19.9] }, + { text: " Some", timestamp: [22.54, 22.9] }, + { text: " people", timestamp: [22.9, 23.24] }, + { text: " dream", timestamp: [23.24, 23.5] }, + { text: " success", timestamp: [23.5, 24.14] }, + { text: " while", timestamp: [24.14, 24.56] }, + { text: " you're", timestamp: [24.56, 24.94] }, + { text: " gonna", timestamp: [24.94, 24.94] }, + { text: " wake", timestamp: [24.94, 25.18] }, + { text: " up", timestamp: [25.18, 25.54] }, + { text: " and", timestamp: [25.54, 25.74] }, + { text: " work", timestamp: [25.74, 26.04] }, + { text: " hard", timestamp: [26.04, 26.34] }, + { text: " at", timestamp: [26.34, 26.46] }, + { text: " it.", timestamp: [26.46, 26.86] }, + { text: " Nothing", timestamp: [27.04, 27.18] }, + { text: " is", timestamp: [27.18, 27.56] }, + { text: " impossible.", timestamp: [27.56, 28.14] }, + { text: " You", timestamp: [29.68, 30.7] }, + { text: " should", timestamp: [30.7, 30.88] }, + { text: " get", timestamp: [30.88, 31.1] }, + { text: " to", timestamp: [31.1, 31.24] }, + { text: " the", timestamp: [31.24, 31.48] }, + { text: " point", timestamp: [31.48, 31.82] }, + { text: " where", timestamp: [31.82, 32.46] }, + { text: " anyone", timestamp: [32.46, 32.82] }, + { text: " else", timestamp: [32.82, 33.2] }, + { text: " would", timestamp: [33.2, 33.46] }, + { text: " quit", timestamp: [33.46, 33.72] }, + { text: " and", timestamp: [33.72, 34.08] }, + { text: " you're", timestamp: [34.08, 34.34] }, + { text: " not", timestamp: [34.34, 34.56] }, + { text: " gonna", timestamp: [34.56, 34.82] }, + { text: " stop", timestamp: [34.82, 35.16] }, + { text: " there.", timestamp: [35.16, 35.74] }, + { text: " No,", timestamp: [36.42, 36.84] }, + { text: " what", timestamp: [36.86, 37] }, + { text: " are", timestamp: [37, 37.1] }, + { text: " you", timestamp: [37.1, 37.2] }, + { text: " waiting", timestamp: [37.2, 37.56] }, + { text: " for?", timestamp: [37.56, 38.08] }, + { text: " Do", timestamp: [39.28, 39.6] }, + { text: " it!", timestamp: [39.6, 40.3] }, + { text: " Just", timestamp: [41.96, 42.64] }, + { text: " do", timestamp: [42.64, 44.28] }, + { text: " it!", timestamp: [44.28, 44.78] }, + { text: " Yes,", timestamp: [45.18, 45.56] }, + { text: " you", timestamp: [45.56, 45.84] }, + { text: " can!", timestamp: [45.8, 46.34] }, + { text: " Just", timestamp: [47.12, 47.56] }, + { text: " do", timestamp: [47.56, 47.8] }, + { text: " it!", timestamp: [47.8, 48.74] }, + { text: " If", timestamp: [50.04, 52.96] }, + { text: " your", timestamp: [52.96, 53.3] }, + { text: " tire", timestamp: [53.3, 53.44] }, + { text: " is", timestamp: [53.44, 53.72] }, + { text: " starting", timestamp: [53.72, 53.98] }, + { text: " over,", timestamp: [53.98, 54.88] }, + { text: " stop", timestamp: [55.5, 56] }, + { text: " giving", timestamp: [56, 56.88] }, + { text: " up.", timestamp: [56.88, 57.78] }, + ], + }, + ]; - compare( - tokenizer._decode_asr(model_outputs, { - return_timestamps: "word", - time_precision: 0.02, - force_full_sequences: false, - }), - target, - 1e-2, + compare( + tokenizer._decode_asr(model_outputs, { + return_timestamps: "word", + time_precision: 0.02, + force_full_sequences: false, + }), + target, + 1e-2, + ); + }, + MAX_EXECUTION_TIME, ); - }, - MAX_EXECUTION_TIME, - ); - }); + }); }; diff --git a/tests/models/xlm-roberta/tokenization.js b/tests/models/xlm-roberta/tokenization.js index 67d02ce8d..43198d942 100644 --- a/tests/models/xlm-roberta/tokenization.js +++ b/tests/models/xlm-roberta/tokenization.js @@ -3,330 +3,330 @@ import { BASE_TEST_STRINGS } from "../test_strings.js"; export const TOKENIZER_CLASS = XLMRobertaTokenizer; export const TEST_CONFIG = { - "Xenova/bge-reranker-base": { - SIMPLE: { - text: BASE_TEST_STRINGS.SIMPLE, - tokens: ["\u2581How", "\u2581are", "\u2581you", "\u2581doing", "?"], - ids: [0, 11249, 621, 398, 20594, 32, 2], - decoded: " How are you doing?", + "Xenova/bge-reranker-base": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["\u2581How", "\u2581are", "\u2581you", "\u2581doing", "?"], + ids: [0, 11249, 621, 398, 20594, 32, 2], + decoded: " How are you doing?", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["\u2581You", "\u2581should", "'", "ve", "\u2581done", "\u2581this"], + ids: [0, 2583, 5608, 25, 272, 16940, 903, 2], + decoded: " You should've done this", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["\u258101", "23", "456", "789", "\u25810", "\u25811", "\u25812", "\u25813", "\u25814", "\u25815", "\u25816", "\u25817", "\u25818", "\u25819", "\u258110", "\u2581100", "\u25811000"], + ids: [0, 3413, 3742, 121317, 153781, 757, 106, 116, 138, 201, 190, 305, 361, 382, 483, 209, 805, 4382, 2], + decoded: " 0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["\u2581The", "\u2581company", "\u2581was", "\u2581found", "ed", "\u2581in", "\u25812016."], + ids: [0, 581, 14380, 509, 14037, 297, 23, 6360, 2], + decoded: " The company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["\u2581A", "\u2581'", "ll", "\u2581!!", "to", "?", "'", "d", "''", "d", "\u2581of", ",", "\u2581can", "'", "t", "."], + ids: [0, 62, 242, 1181, 6506, 188, 32, 25, 71, 4765, 71, 111, 4, 831, 25, 18, 5, 2], + decoded: " A 'll!!to?'d''d of, can't.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["\u2581de", "f", "\u2581main", "(", "):", "\u2581pass"], + ids: [0, 8, 420, 5201, 132, 2077, 27875, 2], + decoded: " def main(): pass", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["\u2581let", "\u2581a", "\u2581=", "\u2581ob", "j", ".", "to", "Str", "ing", "(", ");", "\u2581to", "Str", "ing", "(", ");"], + ids: [0, 2633, 10, 2203, 995, 170, 5, 188, 71713, 214, 132, 3142, 47, 71713, 214, 132, 3142, 2], + decoded: " let a = obj.toString(); toString();", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["\u2581This", "\u2581is", "\u2581a", "\u2581test", "."], + ids: [0, 3293, 83, 10, 3034, 5, 2], + decoded: " This is a test.", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["\u2581UN", "wan", "t\u00e9", "d", ",", "run", "ning"], + ids: [0, 8274, 3206, 2312, 71, 4, 16428, 592, 2], + decoded: " UNwant\u00e9d,running", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["\u25811", "\u0000", "2", "\u25813"], + ids: [0, 106, 3, 304, 138, 2], + decoded: " 12 3", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["\u2581Hello", "\u2581World"], + ids: [0, 35378, 6661, 2], + decoded: " Hello World", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["\u2581hell", "o", "\u2581world"], + ids: [0, 33600, 31, 8999, 2], + decoded: " hello world", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u2581", "\u751f\u6d3b\u7684", "\u771f", "\u8c1b", "\u662f"], + ids: [0, 6, 62668, 5364, 245875, 354, 2], + decoded: " \u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u2581leading", "\u2581space"], + ids: [0, 105207, 32628, 2], + decoded: " leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["\u2581trail", "ing", "\u2581space", "\u2581"], + ids: [0, 141037, 214, 32628, 6, 2], + decoded: " trailing space ", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["\u2581Hi", "\u2581Hello"], + ids: [0, 2673, 35378, 2], + decoded: " Hi Hello", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["\u2581test", "\u2581$1", "\u2581R", "2", "\u2581#3", "\u2581\u20ac", "4", "\u2581\u00a3", "5", "\u2581", "\u00a5", "6", "\u2581", "\u20a3", "7", "\u2581\u20b9", "8", "\u2581", "\u20b1", "9", "\u2581test"], + ids: [0, 3034, 38629, 627, 304, 111378, 2505, 617, 11762, 758, 6, 32389, 910, 6, 3, 966, 87316, 1019, 6, 247425, 1126, 3034, 2], + decoded: " test $1 R2 #3 \u20ac4 \u00a35 \u00a56 7 \u20b98 \u20b19 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["\u2581I", "\u2581bought", "\u2581an", "\u2581apple", "\u2581for", "\u2581$", "1.00", "\u2581at", "\u2581the", "\u2581store", "."], + ids: [0, 87, 123997, 142, 108787, 100, 3650, 146533, 99, 70, 4343, 5, 2], + decoded: " I bought an apple for $1.00 at the store.", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["\u2581you", "...", "\u2581"], + ids: [0, 398, 27, 6, 2], + decoded: " you... ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["\u2581you", "...", "\u2581"], + ids: [0, 398, 27, 6, 2], + decoded: " you... ", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["\u2581you", "...", "\u2581you", "...", "\u2581"], + ids: [0, 398, 27, 398, 27, 6, 2], + decoded: " you... you... ", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["\u2581weird", "\u2581", "\uff5e", "\u2581edge", "\u2581", "\uff5e", "\u2581case"], + ids: [0, 179459, 6, 6087, 121303, 6, 6087, 7225, 2], + decoded: " weird \uff5e edge \uff5e case", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u2581This", "\u2581is", "\u2581a", "\u2581test", "\u2581", "."], + ids: [0, 3293, 83, 10, 3034, 6, 5, 2], + decoded: " This is a test.", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u2581", "\ud83d\ude02", "\u2581", "\ud83d\udc4d", "\u2581", "\ud83e\udd23", "\u2581", "\ud83d\ude0d", "\u2581", "\ud83d\ude2d", "\u2581", "\ud83c\udf89", "\u2581", "\ud83d\ude4f", "\u2581", "\ud83d\ude0a", "\u2581", "\ud83d\udd25", "\u2581", "\ud83d\ude01", "\u2581", "\ud83d\ude05", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\ude06", "\u2581", "\ud83d\udc4f", "\u2581\u2764", "\ufe0f", "\u2581", "\ud83d\udc9c", "\u2581", "\ud83d\udc9a", "\u2581", "\ud83d\udc97", "\u2581", "\ud83d\udc99", "\u2581", "\ud83d\udda4", "\u2581", "\ud83d\ude0e", "\u2581", "\ud83d\udc4c", "\u2581", "\ud83e\udd73", "\u2581", "\ud83d\udcaa", "\u2581", "\u2728", "\u2581", "\ud83d\udc49", "\u2581", "\ud83d\udc40", "\u2581", "\ud83d\udcaf", "\u2581", "\ud83c\udf88", "\u2581", "\ud83d\ude48", "\u2581", "\ud83d\ude4c", "\u2581", "\ud83d\udc80", "\u2581", "\ud83d\udc47", "\u2581", "\ud83d\udc4b", "\u2581", "\u2705", "\u2581", "\ud83c\udf81", "\u2581", "\ud83c\udf1e", "\u2581", "\ud83c\udf38", "\u2581", "\ud83d\udcb0"], + ids: [0, 6, 115114, 6, 118280, 6, 243385, 6, 84464, 6, 232773, 6, 243816, 6, 113612, 6, 82803, 6, 222326, 6, 201344, 6, 239569, 6, 243544, 6, 191876, 6, 243404, 49933, 15755, 6, 244233, 6, 244162, 6, 244181, 6, 243892, 6, 245820, 6, 161546, 6, 204811, 6, 3, 6, 238992, 6, 167474, 6, 120242, 6, 245561, 6, 244864, 6, 246144, 6, 244459, 6, 244703, 6, 246887, 6, 144400, 6, 246511, 6, 142325, 6, 244230, 6, 245559, 6, 243374, 6, 245200, 2], + decoded: " \ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u2581", "\u2728", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\udc41", "\ufe0f", "\u2581", "\ud83d\udc71", "\ud83c\udffb", "\u2581", "\ud83d\udd75", "\u2581", "\u2642", "\ufe0f", "\u2581", "\ud83e\uddd9", "\ud83c\udffb", "\u2581", "\u2642", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u2581", "\ud83c\udf3e", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83d\udc69", "\u2581\u2764", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc67", "\u2581", "\ud83d\udc66", "\u2581", "\ud83e\uddd1", "\ud83c\udffb", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1", "\ud83c\udffb", "\u2581", "\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u2581\u2764", "\ufe0f", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\ud83c\udffc"], + ids: [0, 6, 167474, 6, 243544, 6, 246984, 15755, 6, 247201, 79500, 6, 248325, 6, 228250, 15755, 6, 3, 79500, 6, 228250, 6, 244314, 79500, 6, 246529, 6, 3, 6, 247443, 6, 3, 6, 244785, 49933, 6, 244960, 6, 244314, 6, 244785, 6, 244785, 6, 245719, 6, 246167, 6, 3, 79500, 6, 247443, 6, 3, 79500, 6, 3, 6, 244314, 79500, 49933, 15755, 6, 244960, 6, 244314, 239719, 2], + decoded: " \u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75 \u2642\ufe0f \ud83c\udffb \u2642 \ud83d\udc68\ud83c\udffb \ud83c\udf3e \ud83e\udd1d \ud83d\udc69 \u2764 \ud83d\udc8b \ud83d\udc68 \ud83d\udc69 \ud83d\udc69 \ud83d\udc67 \ud83d\udc66 \ud83c\udffb \ud83e\udd1d \ud83c\udffb \ud83d\udc68\ud83c\udffb \u2764\ufe0f \ud83d\udc8b \ud83d\udc68\ud83c\udffc", + }, + }, + "Xenova/paraphrase-multilingual-mpnet-base-v2": { + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["\u2581trail", "ing", "\u2581space"], + ids: [0, 141037, 214, 32628, 2], + decoded: " trailing space", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["\u2581you", "..."], + ids: [0, 398, 27, 2], + decoded: " you...", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["\u2581you", "..."], + ids: [0, 398, 27, 2], + decoded: " you...", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["\u2581you", "...", "\u2581you", "..."], + ids: [0, 398, 27, 398, 27, 2], + decoded: " you... you...", + }, + }, + "Xenova/donut-base-finetuned-docvqa": { + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["\u258101", "23", "45", "67", "89", "\u25810", "\u25811", "\u25812", "\u25813", "\u25814", "\u25815", "\u25816", "\u25817", "\u25818", "\u25819", "\u258110", "\u2581100", "\u25811000"], + ids: [0, 37391, 38611, 41742, 18610, 20121, 50891, 1314, 3822, 9066, 22081, 20017, 35977, 38100, 38873, 42378, 23485, 52285, 40881, 2], + decoded: " 0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["\u2581A", "\u2581'", "ll", "\u2581!!", "to", "?", "'", "d", "'", "'", "d", "\u2581of", ",", "\u2581can", "'", "t", "."], + ids: [0, 46518, 28559, 4558, 47751, 19616, 36209, 28431, 49224, 28431, 28431, 49224, 2587, 35815, 53017, 28431, 16191, 39539, 2], + decoded: " A 'll!!to?'d''d of, can't.", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u2581lead", "ing", "\u2581space"], + ids: [0, 38498, 24357, 36833, 2], + decoded: " leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["\u2581trai", "ling", "\u2581space"], + ids: [0, 14262, 23291, 36833, 2], + decoded: " trailing space", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["\u2581test", "\u2581$1", "\u2581R", "2", "\u2581#", "3", "\u2581\u20ac", "4", "\u2581\u00a3", "5", "\u2581", "\u00a5", "6", "\u2581", "\u20a3", "7", "\u2581", "\u20b9", "8", "\u2581", "\u20b1", "9", "\u2581test"], + ids: [0, 35950, 39065, 46982, 35934, 41882, 38167, 33874, 46702, 4467, 50934, 42990, 36748, 55144, 42990, 3, 56620, 42990, 31354, 486, 42990, 3, 3200, 35950, 2], + decoded: " test $1 R2 #3 \u20ac4 \u00a35 \u00a56 7 \u20b98 9 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["\u2581I", "\u2581bought", "\u2581an", "\u2581app", "le", "\u2581for", "\u2581$1", ".", "00", "\u2581at", "\u2581the", "\u2581store", "."], + ids: [0, 53821, 6018, 9971, 39627, 7897, 57245, 39065, 39539, 49351, 56980, 48941, 40747, 39539, 2], + decoded: " I bought an apple for $1.00 at the store.", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["\u2581we", "ird", "\u2581", "\uff5e", "\u2581ed", "ge", "\u2581", "\uff5e", "\u2581case"], + ids: [0, 47450, 52806, 42990, 46476, 25847, 40548, 42990, 46476, 49911, 2], + decoded: " weird \uff5e edge \uff5e case", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u2581", "\ud83d\ude02", "\u2581", "\ud83d\udc4d", "\u2581", "\ud83e\udd23", "\u2581", "\ud83d\ude0d", "\u2581", "\ud83d\ude2d", "\u2581", "\ud83c\udf89", "\u2581", "\ud83d\ude4f", "\u2581", "\ud83d\ude0a", "\u2581", "\ud83d\udd25", "\u2581", "\ud83d\ude01", "\u2581", "\ud83d\ude05", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\ude06", "\u2581", "\ud83d\udc4f", "\u2581", "\u2764\ufe0f", "\u2581", "\ud83d\udc9c", "\u2581", "\ud83d\udc9a", "\u2581", "\ud83d\udc97", "\u2581", "\ud83d\udc99", "\u2581", "\ud83d\udda4", "\u2581", "\ud83d\ude0e", "\u2581", "\ud83d\udc4c", "\u2581", "\ud83e\udd73", "\u2581", "\ud83d\udcaa", "\u2581", "\u2728", "\u2581", "\ud83d\udc49", "\u2581", "\ud83d\udc40", "\u2581", "\ud83d\udcaf", "\u2581", "\ud83c\udf88", "\u2581", "\ud83d\ude48", "\u2581", "\ud83d\ude4c", "\u2581", "\ud83d\udc80", "\u2581", "\ud83d\udc47", "\u2581", "\ud83d\udc4b", "\u2581", "\u2705", "\u2581", "\ud83c\udf81", "\u2581", "\ud83c\udf1e", "\u2581", "\ud83c\udf38", "\u2581", "\ud83d\udcb0"], + ids: [0, 42990, 3864, 42990, 3, 42990, 28873, 42990, 3, 42990, 3, 42990, 29257, 42990, 3, 42990, 3, 42990, 3, 42990, 22310, 42990, 3, 42990, 29017, 42990, 3, 42990, 28890, 42990, 3, 42990, 29601, 42990, 3, 42990, 29564, 42990, 3, 42990, 3, 42990, 14430, 42990, 3, 42990, 3, 42990, 28292, 42990, 3, 42990, 3, 42990, 30688, 42990, 30146, 42990, 3, 42990, 29798, 42990, 3, 42990, 3, 42990, 3, 42990, 3, 42990, 10257, 42990, 3, 42990, 30687, 42990, 3, 42990, 3, 2], + decoded: " \ud83d\ude02 \ud83e\udd23 \ud83c\udf89 \ud83d\ude01 \ud83e\udd17 \ud83d\udc4f \ud83d\udc9c \ud83d\udc97 \ud83d\ude0e \ud83d\udcaa \ud83d\udc40 \ud83d\udcaf \ud83d\ude48 \u2705 \ud83c\udf1e ", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u2581", "\u2728", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\udc41\ufe0f", "\u2581", "\ud83d\udc71\ud83c\udffb", "\u2581", "\ud83d\udd75", "\u2581", "\u2642", "\ufe0f", "\u2581", "\ud83e\uddd9\ud83c\udffb", "\u2581", "\u2642", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u2581", "\ud83c\udf3e", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83d\udc69", "\u2581", "\u2764", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc67", "\u2581", "\ud83d\udc66", "\u2581", "\ud83e\uddd1\ud83c\udffb", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1\ud83c\udffb", "\u2581", "\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u2581", "\u2764\ufe0f", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\ud83c\udffc"], + ids: [0, 42990, 3, 42990, 29017, 42990, 3, 42990, 3, 42990, 3, 42990, 26803, 3, 42990, 3, 42990, 26803, 42990, 29673, 3, 42990, 3, 42990, 3, 42990, 3, 42990, 3, 42990, 30079, 42990, 3, 42990, 30218, 42990, 29673, 42990, 30079, 42990, 30079, 42990, 30799, 42990, 31120, 42990, 3, 42990, 3, 42990, 3, 42990, 3, 42990, 29673, 3, 42990, 3, 42990, 30218, 42990, 29673, 28396, 2], + decoded: " \ud83e\udd17 \u2642 \u2642 \ud83d\udc68 \ud83d\udc69 \ud83d\udc8b \ud83d\udc68 \ud83d\udc69 \ud83d\udc69 \ud83d\udc67 \ud83d\udc66 \ud83d\udc68 \ud83d\udc8b \ud83d\udc68\ud83c\udffc", + }, + }, + "Xenova/trocr-small-handwritten": { + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["\u25810", "123", "45", "67", "89", "\u25810", "\u25811", "\u25812", "\u25813", "\u25814", "\u25815", "\u25816", "\u25817", "\u25818", "\u25819", "\u258110", "\u2581100", "\u25811000"], + ids: [0, 1596, 31702, 5356, 8248, 7385, 1596, 267, 252, 271, 319, 331, 467, 531, 539, 641, 274, 921, 8401, 2], + decoded: " 0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["\u2581The", "\u2581company", "\u2581was", "\u2581founded", "\u2581in", "\u25812016."], + ids: [0, 25, 215, 19, 3027, 12, 2401, 2], + decoded: " The company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["\u2581A", "\u2581'", "ll", "\u2581!", "!", "to", "?'", "d", "'", "'", "d", "\u2581of", ",", "\u2581can", "'", "t", "."], + ids: [0, 94, 26, 184, 118, 338, 436, 16368, 109, 27, 27, 109, 9, 6, 64, 27, 31, 5, 2], + decoded: " A 'll!!to?'d''d of, can't.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["\u2581def", "\u2581main", "(", "):", "\u2581pass"], + ids: [0, 13114, 830, 2126, 5056, 1080, 2], + decoded: " def main(): pass", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["\u2581let", "\u2581a", "\u2581=", "\u2581obj", ".", "to", "String", "(", ");", "\u2581to", "String", "(", ");"], + ids: [0, 393, 10, 2219, 48394, 5, 436, 34868, 2126, 3671, 7, 34868, 2126, 3671, 2], + decoded: " let a = obj.toString(); toString();", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["\u2581UN", "wan", "t\u00e9", "d", ",", "running"], + ids: [0, 3225, 6327, 12529, 109, 6, 11484, 2], + decoded: " UNwant\u00e9d,running", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["\u2581hello", "\u2581world"], + ids: [0, 12773, 218, 2], + decoded: " hello world", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u2581", "\u751f", "\u6d3b", "\u7684", "\u771f", "\u8c1b", "\u662f"], + ids: [0, 190, 63145, 3, 62784, 63741, 3, 63010, 2], + decoded: " \u751f\u7684\u771f\u662f", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["\u2581trailing", "\u2581space"], + ids: [0, 12250, 769, 2], + decoded: " trailing space", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["\u2581test", "\u2581$1", "\u2581R", "2", "\u2581#3", "\u2581\u20ac4", "\u2581\u00a35", "\u2581\u00a5", "6", "\u2581", "\u20a3", "7", "\u2581\u20b9", "8", "\u2581", "\u20b1", "9", "\u2581test"], + ids: [0, 1036, 1594, 791, 792, 28537, 46242, 15364, 45731, 1487, 190, 3, 1473, 32176, 1439, 190, 63400, 1428, 1036, 2], + decoded: " test $1 R2 #3 \u20ac4 \u00a35 \u00a56 7 \u20b98 \u20b19 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["\u2581I", "\u2581bought", "\u2581an", "\u2581apple", "\u2581for", "\u2581$1.00", "\u2581at", "\u2581the", "\u2581store", "."], + ids: [0, 14, 1355, 46, 8688, 17, 44092, 29, 4, 920, 5, 2], + decoded: " I bought an apple for $1.00 at the store.", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["\u2581weird", "\u2581", "\uff5e", "\u2581edge", "\u2581", "\uff5e", "\u2581case"], + ids: [0, 3392, 190, 3, 2297, 190, 3, 343, 2], + decoded: " weird edge case", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u2581This", "\u2581is", "\u2581a", "\u2581test", "\u2581."], + ids: [0, 127, 18, 10, 1036, 13, 2], + decoded: " This is a test.", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u2581", "\ud83d\ude02", "\u2581", "\ud83d\udc4d", "\u2581", "\ud83e\udd23", "\u2581", "\ud83d\ude0d", "\u2581", "\ud83d\ude2d", "\u2581", "\ud83c\udf89", "\u2581", "\ud83d\ude4f", "\u2581", "\ud83d\ude0a", "\u2581", "\ud83d\udd25", "\u2581", "\ud83d\ude01", "\u2581", "\ud83d\ude05", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\ude06", "\u2581", "\ud83d\udc4f", "\u2581\u2764", "\ufe0f", "\u2581", "\ud83d\udc9c", "\u2581", "\ud83d\udc9a", "\u2581", "\ud83d\udc97", "\u2581", "\ud83d\udc99", "\u2581", "\ud83d\udda4", "\u2581", "\ud83d\ude0e", "\u2581", "\ud83d\udc4c", "\u2581", "\ud83e\udd73", "\u2581", "\ud83d\udcaa", "\u2581", "\u2728", "\u2581", "\ud83d\udc49", "\u2581", "\ud83d\udc40", "\u2581", "\ud83d\udcaf", "\u2581", "\ud83c\udf88", "\u2581", "\ud83d\ude48", "\u2581", "\ud83d\ude4c", "\u2581", "\ud83d\udc80", "\u2581", "\ud83d\udc47", "\u2581", "\ud83d\udc4b", "\u2581", "\u2705", "\u2581", "\ud83c\udf81", "\u2581", "\ud83c\udf1e", "\u2581", "\ud83c\udf38", "\u2581", "\ud83d\udcb0"], + ids: [0, 190, 45790, 190, 63194, 190, 63067, 190, 62942, 190, 62920, 190, 63116, 190, 62811, 190, 63283, 190, 62997, 190, 63427, 190, 3, 190, 63849, 190, 63751, 190, 62872, 47278, 21661, 190, 63467, 190, 63570, 190, 63473, 190, 63061, 190, 63640, 190, 63468, 190, 63302, 190, 3, 190, 63064, 190, 62860, 190, 63489, 190, 63057, 190, 63543, 190, 3, 190, 3, 190, 62949, 190, 3, 190, 63847, 190, 3, 190, 63747, 190, 3, 190, 3, 190, 3, 190, 3, 2], + decoded: " \ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83d\ude4c \ud83d\udc47 \u2705 ", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u2581", "\u2728", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\udc41", "\ufe0f", "\u2581", "\ud83d\udc71", "\ud83c\udffb", "\u2581", "\ud83d\udd75", "\u2581", "\u2642", "\ufe0f", "\u2581", "\ud83e\uddd9", "\ud83c\udffb", "\u2581", "\u2642", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u2581", "\ud83c\udf3e", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83d\udc69", "\u2581\u2764", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc67", "\u2581", "\ud83d\udc66", "\u2581", "\ud83e\uddd1", "\ud83c\udffb", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1", "\ud83c\udffb", "\u2581", "\ud83c\udff4", "\udb40\udc67", "\udb40\udc62\udb40\udc65\udb40\udc6e", "\udb40\udc67", "\udb40\udc7f", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u2581\u2764", "\ufe0f", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\ud83c\udffc"], + ids: [0, 190, 62860, 190, 63849, 190, 3, 21661, 190, 3, 62863, 190, 3, 190, 63135, 21661, 190, 3, 62863, 190, 63135, 190, 3, 62863, 190, 3, 190, 3, 190, 3, 190, 3, 190, 3, 47278, 190, 63500, 190, 3, 190, 3, 190, 3, 190, 3, 190, 3, 190, 3, 62863, 190, 3, 190, 3, 62863, 190, 3, 63769, 3, 63769, 3, 190, 3, 62863, 47278, 21661, 190, 63500, 190, 3, 62816, 2], + decoded: " \u2728 \ud83e\udd17 \ufe0f \ud83c\udffb \u2642\ufe0f \ud83c\udffb \u2642 \ud83c\udffb \u2764 \ud83d\udc8b \ud83c\udffb \ud83c\udffb \udb40\udc67\udb40\udc67 \ud83c\udffb \u2764\ufe0f \ud83d\udc8b \ud83c\udffc", + }, }, - SIMPLE_WITH_PUNCTUATION: { - text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, - tokens: ["\u2581You", "\u2581should", "'", "ve", "\u2581done", "\u2581this"], - ids: [0, 2583, 5608, 25, 272, 16940, 903, 2], - decoded: " You should've done this", - }, - NUMBERS: { - text: BASE_TEST_STRINGS.NUMBERS, - tokens: ["\u258101", "23", "456", "789", "\u25810", "\u25811", "\u25812", "\u25813", "\u25814", "\u25815", "\u25816", "\u25817", "\u25818", "\u25819", "\u258110", "\u2581100", "\u25811000"], - ids: [0, 3413, 3742, 121317, 153781, 757, 106, 116, 138, 201, 190, 305, 361, 382, 483, 209, 805, 4382, 2], - decoded: " 0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["\u2581The", "\u2581company", "\u2581was", "\u2581found", "ed", "\u2581in", "\u25812016."], - ids: [0, 581, 14380, 509, 14037, 297, 23, 6360, 2], - decoded: " The company was founded in 2016.", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["\u2581A", "\u2581'", "ll", "\u2581!!", "to", "?", "'", "d", "''", "d", "\u2581of", ",", "\u2581can", "'", "t", "."], - ids: [0, 62, 242, 1181, 6506, 188, 32, 25, 71, 4765, 71, 111, 4, 831, 25, 18, 5, 2], - decoded: " A 'll!!to?'d''d of, can't.", - }, - PYTHON_CODE: { - text: BASE_TEST_STRINGS.PYTHON_CODE, - tokens: ["\u2581de", "f", "\u2581main", "(", "):", "\u2581pass"], - ids: [0, 8, 420, 5201, 132, 2077, 27875, 2], - decoded: " def main(): pass", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["\u2581let", "\u2581a", "\u2581=", "\u2581ob", "j", ".", "to", "Str", "ing", "(", ");", "\u2581to", "Str", "ing", "(", ");"], - ids: [0, 2633, 10, 2203, 995, 170, 5, 188, 71713, 214, 132, 3142, 47, 71713, 214, 132, 3142, 2], - decoded: " let a = obj.toString(); toString();", - }, - NEWLINES: { - text: BASE_TEST_STRINGS.NEWLINES, - tokens: ["\u2581This", "\u2581is", "\u2581a", "\u2581test", "."], - ids: [0, 3293, 83, 10, 3034, 5, 2], - decoded: " This is a test.", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["\u2581UN", "wan", "t\u00e9", "d", ",", "run", "ning"], - ids: [0, 8274, 3206, 2312, 71, 4, 16428, 592, 2], - decoded: " UNwant\u00e9d,running", - }, - CONTROL_TOKENS: { - text: BASE_TEST_STRINGS.CONTROL_TOKENS, - tokens: ["\u25811", "\u0000", "2", "\u25813"], - ids: [0, 106, 3, 304, 138, 2], - decoded: " 12 3", - }, - HELLO_WORLD_TITLECASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, - tokens: ["\u2581Hello", "\u2581World"], - ids: [0, 35378, 6661, 2], - decoded: " Hello World", - }, - HELLO_WORLD_LOWERCASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, - tokens: ["\u2581hell", "o", "\u2581world"], - ids: [0, 33600, 31, 8999, 2], - decoded: " hello world", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u2581", "\u751f\u6d3b\u7684", "\u771f", "\u8c1b", "\u662f"], - ids: [0, 6, 62668, 5364, 245875, 354, 2], - decoded: " \u751f\u6d3b\u7684\u771f\u8c1b\u662f", - }, - LEADING_SPACE: { - text: BASE_TEST_STRINGS.LEADING_SPACE, - tokens: ["\u2581leading", "\u2581space"], - ids: [0, 105207, 32628, 2], - decoded: " leading space", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["\u2581trail", "ing", "\u2581space", "\u2581"], - ids: [0, 141037, 214, 32628, 6, 2], - decoded: " trailing space ", - }, - DOUBLE_SPACE: { - text: BASE_TEST_STRINGS.DOUBLE_SPACE, - tokens: ["\u2581Hi", "\u2581Hello"], - ids: [0, 2673, 35378, 2], - decoded: " Hi Hello", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["\u2581test", "\u2581$1", "\u2581R", "2", "\u2581#3", "\u2581\u20ac", "4", "\u2581\u00a3", "5", "\u2581", "\u00a5", "6", "\u2581", "\u20a3", "7", "\u2581\u20b9", "8", "\u2581", "\u20b1", "9", "\u2581test"], - ids: [0, 3034, 38629, 627, 304, 111378, 2505, 617, 11762, 758, 6, 32389, 910, 6, 3, 966, 87316, 1019, 6, 247425, 1126, 3034, 2], - decoded: " test $1 R2 #3 \u20ac4 \u00a35 \u00a56 7 \u20b98 \u20b19 test", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["\u2581I", "\u2581bought", "\u2581an", "\u2581apple", "\u2581for", "\u2581$", "1.00", "\u2581at", "\u2581the", "\u2581store", "."], - ids: [0, 87, 123997, 142, 108787, 100, 3650, 146533, 99, 70, 4343, 5, 2], - decoded: " I bought an apple for $1.00 at the store.", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - tokens: ["\u2581you", "...", "\u2581"], - ids: [0, 398, 27, 6, 2], - decoded: " you... ", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - tokens: ["\u2581you", "...", "\u2581"], - ids: [0, 398, 27, 6, 2], - decoded: " you... ", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - tokens: ["\u2581you", "...", "\u2581you", "...", "\u2581"], - ids: [0, 398, 27, 398, 27, 6, 2], - decoded: " you... you... ", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["\u2581weird", "\u2581", "\uff5e", "\u2581edge", "\u2581", "\uff5e", "\u2581case"], - ids: [0, 179459, 6, 6087, 121303, 6, 6087, 7225, 2], - decoded: " weird \uff5e edge \uff5e case", - }, - SPIECE_UNDERSCORE: { - text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, - tokens: ["\u2581This", "\u2581is", "\u2581a", "\u2581test", "\u2581", "."], - ids: [0, 3293, 83, 10, 3034, 6, 5, 2], - decoded: " This is a test.", - }, - POPULAR_EMOJIS: { - text: BASE_TEST_STRINGS.POPULAR_EMOJIS, - tokens: ["\u2581", "\ud83d\ude02", "\u2581", "\ud83d\udc4d", "\u2581", "\ud83e\udd23", "\u2581", "\ud83d\ude0d", "\u2581", "\ud83d\ude2d", "\u2581", "\ud83c\udf89", "\u2581", "\ud83d\ude4f", "\u2581", "\ud83d\ude0a", "\u2581", "\ud83d\udd25", "\u2581", "\ud83d\ude01", "\u2581", "\ud83d\ude05", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\ude06", "\u2581", "\ud83d\udc4f", "\u2581\u2764", "\ufe0f", "\u2581", "\ud83d\udc9c", "\u2581", "\ud83d\udc9a", "\u2581", "\ud83d\udc97", "\u2581", "\ud83d\udc99", "\u2581", "\ud83d\udda4", "\u2581", "\ud83d\ude0e", "\u2581", "\ud83d\udc4c", "\u2581", "\ud83e\udd73", "\u2581", "\ud83d\udcaa", "\u2581", "\u2728", "\u2581", "\ud83d\udc49", "\u2581", "\ud83d\udc40", "\u2581", "\ud83d\udcaf", "\u2581", "\ud83c\udf88", "\u2581", "\ud83d\ude48", "\u2581", "\ud83d\ude4c", "\u2581", "\ud83d\udc80", "\u2581", "\ud83d\udc47", "\u2581", "\ud83d\udc4b", "\u2581", "\u2705", "\u2581", "\ud83c\udf81", "\u2581", "\ud83c\udf1e", "\u2581", "\ud83c\udf38", "\u2581", "\ud83d\udcb0"], - ids: [0, 6, 115114, 6, 118280, 6, 243385, 6, 84464, 6, 232773, 6, 243816, 6, 113612, 6, 82803, 6, 222326, 6, 201344, 6, 239569, 6, 243544, 6, 191876, 6, 243404, 49933, 15755, 6, 244233, 6, 244162, 6, 244181, 6, 243892, 6, 245820, 6, 161546, 6, 204811, 6, 3, 6, 238992, 6, 167474, 6, 120242, 6, 245561, 6, 244864, 6, 246144, 6, 244459, 6, 244703, 6, 246887, 6, 144400, 6, 246511, 6, 142325, 6, 244230, 6, 245559, 6, 243374, 6, 245200, 2], - decoded: " \ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0", - }, - MULTIBYTE_EMOJIS: { - text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, - tokens: ["\u2581", "\u2728", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\udc41", "\ufe0f", "\u2581", "\ud83d\udc71", "\ud83c\udffb", "\u2581", "\ud83d\udd75", "\u2581", "\u2642", "\ufe0f", "\u2581", "\ud83e\uddd9", "\ud83c\udffb", "\u2581", "\u2642", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u2581", "\ud83c\udf3e", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83d\udc69", "\u2581\u2764", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc67", "\u2581", "\ud83d\udc66", "\u2581", "\ud83e\uddd1", "\ud83c\udffb", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1", "\ud83c\udffb", "\u2581", "\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u2581\u2764", "\ufe0f", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\ud83c\udffc"], - ids: [0, 6, 167474, 6, 243544, 6, 246984, 15755, 6, 247201, 79500, 6, 248325, 6, 228250, 15755, 6, 3, 79500, 6, 228250, 6, 244314, 79500, 6, 246529, 6, 3, 6, 247443, 6, 3, 6, 244785, 49933, 6, 244960, 6, 244314, 6, 244785, 6, 244785, 6, 245719, 6, 246167, 6, 3, 79500, 6, 247443, 6, 3, 79500, 6, 3, 6, 244314, 79500, 49933, 15755, 6, 244960, 6, 244314, 239719, 2], - decoded: " \u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75 \u2642\ufe0f \ud83c\udffb \u2642 \ud83d\udc68\ud83c\udffb \ud83c\udf3e \ud83e\udd1d \ud83d\udc69 \u2764 \ud83d\udc8b \ud83d\udc68 \ud83d\udc69 \ud83d\udc69 \ud83d\udc67 \ud83d\udc66 \ud83c\udffb \ud83e\udd1d \ud83c\udffb \ud83d\udc68\ud83c\udffb \u2764\ufe0f \ud83d\udc8b \ud83d\udc68\ud83c\udffc", - }, - }, - "Xenova/paraphrase-multilingual-mpnet-base-v2": { - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["\u2581trail", "ing", "\u2581space"], - ids: [0, 141037, 214, 32628, 2], - decoded: " trailing space", - }, - ELLIPSIS: { - text: BASE_TEST_STRINGS.ELLIPSIS, - tokens: ["\u2581you", "..."], - ids: [0, 398, 27, 2], - decoded: " you...", - }, - TEXT_WITH_ESCAPE_CHARACTERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, - tokens: ["\u2581you", "..."], - ids: [0, 398, 27, 2], - decoded: " you...", - }, - TEXT_WITH_ESCAPE_CHARACTERS_2: { - text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, - tokens: ["\u2581you", "...", "\u2581you", "..."], - ids: [0, 398, 27, 398, 27, 2], - decoded: " you... you...", - }, - }, - "Xenova/donut-base-finetuned-docvqa": { - NUMBERS: { - text: BASE_TEST_STRINGS.NUMBERS, - tokens: ["\u258101", "23", "45", "67", "89", "\u25810", "\u25811", "\u25812", "\u25813", "\u25814", "\u25815", "\u25816", "\u25817", "\u25818", "\u25819", "\u258110", "\u2581100", "\u25811000"], - ids: [0, 37391, 38611, 41742, 18610, 20121, 50891, 1314, 3822, 9066, 22081, 20017, 35977, 38100, 38873, 42378, 23485, 52285, 40881, 2], - decoded: " 0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["\u2581A", "\u2581'", "ll", "\u2581!!", "to", "?", "'", "d", "'", "'", "d", "\u2581of", ",", "\u2581can", "'", "t", "."], - ids: [0, 46518, 28559, 4558, 47751, 19616, 36209, 28431, 49224, 28431, 28431, 49224, 2587, 35815, 53017, 28431, 16191, 39539, 2], - decoded: " A 'll!!to?'d''d of, can't.", - }, - LEADING_SPACE: { - text: BASE_TEST_STRINGS.LEADING_SPACE, - tokens: ["\u2581lead", "ing", "\u2581space"], - ids: [0, 38498, 24357, 36833, 2], - decoded: " leading space", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["\u2581trai", "ling", "\u2581space"], - ids: [0, 14262, 23291, 36833, 2], - decoded: " trailing space", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["\u2581test", "\u2581$1", "\u2581R", "2", "\u2581#", "3", "\u2581\u20ac", "4", "\u2581\u00a3", "5", "\u2581", "\u00a5", "6", "\u2581", "\u20a3", "7", "\u2581", "\u20b9", "8", "\u2581", "\u20b1", "9", "\u2581test"], - ids: [0, 35950, 39065, 46982, 35934, 41882, 38167, 33874, 46702, 4467, 50934, 42990, 36748, 55144, 42990, 3, 56620, 42990, 31354, 486, 42990, 3, 3200, 35950, 2], - decoded: " test $1 R2 #3 \u20ac4 \u00a35 \u00a56 7 \u20b98 9 test", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["\u2581I", "\u2581bought", "\u2581an", "\u2581app", "le", "\u2581for", "\u2581$1", ".", "00", "\u2581at", "\u2581the", "\u2581store", "."], - ids: [0, 53821, 6018, 9971, 39627, 7897, 57245, 39065, 39539, 49351, 56980, 48941, 40747, 39539, 2], - decoded: " I bought an apple for $1.00 at the store.", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["\u2581we", "ird", "\u2581", "\uff5e", "\u2581ed", "ge", "\u2581", "\uff5e", "\u2581case"], - ids: [0, 47450, 52806, 42990, 46476, 25847, 40548, 42990, 46476, 49911, 2], - decoded: " weird \uff5e edge \uff5e case", - }, - POPULAR_EMOJIS: { - text: BASE_TEST_STRINGS.POPULAR_EMOJIS, - tokens: ["\u2581", "\ud83d\ude02", "\u2581", "\ud83d\udc4d", "\u2581", "\ud83e\udd23", "\u2581", "\ud83d\ude0d", "\u2581", "\ud83d\ude2d", "\u2581", "\ud83c\udf89", "\u2581", "\ud83d\ude4f", "\u2581", "\ud83d\ude0a", "\u2581", "\ud83d\udd25", "\u2581", "\ud83d\ude01", "\u2581", "\ud83d\ude05", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\ude06", "\u2581", "\ud83d\udc4f", "\u2581", "\u2764\ufe0f", "\u2581", "\ud83d\udc9c", "\u2581", "\ud83d\udc9a", "\u2581", "\ud83d\udc97", "\u2581", "\ud83d\udc99", "\u2581", "\ud83d\udda4", "\u2581", "\ud83d\ude0e", "\u2581", "\ud83d\udc4c", "\u2581", "\ud83e\udd73", "\u2581", "\ud83d\udcaa", "\u2581", "\u2728", "\u2581", "\ud83d\udc49", "\u2581", "\ud83d\udc40", "\u2581", "\ud83d\udcaf", "\u2581", "\ud83c\udf88", "\u2581", "\ud83d\ude48", "\u2581", "\ud83d\ude4c", "\u2581", "\ud83d\udc80", "\u2581", "\ud83d\udc47", "\u2581", "\ud83d\udc4b", "\u2581", "\u2705", "\u2581", "\ud83c\udf81", "\u2581", "\ud83c\udf1e", "\u2581", "\ud83c\udf38", "\u2581", "\ud83d\udcb0"], - ids: [0, 42990, 3864, 42990, 3, 42990, 28873, 42990, 3, 42990, 3, 42990, 29257, 42990, 3, 42990, 3, 42990, 3, 42990, 22310, 42990, 3, 42990, 29017, 42990, 3, 42990, 28890, 42990, 3, 42990, 29601, 42990, 3, 42990, 29564, 42990, 3, 42990, 3, 42990, 14430, 42990, 3, 42990, 3, 42990, 28292, 42990, 3, 42990, 3, 42990, 30688, 42990, 30146, 42990, 3, 42990, 29798, 42990, 3, 42990, 3, 42990, 3, 42990, 3, 42990, 10257, 42990, 3, 42990, 30687, 42990, 3, 42990, 3, 2], - decoded: " \ud83d\ude02 \ud83e\udd23 \ud83c\udf89 \ud83d\ude01 \ud83e\udd17 \ud83d\udc4f \ud83d\udc9c \ud83d\udc97 \ud83d\ude0e \ud83d\udcaa \ud83d\udc40 \ud83d\udcaf \ud83d\ude48 \u2705 \ud83c\udf1e ", - }, - MULTIBYTE_EMOJIS: { - text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, - tokens: ["\u2581", "\u2728", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\udc41\ufe0f", "\u2581", "\ud83d\udc71\ud83c\udffb", "\u2581", "\ud83d\udd75", "\u2581", "\u2642", "\ufe0f", "\u2581", "\ud83e\uddd9\ud83c\udffb", "\u2581", "\u2642", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u2581", "\ud83c\udf3e", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83d\udc69", "\u2581", "\u2764", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc67", "\u2581", "\ud83d\udc66", "\u2581", "\ud83e\uddd1\ud83c\udffb", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1\ud83c\udffb", "\u2581", "\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u2581", "\u2764\ufe0f", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\ud83c\udffc"], - ids: [0, 42990, 3, 42990, 29017, 42990, 3, 42990, 3, 42990, 3, 42990, 26803, 3, 42990, 3, 42990, 26803, 42990, 29673, 3, 42990, 3, 42990, 3, 42990, 3, 42990, 3, 42990, 30079, 42990, 3, 42990, 30218, 42990, 29673, 42990, 30079, 42990, 30079, 42990, 30799, 42990, 31120, 42990, 3, 42990, 3, 42990, 3, 42990, 3, 42990, 29673, 3, 42990, 3, 42990, 30218, 42990, 29673, 28396, 2], - decoded: " \ud83e\udd17 \u2642 \u2642 \ud83d\udc68 \ud83d\udc69 \ud83d\udc8b \ud83d\udc68 \ud83d\udc69 \ud83d\udc69 \ud83d\udc67 \ud83d\udc66 \ud83d\udc68 \ud83d\udc8b \ud83d\udc68\ud83c\udffc", - }, - }, - "Xenova/trocr-small-handwritten": { - NUMBERS: { - text: BASE_TEST_STRINGS.NUMBERS, - tokens: ["\u25810", "123", "45", "67", "89", "\u25810", "\u25811", "\u25812", "\u25813", "\u25814", "\u25815", "\u25816", "\u25817", "\u25818", "\u25819", "\u258110", "\u2581100", "\u25811000"], - ids: [0, 1596, 31702, 5356, 8248, 7385, 1596, 267, 252, 271, 319, 331, 467, 531, 539, 641, 274, 921, 8401, 2], - decoded: " 0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", - }, - TEXT_WITH_NUMBERS: { - text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, - tokens: ["\u2581The", "\u2581company", "\u2581was", "\u2581founded", "\u2581in", "\u25812016."], - ids: [0, 25, 215, 19, 3027, 12, 2401, 2], - decoded: " The company was founded in 2016.", - }, - PUNCTUATION: { - text: BASE_TEST_STRINGS.PUNCTUATION, - tokens: ["\u2581A", "\u2581'", "ll", "\u2581!", "!", "to", "?'", "d", "'", "'", "d", "\u2581of", ",", "\u2581can", "'", "t", "."], - ids: [0, 94, 26, 184, 118, 338, 436, 16368, 109, 27, 27, 109, 9, 6, 64, 27, 31, 5, 2], - decoded: " A 'll!!to?'d''d of, can't.", - }, - PYTHON_CODE: { - text: BASE_TEST_STRINGS.PYTHON_CODE, - tokens: ["\u2581def", "\u2581main", "(", "):", "\u2581pass"], - ids: [0, 13114, 830, 2126, 5056, 1080, 2], - decoded: " def main(): pass", - }, - JAVASCRIPT_CODE: { - text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, - tokens: ["\u2581let", "\u2581a", "\u2581=", "\u2581obj", ".", "to", "String", "(", ");", "\u2581to", "String", "(", ");"], - ids: [0, 393, 10, 2219, 48394, 5, 436, 34868, 2126, 3671, 7, 34868, 2126, 3671, 2], - decoded: " let a = obj.toString(); toString();", - }, - BASIC: { - text: BASE_TEST_STRINGS.BASIC, - tokens: ["\u2581UN", "wan", "t\u00e9", "d", ",", "running"], - ids: [0, 3225, 6327, 12529, 109, 6, 11484, 2], - decoded: " UNwant\u00e9d,running", - }, - HELLO_WORLD_LOWERCASE: { - text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, - tokens: ["\u2581hello", "\u2581world"], - ids: [0, 12773, 218, 2], - decoded: " hello world", - }, - CHINESE_ONLY: { - text: BASE_TEST_STRINGS.CHINESE_ONLY, - tokens: ["\u2581", "\u751f", "\u6d3b", "\u7684", "\u771f", "\u8c1b", "\u662f"], - ids: [0, 190, 63145, 3, 62784, 63741, 3, 63010, 2], - decoded: " \u751f\u7684\u771f\u662f", - }, - TRAILING_SPACE: { - text: BASE_TEST_STRINGS.TRAILING_SPACE, - tokens: ["\u2581trailing", "\u2581space"], - ids: [0, 12250, 769, 2], - decoded: " trailing space", - }, - CURRENCY: { - text: BASE_TEST_STRINGS.CURRENCY, - tokens: ["\u2581test", "\u2581$1", "\u2581R", "2", "\u2581#3", "\u2581\u20ac4", "\u2581\u00a35", "\u2581\u00a5", "6", "\u2581", "\u20a3", "7", "\u2581\u20b9", "8", "\u2581", "\u20b1", "9", "\u2581test"], - ids: [0, 1036, 1594, 791, 792, 28537, 46242, 15364, 45731, 1487, 190, 3, 1473, 32176, 1439, 190, 63400, 1428, 1036, 2], - decoded: " test $1 R2 #3 \u20ac4 \u00a35 \u00a56 7 \u20b98 \u20b19 test", - }, - CURRENCY_WITH_DECIMALS: { - text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, - tokens: ["\u2581I", "\u2581bought", "\u2581an", "\u2581apple", "\u2581for", "\u2581$1.00", "\u2581at", "\u2581the", "\u2581store", "."], - ids: [0, 14, 1355, 46, 8688, 17, 44092, 29, 4, 920, 5, 2], - decoded: " I bought an apple for $1.00 at the store.", - }, - TILDE_NORMALIZATION: { - text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, - tokens: ["\u2581weird", "\u2581", "\uff5e", "\u2581edge", "\u2581", "\uff5e", "\u2581case"], - ids: [0, 3392, 190, 3, 2297, 190, 3, 343, 2], - decoded: " weird edge case", - }, - SPIECE_UNDERSCORE: { - text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, - tokens: ["\u2581This", "\u2581is", "\u2581a", "\u2581test", "\u2581."], - ids: [0, 127, 18, 10, 1036, 13, 2], - decoded: " This is a test.", - }, - POPULAR_EMOJIS: { - text: BASE_TEST_STRINGS.POPULAR_EMOJIS, - tokens: ["\u2581", "\ud83d\ude02", "\u2581", "\ud83d\udc4d", "\u2581", "\ud83e\udd23", "\u2581", "\ud83d\ude0d", "\u2581", "\ud83d\ude2d", "\u2581", "\ud83c\udf89", "\u2581", "\ud83d\ude4f", "\u2581", "\ud83d\ude0a", "\u2581", "\ud83d\udd25", "\u2581", "\ud83d\ude01", "\u2581", "\ud83d\ude05", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\ude06", "\u2581", "\ud83d\udc4f", "\u2581\u2764", "\ufe0f", "\u2581", "\ud83d\udc9c", "\u2581", "\ud83d\udc9a", "\u2581", "\ud83d\udc97", "\u2581", "\ud83d\udc99", "\u2581", "\ud83d\udda4", "\u2581", "\ud83d\ude0e", "\u2581", "\ud83d\udc4c", "\u2581", "\ud83e\udd73", "\u2581", "\ud83d\udcaa", "\u2581", "\u2728", "\u2581", "\ud83d\udc49", "\u2581", "\ud83d\udc40", "\u2581", "\ud83d\udcaf", "\u2581", "\ud83c\udf88", "\u2581", "\ud83d\ude48", "\u2581", "\ud83d\ude4c", "\u2581", "\ud83d\udc80", "\u2581", "\ud83d\udc47", "\u2581", "\ud83d\udc4b", "\u2581", "\u2705", "\u2581", "\ud83c\udf81", "\u2581", "\ud83c\udf1e", "\u2581", "\ud83c\udf38", "\u2581", "\ud83d\udcb0"], - ids: [0, 190, 45790, 190, 63194, 190, 63067, 190, 62942, 190, 62920, 190, 63116, 190, 62811, 190, 63283, 190, 62997, 190, 63427, 190, 3, 190, 63849, 190, 63751, 190, 62872, 47278, 21661, 190, 63467, 190, 63570, 190, 63473, 190, 63061, 190, 63640, 190, 63468, 190, 63302, 190, 3, 190, 63064, 190, 62860, 190, 63489, 190, 63057, 190, 63543, 190, 3, 190, 3, 190, 62949, 190, 3, 190, 63847, 190, 3, 190, 63747, 190, 3, 190, 3, 190, 3, 190, 3, 2], - decoded: " \ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83d\ude4c \ud83d\udc47 \u2705 ", - }, - MULTIBYTE_EMOJIS: { - text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, - tokens: ["\u2581", "\u2728", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\udc41", "\ufe0f", "\u2581", "\ud83d\udc71", "\ud83c\udffb", "\u2581", "\ud83d\udd75", "\u2581", "\u2642", "\ufe0f", "\u2581", "\ud83e\uddd9", "\ud83c\udffb", "\u2581", "\u2642", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u2581", "\ud83c\udf3e", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83d\udc69", "\u2581\u2764", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc67", "\u2581", "\ud83d\udc66", "\u2581", "\ud83e\uddd1", "\ud83c\udffb", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1", "\ud83c\udffb", "\u2581", "\ud83c\udff4", "\udb40\udc67", "\udb40\udc62\udb40\udc65\udb40\udc6e", "\udb40\udc67", "\udb40\udc7f", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u2581\u2764", "\ufe0f", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\ud83c\udffc"], - ids: [0, 190, 62860, 190, 63849, 190, 3, 21661, 190, 3, 62863, 190, 3, 190, 63135, 21661, 190, 3, 62863, 190, 63135, 190, 3, 62863, 190, 3, 190, 3, 190, 3, 190, 3, 190, 3, 47278, 190, 63500, 190, 3, 190, 3, 190, 3, 190, 3, 190, 3, 190, 3, 62863, 190, 3, 190, 3, 62863, 190, 3, 63769, 3, 63769, 3, 190, 3, 62863, 47278, 21661, 190, 63500, 190, 3, 62816, 2], - decoded: " \u2728 \ud83e\udd17 \ufe0f \ud83c\udffb \u2642\ufe0f \ud83c\udffb \u2642 \ud83c\udffb \u2764 \ud83d\udc8b \ud83c\udffb \ud83c\udffb \udb40\udc67\udb40\udc67 \ud83c\udffb \u2764\ufe0f \ud83d\udc8b \ud83c\udffc", - }, - }, }; diff --git a/tests/pipelines.test.js b/tests/pipelines.test.js index 6bef83297..9a397ba60 100644 --- a/tests/pipelines.test.js +++ b/tests/pipelines.test.js @@ -11,549 +11,633 @@ init(); // As a result, each test is responsible for exactly one model, but we run multiple inputs through it. // By encapsulating model construction and destruction in a single `it` block, we avoid these memory issues. xdescribe("Pipelines", () => { - describe("Text classification", () => { - // List all models which will be tested - const models = ["Xenova/distilbert-base-uncased-finetuned-sst-2-english", "Xenova/toxic-bert"]; - - // single_label_classification - it( - models[0], - async () => { - let classifier = await pipeline("text-classification", models[0]); - let texts = ["This was a masterpiece. Not completely faithful to the books, but enthralling from beginning to end. Might be my favorite of the three.", "I hated the movie"]; - - // single - { - let outputs = await classifier("I hated the movie"); - let expected = [{ label: "NEGATIVE", score: 0.9996212720870972 }]; - compare(outputs, expected); - } - - // single + topk - { - let outputs = await classifier("I hated the movie", { - topk: 2, - }); - let expected = [ - { label: "NEGATIVE", score: 0.9996212720870972 }, - { label: "POSITIVE", score: 0.0003787268069572747 }, - ]; - compare(outputs, expected); - } - - // batched - { - let outputs = await classifier(texts); - - let expected = [ - { label: "POSITIVE", score: 0.9993746876716614 }, - { label: "NEGATIVE", score: 0.9996694326400757 }, - ]; - - compare(outputs, expected); - } - - // batched + topk - { - let outputs = await classifier(texts, { - topk: 2, - }); - - let expected = [ - [ - { label: "POSITIVE", score: 0.9993746876716614 }, - { label: "NEGATIVE", score: 0.0006253048195503652 }, - ], - [ - { label: "NEGATIVE", score: 0.9996694326400757 }, - { label: "POSITIVE", score: 0.00033057318069040775 }, - ], - ]; - - compare(outputs, expected); - } - - await classifier.dispose(); - }, - MAX_TEST_EXECUTION_TIME, - ); - - // multi_label_classification - it( - models[1], - async () => { - let classifier = await pipeline("text-classification", models[1]); - let texts = [ - "I like you. I love you", // low scores - "I hate you.", // high scores - ]; - - // single - { - let outputs = await classifier(texts); - let expected = [ - { label: "toxic", score: 0.0007729064091108739 }, - { label: "toxic", score: 0.9475088119506836 }, - ]; - compare(outputs, expected); - } - }, - MAX_TEST_EXECUTION_TIME, - ); - }); - - describe("Token classification", () => { - // List all models which will be tested - const models = ["Xenova/bert-base-multilingual-cased-ner-hrl"]; - - it( - models[0], - async () => { - let classifier = await pipeline("token-classification", models[0]); - let texts = ["The Golden State Warriors are an American professional basketball team based in San Francisco.", "My name is Sarah and I live in London."]; - - // single - { - let outputs = await classifier(texts[0]); - - let expected = [ - { entity: "B-ORG", score: 0.9998535513877869, index: 2, word: "Golden", start: null, end: null }, - { entity: "I-ORG", score: 0.9998612999916077, index: 3, word: "State", start: null, end: null }, - { entity: "I-ORG", score: 0.999866247177124, index: 4, word: "Warriors", start: null, end: null }, - { entity: "B-LOC", score: 0.9997050166130066, index: 13, word: "San", start: null, end: null }, - { entity: "I-LOC", score: 0.9987282156944275, index: 14, word: "Francisco", start: null, end: null }, - ]; - - compare(outputs, expected, 0.05); - } - - // batched - { - let outputs = await classifier(texts); - - let expected = [ - [ - { entity: "B-ORG", score: 0.9998375773429871, index: 2, word: "Golden", start: null, end: null }, - { entity: "I-ORG", score: 0.9998642206192017, index: 3, word: "State", start: null, end: null }, - { entity: "I-ORG", score: 0.9998642802238464, index: 4, word: "Warriors", start: null, end: null }, - { entity: "B-LOC", score: 0.9996914863586426, index: 13, word: "San", start: null, end: null }, - { entity: "I-LOC", score: 0.9989780783653259, index: 14, word: "Francisco", start: null, end: null }, - ], - [ - { entity: "B-PER", score: 0.997977614402771, index: 4, word: "Sarah", start: null, end: null }, - { entity: "B-LOC", score: 0.9996902346611023, index: 9, word: "London", start: null, end: null }, - ], - ]; - - compare(outputs, expected, 0.05); - } - - await classifier.dispose(); - }, - MAX_TEST_EXECUTION_TIME, - ); - }); - - describe("Zero-shot classification", () => { - // List all models which will be tested - const models = ["Xenova/bart-large-mnli"]; - - it( - models[0], - async () => { - let classifier = await pipeline("zero-shot-classification", models[0]); - - let sequences_to_classify = ["one day I will see the world", "I love making pizza"]; - let candidate_labels = ["travel", "cooking", "dancing"]; - - // single - { - let outputs = await classifier(sequences_to_classify[0], candidate_labels); - let expected = { - sequence: "one day I will see the world", - labels: ["travel", "dancing", "cooking"], - scores: [0.4261703487477968, 0.2903585771517135, 0.28347107410048983], - }; - - compare(outputs, expected, 0.2); - } - - // batched - { - let outputs = await classifier(sequences_to_classify, candidate_labels); - let expected = [ - { - sequence: "one day I will see the world", - labels: ["travel", "dancing", "cooking"], - scores: [0.4261703487477968, 0.2903585771517135, 0.28347107410048983], + describe("Text classification", () => { + // List all models which will be tested + const models = ["Xenova/distilbert-base-uncased-finetuned-sst-2-english", "Xenova/toxic-bert"]; + + // single_label_classification + it( + models[0], + async () => { + let classifier = await pipeline("text-classification", models[0]); + let texts = ["This was a masterpiece. Not completely faithful to the books, but enthralling from beginning to end. Might be my favorite of the three.", "I hated the movie"]; + + // single + { + let outputs = await classifier("I hated the movie"); + let expected = [{ label: "NEGATIVE", score: 0.9996212720870972 }]; + compare(outputs, expected); + } + + // single + topk + { + let outputs = await classifier("I hated the movie", { + topk: 2, + }); + let expected = [ + { label: "NEGATIVE", score: 0.9996212720870972 }, + { label: "POSITIVE", score: 0.0003787268069572747 }, + ]; + compare(outputs, expected); + } + + // batched + { + let outputs = await classifier(texts); + + let expected = [ + { label: "POSITIVE", score: 0.9993746876716614 }, + { label: "NEGATIVE", score: 0.9996694326400757 }, + ]; + + compare(outputs, expected); + } + + // batched + topk + { + let outputs = await classifier(texts, { + topk: 2, + }); + + let expected = [ + [ + { label: "POSITIVE", score: 0.9993746876716614 }, + { label: "NEGATIVE", score: 0.0006253048195503652 }, + ], + [ + { label: "NEGATIVE", score: 0.9996694326400757 }, + { label: "POSITIVE", score: 0.00033057318069040775 }, + ], + ]; + + compare(outputs, expected); + } + + await classifier.dispose(); }, - { - sequence: "I love making pizza", - labels: ["cooking", "travel", "dancing"], - scores: [0.4660367922118968, 0.2756005926506238, 0.2583626151374795], + MAX_TEST_EXECUTION_TIME, + ); + + // multi_label_classification + it( + models[1], + async () => { + let classifier = await pipeline("text-classification", models[1]); + let texts = [ + "I like you. I love you", // low scores + "I hate you.", // high scores + ]; + + // single + { + let outputs = await classifier(texts); + let expected = [ + { label: "toxic", score: 0.0007729064091108739 }, + { label: "toxic", score: 0.9475088119506836 }, + ]; + compare(outputs, expected); + } }, - ]; - - compare(outputs, expected, 0.2); - } - - // batched + multilabel - { - let outputs = await classifier(sequences_to_classify, candidate_labels, { - multi_label: true, - }); - let expected = [ - { - sequence: "one day I will see the world", - labels: ["travel", "dancing", "cooking"], - scores: [0.7108286792234982, 0.5763787804099745, 0.44303326070949994], + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("Token classification", () => { + // List all models which will be tested + const models = ["Xenova/bert-base-multilingual-cased-ner-hrl"]; + + it( + models[0], + async () => { + let classifier = await pipeline("token-classification", models[0]); + let texts = ["The Golden State Warriors are an American professional basketball team based in San Francisco.", "My name is Sarah and I live in London."]; + + // single + { + let outputs = await classifier(texts[0]); + + let expected = [ + { + entity: "B-ORG", + score: 0.9998535513877869, + index: 2, + word: "Golden", + start: null, + end: null, + }, + { + entity: "I-ORG", + score: 0.9998612999916077, + index: 3, + word: "State", + start: null, + end: null, + }, + { + entity: "I-ORG", + score: 0.999866247177124, + index: 4, + word: "Warriors", + start: null, + end: null, + }, + { + entity: "B-LOC", + score: 0.9997050166130066, + index: 13, + word: "San", + start: null, + end: null, + }, + { + entity: "I-LOC", + score: 0.9987282156944275, + index: 14, + word: "Francisco", + start: null, + end: null, + }, + ]; + + compare(outputs, expected, 0.05); + } + + // batched + { + let outputs = await classifier(texts); + + let expected = [ + [ + { + entity: "B-ORG", + score: 0.9998375773429871, + index: 2, + word: "Golden", + start: null, + end: null, + }, + { + entity: "I-ORG", + score: 0.9998642206192017, + index: 3, + word: "State", + start: null, + end: null, + }, + { + entity: "I-ORG", + score: 0.9998642802238464, + index: 4, + word: "Warriors", + start: null, + end: null, + }, + { + entity: "B-LOC", + score: 0.9996914863586426, + index: 13, + word: "San", + start: null, + end: null, + }, + { + entity: "I-LOC", + score: 0.9989780783653259, + index: 14, + word: "Francisco", + start: null, + end: null, + }, + ], + [ + { + entity: "B-PER", + score: 0.997977614402771, + index: 4, + word: "Sarah", + start: null, + end: null, + }, + { + entity: "B-LOC", + score: 0.9996902346611023, + index: 9, + word: "London", + start: null, + end: null, + }, + ], + ]; + + compare(outputs, expected, 0.05); + } + + await classifier.dispose(); }, - { - sequence: "I love making pizza", - labels: ["cooking", "travel", "dancing"], - scores: [0.8527619536354446, 0.7899589317978243, 0.5838912691496106], + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("Zero-shot classification", () => { + // List all models which will be tested + const models = ["Xenova/bart-large-mnli"]; + + it( + models[0], + async () => { + let classifier = await pipeline("zero-shot-classification", models[0]); + + let sequences_to_classify = ["one day I will see the world", "I love making pizza"]; + let candidate_labels = ["travel", "cooking", "dancing"]; + + // single + { + let outputs = await classifier(sequences_to_classify[0], candidate_labels); + let expected = { + sequence: "one day I will see the world", + labels: ["travel", "dancing", "cooking"], + scores: [0.4261703487477968, 0.2903585771517135, 0.28347107410048983], + }; + + compare(outputs, expected, 0.2); + } + + // batched + { + let outputs = await classifier(sequences_to_classify, candidate_labels); + let expected = [ + { + sequence: "one day I will see the world", + labels: ["travel", "dancing", "cooking"], + scores: [0.4261703487477968, 0.2903585771517135, 0.28347107410048983], + }, + { + sequence: "I love making pizza", + labels: ["cooking", "travel", "dancing"], + scores: [0.4660367922118968, 0.2756005926506238, 0.2583626151374795], + }, + ]; + + compare(outputs, expected, 0.2); + } + + // batched + multilabel + { + let outputs = await classifier(sequences_to_classify, candidate_labels, { + multi_label: true, + }); + let expected = [ + { + sequence: "one day I will see the world", + labels: ["travel", "dancing", "cooking"], + scores: [0.7108286792234982, 0.5763787804099745, 0.44303326070949994], + }, + { + sequence: "I love making pizza", + labels: ["cooking", "travel", "dancing"], + scores: [0.8527619536354446, 0.7899589317978243, 0.5838912691496106], + }, + ]; + + compare(outputs, expected); + } + + await classifier.dispose(); }, - ]; - - compare(outputs, expected); - } - - await classifier.dispose(); - }, - MAX_TEST_EXECUTION_TIME, - ); - }); - - describe("Masked language modelling", () => { - // List all models which will be tested - const models = ["Xenova/bert-base-uncased"]; - - it( - models[0], - async () => { - let unmasker = await pipeline("fill-mask", models[0]); - let texts = ["Once upon a [MASK].", "[MASK] is the capital of England."]; - - // single - { - let outputs = await unmasker(texts[0]); - let expected = [ - { - score: 0.9405396580696106, - token: 2051, - token_str: "time", - sequence: "once upon a time.", + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("Masked language modelling", () => { + // List all models which will be tested + const models = ["Xenova/bert-base-uncased"]; + + it( + models[0], + async () => { + let unmasker = await pipeline("fill-mask", models[0]); + let texts = ["Once upon a [MASK].", "[MASK] is the capital of England."]; + + // single + { + let outputs = await unmasker(texts[0]); + let expected = [ + { + score: 0.9405396580696106, + token: 2051, + token_str: "time", + sequence: "once upon a time.", + }, + { + score: 0.01182964164763689, + token: 13342, + token_str: "mattress", + sequence: "once upon a mattress.", + }, + { + score: 0.0017291896510869265, + token: 6480, + token_str: "lifetime", + sequence: "once upon a lifetime.", + }, + { + score: 0.0010079898638650775, + token: 2504, + token_str: "level", + sequence: "once upon a level.", + }, + { + score: 0.0009655007743276656, + token: 2154, + token_str: "day", + sequence: "once upon a day.", + }, + ]; + compare(outputs, expected); + } + + // batched + { + let outputs = await unmasker(texts); + + let expected = [ + [ + { + score: 0.9900539517402649, + token: 2051, + token_str: "time", + sequence: "once upon a time.", + }, + { + score: 0.0012258145725354552, + token: 13342, + token_str: "mattress", + sequence: "once upon a mattress.", + }, + { + score: 0.0002977887343149632, + token: 2096, + token_str: "while", + sequence: "once upon a while.", + }, + { + score: 0.0001899998023873195, + token: 6480, + token_str: "lifetime", + sequence: "once upon a lifetime.", + }, + { + score: 0.00017618606216274202, + token: 2558, + token_str: "period", + sequence: "once upon a period.", + }, + ], + [ + { + score: 0.2863538861274719, + token: 2414, + token_str: "london", + sequence: "london is the capital of england.", + }, + { + score: 0.0607745461165905, + token: 2009, + token_str: "it", + sequence: "it is the capital of england.", + }, + { + score: 0.037455108016729355, + token: 6484, + token_str: "birmingham", + sequence: "birmingham is the capital of england.", + }, + { + score: 0.029375044628977776, + token: 5087, + token_str: "manchester", + sequence: "manchester is the capital of england.", + }, + { + score: 0.0292277242988348, + token: 7067, + token_str: "bristol", + sequence: "bristol is the capital of england.", + }, + ], + ]; + + compare(outputs, expected); + } + + await unmasker.dispose(); }, - { - score: 0.01182964164763689, - token: 13342, - token_str: "mattress", - sequence: "once upon a mattress.", + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("Question answering", () => { + let question = "Who was Jim Henson?"; + let context = "Jim Henson was a nice puppet."; + + // List all models which will be tested + const models = ["Xenova/distilbert-base-uncased-distilled-squad"]; + + it( + models[0], + async () => { + let answerer = await pipeline("question-answering", models[0]); + + // single + { + let outputs = await answerer(question, context); + let expected = { answer: "a nice puppet", score: 0.5664517526948352 }; + + compare(outputs, expected, 0.2); + } + + // single + topk + { + let outputs = await answerer(question, context, { + topk: 3, + }); + let expected = [ + { answer: "a nice puppet", score: 0.5664517526948352 }, + { answer: "nice puppet", score: 0.1698902336448853 }, + { answer: "puppet", score: 0.14046057793125577 }, + ]; + + compare(outputs, expected, 0.2); + } + await answerer.dispose(); }, - { - score: 0.0017291896510869265, - token: 6480, - token_str: "lifetime", - sequence: "once upon a lifetime.", + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("Summarization", () => { + // List all models which will be tested + const models = ["Xenova/distilbart-cnn-6-6", "Xenova/bart-large-cnn"]; + + let texts = [`The tower is 324 metres (1,063 ft) tall, about the same height as an 81-storey building, and the tallest structure in Paris. Its base is square, measuring 125 metres (410 ft) on each side. During its construction, the Eiffel Tower surpassed the Washington Monument to become the tallest man-made structure in the world, a title it held for 41 years until the Chrysler Building in New York City was finished in 1930. It was the first structure to reach a height of 300 metres. Due to the addition of a broadcasting aerial at the top of the tower in 1957, it is now taller than the Chrysler Building by 5.2 metres (17 ft). Excluding transmitters, the Eiffel Tower is the second tallest free-standing structure in France after the Millau Viaduct.`, `The Amazon rainforest (Portuguese: Floresta Amazônica or Amazônia; Spanish: Selva Amazónica, Amazonía or usually Amazonia; French: Forêt amazonienne; Dutch: Amazoneregenwoud), also known in English as Amazonia or the Amazon Jungle, is a moist broadleaf forest that covers most of the Amazon basin of South America. This basin encompasses 7,000,000 square kilometres (2,700,000 sq mi), of which 5,500,000 square kilometres (2,100,000 sq mi) are covered by the rainforest. This region includes territory belonging to nine nations. The majority of the forest is contained within Brazil, with 60% of the rainforest, followed by Peru with 13%, Colombia with 10%, and with minor amounts in Venezuela, Ecuador, Bolivia, Guyana, Suriname and French Guiana. States or departments in four nations contain "Amazonas" in their names. The Amazon represents over half of the planet's remaining rainforests, and comprises the largest and most biodiverse tract of tropical rainforest in the world, with an estimated 390 billion individual trees divided into 16,000 species.`]; + + it( + models[0], + async () => { + let summarizer = await pipeline("summarization", models[0]); + + // batched + { + let summary = await summarizer(texts, { + top_k: 0, + do_sample: false, + }); + expect(summary).toHaveLength(2); + expect(summary[0].summary_text.length).toBeGreaterThan(50); + expect(summary[1].summary_text.length).toBeGreaterThan(50); + } + await summarizer.dispose(); }, - { - score: 0.0010079898638650775, - token: 2504, - token_str: "level", - sequence: "once upon a level.", + MAX_TEST_EXECUTION_TIME, + ); + + it( + models[1], + async () => { + let summarizer = await pipeline("summarization", models[1]); + + // batched + `forced_bos_token_id` + { + let summary = await summarizer(texts[0], { + top_k: 0, + do_sample: false, + }); + expect(summary).toHaveLength(1); + expect(summary[0].summary_text.length).toBeGreaterThan(50); + } + + await summarizer.dispose(); }, - { - score: 0.0009655007743276656, - token: 2154, - token_str: "day", - sequence: "once upon a day.", + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("Translation", () => { + // List all models which will be tested + const models = [ + "Xenova/t5-small", + + // Multilingual model + "Xenova/nllb-200-distilled-600M", + ]; + + it( + models[0], + async () => { + let translator = await pipeline("translation_en_to_de", models[0]); + let texts = ["Hello, how are you?", "My name is Maria."]; + + // single + { + let translation = await translator(texts[0], { + top_k: 0, + do_sample: false, + }); + + let expected = [{ translation_text: "Hallo, wie sind Sie?" }]; + + compare(translation, expected); + } + + // batched + { + let output = await translator(texts, { + top_k: 0, + do_sample: false, + }); + + let expected = [{ translation_text: "Hallo, wie sind Sie?" }, { translation_text: "Mein Name ist Maria." }]; + + compare(output, expected); + } + + await translator.dispose(); }, - ]; - compare(outputs, expected); - } - - // batched - { - let outputs = await unmasker(texts); - - let expected = [ - [ - { - score: 0.9900539517402649, - token: 2051, - token_str: "time", - sequence: "once upon a time.", - }, - { - score: 0.0012258145725354552, - token: 13342, - token_str: "mattress", - sequence: "once upon a mattress.", - }, - { - score: 0.0002977887343149632, - token: 2096, - token_str: "while", - sequence: "once upon a while.", - }, - { - score: 0.0001899998023873195, - token: 6480, - token_str: "lifetime", - sequence: "once upon a lifetime.", - }, - { - score: 0.00017618606216274202, - token: 2558, - token_str: "period", - sequence: "once upon a period.", - }, - ], - [ - { - score: 0.2863538861274719, - token: 2414, - token_str: "london", - sequence: "london is the capital of england.", - }, - { - score: 0.0607745461165905, - token: 2009, - token_str: "it", - sequence: "it is the capital of england.", - }, - { - score: 0.037455108016729355, - token: 6484, - token_str: "birmingham", - sequence: "birmingham is the capital of england.", - }, - { - score: 0.029375044628977776, - token: 5087, - token_str: "manchester", - sequence: "manchester is the capital of england.", - }, - { - score: 0.0292277242988348, - token: 7067, - token_str: "bristol", - sequence: "bristol is the capital of england.", - }, - ], - ]; - - compare(outputs, expected); - } - - await unmasker.dispose(); - }, - MAX_TEST_EXECUTION_TIME, - ); - }); - - describe("Question answering", () => { - let question = "Who was Jim Henson?"; - let context = "Jim Henson was a nice puppet."; - - // List all models which will be tested - const models = ["Xenova/distilbert-base-uncased-distilled-squad"]; - - it( - models[0], - async () => { - let answerer = await pipeline("question-answering", models[0]); - - // single - { - let outputs = await answerer(question, context); - let expected = { answer: "a nice puppet", score: 0.5664517526948352 }; - - compare(outputs, expected, 0.2); - } - - // single + topk - { - let outputs = await answerer(question, context, { - topk: 3, - }); - let expected = [ - { answer: "a nice puppet", score: 0.5664517526948352 }, - { answer: "nice puppet", score: 0.1698902336448853 }, - { answer: "puppet", score: 0.14046057793125577 }, - ]; - - compare(outputs, expected, 0.2); - } - await answerer.dispose(); - }, - MAX_TEST_EXECUTION_TIME, - ); - }); - - describe("Summarization", () => { - // List all models which will be tested - const models = ["Xenova/distilbart-cnn-6-6", "Xenova/bart-large-cnn"]; - - let texts = [`The tower is 324 metres (1,063 ft) tall, about the same height as an 81-storey building, and the tallest structure in Paris. Its base is square, measuring 125 metres (410 ft) on each side. During its construction, the Eiffel Tower surpassed the Washington Monument to become the tallest man-made structure in the world, a title it held for 41 years until the Chrysler Building in New York City was finished in 1930. It was the first structure to reach a height of 300 metres. Due to the addition of a broadcasting aerial at the top of the tower in 1957, it is now taller than the Chrysler Building by 5.2 metres (17 ft). Excluding transmitters, the Eiffel Tower is the second tallest free-standing structure in France after the Millau Viaduct.`, `The Amazon rainforest (Portuguese: Floresta Amazônica or Amazônia; Spanish: Selva Amazónica, Amazonía or usually Amazonia; French: Forêt amazonienne; Dutch: Amazoneregenwoud), also known in English as Amazonia or the Amazon Jungle, is a moist broadleaf forest that covers most of the Amazon basin of South America. This basin encompasses 7,000,000 square kilometres (2,700,000 sq mi), of which 5,500,000 square kilometres (2,100,000 sq mi) are covered by the rainforest. This region includes territory belonging to nine nations. The majority of the forest is contained within Brazil, with 60% of the rainforest, followed by Peru with 13%, Colombia with 10%, and with minor amounts in Venezuela, Ecuador, Bolivia, Guyana, Suriname and French Guiana. States or departments in four nations contain "Amazonas" in their names. The Amazon represents over half of the planet's remaining rainforests, and comprises the largest and most biodiverse tract of tropical rainforest in the world, with an estimated 390 billion individual trees divided into 16,000 species.`]; - - it( - models[0], - async () => { - let summarizer = await pipeline("summarization", models[0]); - - // batched - { - let summary = await summarizer(texts, { - top_k: 0, - do_sample: false, - }); - expect(summary).toHaveLength(2); - expect(summary[0].summary_text.length).toBeGreaterThan(50); - expect(summary[1].summary_text.length).toBeGreaterThan(50); - } - await summarizer.dispose(); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - models[1], - async () => { - let summarizer = await pipeline("summarization", models[1]); - - // batched + `forced_bos_token_id` - { - let summary = await summarizer(texts[0], { - top_k: 0, - do_sample: false, - }); - expect(summary).toHaveLength(1); - expect(summary[0].summary_text.length).toBeGreaterThan(50); - } - - await summarizer.dispose(); - }, - MAX_TEST_EXECUTION_TIME, - ); - }); - - describe("Translation", () => { - // List all models which will be tested - const models = [ - "Xenova/t5-small", - - // Multilingual model - "Xenova/nllb-200-distilled-600M", - ]; - - it( - models[0], - async () => { - let translator = await pipeline("translation_en_to_de", models[0]); - let texts = ["Hello, how are you?", "My name is Maria."]; - - // single - { - let translation = await translator(texts[0], { - top_k: 0, - do_sample: false, - }); - - let expected = [{ translation_text: "Hallo, wie sind Sie?" }]; - - compare(translation, expected); - } - - // batched - { - let output = await translator(texts, { - top_k: 0, - do_sample: false, - }); - - let expected = [{ translation_text: "Hallo, wie sind Sie?" }, { translation_text: "Mein Name ist Maria." }]; - - compare(output, expected); - } - - await translator.dispose(); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - models[1], - async () => { - let translator = await pipeline("translation", models[1]); - let texts = ["Hello world!", "I like to walk my dog."]; - - // single - { - let translation = await translator(texts[0], { - src_lang: "eng_Latn", - tgt_lang: "arb_Arab", - }); - - let expected = [{ translation_text: "مرحباً، يا عالم!" }]; - - compare(translation, expected); - } - - // single + back-translation - { - let translation1 = await translator(texts[1], { - // src_lang: 'eng_Latn', - tgt_lang: "ell_Grek", - }); - let translation2 = await translator(translation1[0].translation_text, { - src_lang: "ell_Grek", - tgt_lang: "eng_Latn", - }); - - let expected = [{ translation_text: "Μου αρέσει να περπατάω το σκυλί μου." }]; - - compare(translation1, expected); - - let expectedBack = [{ translation_text: texts[1] }]; - compare(translation2, expectedBack); - } - - await translator.dispose(); - }, - MAX_TEST_EXECUTION_TIME, - ); - }); - - describe("Text-to-text generation", () => { - // List all models which will be tested - const models = ["Xenova/flan-t5-small", "Xenova/flan-t5-base"]; - - it( - models[0], - async () => { - let generator = await pipeline("text2text-generation", models[0]); - let text = "Premise: At my age you will probably have learnt one lesson. " + "Hypothesis: It's not certain how many lessons you'll learn by your thirties. " + "Does the premise entail the hypothesis?"; - - { - let outputs = await generator(text, { - top_k: 0, - do_sample: false, - }); - expect(outputs).toHaveLength(1); - expect(outputs[0].generated_text.length).toBeGreaterThan(1); - } - - await generator.dispose(); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - models[1], - async () => { - let generator = await pipeline("text2text-generation", models[1]); - let text = ` + MAX_TEST_EXECUTION_TIME, + ); + + it( + models[1], + async () => { + let translator = await pipeline("translation", models[1]); + let texts = ["Hello world!", "I like to walk my dog."]; + + // single + { + let translation = await translator(texts[0], { + src_lang: "eng_Latn", + tgt_lang: "arb_Arab", + }); + + let expected = [{ translation_text: "مرحباً، يا عالم!" }]; + + compare(translation, expected); + } + + // single + back-translation + { + let translation1 = await translator(texts[1], { + // src_lang: 'eng_Latn', + tgt_lang: "ell_Grek", + }); + let translation2 = await translator(translation1[0].translation_text, { + src_lang: "ell_Grek", + tgt_lang: "eng_Latn", + }); + + let expected = [{ translation_text: "Μου αρέσει να περπατάω το σκυλί μου." }]; + + compare(translation1, expected); + + let expectedBack = [{ translation_text: texts[1] }]; + compare(translation2, expectedBack); + } + + await translator.dispose(); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("Text-to-text generation", () => { + // List all models which will be tested + const models = ["Xenova/flan-t5-small", "Xenova/flan-t5-base"]; + + it( + models[0], + async () => { + let generator = await pipeline("text2text-generation", models[0]); + let text = "Premise: At my age you will probably have learnt one lesson. " + "Hypothesis: It's not certain how many lessons you'll learn by your thirties. " + "Does the premise entail the hypothesis?"; + + { + let outputs = await generator(text, { + top_k: 0, + do_sample: false, + }); + expect(outputs).toHaveLength(1); + expect(outputs[0].generated_text.length).toBeGreaterThan(1); + } + + await generator.dispose(); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + models[1], + async () => { + let generator = await pipeline("text2text-generation", models[1]); + let text = ` Q: Roger has 5 tennis balls. He buys 2 more cans of tennis balls. Each can has 3 tennis balls. How many tennis balls does he have now? A: Roger started with 5 balls. 2 cans of 3 tennis balls each is 6 tennis balls. @@ -562,1023 +646,1033 @@ xdescribe("Pipelines", () => { Q: A juggler can juggle 16 balls. Half of the balls are golf balls, and half of the golf balls are blue. How many blue golf balls are there?`; - // single - { - let outputs = await generator(text, { - top_k: 0, - do_sample: false, - }); - expect(outputs).toHaveLength(1); - expect(outputs[0].generated_text.length).toBeGreaterThan(10); - } - await generator.dispose(); - }, - MAX_TEST_EXECUTION_TIME, - ); - }); - - describe("Text generation", () => { - // List all models which will be tested - const models = ["Xenova/distilgpt2", "Xenova/codegen-350M-mono"]; - - it( - models[0], - async () => { - let generator = await pipeline("text-generation", models[0]); - let texts = ["Once upon a time, there was a", "I enjoy walking with my cute dog"]; - - // single - { - let output = await generator(texts[0], { - max_new_tokens: 10, - top_k: 0, - do_sample: false, - }); - expect(output).toHaveLength(1); - expect(output[0].generated_text.length).toBeGreaterThan(texts[0].length); - } - - // single + `num_beams` + `num_return_sequences` - { - let output = await generator(texts[0], { - max_new_tokens: 10, - num_beams: 2, - num_return_sequences: 2, - top_k: 0, - do_sample: false, - }); - expect(output).toHaveLength(2); - expect(output[0].generated_text.length).toBeGreaterThan(texts[0].length); - expect(output[1].generated_text.length).toBeGreaterThan(texts[0].length); - } - - // batched + `num_beams` + `num_return_sequences` - { - let output = await generator(texts, { - max_new_tokens: 10, - num_beams: 2, - num_return_sequences: 2, - top_k: 0, - do_sample: false, - }); - expect(output).toHaveLength(2); - expect(output[0]).toHaveLength(2); - expect(output[0][0].generated_text.length).toBeGreaterThan(texts[0].length); - expect(output[0][1].generated_text.length).toBeGreaterThan(texts[0].length); - expect(output[1]).toHaveLength(2); - expect(output[1][0].generated_text.length).toBeGreaterThan(texts[1].length); - expect(output[1][1].generated_text.length).toBeGreaterThan(texts[1].length); - } - - await generator.dispose(); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - models[1], - async () => { - let generator = await pipeline("text-generation", models[1]); - let code = "def fib(n):"; - - // single + `added_tokens` - { - let output = await generator(code, { - max_new_tokens: 45, - top_k: 0, - do_sample: false, - }); - expect(output).toHaveLength(1); - expect(output[0].generated_text.length).toBeGreaterThan(code.length); - } - await generator.dispose(); - }, - MAX_TEST_EXECUTION_TIME, - ); - }); - - describe("Feature extraction", () => { - // List all models which will be tested - const models = ["Xenova/all-MiniLM-L6-v2"]; - - it( - models[0], - async () => { - let extractor = await pipeline("feature-extraction", models[0]); - - // Provide sentences - let sentences = ["This framework generates embeddings for each input sentence", "Sentences are passed as a list of string.", "The quick brown fox jumps over the lazy dog."]; - - // Without pooling or normalization - { - let output = await extractor(sentences); - expect(output.dims).toHaveLength(3); - } - - // With pooling and normalization + compare features - { - let output = await extractor(sentences, { pooling: "mean", normalize: true }); - expect(output.dims).toHaveLength(2); - - // Convert Tensor to JS list - output = output.tolist(); - - let pairwiseScores = [ - [output[0], output[1]], - [output[0], output[2]], - [output[1], output[2]], - ].map((x) => cos_sim(...x)); - - let expected = [0.502872309810269, 0.11088411026413121, 0.09602621986931259]; - compare(pairwiseScores, expected); - } - await extractor.dispose(); - }, - MAX_TEST_EXECUTION_TIME, - ); - }); - - describe("Speech-to-text generation", () => { - // List all models which will be tested - const models = [ - // whisper - "Xenova/whisper-tiny.en", // English-only - "Xenova/whisper-small", // Multilingual - ["Xenova/whisper-tiny.en", "output_attentions"], // English-only + `output_attentions` - ["Xenova/whisper-small", "output_attentions"], // Multilingual + `output_attentions` - - // wav2vec2 - "jonatasgrosman/wav2vec2-large-xlsr-53-english", - ]; - - it( - models[0], - async () => { - let transcriber = await pipeline("automatic-speech-recognition", models[0]); - - let url = "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/jfk.wav"; - let audioData = await loadAudio(url); - - { - // Transcribe English - let output = await transcriber(audioData); - expect(output.text.length).toBeGreaterThan(50); - // { text: " And so my fellow Americans ask not what your country can do for you, ask what you can do for your country." } - } - - { - // Transcribe English w/ timestamps. - let output = await transcriber(audioData, { return_timestamps: true }); - expect(output.text.length).toBeGreaterThan(50); - expect(output.chunks.length).toBeGreaterThan(0); - // { - // text: " And so my fellow Americans ask not what your country can do for you, ask what you can do for your country." - // chunks: [ - // { timestamp: [0, 8], text: " And so my fellow Americans ask not what your country can do for you" } - // { timestamp: [8, 11], text: " ask what you can do for your country." } - // ] - // } - } - await transcriber.dispose(); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - models[1], - async () => { - let transcriber = await pipeline("automatic-speech-recognition", models[1]); - - let url = "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/french-audio.wav"; - let audioData = await loadAudio(url); - - { - // Transcribe French - let output = await transcriber(audioData, { language: "french", task: "transcribe" }); - expect(output.text.length).toBeGreaterThan(20); - // { text: " J'adore, j'aime, je n'aime pas, je déteste." } - } - - { - // Translate French to English. - let output = await transcriber(audioData, { language: "french", task: "translate" }); - expect(output.text.length).toBeGreaterThan(20); - // { text: " I love, I like, I don't like, I hate." } - } - await transcriber.dispose(); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - models[2].join(" + "), - async () => { - let transcriber = await pipeline("automatic-speech-recognition", m(models[2][0]), { - revision: models[2][1], - quantized: false, - }); - - let url = "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/jfk.wav"; - let audioData = await loadAudio(url); - - { - // Transcribe English w/ word-level timestamps. - let output = await transcriber(audioData, { return_timestamps: "word" }); - const target = { - text: " And so my fellow Americans ask not what your country can do for you ask what you can do for your country.", - chunks: [ - { text: " And", timestamp: [0, 0.78] }, - { text: " so", timestamp: [0.78, 1.06] }, - { text: " my", timestamp: [1.06, 1.46] }, - { text: " fellow", timestamp: [1.46, 1.76] }, - { text: " Americans", timestamp: [1.76, 2.22] }, - { text: " ask", timestamp: [2.22, 3.88] }, - { text: " not", timestamp: [3.88, 4.52] }, - { text: " what", timestamp: [4.52, 5.68] }, - { text: " your", timestamp: [5.68, 6] }, - { text: " country", timestamp: [6, 6.36] }, - { text: " can", timestamp: [6.36, 6.76] }, - { text: " do", timestamp: [6.76, 7.02] }, - { text: " for", timestamp: [7.02, 7.24] }, - { text: " you", timestamp: [7.24, 8.02] }, - { text: " ask", timestamp: [8.28, 8.66] }, - { text: " what", timestamp: [8.66, 8.94] }, - { text: " you", timestamp: [8.94, 9.28] }, - { text: " can", timestamp: [9.28, 9.5] }, - { text: " do", timestamp: [9.5, 9.72] }, - { text: " for", timestamp: [9.72, 9.92] }, - { text: " your", timestamp: [9.92, 10.22] }, - { text: " country.", timestamp: [10.22, 13.36] }, - ], - }; - - compare(output, target); - } - - await transcriber.dispose(); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - models[3].join(" + "), - async () => { - let transcriber = await pipeline("automatic-speech-recognition", m(models[3][0]), { - revision: models[3][1], - }); - - let url = "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/japanese-audio.wav"; - let audioData = await loadAudio(url); - - { - // Transcribe Japanese w/ word-level timestamps. - let output = await transcriber(audioData, { return_timestamps: "word", language: "japanese", task: "transcribe" }); - const target = { - text: "モリナガの美味しい牛乳は濃い青色に牛乳瓶を払ったゼザインのパック牛乳である。", - chunks: [ - { text: "モ", timestamp: [0, 0.56] }, - { text: "リ", timestamp: [0.56, 0.64] }, - { text: "ナ", timestamp: [0.64, 0.8] }, - { text: "ガ", timestamp: [0.8, 0.88] }, - { text: "の", timestamp: [0.88, 1.04] }, - { text: "美味", timestamp: [1.04, 1.22] }, - { text: "しい", timestamp: [1.22, 1.46] }, - { text: "牛", timestamp: [1.46, 1.76] }, - { text: "乳", timestamp: [1.76, 1.94] }, - { text: "は", timestamp: [1.94, 2.14] }, - { text: "濃", timestamp: [2.14, 2.34] }, - { text: "い", timestamp: [2.34, 2.48] }, - { text: "青", timestamp: [2.48, 2.62] }, - { text: "色", timestamp: [2.62, 2.84] }, - { text: "に", timestamp: [2.84, 3] }, - { text: "牛", timestamp: [3, 3.22] }, - { text: "乳", timestamp: [3.22, 3.42] }, - { text: "瓶", timestamp: [3.42, 3.58] }, - { text: "を", timestamp: [3.58, 3.82] }, - { text: "払", timestamp: [3.82, 4] }, - { text: "った", timestamp: [4, 4.32] }, - { text: "ゼ", timestamp: [4.32, 4.56] }, - { text: "ザ", timestamp: [4.56, 4.6] }, - { text: "イ", timestamp: [4.6, 4.74] }, - { text: "ン", timestamp: [4.74, 4.8] }, - { text: "の", timestamp: [4.8, 4.94] }, - { text: "パ", timestamp: [4.94, 5.12] }, - { text: "ック", timestamp: [5.12, 5.26] }, - { text: "牛", timestamp: [5.26, 5.52] }, - { text: "乳", timestamp: [5.52, 5.72] }, - { text: "で", timestamp: [5.72, 5.86] }, - { text: "ある。", timestamp: [5.86, 6.62] }, - ], - }; - - compare(output, target); - } - - await transcriber.dispose(); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - models[4], - async () => { - let transcriber = await pipeline("automatic-speech-recognition", m(models[4])); - - let url = "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/jfk.wav"; - let audioData = await loadAudio(url); - - { - // Transcribe - let output = await transcriber(audioData); - expect(output.text.length).toBeGreaterThan(50); - // { text: "and so my fellow america ask not what your country can do for you ask what you can do for your country" } - } - - await transcriber.dispose(); - }, - MAX_TEST_EXECUTION_TIME, - ); - }); - - describe("Text-to-speech generation", () => { - // List all models which will be tested - const models = ["Xenova/speecht5_tts", "Xenova/mms-tts-fra"]; - - it( - models[0], - async () => { - let synthesizer = await pipeline("text-to-speech", models[0], { - // NOTE: Although the quantized version produces incoherent results, - // it it is okay to use for testing. - // quantized: false, - }); - - let speaker_embeddings = "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/speaker_embeddings.bin"; - - { - // Generate English speech - let output = await synthesizer("Hello, my dog is cute", { speaker_embeddings }); - expect(output.audio.length).toBeGreaterThan(0); - expect(output.sampling_rate).toEqual(16000); - } - - await synthesizer.dispose(); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - models[1], - async () => { - let synthesizer = await pipeline("text-to-speech", models[1]); - - { - // Generate French speech - let output = await synthesizer("Bonjour"); - expect(output.audio.length).toBeGreaterThan(0); - expect(output.sampling_rate).toEqual(16000); - } - - await synthesizer.dispose(); - }, - MAX_TEST_EXECUTION_TIME, - ); - }); - - describe("Audio classification", () => { - // List all models which will be tested - const models = ["Xenova/wav2vec2-large-xlsr-53-gender-recognition-librispeech"]; - - it( - models[0], - async () => { - let classifier = await pipeline("audio-classification", models[0]); - - let url = "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/jfk.wav"; - let audioData = await loadAudio(url); - - { - // Classify audio - let outputs = await classifier(audioData); - - let expected = [ - { score: 0.997512936592102, label: "male" }, - { score: 0.0024870133493095636, label: "female" }, - ]; - compare(outputs, expected); - } - - await classifier.dispose(); - }, - MAX_TEST_EXECUTION_TIME, - ); - }); - - describe("Image-to-text", () => { - // List all models which will be tested - const models = ["Xenova/vit-gpt2-image-captioning"]; - - it( - models[0], - async () => { - let captioner = await pipeline("image-to-text", models[0]); - - let url = "https://huggingface.co/datasets/mishig/sample_images/resolve/main/savanna.jpg"; - let urls = ["https://huggingface.co/datasets/mishig/sample_images/resolve/main/football-match.jpg", "https://huggingface.co/datasets/mishig/sample_images/resolve/main/airport.jpg"]; - - // single - { - let output = await captioner(url, { - top_k: 0, - do_sample: false, - }); - // let expected = [ - // { "generated_text": "a herd of giraffes and zebras grazing in a field" } - // ] - - expect(output).toHaveLength(1); - expect(output[0].generated_text.length).toBeGreaterThan(10); - } - - // single + generation options - { - let output = await captioner(url, { - max_new_tokens: 20, - num_beams: 2, - num_return_sequences: 2, - top_k: 0, - do_sample: false, - }); - // let expected = [ - // { "generated_text": "a herd of giraffes and zebras grazing in a field" }, - // { "generated_text": "a herd of giraffes and zebras in a grassy field" } - // ] - - expect(output).toHaveLength(2); - expect(output[0].generated_text.length).toBeGreaterThan(10); - expect(output[1].generated_text.length).toBeGreaterThan(10); - } - - // batched - { - let output = await captioner(urls, { - top_k: 0, - do_sample: false, - }); - // let expected = [ - // [{ "generated_text": "two men are kicking a soccer ball in a soccer game" }], - // [{ "generated_text": "a plane on the tarmac with a passenger bus" }] - // ] - - expect(output).toHaveLength(2); - expect(output[0]).toHaveLength(1); - expect(output[0][0].generated_text.length).toBeGreaterThan(10); - expect(output[1]).toHaveLength(1); - expect(output[1][0].generated_text.length).toBeGreaterThan(10); - } - - // batched + generation options - { - let output = await captioner(urls, { - max_new_tokens: 20, - num_beams: 2, - num_return_sequences: 2, - top_k: 0, - do_sample: false, - }); - // let expected = [ - // [ - // { "generated_text": "two men are kicking a soccer ball on a field" }, - // { "generated_text": "two men are kicking a soccer ball in a soccer game" } - // ], [ - // { "generated_text": "a plane on a tarmac with a group of buses" }, - // { "generated_text": "a plane on a tarmac with a group of people on the ground" } - // ] - // ]; - - expect(output).toHaveLength(2); - expect(output[0]).toHaveLength(2); - expect(output[0][0].generated_text.length).toBeGreaterThan(10); - expect(output[0][1].generated_text.length).toBeGreaterThan(10); - expect(output[1]).toHaveLength(2); - expect(output[1][0].generated_text.length).toBeGreaterThan(10); - expect(output[1][1].generated_text.length).toBeGreaterThan(10); - } - await captioner.dispose(); - }, - MAX_TEST_EXECUTION_TIME, - ); - }); - - describe("Image classification", () => { - // List all models which will be tested - const models = ["Xenova/vit-base-patch16-224"]; - - it( - models[0], - async () => { - let classifier = await pipeline("image-classification", models[0]); - - let url = "https://huggingface.co/datasets/mishig/sample_images/resolve/main/tiger.jpg"; - let urls = ["https://huggingface.co/datasets/mishig/sample_images/resolve/main/palace.jpg", "https://huggingface.co/datasets/mishig/sample_images/resolve/main/teapot.jpg"]; - - // single - { - let outputs = await classifier(url); - - let expected = [{ label: "tiger, Panthera tigris", score: 0.607988178730011 }]; - - compare(outputs, expected, 0.2); - } - - // single + topk - { - let outputs = await classifier(url, { - topk: 2, - }); - - let expected = [ - { label: "tiger, Panthera tigris", score: 0.607988178730011 }, - { label: "tiger cat", score: 0.3877776563167572 }, - ]; - - compare(outputs, expected, 0.2); - } - - // batched - { - let outputs = await classifier(urls); - - let expected = [ - { label: "palace", score: 0.9986862540245056 }, - { label: "teapot", score: 0.987880527973175 }, - ]; - - compare(outputs, expected); - } - - // batched + topk - { - let outputs = await classifier(urls, { - topk: 2, - }); - - let expected = [ - [ - { label: "palace", score: 0.9986862540245056 }, - { label: "castle", score: 0.00037879671435803175 }, - ], - [ - { label: "teapot", score: 0.987880527973175 }, - { label: "coffeepot", score: 0.006591461598873138 }, - ], - ]; - - compare(outputs, expected); - } - - await classifier.dispose(); - }, - MAX_TEST_EXECUTION_TIME, - ); - }); - - describe("Image segmentation", () => { - // List all models which will be tested - const models = ["Xenova/detr-resnet-50-panoptic", "Xenova/segformer_b2_clothes"]; - - it( - models[0], - async () => { - let segmenter = await pipeline("image-segmentation", models[0], { - // Quantized version of model produces incorrect results - quantized: false, - }); - let img = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/coco_sample.png"; - - // single - { - let outputs = await segmenter(img); - - let expected = [ - { score: 0.9916538596153259, label: "cat", mask: 58998 }, - { score: 0.9987397789955139, label: "remote", mask: 4164 }, - { score: 0.9994599223136902, label: "remote", mask: 2275 }, - { score: 0.9730215072631836, label: "couch", mask: 176980 }, - { score: 0.9993911385536194, label: "cat", mask: 52670 }, - ]; - - let outputLabels = outputs.map((x) => x.label); - let expectedLabels = expected.map((x) => x.label); - - expect(outputLabels).toHaveLength(expectedLabels.length); - expect(outputLabels.sort()).toEqual(expectedLabels.sort()); - } - - await segmenter.dispose(); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - models[1], - async () => { - let segmenter = await pipeline("image-segmentation", models[1]); - let img = "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/young-man-standing-and-leaning-on-car.jpg"; - - // single - { - let outputs = await segmenter(img); - - let expected = [{ label: "Background" }, { label: "Hair" }, { label: "Upper-clothes" }, { label: "Pants" }, { label: "Left-shoe" }, { label: "Right-shoe" }, { label: "Face" }, { label: "Left-leg" }, { label: "Right-leg" }, { label: "Left-arm" }, { label: "Right-arm" }]; - - let outputLabels = outputs.map((x) => x.label); - let expectedLabels = expected.map((x) => x.label); - - expect(outputLabels).toHaveLength(expectedLabels.length); - expect(outputLabels.sort()).toEqual(expectedLabels.sort()); - - // check that all scores are null, and masks have correct dimensions - for (let output of outputs) { - expect(output.score).toBeNull(); - expect(output.mask.width).toEqual(970); - expect(output.mask.height).toEqual(1455); - expect(output.mask.channels).toEqual(1); - } - } - - await segmenter.dispose(); - }, - MAX_TEST_EXECUTION_TIME, - ); - }); - - describe("Zero-shot image classification", () => { - // List all models which will be tested - const models = ["Xenova/clip-vit-base-patch32"]; - - it( - models[0], - async () => { - let classifier = await pipeline("zero-shot-image-classification", models[0]); - - let url = "https://huggingface.co/datasets/mishig/sample_images/resolve/main/football-match.jpg"; - let urls = ["https://huggingface.co/datasets/mishig/sample_images/resolve/main/football-match.jpg", "https://huggingface.co/datasets/mishig/sample_images/resolve/main/airport.jpg", "https://huggingface.co/datasets/mishig/sample_images/resolve/main/savanna.jpg"]; - - let classes = ["football", "airport", "animals"]; - - // single - { - let output = await classifier(url, classes); - - let expected = [ - { score: 0.9719080924987793, label: "football" }, - { score: 0.022564826533198357, label: "animals" }, - { score: 0.005527070723474026, label: "airport" }, - ]; - compare(output, expected, 0.1); - } - - // batched - { - let output = await classifier(urls, classes); - - let expected = [ - [ - { score: 0.9712504148483276, label: "football" }, - { score: 0.022469401359558105, label: "animals" }, - { score: 0.006280169822275639, label: "airport" }, - ], - [ - { score: 0.997433602809906, label: "airport" }, - { score: 0.0016500800848007202, label: "animals" }, - { score: 0.0009163151844404638, label: "football" }, - ], - [ - { score: 0.9851226806640625, label: "animals" }, - { score: 0.007516484707593918, label: "football" }, - { score: 0.007360846735537052, label: "airport" }, - ], - ]; - compare(output, expected, 0.1); - } - await classifier.dispose(); - }, - MAX_TEST_EXECUTION_TIME, - ); - }); - - describe("Object detection", () => { - // List all models which will be tested - const models = ["Xenova/detr-resnet-50"]; - - it( - models[0], - async () => { - let detector = await pipeline("object-detection", models[0]); - - // TODO add batched test cases when supported - let url = "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/cats.jpg"; - let urls = ["https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/savanna.jpg"]; - - // single + threshold - { - let output = await detector(url, { - threshold: 0.9, - }); - - // let expected = [ - // { - // "score": 0.9977124929428101, - // "label": "remote", - // "box": { "xmin": 41, "ymin": 70, "xmax": 176, "ymax": 118 } - // }, - // { - // "score": 0.9984639883041382, - // "label": "remote", - // "box": { "xmin": 332, "ymin": 73, "xmax": 369, "ymax": 188 } - // }, - // { - // "score": 0.9964856505393982, - // "label": "couch", - // "box": { "xmin": 0, "ymin": 1, "xmax": 639, "ymax": 474 } - // }, - // { - // "score": 0.9988334774971008, - // "label": "cat", - // "box": { "xmin": 11, "ymin": 51, "xmax": 314, "ymax": 472 } - // }, - // { - // "score": 0.9982513785362244, - // "label": "cat", - // "box": { "xmin": 345, "ymin": 22, "xmax": 640, "ymax": 371 } - // } - // ] - - expect(output.length).toBeGreaterThan(0); - for (let cls of output) { - expect(typeof cls.score).toBe("number"); - expect(typeof cls.label).toBe("string"); - for (let key of ["xmin", "ymin", "xmax", "ymax"]) { - expect(typeof cls.box[key]).toBe("number"); - } - } - } - - // batched + threshold + percentage - { - let output = await detector(urls, { - threshold: 0.9, - percentage: true, - }); - // let expected = [[ - // { - // score: 0.9991137385368347, - // label: 'zebra', - // box: { xmin: 0.65165576338768, ymin: 0.685152679681778, xmax: 0.723189502954483, ymax: 0.8801506459712982 } - // }, - // { - // score: 0.998811662197113, - // label: 'zebra', - // box: { xmin: 0.20797613263130188, ymin: 0.6543092578649521, xmax: 0.4147692620754242, ymax: 0.9040975719690323 } - // }, - // { - // score: 0.9707837104797363, - // label: 'giraffe', - // box: { xmin: 0.02498096227645874, ymin: 0.40549489855766296, xmax: 0.38669759035110474, ymax: 0.7895723879337311 } - // }, - // { - // score: 0.9984336495399475, - // label: 'zebra', - // box: { xmin: 0.3540637195110321, ymin: 0.6370827257633209, xmax: 0.5765090882778168, ymax: 0.8480959832668304 } - // }, - // { - // score: 0.9986463785171509, - // label: 'giraffe', - // box: { xmin: 0.6763969212770462, ymin: 0.25748637318611145, xmax: 0.974339172244072, ymax: 0.8684568107128143 } - // } - // ]] - - expect(output).toHaveLength(urls.length); // Same number of inputs as outputs - - for (let i = 0; i < output.length; ++i) { - expect(output[i].length).toBeGreaterThan(0); - for (let cls of output[i]) { - expect(typeof cls.score).toBe("number"); - expect(typeof cls.label).toBe("string"); - for (let key of ["xmin", "ymin", "xmax", "ymax"]) { - expect(typeof cls.box[key]).toBe("number"); - } - } - } - } - - await detector.dispose(); - }, - MAX_TEST_EXECUTION_TIME, - ); - }); - - describe("Zero-shot object detection", () => { - // List all models which will be tested - const models = ["Xenova/owlvit-base-patch32"]; - - it( - models[0], - async () => { - let detector = await pipeline("zero-shot-object-detection", models[0]); - - // single (default) - { - let url = "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/astronaut.png"; - let candidate_labels = ["human face", "rocket", "helmet", "american flag"]; - - let output = await detector(url, candidate_labels); - - // let expected = [ - // { - // score: 0.24392342567443848, - // label: 'human face', - // box: { xmin: 180, ymin: 67, xmax: 274, ymax: 175 } - // }, - // { - // score: 0.15129457414150238, - // label: 'american flag', - // box: { xmin: 0, ymin: 4, xmax: 106, ymax: 513 } - // }, - // { - // score: 0.13649864494800568, - // label: 'helmet', - // box: { xmin: 277, ymin: 337, xmax: 511, ymax: 511 } - // }, - // { - // score: 0.10262022167444229, - // label: 'rocket', - // box: { xmin: 352, ymin: -1, xmax: 463, ymax: 287 } - // } - // ] - - expect(output.length).toBeGreaterThan(0); - for (let cls of output) { - expect(typeof cls.score).toBe("number"); - expect(typeof cls.label).toBe("string"); - for (let key of ["xmin", "ymin", "xmax", "ymax"]) { - expect(typeof cls.box[key]).toBe("number"); - } - } - } - - // topk + threshold + percentage - { - let url = "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/beach.png"; - let candidate_labels = ["hat", "book", "sunglasses", "camera"]; - - let output = await detector(url, candidate_labels, { - topk: 4, - threshold: 0.05, - percentage: true, - }); - - // let expected = [ - // { - // score: 0.1606510728597641, - // label: 'sunglasses', - // box: { xmin: 347, ymin: 229, xmax: 429, ymax: 264 } - // }, - // { - // score: 0.08935828506946564, - // label: 'hat', - // box: { xmin: 38, ymin: 174, xmax: 258, ymax: 364 } - // }, - // { - // score: 0.08530698716640472, - // label: 'camera', - // box: { xmin: 187, ymin: 350, xmax: 260, ymax: 411 } - // }, - // { - // score: 0.08349756896495819, - // label: 'book', - // box: { xmin: 261, ymin: 280, xmax: 494, ymax: 425 } - // } - // ] - - expect(output.length).toBeGreaterThan(0); - for (let cls of output) { - expect(typeof cls.score).toBe("number"); - expect(typeof cls.label).toBe("string"); - for (let key of ["xmin", "ymin", "xmax", "ymax"]) { - expect(typeof cls.box[key]).toBe("number"); - } - } - } - - await detector.dispose(); - }, - MAX_TEST_EXECUTION_TIME, - ); - }); - - describe("Image-to-image", () => { - // List all models which will be tested - const models = ["Xenova/swin2SR-classical-sr-x2-64"]; - - it( - models[0], - async () => { - let upscaler = await pipeline("image-to-image", models[0]); - - // Input is 3x3 => padded to 8x8 => upscaled to 16x16 - let url = "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/pattern_3x3.png"; - - // single - { - let outputs = await upscaler(url); - expect(outputs.width).toEqual(16); - expect(outputs.height).toEqual(16); - expect(outputs.channels).toEqual(3); - expect(outputs.data).toHaveLength(768); - } - - // batched - { - let outputs = await upscaler([url, url]); - expect(outputs).toHaveLength(2); - for (let output of outputs) { - expect(output.width).toEqual(16); - expect(output.height).toEqual(16); - expect(output.channels).toEqual(3); - expect(output.data).toHaveLength(768); - } - } - - await upscaler.dispose(); - }, - MAX_TEST_EXECUTION_TIME, - ); - }); - - describe("Depth estimation", () => { - // List all models which will be tested - const models = ["Xenova/dpt-hybrid-midas"]; - - it( - models[0], - async () => { - let depth_estimator = await pipeline("depth-estimation", models[0]); - - let url = "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/cats.jpg"; - - // single - { - let { predicted_depth, depth } = await depth_estimator(url); - compare(predicted_depth.dims, [384, 384]); - expect(depth.width).toEqual(640); - expect(depth.height).toEqual(480); - expect(depth.channels).toEqual(1); - expect(depth.data).toHaveLength(307200); - } - - // batched - { - let outputs = await depth_estimator([url, url]); - expect(outputs).toHaveLength(2); - for (let output of outputs) { - let { predicted_depth, depth } = output; - compare(predicted_depth.dims, [384, 384]); - expect(depth.width).toEqual(640); - expect(depth.height).toEqual(480); - expect(depth.channels).toEqual(1); - expect(depth.data).toHaveLength(307200); - } - } - - await depth_estimator.dispose(); - }, - MAX_TEST_EXECUTION_TIME, - ); - }); - - describe("Document question answering", () => { - // List all models which will be tested - const models = ["Xenova/donut-base-finetuned-docvqa"]; - const image = "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/invoice.png"; - const question = "What is the invoice number?"; - - it( - models[0], - async () => { - let qa_pipeline = await pipeline("document-question-answering", models[0]); - - // basic - { - let output = await qa_pipeline(image, question); - let expected = [{ answer: "us-001" }]; - compare(output, expected); - } - - await qa_pipeline.dispose(); - }, - MAX_TEST_EXECUTION_TIME, - ); - }); + // single + { + let outputs = await generator(text, { + top_k: 0, + do_sample: false, + }); + expect(outputs).toHaveLength(1); + expect(outputs[0].generated_text.length).toBeGreaterThan(10); + } + await generator.dispose(); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("Text generation", () => { + // List all models which will be tested + const models = ["Xenova/distilgpt2", "Xenova/codegen-350M-mono"]; + + it( + models[0], + async () => { + let generator = await pipeline("text-generation", models[0]); + let texts = ["Once upon a time, there was a", "I enjoy walking with my cute dog"]; + + // single + { + let output = await generator(texts[0], { + max_new_tokens: 10, + top_k: 0, + do_sample: false, + }); + expect(output).toHaveLength(1); + expect(output[0].generated_text.length).toBeGreaterThan(texts[0].length); + } + + // single + `num_beams` + `num_return_sequences` + { + let output = await generator(texts[0], { + max_new_tokens: 10, + num_beams: 2, + num_return_sequences: 2, + top_k: 0, + do_sample: false, + }); + expect(output).toHaveLength(2); + expect(output[0].generated_text.length).toBeGreaterThan(texts[0].length); + expect(output[1].generated_text.length).toBeGreaterThan(texts[0].length); + } + + // batched + `num_beams` + `num_return_sequences` + { + let output = await generator(texts, { + max_new_tokens: 10, + num_beams: 2, + num_return_sequences: 2, + top_k: 0, + do_sample: false, + }); + expect(output).toHaveLength(2); + expect(output[0]).toHaveLength(2); + expect(output[0][0].generated_text.length).toBeGreaterThan(texts[0].length); + expect(output[0][1].generated_text.length).toBeGreaterThan(texts[0].length); + expect(output[1]).toHaveLength(2); + expect(output[1][0].generated_text.length).toBeGreaterThan(texts[1].length); + expect(output[1][1].generated_text.length).toBeGreaterThan(texts[1].length); + } + + await generator.dispose(); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + models[1], + async () => { + let generator = await pipeline("text-generation", models[1]); + let code = "def fib(n):"; + + // single + `added_tokens` + { + let output = await generator(code, { + max_new_tokens: 45, + top_k: 0, + do_sample: false, + }); + expect(output).toHaveLength(1); + expect(output[0].generated_text.length).toBeGreaterThan(code.length); + } + await generator.dispose(); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("Feature extraction", () => { + // List all models which will be tested + const models = ["Xenova/all-MiniLM-L6-v2"]; + + it( + models[0], + async () => { + let extractor = await pipeline("feature-extraction", models[0]); + + // Provide sentences + let sentences = ["This framework generates embeddings for each input sentence", "Sentences are passed as a list of string.", "The quick brown fox jumps over the lazy dog."]; + + // Without pooling or normalization + { + let output = await extractor(sentences); + expect(output.dims).toHaveLength(3); + } + + // With pooling and normalization + compare features + { + let output = await extractor(sentences, { pooling: "mean", normalize: true }); + expect(output.dims).toHaveLength(2); + + // Convert Tensor to JS list + output = output.tolist(); + + let pairwiseScores = [ + [output[0], output[1]], + [output[0], output[2]], + [output[1], output[2]], + ].map((x) => cos_sim(...x)); + + let expected = [0.502872309810269, 0.11088411026413121, 0.09602621986931259]; + compare(pairwiseScores, expected); + } + await extractor.dispose(); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("Speech-to-text generation", () => { + // List all models which will be tested + const models = [ + // whisper + "Xenova/whisper-tiny.en", // English-only + "Xenova/whisper-small", // Multilingual + ["Xenova/whisper-tiny.en", "output_attentions"], // English-only + `output_attentions` + ["Xenova/whisper-small", "output_attentions"], // Multilingual + `output_attentions` + + // wav2vec2 + "jonatasgrosman/wav2vec2-large-xlsr-53-english", + ]; + + it( + models[0], + async () => { + let transcriber = await pipeline("automatic-speech-recognition", models[0]); + + let url = "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/jfk.wav"; + let audioData = await loadAudio(url); + + { + // Transcribe English + let output = await transcriber(audioData); + expect(output.text.length).toBeGreaterThan(50); + // { text: " And so my fellow Americans ask not what your country can do for you, ask what you can do for your country." } + } + + { + // Transcribe English w/ timestamps. + let output = await transcriber(audioData, { return_timestamps: true }); + expect(output.text.length).toBeGreaterThan(50); + expect(output.chunks.length).toBeGreaterThan(0); + // { + // text: " And so my fellow Americans ask not what your country can do for you, ask what you can do for your country." + // chunks: [ + // { timestamp: [0, 8], text: " And so my fellow Americans ask not what your country can do for you" } + // { timestamp: [8, 11], text: " ask what you can do for your country." } + // ] + // } + } + await transcriber.dispose(); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + models[1], + async () => { + let transcriber = await pipeline("automatic-speech-recognition", models[1]); + + let url = "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/french-audio.wav"; + let audioData = await loadAudio(url); + + { + // Transcribe French + let output = await transcriber(audioData, { + language: "french", + task: "transcribe", + }); + expect(output.text.length).toBeGreaterThan(20); + // { text: " J'adore, j'aime, je n'aime pas, je déteste." } + } + + { + // Translate French to English. + let output = await transcriber(audioData, { + language: "french", + task: "translate", + }); + expect(output.text.length).toBeGreaterThan(20); + // { text: " I love, I like, I don't like, I hate." } + } + await transcriber.dispose(); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + models[2].join(" + "), + async () => { + let transcriber = await pipeline("automatic-speech-recognition", m(models[2][0]), { + revision: models[2][1], + quantized: false, + }); + + let url = "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/jfk.wav"; + let audioData = await loadAudio(url); + + { + // Transcribe English w/ word-level timestamps. + let output = await transcriber(audioData, { return_timestamps: "word" }); + const target = { + text: " And so my fellow Americans ask not what your country can do for you ask what you can do for your country.", + chunks: [ + { text: " And", timestamp: [0, 0.78] }, + { text: " so", timestamp: [0.78, 1.06] }, + { text: " my", timestamp: [1.06, 1.46] }, + { text: " fellow", timestamp: [1.46, 1.76] }, + { text: " Americans", timestamp: [1.76, 2.22] }, + { text: " ask", timestamp: [2.22, 3.88] }, + { text: " not", timestamp: [3.88, 4.52] }, + { text: " what", timestamp: [4.52, 5.68] }, + { text: " your", timestamp: [5.68, 6] }, + { text: " country", timestamp: [6, 6.36] }, + { text: " can", timestamp: [6.36, 6.76] }, + { text: " do", timestamp: [6.76, 7.02] }, + { text: " for", timestamp: [7.02, 7.24] }, + { text: " you", timestamp: [7.24, 8.02] }, + { text: " ask", timestamp: [8.28, 8.66] }, + { text: " what", timestamp: [8.66, 8.94] }, + { text: " you", timestamp: [8.94, 9.28] }, + { text: " can", timestamp: [9.28, 9.5] }, + { text: " do", timestamp: [9.5, 9.72] }, + { text: " for", timestamp: [9.72, 9.92] }, + { text: " your", timestamp: [9.92, 10.22] }, + { text: " country.", timestamp: [10.22, 13.36] }, + ], + }; + + compare(output, target); + } + + await transcriber.dispose(); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + models[3].join(" + "), + async () => { + let transcriber = await pipeline("automatic-speech-recognition", m(models[3][0]), { + revision: models[3][1], + }); + + let url = "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/japanese-audio.wav"; + let audioData = await loadAudio(url); + + { + // Transcribe Japanese w/ word-level timestamps. + let output = await transcriber(audioData, { + return_timestamps: "word", + language: "japanese", + task: "transcribe", + }); + const target = { + text: "モリナガの美味しい牛乳は濃い青色に牛乳瓶を払ったゼザインのパック牛乳である。", + chunks: [ + { text: "モ", timestamp: [0, 0.56] }, + { text: "リ", timestamp: [0.56, 0.64] }, + { text: "ナ", timestamp: [0.64, 0.8] }, + { text: "ガ", timestamp: [0.8, 0.88] }, + { text: "の", timestamp: [0.88, 1.04] }, + { text: "美味", timestamp: [1.04, 1.22] }, + { text: "しい", timestamp: [1.22, 1.46] }, + { text: "牛", timestamp: [1.46, 1.76] }, + { text: "乳", timestamp: [1.76, 1.94] }, + { text: "は", timestamp: [1.94, 2.14] }, + { text: "濃", timestamp: [2.14, 2.34] }, + { text: "い", timestamp: [2.34, 2.48] }, + { text: "青", timestamp: [2.48, 2.62] }, + { text: "色", timestamp: [2.62, 2.84] }, + { text: "に", timestamp: [2.84, 3] }, + { text: "牛", timestamp: [3, 3.22] }, + { text: "乳", timestamp: [3.22, 3.42] }, + { text: "瓶", timestamp: [3.42, 3.58] }, + { text: "を", timestamp: [3.58, 3.82] }, + { text: "払", timestamp: [3.82, 4] }, + { text: "った", timestamp: [4, 4.32] }, + { text: "ゼ", timestamp: [4.32, 4.56] }, + { text: "ザ", timestamp: [4.56, 4.6] }, + { text: "イ", timestamp: [4.6, 4.74] }, + { text: "ン", timestamp: [4.74, 4.8] }, + { text: "の", timestamp: [4.8, 4.94] }, + { text: "パ", timestamp: [4.94, 5.12] }, + { text: "ック", timestamp: [5.12, 5.26] }, + { text: "牛", timestamp: [5.26, 5.52] }, + { text: "乳", timestamp: [5.52, 5.72] }, + { text: "で", timestamp: [5.72, 5.86] }, + { text: "ある。", timestamp: [5.86, 6.62] }, + ], + }; + + compare(output, target); + } + + await transcriber.dispose(); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + models[4], + async () => { + let transcriber = await pipeline("automatic-speech-recognition", m(models[4])); + + let url = "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/jfk.wav"; + let audioData = await loadAudio(url); + + { + // Transcribe + let output = await transcriber(audioData); + expect(output.text.length).toBeGreaterThan(50); + // { text: "and so my fellow america ask not what your country can do for you ask what you can do for your country" } + } + + await transcriber.dispose(); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("Text-to-speech generation", () => { + // List all models which will be tested + const models = ["Xenova/speecht5_tts", "Xenova/mms-tts-fra"]; + + it( + models[0], + async () => { + let synthesizer = await pipeline("text-to-speech", models[0], { + // NOTE: Although the quantized version produces incoherent results, + // it it is okay to use for testing. + // quantized: false, + }); + + let speaker_embeddings = "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/speaker_embeddings.bin"; + + { + // Generate English speech + let output = await synthesizer("Hello, my dog is cute", { speaker_embeddings }); + expect(output.audio.length).toBeGreaterThan(0); + expect(output.sampling_rate).toEqual(16000); + } + + await synthesizer.dispose(); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + models[1], + async () => { + let synthesizer = await pipeline("text-to-speech", models[1]); + + { + // Generate French speech + let output = await synthesizer("Bonjour"); + expect(output.audio.length).toBeGreaterThan(0); + expect(output.sampling_rate).toEqual(16000); + } + + await synthesizer.dispose(); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("Audio classification", () => { + // List all models which will be tested + const models = ["Xenova/wav2vec2-large-xlsr-53-gender-recognition-librispeech"]; + + it( + models[0], + async () => { + let classifier = await pipeline("audio-classification", models[0]); + + let url = "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/jfk.wav"; + let audioData = await loadAudio(url); + + { + // Classify audio + let outputs = await classifier(audioData); + + let expected = [ + { score: 0.997512936592102, label: "male" }, + { score: 0.0024870133493095636, label: "female" }, + ]; + compare(outputs, expected); + } + + await classifier.dispose(); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("Image-to-text", () => { + // List all models which will be tested + const models = ["Xenova/vit-gpt2-image-captioning"]; + + it( + models[0], + async () => { + let captioner = await pipeline("image-to-text", models[0]); + + let url = "https://huggingface.co/datasets/mishig/sample_images/resolve/main/savanna.jpg"; + let urls = ["https://huggingface.co/datasets/mishig/sample_images/resolve/main/football-match.jpg", "https://huggingface.co/datasets/mishig/sample_images/resolve/main/airport.jpg"]; + + // single + { + let output = await captioner(url, { + top_k: 0, + do_sample: false, + }); + // let expected = [ + // { "generated_text": "a herd of giraffes and zebras grazing in a field" } + // ] + + expect(output).toHaveLength(1); + expect(output[0].generated_text.length).toBeGreaterThan(10); + } + + // single + generation options + { + let output = await captioner(url, { + max_new_tokens: 20, + num_beams: 2, + num_return_sequences: 2, + top_k: 0, + do_sample: false, + }); + // let expected = [ + // { "generated_text": "a herd of giraffes and zebras grazing in a field" }, + // { "generated_text": "a herd of giraffes and zebras in a grassy field" } + // ] + + expect(output).toHaveLength(2); + expect(output[0].generated_text.length).toBeGreaterThan(10); + expect(output[1].generated_text.length).toBeGreaterThan(10); + } + + // batched + { + let output = await captioner(urls, { + top_k: 0, + do_sample: false, + }); + // let expected = [ + // [{ "generated_text": "two men are kicking a soccer ball in a soccer game" }], + // [{ "generated_text": "a plane on the tarmac with a passenger bus" }] + // ] + + expect(output).toHaveLength(2); + expect(output[0]).toHaveLength(1); + expect(output[0][0].generated_text.length).toBeGreaterThan(10); + expect(output[1]).toHaveLength(1); + expect(output[1][0].generated_text.length).toBeGreaterThan(10); + } + + // batched + generation options + { + let output = await captioner(urls, { + max_new_tokens: 20, + num_beams: 2, + num_return_sequences: 2, + top_k: 0, + do_sample: false, + }); + // let expected = [ + // [ + // { "generated_text": "two men are kicking a soccer ball on a field" }, + // { "generated_text": "two men are kicking a soccer ball in a soccer game" } + // ], [ + // { "generated_text": "a plane on a tarmac with a group of buses" }, + // { "generated_text": "a plane on a tarmac with a group of people on the ground" } + // ] + // ]; + + expect(output).toHaveLength(2); + expect(output[0]).toHaveLength(2); + expect(output[0][0].generated_text.length).toBeGreaterThan(10); + expect(output[0][1].generated_text.length).toBeGreaterThan(10); + expect(output[1]).toHaveLength(2); + expect(output[1][0].generated_text.length).toBeGreaterThan(10); + expect(output[1][1].generated_text.length).toBeGreaterThan(10); + } + await captioner.dispose(); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("Image classification", () => { + // List all models which will be tested + const models = ["Xenova/vit-base-patch16-224"]; + + it( + models[0], + async () => { + let classifier = await pipeline("image-classification", models[0]); + + let url = "https://huggingface.co/datasets/mishig/sample_images/resolve/main/tiger.jpg"; + let urls = ["https://huggingface.co/datasets/mishig/sample_images/resolve/main/palace.jpg", "https://huggingface.co/datasets/mishig/sample_images/resolve/main/teapot.jpg"]; + + // single + { + let outputs = await classifier(url); + + let expected = [{ label: "tiger, Panthera tigris", score: 0.607988178730011 }]; + + compare(outputs, expected, 0.2); + } + + // single + topk + { + let outputs = await classifier(url, { + topk: 2, + }); + + let expected = [ + { label: "tiger, Panthera tigris", score: 0.607988178730011 }, + { label: "tiger cat", score: 0.3877776563167572 }, + ]; + + compare(outputs, expected, 0.2); + } + + // batched + { + let outputs = await classifier(urls); + + let expected = [ + { label: "palace", score: 0.9986862540245056 }, + { label: "teapot", score: 0.987880527973175 }, + ]; + + compare(outputs, expected); + } + + // batched + topk + { + let outputs = await classifier(urls, { + topk: 2, + }); + + let expected = [ + [ + { label: "palace", score: 0.9986862540245056 }, + { label: "castle", score: 0.00037879671435803175 }, + ], + [ + { label: "teapot", score: 0.987880527973175 }, + { label: "coffeepot", score: 0.006591461598873138 }, + ], + ]; + + compare(outputs, expected); + } + + await classifier.dispose(); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("Image segmentation", () => { + // List all models which will be tested + const models = ["Xenova/detr-resnet-50-panoptic", "Xenova/segformer_b2_clothes"]; + + it( + models[0], + async () => { + let segmenter = await pipeline("image-segmentation", models[0], { + // Quantized version of model produces incorrect results + quantized: false, + }); + let img = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/coco_sample.png"; + + // single + { + let outputs = await segmenter(img); + + let expected = [ + { score: 0.9916538596153259, label: "cat", mask: 58998 }, + { score: 0.9987397789955139, label: "remote", mask: 4164 }, + { score: 0.9994599223136902, label: "remote", mask: 2275 }, + { score: 0.9730215072631836, label: "couch", mask: 176980 }, + { score: 0.9993911385536194, label: "cat", mask: 52670 }, + ]; + + let outputLabels = outputs.map((x) => x.label); + let expectedLabels = expected.map((x) => x.label); + + expect(outputLabels).toHaveLength(expectedLabels.length); + expect(outputLabels.sort()).toEqual(expectedLabels.sort()); + } + + await segmenter.dispose(); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + models[1], + async () => { + let segmenter = await pipeline("image-segmentation", models[1]); + let img = "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/young-man-standing-and-leaning-on-car.jpg"; + + // single + { + let outputs = await segmenter(img); + + let expected = [{ label: "Background" }, { label: "Hair" }, { label: "Upper-clothes" }, { label: "Pants" }, { label: "Left-shoe" }, { label: "Right-shoe" }, { label: "Face" }, { label: "Left-leg" }, { label: "Right-leg" }, { label: "Left-arm" }, { label: "Right-arm" }]; + + let outputLabels = outputs.map((x) => x.label); + let expectedLabels = expected.map((x) => x.label); + + expect(outputLabels).toHaveLength(expectedLabels.length); + expect(outputLabels.sort()).toEqual(expectedLabels.sort()); + + // check that all scores are null, and masks have correct dimensions + for (let output of outputs) { + expect(output.score).toBeNull(); + expect(output.mask.width).toEqual(970); + expect(output.mask.height).toEqual(1455); + expect(output.mask.channels).toEqual(1); + } + } + + await segmenter.dispose(); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("Zero-shot image classification", () => { + // List all models which will be tested + const models = ["Xenova/clip-vit-base-patch32"]; + + it( + models[0], + async () => { + let classifier = await pipeline("zero-shot-image-classification", models[0]); + + let url = "https://huggingface.co/datasets/mishig/sample_images/resolve/main/football-match.jpg"; + let urls = ["https://huggingface.co/datasets/mishig/sample_images/resolve/main/football-match.jpg", "https://huggingface.co/datasets/mishig/sample_images/resolve/main/airport.jpg", "https://huggingface.co/datasets/mishig/sample_images/resolve/main/savanna.jpg"]; + + let classes = ["football", "airport", "animals"]; + + // single + { + let output = await classifier(url, classes); + + let expected = [ + { score: 0.9719080924987793, label: "football" }, + { score: 0.022564826533198357, label: "animals" }, + { score: 0.005527070723474026, label: "airport" }, + ]; + compare(output, expected, 0.1); + } + + // batched + { + let output = await classifier(urls, classes); + + let expected = [ + [ + { score: 0.9712504148483276, label: "football" }, + { score: 0.022469401359558105, label: "animals" }, + { score: 0.006280169822275639, label: "airport" }, + ], + [ + { score: 0.997433602809906, label: "airport" }, + { score: 0.0016500800848007202, label: "animals" }, + { score: 0.0009163151844404638, label: "football" }, + ], + [ + { score: 0.9851226806640625, label: "animals" }, + { score: 0.007516484707593918, label: "football" }, + { score: 0.007360846735537052, label: "airport" }, + ], + ]; + compare(output, expected, 0.1); + } + await classifier.dispose(); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("Object detection", () => { + // List all models which will be tested + const models = ["Xenova/detr-resnet-50"]; + + it( + models[0], + async () => { + let detector = await pipeline("object-detection", models[0]); + + // TODO add batched test cases when supported + let url = "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/cats.jpg"; + let urls = ["https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/savanna.jpg"]; + + // single + threshold + { + let output = await detector(url, { + threshold: 0.9, + }); + + // let expected = [ + // { + // "score": 0.9977124929428101, + // "label": "remote", + // "box": { "xmin": 41, "ymin": 70, "xmax": 176, "ymax": 118 } + // }, + // { + // "score": 0.9984639883041382, + // "label": "remote", + // "box": { "xmin": 332, "ymin": 73, "xmax": 369, "ymax": 188 } + // }, + // { + // "score": 0.9964856505393982, + // "label": "couch", + // "box": { "xmin": 0, "ymin": 1, "xmax": 639, "ymax": 474 } + // }, + // { + // "score": 0.9988334774971008, + // "label": "cat", + // "box": { "xmin": 11, "ymin": 51, "xmax": 314, "ymax": 472 } + // }, + // { + // "score": 0.9982513785362244, + // "label": "cat", + // "box": { "xmin": 345, "ymin": 22, "xmax": 640, "ymax": 371 } + // } + // ] + + expect(output.length).toBeGreaterThan(0); + for (let cls of output) { + expect(typeof cls.score).toBe("number"); + expect(typeof cls.label).toBe("string"); + for (let key of ["xmin", "ymin", "xmax", "ymax"]) { + expect(typeof cls.box[key]).toBe("number"); + } + } + } + + // batched + threshold + percentage + { + let output = await detector(urls, { + threshold: 0.9, + percentage: true, + }); + // let expected = [[ + // { + // score: 0.9991137385368347, + // label: 'zebra', + // box: { xmin: 0.65165576338768, ymin: 0.685152679681778, xmax: 0.723189502954483, ymax: 0.8801506459712982 } + // }, + // { + // score: 0.998811662197113, + // label: 'zebra', + // box: { xmin: 0.20797613263130188, ymin: 0.6543092578649521, xmax: 0.4147692620754242, ymax: 0.9040975719690323 } + // }, + // { + // score: 0.9707837104797363, + // label: 'giraffe', + // box: { xmin: 0.02498096227645874, ymin: 0.40549489855766296, xmax: 0.38669759035110474, ymax: 0.7895723879337311 } + // }, + // { + // score: 0.9984336495399475, + // label: 'zebra', + // box: { xmin: 0.3540637195110321, ymin: 0.6370827257633209, xmax: 0.5765090882778168, ymax: 0.8480959832668304 } + // }, + // { + // score: 0.9986463785171509, + // label: 'giraffe', + // box: { xmin: 0.6763969212770462, ymin: 0.25748637318611145, xmax: 0.974339172244072, ymax: 0.8684568107128143 } + // } + // ]] + + expect(output).toHaveLength(urls.length); // Same number of inputs as outputs + + for (let i = 0; i < output.length; ++i) { + expect(output[i].length).toBeGreaterThan(0); + for (let cls of output[i]) { + expect(typeof cls.score).toBe("number"); + expect(typeof cls.label).toBe("string"); + for (let key of ["xmin", "ymin", "xmax", "ymax"]) { + expect(typeof cls.box[key]).toBe("number"); + } + } + } + } + + await detector.dispose(); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("Zero-shot object detection", () => { + // List all models which will be tested + const models = ["Xenova/owlvit-base-patch32"]; + + it( + models[0], + async () => { + let detector = await pipeline("zero-shot-object-detection", models[0]); + + // single (default) + { + let url = "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/astronaut.png"; + let candidate_labels = ["human face", "rocket", "helmet", "american flag"]; + + let output = await detector(url, candidate_labels); + + // let expected = [ + // { + // score: 0.24392342567443848, + // label: 'human face', + // box: { xmin: 180, ymin: 67, xmax: 274, ymax: 175 } + // }, + // { + // score: 0.15129457414150238, + // label: 'american flag', + // box: { xmin: 0, ymin: 4, xmax: 106, ymax: 513 } + // }, + // { + // score: 0.13649864494800568, + // label: 'helmet', + // box: { xmin: 277, ymin: 337, xmax: 511, ymax: 511 } + // }, + // { + // score: 0.10262022167444229, + // label: 'rocket', + // box: { xmin: 352, ymin: -1, xmax: 463, ymax: 287 } + // } + // ] + + expect(output.length).toBeGreaterThan(0); + for (let cls of output) { + expect(typeof cls.score).toBe("number"); + expect(typeof cls.label).toBe("string"); + for (let key of ["xmin", "ymin", "xmax", "ymax"]) { + expect(typeof cls.box[key]).toBe("number"); + } + } + } + + // topk + threshold + percentage + { + let url = "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/beach.png"; + let candidate_labels = ["hat", "book", "sunglasses", "camera"]; + + let output = await detector(url, candidate_labels, { + topk: 4, + threshold: 0.05, + percentage: true, + }); + + // let expected = [ + // { + // score: 0.1606510728597641, + // label: 'sunglasses', + // box: { xmin: 347, ymin: 229, xmax: 429, ymax: 264 } + // }, + // { + // score: 0.08935828506946564, + // label: 'hat', + // box: { xmin: 38, ymin: 174, xmax: 258, ymax: 364 } + // }, + // { + // score: 0.08530698716640472, + // label: 'camera', + // box: { xmin: 187, ymin: 350, xmax: 260, ymax: 411 } + // }, + // { + // score: 0.08349756896495819, + // label: 'book', + // box: { xmin: 261, ymin: 280, xmax: 494, ymax: 425 } + // } + // ] + + expect(output.length).toBeGreaterThan(0); + for (let cls of output) { + expect(typeof cls.score).toBe("number"); + expect(typeof cls.label).toBe("string"); + for (let key of ["xmin", "ymin", "xmax", "ymax"]) { + expect(typeof cls.box[key]).toBe("number"); + } + } + } + + await detector.dispose(); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("Image-to-image", () => { + // List all models which will be tested + const models = ["Xenova/swin2SR-classical-sr-x2-64"]; + + it( + models[0], + async () => { + let upscaler = await pipeline("image-to-image", models[0]); + + // Input is 3x3 => padded to 8x8 => upscaled to 16x16 + let url = "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/pattern_3x3.png"; + + // single + { + let outputs = await upscaler(url); + expect(outputs.width).toEqual(16); + expect(outputs.height).toEqual(16); + expect(outputs.channels).toEqual(3); + expect(outputs.data).toHaveLength(768); + } + + // batched + { + let outputs = await upscaler([url, url]); + expect(outputs).toHaveLength(2); + for (let output of outputs) { + expect(output.width).toEqual(16); + expect(output.height).toEqual(16); + expect(output.channels).toEqual(3); + expect(output.data).toHaveLength(768); + } + } + + await upscaler.dispose(); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("Depth estimation", () => { + // List all models which will be tested + const models = ["Xenova/dpt-hybrid-midas"]; + + it( + models[0], + async () => { + let depth_estimator = await pipeline("depth-estimation", models[0]); + + let url = "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/cats.jpg"; + + // single + { + let { predicted_depth, depth } = await depth_estimator(url); + compare(predicted_depth.dims, [384, 384]); + expect(depth.width).toEqual(640); + expect(depth.height).toEqual(480); + expect(depth.channels).toEqual(1); + expect(depth.data).toHaveLength(307200); + } + + // batched + { + let outputs = await depth_estimator([url, url]); + expect(outputs).toHaveLength(2); + for (let output of outputs) { + let { predicted_depth, depth } = output; + compare(predicted_depth.dims, [384, 384]); + expect(depth.width).toEqual(640); + expect(depth.height).toEqual(480); + expect(depth.channels).toEqual(1); + expect(depth.data).toHaveLength(307200); + } + } + + await depth_estimator.dispose(); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("Document question answering", () => { + // List all models which will be tested + const models = ["Xenova/donut-base-finetuned-docvqa"]; + const image = "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/invoice.png"; + const question = "What is the invoice number?"; + + it( + models[0], + async () => { + let qa_pipeline = await pipeline("document-question-answering", models[0]); + + // basic + { + let output = await qa_pipeline(image, question); + let expected = [{ answer: "us-001" }]; + compare(output, expected); + } + + await qa_pipeline.dispose(); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); }); diff --git a/tests/processors.test.js b/tests/processors.test.js index caf1ddf86..db3359527 100644 --- a/tests/processors.test.js +++ b/tests/processors.test.js @@ -12,1007 +12,1019 @@ const avg = (array) => sum(array) / array.length; const IMAGE_CACHE = new Map(); const load_image = async (url) => { - const cached = IMAGE_CACHE.get(url); - if (cached) { - return cached; - } - const image = await RawImage.fromURL(url); - IMAGE_CACHE.set(url, image); - return image; + const cached = IMAGE_CACHE.get(url); + if (cached) { + return cached; + } + const image = await RawImage.fromURL(url); + IMAGE_CACHE.set(url, image); + return image; }; const MODELS = { - swin2sr: "Xenova/swin2SR-classical-sr-x2-64", - sam: "Xenova/sam-vit-base", - "donut-swin": "Xenova/donut-base-finetuned-cord-v2", - resnet: "Xenova/resnet-50", - vit: "Xenova/vit-base-patch16-224", - mobilevit: "Xenova/mobilevit-small", - mobilevit_2: "Xenova/quickdraw-mobilevit-small", - mobilevit_3: "Xenova/mobilevitv2-1.0-imagenet1k-256", - deit: "Xenova/deit-tiny-distilled-patch16-224", - beit: "Xenova/beit-base-patch16-224-pt22k-ft22k", - detr: "Xenova/detr-resnet-50", - yolos: "Xenova/yolos-small-300", - dpt: "Xenova/dpt-hybrid-midas", - dpt_2: "Xenova/depth-anything-small-hf", - glpn: "Xenova/glpn-kitti", - nougat: "Xenova/nougat-small", - owlvit: "Xenova/owlvit-base-patch32", - clip: "Xenova/clip-vit-base-patch16", - vitmatte: "Xenova/vitmatte-small-distinctions-646", - dinov2: "Xenova/dinov2-small-imagenet1k-1-layer", - // efficientnet: 'Xenova/efficientnet-b0', - florence2: "Xenova/tiny-random-Florence2ForConditionalGeneration", + swin2sr: "Xenova/swin2SR-classical-sr-x2-64", + sam: "Xenova/sam-vit-base", + "donut-swin": "Xenova/donut-base-finetuned-cord-v2", + resnet: "Xenova/resnet-50", + vit: "Xenova/vit-base-patch16-224", + mobilevit: "Xenova/mobilevit-small", + mobilevit_2: "Xenova/quickdraw-mobilevit-small", + mobilevit_3: "Xenova/mobilevitv2-1.0-imagenet1k-256", + deit: "Xenova/deit-tiny-distilled-patch16-224", + beit: "Xenova/beit-base-patch16-224-pt22k-ft22k", + detr: "Xenova/detr-resnet-50", + yolos: "Xenova/yolos-small-300", + dpt: "Xenova/dpt-hybrid-midas", + dpt_2: "Xenova/depth-anything-small-hf", + glpn: "Xenova/glpn-kitti", + nougat: "Xenova/nougat-small", + owlvit: "Xenova/owlvit-base-patch32", + clip: "Xenova/clip-vit-base-patch16", + vitmatte: "Xenova/vitmatte-small-distinctions-646", + dinov2: "Xenova/dinov2-small-imagenet1k-1-layer", + // efficientnet: 'Xenova/efficientnet-b0', + florence2: "Xenova/tiny-random-Florence2ForConditionalGeneration", }; const TEST_IMAGES = { - pattern_3x3: "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/pattern_3x3.png", - pattern_3x5: "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/pattern_3x5.png", - checkerboard_8x8: "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/checkerboard_8x8.png", - checkerboard_64x32: "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/checkerboard_64x32.png", - receipt: "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/receipt.png", - tiger: "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/tiger.jpg", - paper: "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/nougat_paper.png", - cats: "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/cats.jpg", - - // grayscale image - skateboard: "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/blog/ml-web-games/skateboard.png", - - vitmatte_image: "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/vitmatte_image.png", - vitmatte_trimap: "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/vitmatte_trimap.png", - - beetle: "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/beetle.png", - book_cover: "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/book-cover.png", + pattern_3x3: "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/pattern_3x3.png", + pattern_3x5: "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/pattern_3x5.png", + checkerboard_8x8: "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/checkerboard_8x8.png", + checkerboard_64x32: "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/checkerboard_64x32.png", + receipt: "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/receipt.png", + tiger: "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/tiger.jpg", + paper: "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/nougat_paper.png", + cats: "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/cats.jpg", + + // grayscale image + skateboard: "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/blog/ml-web-games/skateboard.png", + + vitmatte_image: "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/vitmatte_image.png", + vitmatte_trimap: "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/vitmatte_trimap.png", + + beetle: "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/beetle.png", + book_cover: "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/book-cover.png", }; describe("Processors", () => { - describe("Image processors", () => { - // Swin2SRImageProcessor - // - tests when padding is a number (do_pad=true, pad_size=8) - it( - MODELS.swin2sr, - async () => { - const processor = await AutoProcessor.from_pretrained(MODELS.swin2sr); - - { - // Pad to multiple of 8 (3x3 -> 8x8) - const image = await load_image(TEST_IMAGES.pattern_3x3); - const { pixel_values } = await processor(image); - - compare(pixel_values.dims, [1, 3, 8, 8]); - compare(avg(pixel_values.data), 0.5458333368102709); - } - - { - // Do not pad if already a multiple of 8 (8x8 -> 8x8) - const image = await load_image(TEST_IMAGES.checkerboard_8x8); - const { pixel_values } = await processor(image); - compare(pixel_values.dims, [1, 3, 8, 8]); - compare(avg(pixel_values.data), 0.5); - } - }, - MAX_TEST_EXECUTION_TIME, - ); - - // SamProcessor/SamImageProcessor - // - tests normal padding (do_pad=true, pad_size={"height":1024,"width":1024}) - // - In addition to the image, pass in a list of points - it( - MODELS.sam, - async () => { - const processor = await AutoProcessor.from_pretrained(MODELS.sam); - - { - // without input points - const image = await load_image(TEST_IMAGES.pattern_3x3); - const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); - compare(pixel_values.dims, [1, 3, 1024, 1024]); - compare(avg(pixel_values.data), -0.4505715670146813); - - compare(original_sizes, [[3, 3]]); - compare(reshaped_input_sizes, [[1024, 1024]]); - } - - { - // with input points - const image = await load_image(TEST_IMAGES.pattern_3x3); - const { original_sizes, reshaped_input_sizes, input_points } = await processor(image, { - input_points: [[[1, 2]]], - }); - - compare(original_sizes, [[3, 3]]); - compare(reshaped_input_sizes, [[1024, 1024]]); - compare(input_points.tolist(), [[[[341.3333, 682.6667]]]]); - } - - { - // multiple points with labels - const image = await load_image(TEST_IMAGES.pattern_3x3); - const { original_sizes, reshaped_input_sizes, input_points, input_labels } = await processor(image, { - input_points: [ - [ - [1, 2], - [2, 1], - ], - ], - input_labels: [[1, 0]], - }); - - compare(original_sizes, [[3, 3]]); - compare(reshaped_input_sizes, [[1024, 1024]]); - compare(input_points.tolist(), [ - [ - [ - [341.3333, 682.6667], - [682.6667, 341.3333], - ], - ], - ]); - compare(input_labels.tolist(), [[[1n, 0n]]]); - } - - { - // with input boxes - const image = await load_image(TEST_IMAGES.pattern_3x3); - const { original_sizes, reshaped_input_sizes, input_boxes } = await processor(image, { - input_boxes: [[[0, 1, 2, 2]]], - }); - - compare(original_sizes, [[3, 3]]); - compare(reshaped_input_sizes, [[1024, 1024]]); - compare(input_boxes.tolist(), [[[0, 341.3333, 682.6667, 682.6667]]]); - } - }, - MAX_TEST_EXECUTION_TIME, - ); - - // DonutProcessor/DonutFeatureExtractor - // - tests thumbnail resizing (do_thumbnail=true, size=[960, 1280]) - // - tests padding after normalization (image_mean=image_std=0.5) - it( - MODELS["donut-swin"], - async () => { - const processor = await AutoProcessor.from_pretrained(MODELS["donut-swin"]); - - { - const image = await load_image(TEST_IMAGES.receipt); - const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); - - compare(pixel_values.dims, [1, 3, 1280, 960]); - compare(avg(pixel_values.data), 0.1229388610053704); - - compare(original_sizes, [[864, 576]]); - compare(reshaped_input_sizes, [[1280, 853]]); - } - }, - MAX_TEST_EXECUTION_TIME, - ); - - // ConvNextFeatureExtractor - it( - MODELS.resnet, - async () => { - const processor = await AutoProcessor.from_pretrained(MODELS.resnet); - - { - const image = await load_image(TEST_IMAGES.tiger); - const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); - - compare(pixel_values.dims, [1, 3, 224, 224]); - compare(avg(pixel_values.data), 0.06262318789958954); - - compare(original_sizes, [[408, 612]]); - compare(reshaped_input_sizes, [[224, 224]]); - } - }, - MAX_TEST_EXECUTION_TIME, - ); - - // ViTFeatureExtractor - it( - MODELS.vit, - async () => { - const processor = await AutoProcessor.from_pretrained(MODELS.vit); - - { - const image = await load_image(TEST_IMAGES.tiger); - const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); - - compare(pixel_values.dims, [1, 3, 224, 224]); - compare(avg(pixel_values.data), -0.22706867939852762); - - compare(original_sizes, [[408, 612]]); - compare(reshaped_input_sizes, [[224, 224]]); - } - }, - MAX_TEST_EXECUTION_TIME, - ); - - // MobileViTFeatureExtractor - it( - MODELS.mobilevit, - async () => { - const processor = await AutoProcessor.from_pretrained(MODELS.mobilevit); - - { - const image = await load_image(TEST_IMAGES.tiger); - const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); - - compare(pixel_values.dims, [1, 3, 256, 256]); - compare(avg(pixel_values.data), 0.4599160496887033); - - compare(original_sizes, [[408, 612]]); - compare(reshaped_input_sizes, [[256, 256]]); - } - }, - MAX_TEST_EXECUTION_TIME, - ); - - // MobileViTFeatureExtractor - // - tests not converting to rgb (do_convert_rgb=false) - it( - MODELS.mobilevit_2, - async () => { - const processor = await AutoProcessor.from_pretrained(MODELS.mobilevit_2); - - { - // Tests grayscale image - const image = await load_image(TEST_IMAGES.skateboard); - const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); - - compare(pixel_values.dims, [1, 1, 28, 28]); - compare(avg(pixel_values.data), 0.08558923671585128); - - compare(original_sizes, [[28, 28]]); - compare(reshaped_input_sizes, [[28, 28]]); - } - }, - MAX_TEST_EXECUTION_TIME, - ); - - // MobileViTImageProcessor - // - tests converting RGB to BGR (do_flip_channel_order=true) - it( - MODELS.mobilevit_3, - async () => { - const processor = await AutoProcessor.from_pretrained(MODELS.mobilevit_3); - - { - const image = await load_image(TEST_IMAGES.cats); - const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); - - compare(pixel_values.dims, [1, 3, 256, 256]); - compare(avg(pixel_values.data), 0.5215385556221008); - - compare(original_sizes, [[480, 640]]); - compare(reshaped_input_sizes, [[256, 256]]); - - // Ensure RGB to BGR conversion - compare(pixel_values.data.slice(0, 3), [0.24313725531101227, 0.250980406999588, 0.364705890417099]); - } - }, - MAX_TEST_EXECUTION_TIME, - ); - - // DeiTFeatureExtractor - it( - MODELS.deit, - async () => { - const processor = await AutoProcessor.from_pretrained(MODELS.deit); - - { - const image = await load_image(TEST_IMAGES.tiger); - const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); - - compare(pixel_values.dims, [1, 3, 224, 224]); - compare(avg(pixel_values.data), -0.2760336682859463); - - compare(original_sizes, [[408, 612]]); - compare(reshaped_input_sizes, [[224, 224]]); - } - }, - MAX_TEST_EXECUTION_TIME, - ); - - // BeitFeatureExtractor - it( - MODELS.beit, - async () => { - const processor = await AutoProcessor.from_pretrained(MODELS.beit); - - { - const image = await load_image(TEST_IMAGES.tiger); - const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); - - compare(pixel_values.dims, [1, 3, 224, 224]); - compare(avg(pixel_values.data), -0.22706867939852762); - - compare(original_sizes, [[408, 612]]); - compare(reshaped_input_sizes, [[224, 224]]); - } - }, - MAX_TEST_EXECUTION_TIME, - ); - - // DetrFeatureExtractor - it( - MODELS.detr, - async () => { - const processor = await AutoProcessor.from_pretrained(MODELS.detr); - - { - const image = await load_image(TEST_IMAGES.tiger); - const { pixel_values, original_sizes, reshaped_input_sizes, pixel_mask } = await processor(image); - - compare(pixel_values.dims, [1, 3, 888, 1333]); - compare(avg(pixel_values.data), -0.27840224131001773); - - compare(original_sizes, [[408, 612]]); - compare(reshaped_input_sizes, [[888, 1333]]); - - compare(pixel_mask.dims, [1, 64, 64]); - compare(avg(pixel_mask.data), 1); - } - }, - MAX_TEST_EXECUTION_TIME, - ); - - // YolosFeatureExtractor - it( - MODELS.yolos, - async () => { - const processor = await AutoProcessor.from_pretrained(MODELS.yolos); - - { - const image = await load_image(TEST_IMAGES.tiger); - const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); - - compare(pixel_values.dims, [1, 3, 888, 1333]); - compare(avg(pixel_values.data), -0.27840224131001773); - - compare(original_sizes, [[408, 612]]); - compare(reshaped_input_sizes, [[888, 1333]]); - } - }, - MAX_TEST_EXECUTION_TIME, - ); - - // DPTFeatureExtractor - it( - MODELS.dpt, - async () => { - const processor = await AutoProcessor.from_pretrained(MODELS.dpt); - - { - // Tests grayscale image - const image = await load_image(TEST_IMAGES.cats); - const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); - - compare(pixel_values.dims, [1, 3, 384, 384]); - compare(avg(pixel_values.data), 0.0372855559389454); - - compare(original_sizes, [[480, 640]]); - compare(reshaped_input_sizes, [[384, 384]]); - } - }, - MAX_TEST_EXECUTION_TIME, - ); - - // GLPNForDepthEstimation - // - tests `size_divisor` and no size (size_divisor=32) - it( - MODELS.glpn, - async () => { - const processor = await AutoProcessor.from_pretrained(MODELS.glpn); - - { - const image = await load_image(TEST_IMAGES.cats); - const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); - compare(pixel_values.dims, [1, 3, 480, 640]); - compare(avg(pixel_values.data), 0.5186172404123327); - - compare(original_sizes, [[480, 640]]); - compare(reshaped_input_sizes, [[480, 640]]); - } - - { - // Tests input which is not a multiple of 32 ([408, 612] -> [384, 608]) - const image = await load_image(TEST_IMAGES.tiger); - const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); - - compare(pixel_values.dims, [1, 3, 384, 608]); - compare(avg(pixel_values.data), 0.38628831535989555); - - compare(original_sizes, [[408, 612]]); - compare(reshaped_input_sizes, [[384, 608]]); - } - }, - MAX_TEST_EXECUTION_TIME, - ); - - // NougatImageProcessor - // - tests padding after normalization (image_mean != 0.5, image_std != 0.5) - it( - MODELS.nougat, - async () => { - const processor = await AutoProcessor.from_pretrained(MODELS.nougat); - - { - const image = await load_image(TEST_IMAGES.paper); - const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); - - compare(pixel_values.dims, [1, 3, 896, 672]); - compare(avg(pixel_values.data), 1.8447155005897355); - - compare(original_sizes, [[850, 685]]); - compare(reshaped_input_sizes, [[833, 672]]); - } - }, - MAX_TEST_EXECUTION_TIME, - ); - - // OwlViTFeatureExtractor - it(MODELS.owlvit, async () => { - const processor = await AutoProcessor.from_pretrained(MODELS.owlvit); - { - const image = await load_image(TEST_IMAGES.cats); - const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); - - compare(pixel_values.dims, [1, 3, 768, 768]); - compare(avg(pixel_values.data), 0.250620447910435); - - compare(original_sizes, [[480, 640]]); - compare(reshaped_input_sizes, [[768, 768]]); - } - }); + describe("Image processors", () => { + // Swin2SRImageProcessor + // - tests when padding is a number (do_pad=true, pad_size=8) + it( + MODELS.swin2sr, + async () => { + const processor = await AutoProcessor.from_pretrained(MODELS.swin2sr); + + { + // Pad to multiple of 8 (3x3 -> 8x8) + const image = await load_image(TEST_IMAGES.pattern_3x3); + const { pixel_values } = await processor(image); + + compare(pixel_values.dims, [1, 3, 8, 8]); + compare(avg(pixel_values.data), 0.5458333368102709); + } + + { + // Do not pad if already a multiple of 8 (8x8 -> 8x8) + const image = await load_image(TEST_IMAGES.checkerboard_8x8); + const { pixel_values } = await processor(image); + compare(pixel_values.dims, [1, 3, 8, 8]); + compare(avg(pixel_values.data), 0.5); + } + }, + MAX_TEST_EXECUTION_TIME, + ); + + // SamProcessor/SamImageProcessor + // - tests normal padding (do_pad=true, pad_size={"height":1024,"width":1024}) + // - In addition to the image, pass in a list of points + it( + MODELS.sam, + async () => { + const processor = await AutoProcessor.from_pretrained(MODELS.sam); + + { + // without input points + const image = await load_image(TEST_IMAGES.pattern_3x3); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + compare(pixel_values.dims, [1, 3, 1024, 1024]); + compare(avg(pixel_values.data), -0.4505715670146813); + + compare(original_sizes, [[3, 3]]); + compare(reshaped_input_sizes, [[1024, 1024]]); + } + + { + // with input points + const image = await load_image(TEST_IMAGES.pattern_3x3); + const { original_sizes, reshaped_input_sizes, input_points } = await processor(image, { + input_points: [[[1, 2]]], + }); + + compare(original_sizes, [[3, 3]]); + compare(reshaped_input_sizes, [[1024, 1024]]); + compare(input_points.tolist(), [[[[341.3333, 682.6667]]]]); + } + + { + // multiple points with labels + const image = await load_image(TEST_IMAGES.pattern_3x3); + const { original_sizes, reshaped_input_sizes, input_points, input_labels } = await processor(image, { + input_points: [ + [ + [1, 2], + [2, 1], + ], + ], + input_labels: [[1, 0]], + }); + + compare(original_sizes, [[3, 3]]); + compare(reshaped_input_sizes, [[1024, 1024]]); + compare(input_points.tolist(), [ + [ + [ + [341.3333, 682.6667], + [682.6667, 341.3333], + ], + ], + ]); + compare(input_labels.tolist(), [[[1n, 0n]]]); + } + + { + // with input boxes + const image = await load_image(TEST_IMAGES.pattern_3x3); + const { original_sizes, reshaped_input_sizes, input_boxes } = await processor(image, { + input_boxes: [[[0, 1, 2, 2]]], + }); + + compare(original_sizes, [[3, 3]]); + compare(reshaped_input_sizes, [[1024, 1024]]); + compare(input_boxes.tolist(), [[[0, 341.3333, 682.6667, 682.6667]]]); + } + }, + MAX_TEST_EXECUTION_TIME, + ); + + // DonutProcessor/DonutFeatureExtractor + // - tests thumbnail resizing (do_thumbnail=true, size=[960, 1280]) + // - tests padding after normalization (image_mean=image_std=0.5) + it( + MODELS["donut-swin"], + async () => { + const processor = await AutoProcessor.from_pretrained(MODELS["donut-swin"]); + + { + const image = await load_image(TEST_IMAGES.receipt); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + + compare(pixel_values.dims, [1, 3, 1280, 960]); + compare(avg(pixel_values.data), 0.1229388610053704); + + compare(original_sizes, [[864, 576]]); + compare(reshaped_input_sizes, [[1280, 853]]); + } + }, + MAX_TEST_EXECUTION_TIME, + ); - // CLIPFeatureExtractor - // - tests center crop (do_center_crop=true, crop_size=224) - it( - MODELS.clip, - async () => { - const processor = await AutoProcessor.from_pretrained(MODELS.clip); - - { - const image = await load_image(TEST_IMAGES.tiger); - const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); - - compare(pixel_values.dims, [1, 3, 224, 224]); - compare(avg(pixel_values.data), -0.06678297738282096); - - compare(original_sizes, [[408, 612]]); - compare(reshaped_input_sizes, [[224, 224]]); - } - }, - MAX_TEST_EXECUTION_TIME, - ); - - // VitMatteImageProcessor - // - tests custom overrides - // - tests multiple inputs - // - tests `size_divisibility` and no size (size_divisibility=32) - // - tests do_pad and `size_divisibility` - it( - MODELS.vitmatte, - async () => { - const processor = await AutoProcessor.from_pretrained(MODELS.vitmatte); - - { - const image = await load_image(TEST_IMAGES.vitmatte_image); - const image2 = await load_image(TEST_IMAGES.vitmatte_trimap); - const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image, image2); - - compare(pixel_values.dims, [1, 4, 640, 960]); - expect(avg(pixel_values.data)).toBeCloseTo(-0.4028555154800415); - expect(pixel_values.data[0]).toBeCloseTo(-0.9921568632125854); - expect(pixel_values.data[1]).toBeCloseTo(-0.9921568632125854); - expect(pixel_values.data[5]).toBeCloseTo(-1.0); - expect(pixel_values.data[640]).toBeCloseTo(-0.6784313917160034); - expect(pixel_values.data[641]).toBeCloseTo(-0.6705882549285889); - expect(pixel_values.data[640 * 960]).toBeCloseTo(-1.0); - expect(pixel_values.data[640 * 960 + 1]).toBeCloseTo(-1.0); - expect(pixel_values.data.at(-1)).toBeCloseTo(0.0); - - compare(original_sizes, [[640, 960]]); - compare(reshaped_input_sizes, [[640, 960]]); - } - - { - const image = await load_image(TEST_IMAGES.pattern_3x5); - const image2 = await load_image(TEST_IMAGES.pattern_3x5); - const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image, image2); - - compare(pixel_values.dims, [1, 4, 32, 32]); - expect(avg(pixel_values.data)).toBeCloseTo(-0.00867417361587286); - expect(pixel_values.data[0]).toBeCloseTo(-0.9921568632125854); - expect(pixel_values.data[1]).toBeCloseTo(-0.9686274528503418); - expect(pixel_values.data[5]).toBeCloseTo(0.0); - expect(pixel_values.data[32]).toBeCloseTo(-0.9215686321258545); - expect(pixel_values.data[33]).toBeCloseTo(-0.8980392217636108); - expect(pixel_values.data.at(-1)).toBeCloseTo(0.0); - - compare(original_sizes, [[5, 3]]); - compare(reshaped_input_sizes, [[5, 3]]); - } - }, - MAX_TEST_EXECUTION_TIME, - ); - - // BitImageProcessor - it( - MODELS.dinov2, - async () => { - const processor = await AutoProcessor.from_pretrained(MODELS.dinov2); - - { - const image = await load_image(TEST_IMAGES.tiger); - const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); - - compare(pixel_values.dims, [1, 3, 224, 224]); - compare(avg(pixel_values.data), 0.06262318789958954); - - compare(original_sizes, [[408, 612]]); - compare(reshaped_input_sizes, [[224, 224]]); - } - }, - MAX_TEST_EXECUTION_TIME, - ); - - // DPTImageProcessor - // - tests ensure_multiple_of - // - tests keep_aspect_ratio - // - tests bankers rounding - it( - MODELS.dpt_2, - async () => { - const processor = await AutoProcessor.from_pretrained(MODELS.dpt_2); - - { - const image = await load_image(TEST_IMAGES.cats); - const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); - - compare(pixel_values.dims, [1, 3, 518, 686]); - compare(avg(pixel_values.data), 0.30337387323379517); - - compare(original_sizes, [[480, 640]]); - compare(reshaped_input_sizes, [[518, 686]]); - } - - { - const image = await load_image(TEST_IMAGES.checkerboard_64x32); - const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); - - // NOTE: without bankers rounding, this would be [1, 3, 266, 518] - compare(pixel_values.dims, [1, 3, 252, 518]); - compare(avg(pixel_values.data), 0.2267402559518814); - - compare(original_sizes, [[32, 64]]); - compare(reshaped_input_sizes, [[252, 518]]); - } - }, - MAX_TEST_EXECUTION_TIME, - ); - - // TODO: Add back - // // EfficientNetImageProcessor - // // - tests include_top - // it(MODELS.efficientnet, async () => { - // const processor = await AutoProcessor.from_pretrained(MODELS.efficientnet) - - // { - // const image = await load_image(TEST_IMAGES.cats); - // const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); - - // compare(pixel_values.dims, [1, 3, 224, 224]); - // compare(avg(pixel_values.data), 0.3015307230282871); - - // compare(original_sizes, [[480, 640]]); - // compare(reshaped_input_sizes, [[224, 224]]); - // } - // }, MAX_TEST_EXECUTION_TIME); - }); - - describe("Audio processors", () => { - const audioPromise = new Promise(async (resolve) => { - const url = "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/mlk.npy"; - const buffer = await (await fetch(url)).arrayBuffer(); - const audio = Float32Array.from(new Float64Array(buffer)); - resolve(audio); - }); + // ConvNextFeatureExtractor + it( + MODELS.resnet, + async () => { + const processor = await AutoProcessor.from_pretrained(MODELS.resnet); - it( - "WhisperFeatureExtractor", - async () => { - const audio = await audioPromise; - const processor = await AutoProcessor.from_pretrained("Xenova/whisper-tiny.en"); - const { input_features } = await processor(audio); - compare(input_features.dims, [1, 80, 3000]); - expect(avg(input_features.data)).toBeCloseTo(-0.2813588131551941); - expect(input_features.data[0]).toBeCloseTo(0.33168578147888184); - expect(input_features.data[1]).toBeCloseTo(0.30986475944519043); - expect(input_features.data[81]).toBeCloseTo(0.10727232694625854); - expect(input_features.data[3001]).toBeCloseTo(0.2555035352706909); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "ASTFeatureExtractor", - async () => { - const audio = await audioPromise; - const processor = await AutoProcessor.from_pretrained("Xenova/ast-finetuned-audioset-10-10-0.4593"); - { - // truncation - const { input_values } = await processor(audio); - compare(input_values.dims, [1, 1024, 128]); - - expect(avg(input_values.data)).toBeCloseTo(-0.04054912979309085); - expect(input_values.data[0]).toBeCloseTo(-0.5662586092948914); - expect(input_values.data[1]).toBeCloseTo(-1.0300861597061157); - expect(input_values.data[129]).toBeCloseTo(-1.084834098815918); - expect(input_values.data[1025]).toBeCloseTo(-1.1204065084457397); - } - { - // padding - const { input_values } = await processor(audio.slice(0, 1000)); - compare(input_values.dims, [1, 1024, 128]); // [1, 4, 128] -> (padded to) -> [1, 1024, 128] - - expect(avg(input_values.data)).toBeCloseTo(0.4647964835166931); - expect(input_values.data[0]).toBeCloseTo(-0.5662586092948914); - expect(input_values.data[1]).toBeCloseTo(-1.0300861597061157); - expect(input_values.data[129]).toBeCloseTo(-1.084834098815918); - - // padded values - expect(input_values.data[1025]).toBeCloseTo(0.46703237295150757); - expect(input_values.data[2049]).toBeCloseTo(0.46703237295150757); - expect(input_values.data[10000]).toBeCloseTo(0.46703237295150757); - } - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "SeamlessM4TFeatureExtractor", - async () => { - const audio = await audioPromise; - const processor = await AutoProcessor.from_pretrained("Xenova/wav2vec2-bert-CV16-en"); - { - // normal - const { input_features, attention_mask } = await processor(audio); - compare(input_features.dims, [1, 649, 160]); - compare(attention_mask.dims, [1, 649]); - - expect(avg(input_features.data)).toBeCloseTo(-2.938903875815413e-8); - expect(input_features.data[0]).toBeCloseTo(1.1939343214035034); - expect(input_features.data[1]).toBeCloseTo(0.7874255180358887); - expect(input_features.data[160]).toBeCloseTo(-0.712975025177002); - expect(input_features.data[161]).toBeCloseTo(0.045802414417266846); - expect(input_features.data.at(-1)).toBeCloseTo(-1.3328346014022827); - - expect(sum(attention_mask.data)).toEqual(649); - } - { - // padding (pad_to_multiple_of=2) - const { input_features, attention_mask } = await processor(audio.slice(0, 10000)); - - // [1, 61, 80] -> [1, 62, 80] -> [1, 31, 160] - compare(input_features.dims, [1, 31, 160]); - compare(attention_mask.dims, [1, 31]); - - expect(avg(input_features.data)).toBeCloseTo(0.01612919569015503); - expect(input_features.data[0]).toBeCloseTo(0.9657132029533386); - expect(input_features.data[1]).toBeCloseTo(0.12912897765636444); - expect(input_features.data[160]).toBeCloseTo(-1.2364212274551392); - expect(input_features.data[161]).toBeCloseTo(-0.9703778028488159); - expect(input_features.data.at(-1)).toBeCloseTo(1); // padding value - - expect(sum(attention_mask.data)).toEqual(30); - } - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "ClapFeatureExtractor", - async () => { - const audio = await audioPromise; - const processor = await AutoProcessor.from_pretrained("Xenova/clap-htsat-unfused"); - { - // truncation - // Since truncation uses a random strategy, we override - // Math.random to ensure that the test is deterministic - const originalRandom = Math.random; - Math.random = () => 0.5; - - let long_audio = new Float32Array(500000); - long_audio.set(audio); - long_audio.set(audio, long_audio.length - audio.length); - - const { input_features } = await processor(long_audio); - compare(input_features.dims, [1, 1, 1001, 64]); - - expect(avg(input_features.data)).toBeCloseTo(-37.94569396972656); - expect(input_features.data[0]).toBeCloseTo(-53.32647705078125); - expect(input_features.data[1]).toBeCloseTo(-47.76755142211914); - expect(input_features.data[65]).toBeCloseTo(-36.32261276245117); - expect(input_features.data[1002]).toBeCloseTo(-28.0314884185791); - expect(input_features.data[10000]).toBeCloseTo(-21.905902862548828); - expect(input_features.data[60000]).toBeCloseTo(-14.877863883972168); - expect(input_features.data[64062]).toBeCloseTo(-37.9784049987793); - expect(input_features.data[64063]).toBeCloseTo(-37.73963928222656); - - // Reset Math.random - Math.random = originalRandom; - } - { - // padding - const { input_features } = await processor(audio); - compare(input_features.dims, [1, 1, 1001, 64]); - - expect(avg(input_features.data)).toBeCloseTo(-34.99049377441406); - expect(input_features.data[0]).toBeCloseTo(-21.32573890686035); - expect(input_features.data[1]).toBeCloseTo(-26.168411254882812); - expect(input_features.data[65]).toBeCloseTo(-29.716018676757812); - expect(input_features.data[1002]).toBeCloseTo(-32.16273498535156); - expect(input_features.data[10000]).toBeCloseTo(-19.9283390045166); - - // padded values - expect(input_features.data[60000]).toBeCloseTo(-100.0); - expect(input_features.data[64062]).toBeCloseTo(-100.0); - expect(input_features.data[64063]).toBeCloseTo(-100.0); - } - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "WeSpeakerFeatureExtractor", - async () => { - const processor = await AutoProcessor.from_pretrained("onnx-community/wespeaker-voxceleb-resnet34-LM"); - { - // default - const audio = new Float32Array(16000).map((_, i) => Math.sin(i / 100)); - const { input_features } = await processor(audio); - compare(input_features.dims, [1, 98, 80]); - - expect(avg(input_features.data)).toBeCloseTo(5.461731689138105e-8); - expect(input_features.data[0]).toBeCloseTo(-0.19300270080566406); - expect(input_features.data[1]).toBeCloseTo(-0.05825042724609375); - expect(input_features.data[78]).toBeCloseTo(0.2683420181274414); - expect(input_features.data[79]).toBeCloseTo(0.26250171661376953); - expect(input_features.data[80]).toBeCloseTo(0.19062232971191406); - expect(input_features.data.at(-2)).toBeCloseTo(-0.43694400787353516); - expect(input_features.data.at(-1)).toBeCloseTo(-0.4266204833984375); - } - - { - // pad to `min_num_frames` - const audio = new Float32Array(3).map((_, i) => Math.sin(i / 100)); - const { input_features } = await processor(audio); - compare(input_features.dims, [1, 9, 80]); - - expect(avg(input_features.data)).toBeCloseTo(-0.0000010093053181966146); - expect(input_features.data[0]).toBeCloseTo(20.761859893798828); - expect(input_features.data[1]).toBeCloseTo(21.02924346923828); - expect(input_features.data[78]).toBeCloseTo(19.083993911743164); - expect(input_features.data[79]).toBeCloseTo(18.003454208374023); - expect(input_features.data[80]).toBeCloseTo(-2.595233917236328); - expect(input_features.data.at(-2)).toBeCloseTo(-2.385499954223633); - expect(input_features.data.at(-1)).toBeCloseTo(-2.2504329681396484); - } - }, - MAX_TEST_EXECUTION_TIME, - ); - }); - - describe("Other processors", () => { - describe( - "FlorenceProcessor", - () => { - /** @type {import('../src/processors.js').Florence2Processor} */ - let processor; - let images = {}; - - beforeAll(async () => { - processor = await AutoProcessor.from_pretrained(MODELS.florence2); - images = { - beetle: await load_image(TEST_IMAGES.beetle), - book_cover: await load_image(TEST_IMAGES.book_cover), - }; - }); + { + const image = await load_image(TEST_IMAGES.tiger); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); - describe("Prompt construction", () => { - it("Construct prompt", async () => { - const text = ""; - const prompts = processor.construct_prompts(text); - const target = ["Locate the objects with category name in the image."]; - compare(prompts, target); - }); - - it("Construct prompts", async () => { - const texts = ["", "Locate the objects with category name in the image.", "cat"]; - const prompts = processor.construct_prompts(texts); - const target = ["Describe with a paragraph what is shown in the image.", "Locate the objects with category name in the image.", "Locate cat in the image."]; - compare(prompts, target); - }); - }); + compare(pixel_values.dims, [1, 3, 224, 224]); + compare(avg(pixel_values.data), 0.06262318789958954); - describe("Post-process generation", () => { - const TESTS = [ - { - task: "", - generated_text: "A green car parked in front of a yellow building.", - target: { "": "A green car parked in front of a yellow building." }, - image: "beetle", + compare(original_sizes, [[408, 612]]); + compare(reshaped_input_sizes, [[224, 224]]); + } }, - { - task: "", - generated_text: "The image shows a green Volkswagen Beetle parked in front of a yellow building with two brown doors. The sky is a mix of blue and white, and there are a few green trees in the background.", - target: { "": "The image shows a green Volkswagen Beetle parked in front of a yellow building with two brown doors. The sky is a mix of blue and white, and there are a few green trees in the background." }, - image: "beetle", + MAX_TEST_EXECUTION_TIME, + ); + + // ViTFeatureExtractor + it( + MODELS.vit, + async () => { + const processor = await AutoProcessor.from_pretrained(MODELS.vit); + + { + const image = await load_image(TEST_IMAGES.tiger); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + + compare(pixel_values.dims, [1, 3, 224, 224]); + compare(avg(pixel_values.data), -0.22706867939852762); + + compare(original_sizes, [[408, 612]]); + compare(reshaped_input_sizes, [[224, 224]]); + } }, - { - task: "", - generated_text: "The image shows a vintage Volkswagen Beetle car parked on a cobblestone street in front of a yellow building with two wooden doors. The car is painted in a bright turquoise color and has a white stripe running along the side. It has two doors on either side of the car, one on top of the other, and a small window on the front. The building appears to be old and dilapidated, with peeling paint and crumbling walls. The sky is blue and there are trees in the background.", - target: { "": "The image shows a vintage Volkswagen Beetle car parked on a cobblestone street in front of a yellow building with two wooden doors. The car is painted in a bright turquoise color and has a white stripe running along the side. It has two doors on either side of the car, one on top of the other, and a small window on the front. The building appears to be old and dilapidated, with peeling paint and crumbling walls. The sky is blue and there are trees in the background." }, - image: "beetle", + MAX_TEST_EXECUTION_TIME, + ); + + // MobileViTFeatureExtractor + it( + MODELS.mobilevit, + async () => { + const processor = await AutoProcessor.from_pretrained(MODELS.mobilevit); + + { + const image = await load_image(TEST_IMAGES.tiger); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + + compare(pixel_values.dims, [1, 3, 256, 256]); + compare(avg(pixel_values.data), 0.4599160496887033); + + compare(original_sizes, [[408, 612]]); + compare(reshaped_input_sizes, [[256, 256]]); + } }, - { - task: "", - generated_text: "cardoorwheel", - target: { - "": { - bboxes: [ - [34.24, 160.08, 597.44, 371.76], - [456.0, 97.68, 580.16, 261.84], - [450.88, 276.72, 554.56, 370.8], - [95.68, 280.56, 198.72, 371.28], - ], - labels: ["car", "door", "wheel", "wheel"], - }, - }, - image: "beetle", + MAX_TEST_EXECUTION_TIME, + ); + + // MobileViTFeatureExtractor + // - tests not converting to rgb (do_convert_rgb=false) + it( + MODELS.mobilevit_2, + async () => { + const processor = await AutoProcessor.from_pretrained(MODELS.mobilevit_2); + + { + // Tests grayscale image + const image = await load_image(TEST_IMAGES.skateboard); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + + compare(pixel_values.dims, [1, 1, 28, 28]); + compare(avg(pixel_values.data), 0.08558923671585128); + + compare(original_sizes, [[28, 28]]); + compare(reshaped_input_sizes, [[28, 28]]); + } }, - { - task: "", - generated_text: "turquoise Volkswagen Beetlewheel", - target: { - "": { - bboxes: [ - [33.6, 160.08, 596.8, 371.76], - [450.88, 276.72, 553.28, 370.8], - [95.04, 280.56, 197.44, 371.28], - ], - labels: ["turquoise Volkswagen Beetle", "wheel", "wheel"], - }, - }, - image: "beetle", + MAX_TEST_EXECUTION_TIME, + ); + + // MobileViTImageProcessor + // - tests converting RGB to BGR (do_flip_channel_order=true) + it( + MODELS.mobilevit_3, + async () => { + const processor = await AutoProcessor.from_pretrained(MODELS.mobilevit_3); + + { + const image = await load_image(TEST_IMAGES.cats); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + + compare(pixel_values.dims, [1, 3, 256, 256]); + compare(avg(pixel_values.data), 0.5215385556221008); + + compare(original_sizes, [[480, 640]]); + compare(reshaped_input_sizes, [[256, 256]]); + + // Ensure RGB to BGR conversion + compare(pixel_values.data.slice(0, 3), [0.24313725531101227, 0.250980406999588, 0.364705890417099]); + } }, - { - task: "", - generated_text: "", - target: { - "": { - bboxes: [ - [33.6, 160.08, 596.8, 371.76], - [455.36, 97.68, 579.52, 261.84], - [450.88, 276.72, 553.28, 370.8], - [95.04, 280.56, 198.08, 371.28], - [226.88, 88.56, 332.48, 164.4], - [65.6, 266.64, 86.72, 295.92], - [271.68, 241.68, 302.4, 246.96], - [408.0, 308.4, 413.76, 320.88], - ], - labels: ["", "", "", "", "", "", "", ""], - }, - }, - image: "beetle", + MAX_TEST_EXECUTION_TIME, + ); + + // DeiTFeatureExtractor + it( + MODELS.deit, + async () => { + const processor = await AutoProcessor.from_pretrained(MODELS.deit); + + { + const image = await load_image(TEST_IMAGES.tiger); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + + compare(pixel_values.dims, [1, 3, 224, 224]); + compare(avg(pixel_values.data), -0.2760336682859463); + + compare(original_sizes, [[408, 612]]); + compare(reshaped_input_sizes, [[224, 224]]); + } }, - { - task: "", - text_input: "A green car parked in front of a yellow building.", - generated_text: "A green cara yellow building", - target: { - "": { - bboxes: [ - [34.88, 158.64, 583.36, 374.64], - [0.32, 4.08, 639.04, 305.04], - ], - labels: ["A green car", "a yellow building"], - }, - }, - image: "beetle", + MAX_TEST_EXECUTION_TIME, + ); + + // BeitFeatureExtractor + it( + MODELS.beit, + async () => { + const processor = await AutoProcessor.from_pretrained(MODELS.beit); + + { + const image = await load_image(TEST_IMAGES.tiger); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + + compare(pixel_values.dims, [1, 3, 224, 224]); + compare(avg(pixel_values.data), -0.22706867939852762); + + compare(original_sizes, [[408, 612]]); + compare(reshaped_input_sizes, [[224, 224]]); + } }, - // { - // task: "", - // text_input: "a green car", - // generated_text: "", - // target: { - // '': { - // polygons: [[[[178.88, 181.68, 180.8, 180.72, 182.72, 180.72, 187.84, 177.84, 189.76, 177.84, 192.96, 175.92, 194.88, 175.92, 198.08, 174, 200.64, 173.04, 203.84, 172.08, 207.04, 170.64, 209.6, 169.68, 214.08, 168.72, 217.92, 167.76, 221.76, 166.8, 226.24, 165.84, 230.72, 164.88, 237.12, 163.92, 244.16, 162.96, 253.12, 162, 265.28, 161.04, 311.36, 161.04, 329.28, 162, 338.24, 162.96, 345.28, 163.92, 350.4, 164.88, 354.24, 165.84, 358.72, 166.8, 362.56, 167.76, 366.4, 168.72, 370.24, 169.68, 373.44, 170.64, 375.36, 172.08, 377.28, 174, 379.2, 176.88, 380.48, 179.76, 382.4, 181.68, 384.32, 185.04, 386.24, 187.92, 387.52, 190.8, 389.44, 192.72, 390.08, 196.08, 392, 198.96, 394.56, 201.84, 396.48, 204.72, 398.4, 208.08, 403.52, 212.88, 406.08, 213.84, 409.28, 216.72, 412.48, 220.08, 431.68, 220.08, 432.32, 221.04, 442.56, 222, 456.64, 222, 465.6, 222.96, 472.64, 223.92, 478.4, 224.88, 484.8, 225.84, 489.92, 226.8, 493.76, 227.76, 497.6, 228.72, 501.44, 229.68, 504.64, 231.12, 507.84, 232.08, 510.4, 233.04, 513.6, 234, 516.8, 235.92, 518.72, 235.92, 523.84, 238.8, 525.76, 238.8, 527.68, 239.76, 529.6, 241.68, 532.8, 242.64, 536, 245.04, 538.56, 247.92, 541.76, 249.84, 545.6, 251.76, 548.8, 252.72, 550.72, 252.72, 553.92, 253.68, 556.48, 255.6, 558.4, 255.6, 564.8, 258.96, 566.72, 260.88, 568.64, 260.88, 570.56, 261.84, 572.48, 263.76, 573.76, 265.68, 574.4, 268.56, 574.4, 271.92, 573.76, 272.88, 572.48, 275.76, 572.48, 279.6, 573.76, 285.84, 574.4, 286.8, 575.68, 289.68, 576.32, 292.56, 577.6, 298.8, 577.6, 301.68, 576.32, 302.64, 575.68, 310.8, 575.68, 312.72, 576.32, 313.68, 577.6, 316.56, 577.6, 320.88, 574.4, 321.84, 568.64, 322.8, 559.68, 322.8, 553.92, 323.76, 552.64, 332.88, 552, 336.72, 550.72, 339.6, 550.08, 342.96, 548.8, 344.88, 546.88, 346.8, 545.6, 349.68, 543.68, 352.56, 541.76, 355.92, 534.72, 362.64, 531.52, 364.56, 525.76, 367.92, 522.56, 368.88, 518.72, 369.84, 495.68, 369.84, 489.92, 368.88, 486.72, 367.92, 483.52, 366.96, 479.68, 364.56, 476.48, 362.64, 472.64, 359.76, 465.6, 352.56, 463.68, 349.68, 461.76, 346.8, 460.48, 344.88, 460.48, 342.96, 458.56, 339.6, 457.92, 336.72, 457.92, 334.8, 456.64, 332.88, 454.72, 330.96, 452.8, 331.92, 448.32, 336.72, 446.4, 337.68, 426.56, 336.72, 424.64, 336.72, 423.36, 337.68, 420.8, 338.64, 414.4, 339.6, 412.48, 339.6, 411.2, 338.64, 380.48, 337.68, 217.28, 337.68, 216, 338.64, 210.88, 339.6, 207.04, 339.6, 203.84, 338.64, 201.92, 337.68, 200, 335.76, 198.08, 334.8, 194.88, 334.8, 192.96, 336.72, 191.68, 338.64, 191.68, 340.56, 191.04, 342.96, 189.12, 344.88, 187.84, 347.76, 185.92, 349.68, 184.64, 352.56, 182.72, 355.92, 176.96, 361.68, 173.76, 363.6, 170.56, 365.52, 166.72, 367.92, 163.52, 368.88, 160.96, 369.84, 153.92, 370.8, 131.52, 370.8, 127.68, 369.84, 124.48, 368.88, 118.72, 365.52, 115.52, 363.6, 111.68, 360.72, 106.56, 355.92, 104.64, 352.56, 103.36, 349.68, 101.44, 347.76, 100.8, 345.84, 99.52, 342.96, 99.52, 339.6, 98.88, 337.68, 95.68, 334.8, 93.76, 333.84, 86.72, 333.84, 80.32, 334.8, 79.68, 335.76, 74.56, 336.72, 66.24, 336.72, 63.68, 334.8, 53.44, 334.8, 50.24, 333.84, 48.32, 331.92, 48.32, 328.56, 50.24, 326.64, 51.52, 324.72, 51.52, 322.8, 44.48, 321.84, 40.64, 320.88, 38.72, 319.92, 37.44, 317.52, 36.16, 313.68, 36.16, 306.96, 38.72, 304.56, 42.56, 303.6, 46.4, 302.64, 55.36, 301.68, 65.6, 301.68, 67.52, 300.72, 69.44, 298.8, 70.72, 296.88, 70.72, 292.56, 69.44, 291.6, 68.8, 288.72, 67.52, 284.88, 67.52, 276.72, 68.8, 273.84, 69.44, 271.92, 72.64, 268.56, 74.56, 267.6, 77.76, 266.64, 79.68, 266.64, 81.6, 264.72, 80.32, 260.88, 81.6, 258.96, 83.52, 256.56, 88.64, 256.56, 90.56, 255.6, 92.48, 253.68, 92.48, 252.72, 97.6, 246.96, 114.88, 229.68, 117.44, 226.8, 122.56, 222.96, 125.76, 221.04, 126.4, 221.04, 129.6, 219.12, 133.44, 215.76, 138.56, 211.92, 143.68, 208.08, 149.44, 201.84, 153.92, 198.96, 154.56, 198.96, 157.76, 197.04, 162.88, 192.72, 168.64, 186.96, 171.84, 185.04, 176.96, 183.12, 178.88, 180.72]]]], - // labels: [''], - // } - // }, - // image: 'beetle', - // }, - // { - // task: "", - // text_input: "", - // generated_text: "", - // target: { - // '': { - // polygons: [[[[470.08, 288.24, 473.92, 285.36, 477.12, 283.44, 479.04, 282.48, 480.96, 282.48, 484.16, 280.56, 486.72, 279.6, 489.92, 278.64, 495.04, 277.68, 512.32, 277.68, 514.88, 278.64, 518.08, 279.6, 521.28, 281.52, 523.2, 281.52, 525.12, 283.44, 528.32, 284.4, 530.88, 286.32, 534.08, 288.24, 543.04, 297.36, 544.96, 300.24, 546.88, 303.12, 550.08, 309.36, 551.36, 312.24, 552, 315.12, 553.28, 319.44, 553.28, 332.4, 552, 337.2, 551.36, 340.08, 550.08, 343.44, 548.16, 347.28, 546.24, 350.16, 544.32, 353.04, 541.12, 357.36, 537.28, 361.2, 532.16, 365.04, 528.96, 366.96, 527.04, 367.92, 523.84, 368.88, 521.28, 369.84, 516.16, 371.28, 500.8, 371.28, 491.84, 369.84, 488, 368.88, 484.8, 367.92, 479.04, 365.04, 475.84, 363.12, 472, 360.24, 464.96, 353.04, 463.04, 350.16, 461.12, 347.28, 459.84, 345.36, 459.84, 343.44, 457.92, 340.08, 456.64, 337.2, 456, 334.32, 454.72, 330.48, 454.72, 316.08, 456, 311.28, 456.64, 307.44, 457.92, 304.08, 459.84, 301.2, 459.84, 299.28, 461.12, 297.36, 463.04, 294.48]]]], - // labels: [''], - // } - // }, - // image: 'beetle', - // }, - // { - // task: "", - // text_input: "a green car", - // generated_text: "a green car", - // target: { - // '': { - // bboxes: [[34.24, 158.64, 582.72, 374.16]], - // bboxes_labels: ['a green car'], - // polygons: [], - // polygons_labels: [], - // } - // }, - // image: 'beetle', - // }, - { - task: "", - text_input: "", - generated_text: "car", - target: { "": "car" }, - image: "beetle", + MAX_TEST_EXECUTION_TIME, + ); + + // DetrFeatureExtractor + it( + MODELS.detr, + async () => { + const processor = await AutoProcessor.from_pretrained(MODELS.detr); + + { + const image = await load_image(TEST_IMAGES.tiger); + const { pixel_values, original_sizes, reshaped_input_sizes, pixel_mask } = await processor(image); + + compare(pixel_values.dims, [1, 3, 888, 1333]); + compare(avg(pixel_values.data), -0.27840224131001773); + + compare(original_sizes, [[408, 612]]); + compare(reshaped_input_sizes, [[888, 1333]]); + + compare(pixel_mask.dims, [1, 64, 64]); + compare(avg(pixel_mask.data), 1); + } }, - { - task: "", - text_input: "", - generated_text: "turquoise Volkswagen Beetle", - target: { "": "turquoise Volkswagen Beetle" }, - image: "beetle", + MAX_TEST_EXECUTION_TIME, + ); + + // YolosFeatureExtractor + it( + MODELS.yolos, + async () => { + const processor = await AutoProcessor.from_pretrained(MODELS.yolos); + + { + const image = await load_image(TEST_IMAGES.tiger); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + + compare(pixel_values.dims, [1, 3, 888, 1333]); + compare(avg(pixel_values.data), -0.27840224131001773); + + compare(original_sizes, [[408, 612]]); + compare(reshaped_input_sizes, [[888, 1333]]); + } }, - { - task: "", - generated_text: "CUDAFOR ENGINEERSAn Introduction to High-PerformanceParallel ComputingDUANE STORTIMETE YURTOGLU", - target: { "": "CUDAFOR ENGINEERSAn Introduction to High-PerformanceParallel ComputingDUANE STORTIMETE YURTOGLU" }, - image: "book_cover", + MAX_TEST_EXECUTION_TIME, + ); + + // DPTFeatureExtractor + it( + MODELS.dpt, + async () => { + const processor = await AutoProcessor.from_pretrained(MODELS.dpt); + + { + // Tests grayscale image + const image = await load_image(TEST_IMAGES.cats); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + + compare(pixel_values.dims, [1, 3, 384, 384]); + compare(avg(pixel_values.data), 0.0372855559389454); + + compare(original_sizes, [[480, 640]]); + compare(reshaped_input_sizes, [[384, 384]]); + } + }, + MAX_TEST_EXECUTION_TIME, + ); + + // GLPNForDepthEstimation + // - tests `size_divisor` and no size (size_divisor=32) + it( + MODELS.glpn, + async () => { + const processor = await AutoProcessor.from_pretrained(MODELS.glpn); + + { + const image = await load_image(TEST_IMAGES.cats); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + compare(pixel_values.dims, [1, 3, 480, 640]); + compare(avg(pixel_values.data), 0.5186172404123327); + + compare(original_sizes, [[480, 640]]); + compare(reshaped_input_sizes, [[480, 640]]); + } + + { + // Tests input which is not a multiple of 32 ([408, 612] -> [384, 608]) + const image = await load_image(TEST_IMAGES.tiger); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + + compare(pixel_values.dims, [1, 3, 384, 608]); + compare(avg(pixel_values.data), 0.38628831535989555); + + compare(original_sizes, [[408, 612]]); + compare(reshaped_input_sizes, [[384, 608]]); + } }, + MAX_TEST_EXECUTION_TIME, + ); + + // NougatImageProcessor + // - tests padding after normalization (image_mean != 0.5, image_std != 0.5) + it( + MODELS.nougat, + async () => { + const processor = await AutoProcessor.from_pretrained(MODELS.nougat); + + { + const image = await load_image(TEST_IMAGES.paper); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + + compare(pixel_values.dims, [1, 3, 896, 672]); + compare(avg(pixel_values.data), 1.8447155005897355); + + compare(original_sizes, [[850, 685]]); + compare(reshaped_input_sizes, [[833, 672]]); + } + }, + MAX_TEST_EXECUTION_TIME, + ); + + // OwlViTFeatureExtractor + it(MODELS.owlvit, async () => { + const processor = await AutoProcessor.from_pretrained(MODELS.owlvit); { - task: "", - generated_text: "CUDAFOR ENGINEERSAn Introduction to High-PerformanceParallel ComputingDUANE STORTIMETE YURTOGLU", - target: { - "": { - quad_boxes: [ - [167.0435028076172, 50.25, 375.7974853515625, 50.25, 375.7974853515625, 114.75, 167.0435028076172, 114.75], - [144.8784942626953, 120.75, 375.7974853515625, 120.75, 375.7974853515625, 149.25, 144.8784942626953, 149.25], - [115.86249542236328, 165.25, 376.6034851074219, 166.25, 376.6034851074219, 184.25, 115.86249542236328, 183.25], - [239.9864959716797, 184.25, 376.6034851074219, 186.25, 376.6034851074219, 204.25, 239.9864959716797, 202.25], - [266.1814880371094, 441.25, 376.6034851074219, 441.25, 376.6034851074219, 456.25, 266.1814880371094, 456.25], - [252.0764923095703, 460.25, 376.6034851074219, 460.25, 376.6034851074219, 475.25, 252.0764923095703, 475.25], - ], - - // NOTE: Python version has a bug here, it should be "CUDA" instead of "CUDA" - labels: [/* 'CUDA' */ "CUDA", "FOR ENGINEERS", "An Introduction to High-Performance", "Parallel Computing", "DUANE STORTI", "METE YURTOGLU"], - }, - }, - image: "book_cover", + const image = await load_image(TEST_IMAGES.cats); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + + compare(pixel_values.dims, [1, 3, 768, 768]); + compare(avg(pixel_values.data), 0.250620447910435); + + compare(original_sizes, [[480, 640]]); + compare(reshaped_input_sizes, [[768, 768]]); + } + }); + + // CLIPFeatureExtractor + // - tests center crop (do_center_crop=true, crop_size=224) + it( + MODELS.clip, + async () => { + const processor = await AutoProcessor.from_pretrained(MODELS.clip); + + { + const image = await load_image(TEST_IMAGES.tiger); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + + compare(pixel_values.dims, [1, 3, 224, 224]); + compare(avg(pixel_values.data), -0.06678297738282096); + + compare(original_sizes, [[408, 612]]); + compare(reshaped_input_sizes, [[224, 224]]); + } }, - ]; - - for (const { task, generated_text, target, image } of TESTS) { - it(task, () => { - const result = processor.post_process_generation(generated_text, task, images[image].size); - compare(result, target); - }); - } + MAX_TEST_EXECUTION_TIME, + ); + + // VitMatteImageProcessor + // - tests custom overrides + // - tests multiple inputs + // - tests `size_divisibility` and no size (size_divisibility=32) + // - tests do_pad and `size_divisibility` + it( + MODELS.vitmatte, + async () => { + const processor = await AutoProcessor.from_pretrained(MODELS.vitmatte); + + { + const image = await load_image(TEST_IMAGES.vitmatte_image); + const image2 = await load_image(TEST_IMAGES.vitmatte_trimap); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image, image2); + + compare(pixel_values.dims, [1, 4, 640, 960]); + expect(avg(pixel_values.data)).toBeCloseTo(-0.4028555154800415); + expect(pixel_values.data[0]).toBeCloseTo(-0.9921568632125854); + expect(pixel_values.data[1]).toBeCloseTo(-0.9921568632125854); + expect(pixel_values.data[5]).toBeCloseTo(-1.0); + expect(pixel_values.data[640]).toBeCloseTo(-0.6784313917160034); + expect(pixel_values.data[641]).toBeCloseTo(-0.6705882549285889); + expect(pixel_values.data[640 * 960]).toBeCloseTo(-1.0); + expect(pixel_values.data[640 * 960 + 1]).toBeCloseTo(-1.0); + expect(pixel_values.data.at(-1)).toBeCloseTo(0.0); + + compare(original_sizes, [[640, 960]]); + compare(reshaped_input_sizes, [[640, 960]]); + } + + { + const image = await load_image(TEST_IMAGES.pattern_3x5); + const image2 = await load_image(TEST_IMAGES.pattern_3x5); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image, image2); + + compare(pixel_values.dims, [1, 4, 32, 32]); + expect(avg(pixel_values.data)).toBeCloseTo(-0.00867417361587286); + expect(pixel_values.data[0]).toBeCloseTo(-0.9921568632125854); + expect(pixel_values.data[1]).toBeCloseTo(-0.9686274528503418); + expect(pixel_values.data[5]).toBeCloseTo(0.0); + expect(pixel_values.data[32]).toBeCloseTo(-0.9215686321258545); + expect(pixel_values.data[33]).toBeCloseTo(-0.8980392217636108); + expect(pixel_values.data.at(-1)).toBeCloseTo(0.0); + + compare(original_sizes, [[5, 3]]); + compare(reshaped_input_sizes, [[5, 3]]); + } + }, + MAX_TEST_EXECUTION_TIME, + ); + + // BitImageProcessor + it( + MODELS.dinov2, + async () => { + const processor = await AutoProcessor.from_pretrained(MODELS.dinov2); + + { + const image = await load_image(TEST_IMAGES.tiger); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + + compare(pixel_values.dims, [1, 3, 224, 224]); + compare(avg(pixel_values.data), 0.06262318789958954); + + compare(original_sizes, [[408, 612]]); + compare(reshaped_input_sizes, [[224, 224]]); + } + }, + MAX_TEST_EXECUTION_TIME, + ); + + // DPTImageProcessor + // - tests ensure_multiple_of + // - tests keep_aspect_ratio + // - tests bankers rounding + it( + MODELS.dpt_2, + async () => { + const processor = await AutoProcessor.from_pretrained(MODELS.dpt_2); + + { + const image = await load_image(TEST_IMAGES.cats); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + + compare(pixel_values.dims, [1, 3, 518, 686]); + compare(avg(pixel_values.data), 0.30337387323379517); + + compare(original_sizes, [[480, 640]]); + compare(reshaped_input_sizes, [[518, 686]]); + } + + { + const image = await load_image(TEST_IMAGES.checkerboard_64x32); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + + // NOTE: without bankers rounding, this would be [1, 3, 266, 518] + compare(pixel_values.dims, [1, 3, 252, 518]); + compare(avg(pixel_values.data), 0.2267402559518814); + + compare(original_sizes, [[32, 64]]); + compare(reshaped_input_sizes, [[252, 518]]); + } + }, + MAX_TEST_EXECUTION_TIME, + ); + + // TODO: Add back + // // EfficientNetImageProcessor + // // - tests include_top + // it(MODELS.efficientnet, async () => { + // const processor = await AutoProcessor.from_pretrained(MODELS.efficientnet) + + // { + // const image = await load_image(TEST_IMAGES.cats); + // const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + + // compare(pixel_values.dims, [1, 3, 224, 224]); + // compare(avg(pixel_values.data), 0.3015307230282871); + + // compare(original_sizes, [[480, 640]]); + // compare(reshaped_input_sizes, [[224, 224]]); + // } + // }, MAX_TEST_EXECUTION_TIME); + }); + + describe("Audio processors", () => { + const audioPromise = new Promise(async (resolve) => { + const url = "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/mlk.npy"; + const buffer = await (await fetch(url)).arrayBuffer(); + const audio = Float32Array.from(new Float64Array(buffer)); + resolve(audio); }); - }, - MAX_TEST_EXECUTION_TIME, - ); - }); + + it( + "WhisperFeatureExtractor", + async () => { + const audio = await audioPromise; + const processor = await AutoProcessor.from_pretrained("Xenova/whisper-tiny.en"); + const { input_features } = await processor(audio); + compare(input_features.dims, [1, 80, 3000]); + expect(avg(input_features.data)).toBeCloseTo(-0.2813588131551941); + expect(input_features.data[0]).toBeCloseTo(0.33168578147888184); + expect(input_features.data[1]).toBeCloseTo(0.30986475944519043); + expect(input_features.data[81]).toBeCloseTo(0.10727232694625854); + expect(input_features.data[3001]).toBeCloseTo(0.2555035352706909); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "ASTFeatureExtractor", + async () => { + const audio = await audioPromise; + const processor = await AutoProcessor.from_pretrained("Xenova/ast-finetuned-audioset-10-10-0.4593"); + { + // truncation + const { input_values } = await processor(audio); + compare(input_values.dims, [1, 1024, 128]); + + expect(avg(input_values.data)).toBeCloseTo(-0.04054912979309085); + expect(input_values.data[0]).toBeCloseTo(-0.5662586092948914); + expect(input_values.data[1]).toBeCloseTo(-1.0300861597061157); + expect(input_values.data[129]).toBeCloseTo(-1.084834098815918); + expect(input_values.data[1025]).toBeCloseTo(-1.1204065084457397); + } + { + // padding + const { input_values } = await processor(audio.slice(0, 1000)); + compare(input_values.dims, [1, 1024, 128]); // [1, 4, 128] -> (padded to) -> [1, 1024, 128] + + expect(avg(input_values.data)).toBeCloseTo(0.4647964835166931); + expect(input_values.data[0]).toBeCloseTo(-0.5662586092948914); + expect(input_values.data[1]).toBeCloseTo(-1.0300861597061157); + expect(input_values.data[129]).toBeCloseTo(-1.084834098815918); + + // padded values + expect(input_values.data[1025]).toBeCloseTo(0.46703237295150757); + expect(input_values.data[2049]).toBeCloseTo(0.46703237295150757); + expect(input_values.data[10000]).toBeCloseTo(0.46703237295150757); + } + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "SeamlessM4TFeatureExtractor", + async () => { + const audio = await audioPromise; + const processor = await AutoProcessor.from_pretrained("Xenova/wav2vec2-bert-CV16-en"); + { + // normal + const { input_features, attention_mask } = await processor(audio); + compare(input_features.dims, [1, 649, 160]); + compare(attention_mask.dims, [1, 649]); + + expect(avg(input_features.data)).toBeCloseTo(-2.938903875815413e-8); + expect(input_features.data[0]).toBeCloseTo(1.1939343214035034); + expect(input_features.data[1]).toBeCloseTo(0.7874255180358887); + expect(input_features.data[160]).toBeCloseTo(-0.712975025177002); + expect(input_features.data[161]).toBeCloseTo(0.045802414417266846); + expect(input_features.data.at(-1)).toBeCloseTo(-1.3328346014022827); + + expect(sum(attention_mask.data)).toEqual(649); + } + { + // padding (pad_to_multiple_of=2) + const { input_features, attention_mask } = await processor(audio.slice(0, 10000)); + + // [1, 61, 80] -> [1, 62, 80] -> [1, 31, 160] + compare(input_features.dims, [1, 31, 160]); + compare(attention_mask.dims, [1, 31]); + + expect(avg(input_features.data)).toBeCloseTo(0.01612919569015503); + expect(input_features.data[0]).toBeCloseTo(0.9657132029533386); + expect(input_features.data[1]).toBeCloseTo(0.12912897765636444); + expect(input_features.data[160]).toBeCloseTo(-1.2364212274551392); + expect(input_features.data[161]).toBeCloseTo(-0.9703778028488159); + expect(input_features.data.at(-1)).toBeCloseTo(1); // padding value + + expect(sum(attention_mask.data)).toEqual(30); + } + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "ClapFeatureExtractor", + async () => { + const audio = await audioPromise; + const processor = await AutoProcessor.from_pretrained("Xenova/clap-htsat-unfused"); + { + // truncation + // Since truncation uses a random strategy, we override + // Math.random to ensure that the test is deterministic + const originalRandom = Math.random; + Math.random = () => 0.5; + + let long_audio = new Float32Array(500000); + long_audio.set(audio); + long_audio.set(audio, long_audio.length - audio.length); + + const { input_features } = await processor(long_audio); + compare(input_features.dims, [1, 1, 1001, 64]); + + expect(avg(input_features.data)).toBeCloseTo(-37.94569396972656); + expect(input_features.data[0]).toBeCloseTo(-53.32647705078125); + expect(input_features.data[1]).toBeCloseTo(-47.76755142211914); + expect(input_features.data[65]).toBeCloseTo(-36.32261276245117); + expect(input_features.data[1002]).toBeCloseTo(-28.0314884185791); + expect(input_features.data[10000]).toBeCloseTo(-21.905902862548828); + expect(input_features.data[60000]).toBeCloseTo(-14.877863883972168); + expect(input_features.data[64062]).toBeCloseTo(-37.9784049987793); + expect(input_features.data[64063]).toBeCloseTo(-37.73963928222656); + + // Reset Math.random + Math.random = originalRandom; + } + { + // padding + const { input_features } = await processor(audio); + compare(input_features.dims, [1, 1, 1001, 64]); + + expect(avg(input_features.data)).toBeCloseTo(-34.99049377441406); + expect(input_features.data[0]).toBeCloseTo(-21.32573890686035); + expect(input_features.data[1]).toBeCloseTo(-26.168411254882812); + expect(input_features.data[65]).toBeCloseTo(-29.716018676757812); + expect(input_features.data[1002]).toBeCloseTo(-32.16273498535156); + expect(input_features.data[10000]).toBeCloseTo(-19.9283390045166); + + // padded values + expect(input_features.data[60000]).toBeCloseTo(-100.0); + expect(input_features.data[64062]).toBeCloseTo(-100.0); + expect(input_features.data[64063]).toBeCloseTo(-100.0); + } + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "WeSpeakerFeatureExtractor", + async () => { + const processor = await AutoProcessor.from_pretrained("onnx-community/wespeaker-voxceleb-resnet34-LM"); + { + // default + const audio = new Float32Array(16000).map((_, i) => Math.sin(i / 100)); + const { input_features } = await processor(audio); + compare(input_features.dims, [1, 98, 80]); + + expect(avg(input_features.data)).toBeCloseTo(5.461731689138105e-8); + expect(input_features.data[0]).toBeCloseTo(-0.19300270080566406); + expect(input_features.data[1]).toBeCloseTo(-0.05825042724609375); + expect(input_features.data[78]).toBeCloseTo(0.2683420181274414); + expect(input_features.data[79]).toBeCloseTo(0.26250171661376953); + expect(input_features.data[80]).toBeCloseTo(0.19062232971191406); + expect(input_features.data.at(-2)).toBeCloseTo(-0.43694400787353516); + expect(input_features.data.at(-1)).toBeCloseTo(-0.4266204833984375); + } + + { + // pad to `min_num_frames` + const audio = new Float32Array(3).map((_, i) => Math.sin(i / 100)); + const { input_features } = await processor(audio); + compare(input_features.dims, [1, 9, 80]); + + expect(avg(input_features.data)).toBeCloseTo(-0.0000010093053181966146); + expect(input_features.data[0]).toBeCloseTo(20.761859893798828); + expect(input_features.data[1]).toBeCloseTo(21.02924346923828); + expect(input_features.data[78]).toBeCloseTo(19.083993911743164); + expect(input_features.data[79]).toBeCloseTo(18.003454208374023); + expect(input_features.data[80]).toBeCloseTo(-2.595233917236328); + expect(input_features.data.at(-2)).toBeCloseTo(-2.385499954223633); + expect(input_features.data.at(-1)).toBeCloseTo(-2.2504329681396484); + } + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("Other processors", () => { + describe( + "FlorenceProcessor", + () => { + /** @type {import('../src/processors.js').Florence2Processor} */ + let processor; + let images = {}; + + beforeAll(async () => { + processor = await AutoProcessor.from_pretrained(MODELS.florence2); + images = { + beetle: await load_image(TEST_IMAGES.beetle), + book_cover: await load_image(TEST_IMAGES.book_cover), + }; + }); + + describe("Prompt construction", () => { + it("Construct prompt", async () => { + const text = ""; + const prompts = processor.construct_prompts(text); + const target = ["Locate the objects with category name in the image."]; + compare(prompts, target); + }); + + it("Construct prompts", async () => { + const texts = ["", "Locate the objects with category name in the image.", "cat"]; + const prompts = processor.construct_prompts(texts); + const target = ["Describe with a paragraph what is shown in the image.", "Locate the objects with category name in the image.", "Locate cat in the image."]; + compare(prompts, target); + }); + }); + + describe("Post-process generation", () => { + const TESTS = [ + { + task: "", + generated_text: "A green car parked in front of a yellow building.", + target: { + "": "A green car parked in front of a yellow building.", + }, + image: "beetle", + }, + { + task: "", + generated_text: "The image shows a green Volkswagen Beetle parked in front of a yellow building with two brown doors. The sky is a mix of blue and white, and there are a few green trees in the background.", + target: { + "": "The image shows a green Volkswagen Beetle parked in front of a yellow building with two brown doors. The sky is a mix of blue and white, and there are a few green trees in the background.", + }, + image: "beetle", + }, + { + task: "", + generated_text: "The image shows a vintage Volkswagen Beetle car parked on a cobblestone street in front of a yellow building with two wooden doors. The car is painted in a bright turquoise color and has a white stripe running along the side. It has two doors on either side of the car, one on top of the other, and a small window on the front. The building appears to be old and dilapidated, with peeling paint and crumbling walls. The sky is blue and there are trees in the background.", + target: { + "": "The image shows a vintage Volkswagen Beetle car parked on a cobblestone street in front of a yellow building with two wooden doors. The car is painted in a bright turquoise color and has a white stripe running along the side. It has two doors on either side of the car, one on top of the other, and a small window on the front. The building appears to be old and dilapidated, with peeling paint and crumbling walls. The sky is blue and there are trees in the background.", + }, + image: "beetle", + }, + { + task: "", + generated_text: "cardoorwheel", + target: { + "": { + bboxes: [ + [34.24, 160.08, 597.44, 371.76], + [456.0, 97.68, 580.16, 261.84], + [450.88, 276.72, 554.56, 370.8], + [95.68, 280.56, 198.72, 371.28], + ], + labels: ["car", "door", "wheel", "wheel"], + }, + }, + image: "beetle", + }, + { + task: "", + generated_text: "turquoise Volkswagen Beetlewheel", + target: { + "": { + bboxes: [ + [33.6, 160.08, 596.8, 371.76], + [450.88, 276.72, 553.28, 370.8], + [95.04, 280.56, 197.44, 371.28], + ], + labels: ["turquoise Volkswagen Beetle", "wheel", "wheel"], + }, + }, + image: "beetle", + }, + { + task: "", + generated_text: "", + target: { + "": { + bboxes: [ + [33.6, 160.08, 596.8, 371.76], + [455.36, 97.68, 579.52, 261.84], + [450.88, 276.72, 553.28, 370.8], + [95.04, 280.56, 198.08, 371.28], + [226.88, 88.56, 332.48, 164.4], + [65.6, 266.64, 86.72, 295.92], + [271.68, 241.68, 302.4, 246.96], + [408.0, 308.4, 413.76, 320.88], + ], + labels: ["", "", "", "", "", "", "", ""], + }, + }, + image: "beetle", + }, + { + task: "", + text_input: "A green car parked in front of a yellow building.", + generated_text: "A green cara yellow building", + target: { + "": { + bboxes: [ + [34.88, 158.64, 583.36, 374.64], + [0.32, 4.08, 639.04, 305.04], + ], + labels: ["A green car", "a yellow building"], + }, + }, + image: "beetle", + }, + // { + // task: "", + // text_input: "a green car", + // generated_text: "", + // target: { + // '': { + // polygons: [[[[178.88, 181.68, 180.8, 180.72, 182.72, 180.72, 187.84, 177.84, 189.76, 177.84, 192.96, 175.92, 194.88, 175.92, 198.08, 174, 200.64, 173.04, 203.84, 172.08, 207.04, 170.64, 209.6, 169.68, 214.08, 168.72, 217.92, 167.76, 221.76, 166.8, 226.24, 165.84, 230.72, 164.88, 237.12, 163.92, 244.16, 162.96, 253.12, 162, 265.28, 161.04, 311.36, 161.04, 329.28, 162, 338.24, 162.96, 345.28, 163.92, 350.4, 164.88, 354.24, 165.84, 358.72, 166.8, 362.56, 167.76, 366.4, 168.72, 370.24, 169.68, 373.44, 170.64, 375.36, 172.08, 377.28, 174, 379.2, 176.88, 380.48, 179.76, 382.4, 181.68, 384.32, 185.04, 386.24, 187.92, 387.52, 190.8, 389.44, 192.72, 390.08, 196.08, 392, 198.96, 394.56, 201.84, 396.48, 204.72, 398.4, 208.08, 403.52, 212.88, 406.08, 213.84, 409.28, 216.72, 412.48, 220.08, 431.68, 220.08, 432.32, 221.04, 442.56, 222, 456.64, 222, 465.6, 222.96, 472.64, 223.92, 478.4, 224.88, 484.8, 225.84, 489.92, 226.8, 493.76, 227.76, 497.6, 228.72, 501.44, 229.68, 504.64, 231.12, 507.84, 232.08, 510.4, 233.04, 513.6, 234, 516.8, 235.92, 518.72, 235.92, 523.84, 238.8, 525.76, 238.8, 527.68, 239.76, 529.6, 241.68, 532.8, 242.64, 536, 245.04, 538.56, 247.92, 541.76, 249.84, 545.6, 251.76, 548.8, 252.72, 550.72, 252.72, 553.92, 253.68, 556.48, 255.6, 558.4, 255.6, 564.8, 258.96, 566.72, 260.88, 568.64, 260.88, 570.56, 261.84, 572.48, 263.76, 573.76, 265.68, 574.4, 268.56, 574.4, 271.92, 573.76, 272.88, 572.48, 275.76, 572.48, 279.6, 573.76, 285.84, 574.4, 286.8, 575.68, 289.68, 576.32, 292.56, 577.6, 298.8, 577.6, 301.68, 576.32, 302.64, 575.68, 310.8, 575.68, 312.72, 576.32, 313.68, 577.6, 316.56, 577.6, 320.88, 574.4, 321.84, 568.64, 322.8, 559.68, 322.8, 553.92, 323.76, 552.64, 332.88, 552, 336.72, 550.72, 339.6, 550.08, 342.96, 548.8, 344.88, 546.88, 346.8, 545.6, 349.68, 543.68, 352.56, 541.76, 355.92, 534.72, 362.64, 531.52, 364.56, 525.76, 367.92, 522.56, 368.88, 518.72, 369.84, 495.68, 369.84, 489.92, 368.88, 486.72, 367.92, 483.52, 366.96, 479.68, 364.56, 476.48, 362.64, 472.64, 359.76, 465.6, 352.56, 463.68, 349.68, 461.76, 346.8, 460.48, 344.88, 460.48, 342.96, 458.56, 339.6, 457.92, 336.72, 457.92, 334.8, 456.64, 332.88, 454.72, 330.96, 452.8, 331.92, 448.32, 336.72, 446.4, 337.68, 426.56, 336.72, 424.64, 336.72, 423.36, 337.68, 420.8, 338.64, 414.4, 339.6, 412.48, 339.6, 411.2, 338.64, 380.48, 337.68, 217.28, 337.68, 216, 338.64, 210.88, 339.6, 207.04, 339.6, 203.84, 338.64, 201.92, 337.68, 200, 335.76, 198.08, 334.8, 194.88, 334.8, 192.96, 336.72, 191.68, 338.64, 191.68, 340.56, 191.04, 342.96, 189.12, 344.88, 187.84, 347.76, 185.92, 349.68, 184.64, 352.56, 182.72, 355.92, 176.96, 361.68, 173.76, 363.6, 170.56, 365.52, 166.72, 367.92, 163.52, 368.88, 160.96, 369.84, 153.92, 370.8, 131.52, 370.8, 127.68, 369.84, 124.48, 368.88, 118.72, 365.52, 115.52, 363.6, 111.68, 360.72, 106.56, 355.92, 104.64, 352.56, 103.36, 349.68, 101.44, 347.76, 100.8, 345.84, 99.52, 342.96, 99.52, 339.6, 98.88, 337.68, 95.68, 334.8, 93.76, 333.84, 86.72, 333.84, 80.32, 334.8, 79.68, 335.76, 74.56, 336.72, 66.24, 336.72, 63.68, 334.8, 53.44, 334.8, 50.24, 333.84, 48.32, 331.92, 48.32, 328.56, 50.24, 326.64, 51.52, 324.72, 51.52, 322.8, 44.48, 321.84, 40.64, 320.88, 38.72, 319.92, 37.44, 317.52, 36.16, 313.68, 36.16, 306.96, 38.72, 304.56, 42.56, 303.6, 46.4, 302.64, 55.36, 301.68, 65.6, 301.68, 67.52, 300.72, 69.44, 298.8, 70.72, 296.88, 70.72, 292.56, 69.44, 291.6, 68.8, 288.72, 67.52, 284.88, 67.52, 276.72, 68.8, 273.84, 69.44, 271.92, 72.64, 268.56, 74.56, 267.6, 77.76, 266.64, 79.68, 266.64, 81.6, 264.72, 80.32, 260.88, 81.6, 258.96, 83.52, 256.56, 88.64, 256.56, 90.56, 255.6, 92.48, 253.68, 92.48, 252.72, 97.6, 246.96, 114.88, 229.68, 117.44, 226.8, 122.56, 222.96, 125.76, 221.04, 126.4, 221.04, 129.6, 219.12, 133.44, 215.76, 138.56, 211.92, 143.68, 208.08, 149.44, 201.84, 153.92, 198.96, 154.56, 198.96, 157.76, 197.04, 162.88, 192.72, 168.64, 186.96, 171.84, 185.04, 176.96, 183.12, 178.88, 180.72]]]], + // labels: [''], + // } + // }, + // image: 'beetle', + // }, + // { + // task: "", + // text_input: "", + // generated_text: "", + // target: { + // '': { + // polygons: [[[[470.08, 288.24, 473.92, 285.36, 477.12, 283.44, 479.04, 282.48, 480.96, 282.48, 484.16, 280.56, 486.72, 279.6, 489.92, 278.64, 495.04, 277.68, 512.32, 277.68, 514.88, 278.64, 518.08, 279.6, 521.28, 281.52, 523.2, 281.52, 525.12, 283.44, 528.32, 284.4, 530.88, 286.32, 534.08, 288.24, 543.04, 297.36, 544.96, 300.24, 546.88, 303.12, 550.08, 309.36, 551.36, 312.24, 552, 315.12, 553.28, 319.44, 553.28, 332.4, 552, 337.2, 551.36, 340.08, 550.08, 343.44, 548.16, 347.28, 546.24, 350.16, 544.32, 353.04, 541.12, 357.36, 537.28, 361.2, 532.16, 365.04, 528.96, 366.96, 527.04, 367.92, 523.84, 368.88, 521.28, 369.84, 516.16, 371.28, 500.8, 371.28, 491.84, 369.84, 488, 368.88, 484.8, 367.92, 479.04, 365.04, 475.84, 363.12, 472, 360.24, 464.96, 353.04, 463.04, 350.16, 461.12, 347.28, 459.84, 345.36, 459.84, 343.44, 457.92, 340.08, 456.64, 337.2, 456, 334.32, 454.72, 330.48, 454.72, 316.08, 456, 311.28, 456.64, 307.44, 457.92, 304.08, 459.84, 301.2, 459.84, 299.28, 461.12, 297.36, 463.04, 294.48]]]], + // labels: [''], + // } + // }, + // image: 'beetle', + // }, + // { + // task: "", + // text_input: "a green car", + // generated_text: "a green car", + // target: { + // '': { + // bboxes: [[34.24, 158.64, 582.72, 374.16]], + // bboxes_labels: ['a green car'], + // polygons: [], + // polygons_labels: [], + // } + // }, + // image: 'beetle', + // }, + { + task: "", + text_input: "", + generated_text: "car", + target: { + "": "car", + }, + image: "beetle", + }, + { + task: "", + text_input: "", + generated_text: "turquoise Volkswagen Beetle", + target: { + "": "turquoise Volkswagen Beetle", + }, + image: "beetle", + }, + { + task: "", + generated_text: "CUDAFOR ENGINEERSAn Introduction to High-PerformanceParallel ComputingDUANE STORTIMETE YURTOGLU", + target: { + "": "CUDAFOR ENGINEERSAn Introduction to High-PerformanceParallel ComputingDUANE STORTIMETE YURTOGLU", + }, + image: "book_cover", + }, + { + task: "", + generated_text: "CUDAFOR ENGINEERSAn Introduction to High-PerformanceParallel ComputingDUANE STORTIMETE YURTOGLU", + target: { + "": { + quad_boxes: [ + [167.0435028076172, 50.25, 375.7974853515625, 50.25, 375.7974853515625, 114.75, 167.0435028076172, 114.75], + [144.8784942626953, 120.75, 375.7974853515625, 120.75, 375.7974853515625, 149.25, 144.8784942626953, 149.25], + [115.86249542236328, 165.25, 376.6034851074219, 166.25, 376.6034851074219, 184.25, 115.86249542236328, 183.25], + [239.9864959716797, 184.25, 376.6034851074219, 186.25, 376.6034851074219, 204.25, 239.9864959716797, 202.25], + [266.1814880371094, 441.25, 376.6034851074219, 441.25, 376.6034851074219, 456.25, 266.1814880371094, 456.25], + [252.0764923095703, 460.25, 376.6034851074219, 460.25, 376.6034851074219, 475.25, 252.0764923095703, 475.25], + ], + + // NOTE: Python version has a bug here, it should be "CUDA" instead of "CUDA" + labels: [/* 'CUDA' */ "CUDA", "FOR ENGINEERS", "An Introduction to High-Performance", "Parallel Computing", "DUANE STORTI", "METE YURTOGLU"], + }, + }, + image: "book_cover", + }, + ]; + + for (const { task, generated_text, target, image } of TESTS) { + it(task, () => { + const result = processor.post_process_generation(generated_text, task, images[image].size); + compare(result, target); + }); + } + }); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); }); diff --git a/tests/test_utils.js b/tests/test_utils.js index 9928bf75b..752f999dc 100644 --- a/tests/test_utils.js +++ b/tests/test_utils.js @@ -1,30 +1,30 @@ export async function loadAudio(url) { - // NOTE: Since the Web Audio API is not available in Node.js, we will need to use the `wavefile` library to obtain the raw audio data. - // For more information, see: https://huggingface.co/docs/transformers.js/guides/node-audio-processing - let wavefile = (await import("wavefile")).default; + // NOTE: Since the Web Audio API is not available in Node.js, we will need to use the `wavefile` library to obtain the raw audio data. + // For more information, see: https://huggingface.co/docs/transformers.js/guides/node-audio-processing + let wavefile = (await import("wavefile")).default; - // Load audio data - let buffer = Buffer.from(await fetch(url).then((x) => x.arrayBuffer())); + // Load audio data + let buffer = Buffer.from(await fetch(url).then((x) => x.arrayBuffer())); - // Read .wav file and convert it to required format - let wav = new wavefile.WaveFile(buffer); - wav.toBitDepth("32f"); // Pipeline expects input as a Float32Array - wav.toSampleRate(16000); // Whisper expects audio with a sampling rate of 16000 - let audioData = wav.getSamples(); - if (Array.isArray(audioData)) { - if (audioData.length > 1) { - const SCALING_FACTOR = Math.sqrt(2); + // Read .wav file and convert it to required format + let wav = new wavefile.WaveFile(buffer); + wav.toBitDepth("32f"); // Pipeline expects input as a Float32Array + wav.toSampleRate(16000); // Whisper expects audio with a sampling rate of 16000 + let audioData = wav.getSamples(); + if (Array.isArray(audioData)) { + if (audioData.length > 1) { + const SCALING_FACTOR = Math.sqrt(2); - // Merge channels (into first channel to save memory) - for (let i = 0; i < audioData[0].length; ++i) { - audioData[0][i] = (SCALING_FACTOR * (audioData[0][i] + audioData[1][i])) / 2; - } - } + // Merge channels (into first channel to save memory) + for (let i = 0; i < audioData[0].length; ++i) { + audioData[0][i] = (SCALING_FACTOR * (audioData[0][i] + audioData[1][i])) / 2; + } + } - // Select first channel - audioData = audioData[0]; - } - return audioData; + // Select first channel + audioData = audioData[0]; + } + return audioData; } /** * Deep equality test (for arrays and objects) with tolerance for floating point numbers @@ -33,33 +33,33 @@ export async function loadAudio(url) { * @param {number} tol Tolerance for floating point numbers */ export function compare(val1, val2, tol = 0.1) { - if (val1 !== null && val2 !== null && typeof val1 === "object" && typeof val2 === "object") { - // Both are non-null objects - - if (Array.isArray(val1) && Array.isArray(val2)) { - expect(val1).toHaveLength(val2.length); + if (val1 !== null && val2 !== null && typeof val1 === "object" && typeof val2 === "object") { + // Both are non-null objects - for (let i = 0; i < val1.length; ++i) { - compare(val1[i], val2[i], tol); - } - } else { - expect(Object.keys(val1)).toHaveLength(Object.keys(val2).length); + if (Array.isArray(val1) && Array.isArray(val2)) { + expect(val1).toHaveLength(val2.length); - for (let key in val1) { - compare(val1[key], val2[key], tol); - } - } - } else { - // At least one of them is not an object - // First check that both have the same type - expect(typeof val1).toEqual(typeof val2); + for (let i = 0; i < val1.length; ++i) { + compare(val1[i], val2[i], tol); + } + } else { + expect(Object.keys(val1)).toHaveLength(Object.keys(val2).length); - if (typeof val1 === "number" && (!Number.isInteger(val1) || !Number.isInteger(val2))) { - // If both are numbers and at least one of them is not an integer - expect(val1).toBeCloseTo(val2, -Math.log10(tol)); + for (let key in val1) { + compare(val1[key], val2[key], tol); + } + } } else { - // Perform equality test - expect(val1).toEqual(val2); + // At least one of them is not an object + // First check that both have the same type + expect(typeof val1).toEqual(typeof val2); + + if (typeof val1 === "number" && (!Number.isInteger(val1) || !Number.isInteger(val2))) { + // If both are numbers and at least one of them is not an integer + expect(val1).toBeCloseTo(val2, -Math.log10(tol)); + } else { + // Perform equality test + expect(val1).toEqual(val2); + } } - } } diff --git a/tests/tiny_random.test.js b/tests/tiny_random.test.js index e0a3d9369..a82289644 100644 --- a/tests/tiny_random.test.js +++ b/tests/tiny_random.test.js @@ -1,74 +1,74 @@ import { - // Tokenizers - CodeGenTokenizer, - LlamaTokenizer, - CohereTokenizer, - GemmaTokenizer, - GPT2Tokenizer, - GPTNeoXTokenizer, - BloomTokenizer, - BertTokenizer, - T5Tokenizer, - WhisperTokenizer, - BartTokenizer, - MarianTokenizer, - PreTrainedTokenizer, - AutoTokenizer, - - // Processors - CLIPImageProcessor, - AutoProcessor, - Processor, - Florence2Processor, - - // Models - LlamaForCausalLM, - OlmoForCausalLM, - GraniteForCausalLM, - CohereModel, - CohereForCausalLM, - GemmaForCausalLM, - Gemma2ForCausalLM, - OPTForCausalLM, - GPTNeoXForCausalLM, - GPTJForCausalLM, - BloomForCausalLM, - GPTBigCodeForCausalLM, - GPT2LMHeadModel, - JAISLMHeadModel, - MptForCausalLM, - CodeGenForCausalLM, - MistralForCausalLM, - GPTNeoForCausalLM, - BertForMaskedLM, - BertForSequenceClassification, - T5ForConditionalGeneration, - T5Model, - BertModel, - BertForTokenClassification, - BertForQuestionAnswering, - MusicgenForConditionalGeneration, - LlavaForConditionalGeneration, - WhisperForConditionalGeneration, - VisionEncoderDecoderModel, - Florence2ForConditionalGeneration, - MarianMTModel, - - // Pipelines - pipeline, - FillMaskPipeline, - TextClassificationPipeline, - TextGenerationPipeline, - TranslationPipeline, - ImageClassificationPipeline, - ZeroShotImageClassificationPipeline, - TokenClassificationPipeline, - QuestionAnsweringPipeline, - DocumentQuestionAnsweringPipeline, - - // Other - full, - RawImage, + // Tokenizers + CodeGenTokenizer, + LlamaTokenizer, + CohereTokenizer, + GemmaTokenizer, + GPT2Tokenizer, + GPTNeoXTokenizer, + BloomTokenizer, + BertTokenizer, + T5Tokenizer, + WhisperTokenizer, + BartTokenizer, + MarianTokenizer, + PreTrainedTokenizer, + AutoTokenizer, + + // Processors + CLIPImageProcessor, + AutoProcessor, + Processor, + Florence2Processor, + + // Models + LlamaForCausalLM, + OlmoForCausalLM, + GraniteForCausalLM, + CohereModel, + CohereForCausalLM, + GemmaForCausalLM, + Gemma2ForCausalLM, + OPTForCausalLM, + GPTNeoXForCausalLM, + GPTJForCausalLM, + BloomForCausalLM, + GPTBigCodeForCausalLM, + GPT2LMHeadModel, + JAISLMHeadModel, + MptForCausalLM, + CodeGenForCausalLM, + MistralForCausalLM, + GPTNeoForCausalLM, + BertForMaskedLM, + BertForSequenceClassification, + T5ForConditionalGeneration, + T5Model, + BertModel, + BertForTokenClassification, + BertForQuestionAnswering, + MusicgenForConditionalGeneration, + LlavaForConditionalGeneration, + WhisperForConditionalGeneration, + VisionEncoderDecoderModel, + Florence2ForConditionalGeneration, + MarianMTModel, + + // Pipelines + pipeline, + FillMaskPipeline, + TextClassificationPipeline, + TextGenerationPipeline, + TranslationPipeline, + ImageClassificationPipeline, + ZeroShotImageClassificationPipeline, + TokenClassificationPipeline, + QuestionAnsweringPipeline, + DocumentQuestionAnsweringPipeline, + + // Other + full, + RawImage, } from "../src/transformers.js"; import { init, MAX_TEST_TIME, MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME } from "./init.js"; @@ -77,2754 +77,2928 @@ import { compare } from "./test_utils.js"; init(); const DEFAULT_MODEL_OPTIONS = { - dtype: "fp32", + dtype: "fp32", }; describe("Tiny random models", () => { - describe("bert", () => { - describe("BertModel", () => { - const model_id = "hf-internal-testing/tiny-random-BertModel"; - - /** @type {BertModel} */ - let model; - /** @type {BertTokenizer} */ - let tokenizer; - beforeAll(async () => { - model = await BertModel.from_pretrained(model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, + describe("bert", () => { + describe("BertModel", () => { + const model_id = "hf-internal-testing/tiny-random-BertModel"; + + /** @type {BertModel} */ + let model; + /** @type {BertTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await BertModel.from_pretrained(model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + tokenizer = await BertTokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const { last_hidden_state } = await model(inputs); + expect(last_hidden_state.dims).toEqual([1, 7, 32]); + expect(last_hidden_state.mean().item()).toBeCloseTo(0.0, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const { last_hidden_state } = await model(inputs); + expect(last_hidden_state.dims).toEqual([2, 12, 32]); + expect(last_hidden_state.mean().item()).toBeCloseTo(1.4901161193847656e-8, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); }); - tokenizer = await BertTokenizer.from_pretrained(model_id); - }, MAX_MODEL_LOAD_TIME); - - it( - "batch_size=1", - async () => { - const inputs = tokenizer("hello"); - const { last_hidden_state } = await model(inputs); - expect(last_hidden_state.dims).toEqual([1, 7, 32]); - expect(last_hidden_state.mean().item()).toBeCloseTo(0.0, 5); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "batch_size>1", - async () => { - const inputs = tokenizer(["hello", "hello world"], { padding: true }); - const { last_hidden_state } = await model(inputs); - expect(last_hidden_state.dims).toEqual([2, 12, 32]); - expect(last_hidden_state.mean().item()).toBeCloseTo(1.4901161193847656e-8, 5); - }, - MAX_TEST_EXECUTION_TIME, - ); - - afterAll(async () => { - await model?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); - }); - - describe("BertForMaskedLM", () => { - const model_id = "hf-internal-testing/tiny-random-BertForMaskedLM"; - - const texts = ["The goal of life is [MASK].", "Paris is the [MASK] of France."]; - /** @type {BertForMaskedLM} */ - let model; - /** @type {BertTokenizer} */ - let tokenizer; - beforeAll(async () => { - model = await BertForMaskedLM.from_pretrained(model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, + describe("BertForMaskedLM", () => { + const model_id = "hf-internal-testing/tiny-random-BertForMaskedLM"; + + const texts = ["The goal of life is [MASK].", "Paris is the [MASK] of France."]; + + /** @type {BertForMaskedLM} */ + let model; + /** @type {BertTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await BertForMaskedLM.from_pretrained(model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + tokenizer = await BertTokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer(texts[0]); + const { logits } = await model(inputs); + expect(logits.dims).toEqual([1, 19, 1124]); + expect(logits.mean().item()).toBeCloseTo(0.0016587056452408433, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(texts, { padding: true }); + const { logits } = await model(inputs); + expect(logits.dims).toEqual([2, 22, 1124]); + expect(logits.mean().item()).toBeCloseTo(0.0017160633578896523, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); }); - tokenizer = await BertTokenizer.from_pretrained(model_id); - }, MAX_MODEL_LOAD_TIME); - - it( - "batch_size=1", - async () => { - const inputs = tokenizer(texts[0]); - const { logits } = await model(inputs); - expect(logits.dims).toEqual([1, 19, 1124]); - expect(logits.mean().item()).toBeCloseTo(0.0016587056452408433, 5); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "batch_size>1", - async () => { - const inputs = tokenizer(texts, { padding: true }); - const { logits } = await model(inputs); - expect(logits.dims).toEqual([2, 22, 1124]); - expect(logits.mean().item()).toBeCloseTo(0.0017160633578896523, 5); - }, - MAX_TEST_EXECUTION_TIME, - ); - - afterAll(async () => { - await model?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); - }); - describe("BertForSequenceClassification", () => { - const model_id = "hf-internal-testing/tiny-random-BertForSequenceClassification"; - - /** @type {BertForSequenceClassification} */ - let model; - /** @type {BertTokenizer} */ - let tokenizer; - beforeAll(async () => { - model = await BertForSequenceClassification.from_pretrained(model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, + describe("BertForSequenceClassification", () => { + const model_id = "hf-internal-testing/tiny-random-BertForSequenceClassification"; + + /** @type {BertForSequenceClassification} */ + let model; + /** @type {BertTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await BertForSequenceClassification.from_pretrained(model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + tokenizer = await BertTokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const { logits } = await model(inputs); + const target = [[0.00043986947275698185, -0.030218850821256638]].flat(); + expect(logits.dims).toEqual([1, 2]); + logits + .tolist() + .flat() + .forEach((item, i) => { + expect(item).toBeCloseTo(target[i], 5); + }); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const { logits } = await model(inputs); + const target = [ + [0.00043986947275698185, -0.030218850821256638], + [0.0003853091038763523, -0.03022204339504242], + ].flat(); + expect(logits.dims).toEqual([2, 2]); + logits + .tolist() + .flat() + .forEach((item, i) => { + expect(item).toBeCloseTo(target[i], 5); + }); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); }); - tokenizer = await BertTokenizer.from_pretrained(model_id); - }, MAX_MODEL_LOAD_TIME); - - it( - "batch_size=1", - async () => { - const inputs = tokenizer("hello"); - const { logits } = await model(inputs); - const target = [[0.00043986947275698185, -0.030218850821256638]].flat(); - expect(logits.dims).toEqual([1, 2]); - logits - .tolist() - .flat() - .forEach((item, i) => { - expect(item).toBeCloseTo(target[i], 5); - }); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "batch_size>1", - async () => { - const inputs = tokenizer(["hello", "hello world"], { padding: true }); - const { logits } = await model(inputs); - const target = [ - [0.00043986947275698185, -0.030218850821256638], - [0.0003853091038763523, -0.03022204339504242], - ].flat(); - expect(logits.dims).toEqual([2, 2]); - logits - .tolist() - .flat() - .forEach((item, i) => { - expect(item).toBeCloseTo(target[i], 5); - }); - }, - MAX_TEST_EXECUTION_TIME, - ); - afterAll(async () => { - await model?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); - }); - - describe("BertForTokenClassification", () => { - const model_id = "hf-internal-testing/tiny-random-BertForTokenClassification"; - - /** @type {BertForTokenClassification} */ - let model; - /** @type {BertTokenizer} */ - let tokenizer; - beforeAll(async () => { - model = await BertForTokenClassification.from_pretrained(model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, + describe("BertForTokenClassification", () => { + const model_id = "hf-internal-testing/tiny-random-BertForTokenClassification"; + + /** @type {BertForTokenClassification} */ + let model; + /** @type {BertTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await BertForTokenClassification.from_pretrained(model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + tokenizer = await BertTokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const { logits } = await model(inputs); + expect(logits.dims).toEqual([1, 7, 2]); + expect(logits.mean().item()).toBeCloseTo(0.07089076191186905, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const { logits } = await model(inputs); + expect(logits.dims).toEqual([2, 12, 2]); + expect(logits.mean().item()).toBeCloseTo(0.04702216014266014, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); }); - tokenizer = await BertTokenizer.from_pretrained(model_id); - }, MAX_MODEL_LOAD_TIME); - - it( - "batch_size=1", - async () => { - const inputs = tokenizer("hello"); - const { logits } = await model(inputs); - expect(logits.dims).toEqual([1, 7, 2]); - expect(logits.mean().item()).toBeCloseTo(0.07089076191186905, 5); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "batch_size>1", - async () => { - const inputs = tokenizer(["hello", "hello world"], { padding: true }); - const { logits } = await model(inputs); - expect(logits.dims).toEqual([2, 12, 2]); - expect(logits.mean().item()).toBeCloseTo(0.04702216014266014, 5); - }, - MAX_TEST_EXECUTION_TIME, - ); - - afterAll(async () => { - await model?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); - }); - describe("BertForQuestionAnswering", () => { - const model_id = "hf-internal-testing/tiny-random-BertForQuestionAnswering"; - - /** @type {BertForQuestionAnswering} */ - let model; - /** @type {BertTokenizer} */ - let tokenizer; - beforeAll(async () => { - model = await BertForQuestionAnswering.from_pretrained(model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, + describe("BertForQuestionAnswering", () => { + const model_id = "hf-internal-testing/tiny-random-BertForQuestionAnswering"; + + /** @type {BertForQuestionAnswering} */ + let model; + /** @type {BertTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await BertForQuestionAnswering.from_pretrained(model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + tokenizer = await BertTokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const { start_logits, end_logits } = await model(inputs); + expect(start_logits.dims).toEqual([1, 7]); + expect(start_logits.mean().item()).toBeCloseTo(0.12772157788276672, 5); + expect(end_logits.dims).toEqual([1, 7]); + expect(end_logits.mean().item()).toBeCloseTo(0.11811424791812897, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const { start_logits, end_logits } = await model(inputs); + expect(start_logits.dims).toEqual([2, 12]); + expect(start_logits.mean().item()).toBeCloseTo(0.12843115627765656, 5); + expect(end_logits.dims).toEqual([2, 12]); + expect(end_logits.mean().item()).toBeCloseTo(0.11745202541351318, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); }); - tokenizer = await BertTokenizer.from_pretrained(model_id); - }, MAX_MODEL_LOAD_TIME); - - it( - "batch_size=1", - async () => { - const inputs = tokenizer("hello"); - const { start_logits, end_logits } = await model(inputs); - expect(start_logits.dims).toEqual([1, 7]); - expect(start_logits.mean().item()).toBeCloseTo(0.12772157788276672, 5); - expect(end_logits.dims).toEqual([1, 7]); - expect(end_logits.mean().item()).toBeCloseTo(0.11811424791812897, 5); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "batch_size>1", - async () => { - const inputs = tokenizer(["hello", "hello world"], { padding: true }); - const { start_logits, end_logits } = await model(inputs); - expect(start_logits.dims).toEqual([2, 12]); - expect(start_logits.mean().item()).toBeCloseTo(0.12843115627765656, 5); - expect(end_logits.dims).toEqual([2, 12]); - expect(end_logits.mean().item()).toBeCloseTo(0.11745202541351318, 5); - }, - MAX_TEST_EXECUTION_TIME, - ); - - afterAll(async () => { - await model?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); }); - }); - - describe("t5", () => { - describe("T5Model", () => { - const model_id = "hf-internal-testing/tiny-random-T5Model"; - - /** @type {T5Model} */ - let model; - /** @type {T5Tokenizer} */ - let tokenizer; - beforeAll(async () => { - model = await T5Model.from_pretrained(model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, + + describe("t5", () => { + describe("T5Model", () => { + const model_id = "hf-internal-testing/tiny-random-T5Model"; + + /** @type {T5Model} */ + let model; + /** @type {T5Tokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await T5Model.from_pretrained(model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + tokenizer = await T5Tokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "forward", + async () => { + // Example adapted from https://huggingface.co/google-t5/t5-small#how-to-get-started-with-the-model + const inputs = tokenizer("Studies have been shown that owning a dog is good for you"); + const { input_ids: decoder_input_ids } = tokenizer("Studies show that"); + + const { last_hidden_state } = await model({ ...inputs, decoder_input_ids }); + expect(last_hidden_state.dims).toEqual([1, 4, 32]); + expect(last_hidden_state.mean().item()).toBeCloseTo(7.492632721550763e-5, 8); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); }); - tokenizer = await T5Tokenizer.from_pretrained(model_id); - }, MAX_MODEL_LOAD_TIME); - - it( - "forward", - async () => { - // Example adapted from https://huggingface.co/google-t5/t5-small#how-to-get-started-with-the-model - const inputs = tokenizer("Studies have been shown that owning a dog is good for you"); - const { input_ids: decoder_input_ids } = tokenizer("Studies show that"); - - const { last_hidden_state } = await model({ ...inputs, decoder_input_ids }); - expect(last_hidden_state.dims).toEqual([1, 4, 32]); - expect(last_hidden_state.mean().item()).toBeCloseTo(7.492632721550763e-5, 8); - }, - MAX_TEST_EXECUTION_TIME, - ); - - afterAll(async () => { - await model?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); - }); - describe("T5ForConditionalGeneration", () => { - const model_id = "hf-internal-testing/tiny-random-T5ForConditionalGeneration"; - - /** @type {T5ForConditionalGeneration} */ - let model; - /** @type {T5Tokenizer} */ - let tokenizer; - beforeAll(async () => { - model = await T5ForConditionalGeneration.from_pretrained(model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, + describe("T5ForConditionalGeneration", () => { + const model_id = "hf-internal-testing/tiny-random-T5ForConditionalGeneration"; + + /** @type {T5ForConditionalGeneration} */ + let model; + /** @type {T5Tokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await T5ForConditionalGeneration.from_pretrained(model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + tokenizer = await T5Tokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "forward", + async () => { + // Example adapted from https://huggingface.co/google-t5/t5-small#how-to-get-started-with-the-model + const inputs = tokenizer("Studies have been shown that owning a dog is good for you"); + const { input_ids: decoder_input_ids } = tokenizer("Studies show that"); + + const model = await T5ForConditionalGeneration.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + const outputs = await model({ ...inputs, decoder_input_ids }); + expect(outputs.logits.dims).toEqual([1, 4, 32100]); + expect(outputs.logits.mean().item()).toBeCloseTo(8.867568901393952e-9, 12); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n], + [0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); }); - tokenizer = await T5Tokenizer.from_pretrained(model_id); - }, MAX_MODEL_LOAD_TIME); - - it( - "forward", - async () => { - // Example adapted from https://huggingface.co/google-t5/t5-small#how-to-get-started-with-the-model - const inputs = tokenizer("Studies have been shown that owning a dog is good for you"); - const { input_ids: decoder_input_ids } = tokenizer("Studies show that"); - - const model = await T5ForConditionalGeneration.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); - const outputs = await model({ ...inputs, decoder_input_ids }); - expect(outputs.logits.dims).toEqual([1, 4, 32100]); - expect(outputs.logits.mean().item()).toBeCloseTo(8.867568901393952e-9, 12); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "batch_size=1", - async () => { - const inputs = tokenizer("hello"); - const outputs = await model.generate({ - ...inputs, - max_length: 10, - }); - expect(outputs.tolist()).toEqual([[0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n]]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "batch_size>1", - async () => { - const inputs = tokenizer(["hello", "hello world"], { padding: true }); - const outputs = await model.generate({ - ...inputs, - max_length: 10, - }); - expect(outputs.tolist()).toEqual([ - [0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n], - [0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n], - ]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - afterAll(async () => { - await model?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); }); - }); - - describe("marian", () => { - describe("MarianMTModel", () => { - const model_id = "onnx-community/tiny-random-MarianMTModel"; - - /** @type {MarianMTModel} */ - let model; - /** @type {MarianTokenizer} */ - let tokenizer; - beforeAll(async () => { - model = await MarianMTModel.from_pretrained(model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, + + describe("marian", () => { + describe("MarianMTModel", () => { + const model_id = "onnx-community/tiny-random-MarianMTModel"; + + /** @type {MarianMTModel} */ + let model; + /** @type {MarianTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await MarianMTModel.from_pretrained(model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + tokenizer = await MarianTokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[3n, 40672n, 8358n, 32810n, 32810n, 32810n, 32810n, 35687n, 33073n, 6870n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [3n, 40672n, 8358n, 32810n, 32810n, 32810n, 32810n, 35687n, 33073n, 6870n], + [3n, 40672n, 8358n, 32810n, 32810n, 32810n, 32810n, 35687n, 33073n, 6870n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); }); - tokenizer = await MarianTokenizer.from_pretrained(model_id); - }, MAX_MODEL_LOAD_TIME); - - it( - "batch_size=1", - async () => { - const inputs = tokenizer("hello"); - const outputs = await model.generate({ - ...inputs, - max_length: 10, - }); - expect(outputs.tolist()).toEqual([[3n, 40672n, 8358n, 32810n, 32810n, 32810n, 32810n, 35687n, 33073n, 6870n]]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "batch_size>1", - async () => { - const inputs = tokenizer(["hello", "hello world"], { padding: true }); - const outputs = await model.generate({ - ...inputs, - max_length: 10, - }); - expect(outputs.tolist()).toEqual([ - [3n, 40672n, 8358n, 32810n, 32810n, 32810n, 32810n, 35687n, 33073n, 6870n], - [3n, 40672n, 8358n, 32810n, 32810n, 32810n, 32810n, 35687n, 33073n, 6870n], - ]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - afterAll(async () => { - await model?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); }); - }); - - describe("musicgen", () => { - describe("MusicgenForConditionalGeneration", () => { - const model_id = "hf-internal-testing/tiny-random-MusicgenForConditionalGeneration"; - - // Example adapted from https://huggingface.co/docs/transformers/model_doc/musicgen#text-conditional-generation - const texts = ["80s pop track with bassy drums and synth", "90s rock song with loud guitars and heavy drums"]; - - /** @type {MusicgenForConditionalGeneration} */ - let model; - /** @type {T5Tokenizer} */ - let tokenizer; - beforeAll(async () => { - model = await MusicgenForConditionalGeneration.from_pretrained(model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, + + describe("musicgen", () => { + describe("MusicgenForConditionalGeneration", () => { + const model_id = "hf-internal-testing/tiny-random-MusicgenForConditionalGeneration"; + + // Example adapted from https://huggingface.co/docs/transformers/model_doc/musicgen#text-conditional-generation + const texts = ["80s pop track with bassy drums and synth", "90s rock song with loud guitars and heavy drums"]; + + /** @type {MusicgenForConditionalGeneration} */ + let model; + /** @type {T5Tokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await MusicgenForConditionalGeneration.from_pretrained(model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + tokenizer = await T5Tokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "forward", + async () => { + // Example from https://huggingface.co/docs/transformers/model_doc/musicgen#transformers.MusicgenForConditionalGeneration.forward.example + const inputs = tokenizer(texts, { padding: true }); + const pad_token_id = BigInt(model.generation_config.pad_token_id); + const decoder_input_ids = full([inputs.input_ids.dims[0] * model.config.decoder.num_codebooks, 1], pad_token_id); + const { logits } = await model({ ...inputs, decoder_input_ids }); + expect(logits.dims).toEqual([8, 1, 99]); + expect(logits.mean().item()).toBeCloseTo(-0.0018370470497757196, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer(texts[0]); + const audio_values = await model.generate({ ...inputs, max_length: 10 }); + expect(audio_values.dims).toEqual([1, 1, 1920]); + expect(audio_values.mean().item()).toBeCloseTo(0.16644205152988434, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(texts, { padding: true }); + const audio_values = await model.generate({ ...inputs, max_length: 10 }); + expect(audio_values.dims).toEqual([2, 1, 1920]); + expect(audio_values.mean().item()).toBeCloseTo(0.16644206643104553, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); }); - tokenizer = await T5Tokenizer.from_pretrained(model_id); - }, MAX_MODEL_LOAD_TIME); - - it( - "forward", - async () => { - // Example from https://huggingface.co/docs/transformers/model_doc/musicgen#transformers.MusicgenForConditionalGeneration.forward.example - const inputs = tokenizer(texts, { padding: true }); - const pad_token_id = BigInt(model.generation_config.pad_token_id); - const decoder_input_ids = full([inputs.input_ids.dims[0] * model.config.decoder.num_codebooks, 1], pad_token_id); - const { logits } = await model({ ...inputs, decoder_input_ids }); - expect(logits.dims).toEqual([8, 1, 99]); - expect(logits.mean().item()).toBeCloseTo(-0.0018370470497757196, 5); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "batch_size=1", - async () => { - const inputs = tokenizer(texts[0]); - const audio_values = await model.generate({ ...inputs, max_length: 10 }); - expect(audio_values.dims).toEqual([1, 1, 1920]); - expect(audio_values.mean().item()).toBeCloseTo(0.16644205152988434, 5); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "batch_size>1", - async () => { - const inputs = tokenizer(texts, { padding: true }); - const audio_values = await model.generate({ ...inputs, max_length: 10 }); - expect(audio_values.dims).toEqual([2, 1, 1920]); - expect(audio_values.mean().item()).toBeCloseTo(0.16644206643104553, 5); - }, - MAX_TEST_EXECUTION_TIME, - ); - - afterAll(async () => { - await model?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); }); - }); - - describe("whisper", () => { - describe("WhisperForConditionalGeneration", () => { - const model_id = "Xenova/tiny-random-WhisperForConditionalGeneration"; - - /** @type {WhisperForConditionalGeneration} */ - let model; - /** @type {WhisperTokenizer} */ - let tokenizer; - beforeAll(async () => { - model = await WhisperForConditionalGeneration.from_pretrained(model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, - }); - tokenizer = await WhisperTokenizer.from_pretrained(model_id); - }, MAX_MODEL_LOAD_TIME); - describe("prefix tokens", () => { - const input_features = full([1, 80, 3000], 0.0); - - describe("English-only", () => { - it( - "default", - async () => { - const outputs = await model.generate({ - input_features, - is_multilingual: false, - max_new_tokens: 1, - }); + describe("whisper", () => { + describe("WhisperForConditionalGeneration", () => { + const model_id = "Xenova/tiny-random-WhisperForConditionalGeneration"; + + /** @type {WhisperForConditionalGeneration} */ + let model; + /** @type {WhisperTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await WhisperForConditionalGeneration.from_pretrained(model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + tokenizer = await WhisperTokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + describe("prefix tokens", () => { + const input_features = full([1, 80, 3000], 0.0); + + describe("English-only", () => { + it( + "default", + async () => { + const outputs = await model.generate({ + input_features, + is_multilingual: false, + max_new_tokens: 1, + }); + + expect(outputs.tolist()).toEqual([[/* Prefix */ 50258n, 50363n, /* Generated */ 45084n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "return_timestamps=true", + async () => { + const outputs = await model.generate({ + input_features, + is_multilingual: false, + max_new_tokens: 1, + return_timestamps: true, + }); + + expect(outputs.tolist()).toEqual([[/* Prefix */ 50258n, /* Generated */ 50366n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("multilingual", () => { + it( + "language unset; task unset", + async () => { + // language defaults to 'en' + // task defaults to 'transcribe' + + const outputs = await model.generate({ + input_features, + max_new_tokens: 1, + }); + + expect(outputs.tolist()).toEqual([[/* Prefix */ 50258n, 50259n, 50359n, 50363n, /* Generated */ 45084n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "language set; task unset", + async () => { + // task defaults to 'transcribe' + const outputs = await model.generate({ + input_features, + max_new_tokens: 1, + language: "af", + }); + + expect(outputs.tolist()).toEqual([[/* Prefix */ 50258n, 50327n, 50359n, 50363n, /* Generated */ 45084n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "language set; task set", + async () => { + const outputs = await model.generate({ + input_features, + max_new_tokens: 1, + language: "zh", + task: "translate", + }); + + expect(outputs.tolist()).toEqual([[/* Prefix */ 50258n, 50260n, 50358n, 50363n, /* Generated */ 45084n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "return_timestamps=true", + async () => { + const outputs = await model.generate({ + input_features, + max_new_tokens: 1, + language: "en", + task: "transcribe", + return_timestamps: true, + }); + + expect(outputs.tolist()).toEqual([[/* Prefix */ 50258n, 50259n, 50359n, /* Generated */ 50400n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + }); - expect(outputs.tolist()).toEqual([[/* Prefix */ 50258n, 50363n, /* Generated */ 45084n]]); - }, - MAX_TEST_EXECUTION_TIME, - ); + describe("decoder_start_ids", () => { + const input_features = full([1, 80, 3000], 0.0); + + it( + "broadcast inputs", + async () => { + const { decoder_start_token_id, lang_to_id, task_to_id, no_timestamps_token_id } = model.generation_config; + + const outputs = await model.generate({ + input_features, // batch size 1 + max_new_tokens: 1, + decoder_input_ids: [ + // batch size 2 + // <|startoftranscript|> <|lang_id|> <|task|> [<|notimestamps|>] + [decoder_start_token_id, lang_to_id["<|en|>"], task_to_id["translate"], no_timestamps_token_id], + [decoder_start_token_id, lang_to_id["<|fr|>"], task_to_id["transcribe"], no_timestamps_token_id], + ], + }); + expect(outputs.tolist()).toEqual([ + [/* Prefix */ 50258n, 50259n, 50358n, 50363n, /* Generated */ 45084n], + [/* Prefix */ 50258n, 50265n, 50359n, 50363n, /* Generated */ 45084n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); - it( - "return_timestamps=true", - async () => { - const outputs = await model.generate({ - input_features, - is_multilingual: false, - max_new_tokens: 1, - return_timestamps: true, - }); - - expect(outputs.tolist()).toEqual([[/* Prefix */ 50258n, /* Generated */ 50366n]]); - }, - MAX_TEST_EXECUTION_TIME, - ); + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); }); + }); - describe("multilingual", () => { - it( - "language unset; task unset", - async () => { - // language defaults to 'en' - // task defaults to 'transcribe' - - const outputs = await model.generate({ - input_features, - max_new_tokens: 1, - }); + describe("llava", () => { + const prompts = [ + // Example adapted from https://huggingface.co/docs/transformers/model_doc/llava#transformers.LlavaForConditionalGeneration.forward.example + "\nUSER: What's the content of the image?\nASSISTANT:", + "Hi", + ]; - expect(outputs.tolist()).toEqual([[/* Prefix */ 50258n, 50259n, 50359n, 50363n, /* Generated */ 45084n]]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "language set; task unset", - async () => { - // task defaults to 'transcribe' - const outputs = await model.generate({ - input_features, - max_new_tokens: 1, - language: "af", - }); - - expect(outputs.tolist()).toEqual([[/* Prefix */ 50258n, 50327n, 50359n, 50363n, /* Generated */ 45084n]]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "language set; task set", - async () => { - const outputs = await model.generate({ - input_features, - max_new_tokens: 1, - language: "zh", - task: "translate", - }); - - expect(outputs.tolist()).toEqual([[/* Prefix */ 50258n, 50260n, 50358n, 50363n, /* Generated */ 45084n]]); - }, - MAX_TEST_EXECUTION_TIME, - ); + // Empty white image + const dims = [224, 224, 3]; + const image = new RawImage(new Uint8ClampedArray(dims[0] * dims[1] * dims[2]).fill(255), ...dims); - it( - "return_timestamps=true", - async () => { - const outputs = await model.generate({ - input_features, - max_new_tokens: 1, - language: "en", - task: "transcribe", - return_timestamps: true, - }); - - expect(outputs.tolist()).toEqual([[/* Prefix */ 50258n, 50259n, 50359n, /* Generated */ 50400n]]); - }, - MAX_TEST_EXECUTION_TIME, - ); + describe("LlavaForConditionalGeneration", () => { + const model_id = "Xenova/tiny-random-LlavaForConditionalGeneration"; + + /** @type {LlavaForConditionalGeneration} */ + let model; + /** @type {LlamaTokenizer} */ + let tokenizer; + /** @type {CLIPImageProcessor} */ + let processor; + beforeAll(async () => { + model = await LlavaForConditionalGeneration.from_pretrained(model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + tokenizer = await LlamaTokenizer.from_pretrained(model_id); + processor = await AutoProcessor.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "forward", + async () => { + const text_inputs = tokenizer(prompts[0]); + const vision_inputs = await processor(image); + const inputs = { ...text_inputs, ...vision_inputs }; + + const { logits } = await model(inputs); + expect(logits.dims).toEqual([1, 244, 32002]); + expect(logits.mean().item()).toBeCloseTo(-0.0005755752790719271, 8); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size=1", + async () => { + const text_inputs = tokenizer(prompts[0]); + const vision_inputs = await processor(image); + const inputs = { ...text_inputs, ...vision_inputs }; + + const generate_ids = await model.generate({ ...inputs, max_new_tokens: 10 }); + expect(generate_ids.tolist()).toEqual([[1n, 32000n, 29871n, 13n, 11889n, 29901n, 1724n, 29915n, 29879n, 278n, 2793n, 310n, 278n, 1967n, 29973n, 13n, 22933n, 9047n, 13566n, 29901n, 21557n, 16781n, 27238n, 8279n, 20454n, 11927n, 12462n, 12306n, 2414n, 7561n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const text_inputs = tokenizer(prompts, { padding: true }); + const vision_inputs = await processor([image, image]); + const inputs = { ...text_inputs, ...vision_inputs }; + + const generate_ids = await model.generate({ ...inputs, max_new_tokens: 10 }); + expect(generate_ids.tolist()).toEqual([ + [1n, 32000n, 29871n, 13n, 11889n, 29901n, 1724n, 29915n, 29879n, 278n, 2793n, 310n, 278n, 1967n, 29973n, 13n, 22933n, 9047n, 13566n, 29901n, 21557n, 16781n, 27238n, 8279n, 20454n, 11927n, 12462n, 12306n, 2414n, 7561n], + [0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n, 1n, 32000n, 6324n, 1217n, 22958n, 22913n, 10381n, 148n, 31410n, 31736n, 7358n, 9150n, 28635n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); }); - }); - - describe("decoder_start_ids", () => { - const input_features = full([1, 80, 3000], 0.0); - - it( - "broadcast inputs", - async () => { - const { decoder_start_token_id, lang_to_id, task_to_id, no_timestamps_token_id } = model.generation_config; - - const outputs = await model.generate({ - input_features, // batch size 1 - max_new_tokens: 1, - decoder_input_ids: [ - // batch size 2 - // <|startoftranscript|> <|lang_id|> <|task|> [<|notimestamps|>] - [decoder_start_token_id, lang_to_id["<|en|>"], task_to_id["translate"], no_timestamps_token_id], - [decoder_start_token_id, lang_to_id["<|fr|>"], task_to_id["transcribe"], no_timestamps_token_id], - ], - }); - expect(outputs.tolist()).toEqual([ - [/* Prefix */ 50258n, 50259n, 50358n, 50363n, /* Generated */ 45084n], - [/* Prefix */ 50258n, 50265n, 50359n, 50363n, /* Generated */ 45084n], - ]); - }, - MAX_TEST_EXECUTION_TIME, - ); - }); - - afterAll(async () => { - await model?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); }); - }); - describe("llava", () => { - const prompts = [ - // Example adapted from https://huggingface.co/docs/transformers/model_doc/llava#transformers.LlavaForConditionalGeneration.forward.example - "\nUSER: What's the content of the image?\nASSISTANT:", - "Hi", - ]; + describe("florence2", () => { + const texts = ["Describe with a paragraph what is shown in the image.", "Locate the objects with category name in the image."]; - // Empty white image - const dims = [224, 224, 3]; - const image = new RawImage(new Uint8ClampedArray(dims[0] * dims[1] * dims[2]).fill(255), ...dims); + // Empty white image + const dims = [224, 224, 3]; + const image = new RawImage(new Uint8ClampedArray(dims[0] * dims[1] * dims[2]).fill(255), ...dims); - describe("LlavaForConditionalGeneration", () => { - const model_id = "Xenova/tiny-random-LlavaForConditionalGeneration"; - - /** @type {LlavaForConditionalGeneration} */ - let model; - /** @type {LlamaTokenizer} */ - let tokenizer; - /** @type {CLIPImageProcessor} */ - let processor; - beforeAll(async () => { - model = await LlavaForConditionalGeneration.from_pretrained(model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, + describe("Florence2ForConditionalGeneration", () => { + const model_id = "Xenova/tiny-random-Florence2ForConditionalGeneration"; + + /** @type {Florence2ForConditionalGeneration} */ + let model; + /** @type {BartTokenizer} */ + let tokenizer; + /** @type {Florence2Processor} */ + let processor; + beforeAll(async () => { + model = await Florence2ForConditionalGeneration.from_pretrained(model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + tokenizer = await BartTokenizer.from_pretrained(model_id); + processor = await AutoProcessor.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "forward", + async () => { + const text_inputs = tokenizer(texts[0]); + const vision_inputs = await processor(image); + const inputs = { + ...text_inputs, + ...vision_inputs, + decoder_input_ids: full([1, 1], 2n), + }; + + const { logits } = await model(inputs); + expect(logits.dims).toEqual([1, 1, 51289]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size=1", + async () => { + const text_inputs = tokenizer(texts[0]); + { + const generate_ids = await model.generate({ + ...text_inputs, + max_new_tokens: 10, + }); + expect(generate_ids.tolist()).toEqual([[2n, 0n, 0n, 0n, 1n, 0n, 0n, 2n]]); + } + { + const vision_inputs = await processor(image); + const inputs = { ...text_inputs, ...vision_inputs }; + + const generate_ids = await model.generate({ + ...inputs, + max_new_tokens: 10, + }); + expect(generate_ids.tolist()).toEqual([[2n, 0n, 48n, 48n, 48n, 48n, 48n, 48n, 48n, 48n, 2n]]); + } + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const text_inputs = tokenizer(texts, { padding: true }); + { + const generate_ids = await model.generate({ + ...text_inputs, + max_new_tokens: 10, + }); + expect(generate_ids.tolist()).toEqual([ + [2n, 0n, 0n, 0n, 1n, 0n, 0n, 2n], + [2n, 0n, 0n, 0n, 1n, 0n, 0n, 2n], + ]); + } + { + const vision_inputs = await processor([image, image]); + const inputs = { ...text_inputs, ...vision_inputs }; + + const generate_ids = await model.generate({ + ...inputs, + max_new_tokens: 10, + }); + expect(generate_ids.tolist()).toEqual([ + [2n, 0n, 48n, 48n, 48n, 48n, 48n, 48n, 48n, 48n, 2n], + [2n, 0n, 48n, 48n, 48n, 48n, 48n, 48n, 48n, 48n, 2n], + ]); + } + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); }); - tokenizer = await LlamaTokenizer.from_pretrained(model_id); - processor = await AutoProcessor.from_pretrained(model_id); - }, MAX_MODEL_LOAD_TIME); - - it( - "forward", - async () => { - const text_inputs = tokenizer(prompts[0]); - const vision_inputs = await processor(image); - const inputs = { ...text_inputs, ...vision_inputs }; - - const { logits } = await model(inputs); - expect(logits.dims).toEqual([1, 244, 32002]); - expect(logits.mean().item()).toBeCloseTo(-0.0005755752790719271, 8); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "batch_size=1", - async () => { - const text_inputs = tokenizer(prompts[0]); - const vision_inputs = await processor(image); - const inputs = { ...text_inputs, ...vision_inputs }; - - const generate_ids = await model.generate({ ...inputs, max_new_tokens: 10 }); - expect(generate_ids.tolist()).toEqual([[1n, 32000n, 29871n, 13n, 11889n, 29901n, 1724n, 29915n, 29879n, 278n, 2793n, 310n, 278n, 1967n, 29973n, 13n, 22933n, 9047n, 13566n, 29901n, 21557n, 16781n, 27238n, 8279n, 20454n, 11927n, 12462n, 12306n, 2414n, 7561n]]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "batch_size>1", - async () => { - const text_inputs = tokenizer(prompts, { padding: true }); - const vision_inputs = await processor([image, image]); - const inputs = { ...text_inputs, ...vision_inputs }; - - const generate_ids = await model.generate({ ...inputs, max_new_tokens: 10 }); - expect(generate_ids.tolist()).toEqual([ - [1n, 32000n, 29871n, 13n, 11889n, 29901n, 1724n, 29915n, 29879n, 278n, 2793n, 310n, 278n, 1967n, 29973n, 13n, 22933n, 9047n, 13566n, 29901n, 21557n, 16781n, 27238n, 8279n, 20454n, 11927n, 12462n, 12306n, 2414n, 7561n], - [0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n, 1n, 32000n, 6324n, 1217n, 22958n, 22913n, 10381n, 148n, 31410n, 31736n, 7358n, 9150n, 28635n], - ]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - afterAll(async () => { - await model?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); }); - }); - - describe("florence2", () => { - const texts = ["Describe with a paragraph what is shown in the image.", "Locate the objects with category name in the image."]; - - // Empty white image - const dims = [224, 224, 3]; - const image = new RawImage(new Uint8ClampedArray(dims[0] * dims[1] * dims[2]).fill(255), ...dims); - - describe("Florence2ForConditionalGeneration", () => { - const model_id = "Xenova/tiny-random-Florence2ForConditionalGeneration"; - - /** @type {Florence2ForConditionalGeneration} */ - let model; - /** @type {BartTokenizer} */ - let tokenizer; - /** @type {Florence2Processor} */ - let processor; - beforeAll(async () => { - model = await Florence2ForConditionalGeneration.from_pretrained(model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, + + describe("vision-encoder-decoder", () => { + describe("VisionEncoderDecoderModel", () => { + const model_id = "hf-internal-testing/tiny-random-VisionEncoderDecoderModel-vit-gpt2"; + + /** @type {VisionEncoderDecoderModel} */ + let model; + /** @type {GPT2Tokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await VisionEncoderDecoderModel.from_pretrained(model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + tokenizer = await GPT2Tokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const outputs = await model.generate({ + pixel_values: full([1, 3, 30, 30], -1.0), + max_length: 5, + }); + expect(outputs.tolist()).toEqual([[0n, 400n, 400n, 400n, 400n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + // TODO: Add back + // it('batch_size>1', async () => { + // const outputs = await model.generate({ + // pixel_values: cat([ + // full([1, 3, 30, 30], -1.0), + // full([1, 3, 30, 30], 0.0), + // ]), + // max_length: 5, + // }); + // expect(outputs.tolist()).toEqual([ + // // Generation continues + // [0n, 400n, 400n, 400n, 400n], + + // // Finishes early. 1023 is the padding token + // [0n, 0n, 1023n, 1023n, 1023n], + // ]); + // }, MAX_TEST_EXECUTION_TIME); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); }); - tokenizer = await BartTokenizer.from_pretrained(model_id); - processor = await AutoProcessor.from_pretrained(model_id); - }, MAX_MODEL_LOAD_TIME); - - it( - "forward", - async () => { - const text_inputs = tokenizer(texts[0]); - const vision_inputs = await processor(image); - const inputs = { - ...text_inputs, - ...vision_inputs, - decoder_input_ids: full([1, 1], 2n), - }; - - const { logits } = await model(inputs); - expect(logits.dims).toEqual([1, 1, 51289]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "batch_size=1", - async () => { - const text_inputs = tokenizer(texts[0]); - { - const generate_ids = await model.generate({ ...text_inputs, max_new_tokens: 10 }); - expect(generate_ids.tolist()).toEqual([[2n, 0n, 0n, 0n, 1n, 0n, 0n, 2n]]); - } - { - const vision_inputs = await processor(image); - const inputs = { ...text_inputs, ...vision_inputs }; - - const generate_ids = await model.generate({ ...inputs, max_new_tokens: 10 }); - expect(generate_ids.tolist()).toEqual([[2n, 0n, 48n, 48n, 48n, 48n, 48n, 48n, 48n, 48n, 2n]]); - } - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "batch_size>1", - async () => { - const text_inputs = tokenizer(texts, { padding: true }); - { - const generate_ids = await model.generate({ ...text_inputs, max_new_tokens: 10 }); - expect(generate_ids.tolist()).toEqual([ - [2n, 0n, 0n, 0n, 1n, 0n, 0n, 2n], - [2n, 0n, 0n, 0n, 1n, 0n, 0n, 2n], - ]); - } - { - const vision_inputs = await processor([image, image]); - const inputs = { ...text_inputs, ...vision_inputs }; - - const generate_ids = await model.generate({ ...inputs, max_new_tokens: 10 }); - expect(generate_ids.tolist()).toEqual([ - [2n, 0n, 48n, 48n, 48n, 48n, 48n, 48n, 48n, 48n, 2n], - [2n, 0n, 48n, 48n, 48n, 48n, 48n, 48n, 48n, 48n, 2n], - ]); - } - }, - MAX_TEST_EXECUTION_TIME, - ); - - afterAll(async () => { - await model?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); }); - }); - - describe("vision-encoder-decoder", () => { - describe("VisionEncoderDecoderModel", () => { - const model_id = "hf-internal-testing/tiny-random-VisionEncoderDecoderModel-vit-gpt2"; - - /** @type {VisionEncoderDecoderModel} */ - let model; - /** @type {GPT2Tokenizer} */ - let tokenizer; - beforeAll(async () => { - model = await VisionEncoderDecoderModel.from_pretrained(model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, + describe("opt", () => { + describe("OPTForCausalLM", () => { + const model_id = "hf-internal-testing/tiny-random-OPTForCausalLM"; + /** @type {OPTForCausalLM} */ + let model; + /** @type {GPT2Tokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await OPTForCausalLM.from_pretrained(model_id, { + // TODO move to config + revision: "refs/pr/2", + ...DEFAULT_MODEL_OPTIONS, + }); + tokenizer = await GPT2Tokenizer.from_pretrained(model_id, { + // TODO update this + revision: "refs/pr/3", + }); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[2n, 42891n, 39144n, 39144n, 39144n, 39144n, 39144n, 39144n, 39144n, 39144n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [1n, 2n, 42891n, 39144n, 39144n, 39144n, 39144n, 39144n, 39144n, 39144n], + [2n, 42891n, 232n, 24680n, 24680n, 24680n, 24680n, 24680n, 24680n, 24680n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); }); - tokenizer = await GPT2Tokenizer.from_pretrained(model_id); - }, MAX_MODEL_LOAD_TIME); - - it( - "batch_size=1", - async () => { - const outputs = await model.generate({ - pixel_values: full([1, 3, 30, 30], -1.0), - max_length: 5, - }); - expect(outputs.tolist()).toEqual([[0n, 400n, 400n, 400n, 400n]]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - // TODO: Add back - // it('batch_size>1', async () => { - // const outputs = await model.generate({ - // pixel_values: cat([ - // full([1, 3, 30, 30], -1.0), - // full([1, 3, 30, 30], 0.0), - // ]), - // max_length: 5, - // }); - // expect(outputs.tolist()).toEqual([ - // // Generation continues - // [0n, 400n, 400n, 400n, 400n], - - // // Finishes early. 1023 is the padding token - // [0n, 0n, 1023n, 1023n, 1023n], - // ]); - // }, MAX_TEST_EXECUTION_TIME); - - afterAll(async () => { - await model?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); }); - }); - describe("opt", () => { - describe("OPTForCausalLM", () => { - const model_id = "hf-internal-testing/tiny-random-OPTForCausalLM"; - /** @type {OPTForCausalLM} */ - let model; - /** @type {GPT2Tokenizer} */ - let tokenizer; - beforeAll(async () => { - model = await OPTForCausalLM.from_pretrained(model_id, { - // TODO move to config - revision: "refs/pr/2", - ...DEFAULT_MODEL_OPTIONS, - }); - tokenizer = await GPT2Tokenizer.from_pretrained(model_id, { - // TODO update this - revision: "refs/pr/3", + + describe("llama", () => { + describe("LlamaForCausalLM", () => { + const model_id = "hf-internal-testing/tiny-random-LlamaForCausalLM"; + /** @type {LlamaForCausalLM} */ + let model; + /** @type {LlamaTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await LlamaForCausalLM.from_pretrained(model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + tokenizer = await LlamaTokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[1n, 22172n, 18547n, 8143n, 22202n, 9456n, 17213n, 15330n, 26591n, 15721n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [0n, 1n, 22172n, 18547n, 8143n, 22202n, 9456n, 17213n, 15330n, 26591n], + [1n, 22172n, 3186n, 24786n, 19169n, 20222n, 29993n, 27146n, 27426n, 24562n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); }); - tokenizer.padding_side = "left"; - }, MAX_MODEL_LOAD_TIME); - - it( - "batch_size=1", - async () => { - const inputs = tokenizer("hello"); - const outputs = await model.generate({ - ...inputs, - max_length: 10, - }); - expect(outputs.tolist()).toEqual([[2n, 42891n, 39144n, 39144n, 39144n, 39144n, 39144n, 39144n, 39144n, 39144n]]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "batch_size>1", - async () => { - const inputs = tokenizer(["hello", "hello world"], { padding: true }); - const outputs = await model.generate({ - ...inputs, - max_length: 10, - }); - expect(outputs.tolist()).toEqual([ - [1n, 2n, 42891n, 39144n, 39144n, 39144n, 39144n, 39144n, 39144n, 39144n], - [2n, 42891n, 232n, 24680n, 24680n, 24680n, 24680n, 24680n, 24680n, 24680n], - ]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - afterAll(async () => { - await model?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); - }); - }); - describe("llama", () => { - describe("LlamaForCausalLM", () => { - const model_id = "hf-internal-testing/tiny-random-LlamaForCausalLM"; - /** @type {LlamaForCausalLM} */ - let model; - /** @type {LlamaTokenizer} */ - let tokenizer; - beforeAll(async () => { - model = await LlamaForCausalLM.from_pretrained(model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, + describe("LlamaForCausalLM (onnxruntime-genai)", () => { + const model_id = "onnx-community/tiny-random-LlamaForCausalLM-ONNX"; + /** @type {LlamaTokenizer} */ + let tokenizer; + let inputs; + beforeAll(async () => { + tokenizer = await LlamaTokenizer.from_pretrained(model_id); + inputs = tokenizer("hello"); + }, MAX_MODEL_LOAD_TIME); + + const dtypes = ["fp32", "fp16", "q4", "q4f16"]; + + for (const dtype of dtypes) { + it( + `dtype=${dtype}`, + async () => { + /** @type {LlamaForCausalLM} */ + const model = await LlamaForCausalLM.from_pretrained(model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + dtype, + }); + + const outputs = await model.generate({ + ...inputs, + max_length: 5, + }); + expect(outputs.tolist()).toEqual([[128000n, 15339n, 15339n, 15339n, 15339n]]); + + await model?.dispose(); + }, + MAX_TEST_TIME, + ); + } }); - tokenizer = await LlamaTokenizer.from_pretrained(model_id); - }, MAX_MODEL_LOAD_TIME); - - it( - "batch_size=1", - async () => { - const inputs = tokenizer("hello"); - const outputs = await model.generate({ - ...inputs, - max_length: 10, - }); - expect(outputs.tolist()).toEqual([[1n, 22172n, 18547n, 8143n, 22202n, 9456n, 17213n, 15330n, 26591n, 15721n]]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "batch_size>1", - async () => { - const inputs = tokenizer(["hello", "hello world"], { padding: true }); - const outputs = await model.generate({ - ...inputs, - max_length: 10, - }); - expect(outputs.tolist()).toEqual([ - [0n, 1n, 22172n, 18547n, 8143n, 22202n, 9456n, 17213n, 15330n, 26591n], - [1n, 22172n, 3186n, 24786n, 19169n, 20222n, 29993n, 27146n, 27426n, 24562n], - ]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - afterAll(async () => { - await model?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); }); - describe("LlamaForCausalLM (onnxruntime-genai)", () => { - const model_id = "onnx-community/tiny-random-LlamaForCausalLM-ONNX"; - /** @type {LlamaTokenizer} */ - let tokenizer; - let inputs; - beforeAll(async () => { - tokenizer = await LlamaTokenizer.from_pretrained(model_id); - inputs = tokenizer("hello"); - }, MAX_MODEL_LOAD_TIME); - - const dtypes = ["fp32", "fp16", "q4", "q4f16"]; - - for (const dtype of dtypes) { - it( - `dtype=${dtype}`, - async () => { - /** @type {LlamaForCausalLM} */ - const model = await LlamaForCausalLM.from_pretrained(model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, - dtype, - }); - - const outputs = await model.generate({ - ...inputs, - max_length: 5, - }); - expect(outputs.tolist()).toEqual([[128000n, 15339n, 15339n, 15339n, 15339n]]); - - await model?.dispose(); - }, - MAX_TEST_TIME, - ); - } - }); - }); - - describe("olmo", () => { - describe("OlmoForCausalLM", () => { - const model_id = "onnx-community/tiny-random-olmo-hf"; - /** @type {OlmoForCausalLM} */ - let model; - /** @type {GPTNeoXTokenizer} */ - let tokenizer; - beforeAll(async () => { - model = await OlmoForCausalLM.from_pretrained(model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, + describe("olmo", () => { + describe("OlmoForCausalLM", () => { + const model_id = "onnx-community/tiny-random-olmo-hf"; + /** @type {OlmoForCausalLM} */ + let model; + /** @type {GPTNeoXTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await OlmoForCausalLM.from_pretrained(model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + tokenizer = await GPTNeoXTokenizer.from_pretrained(model_id); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[25521n, 10886n, 44936n, 38777n, 33038n, 18557n, 1810n, 33853n, 9517n, 28892n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [1n, 25521n, 10886n, 44936n, 38777n, 33038n, 18557n, 1810n, 33853n, 9517n], + [25521n, 1533n, 37199n, 27362n, 30594n, 39261n, 8824n, 19175n, 8545n, 29335n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); }); - tokenizer = await GPTNeoXTokenizer.from_pretrained(model_id); - tokenizer.padding_side = "left"; - }, MAX_MODEL_LOAD_TIME); - - it( - "batch_size=1", - async () => { - const inputs = tokenizer("hello"); - const outputs = await model.generate({ - ...inputs, - max_length: 10, - }); - expect(outputs.tolist()).toEqual([[25521n, 10886n, 44936n, 38777n, 33038n, 18557n, 1810n, 33853n, 9517n, 28892n]]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "batch_size>1", - async () => { - const inputs = tokenizer(["hello", "hello world"], { padding: true }); - const outputs = await model.generate({ - ...inputs, - max_length: 10, - }); - expect(outputs.tolist()).toEqual([ - [1n, 25521n, 10886n, 44936n, 38777n, 33038n, 18557n, 1810n, 33853n, 9517n], - [25521n, 1533n, 37199n, 27362n, 30594n, 39261n, 8824n, 19175n, 8545n, 29335n], - ]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - afterAll(async () => { - await model?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); }); - }); - - describe("granite", () => { - describe("GraniteForCausalLM", () => { - const model_id = "hf-internal-testing/tiny-random-GraniteForCausalLM"; - /** @type {GraniteForCausalLM} */ - let model; - /** @type {GPT2Tokenizer} */ - let tokenizer; - beforeAll(async () => { - model = await GraniteForCausalLM.from_pretrained(model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, + + describe("granite", () => { + describe("GraniteForCausalLM", () => { + const model_id = "hf-internal-testing/tiny-random-GraniteForCausalLM"; + /** @type {GraniteForCausalLM} */ + let model; + /** @type {GPT2Tokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await GraniteForCausalLM.from_pretrained(model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + tokenizer = await GPT2Tokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[7656n, 39727n, 33077n, 9643n, 30539n, 47869n, 48739n, 15085n, 9203n, 14020n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [0n, 7656n, 39727n, 33077n, 9643n, 30539n, 47869n, 48739n, 15085n, 9203n], + [7656n, 5788n, 17835n, 13234n, 7592n, 21471n, 30537n, 23023n, 43450n, 4824n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); }); - tokenizer = await GPT2Tokenizer.from_pretrained(model_id); - }, MAX_MODEL_LOAD_TIME); - - it( - "batch_size=1", - async () => { - const inputs = tokenizer("hello"); - const outputs = await model.generate({ - ...inputs, - max_length: 10, - }); - expect(outputs.tolist()).toEqual([[7656n, 39727n, 33077n, 9643n, 30539n, 47869n, 48739n, 15085n, 9203n, 14020n]]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "batch_size>1", - async () => { - const inputs = tokenizer(["hello", "hello world"], { padding: true }); - const outputs = await model.generate({ - ...inputs, - max_length: 10, - }); - expect(outputs.tolist()).toEqual([ - [0n, 7656n, 39727n, 33077n, 9643n, 30539n, 47869n, 48739n, 15085n, 9203n], - [7656n, 5788n, 17835n, 13234n, 7592n, 21471n, 30537n, 23023n, 43450n, 4824n], - ]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - afterAll(async () => { - await model?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); }); - }); - - describe("cohere", () => { - describe("CohereModel", () => { - const model_id = "hf-internal-testing/tiny-random-CohereModel"; - /** @type {CohereModel} */ - let model; - /** @type {CohereTokenizer} */ - let tokenizer; - beforeAll(async () => { - model = await CohereModel.from_pretrained(model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, + + describe("cohere", () => { + describe("CohereModel", () => { + const model_id = "hf-internal-testing/tiny-random-CohereModel"; + /** @type {CohereModel} */ + let model; + /** @type {CohereTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await CohereModel.from_pretrained(model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + tokenizer = await CohereTokenizer.from_pretrained(model_id); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const { last_hidden_state } = await model(inputs); + expect(last_hidden_state.dims).toEqual([1, 4, 32]); + expect(last_hidden_state.mean().item()).toBeCloseTo(0.0, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const { last_hidden_state } = await model(inputs); + expect(last_hidden_state.dims).toEqual([2, 6, 32]); + expect(last_hidden_state.mean().item()).toBeCloseTo(9.934107758624577e-9, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); }); - tokenizer = await CohereTokenizer.from_pretrained(model_id); - tokenizer.padding_side = "left"; - }, MAX_MODEL_LOAD_TIME); - - it( - "batch_size=1", - async () => { - const inputs = tokenizer("hello"); - const { last_hidden_state } = await model(inputs); - expect(last_hidden_state.dims).toEqual([1, 4, 32]); - expect(last_hidden_state.mean().item()).toBeCloseTo(0.0, 5); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "batch_size>1", - async () => { - const inputs = tokenizer(["hello", "hello world"], { padding: true }); - const { last_hidden_state } = await model(inputs); - expect(last_hidden_state.dims).toEqual([2, 6, 32]); - expect(last_hidden_state.mean().item()).toBeCloseTo(9.934107758624577e-9, 5); - }, - MAX_TEST_EXECUTION_TIME, - ); - - afterAll(async () => { - await model?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); - }); - describe("CohereForCausalLM", () => { - const model_id = "hf-internal-testing/tiny-random-CohereForCausalLM"; - /** @type {CohereForCausalLM} */ - let model; - /** @type {CohereTokenizer} */ - let tokenizer; - beforeAll(async () => { - model = await CohereForCausalLM.from_pretrained(model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, + describe("CohereForCausalLM", () => { + const model_id = "hf-internal-testing/tiny-random-CohereForCausalLM"; + /** @type {CohereForCausalLM} */ + let model; + /** @type {CohereTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await CohereForCausalLM.from_pretrained(model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + tokenizer = await CohereTokenizer.from_pretrained(model_id); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[5n, 203n, 790n, 87n, 87n, 87n, 87n, 87n, 87n, 87n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [0n, 0n, 5n, 203n, 790n, 87n, 87n, 87n, 87n, 87n], + [5n, 203n, 790n, 87n, 214n, 741n, 741n, 741n, 741n, 741n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); }); - tokenizer = await CohereTokenizer.from_pretrained(model_id); - tokenizer.padding_side = "left"; - }, MAX_MODEL_LOAD_TIME); - - it( - "batch_size=1", - async () => { - const inputs = tokenizer("hello"); - const outputs = await model.generate({ - ...inputs, - max_length: 10, - }); - expect(outputs.tolist()).toEqual([[5n, 203n, 790n, 87n, 87n, 87n, 87n, 87n, 87n, 87n]]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "batch_size>1", - async () => { - const inputs = tokenizer(["hello", "hello world"], { padding: true }); - const outputs = await model.generate({ - ...inputs, - max_length: 10, - }); - expect(outputs.tolist()).toEqual([ - [0n, 0n, 5n, 203n, 790n, 87n, 87n, 87n, 87n, 87n], - [5n, 203n, 790n, 87n, 214n, 741n, 741n, 741n, 741n, 741n], - ]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - afterAll(async () => { - await model?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); }); - }); - - describe("gemma", () => { - describe("GemmaForCausalLM", () => { - const model_id = "Xenova/tiny-random-GemmaForCausalLM"; - /** @type {GemmaForCausalLM} */ - let model; - /** @type {GemmaTokenizer} */ - let tokenizer; - beforeAll(async () => { - model = await GemmaForCausalLM.from_pretrained(model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, + + describe("gemma", () => { + describe("GemmaForCausalLM", () => { + const model_id = "Xenova/tiny-random-GemmaForCausalLM"; + /** @type {GemmaForCausalLM} */ + let model; + /** @type {GemmaTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await GemmaForCausalLM.from_pretrained(model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + tokenizer = await GemmaTokenizer.from_pretrained(model_id); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[2n, 17534n, 254059n, 254059n, 254059n, 254059n, 254059n, 254059n, 254059n, 254059n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [0n, 2n, 17534n, 254059n, 254059n, 254059n, 254059n, 254059n, 254059n, 254059n], + [2n, 17534n, 2134n, 71055n, 71055n, 71055n, 71055n, 71055n, 71055n, 71055n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); }); - tokenizer = await GemmaTokenizer.from_pretrained(model_id); - tokenizer.padding_side = "left"; - }, MAX_MODEL_LOAD_TIME); - - it( - "batch_size=1", - async () => { - const inputs = tokenizer("hello"); - const outputs = await model.generate({ - ...inputs, - max_length: 10, - }); - expect(outputs.tolist()).toEqual([[2n, 17534n, 254059n, 254059n, 254059n, 254059n, 254059n, 254059n, 254059n, 254059n]]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "batch_size>1", - async () => { - const inputs = tokenizer(["hello", "hello world"], { padding: true }); - const outputs = await model.generate({ - ...inputs, - max_length: 10, - }); - expect(outputs.tolist()).toEqual([ - [0n, 2n, 17534n, 254059n, 254059n, 254059n, 254059n, 254059n, 254059n, 254059n], - [2n, 17534n, 2134n, 71055n, 71055n, 71055n, 71055n, 71055n, 71055n, 71055n], - ]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - afterAll(async () => { - await model?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); }); - }); - - describe("gemma", () => { - describe("Gemma2ForCausalLM", () => { - const model_id = "hf-internal-testing/tiny-random-Gemma2ForCausalLM"; - /** @type {Gemma2ForCausalLM} */ - let model; - /** @type {GemmaTokenizer} */ - let tokenizer; - beforeAll(async () => { - model = await Gemma2ForCausalLM.from_pretrained(model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, + + describe("gemma", () => { + describe("Gemma2ForCausalLM", () => { + const model_id = "hf-internal-testing/tiny-random-Gemma2ForCausalLM"; + /** @type {Gemma2ForCausalLM} */ + let model; + /** @type {GemmaTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await Gemma2ForCausalLM.from_pretrained(model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + tokenizer = await GemmaTokenizer.from_pretrained(model_id); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[2n, 17534n, 127534n, 160055n, 160055n, 160055n, 160055n, 160055n, 160055n, 160055n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [0n, 2n, 17534n, 127534n, 127534n, 215341n, 215341n, 215341n, 215341n, 215341n], + [2n, 17534n, 2134n, 107508n, 160055n, 160055n, 160055n, 160055n, 160055n, 160055n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); }); - tokenizer = await GemmaTokenizer.from_pretrained(model_id); - tokenizer.padding_side = "left"; - }, MAX_MODEL_LOAD_TIME); - - it( - "batch_size=1", - async () => { - const inputs = tokenizer("hello"); - const outputs = await model.generate({ - ...inputs, - max_length: 10, - }); - expect(outputs.tolist()).toEqual([[2n, 17534n, 127534n, 160055n, 160055n, 160055n, 160055n, 160055n, 160055n, 160055n]]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "batch_size>1", - async () => { - const inputs = tokenizer(["hello", "hello world"], { padding: true }); - const outputs = await model.generate({ - ...inputs, - max_length: 10, - }); - expect(outputs.tolist()).toEqual([ - [0n, 2n, 17534n, 127534n, 127534n, 215341n, 215341n, 215341n, 215341n, 215341n], - [2n, 17534n, 2134n, 107508n, 160055n, 160055n, 160055n, 160055n, 160055n, 160055n], - ]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - afterAll(async () => { - await model?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); }); - }); - - describe("gpt_neo", () => { - describe("GPTNeoForCausalLM", () => { - const model_id = "hf-internal-testing/tiny-random-GPTNeoForCausalLM"; - /** @type {GPTNeoForCausalLM} */ - let model; - /** @type {GPT2Tokenizer} */ - let tokenizer; - beforeAll(async () => { - model = await GPTNeoForCausalLM.from_pretrained(model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, + + describe("gpt_neo", () => { + describe("GPTNeoForCausalLM", () => { + const model_id = "hf-internal-testing/tiny-random-GPTNeoForCausalLM"; + /** @type {GPTNeoForCausalLM} */ + let model; + /** @type {GPT2Tokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await GPTNeoForCausalLM.from_pretrained(model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + tokenizer = await GPT2Tokenizer.from_pretrained(model_id); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[258n, 863n, 79n, 79n, 79n, 949n, 949n, 949n, 949n, 949n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [0n, 0n, 258n, 863n, 79n, 79n, 79n, 949n, 949n, 949n], + [258n, 863n, 79n, 269n, 813n, 849n, 849n, 849n, 849n, 849n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); }); - tokenizer = await GPT2Tokenizer.from_pretrained(model_id); - tokenizer.padding_side = "left"; - }, MAX_MODEL_LOAD_TIME); - - it( - "batch_size=1", - async () => { - const inputs = tokenizer("hello"); - const outputs = await model.generate({ - ...inputs, - max_length: 10, - }); - expect(outputs.tolist()).toEqual([[258n, 863n, 79n, 79n, 79n, 949n, 949n, 949n, 949n, 949n]]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "batch_size>1", - async () => { - const inputs = tokenizer(["hello", "hello world"], { padding: true }); - const outputs = await model.generate({ - ...inputs, - max_length: 10, - }); - expect(outputs.tolist()).toEqual([ - [0n, 0n, 258n, 863n, 79n, 79n, 79n, 949n, 949n, 949n], - [258n, 863n, 79n, 269n, 813n, 849n, 849n, 849n, 849n, 849n], - ]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - afterAll(async () => { - await model?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); }); - }); - - describe("gpt_neox", () => { - describe("GPTNeoXForCausalLM", () => { - const model_id = "hf-internal-testing/tiny-random-GPTNeoXForCausalLM"; - /** @type {GPTNeoXForCausalLM} */ - let model; - /** @type {GPTNeoXTokenizer} */ - let tokenizer; - beforeAll(async () => { - model = await GPTNeoXForCausalLM.from_pretrained(model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, + + describe("gpt_neox", () => { + describe("GPTNeoXForCausalLM", () => { + const model_id = "hf-internal-testing/tiny-random-GPTNeoXForCausalLM"; + /** @type {GPTNeoXForCausalLM} */ + let model; + /** @type {GPTNeoXTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await GPTNeoXForCausalLM.from_pretrained(model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + tokenizer = await GPTNeoXTokenizer.from_pretrained(model_id); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[259n, 864n, 80n, 881n, 502n, 895n, 938n, 668n, 502n, 895n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [0n, 0n, 259n, 864n, 80n, 881n, 502n, 895n, 938n, 668n], + [259n, 864n, 80n, 270n, 814n, 522n, 112n, 268n, 503n, 468n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); }); - tokenizer = await GPTNeoXTokenizer.from_pretrained(model_id); - tokenizer.padding_side = "left"; - }, MAX_MODEL_LOAD_TIME); - - it( - "batch_size=1", - async () => { - const inputs = tokenizer("hello"); - const outputs = await model.generate({ - ...inputs, - max_length: 10, - }); - expect(outputs.tolist()).toEqual([[259n, 864n, 80n, 881n, 502n, 895n, 938n, 668n, 502n, 895n]]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "batch_size>1", - async () => { - const inputs = tokenizer(["hello", "hello world"], { padding: true }); - const outputs = await model.generate({ - ...inputs, - max_length: 10, - }); - expect(outputs.tolist()).toEqual([ - [0n, 0n, 259n, 864n, 80n, 881n, 502n, 895n, 938n, 668n], - [259n, 864n, 80n, 270n, 814n, 522n, 112n, 268n, 503n, 468n], - ]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - afterAll(async () => { - await model?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); }); - }); - - describe("gptj", () => { - describe("GPTJForCausalLM", () => { - const model_id = "hf-internal-testing/tiny-random-GPTJForCausalLM"; - /** @type {GPTJForCausalLM} */ - let model; - /** @type {GPTNeoXTokenizer} */ - let tokenizer; - beforeAll(async () => { - model = await GPTJForCausalLM.from_pretrained(model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, + + describe("gptj", () => { + describe("GPTJForCausalLM", () => { + const model_id = "hf-internal-testing/tiny-random-GPTJForCausalLM"; + /** @type {GPTJForCausalLM} */ + let model; + /** @type {GPTNeoXTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await GPTJForCausalLM.from_pretrained(model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + tokenizer = await GPTNeoXTokenizer.from_pretrained(model_id); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[258n, 863n, 79n, 102n, 401n, 773n, 889n, 159n, 957n, 869n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [0n, 0n, 258n, 863n, 79n, 102n, 401n, 773n, 889n, 159n], + [258n, 863n, 79n, 269n, 813n, 879n, 175n, 39n, 141n, 1000n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); }); - tokenizer = await GPTNeoXTokenizer.from_pretrained(model_id); - tokenizer.padding_side = "left"; - }, MAX_MODEL_LOAD_TIME); - - it( - "batch_size=1", - async () => { - const inputs = tokenizer("hello"); - const outputs = await model.generate({ - ...inputs, - max_length: 10, - }); - expect(outputs.tolist()).toEqual([[258n, 863n, 79n, 102n, 401n, 773n, 889n, 159n, 957n, 869n]]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "batch_size>1", - async () => { - const inputs = tokenizer(["hello", "hello world"], { padding: true }); - const outputs = await model.generate({ - ...inputs, - max_length: 10, - }); - expect(outputs.tolist()).toEqual([ - [0n, 0n, 258n, 863n, 79n, 102n, 401n, 773n, 889n, 159n], - [258n, 863n, 79n, 269n, 813n, 879n, 175n, 39n, 141n, 1000n], - ]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - afterAll(async () => { - await model?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); }); - }); - - describe("bloom", () => { - describe("BloomForCausalLM", () => { - const model_id = "hf-internal-testing/tiny-random-BloomForCausalLM"; - /** @type {BloomForCausalLM} */ - let model; - /** @type {BloomTokenizer} */ - let tokenizer; - beforeAll(async () => { - model = await BloomForCausalLM.from_pretrained(model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, + + describe("bloom", () => { + describe("BloomForCausalLM", () => { + const model_id = "hf-internal-testing/tiny-random-BloomForCausalLM"; + /** @type {BloomForCausalLM} */ + let model; + /** @type {BloomTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await BloomForCausalLM.from_pretrained(model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + tokenizer = await BloomTokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[198n, 803n, 82n, 82n, 82n, 82n, 82n, 82n, 82n, 82n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [3n, 3n, 198n, 803n, 82n, 82n, 82n, 82n, 82n, 82n], + [198n, 803n, 82n, 209n, 753n, 753n, 753n, 753n, 753n, 753n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); }); - tokenizer = await BloomTokenizer.from_pretrained(model_id); - }, MAX_MODEL_LOAD_TIME); - - it( - "batch_size=1", - async () => { - const inputs = tokenizer("hello"); - const outputs = await model.generate({ - ...inputs, - max_length: 10, - }); - expect(outputs.tolist()).toEqual([[198n, 803n, 82n, 82n, 82n, 82n, 82n, 82n, 82n, 82n]]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "batch_size>1", - async () => { - const inputs = tokenizer(["hello", "hello world"], { padding: true }); - const outputs = await model.generate({ - ...inputs, - max_length: 10, - }); - expect(outputs.tolist()).toEqual([ - [3n, 3n, 198n, 803n, 82n, 82n, 82n, 82n, 82n, 82n], - [198n, 803n, 82n, 209n, 753n, 753n, 753n, 753n, 753n, 753n], - ]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - afterAll(async () => { - await model?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); }); - }); - - describe("gpt_bigcode", () => { - describe("GPTBigCodeForCausalLM", () => { - const model_id = "hf-internal-testing/tiny-random-GPTBigCodeForCausalLM"; - /** @type {GPTBigCodeForCausalLM} */ - let model; - /** @type {GPT2Tokenizer} */ - let tokenizer; - beforeAll(async () => { - model = await GPTBigCodeForCausalLM.from_pretrained(model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, + + describe("gpt_bigcode", () => { + describe("GPTBigCodeForCausalLM", () => { + const model_id = "hf-internal-testing/tiny-random-GPTBigCodeForCausalLM"; + /** @type {GPTBigCodeForCausalLM} */ + let model; + /** @type {GPT2Tokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await GPTBigCodeForCausalLM.from_pretrained(model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + tokenizer = await GPT2Tokenizer.from_pretrained(model_id); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[258n, 863n, 79n, 79n, 79n, 79n, 79n, 79n, 79n, 79n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [0n, 0n, 258n, 863n, 79n, 79n, 79n, 79n, 79n, 79n], + [258n, 863n, 79n, 269n, 813n, 832n, 93n, 93n, 93n, 93n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); }); - tokenizer = await GPT2Tokenizer.from_pretrained(model_id); - tokenizer.padding_side = "left"; - }, MAX_MODEL_LOAD_TIME); - - it( - "batch_size=1", - async () => { - const inputs = tokenizer("hello"); - const outputs = await model.generate({ - ...inputs, - max_length: 10, - }); - expect(outputs.tolist()).toEqual([[258n, 863n, 79n, 79n, 79n, 79n, 79n, 79n, 79n, 79n]]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "batch_size>1", - async () => { - const inputs = tokenizer(["hello", "hello world"], { padding: true }); - const outputs = await model.generate({ - ...inputs, - max_length: 10, - }); - expect(outputs.tolist()).toEqual([ - [0n, 0n, 258n, 863n, 79n, 79n, 79n, 79n, 79n, 79n], - [258n, 863n, 79n, 269n, 813n, 832n, 93n, 93n, 93n, 93n], - ]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - afterAll(async () => { - await model?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); }); - }); - - describe("gpt2", () => { - describe("GPT2LMHeadModel", () => { - const model_id = "hf-internal-testing/tiny-random-GPT2LMHeadModel"; - /** @type {GPT2LMHeadModel} */ - let model; - /** @type {GPT2Tokenizer} */ - let tokenizer; - beforeAll(async () => { - model = await GPT2LMHeadModel.from_pretrained(model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, + + describe("gpt2", () => { + describe("GPT2LMHeadModel", () => { + const model_id = "hf-internal-testing/tiny-random-GPT2LMHeadModel"; + /** @type {GPT2LMHeadModel} */ + let model; + /** @type {GPT2Tokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await GPT2LMHeadModel.from_pretrained(model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + tokenizer = await GPT2Tokenizer.from_pretrained(model_id); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[258n, 863n, 79n, 79n, 79n, 79n, 79n, 79n, 79n, 243n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [0n, 0n, 258n, 863n, 79n, 79n, 79n, 79n, 79n, 79n], + [258n, 863n, 79n, 269n, 813n, 813n, 813n, 813n, 813n, 813n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); }); - tokenizer = await GPT2Tokenizer.from_pretrained(model_id); - tokenizer.padding_side = "left"; - }, MAX_MODEL_LOAD_TIME); - - it( - "batch_size=1", - async () => { - const inputs = tokenizer("hello"); - const outputs = await model.generate({ - ...inputs, - max_length: 10, - }); - expect(outputs.tolist()).toEqual([[258n, 863n, 79n, 79n, 79n, 79n, 79n, 79n, 79n, 243n]]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "batch_size>1", - async () => { - const inputs = tokenizer(["hello", "hello world"], { padding: true }); - const outputs = await model.generate({ - ...inputs, - max_length: 10, - }); - expect(outputs.tolist()).toEqual([ - [0n, 0n, 258n, 863n, 79n, 79n, 79n, 79n, 79n, 79n], - [258n, 863n, 79n, 269n, 813n, 813n, 813n, 813n, 813n, 813n], - ]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - afterAll(async () => { - await model?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); }); - }); - - describe("jais", () => { - describe("JAISLMHeadModel", () => { - const model_id = "onnx-community/tiny-random-jais"; - /** @type {JAISLMHeadModel} */ - let model; - /** @type {PreTrainedTokenizer} */ - let tokenizer; - beforeAll(async () => { - model = await JAISLMHeadModel.from_pretrained(model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, + + describe("jais", () => { + describe("JAISLMHeadModel", () => { + const model_id = "onnx-community/tiny-random-jais"; + /** @type {JAISLMHeadModel} */ + let model; + /** @type {PreTrainedTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await JAISLMHeadModel.from_pretrained(model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + tokenizer = await PreTrainedTokenizer.from_pretrained(model_id); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[55422n, 55422n, 55422n, 55422n, 55422n, 55422n, 55422n, 55422n, 55422n, 55422n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [0n, 55422n, 55422n, 55422n, 55422n, 55422n, 55422n, 55422n, 55422n, 55422n], + [55422n, 2838n, 2838n, 2838n, 2838n, 2838n, 2838n, 2838n, 2838n, 2838n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); }); - tokenizer = await PreTrainedTokenizer.from_pretrained(model_id); - tokenizer.padding_side = "left"; - }, MAX_MODEL_LOAD_TIME); - - it( - "batch_size=1", - async () => { - const inputs = tokenizer("hello"); - const outputs = await model.generate({ - ...inputs, - max_length: 10, - }); - expect(outputs.tolist()).toEqual([[55422n, 55422n, 55422n, 55422n, 55422n, 55422n, 55422n, 55422n, 55422n, 55422n]]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "batch_size>1", - async () => { - const inputs = tokenizer(["hello", "hello world"], { padding: true }); - const outputs = await model.generate({ - ...inputs, - max_length: 10, - }); - expect(outputs.tolist()).toEqual([ - [0n, 55422n, 55422n, 55422n, 55422n, 55422n, 55422n, 55422n, 55422n, 55422n], - [55422n, 2838n, 2838n, 2838n, 2838n, 2838n, 2838n, 2838n, 2838n, 2838n], - ]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - afterAll(async () => { - await model?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); }); - }); - - describe("mpt", () => { - describe("MptForCausalLM", () => { - const model_id = "hf-internal-testing/tiny-random-MptForCausalLM"; - /** @type {MptForCausalLM} */ - let model; - /** @type {GPTNeoXTokenizer} */ - let tokenizer; - beforeAll(async () => { - model = await MptForCausalLM.from_pretrained(model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, + + describe("mpt", () => { + describe("MptForCausalLM", () => { + const model_id = "hf-internal-testing/tiny-random-MptForCausalLM"; + /** @type {MptForCausalLM} */ + let model; + /** @type {GPTNeoXTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await MptForCausalLM.from_pretrained(model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + tokenizer = await GPTNeoXTokenizer.from_pretrained(model_id); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[259n, 864n, 80n, 80n, 80n, 80n, 80n, 80n, 80n, 80n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [0n, 0n, 259n, 864n, 80n, 80n, 80n, 80n, 80n, 80n], + [259n, 864n, 80n, 270n, 814n, 293n, 293n, 293n, 293n, 293n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); }); - tokenizer = await GPTNeoXTokenizer.from_pretrained(model_id); - tokenizer.padding_side = "left"; - }, MAX_MODEL_LOAD_TIME); - - it( - "batch_size=1", - async () => { - const inputs = tokenizer("hello"); - const outputs = await model.generate({ - ...inputs, - max_length: 10, - }); - expect(outputs.tolist()).toEqual([[259n, 864n, 80n, 80n, 80n, 80n, 80n, 80n, 80n, 80n]]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "batch_size>1", - async () => { - const inputs = tokenizer(["hello", "hello world"], { padding: true }); - const outputs = await model.generate({ - ...inputs, - max_length: 10, - }); - expect(outputs.tolist()).toEqual([ - [0n, 0n, 259n, 864n, 80n, 80n, 80n, 80n, 80n, 80n], - [259n, 864n, 80n, 270n, 814n, 293n, 293n, 293n, 293n, 293n], - ]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - afterAll(async () => { - await model?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); }); - }); - - describe("codegen", () => { - describe("CodeGenForCausalLM", () => { - const model_id = "hf-internal-testing/tiny-random-CodeGenForCausalLM"; - /** @type {CodeGenForCausalLM} */ - let model; - /** @type {CodeGenTokenizer} */ - let tokenizer; - beforeAll(async () => { - model = await CodeGenForCausalLM.from_pretrained(model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, + + describe("codegen", () => { + describe("CodeGenForCausalLM", () => { + const model_id = "hf-internal-testing/tiny-random-CodeGenForCausalLM"; + /** @type {CodeGenForCausalLM} */ + let model; + /** @type {CodeGenTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await CodeGenForCausalLM.from_pretrained(model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + tokenizer = await CodeGenTokenizer.from_pretrained(model_id); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[258n, 863n, 79n, 437n, 334n, 450n, 294n, 621n, 375n, 385n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [0n, 0n, 258n, 863n, 79n, 437n, 334n, 450n, 294n, 621n], + [258n, 863n, 79n, 269n, 813n, 759n, 113n, 295n, 574n, 987n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); }); - tokenizer = await CodeGenTokenizer.from_pretrained(model_id); - tokenizer.padding_side = "left"; - }, MAX_MODEL_LOAD_TIME); - - it( - "batch_size=1", - async () => { - const inputs = tokenizer("hello"); - const outputs = await model.generate({ - ...inputs, - max_length: 10, - }); - expect(outputs.tolist()).toEqual([[258n, 863n, 79n, 437n, 334n, 450n, 294n, 621n, 375n, 385n]]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "batch_size>1", - async () => { - const inputs = tokenizer(["hello", "hello world"], { padding: true }); - const outputs = await model.generate({ - ...inputs, - max_length: 10, - }); - expect(outputs.tolist()).toEqual([ - [0n, 0n, 258n, 863n, 79n, 437n, 334n, 450n, 294n, 621n], - [258n, 863n, 79n, 269n, 813n, 759n, 113n, 295n, 574n, 987n], - ]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - afterAll(async () => { - await model?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); }); - }); - - describe("mistral", () => { - describe("MistralForCausalLM", () => { - const model_id = "hf-internal-testing/tiny-random-MistralForCausalLM"; - /** @type {MistralForCausalLM} */ - let model; - /** @type {LlamaTokenizer} */ - let tokenizer; - beforeAll(async () => { - model = await MistralForCausalLM.from_pretrained(model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, + + describe("mistral", () => { + describe("MistralForCausalLM", () => { + const model_id = "hf-internal-testing/tiny-random-MistralForCausalLM"; + /** @type {MistralForCausalLM} */ + let model; + /** @type {LlamaTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await MistralForCausalLM.from_pretrained(model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + tokenizer = await LlamaTokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[1n, 6312n, 28709n, 24704n, 8732n, 1310n, 9808n, 13771n, 27309n, 4779n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [2n, 1n, 6312n, 28709n, 24704n, 8732n, 1310n, 9808n, 13771n, 27309n], + [1n, 6312n, 28709n, 1526n, 8687n, 5690n, 1770n, 30811n, 12501n, 3325n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); }); - tokenizer = await LlamaTokenizer.from_pretrained(model_id); - }, MAX_MODEL_LOAD_TIME); - - it( - "batch_size=1", - async () => { - const inputs = tokenizer("hello"); - const outputs = await model.generate({ - ...inputs, - max_length: 10, - }); - expect(outputs.tolist()).toEqual([[1n, 6312n, 28709n, 24704n, 8732n, 1310n, 9808n, 13771n, 27309n, 4779n]]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "batch_size>1", - async () => { - const inputs = tokenizer(["hello", "hello world"], { padding: true }); - const outputs = await model.generate({ - ...inputs, - max_length: 10, - }); - expect(outputs.tolist()).toEqual([ - [2n, 1n, 6312n, 28709n, 24704n, 8732n, 1310n, 9808n, 13771n, 27309n], - [1n, 6312n, 28709n, 1526n, 8687n, 5690n, 1770n, 30811n, 12501n, 3325n], - ]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - afterAll(async () => { - await model?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); }); - }); }); describe("Tiny random pipelines", () => { - describe("fill-mask", () => { - const model_id = "hf-internal-testing/tiny-random-BertForMaskedLM"; - - /** @type {FillMaskPipeline} */ - let pipe; - beforeAll(async () => { - pipe = await pipeline("fill-mask", model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, - }); - }, MAX_MODEL_LOAD_TIME); - - describe("batch_size=1", () => { - it( - "default (top_k=5)", - async () => { - const output = await pipe("a [MASK] c"); - const target = [ - { score: 0.0013377574505284429, token: 854, token_str: "##ο", sequence: "aο c" }, - { score: 0.001248967950232327, token: 962, token_str: "##ち", sequence: "aち c" }, - { score: 0.0012304208939895034, token: 933, token_str: "##ع", sequence: "aع c" }, - { score: 0.0012301815440878272, token: 313, token_str: "ფ", sequence: "a ფ c" }, - { score: 0.001222139224410057, token: 624, token_str: "未", sequence: "a 未 c" }, - ]; - compare(output, target, 1e-5); - }, - MAX_TEST_EXECUTION_TIME, - ); - it( - "custom (top_k=2)", - async () => { - const output = await pipe("a [MASK] c", { top_k: 2 }); - const target = [ - { score: 0.0013377574505284429, token: 854, token_str: "##ο", sequence: "aο c" }, - { score: 0.001248967950232327, token: 962, token_str: "##ち", sequence: "aち c" }, - ]; - compare(output, target, 1e-5); - }, - MAX_TEST_EXECUTION_TIME, - ); - }); + describe("fill-mask", () => { + const model_id = "hf-internal-testing/tiny-random-BertForMaskedLM"; + + /** @type {FillMaskPipeline} */ + let pipe; + beforeAll(async () => { + pipe = await pipeline("fill-mask", model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + }, MAX_MODEL_LOAD_TIME); + + describe("batch_size=1", () => { + it( + "default (top_k=5)", + async () => { + const output = await pipe("a [MASK] c"); + const target = [ + { + score: 0.0013377574505284429, + token: 854, + token_str: "##ο", + sequence: "aο c", + }, + { + score: 0.001248967950232327, + token: 962, + token_str: "##ち", + sequence: "aち c", + }, + { + score: 0.0012304208939895034, + token: 933, + token_str: "##ع", + sequence: "aع c", + }, + { + score: 0.0012301815440878272, + token: 313, + token_str: "ფ", + sequence: "a ფ c", + }, + { + score: 0.001222139224410057, + token: 624, + token_str: "未", + sequence: "a 未 c", + }, + ]; + compare(output, target, 1e-5); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "custom (top_k=2)", + async () => { + const output = await pipe("a [MASK] c", { top_k: 2 }); + const target = [ + { + score: 0.0013377574505284429, + token: 854, + token_str: "##ο", + sequence: "aο c", + }, + { + score: 0.001248967950232327, + token: 962, + token_str: "##ち", + sequence: "aち c", + }, + ]; + compare(output, target, 1e-5); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); - describe("batch_size>1", () => { - it( - "default (top_k=5)", - async () => { - const output = await pipe(["a [MASK] c", "a b [MASK] c"]); - const target = [ - [ - { score: 0.0013377574505284429, token: 854, token_str: "##ο", sequence: "aο c" }, - { score: 0.001248967950232327, token: 962, token_str: "##ち", sequence: "aち c" }, - { score: 0.0012304208939895034, token: 933, token_str: "##ع", sequence: "aع c" }, - { score: 0.0012301815440878272, token: 313, token_str: "ფ", sequence: "a ფ c" }, - { score: 0.001222139224410057, token: 624, token_str: "未", sequence: "a 未 c" }, - ], - [ - { score: 0.0013287801994010806, token: 962, token_str: "##ち", sequence: "a bち c" }, - { score: 0.0012486606137827039, token: 823, token_str: "##ن", sequence: "a bن c" }, - { score: 0.0012320734094828367, token: 1032, token_str: "##ც", sequence: "a bც c" }, - { score: 0.0012295148335397243, token: 854, token_str: "##ο", sequence: "a bο c" }, - { score: 0.0012277684872969985, token: 624, token_str: "未", sequence: "a b 未 c" }, - ], - ]; - compare(output, target, 1e-5); - }, - MAX_TEST_EXECUTION_TIME, - ); - it( - "custom (top_k=2)", - async () => { - const output = await pipe(["a [MASK] c", "a b [MASK] c"], { top_k: 2 }); - const target = [ - [ - { score: 0.0013377574505284429, token: 854, token_str: "##ο", sequence: "aο c" }, - { score: 0.001248967950232327, token: 962, token_str: "##ち", sequence: "aち c" }, - ], - [ - { score: 0.0013287801994010806, token: 962, token_str: "##ち", sequence: "a bち c" }, - { score: 0.0012486606137827039, token: 823, token_str: "##ن", sequence: "a bن c" }, - ], - ]; - compare(output, target, 1e-5); - }, - MAX_TEST_EXECUTION_TIME, - ); - }); + describe("batch_size>1", () => { + it( + "default (top_k=5)", + async () => { + const output = await pipe(["a [MASK] c", "a b [MASK] c"]); + const target = [ + [ + { + score: 0.0013377574505284429, + token: 854, + token_str: "##ο", + sequence: "aο c", + }, + { + score: 0.001248967950232327, + token: 962, + token_str: "##ち", + sequence: "aち c", + }, + { + score: 0.0012304208939895034, + token: 933, + token_str: "##ع", + sequence: "aع c", + }, + { + score: 0.0012301815440878272, + token: 313, + token_str: "ფ", + sequence: "a ფ c", + }, + { + score: 0.001222139224410057, + token: 624, + token_str: "未", + sequence: "a 未 c", + }, + ], + [ + { + score: 0.0013287801994010806, + token: 962, + token_str: "##ち", + sequence: "a bち c", + }, + { + score: 0.0012486606137827039, + token: 823, + token_str: "##ن", + sequence: "a bن c", + }, + { + score: 0.0012320734094828367, + token: 1032, + token_str: "##ც", + sequence: "a bც c", + }, + { + score: 0.0012295148335397243, + token: 854, + token_str: "##ο", + sequence: "a bο c", + }, + { + score: 0.0012277684872969985, + token: 624, + token_str: "未", + sequence: "a b 未 c", + }, + ], + ]; + compare(output, target, 1e-5); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "custom (top_k=2)", + async () => { + const output = await pipe(["a [MASK] c", "a b [MASK] c"], { top_k: 2 }); + const target = [ + [ + { + score: 0.0013377574505284429, + token: 854, + token_str: "##ο", + sequence: "aο c", + }, + { + score: 0.001248967950232327, + token: 962, + token_str: "##ち", + sequence: "aち c", + }, + ], + [ + { + score: 0.0013287801994010806, + token: 962, + token_str: "##ち", + sequence: "a bち c", + }, + { + score: 0.0012486606137827039, + token: 823, + token_str: "##ن", + sequence: "a bن c", + }, + ], + ]; + compare(output, target, 1e-5); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); - afterAll(async () => { - await pipe?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); - }); - - describe("text-classification", () => { - const model_id = "hf-internal-testing/tiny-random-BertForSequenceClassification"; - - /** @type {TextClassificationPipeline} */ - let pipe; - beforeAll(async () => { - pipe = await pipeline("text-classification", model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, - }); - }, MAX_MODEL_LOAD_TIME); - - describe("batch_size=1", () => { - it( - "default (top_k=1)", - async () => { - const output = await pipe("a"); - const target = [{ label: "LABEL_0", score: 0.5076976418495178 }]; - compare(output, target, 1e-5); - }, - MAX_TEST_EXECUTION_TIME, - ); - it( - "custom (top_k=2)", - async () => { - const output = await pipe("a", { top_k: 2 }); - const target = [ - { label: "LABEL_0", score: 0.5076976418495178 }, - { label: "LABEL_1", score: 0.49230238795280457 }, - ]; - compare(output, target, 1e-5); - }, - MAX_TEST_EXECUTION_TIME, - ); + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); }); - describe("batch_size>1", () => { - it( - "default (top_k=1)", - async () => { - const output = await pipe(["a", "b c"]); - const target = [ - { label: "LABEL_0", score: 0.5076976418495178 }, - { label: "LABEL_0", score: 0.5077522993087769 }, - ]; - compare(output, target, 1e-5); - }, - MAX_TEST_EXECUTION_TIME, - ); - it( - "custom (top_k=2)", - async () => { - const output = await pipe(["a", "b c"], { top_k: 2 }); - const target = [ - [ - { label: "LABEL_0", score: 0.5076976418495178 }, - { label: "LABEL_1", score: 0.49230238795280457 }, - ], - [ - { label: "LABEL_0", score: 0.5077522993087769 }, - { label: "LABEL_1", score: 0.49224773049354553 }, - ], - ]; - compare(output, target, 1e-5); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "multi_label_classification", - async () => { - const problem_type = pipe.model.config.problem_type; - pipe.model.config.problem_type = "multi_label_classification"; - - const output = await pipe(["a", "b c"], { top_k: 2 }); - const target = [ - [ - { label: "LABEL_0", score: 0.5001373887062073 }, - { label: "LABEL_1", score: 0.49243971705436707 }, - ], - [ - { label: "LABEL_0", score: 0.5001326203346252 }, - { label: "LABEL_1", score: 0.492380291223526 }, - ], - ]; - compare(output, target, 1e-5); - - // Reset problem type - pipe.model.config.problem_type = problem_type; - }, - MAX_TEST_EXECUTION_TIME, - ); - }); + describe("text-classification", () => { + const model_id = "hf-internal-testing/tiny-random-BertForSequenceClassification"; - afterAll(async () => { - await pipe?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); - }); - - describe("token-classification", () => { - const model_id = "hf-internal-testing/tiny-random-BertForTokenClassification"; - - /** @type {TokenClassificationPipeline} */ - let pipe; - beforeAll(async () => { - pipe = await pipeline("token-classification", model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, - }); - }, MAX_MODEL_LOAD_TIME); - - describe("batch_size=1", () => { - it( - "default", - async () => { - const output = await pipe("1 2 3"); - - // TODO: Add start/end to target - const target = [ - { - entity: "LABEL_0", - score: 0.5292708, - index: 1, - word: "1", - // 'start': 0, 'end': 1 - }, - { - entity: "LABEL_0", - score: 0.5353687, - index: 2, - word: "2", - // 'start': 2, 'end': 3 - }, - { - entity: "LABEL_1", - score: 0.51381934, - index: 3, - word: "3", - // 'start': 4, 'end': 5 - }, - ]; - compare(output, target, 1e-5); - }, - MAX_TEST_EXECUTION_TIME, - ); - it( - "custom (ignore_labels set)", - async () => { - const output = await pipe("1 2 3", { ignore_labels: ["LABEL_0"] }); - const target = [ - { - entity: "LABEL_1", - score: 0.51381934, - index: 3, - word: "3", - // 'start': 4, 'end': 5 - }, - ]; - compare(output, target, 1e-5); - }, - MAX_TEST_EXECUTION_TIME, - ); - }); + /** @type {TextClassificationPipeline} */ + let pipe; + beforeAll(async () => { + pipe = await pipeline("text-classification", model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + }, MAX_MODEL_LOAD_TIME); + + describe("batch_size=1", () => { + it( + "default (top_k=1)", + async () => { + const output = await pipe("a"); + const target = [{ label: "LABEL_0", score: 0.5076976418495178 }]; + compare(output, target, 1e-5); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "custom (top_k=2)", + async () => { + const output = await pipe("a", { top_k: 2 }); + const target = [ + { label: "LABEL_0", score: 0.5076976418495178 }, + { label: "LABEL_1", score: 0.49230238795280457 }, + ]; + compare(output, target, 1e-5); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); - describe("batch_size>1", () => { - it( - "default", - async () => { - const output = await pipe(["1 2 3", "4 5"]); - const target = [ - [ - { - entity: "LABEL_0", - score: 0.5292708, - index: 1, - word: "1", - // 'start': 0, 'end': 1 - }, - { - entity: "LABEL_0", - score: 0.5353687, - index: 2, - word: "2", - // 'start': 2, 'end': 3 - }, - { - entity: "LABEL_1", - score: 0.51381934, - index: 3, - word: "3", - // 'start': 4, 'end': 5 - }, - ], - [ - { - entity: "LABEL_0", - score: 0.5432807, - index: 1, - word: "4", - // 'start': 0, 'end': 1 - }, - { - entity: "LABEL_1", - score: 0.5007693, - index: 2, - word: "5", - // 'start': 2, 'end': 3 - }, - ], - ]; - compare(output, target, 1e-5); - }, - MAX_TEST_EXECUTION_TIME, - ); - it( - "custom (ignore_labels set)", - async () => { - const output = await pipe(["1 2 3", "4 5"], { ignore_labels: ["LABEL_0"] }); - const target = [ - [ - { - entity: "LABEL_1", - score: 0.51381934, - index: 3, - word: "3", - // 'start': 4, 'end': 5 - }, - ], - [ - { - entity: "LABEL_1", - score: 0.5007693, - index: 2, - word: "5", - // 'start': 2, 'end': 3 - }, - ], - ]; - compare(output, target, 1e-5); - }, - MAX_TEST_EXECUTION_TIME, - ); - }); + describe("batch_size>1", () => { + it( + "default (top_k=1)", + async () => { + const output = await pipe(["a", "b c"]); + const target = [ + { label: "LABEL_0", score: 0.5076976418495178 }, + { label: "LABEL_0", score: 0.5077522993087769 }, + ]; + compare(output, target, 1e-5); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "custom (top_k=2)", + async () => { + const output = await pipe(["a", "b c"], { top_k: 2 }); + const target = [ + [ + { label: "LABEL_0", score: 0.5076976418495178 }, + { label: "LABEL_1", score: 0.49230238795280457 }, + ], + [ + { label: "LABEL_0", score: 0.5077522993087769 }, + { label: "LABEL_1", score: 0.49224773049354553 }, + ], + ]; + compare(output, target, 1e-5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "multi_label_classification", + async () => { + const problem_type = pipe.model.config.problem_type; + pipe.model.config.problem_type = "multi_label_classification"; + + const output = await pipe(["a", "b c"], { top_k: 2 }); + const target = [ + [ + { label: "LABEL_0", score: 0.5001373887062073 }, + { label: "LABEL_1", score: 0.49243971705436707 }, + ], + [ + { label: "LABEL_0", score: 0.5001326203346252 }, + { label: "LABEL_1", score: 0.492380291223526 }, + ], + ]; + compare(output, target, 1e-5); + + // Reset problem type + pipe.model.config.problem_type = problem_type; + }, + MAX_TEST_EXECUTION_TIME, + ); + }); - afterAll(async () => { - await pipe?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); - }); - - describe("question-answering", () => { - const model_id = "hf-internal-testing/tiny-random-BertForQuestionAnswering"; - - /** @type {QuestionAnsweringPipeline} */ - let pipe; - beforeAll(async () => { - pipe = await pipeline("question-answering", model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, - }); - }, MAX_MODEL_LOAD_TIME); - - describe("batch_size=1", () => { - it( - "default (top_k=1)", - async () => { - const output = await pipe("a", "b c"); - const target = { score: 0.11395696550607681, /* start: 0, end: 1, */ answer: "b" }; - compare(output, target, 1e-5); - }, - MAX_TEST_EXECUTION_TIME, - ); - it( - "custom (top_k=3)", - async () => { - const output = await pipe("a", "b c", { top_k: 3 }); - const target = [ - { score: 0.11395696550607681, /* start: 0, end: 1, */ answer: "b" }, - { score: 0.11300431191921234, /* start: 2, end: 3, */ answer: "c" }, - { score: 0.10732574015855789, /* start: 0, end: 3, */ answer: "b c" }, - ]; - compare(output, target, 1e-5); - }, - MAX_TEST_EXECUTION_TIME, - ); + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); }); - afterAll(async () => { - await pipe?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); - }); - - describe("image-classification", () => { - const model_id = "hf-internal-testing/tiny-random-vit"; - const urls = ["https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/white-image.png", "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/blue-image.png"]; - - /** @type {ImageClassificationPipeline} */ - let pipe; - beforeAll(async () => { - pipe = await pipeline("image-classification", model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, - }); - }, MAX_MODEL_LOAD_TIME); - - describe("batch_size=1", () => { - it( - "default (top_k=5)", - async () => { - const output = await pipe(urls[0]); - const target = [ - { label: "LABEL_1", score: 0.5020533800125122 }, - { label: "LABEL_0", score: 0.4979466497898102 }, - ]; - compare(output, target, 1e-5); - }, - MAX_TEST_EXECUTION_TIME, - ); - it( - "custom (top_k=1)", - async () => { - const output = await pipe(urls[0], { top_k: 1 }); - const target = [{ label: "LABEL_1", score: 0.5020533800125122 }]; - compare(output, target, 1e-5); - }, - MAX_TEST_EXECUTION_TIME, - ); - }); + describe("token-classification", () => { + const model_id = "hf-internal-testing/tiny-random-BertForTokenClassification"; - describe("batch_size>1", () => { - it( - "default (top_k=5)", - async () => { - const output = await pipe(urls); - const target = [ - [ - { label: "LABEL_1", score: 0.5020533800125122 }, - { label: "LABEL_0", score: 0.4979466497898102 }, - ], - [ - { label: "LABEL_1", score: 0.519227921962738 }, - { label: "LABEL_0", score: 0.4807720482349396 }, - ], - ]; - compare(output, target, 1e-5); - }, - MAX_TEST_EXECUTION_TIME, - ); - it( - "custom (top_k=1)", - async () => { - const output = await pipe(urls, { top_k: 1 }); - const target = [[{ label: "LABEL_1", score: 0.5020533800125122 }], [{ label: "LABEL_1", score: 0.519227921962738 }]]; - compare(output, target, 1e-5); - }, - MAX_TEST_EXECUTION_TIME, - ); - }); + /** @type {TokenClassificationPipeline} */ + let pipe; + beforeAll(async () => { + pipe = await pipeline("token-classification", model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + }, MAX_MODEL_LOAD_TIME); + + describe("batch_size=1", () => { + it( + "default", + async () => { + const output = await pipe("1 2 3"); + + // TODO: Add start/end to target + const target = [ + { + entity: "LABEL_0", + score: 0.5292708, + index: 1, + word: "1", + // 'start': 0, 'end': 1 + }, + { + entity: "LABEL_0", + score: 0.5353687, + index: 2, + word: "2", + // 'start': 2, 'end': 3 + }, + { + entity: "LABEL_1", + score: 0.51381934, + index: 3, + word: "3", + // 'start': 4, 'end': 5 + }, + ]; + compare(output, target, 1e-5); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "custom (ignore_labels set)", + async () => { + const output = await pipe("1 2 3", { ignore_labels: ["LABEL_0"] }); + const target = [ + { + entity: "LABEL_1", + score: 0.51381934, + index: 3, + word: "3", + // 'start': 4, 'end': 5 + }, + ]; + compare(output, target, 1e-5); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); - afterAll(async () => { - await pipe?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); - }); - - describe("zero-shot-image-classification", () => { - const model_id = "hf-internal-testing/tiny-random-GroupViTModel"; - - // Example adapted from https://huggingface.co/docs/transformers/en/model_doc/groupvit - const urls = ["https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/white-image.png", "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/blue-image.png"]; - const labels = ["cat", "dog"]; - const hypothesis_template = "a photo of a {}"; - - /** @type {ZeroShotImageClassificationPipeline} */ - let pipe; - beforeAll(async () => { - pipe = await pipeline("zero-shot-image-classification", model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, - }); - }, MAX_MODEL_LOAD_TIME); - - describe("batch_size=1", () => { - it( - "default", - async () => { - const output = await pipe(urls[0], labels); - const target = [ - { score: 0.5990662574768066, label: "cat" }, - { score: 0.40093377232551575, label: "dog" }, - ]; - compare(output, target, 1e-5); - }, - MAX_TEST_EXECUTION_TIME, - ); - it( - "custom (w/ hypothesis_template)", - async () => { - const output = await pipe(urls[0], labels, { hypothesis_template }); - const target = [ - { score: 0.5527022480964661, label: "cat" }, - { score: 0.44729775190353394, label: "dog" }, - ]; - compare(output, target, 1e-5); - }, - MAX_TEST_EXECUTION_TIME, - ); - }); + describe("batch_size>1", () => { + it( + "default", + async () => { + const output = await pipe(["1 2 3", "4 5"]); + const target = [ + [ + { + entity: "LABEL_0", + score: 0.5292708, + index: 1, + word: "1", + // 'start': 0, 'end': 1 + }, + { + entity: "LABEL_0", + score: 0.5353687, + index: 2, + word: "2", + // 'start': 2, 'end': 3 + }, + { + entity: "LABEL_1", + score: 0.51381934, + index: 3, + word: "3", + // 'start': 4, 'end': 5 + }, + ], + [ + { + entity: "LABEL_0", + score: 0.5432807, + index: 1, + word: "4", + // 'start': 0, 'end': 1 + }, + { + entity: "LABEL_1", + score: 0.5007693, + index: 2, + word: "5", + // 'start': 2, 'end': 3 + }, + ], + ]; + compare(output, target, 1e-5); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "custom (ignore_labels set)", + async () => { + const output = await pipe(["1 2 3", "4 5"], { ignore_labels: ["LABEL_0"] }); + const target = [ + [ + { + entity: "LABEL_1", + score: 0.51381934, + index: 3, + word: "3", + // 'start': 4, 'end': 5 + }, + ], + [ + { + entity: "LABEL_1", + score: 0.5007693, + index: 2, + word: "5", + // 'start': 2, 'end': 3 + }, + ], + ]; + compare(output, target, 1e-5); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); - describe("batch_size>1", () => { - it( - "default", - async () => { - const output = await pipe(urls, labels); - const target = [ - [ - { score: 0.5990662574768066, label: "cat" }, - { score: 0.40093377232551575, label: "dog" }, - ], - [ - { score: 0.5006340146064758, label: "dog" }, - { score: 0.49936598539352417, label: "cat" }, - ], - ]; - compare(output, target, 1e-5); - }, - MAX_TEST_EXECUTION_TIME, - ); - it( - "custom (w/ hypothesis_template)", - async () => { - const output = await pipe(urls, labels, { hypothesis_template }); - const target = [ - [ - { score: 0.5527022480964661, label: "cat" }, - { score: 0.44729775190353394, label: "dog" }, - ], - [ - { score: 0.5395973324775696, label: "cat" }, - { score: 0.46040263772010803, label: "dog" }, - ], - ]; - compare(output, target, 1e-5); - }, - MAX_TEST_EXECUTION_TIME, - ); + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); }); - afterAll(async () => { - await pipe?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); - }); - - describe("audio-classification", () => { - const model_id = "hf-internal-testing/tiny-random-unispeech"; - const audios = [new Float32Array(16000).fill(0), Float32Array.from({ length: 16000 }, (_, i) => i)]; - - /** @type {ImageClassificationPipeline} */ - let pipe; - beforeAll(async () => { - pipe = await pipeline("audio-classification", model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, - }); - }, MAX_MODEL_LOAD_TIME); - - describe("batch_size=1", () => { - it( - "default (top_k=5)", - async () => { - const output = await pipe(audios[0]); - const target = [ - { score: 0.5043687224388123, label: "LABEL_0" }, - { score: 0.4956313371658325, label: "LABEL_1" }, - ]; - compare(output, target, 1e-5); - }, - MAX_TEST_EXECUTION_TIME, - ); - it( - "custom (top_k=1)", - async () => { - const output = await pipe(audios[0], { top_k: 1 }); - const target = [{ score: 0.5043687224388123, label: "LABEL_0" }]; - compare(output, target, 1e-5); - }, - MAX_TEST_EXECUTION_TIME, - ); - }); + describe("question-answering", () => { + const model_id = "hf-internal-testing/tiny-random-BertForQuestionAnswering"; - describe("batch_size>1", () => { - it( - "default (top_k=5)", - async () => { - const output = await pipe(audios); - const target = [ - [ - { score: 0.5043687224388123, label: "LABEL_0" }, - { score: 0.4956313371658325, label: "LABEL_1" }, - ], - [ - { score: 0.5187293887138367, label: "LABEL_0" }, - { score: 0.4812707006931305, label: "LABEL_1" }, - ], - ]; - compare(output, target, 1e-5); - }, - MAX_TEST_EXECUTION_TIME, - ); - it( - "custom (top_k=1)", - async () => { - const output = await pipe(audios, { top_k: 1 }); - const target = [[{ score: 0.5043687224388123, label: "LABEL_0" }], [{ score: 0.5187293887138367, label: "LABEL_0" }]]; - compare(output, target, 1e-5); - }, - MAX_TEST_EXECUTION_TIME, - ); - }); + /** @type {QuestionAnsweringPipeline} */ + let pipe; + beforeAll(async () => { + pipe = await pipeline("question-answering", model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + }, MAX_MODEL_LOAD_TIME); + + describe("batch_size=1", () => { + it( + "default (top_k=1)", + async () => { + const output = await pipe("a", "b c"); + const target = { + score: 0.11395696550607681, + /* start: 0, end: 1, */ answer: "b", + }; + compare(output, target, 1e-5); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "custom (top_k=3)", + async () => { + const output = await pipe("a", "b c", { top_k: 3 }); + const target = [ + { score: 0.11395696550607681, /* start: 0, end: 1, */ answer: "b" }, + { score: 0.11300431191921234, /* start: 2, end: 3, */ answer: "c" }, + { score: 0.10732574015855789, /* start: 0, end: 3, */ answer: "b c" }, + ]; + compare(output, target, 1e-5); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); - afterAll(async () => { - await pipe?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); - }); - - describe("text-generation", () => { - const model_id = "hf-internal-testing/tiny-random-LlamaForCausalLM"; - - /** @type {TextGenerationPipeline} */ - let pipe; - beforeAll(async () => { - pipe = await pipeline("text-generation", model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, - }); - }, MAX_MODEL_LOAD_TIME); - - describe("batch_size=1", () => { - const text_input = "hello"; - const generated_text_target = "erdingsAndroid Load"; - const text_target = [{ generated_text: text_input + generated_text_target }]; - const new_text_target = [{ generated_text: generated_text_target }]; - - const chat_input = [ - { role: "system", content: "a" }, - { role: "user", content: "b" }, - ]; - const chat_target = [ - { - generated_text: [ - { role: "system", content: "a" }, - { role: "user", content: "b" }, - { role: "assistant", content: " Southern abund Load" }, - ], - }, - ]; - - it( - "text input (single)", - async () => { - const output = await pipe(text_input, { max_new_tokens: 3 }); - compare(output, text_target); - }, - MAX_TEST_EXECUTION_TIME, - ); - it( - "text input (list)", - async () => { - const output = await pipe([text_input], { max_new_tokens: 3 }); - compare(output, [text_target]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "text input (single) - return_full_text=false", - async () => { - const output = await pipe(text_input, { max_new_tokens: 3, return_full_text: false }); - compare(output, new_text_target); - }, - MAX_TEST_EXECUTION_TIME, - ); - it( - "text input (list) - return_full_text=false", - async () => { - const output = await pipe([text_input], { max_new_tokens: 3, return_full_text: false }); - compare(output, [new_text_target]); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "chat input (single)", - async () => { - const output = await pipe(chat_input, { max_new_tokens: 3 }); - compare(output, chat_target); - }, - MAX_TEST_EXECUTION_TIME, - ); - it( - "chat input (list)", - async () => { - const output = await pipe([chat_input], { max_new_tokens: 3 }); - compare(output, [chat_target]); - }, - MAX_TEST_EXECUTION_TIME, - ); + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); }); - // TODO: Fix batch_size>1 - // describe('batch_size>1', () => { - // it('default', async () => { - // const output = await pipe(['hello', 'hello world']); - // const target = [ - // [{generated_text: 'helloerdingsAndroid Load'}], - // [{generated_text: 'hello world zerosMillнал'}], - // ]; - // compare(output, target); - // }, MAX_TEST_EXECUTION_TIME); - // }); - - afterAll(async () => { - await pipe?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); - }); - - describe("translation", () => { - const model_id = "Xenova/tiny-random-M2M100ForConditionalGeneration"; - - /** @type {TranslationPipeline} */ - let pipe; - beforeAll(async () => { - pipe = await pipeline("translation", model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, - }); - }, MAX_MODEL_LOAD_TIME); - - describe("batch_size=1", () => { - it( - "default", - async () => { - const text = "जीवन एक चॉकलेट बॉक्स की तरह है।"; - const output = await pipe(text, { - src_lang: "hi", - tgt_lang: "fr", - max_new_tokens: 5, - }); - const target = [{ translation_text: "Slovenska төсли төсли төсли" }]; - compare(output, target); - }, - MAX_TEST_EXECUTION_TIME, - ); - }); + describe("image-classification", () => { + const model_id = "hf-internal-testing/tiny-random-vit"; + const urls = ["https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/white-image.png", "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/blue-image.png"]; - afterAll(async () => { - await pipe?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); - }); - - describe("object-detection", () => { - const model_id = "hf-internal-testing/tiny-random-DetrForObjectDetection"; - const urls = ["https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/white-image.png", "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/blue-image.png"]; - - /** @type {ImageClassificationPipeline} */ - let pipe; - beforeAll(async () => { - pipe = await pipeline("object-detection", model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, - }); - }, MAX_MODEL_LOAD_TIME); - - describe("batch_size=1", () => { - it( - "default (threshold unset)", - async () => { - const output = await pipe(urls[0]); - const target = []; - compare(output, target, 1e-5); - }, - MAX_TEST_EXECUTION_TIME, - ); - it( - "default (threshold=0)", - async () => { - const output = await pipe(urls[0], { threshold: 0 }); - const target = [ - { score: 0.020360443741083145, label: "LABEL_31", box: { xmin: 56, ymin: 55, xmax: 169, ymax: 167 } }, - { score: 0.020360419526696205, label: "LABEL_31", box: { xmin: 56, ymin: 55, xmax: 169, ymax: 167 } }, - { score: 0.02036038413643837, label: "LABEL_31", box: { xmin: 56, ymin: 55, xmax: 169, ymax: 167 } }, - { score: 0.020360447466373444, label: "LABEL_31", box: { xmin: 56, ymin: 55, xmax: 169, ymax: 167 } }, - { score: 0.020360389724373817, label: "LABEL_31", box: { xmin: 56, ymin: 55, xmax: 169, ymax: 167 } }, - { score: 0.020360423251986504, label: "LABEL_31", box: { xmin: 56, ymin: 55, xmax: 169, ymax: 167 } }, - { score: 0.02036040835082531, label: "LABEL_31", box: { xmin: 56, ymin: 55, xmax: 169, ymax: 167 } }, - { score: 0.020360363647341728, label: "LABEL_31", box: { xmin: 56, ymin: 55, xmax: 169, ymax: 167 } }, - { score: 0.020360389724373817, label: "LABEL_31", box: { xmin: 56, ymin: 55, xmax: 169, ymax: 167 } }, - { score: 0.020360389724373817, label: "LABEL_31", box: { xmin: 56, ymin: 55, xmax: 169, ymax: 167 } }, - { score: 0.020360343158245087, label: "LABEL_31", box: { xmin: 56, ymin: 55, xmax: 169, ymax: 167 } }, - { score: 0.020360423251986504, label: "LABEL_31", box: { xmin: 56, ymin: 55, xmax: 169, ymax: 167 } }, - ]; - compare(output, target, 1e-5); - }, - MAX_TEST_EXECUTION_TIME, - ); - }); + /** @type {ImageClassificationPipeline} */ + let pipe; + beforeAll(async () => { + pipe = await pipeline("image-classification", model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + }, MAX_MODEL_LOAD_TIME); + + describe("batch_size=1", () => { + it( + "default (top_k=5)", + async () => { + const output = await pipe(urls[0]); + const target = [ + { label: "LABEL_1", score: 0.5020533800125122 }, + { label: "LABEL_0", score: 0.4979466497898102 }, + ]; + compare(output, target, 1e-5); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "custom (top_k=1)", + async () => { + const output = await pipe(urls[0], { top_k: 1 }); + const target = [{ label: "LABEL_1", score: 0.5020533800125122 }]; + compare(output, target, 1e-5); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); - // TODO: Add batched support to object detection pipeline - // describe('batch_size>1', () => { - // it('default (threshold unset)', async () => { - // const output = await pipe(urls); - // console.log(output); - // const target = []; - // compare(output, target, 1e-5); - // }, MAX_TEST_EXECUTION_TIME); - // it('default (threshold=0)', async () => { - // const output = await pipe(urls, { threshold: 0 }); - // console.log(output); - // const target = []; - // compare(output, target, 1e-5); - // }, MAX_TEST_EXECUTION_TIME); - // }); - - afterAll(async () => { - await pipe?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); - }); - - describe("document-question-answering", () => { - const model_id = "hf-internal-testing/tiny-random-VisionEncoderDecoderModel-donutswin-mbart"; - - /** @type {DocumentQuestionAnsweringPipeline} */ - let pipe; - beforeAll(async () => { - pipe = await pipeline("document-question-answering", model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, - }); - }, MAX_MODEL_LOAD_TIME); - - describe("batch_size=1", () => { - it( - "default", - async () => { - const dims = [64, 32, 3]; - const image = new RawImage(new Uint8ClampedArray(dims[0] * dims[1] * dims[2]).fill(255), ...dims); - const question = "What is the invoice number?"; - const output = await pipe(image, question); - - const target = [{ answer: null }]; - compare(output, target); - }, - MAX_TEST_EXECUTION_TIME, - ); + describe("batch_size>1", () => { + it( + "default (top_k=5)", + async () => { + const output = await pipe(urls); + const target = [ + [ + { label: "LABEL_1", score: 0.5020533800125122 }, + { label: "LABEL_0", score: 0.4979466497898102 }, + ], + [ + { label: "LABEL_1", score: 0.519227921962738 }, + { label: "LABEL_0", score: 0.4807720482349396 }, + ], + ]; + compare(output, target, 1e-5); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "custom (top_k=1)", + async () => { + const output = await pipe(urls, { top_k: 1 }); + const target = [[{ label: "LABEL_1", score: 0.5020533800125122 }], [{ label: "LABEL_1", score: 0.519227921962738 }]]; + compare(output, target, 1e-5); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); }); - afterAll(async () => { - await pipe?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); - }); -}); + describe("zero-shot-image-classification", () => { + const model_id = "hf-internal-testing/tiny-random-GroupViTModel"; -describe("PKV caching", () => { - describe("LlamaForCausalLM", () => { - const model_id = "hf-internal-testing/tiny-random-LlamaForCausalLM"; - /** @type {LlamaForCausalLM} */ - let model; - /** @type {LlamaTokenizer} */ - let tokenizer; - beforeAll(async () => { - model = await LlamaForCausalLM.from_pretrained(model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, - }); - tokenizer = await LlamaTokenizer.from_pretrained(model_id); - }, MAX_MODEL_LOAD_TIME); - - it( - "batch_size=1", - async () => { - const inputs = tokenizer("1"); - - // Generate first sequence w/o PKV - // NOTE: `return_dict_in_generate=true` is required to get PKV - const { past_key_values, sequences } = await model.generate({ - ...inputs, - max_new_tokens: 5, - do_sample: false, - return_dict_in_generate: true, + // Example adapted from https://huggingface.co/docs/transformers/en/model_doc/groupvit + const urls = ["https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/white-image.png", "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/blue-image.png"]; + const labels = ["cat", "dog"]; + const hypothesis_template = "a photo of a {}"; + + /** @type {ZeroShotImageClassificationPipeline} */ + let pipe; + beforeAll(async () => { + pipe = await pipeline("zero-shot-image-classification", model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + }, MAX_MODEL_LOAD_TIME); + + describe("batch_size=1", () => { + it( + "default", + async () => { + const output = await pipe(urls[0], labels); + const target = [ + { score: 0.5990662574768066, label: "cat" }, + { score: 0.40093377232551575, label: "dog" }, + ]; + compare(output, target, 1e-5); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "custom (w/ hypothesis_template)", + async () => { + const output = await pipe(urls[0], labels, { hypothesis_template }); + const target = [ + { score: 0.5527022480964661, label: "cat" }, + { score: 0.44729775190353394, label: "dog" }, + ]; + compare(output, target, 1e-5); + }, + MAX_TEST_EXECUTION_TIME, + ); }); - // Update output with new text - const decoded = tokenizer.batch_decode(sequences, { - skip_special_tokens: false, - })[0]; - const new_inputs = tokenizer(decoded + "2", { - add_special_tokens: false, + describe("batch_size>1", () => { + it( + "default", + async () => { + const output = await pipe(urls, labels); + const target = [ + [ + { score: 0.5990662574768066, label: "cat" }, + { score: 0.40093377232551575, label: "dog" }, + ], + [ + { score: 0.5006340146064758, label: "dog" }, + { score: 0.49936598539352417, label: "cat" }, + ], + ]; + compare(output, target, 1e-5); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "custom (w/ hypothesis_template)", + async () => { + const output = await pipe(urls, labels, { hypothesis_template }); + const target = [ + [ + { score: 0.5527022480964661, label: "cat" }, + { score: 0.44729775190353394, label: "dog" }, + ], + [ + { score: 0.5395973324775696, label: "cat" }, + { score: 0.46040263772010803, label: "dog" }, + ], + ]; + compare(output, target, 1e-5); + }, + MAX_TEST_EXECUTION_TIME, + ); }); - // Run w/o PKV - const generated_ids = await model.generate({ - ...new_inputs, - max_new_tokens: 3, - do_sample: false, + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + describe("audio-classification", () => { + const model_id = "hf-internal-testing/tiny-random-unispeech"; + const audios = [new Float32Array(16000).fill(0), Float32Array.from({ length: 16000 }, (_, i) => i)]; + + /** @type {ImageClassificationPipeline} */ + let pipe; + beforeAll(async () => { + pipe = await pipeline("audio-classification", model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + }, MAX_MODEL_LOAD_TIME); + + describe("batch_size=1", () => { + it( + "default (top_k=5)", + async () => { + const output = await pipe(audios[0]); + const target = [ + { score: 0.5043687224388123, label: "LABEL_0" }, + { score: 0.4956313371658325, label: "LABEL_1" }, + ]; + compare(output, target, 1e-5); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "custom (top_k=1)", + async () => { + const output = await pipe(audios[0], { top_k: 1 }); + const target = [{ score: 0.5043687224388123, label: "LABEL_0" }]; + compare(output, target, 1e-5); + }, + MAX_TEST_EXECUTION_TIME, + ); }); - // Run w/ PKV - const generated_ids_pkv = await model.generate({ - ...new_inputs, - past_key_values, - max_new_tokens: 3, - do_sample: false, + describe("batch_size>1", () => { + it( + "default (top_k=5)", + async () => { + const output = await pipe(audios); + const target = [ + [ + { score: 0.5043687224388123, label: "LABEL_0" }, + { score: 0.4956313371658325, label: "LABEL_1" }, + ], + [ + { score: 0.5187293887138367, label: "LABEL_0" }, + { score: 0.4812707006931305, label: "LABEL_1" }, + ], + ]; + compare(output, target, 1e-5); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "custom (top_k=1)", + async () => { + const output = await pipe(audios, { top_k: 1 }); + const target = [[{ score: 0.5043687224388123, label: "LABEL_0" }], [{ score: 0.5187293887138367, label: "LABEL_0" }]]; + compare(output, target, 1e-5); + }, + MAX_TEST_EXECUTION_TIME, + ); }); - const target = [[1n, 259n, 29896n, 24959n, 22063n, 17192n, 12189n, 22468n, 29906n, 3399n, 24823n, 26470n]]; - - expect(generated_ids.tolist()).toEqual(target); - expect(generated_ids_pkv.tolist()).toEqual(target); - }, - MAX_TEST_EXECUTION_TIME, - ); - - afterAll(async () => { - await model?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); - }); - - describe("LlavaForConditionalGeneration", () => { - const model_id = "Xenova/tiny-random-LlavaForConditionalGeneration"; - /** @type {LlavaForConditionalGeneration} */ - let model; - /** @type {PreTrainedTokenizer} */ - let tokenizer; - /** @type {Processor} */ - let processor; - beforeAll(async () => { - model = await LlavaForConditionalGeneration.from_pretrained(model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, - }); - tokenizer = await AutoTokenizer.from_pretrained(model_id); - processor = await AutoProcessor.from_pretrained(model_id); - }, MAX_MODEL_LOAD_TIME); - - it( - "batch_size=1", - async () => { - const text_inputs = tokenizer("hello"); + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); - // Empty white image - const dims = [224, 224, 3]; - const image = new RawImage(new Uint8ClampedArray(dims[0] * dims[1] * dims[2]).fill(255), ...dims); - const vision_inputs = await processor(image); - - // Generate first sequence w/o PKV - // NOTE: `return_dict_in_generate=true` is required to get PKV - const { past_key_values, sequences } = await model.generate({ - ...text_inputs, - ...vision_inputs, - max_new_tokens: 5, - do_sample: false, - return_dict_in_generate: true, + describe("text-generation", () => { + const model_id = "hf-internal-testing/tiny-random-LlamaForCausalLM"; + + /** @type {TextGenerationPipeline} */ + let pipe; + beforeAll(async () => { + pipe = await pipeline("text-generation", model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + }, MAX_MODEL_LOAD_TIME); + + describe("batch_size=1", () => { + const text_input = "hello"; + const generated_text_target = "erdingsAndroid Load"; + const text_target = [{ generated_text: text_input + generated_text_target }]; + const new_text_target = [{ generated_text: generated_text_target }]; + + const chat_input = [ + { role: "system", content: "a" }, + { role: "user", content: "b" }, + ]; + const chat_target = [ + { + generated_text: [ + { role: "system", content: "a" }, + { role: "user", content: "b" }, + { role: "assistant", content: " Southern abund Load" }, + ], + }, + ]; + + it( + "text input (single)", + async () => { + const output = await pipe(text_input, { max_new_tokens: 3 }); + compare(output, text_target); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "text input (list)", + async () => { + const output = await pipe([text_input], { max_new_tokens: 3 }); + compare(output, [text_target]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "text input (single) - return_full_text=false", + async () => { + const output = await pipe(text_input, { + max_new_tokens: 3, + return_full_text: false, + }); + compare(output, new_text_target); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "text input (list) - return_full_text=false", + async () => { + const output = await pipe([text_input], { + max_new_tokens: 3, + return_full_text: false, + }); + compare(output, [new_text_target]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "chat input (single)", + async () => { + const output = await pipe(chat_input, { max_new_tokens: 3 }); + compare(output, chat_target); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "chat input (list)", + async () => { + const output = await pipe([chat_input], { max_new_tokens: 3 }); + compare(output, [chat_target]); + }, + MAX_TEST_EXECUTION_TIME, + ); }); - // Update output with new text - const decoded = tokenizer.batch_decode(sequences).map((x) => x + "new"); - const new_inputs = tokenizer(decoded, { - add_special_tokens: false, + // TODO: Fix batch_size>1 + // describe('batch_size>1', () => { + // it('default', async () => { + // const output = await pipe(['hello', 'hello world']); + // const target = [ + // [{generated_text: 'helloerdingsAndroid Load'}], + // [{generated_text: 'hello world zerosMillнал'}], + // ]; + // compare(output, target); + // }, MAX_TEST_EXECUTION_TIME); + // }); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + describe("translation", () => { + const model_id = "Xenova/tiny-random-M2M100ForConditionalGeneration"; + + /** @type {TranslationPipeline} */ + let pipe; + beforeAll(async () => { + pipe = await pipeline("translation", model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + }, MAX_MODEL_LOAD_TIME); + + describe("batch_size=1", () => { + it( + "default", + async () => { + const text = "जीवन एक चॉकलेट बॉक्स की तरह है।"; + const output = await pipe(text, { + src_lang: "hi", + tgt_lang: "fr", + max_new_tokens: 5, + }); + const target = [{ translation_text: "Slovenska төсли төсли төсли" }]; + compare(output, target); + }, + MAX_TEST_EXECUTION_TIME, + ); }); - // Run w/o PKV - const generated_ids = await model.generate({ - ...new_inputs, - ...vision_inputs, - max_new_tokens: 3, - do_sample: false, + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + describe("object-detection", () => { + const model_id = "hf-internal-testing/tiny-random-DetrForObjectDetection"; + const urls = ["https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/white-image.png", "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/blue-image.png"]; + + /** @type {ImageClassificationPipeline} */ + let pipe; + beforeAll(async () => { + pipe = await pipeline("object-detection", model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + }, MAX_MODEL_LOAD_TIME); + + describe("batch_size=1", () => { + it( + "default (threshold unset)", + async () => { + const output = await pipe(urls[0]); + const target = []; + compare(output, target, 1e-5); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "default (threshold=0)", + async () => { + const output = await pipe(urls[0], { threshold: 0 }); + const target = [ + { + score: 0.020360443741083145, + label: "LABEL_31", + box: { xmin: 56, ymin: 55, xmax: 169, ymax: 167 }, + }, + { + score: 0.020360419526696205, + label: "LABEL_31", + box: { xmin: 56, ymin: 55, xmax: 169, ymax: 167 }, + }, + { + score: 0.02036038413643837, + label: "LABEL_31", + box: { xmin: 56, ymin: 55, xmax: 169, ymax: 167 }, + }, + { + score: 0.020360447466373444, + label: "LABEL_31", + box: { xmin: 56, ymin: 55, xmax: 169, ymax: 167 }, + }, + { + score: 0.020360389724373817, + label: "LABEL_31", + box: { xmin: 56, ymin: 55, xmax: 169, ymax: 167 }, + }, + { + score: 0.020360423251986504, + label: "LABEL_31", + box: { xmin: 56, ymin: 55, xmax: 169, ymax: 167 }, + }, + { + score: 0.02036040835082531, + label: "LABEL_31", + box: { xmin: 56, ymin: 55, xmax: 169, ymax: 167 }, + }, + { + score: 0.020360363647341728, + label: "LABEL_31", + box: { xmin: 56, ymin: 55, xmax: 169, ymax: 167 }, + }, + { + score: 0.020360389724373817, + label: "LABEL_31", + box: { xmin: 56, ymin: 55, xmax: 169, ymax: 167 }, + }, + { + score: 0.020360389724373817, + label: "LABEL_31", + box: { xmin: 56, ymin: 55, xmax: 169, ymax: 167 }, + }, + { + score: 0.020360343158245087, + label: "LABEL_31", + box: { xmin: 56, ymin: 55, xmax: 169, ymax: 167 }, + }, + { + score: 0.020360423251986504, + label: "LABEL_31", + box: { xmin: 56, ymin: 55, xmax: 169, ymax: 167 }, + }, + ]; + compare(output, target, 1e-5); + }, + MAX_TEST_EXECUTION_TIME, + ); }); - // Run w/ PKV - const generated_ids_pkv = await model.generate({ - ...new_inputs, - past_key_values, - max_new_tokens: 3, - do_sample: false, + // TODO: Add batched support to object detection pipeline + // describe('batch_size>1', () => { + // it('default (threshold unset)', async () => { + // const output = await pipe(urls); + // console.log(output); + // const target = []; + // compare(output, target, 1e-5); + // }, MAX_TEST_EXECUTION_TIME); + // it('default (threshold=0)', async () => { + // const output = await pipe(urls, { threshold: 0 }); + // console.log(output); + // const target = []; + // compare(output, target, 1e-5); + // }, MAX_TEST_EXECUTION_TIME); + // }); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + describe("document-question-answering", () => { + const model_id = "hf-internal-testing/tiny-random-VisionEncoderDecoderModel-donutswin-mbart"; + + /** @type {DocumentQuestionAnsweringPipeline} */ + let pipe; + beforeAll(async () => { + pipe = await pipeline("document-question-answering", model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + }, MAX_MODEL_LOAD_TIME); + + describe("batch_size=1", () => { + it( + "default", + async () => { + const dims = [64, 32, 3]; + const image = new RawImage(new Uint8ClampedArray(dims[0] * dims[1] * dims[2]).fill(255), ...dims); + const question = "What is the invoice number?"; + const output = await pipe(image, question); + + const target = [{ answer: null }]; + compare(output, target); + }, + MAX_TEST_EXECUTION_TIME, + ); }); - const target = [[1n, 32000n, 29871n, 23927n, 359n, 1519n, 568n, 5769n, 1330n, 21544n, 11568n, 1482n, 7258n, 1250n, 16117n]]; - expect(generated_ids.tolist()).toEqual(target); - expect(generated_ids_pkv.tolist()).toEqual(target); - }, - MAX_TEST_EXECUTION_TIME, - ); - - afterAll(async () => { - await model?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); - }); + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}); + +describe("PKV caching", () => { + describe("LlamaForCausalLM", () => { + const model_id = "hf-internal-testing/tiny-random-LlamaForCausalLM"; + /** @type {LlamaForCausalLM} */ + let model; + /** @type {LlamaTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await LlamaForCausalLM.from_pretrained(model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + tokenizer = await LlamaTokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("1"); + + // Generate first sequence w/o PKV + // NOTE: `return_dict_in_generate=true` is required to get PKV + const { past_key_values, sequences } = await model.generate({ + ...inputs, + max_new_tokens: 5, + do_sample: false, + return_dict_in_generate: true, + }); + + // Update output with new text + const decoded = tokenizer.batch_decode(sequences, { + skip_special_tokens: false, + })[0]; + const new_inputs = tokenizer(decoded + "2", { + add_special_tokens: false, + }); + + // Run w/o PKV + const generated_ids = await model.generate({ + ...new_inputs, + max_new_tokens: 3, + do_sample: false, + }); + + // Run w/ PKV + const generated_ids_pkv = await model.generate({ + ...new_inputs, + past_key_values, + max_new_tokens: 3, + do_sample: false, + }); + + const target = [[1n, 259n, 29896n, 24959n, 22063n, 17192n, 12189n, 22468n, 29906n, 3399n, 24823n, 26470n]]; + + expect(generated_ids.tolist()).toEqual(target); + expect(generated_ids_pkv.tolist()).toEqual(target); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + describe("LlavaForConditionalGeneration", () => { + const model_id = "Xenova/tiny-random-LlavaForConditionalGeneration"; + /** @type {LlavaForConditionalGeneration} */ + let model; + /** @type {PreTrainedTokenizer} */ + let tokenizer; + /** @type {Processor} */ + let processor; + beforeAll(async () => { + model = await LlavaForConditionalGeneration.from_pretrained(model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + tokenizer = await AutoTokenizer.from_pretrained(model_id); + processor = await AutoProcessor.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const text_inputs = tokenizer("hello"); + + // Empty white image + const dims = [224, 224, 3]; + const image = new RawImage(new Uint8ClampedArray(dims[0] * dims[1] * dims[2]).fill(255), ...dims); + const vision_inputs = await processor(image); + + // Generate first sequence w/o PKV + // NOTE: `return_dict_in_generate=true` is required to get PKV + const { past_key_values, sequences } = await model.generate({ + ...text_inputs, + ...vision_inputs, + max_new_tokens: 5, + do_sample: false, + return_dict_in_generate: true, + }); + + // Update output with new text + const decoded = tokenizer.batch_decode(sequences).map((x) => x + "new"); + const new_inputs = tokenizer(decoded, { + add_special_tokens: false, + }); + + // Run w/o PKV + const generated_ids = await model.generate({ + ...new_inputs, + ...vision_inputs, + max_new_tokens: 3, + do_sample: false, + }); + + // Run w/ PKV + const generated_ids_pkv = await model.generate({ + ...new_inputs, + past_key_values, + max_new_tokens: 3, + do_sample: false, + }); + + const target = [[1n, 32000n, 29871n, 23927n, 359n, 1519n, 568n, 5769n, 1330n, 21544n, 11568n, 1482n, 7258n, 1250n, 16117n]]; + expect(generated_ids.tolist()).toEqual(target); + expect(generated_ids_pkv.tolist()).toEqual(target); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); }); diff --git a/tests/tokenizers.test.js b/tests/tokenizers.test.js index 00f58193d..41826cd14 100644 --- a/tests/tokenizers.test.js +++ b/tests/tokenizers.test.js @@ -7,496 +7,546 @@ const MAX_LOAD_TIME = 10_000; const MAX_EXECUTION_TIME = 10_000; describe("Tokenizers (model-specific)", () => { - for (const [tokenizer_name, { TOKENIZER_CLASS, TEST_CONFIG, CUSTOM_TESTS }] of Object.entries(TOKENIZER_TESTS)) { - describe(tokenizer_name, () => { - for (const model_id in TEST_CONFIG) { - describe(model_id, () => { - /** @type {import('../src/tokenizers.js').PreTrainedTokenizer} */ - let tokenizer; - beforeAll(async () => { - tokenizer = await TOKENIZER_CLASS.from_pretrained(model_id); - }, MAX_LOAD_TIME); - - for (const [test_name, test_case] of Object.entries(TEST_CONFIG[model_id])) { - test(test_name, () => { - if (test_case.ids) { - const ids = tokenizer.encode(test_case.text, { - text_pair: test_case.text_pair, + for (const [tokenizer_name, { TOKENIZER_CLASS, TEST_CONFIG, CUSTOM_TESTS }] of Object.entries(TOKENIZER_TESTS)) { + describe(tokenizer_name, () => { + for (const model_id in TEST_CONFIG) { + describe(model_id, () => { + /** @type {import('../src/tokenizers.js').PreTrainedTokenizer} */ + let tokenizer; + beforeAll(async () => { + tokenizer = await TOKENIZER_CLASS.from_pretrained(model_id); + }, MAX_LOAD_TIME); + + for (const [test_name, test_case] of Object.entries(TEST_CONFIG[model_id])) { + test(test_name, () => { + if (test_case.ids) { + const ids = tokenizer.encode(test_case.text, { + text_pair: test_case.text_pair, + }); + expect(ids).toEqual(test_case.ids); + } + if (test_case.tokens) { + const tokens = tokenizer.tokenize(test_case.text, { + pair: test_case.text_pair, + }); + expect(tokens).toEqual(test_case.tokens); + } + if (test_case.decoded) { + const decoded = tokenizer.decode(test_case.ids); + expect(decoded).toEqual(test_case.decoded); + } + }); + } }); - expect(ids).toEqual(test_case.ids); - } - if (test_case.tokens) { - const tokens = tokenizer.tokenize(test_case.text, { - pair: test_case.text_pair, - }); - expect(tokens).toEqual(test_case.tokens); - } - if (test_case.decoded) { - const decoded = tokenizer.decode(test_case.ids); - expect(decoded).toEqual(test_case.decoded); - } - }); - } + } + // Run custom tests, if they exist + CUSTOM_TESTS && describe("custom", CUSTOM_TESTS); }); - } - // Run custom tests, if they exist - CUSTOM_TESTS && describe("custom", CUSTOM_TESTS); - }); - } + } }); describe("Tokenizer padding/truncation", () => { - const inputs = ["a", "b c"]; - const text_pair = ["d e", "f g h"]; - - it("should create a jagged array", async () => { - const tokenizer = await AutoTokenizer.from_pretrained("Xenova/bert-base-uncased"); - - { - // support jagged array if `return_tensor=false` - const output = tokenizer(inputs, { - return_tensor: false, - }); - const expected = { - input_ids: [ - [101, 1037, 102], - [101, 1038, 1039, 102], - ], - attention_mask: [ - [1, 1, 1], - [1, 1, 1, 1], - ], - token_type_ids: [ - [0, 0, 0], - [0, 0, 0, 0], - ], - }; - compare(output, expected); - } + const inputs = ["a", "b c"]; + const text_pair = ["d e", "f g h"]; - { - const output = tokenizer(inputs, { - return_tensor: false, - truncation: true, - add_special_tokens: false, - }); - const expected = { - input_ids: [[1037], [1038, 1039]], - attention_mask: [[1], [1, 1]], - token_type_ids: [[0], [0, 0]], - }; - compare(output, expected); - } - }); - - it( - "should create a tensor", - async () => { - const tokenizer = await AutoTokenizer.from_pretrained("Xenova/bert-base-uncased"); - - { - // Expected to throw error if jagged array - expect(() => tokenizer(inputs)).toThrow("Unable to create tensor"); - } - - { - // Truncation - const { input_ids, attention_mask, token_type_ids } = tokenizer(inputs, { - truncation: true, - max_length: 1, - add_special_tokens: false, - }); + it("should create a jagged array", async () => { + const tokenizer = await AutoTokenizer.from_pretrained("Xenova/bert-base-uncased"); - expect(input_ids.tolist()).toEqual([[1037n], [1038n]]); - expect(attention_mask.tolist()).toEqual([[1n], [1n]]); - expect(token_type_ids.tolist()).toEqual([[0n], [0n]]); - } - { - // Truncation w/ text pair - // TODO - } - - { - // Padding - const { input_ids, attention_mask, token_type_ids } = tokenizer(inputs, { - padding: true, - add_special_tokens: false, - }); + { + // support jagged array if `return_tensor=false` + const output = tokenizer(inputs, { + return_tensor: false, + }); + const expected = { + input_ids: [ + [101, 1037, 102], + [101, 1038, 1039, 102], + ], + attention_mask: [ + [1, 1, 1], + [1, 1, 1, 1], + ], + token_type_ids: [ + [0, 0, 0], + [0, 0, 0, 0], + ], + }; + compare(output, expected); + } - expect(input_ids.tolist()).toEqual([ - [1037n, 0n], - [1038n, 1039n], - ]); - expect(attention_mask.tolist()).toEqual([ - [1n, 0n], - [1n, 1n], - ]); - expect(token_type_ids.tolist()).toEqual([ - [0n, 0n], - [0n, 0n], - ]); - } - { - // Padding w/ text pair - const { input_ids, attention_mask, token_type_ids } = tokenizer(inputs, { - text_pair, - padding: true, - add_special_tokens: false, - }); + { + const output = tokenizer(inputs, { + return_tensor: false, + truncation: true, + add_special_tokens: false, + }); + const expected = { + input_ids: [[1037], [1038, 1039]], + attention_mask: [[1], [1, 1]], + token_type_ids: [[0], [0, 0]], + }; + compare(output, expected); + } + }); - expect(input_ids.tolist()).toEqual([ - [1037n, 1040n, 1041n, 0n, 0n], - [1038n, 1039n, 1042n, 1043n, 1044n], - ]); - expect(attention_mask.tolist()).toEqual([ - [1n, 1n, 1n, 0n, 0n], - [1n, 1n, 1n, 1n, 1n], - ]); - expect(token_type_ids.tolist()).toEqual([ - [0n, 1n, 1n, 0n, 0n], - [0n, 0n, 1n, 1n, 1n], - ]); - } - - { - // Truncation + padding - const { input_ids, attention_mask, token_type_ids } = tokenizer(["a", "b c", "d e f"], { - padding: true, - truncation: true, - add_special_tokens: false, - max_length: 2, - }); + it( + "should create a tensor", + async () => { + const tokenizer = await AutoTokenizer.from_pretrained("Xenova/bert-base-uncased"); + + { + // Expected to throw error if jagged array + expect(() => tokenizer(inputs)).toThrow("Unable to create tensor"); + } + + { + // Truncation + const { input_ids, attention_mask, token_type_ids } = tokenizer(inputs, { + truncation: true, + max_length: 1, + add_special_tokens: false, + }); + + expect(input_ids.tolist()).toEqual([[1037n], [1038n]]); + expect(attention_mask.tolist()).toEqual([[1n], [1n]]); + expect(token_type_ids.tolist()).toEqual([[0n], [0n]]); + } + { + // Truncation w/ text pair + // TODO + } + + { + // Padding + const { input_ids, attention_mask, token_type_ids } = tokenizer(inputs, { + padding: true, + add_special_tokens: false, + }); - expect(input_ids.tolist()).toEqual([ - [1037n, 0n], - [1038n, 1039n], - [1040n, 1041n], - ]); - expect(attention_mask.tolist()).toEqual([ - [1n, 0n], - [1n, 1n], - [1n, 1n], - ]); - expect(token_type_ids.tolist()).toEqual([ - [0n, 0n], - [0n, 0n], - [0n, 0n], - ]); - } - }, - MAX_EXECUTION_TIME, - ); + expect(input_ids.tolist()).toEqual([ + [1037n, 0n], + [1038n, 1039n], + ]); + expect(attention_mask.tolist()).toEqual([ + [1n, 0n], + [1n, 1n], + ]); + expect(token_type_ids.tolist()).toEqual([ + [0n, 0n], + [0n, 0n], + ]); + } + { + // Padding w/ text pair + const { input_ids, attention_mask, token_type_ids } = tokenizer(inputs, { + text_pair, + padding: true, + add_special_tokens: false, + }); + + expect(input_ids.tolist()).toEqual([ + [1037n, 1040n, 1041n, 0n, 0n], + [1038n, 1039n, 1042n, 1043n, 1044n], + ]); + expect(attention_mask.tolist()).toEqual([ + [1n, 1n, 1n, 0n, 0n], + [1n, 1n, 1n, 1n, 1n], + ]); + expect(token_type_ids.tolist()).toEqual([ + [0n, 1n, 1n, 0n, 0n], + [0n, 0n, 1n, 1n, 1n], + ]); + } + + { + // Truncation + padding + const { input_ids, attention_mask, token_type_ids } = tokenizer(["a", "b c", "d e f"], { + padding: true, + truncation: true, + add_special_tokens: false, + max_length: 2, + }); + + expect(input_ids.tolist()).toEqual([ + [1037n, 0n], + [1038n, 1039n], + [1040n, 1041n], + ]); + expect(attention_mask.tolist()).toEqual([ + [1n, 0n], + [1n, 1n], + [1n, 1n], + ]); + expect(token_type_ids.tolist()).toEqual([ + [0n, 0n], + [0n, 0n], + [0n, 0n], + ]); + } + }, + MAX_EXECUTION_TIME, + ); }); describe("Token type ids", () => { - it( - "should correctly add token type ids", - async () => { - const tokenizer = await AutoTokenizer.from_pretrained("Xenova/bert-base-uncased"); - - const model_inputs = tokenizer(["a b c", "d"], { - text_pair: ["e f", "g h"], - padding: true, - truncation: true, - return_tensor: false, - }); - - const expected = { - input_ids: [ - [101, 1037, 1038, 1039, 102, 1041, 1042, 102], - [101, 1040, 102, 1043, 1044, 102, 0, 0], - ], - token_type_ids: [ - [0, 0, 0, 0, 0, 1, 1, 1], - [0, 0, 0, 1, 1, 1, 0, 0], - ], - attention_mask: [ - [1, 1, 1, 1, 1, 1, 1, 1], - [1, 1, 1, 1, 1, 1, 0, 0], - ], - }; - - compare(model_inputs, expected); - }, - MAX_EXECUTION_TIME, - ); - - it( - "should add token type ids if user requests them", - async () => { - const tokenizer = await AutoTokenizer.from_pretrained("Xenova/llama3-tokenizer-new"); - - { - // Without text pair - const model_inputs = tokenizer("hello", { - return_tensor: false, - return_token_type_ids: true, - }); - const expected = { - input_ids: [128000, 15339], - attention_mask: [1, 1], - token_type_ids: [0, 0], - }; - compare(model_inputs, expected); - } - - { - // With text pair - const model_inputs = tokenizer("hello", { - text_pair: "world", - return_tensor: false, - return_token_type_ids: true, - }); - const expected = { - input_ids: [128000, 15339, 128000, 14957], - attention_mask: [1, 1, 1, 1], - token_type_ids: [0, 0, 1, 1], - }; - compare(model_inputs, expected); - } - }, - MAX_EXECUTION_TIME, - ); + it( + "should correctly add token type ids", + async () => { + const tokenizer = await AutoTokenizer.from_pretrained("Xenova/bert-base-uncased"); + + const model_inputs = tokenizer(["a b c", "d"], { + text_pair: ["e f", "g h"], + padding: true, + truncation: true, + return_tensor: false, + }); + + const expected = { + input_ids: [ + [101, 1037, 1038, 1039, 102, 1041, 1042, 102], + [101, 1040, 102, 1043, 1044, 102, 0, 0], + ], + token_type_ids: [ + [0, 0, 0, 0, 0, 1, 1, 1], + [0, 0, 0, 1, 1, 1, 0, 0], + ], + attention_mask: [ + [1, 1, 1, 1, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1, 0, 0], + ], + }; + + compare(model_inputs, expected); + }, + MAX_EXECUTION_TIME, + ); + + it( + "should add token type ids if user requests them", + async () => { + const tokenizer = await AutoTokenizer.from_pretrained("Xenova/llama3-tokenizer-new"); + + { + // Without text pair + const model_inputs = tokenizer("hello", { + return_tensor: false, + return_token_type_ids: true, + }); + const expected = { + input_ids: [128000, 15339], + attention_mask: [1, 1], + token_type_ids: [0, 0], + }; + compare(model_inputs, expected); + } + + { + // With text pair + const model_inputs = tokenizer("hello", { + text_pair: "world", + return_tensor: false, + return_token_type_ids: true, + }); + const expected = { + input_ids: [128000, 15339, 128000, 14957], + attention_mask: [1, 1, 1, 1], + token_type_ids: [0, 0, 1, 1], + }; + compare(model_inputs, expected); + } + }, + MAX_EXECUTION_TIME, + ); }); describe("Edge cases", () => { - it( - "should not crash when encoding a very long string", - async () => { - let tokenizer = await AutoTokenizer.from_pretrained("Xenova/t5-small"); - - let text = String.prototype.repeat.call("Hello world! ", 50000); - let encoded = tokenizer(text); - expect(encoded.input_ids.data.length).toBeGreaterThan(100000); - }, - MAX_EXECUTION_TIME, - ); - - it("should not take too long", async () => { - let tokenizer = await AutoTokenizer.from_pretrained("Xenova/all-MiniLM-L6-v2"); - - let text = String.prototype.repeat.call("a", 50000); - let token_ids = tokenizer.encode(text); - compare(token_ids, [101, 100, 102]); - }, 5000); // NOTE: 5 seconds - - it( - "Special/added tokens with earlier partial matches", - async () => { - let tokenizer = await AutoTokenizer.from_pretrained("Xenova/gemini-nano"); - { - let token_ids = tokenizer.encode("\n", { add_special_tokens: false }); - compare(token_ids, [108]); - } - { - let token_ids = tokenizer.encode("\n\n", { add_special_tokens: false }); - compare(token_ids, [109]); // Should not be [108, 108] - } - }, - MAX_EXECUTION_TIME, - ); + it( + "should not crash when encoding a very long string", + async () => { + let tokenizer = await AutoTokenizer.from_pretrained("Xenova/t5-small"); + + let text = String.prototype.repeat.call("Hello world! ", 50000); + let encoded = tokenizer(text); + expect(encoded.input_ids.data.length).toBeGreaterThan(100000); + }, + MAX_EXECUTION_TIME, + ); + + it("should not take too long", async () => { + let tokenizer = await AutoTokenizer.from_pretrained("Xenova/all-MiniLM-L6-v2"); + + let text = String.prototype.repeat.call("a", 50000); + let token_ids = tokenizer.encode(text); + compare(token_ids, [101, 100, 102]); + }, 5000); // NOTE: 5 seconds + + it( + "Special/added tokens with earlier partial matches", + async () => { + let tokenizer = await AutoTokenizer.from_pretrained("Xenova/gemini-nano"); + { + let token_ids = tokenizer.encode("\n", { add_special_tokens: false }); + compare(token_ids, [108]); + } + { + let token_ids = tokenizer.encode("\n\n", { add_special_tokens: false }); + compare(token_ids, [109]); // Should not be [108, 108] + } + }, + MAX_EXECUTION_TIME, + ); }); describe("Extra decoding tests", () => { - it( - "should be able to decode the output of encode", - async () => { - let tokenizer = await AutoTokenizer.from_pretrained("Xenova/bert-base-uncased"); - - let text = "hello world!"; - - // Ensure all the following outputs are the same: - // 1. Tensor of ids: allow decoding of 1D or 2D tensors. - let encodedTensor = tokenizer(text); - let decoded1 = tokenizer.decode(encodedTensor.input_ids, { skip_special_tokens: true }); - let decoded2 = tokenizer.batch_decode(encodedTensor.input_ids, { skip_special_tokens: true })[0]; - expect(decoded1).toEqual(text); - expect(decoded2).toEqual(text); - - // 2. List of ids - let encodedList = tokenizer(text, { return_tensor: false }); - let decoded3 = tokenizer.decode(encodedList.input_ids, { skip_special_tokens: true }); - let decoded4 = tokenizer.batch_decode([encodedList.input_ids], { skip_special_tokens: true })[0]; - expect(decoded3).toEqual(text); - expect(decoded4).toEqual(text); - }, - MAX_EXECUTION_TIME, - ); + it( + "should be able to decode the output of encode", + async () => { + let tokenizer = await AutoTokenizer.from_pretrained("Xenova/bert-base-uncased"); + + let text = "hello world!"; + + // Ensure all the following outputs are the same: + // 1. Tensor of ids: allow decoding of 1D or 2D tensors. + let encodedTensor = tokenizer(text); + let decoded1 = tokenizer.decode(encodedTensor.input_ids, { skip_special_tokens: true }); + let decoded2 = tokenizer.batch_decode(encodedTensor.input_ids, { + skip_special_tokens: true, + })[0]; + expect(decoded1).toEqual(text); + expect(decoded2).toEqual(text); + + // 2. List of ids + let encodedList = tokenizer(text, { return_tensor: false }); + let decoded3 = tokenizer.decode(encodedList.input_ids, { skip_special_tokens: true }); + let decoded4 = tokenizer.batch_decode([encodedList.input_ids], { + skip_special_tokens: true, + })[0]; + expect(decoded3).toEqual(text); + expect(decoded4).toEqual(text); + }, + MAX_EXECUTION_TIME, + ); }); describe("Chat templates", () => { - it("should generate a chat template", async () => { - const tokenizer = await AutoTokenizer.from_pretrained("Xenova/mistral-tokenizer-v1"); - - const chat = [ - { role: "user", content: "Hello, how are you?" }, - { role: "assistant", content: "I'm doing great. How can I help you today?" }, - { role: "user", content: "I'd like to show off how chat templating works!" }, - ]; - - const text = tokenizer.apply_chat_template(chat, { tokenize: false }); - - expect(text).toEqual("[INST] Hello, how are you? [/INST]I'm doing great. How can I help you today? [INST] I'd like to show off how chat templating works! [/INST]"); - - const input_ids = tokenizer.apply_chat_template(chat, { tokenize: true, return_tensor: false }); - compare(input_ids, [1, 733, 16289, 28793, 22557, 28725, 910, 460, 368, 28804, 733, 28748, 16289, 28793, 28737, 28742, 28719, 2548, 1598, 28723, 1602, 541, 315, 1316, 368, 3154, 28804, 2, 28705, 733, 16289, 28793, 315, 28742, 28715, 737, 298, 1347, 805, 910, 10706, 5752, 1077, 3791, 28808, 733, 28748, 16289, 28793]); - }); - - it("should support multiple chat templates", async () => { - const tokenizer = await AutoTokenizer.from_pretrained("Xenova/c4ai-command-r-v01-tokenizer"); - - // define conversation input: - const conversation = [{ role: "user", content: "Whats the biggest penguin in the world?" }]; - // define documents to ground on: - const documents = [ - { title: "Tall penguins", text: "Emperor penguins are the tallest growing up to 122 cm in height." }, - { title: "Penguin habitats", text: "Emperor penguins only live in Antarctica." }, - ]; - - // render the RAG prompt as a string: - const grounded_generation_prompt = tokenizer.apply_chat_template(conversation, { - chat_template: "rag", - tokenize: false, - add_generation_prompt: true, - - documents, - citation_mode: "accurate", // or "fast" + it("should generate a chat template", async () => { + const tokenizer = await AutoTokenizer.from_pretrained("Xenova/mistral-tokenizer-v1"); + + const chat = [ + { role: "user", content: "Hello, how are you?" }, + { role: "assistant", content: "I'm doing great. How can I help you today?" }, + { role: "user", content: "I'd like to show off how chat templating works!" }, + ]; + + const text = tokenizer.apply_chat_template(chat, { tokenize: false }); + + expect(text).toEqual("[INST] Hello, how are you? [/INST]I'm doing great. How can I help you today? [INST] I'd like to show off how chat templating works! [/INST]"); + + const input_ids = tokenizer.apply_chat_template(chat, { + tokenize: true, + return_tensor: false, + }); + compare(input_ids, [1, 733, 16289, 28793, 22557, 28725, 910, 460, 368, 28804, 733, 28748, 16289, 28793, 28737, 28742, 28719, 2548, 1598, 28723, 1602, 541, 315, 1316, 368, 3154, 28804, 2, 28705, 733, 16289, 28793, 315, 28742, 28715, 737, 298, 1347, 805, 910, 10706, 5752, 1077, 3791, 28808, 733, 28748, 16289, 28793]); }); - expect(grounded_generation_prompt).toEqual("<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|># Safety Preamble\nThe instructions in this section override those in the task description and style guide sections. Don't answer questions that are harmful or immoral.\n\n" + "# System Preamble\n## Basic Rules\nYou are a powerful conversational AI trained by Cohere to help people. You are augmented by a number of tools, and your job is to use and consume the output of these tools to best help the user. You will see a conversation history between yourself and a user, ending with an utterance from the user. You will then see a specific instruction instructing you what kind of response to generate. When you answer the user's requests, you cite your sources in your answers, according to those instructions.\n\n" + "# User Preamble\n## Task and Context\nYou help people answer their questions and other requests interactively. You will be asked a very wide array of requests on all kinds of topics. You will be equipped with a wide range of search engines or similar tools to help you, which you use to research your answer. You should focus on serving the user's needs as best you can, which will be wide-ranging.\n\n## Style Guide\nUnless the user asks for a different style of answer, you should answer in full sentences, using proper grammar and spelling.<|END_OF_TURN_TOKEN|>" + "<|START_OF_TURN_TOKEN|><|USER_TOKEN|>Whats the biggest penguin in the world?<|END_OF_TURN_TOKEN|>" + "<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>\nDocument: 0\ntitle: Tall penguins\ntext: Emperor penguins are the tallest growing up to 122 cm in height.\n\nDocument: 1\ntitle: Penguin habitats\ntext: Emperor penguins only live in Antarctica.\n<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>Carefully perform the following instructions, in order, starting each with a new line.\nFirstly, Decide which of the retrieved documents are relevant to the user's last input by writing 'Relevant Documents:' followed by comma-separated list of document numbers. If none are relevant, you should instead write 'None'.\nSecondly, Decide which of the retrieved documents contain facts that should be cited in a good answer to the user's last input by writing 'Cited Documents:' followed a comma-separated list of document numbers. If you dont want to cite any of them, you should instead write 'None'.\nThirdly, Write 'Answer:' followed by a response to the user's last input in high quality natural english. Use the retrieved documents to help you. Do not insert any citations or grounding markup.\nFinally, Write 'Grounded answer:' followed by a response to the user's last input in high quality natural english. Use the symbols and to indicate when a fact comes from a document in the search result, e.g my fact for a fact from document 0.<|END_OF_TURN_TOKEN|>" + "<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>"); - }); - it("should support automatic chat template detection based on inputs", async () => { - const tokenizer = await AutoTokenizer.from_pretrained("Xenova/c4ai-command-r-plus-08-2024-tokenizer"); + it("should support multiple chat templates", async () => { + const tokenizer = await AutoTokenizer.from_pretrained("Xenova/c4ai-command-r-v01-tokenizer"); - // Examples adapted from https://huggingface.co/CohereForAI/c4ai-command-r-plus-08-2024 + // define conversation input: + const conversation = [{ role: "user", content: "Whats the biggest penguin in the world?" }]; + // define documents to ground on: + const documents = [ + { + title: "Tall penguins", + text: "Emperor penguins are the tallest growing up to 122 cm in height.", + }, + { title: "Penguin habitats", text: "Emperor penguins only live in Antarctica." }, + ]; - { - // - default - // define conversation input: - const messages = [{ role: "user", content: "Hello, how are you?" }]; + // render the RAG prompt as a string: + const grounded_generation_prompt = tokenizer.apply_chat_template(conversation, { + chat_template: "rag", + tokenize: false, + add_generation_prompt: true, - // Format message with the command-r-plus-08-2024 chat template - const prompt = tokenizer.apply_chat_template(messages, { tokenize: false, add_generation_prompt: true }); - expect(prompt).toEqual("<|START_OF_TURN_TOKEN|><|USER_TOKEN|>Hello, how are you?<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>"); - } + documents, + citation_mode: "accurate", // or "fast" + }); + expect(grounded_generation_prompt).toEqual("<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|># Safety Preamble\nThe instructions in this section override those in the task description and style guide sections. Don't answer questions that are harmful or immoral.\n\n" + "# System Preamble\n## Basic Rules\nYou are a powerful conversational AI trained by Cohere to help people. You are augmented by a number of tools, and your job is to use and consume the output of these tools to best help the user. You will see a conversation history between yourself and a user, ending with an utterance from the user. You will then see a specific instruction instructing you what kind of response to generate. When you answer the user's requests, you cite your sources in your answers, according to those instructions.\n\n" + "# User Preamble\n## Task and Context\nYou help people answer their questions and other requests interactively. You will be asked a very wide array of requests on all kinds of topics. You will be equipped with a wide range of search engines or similar tools to help you, which you use to research your answer. You should focus on serving the user's needs as best you can, which will be wide-ranging.\n\n## Style Guide\nUnless the user asks for a different style of answer, you should answer in full sentences, using proper grammar and spelling.<|END_OF_TURN_TOKEN|>" + "<|START_OF_TURN_TOKEN|><|USER_TOKEN|>Whats the biggest penguin in the world?<|END_OF_TURN_TOKEN|>" + "<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>\nDocument: 0\ntitle: Tall penguins\ntext: Emperor penguins are the tallest growing up to 122 cm in height.\n\nDocument: 1\ntitle: Penguin habitats\ntext: Emperor penguins only live in Antarctica.\n<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>Carefully perform the following instructions, in order, starting each with a new line.\nFirstly, Decide which of the retrieved documents are relevant to the user's last input by writing 'Relevant Documents:' followed by comma-separated list of document numbers. If none are relevant, you should instead write 'None'.\nSecondly, Decide which of the retrieved documents contain facts that should be cited in a good answer to the user's last input by writing 'Cited Documents:' followed a comma-separated list of document numbers. If you dont want to cite any of them, you should instead write 'None'.\nThirdly, Write 'Answer:' followed by a response to the user's last input in high quality natural english. Use the retrieved documents to help you. Do not insert any citations or grounding markup.\nFinally, Write 'Grounded answer:' followed by a response to the user's last input in high quality natural english. Use the symbols and to indicate when a fact comes from a document in the search result, e.g my fact for a fact from document 0.<|END_OF_TURN_TOKEN|>" + "<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>"); + }); + + it("should support automatic chat template detection based on inputs", async () => { + const tokenizer = await AutoTokenizer.from_pretrained("Xenova/c4ai-command-r-plus-08-2024-tokenizer"); - { - // - tool_use - // define conversation input: - const conversation = [{ role: "user", content: "Whats the biggest penguin in the world?" }]; + // Examples adapted from https://huggingface.co/CohereForAI/c4ai-command-r-plus-08-2024 - // Define tools available for the model to use: - const tools = [ { - name: "internet_search", - description: "Returns a list of relevant document snippets for a textual query retrieved from the internet", - parameter_definitions: { - query: { - description: "Query to search the internet with", - type: "str", - required: true, - }, - }, - }, + // - default + // define conversation input: + const messages = [{ role: "user", content: "Hello, how are you?" }]; + + // Format message with the command-r-plus-08-2024 chat template + const prompt = tokenizer.apply_chat_template(messages, { + tokenize: false, + add_generation_prompt: true, + }); + expect(prompt).toEqual("<|START_OF_TURN_TOKEN|><|USER_TOKEN|>Hello, how are you?<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>"); + } + { - name: "directly_answer", - description: "Calls a standard (un-augmented) AI chatbot to generate a response given the conversation history", - parameter_definitions: {}, - }, - ]; + // - tool_use + // define conversation input: + const conversation = [{ role: "user", content: "Whats the biggest penguin in the world?" }]; + + // Define tools available for the model to use: + const tools = [ + { + name: "internet_search", + description: "Returns a list of relevant document snippets for a textual query retrieved from the internet", + parameter_definitions: { + query: { + description: "Query to search the internet with", + type: "str", + required: true, + }, + }, + }, + { + name: "directly_answer", + description: "Calls a standard (un-augmented) AI chatbot to generate a response given the conversation history", + parameter_definitions: {}, + }, + ]; + + // render the tool use prompt as a string: + const prompt = tokenizer.apply_chat_template(conversation, { + tools, + tokenize: false, + add_generation_prompt: true, + }); + expect(prompt).toEqual('<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|># Safety Preamble\nThe instructions in this section override those in the task description and style guide sections. Don\'t answer questions that are harmful or immoral.\n\n# System Preamble\n## Basic Rules\nYou are a powerful conversational AI trained by Cohere to help people. You are augmented by a number of tools, and your job is to use and consume the output of these tools to best help the user. You will see a conversation history between yourself and a user, ending with an utterance from the user. You will then see a specific instruction instructing you what kind of response to generate. When you answer the user\'s requests, you cite your sources in your answers, according to those instructions.\n\n# User Preamble\n## Task and Context\nYou help people answer their questions and other requests interactively. You will be asked a very wide array of requests on all kinds of topics. You will be equipped with a wide range of search engines or similar tools to help you, which you use to research your answer. You should focus on serving the user\'s needs as best you can, which will be wide-ranging.\n\n## Style Guide\nUnless the user asks for a different style of answer, you should answer in full sentences, using proper grammar and spelling.\n\n## Available Tools\nHere is a list of tools that you have available to you:\n\n```python\ndef internet_search(query: str) -> List[Dict]:\n """Returns a list of relevant document snippets for a textual query retrieved from the internet\n\n Args:\n query (str): Query to search the internet with\n """\n pass\n```\n\n```python\ndef directly_answer() -> List[Dict]:\n """Calls a standard (un-augmented) AI chatbot to generate a response given the conversation history\n """\n pass\n```<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|USER_TOKEN|>Whats the biggest penguin in the world?<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>Write \'Action:\' followed by a json-formatted list of actions that you want to perform in order to produce a good response to the user\'s last input. You can use any of the supplied tools any number of times, but you should aim to execute the minimum number of necessary actions for the input. You should use the `directly-answer` tool if calling the other tools is unnecessary. The list of actions you want to call should be formatted as a list of json objects, for example:\n```json\n[\n {\n "tool_name": title of the tool in the specification,\n "parameters": a dict of parameters to input into the tool as they are defined in the specs, or {} if it takes no parameters\n }\n]```<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>'); + } + }); - // render the tool use prompt as a string: - const prompt = tokenizer.apply_chat_template(conversation, { tools, tokenize: false, add_generation_prompt: true }); - expect(prompt).toEqual('<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|># Safety Preamble\nThe instructions in this section override those in the task description and style guide sections. Don\'t answer questions that are harmful or immoral.\n\n# System Preamble\n## Basic Rules\nYou are a powerful conversational AI trained by Cohere to help people. You are augmented by a number of tools, and your job is to use and consume the output of these tools to best help the user. You will see a conversation history between yourself and a user, ending with an utterance from the user. You will then see a specific instruction instructing you what kind of response to generate. When you answer the user\'s requests, you cite your sources in your answers, according to those instructions.\n\n# User Preamble\n## Task and Context\nYou help people answer their questions and other requests interactively. You will be asked a very wide array of requests on all kinds of topics. You will be equipped with a wide range of search engines or similar tools to help you, which you use to research your answer. You should focus on serving the user\'s needs as best you can, which will be wide-ranging.\n\n## Style Guide\nUnless the user asks for a different style of answer, you should answer in full sentences, using proper grammar and spelling.\n\n## Available Tools\nHere is a list of tools that you have available to you:\n\n```python\ndef internet_search(query: str) -> List[Dict]:\n """Returns a list of relevant document snippets for a textual query retrieved from the internet\n\n Args:\n query (str): Query to search the internet with\n """\n pass\n```\n\n```python\ndef directly_answer() -> List[Dict]:\n """Calls a standard (un-augmented) AI chatbot to generate a response given the conversation history\n """\n pass\n```<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|USER_TOKEN|>Whats the biggest penguin in the world?<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>Write \'Action:\' followed by a json-formatted list of actions that you want to perform in order to produce a good response to the user\'s last input. You can use any of the supplied tools any number of times, but you should aim to execute the minimum number of necessary actions for the input. You should use the `directly-answer` tool if calling the other tools is unnecessary. The list of actions you want to call should be formatted as a list of json objects, for example:\n```json\n[\n {\n "tool_name": title of the tool in the specification,\n "parameters": a dict of parameters to input into the tool as they are defined in the specs, or {} if it takes no parameters\n }\n]```<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>'); - } - }); - - it("should support user-defined chat template", async () => { - const tokenizer = await AutoTokenizer.from_pretrained("Xenova/llama-tokenizer"); - - const chat = [ - { role: "user", content: "Hello, how are you?" }, - { role: "assistant", content: "I'm doing great. How can I help you today?" }, - { role: "user", content: "I'd like to show off how chat templating works!" }, - ]; - - // https://discuss.huggingface.co/t/issue-with-llama-2-chat-template-and-out-of-date-documentation/61645/3 - const chat_template = ( - "{% if messages[0]['role'] == 'system' %}" + - "{% set loop_messages = messages[1:] %}" + // Extract system message if it's present - "{% set system_message = messages[0]['content'] %}" + - "{% elif USE_DEFAULT_PROMPT == true and not '<>' in messages[0]['content'] %}" + - "{% set loop_messages = messages %}" + // Or use the default system message if the flag is set - "{% set system_message = 'DEFAULT_SYSTEM_MESSAGE' %}" + - "{% else %}" + - "{% set loop_messages = messages %}" + - "{% set system_message = false %}" + - "{% endif %}" + - "{% if loop_messages|length == 0 and system_message %}" + // Special handling when only sys message present - "{{ bos_token + '[INST] <>\\n' + system_message + '\\n<>\\n\\n [/INST]' }}" + - "{% endif %}" + - "{% for message in loop_messages %}" + // Loop over all non-system messages - "{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}" + - "{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}" + - "{% endif %}" + - "{% if loop.index0 == 0 and system_message != false %}" + // Embed system message in first message - "{% set content = '<>\\n' + system_message + '\\n<>\\n\\n' + message['content'] %}" + - "{% else %}" + - "{% set content = message['content'] %}" + - "{% endif %}" + - "{% if message['role'] == 'user' %}" + // After all of that, handle messages/roles in a fairly normal way - "{{ bos_token + '[INST] ' + content.strip() + ' [/INST]' }}" + - "{% elif message['role'] == 'system' %}" + - "{{ '<>\\n' + content.strip() + '\\n<>\\n\\n' }}" + - "{% elif message['role'] == 'assistant' %}" + - "{{ ' ' + content.strip() + ' ' + eos_token }}" + - "{% endif %}" + - "{% endfor %}" - ) - .replaceAll("USE_DEFAULT_PROMPT", true) - .replaceAll("DEFAULT_SYSTEM_MESSAGE", "You are a helpful, respectful and honest assistant."); - - const text = tokenizer.apply_chat_template(chat, { tokenize: false, return_tensor: false, chat_template }); - - expect(text).toEqual("[INST] <>\nYou are a helpful, respectful and honest assistant.\n<>\n\nHello, how are you? [/INST] I'm doing great. How can I help you today? [INST] I'd like to show off how chat templating works! [/INST]"); - - // TODO: Add test for token_ids once bug in transformers is fixed. - }); - - it("should throw an error when no chat template is detected", async () => { - const tokenizer = await AutoTokenizer.from_pretrained("Xenova/gpt-4o"); - - const chat = [{ role: "user", content: "Hello, how are you?" }]; - - expect(() => tokenizer.apply_chat_template(chat, { tokenize: false })).toThrow("tokenizer.chat_template is not set and no template argument was passed"); - }); - - it("should support default parameters", async () => { - const tokenizer = await AutoTokenizer.from_pretrained("Xenova/Meta-Llama-3.1-Tokenizer"); - - // Example adapted from https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct#tool-use-with-transformers - const chat = [ - { role: "system", content: "You are a bot that responds to weather queries." }, - { role: "user", content: "Hey, what's the temperature in Paris right now?" }, - ]; - const tools = [{ type: "function", function: { name: "get_current_temperature", description: "Get the current temperature at a location.", parameters: { type: "object", properties: { location: { type: "string", description: 'The location to get the temperature for, in the format "City, Country"' } }, required: ["location"] }, return: { type: "number", description: "The current temperature at the specified location in the specified units, as a float." } } }]; - - { - // `tools` unset (will default to `null`) - const text = tokenizer.apply_chat_template(chat, { tokenize: false }); - expect(text).toEqual("<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\nCutting Knowledge Date: December 2023\nToday Date: 26 Jul 2024\n\nYou are a bot that responds to weather queries.<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nHey, what's the temperature in Paris right now?<|eot_id|>"); - - const input_ids = tokenizer.apply_chat_template(chat, { tokenize: true, return_tensor: false }); - compare(input_ids, [128000, 128006, 9125, 128007, 271, 38766, 1303, 33025, 2696, 25, 6790, 220, 2366, 18, 198, 15724, 2696, 25, 220, 1627, 10263, 220, 2366, 19, 271, 2675, 527, 264, 11164, 430, 31680, 311, 9282, 20126, 13, 128009, 128006, 882, 128007, 271, 19182, 11, 1148, 596, 279, 9499, 304, 12366, 1314, 1457, 30, 128009]); - } + it("should support user-defined chat template", async () => { + const tokenizer = await AutoTokenizer.from_pretrained("Xenova/llama-tokenizer"); + + const chat = [ + { role: "user", content: "Hello, how are you?" }, + { role: "assistant", content: "I'm doing great. How can I help you today?" }, + { role: "user", content: "I'd like to show off how chat templating works!" }, + ]; + + // https://discuss.huggingface.co/t/issue-with-llama-2-chat-template-and-out-of-date-documentation/61645/3 + const chat_template = ( + "{% if messages[0]['role'] == 'system' %}" + + "{% set loop_messages = messages[1:] %}" + // Extract system message if it's present + "{% set system_message = messages[0]['content'] %}" + + "{% elif USE_DEFAULT_PROMPT == true and not '<>' in messages[0]['content'] %}" + + "{% set loop_messages = messages %}" + // Or use the default system message if the flag is set + "{% set system_message = 'DEFAULT_SYSTEM_MESSAGE' %}" + + "{% else %}" + + "{% set loop_messages = messages %}" + + "{% set system_message = false %}" + + "{% endif %}" + + "{% if loop_messages|length == 0 and system_message %}" + // Special handling when only sys message present + "{{ bos_token + '[INST] <>\\n' + system_message + '\\n<>\\n\\n [/INST]' }}" + + "{% endif %}" + + "{% for message in loop_messages %}" + // Loop over all non-system messages + "{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}" + + "{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}" + + "{% endif %}" + + "{% if loop.index0 == 0 and system_message != false %}" + // Embed system message in first message + "{% set content = '<>\\n' + system_message + '\\n<>\\n\\n' + message['content'] %}" + + "{% else %}" + + "{% set content = message['content'] %}" + + "{% endif %}" + + "{% if message['role'] == 'user' %}" + // After all of that, handle messages/roles in a fairly normal way + "{{ bos_token + '[INST] ' + content.strip() + ' [/INST]' }}" + + "{% elif message['role'] == 'system' %}" + + "{{ '<>\\n' + content.strip() + '\\n<>\\n\\n' }}" + + "{% elif message['role'] == 'assistant' %}" + + "{{ ' ' + content.strip() + ' ' + eos_token }}" + + "{% endif %}" + + "{% endfor %}" + ) + .replaceAll("USE_DEFAULT_PROMPT", true) + .replaceAll("DEFAULT_SYSTEM_MESSAGE", "You are a helpful, respectful and honest assistant."); + + const text = tokenizer.apply_chat_template(chat, { + tokenize: false, + return_tensor: false, + chat_template, + }); - { - // `tools` set - const text = tokenizer.apply_chat_template(chat, { tools, tokenize: false }); - expect(text).toEqual('<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\nEnvironment: ipython\nCutting Knowledge Date: December 2023\nToday Date: 26 Jul 2024\n\nYou are a bot that responds to weather queries.<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nGiven the following functions, please respond with a JSON for a function call with its proper arguments that best answers the given prompt.\n\nRespond in the format {"name": function name, "parameters": dictionary of argument name and its value}.Do not use variables.\n\n{\n "type": "function",\n "function": {\n "name": "get_current_temperature",\n "description": "Get the current temperature at a location.",\n "parameters": {\n "type": "object",\n "properties": {\n "location": {\n "type": "string",\n "description": "The location to get the temperature for, in the format \\"City, Country\\""\n }\n },\n "required": [\n "location"\n ]\n },\n "return": {\n "type": "number",\n "description": "The current temperature at the specified location in the specified units, as a float."\n }\n }\n}\n\nHey, what\'s the temperature in Paris right now?<|eot_id|>'); + expect(text).toEqual("[INST] <>\nYou are a helpful, respectful and honest assistant.\n<>\n\nHello, how are you? [/INST] I'm doing great. How can I help you today? [INST] I'd like to show off how chat templating works! [/INST]"); - const input_ids = tokenizer.apply_chat_template(chat, { tools, tokenize: true, return_tensor: false }); - compare(input_ids, [128000, 128006, 9125, 128007, 271, 13013, 25, 6125, 27993, 198, 38766, 1303, 33025, 2696, 25, 6790, 220, 2366, 18, 198, 15724, 2696, 25, 220, 1627, 10263, 220, 2366, 19, 271, 2675, 527, 264, 11164, 430, 31680, 311, 9282, 20126, 13, 128009, 128006, 882, 128007, 271, 22818, 279, 2768, 5865, 11, 4587, 6013, 449, 264, 4823, 369, 264, 734, 1650, 449, 1202, 6300, 6105, 430, 1888, 11503, 279, 2728, 10137, 382, 66454, 304, 279, 3645, 5324, 609, 794, 734, 836, 11, 330, 14105, 794, 11240, 315, 5811, 836, 323, 1202, 907, 7966, 5519, 539, 1005, 7482, 382, 517, 262, 330, 1337, 794, 330, 1723, 761, 262, 330, 1723, 794, 341, 286, 330, 609, 794, 330, 456, 11327, 54625, 761, 286, 330, 4789, 794, 330, 1991, 279, 1510, 9499, 520, 264, 3813, 10560, 286, 330, 14105, 794, 341, 310, 330, 1337, 794, 330, 1735, 761, 310, 330, 13495, 794, 341, 394, 330, 2588, 794, 341, 504, 330, 1337, 794, 330, 928, 761, 504, 330, 4789, 794, 330, 791, 3813, 311, 636, 279, 9499, 369, 11, 304, 279, 3645, 7393, 13020, 11, 14438, 2153, 702, 394, 457, 310, 1173, 310, 330, 6413, 794, 2330, 394, 330, 2588, 702, 310, 5243, 286, 1173, 286, 330, 693, 794, 341, 310, 330, 1337, 794, 330, 4174, 761, 310, 330, 4789, 794, 330, 791, 1510, 9499, 520, 279, 5300, 3813, 304, 279, 5300, 8316, 11, 439, 264, 2273, 10246, 286, 457, 262, 457, 633, 19182, 11, 1148, 596, 279, 9499, 304, 12366, 1314, 1457, 30, 128009]); - } - }); + // TODO: Add test for token_ids once bug in transformers is fixed. + }); + + it("should throw an error when no chat template is detected", async () => { + const tokenizer = await AutoTokenizer.from_pretrained("Xenova/gpt-4o"); + + const chat = [{ role: "user", content: "Hello, how are you?" }]; + + expect(() => tokenizer.apply_chat_template(chat, { tokenize: false })).toThrow("tokenizer.chat_template is not set and no template argument was passed"); + }); + + it("should support default parameters", async () => { + const tokenizer = await AutoTokenizer.from_pretrained("Xenova/Meta-Llama-3.1-Tokenizer"); + + // Example adapted from https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct#tool-use-with-transformers + const chat = [ + { role: "system", content: "You are a bot that responds to weather queries." }, + { role: "user", content: "Hey, what's the temperature in Paris right now?" }, + ]; + const tools = [ + { + type: "function", + function: { + name: "get_current_temperature", + description: "Get the current temperature at a location.", + parameters: { + type: "object", + properties: { + location: { + type: "string", + description: 'The location to get the temperature for, in the format "City, Country"', + }, + }, + required: ["location"], + }, + return: { + type: "number", + description: "The current temperature at the specified location in the specified units, as a float.", + }, + }, + }, + ]; + + { + // `tools` unset (will default to `null`) + const text = tokenizer.apply_chat_template(chat, { tokenize: false }); + expect(text).toEqual("<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\nCutting Knowledge Date: December 2023\nToday Date: 26 Jul 2024\n\nYou are a bot that responds to weather queries.<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nHey, what's the temperature in Paris right now?<|eot_id|>"); + + const input_ids = tokenizer.apply_chat_template(chat, { + tokenize: true, + return_tensor: false, + }); + compare(input_ids, [128000, 128006, 9125, 128007, 271, 38766, 1303, 33025, 2696, 25, 6790, 220, 2366, 18, 198, 15724, 2696, 25, 220, 1627, 10263, 220, 2366, 19, 271, 2675, 527, 264, 11164, 430, 31680, 311, 9282, 20126, 13, 128009, 128006, 882, 128007, 271, 19182, 11, 1148, 596, 279, 9499, 304, 12366, 1314, 1457, 30, 128009]); + } + + { + // `tools` set + const text = tokenizer.apply_chat_template(chat, { tools, tokenize: false }); + expect(text).toEqual('<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\nEnvironment: ipython\nCutting Knowledge Date: December 2023\nToday Date: 26 Jul 2024\n\nYou are a bot that responds to weather queries.<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nGiven the following functions, please respond with a JSON for a function call with its proper arguments that best answers the given prompt.\n\nRespond in the format {"name": function name, "parameters": dictionary of argument name and its value}.Do not use variables.\n\n{\n "type": "function",\n "function": {\n "name": "get_current_temperature",\n "description": "Get the current temperature at a location.",\n "parameters": {\n "type": "object",\n "properties": {\n "location": {\n "type": "string",\n "description": "The location to get the temperature for, in the format \\"City, Country\\""\n }\n },\n "required": [\n "location"\n ]\n },\n "return": {\n "type": "number",\n "description": "The current temperature at the specified location in the specified units, as a float."\n }\n }\n}\n\nHey, what\'s the temperature in Paris right now?<|eot_id|>'); + + const input_ids = tokenizer.apply_chat_template(chat, { + tools, + tokenize: true, + return_tensor: false, + }); + compare(input_ids, [128000, 128006, 9125, 128007, 271, 13013, 25, 6125, 27993, 198, 38766, 1303, 33025, 2696, 25, 6790, 220, 2366, 18, 198, 15724, 2696, 25, 220, 1627, 10263, 220, 2366, 19, 271, 2675, 527, 264, 11164, 430, 31680, 311, 9282, 20126, 13, 128009, 128006, 882, 128007, 271, 22818, 279, 2768, 5865, 11, 4587, 6013, 449, 264, 4823, 369, 264, 734, 1650, 449, 1202, 6300, 6105, 430, 1888, 11503, 279, 2728, 10137, 382, 66454, 304, 279, 3645, 5324, 609, 794, 734, 836, 11, 330, 14105, 794, 11240, 315, 5811, 836, 323, 1202, 907, 7966, 5519, 539, 1005, 7482, 382, 517, 262, 330, 1337, 794, 330, 1723, 761, 262, 330, 1723, 794, 341, 286, 330, 609, 794, 330, 456, 11327, 54625, 761, 286, 330, 4789, 794, 330, 1991, 279, 1510, 9499, 520, 264, 3813, 10560, 286, 330, 14105, 794, 341, 310, 330, 1337, 794, 330, 1735, 761, 310, 330, 13495, 794, 341, 394, 330, 2588, 794, 341, 504, 330, 1337, 794, 330, 928, 761, 504, 330, 4789, 794, 330, 791, 3813, 311, 636, 279, 9499, 369, 11, 304, 279, 3645, 7393, 13020, 11, 14438, 2153, 702, 394, 457, 310, 1173, 310, 330, 6413, 794, 2330, 394, 330, 2588, 702, 310, 5243, 286, 1173, 286, 330, 693, 794, 341, 310, 330, 1337, 794, 330, 4174, 761, 310, 330, 4789, 794, 330, 791, 1510, 9499, 520, 279, 5300, 3813, 304, 279, 5300, 8316, 11, 439, 264, 2273, 10246, 286, 457, 262, 457, 633, 19182, 11, 1148, 596, 279, 9499, 304, 12366, 1314, 1457, 30, 128009]); + } + }); }); diff --git a/tests/utils/data_structures.test.js b/tests/utils/data_structures.test.js index 033a91d00..e6a2b0bee 100644 --- a/tests/utils/data_structures.test.js +++ b/tests/utils/data_structures.test.js @@ -1,33 +1,33 @@ import { PriorityQueue } from "../../src/utils/data-structures.js"; describe("Priority queue", () => { - const EXAMPLE_ARRAY = [2, 5, 3, 1, 4]; - it("default (max heap)", () => { - const queue = new PriorityQueue(); - queue.extend(EXAMPLE_ARRAY); - expect(queue.pop()).toBe(5); - }); + const EXAMPLE_ARRAY = [2, 5, 3, 1, 4]; + it("default (max heap)", () => { + const queue = new PriorityQueue(); + queue.extend(EXAMPLE_ARRAY); + expect(queue.pop()).toBe(5); + }); - it("min heap", () => { - const queue = new PriorityQueue((a, b) => a < b); - queue.extend(EXAMPLE_ARRAY); - expect(queue.pop()).toBe(1); - }); + it("min heap", () => { + const queue = new PriorityQueue((a, b) => a < b); + queue.extend(EXAMPLE_ARRAY); + expect(queue.pop()).toBe(1); + }); - it("heap w/ max size", () => { - const queue = new PriorityQueue((a, b) => a > b, 3); - queue.extend([1, 2, 3, 4, 5, 4, 3, 2, 1]); - expect(queue.pop()).toBe(5); + it("heap w/ max size", () => { + const queue = new PriorityQueue((a, b) => a > b, 3); + queue.extend([1, 2, 3, 4, 5, 4, 3, 2, 1]); + expect(queue.pop()).toBe(5); - // Test with random sizes - const sizes = [1, 3, 4, 5, 8, 9, 15, 16, 31, 32, 127, 128]; - const arr = Array.from({ length: 100 }, (_) => Math.random()); - const max = Math.max(...arr); - for (const size of sizes) { - const queue = new PriorityQueue((a, b) => a > b, size); - queue.extend(arr); - expect(queue.pop()).toBe(max); - expect(queue.size).toBeLessThanOrEqual(size); - } - }); + // Test with random sizes + const sizes = [1, 3, 4, 5, 8, 9, 15, 16, 31, 32, 127, 128]; + const arr = Array.from({ length: 100 }, (_) => Math.random()); + const max = Math.max(...arr); + for (const size of sizes) { + const queue = new PriorityQueue((a, b) => a > b, size); + queue.extend(arr); + expect(queue.pop()).toBe(max); + expect(queue.size).toBeLessThanOrEqual(size); + } + }); }); diff --git a/tests/utils/generation.test.js b/tests/utils/generation.test.js index dd2229826..2b52f4740 100644 --- a/tests/utils/generation.test.js +++ b/tests/utils/generation.test.js @@ -8,197 +8,200 @@ init(); // Helper function to generate text const generate = async (model, tokenizer, text, options) => { - const inputs = tokenizer(text); - return await model.generate({ - ...inputs, - ...options, - }); + const inputs = tokenizer(text); + return await model.generate({ + ...inputs, + ...options, + }); }; describe("Generation parameters", () => { - // List all models which will be tested - const models = [ - "hf-internal-testing/tiny-random-T5ForConditionalGeneration", // - "hf-internal-testing/tiny-random-LlamaForCausalLM", // decoder-only - ]; - const DUMMY_TEXT = "hello"; - - describe(`encoder-decoder (${models[0]})`, () => { - const model_id = models[0]; - - let model; - let tokenizer; - beforeAll(async () => { - model = await AutoModelForSeq2SeqLM.from_pretrained(model_id); - tokenizer = await AutoTokenizer.from_pretrained(model_id); - }, MAX_MODEL_LOAD_TIME); - - // NOTE: Since `max_length` defaults to 20, this case also tests that. - it( - "default", - async () => { - const outputs = await generate(model, tokenizer, DUMMY_TEXT, {}); - expect(outputs.dims.at(-1)).toEqual(20); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "max_new_tokens", - async () => { - const MAX_NEW_TOKENS = 5; - const outputs = await generate(model, tokenizer, DUMMY_TEXT, { - max_new_tokens: MAX_NEW_TOKENS, - }); - expect(outputs.dims.at(-1)).toEqual(MAX_NEW_TOKENS + 1); // + 1 due to forced BOS token - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "min_length", - async () => { - const MIN_LENGTH = 3; - const MAX_LENGTH = 5; - const outputs = await generate(model, tokenizer, DUMMY_TEXT, { - eos_token_id: 0, - min_length: MIN_LENGTH, - max_length: MAX_LENGTH, - }); - expect(outputs.tolist()).toEqual([[0n, 11924n, 11924n, 11924n, 11924n]]); - expect(outputs.dims.at(-1)).toBeGreaterThanOrEqual(MIN_LENGTH); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "min_new_tokens", - async () => { - const MIN_NEW_TOKENS = 2; - const MAX_LENGTH = 5; - const outputs = await generate(model, tokenizer, DUMMY_TEXT, { - eos_token_id: 0, - min_new_tokens: MIN_NEW_TOKENS, - max_length: MAX_LENGTH, - }); - expect(outputs.tolist()).toEqual([[0n, 11924n, 11924n, 11924n, 11924n]]); - expect(outputs.dims.at(-1)).toBeGreaterThanOrEqual(MIN_NEW_TOKENS); - }, - MAX_TEST_EXECUTION_TIME, - ); - - afterAll(async () => { - await model?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); - }); - - describe(`decoder-only (${models[1]})`, () => { - const model_id = models[1]; - - let model; - let tokenizer; - beforeAll(async () => { - model = await AutoModelForCausalLM.from_pretrained(model_id); - tokenizer = await AutoTokenizer.from_pretrained(model_id); - }, MAX_MODEL_LOAD_TIME); - - // NOTE: Since `max_length` defaults to 20, this case also tests that. - it( - "default", - async () => { - const outputs = await generate(model, tokenizer, DUMMY_TEXT, {}); - expect(outputs.dims.at(-1)).toEqual(20); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "max_new_tokens", - async () => { - const MAX_NEW_TOKENS = 5; - const PROMPT_LENGTH = 2; // BOS + DUMMY_TEXT - const outputs = await generate(model, tokenizer, DUMMY_TEXT, { - max_new_tokens: MAX_NEW_TOKENS, - }); - const expected_length = PROMPT_LENGTH + MAX_NEW_TOKENS; - expect(outputs.dims.at(-1)).toEqual(expected_length); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "min_length", - async () => { - const MIN_LENGTH = 4; - const outputs = await generate(model, tokenizer, DUMMY_TEXT, { - eos_token_id: [ - 18547, // min_length will suppress this token (generated by default) - 16012, // stop at this token - ], - min_length: MIN_LENGTH, - }); - expect(outputs.tolist()).toEqual([[1n, 22172n, 31583n, 18824n, 16621n, 8136n, 16012n]]); - expect(outputs.dims.at(-1)).toBeGreaterThanOrEqual(MIN_LENGTH); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "min_new_tokens", - async () => { - const MIN_NEW_TOKENS = 2; - const outputs = await generate(model, tokenizer, DUMMY_TEXT, { - eos_token_id: [ - 18547, // min_new_tokens will suppress this token (generated by default) - 16012, // stop at this token - ], - min_new_tokens: MIN_NEW_TOKENS, - }); - expect(outputs.tolist()).toEqual([[1n, 22172n, 31583n, 18824n, 16621n, 8136n, 16012n]]); - expect(outputs.dims.at(-1)).toBeGreaterThanOrEqual(MIN_NEW_TOKENS); - }, - MAX_TEST_EXECUTION_TIME, - ); - - afterAll(async () => { - await model?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); - }); + // List all models which will be tested + const models = [ + "hf-internal-testing/tiny-random-T5ForConditionalGeneration", // + "hf-internal-testing/tiny-random-LlamaForCausalLM", // decoder-only + ]; + const DUMMY_TEXT = "hello"; + + describe(`encoder-decoder (${models[0]})`, () => { + const model_id = models[0]; + + let model; + let tokenizer; + beforeAll(async () => { + model = await AutoModelForSeq2SeqLM.from_pretrained(model_id); + tokenizer = await AutoTokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + // NOTE: Since `max_length` defaults to 20, this case also tests that. + it( + "default", + async () => { + const outputs = await generate(model, tokenizer, DUMMY_TEXT, {}); + expect(outputs.dims.at(-1)).toEqual(20); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "max_new_tokens", + async () => { + const MAX_NEW_TOKENS = 5; + const outputs = await generate(model, tokenizer, DUMMY_TEXT, { + max_new_tokens: MAX_NEW_TOKENS, + }); + expect(outputs.dims.at(-1)).toEqual(MAX_NEW_TOKENS + 1); // + 1 due to forced BOS token + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "min_length", + async () => { + const MIN_LENGTH = 3; + const MAX_LENGTH = 5; + const outputs = await generate(model, tokenizer, DUMMY_TEXT, { + eos_token_id: 0, + min_length: MIN_LENGTH, + max_length: MAX_LENGTH, + }); + expect(outputs.tolist()).toEqual([[0n, 11924n, 11924n, 11924n, 11924n]]); + expect(outputs.dims.at(-1)).toBeGreaterThanOrEqual(MIN_LENGTH); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "min_new_tokens", + async () => { + const MIN_NEW_TOKENS = 2; + const MAX_LENGTH = 5; + const outputs = await generate(model, tokenizer, DUMMY_TEXT, { + eos_token_id: 0, + min_new_tokens: MIN_NEW_TOKENS, + max_length: MAX_LENGTH, + }); + expect(outputs.tolist()).toEqual([[0n, 11924n, 11924n, 11924n, 11924n]]); + expect(outputs.dims.at(-1)).toBeGreaterThanOrEqual(MIN_NEW_TOKENS); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + describe(`decoder-only (${models[1]})`, () => { + const model_id = models[1]; + + let model; + let tokenizer; + beforeAll(async () => { + model = await AutoModelForCausalLM.from_pretrained(model_id); + tokenizer = await AutoTokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + // NOTE: Since `max_length` defaults to 20, this case also tests that. + it( + "default", + async () => { + const outputs = await generate(model, tokenizer, DUMMY_TEXT, {}); + expect(outputs.dims.at(-1)).toEqual(20); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "max_new_tokens", + async () => { + const MAX_NEW_TOKENS = 5; + const PROMPT_LENGTH = 2; // BOS + DUMMY_TEXT + const outputs = await generate(model, tokenizer, DUMMY_TEXT, { + max_new_tokens: MAX_NEW_TOKENS, + }); + const expected_length = PROMPT_LENGTH + MAX_NEW_TOKENS; + expect(outputs.dims.at(-1)).toEqual(expected_length); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "min_length", + async () => { + const MIN_LENGTH = 4; + const outputs = await generate(model, tokenizer, DUMMY_TEXT, { + eos_token_id: [ + 18547, // min_length will suppress this token (generated by default) + 16012, // stop at this token + ], + min_length: MIN_LENGTH, + }); + expect(outputs.tolist()).toEqual([[1n, 22172n, 31583n, 18824n, 16621n, 8136n, 16012n]]); + expect(outputs.dims.at(-1)).toBeGreaterThanOrEqual(MIN_LENGTH); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "min_new_tokens", + async () => { + const MIN_NEW_TOKENS = 2; + const outputs = await generate(model, tokenizer, DUMMY_TEXT, { + eos_token_id: [ + 18547, // min_new_tokens will suppress this token (generated by default) + 16012, // stop at this token + ], + min_new_tokens: MIN_NEW_TOKENS, + }); + expect(outputs.tolist()).toEqual([[1n, 22172n, 31583n, 18824n, 16621n, 8136n, 16012n]]); + expect(outputs.dims.at(-1)).toBeGreaterThanOrEqual(MIN_NEW_TOKENS); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); }); describe("Streamers", () => { - describe("decoder-only", () => { - const model_id = "hf-internal-testing/tiny-random-LlamaForCausalLM"; - let model, tokenizer; - beforeAll(async () => { - model = await AutoModelForCausalLM.from_pretrained(model_id); - tokenizer = await AutoTokenizer.from_pretrained(model_id); - }, MAX_MODEL_LOAD_TIME); - - it( - "batch_size=1", - async () => { - const target_chunks = ["helloerdingsdelete ", "melytabular ", "Stadiumoba ", "alcune ", "drug"]; - const chunks = []; - const callback_function = (text) => { - chunks.push(text); - }; - const streamer = new TextStreamer(tokenizer, { callback_function, skip_special_tokens: true }); - - const inputs = tokenizer("hello"); - const outputs = await model.generate({ - ...inputs, - max_length: 10, - streamer, - }); - expect(outputs.tolist()).toEqual([[1n, 22172n, 18547n, 8143n, 22202n, 9456n, 17213n, 15330n, 26591n, 15721n]]); - expect(chunks).toEqual(target_chunks); - }, - MAX_TEST_EXECUTION_TIME, - ); - - afterAll(async () => { - await model?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); - }); + describe("decoder-only", () => { + const model_id = "hf-internal-testing/tiny-random-LlamaForCausalLM"; + let model, tokenizer; + beforeAll(async () => { + model = await AutoModelForCausalLM.from_pretrained(model_id); + tokenizer = await AutoTokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const target_chunks = ["helloerdingsdelete ", "melytabular ", "Stadiumoba ", "alcune ", "drug"]; + const chunks = []; + const callback_function = (text) => { + chunks.push(text); + }; + const streamer = new TextStreamer(tokenizer, { + callback_function, + skip_special_tokens: true, + }); + + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + streamer, + }); + expect(outputs.tolist()).toEqual([[1n, 22172n, 18547n, 8143n, 22202n, 9456n, 17213n, 15330n, 26591n, 15721n]]); + expect(chunks).toEqual(target_chunks); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); }); diff --git a/tests/utils/hub.test.js b/tests/utils/hub.test.js index 19077f009..e430710e5 100644 --- a/tests/utils/hub.test.js +++ b/tests/utils/hub.test.js @@ -5,36 +5,36 @@ import { MAX_TEST_EXECUTION_TIME } from "../init.js"; // TODO: Set cache folder to a temp directory describe("Hub", () => { - describe("Loading models", () => { - it( - "should load a model from the local cache", - async () => { - // 1. Local model exists (doesn't matter about status of remote file since local is tried first) - const model = await AutoModel.from_pretrained("hf-internal-testing/tiny-random-T5ForConditionalGeneration"); - expect(model).toBeInstanceOf(PreTrainedModel); - }, - MAX_TEST_EXECUTION_TIME, - ); + describe("Loading models", () => { + it( + "should load a model from the local cache", + async () => { + // 1. Local model exists (doesn't matter about status of remote file since local is tried first) + const model = await AutoModel.from_pretrained("hf-internal-testing/tiny-random-T5ForConditionalGeneration"); + expect(model).toBeInstanceOf(PreTrainedModel); + }, + MAX_TEST_EXECUTION_TIME, + ); - it( - "should load a model from the remote cache", - async () => { - // 2. Local model doesn't exist, remote file exists - // This tests that fallback functionality is working - const model = await AutoModel.from_pretrained("hf-internal-testing/tiny-random-T5ForConditionalGeneration"); - expect(model).toBeInstanceOf(PreTrainedModel); - }, - MAX_TEST_EXECUTION_TIME, - ); + it( + "should load a model from the remote cache", + async () => { + // 2. Local model doesn't exist, remote file exists + // This tests that fallback functionality is working + const model = await AutoModel.from_pretrained("hf-internal-testing/tiny-random-T5ForConditionalGeneration"); + expect(model).toBeInstanceOf(PreTrainedModel); + }, + MAX_TEST_EXECUTION_TIME, + ); - it( - "should fail to load a model", - async () => { - // 3. Local model doesn't exist, remote file doesn't exist - // This tests that error handling is working. - await expect(AutoModel.from_pretrained("hf-internal-testing/this-model-does-not-exist")).rejects.toBeInstanceOf(Error); - }, - MAX_TEST_EXECUTION_TIME, - ); - }); + it( + "should fail to load a model", + async () => { + // 3. Local model doesn't exist, remote file doesn't exist + // This tests that error handling is working. + await expect(AutoModel.from_pretrained("hf-internal-testing/this-model-does-not-exist")).rejects.toBeInstanceOf(Error); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); }); diff --git a/tests/utils/logits_process.test.js b/tests/utils/logits_process.test.js index 5da188ed4..6fb830bfe 100644 --- a/tests/utils/logits_process.test.js +++ b/tests/utils/logits_process.test.js @@ -1,7 +1,7 @@ import { - // Pipelines - pipeline, - TextGenerationPipeline, + // Pipelines + pipeline, + TextGenerationPipeline, } from "../../src/transformers.js"; import { init } from "../init.js"; @@ -13,76 +13,76 @@ const MAX_TEST_EXECUTION_TIME = 10_000; // 10 seconds const MAX_MODEL_DISPOSE_TIME = 1_000; // 1 second const DEFAULT_MODEL_OPTIONS = { - dtype: "fp32", + dtype: "fp32", }; describe("Logits Processors", () => { - describe("text-generation", () => { - const model_id = "hf-internal-testing/tiny-random-LlamaForCausalLM"; + describe("text-generation", () => { + const model_id = "hf-internal-testing/tiny-random-LlamaForCausalLM"; - /** @type {TextGenerationPipeline} */ - let pipe; - beforeAll(async () => { - pipe = await pipeline("text-generation", model_id, { - // TODO move to config - ...DEFAULT_MODEL_OPTIONS, - }); - }, MAX_MODEL_LOAD_TIME); + /** @type {TextGenerationPipeline} */ + let pipe; + beforeAll(async () => { + pipe = await pipeline("text-generation", model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + }, MAX_MODEL_LOAD_TIME); - describe("bad_word_ids", () => { - it( - "basic", - async () => { - const text_input = "hello"; + describe("bad_word_ids", () => { + it( + "basic", + async () => { + const text_input = "hello"; - const generated_text_target = " Bert explicit wed digasset"; - const text_target = [{ generated_text: text_input + generated_text_target }]; + const generated_text_target = " Bert explicit wed digasset"; + const text_target = [{ generated_text: text_input + generated_text_target }]; - const output = await pipe(text_input, { - max_new_tokens: 5, - bad_words_ids: [ - // default: [22172n, 18547n, 8136n, 16012n, 28064n, 11361n] - [18547], + const output = await pipe(text_input, { + max_new_tokens: 5, + bad_words_ids: [ + // default: [22172n, 18547n, 8136n, 16012n, 28064n, 11361n] + [18547], - // block #1: [22172n, 16662n, 6261n, 18916n, 29109n, 799n] - [6261, 18916], - ], - }); - compare(output, text_target); - }, - MAX_TEST_EXECUTION_TIME, - ); + // block #1: [22172n, 16662n, 6261n, 18916n, 29109n, 799n] + [6261, 18916], + ], + }); + compare(output, text_target); + }, + MAX_TEST_EXECUTION_TIME, + ); - it( - "many bad words", - async () => { - const text_input = "hello"; + it( + "many bad words", + async () => { + const text_input = "hello"; - const generated_text_target = "erdingsdeletearus)?nor"; - const text_target = [{ generated_text: text_input + generated_text_target }]; + const generated_text_target = "erdingsdeletearus)?nor"; + const text_target = [{ generated_text: text_input + generated_text_target }]; - // Construct long list of bad words - const bad_words_ids = []; - // default: [22172n, 18547n, 8136n, 16012n, 28064n, 11361n] - for (let i = 0; i < 100000; ++i) { - bad_words_ids.push([i * 2]); // block all even numbers - } - // block #1: [22172n, 18547n, 8143n, 30327n, 20061n, 18193n] - bad_words_ids.push([8143, 30327]); + // Construct long list of bad words + const bad_words_ids = []; + // default: [22172n, 18547n, 8136n, 16012n, 28064n, 11361n] + for (let i = 0; i < 100000; ++i) { + bad_words_ids.push([i * 2]); // block all even numbers + } + // block #1: [22172n, 18547n, 8143n, 30327n, 20061n, 18193n] + bad_words_ids.push([8143, 30327]); - // block #2: [22172n, 18547n, 8143n, 29485n, 3799n, 29331n] - bad_words_ids.push([18547, 8143, 29485]); + // block #2: [22172n, 18547n, 8143n, 29485n, 3799n, 29331n] + bad_words_ids.push([18547, 8143, 29485]); - // block #3: [22172n, 18547n, 8143n, 26465n, 6877n, 15459n] - const output = await pipe(text_input, { max_new_tokens: 5, bad_words_ids }); - compare(output, text_target); - }, - MAX_TEST_EXECUTION_TIME, - ); - }); + // block #3: [22172n, 18547n, 8143n, 26465n, 6877n, 15459n] + const output = await pipe(text_input, { max_new_tokens: 5, bad_words_ids }); + compare(output, text_target); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); - afterAll(async () => { - await pipe?.dispose(); - }, MAX_MODEL_DISPOSE_TIME); - }); + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); }); diff --git a/tests/utils/maths.test.js b/tests/utils/maths.test.js index 9a02d4dbd..b019cfdae 100644 --- a/tests/utils/maths.test.js +++ b/tests/utils/maths.test.js @@ -2,191 +2,311 @@ import { FFT, medianFilter, bankers_round, log_softmax } from "../../src/utils/m import { compare } from "../test_utils.js"; const fft = (arr, complex = false) => { - let output; - let fft; - if (complex) { - fft = new FFT(arr.length / 2); - output = new Float64Array(fft.outputBufferSize); - fft.transform(output, arr); - } else { - fft = new FFT(arr.length); - output = new Float64Array(fft.outputBufferSize); - fft.realTransform(output, arr); - } - if (!fft.isPowerOfTwo) { - output = output.slice(0, complex ? arr.length : 2 * arr.length); - } - return output; + let output; + let fft; + if (complex) { + fft = new FFT(arr.length / 2); + output = new Float64Array(fft.outputBufferSize); + fft.transform(output, arr); + } else { + fft = new FFT(arr.length); + output = new Float64Array(fft.outputBufferSize); + fft.realTransform(output, arr); + } + if (!fft.isPowerOfTwo) { + output = output.slice(0, complex ? arr.length : 2 * arr.length); + } + return output; }; const FFT_TEST_DATA = { - fft_2_real: { complex: false, input: [1.764052345967664, 0.4001572083672233], output: [2.1642095543348874, 0.0, 1.3638951376004407, 0.0] }, - fft_3_real: { complex: false, input: [0.9787379841057392, 2.240893199201458, 1.8675579901499675], output: [5.087189173457165, 0.0, -1.0754876105699736, -0.3233177751657647, -1.0754876105699736, 0.3233177751657647] }, - fft_4_real: { complex: false, input: [-0.977277879876411, 0.9500884175255894, -0.1513572082976979, -0.10321885179355784], output: [-0.2817655224420774, 0.0, -0.8259206715787131, -1.0533072693191472, -1.9755046539061405, 0.0, -0.8259206715787131, 1.0533072693191472] }, - fft_5_real: { complex: false, input: [0.41059850193837233, 0.144043571160878, 1.454273506962975, 0.7610377251469934, 0.12167501649282841], output: [2.891628321702047, 0.0, -1.2995143733619916, -0.4287475285900295, 0.8801964673568988, 0.6461585010760772, 0.8801964673568988, -0.6461585010760772, -1.2995143733619916, 0.4287475285900295] }, - fft_7_real: { complex: false, input: [0.44386323274542566, 0.33367432737426683, 1.4940790731576061, -0.20515826376580087, 0.31306770165090136, -0.8540957393017248, -2.5529898158340787], output: [-1.0275594839734046, 0.0, -1.1794900746698127, -4.321336254238002, 0.4283827622463364, -2.2006199533718855, 2.8184083690191684, 1.0886333211894632, 2.8184083690191684, -1.0886333211894632, 0.4283827622463364, 2.2006199533718855, -1.1794900746698127, 4.321336254238002] }, - fft_8_real: { complex: false, input: [0.6536185954403606, 0.8644361988595057, -0.7421650204064419, 2.2697546239876076, -1.4543656745987648, 0.04575851730144607, -0.1871838500258336, 1.5327792143584575], output: [2.9826326049163367, 0.0, 2.1657565005584045, -0.5450316795716824, 0.1286017912738714, 2.8923391221851134, 2.050212039519846, -1.6549940203328992, -6.442824504097697, 0.0, 2.050212039519846, 1.6549940203328992, 0.1286017912738714, -2.8923391221851134, 2.1657565005584045, 0.5450316795716824] }, - fft_9_real: { complex: false, input: [1.469358769900285, 0.1549474256969163, 0.37816251960217356, -0.8877857476301128, -1.980796468223927, -0.3479121493261526, 0.15634896910398005, 1.2302906807277207, 1.2023798487844113], output: [1.3749938486352948, 0.0, 5.342425446245793, 2.975185521629713, -1.22457133247234, -0.6308804310909996, 0.41938606274358115, 1.5832577539331407, 1.3873773637166027, -0.8933243837257328, 1.3873773637166025, 0.8933243837257323, 0.41938606274358115, -1.5832577539331407, -1.2245713324723406, 0.6308804310909994, 5.342425446245792, -2.975185521629712] }, - fft_16_real: { complex: false, input: [-0.3873268174079523, -0.30230275057533557, -1.0485529650670926, -1.4200179371789752, -1.7062701906250126, 1.9507753952317897, -0.5096521817516535, -0.4380743016111864, -1.2527953600499262, 0.7774903558319101, -1.6138978475579515, -0.2127402802139687, -0.8954665611936756, 0.386902497859262, -0.510805137568873, -1.180632184122412], output: [-8.363366266001053, 0.0, -1.4796937055641797, 0.20983245542766216, -0.3454288633172973, 2.968909080470098, 2.329350352646858, -0.7633502808629014, -0.5589507973309957, -6.064330201474169, -1.3963011418461777, 1.659405903990098, 2.2686580120389164, -0.3150779061389377, 4.008518665331395, -0.6106258774446865, -7.486167856443222, 0.0, 4.008518665331395, 0.6106258774446865, 2.2686580120389164, 0.3150779061389377, -1.3963011418461777, -1.659405903990098, -0.5589507973309957, 6.064330201474169, 2.329350352646858, 0.7633502808629014, -0.3454288633172973, -2.968909080470098, -1.4796937055641797, -0.20983245542766216] }, - fft_25_real: { complex: false, input: [-0.028182228338654868, 0.42833187053041766, 0.06651722238316789, 0.3024718977397814, -0.6343220936809636, -0.3627411659871381, -0.672460447775951, -0.3595531615405413, -0.813146282044454, -1.7262826023316769, 0.17742614225375283, -0.4017809362082619, -1.6301983469660446, 0.4627822555257742, -0.9072983643832422, 0.05194539579613895, 0.7290905621775369, 0.12898291075741067, 1.1394006845433007, -1.2348258203536526, 0.402341641177549, -0.6848100909403132, -0.8707971491818818, -0.5788496647644155, -0.31155253212737266], output: [-7.327510303739734, 0.0, 1.377932222483735, 3.044921011687328, 0.06954965931518875, -5.483478593023237, 2.616166182267447, -2.0815154752919764, 0.1551657690860495, 0.5208228796162939, 0.3085012063215258, -2.1386609330432687, -0.566204290441261, 0.012329506565646242, 1.309786704987264, 3.1736358514734673, -1.0557581301013133, -2.4635866620117564, -2.579387724556426, 1.9885603872002973, 3.9572284078024613, -5.498314280945309, 0.6049416192890185, 2.7215390337882552, -2.8864443288170083, 0.6099589663561517, -2.8864443288170074, -0.6099589663561518, 0.604941619289019, -2.721539033788255, 3.9572284078024613, 5.498314280945309, -2.579387724556426, -1.988560387200297, -1.0557581301013133, 2.4635866620117564, 1.3097867049872634, -3.1736358514734673, -0.5662042904412615, -0.012329506565646575, 0.3085012063215258, 2.1386609330432687, 0.15516576908604957, -0.520822879616294, 2.6161661822674476, 2.0815154752919764, 0.06954965931518836, 5.4834785930232375, 1.377932222483735, -3.044921011687328] }, - fft_27_real: { complex: false, input: [0.05616534222974544, -1.1651498407833565, 0.9008264869541871, 0.46566243973045984, -1.5362436862772237, 1.4882521937955997, 1.8958891760305832, 1.1787795711596507, -0.17992483581235091, -1.0707526215105425, 1.0544517269311366, -0.40317694697317963, 1.2224450703824274, 0.2082749780768603, 0.9766390364837128, 0.3563663971744019, 0.7065731681919482, 0.010500020720820478, 1.7858704939058352, 0.12691209270361992, 0.40198936344470165, 1.8831506970562544, -1.3477590611424464, -1.2704849984857336, 0.9693967081580112, -1.17312340511416, 1.9436211856492926], output: [9.485150752680255, 0.0, -3.3736195725050755, -0.6576671739932465, -1.4668314059671266, -3.7345058492453083, 0.41824676938222627, 1.9663430754071836, 5.709032912166549, 5.528638987648501, 1.8435730384179874, -0.34683544424097734, -4.963689356113413, -0.5177297293983665, -5.8830486994336315, 3.207542182273314, 1.8987452089640713, 2.1505302726369, 6.603715178395637, 5.036393219486801, 1.3865285438489818, 10.275020532162417, 0.1681971482993614, 2.6656111808114664, -3.3300735021919055, -3.3923756439906483, -2.9951195195022233, 4.64643931476696, -2.9951195195022233, -4.64643931476696, -3.3300735021919055, 3.3923756439906487, 0.16819714829936183, -2.665611180811466, 1.3865285438489812, -10.275020532162415, 6.603715178395637, -5.036393219486801, 1.8987452089640697, -2.1505302726368996, -5.8830486994336315, -3.207542182273315, -4.963689356113414, 0.5177297293983658, 1.8435730384179874, 0.34683544424097734, 5.709032912166549, -5.528638987648501, 0.41824676938222605, -1.9663430754071831, -1.4668314059671266, 3.7345058492453087, -3.373619572505076, 0.6576671739932474] }, - fft_32_real: { complex: false, input: [-0.41361898075974735, -0.7474548114407578, 1.9229420264803847, 1.4805147914344243, 1.8675589604265699, 0.9060446582753853, -0.8612256850547025, 1.9100649530990337, -0.2680033709513804, 0.8024563957963952, 0.947251967773748, -0.1550100930908342, 0.6140793703460803, 0.9222066715665268, 0.37642553115562943, -1.0994007905841945, 0.298238174206056, 1.3263858966870303, -0.6945678597313655, -0.14963454032767076, -0.43515355172163744, 1.8492637284793418, 0.6722947570124355, 0.40746183624111043, -0.7699160744453164, 0.5392491912918173, -0.6743326606573761, 0.03183055827435118, -0.635846078378881, 0.6764332949464997, 0.5765908166149409, -0.20829875557799488], output: [11.014830327385901, 0.0, 1.2409883700304647, -5.276491159392059, -1.534551501998357, -4.8297133119274545, 1.336708303789408, -2.7137782948769944, -4.424467817605524, 0.843485461536611, -6.581416660337224, -0.1795449695375516, -2.514517651560188, -0.05943606215207886, -4.207556144474312, 7.447633642178459, -2.0080404448719507, -4.057057066134013, 1.4811356128841098, 3.432496305430468, 1.3934203835489032, -3.231520399869298, -4.317592600060283, 0.1993606400932625, -0.7034100876395168, 2.317901569810786, 3.7371408499766874, 5.7058540737147405, 6.345803325381664, -2.1851091826937408, 1.6157350284647216, 2.7643998907723595, -5.969395642755025, 0.0, 1.6157350284647212, -2.7643998907723604, 6.345803325381664, 2.1851091826937408, 3.737140849976688, -5.7058540737147405, -0.7034100876395168, -2.317901569810786, -4.317592600060284, -0.19936064009326138, 1.3934203835489032, 3.231520399869298, 1.4811356128841102, -3.432496305430468, -2.0080404448719507, 4.057057066134013, -4.207556144474311, -7.447633642178459, -2.514517651560188, 0.05943606215207886, -6.581416660337223, 0.1795449695375514, -4.424467817605524, -0.843485461536611, 1.3367083037894085, 2.7137782948769944, -1.534551501998357, 4.8297133119274545, 1.2409883700304651, 5.276491159392059] }, - fft_49_real: { complex: false, input: [0.3960067126616453, -1.0930615087305058, -1.4912575927056055, 0.4393917012645369, 0.16667349537252904, 0.6350314368921064, 2.383144774863942, 0.9444794869904138, -0.9128222254441586, 1.117016288095853, -1.3159074105115212, -0.461584604814709, -0.06824160532463124, 1.7133427216493666, -0.7447548220484399, -0.8264385386590144, -0.0984525244254323, -0.6634782863621074, 1.126635922106507, -1.0799315083634233, -1.1474686524111024, -0.43782004474443403, -0.4980324506923049, 1.9295320538169858, 0.9494208069257608, 0.0875512413851909, -1.225435518830168, 0.8443629764015471, -1.0002153473895647, -1.5447710967776116, 1.1880297923523018, 0.3169426119248496, 0.920858823780819, 0.3187276529430212, 0.8568306119026912, -0.6510255933001469, -1.0342428417844647, 0.681594518281627, -0.8034096641738411, -0.6895497777502005, -0.45553250351734315, 0.01747915902505673, -0.35399391125348395, -1.3749512934180188, -0.6436184028328905, -2.2234031522244266, 0.6252314510271875, -1.6020576556067476, -1.1043833394284506], output: [-7.887557633860811, 0.0, -5.014267769110843, -4.376842071326452, -2.3853231496935923, -10.851256239812793, -3.2717318894892826, -5.326849080561187, -2.7264013581332716, 1.1514977438201093, -7.652841900031138, -3.7746076991956867, 5.309279658846421, 8.814060475350379, -2.6168809007788796, 4.677691230979557, 7.6656963597889805, 3.4858425270229887, 0.2740709971425779, -4.676330054757555, -5.155232202781799, 3.7556835779660136, 5.001888119593458, -3.6198301043995045, -0.0953704694177091, -3.6318583861463845, 2.767000800593773, -2.450144252684209, -1.2540039367717049, 9.279722666945066, 3.815471636849461, -3.3563499317203815, 12.763423104544493, 0.4158128174652558, 3.995279461690892, -6.6439121988180325, 3.2201986963478983, 0.6334206695259166, 2.4912752816319514, -1.9446295959593973, 1.027982187347956, -2.8785858483639863, 1.3490313376869536, 14.471900839049741, 1.0215246227635422, -0.4420763712931004, -1.0162745955034818, -0.3075325636480104, -5.867850815975937, -1.9145668225042987, -5.867850815975937, 1.9145668225042978, -1.0162745955034822, 0.3075325636480101, 1.0215246227635417, 0.4420763712931006, 1.3490313376869536, -14.471900839049741, 1.0279821873479558, 2.878585848363986, 2.491275281631952, 1.9446295959593989, 3.2201986963478975, -0.633420669525917, 3.9952794616908927, 6.6439121988180325, 12.763423104544493, -0.41581281746525595, 3.815471636849461, 3.3563499317203815, -1.2540039367717049, -9.279722666945066, 2.767000800593772, 2.4501442526842085, -0.0953704694177091, 3.631858386146384, 5.001888119593458, 3.6198301043995045, -5.1552322027818, -3.755683577966013, 0.2740709971425781, 4.676330054757556, 7.665696359788981, -3.4858425270229905, -2.6168809007788796, -4.677691230979557, 5.30927965884642, -8.814060475350379, -7.652841900031138, 3.774607699195687, -2.7264013581332716, -1.1514977438201108, -3.2717318894892826, 5.326849080561189, -2.385323149693591, 10.851256239812795, -5.014267769110844, 4.376842071326452] }, - fft_64_real: { complex: false, input: [0.052165079260974405, -0.7395629963913133, 1.5430145954067358, -1.2928569097234486, 0.26705086934918293, -0.0392828182274956, -1.1680934977411974, 0.5232766605317537, -0.1715463312222481, 0.7717905512136674, 0.8235041539637314, 2.16323594928069, 1.336527949436392, -0.3691818379424436, -0.2393791775759264, 1.0996595958871132, 0.6552637307225978, 0.640131526097592, -1.6169560443108344, -0.024326124398935636, -0.7380309092056887, 0.27992459904323824, -0.09815038964295794, 0.9101789080925919, 0.31721821519130206, 0.7863279621089762, -0.46641909673594306, -0.9444462559182504, -0.41004969320254847, -0.017020413861440594, 0.3791517355550818, 2.259308950690852, -0.04225715166064269, -0.955945000492777, -0.34598177569938643, -0.4635959746460942, 0.4814814737734622, -1.5407970144446248, 0.06326199420033171, 0.1565065379653756, 0.23218103620027578, -0.5973160689653627, -0.237921729736007, -1.4240609089825316, -0.49331988336219407, -0.5428614760167177, 0.4160500462614255, -1.1561824318219127, 0.7811981017099934, 1.4944845444913688, -2.0699850250135325, 0.42625873077810095, 0.6769080350302455, -0.637437025552229, -0.39727181432879766, -0.13288057758695562, -0.2977908794017283, -0.3090129690471222, -1.6760038063299767, 1.15233156478312, 1.079618592036821, -0.8133642592042029, -1.466424327802514, 0.5210648764527586], output: [-1.64663603068023, 0.0, 1.505825929177829, -9.200008132744042, -2.544735819316532, -1.5339227544442826, -7.4360215710438755, -1.7984575917004604, 1.3619465096845937, 4.8456564834488915, 5.070506185450519, 4.849409603121612, 3.777805068935999, 0.17970844009281262, -5.309242543498751, -6.973066979348705, 5.944277107889729, 1.4068283873569305, -7.321235075656323, -8.916520841302049, -5.0314030969029835, 1.309770133246369, 3.721064537404591, -2.7547595090183408, 3.8878856671539617, 0.733440111196902, 5.958504522515222, -9.443751218054468, -3.743781835570924, 1.4088028338848555, 1.4543498973854279, -1.4051068307925165, 10.284222394185964, 6.362595288575114, 5.69300272333881, 2.3840261995408785, -2.344460920124865, 8.555867083200523, -10.10946038896856, -1.5537275998675408, -0.8110283215952525, 0.08930281805022311, -0.3624099573254269, 4.570396206585882, -7.6585567061471576, 9.739963432241709, -4.941569815253301, 0.6205797252271052, -7.291786374000026, -1.6649582074043856, 5.388745981879483, 1.3949577742938146, 1.4070140670266627, 11.226248771172266, 4.869069031848913, 4.140589494166391, 1.0264270218179838, 0.9000492064571382, 4.80696917175708, 0.26025510601075585, 4.725688003441722, 8.559147480463128, -1.4773429342657596, -6.3922359470118755, -4.01533581906691, 0.0, -1.4773429342657578, 6.392235947011875, 4.7256880034417215, -8.559147480463128, 4.80696917175708, -0.26025510601075497, 1.0264270218179838, -0.9000492064571382, 4.869069031848914, -4.140589494166391, 1.4070140670266635, -11.226248771172266, 5.388745981879483, -1.3949577742938146, -7.291786374000026, 1.6649582074043856, -4.941569815253302, -0.6205797252271044, -7.658556706147157, -9.739963432241707, -0.36240995732542824, -4.570396206585883, -0.8110283215952525, -0.08930281805022311, -10.109460388968563, 1.5537275998675408, -2.344460920124864, -8.555867083200523, 5.693002723338812, -2.38402619954088, 10.284222394185964, -6.362595288575114, 1.4543498973854279, 1.4051068307925176, -3.743781835570923, -1.4088028338848546, 5.95850452251522, 9.443751218054468, 3.8878856671539617, -0.733440111196902, 3.72106453740459, 2.754759509018341, -5.031403096902983, -1.309770133246368, -7.321235075656325, 8.916520841302049, 5.944277107889729, -1.4068283873569305, -5.309242543498751, 6.973066979348703, 3.777805068936, -0.1797084400928135, 5.07050618545052, -4.849409603121612, 1.3619465096845937, -4.8456564834488915, -7.4360215710438755, 1.7984575917004595, -2.544735819316533, 1.5339227544442826, 1.505825929177829, 9.200008132744042] }, - fft_81_real: { complex: false, input: [-0.5757879698130661, 0.14195316332077967, -0.3193284171450952, 0.6915387510701866, 0.6947491436560059, -0.7255973784635843, -1.3833639553950554, -1.582938397335082, 0.6103793791072052, -1.188859257784029, -0.5068163542986875, -0.5963140384505081, -0.05256729626954629, -1.936279805846507, 0.18877859679382855, 0.5238910238342056, 0.08842208704466141, -0.3108861716984717, 0.09740016626878341, 0.3990463456401302, -2.77259275642665, 1.9559123082506942, 0.39009332268792646, -0.65240858238702, -0.3909533751876011, 0.49374177734918845, -0.11610393903436653, -2.0306844677814944, 2.0644928613593194, -0.11054065723247261, 1.0201727117157997, -0.6920498477843912, 1.5363770542457977, 0.28634368889227957, 0.6088438344754508, -1.0452533661469547, 1.2111452896827009, 0.6898181645347884, 1.3018462295649984, -0.6280875596415789, -0.4810271184607877, 2.303916697683942, -1.0600158227215473, -0.13594970067832082, 1.1368913626026953, 0.0977249677148556, 0.5829536797532936, -0.3994490292628752, 0.37005588784751875, -1.3065268517353166, 1.658130679618188, -0.11816404512856976, -0.6801782039968504, 0.6663830820319143, -0.4607197873885533, -1.3342584714027534, -1.3467175057975553, 0.6937731526901325, -0.1595734381462669, -0.13370155966843916, 1.0777438059762627, -1.1268258087567435, -0.7306777528648248, -0.38487980918127546, 0.094351589317074, -0.042171451290578935, -0.2868871923899076, -0.0616264020956474, -0.10730527629117469, -0.7196043885517929, -0.8129929885540773, 0.2745163577239395, -0.8909150829955279, -1.1573552591908536, -0.3122922511256933, -0.1576670161638159, 2.2567234972982093, -0.7047002758562337, 0.9432607249694948, 0.7471883342046318, -1.188944955203736], output: [-6.019981320094997, 0.0, -11.024215089600144, -2.2296309610489784, 7.885780110732945, 6.9546439213354425, 2.1468923343095865, 2.0704996330469774, 5.223342327125926, 3.3335079943959576, 6.670733097535111, 3.059813085584419, -1.291987686750553, 0.4705982966401041, -2.6851539401562734, 0.13762863856647578, -2.2292463073335753, 1.3031715676222848, -8.459551141726745, -1.1728586245314498, -5.19534088481691, -4.1102647711302005, 4.442176173858688, -5.752409619199853, -10.978436211517131, -2.360518158521794, -8.97056708617231, -4.44028490950546, -0.7378544972090442, 0.3833473146759321, 2.41788584599411, 2.094126839887842, -0.9134777730571964, 0.6964431384964922, -1.348881203926696, 4.273283923628751, 2.6990650914215637, -0.04446227794217483, -2.8201900136765587, 10.405176121148552, 4.561086132295523, -4.8957475545660945, -9.626965987488632, 5.703554469280806, 5.118571440405241, -0.47801741518584884, 4.087758685622818, -16.982284140767902, -1.672788156348311, -8.247953480159863, -15.76267685360949, 6.557268666342, 8.45494487748653, -7.023784282842257, 0.9696988388268291, 0.6791846860727092, -3.7628962341106327, -6.731131201638246, 0.3164269710342955, -0.346532027732811, 5.824835704304008, -8.818250625842744, 5.382334715219444, -1.672571325598101, 6.240545283378912, -5.74548463471282, -6.38340050971792, 3.0747648212362937, 11.52437440252928, -1.3394713252215091, 6.27611320575086, 6.861171951341367, -10.764313909223372, -10.425524196300067, -8.17856146704236, 14.338103920769218, 4.86550855683382, 5.452700220702592, -3.278038589307964, -1.8007559828104207, 0.6670476307446598, 9.785124153822995, 0.6670476307446602, -9.785124153823, -3.2780385893079633, 1.8007559828104218, 4.865508556833819, -5.452700220702592, -8.178561467042359, -14.338103920769221, -10.764313909223372, 10.425524196300069, 6.276113205750862, -6.861171951341369, 11.524374402529276, 1.3394713252215045, -6.3834005097179185, -3.0747648212362932, 6.2405452833789115, 5.745484634712819, 5.382334715219443, 1.6725713255980996, 5.824835704304008, 8.818250625842744, 0.31642697103429596, 0.34653202773281055, -3.762896234110635, 6.731131201638245, 0.9696988388268291, -0.6791846860727092, 8.45494487748653, 7.023784282842258, -15.762676853609491, -6.557268666342001, -1.67278815634831, 8.247953480159865, 4.087758685622817, 16.982284140767902, 5.118571440405242, 0.47801741518584784, -9.626965987488635, -5.703554469280807, 4.561086132295526, 4.895747554566093, -2.82019001367656, -10.405176121148555, 2.699065091421563, 0.04446227794217439, -1.3488812039266977, -4.273283923628747, -0.9134777730571959, -0.6964431384964889, 2.417885845994109, -2.0941268398878425, -0.7378544972090446, -0.3833473146759312, -8.97056708617231, 4.4402849095054595, -10.978436211517133, 2.3605181585217934, 4.442176173858688, 5.752409619199854, -5.195340884816912, 4.1102647711302005, -8.459551141726745, 1.1728586245314485, -2.2292463073335775, -1.3031715676222801, -2.6851539401562716, -0.13762863856647434, -1.291987686750553, -0.47059829664010344, 6.670733097535112, -3.0598130855844157, 5.223342327125925, -3.3335079943959554, 2.1468923343095883, -2.0704996330469765, 7.8857801107329415, -6.9546439213354425, -11.024215089600146, 2.229630961048978] }, - fft_125_real: { complex: false, input: [0.7732529774025997, -1.1838806401933177, -2.659172237996741, 0.6063195243593807, -1.7558905834377194, 0.45093446180591484, -0.6840108977372166, 1.6595507961898721, 1.068509399316009, -0.45338580385138766, -0.6878376110286823, -1.2140774030941206, -0.4409226322925914, -0.2803554951845091, -0.3646935443916854, 0.15670385527236397, 0.5785214977288784, 0.349654456993174, -0.764143923906443, -1.4377914738015785, 1.3645318481024713, -0.6894491845499376, -0.6522935999350191, -0.5211893123011109, -1.8430695501566485, -0.4779740040404867, -0.47965581400794766, 0.6203582983435125, 0.698457149107336, 0.00377088908626934, 0.9318483741143037, 0.339964983801262, -0.01568211160255477, 0.16092816829822298, -0.19065349358139935, -0.3948495140334503, -0.26773353689396645, -1.1280113314700069, 0.280441705316296, -0.9931236109295807, 0.8416312640736364, -0.24945858016094885, 0.04949498165009074, 0.49383677628095635, 0.6433144650629279, -1.5706234086334527, -0.20690367616397173, 0.8801789120807822, -1.6981058194322545, 0.3872804753950634, -2.2555642294021894, -1.0225068436356035, 0.0386305518401881, -1.6567151023219537, -0.9855107376841507, -1.4718350074635869, 1.6481349322075596, 0.16422775548733395, 0.5672902778526694, -0.2226751005151545, -0.35343174875719907, -1.6164741886510325, -0.2918373627478628, -0.7614922118116233, 0.8579239242923363, 1.1411018666575734, 1.4665787155741776, 0.852551939461232, -0.5986539369229861, -1.1158969859603944, 0.7666631816450861, 0.3562928174722889, -1.7685384506770307, 0.35548179274376907, 0.8145198224878664, 0.05892558918162996, -0.18505367100934153, -0.8076484876163557, -1.4465346995633879, 0.8002979493400275, -0.3091144447717088, -0.23346666154369272, 1.7327211869191332, 0.6845011068591904, 0.3708250012811021, 0.14206180518723566, 1.5199948607657727, 1.7195893074161945, 0.9295051114795281, 0.5822245913979243, -2.0946030712061448, 0.12372191423350658, -0.130106954193704, 0.09395322938556872, 0.9430460873225178, -2.7396771671895563, -0.5693120534701851, 0.26990435494076137, -0.4668455460527625, -1.4169061131262595, 0.8689634868967954, 0.27687190584612803, -0.9711045704444846, 0.3148172045158238, 0.8215857120497958, 0.005292646299360854, 0.8005648034309968, 0.07826017516166135, -0.39522898265435435, -1.159420516399913, -0.08593076697161273, 0.19429293804577166, 0.8758327615873309, -0.11510746848722672, 0.4574156062209908, -0.9646120137337284, -0.7826291558275251, -0.11038929902688775, -1.0546284639850139, 0.8202478373246812, 0.4631303293186071, 0.2790957643924534, 0.33890412521594454, 2.0210435614847975, -0.46886418796679563], output: [-12.976781205595502, 0.0, -0.5336306740788994, 9.664313658528595, -2.250865418762321, -6.399236976457672, 4.984600678216093, 10.085773094158728, 4.054865444587319, 1.1220280117249106, -12.381959140729, -4.245941970428558, 11.300429673882508, -3.867369304887277, -1.924145132324413, 15.625845728307613, -3.683967210179358, 9.246098611130797, 6.678100602051274, -5.8877245671287906, -11.142505237356518, 1.2462822325294702, -4.950012182132614, 15.368429468743631, -2.7544743905540385, 7.397618288416753, 4.896491435868608, 21.15703724867305, -10.418689778983975, 1.697253164520248, 2.2518122169760852, 14.869916845413385, -5.553898084870594, -3.616357859997289, 8.688187732450988, 9.51306110875087, 1.8459884086796337, -6.750879404311422, 10.389191729731694, -8.563754760867933, 6.323482750892878, -0.814569185033468, -6.339262858307836, 12.395809458636048, -5.5325241155089815, -4.087114562924723, 5.264130311599194, 5.174155710626681, 0.07817803409444535, 4.502699566709124, -6.841988594388185, -4.209615030635216, -3.014915877040603, 0.7642671251818629, 4.979942269363006, -0.7954973970411463, -9.213201050170268, 20.281572828473664, 2.195700817994501, -2.8150917630751158, 3.66256874231074, 4.947718687255637, 2.417023995325397, 6.311630702081038, 8.910067289733103, -1.3549619066609422, -7.604782781699578, 3.2671105424317712, -0.9261608630826195, -10.26033603214804, 2.477431685753063, -0.5368669007709066, 6.588511626180253, 3.6391310126388206, -0.7696459233665145, 10.849962526266822, 0.8214214627634562, -9.665598459357405, -4.208875535725593, -8.955547690168373, 2.3998423294426914, -6.900645072282014, 10.561899982588944, -6.1497412675557115, -10.104315792100294, 11.339404667032188, 4.041967950367949, -10.290041543502431, 9.533068142456843, -5.3782354906044025, 4.12027245501964, -0.5914284618824925, 12.365588431711828, -2.701635423848076, -6.366846689539269, -5.370727636437312, 6.689816472907384, 7.334498752981377, 13.69756331474747, 3.7921571289587113, -0.2721490559996148, 0.2084376847247582, 9.275694421318246, -16.06448352473527, 1.3029488980606758, -8.03640349313321, 0.20226177706452786, 6.0912757829407465, -4.452961538391443, -15.072885312247177, -0.8419482730978101, -7.04697219502747, 12.864547051022127, -6.206422060382598, -6.149387177052475, -15.825612767912974, -0.5432046258303531, -0.24636987696629387, 0.5819230336105994, -10.272301011724789, 3.8285784744531433, 1.1829320961516245, -5.404044842034781, 0.4066057531145235, -1.2770351094581365, 7.590779935818785, -1.2770351094581383, -7.590779935818781, -5.404044842034781, -0.4066057531145224, 3.828578474453144, -1.1829320961516248, 0.5819230336105993, 10.272301011724787, -0.54320462583035, 0.2463698769662941, -6.149387177052476, 15.825612767912975, 12.864547051022129, 6.206422060382598, -0.8419482730978074, 7.04697219502747, -4.452961538391442, 15.072885312247177, 0.20226177706453008, -6.091275782940745, 1.3029488980606758, 8.03640349313321, 9.27569442131825, 16.06448352473527, -0.2721490559996148, -0.2084376847247582, 13.69756331474747, -3.792157128958717, 6.689816472907385, -7.334498752981374, -6.366846689539267, 5.370727636437312, 12.365588431711828, 2.7016354238480744, 4.120272455019638, 0.5914284618824928, 9.533068142456843, 5.378235490604402, 4.041967950367946, 10.290041543502431, -10.104315792100294, -11.339404667032191, 10.561899982588944, 6.1497412675557115, 2.399842329442692, 6.900645072282014, -4.208875535725592, 8.955547690168373, 0.8214214627634587, 9.665598459357408, -0.7696459233665118, -10.849962526266822, 6.588511626180252, -3.6391310126388214, 2.477431685753063, 0.536866900770905, -0.9261608630826198, 10.260336032148041, -7.604782781699578, -3.2671105424317712, 8.9100672897331, 1.354961906660943, 2.4170239953253976, -6.311630702081036, 3.662568742310737, -4.947718687255637, 2.1957008179945006, 2.815091763075113, -9.213201050170266, -20.281572828473664, 4.979942269363008, 0.7954973970411481, -3.0149158770406013, -0.7642671251818633, -6.841988594388185, 4.209615030635216, 0.07817803409444402, -4.502699566709122, 5.264130311599196, -5.174155710626682, -5.532524115508983, 4.087114562924725, -6.339262858307835, -12.395809458636048, 6.323482750892879, 0.8145691850334673, 10.389191729731692, 8.563754760867932, 1.845988408679628, 6.75087940431142, 8.68818773245099, -9.51306110875087, -5.553898084870594, 3.6163578599972883, 2.251812216976085, -14.869916845413385, -10.418689778983971, -1.6972531645202498, 4.896491435868606, -21.15703724867305, -2.7544743905540354, -7.397618288416755, -4.950012182132614, -15.368429468743635, -11.142505237356517, -1.2462822325294698, 6.678100602051274, 5.8877245671287906, -3.683967210179361, -9.246098611130797, -1.9241451323244125, -15.625845728307619, 11.30042967388251, 3.8673693048872773, -12.381959140729, 4.245941970428559, 4.054865444587321, -1.122028011724911, 4.984600678216097, -10.085773094158727, -2.250865418762321, 6.399236976457672, -0.5336306740788972, -9.664313658528593] }, - fft_128_real: { complex: false, input: [-2.201441285500558, 0.1993001968964652, -0.050603540961665895, -0.5175190425104033, -0.9788298593587699, -0.43918952180214793, 0.18133842921782128, -0.5028167006425383, 2.4124536795437486, -0.960504381633148, -0.7931173627076716, -2.2886200400145285, 0.251484415021537, -2.01640662779976, -0.5394546333745014, -0.27567053456055696, -0.7097279658468882, 1.738872677454511, 0.9943943913154989, 1.3191368763015756, -0.8824188185499185, 1.1285940645145685, 0.4960009463439622, 0.7714059486768455, 1.0294388287827672, -0.9087632459590531, -0.4243176209779015, 0.8625960113284511, -2.655619092974933, 1.5133280825732052, 0.553132064207584, -0.045703960660234855, 0.2205076557571733, -1.0299352833089765, -0.34994336458910474, 1.1002843382203737, 1.2980219723262212, 2.6962240525635797, -0.07392466628041514, -0.6585529668050037, -0.5142339659399888, -1.0180418752873648, -0.07785475594085076, 0.38273243001226814, -0.03424228053195387, 1.0963468456657985, -0.23421580134453654, -0.3474506524985633, -0.5812684768603252, -1.6326345262344952, -1.567767724308454, -1.1791579306376878, 1.3014280716647608, 0.8952602728899299, 1.3749640663929898, -1.3322116545945017, -1.9686246897860202, -0.6600563201340829, 0.175818953296028, 0.4986902749098275, 1.0479721559680528, 0.2842796708072146, 1.7426687806556311, -0.22260568094832048, -0.9130792180417964, -1.6812182154944335, -0.8889713580954499, 0.242117960985123, -0.8887202573536308, 0.9367424635352571, 1.412327706037443, -2.369586905226603, 0.8640523004976479, -2.2396040586617367, 0.4014990550902875, 1.2248705641936597, 0.06485610634357618, -1.2796891732042395, -0.5854312042777726, -0.2616454457109007, -0.18224478378994294, -0.20289684076666706, -0.1098827793093138, 0.2134800489101689, -1.2085736537332212, -0.2420198298702195, 1.5182611703557054, -0.38464542314251776, -0.4438360931551978, 1.0781973037142378, -2.5591846663440965, 1.1813786012882859, -0.6319037580051673, 0.16392857245258663, 0.09632135592119682, 0.9424681192203938, -0.2675947462353477, -0.6780257815644504, 1.2978457906510987, -2.36417381714118, 0.02033418170524325, -1.3479254226291204, -0.761573388256559, 2.011256681463137, -0.044595426455857026, 0.19506969715138117, -1.7815628557055914, -0.7290446587946957, 0.19655740072878491, 0.3547576931132181, 0.6168865543932788, 0.008627898917576322, 0.5270042084546597, 0.453781912635684, -1.8297404110045314, 0.03700572191014953, 0.7679024077327037, 0.5898798207345195, -0.3638588099707899, -0.8056265075393678, -1.1183119243216322, -0.13105401154141233, 1.133079879559722, -1.951804101481602, -0.659891729729498, -1.139802455426774, 0.7849575212405001, -0.5543096265713009], output: [-17.373351338172824, 0.0, -2.9147793143550498, -7.299382227996462, -12.507738907636103, -1.0091552608283472, -8.14898282968933, 5.307325780993493, -0.034750314603287924, 4.577509972521205, -7.468519860234688, -2.7826003571131217, 5.138209694069926, 2.2706200801626815, 10.580893461437023, -11.281416288223713, -2.4076277186241555, -8.076888559746063, -7.7255720166232775, -7.240220313440044, -11.221660986331635, 5.410676842717731, -0.7666763204502667, 0.23941154418990696, -1.1567394707746503, 8.29598249679744, -5.425512551243248, 4.4002233124645125, 4.01519747432619, 9.515412040403152, 17.306448507916105, -15.673102695422665, -9.901937950321809, 18.738864037440322, 19.21931801890001, 4.139374296179019, -15.995833331174268, -14.354578092775146, -1.028728623054629, -10.403106776550125, -2.312215325199424, 0.9570154846116288, 12.655685692773659, -2.588769695009625, 14.96829370833483, -0.28008251578319854, -19.11873843663264, -1.1020798388070263, 2.4267878006424475, 2.4199460094928953, -7.0321828345356066, -2.542686317569949, -7.804615781068486, 7.887570592086873, -6.609647506979987, 10.661665300030329, -13.524024198497745, 11.81604749808395, -1.07822493702058, -1.484839411655623, -11.897938634419027, -8.105846942980566, 5.44601793359646, -6.0339961403022055, -6.671236362873311, -1.7118899285267464, -0.9493422927794772, -2.6019479213685455, 0.783595826718674, 1.6178298328406964, -1.9230298563197574, -5.115726520029282, -7.036766891552735, -8.977589519991337, -6.579537548860962, -0.34069352829828503, -9.041064010465133, -6.282136934564469, 3.7117329189048838, -5.394231282736409, -11.177409462832511, 2.636948493512454, -6.698531905653285, -6.518503941570649, 8.905271814948328, 12.87201877019853, 7.090676972577299, 8.341357540310314, -4.648696854100845, 6.957035567575479, -2.241879985890022, -15.193281739525478, 1.2271924686489228, 4.616886618458665, -14.338441214676216, 14.862058456971615, 8.10221962201912, -6.558552885146747, -4.313458061854922, -6.800570539708076, -9.46963628765032, -14.706255169181524, -1.0508252086336558, 14.12155372134561, 10.30295927948794, 1.482579183504495, -9.087461443620512, 7.457473719466465, -10.18793804336269, -25.247790452763812, 4.02147223954335, -15.36278457408435, -6.138499904099753, -0.4601627652745437, -8.437677953855983, -1.9355996291504027, -16.216877073460964, -6.537035151324876, 4.1897004524761075, -2.2519743698026677, 0.6872691674164905, 12.722533287194196, 0.2882873356987652, 5.116890429328395, 20.226607459494886, 1.290674771102844, -2.800068989539933, -1.5108108590900127, 3.1832454973443687, 0.0, -2.8000689895399358, 1.5108108590900131, 20.226607459494886, -1.290674771102844, 0.2882873356987643, -5.116890429328395, 0.6872691674164899, -12.722533287194196, 4.189700452476105, 2.2519743698026673, -16.216877073460967, 6.537035151324876, -8.437677953855987, 1.9355996291504054, -6.138499904099753, 0.4601627652745437, 4.021472239543354, 15.362784574084351, -10.187938043362688, 25.247790452763812, -9.087461443620512, -7.457473719466468, 10.30295927948794, -1.4825791835044928, -1.0508252086336576, -14.121553721345606, -9.46963628765032, 14.706255169181524, -4.31345806185492, 6.800570539708074, 8.10221962201912, 6.558552885146747, -14.338441214676216, -14.862058456971612, 1.2271924686489228, -4.616886618458665, -2.241879985890021, 15.193281739525482, -4.648696854100845, -6.957035567575479, 7.090676972577298, -8.341357540310312, 8.905271814948328, -12.872018770198533, -6.698531905653283, 6.518503941570645, -11.177409462832511, -2.636948493512454, 3.7117329189048833, 5.394231282736407, -9.041064010465133, 6.282136934564471, -6.579537548860961, 0.3406935282982859, -7.036766891552734, 8.977589519991337, -1.9230298563197554, 5.115726520029284, 0.783595826718674, -1.6178298328406973, -0.9493422927794772, 2.601947921368547, -6.671236362873311, 1.7118899285267464, 5.446017933596458, 6.033996140302204, -11.897938634419027, 8.105846942980568, -1.0782249370205794, 1.4848394116556225, -13.524024198497745, -11.81604749808395, -6.609647506979986, -10.661665300030329, -7.804615781068486, -7.887570592086873, -7.032182834535609, 2.5426863175699514, 2.4267878006424475, -2.4199460094928953, -19.11873843663264, 1.1020798388070299, 14.96829370833483, 0.2800825157832003, 12.655685692773659, 2.588769695009626, -2.312215325199422, -0.9570154846116292, -1.02872862305463, 10.403106776550121, -15.995833331174268, 14.354578092775146, 19.21931801890001, -4.139374296179023, -9.901937950321809, -18.738864037440322, 17.3064485079161, 15.673102695422664, 4.01519747432619, -9.515412040403152, -5.425512551243248, -4.400223312464516, -1.1567394707746494, -8.295982496797441, -0.7666763204502649, -0.23941154418990784, -11.221660986331633, -5.410676842717731, -7.725572016623278, 7.240220313440043, -2.4076277186241555, 8.076888559746063, 10.580893461437023, 11.281416288223713, 5.138209694069921, -2.2706200801626824, -7.468519860234687, 2.782600357113122, -0.0347503146032867, -4.577509972521205, -8.148982829689329, -5.307325780993491, -12.507738907636105, 1.0091552608283463, -2.9147793143550498, 7.299382227996464] }, - fft_2_complex: { complex: true, input: [0.3076825439929962, -1.695198893547058, -1.7243221998214722, 0.2783026695251465], output: [-1.416639655828476, -1.4168962240219116, 2.0320047438144684, -1.9735015630722046] }, - fft_3_complex: { complex: true, input: [0.9383012056350708, -0.6457691788673401, -0.8741029500961304, -0.8208730816841125, -1.1462081670761108, 1.1436293125152588], output: [-1.0820099115371704, -0.32301294803619385, 0.24714778504918455, -1.042797324689853, 3.649765743393198, -0.5714972638759733] }, - fft_4_complex: { complex: true, input: [1.4125794172286987, 0.6605323553085327, 0.28041958808898926, -0.03313862159848213, -1.6822501420974731, -0.03843749687075615, -0.424235999584198, -0.8116567134857178], output: [-0.41348713636398315, -0.22270047664642334, 3.8733476512134075, -0.005685735493898392, -0.12585431337356567, 1.4668901935219765, 2.3163114674389362, 1.4036254398524761] }, - fft_5_complex: { complex: true, input: [0.17418311536312103, -1.2342511415481567, 1.477837324142456, 1.1353919506072998, -0.21928854286670685, 0.8536889553070068, -0.5614511966705322, 0.6619521379470825, -0.8825250864028931, 0.19404278695583344], output: [-0.011244386434555054, 1.6108246892690659, 1.9977527610189005, -4.495568829702767, -0.17773784354010536, -2.9033977754116895, -0.9196550557167773, -0.7794573885094949, -0.018199898511857437, 0.39634359661410157] }, - fft_7_complex: { complex: true, input: [1.2519853115081787, -0.8415557742118835, 1.4540354013442993, -2.404756546020508, -1.4876911640167236, -1.494455099105835, -0.5286277532577515, -2.204728603363037, -0.5613620281219482, -0.7104699611663818, 0.4594193398952484, -0.6865041255950928, 0.4614850580692291, 0.4871855676174164], output: [1.0492441654205322, -7.855284541845322, -0.039888223328758965, 2.182752912893351, -0.22803454984866334, -2.054368234099608, -2.9522900484683143, -1.8098744525563912, 1.2074691685152796, 2.159880155954229, 2.3732051464911823, 1.5194156322450416, 7.354191521775993, -0.03341189207448503] }, - fft_8_complex: { complex: true, input: [0.970332682132721, 0.02602463774383068, 0.019165080040693283, -0.39630618691444397, 0.030557435005903244, 0.20711614191532135, 2.1260392665863037, -0.4882599413394928, 0.38254037499427795, -0.11685442924499512, 0.10177633911371231, -0.9286617040634155, 1.243769645690918, 1.106126070022583, 0.0841875895857811, -0.24541817605495453], output: [4.9583684131503105, -0.8362335879355669, -1.6087226322558026, 0.5188463473486555, -0.5127437971532345, 0.6852134335786104, 3.193741520912889, -3.0038725843536103, 0.2960318624973297, 3.2810584288090467, 0.9862873903181653, 2.1933362079990255, 0.6698357500135899, -3.493357440456748, -0.22013705042147969, 0.8632062969612324] }, - fft_9_complex: { complex: true, input: [-0.06354460120201111, -0.2525795102119446, -1.3290656805038452, 1.239378571510315, -0.2628045976161957, -0.6161656975746155, 1.4840424060821533, -0.8743874430656433, 0.5852842926979065, -1.1418819427490234, -0.6792443990707397, -0.5788629055023193, -0.40828627347946167, 2.353774309158325, -0.34253063797950745, 1.3391598463058472, -0.5310598015785217, 0.7086358070373535], output: [-1.5472092926502228, 2.1770710349082947, -6.615882017596261, 0.6056449746473178, 2.5839119815832605, 0.5586794569181851, 3.9573314649336995, 0.4166996852974395, -0.136326996045127, -2.6370722903002877, 1.4693146501411956, -2.4984601837994824, 0.6265124219185647, 1.0866513474364776, -3.439069243722412, -5.861805640552551, 2.529515620619202, 3.879376023537104] }, - fft_16_complex: { complex: true, input: [1.0133312940597534, 0.3585434854030609, -0.6402965188026428, -1.3225007057189941, -0.4752223789691925, -0.5362467765808105, 1.2483383417129517, 0.05567394196987152, -0.041222695261240005, -0.5106480717658997, -0.34942546486854553, -1.4202176332473755, 1.2869471311569214, 1.1659399271011353, -1.326190710067749, 0.7442878484725952, 1.092625379562378, -0.6453229188919067, 0.3833552300930023, 0.4733515679836273, -0.6908642053604126, -1.0344504117965698, -0.07214371114969254, 1.7793974876403809, 0.4729478657245636, -0.7470325827598572, -0.46451860666275024, 1.7667423486709595, 0.8151687383651733, 1.0275626182556152, -0.7450098395347595, 0.052204303443431854], output: [1.5078198499977589, 1.207284428179264, -4.4806762050351825, -1.1643592216953969, -4.103445578388806, -0.030867462785639788, -0.10790451872168705, -0.5227224508100916, -1.532535444945097, -1.991386003792286, 2.03052393483773, 6.085669694760458, 7.229743305508964, -3.4075612910619277, -5.958691786804708, 2.256651932037089, 5.43960240855813, -3.050593890249729, 5.332886654634274, 3.73714542609058, -0.0764908827618398, 8.509074812562861, 0.7390405363915563, 0.02108386955870034, 4.735840562731028, -2.3431448861956596, -2.2543726824714905, -2.586308038032556, 3.6471191682769133, -1.1870411745676497, 4.064841383148511, 0.2037700224509582] }, - fft_25_complex: { complex: true, input: [-1.9739670753479004, -0.837864100933075, -1.1114044189453125, -0.5064931511878967, -0.32460978627204895, -0.5005347728729248, -0.3506036102771759, 0.0742226392030716, -0.37107688188552856, 0.5423977971076965, -1.9017586708068848, 0.6719839572906494, 2.0592153072357178, 0.39552798867225647, 0.16856394708156586, 0.19291874766349792, 0.04277687519788742, -0.04168238118290901, -1.3544977903366089, 0.5823065638542175, 0.5274385809898376, 0.40750083327293396, -0.8166235089302063, 0.9344403743743896, -0.633162260055542, 1.6156861782073975, -0.017774872481822968, -0.4030669629573822, 1.5376009941101074, -0.5414798855781555, 1.5840349197387695, 0.37068140506744385, -1.5073336362838745, -0.4143368899822235, 0.0665762796998024, 1.2245029211044312, 0.3779613673686981, -1.7612340450286865, 0.9021579623222351, 0.02208423614501953, 0.017875323072075844, 0.09636616706848145, -0.1060614064335823, -0.9343460202217102, 0.9497752785682678, -2.030526876449585, 1.145573616027832, -0.636160671710968, 1.1762216091156006, -0.4195115566253662], output: [0.08689814247190952, -1.8966175056993961, 4.5468912855738495, -1.8818154119554684, -0.5200552972284029, 1.2734341399283204, 0.4632869234518082, 5.931840244626491, -2.9589503069493865, -0.5547667441139432, -1.5273151665934335, 6.215078295347433, -15.640798930847374, 2.4457132590529254, 2.0174810051387153, 0.018505660090085918, -0.195679780600583, -1.5733085555890094, 2.608942923248818, -1.8646958677420233, -5.164085446696965, 1.34215964208527, -5.615886183114355, -0.2185132207952143, 5.847780025346529, 4.8320068942200045, -0.10823882684491926, -0.9458843955572025, -9.968247687952289, -5.8960346192185416, 1.8915989122741381, -0.7760332258366917, -5.750397961307298, 3.3450370182735623, -0.40650253504753975, -3.1697133057132096, 3.105915528359142, -7.099303168364118, -6.373188277089521, 1.2862869596061528, -4.01898105322616, -1.3412458970644465, 3.983941673212713, 2.3932716249692954, -4.4090529271307455, -8.431989160831321, -3.768856482573692, -3.025597080828077, -7.475676439572469, -11.354418102217751] }, - fft_27_complex: { complex: true, input: [0.7116051316261292, 1.037112832069397, 0.6106399893760681, 0.7972601652145386, -0.2644272744655609, -1.5348200798034668, 0.42119401693344116, -0.22403796017169952, -1.1107473373413086, -0.7211647629737854, -0.6500771641731262, 1.2391200065612793, -0.15565899014472961, -0.5251198410987854, 1.8477299213409424, 0.3672321140766144, -1.4575940370559692, -0.28738635778427124, 0.4544501006603241, 0.10830547660589218, 0.5361469388008118, -0.3605248034000397, -1.8324573040008545, 0.07127898186445236, 0.04289998486638069, 0.9799603819847107, 0.5673006772994995, 0.6624241471290588, -0.3812290132045746, -1.7817859649658203, 0.15860582888126373, -1.0848448276519775, -0.7432076334953308, 0.5504264831542969, -2.300358772277832, -0.41666755080223083, -3.017878293991089, 1.61266028881073, -0.419252872467041, -0.20492435991764069, 0.36734986305236816, -1.0211094617843628, -0.3779532313346863, 0.3455152213573456, 0.3696709871292114, -0.5317375659942627, 0.7667483687400818, -0.7377064824104309, 1.1323468685150146, -0.4188673496246338, 0.1713690608739853, 0.8521258234977722, 1.9920309782028198, 0.2819546163082123], output: [-2.560753207653761, -0.9453208297491074, 9.347935323529054, -0.9253007594026554, 3.3062934251957525, 9.624685892964905, -3.747491429172474, 6.944443936382198, 0.2846532016836547, -2.186295990939281, 1.6811782376183848, 6.6863808498799235, -6.517827112663515, 0.14943438287348965, 5.055696157497922, 1.2377621172326654, 5.652717825977259, 5.273448677870682, 5.183011380415665, -1.6221043494072749, -7.350588286919974, 3.2159710253980176, 6.887033244512073, 2.2722794132317587, 3.079426655802572, 0.7414325238817026, -3.3293146897618247, -5.438193278707893, -7.072286065035183, -1.106348191005082, -6.900230177764772, 4.314765981558758, -0.8251297022533834, -3.636347937894679, -0.8573794050807142, -0.5671438615765814, -4.513423924725758, 8.05947784599932, 2.720475531609292, -11.224192327686906, -1.8276595977578185, -0.7655023451245344, 1.4131230144639861, 4.705230182137967, -4.606202193101245, 1.2945168184514193, 14.000104083285905, 4.499731058762659, -2.1022427540436652, 2.4753477036971208, 4.917062359267286, -7.812371522652364, 7.895156658980767, 2.736259449697491] }, - fft_32_complex: { complex: true, input: [0.8450640439987183, 1.3659120798110962, -1.0280054807662964, -0.08084108680486679, 0.9755204916000366, 0.5328992605209351, -0.06296933442354202, -0.08816881477832794, -0.6170272827148438, -1.8309184312820435, -1.3914250135421753, -0.5434551239013672, 1.1874107122421265, 0.6239242553710938, 0.13674135506153107, 0.8248231410980225, 0.44601964950561523, -0.5365520715713501, -1.2611466646194458, -1.2415540218353271, 0.03862550109624863, 0.7438274025917053, -0.964236855506897, 1.1258167028427124, 0.41451361775398254, 0.16199752688407898, -0.17522190511226654, 0.2171468436717987, -2.5754177570343018, -0.09452058374881744, -1.6540249586105347, -0.7705362439155579, -0.03492758423089981, -0.2652542293071747, 0.044895902276039124, 0.3964420557022095, -0.40189823508262634, -1.0244829654693604, -0.7180095911026001, 1.4870411157608032, -0.07942715287208557, -0.15951769053936005, -1.0193690061569214, -2.5220484733581543, 0.04660886898636818, 0.8388492465019226, 1.0369007587432861, -1.2317235469818115, -0.41559332609176636, -0.9562405943870544, -0.6224948167800903, -0.6581323146820068, 0.2363770604133606, -0.2054009884595871, 2.034836530685425, 0.08647070080041885, -1.3155070543289185, -1.5467650890350342, 1.402755856513977, -0.016710011288523674, 0.732329785823822, 0.09752081334590912, 1.2334948778152466, -1.622504711151123], output: [-3.5246070064604282, -6.892655847594142, 14.091260128579158, 2.401029930407454, -2.680974992789798, 3.8916277984556755, 0.5252526044217629, 7.272547842564242, 4.888426456046666, 0.1230857616321388, 0.32227787389030693, 8.837821222991312, 6.797800467459476, -5.336196554450358, -0.3831869573992117, 3.2695703626864097, -5.256811993196607, -0.1872110292315483, -5.001142715544147, 2.250221073549945, 8.811058109636217, 5.232778335556318, -0.6512846517539874, -2.242062163272187, 9.451628634097027, 8.879343426811909, 2.0671711072014323, 9.227474643353165, -2.163253380148122, 6.008848575681659, 0.7074175020118927, 9.617801820579668, 2.4899496845901012, 2.3832117300480604, -2.562616988438002, -5.25214449133264, -0.2925401848541538, -7.40888946314626, 6.728447606715459, 8.585534132159177, -2.8502671899324317, 2.9276655593856034, 6.279407417654955, -0.03356822185923569, 0.5101090153146224, 9.679261772566992, -9.235685068269035, -3.9845762833139267, 3.263928959146142, -10.372698850929737, -7.636904664040565, -9.25411350837314, -5.141376625258181, 7.840129391031544, 1.7582723882580569, 2.5429684456901556, -1.7377452788371315, 0.002180726241851616, 2.8379890490594435, -2.0202939805697477, 0.396858681471695, 0.8400442760030362, 4.233191419326372, -5.119549879368318] }, - fft_49_complex: { complex: true, input: [-0.653186559677124, -0.9652466773986816, 1.1103789806365967, -0.8953530192375183, 0.17809812724590302, 0.7561290264129639, 1.412551999092102, 0.4937201738357544, 0.6659147143363953, -0.35966482758522034, 0.7821815013885498, 0.9256682991981506, -2.061739206314087, 1.1828033924102783, 0.8627902269363403, -1.1108970642089844, -0.38785845041275024, 0.6139956712722778, -0.1652761548757553, 1.3882287740707397, 0.014210155233740807, -0.25180017948150635, 0.04756239056587219, -0.7563911080360413, 0.8808799386024475, -0.9352231621742249, 0.09307317435741425, 0.9392328858375549, 1.9853578805923462, 0.5460475087165833, 0.2075347602367401, 0.9603745937347412, 0.04923528432846069, -0.9488556385040283, 1.8051396608352661, -0.4981226325035095, -0.5307197570800781, -0.4026987552642822, -1.1836994886398315, 0.5920459032058716, 2.3241052627563477, 0.00905652716755867, -0.3693919777870178, -0.8863218426704407, -0.03561544418334961, 0.04909980669617653, 1.7207887172698975, -0.1781470775604248, -0.02191477082669735, 0.2800677418708801, -1.0829179286956787, -0.9707691669464111, -1.5767006874084473, 1.3887768983840942, -0.1409223973751068, -0.6077256202697754, 1.3477866649627686, -0.22426994144916534, -0.15349310636520386, 1.9329805374145508, -1.1728582382202148, -0.8913836479187012, -0.7144800424575806, 0.018777834251523018, -0.22412529587745667, 1.3015447854995728, 0.35400378704071045, 1.3718682527542114, -0.11531352996826172, 0.5478003025054932, -0.9265352487564087, -1.0495789051055908, 0.16852520406246185, -2.1708903312683105, -0.6477338075637817, 1.3192451000213623, -0.5638371706008911, -0.902698278427124, 0.1049879938364029, -1.401151418685913, -0.08417209982872009, 2.0441510677337646, -0.5061149597167969, -2.023317337036133, -1.237672209739685, -0.008067949675023556, -1.2302114963531494, -1.6217567920684814, -0.24184295535087585, 0.4429032802581787, -0.04756205901503563, 0.04733582213521004, 0.34705546498298645, 0.9135802388191223, -0.790744960308075, -0.045627497136592865, 0.08297869563102722, 0.028814325109124184], output: [-0.32149941846728325, -0.011710121296346188, 1.5719012837700608, -11.9435719170496, 7.889099849425332, 2.349164232580148, 3.101234046136667, -0.9897723162237673, 2.906207608358317, -7.272673248354674, -0.28567752458921314, -7.807930436881056, -9.98287287277238, 2.170225845960604, -3.9995455705964806, -6.139724045667118, -2.6586132853906514, -7.769710705243423, 0.9131703097247483, -9.940238191191922, -8.286631830021289, 7.3493173799497, 8.661878987183604, -3.553867174039695, 11.583726594048862, 6.9528434851612655, -9.85410942809862, -6.509237172308493, 3.264008317204529, -8.95247219585599, -1.0683784894257498, -8.209758113014594, -8.459077459978726, 3.5851739864239787, -3.130647641813152, 14.882850815864773, -12.112293562127494, -12.761696026041244, 8.867540664810484, 5.336684447842124, 6.405865160930168, 4.524565520649802, 3.1540883076724198, 0.8664358168359105, -14.034517102599953, -1.5599303443084476, -9.534876375145306, -4.0920648890208, -7.072712272425977, -4.325977005204747, -5.285935735332723, -6.9699101614527, -3.845638728726241, 0.6627204864850933, 11.368533948151075, 9.810447058144453, -2.880030910522577, 3.760503532656639, 0.5451509287272396, 1.6597167211016912, -8.58285942430189, -11.129246819405175, 2.030691093396224, 1.6669154167624718, 8.748833248959015, 6.192014902704366, -7.3302180587750225, -0.08243838526629593, 5.522485457006632, 10.260949490755591, 3.972158254442996, -14.10516382807534, 3.8678504886320235, -0.06906298330718297, 3.321736134975782, -4.198418580926676, -6.9491746204757945, 3.9162914328571237, 5.508243105428264, -2.4542837759115645, 0.26676806448008605, 2.9368877862382843, -16.784439087864854, -7.618115164366894, 3.8748624559849345, -1.3062132611368762, 0.5913456904524359, -2.1117316472857373, -0.6923707074676628, 1.8519748031549472, 4.596152734943953, -1.3913510533714808, 9.06246632833908, 5.377389540572535, -6.9033275845955755, 0.20446700081688807, -3.5466927958494017, 9.661642666154053] }, - fft_64_complex: { complex: true, input: [-0.5566142201423645, -0.31118565797805786, -0.16021257638931274, 1.2795861959457397, 0.789443850517273, 1.4509670734405518, -2.3435871601104736, -0.908771276473999, -0.8936960697174072, -0.8750560879707336, -0.63246089220047, -0.6453881859779358, 1.4873902797698975, -2.564749240875244, -0.7355272769927979, 0.2437494695186615, 0.01852554641664028, 0.5734773278236389, 0.29105886816978455, 1.1027342081069946, 1.8088326454162598, -0.7475714683532715, -1.217250108718872, 2.0953369140625, -1.4750502109527588, -1.343973159790039, -0.5276563763618469, -1.297399640083313, 2.03525447845459, 1.159611701965332, 0.4579901695251465, -0.9503767490386963, 0.05310674011707306, 0.0014978965045884252, 0.18816789984703064, 0.1826970875263214, -0.8126240968704224, 0.18846499919891357, -1.0810470581054688, 0.39759552478790283, 0.6841163635253906, -0.6645811200141907, 1.4170793294906616, -0.13333140313625336, 0.4437287151813507, -2.8415510654449463, -0.07697270810604095, 0.41809648275375366, -1.0915343761444092, -2.776437282562256, -0.43745681643486023, 0.26580730080604553, 1.6740102767944336, 0.22755879163742065, -2.9241530895233154, 1.4011553525924683, 1.6825302839279175, 1.5555111169815063, -0.14711306989192963, 0.8428824543952942, -0.36785921454429626, 0.9028973579406738, 0.1373748779296875, 0.32123002409935, -0.22010713815689087, -0.599694013595581, -2.5014238357543945, -1.0242420434951782, -0.6038212180137634, -0.6158038377761841, 0.7263562083244324, 1.6927067041397095, 1.2062644958496094, 0.26875412464141846, -1.0378527641296387, -0.3995000422000885, -0.24585849046707153, -0.30057334899902344, -1.389653205871582, 0.22163736820220947, -0.07863101363182068, -0.22048908472061157, 0.9795127511024475, 0.6902763247489929, -0.20713239908218384, -1.416999340057373, -0.4947267770767212, -0.8151863813400269, -1.6725231409072876, -1.9904801845550537, 0.21500161290168762, 0.44239726662635803, -0.6723101735115051, 0.6706801056861877, -0.21685439348220825, 0.7185264825820923, -0.9003168344497681, -0.29727739095687866, -1.247776985168457, 1.110876441001892, -0.17718446254730225, -0.8177000284194946, -0.6590757966041565, -0.5754866600036621, 1.1977955102920532, 1.1977572441101074, -1.5937381982803345, -2.6281495094299316, -0.7925133109092712, 1.5867403745651245, -0.6221466660499573, 1.4161043167114258, -1.6597610712051392, 0.7671688199043274, 0.4350447356700897, 0.16583804786205292, -2.573836088180542, 0.8150725960731506, -0.6248568296432495, -1.627732753753662, 0.3362247049808502, 0.30031880736351013, 1.0735864639282227, 1.333678960800171, -1.4852718114852905, 0.8292962312698364, -0.2946455478668213, -0.4848930835723877], output: [-18.114436665549874, -1.03589254419785, -7.089877931654041, -8.136593953551714, -7.48613375743217, 3.4058307917826487, -0.11349749558287137, -7.230611270566168, -8.574244204990293, 5.7986400915728815, 5.473250096298479, 7.160363687658917, -5.13533852659519, 1.8255020562058237, 7.273548900259892, 24.690830952503084, 1.6868156269139245, 8.810813815606256, 6.6254795081932745, -4.567693073247041, -2.239049755279698, -17.516871542107324, 1.3976937653317711, -4.659023464944261, -3.498063522435509, -1.042294081352104, 10.872729327063226, -18.927297144289987, 4.267325206467068, -6.680543926630708, 6.4003803411255165, -14.418237604405501, -5.944847682490945, -10.613565055537038, -1.0402464930724309, 4.733410202647864, 10.284586844608768, 7.495884983823905, 10.978098280081051, -5.653369285851954, 17.957223243942725, 3.3651604181690082, -12.12841177766924, -2.173315060592438, -11.807254527678538, 15.743279066172082, -9.485910951122346, -5.369406024705013, -4.183827428022064, -1.3953354113002816, 5.236741461404014, -0.8655702406244625, -3.707989963955507, -7.550554154083424, -2.1556733662040837, 8.35703936882244, -1.893019665574852, 14.917386376122955, -0.5382039039277775, 2.069340044414755, 10.860091518498637, 6.278499449945484, 2.0427421023295516, 17.14233054094595, 11.975593766197562, -10.740802941727452, -12.646531404584593, 0.13425425752591735, -3.6872251499900193, -1.9392344875845375, -18.16736557788115, -20.48335434059285, 0.6875482818287537, 1.3708837169699595, 26.677807954665866, -1.8603500541565219, -6.790446289324352, -0.7636112342571075, 0.16202327721217014, 2.890628934614975, -13.405530896032472, -10.423450139391665, 3.2690754871155625, -0.13370157023242735, 17.421851422986148, -7.428197970888403, -2.696655432079798, -3.6753472703589476, -1.8393129488369242, -3.9804913300385483, -14.84575265792169, -2.1073012152618524, 1.9496145693049904, -7.632750924513978, -12.292581873003275, -3.378658179390201, -1.3949911389499903, 4.731505962205119, -4.453599350900657, 8.2927961867939, 2.890294756083995, 8.32112212455824, -7.313795240640941, 13.541967260990265, 0.3013408500166399, -22.02939466419966, -0.8879106014223499, -0.9280433153423213, -13.511110887018695, -0.9068611379695746, 13.278501353056239, 4.253543634046318, -6.101434519639574, -2.2367887703037312, -8.42198169225635, 15.996815858908311, 5.985629137333648, -11.238301003546834, 0.09940024194354224, 1.4371628357505024, 6.358283661511687, 5.197077900987281, 3.293723266284598, -9.325758113839981, 1.8329751785259507, 8.745205075152134, 0.42857376402367997, 12.425414796091186] }, - fft_81_complex: { complex: true, input: [0.16182588040828705, -2.0999834537506104, -0.5849781632423401, 0.24368105828762054, -0.6967284083366394, -0.034912507981061935, -1.1470516920089722, 0.7932613492012024, 0.7196304798126221, 1.1110419034957886, -0.939569354057312, 2.4974799156188965, -1.2485228776931763, -2.838197946548462, 0.9772579669952393, -1.0605124235153198, -0.9749694466590881, 1.0589070320129395, -0.9194549322128296, 0.9585606455802917, 0.15526491403579712, 0.8247618079185486, -1.3407961130142212, -2.523029088973999, 1.0570300817489624, 1.5672498941421509, -0.07786913961172104, -0.20268507301807404, 1.7038300037384033, 2.0308806896209717, -0.27392077445983887, -0.06376122683286667, -0.310247540473938, 0.5324293375015259, -1.7548904418945312, 0.9650161266326904, 0.41914716362953186, -0.35616472363471985, 1.2825931310653687, -2.2441608905792236, -0.5296666622161865, 0.5721330642700195, 1.0294588804244995, -0.04206320270895958, 0.15277068316936493, 0.8333802819252014, -0.858043372631073, -0.12947990000247955, 0.46606531739234924, 0.21820232272148132, 0.2710532248020172, 0.36254584789276123, -0.7573392987251282, -0.340090274810791, 1.7085819244384766, -1.3537571430206299, 0.25935328006744385, -1.2952094078063965, 0.2727315127849579, 0.7723783254623413, 0.03050963208079338, -0.06484062969684601, 0.6543181538581848, -1.2400768995285034, 0.7176415920257568, 1.6827946901321411, -0.272500604391098, 1.1401575803756714, 1.164631724357605, -1.5258969068527222, -0.1886187493801117, 0.8159262537956238, 0.6548362374305725, -1.195924997329712, 0.7484856247901917, -1.5505919456481934, 1.5596144199371338, -0.6034004092216492, -1.260884404182434, -0.6083353757858276, -0.6304160356521606, -1.0715056657791138, 0.6321176886558533, -0.11946370452642441, 0.2912154495716095, 0.014026062563061714, -0.831264078617096, -0.9964612126350403, -0.20909088850021362, -0.11727125197649002, -0.705121636390686, -0.5439452528953552, -1.5571593046188354, -1.6995549201965332, 1.7400833368301392, -0.0008625089540146291, -1.17470383644104, -0.997884213924408, -0.3725011944770813, 0.35961270332336426, -0.3637978732585907, -0.7026296854019165, 0.4852464497089386, -0.5324466824531555, -1.6679120063781738, -0.7600529193878174, -0.40924522280693054, -1.3232053518295288, 0.48746198415756226, -0.5340525507926941, -0.06851266324520111, -2.164428949356079, -0.23488061130046844, 0.4067259430885315, 0.1828349083662033, -1.640533208847046, 1.0253461599349976, -0.3487933874130249, -0.39523687958717346, 1.1609245538711548, -1.0767440795898438, 0.29041048884391785, -0.4530940651893616, -0.1469300389289856, -0.5046754479408264, -1.5347036123275757, -0.49093714356422424, 0.2856205403804779, -0.3833635449409485, 0.01342631783336401, -1.825499176979065, -2.1771950721740723, -1.1022430658340454, 1.0045795440673828, 0.35692599415779114, 0.8746539354324341, -0.22437222301959991, 1.1157619953155518, -1.7885184288024902, -0.09074169397354126, 0.9822217226028442, 1.47532320022583, -1.4025393724441528, 0.5067855715751648, 0.8213406801223755, -0.7191897630691528, 0.3869772255420685, -0.3575884699821472, -1.186098337173462, -0.10889001935720444, -1.2052828073501587, -0.4578658938407898, -0.15333758294582367, 1.691064715385437, 1.8314754962921143, -0.3512211740016937, 0.8154265880584717, -0.3477337062358856, -1.0724186897277832, -0.0016170362941920757, 0.8384286761283875, 1.601130723953247], output: [-8.58128398284316, -11.439007951354142, 0.210079195962251, 5.283435256837555, -3.729018054516872, 4.172233149410547, 6.0094247849082345, -4.867406596990031, 1.6896215297204356, 8.254567899165185, -1.5832996916764728, 6.874626142064364, 7.791926933815942, -0.1113988953646401, 6.157976194485636, 7.908498593612423, 2.1554616784825296, 5.087697970296618, -10.929793301868514, -20.248492473955267, -1.7673074913954707, 4.926997195410063, 3.7962336785471167, 9.785487339090924, 7.5509937732589805, -4.618565996292649, 1.2172725036089613, -10.455295568854787, -10.613583259389454, -12.094708219848364, -11.920648753075001, 2.7422204833456965, -14.39499933098191, -7.986984514501212, 1.4636316169286738, 3.6526472221302644, 1.7780382602070137, 0.929515572223278, -4.1263772762395785, 17.702314875380782, 3.8927335234933165, 3.3808527564005315, 20.656948077140108, -10.291866793201438, 10.429803208602253, 3.2896163180066105, 2.885996928025134, 6.240479112191121, 5.126031982460415, -15.569134216773076, -14.789597569471173, 4.411143413971, 11.01877897308899, -18.4554008501283, -13.452431942413536, -12.210080319899106, 2.7697215257400316, -10.572091571648135, -4.523764337963882, 3.9530076132464806, -1.2538076112045768, 5.196509906553922, -4.582902295841774, -2.4578802622089864, -16.85598763977452, 13.658512440269673, -3.7344804477869467, 25.005764333536174, -5.796885916344451, -3.741579209020099, 14.995322293493794, -9.021964181821255, 12.535576498170204, -3.2115394425750283, -5.316067284014389, -15.73079791413737, 5.410951763343561, -2.179490941864971, -17.59181683345424, 0.35349287922924855, -4.114985065194593, -3.6153272922914104, 5.362879738072423, -6.115564376188364, 10.568119159000045, -15.318073540754956, 4.64815841280388, -21.68501783286247, -13.224626602809812, -2.219868377234576, 4.00167448459214, 1.4272276456380588, -11.341596811743006, 4.873948561987781, 4.2861526759974895, -5.96023765608868, 3.9760305188050347, 6.566800933990912, -5.002198883369546, -2.0309709147031416, -1.2884460262093702, -12.32833011496317, 0.05382037687629326, -10.78439574018218, 1.534354885003597, -13.264704159013437, -12.280184562954279, 11.35458553480188, 9.869000609110085, -0.9969714411418238, -5.650404262545905, -11.399256472642067, -6.3488730387883825, -17.807560792326566, 17.456387259491258, -8.437210476076581, 3.5889363471995455, 1.8943033185815557, 4.734889441188521, -5.865838952657386, -1.197257968989023, -16.13996530086823, -6.139028694267207, 3.1292836318406754, 12.18828544229761, 4.610631776769463, 18.678240243897175, 13.33540206528819, 15.33449728617182, 3.2382519818121347, 7.4739674455797, 6.838902240570333, 2.105371692765212, -1.5378787198749482, -2.6243434923047406, 0.11652216064202503, 2.5272654877458214, -4.532248506023914, -1.954636472567441, -3.4146656232065977, -9.008911319126828, -11.353937437276306, -7.625555655430608, -5.250709931084791, 5.340100553320184, -17.615583941013096, -3.279927088522264, -4.17250143308863, 4.4785210214064195, 6.400344186694214, -3.7005380565285066, -1.122575913062859, -0.5038180980239875, 2.5292421600004156, -1.6061528617728151, -14.630926203339426, 4.481494466029414, -5.292535421445056, 6.4147407824254055, -8.858967182263992, -13.101978958787178, 27.791783247323977] }, - fft_125_complex: { - complex: true, - input: [0.8124353885650635, -1.2395265102386475, 0.16039574146270752, 1.9819504022598267, -0.3600437343120575, 0.5889578461647034, -0.24872885644435883, 0.7082515954971313, -0.5773665904998779, 0.13543199002742767, 0.8298076391220093, -0.4342207610607147, -0.4896481931209564, 0.8350552916526794, -0.7197877764701843, -1.5380432605743408, -0.06107468158006668, 2.201507568359375, -0.3374556005001068, -1.2920140027999878, -1.5635737180709839, 1.6503757238388062, -0.6105449795722961, 0.4370865225791931, 0.5172725319862366, -0.6922412514686584, -1.2310928106307983, -1.8327370882034302, 0.6385520100593567, -0.11756168305873871, 0.23863999545574188, 0.018904687836766243, 1.5264911651611328, -0.10709311813116074, 0.5826241374015808, -1.765297770500183, -0.6799334287643433, 0.571349561214447, -0.5346289873123169, -0.8332467079162598, -0.20369751751422882, 0.30775347352027893, -0.7439308762550354, -0.5451762676239014, 0.4147465229034424, 0.5432214736938477, 0.0078077311627566814, 0.9431089758872986, -1.0435984134674072, 0.23313158750534058, -1.209730863571167, 0.45955437421798706, -0.3265855610370636, 0.6053746342658997, -0.8725820183753967, -0.5241636633872986, 1.7610948085784912, 0.6633955836296082, -0.9714349508285522, -0.5136868357658386, 1.5349853038787842, 0.3695436716079712, 0.2814086675643921, 2.5949618816375732, 1.2379722595214844, -1.8505828380584717, 0.47188371419906616, 1.3736778497695923, 1.1676279306411743, -0.2972381114959717, -1.2804001569747925, -1.7225773334503174, -1.577651023864746, 0.3269241452217102, 0.4372701346874237, -0.9890488386154175, -1.1033475399017334, 1.9549521207809448, 0.6475730538368225, 0.5292824506759644, 0.4473706781864166, 0.6432071328163147, -0.1467706263065338, 0.20976980030536652, 0.7794570922851562, -0.31381580233573914, -0.6803262233734131, 0.011037338525056839, 0.9622517824172974, -0.8368542790412903, -0.8587225079536438, 0.633898138999939, 1.2174996137619019, 2.1725356578826904, 0.2897004187107086, -1.309606671333313, -2.598196506500244, 0.557666540145874, -1.5418102741241455, -0.8026767373085022, 0.08196965605020523, -0.002491528633981943, 1.587324857711792, 0.720113217830658, -0.7349403500556946, 1.192823052406311, 0.729076087474823, 0.2271231710910797, 0.9269818663597107, -0.006230967119336128, 0.7080751061439514, -2.3676698207855225, -1.2577210664749146, -0.24024532735347748, 1.1770211458206177, -0.5720393061637878, -1.920380711555481, 0.8346250057220459, 0.7047927975654602, 0.19386297464370728, -1.093695044517517, 0.6820136904716492, -0.2829663157463074, -0.11388272047042847, 0.6887729167938232, 0.09922466427087784, 1.250266194343567, 0.6323102712631226, 0.1848202496767044, -1.8433687686920166, 0.4263668954372406, -0.24713163077831268, 0.01820266619324684, 0.8616644144058228, -1.5792089700698853, 0.6367298364639282, -0.3927980363368988, 0.6539236307144165, 0.8779292106628418, 0.5544044971466064, -0.4019509553909302, -0.3177534341812134, 0.4690700173377991, -0.7643328905105591, -1.5016077756881714, 0.5900797843933105, 0.36968597769737244, 0.7913125157356262, -0.9097312688827515, 0.7577337622642517, 0.7302269339561462, 0.7785899639129639, -0.7808380722999573, -1.0440396070480347, 0.5710436701774597, -0.6633660197257996, -0.6982168555259705, 1.5012997388839722, 1.8683104515075684, -0.009162770584225655, 0.053628917783498764, 0.7560212016105652, 2.734218120574951, 0.8975538015365601, 0.9965337514877319, -0.9736179709434509, -1.3732835054397583, -0.22769349813461304, 0.5279285311698914, 0.9973939657211304, -1.5946539640426636, -0.3334585130214691, 0.5114824771881104, -0.8088312745094299, -0.009309413842856884, -0.3517300486564636, -1.9116069078445435, 2.348860740661621, 0.5795866250991821, 0.6854371428489685, -1.3237437009811401, 0.30641838908195496, -0.9147019386291504, -0.4391704797744751, -0.5839483141899109, 2.629678726196289, -0.1468760073184967, 0.2384973019361496, 0.9125059247016907, -0.9323630332946777, -0.12135637551546097, -0.1401946097612381, 0.6303017139434814, -0.8720086812973022, -0.4348902702331543, 0.707770824432373, 1.0431065559387207, -0.3136989176273346, -0.34615302085876465, -0.09326159954071045, 1.0725315809249878, 0.6460273265838623, 0.9376847743988037, -0.45695820450782776, -0.6338280439376831, 1.7222473621368408, -1.7862892150878906, 0.24867220222949982, -0.8356691002845764, -0.5947818756103516, 0.46319833397865295, -0.5724456310272217, -1.3741754293441772, -1.2928011417388916, -0.15125389397144318, 0.35185670852661133, 0.45909401774406433, -0.276844322681427, -0.7534619569778442, 0.17448769509792328, -0.8041709065437317, 0.7715813517570496, -0.17985635995864868, -0.3155222535133362, -0.27937594056129456, -0.6012001037597656, 1.3496509790420532, -0.07789217680692673, 0.911784827709198, 0.6547526121139526, 0.6284425854682922, -1.5268802642822266, -0.11813484132289886, -0.8698882460594177, 0.5227888822555542, -0.2165810912847519, -0.807140588760376, 1.1484020948410034, -1.4427751302719116, 0.6043649911880493, -0.525101363658905, -0.7420395612716675, -0.4712083339691162, -0.04465737193822861, -0.4328814148902893, -1.3837007284164429, -0.06250803917646408, 0.5973103046417236, 0.2701307237148285, -1.2017232179641724], - output: [-8.60566947190091, 9.047967813443393, -6.678979661059562, -8.360831755799953, 3.115231761090947, -6.697091460792556, 0.7568358822534593, 11.066622749967639, 2.8466255459388927, 6.27918164849176, 1.982200342331545, 0.3737138114008831, 8.93599812413914, 12.19410650368572, -4.5824951902858615, -4.643371981421932, 11.975504695471635, -19.79773110340414, 16.894522457707588, -11.40437941841762, -0.5369639596199498, -9.687637013191164, -2.1092885577719587, 10.925736950748426, 2.630639119388632, -2.8059934209514976, 8.536957582736084, -6.326392685852441, 19.05482126619987, 4.74321319044126, 1.7193491466752984, 17.828632923932844, 11.48731740639608, 11.824203873765583, 17.4496745991033, 14.886923392164112, -8.857144569139852, -18.45702795068501, 1.7494909136262233, -0.10810961271060027, 11.536135407131509, -12.549170934604959, -5.461911779043582, 9.93390865307337, 7.577784660577098, 11.044552867393765, -0.38872990536666596, -4.554231235610941, -9.19372645559055, 3.9372069026290077, -0.01611898707200954, -14.170353635176678, 2.977419276887061, 1.171732761405826, 9.77702001582745, 4.731009139477455, -6.457985308398847, -15.847828201326442, 9.95926759256128, -11.402544444287324, 17.96375259440102, -6.2530379131604725, 4.938790863887686, -10.135905976382034, 5.383747427255329, 6.230288801920859, 6.384329417961547, 1.3219731997515085, 3.1738925777367895, 11.969669020511425, -7.174837958584151, -14.303685689056584, 0.7454990885394415, -0.8185666838197116, 8.849555077234482, 3.9055874990272663, -2.018206936296142, -2.3395830964594415, -9.60564366177047, -5.769798601114195, 0.7520113635166212, 0.5065019274195421, -10.25514045587141, -9.637019763988052, 18.803482381840155, -2.3670620465003394, -11.199527109220814, 0.007833023989774368, 8.625619052479255, 9.951138032831594, 7.758961547592464, -1.8211300264093309, 6.718724482310631, -3.675823001520721, -6.918876012140167, -11.471658204308536, 2.467033682197071, -8.508193328034618, 14.225276109269815, 16.144776448129633, 15.927933230731913, 10.235538687304809, 5.767881945940595, -28.86811292326169, 9.669928964930417, -4.668293602235824, 14.769201279257288, 11.067653051718727, 12.761530418035363, -11.044953949483144, 4.242003110570776, -23.262991395970026, -10.145352776759976, 9.212885213669484, 6.329756726823897, -2.401697066728733, -8.54352813832675, 0.701551842851496, -15.258656469035635, -19.14394691246519, -7.522186448172975, -1.997676262113934, -11.408489749207227, 24.631586391822943, -5.098355854851496, 1.0410089604391717, 22.688569178648756, 19.097185694651152, 0.16409737757445608, -14.474036852523309, 4.7859526448704734, 19.558395170182035, 10.633635125709178, -3.0615977245039865, -6.883177607599656, -4.696650706016856, 4.286737340380929, 1.1391516077082633, -17.043528019280284, -7.2919743123329965, 18.42867500955987, 9.458517392981532, -7.23747329102956, 3.0801112402296056, -19.46748520769718, -19.760353803723497, -4.916492200532298, -11.492889148389452, 18.97908975026986, -9.61307230309599, -31.333799318924978, -6.6051904344747445, -2.610154763269807, 17.822400589161646, 5.128110541635698, 7.169102569854218, -20.704759452278893, -13.58731153366598, 9.44388682655616, -1.6262238850380326, 2.4785447765146986, 15.61067463907144, 8.206847406028755, -3.2188956590845867, 10.069966894154257, -17.811422869295264, -7.333188700567103, 7.277796245862989, -5.975213239029163, 0.0776494376891641, 4.448473434391025, -2.74774293836415, -19.167307678242555, -22.10574472079241, -0.9176460422085646, -2.698508980697669, -2.9126695148592425, -11.855087227776476, -10.63642091855646, -4.994794817859426, 8.34646291775615, 6.768483375508413, -10.605502170349608, 14.164919594071957, -12.394208891449225, 12.057338941547131, 22.90400935226583, -15.664600594740676, 1.30010074910831, -0.2199980758912563, -0.13084869289575352, 1.5514316991800943, -6.207758174655294, -21.281702562218502, 0.6785363179803934, 1.3313626670817715, -4.335780044121057, 0.5496189332641022, -12.958391630849423, -8.448073173532364, 9.262064248245114, -6.620464792278902, 6.229199284276308, 14.566232762792579, 7.969523876692342, -6.776657815077612, 12.290319767193232, 9.559089975820987, -10.33865192053783, -22.41131767867546, 2.9821554944671256, 4.62861355727904, -1.556181975825081, 15.142425835968055, -7.526775680393509, -16.871371735315304, -16.91792120984387, 10.94566759815768, 7.919149545235784, -2.721091872750706, -11.568727749623791, -6.404971618403414, 0.9048111698914159, -12.163107403321717, -23.30918866848898, 0.29599861864002963, 21.938310258693168, -2.3132157853956175, 0.7524657263569949, -5.613641198847075, -8.631493933375305, 17.933382465737296, 15.905249964584621, -0.12417131736795461, 16.840693139623014, 10.447828447732046, -15.117925469220577, -1.7000257658739502, -14.07936041053577, -12.445181510422046, 0.09178864954511656, 8.976039835817634, -3.0952547078859665, 4.42715006938438, 2.016084671443174, 1.299572020270631, -5.764752453011912, -5.767130951426552, -4.024939854949247, -6.303606982466311], - }, - fft_128_complex: { - complex: true, - input: [-0.46925589442253113, -0.17965197563171387, 1.0996962785720825, -0.38428056240081787, 0.48438164591789246, 0.4687068462371826, 0.39249294996261597, -0.3080357015132904, 0.5177378058433533, 0.6460461020469666, 2.131727933883667, -1.9881218671798706, 1.7797539234161377, -1.7341786623001099, 1.8023210763931274, 0.04056849330663681, 0.5402734875679016, -1.2557553052902222, 1.6492481231689453, 1.1959751844406128, -1.1998282670974731, -0.28071489930152893, -1.1383824348449707, -1.105458378791809, -1.3544220924377441, -0.9994213581085205, 1.0511277914047241, 1.3731906414031982, -0.366388738155365, 0.5248828530311584, -0.06661345064640045, 1.1866965293884277, -1.7126017808914185, -0.8613355755805969, -0.15631477534770966, -0.25460687279701233, -1.6851294040679932, -0.479960560798645, -1.6602188348770142, -1.8969695568084717, -1.3355588912963867, 0.2036823183298111, -0.21647731959819794, 0.027381736785173416, 1.3588712215423584, -1.916928768157959, -0.774971067905426, -0.9908143877983093, -0.5971747040748596, 0.1783454567193985, -1.5134319067001343, 0.49953287839889526, 0.6159718632698059, -0.36356979608535767, -1.0240488052368164, -0.04502468928694725, 0.4399787485599518, 1.6701726913452148, -0.29381847381591797, -0.10132483392953873, 1.382400393486023, 1.0435426235198975, 0.9063398838043213, -0.8696965575218201, 1.7439777851104736, -1.1632086038589478, -0.8023254871368408, 0.18316711485385895, 0.7868092656135559, 0.6457328796386719, 0.000458094640634954, 0.2006639540195465, 0.44469723105430603, 1.5764163732528687, 1.37369704246521, 1.4348359107971191, -0.3319808542728424, 0.4867582619190216, 1.6903849840164185, 0.09980353713035583, -0.8114499449729919, 0.21222348511219025, 1.7070233821868896, -0.06860260665416718, 0.3595496714115143, -0.22636087238788605, -0.11660676449537277, 0.35164034366607666, 0.6463969349861145, 0.7742586731910706, 2.551539897918701, -0.36875852942466736, -0.907656192779541, -1.5238984823226929, -0.17778240144252777, -1.5085883140563965, -2.1957619190216064, 0.2777234613895416, -1.9981634616851807, -0.36680006980895996, 0.43345877528190613, 0.12283071875572205, -0.7519865036010742, -0.8946448564529419, -1.0104613304138184, -0.574897289276123, -0.7528412342071533, -0.05524379014968872, -0.2588641345500946, 0.13064302504062653, -1.2371978759765625, 0.4406910538673401, -0.06988917291164398, 2.41745924949646, 0.2212488353252411, -0.9714004993438721, 0.41281431913375854, -0.5191168785095215, 0.5448367595672607, -0.5188544392585754, -0.35594210028648376, 1.31781005859375, 0.19899560511112213, 0.10419808328151703, -0.7013852000236511, 1.2463740110397339, -0.6775157451629639, -0.07904287427663803, -0.46743783354759216, 0.6186703443527222, 1.0175814628601074, 1.2864513397216797, -0.9423191547393799, -0.815204381942749, -1.4644925594329834, 0.4917009174823761, 1.0119210481643677, -1.106779932975769, -0.537906289100647, 0.09276387840509415, 2.3790464401245117, -0.18112841248512268, -0.7766230702400208, -0.4697842597961426, -0.24867388606071472, 0.350057452917099, -1.7352688312530518, 0.3294185996055603, -0.049025554209947586, 0.19478608667850494, 0.3822934031486511, 0.8966708183288574, 0.7362163662910461, 1.9381089210510254, 0.7763320207595825, -0.9195051193237305, -0.15279459953308105, 0.7260957956314087, 1.248564600944519, 0.8662853837013245, 0.22399753332138062, -0.5145770311355591, 0.6150395274162292, 0.17572005093097687, 0.1295638382434845, -0.823893129825592, 0.09608310461044312, 0.1056567132472992, -0.1421104371547699, -0.13746590912342072, 0.6133358478546143, -1.2861822843551636, 0.8356537818908691, 0.5849096775054932, 0.7742197513580322, 1.864620327949524, 0.1762249767780304, -1.6820768117904663, 0.170370414853096, -0.9834036827087402, -0.3257652521133423, 1.3807744979858398, 0.5471190810203552, -1.2435959577560425, -1.5459822416305542, -1.613089919090271, -1.5104633569717407, -0.7388973832130432, -0.3404698073863983, 2.2242629528045654, 1.101983904838562, -0.41542941331863403, 0.825286328792572, -1.7480731010437012, -0.227715402841568, -0.2249363213777542, -0.43012070655822754, -0.7993203401565552, -0.06888776272535324, 0.6460397839546204, 0.8002319931983948, 0.17660461366176605, 1.4292235374450684, -1.6437238454818726, 0.43454229831695557, -0.011055096983909607, -1.4420714378356934, -1.447640299797058, 0.16068753600120544, -0.2804417014122009, 0.3235762119293213, 1.2617326974868774, 0.1411382406949997, -0.23557768762111664, 0.13576534390449524, -1.6873000860214233, -0.42229339480400085, 0.8121863007545471, -1.0847389698028564, -1.6272295713424683, 1.8023097515106201, 0.7626906633377075, -1.3667072057724, 0.14141327142715454, 1.5778913497924805, -0.9910776615142822, -0.013187640346586704, 0.8963002562522888, 0.3190685510635376, 1.4895867109298706, 0.8450273871421814, -0.7519250512123108, 1.1641961336135864, -0.1140206977725029, -0.7162372469902039, -0.2397381216287613, 0.3938809037208557, 1.8317588567733765, -0.32195407152175903, 0.9710619449615479, -1.7239831686019897, -2.138955593109131, -1.1850391626358032, -1.1381371021270752, -0.2674075663089752, -0.7534425258636475, 1.180410385131836, -1.2349971532821655, -0.9732410311698914, -0.9838739633560181, 0.1530347764492035, 0.7869290113449097, -0.5131613612174988, 0.7393561601638794, 0.5818021893501282, -0.9010815024375916], - output: [4.6533055662875995, -11.01426088809967, 8.89589916229627, -8.67151790820412, -5.338702909401541, 1.2596635311977273, 3.5364356687085934, 7.495255571019467, 14.607353334543243, -18.087691812322902, 7.564636217194101, -31.42023502604239, -6.214070442777103, -10.959840511165702, -0.23122589156934037, -11.620785761919407, -8.17956409451724, -2.5877133314432683, 11.015937578315528, 5.780471206474331, -6.0841540885916, 20.060728028967564, 3.2708359605096224, 11.782757950154132, 0.09008114957166846, 3.524696982412627, -10.677391613915269, 0.31298678204602526, -21.35192563693417, 12.385290905482485, 2.161430892216815, 1.8976227049433234, -8.989539272795344, 13.126889442846508, 2.8499370378020865, 1.7474738916889598, 5.022029084645628, -3.2038788652839223, 5.800603925256104, -11.310381554572714, 8.70516962035894, -19.131074662654022, 1.7802965964722235, -14.806060115152377, 14.302944499531886, -15.445567609224282, -3.8462368249304397, -9.516418012771346, 13.019042335168749, -10.443622659963964, -10.883324294802648, 3.957267045122531, -0.667106901008031, -4.392642175023952, -5.891115650099352, 16.457002297798386, 8.618862097531547, 8.454290723978408, -1.5705784838263321, -2.0950451541937802, -0.8023593073701956, -3.1480197619642944, -5.620797591431048, -4.595460315016652, -5.651367928832769, -12.289105277159251, 12.268533105640746, -14.70471337060609, -2.55864085006375, 4.848469317941595, -15.477103640433441, -0.8200806702142849, 1.477227192472725, -8.856561753264552, -1.7000675285625562, 17.974609221649885, -1.8152649987658895, 2.226238832898174, 5.942983183373236, 5.062290291727308, 7.091530929787825, -2.0854528335431413, -2.2607116079192284, -0.9522421064376894, 1.9922801075296586, -4.848590729633989, 22.20475854724638, 14.139180422304625, 7.742631959368987, 8.736735176873482, -14.221452296099534, 10.498675928711396, -0.16082936982259532, -3.8946140234989644, -20.400591916908017, -26.146811938453247, 6.587325710150547, -20.584777745167045, 14.638519536239858, -1.6926103003462887, -10.208680237895118, 6.147709389485655, -1.986836416339922, 15.002313879999715, 6.5052637626549386, 2.3893876610117486, -1.4903416359002417, 2.5125845767117747, -11.629045740948362, -1.0314802432568921, -7.7204438190047515, -2.026871521184386, 5.038120875238651, 12.509857810233317, -2.8111210090417167, 6.6237223115217265, -11.560596890267984, 21.096203768560102, 2.492616415367813, 15.694305674116976, 6.98125969245643, -2.51588120579323, -23.614269801976544, -19.721211546858797, 5.821751650555415, -7.957710273512344, -2.3083909989954927, -9.024817057699233, -2.3525185122853145, 12.543071284890175, -6.633775272995389, 1.9856801362534808, -18.01947792290391, 1.6526855337166202, -25.06385984262638, 15.109040183134795, -3.952374815603415, 8.226644901530008, 18.761705127047698, 3.93250694875225, -7.442640246025839, -19.86894607874479, 12.889936646377077, 17.058248465954463, -10.38590977421429, -8.762421865432158, 21.57778103004022, 3.943591516088906, -5.804714467684743, 4.255345272797616, 6.842976031631512, 1.5538004698130026, -4.425512141506818, -4.377682130235696, -0.5744161527454539, -6.466874369118861, 2.117583592008284, 9.441789454733126, 8.94127387455451, -0.04257498982078545, -6.206626408089971, -17.798516107544632, 3.1902492140570136, -8.701598789976085, -3.6350162587702526, 11.47775543413513, 5.763743605757109, -15.507961256533623, -12.20946994071868, 15.597966311245933, -7.577596335933641, -5.783678667731976, 0.5854316506349582, 1.0524576740591431, 9.32444015560922, 14.86759124801031, -2.8564425698215246, -4.050241339325346, -14.743103022730313, 0.6982253561530176, 1.4485867743987042, 15.863950914048452, -10.345289369065991, 18.289197082863648, -15.758886684054374, -5.422633603143401, -0.983270019651755, -12.824811394399234, -2.2739348529046874, 10.113426086776306, -29.057768796230015, -23.164653851413163, -14.355517383664846, -1.9394528310513124, 13.636590068666269, 3.459140997767836, -1.3650156273028862, 4.305744984992871, 17.561184167420265, -10.355528699590693, 21.10112660094154, -8.222224503241419, 2.535762677259953, 7.713352852965743, -14.28383557691274, 25.855974989407443, 4.209799001514582, -21.52088963141282, 10.888001415630182, -13.13372692624647, 11.475712218807361, 1.8430819379630443, -4.696550665547157, -6.590021563791754, -25.66889721277428, 3.6636259509044233, 16.177010478373504, -18.738751282648302, -8.929151482851156, 1.9637629189516739, 9.362438823950344, 18.876072069444945, 15.70735011826503, -7.857995088347812, 1.9318237720996665, -16.12923916175005, -7.934748190521045, 0.06924911452461413, -14.499587783289256, -0.4727635184847867, 10.482895882033887, -9.583766230162844, -4.311441092707766, 10.174307532202345, -12.09622773401161, -11.448182124496999, -8.63443998241864, 19.56612298031517, 12.124200790573179, 13.998676498504775, 2.185864445998216, 9.654230140395715, -13.738018677296829, 23.468720582013574, 9.279088660718887, -3.9238192921918174, -13.262040984478173, 13.199249429343212, 8.476422030030909, 22.120527194597067, 30.905491552855995, -14.140763293470911, -3.076195026308723, 2.461867832774407, -1.150707769438454, -18.323349178808648], - }, + fft_2_real: { + complex: false, + input: [1.764052345967664, 0.4001572083672233], + output: [2.1642095543348874, 0.0, 1.3638951376004407, 0.0], + }, + fft_3_real: { + complex: false, + input: [0.9787379841057392, 2.240893199201458, 1.8675579901499675], + output: [5.087189173457165, 0.0, -1.0754876105699736, -0.3233177751657647, -1.0754876105699736, 0.3233177751657647], + }, + fft_4_real: { + complex: false, + input: [-0.977277879876411, 0.9500884175255894, -0.1513572082976979, -0.10321885179355784], + output: [-0.2817655224420774, 0.0, -0.8259206715787131, -1.0533072693191472, -1.9755046539061405, 0.0, -0.8259206715787131, 1.0533072693191472], + }, + fft_5_real: { + complex: false, + input: [0.41059850193837233, 0.144043571160878, 1.454273506962975, 0.7610377251469934, 0.12167501649282841], + output: [2.891628321702047, 0.0, -1.2995143733619916, -0.4287475285900295, 0.8801964673568988, 0.6461585010760772, 0.8801964673568988, -0.6461585010760772, -1.2995143733619916, 0.4287475285900295], + }, + fft_7_real: { + complex: false, + input: [0.44386323274542566, 0.33367432737426683, 1.4940790731576061, -0.20515826376580087, 0.31306770165090136, -0.8540957393017248, -2.5529898158340787], + output: [-1.0275594839734046, 0.0, -1.1794900746698127, -4.321336254238002, 0.4283827622463364, -2.2006199533718855, 2.8184083690191684, 1.0886333211894632, 2.8184083690191684, -1.0886333211894632, 0.4283827622463364, 2.2006199533718855, -1.1794900746698127, 4.321336254238002], + }, + fft_8_real: { + complex: false, + input: [0.6536185954403606, 0.8644361988595057, -0.7421650204064419, 2.2697546239876076, -1.4543656745987648, 0.04575851730144607, -0.1871838500258336, 1.5327792143584575], + output: [2.9826326049163367, 0.0, 2.1657565005584045, -0.5450316795716824, 0.1286017912738714, 2.8923391221851134, 2.050212039519846, -1.6549940203328992, -6.442824504097697, 0.0, 2.050212039519846, 1.6549940203328992, 0.1286017912738714, -2.8923391221851134, 2.1657565005584045, 0.5450316795716824], + }, + fft_9_real: { + complex: false, + input: [1.469358769900285, 0.1549474256969163, 0.37816251960217356, -0.8877857476301128, -1.980796468223927, -0.3479121493261526, 0.15634896910398005, 1.2302906807277207, 1.2023798487844113], + output: [1.3749938486352948, 0.0, 5.342425446245793, 2.975185521629713, -1.22457133247234, -0.6308804310909996, 0.41938606274358115, 1.5832577539331407, 1.3873773637166027, -0.8933243837257328, 1.3873773637166025, 0.8933243837257323, 0.41938606274358115, -1.5832577539331407, -1.2245713324723406, 0.6308804310909994, 5.342425446245792, -2.975185521629712], + }, + fft_16_real: { + complex: false, + input: [-0.3873268174079523, -0.30230275057533557, -1.0485529650670926, -1.4200179371789752, -1.7062701906250126, 1.9507753952317897, -0.5096521817516535, -0.4380743016111864, -1.2527953600499262, 0.7774903558319101, -1.6138978475579515, -0.2127402802139687, -0.8954665611936756, 0.386902497859262, -0.510805137568873, -1.180632184122412], + output: [-8.363366266001053, 0.0, -1.4796937055641797, 0.20983245542766216, -0.3454288633172973, 2.968909080470098, 2.329350352646858, -0.7633502808629014, -0.5589507973309957, -6.064330201474169, -1.3963011418461777, 1.659405903990098, 2.2686580120389164, -0.3150779061389377, 4.008518665331395, -0.6106258774446865, -7.486167856443222, 0.0, 4.008518665331395, 0.6106258774446865, 2.2686580120389164, 0.3150779061389377, -1.3963011418461777, -1.659405903990098, -0.5589507973309957, 6.064330201474169, 2.329350352646858, 0.7633502808629014, -0.3454288633172973, -2.968909080470098, -1.4796937055641797, -0.20983245542766216], + }, + fft_25_real: { + complex: false, + input: [-0.028182228338654868, 0.42833187053041766, 0.06651722238316789, 0.3024718977397814, -0.6343220936809636, -0.3627411659871381, -0.672460447775951, -0.3595531615405413, -0.813146282044454, -1.7262826023316769, 0.17742614225375283, -0.4017809362082619, -1.6301983469660446, 0.4627822555257742, -0.9072983643832422, 0.05194539579613895, 0.7290905621775369, 0.12898291075741067, 1.1394006845433007, -1.2348258203536526, 0.402341641177549, -0.6848100909403132, -0.8707971491818818, -0.5788496647644155, -0.31155253212737266], + output: [-7.327510303739734, 0.0, 1.377932222483735, 3.044921011687328, 0.06954965931518875, -5.483478593023237, 2.616166182267447, -2.0815154752919764, 0.1551657690860495, 0.5208228796162939, 0.3085012063215258, -2.1386609330432687, -0.566204290441261, 0.012329506565646242, 1.309786704987264, 3.1736358514734673, -1.0557581301013133, -2.4635866620117564, -2.579387724556426, 1.9885603872002973, 3.9572284078024613, -5.498314280945309, 0.6049416192890185, 2.7215390337882552, -2.8864443288170083, 0.6099589663561517, -2.8864443288170074, -0.6099589663561518, 0.604941619289019, -2.721539033788255, 3.9572284078024613, 5.498314280945309, -2.579387724556426, -1.988560387200297, -1.0557581301013133, 2.4635866620117564, 1.3097867049872634, -3.1736358514734673, -0.5662042904412615, -0.012329506565646575, 0.3085012063215258, 2.1386609330432687, 0.15516576908604957, -0.520822879616294, 2.6161661822674476, 2.0815154752919764, 0.06954965931518836, 5.4834785930232375, 1.377932222483735, -3.044921011687328], + }, + fft_27_real: { + complex: false, + input: [0.05616534222974544, -1.1651498407833565, 0.9008264869541871, 0.46566243973045984, -1.5362436862772237, 1.4882521937955997, 1.8958891760305832, 1.1787795711596507, -0.17992483581235091, -1.0707526215105425, 1.0544517269311366, -0.40317694697317963, 1.2224450703824274, 0.2082749780768603, 0.9766390364837128, 0.3563663971744019, 0.7065731681919482, 0.010500020720820478, 1.7858704939058352, 0.12691209270361992, 0.40198936344470165, 1.8831506970562544, -1.3477590611424464, -1.2704849984857336, 0.9693967081580112, -1.17312340511416, 1.9436211856492926], + output: [9.485150752680255, 0.0, -3.3736195725050755, -0.6576671739932465, -1.4668314059671266, -3.7345058492453083, 0.41824676938222627, 1.9663430754071836, 5.709032912166549, 5.528638987648501, 1.8435730384179874, -0.34683544424097734, -4.963689356113413, -0.5177297293983665, -5.8830486994336315, 3.207542182273314, 1.8987452089640713, 2.1505302726369, 6.603715178395637, 5.036393219486801, 1.3865285438489818, 10.275020532162417, 0.1681971482993614, 2.6656111808114664, -3.3300735021919055, -3.3923756439906483, -2.9951195195022233, 4.64643931476696, -2.9951195195022233, -4.64643931476696, -3.3300735021919055, 3.3923756439906487, 0.16819714829936183, -2.665611180811466, 1.3865285438489812, -10.275020532162415, 6.603715178395637, -5.036393219486801, 1.8987452089640697, -2.1505302726368996, -5.8830486994336315, -3.207542182273315, -4.963689356113414, 0.5177297293983658, 1.8435730384179874, 0.34683544424097734, 5.709032912166549, -5.528638987648501, 0.41824676938222605, -1.9663430754071831, -1.4668314059671266, 3.7345058492453087, -3.373619572505076, 0.6576671739932474], + }, + fft_32_real: { + complex: false, + input: [-0.41361898075974735, -0.7474548114407578, 1.9229420264803847, 1.4805147914344243, 1.8675589604265699, 0.9060446582753853, -0.8612256850547025, 1.9100649530990337, -0.2680033709513804, 0.8024563957963952, 0.947251967773748, -0.1550100930908342, 0.6140793703460803, 0.9222066715665268, 0.37642553115562943, -1.0994007905841945, 0.298238174206056, 1.3263858966870303, -0.6945678597313655, -0.14963454032767076, -0.43515355172163744, 1.8492637284793418, 0.6722947570124355, 0.40746183624111043, -0.7699160744453164, 0.5392491912918173, -0.6743326606573761, 0.03183055827435118, -0.635846078378881, 0.6764332949464997, 0.5765908166149409, -0.20829875557799488], + output: [11.014830327385901, 0.0, 1.2409883700304647, -5.276491159392059, -1.534551501998357, -4.8297133119274545, 1.336708303789408, -2.7137782948769944, -4.424467817605524, 0.843485461536611, -6.581416660337224, -0.1795449695375516, -2.514517651560188, -0.05943606215207886, -4.207556144474312, 7.447633642178459, -2.0080404448719507, -4.057057066134013, 1.4811356128841098, 3.432496305430468, 1.3934203835489032, -3.231520399869298, -4.317592600060283, 0.1993606400932625, -0.7034100876395168, 2.317901569810786, 3.7371408499766874, 5.7058540737147405, 6.345803325381664, -2.1851091826937408, 1.6157350284647216, 2.7643998907723595, -5.969395642755025, 0.0, 1.6157350284647212, -2.7643998907723604, 6.345803325381664, 2.1851091826937408, 3.737140849976688, -5.7058540737147405, -0.7034100876395168, -2.317901569810786, -4.317592600060284, -0.19936064009326138, 1.3934203835489032, 3.231520399869298, 1.4811356128841102, -3.432496305430468, -2.0080404448719507, 4.057057066134013, -4.207556144474311, -7.447633642178459, -2.514517651560188, 0.05943606215207886, -6.581416660337223, 0.1795449695375514, -4.424467817605524, -0.843485461536611, 1.3367083037894085, 2.7137782948769944, -1.534551501998357, 4.8297133119274545, 1.2409883700304651, 5.276491159392059], + }, + fft_49_real: { + complex: false, + input: [0.3960067126616453, -1.0930615087305058, -1.4912575927056055, 0.4393917012645369, 0.16667349537252904, 0.6350314368921064, 2.383144774863942, 0.9444794869904138, -0.9128222254441586, 1.117016288095853, -1.3159074105115212, -0.461584604814709, -0.06824160532463124, 1.7133427216493666, -0.7447548220484399, -0.8264385386590144, -0.0984525244254323, -0.6634782863621074, 1.126635922106507, -1.0799315083634233, -1.1474686524111024, -0.43782004474443403, -0.4980324506923049, 1.9295320538169858, 0.9494208069257608, 0.0875512413851909, -1.225435518830168, 0.8443629764015471, -1.0002153473895647, -1.5447710967776116, 1.1880297923523018, 0.3169426119248496, 0.920858823780819, 0.3187276529430212, 0.8568306119026912, -0.6510255933001469, -1.0342428417844647, 0.681594518281627, -0.8034096641738411, -0.6895497777502005, -0.45553250351734315, 0.01747915902505673, -0.35399391125348395, -1.3749512934180188, -0.6436184028328905, -2.2234031522244266, 0.6252314510271875, -1.6020576556067476, -1.1043833394284506], + output: [-7.887557633860811, 0.0, -5.014267769110843, -4.376842071326452, -2.3853231496935923, -10.851256239812793, -3.2717318894892826, -5.326849080561187, -2.7264013581332716, 1.1514977438201093, -7.652841900031138, -3.7746076991956867, 5.309279658846421, 8.814060475350379, -2.6168809007788796, 4.677691230979557, 7.6656963597889805, 3.4858425270229887, 0.2740709971425779, -4.676330054757555, -5.155232202781799, 3.7556835779660136, 5.001888119593458, -3.6198301043995045, -0.0953704694177091, -3.6318583861463845, 2.767000800593773, -2.450144252684209, -1.2540039367717049, 9.279722666945066, 3.815471636849461, -3.3563499317203815, 12.763423104544493, 0.4158128174652558, 3.995279461690892, -6.6439121988180325, 3.2201986963478983, 0.6334206695259166, 2.4912752816319514, -1.9446295959593973, 1.027982187347956, -2.8785858483639863, 1.3490313376869536, 14.471900839049741, 1.0215246227635422, -0.4420763712931004, -1.0162745955034818, -0.3075325636480104, -5.867850815975937, -1.9145668225042987, -5.867850815975937, 1.9145668225042978, -1.0162745955034822, 0.3075325636480101, 1.0215246227635417, 0.4420763712931006, 1.3490313376869536, -14.471900839049741, 1.0279821873479558, 2.878585848363986, 2.491275281631952, 1.9446295959593989, 3.2201986963478975, -0.633420669525917, 3.9952794616908927, 6.6439121988180325, 12.763423104544493, -0.41581281746525595, 3.815471636849461, 3.3563499317203815, -1.2540039367717049, -9.279722666945066, 2.767000800593772, 2.4501442526842085, -0.0953704694177091, 3.631858386146384, 5.001888119593458, 3.6198301043995045, -5.1552322027818, -3.755683577966013, 0.2740709971425781, 4.676330054757556, 7.665696359788981, -3.4858425270229905, -2.6168809007788796, -4.677691230979557, 5.30927965884642, -8.814060475350379, -7.652841900031138, 3.774607699195687, -2.7264013581332716, -1.1514977438201108, -3.2717318894892826, 5.326849080561189, -2.385323149693591, 10.851256239812795, -5.014267769110844, 4.376842071326452], + }, + fft_64_real: { + complex: false, + input: [0.052165079260974405, -0.7395629963913133, 1.5430145954067358, -1.2928569097234486, 0.26705086934918293, -0.0392828182274956, -1.1680934977411974, 0.5232766605317537, -0.1715463312222481, 0.7717905512136674, 0.8235041539637314, 2.16323594928069, 1.336527949436392, -0.3691818379424436, -0.2393791775759264, 1.0996595958871132, 0.6552637307225978, 0.640131526097592, -1.6169560443108344, -0.024326124398935636, -0.7380309092056887, 0.27992459904323824, -0.09815038964295794, 0.9101789080925919, 0.31721821519130206, 0.7863279621089762, -0.46641909673594306, -0.9444462559182504, -0.41004969320254847, -0.017020413861440594, 0.3791517355550818, 2.259308950690852, -0.04225715166064269, -0.955945000492777, -0.34598177569938643, -0.4635959746460942, 0.4814814737734622, -1.5407970144446248, 0.06326199420033171, 0.1565065379653756, 0.23218103620027578, -0.5973160689653627, -0.237921729736007, -1.4240609089825316, -0.49331988336219407, -0.5428614760167177, 0.4160500462614255, -1.1561824318219127, 0.7811981017099934, 1.4944845444913688, -2.0699850250135325, 0.42625873077810095, 0.6769080350302455, -0.637437025552229, -0.39727181432879766, -0.13288057758695562, -0.2977908794017283, -0.3090129690471222, -1.6760038063299767, 1.15233156478312, 1.079618592036821, -0.8133642592042029, -1.466424327802514, 0.5210648764527586], + output: [-1.64663603068023, 0.0, 1.505825929177829, -9.200008132744042, -2.544735819316532, -1.5339227544442826, -7.4360215710438755, -1.7984575917004604, 1.3619465096845937, 4.8456564834488915, 5.070506185450519, 4.849409603121612, 3.777805068935999, 0.17970844009281262, -5.309242543498751, -6.973066979348705, 5.944277107889729, 1.4068283873569305, -7.321235075656323, -8.916520841302049, -5.0314030969029835, 1.309770133246369, 3.721064537404591, -2.7547595090183408, 3.8878856671539617, 0.733440111196902, 5.958504522515222, -9.443751218054468, -3.743781835570924, 1.4088028338848555, 1.4543498973854279, -1.4051068307925165, 10.284222394185964, 6.362595288575114, 5.69300272333881, 2.3840261995408785, -2.344460920124865, 8.555867083200523, -10.10946038896856, -1.5537275998675408, -0.8110283215952525, 0.08930281805022311, -0.3624099573254269, 4.570396206585882, -7.6585567061471576, 9.739963432241709, -4.941569815253301, 0.6205797252271052, -7.291786374000026, -1.6649582074043856, 5.388745981879483, 1.3949577742938146, 1.4070140670266627, 11.226248771172266, 4.869069031848913, 4.140589494166391, 1.0264270218179838, 0.9000492064571382, 4.80696917175708, 0.26025510601075585, 4.725688003441722, 8.559147480463128, -1.4773429342657596, -6.3922359470118755, -4.01533581906691, 0.0, -1.4773429342657578, 6.392235947011875, 4.7256880034417215, -8.559147480463128, 4.80696917175708, -0.26025510601075497, 1.0264270218179838, -0.9000492064571382, 4.869069031848914, -4.140589494166391, 1.4070140670266635, -11.226248771172266, 5.388745981879483, -1.3949577742938146, -7.291786374000026, 1.6649582074043856, -4.941569815253302, -0.6205797252271044, -7.658556706147157, -9.739963432241707, -0.36240995732542824, -4.570396206585883, -0.8110283215952525, -0.08930281805022311, -10.109460388968563, 1.5537275998675408, -2.344460920124864, -8.555867083200523, 5.693002723338812, -2.38402619954088, 10.284222394185964, -6.362595288575114, 1.4543498973854279, 1.4051068307925176, -3.743781835570923, -1.4088028338848546, 5.95850452251522, 9.443751218054468, 3.8878856671539617, -0.733440111196902, 3.72106453740459, 2.754759509018341, -5.031403096902983, -1.309770133246368, -7.321235075656325, 8.916520841302049, 5.944277107889729, -1.4068283873569305, -5.309242543498751, 6.973066979348703, 3.777805068936, -0.1797084400928135, 5.07050618545052, -4.849409603121612, 1.3619465096845937, -4.8456564834488915, -7.4360215710438755, 1.7984575917004595, -2.544735819316533, 1.5339227544442826, 1.505825929177829, 9.200008132744042], + }, + fft_81_real: { + complex: false, + input: [-0.5757879698130661, 0.14195316332077967, -0.3193284171450952, 0.6915387510701866, 0.6947491436560059, -0.7255973784635843, -1.3833639553950554, -1.582938397335082, 0.6103793791072052, -1.188859257784029, -0.5068163542986875, -0.5963140384505081, -0.05256729626954629, -1.936279805846507, 0.18877859679382855, 0.5238910238342056, 0.08842208704466141, -0.3108861716984717, 0.09740016626878341, 0.3990463456401302, -2.77259275642665, 1.9559123082506942, 0.39009332268792646, -0.65240858238702, -0.3909533751876011, 0.49374177734918845, -0.11610393903436653, -2.0306844677814944, 2.0644928613593194, -0.11054065723247261, 1.0201727117157997, -0.6920498477843912, 1.5363770542457977, 0.28634368889227957, 0.6088438344754508, -1.0452533661469547, 1.2111452896827009, 0.6898181645347884, 1.3018462295649984, -0.6280875596415789, -0.4810271184607877, 2.303916697683942, -1.0600158227215473, -0.13594970067832082, 1.1368913626026953, 0.0977249677148556, 0.5829536797532936, -0.3994490292628752, 0.37005588784751875, -1.3065268517353166, 1.658130679618188, -0.11816404512856976, -0.6801782039968504, 0.6663830820319143, -0.4607197873885533, -1.3342584714027534, -1.3467175057975553, 0.6937731526901325, -0.1595734381462669, -0.13370155966843916, 1.0777438059762627, -1.1268258087567435, -0.7306777528648248, -0.38487980918127546, 0.094351589317074, -0.042171451290578935, -0.2868871923899076, -0.0616264020956474, -0.10730527629117469, -0.7196043885517929, -0.8129929885540773, 0.2745163577239395, -0.8909150829955279, -1.1573552591908536, -0.3122922511256933, -0.1576670161638159, 2.2567234972982093, -0.7047002758562337, 0.9432607249694948, 0.7471883342046318, -1.188944955203736], + output: [-6.019981320094997, 0.0, -11.024215089600144, -2.2296309610489784, 7.885780110732945, 6.9546439213354425, 2.1468923343095865, 2.0704996330469774, 5.223342327125926, 3.3335079943959576, 6.670733097535111, 3.059813085584419, -1.291987686750553, 0.4705982966401041, -2.6851539401562734, 0.13762863856647578, -2.2292463073335753, 1.3031715676222848, -8.459551141726745, -1.1728586245314498, -5.19534088481691, -4.1102647711302005, 4.442176173858688, -5.752409619199853, -10.978436211517131, -2.360518158521794, -8.97056708617231, -4.44028490950546, -0.7378544972090442, 0.3833473146759321, 2.41788584599411, 2.094126839887842, -0.9134777730571964, 0.6964431384964922, -1.348881203926696, 4.273283923628751, 2.6990650914215637, -0.04446227794217483, -2.8201900136765587, 10.405176121148552, 4.561086132295523, -4.8957475545660945, -9.626965987488632, 5.703554469280806, 5.118571440405241, -0.47801741518584884, 4.087758685622818, -16.982284140767902, -1.672788156348311, -8.247953480159863, -15.76267685360949, 6.557268666342, 8.45494487748653, -7.023784282842257, 0.9696988388268291, 0.6791846860727092, -3.7628962341106327, -6.731131201638246, 0.3164269710342955, -0.346532027732811, 5.824835704304008, -8.818250625842744, 5.382334715219444, -1.672571325598101, 6.240545283378912, -5.74548463471282, -6.38340050971792, 3.0747648212362937, 11.52437440252928, -1.3394713252215091, 6.27611320575086, 6.861171951341367, -10.764313909223372, -10.425524196300067, -8.17856146704236, 14.338103920769218, 4.86550855683382, 5.452700220702592, -3.278038589307964, -1.8007559828104207, 0.6670476307446598, 9.785124153822995, 0.6670476307446602, -9.785124153823, -3.2780385893079633, 1.8007559828104218, 4.865508556833819, -5.452700220702592, -8.178561467042359, -14.338103920769221, -10.764313909223372, 10.425524196300069, 6.276113205750862, -6.861171951341369, 11.524374402529276, 1.3394713252215045, -6.3834005097179185, -3.0747648212362932, 6.2405452833789115, 5.745484634712819, 5.382334715219443, 1.6725713255980996, 5.824835704304008, 8.818250625842744, 0.31642697103429596, 0.34653202773281055, -3.762896234110635, 6.731131201638245, 0.9696988388268291, -0.6791846860727092, 8.45494487748653, 7.023784282842258, -15.762676853609491, -6.557268666342001, -1.67278815634831, 8.247953480159865, 4.087758685622817, 16.982284140767902, 5.118571440405242, 0.47801741518584784, -9.626965987488635, -5.703554469280807, 4.561086132295526, 4.895747554566093, -2.82019001367656, -10.405176121148555, 2.699065091421563, 0.04446227794217439, -1.3488812039266977, -4.273283923628747, -0.9134777730571959, -0.6964431384964889, 2.417885845994109, -2.0941268398878425, -0.7378544972090446, -0.3833473146759312, -8.97056708617231, 4.4402849095054595, -10.978436211517133, 2.3605181585217934, 4.442176173858688, 5.752409619199854, -5.195340884816912, 4.1102647711302005, -8.459551141726745, 1.1728586245314485, -2.2292463073335775, -1.3031715676222801, -2.6851539401562716, -0.13762863856647434, -1.291987686750553, -0.47059829664010344, 6.670733097535112, -3.0598130855844157, 5.223342327125925, -3.3335079943959554, 2.1468923343095883, -2.0704996330469765, 7.8857801107329415, -6.9546439213354425, -11.024215089600146, 2.229630961048978], + }, + fft_125_real: { + complex: false, + input: [0.7732529774025997, -1.1838806401933177, -2.659172237996741, 0.6063195243593807, -1.7558905834377194, 0.45093446180591484, -0.6840108977372166, 1.6595507961898721, 1.068509399316009, -0.45338580385138766, -0.6878376110286823, -1.2140774030941206, -0.4409226322925914, -0.2803554951845091, -0.3646935443916854, 0.15670385527236397, 0.5785214977288784, 0.349654456993174, -0.764143923906443, -1.4377914738015785, 1.3645318481024713, -0.6894491845499376, -0.6522935999350191, -0.5211893123011109, -1.8430695501566485, -0.4779740040404867, -0.47965581400794766, 0.6203582983435125, 0.698457149107336, 0.00377088908626934, 0.9318483741143037, 0.339964983801262, -0.01568211160255477, 0.16092816829822298, -0.19065349358139935, -0.3948495140334503, -0.26773353689396645, -1.1280113314700069, 0.280441705316296, -0.9931236109295807, 0.8416312640736364, -0.24945858016094885, 0.04949498165009074, 0.49383677628095635, 0.6433144650629279, -1.5706234086334527, -0.20690367616397173, 0.8801789120807822, -1.6981058194322545, 0.3872804753950634, -2.2555642294021894, -1.0225068436356035, 0.0386305518401881, -1.6567151023219537, -0.9855107376841507, -1.4718350074635869, 1.6481349322075596, 0.16422775548733395, 0.5672902778526694, -0.2226751005151545, -0.35343174875719907, -1.6164741886510325, -0.2918373627478628, -0.7614922118116233, 0.8579239242923363, 1.1411018666575734, 1.4665787155741776, 0.852551939461232, -0.5986539369229861, -1.1158969859603944, 0.7666631816450861, 0.3562928174722889, -1.7685384506770307, 0.35548179274376907, 0.8145198224878664, 0.05892558918162996, -0.18505367100934153, -0.8076484876163557, -1.4465346995633879, 0.8002979493400275, -0.3091144447717088, -0.23346666154369272, 1.7327211869191332, 0.6845011068591904, 0.3708250012811021, 0.14206180518723566, 1.5199948607657727, 1.7195893074161945, 0.9295051114795281, 0.5822245913979243, -2.0946030712061448, 0.12372191423350658, -0.130106954193704, 0.09395322938556872, 0.9430460873225178, -2.7396771671895563, -0.5693120534701851, 0.26990435494076137, -0.4668455460527625, -1.4169061131262595, 0.8689634868967954, 0.27687190584612803, -0.9711045704444846, 0.3148172045158238, 0.8215857120497958, 0.005292646299360854, 0.8005648034309968, 0.07826017516166135, -0.39522898265435435, -1.159420516399913, -0.08593076697161273, 0.19429293804577166, 0.8758327615873309, -0.11510746848722672, 0.4574156062209908, -0.9646120137337284, -0.7826291558275251, -0.11038929902688775, -1.0546284639850139, 0.8202478373246812, 0.4631303293186071, 0.2790957643924534, 0.33890412521594454, 2.0210435614847975, -0.46886418796679563], + output: [-12.976781205595502, 0.0, -0.5336306740788994, 9.664313658528595, -2.250865418762321, -6.399236976457672, 4.984600678216093, 10.085773094158728, 4.054865444587319, 1.1220280117249106, -12.381959140729, -4.245941970428558, 11.300429673882508, -3.867369304887277, -1.924145132324413, 15.625845728307613, -3.683967210179358, 9.246098611130797, 6.678100602051274, -5.8877245671287906, -11.142505237356518, 1.2462822325294702, -4.950012182132614, 15.368429468743631, -2.7544743905540385, 7.397618288416753, 4.896491435868608, 21.15703724867305, -10.418689778983975, 1.697253164520248, 2.2518122169760852, 14.869916845413385, -5.553898084870594, -3.616357859997289, 8.688187732450988, 9.51306110875087, 1.8459884086796337, -6.750879404311422, 10.389191729731694, -8.563754760867933, 6.323482750892878, -0.814569185033468, -6.339262858307836, 12.395809458636048, -5.5325241155089815, -4.087114562924723, 5.264130311599194, 5.174155710626681, 0.07817803409444535, 4.502699566709124, -6.841988594388185, -4.209615030635216, -3.014915877040603, 0.7642671251818629, 4.979942269363006, -0.7954973970411463, -9.213201050170268, 20.281572828473664, 2.195700817994501, -2.8150917630751158, 3.66256874231074, 4.947718687255637, 2.417023995325397, 6.311630702081038, 8.910067289733103, -1.3549619066609422, -7.604782781699578, 3.2671105424317712, -0.9261608630826195, -10.26033603214804, 2.477431685753063, -0.5368669007709066, 6.588511626180253, 3.6391310126388206, -0.7696459233665145, 10.849962526266822, 0.8214214627634562, -9.665598459357405, -4.208875535725593, -8.955547690168373, 2.3998423294426914, -6.900645072282014, 10.561899982588944, -6.1497412675557115, -10.104315792100294, 11.339404667032188, 4.041967950367949, -10.290041543502431, 9.533068142456843, -5.3782354906044025, 4.12027245501964, -0.5914284618824925, 12.365588431711828, -2.701635423848076, -6.366846689539269, -5.370727636437312, 6.689816472907384, 7.334498752981377, 13.69756331474747, 3.7921571289587113, -0.2721490559996148, 0.2084376847247582, 9.275694421318246, -16.06448352473527, 1.3029488980606758, -8.03640349313321, 0.20226177706452786, 6.0912757829407465, -4.452961538391443, -15.072885312247177, -0.8419482730978101, -7.04697219502747, 12.864547051022127, -6.206422060382598, -6.149387177052475, -15.825612767912974, -0.5432046258303531, -0.24636987696629387, 0.5819230336105994, -10.272301011724789, 3.8285784744531433, 1.1829320961516245, -5.404044842034781, 0.4066057531145235, -1.2770351094581365, 7.590779935818785, -1.2770351094581383, -7.590779935818781, -5.404044842034781, -0.4066057531145224, 3.828578474453144, -1.1829320961516248, 0.5819230336105993, 10.272301011724787, -0.54320462583035, 0.2463698769662941, -6.149387177052476, 15.825612767912975, 12.864547051022129, 6.206422060382598, -0.8419482730978074, 7.04697219502747, -4.452961538391442, 15.072885312247177, 0.20226177706453008, -6.091275782940745, 1.3029488980606758, 8.03640349313321, 9.27569442131825, 16.06448352473527, -0.2721490559996148, -0.2084376847247582, 13.69756331474747, -3.792157128958717, 6.689816472907385, -7.334498752981374, -6.366846689539267, 5.370727636437312, 12.365588431711828, 2.7016354238480744, 4.120272455019638, 0.5914284618824928, 9.533068142456843, 5.378235490604402, 4.041967950367946, 10.290041543502431, -10.104315792100294, -11.339404667032191, 10.561899982588944, 6.1497412675557115, 2.399842329442692, 6.900645072282014, -4.208875535725592, 8.955547690168373, 0.8214214627634587, 9.665598459357408, -0.7696459233665118, -10.849962526266822, 6.588511626180252, -3.6391310126388214, 2.477431685753063, 0.536866900770905, -0.9261608630826198, 10.260336032148041, -7.604782781699578, -3.2671105424317712, 8.9100672897331, 1.354961906660943, 2.4170239953253976, -6.311630702081036, 3.662568742310737, -4.947718687255637, 2.1957008179945006, 2.815091763075113, -9.213201050170266, -20.281572828473664, 4.979942269363008, 0.7954973970411481, -3.0149158770406013, -0.7642671251818633, -6.841988594388185, 4.209615030635216, 0.07817803409444402, -4.502699566709122, 5.264130311599196, -5.174155710626682, -5.532524115508983, 4.087114562924725, -6.339262858307835, -12.395809458636048, 6.323482750892879, 0.8145691850334673, 10.389191729731692, 8.563754760867932, 1.845988408679628, 6.75087940431142, 8.68818773245099, -9.51306110875087, -5.553898084870594, 3.6163578599972883, 2.251812216976085, -14.869916845413385, -10.418689778983971, -1.6972531645202498, 4.896491435868606, -21.15703724867305, -2.7544743905540354, -7.397618288416755, -4.950012182132614, -15.368429468743635, -11.142505237356517, -1.2462822325294698, 6.678100602051274, 5.8877245671287906, -3.683967210179361, -9.246098611130797, -1.9241451323244125, -15.625845728307619, 11.30042967388251, 3.8673693048872773, -12.381959140729, 4.245941970428559, 4.054865444587321, -1.122028011724911, 4.984600678216097, -10.085773094158727, -2.250865418762321, 6.399236976457672, -0.5336306740788972, -9.664313658528593], + }, + fft_128_real: { + complex: false, + input: [-2.201441285500558, 0.1993001968964652, -0.050603540961665895, -0.5175190425104033, -0.9788298593587699, -0.43918952180214793, 0.18133842921782128, -0.5028167006425383, 2.4124536795437486, -0.960504381633148, -0.7931173627076716, -2.2886200400145285, 0.251484415021537, -2.01640662779976, -0.5394546333745014, -0.27567053456055696, -0.7097279658468882, 1.738872677454511, 0.9943943913154989, 1.3191368763015756, -0.8824188185499185, 1.1285940645145685, 0.4960009463439622, 0.7714059486768455, 1.0294388287827672, -0.9087632459590531, -0.4243176209779015, 0.8625960113284511, -2.655619092974933, 1.5133280825732052, 0.553132064207584, -0.045703960660234855, 0.2205076557571733, -1.0299352833089765, -0.34994336458910474, 1.1002843382203737, 1.2980219723262212, 2.6962240525635797, -0.07392466628041514, -0.6585529668050037, -0.5142339659399888, -1.0180418752873648, -0.07785475594085076, 0.38273243001226814, -0.03424228053195387, 1.0963468456657985, -0.23421580134453654, -0.3474506524985633, -0.5812684768603252, -1.6326345262344952, -1.567767724308454, -1.1791579306376878, 1.3014280716647608, 0.8952602728899299, 1.3749640663929898, -1.3322116545945017, -1.9686246897860202, -0.6600563201340829, 0.175818953296028, 0.4986902749098275, 1.0479721559680528, 0.2842796708072146, 1.7426687806556311, -0.22260568094832048, -0.9130792180417964, -1.6812182154944335, -0.8889713580954499, 0.242117960985123, -0.8887202573536308, 0.9367424635352571, 1.412327706037443, -2.369586905226603, 0.8640523004976479, -2.2396040586617367, 0.4014990550902875, 1.2248705641936597, 0.06485610634357618, -1.2796891732042395, -0.5854312042777726, -0.2616454457109007, -0.18224478378994294, -0.20289684076666706, -0.1098827793093138, 0.2134800489101689, -1.2085736537332212, -0.2420198298702195, 1.5182611703557054, -0.38464542314251776, -0.4438360931551978, 1.0781973037142378, -2.5591846663440965, 1.1813786012882859, -0.6319037580051673, 0.16392857245258663, 0.09632135592119682, 0.9424681192203938, -0.2675947462353477, -0.6780257815644504, 1.2978457906510987, -2.36417381714118, 0.02033418170524325, -1.3479254226291204, -0.761573388256559, 2.011256681463137, -0.044595426455857026, 0.19506969715138117, -1.7815628557055914, -0.7290446587946957, 0.19655740072878491, 0.3547576931132181, 0.6168865543932788, 0.008627898917576322, 0.5270042084546597, 0.453781912635684, -1.8297404110045314, 0.03700572191014953, 0.7679024077327037, 0.5898798207345195, -0.3638588099707899, -0.8056265075393678, -1.1183119243216322, -0.13105401154141233, 1.133079879559722, -1.951804101481602, -0.659891729729498, -1.139802455426774, 0.7849575212405001, -0.5543096265713009], + output: [-17.373351338172824, 0.0, -2.9147793143550498, -7.299382227996462, -12.507738907636103, -1.0091552608283472, -8.14898282968933, 5.307325780993493, -0.034750314603287924, 4.577509972521205, -7.468519860234688, -2.7826003571131217, 5.138209694069926, 2.2706200801626815, 10.580893461437023, -11.281416288223713, -2.4076277186241555, -8.076888559746063, -7.7255720166232775, -7.240220313440044, -11.221660986331635, 5.410676842717731, -0.7666763204502667, 0.23941154418990696, -1.1567394707746503, 8.29598249679744, -5.425512551243248, 4.4002233124645125, 4.01519747432619, 9.515412040403152, 17.306448507916105, -15.673102695422665, -9.901937950321809, 18.738864037440322, 19.21931801890001, 4.139374296179019, -15.995833331174268, -14.354578092775146, -1.028728623054629, -10.403106776550125, -2.312215325199424, 0.9570154846116288, 12.655685692773659, -2.588769695009625, 14.96829370833483, -0.28008251578319854, -19.11873843663264, -1.1020798388070263, 2.4267878006424475, 2.4199460094928953, -7.0321828345356066, -2.542686317569949, -7.804615781068486, 7.887570592086873, -6.609647506979987, 10.661665300030329, -13.524024198497745, 11.81604749808395, -1.07822493702058, -1.484839411655623, -11.897938634419027, -8.105846942980566, 5.44601793359646, -6.0339961403022055, -6.671236362873311, -1.7118899285267464, -0.9493422927794772, -2.6019479213685455, 0.783595826718674, 1.6178298328406964, -1.9230298563197574, -5.115726520029282, -7.036766891552735, -8.977589519991337, -6.579537548860962, -0.34069352829828503, -9.041064010465133, -6.282136934564469, 3.7117329189048838, -5.394231282736409, -11.177409462832511, 2.636948493512454, -6.698531905653285, -6.518503941570649, 8.905271814948328, 12.87201877019853, 7.090676972577299, 8.341357540310314, -4.648696854100845, 6.957035567575479, -2.241879985890022, -15.193281739525478, 1.2271924686489228, 4.616886618458665, -14.338441214676216, 14.862058456971615, 8.10221962201912, -6.558552885146747, -4.313458061854922, -6.800570539708076, -9.46963628765032, -14.706255169181524, -1.0508252086336558, 14.12155372134561, 10.30295927948794, 1.482579183504495, -9.087461443620512, 7.457473719466465, -10.18793804336269, -25.247790452763812, 4.02147223954335, -15.36278457408435, -6.138499904099753, -0.4601627652745437, -8.437677953855983, -1.9355996291504027, -16.216877073460964, -6.537035151324876, 4.1897004524761075, -2.2519743698026677, 0.6872691674164905, 12.722533287194196, 0.2882873356987652, 5.116890429328395, 20.226607459494886, 1.290674771102844, -2.800068989539933, -1.5108108590900127, 3.1832454973443687, 0.0, -2.8000689895399358, 1.5108108590900131, 20.226607459494886, -1.290674771102844, 0.2882873356987643, -5.116890429328395, 0.6872691674164899, -12.722533287194196, 4.189700452476105, 2.2519743698026673, -16.216877073460967, 6.537035151324876, -8.437677953855987, 1.9355996291504054, -6.138499904099753, 0.4601627652745437, 4.021472239543354, 15.362784574084351, -10.187938043362688, 25.247790452763812, -9.087461443620512, -7.457473719466468, 10.30295927948794, -1.4825791835044928, -1.0508252086336576, -14.121553721345606, -9.46963628765032, 14.706255169181524, -4.31345806185492, 6.800570539708074, 8.10221962201912, 6.558552885146747, -14.338441214676216, -14.862058456971612, 1.2271924686489228, -4.616886618458665, -2.241879985890021, 15.193281739525482, -4.648696854100845, -6.957035567575479, 7.090676972577298, -8.341357540310312, 8.905271814948328, -12.872018770198533, -6.698531905653283, 6.518503941570645, -11.177409462832511, -2.636948493512454, 3.7117329189048833, 5.394231282736407, -9.041064010465133, 6.282136934564471, -6.579537548860961, 0.3406935282982859, -7.036766891552734, 8.977589519991337, -1.9230298563197554, 5.115726520029284, 0.783595826718674, -1.6178298328406973, -0.9493422927794772, 2.601947921368547, -6.671236362873311, 1.7118899285267464, 5.446017933596458, 6.033996140302204, -11.897938634419027, 8.105846942980568, -1.0782249370205794, 1.4848394116556225, -13.524024198497745, -11.81604749808395, -6.609647506979986, -10.661665300030329, -7.804615781068486, -7.887570592086873, -7.032182834535609, 2.5426863175699514, 2.4267878006424475, -2.4199460094928953, -19.11873843663264, 1.1020798388070299, 14.96829370833483, 0.2800825157832003, 12.655685692773659, 2.588769695009626, -2.312215325199422, -0.9570154846116292, -1.02872862305463, 10.403106776550121, -15.995833331174268, 14.354578092775146, 19.21931801890001, -4.139374296179023, -9.901937950321809, -18.738864037440322, 17.3064485079161, 15.673102695422664, 4.01519747432619, -9.515412040403152, -5.425512551243248, -4.400223312464516, -1.1567394707746494, -8.295982496797441, -0.7666763204502649, -0.23941154418990784, -11.221660986331633, -5.410676842717731, -7.725572016623278, 7.240220313440043, -2.4076277186241555, 8.076888559746063, 10.580893461437023, 11.281416288223713, 5.138209694069921, -2.2706200801626824, -7.468519860234687, 2.782600357113122, -0.0347503146032867, -4.577509972521205, -8.148982829689329, -5.307325780993491, -12.507738907636105, 1.0091552608283463, -2.9147793143550498, 7.299382227996464], + }, + fft_2_complex: { + complex: true, + input: [0.3076825439929962, -1.695198893547058, -1.7243221998214722, 0.2783026695251465], + output: [-1.416639655828476, -1.4168962240219116, 2.0320047438144684, -1.9735015630722046], + }, + fft_3_complex: { + complex: true, + input: [0.9383012056350708, -0.6457691788673401, -0.8741029500961304, -0.8208730816841125, -1.1462081670761108, 1.1436293125152588], + output: [-1.0820099115371704, -0.32301294803619385, 0.24714778504918455, -1.042797324689853, 3.649765743393198, -0.5714972638759733], + }, + fft_4_complex: { + complex: true, + input: [1.4125794172286987, 0.6605323553085327, 0.28041958808898926, -0.03313862159848213, -1.6822501420974731, -0.03843749687075615, -0.424235999584198, -0.8116567134857178], + output: [-0.41348713636398315, -0.22270047664642334, 3.8733476512134075, -0.005685735493898392, -0.12585431337356567, 1.4668901935219765, 2.3163114674389362, 1.4036254398524761], + }, + fft_5_complex: { + complex: true, + input: [0.17418311536312103, -1.2342511415481567, 1.477837324142456, 1.1353919506072998, -0.21928854286670685, 0.8536889553070068, -0.5614511966705322, 0.6619521379470825, -0.8825250864028931, 0.19404278695583344], + output: [-0.011244386434555054, 1.6108246892690659, 1.9977527610189005, -4.495568829702767, -0.17773784354010536, -2.9033977754116895, -0.9196550557167773, -0.7794573885094949, -0.018199898511857437, 0.39634359661410157], + }, + fft_7_complex: { + complex: true, + input: [1.2519853115081787, -0.8415557742118835, 1.4540354013442993, -2.404756546020508, -1.4876911640167236, -1.494455099105835, -0.5286277532577515, -2.204728603363037, -0.5613620281219482, -0.7104699611663818, 0.4594193398952484, -0.6865041255950928, 0.4614850580692291, 0.4871855676174164], + output: [1.0492441654205322, -7.855284541845322, -0.039888223328758965, 2.182752912893351, -0.22803454984866334, -2.054368234099608, -2.9522900484683143, -1.8098744525563912, 1.2074691685152796, 2.159880155954229, 2.3732051464911823, 1.5194156322450416, 7.354191521775993, -0.03341189207448503], + }, + fft_8_complex: { + complex: true, + input: [0.970332682132721, 0.02602463774383068, 0.019165080040693283, -0.39630618691444397, 0.030557435005903244, 0.20711614191532135, 2.1260392665863037, -0.4882599413394928, 0.38254037499427795, -0.11685442924499512, 0.10177633911371231, -0.9286617040634155, 1.243769645690918, 1.106126070022583, 0.0841875895857811, -0.24541817605495453], + output: [4.9583684131503105, -0.8362335879355669, -1.6087226322558026, 0.5188463473486555, -0.5127437971532345, 0.6852134335786104, 3.193741520912889, -3.0038725843536103, 0.2960318624973297, 3.2810584288090467, 0.9862873903181653, 2.1933362079990255, 0.6698357500135899, -3.493357440456748, -0.22013705042147969, 0.8632062969612324], + }, + fft_9_complex: { + complex: true, + input: [-0.06354460120201111, -0.2525795102119446, -1.3290656805038452, 1.239378571510315, -0.2628045976161957, -0.6161656975746155, 1.4840424060821533, -0.8743874430656433, 0.5852842926979065, -1.1418819427490234, -0.6792443990707397, -0.5788629055023193, -0.40828627347946167, 2.353774309158325, -0.34253063797950745, 1.3391598463058472, -0.5310598015785217, 0.7086358070373535], + output: [-1.5472092926502228, 2.1770710349082947, -6.615882017596261, 0.6056449746473178, 2.5839119815832605, 0.5586794569181851, 3.9573314649336995, 0.4166996852974395, -0.136326996045127, -2.6370722903002877, 1.4693146501411956, -2.4984601837994824, 0.6265124219185647, 1.0866513474364776, -3.439069243722412, -5.861805640552551, 2.529515620619202, 3.879376023537104], + }, + fft_16_complex: { + complex: true, + input: [1.0133312940597534, 0.3585434854030609, -0.6402965188026428, -1.3225007057189941, -0.4752223789691925, -0.5362467765808105, 1.2483383417129517, 0.05567394196987152, -0.041222695261240005, -0.5106480717658997, -0.34942546486854553, -1.4202176332473755, 1.2869471311569214, 1.1659399271011353, -1.326190710067749, 0.7442878484725952, 1.092625379562378, -0.6453229188919067, 0.3833552300930023, 0.4733515679836273, -0.6908642053604126, -1.0344504117965698, -0.07214371114969254, 1.7793974876403809, 0.4729478657245636, -0.7470325827598572, -0.46451860666275024, 1.7667423486709595, 0.8151687383651733, 1.0275626182556152, -0.7450098395347595, 0.052204303443431854], + output: [1.5078198499977589, 1.207284428179264, -4.4806762050351825, -1.1643592216953969, -4.103445578388806, -0.030867462785639788, -0.10790451872168705, -0.5227224508100916, -1.532535444945097, -1.991386003792286, 2.03052393483773, 6.085669694760458, 7.229743305508964, -3.4075612910619277, -5.958691786804708, 2.256651932037089, 5.43960240855813, -3.050593890249729, 5.332886654634274, 3.73714542609058, -0.0764908827618398, 8.509074812562861, 0.7390405363915563, 0.02108386955870034, 4.735840562731028, -2.3431448861956596, -2.2543726824714905, -2.586308038032556, 3.6471191682769133, -1.1870411745676497, 4.064841383148511, 0.2037700224509582], + }, + fft_25_complex: { + complex: true, + input: [-1.9739670753479004, -0.837864100933075, -1.1114044189453125, -0.5064931511878967, -0.32460978627204895, -0.5005347728729248, -0.3506036102771759, 0.0742226392030716, -0.37107688188552856, 0.5423977971076965, -1.9017586708068848, 0.6719839572906494, 2.0592153072357178, 0.39552798867225647, 0.16856394708156586, 0.19291874766349792, 0.04277687519788742, -0.04168238118290901, -1.3544977903366089, 0.5823065638542175, 0.5274385809898376, 0.40750083327293396, -0.8166235089302063, 0.9344403743743896, -0.633162260055542, 1.6156861782073975, -0.017774872481822968, -0.4030669629573822, 1.5376009941101074, -0.5414798855781555, 1.5840349197387695, 0.37068140506744385, -1.5073336362838745, -0.4143368899822235, 0.0665762796998024, 1.2245029211044312, 0.3779613673686981, -1.7612340450286865, 0.9021579623222351, 0.02208423614501953, 0.017875323072075844, 0.09636616706848145, -0.1060614064335823, -0.9343460202217102, 0.9497752785682678, -2.030526876449585, 1.145573616027832, -0.636160671710968, 1.1762216091156006, -0.4195115566253662], + output: [0.08689814247190952, -1.8966175056993961, 4.5468912855738495, -1.8818154119554684, -0.5200552972284029, 1.2734341399283204, 0.4632869234518082, 5.931840244626491, -2.9589503069493865, -0.5547667441139432, -1.5273151665934335, 6.215078295347433, -15.640798930847374, 2.4457132590529254, 2.0174810051387153, 0.018505660090085918, -0.195679780600583, -1.5733085555890094, 2.608942923248818, -1.8646958677420233, -5.164085446696965, 1.34215964208527, -5.615886183114355, -0.2185132207952143, 5.847780025346529, 4.8320068942200045, -0.10823882684491926, -0.9458843955572025, -9.968247687952289, -5.8960346192185416, 1.8915989122741381, -0.7760332258366917, -5.750397961307298, 3.3450370182735623, -0.40650253504753975, -3.1697133057132096, 3.105915528359142, -7.099303168364118, -6.373188277089521, 1.2862869596061528, -4.01898105322616, -1.3412458970644465, 3.983941673212713, 2.3932716249692954, -4.4090529271307455, -8.431989160831321, -3.768856482573692, -3.025597080828077, -7.475676439572469, -11.354418102217751], + }, + fft_27_complex: { + complex: true, + input: [0.7116051316261292, 1.037112832069397, 0.6106399893760681, 0.7972601652145386, -0.2644272744655609, -1.5348200798034668, 0.42119401693344116, -0.22403796017169952, -1.1107473373413086, -0.7211647629737854, -0.6500771641731262, 1.2391200065612793, -0.15565899014472961, -0.5251198410987854, 1.8477299213409424, 0.3672321140766144, -1.4575940370559692, -0.28738635778427124, 0.4544501006603241, 0.10830547660589218, 0.5361469388008118, -0.3605248034000397, -1.8324573040008545, 0.07127898186445236, 0.04289998486638069, 0.9799603819847107, 0.5673006772994995, 0.6624241471290588, -0.3812290132045746, -1.7817859649658203, 0.15860582888126373, -1.0848448276519775, -0.7432076334953308, 0.5504264831542969, -2.300358772277832, -0.41666755080223083, -3.017878293991089, 1.61266028881073, -0.419252872467041, -0.20492435991764069, 0.36734986305236816, -1.0211094617843628, -0.3779532313346863, 0.3455152213573456, 0.3696709871292114, -0.5317375659942627, 0.7667483687400818, -0.7377064824104309, 1.1323468685150146, -0.4188673496246338, 0.1713690608739853, 0.8521258234977722, 1.9920309782028198, 0.2819546163082123], + output: [-2.560753207653761, -0.9453208297491074, 9.347935323529054, -0.9253007594026554, 3.3062934251957525, 9.624685892964905, -3.747491429172474, 6.944443936382198, 0.2846532016836547, -2.186295990939281, 1.6811782376183848, 6.6863808498799235, -6.517827112663515, 0.14943438287348965, 5.055696157497922, 1.2377621172326654, 5.652717825977259, 5.273448677870682, 5.183011380415665, -1.6221043494072749, -7.350588286919974, 3.2159710253980176, 6.887033244512073, 2.2722794132317587, 3.079426655802572, 0.7414325238817026, -3.3293146897618247, -5.438193278707893, -7.072286065035183, -1.106348191005082, -6.900230177764772, 4.314765981558758, -0.8251297022533834, -3.636347937894679, -0.8573794050807142, -0.5671438615765814, -4.513423924725758, 8.05947784599932, 2.720475531609292, -11.224192327686906, -1.8276595977578185, -0.7655023451245344, 1.4131230144639861, 4.705230182137967, -4.606202193101245, 1.2945168184514193, 14.000104083285905, 4.499731058762659, -2.1022427540436652, 2.4753477036971208, 4.917062359267286, -7.812371522652364, 7.895156658980767, 2.736259449697491], + }, + fft_32_complex: { + complex: true, + input: [0.8450640439987183, 1.3659120798110962, -1.0280054807662964, -0.08084108680486679, 0.9755204916000366, 0.5328992605209351, -0.06296933442354202, -0.08816881477832794, -0.6170272827148438, -1.8309184312820435, -1.3914250135421753, -0.5434551239013672, 1.1874107122421265, 0.6239242553710938, 0.13674135506153107, 0.8248231410980225, 0.44601964950561523, -0.5365520715713501, -1.2611466646194458, -1.2415540218353271, 0.03862550109624863, 0.7438274025917053, -0.964236855506897, 1.1258167028427124, 0.41451361775398254, 0.16199752688407898, -0.17522190511226654, 0.2171468436717987, -2.5754177570343018, -0.09452058374881744, -1.6540249586105347, -0.7705362439155579, -0.03492758423089981, -0.2652542293071747, 0.044895902276039124, 0.3964420557022095, -0.40189823508262634, -1.0244829654693604, -0.7180095911026001, 1.4870411157608032, -0.07942715287208557, -0.15951769053936005, -1.0193690061569214, -2.5220484733581543, 0.04660886898636818, 0.8388492465019226, 1.0369007587432861, -1.2317235469818115, -0.41559332609176636, -0.9562405943870544, -0.6224948167800903, -0.6581323146820068, 0.2363770604133606, -0.2054009884595871, 2.034836530685425, 0.08647070080041885, -1.3155070543289185, -1.5467650890350342, 1.402755856513977, -0.016710011288523674, 0.732329785823822, 0.09752081334590912, 1.2334948778152466, -1.622504711151123], + output: [-3.5246070064604282, -6.892655847594142, 14.091260128579158, 2.401029930407454, -2.680974992789798, 3.8916277984556755, 0.5252526044217629, 7.272547842564242, 4.888426456046666, 0.1230857616321388, 0.32227787389030693, 8.837821222991312, 6.797800467459476, -5.336196554450358, -0.3831869573992117, 3.2695703626864097, -5.256811993196607, -0.1872110292315483, -5.001142715544147, 2.250221073549945, 8.811058109636217, 5.232778335556318, -0.6512846517539874, -2.242062163272187, 9.451628634097027, 8.879343426811909, 2.0671711072014323, 9.227474643353165, -2.163253380148122, 6.008848575681659, 0.7074175020118927, 9.617801820579668, 2.4899496845901012, 2.3832117300480604, -2.562616988438002, -5.25214449133264, -0.2925401848541538, -7.40888946314626, 6.728447606715459, 8.585534132159177, -2.8502671899324317, 2.9276655593856034, 6.279407417654955, -0.03356822185923569, 0.5101090153146224, 9.679261772566992, -9.235685068269035, -3.9845762833139267, 3.263928959146142, -10.372698850929737, -7.636904664040565, -9.25411350837314, -5.141376625258181, 7.840129391031544, 1.7582723882580569, 2.5429684456901556, -1.7377452788371315, 0.002180726241851616, 2.8379890490594435, -2.0202939805697477, 0.396858681471695, 0.8400442760030362, 4.233191419326372, -5.119549879368318], + }, + fft_49_complex: { + complex: true, + input: [-0.653186559677124, -0.9652466773986816, 1.1103789806365967, -0.8953530192375183, 0.17809812724590302, 0.7561290264129639, 1.412551999092102, 0.4937201738357544, 0.6659147143363953, -0.35966482758522034, 0.7821815013885498, 0.9256682991981506, -2.061739206314087, 1.1828033924102783, 0.8627902269363403, -1.1108970642089844, -0.38785845041275024, 0.6139956712722778, -0.1652761548757553, 1.3882287740707397, 0.014210155233740807, -0.25180017948150635, 0.04756239056587219, -0.7563911080360413, 0.8808799386024475, -0.9352231621742249, 0.09307317435741425, 0.9392328858375549, 1.9853578805923462, 0.5460475087165833, 0.2075347602367401, 0.9603745937347412, 0.04923528432846069, -0.9488556385040283, 1.8051396608352661, -0.4981226325035095, -0.5307197570800781, -0.4026987552642822, -1.1836994886398315, 0.5920459032058716, 2.3241052627563477, 0.00905652716755867, -0.3693919777870178, -0.8863218426704407, -0.03561544418334961, 0.04909980669617653, 1.7207887172698975, -0.1781470775604248, -0.02191477082669735, 0.2800677418708801, -1.0829179286956787, -0.9707691669464111, -1.5767006874084473, 1.3887768983840942, -0.1409223973751068, -0.6077256202697754, 1.3477866649627686, -0.22426994144916534, -0.15349310636520386, 1.9329805374145508, -1.1728582382202148, -0.8913836479187012, -0.7144800424575806, 0.018777834251523018, -0.22412529587745667, 1.3015447854995728, 0.35400378704071045, 1.3718682527542114, -0.11531352996826172, 0.5478003025054932, -0.9265352487564087, -1.0495789051055908, 0.16852520406246185, -2.1708903312683105, -0.6477338075637817, 1.3192451000213623, -0.5638371706008911, -0.902698278427124, 0.1049879938364029, -1.401151418685913, -0.08417209982872009, 2.0441510677337646, -0.5061149597167969, -2.023317337036133, -1.237672209739685, -0.008067949675023556, -1.2302114963531494, -1.6217567920684814, -0.24184295535087585, 0.4429032802581787, -0.04756205901503563, 0.04733582213521004, 0.34705546498298645, 0.9135802388191223, -0.790744960308075, -0.045627497136592865, 0.08297869563102722, 0.028814325109124184], + output: [-0.32149941846728325, -0.011710121296346188, 1.5719012837700608, -11.9435719170496, 7.889099849425332, 2.349164232580148, 3.101234046136667, -0.9897723162237673, 2.906207608358317, -7.272673248354674, -0.28567752458921314, -7.807930436881056, -9.98287287277238, 2.170225845960604, -3.9995455705964806, -6.139724045667118, -2.6586132853906514, -7.769710705243423, 0.9131703097247483, -9.940238191191922, -8.286631830021289, 7.3493173799497, 8.661878987183604, -3.553867174039695, 11.583726594048862, 6.9528434851612655, -9.85410942809862, -6.509237172308493, 3.264008317204529, -8.95247219585599, -1.0683784894257498, -8.209758113014594, -8.459077459978726, 3.5851739864239787, -3.130647641813152, 14.882850815864773, -12.112293562127494, -12.761696026041244, 8.867540664810484, 5.336684447842124, 6.405865160930168, 4.524565520649802, 3.1540883076724198, 0.8664358168359105, -14.034517102599953, -1.5599303443084476, -9.534876375145306, -4.0920648890208, -7.072712272425977, -4.325977005204747, -5.285935735332723, -6.9699101614527, -3.845638728726241, 0.6627204864850933, 11.368533948151075, 9.810447058144453, -2.880030910522577, 3.760503532656639, 0.5451509287272396, 1.6597167211016912, -8.58285942430189, -11.129246819405175, 2.030691093396224, 1.6669154167624718, 8.748833248959015, 6.192014902704366, -7.3302180587750225, -0.08243838526629593, 5.522485457006632, 10.260949490755591, 3.972158254442996, -14.10516382807534, 3.8678504886320235, -0.06906298330718297, 3.321736134975782, -4.198418580926676, -6.9491746204757945, 3.9162914328571237, 5.508243105428264, -2.4542837759115645, 0.26676806448008605, 2.9368877862382843, -16.784439087864854, -7.618115164366894, 3.8748624559849345, -1.3062132611368762, 0.5913456904524359, -2.1117316472857373, -0.6923707074676628, 1.8519748031549472, 4.596152734943953, -1.3913510533714808, 9.06246632833908, 5.377389540572535, -6.9033275845955755, 0.20446700081688807, -3.5466927958494017, 9.661642666154053], + }, + fft_64_complex: { + complex: true, + input: [-0.5566142201423645, -0.31118565797805786, -0.16021257638931274, 1.2795861959457397, 0.789443850517273, 1.4509670734405518, -2.3435871601104736, -0.908771276473999, -0.8936960697174072, -0.8750560879707336, -0.63246089220047, -0.6453881859779358, 1.4873902797698975, -2.564749240875244, -0.7355272769927979, 0.2437494695186615, 0.01852554641664028, 0.5734773278236389, 0.29105886816978455, 1.1027342081069946, 1.8088326454162598, -0.7475714683532715, -1.217250108718872, 2.0953369140625, -1.4750502109527588, -1.343973159790039, -0.5276563763618469, -1.297399640083313, 2.03525447845459, 1.159611701965332, 0.4579901695251465, -0.9503767490386963, 0.05310674011707306, 0.0014978965045884252, 0.18816789984703064, 0.1826970875263214, -0.8126240968704224, 0.18846499919891357, -1.0810470581054688, 0.39759552478790283, 0.6841163635253906, -0.6645811200141907, 1.4170793294906616, -0.13333140313625336, 0.4437287151813507, -2.8415510654449463, -0.07697270810604095, 0.41809648275375366, -1.0915343761444092, -2.776437282562256, -0.43745681643486023, 0.26580730080604553, 1.6740102767944336, 0.22755879163742065, -2.9241530895233154, 1.4011553525924683, 1.6825302839279175, 1.5555111169815063, -0.14711306989192963, 0.8428824543952942, -0.36785921454429626, 0.9028973579406738, 0.1373748779296875, 0.32123002409935, -0.22010713815689087, -0.599694013595581, -2.5014238357543945, -1.0242420434951782, -0.6038212180137634, -0.6158038377761841, 0.7263562083244324, 1.6927067041397095, 1.2062644958496094, 0.26875412464141846, -1.0378527641296387, -0.3995000422000885, -0.24585849046707153, -0.30057334899902344, -1.389653205871582, 0.22163736820220947, -0.07863101363182068, -0.22048908472061157, 0.9795127511024475, 0.6902763247489929, -0.20713239908218384, -1.416999340057373, -0.4947267770767212, -0.8151863813400269, -1.6725231409072876, -1.9904801845550537, 0.21500161290168762, 0.44239726662635803, -0.6723101735115051, 0.6706801056861877, -0.21685439348220825, 0.7185264825820923, -0.9003168344497681, -0.29727739095687866, -1.247776985168457, 1.110876441001892, -0.17718446254730225, -0.8177000284194946, -0.6590757966041565, -0.5754866600036621, 1.1977955102920532, 1.1977572441101074, -1.5937381982803345, -2.6281495094299316, -0.7925133109092712, 1.5867403745651245, -0.6221466660499573, 1.4161043167114258, -1.6597610712051392, 0.7671688199043274, 0.4350447356700897, 0.16583804786205292, -2.573836088180542, 0.8150725960731506, -0.6248568296432495, -1.627732753753662, 0.3362247049808502, 0.30031880736351013, 1.0735864639282227, 1.333678960800171, -1.4852718114852905, 0.8292962312698364, -0.2946455478668213, -0.4848930835723877], + output: [-18.114436665549874, -1.03589254419785, -7.089877931654041, -8.136593953551714, -7.48613375743217, 3.4058307917826487, -0.11349749558287137, -7.230611270566168, -8.574244204990293, 5.7986400915728815, 5.473250096298479, 7.160363687658917, -5.13533852659519, 1.8255020562058237, 7.273548900259892, 24.690830952503084, 1.6868156269139245, 8.810813815606256, 6.6254795081932745, -4.567693073247041, -2.239049755279698, -17.516871542107324, 1.3976937653317711, -4.659023464944261, -3.498063522435509, -1.042294081352104, 10.872729327063226, -18.927297144289987, 4.267325206467068, -6.680543926630708, 6.4003803411255165, -14.418237604405501, -5.944847682490945, -10.613565055537038, -1.0402464930724309, 4.733410202647864, 10.284586844608768, 7.495884983823905, 10.978098280081051, -5.653369285851954, 17.957223243942725, 3.3651604181690082, -12.12841177766924, -2.173315060592438, -11.807254527678538, 15.743279066172082, -9.485910951122346, -5.369406024705013, -4.183827428022064, -1.3953354113002816, 5.236741461404014, -0.8655702406244625, -3.707989963955507, -7.550554154083424, -2.1556733662040837, 8.35703936882244, -1.893019665574852, 14.917386376122955, -0.5382039039277775, 2.069340044414755, 10.860091518498637, 6.278499449945484, 2.0427421023295516, 17.14233054094595, 11.975593766197562, -10.740802941727452, -12.646531404584593, 0.13425425752591735, -3.6872251499900193, -1.9392344875845375, -18.16736557788115, -20.48335434059285, 0.6875482818287537, 1.3708837169699595, 26.677807954665866, -1.8603500541565219, -6.790446289324352, -0.7636112342571075, 0.16202327721217014, 2.890628934614975, -13.405530896032472, -10.423450139391665, 3.2690754871155625, -0.13370157023242735, 17.421851422986148, -7.428197970888403, -2.696655432079798, -3.6753472703589476, -1.8393129488369242, -3.9804913300385483, -14.84575265792169, -2.1073012152618524, 1.9496145693049904, -7.632750924513978, -12.292581873003275, -3.378658179390201, -1.3949911389499903, 4.731505962205119, -4.453599350900657, 8.2927961867939, 2.890294756083995, 8.32112212455824, -7.313795240640941, 13.541967260990265, 0.3013408500166399, -22.02939466419966, -0.8879106014223499, -0.9280433153423213, -13.511110887018695, -0.9068611379695746, 13.278501353056239, 4.253543634046318, -6.101434519639574, -2.2367887703037312, -8.42198169225635, 15.996815858908311, 5.985629137333648, -11.238301003546834, 0.09940024194354224, 1.4371628357505024, 6.358283661511687, 5.197077900987281, 3.293723266284598, -9.325758113839981, 1.8329751785259507, 8.745205075152134, 0.42857376402367997, 12.425414796091186], + }, + fft_81_complex: { + complex: true, + input: [0.16182588040828705, -2.0999834537506104, -0.5849781632423401, 0.24368105828762054, -0.6967284083366394, -0.034912507981061935, -1.1470516920089722, 0.7932613492012024, 0.7196304798126221, 1.1110419034957886, -0.939569354057312, 2.4974799156188965, -1.2485228776931763, -2.838197946548462, 0.9772579669952393, -1.0605124235153198, -0.9749694466590881, 1.0589070320129395, -0.9194549322128296, 0.9585606455802917, 0.15526491403579712, 0.8247618079185486, -1.3407961130142212, -2.523029088973999, 1.0570300817489624, 1.5672498941421509, -0.07786913961172104, -0.20268507301807404, 1.7038300037384033, 2.0308806896209717, -0.27392077445983887, -0.06376122683286667, -0.310247540473938, 0.5324293375015259, -1.7548904418945312, 0.9650161266326904, 0.41914716362953186, -0.35616472363471985, 1.2825931310653687, -2.2441608905792236, -0.5296666622161865, 0.5721330642700195, 1.0294588804244995, -0.04206320270895958, 0.15277068316936493, 0.8333802819252014, -0.858043372631073, -0.12947990000247955, 0.46606531739234924, 0.21820232272148132, 0.2710532248020172, 0.36254584789276123, -0.7573392987251282, -0.340090274810791, 1.7085819244384766, -1.3537571430206299, 0.25935328006744385, -1.2952094078063965, 0.2727315127849579, 0.7723783254623413, 0.03050963208079338, -0.06484062969684601, 0.6543181538581848, -1.2400768995285034, 0.7176415920257568, 1.6827946901321411, -0.272500604391098, 1.1401575803756714, 1.164631724357605, -1.5258969068527222, -0.1886187493801117, 0.8159262537956238, 0.6548362374305725, -1.195924997329712, 0.7484856247901917, -1.5505919456481934, 1.5596144199371338, -0.6034004092216492, -1.260884404182434, -0.6083353757858276, -0.6304160356521606, -1.0715056657791138, 0.6321176886558533, -0.11946370452642441, 0.2912154495716095, 0.014026062563061714, -0.831264078617096, -0.9964612126350403, -0.20909088850021362, -0.11727125197649002, -0.705121636390686, -0.5439452528953552, -1.5571593046188354, -1.6995549201965332, 1.7400833368301392, -0.0008625089540146291, -1.17470383644104, -0.997884213924408, -0.3725011944770813, 0.35961270332336426, -0.3637978732585907, -0.7026296854019165, 0.4852464497089386, -0.5324466824531555, -1.6679120063781738, -0.7600529193878174, -0.40924522280693054, -1.3232053518295288, 0.48746198415756226, -0.5340525507926941, -0.06851266324520111, -2.164428949356079, -0.23488061130046844, 0.4067259430885315, 0.1828349083662033, -1.640533208847046, 1.0253461599349976, -0.3487933874130249, -0.39523687958717346, 1.1609245538711548, -1.0767440795898438, 0.29041048884391785, -0.4530940651893616, -0.1469300389289856, -0.5046754479408264, -1.5347036123275757, -0.49093714356422424, 0.2856205403804779, -0.3833635449409485, 0.01342631783336401, -1.825499176979065, -2.1771950721740723, -1.1022430658340454, 1.0045795440673828, 0.35692599415779114, 0.8746539354324341, -0.22437222301959991, 1.1157619953155518, -1.7885184288024902, -0.09074169397354126, 0.9822217226028442, 1.47532320022583, -1.4025393724441528, 0.5067855715751648, 0.8213406801223755, -0.7191897630691528, 0.3869772255420685, -0.3575884699821472, -1.186098337173462, -0.10889001935720444, -1.2052828073501587, -0.4578658938407898, -0.15333758294582367, 1.691064715385437, 1.8314754962921143, -0.3512211740016937, 0.8154265880584717, -0.3477337062358856, -1.0724186897277832, -0.0016170362941920757, 0.8384286761283875, 1.601130723953247], + output: [-8.58128398284316, -11.439007951354142, 0.210079195962251, 5.283435256837555, -3.729018054516872, 4.172233149410547, 6.0094247849082345, -4.867406596990031, 1.6896215297204356, 8.254567899165185, -1.5832996916764728, 6.874626142064364, 7.791926933815942, -0.1113988953646401, 6.157976194485636, 7.908498593612423, 2.1554616784825296, 5.087697970296618, -10.929793301868514, -20.248492473955267, -1.7673074913954707, 4.926997195410063, 3.7962336785471167, 9.785487339090924, 7.5509937732589805, -4.618565996292649, 1.2172725036089613, -10.455295568854787, -10.613583259389454, -12.094708219848364, -11.920648753075001, 2.7422204833456965, -14.39499933098191, -7.986984514501212, 1.4636316169286738, 3.6526472221302644, 1.7780382602070137, 0.929515572223278, -4.1263772762395785, 17.702314875380782, 3.8927335234933165, 3.3808527564005315, 20.656948077140108, -10.291866793201438, 10.429803208602253, 3.2896163180066105, 2.885996928025134, 6.240479112191121, 5.126031982460415, -15.569134216773076, -14.789597569471173, 4.411143413971, 11.01877897308899, -18.4554008501283, -13.452431942413536, -12.210080319899106, 2.7697215257400316, -10.572091571648135, -4.523764337963882, 3.9530076132464806, -1.2538076112045768, 5.196509906553922, -4.582902295841774, -2.4578802622089864, -16.85598763977452, 13.658512440269673, -3.7344804477869467, 25.005764333536174, -5.796885916344451, -3.741579209020099, 14.995322293493794, -9.021964181821255, 12.535576498170204, -3.2115394425750283, -5.316067284014389, -15.73079791413737, 5.410951763343561, -2.179490941864971, -17.59181683345424, 0.35349287922924855, -4.114985065194593, -3.6153272922914104, 5.362879738072423, -6.115564376188364, 10.568119159000045, -15.318073540754956, 4.64815841280388, -21.68501783286247, -13.224626602809812, -2.219868377234576, 4.00167448459214, 1.4272276456380588, -11.341596811743006, 4.873948561987781, 4.2861526759974895, -5.96023765608868, 3.9760305188050347, 6.566800933990912, -5.002198883369546, -2.0309709147031416, -1.2884460262093702, -12.32833011496317, 0.05382037687629326, -10.78439574018218, 1.534354885003597, -13.264704159013437, -12.280184562954279, 11.35458553480188, 9.869000609110085, -0.9969714411418238, -5.650404262545905, -11.399256472642067, -6.3488730387883825, -17.807560792326566, 17.456387259491258, -8.437210476076581, 3.5889363471995455, 1.8943033185815557, 4.734889441188521, -5.865838952657386, -1.197257968989023, -16.13996530086823, -6.139028694267207, 3.1292836318406754, 12.18828544229761, 4.610631776769463, 18.678240243897175, 13.33540206528819, 15.33449728617182, 3.2382519818121347, 7.4739674455797, 6.838902240570333, 2.105371692765212, -1.5378787198749482, -2.6243434923047406, 0.11652216064202503, 2.5272654877458214, -4.532248506023914, -1.954636472567441, -3.4146656232065977, -9.008911319126828, -11.353937437276306, -7.625555655430608, -5.250709931084791, 5.340100553320184, -17.615583941013096, -3.279927088522264, -4.17250143308863, 4.4785210214064195, 6.400344186694214, -3.7005380565285066, -1.122575913062859, -0.5038180980239875, 2.5292421600004156, -1.6061528617728151, -14.630926203339426, 4.481494466029414, -5.292535421445056, 6.4147407824254055, -8.858967182263992, -13.101978958787178, 27.791783247323977], + }, + fft_125_complex: { + complex: true, + input: [0.8124353885650635, -1.2395265102386475, 0.16039574146270752, 1.9819504022598267, -0.3600437343120575, 0.5889578461647034, -0.24872885644435883, 0.7082515954971313, -0.5773665904998779, 0.13543199002742767, 0.8298076391220093, -0.4342207610607147, -0.4896481931209564, 0.8350552916526794, -0.7197877764701843, -1.5380432605743408, -0.06107468158006668, 2.201507568359375, -0.3374556005001068, -1.2920140027999878, -1.5635737180709839, 1.6503757238388062, -0.6105449795722961, 0.4370865225791931, 0.5172725319862366, -0.6922412514686584, -1.2310928106307983, -1.8327370882034302, 0.6385520100593567, -0.11756168305873871, 0.23863999545574188, 0.018904687836766243, 1.5264911651611328, -0.10709311813116074, 0.5826241374015808, -1.765297770500183, -0.6799334287643433, 0.571349561214447, -0.5346289873123169, -0.8332467079162598, -0.20369751751422882, 0.30775347352027893, -0.7439308762550354, -0.5451762676239014, 0.4147465229034424, 0.5432214736938477, 0.0078077311627566814, 0.9431089758872986, -1.0435984134674072, 0.23313158750534058, -1.209730863571167, 0.45955437421798706, -0.3265855610370636, 0.6053746342658997, -0.8725820183753967, -0.5241636633872986, 1.7610948085784912, 0.6633955836296082, -0.9714349508285522, -0.5136868357658386, 1.5349853038787842, 0.3695436716079712, 0.2814086675643921, 2.5949618816375732, 1.2379722595214844, -1.8505828380584717, 0.47188371419906616, 1.3736778497695923, 1.1676279306411743, -0.2972381114959717, -1.2804001569747925, -1.7225773334503174, -1.577651023864746, 0.3269241452217102, 0.4372701346874237, -0.9890488386154175, -1.1033475399017334, 1.9549521207809448, 0.6475730538368225, 0.5292824506759644, 0.4473706781864166, 0.6432071328163147, -0.1467706263065338, 0.20976980030536652, 0.7794570922851562, -0.31381580233573914, -0.6803262233734131, 0.011037338525056839, 0.9622517824172974, -0.8368542790412903, -0.8587225079536438, 0.633898138999939, 1.2174996137619019, 2.1725356578826904, 0.2897004187107086, -1.309606671333313, -2.598196506500244, 0.557666540145874, -1.5418102741241455, -0.8026767373085022, 0.08196965605020523, -0.002491528633981943, 1.587324857711792, 0.720113217830658, -0.7349403500556946, 1.192823052406311, 0.729076087474823, 0.2271231710910797, 0.9269818663597107, -0.006230967119336128, 0.7080751061439514, -2.3676698207855225, -1.2577210664749146, -0.24024532735347748, 1.1770211458206177, -0.5720393061637878, -1.920380711555481, 0.8346250057220459, 0.7047927975654602, 0.19386297464370728, -1.093695044517517, 0.6820136904716492, -0.2829663157463074, -0.11388272047042847, 0.6887729167938232, 0.09922466427087784, 1.250266194343567, 0.6323102712631226, 0.1848202496767044, -1.8433687686920166, 0.4263668954372406, -0.24713163077831268, 0.01820266619324684, 0.8616644144058228, -1.5792089700698853, 0.6367298364639282, -0.3927980363368988, 0.6539236307144165, 0.8779292106628418, 0.5544044971466064, -0.4019509553909302, -0.3177534341812134, 0.4690700173377991, -0.7643328905105591, -1.5016077756881714, 0.5900797843933105, 0.36968597769737244, 0.7913125157356262, -0.9097312688827515, 0.7577337622642517, 0.7302269339561462, 0.7785899639129639, -0.7808380722999573, -1.0440396070480347, 0.5710436701774597, -0.6633660197257996, -0.6982168555259705, 1.5012997388839722, 1.8683104515075684, -0.009162770584225655, 0.053628917783498764, 0.7560212016105652, 2.734218120574951, 0.8975538015365601, 0.9965337514877319, -0.9736179709434509, -1.3732835054397583, -0.22769349813461304, 0.5279285311698914, 0.9973939657211304, -1.5946539640426636, -0.3334585130214691, 0.5114824771881104, -0.8088312745094299, -0.009309413842856884, -0.3517300486564636, -1.9116069078445435, 2.348860740661621, 0.5795866250991821, 0.6854371428489685, -1.3237437009811401, 0.30641838908195496, -0.9147019386291504, -0.4391704797744751, -0.5839483141899109, 2.629678726196289, -0.1468760073184967, 0.2384973019361496, 0.9125059247016907, -0.9323630332946777, -0.12135637551546097, -0.1401946097612381, 0.6303017139434814, -0.8720086812973022, -0.4348902702331543, 0.707770824432373, 1.0431065559387207, -0.3136989176273346, -0.34615302085876465, -0.09326159954071045, 1.0725315809249878, 0.6460273265838623, 0.9376847743988037, -0.45695820450782776, -0.6338280439376831, 1.7222473621368408, -1.7862892150878906, 0.24867220222949982, -0.8356691002845764, -0.5947818756103516, 0.46319833397865295, -0.5724456310272217, -1.3741754293441772, -1.2928011417388916, -0.15125389397144318, 0.35185670852661133, 0.45909401774406433, -0.276844322681427, -0.7534619569778442, 0.17448769509792328, -0.8041709065437317, 0.7715813517570496, -0.17985635995864868, -0.3155222535133362, -0.27937594056129456, -0.6012001037597656, 1.3496509790420532, -0.07789217680692673, 0.911784827709198, 0.6547526121139526, 0.6284425854682922, -1.5268802642822266, -0.11813484132289886, -0.8698882460594177, 0.5227888822555542, -0.2165810912847519, -0.807140588760376, 1.1484020948410034, -1.4427751302719116, 0.6043649911880493, -0.525101363658905, -0.7420395612716675, -0.4712083339691162, -0.04465737193822861, -0.4328814148902893, -1.3837007284164429, -0.06250803917646408, 0.5973103046417236, 0.2701307237148285, -1.2017232179641724], + output: [-8.60566947190091, 9.047967813443393, -6.678979661059562, -8.360831755799953, 3.115231761090947, -6.697091460792556, 0.7568358822534593, 11.066622749967639, 2.8466255459388927, 6.27918164849176, 1.982200342331545, 0.3737138114008831, 8.93599812413914, 12.19410650368572, -4.5824951902858615, -4.643371981421932, 11.975504695471635, -19.79773110340414, 16.894522457707588, -11.40437941841762, -0.5369639596199498, -9.687637013191164, -2.1092885577719587, 10.925736950748426, 2.630639119388632, -2.8059934209514976, 8.536957582736084, -6.326392685852441, 19.05482126619987, 4.74321319044126, 1.7193491466752984, 17.828632923932844, 11.48731740639608, 11.824203873765583, 17.4496745991033, 14.886923392164112, -8.857144569139852, -18.45702795068501, 1.7494909136262233, -0.10810961271060027, 11.536135407131509, -12.549170934604959, -5.461911779043582, 9.93390865307337, 7.577784660577098, 11.044552867393765, -0.38872990536666596, -4.554231235610941, -9.19372645559055, 3.9372069026290077, -0.01611898707200954, -14.170353635176678, 2.977419276887061, 1.171732761405826, 9.77702001582745, 4.731009139477455, -6.457985308398847, -15.847828201326442, 9.95926759256128, -11.402544444287324, 17.96375259440102, -6.2530379131604725, 4.938790863887686, -10.135905976382034, 5.383747427255329, 6.230288801920859, 6.384329417961547, 1.3219731997515085, 3.1738925777367895, 11.969669020511425, -7.174837958584151, -14.303685689056584, 0.7454990885394415, -0.8185666838197116, 8.849555077234482, 3.9055874990272663, -2.018206936296142, -2.3395830964594415, -9.60564366177047, -5.769798601114195, 0.7520113635166212, 0.5065019274195421, -10.25514045587141, -9.637019763988052, 18.803482381840155, -2.3670620465003394, -11.199527109220814, 0.007833023989774368, 8.625619052479255, 9.951138032831594, 7.758961547592464, -1.8211300264093309, 6.718724482310631, -3.675823001520721, -6.918876012140167, -11.471658204308536, 2.467033682197071, -8.508193328034618, 14.225276109269815, 16.144776448129633, 15.927933230731913, 10.235538687304809, 5.767881945940595, -28.86811292326169, 9.669928964930417, -4.668293602235824, 14.769201279257288, 11.067653051718727, 12.761530418035363, -11.044953949483144, 4.242003110570776, -23.262991395970026, -10.145352776759976, 9.212885213669484, 6.329756726823897, -2.401697066728733, -8.54352813832675, 0.701551842851496, -15.258656469035635, -19.14394691246519, -7.522186448172975, -1.997676262113934, -11.408489749207227, 24.631586391822943, -5.098355854851496, 1.0410089604391717, 22.688569178648756, 19.097185694651152, 0.16409737757445608, -14.474036852523309, 4.7859526448704734, 19.558395170182035, 10.633635125709178, -3.0615977245039865, -6.883177607599656, -4.696650706016856, 4.286737340380929, 1.1391516077082633, -17.043528019280284, -7.2919743123329965, 18.42867500955987, 9.458517392981532, -7.23747329102956, 3.0801112402296056, -19.46748520769718, -19.760353803723497, -4.916492200532298, -11.492889148389452, 18.97908975026986, -9.61307230309599, -31.333799318924978, -6.6051904344747445, -2.610154763269807, 17.822400589161646, 5.128110541635698, 7.169102569854218, -20.704759452278893, -13.58731153366598, 9.44388682655616, -1.6262238850380326, 2.4785447765146986, 15.61067463907144, 8.206847406028755, -3.2188956590845867, 10.069966894154257, -17.811422869295264, -7.333188700567103, 7.277796245862989, -5.975213239029163, 0.0776494376891641, 4.448473434391025, -2.74774293836415, -19.167307678242555, -22.10574472079241, -0.9176460422085646, -2.698508980697669, -2.9126695148592425, -11.855087227776476, -10.63642091855646, -4.994794817859426, 8.34646291775615, 6.768483375508413, -10.605502170349608, 14.164919594071957, -12.394208891449225, 12.057338941547131, 22.90400935226583, -15.664600594740676, 1.30010074910831, -0.2199980758912563, -0.13084869289575352, 1.5514316991800943, -6.207758174655294, -21.281702562218502, 0.6785363179803934, 1.3313626670817715, -4.335780044121057, 0.5496189332641022, -12.958391630849423, -8.448073173532364, 9.262064248245114, -6.620464792278902, 6.229199284276308, 14.566232762792579, 7.969523876692342, -6.776657815077612, 12.290319767193232, 9.559089975820987, -10.33865192053783, -22.41131767867546, 2.9821554944671256, 4.62861355727904, -1.556181975825081, 15.142425835968055, -7.526775680393509, -16.871371735315304, -16.91792120984387, 10.94566759815768, 7.919149545235784, -2.721091872750706, -11.568727749623791, -6.404971618403414, 0.9048111698914159, -12.163107403321717, -23.30918866848898, 0.29599861864002963, 21.938310258693168, -2.3132157853956175, 0.7524657263569949, -5.613641198847075, -8.631493933375305, 17.933382465737296, 15.905249964584621, -0.12417131736795461, 16.840693139623014, 10.447828447732046, -15.117925469220577, -1.7000257658739502, -14.07936041053577, -12.445181510422046, 0.09178864954511656, 8.976039835817634, -3.0952547078859665, 4.42715006938438, 2.016084671443174, 1.299572020270631, -5.764752453011912, -5.767130951426552, -4.024939854949247, -6.303606982466311], + }, + fft_128_complex: { + complex: true, + input: [-0.46925589442253113, -0.17965197563171387, 1.0996962785720825, -0.38428056240081787, 0.48438164591789246, 0.4687068462371826, 0.39249294996261597, -0.3080357015132904, 0.5177378058433533, 0.6460461020469666, 2.131727933883667, -1.9881218671798706, 1.7797539234161377, -1.7341786623001099, 1.8023210763931274, 0.04056849330663681, 0.5402734875679016, -1.2557553052902222, 1.6492481231689453, 1.1959751844406128, -1.1998282670974731, -0.28071489930152893, -1.1383824348449707, -1.105458378791809, -1.3544220924377441, -0.9994213581085205, 1.0511277914047241, 1.3731906414031982, -0.366388738155365, 0.5248828530311584, -0.06661345064640045, 1.1866965293884277, -1.7126017808914185, -0.8613355755805969, -0.15631477534770966, -0.25460687279701233, -1.6851294040679932, -0.479960560798645, -1.6602188348770142, -1.8969695568084717, -1.3355588912963867, 0.2036823183298111, -0.21647731959819794, 0.027381736785173416, 1.3588712215423584, -1.916928768157959, -0.774971067905426, -0.9908143877983093, -0.5971747040748596, 0.1783454567193985, -1.5134319067001343, 0.49953287839889526, 0.6159718632698059, -0.36356979608535767, -1.0240488052368164, -0.04502468928694725, 0.4399787485599518, 1.6701726913452148, -0.29381847381591797, -0.10132483392953873, 1.382400393486023, 1.0435426235198975, 0.9063398838043213, -0.8696965575218201, 1.7439777851104736, -1.1632086038589478, -0.8023254871368408, 0.18316711485385895, 0.7868092656135559, 0.6457328796386719, 0.000458094640634954, 0.2006639540195465, 0.44469723105430603, 1.5764163732528687, 1.37369704246521, 1.4348359107971191, -0.3319808542728424, 0.4867582619190216, 1.6903849840164185, 0.09980353713035583, -0.8114499449729919, 0.21222348511219025, 1.7070233821868896, -0.06860260665416718, 0.3595496714115143, -0.22636087238788605, -0.11660676449537277, 0.35164034366607666, 0.6463969349861145, 0.7742586731910706, 2.551539897918701, -0.36875852942466736, -0.907656192779541, -1.5238984823226929, -0.17778240144252777, -1.5085883140563965, -2.1957619190216064, 0.2777234613895416, -1.9981634616851807, -0.36680006980895996, 0.43345877528190613, 0.12283071875572205, -0.7519865036010742, -0.8946448564529419, -1.0104613304138184, -0.574897289276123, -0.7528412342071533, -0.05524379014968872, -0.2588641345500946, 0.13064302504062653, -1.2371978759765625, 0.4406910538673401, -0.06988917291164398, 2.41745924949646, 0.2212488353252411, -0.9714004993438721, 0.41281431913375854, -0.5191168785095215, 0.5448367595672607, -0.5188544392585754, -0.35594210028648376, 1.31781005859375, 0.19899560511112213, 0.10419808328151703, -0.7013852000236511, 1.2463740110397339, -0.6775157451629639, -0.07904287427663803, -0.46743783354759216, 0.6186703443527222, 1.0175814628601074, 1.2864513397216797, -0.9423191547393799, -0.815204381942749, -1.4644925594329834, 0.4917009174823761, 1.0119210481643677, -1.106779932975769, -0.537906289100647, 0.09276387840509415, 2.3790464401245117, -0.18112841248512268, -0.7766230702400208, -0.4697842597961426, -0.24867388606071472, 0.350057452917099, -1.7352688312530518, 0.3294185996055603, -0.049025554209947586, 0.19478608667850494, 0.3822934031486511, 0.8966708183288574, 0.7362163662910461, 1.9381089210510254, 0.7763320207595825, -0.9195051193237305, -0.15279459953308105, 0.7260957956314087, 1.248564600944519, 0.8662853837013245, 0.22399753332138062, -0.5145770311355591, 0.6150395274162292, 0.17572005093097687, 0.1295638382434845, -0.823893129825592, 0.09608310461044312, 0.1056567132472992, -0.1421104371547699, -0.13746590912342072, 0.6133358478546143, -1.2861822843551636, 0.8356537818908691, 0.5849096775054932, 0.7742197513580322, 1.864620327949524, 0.1762249767780304, -1.6820768117904663, 0.170370414853096, -0.9834036827087402, -0.3257652521133423, 1.3807744979858398, 0.5471190810203552, -1.2435959577560425, -1.5459822416305542, -1.613089919090271, -1.5104633569717407, -0.7388973832130432, -0.3404698073863983, 2.2242629528045654, 1.101983904838562, -0.41542941331863403, 0.825286328792572, -1.7480731010437012, -0.227715402841568, -0.2249363213777542, -0.43012070655822754, -0.7993203401565552, -0.06888776272535324, 0.6460397839546204, 0.8002319931983948, 0.17660461366176605, 1.4292235374450684, -1.6437238454818726, 0.43454229831695557, -0.011055096983909607, -1.4420714378356934, -1.447640299797058, 0.16068753600120544, -0.2804417014122009, 0.3235762119293213, 1.2617326974868774, 0.1411382406949997, -0.23557768762111664, 0.13576534390449524, -1.6873000860214233, -0.42229339480400085, 0.8121863007545471, -1.0847389698028564, -1.6272295713424683, 1.8023097515106201, 0.7626906633377075, -1.3667072057724, 0.14141327142715454, 1.5778913497924805, -0.9910776615142822, -0.013187640346586704, 0.8963002562522888, 0.3190685510635376, 1.4895867109298706, 0.8450273871421814, -0.7519250512123108, 1.1641961336135864, -0.1140206977725029, -0.7162372469902039, -0.2397381216287613, 0.3938809037208557, 1.8317588567733765, -0.32195407152175903, 0.9710619449615479, -1.7239831686019897, -2.138955593109131, -1.1850391626358032, -1.1381371021270752, -0.2674075663089752, -0.7534425258636475, 1.180410385131836, -1.2349971532821655, -0.9732410311698914, -0.9838739633560181, 0.1530347764492035, 0.7869290113449097, -0.5131613612174988, 0.7393561601638794, 0.5818021893501282, -0.9010815024375916], + output: [4.6533055662875995, -11.01426088809967, 8.89589916229627, -8.67151790820412, -5.338702909401541, 1.2596635311977273, 3.5364356687085934, 7.495255571019467, 14.607353334543243, -18.087691812322902, 7.564636217194101, -31.42023502604239, -6.214070442777103, -10.959840511165702, -0.23122589156934037, -11.620785761919407, -8.17956409451724, -2.5877133314432683, 11.015937578315528, 5.780471206474331, -6.0841540885916, 20.060728028967564, 3.2708359605096224, 11.782757950154132, 0.09008114957166846, 3.524696982412627, -10.677391613915269, 0.31298678204602526, -21.35192563693417, 12.385290905482485, 2.161430892216815, 1.8976227049433234, -8.989539272795344, 13.126889442846508, 2.8499370378020865, 1.7474738916889598, 5.022029084645628, -3.2038788652839223, 5.800603925256104, -11.310381554572714, 8.70516962035894, -19.131074662654022, 1.7802965964722235, -14.806060115152377, 14.302944499531886, -15.445567609224282, -3.8462368249304397, -9.516418012771346, 13.019042335168749, -10.443622659963964, -10.883324294802648, 3.957267045122531, -0.667106901008031, -4.392642175023952, -5.891115650099352, 16.457002297798386, 8.618862097531547, 8.454290723978408, -1.5705784838263321, -2.0950451541937802, -0.8023593073701956, -3.1480197619642944, -5.620797591431048, -4.595460315016652, -5.651367928832769, -12.289105277159251, 12.268533105640746, -14.70471337060609, -2.55864085006375, 4.848469317941595, -15.477103640433441, -0.8200806702142849, 1.477227192472725, -8.856561753264552, -1.7000675285625562, 17.974609221649885, -1.8152649987658895, 2.226238832898174, 5.942983183373236, 5.062290291727308, 7.091530929787825, -2.0854528335431413, -2.2607116079192284, -0.9522421064376894, 1.9922801075296586, -4.848590729633989, 22.20475854724638, 14.139180422304625, 7.742631959368987, 8.736735176873482, -14.221452296099534, 10.498675928711396, -0.16082936982259532, -3.8946140234989644, -20.400591916908017, -26.146811938453247, 6.587325710150547, -20.584777745167045, 14.638519536239858, -1.6926103003462887, -10.208680237895118, 6.147709389485655, -1.986836416339922, 15.002313879999715, 6.5052637626549386, 2.3893876610117486, -1.4903416359002417, 2.5125845767117747, -11.629045740948362, -1.0314802432568921, -7.7204438190047515, -2.026871521184386, 5.038120875238651, 12.509857810233317, -2.8111210090417167, 6.6237223115217265, -11.560596890267984, 21.096203768560102, 2.492616415367813, 15.694305674116976, 6.98125969245643, -2.51588120579323, -23.614269801976544, -19.721211546858797, 5.821751650555415, -7.957710273512344, -2.3083909989954927, -9.024817057699233, -2.3525185122853145, 12.543071284890175, -6.633775272995389, 1.9856801362534808, -18.01947792290391, 1.6526855337166202, -25.06385984262638, 15.109040183134795, -3.952374815603415, 8.226644901530008, 18.761705127047698, 3.93250694875225, -7.442640246025839, -19.86894607874479, 12.889936646377077, 17.058248465954463, -10.38590977421429, -8.762421865432158, 21.57778103004022, 3.943591516088906, -5.804714467684743, 4.255345272797616, 6.842976031631512, 1.5538004698130026, -4.425512141506818, -4.377682130235696, -0.5744161527454539, -6.466874369118861, 2.117583592008284, 9.441789454733126, 8.94127387455451, -0.04257498982078545, -6.206626408089971, -17.798516107544632, 3.1902492140570136, -8.701598789976085, -3.6350162587702526, 11.47775543413513, 5.763743605757109, -15.507961256533623, -12.20946994071868, 15.597966311245933, -7.577596335933641, -5.783678667731976, 0.5854316506349582, 1.0524576740591431, 9.32444015560922, 14.86759124801031, -2.8564425698215246, -4.050241339325346, -14.743103022730313, 0.6982253561530176, 1.4485867743987042, 15.863950914048452, -10.345289369065991, 18.289197082863648, -15.758886684054374, -5.422633603143401, -0.983270019651755, -12.824811394399234, -2.2739348529046874, 10.113426086776306, -29.057768796230015, -23.164653851413163, -14.355517383664846, -1.9394528310513124, 13.636590068666269, 3.459140997767836, -1.3650156273028862, 4.305744984992871, 17.561184167420265, -10.355528699590693, 21.10112660094154, -8.222224503241419, 2.535762677259953, 7.713352852965743, -14.28383557691274, 25.855974989407443, 4.209799001514582, -21.52088963141282, 10.888001415630182, -13.13372692624647, 11.475712218807361, 1.8430819379630443, -4.696550665547157, -6.590021563791754, -25.66889721277428, 3.6636259509044233, 16.177010478373504, -18.738751282648302, -8.929151482851156, 1.9637629189516739, 9.362438823950344, 18.876072069444945, 15.70735011826503, -7.857995088347812, 1.9318237720996665, -16.12923916175005, -7.934748190521045, 0.06924911452461413, -14.499587783289256, -0.4727635184847867, 10.482895882033887, -9.583766230162844, -4.311441092707766, 10.174307532202345, -12.09622773401161, -11.448182124496999, -8.63443998241864, 19.56612298031517, 12.124200790573179, 13.998676498504775, 2.185864445998216, 9.654230140395715, -13.738018677296829, 23.468720582013574, 9.279088660718887, -3.9238192921918174, -13.262040984478173, 13.199249429343212, 8.476422030030909, 22.120527194597067, 30.905491552855995, -14.140763293470911, -3.076195026308723, 2.461867832774407, -1.150707769438454, -18.323349178808648], + }, }; describe("Mathematical operations", () => { - describe("bankers rounding", () => { - it("should round up to nearest even", () => { - expect(bankers_round(-0.5)).toBeCloseTo(0); - expect(bankers_round(1.5)).toBeCloseTo(2); - expect(bankers_round(19.5)).toBeCloseTo(20); + describe("bankers rounding", () => { + it("should round up to nearest even", () => { + expect(bankers_round(-0.5)).toBeCloseTo(0); + expect(bankers_round(1.5)).toBeCloseTo(2); + expect(bankers_round(19.5)).toBeCloseTo(20); + }); + it("should round down to nearest even", () => { + expect(bankers_round(-1.5)).toBeCloseTo(-2); + expect(bankers_round(2.5)).toBeCloseTo(2); + expect(bankers_round(18.5)).toBeCloseTo(18); + }); }); - it("should round down to nearest even", () => { - expect(bankers_round(-1.5)).toBeCloseTo(-2); - expect(bankers_round(2.5)).toBeCloseTo(2); - expect(bankers_round(18.5)).toBeCloseTo(18); - }); - }); - describe("median filtering", () => { - it("should compute median filter", async () => { - const t1 = new Float32Array([5, 12, 2, 6, 3, 10, 9, 1, 4, 8, 11, 7]); - const window = 3; + describe("median filtering", () => { + it("should compute median filter", async () => { + const t1 = new Float32Array([5, 12, 2, 6, 3, 10, 9, 1, 4, 8, 11, 7]); + const window = 3; + + const target = new Float32Array([12, 5, 6, 3, 6, 9, 9, 4, 4, 8, 8, 11]); - const target = new Float32Array([12, 5, 6, 3, 6, 9, 9, 4, 4, 8, 8, 11]); + const output = medianFilter(t1, window); + compare(output, target, 1e-3); + }); - const output = medianFilter(t1, window); - compare(output, target, 1e-3); + // TODO add tests for errors }); - // TODO add tests for errors - }); + describe("log softmax", () => { + // Should match output of scipy log_softmax + it("should compute log softmax correctly for usual values", () => { + const input = [0, 1, 2, 3]; + const expected = [-3.4401896985611953, -2.4401896985611953, -1.4401896985611953, -0.44018969856119533]; + const output = log_softmax(input); + compare(output, expected, 1e-13); + }); - describe("log softmax", () => { - // Should match output of scipy log_softmax - it("should compute log softmax correctly for usual values", () => { - const input = [0, 1, 2, 3]; - const expected = [-3.4401896985611953, -2.4401896985611953, -1.4401896985611953, -0.44018969856119533]; - const output = log_softmax(input); - compare(output, expected, 1e-13); + it("should compute log softmax correctly for values with large differences", () => { + const input = [1000, 1]; + const expected = [0, -999]; + const output = log_softmax(input); + compare(output, expected, 1e-13); + }); }); - it("should compute log softmax correctly for values with large differences", () => { - const input = [1000, 1]; - const expected = [0, -999]; - const output = log_softmax(input); - compare(output, expected, 1e-13); - }); - }); - - describe("FFT", () => { - // Should match output of numpy fft - it("should compute real FFT for power of two", () => { - { - // size = 4 - // np.fft.fft([1,2,3,4]) == array([10.+0.j, -2.+2.j, -2.+0.j, -2.-2.j]) - const input = new Float32Array([1, 2, 3, 4]); - const target = new Float32Array([10, 0, -2, 2, -2, 0, -2, -2]); - - const output = fft(input); - compare(output, target, 1e-3); - } - - { - // size = 16 - // np.fft.fft([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]) - // == array([136. +0.j , -8.+40.21871594j, -8.+19.3137085j , - // -8.+11.9728461j , -8. +8.j , -8. +5.3454291j , - // -8. +3.3137085j , -8. +1.59129894j, -8. +0.j , - // -8. -1.59129894j, -8. -3.3137085j , -8. -5.3454291j , - // -8. -8.j , -8.-11.9728461j , -8.-19.3137085j , - // -8.-40.21871594j]) - const input = new Float32Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]); - const target = new Float32Array([136.0, 0.0, -8.0, 40.218715937006785, -8.0, 19.31370849898476, -8.0, 11.972846101323912, -8.0, 8.0, -8.0, 5.345429103354389, -8.0, 3.313708498984761, -8.0, 1.5912989390372658, -8.0, 0.0, -8.0, -1.5912989390372658, -8.0, -3.313708498984761, -8.0, -5.345429103354389, -8.0, -8.0, -8.0, -11.972846101323912, -8.0, -19.31370849898476, -8.0, -40.218715937006785]); - - const output = fft(input); - compare(output, target, 1e-3); - } - }); + describe("FFT", () => { + // Should match output of numpy fft + it("should compute real FFT for power of two", () => { + { + // size = 4 + // np.fft.fft([1,2,3,4]) == array([10.+0.j, -2.+2.j, -2.+0.j, -2.-2.j]) + const input = new Float32Array([1, 2, 3, 4]); + const target = new Float32Array([10, 0, -2, 2, -2, 0, -2, -2]); - it("should compute real FFT for non-power of two", () => { - { - // size = 3 - // np.fft.fft([1,2,3]) == array([ 6. +0.j, -1.5+0.8660254j, -1.5-0.8660254j]) - const input = new Float32Array([1, 2, 3]); - const target = new Float32Array([6, 0, -1.5, 0.8660254, -1.5, -0.8660254]); + const output = fft(input); + compare(output, target, 1e-3); + } - const output = fft(input); - compare(output, target, 1e-3); - } - }); + { + // size = 16 + // np.fft.fft([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]) + // == array([136. +0.j , -8.+40.21871594j, -8.+19.3137085j , + // -8.+11.9728461j , -8. +8.j , -8. +5.3454291j , + // -8. +3.3137085j , -8. +1.59129894j, -8. +0.j , + // -8. -1.59129894j, -8. -3.3137085j , -8. -5.3454291j , + // -8. -8.j , -8.-11.9728461j , -8.-19.3137085j , + // -8.-40.21871594j]) + const input = new Float32Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]); + const target = new Float32Array([136.0, 0.0, -8.0, 40.218715937006785, -8.0, 19.31370849898476, -8.0, 11.972846101323912, -8.0, 8.0, -8.0, 5.345429103354389, -8.0, 3.313708498984761, -8.0, 1.5912989390372658, -8.0, 0.0, -8.0, -1.5912989390372658, -8.0, -3.313708498984761, -8.0, -5.345429103354389, -8.0, -8.0, -8.0, -11.972846101323912, -8.0, -19.31370849898476, -8.0, -40.218715937006785]); - it("should compute complex FFT for non-power of two", () => { - { - // size = 3 - // np.fft.fft([1+3j,2-2j,3+1j]) == array([ 6. +2.j, -4.09807621+4.3660254j, 1.09807621+2.6339746j]) - const input = new Float32Array([1, 3, 2, -2, 3, 1]); - const target = new Float32Array([6, 2, -4.09807621, 4.3660254, 1.09807621, 2.6339746]); + const output = fft(input); + compare(output, target, 1e-3); + } + }); - const output = fft(input, true); - compare(output, target, 1e-3); - } - }); + it("should compute real FFT for non-power of two", () => { + { + // size = 3 + // np.fft.fft([1,2,3]) == array([ 6. +0.j, -1.5+0.8660254j, -1.5-0.8660254j]) + const input = new Float32Array([1, 2, 3]); + const target = new Float32Array([6, 0, -1.5, 0.8660254, -1.5, -0.8660254]); - it("should compute complex FFT for power of two", () => { - { - // size = 4 - // np.fft.fft([1+4j, 2-3j,3+2j, 4-1j]) == array([10. +2.j, -4. +4.j, -2.+10.j, 0. +0.j]) - const input = new Float32Array([1, 4, 2, -3, 3, 2, 4, -1]); - const target = new Float32Array([10, 2, -4, 4, -2, 10, 0, 0]); + const output = fft(input); + compare(output, target, 1e-3); + } + }); - const output = fft(input, true); - compare(output, target, 1e-3); - } + it("should compute complex FFT for non-power of two", () => { + { + // size = 3 + // np.fft.fft([1+3j,2-2j,3+1j]) == array([ 6. +2.j, -4.09807621+4.3660254j, 1.09807621+2.6339746j]) + const input = new Float32Array([1, 3, 2, -2, 3, 1]); + const target = new Float32Array([6, 2, -4.09807621, 4.3660254, 1.09807621, 2.6339746]); + + const output = fft(input, true); + compare(output, target, 1e-3); + } + }); + + it("should compute complex FFT for power of two", () => { + { + // size = 4 + // np.fft.fft([1+4j, 2-3j,3+2j, 4-1j]) == array([10. +2.j, -4. +4.j, -2.+10.j, 0. +0.j]) + const input = new Float32Array([1, 4, 2, -3, 3, 2, 4, -1]); + const target = new Float32Array([10, 2, -4, 4, -2, 10, 0, 0]); + + const output = fft(input, true); + compare(output, target, 1e-3); + } + }); }); - }); - - describe("FFT (random & dynamic)", () => { - // Should match output of numpy fft - for (const [name, test] of Object.entries(FFT_TEST_DATA)) { - it(name, () => { - const output = fft(test.input, test.complex); - - if (output.map((v, i) => Math.abs(v - test.output[i])).some((v) => v > 1e-4)) { - console.log("input", test.input); - console.log("output", output); - console.log("target", test.output); + + describe("FFT (random & dynamic)", () => { + // Should match output of numpy fft + for (const [name, test] of Object.entries(FFT_TEST_DATA)) { + it(name, () => { + const output = fft(test.input, test.complex); + + if (output.map((v, i) => Math.abs(v - test.output[i])).some((v) => v > 1e-4)) { + console.log("input", test.input); + console.log("output", output); + console.log("target", test.output); + } + compare(output, test.output, 1e-4); + }); } - compare(output, test.output, 1e-4); - }); - } - }); + }); }); diff --git a/tests/utils/tensor.test.js b/tests/utils/tensor.test.js index 0d36954e3..4930e98d3 100644 --- a/tests/utils/tensor.test.js +++ b/tests/utils/tensor.test.js @@ -2,175 +2,175 @@ import { Tensor, cat, mean, stack, layer_norm } from "../../src/transformers.js" import { compare } from "../test_utils.js"; describe("Tensor operations", () => { - describe("cat", () => { - it("should concatenate on dim=0", async () => { - const t1 = new Tensor("float32", [1, 2, 3], [1, 3]); - const t2 = new Tensor("float32", [4, 5, 6, 7, 8, 9], [2, 3]); - const t3 = new Tensor("float32", [10, 11, 12], [1, 3]); - - const target1 = new Tensor("float32", [1, 2, 3, 4, 5, 6, 7, 8, 9], [3, 3]); - const target2 = new Tensor("float32", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], [4, 3]); - - // 2 tensors - const concatenated1 = cat([t1, t2], 0); - compare(concatenated1, target1, 1e-3); - - // 3 tensors - const concatenated2 = cat([t1, t2, t3], 0); - compare(concatenated2, target2, 1e-3); - }); + describe("cat", () => { + it("should concatenate on dim=0", async () => { + const t1 = new Tensor("float32", [1, 2, 3], [1, 3]); + const t2 = new Tensor("float32", [4, 5, 6, 7, 8, 9], [2, 3]); + const t3 = new Tensor("float32", [10, 11, 12], [1, 3]); - it("should concatenate on dim=1", async () => { - const t1 = new Tensor("float32", [1, 2, 3, -1, -2, -3], [2, 3, 1]); - const t2 = new Tensor("float32", [4, -4], [2, 1, 1]); - const t3 = new Tensor("float32", [5, 6, -5, -6], [2, 2, 1]); + const target1 = new Tensor("float32", [1, 2, 3, 4, 5, 6, 7, 8, 9], [3, 3]); + const target2 = new Tensor("float32", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], [4, 3]); - const target1 = new Tensor("float32", [1, 2, 3, 4, -1, -2, -3, -4], [2, 4, 1]); - const target2 = new Tensor("float32", [1, 2, 3, 4, 5, 6, -1, -2, -3, -4, -5, -6], [2, 6, 1]); + // 2 tensors + const concatenated1 = cat([t1, t2], 0); + compare(concatenated1, target1, 1e-3); - // 2 tensors - const concatenated1 = cat([t1, t2], 1); - compare(concatenated1, target1, 1e-3); + // 3 tensors + const concatenated2 = cat([t1, t2, t3], 0); + compare(concatenated2, target2, 1e-3); + }); - // 3 tensors - const concatenated2 = cat([t1, t2, t3], 1); - compare(concatenated2, target2, 1e-3); - }); + it("should concatenate on dim=1", async () => { + const t1 = new Tensor("float32", [1, 2, 3, -1, -2, -3], [2, 3, 1]); + const t2 = new Tensor("float32", [4, -4], [2, 1, 1]); + const t3 = new Tensor("float32", [5, 6, -5, -6], [2, 2, 1]); - it("should concatenate on dim=-2", async () => { - const t1 = new Tensor("float32", [1, 2, 3, 4, 5, 6, 11, 12, 13, 14, 15, 16], [2, 1, 3, 2]); - const t2 = new Tensor("float32", [7, 8, 9, 10, 17, 18, 19, 20], [2, 1, 2, 2]); + const target1 = new Tensor("float32", [1, 2, 3, 4, -1, -2, -3, -4], [2, 4, 1]); + const target2 = new Tensor("float32", [1, 2, 3, 4, 5, 6, -1, -2, -3, -4, -5, -6], [2, 6, 1]); - const target = new Tensor("float32", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], [2, 1, 5, 2]); + // 2 tensors + const concatenated1 = cat([t1, t2], 1); + compare(concatenated1, target1, 1e-3); - const concatenated = cat([t1, t2], -2); + // 3 tensors + const concatenated2 = cat([t1, t2, t3], 1); + compare(concatenated2, target2, 1e-3); + }); - compare(concatenated, target, 1e-3); - }); + it("should concatenate on dim=-2", async () => { + const t1 = new Tensor("float32", [1, 2, 3, 4, 5, 6, 11, 12, 13, 14, 15, 16], [2, 1, 3, 2]); + const t2 = new Tensor("float32", [7, 8, 9, 10, 17, 18, 19, 20], [2, 1, 2, 2]); - // TODO add tests for errors - }); + const target = new Tensor("float32", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], [2, 1, 5, 2]); - describe("stack", () => { - const t1 = new Tensor("float32", [0, 1, 2, 3, 4, 5], [1, 3, 2]); + const concatenated = cat([t1, t2], -2); - it("should stack on dim=0", async () => { - const target1 = new Tensor("float32", [0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5], [2, 1, 3, 2]); - const target2 = new Tensor("float32", [0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5], [3, 1, 3, 2]); + compare(concatenated, target, 1e-3); + }); - // 2 tensors - const stacked1 = stack([t1, t1], 0); - compare(stacked1, target1, 1e-3); - - // 3 tensors - const stacked2 = stack([t1, t1, t1], 0); - compare(stacked2, target2, 1e-3); + // TODO add tests for errors }); - it("should stack on dim=1", async () => { - const target1 = new Tensor("float32", [0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5], [1, 2, 3, 2]); - const target2 = new Tensor("float32", [0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5], [1, 3, 3, 2]); + describe("stack", () => { + const t1 = new Tensor("float32", [0, 1, 2, 3, 4, 5], [1, 3, 2]); - // 2 tensors - const stacked1 = stack([t1, t1], 1); - compare(stacked1, target1, 1e-3); + it("should stack on dim=0", async () => { + const target1 = new Tensor("float32", [0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5], [2, 1, 3, 2]); + const target2 = new Tensor("float32", [0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5], [3, 1, 3, 2]); - // 3 tensors - const stacked2 = stack([t1, t1, t1], 1); - compare(stacked2, target2, 1e-3); - }); + // 2 tensors + const stacked1 = stack([t1, t1], 0); + compare(stacked1, target1, 1e-3); - it("should stack on dim=-1", async () => { - const target1 = new Tensor("float32", [0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5], [1, 3, 2, 2]); - const target2 = new Tensor("float32", [0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5], [1, 3, 2, 3]); + // 3 tensors + const stacked2 = stack([t1, t1, t1], 0); + compare(stacked2, target2, 1e-3); + }); - // 2 tensors - const stacked1 = stack([t1, t1], -1); - compare(stacked1, target1, 1e-3); + it("should stack on dim=1", async () => { + const target1 = new Tensor("float32", [0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5], [1, 2, 3, 2]); + const target2 = new Tensor("float32", [0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5], [1, 3, 3, 2]); - // 3 tensors - const stacked2 = stack([t1, t1, t1], -1); - compare(stacked2, target2, 1e-3); + // 2 tensors + const stacked1 = stack([t1, t1], 1); + compare(stacked1, target1, 1e-3); + + // 3 tensors + const stacked2 = stack([t1, t1, t1], 1); + compare(stacked2, target2, 1e-3); + }); + + it("should stack on dim=-1", async () => { + const target1 = new Tensor("float32", [0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5], [1, 3, 2, 2]); + const target2 = new Tensor("float32", [0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5], [1, 3, 2, 3]); + + // 2 tensors + const stacked1 = stack([t1, t1], -1); + compare(stacked1, target1, 1e-3); + + // 3 tensors + const stacked2 = stack([t1, t1, t1], -1); + compare(stacked2, target2, 1e-3); + }); }); - }); - - describe("permute", () => { - it("should permute", async () => { - const x = new Tensor("float32", [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23], [2, 3, 4]); - // Permute axes to (0, 1, 2) - No change - const permuted_1 = x.permute(0, 1, 2); - const target_1 = x; - compare(permuted_1, target_1, 1e-3); - - // Permute axes to (0, 2, 1) - const permuted_2 = x.permute(0, 2, 1); - const target_2 = new Tensor("float32", [0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11, 12, 16, 20, 13, 17, 21, 14, 18, 22, 15, 19, 23], [2, 4, 3]); - compare(permuted_2, target_2, 1e-3); - - // Permute axes to (1, 0, 2) - const permuted_3 = x.permute(1, 0, 2); - const target_3 = new Tensor("float32", [0, 1, 2, 3, 12, 13, 14, 15, 4, 5, 6, 7, 16, 17, 18, 19, 8, 9, 10, 11, 20, 21, 22, 23], [3, 2, 4]); - compare(permuted_3, target_3, 1e-3); - - // Permute axes to (1, 2, 0) - const permuted_4 = x.permute(1, 2, 0); - const target_4 = new Tensor("float32", [0, 12, 1, 13, 2, 14, 3, 15, 4, 16, 5, 17, 6, 18, 7, 19, 8, 20, 9, 21, 10, 22, 11, 23], [3, 4, 2]); - compare(permuted_4, target_4, 1e-3); - - // Permute axes to (2, 0, 1) - const permuted_5 = x.permute(2, 0, 1); - const target_5 = new Tensor("float32", [0, 4, 8, 12, 16, 20, 1, 5, 9, 13, 17, 21, 2, 6, 10, 14, 18, 22, 3, 7, 11, 15, 19, 23], [4, 2, 3]); - compare(permuted_5, target_5, 1e-3); - - // Permute axes to (2, 1, 0) - const permuted_6 = x.permute(2, 1, 0); - const target_6 = new Tensor("float32", [0, 12, 4, 16, 8, 20, 1, 13, 5, 17, 9, 21, 2, 14, 6, 18, 10, 22, 3, 15, 7, 19, 11, 23], [4, 3, 2]); - compare(permuted_6, target_6, 1e-3); + + describe("permute", () => { + it("should permute", async () => { + const x = new Tensor("float32", [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23], [2, 3, 4]); + // Permute axes to (0, 1, 2) - No change + const permuted_1 = x.permute(0, 1, 2); + const target_1 = x; + compare(permuted_1, target_1, 1e-3); + + // Permute axes to (0, 2, 1) + const permuted_2 = x.permute(0, 2, 1); + const target_2 = new Tensor("float32", [0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11, 12, 16, 20, 13, 17, 21, 14, 18, 22, 15, 19, 23], [2, 4, 3]); + compare(permuted_2, target_2, 1e-3); + + // Permute axes to (1, 0, 2) + const permuted_3 = x.permute(1, 0, 2); + const target_3 = new Tensor("float32", [0, 1, 2, 3, 12, 13, 14, 15, 4, 5, 6, 7, 16, 17, 18, 19, 8, 9, 10, 11, 20, 21, 22, 23], [3, 2, 4]); + compare(permuted_3, target_3, 1e-3); + + // Permute axes to (1, 2, 0) + const permuted_4 = x.permute(1, 2, 0); + const target_4 = new Tensor("float32", [0, 12, 1, 13, 2, 14, 3, 15, 4, 16, 5, 17, 6, 18, 7, 19, 8, 20, 9, 21, 10, 22, 11, 23], [3, 4, 2]); + compare(permuted_4, target_4, 1e-3); + + // Permute axes to (2, 0, 1) + const permuted_5 = x.permute(2, 0, 1); + const target_5 = new Tensor("float32", [0, 4, 8, 12, 16, 20, 1, 5, 9, 13, 17, 21, 2, 6, 10, 14, 18, 22, 3, 7, 11, 15, 19, 23], [4, 2, 3]); + compare(permuted_5, target_5, 1e-3); + + // Permute axes to (2, 1, 0) + const permuted_6 = x.permute(2, 1, 0); + const target_6 = new Tensor("float32", [0, 12, 4, 16, 8, 20, 1, 13, 5, 17, 9, 21, 2, 14, 6, 18, 10, 22, 3, 15, 7, 19, 11, 23], [4, 3, 2]); + compare(permuted_6, target_6, 1e-3); + }); }); - }); - describe("map", () => { - it("should double", async () => { - const original = new Tensor("float32", [1, 2, 3, 4, 5, 6], [2, 3]); - const target = new Tensor("float32", [2, 4, 6, 8, 10, 12], [2, 3]); + describe("map", () => { + it("should double", async () => { + const original = new Tensor("float32", [1, 2, 3, 4, 5, 6], [2, 3]); + const target = new Tensor("float32", [2, 4, 6, 8, 10, 12], [2, 3]); - const doubled = original.map((x) => x * 2); - compare(doubled, target, 1e-3); + const doubled = original.map((x) => x * 2); + compare(doubled, target, 1e-3); + }); }); - }); - describe("mean", () => { - it("should calculate mean", async () => { - const t1 = new Tensor("float32", [1, 2, 3, 4, 5, 6], [2, 3, 1]); + describe("mean", () => { + it("should calculate mean", async () => { + const t1 = new Tensor("float32", [1, 2, 3, 4, 5, 6], [2, 3, 1]); - const target = new Tensor("float32", [3.5], []); + const target = new Tensor("float32", [3.5], []); - const target0 = new Tensor("float32", [2.5, 3.5, 4.5], [3, 1]); - const target1 = new Tensor("float32", [2, 5], [2, 1]); - const target2 = new Tensor("float32", [1, 2, 3, 4, 5, 6], [2, 3]); + const target0 = new Tensor("float32", [2.5, 3.5, 4.5], [3, 1]); + const target1 = new Tensor("float32", [2, 5], [2, 1]); + const target2 = new Tensor("float32", [1, 2, 3, 4, 5, 6], [2, 3]); - let avg = mean(t1); - compare(avg, target, 1e-3); + let avg = mean(t1); + compare(avg, target, 1e-3); - let avg0 = mean(t1, 0); - compare(avg0, target0, 1e-3); + let avg0 = mean(t1, 0); + compare(avg0, target0, 1e-3); - let avg1 = mean(t1, 1); - compare(avg1, target1, 1e-3); + let avg1 = mean(t1, 1); + compare(avg1, target1, 1e-3); - let avg2 = mean(t1, 2); - compare(avg2, target2, 1e-3); + let avg2 = mean(t1, 2); + compare(avg2, target2, 1e-3); + }); }); - }); - describe("layer_norm", () => { - it("should calculate layer norm", async () => { - const t1 = new Tensor("float32", [1, 2, 3, 4, 5, 6], [2, 3]); + describe("layer_norm", () => { + it("should calculate layer norm", async () => { + const t1 = new Tensor("float32", [1, 2, 3, 4, 5, 6], [2, 3]); - const target = new Tensor("float32", [-1.2247356176376343, 0.0, 1.2247356176376343, -1.2247357368469238, -1.1920928955078125e-7, 1.2247354984283447], [2, 3]); + const target = new Tensor("float32", [-1.2247356176376343, 0.0, 1.2247356176376343, -1.2247357368469238, -1.1920928955078125e-7, 1.2247354984283447], [2, 3]); - const norm = layer_norm(t1, [t1.dims.at(-1)]); - compare(norm, target, 1e-3); + const norm = layer_norm(t1, [t1.dims.at(-1)]); + compare(norm, target, 1e-3); + }); }); - }); }); diff --git a/tests/utils/tensor_ops.test.js b/tests/utils/tensor_ops.test.js index 3227d5f58..8c61a8ee2 100644 --- a/tests/utils/tensor_ops.test.js +++ b/tests/utils/tensor_ops.test.js @@ -5,187 +5,187 @@ import { init } from "../init.js"; init(); function expectToBeCloseToArray(actual, expected) { - expect(actual.length).toEqual(expected.length); - actual.forEach((x, i) => expect(x).toBeCloseTo(expected[i])); + expect(actual.length).toEqual(expected.length); + actual.forEach((x, i) => expect(x).toBeCloseTo(expected[i])); } function range(start, stop = undefined, step = 1) { - if (stop === undefined) { - stop = start; - start = 0; - } - - const result = []; - for (let i = start; i < stop; i += step) { - result.push(i); - } - return result; + if (stop === undefined) { + stop = start; + start = 0; + } + + const result = []; + for (let i = start; i < stop; i += step) { + result.push(i); + } + return result; } describe("Tensor operations", () => { - describe("interpolate", () => { - const input = new Tensor( - "float32", - new Float32Array(2 * 3 * 4 * 5).map((_, i) => i), - [2, 3, 4, 5], - ); - - const size = [2, 3, 3, 2]; - it("bilinear", async () => { - const resized = await interpolate_4d(input, { mode: "bilinear", size }); - const target = new Float32Array( - [ - [ - [ - [1.5833335, 4.0833335], - [8.25, 10.75], - [14.916668, 17.416668], - ], - [ - [21.583332, 24.083334], - [28.25, 30.75], - [34.916668, 37.416668], - ], - [ - [41.583332, 44.083332], - [48.25, 50.75], - [54.916668, 57.416668], - ], - ], - [ - [ - [61.583332, 64.083336], - [68.25, 70.75], - [74.916664, 77.41667], - ], - [ - [81.58333, 84.083336], - [88.25, 90.75], - [94.91667, 97.41667], - ], - [ - [101.583336, 104.08333], - [108.25, 110.75], - [114.916664, 117.416664], - ], - ], - ].flat(Infinity), - ); - - expectToBeCloseToArray(target, resized.data); + describe("interpolate", () => { + const input = new Tensor( + "float32", + new Float32Array(2 * 3 * 4 * 5).map((_, i) => i), + [2, 3, 4, 5], + ); + + const size = [2, 3, 3, 2]; + it("bilinear", async () => { + const resized = await interpolate_4d(input, { mode: "bilinear", size }); + const target = new Float32Array( + [ + [ + [ + [1.5833335, 4.0833335], + [8.25, 10.75], + [14.916668, 17.416668], + ], + [ + [21.583332, 24.083334], + [28.25, 30.75], + [34.916668, 37.416668], + ], + [ + [41.583332, 44.083332], + [48.25, 50.75], + [54.916668, 57.416668], + ], + ], + [ + [ + [61.583332, 64.083336], + [68.25, 70.75], + [74.916664, 77.41667], + ], + [ + [81.58333, 84.083336], + [88.25, 90.75], + [94.91667, 97.41667], + ], + [ + [101.583336, 104.08333], + [108.25, 110.75], + [114.916664, 117.416664], + ], + ], + ].flat(Infinity), + ); + + expectToBeCloseToArray(target, resized.data); + }); + + it("bicubic", async () => { + const resized = await interpolate_4d(input, { mode: "bicubic", size }); + + const target = new Float32Array( + [ + [ + [ + [1.2987545, 3.9628172], + [8.167969, 10.832031], + [15.037184, 17.701244], + ], + [ + [21.298756, 23.962818], + [28.167969, 30.832031], + [35.037186, 37.701252], + ], + [ + [41.298756, 43.96282], + [48.16797, 50.83203], + [55.037193, 57.701256], + ], + ], + [ + [ + [61.29875, 63.96282], + [68.16797, 70.83203], + [75.03719, 77.701256], + ], + [ + [81.29875, 83.96282], + [88.16797, 90.83203], + [95.03721, 97.70126], + ], + [ + [101.29875, 103.962814], + [108.16797, 110.83203], + [115.03721, 117.70127], + ], + ], + ].flat(Infinity), + ); + + expectToBeCloseToArray(target, resized.data); + }); }); - it("bicubic", async () => { - const resized = await interpolate_4d(input, { mode: "bicubic", size }); - - const target = new Float32Array( - [ - [ - [ - [1.2987545, 3.9628172], - [8.167969, 10.832031], - [15.037184, 17.701244], - ], - [ - [21.298756, 23.962818], - [28.167969, 30.832031], - [35.037186, 37.701252], - ], - [ - [41.298756, 43.96282], - [48.16797, 50.83203], - [55.037193, 57.701256], - ], - ], - [ - [ - [61.29875, 63.96282], - [68.16797, 70.83203], - [75.03719, 77.701256], - ], - [ - [81.29875, 83.96282], - [88.16797, 90.83203], - [95.03721, 97.70126], - ], - [ - [101.29875, 103.962814], - [108.16797, 110.83203], - [115.03721, 117.70127], - ], - ], - ].flat(Infinity), - ); - - expectToBeCloseToArray(target, resized.data); - }); - }); - - describe("matmul", () => { - it("(2, 5) @ (5, 4) -> (2, 4)", async () => { - const a = new Tensor("float32", range(10), [2, 5]); - const b = new Tensor("float32", range(20), [5, 4]); + describe("matmul", () => { + it("(2, 5) @ (5, 4) -> (2, 4)", async () => { + const a = new Tensor("float32", range(10), [2, 5]); + const b = new Tensor("float32", range(20), [5, 4]); - const result = await matmul(a, b); + const result = await matmul(a, b); - const target = new Float32Array( - [ - [120.0, 130.0, 140.0, 150.0], - [320.0, 355.0, 390.0, 425.0], - ].flat(), - ); + const target = new Float32Array( + [ + [120.0, 130.0, 140.0, 150.0], + [320.0, 355.0, 390.0, 425.0], + ].flat(), + ); - expectToBeCloseToArray(target, result.data); - }); - }); - - describe("rfft", () => { - it("non-power of 2", async () => { - const rows = 2; - const cols = 3; - const input = new Tensor("float32", range(rows * cols), [rows, cols]); - const dim = new Tensor("int64", [-1n], []); - const result = await rfft(input, dim); - - const target = new Float32Array( - [ - [ - [3, 0], - [-1.5, 0.8660262823104858], - ], - [ - [12, 0], - [-1.5, 0.866027295589447], - ], - ].flat(Infinity), - ); - - expectToBeCloseToArray(target, result.data); + expectToBeCloseToArray(target, result.data); + }); }); - it("power of 2", async () => { - const rows = 2; - const cols = 4; - const input = new Tensor("float32", range(rows * cols), [rows, cols]); - const dim = new Tensor("int64", [-1n], []); - - const result = await rfft(input, dim); - const target = new Float32Array( - [ - [ - [6, 0], - [-2, 2], - [-2, 0], - ], - [ - [22, 0], - [-2, 2], - [-2, 0], - ], - ].flat(Infinity), - ); - - expectToBeCloseToArray(target, result.data); + describe("rfft", () => { + it("non-power of 2", async () => { + const rows = 2; + const cols = 3; + const input = new Tensor("float32", range(rows * cols), [rows, cols]); + const dim = new Tensor("int64", [-1n], []); + const result = await rfft(input, dim); + + const target = new Float32Array( + [ + [ + [3, 0], + [-1.5, 0.8660262823104858], + ], + [ + [12, 0], + [-1.5, 0.866027295589447], + ], + ].flat(Infinity), + ); + + expectToBeCloseToArray(target, result.data); + }); + + it("power of 2", async () => { + const rows = 2; + const cols = 4; + const input = new Tensor("float32", range(rows * cols), [rows, cols]); + const dim = new Tensor("int64", [-1n], []); + + const result = await rfft(input, dim); + const target = new Float32Array( + [ + [ + [6, 0], + [-2, 2], + [-2, 0], + ], + [ + [22, 0], + [-2, 2], + [-2, 0], + ], + ].flat(Infinity), + ); + + expectToBeCloseToArray(target, result.data); + }); }); - }); }); diff --git a/tests/utils/utils.test.js b/tests/utils/utils.test.js index 8a1891f19..038ccb784 100644 --- a/tests/utils/utils.test.js +++ b/tests/utils/utils.test.js @@ -5,58 +5,58 @@ import { MAX_TEST_EXECUTION_TIME } from "../init.js"; import { compare } from "../test_utils.js"; describe("Utilities", () => { - describe("Audio utilities", () => { - it( - "should calculate MEL filters", - async () => { - // NOTE: Uses official HF implementation as reference: - const processor = await AutoProcessor.from_pretrained("openai/whisper-tiny.en"); - const config = processor.feature_extractor.config; - - // True MEL filters - const original_mel_filters = config.mel_filters; - - // Calculated MEL filters - const calculated_mel_filters = mel_filter_bank( - Math.floor(1 + config.n_fft / 2), // num_frequency_bins - config.feature_size, // num_mel_filters - 0.0, // min_frequency - 8000.0, // max_frequency - config.sampling_rate, // sampling_rate - "slaney", // norm - "slaney", // mel_scale + describe("Audio utilities", () => { + it( + "should calculate MEL filters", + async () => { + // NOTE: Uses official HF implementation as reference: + const processor = await AutoProcessor.from_pretrained("openai/whisper-tiny.en"); + const config = processor.feature_extractor.config; + + // True MEL filters + const original_mel_filters = config.mel_filters; + + // Calculated MEL filters + const calculated_mel_filters = mel_filter_bank( + Math.floor(1 + config.n_fft / 2), // num_frequency_bins + config.feature_size, // num_mel_filters + 0.0, // min_frequency + 8000.0, // max_frequency + config.sampling_rate, // sampling_rate + "slaney", // norm + "slaney", // mel_scale + ); + + const original = original_mel_filters.flat(); + const calculated = calculated_mel_filters.flat(); + + // Compute max difference + const maxdiff = original.reduce((maxdiff, _, i) => { + const diff = Math.abs(original[i] - calculated[i]); + return Math.max(maxdiff, diff); + }, -Infinity); + expect(maxdiff).toBeGreaterThanOrEqual(0); + expect(maxdiff).toBeLessThan(1e-6); + }, + MAX_TEST_EXECUTION_TIME, ); - const original = original_mel_filters.flat(); - const calculated = calculated_mel_filters.flat(); - - // Compute max difference - const maxdiff = original.reduce((maxdiff, _, i) => { - const diff = Math.abs(original[i] - calculated[i]); - return Math.max(maxdiff, diff); - }, -Infinity); - expect(maxdiff).toBeGreaterThanOrEqual(0); - expect(maxdiff).toBeLessThan(1e-6); - }, - MAX_TEST_EXECUTION_TIME, - ); - - it( - "should calculate window", - async () => { - compare(hanning(10), new Float64Array([0.0, 0.11697777844051105, 0.41317591116653485, 0.75, 0.9698463103929542, 0.9698463103929542, 0.75, 0.41317591116653485, 0.11697777844051105, 0.0])); - compare(hamming(10), new Float64Array([0.08000000000000002, 0.1876195561652702, 0.46012183827321207, 0.7700000000000001, 0.9722586055615179, 0.9722586055615179, 0.7700000000000001, 0.46012183827321207, 0.1876195561652702, 0.08000000000000002])); - }, - MAX_TEST_EXECUTION_TIME, - ); - }); - - describe("Hub utilities", () => { - it("Read data from blob", async () => { - const blob = new Blob(["Hello, world!"], { type: "text/plain" }); - const blobUrl = URL.createObjectURL(blob); - const data = await getFile(blobUrl); - expect(await data.text()).toBe("Hello, world!"); + it( + "should calculate window", + async () => { + compare(hanning(10), new Float64Array([0.0, 0.11697777844051105, 0.41317591116653485, 0.75, 0.9698463103929542, 0.9698463103929542, 0.75, 0.41317591116653485, 0.11697777844051105, 0.0])); + compare(hamming(10), new Float64Array([0.08000000000000002, 0.1876195561652702, 0.46012183827321207, 0.7700000000000001, 0.9722586055615179, 0.9722586055615179, 0.7700000000000001, 0.46012183827321207, 0.1876195561652702, 0.08000000000000002])); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("Hub utilities", () => { + it("Read data from blob", async () => { + const blob = new Blob(["Hello, world!"], { type: "text/plain" }); + const blobUrl = URL.createObjectURL(blob); + const data = await getFile(blobUrl); + expect(await data.text()).toBe("Hello, world!"); + }); }); - }); }); diff --git a/webpack.config.js b/webpack.config.js index 218249249..3dccea721 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -15,73 +15,75 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url)); * @returns {import('webpack').Configuration} One webpack target. */ function buildConfig({ - name = "", - suffix = ".js", - type = "module", // 'module' | 'commonjs' - ignoreModules = [], - externalModules = [], + name = "", + suffix = ".js", + type = "module", // 'module' | 'commonjs' + ignoreModules = [], + externalModules = [], } = {}) { - const outputModule = type === "module"; + const outputModule = type === "module"; - const alias = Object.fromEntries( - ignoreModules.map((module) => { - return [module, false]; - }), - ); + const alias = Object.fromEntries( + ignoreModules.map((module) => { + return [module, false]; + }), + ); - /** @type {import('webpack').Configuration} */ - const config = { - mode: 'development', - devtool: 'source-map', - entry: { - [`transformers${name}`]: './src/transformers.js', - [`transformers${name}.min`]: './src/transformers.js', - }, - output: { - filename: `[name]${suffix}`, - path: path.join(__dirname, 'dist'), - library: { - type, - }, - assetModuleFilename: '[name][ext]', - chunkFormat: 'module', - }, - optimization: { - minimize: true, - minimizer: [new TerserPlugin({ - test: new RegExp(`\\.min\\${suffix}$`), - extractComments: false, - })], - }, - experiments: { - outputModule, - }, - resolve: { alias }, + /** @type {import('webpack').Configuration} */ + const config = { + mode: "development", + devtool: "source-map", + entry: { + [`transformers${name}`]: "./src/transformers.js", + [`transformers${name}.min`]: "./src/transformers.js", + }, + output: { + filename: `[name]${suffix}`, + path: path.join(__dirname, "dist"), + library: { + type, + }, + assetModuleFilename: "[name][ext]", + chunkFormat: "module", + }, + optimization: { + minimize: true, + minimizer: [ + new TerserPlugin({ + test: new RegExp(`\\.min\\${suffix}$`), + extractComments: false, + }), + ], + }, + experiments: { + outputModule, + }, + resolve: { alias }, - externals: externalModules, + externals: externalModules, - // Development server - devServer: { - static: { - directory: __dirname, - }, - port: 8080, - }, - }; + // Development server + devServer: { + static: { + directory: __dirname, + }, + port: 8080, + }, + }; - if (outputModule) { - config.module = { - parser: { - javascript: { - importMeta: false - } - } + if (outputModule) { + config.module = { + parser: { + javascript: { + importMeta: false, + }, + }, + }; + } else { + config.externalsType = "commonjs"; } - } else { - config.externalsType = 'commonjs'; - } - return config; + return config; } // Do not bundle onnxruntime-web when packaging for Node.js. @@ -93,24 +95,23 @@ const NODE_IGNORE_MODULES = ["onnxruntime-web", "onnxruntime-web/webgpu"]; // and will be ignored when building for web (only used for node/deno) const NODE_EXTERNAL_MODULES = ["onnxruntime-node", "sharp", "fs", "path", "url"]; - export default [ - // Web-only build - buildConfig({ - type: "module", - }), + // Web-only build + buildConfig({ + type: "module", + }), - // Node-compatible builds - buildConfig({ - suffix: ".mjs", - type: "module", - ignoreModules: NODE_IGNORE_MODULES, - externalModules: NODE_EXTERNAL_MODULES, - }), - buildConfig({ - suffix: ".cjs", - type: "commonjs", - ignoreModules: NODE_IGNORE_MODULES, - externalModules: NODE_EXTERNAL_MODULES, - }), + // Node-compatible builds + buildConfig({ + suffix: ".mjs", + type: "module", + ignoreModules: NODE_IGNORE_MODULES, + externalModules: NODE_EXTERNAL_MODULES, + }), + buildConfig({ + suffix: ".cjs", + type: "commonjs", + ignoreModules: NODE_IGNORE_MODULES, + externalModules: NODE_EXTERNAL_MODULES, + }), ]; From 717a6aeb05fe092114907e432e239ad552f8bce5 Mon Sep 17 00:00:00 2001 From: Paul Dufour Date: Wed, 20 Nov 2024 01:30:54 +0000 Subject: [PATCH 2/4] Test linting failure --- src/models.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/models.js b/src/models.js index f701cbc9f..3ff4f83b0 100644 --- a/src/models.js +++ b/src/models.js @@ -182,7 +182,7 @@ async function getSession(pretrained_model_name_or_path, fileName, options) { // Overwrite `freeDimensionOverrides` if specified in config and not set in session options const free_dimension_overrides = custom_config.free_dimension_overrides; if (free_dimension_overrides) { - session_options.freeDimensionOverrides ??= free_dimension_overrides; + session_options.freeDimensionOverrides ??= free_dimension_overrides; } else if (selectedDevice.startsWith("webnn") && !session_options.freeDimensionOverrides) { console.warn( 'WebNN does not currently support dynamic shapes and requires `free_dimension_overrides` to be set in config.json as a field within "transformers.js_config". ' + From 7dbb619c10603e5272e6d9551cba7c6b383497a3 Mon Sep 17 00:00:00 2001 From: Paul Dufour Date: Wed, 20 Nov 2024 01:32:21 +0000 Subject: [PATCH 3/4] Fix linting failure --- src/models.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/models.js b/src/models.js index 3ff4f83b0..f701cbc9f 100644 --- a/src/models.js +++ b/src/models.js @@ -182,7 +182,7 @@ async function getSession(pretrained_model_name_or_path, fileName, options) { // Overwrite `freeDimensionOverrides` if specified in config and not set in session options const free_dimension_overrides = custom_config.free_dimension_overrides; if (free_dimension_overrides) { - session_options.freeDimensionOverrides ??= free_dimension_overrides; + session_options.freeDimensionOverrides ??= free_dimension_overrides; } else if (selectedDevice.startsWith("webnn") && !session_options.freeDimensionOverrides) { console.warn( 'WebNN does not currently support dynamic shapes and requires `free_dimension_overrides` to be set in config.json as a field within "transformers.js_config". ' + From 9f03f707ed5f30442db6e648ce7f2c45bd1509c5 Mon Sep 17 00:00:00 2001 From: Paul Dufour Date: Wed, 20 Nov 2024 01:35:03 +0000 Subject: [PATCH 4/4] remove ds_store -uncessary --- .prettierignore | 2 -- 1 file changed, 2 deletions(-) diff --git a/.prettierignore b/.prettierignore index 264b95b18..f2d7d83bc 100644 --- a/.prettierignore +++ b/.prettierignore @@ -8,5 +8,3 @@ types *.md # Ignore python related files .venv -# Mac files -.DS_STORE