Skip to content

Commit

Permalink
Add support for Split pretokenizer w/ behavior=removed & `invert=fa…
Browse files Browse the repository at this point in the history 
…lse`
  • Loading branch information
@xenova @pdufour
xenova authored and pdufour committed Nov 20, 2024
1 parent 7b1ce3c commit 5c2fcbe
Show file tree
Hide file tree
Showing 76 changed files with 31,003 additions and 30,798 deletions.
29 changes: 29 additions & 0 deletions .github/workflows/format-check.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
name: Prettier Check

on:
push:
branches: [ main ]
pull_request:
branches: [ main ]

jobs:
format:
runs-on: ubuntu-latest
strategy:
matrix:
node-version: [22]

steps:
- uses: actions/checkout@v4

- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node-version }}
cache: 'npm'

- name: Install dependencies
run: npm ci

- name: format:check
run: npm run format:check
4 changes: 4 additions & 0 deletions .prettierignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,3 +6,7 @@ examples
scripts
types
*.md
# Ignore python related files
.venv
# Mac files
.DS_STORE
4 changes: 3 additions & 1 deletion .prettierrc
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
{
"tabWidth": 4,
"printWidth": 120,
"overrides": [
{
"files": ["tests/**/*.js"],
"options": {
"printWidth": 10000000
"printWidth": 10000000
}
}
]
Expand Down
263 changes: 130 additions & 133 deletions jest.config.mjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,188 +4,185 @@
*/

export default {
// All imported modules in your tests should be mocked automatically
// automock: false,
// All imported modules in your tests should be mocked automatically
// automock: false,

// Stop running tests after `n` failures
// bail: 0,
// Stop running tests after `n` failures
// bail: 0,

// Automatically clear mock calls, instances, contexts and results before every test
clearMocks: true,
// Automatically clear mock calls, instances, contexts and results before every test
clearMocks: true,

// Indicates whether the coverage information should be collected while executing the test
collectCoverage: true,
// Indicates whether the coverage information should be collected while executing the test
collectCoverage: true,

// An array of glob patterns indicating a set of files for which coverage information should be collected
// collectCoverageFrom: undefined,
// An array of glob patterns indicating a set of files for which coverage information should be collected
// collectCoverageFrom: undefined,

// The directory where Jest should output its coverage files
coverageDirectory: "coverage",
// The directory where Jest should output its coverage files
coverageDirectory: "coverage",

// An array of regexp pattern strings used to skip coverage collection
coveragePathIgnorePatterns: [
"node_modules",
"tests",
],
// An array of regexp pattern strings used to skip coverage collection
coveragePathIgnorePatterns: ["node_modules", "tests"],

// Indicates which provider should be used to instrument code for coverage
coverageProvider: "v8",
// Indicates which provider should be used to instrument code for coverage
coverageProvider: "v8",

// A list of reporter names that Jest uses when writing coverage reports
// coverageReporters: [
// "json",
// "text",
// "lcov",
// "clover"
// ],
// A list of reporter names that Jest uses when writing coverage reports
// coverageReporters: [
// "json",
// "text",
// "lcov",
// "clover"
// ],

// An object that configures minimum threshold enforcement for coverage results
// coverageThreshold: undefined,
// An object that configures minimum threshold enforcement for coverage results
// coverageThreshold: undefined,

// A path to a custom dependency extractor
// dependencyExtractor: undefined,
// A path to a custom dependency extractor
// dependencyExtractor: undefined,

// Make calling deprecated APIs throw helpful error messages
// errorOnDeprecated: false,
// Make calling deprecated APIs throw helpful error messages
// errorOnDeprecated: false,

// The default configuration for fake timers
// fakeTimers: {
// "enableGlobally": false
// },
// The default configuration for fake timers
// fakeTimers: {
// "enableGlobally": false
// },

// Force coverage collection from ignored files using an array of glob patterns
// forceCoverageMatch: [],
// Force coverage collection from ignored files using an array of glob patterns
// forceCoverageMatch: [],

// A path to a module which exports an async function that is triggered once before all test suites
// globalSetup: undefined,
// A path to a module which exports an async function that is triggered once before all test suites
// globalSetup: undefined,

// A path to a module which exports an async function that is triggered once after all test suites
// globalTeardown: undefined,
// A path to a module which exports an async function that is triggered once after all test suites
// globalTeardown: undefined,

// A set of global variables that need to be available in all test environments
// globals: {},
// A set of global variables that need to be available in all test environments
// globals: {},

// The maximum amount of workers used to run your tests. Can be specified as % or a number. E.g. maxWorkers: 10% will use 10% of your CPU amount + 1 as the maximum worker number. maxWorkers: 2 will use a maximum of 2 workers.
// maxWorkers: "50%",
// The maximum amount of workers used to run your tests. Can be specified as % or a number. E.g. maxWorkers: 10% will use 10% of your CPU amount + 1 as the maximum worker number. maxWorkers: 2 will use a maximum of 2 workers.
// maxWorkers: "50%",

// An array of directory names to be searched recursively up from the requiring module's location
// moduleDirectories: [
// "node_modules"
// ],
// An array of directory names to be searched recursively up from the requiring module's location
// moduleDirectories: [
// "node_modules"
// ],

// An array of file extensions your modules use
// moduleFileExtensions: [
// "js",
// "mjs",
// "cjs",
// "jsx",
// "ts",
// "tsx",
// "json",
// "node"
// ],
// An array of file extensions your modules use
// moduleFileExtensions: [
// "js",
// "mjs",
// "cjs",
// "jsx",
// "ts",
// "tsx",
// "json",
// "node"
// ],

// A map from regular expressions to module names or to arrays of module names that allow to stub out resources with a single module
// moduleNameMapper: {},
// A map from regular expressions to module names or to arrays of module names that allow to stub out resources with a single module
// moduleNameMapper: {},

// An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader
// modulePathIgnorePatterns: [],
// An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader
// modulePathIgnorePatterns: [],

// Activates notifications for test results
// notify: false,
// Activates notifications for test results
// notify: false,

// An enum that specifies notification mode. Requires { notify: true }
// notifyMode: "failure-change",
// An enum that specifies notification mode. Requires { notify: true }
// notifyMode: "failure-change",

// A preset that is used as a base for Jest's configuration
// preset: undefined,
// A preset that is used as a base for Jest's configuration
// preset: undefined,

// Run tests from one or more projects
// projects: undefined,
// Run tests from one or more projects
// projects: undefined,

// Use this configuration option to add custom reporters to Jest
// reporters: undefined,
// Use this configuration option to add custom reporters to Jest
// reporters: undefined,

// Automatically reset mock state before every test
// resetMocks: false,
// Automatically reset mock state before every test
// resetMocks: false,

// Reset the module registry before running each individual test
// resetModules: false,
// Reset the module registry before running each individual test
// resetModules: false,

// A path to a custom resolver
// resolver: undefined,
// A path to a custom resolver
// resolver: undefined,

// Automatically restore mock state and implementation before every test
// restoreMocks: false,
// Automatically restore mock state and implementation before every test
// restoreMocks: false,

// The root directory that Jest should scan for tests and modules within
// rootDir: undefined,
// The root directory that Jest should scan for tests and modules within
// rootDir: undefined,

// A list of paths to directories that Jest should use to search for files in
roots: ["./tests/"],
// A list of paths to directories that Jest should use to search for files in
roots: ["./tests/"],

// Allows you to use a custom runner instead of Jest's default test runner
// runner: "jest-runner",
// Allows you to use a custom runner instead of Jest's default test runner
// runner: "jest-runner",

// The paths to modules that run some code to configure or set up the testing environment before each test
// setupFiles: [],
// The paths to modules that run some code to configure or set up the testing environment before each test
// setupFiles: [],

// A list of paths to modules that run some code to configure or set up the testing framework before each test
// setupFilesAfterEnv: [],
// A list of paths to modules that run some code to configure or set up the testing framework before each test
// setupFilesAfterEnv: [],

// The number of seconds after which a test is considered as slow and reported as such in the results.
// slowTestThreshold: 5,
// The number of seconds after which a test is considered as slow and reported as such in the results.
// slowTestThreshold: 5,

// A list of paths to snapshot serializer modules Jest should use for snapshot testing
// snapshotSerializers: [],
// A list of paths to snapshot serializer modules Jest should use for snapshot testing
// snapshotSerializers: [],

// The test environment that will be used for testing
// testEnvironment: "jest-environment-node",
// The test environment that will be used for testing
// testEnvironment: "jest-environment-node",

// Options that will be passed to the testEnvironment
// testEnvironmentOptions: {},
// Options that will be passed to the testEnvironment
// testEnvironmentOptions: {},

// Adds a location field to test results
// testLocationInResults: false,
// Adds a location field to test results
// testLocationInResults: false,

// The glob patterns Jest uses to detect test files
// testMatch: [
// "**/__tests__/**/*.[jt]s?(x)",
// "**/?(*.)+(spec|test).[tj]s?(x)"
// ],
// The glob patterns Jest uses to detect test files
// testMatch: [
// "**/__tests__/**/*.[jt]s?(x)",
// "**/?(*.)+(spec|test).[tj]s?(x)"
// ],

// An array of regexp pattern strings that are matched against all test paths, matched tests are skipped
// testPathIgnorePatterns: [
// "\\\\node_modules\\\\"
// ],
// An array of regexp pattern strings that are matched against all test paths, matched tests are skipped
// testPathIgnorePatterns: [
// "\\\\node_modules\\\\"
// ],

// The regexp pattern or array of patterns that Jest uses to detect test files
// testRegex: [],
// The regexp pattern or array of patterns that Jest uses to detect test files
// testRegex: [],

// This option allows the use of a custom results processor
// testResultsProcessor: undefined,
// This option allows the use of a custom results processor
// testResultsProcessor: undefined,

// This option allows use of a custom test runner
// testRunner: "jest-circus/runner",
// This option allows use of a custom test runner
// testRunner: "jest-circus/runner",

// A map from regular expressions to paths to transformers
transform: {},
// A map from regular expressions to paths to transformers
transform: {},

// An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation
// transformIgnorePatterns: [
// "\\\\node_modules\\\\",
// "\\.pnp\\.[^\\\\]+$"
// ],
// An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation
// transformIgnorePatterns: [
// "\\\\node_modules\\\\",
// "\\.pnp\\.[^\\\\]+$"
// ],

// An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them
// unmockedModulePathPatterns: undefined,
// An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them
// unmockedModulePathPatterns: undefined,

// Indicates whether each individual test should be reported during the run
// verbose: undefined,
// Indicates whether each individual test should be reported during the run
// verbose: undefined,

// An array of regexp patterns that are matched against all source file paths before re-running tests in watch mode
// watchPathIgnorePatterns: [],
// An array of regexp patterns that are matched against all source file paths before re-running tests in watch mode
// watchPathIgnorePatterns: [],

// Whether to use watchman for file crawling
// watchman: true,
// Whether to use watchman for file crawling
// watchman: true,
};
24 changes: 12 additions & 12 deletions jsconfig.json
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
{
// Only include files in the src directory
"include": ["src/**/*"],
"compilerOptions": {
// Tells the compiler to check JS files
"checkJs": true,
"target": "esnext",
"module": "nodenext",
"moduleResolution": "nodenext"
},
"typeAcquisition": {
"include": ["jest"]
}
// Only include files in the src directory
"include": ["src/**/*"],
"compilerOptions": {
// Tells the compiler to check JS files
"checkJs": true,
"target": "esnext",
"module": "nodenext",
"moduleResolution": "nodenext"
},
"typeAcquisition": {
"include": ["jest"]
}
}
Loading

0 comments on commit 5c2fcbe

Please sign in to comment.