diff --git a/.editorconfig b/.editorconfig index 61dca52fd..ad9caa755 100644 --- a/.editorconfig +++ b/.editorconfig @@ -28,9 +28,12 @@ trim_trailing_whitespace = true end_of_line = crlf trim_trailing_whitespace = false -[*.{g,gradle,json,xml,xsd,yml}] +[*.{fix,g,gradle,json,md,mwe2,sh,vim,xml,xsd,xtext,yml}] indent_size = 2 +[*.tmLanguage] +indent_style = tab + [metafacture-io/src/test/resources/org/metafacture/io/compressed.txt] insert_final_newline = false diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f9971375d..e3d4821c9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,6 +1,7 @@ name: Build on: push + jobs: build: runs-on: ubuntu-latest @@ -10,5 +11,12 @@ jobs: uses: actions/setup-java@v1 with: java-version: 11 - - name: Build with gradle + - name: Build with Gradle run: ./gradlew check + env: + METAFIX_LOG_PASSED: true + - name: Install language server + run: ./gradlew installServer + - name: Install fix extension + working-directory: metafix-vsc/ + run: npm install diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 000000000..d177a3f65 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,42 @@ + +name: Build and Deploy + +on: + push: + branches: + - deploy + +jobs: + build-and-deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Node.js + uses: actions/setup-node@v2 + - name: Set up vsce + run: npm install -g vsce + - name: Set up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: 11 + - name: Build with Gradle + run: | + chmod +x gradlew + ./gradlew build + ./gradlew installServer + - name: Create fix extension + working-directory: metafix-vsc/ + run: | + npm install + vsce package -o fix.vsix + - name: Pushes vsix to GitHub Pages + uses: dmnemec/copy_file_to_another_repo_action@v1.0.4 + env: + API_TOKEN_GITHUB: ${{ secrets.API_TOKEN_GITHUB }} + with: + source_file: 'metafix-vsc/fix.vsix' + destination_repo: 'metafacture/metafacture.github.io' + destination_branch: main + destination_folder: 'ide-extensions' + user_email: '${{ github.actor }}@users.noreply.github.com' + user_name: '${{ github.actor }}' diff --git a/.gitignore b/.gitignore index a3d6f2425..3a76a17f8 100644 --- a/.gitignore +++ b/.gitignore @@ -18,9 +18,9 @@ target/ # Ignore temporary gradle files: -.gradle +.gradle/ .gradletasknamecache -build +build/ # Ignore tmp directory tmp @@ -53,3 +53,17 @@ buildbot.keyring # Ignore files with sensitive data gradle.properties secring.gpg + +# Ignore metafacture-fix files +*.jfr +*.vsix +generated/ +node_modules/ +package-lock.json +xtext-gen/ +xtext-server/ +/metafix/src/test/resources/org/metafacture/metafix/integration/**/*.diff +/metafix/src/test/resources/org/metafacture/metafix/integration/**/*.err +/metafix/src/test/resources/org/metafacture/metafix/integration/**/*.out +/metafix/src/test/resources/org/metafacture/metafix/integration/**/output-* +!/metafix/src/test/resources/org/metafacture/metafix/integration/**/expected.err diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2676b1423..96b15e84e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -105,9 +105,9 @@ As a general rule, we don't change public commit history, i.e. we don’t use `` #### Code Formatting and Quality -Please format your code according to [this EditorConfig file](https://github.com/metafacture/metafacture-fix/blob/master/.editorconfig) and consider our current [code quality and style guidelines](https://github.com/metafacture/metafacture-core/wiki/Code-Quality-and-Style). +Please format your code according to [this EditorConfig file](https://github.com/metafacture/metafacture-core/blob/master/.editorconfig) and consider our current [code quality and style guidelines](https://github.com/metafacture/metafacture-core/wiki/Code-Quality-and-Style). -The [metafacture-fix build](https://github.com/metafacture/metafacture-fix/blob/master/build.gradle) performs automated [EditorConfig](https://github.com/metafacture/metafacture-fix/blob/master/.editorconfig) and [Checkstyle](https://github.com/metafacture/metafacture-fix/blob/master/config/checkstyle/checkstyle.xml) checks. +The [metafacture-core build](https://github.com/metafacture/metafacture-core/blob/master/build.gradle) performs automated [EditorConfig](https://github.com/metafacture/metafacture-core/blob/master/.editorconfig) and [Checkstyle](https://github.com/metafacture/metafacture-core/blob/master/config/checkstyle/checkstyle.xml) checks. The code is automatically [quality-checked on sonarcloud.io](https://sonarcloud.io/dashboard?id=org.metafacture%3Ametafacture-core) when pushed to GitHub. diff --git a/LICENSE b/LICENSE index e06d20818..8f71f43fe 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Apache License + Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ diff --git a/README.md b/README.md index 21058556b..f2cfcbcda 100644 --- a/README.md +++ b/README.md @@ -97,12 +97,1318 @@ Building metafacture-core from source is easy. All you need is git and JDK 11: $ ./gradlew assembleDist ``` +(To import the projects in Eclipse, choose `File > Import > Existing Gradle Project` and select the `metafacture-core` directory.) + See [Code Quality and Style](https://github.com/metafacture/metafacture-core/wiki/Code-Quality-and-Style) on the wiki for further information on the sources. +# Metafacture Fix + +Metafacture Fix (Metafix) is work in progress towards tools and an implementation of the Fix language for [Metafacture](https://metafacture.org/) as an alternative to configuring data transformations with [Metamorph](https://github.com/metafacture/metafacture-core/wiki#morph). Inspired by [Catmandu Fix](https://librecat.org/Catmandu/#fix-language), Metafix processes metadata not as a continuous data stream but as discrete records. The basic idea is to rebuild constructs from the (Catmandu) Fix language like [functions](https://librecat.org/Catmandu/#functions), [selectors](https://librecat.org/Catmandu/#selectors) and [binds](https://librecat.org/Catmandu/#binds) in Java and combine with additional functionalities from the Metamorph toolbox. + +See also [Fix Interest Group](https://github.com/elag/FIG) for an initiative towards an implementation-independent specification for the Fix Language. + +The project `metafix` contains the actual implementation of the Fix language as a Metafacture module and related components. It started as an [Xtext](#xtext) web project with a Fix grammar, from which a parser, a web editor, and a language server are generated. This project also contains an extension for VS code/codium based on that language server. (The web editor has effectively been replaced by the [Metafacture Playground](https://metafacture.org/playground), but remains here for its integration into the language server, which [we want to move over](https://github.com/metafacture/metafacture-playground/issues?q=is%3Aissue+language+server+is%3Aopen) to the playground.) + +## Usage + +The project `metafix` contains and uses a new `Metafix` stream module for Metafacture which plays the role of the `Metamorph` module in Fix-based Metafacture workflows. For the current implementation of the `Metafix` stream module see the tests in `metafix/src/test/java`. To play around with some examples, check out the [Metafacture Playground](https://metafacture.org/playground). For real-world usage samples see [openRub.fix](https://gitlab.com/oersi/oersi-etl/-/blob/master/data/production/openRub/openRub.fix) and [duepublico.fix](https://gitlab.com/oersi/oersi-etl/-/blob/master/data/production/duepublico/duepublico.fix). For reference documentation, see [Functions and cookbook](#functions-and-cookbook). + +### Extension + +The project `metafix-vsc` provides an extension for Visual Studio Code / Codium for `fix` via the language server protocol (LSP). In the current state the extension supports auto completion, simple syntax highlighting and auto closing brackets and quotes. This project was created using this [tutorial](https://www.typefox.io/blog/building-a-vs-code-extension-with-xtext-and-the-language-server-protocol) and the corresponding [example](https://github.com/TypeFox/languageserver-example). + +Build extension: + +> [!IMPORTANT] +> There is a problem when building the extension on Windows and installing the extension on a Linux system afterwards. In some cases the Xtext Server won't start. So if you want to use the extension not only on Windows, build the extension on a Linux system or on a Linux Subsystem on Windows. + +1. Install Visual Studio Code / alternative: VS Codium +2. Install Node.js (including npm) +3. In metafacture-core execute: +Unix: `./gradlew installServer` +Windows: `.\gradlew.bat installServer` +4. In `metafix-vsc/` execute (tip: if you use windows, install cygwin to execute npm commands): +`npm install` + +To start the extension in development mode (starting a second code/codium instance), follow A. To create a vsix file to install the extension permanently follow B. + +A) Run in dev mode: +1. Open `metafix-vsc/` in Visual Studio Code / Codium +2. Launch vscode extension by pressing F5 (opens new window of Visual Studio Code) +3. Open new file (file-ending `.fix`) or open existing fix-file (see sample below) + +B) Install vsix file: +1. Install vsce: `npm install -g vsce` +2. In `metafix-vsc/` execute: `vsce package` +vsce will create a vsix file in the vsc directory which can be used for installation: +3. Open VS Code / Codium +4. Click 'Extensions' section +5. Click menu bar and choose 'Install from VSIX...' + +### Web editor + +Start the web server: + +`./gradlew jettyRun` + +Visit [http://localhost:8080/](http://localhost:8080/), and paste this into the editor: + +``` +# Fix is a macro-language for data transformations + +# Simple fixes + +add_field("hello", "world") +remove_field("my.deep.nested.junk") +copy_field("stats", "output.$append") + +# Conditionals + +if exists("error") + set_field("is_valid", "no") + log("error") +elsif exists("warning") + set_field("is_valid", "yes") + log("warning") +else + set_field("is_valid", "yes") +end + +# Loops + +do list(path: "foo", "var": "$i") + add_field("$i.bar", "baz") +end +``` + +Content assist is triggered with Ctrl-Space. The input above is also used in `FixParsingTest.java`. + +Run workflows on the web server, passing `data`, `flux`, and `fix`: + +[http://localhost:8080/xtext-service/run?data='1'{'a': '5', 'z': 10}&flux=as-lines|decode-formeta|fix|encode-formeta(style="multiline")&fix=map(a,b) map(\_else)](http://localhost:8080/xtext-service/run?data=%271%27{%27a%27:%20%275%27,%20%27z%27:%2010}&flux=as-lines|decode-formeta|fix|encode-formeta(style=%22multiline%22)&fix=map(a,c)%20map(\_else)) + +## Functions and cookbook + +### Best practices and guidelines for working with Metafacture Fix + +- We recommend to use double quotation marks for arguments and values in functions, binds and conditionals. +- If using a `list` bind with a variable, the `var` option requires quotation marks (`do list(path: "", "var": "")`). +- Fix turns repeated fields into arrays internally but only marked arrays (with `[]` at the end of the field name) are also emitted as "arrays" (entities with indexed literals), all other arrays are emitted as repeated fields. +- Every Fix file should end with a final newline. + +### Glossary + +#### Array wildcards + +Array wildcards resemble [Catmandu's concept of wildcards](http://librecat.org/Catmandu/#wildcards). + +When working with arrays and repeated fields you can use wildcards instead of an index number to select elements of an array. + +| Wildcard | Meaning | +|----------|:--------| +| `*` | Selects _all_ elements of an array. | +| `$first` | Selects only the _first_ element of an array. | +| `$last` | Selects only the _last_ element of an array. | +| `$prepend` | Selects the position _before_ the first element of an array. Can only be used when adding new elements to an array. | +| `$append` | Selects the position _after_ the last element of an array. Can only be used when adding new elements to an array. | + +#### Path wildcards + +Path wildcards resemble [Metamorph's concept of wildcards](https://github.com/metafacture/metafacture-core/wiki/Metamorph-User-Guide#addressing-pieces-of-data). They are not supported in Catmandu (it has [specialized Fix functions](https://librecat.org/Catmandu/#marc-mab-pica-paths) instead). + +You can use path wildcards to select fields matching a pattern. They only match path _segments_ (field names), though, not _whole_ paths of nested fields. These wildcards cannot be used to add new elements. + +| Wildcard | Meaning | +|----------|:--------| +| `*` | Placeholder for zero or more characters. | +| `?` | Placeholder for exactly one character. | +| `\|` | Alternation of multiple patterns. | +| `[...]` | Enumeration of characters. | + +### Functions + +#### Script-level functions + +##### `include` + +Includes a Fix file and executes it as if its statements were written in place of the function call. + +Parameters: + +- `path` (required): Path to Fix file (if the path starts with a `.`, it is resolved relative to the including file's directory; otherwise, it is resolved relative to the current working directory). + +Options: + +- All options are made available as "dynamic" local variables in the included Fix file. + +```perl +include(""[, ...]) +``` + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+include+{") + +##### `log` + +Sends a message to the logs. + +Parameters: + +- `logMessage` (required): Message to log. + +Options: + +- `level`: Log level to log at (one of `DEBUG`, `INFO`, `WARN` or `ERROR`). (Default: `INFO`) + +```perl +log(""[, level: ""]) +``` + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+log+{") + +##### `nothing` + +Does nothing. It is used for benchmarking in Catmandu. + +```perl +nothing() +``` + +[Example in Playground](https://metafacture.org/playground/?example=nothing) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+nothing+{") + +##### `put_filemap` + +Defines an external map for [lookup](#lookup) from a file or a URL. Maps with more than 2 columns are supported but are reduced to a defined key and a value column. + +```perl +put_filemap("", "", sep_char: "\t") +``` + +[Example in Playground](https://metafacture.org/playground/?example=put_filemap) + +The separator (`sep_char`) will vary depending on the source file, e.g.: + +| Type | Separator | +|------|------------| +| CSV | `,` or `;` | +| TSV | `\t` | + +Options: + +- `allow_empty_values`: Sets whether to allow empty values in the filemap or to ignore these entries. (Default: `false`) +- `compression`: Sets the compression of the file. +- `decompress_concatenated`: Flags whether to use decompress concatenated file compression. +- `encoding`: Sets the encoding used to open the resource. +- `expected_columns`: Sets number of expected columns; lines with different number of columns are ignored. Set to `-1` to disable the check and allow arbitrary number of columns. (Default: `2`) +- `key_column`: Defines the column to be used for keys. Uses zero index. (Default: `0`) +- `value_column`: Defines the column to be used for values. Uses zero index. (Default: `1`) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+put_filemap+{") + +##### `put_map` + +Defines an internal map for [lookup](#lookup) from key/value pairs. + +```perl +put_map("", + "dog": "mammal", + "parrot": "bird", + "shark": "fish" +) +``` + +[Example in Playground](https://metafacture.org/playground/?example=put_map) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+put_map+{") + +##### `put_rdfmap` + +Defines an external RDF map for lookup from a file or an HTTP(S) resource. +As the RDF map is reducing RDF triples to a key/value map it is mandatory to set the target. +The targeted RDF property can optionally be bound by an RDF language tag. + +```perl +put_rdfmap("", "", target: "") +put_rdfmap("", "", target: "", select_language: "") +``` + +[Example in Playground](https://metafacture.org/playground/?example=put_rdfmap) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+put_rdfmap+{") + +##### `put_var` + +Defines a single global variable that can be referenced with `$[]`. + +```perl +put_var("", "") +``` + +[Example in Playground](https://metafacture.org/playground/?example=put_var) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+put_var+{") + +##### `put_vars` + +Defines multiple global variables that can be referenced with `$[]`. + +```perl +put_vars( + "": "", + "": "" +) +``` + +[Example in Playground](https://metafacture.org/playground/?example=put_vars) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+put_vars+{") + +##### `to_var` + +Defines a single global variable that can be referenced with `$[]` and assigns the value of the ``. + +```perl +to_var("", "") +``` + +Options: + +- `default`: Default value if source field does not exist. The option needs to be written in quotation marks because it is a reserved word in Java. (Default: Empty string) + +[Example in Playground](https://metafacture.org/playground/?example=to_var) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+to_var+{") + +#### Record-level functions + +##### `add_array` + +Creates a new array (with optional values). + +```perl +add_array("") +add_array("", ""[, ...]) +``` + +[Example in Playground](https://metafacture.org/playground/?example=add_array) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+add_array+{") + +##### `add_field` + +Creates a field with a defined value. + +```perl +add_field("", "") +``` + +[Example in Playground](https://metafacture.org/playground/?example=add_field) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+add_field+{") + +##### `add_hash` + +Creates a new hash (with optional values). + +```perl +add_hash("") +add_hash("", "subfieldName": ""[, ...]) +``` + +[Example in Playground](https://metafacture.org/playground/?example=add_hash) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+add_hash+{") + +##### `array` + +Converts a hash/object into an array. + +```perl +array("") +``` + +E.g.: + +```perl +array("foo") +# {"name":"value"} => ["name", "value"] +``` + +[Example in Playground](https://metafacture.org/playground/?example=array) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+array+{") + +##### `call_macro` + +Calls a named macro, i.e. a list of statements that have been previously defined with the [`do put_macro`](#do-put_macro) bind. + +Parameters: + +- `name` (required): Unique name of the macro. + +Options: + +- All options are made available as "dynamic" local variables in the macro. + +```perl +do put_macro(""[, ...]) + ... +end +call_macro(""[, ...]) +``` + +[Example in Playground](https://metafacture.org/playground/?example=call_macro) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+call_macro+{") + +##### `copy_field` + +Copies a field from an existing field. + +```perl +copy_field("", "") +``` + +[Example in Playground](https://metafacture.org/playground/?example=copy_field) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+copy_field+{") + +##### `format` + +Replaces the value with a formatted (`sprintf`-like) version. + +---- TODO: THIS NEEDS MORE CONTENT ----- + +```perl +format("", "") +``` + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+format+{") + +##### `hash` + +Converts an array into a hash/object. + +```perl +hash("") +``` + +E.g.: +```perl +hash("foo") +# ["name", "value"] => {"name":"value"} +``` + +[Example in Playground](https://metafacture.org/playground/?example=hash) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+hash+{") + +##### `move_field` + +Moves a field from an existing field. Can be used to rename a field. + +```perl +move_field("", "") +``` + +[Example in Playground](https://metafacture.org/playground/?example=move_field) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+move_field+{") + +##### `parse_text` + +Parses a text into an array or hash of values. + +---- TODO: THIS NEEDS MORE CONTENT ----- + +```perl +parse_text("", "") +``` + +[Example in Playground](https://metafacture.org/playground/?example=parse_text) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+parse_text+{") + +##### `paste` + +Joins multiple field values into a new field. Can be combined with additional literal strings. + +The default `join_char` is a single space. Literal strings have to start with `~`. + +```perl +paste("", ""[, ...][, "join_char": ", "]) +``` + +E.g.: + +```perl +# a: eeny +# b: meeny +# c: miny +# d: moe +paste("my.string", "~Hi", "a", "~how are you?") +# "my.string": "Hi eeny how are you?" +``` + +[Example in Playground](https://metafacture.org/playground/?example=paste) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+paste+{") + +##### `print_record` + +Prints the current record as JSON either to standard output or to a file. + +Parameters: + +- `prefix` (optional): Prefix to print before the record; may include [format directives](https://docs.oracle.com/javase/8/docs/api/java/util/Formatter.html#syntax) for counter and record ID (in that order). (Default: Empty string) + +Options: + +- `append`: Whether to open files in append mode if they exist. (Default: `false`) +- `compression` (file output only): Compression mode. (Default: `auto`) +- `destination`: Destination to write the record to; may include [format directives](https://docs.oracle.com/javase/8/docs/api/java/util/Formatter.html#syntax) for counter and record ID (in that order). (Default: `stdout`) +- `encoding` (file output only): Encoding used by the underlying writer. (Default: `UTF-8`) +- `footer`: Footer which is written at the end of the output. (Default: `\n`) +- `header`: Header which is written at the beginning of the output. (Default: Empty string) +- `id`: Field name which contains the record ID; if found, will be available for inclusion in `prefix` and `destination`. (Default: `_id`) +- `internal`: Whether to print the record's internal representation instead of JSON. (Default: `false`) +- `pretty`: Whether to use pretty printing. (Default: `false`) +- `separator`: Separator which is written after the record. (Default: `\n`) + +```perl +print_record([""][, ...]) +``` + +E.g.: + +```perl +print_record("%d) Before transformation: ") +print_record(destination: "record-%2$s.json", id: "001", pretty: "true") +print_record(destination: "record-%03d.json.gz", header: "After transformation: ") +``` + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+print_record+{") + +##### `random` + +Creates (or replaces) a field with a random number (less than the specified maximum). + +```perl +random("", "") +``` + +[Example in Playground](https://metafacture.org/playground/?example=random) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+random+{") + +##### `remove_field` + +Removes a field. + +```perl +remove_field("") +``` + +[Example in Playground](https://metafacture.org/playground/?example=remove_field) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+remove_field+{") + +##### `rename` + +Replaces a regular expression pattern in subfield names of a field. Does not change the name of the source field itself. + +```perl +rename("", "", "") +``` + +[Example in Playground](https://metafacture.org/playground/?example=rename) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+rename+{") + +##### `retain` + +Deletes all fields except the ones listed (incl. subfields). + +```perl +retain(""[, ...]) +``` + +[Example in Playground](https://metafacture.org/playground/?example=retain) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+retain+{") + +##### `set_array` + +_Currently alias for [`add_array`](#add_array)._ + +We advise you to use [`add_array`](#add_array) instead of `set_array` due to changing behaviour in an upcoming release. For more information see: [#309](https://github.com/metafacture/metafacture-fix/issues/309) + +##### `set_field` + +_Currently alias for [`add_field`](#add_field)._ + +We advise you to use [`add_field`](#add_field) instead of `set_field` due to changing behaviour in an upcoming release. For more information see: [#309](https://github.com/metafacture/metafacture-fix/issues/309) + +##### `set_hash` + +_Currently alias for [`add_hash`](#add_hash)._ + +We advise you to use [`add_hash`](#add_hash) instead of `set_hash` due to changing behaviour in an upcoming release. For more information see: [#309](https://github.com/metafacture/metafacture-fix/issues/309) + +##### `timestamp` + +Creates (or replaces) a field with the current timestamp. + +Options: + +- `format`: Date and time pattern as in [java.text.SimpleDateFormat](https://docs.oracle.com/javase/8/docs/api/java/text/SimpleDateFormat.html). (Default: `timestamp`) +- `timezone`: Time zone as in [java.util.TimeZone](https://docs.oracle.com/javase/8/docs/api/java/util/TimeZone.html). (Default: `UTC`) +- `language`: Language tag as in [java.util.Locale](https://docs.oracle.com/javase/8/docs/api/java/util/Locale.html). (Default: The locale of the host system) + +```perl +timestamp(""[, format: ""][, timezone: ""][, language: ""]) +``` + +[Example in Playground](https://metafacture.org/playground/?example=timestamp) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+timestamp+{") + +##### `vacuum` + +Deletes empty fields, arrays and objects. + +```perl +vacuum() +``` + +[Example in Playground](https://metafacture.org/playground/?example=vacuum) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+vacuum+{") + +#### Field-level functions + +##### `append` + +Adds a string at the end of a field value. + +```perl +append("", "") +``` + +[Example in Playground](https://metafacture.org/playground/?example=append) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+append+{") + +##### `capitalize` + +Upcases the first character in a field value. + +```perl +capitalize("") +``` + +[Example in Playground](https://metafacture.org/playground/?example=capitalize) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+capitalize+{") + +##### `count` + +Counts the number of elements in an array or a hash and replaces the field value with this number. + +```perl +count("") +``` + +[Example in Playground](https://metafacture.org/playground/?example=count) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+count+{") + +##### `downcase` + +Downcases all characters in a field value. + +```perl +downcase("") +``` + +[Example in Playground](https://metafacture.org/playground/?example=downcase) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+downcase+{") + +##### `filter` + +Only keeps field values that match the regular expression pattern. Works only with array of strings/repeated fields. + +```perl +filter("", "") +``` + +[Example in Playground](https://metafacture.org/playground/?example=filter) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+filter+{") + +##### `flatten` + +Flattens a nested array field. + +```perl +flatten("") +``` + +[Example in Playground](https://metafacture.org/playground/?example=flatten) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+flatten+{") + +##### `from_json` + +Replaces the string with its JSON deserialization. + +Options: + +- `error_string`: Error message as a placeholder if the JSON couldn't be parsed. (Default: `null`) + +```perl +from_json(""[, error_string: ""]) +``` + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+from_json+{") + +##### `index` + +Returns the index position of a substring in a field and replaces the field value with this number. + +```perl +index("", "") +``` + +[Example in Playground](https://metafacture.org/playground/?example=index) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+index+{") + +##### `isbn` + +Extracts an ISBN and replaces the field value with the normalized ISBN; optionally converts and/or validates the ISBN. + +Options: + +- `to`: ISBN format to convert to (either `ISBN10` or `ISBN13`). (Default: Only normalize ISBN) +- `verify_check_digit`: Whether the check digit should be verified. (Default: `false`) +- `error_string`: Error message as a placeholder if the ISBN couldn't be validated. (Default: `null`) + +```perl +isbn(""[, to: ""][, verify_check_digit: ""][, error_string: ""]) +``` + +[Example in Playground](https://metafacture.org/playground/?example=isbn) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+isbn+{") + +##### `join_field` + +Joins an array of strings into a single string. + +```perl +join_field("", "") +``` + +[Example in Playground](https://metafacture.org/playground/?example=join_field) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+join_field+{") + +##### `lookup` + +Looks up matching values in a map and replaces the field value with this match. [External files](#put_filemap), [internal maps](#put_map) as well as [RDF resources](#put_rdfmap) can be used. + +Parameters: + +- `path` (required): Field path to look up. +- `map` (optional): Name or path of the map in which to look up values. + +Options: + +- `default`: Default value to use for unknown values. The option needs to be written in quotation marks because it is a reserved word in Java. (Default: Old value) +- `delete`: Whether to delete unknown values. (Default: `false`) +- `print_unknown`: Whether to print unknown values. (Default: `false`) + +Additional options when printing unknown values: + +- `append`: Whether to open files in append mode if they exist. (Default: `true`) +- `compression` (file output only): Compression mode. (Default: `auto`) +- `destination`: Destination to write unknown values to; may include [format directives](https://docs.oracle.com/javase/8/docs/api/java/util/Formatter.html#syntax) for counter and record ID (in that order). (Default: `stdout`) +- `encoding` (file output only): Encoding used by the underlying writer. (Default: `UTF-8`) +- `footer`: Footer which is written at the end of the output. (Default: `\n`) +- `header`: Header which is written at the beginning of the output. (Default: Empty string) +- `id`: Field name which contains the record ID; if found, will be available for inclusion in `prefix` and `destination`. (Default: `_id`) +- `prefix`: Prefix to print before the unknown value; may include [format directives](https://docs.oracle.com/javase/8/docs/api/java/util/Formatter.html#syntax) for counter and record ID (in that order). (Default: Empty string) +- `separator`: Separator which is written after the unknown value. (Default: `\n`) + +```perl +lookup(""[, ][, ...]) +``` + +E.g.: + +```perl +# local (unnamed) map +lookup("path.to.field", key_1: "value_1", ...) + +# internal (named) map +put_map("internal-map", key_1: "value_1", ...) +lookup("path.to.field", "internal-map") + +# external file map (implicit) +lookup("path.to.field", "path/to/file", sep_char: ";") + +# external file map (explicit) +put_filemap("path/to/file", "file-map", sep_char: ";") +lookup("path.to.field", "file-map") + +# RDF map (explicit) +put_rdfmap("path/to/file", "rdf-map", target: "") +lookup("path.to.field", "rdf-map") + +# with default value +lookup("path.to.field", "map-name", "default": "NA") + +# with printing unknown values to a file +lookup("path.to.field", "map-name", print_unknown: "true", destination: "unknown.txt") +``` + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+lookup+{") + +##### `prepend` + +Adds a string at the beginning of a field value. + +```perl +prepend("", "") +``` + +[Example in Playground](https://metafacture.org/playground/?example=prepend) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+prepend+{") + +##### `replace_all` + +Replaces a regular expression pattern in field values with a replacement string. Regexp capturing is possible; refer to capturing groups by number (`$`) or name (`${}`). + +```perl +replace_all("", "", "") +``` + +[Example in Playground](https://metafacture.org/playground/?example=replace_all) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+replace_all+{") + +##### `reverse` + +Reverses the character order of a string or the element order of an array. + +```perl +reverse("") +``` + +[Example in Playground](https://metafacture.org/playground/?example=reverse) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+reverse+{") + +##### `sort_field` + +Sorts strings in an array. Alphabetically and A-Z by default. Optional numerical and reverse sorting. + +```perl +sort_field("") +sort_field("", reverse: "true") +sort_field("", numeric: "true") +``` + +[Example in Playground](https://metafacture.org/playground/?example=sort_field) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+sort_field+{") + +##### `split_field` + +Splits a string into an array and replaces the field value with this array. + +```perl +split_field("", "") +``` + +[Example in Playground](https://metafacture.org/playground/?example=split_field) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+split_field+{") + +##### `substring` + +Replaces a string with its substring as defined by the start position (offset) and length. + +```perl +substring("", "", "") +``` + +[Example in Playground](https://metafacture.org/playground/?example=substring) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+substring+{") + +##### `sum` + +Sums numbers in an array and replaces the field value with this number. + +```perl +sum("") +``` + +[Example in Playground](https://metafacture.org/playground/?example=sum) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+sum+{") + +##### `to_json` + +Replaces the value with its JSON serialization. + +Options: + +- `error_string`: Error message as a placeholder if the JSON couldn't be generated. (Default: `null`) +- `pretty`: Whether to use pretty printing. (Default: `false`) + +```perl +to_json(""[, pretty: ""][, error_string: ""]) +``` + +[Example in Playground](https://metafacture.org/playground/?example=to_json) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+to_json+{") + +##### `to_base64` + +Replaces the value with its Base64 encoding. + +Options: + +-`url_safe`: Perform URL-safe encoding (uses Base64URL format). (Default: `false`) + +```perl +to_base64(""[, url_safe: ""]) +``` + +[Example in Playground](https://metafacture.org/playground/?example=to_base64) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+to_base64+{") + +##### `trim` + +Deletes whitespace at the beginning and the end of a field value. + +```perl +trim("") +``` + +[Example in Playground](https://metafacture.org/playground/?example=trim) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+trim+{") + +##### `uniq` + +Deletes duplicate values in an array. + +```perl +uniq("") +``` + +[Example in Playground](https://metafacture.org/playground/?example=uniq) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+uniq+{") + +##### `upcase` + +Upcases all characters in a field value. + +```perl +upcase("") +``` + +[Example in Playground](https://metafacture.org/playground/?example=upcase) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+upcase+{") + +##### `uri_encode` + +Encodes a field value as URI. Aka percent-encoding. + +Options: + +- `plus_for_space`: Sets whether "space" (` `) will be substituted by a "plus" (`+`) or be percent escaped (`%20`). (Default: `true`) +- `safe_chars`: Sets characters that won't be escaped. Safe characters are the ranges 0..9, a..z and A..Z. These are always safe and should not be specified. (Default: `.-*_`) + +```perl +uri_encode(""[, ...]) +``` + +E.g.: + +```perl +uri_encode("path.to.field", plus_for_space:"false", safe_chars:"") +``` + +[Example in Playground](https://metafacture.org/playground/?example=uri_encode) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+uri_encode+{") + +### Selectors + +#### `reject` + +Ignores records that match a condition. + +```perl +if + reject() +end +``` + +[Example in Playground](https://metafacture.org/playground/?example=reject) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixMethod.java+"+reject+{") + +### Binds + +#### `do list` + +Iterates over each element of an array. In contrast to Catmandu, it can also iterate over a single object or string. + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixBind.java+"+list+{") + +```perl +do list(path: "") + ... +end +``` + +[Example in Playground](https://metafacture.org/playground/?example=do_list) + +Only the current element is accessible in this case (as the root element). + +When specifying a variable name for the current element, the record remains accessible as the root element and the current element is accessible through the variable name: + +```perl +do list(path: "", "var": "") + ... +end +``` + +[Example in Playground](https://metafacture.org/playground/?example=do_list_with_var) + +#### `do list_as` + +Iterates over each _named_ element of an array (like [`do list`](#do-list) with a variable name). If multiple arrays are given, iterates over the _corresponding_ elements from each array (i.e., all elements with the same array index, skipping elements whose arrays have already been exhausted). + +[Example in Playground](https://metafacture.org/playground/?example=do+list_as) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixBind.java+"+list_as+{") + +```perl +do list_as(element_1: ""[, ...]) + ... +end +``` + +E.g.: + +```perl +# "ccm:university":["https://ror.org/0304hq317"] +# "ccm:university_DISPLAYNAME":["Gottfried Wilhelm Leibniz Universität Hannover"] +set_array("sourceOrga[]") +do list_as(orgId: "ccm:university[]", orgName: "ccm:university_DISPLAYNAME[]") + copy_field(orgId, "sourceOrga[].$append.id") + copy_field(orgName, "sourceOrga[].$last.name") +end +# {"sourceOrga":[{"id":"https://ror.org/0304hq317","name":"Gottfried Wilhelm Leibniz Universität Hannover"}]} +``` + +#### `do once` + +Executes the statements only once (when the bind is first encountered), not repeatedly for each record. + +```perl +do once() + ... +end +``` + +[Example in Playground](https://metafacture.org/playground/?example=do_once) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixBind.java+"+once+{") + +In order to execute multiple blocks only once, tag them with unique identifiers: + +```perl +do once("maps setup") + ... +end +do once("vars setup") + ... +end +``` + +#### `do put_macro` + +Defines a named macro, i.e. a list of statements that can be executed later with the [`call_macro`](#call_macro) function. + +Variables can be referenced with `$[]`, in the following order of precedence: + +1. "dynamic" local variables, passed as options to the `call_macro` function; +2. "static" local variables, passed as options to the `do put_macro` bind; +3. global variables, defined via [`put_var`](#put_var)/[`put_vars`](#put_vars). + +Parameters: + +- `name` (required): Unique name of the macro. + +Options: + +- All options are made available as "static" local variables in the macro. + +```perl +do put_macro(""[, ...]) + ... +end +call_macro(""[, ...]) +``` + +[Example in Playground](https://metafacture.org/playground/?example=do_put_macro) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixBind.java+"+put_macro+{") + +### Conditionals + +Conditionals start with `if` in case of affirming the condition or `unless` rejecting the condition. + +Conditionals require a final `end`. + +Additional conditionals can be set with `elsif` and `else`. + +```perl +if + ... +end +``` + +```perl +unless + ... +end +``` + +```perl +if + ... +elsif + ... +else + ... +end +``` + +#### `contain` + +##### `all_contain` + +Executes the functions if/unless the field contains the value. If it is an array or a hash all field values must contain the string. + +[Example in Playground](https://metafacture.org/playground/?example=all_contain) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixConditional.java+"+all_contain+{") + +##### `any_contain` + +Executes the functions if/unless the field contains the value. If it is an array or a hash one or more field values must contain the string. + +[Example in Playground](https://metafacture.org/playground/?example=any_contain) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixConditional.java+"+any_contain+{") + +##### `none_contain` + +Executes the functions if/unless the field does not contain the value. If it is an array or a hash none of the field values may contain the string. + +[Example in Playground](https://metafacture.org/playground/?example=none_contain) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixConditional.java+"+none_contain+{") + +##### `str_contain` + +Executes the functions if/unless the first string contains the second string. + +[Example in Playground](https://metafacture.org/playground/?example=str_contain) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixConditional.java+"+str_contain+{") + +#### `equal` + +##### `all_equal` + +Executes the functions if/unless the field value equals the string. If it is an array or a hash all field values must equal the string. + +[Example in Playground](https://metafacture.org/playground/?example=all_equal) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixConditional.java+"+all_equal+{") + +##### `any_equal` + +Executes the functions if/unless the field value equals the string. If it is an array or a hash one or more field values must equal the string. + +[Example in Playground](https://metafacture.org/playground/?example=any_equal) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixConditional.java+"+any_equal+{") + +##### `none_equal` + +Executes the functions if/unless the field value does not equal the string. If it is an array or a hash none of the field values may equal the string. + +[Example in Playground](https://metafacture.org/playground/?example=none_equal) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixConditional.java+"+none_equal+{") + +##### `str_equal` + +Executes the functions if/unless the first string equals the second string. + +[Example in Playground](https://metafacture.org/playground/?example=str_equal) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixConditional.java+"+str_equal+{") + +#### `exists` + +Executes the functions if/unless the field exists. + +```perl +if exists("") +``` + +[Example in Playground](https://metafacture.org/playground/?example=exists) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixConditional.java+"+exists+{") + +#### `in` + +Executes the functions if/unless the field value [is contained in](https://perldoc.perl.org/perlop#Smartmatch-Operator) the value of the other field. + +_Also aliased as [`is_contained_in`](#is_contained_in)._ + +[Example in Playground](https://metafacture.org/playground/?example=in) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixConditional.java+"+in+{") + +#### `is_contained_in` + +_Alias for [`in`](#in)._ + +#### `is_array` + +Executes the functions if/unless the field value is an array. + +[Example in Playground](https://metafacture.org/playground/?example=is_array) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixConditional.java+"+is_array+{") + +#### `is_empty` + +Executes the functions if/unless the field value is empty. + +[Example in Playground](https://metafacture.org/playground/?example=is_empty) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixConditional.java+"+is_empty+{") + +#### `is_false` + +Executes the functions if/unless the field value equals `false` or `0`. + +[Example in Playground](https://metafacture.org/playground/?example=is_false) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixConditional.java+"+is_false+{") + +#### `is_hash` + +_Alias for [`is_object`](#is_object)._ + +[Example in Playground](https://metafacture.org/playground/?example=is_hash) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixConditional.java+"+is_hash+{") + +#### `is_number` + +Executes the functions if/unless the field value is a number. + +[Example in Playground](https://metafacture.org/playground/?example=is_number) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixConditional.java+"+is_number+{") + +#### `is_object` + +Executes the functions if/unless the field value is a hash (object). + +_Also aliased as [`is_hash`](#is_hash)._ + +#### `is_string` + +Executes the functions if/unless the field value is a string (and not a number). + +[Example in Playground](https://metafacture.org/playground/?example=is_string) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixConditional.java+"+is_string+{") + +#### `is_true` + +Executes the functions if/unless the field value equals `true` or `1`. + +[Example in Playground](https://metafacture.org/playground/?example=is_true) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixConditional.java+"+is_true+{") + + +#### `match` + +##### `all_match` + +Executes the functions if/unless the field value matches the regular expression pattern. If it is an array or a hash all field values must match the regular expression pattern. + +[Example in Playground](https://metafacture.org/playground/?example=all_match) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixConditional.java+"+all_match+{") + +##### `any_match` + +Executes the functions if/unless the field value matches the regular expression pattern. If it is an array or a hash one or more field values must match the regular expression pattern. + +[Example in Playground](https://metafacture.org/playground/?example=any_match) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixConditional.java+"+any_match+{") + +##### `none_match` + +Executes the functions if/unless the field value does not match the regular expression pattern. If it is an array or a hash none of the field values may match the regular expression pattern. + +[Example in Playground](https://metafacture.org/playground/?example=none_match) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixConditional.java+"+none_match+{") + +##### `str_match` + +Executes the functions if/unless the string matches the regular expression pattern. + +[Example in Playground](https://metafacture.org/playground/?example=str_match) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-core+path:FixConditional.java+"+str_match+{") + +## Xtext + +The Metafix projects have been originally set up with [Xtext](https://www.eclipse.org/Xtext/) 2.17.0 and Eclipse for Java 2019-03, following [https://www.eclipse.org/Xtext/documentation/104_jvmdomainmodel.html](https://www.eclipse.org/Xtext/documentation/104_jvmdomainmodel.html). + # Stay updated For support and discussion join the [mailing list](http://lists.dnb.de/mailman/listinfo/metafacture). diff --git a/build.gradle b/build.gradle index 37dcbe95e..3a08dde1c 100644 --- a/build.gradle +++ b/build.gradle @@ -20,21 +20,34 @@ plugins { id 'org.ajoberstar.grgit' version '2.3.0' //id 'org.ec4j.editorconfig' version '0.0.3' id 'org.sonarqube' version '2.6.2' + id 'org.xtext.builder' version '4.0.0' id 'io.github.gradle-nexus.publish-plugin' version '1.3.0' } subprojects { ext { versions = [ - 'assertj_core': '3.11.1', + 'ace': '1.3.3', + 'antlr': '3.2', + 'assertj_core': '3.11.1', 'commons_compress': '1.21', - 'guava': '32.0.1-jre', + 'equalsverifier': '3.8.2', + 'guava': '32.0.1-jre', + 'jackson': '2.13.3', 'jackson_databind': '2.15.1', - 'jdk': '11', - 'junit': '4.12', - 'mockito': '2.27.0', - 'slf4j': '1.7.36', - 'wiremock_jre': '2.35.0' + 'jdk': '11', + 'jena': '3.17.0', + 'jetty': '9.4.14.v20181114', + 'jquery': '3.3.1-1', + 'junit': '4.12', + 'junit_jupiter': '5.8.2', + 'junit_platform': '1.4.2', + 'mockito': '2.27.0', + 'requirejs': '2.3.6', + 'slf4j': '1.7.36', + 'slf4j_mock': '2.3.0', + 'wiremock': '2.35.0', + 'xtext': '2.26.0' ] } } @@ -61,10 +74,21 @@ project(':metafacture-runner') { // '**/*.bgzf', // '**/*.bz2', // '**/*.bzip2', +// '**/*.diff', +// '**/*.err', // '**/*.gzip', +// '**/*.jfr', +// '**/*.out', +// '**/*.vsix', // '**/*.xz', // '**/.*', // '**/bin', +// '**/generated', +// '**/includeBenchmark*100000/expected.json', +// '**/node_modules', +// '**/out', +// '**/output-*', +// '**/xtext-gen', // 'gradlew*' // ] //} @@ -74,12 +98,19 @@ task editorconfigChecker(type: Exec, group: 'Verification') { args('-exclude', '/\\.|/bin/|\\.beacon$|\\.txt\\.|^LICENSE$|^\\.project$|^gradlew.*') } +task installServer(type: Copy) { + dependsOn(':metafix-ide:installDist') + from "metafix-ide/build/install/xtext-server" + into "metafix-vsc/xtext-server" +} + subprojects { - apply plugin: 'signing' - apply plugin: 'jacoco' apply plugin: 'checkstyle' + apply plugin: 'jacoco' apply plugin: 'maven-publish' + apply plugin: 'signing' + //editorconfigCheck.dependsOn(editorconfigChecker) check.dependsOn(editorconfigChecker) check.dependsOn(javadoc) @@ -97,7 +128,7 @@ subprojects { tasks.withType(JavaCompile) { 'all -processing -rawtypes -serial'.split().each { - options.compilerArgs << "-Xlint:${it}".toString() + options.compilerArgs << "-Xlint:${it}" } options.compilerArgs << '-Werror' @@ -123,6 +154,11 @@ subprojects { task allDeps(type: DependencyReportTask) {} javadoc { + // Can't exclude generated files based on source path + // (cf. https://stackoverflow.com/a/47711311). + //exclude '**/xtext-gen' + source = source.filter(f -> !f.getPath().contains('xtext-gen')) + options { addBooleanOption 'Xwerror', true } diff --git a/config/checkstyle/checkstyle.xml b/config/checkstyle/checkstyle.xml index b31045320..e7a3851fb 100644 --- a/config/checkstyle/checkstyle.xml +++ b/config/checkstyle/checkstyle.xml @@ -8,6 +8,14 @@ + + + + + + + + @@ -83,7 +91,7 @@ - + @@ -94,15 +102,15 @@ - + - + @@ -138,7 +146,10 @@ - + + + + @@ -154,8 +165,8 @@ - + diff --git a/docs/xtext-setup-1.png b/docs/xtext-setup-1.png new file mode 100644 index 000000000..4a366fe3d Binary files /dev/null and b/docs/xtext-setup-1.png differ diff --git a/docs/xtext-setup-2.png b/docs/xtext-setup-2.png new file mode 100644 index 000000000..eedf60487 Binary files /dev/null and b/docs/xtext-setup-2.png differ diff --git a/gradle/metafix.gradle b/gradle/metafix.gradle new file mode 100644 index 000000000..2c6995678 --- /dev/null +++ b/gradle/metafix.gradle @@ -0,0 +1,40 @@ +apply plugin: 'eclipse' +apply plugin: 'java' +apply plugin: 'org.xtext.builder' + +dependencies { + implementation platform("org.eclipse.xtext:xtext-dev-bom:${versions.xtext}") +} + +configurations.all { + exclude group: 'asm' +} + +sourceSets { + main { + java.srcDir 'src/main/xtext-gen' + resources.srcDir 'src/main/xtext-gen' + } + test { + java.srcDir 'src/test/xtext-gen' + resources.srcDir 'src/test/xtext-gen' + } +} + +jar { + from('model') { + into('model') + } + from(sourceSets.main.allSource) { + include '**/*.xtext' + } + manifest { + attributes 'Bundle-SymbolicName': project.name + } +} + +plugins.withId('war') { + war { + webAppDirectory = file('src/main/webapp') + } +} diff --git a/metafacture-json/build.gradle b/metafacture-json/build.gradle index 2d68433f0..18deac1e2 100644 --- a/metafacture-json/build.gradle +++ b/metafacture-json/build.gradle @@ -25,6 +25,6 @@ dependencies { implementation 'com.github.erosb:everit-json-schema:1.14.2' testImplementation "junit:junit:${versions.junit}" testImplementation "org.mockito:mockito-core:${versions.mockito}" - testImplementation "com.github.tomakehurst:wiremock-jre8:${versions.wiremock_jre}" + testImplementation "com.github.tomakehurst:wiremock-jre8:${versions.wiremock}" testRuntimeOnly "org.slf4j:slf4j-simple:${versions.slf4j}" } diff --git a/metafacture-runner/build.gradle b/metafacture-runner/build.gradle index 0727a66c4..f40402c5d 100644 --- a/metafacture-runner/build.gradle +++ b/metafacture-runner/build.gradle @@ -15,6 +15,10 @@ */ import org.apache.tools.ant.filters.ReplaceTokens +plugins { + id 'application' +} + ext.mavenName = 'Metafacture Runner' description = 'CLI application for data processing with Flux and Metamorph' @@ -60,6 +64,7 @@ dependencies { plugins project(':metafacture-xml') plugins project(':metafacture-html') plugins project(':metafacture-yaml') + plugins project(':metafix') plugins project(':metamorph') // In a perfect world the slf4j binding would be a provided dependency so that @@ -83,6 +88,10 @@ dependencies { provided 'simple-jndi:simple-jndi:0.11.4.1' // Required for connection pooling with simple-jndi: provided 'commons-dbcp:commons-dbcp:1.4' + + implementation('org.antlr:antlr-runtime') { + version { strictly versions.antlr } + } } jar { @@ -110,10 +119,6 @@ distributions { dirPermissions { unix 0755 } filePermissions { unix 0644 } - into('lib/') { - from jar - from configurations.runtimeClasspath - } into('provided-libs') { from configurations.provided } @@ -157,3 +162,33 @@ task('fluxCommands', type: JavaExec, description: 'Print available Flux commands classpath = sourceSets.main.runtimeClasspath mainClass = 'org.metafacture.flux.HelpPrinter' } + +application { + mainClass = 'org.metafacture.runner.Flux' + + if (project.hasProperty('profile')) { + def file = project.getProperty('profile') ?: project.name + def depth = project.hasProperty('profile.depth') ? project.getProperty('profile.depth') : 8 + + applicationDefaultJvmArgs += [ + "-XX:FlightRecorderOptions=stackdepth=${depth}", + "-XX:StartFlightRecording=dumponexit=true,filename=${file}.jfr,settings=profile" + ] + } + + startScripts { + doLast { + delete outputDir + } + } +} + +tasks.withType(JavaExec) { + doFirst { + def prefix = project.group + '.' + + System.properties.each { k, v -> + if (k.startsWith(prefix)) systemProperties[k] = v + } + } +} diff --git a/metafix-ide/build.gradle b/metafix-ide/build.gradle new file mode 100644 index 000000000..e87878f4e --- /dev/null +++ b/metafix-ide/build.gradle @@ -0,0 +1,46 @@ +plugins { + id 'com.github.johnrengelman.shadow' version '8.1.1' +} + +dependencies { + implementation project(':metafix') + + implementation "org.eclipse.xtext:org.eclipse.xtext.ide:${versions.xtext}" + implementation "org.eclipse.xtext:org.eclipse.xtext.xbase.ide:${versions.xtext}" +} + +apply plugin: 'application' +apply plugin: 'com.github.johnrengelman.shadow' + +apply from: "${rootDir}/gradle/metafix.gradle" + +application { + mainClass = 'org.eclipse.xtext.ide.server.ServerLauncher' + applicationName = 'xtext-server' +} + +shadowJar { + from(sourceSets.main.output) + configurations = [project.configurations.runtimeClasspath] + + exclude( + '*.html', + '*.profile', + '.api_description', + '.options', + 'META-INF/*.DSA', + 'META-INF/*.RSA', + 'META-INF/*.SF', + 'META-INF/INDEX.LIST', + 'about.*', + 'about_files/*', + 'modeling32.png', + 'plugin.xml', + 'profile.list', + 'schema/*', + 'systembundle.properties' + ) + + archiveClassifier = 'ls' + append('plugin.properties') +} diff --git a/metafix-ide/src/main/java/org/metafacture/metafix/ide/FixIdeModule.java b/metafix-ide/src/main/java/org/metafacture/metafix/ide/FixIdeModule.java new file mode 100644 index 000000000..4b9f47859 --- /dev/null +++ b/metafix-ide/src/main/java/org/metafacture/metafix/ide/FixIdeModule.java @@ -0,0 +1,9 @@ +package org.metafacture.metafix.ide; + +/** + * Use this class to register ide components. + */ +public class FixIdeModule extends AbstractFixIdeModule { + public FixIdeModule() { + } +} diff --git a/metafix-ide/src/main/java/org/metafacture/metafix/ide/FixIdeSetup.java b/metafix-ide/src/main/java/org/metafacture/metafix/ide/FixIdeSetup.java new file mode 100644 index 000000000..2dd6f9fca --- /dev/null +++ b/metafix-ide/src/main/java/org/metafacture/metafix/ide/FixIdeSetup.java @@ -0,0 +1,23 @@ +package org.metafacture.metafix.ide; + +import org.metafacture.metafix.FixRuntimeModule; +import org.metafacture.metafix.FixStandaloneSetup; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import org.eclipse.xtext.util.Modules2; + +/** + * Initialization support for running Xtext languages as language servers. + */ +public class FixIdeSetup extends FixStandaloneSetup { + + public FixIdeSetup() { + } + + @Override + public Injector createInjector() { + return Guice.createInjector(Modules2.mixin(new FixRuntimeModule(), new FixIdeModule())); + } + +} diff --git a/metafix-vsc/.vscodeignore b/metafix-vsc/.vscodeignore new file mode 100644 index 000000000..d3f010738 --- /dev/null +++ b/metafix-vsc/.vscodeignore @@ -0,0 +1,8 @@ +.vscode/** +typings/** +out/test/** +test/** +src/** +**/*.map +.gitignore +tsconfig.json diff --git a/metafix-vsc/fix.configuration.json b/metafix-vsc/fix.configuration.json new file mode 100644 index 000000000..69347af5a --- /dev/null +++ b/metafix-vsc/fix.configuration.json @@ -0,0 +1,24 @@ +{ + "comments": { + "lineComment": "#" + }, + "brackets": [ + ["{", "}"], + ["[", "]"], + ["(", ")"] + ], + "autoClosingPairs": [ + ["{", "}"], + ["[", "]"], + ["(", ")"], + ["\"", "\""], + ["'", "'"] + ], + "surroundingPairs": [ + ["{", "}"], + ["[", "]"], + ["(", ")"], + ["\"", "\""], + ["'", "'"] + ] +} diff --git a/metafix-vsc/fix.snippets.json b/metafix-vsc/fix.snippets.json new file mode 100644 index 000000000..1c488d5f1 --- /dev/null +++ b/metafix-vsc/fix.snippets.json @@ -0,0 +1,22 @@ +{ + "do list": { + "prefix": ["do list"], + "body": ["do list(path: \"${1:}\", \"var\": \"${4:}\")", "\t${0:body}", "end"], + "description": "Iterates over each element of an array. In contrast to Catmandu, it can also iterate over a single object or string." + }, + "do list_as": { + "prefix": ["do list_as"], + "body": ["do list_as(\"${1:}\": \"${2:}\")", "\t${0:body}", "end"], + "description": "Iterates over each named element of an array (like do list with a variable name). If multiple arrays are given, iterates over the corresponding elements from each array (i.e., all elements with the same array index, skipping elements whose arrays have already been exhausted)." + }, + "do once": { + "prefix": ["do once"], + "body": ["do once(\"${1:}\")", "\t${0:body}", "end"], + "description": "Executes the statements only once (when the bind is first encountered), not repeatedly for each record." + }, + "do put_macro": { + "prefix": ["do put_macro"], + "body": ["do put_macro(\"${1:}\")", "\t${0:body}", "end"], + "description": "Defines a named macro, i.e. a list of statements that can be executed later with the call_macro function." + } + } diff --git a/metafix-vsc/fix.tmLanguage b/metafix-vsc/fix.tmLanguage new file mode 100644 index 000000000..e435b9670 --- /dev/null +++ b/metafix-vsc/fix.tmLanguage @@ -0,0 +1,73 @@ + + + + + fileTypes + + *.fix + + name + fix + patterns + + + name + keyword.control.fix + match + \b(do|if|elsif|else|end|unless)\b + comment + The fix grammar can be found here: https://github.com/metafacture/metafacture-core/blob/master/metafix/src/main/java/org/metafacture/metafix/Fix.xtext + + + name + entity.name.function.method.fix + match + \b(include|nothing|put_filemap|put_map|put_rdfmap|put_var|put_vars|add_field|array|call_macro|copy_field|format|hash|move_field|parse_text|paste|print_record|random|reject|rename|retain|set_array|set_field|set_hash|timestamp|vacuum|append|capitalize|count|downcase|filter|flatten|from_json|index|isbn|join_field|lookup|prepend|replace_all|reverse|sort_field|split_field|substring|sum|to_json|trim|uniq|upcase)\b + comment + The method names are taken from https://github.com/metafacture/metafacture-core/blob/master/metafix/src/main/java/org/metafacture/metafix/FixMethod.java + + + name + support.function.conditional.fix + match + \b(all_contain|any_contain|none_contain|str_contain|all_equal|any_equal|none_equal|str_equal|exists|in|is_contained_in|is_array|is_empty|is_false|is_hash|is_number|is_object|is_sring|is_true|all_match|any_match|none_match|str_match)\b + comment + The names of the conditionals are taken from https://github.com/metafacture/metafacture-core/blob/master/metafix/src/main/java/org/metafacture/metafix/FixConditional.java + + + name + storage.type.bind.fix + match + \b(list|list_as|once|put_macro)\b + comment + The names of the fix binds are taken from https://github.com/metafacture/metafacture-core/blob/master/metafix/src/main/java/org/metafacture/metafix/FixBind.java + + + name + comment.line.number-sign.fix + begin + # + end + \n + + + name + string.quoted.single.fix + begin + ' + end + ' + + + name + string.quoted.double.fix + begin + " + end + " + + + scopeName + text.fix + + diff --git a/metafix-vsc/package.json b/metafix-vsc/package.json new file mode 100644 index 000000000..0e6a59952 --- /dev/null +++ b/metafix-vsc/package.json @@ -0,0 +1,64 @@ +{ + "name": "fix", + "displayName": "fix", + "description": "Language Support for Metafacture Fix (Xtext Language Server for Metafacture Fix). For more information, visit https://metafacture.org", + "version": "0.3.0", + "publisher": "metafacture", + "license": "Apache-2.0", + "engines": { + "vscode": "^1.18.0" + }, + "categories": [ + "Programming Languages", + "Snippets" + ], + "activationEvents": [ + "onLanguage:fix" + ], + "main": "./out/src/extension", + "contributes": { + "languages": [ + { + "id": "fix", + "aliases": [ + "fix" + ], + "extensions": [ + ".fix" + ], + "configuration": "./fix.configuration.json" + } + ], + "grammars": [ + { + "language": "fix", + "scopeName": "text.fix", + "path": "./fix.tmLanguage" + } + ], + "snippets": [ + { + "language": "fix", + "path": "./fix.snippets.json" + } + ] + + }, + "scripts": { + "vscode:prepublish": "tsc -p ./", + "compile": "tsc -watch -p ./", + "postinstall": "node ./node_modules/vscode/bin/install" + }, + "devDependencies": { + "typescript": "^3.4.3", + "vscode": "^1.0.0", + "@types/node": "^12.12.2" + }, + "dependencies": { + "vscode-languageclient": "^3.0.3" + }, + "repository": { + "type": "git", + "url": "https://github.com/metafacture/metafacture-core.git" + } +} diff --git a/metafix-vsc/src/extension.ts b/metafix-vsc/src/extension.ts new file mode 100644 index 000000000..359b70953 --- /dev/null +++ b/metafix-vsc/src/extension.ts @@ -0,0 +1,41 @@ +/******************************************************************************* + * Copyright (c) 2016 TypeFox GmbH (http://www.typefox.io) and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + *******************************************************************************/ +'use strict'; + +import * as net from 'net'; +import * as path from 'path'; + +import { workspace, Disposable, ExtensionContext } from 'vscode'; +import { LanguageClient, LanguageClientOptions, ServerOptions } from 'vscode-languageclient'; + +export function activate(context: ExtensionContext) { + // The server is a locally installed Java application + const executablExt = process.platform == 'win32' ? '.bat' : ''; + const executable = 'xtext-server' + executablExt; + const command = context.asAbsolutePath(path.join('xtext-server', 'bin', executable)); + const serverOptions = { command }; + + // Options to control the language client + const clientOptions: LanguageClientOptions = { + // Register the server for plain text documents + documentSelector: ['fix'], + synchronize: { + // Synchronize the setting section 'languageServerExample' to the server + configurationSection: 'languageServerExample', + // Notify the server about file changes to '.clientrc files contain in the workspace + fileEvents: workspace.createFileSystemWatcher('**/*.*') + } + } + + // Create the language client and start the client. + const disposable = new LanguageClient('Xtext Server', serverOptions, clientOptions).start(); + + // Push the disposable to the context's subscriptions so that the + // client can be deactivated on extension deactivation + context.subscriptions.push(disposable); +} diff --git a/metafix-vsc/tsconfig.json b/metafix-vsc/tsconfig.json new file mode 100644 index 000000000..0b14cc1db --- /dev/null +++ b/metafix-vsc/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "module": "commonjs", + "target": "es6", + "outDir": "out", + "lib": [ + "es6" + ], + "sourceMap": true, + "rootDir": "." + }, + "exclude": [ + "node_modules", + ".vscode-test" + ] +} diff --git a/metafix-web/build.gradle b/metafix-web/build.gradle new file mode 100644 index 000000000..b2d76658f --- /dev/null +++ b/metafix-web/build.gradle @@ -0,0 +1,36 @@ +plugins { + id 'war' +} + +dependencies { + implementation project(':metafix') + implementation project(':metafix-ide') + + implementation "org.eclipse.xtend:org.eclipse.xtend.lib:${versions.xtext}" + implementation "org.eclipse.xtext:org.eclipse.xtext.web.servlet:${versions.xtext}" + implementation "org.eclipse.xtext:org.eclipse.xtext.xbase.web:${versions.xtext}" + implementation "org.webjars:ace:${versions.ace}" + implementation "org.webjars:jquery:${versions.jquery}" + implementation "org.webjars:requirejs:${versions.requirejs}" + + providedCompile "org.eclipse.jetty:jetty-annotations:${versions.jetty}" + providedCompile "org.slf4j:slf4j-simple:${versions.slf4j}" + + implementation project(':metafacture-commons') + implementation project(':metafacture-formeta') + implementation project(':metafacture-mangling') + implementation project(':metafacture-runner') + implementation project(':metafacture-xml') + implementation project(':metamorph') +} + +apply from: "${rootDir}/gradle/metafix.gradle" + +task jettyRun(type: JavaExec) { + dependsOn(sourceSets.main.runtimeClasspath) + classpath = sourceSets.main.runtimeClasspath.filter { it.exists() } + mainClass = 'org.metafacture.metafix.web.ServerLauncher' + standardInput = System.in + group = 'run' + description = 'Starts an example Jetty server with your language' +} diff --git a/metafix-web/src/main/java/org/metafacture/metafix/web/FixServlet.java b/metafix-web/src/main/java/org/metafacture/metafix/web/FixServlet.java new file mode 100644 index 000000000..ae87a658f --- /dev/null +++ b/metafix-web/src/main/java/org/metafacture/metafix/web/FixServlet.java @@ -0,0 +1,115 @@ +package org.metafacture.metafix.web; + +import org.metafacture.metafix.FixStandaloneSetup; +import org.metafacture.runner.Flux; + +import org.antlr.runtime.RecognitionException; +import org.eclipse.xtext.util.DisposableRegistry; +import org.eclipse.xtext.web.servlet.XtextServlet; +import org.eclipse.xtext.xbase.lib.InputOutput; + +import java.io.IOException; +import java.io.Reader; +import java.io.StringReader; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Map; +import javax.servlet.ServletException; +import javax.servlet.annotation.WebServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** + * Deploy this class into a servlet container to enable DSL-specific services. + */ +@WebServlet(name = "XtextServices", urlPatterns = "/xtext-service/*") +public class FixServlet extends XtextServlet { + + private static final String COMMAND_FIX = "fix"; + + private static final String PARAM_DATA = "data"; + private static final String PARAM_FIX = "fix"; + private static final String PARAM_FLUX = "flux"; + + private DisposableRegistry disposableRegistry; + + public FixServlet() { + } + + @Override + public void init() throws ServletException { + disposableRegistry = new FixWebSetup().createInjectorAndDoEMFRegistration().getInstance(DisposableRegistry.class); + } + + @Override + public void destroy() { + if (disposableRegistry != null) { + disposableRegistry.dispose(); + disposableRegistry = null; + } + + super.destroy(); + } + + @Override + public void doPost(final HttpServletRequest request, final HttpServletResponse response) throws IOException, ServletException { + InputOutput.println("POST Request: " + request); + + if (!request.getPathInfo().endsWith("/run") || !process(request, response)) { + super.doPost(request, response); + } + } + + @Override + public void doGet(final HttpServletRequest request, final HttpServletResponse response) throws IOException, ServletException { + InputOutput.println("GET Request: " + request); + + if (!process(request, response)) { + super.doGet(request, response); + } + } + + private boolean process(final HttpServletRequest request, final HttpServletResponse response) throws IOException { + final Map params = request.getParameterMap(); + + if (!params.containsKey(PARAM_DATA) || !params.containsKey(PARAM_FLUX) || !params.containsKey(PARAM_FIX)) { + return false; + } + + final StringBuilder builder = new StringBuilder(); + + final String inData = request.getParameter(PARAM_DATA); + builder.append(inData == null || inData.isEmpty() ? "" : + "\"" + absPathToTempFile(inData, ".txt") + "\"|open-file|"); + + final String fixFile = absPathToTempFile(request.getParameter(PARAM_FIX), ".fix"); + final String outFile = absPathToTempFile("", ".txt"); + + builder.append(request.getParameter(PARAM_FLUX).replaceAll("\\s?\\|\\s?", "|").replace( + "|" + COMMAND_FIX + "|", + "|org.metafacture.metafix.Metafix(\"" + fixFile + "\")|")); + builder.append("|write(\""); + builder.append(outFile); + builder.append("\");"); + + final String fullFlux = builder.toString(); + InputOutput.println("full flux: " + fullFlux); + + try { + Flux.main(new String[] {absPathToTempFile(fullFlux, ".flux")}); + } + catch (final RecognitionException e) { + throw new RuntimeException(e); + } + + Files.copy(Paths.get(outFile), response.getOutputStream()); + return true; + } + + private String absPathToTempFile(final String content, final String suffix) throws IOException { + try (Reader reader = new StringReader(content)) { + return FixStandaloneSetup.absPathToTempFile(reader, suffix); + } + } + +} diff --git a/metafix-web/src/main/java/org/metafacture/metafix/web/FixWebModule.java b/metafix-web/src/main/java/org/metafacture/metafix/web/FixWebModule.java new file mode 100644 index 000000000..ad61024ed --- /dev/null +++ b/metafix-web/src/main/java/org/metafacture/metafix/web/FixWebModule.java @@ -0,0 +1,10 @@ +package org.metafacture.metafix.web; + +/** + * Use this class to register additional components to be used within the web + * application. + */ +public class FixWebModule extends AbstractFixWebModule { + public FixWebModule() { + } +} diff --git a/metafix-web/src/main/java/org/metafacture/metafix/web/FixWebSetup.java b/metafix-web/src/main/java/org/metafacture/metafix/web/FixWebSetup.java new file mode 100644 index 000000000..ad7c397af --- /dev/null +++ b/metafix-web/src/main/java/org/metafacture/metafix/web/FixWebSetup.java @@ -0,0 +1,24 @@ +package org.metafacture.metafix.web; + +import org.metafacture.metafix.FixRuntimeModule; +import org.metafacture.metafix.FixStandaloneSetup; +import org.metafacture.metafix.ide.FixIdeModule; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import org.eclipse.xtext.util.Modules2; + +/** + * Initialization support for running Xtext languages in web applications. + */ +public class FixWebSetup extends FixStandaloneSetup { + + public FixWebSetup() { + } + + @Override + public Injector createInjector() { + return Guice.createInjector(Modules2.mixin(new FixRuntimeModule(), new FixIdeModule(), new FixWebModule())); + } + +} diff --git a/metafix-web/src/main/java/org/metafacture/metafix/web/ServerLauncher.java b/metafix-web/src/main/java/org/metafacture/metafix/web/ServerLauncher.java new file mode 100644 index 000000000..6b5658a0c --- /dev/null +++ b/metafix-web/src/main/java/org/metafacture/metafix/web/ServerLauncher.java @@ -0,0 +1,73 @@ +package org.metafacture.metafix.web; + +import org.eclipse.jetty.annotations.AnnotationConfiguration; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.util.log.Slf4jLog; +import org.eclipse.jetty.webapp.Configuration; +import org.eclipse.jetty.webapp.MetaInfConfiguration; +import org.eclipse.jetty.webapp.WebAppContext; +import org.eclipse.jetty.webapp.WebInfConfiguration; +import org.eclipse.jetty.webapp.WebXmlConfiguration; +import org.eclipse.xtext.xbase.lib.Exceptions; + +import java.net.InetSocketAddress; + +/** + * This program starts an HTTP server for testing the web integration of your DSL. + * Just execute it and point a web browser to http://localhost:8080/ + */ +public class ServerLauncher { // checkstyle-disable-line ClassDataAbstractionCoupling + + private ServerLauncher() { + throw new IllegalAccessError("Utility class"); + } + + public static void main(final String[] args) { + final WebAppContext context = new WebAppContext(); + context.setResourceBase("src/main/webapp"); + context.setWelcomeFiles(new String[] {"index.html"}); + context.setContextPath("/"); + context.setConfigurations(new Configuration[] {new AnnotationConfiguration(), new WebXmlConfiguration(), new WebInfConfiguration(), new MetaInfConfiguration()}); + context.setAttribute(WebInfConfiguration.CONTAINER_JAR_PATTERN, ".*/org\\.metafacture\\.fix\\.web/.*,.*\\.jar"); + context.setInitParameter("org.mortbay.jetty.servlet.Default.useFileMappedBuffer", "false"); + + final Server server = new Server(new InetSocketAddress("0.0.0.0", 8080)); + server.setHandler(context); + + final Slf4jLog log = new Slf4jLog(ServerLauncher.class.getName()); + + try { + server.start(); + + log.info("Server started " + server.getURI() + "..."); + + new Thread(() -> { + try { + log.info("Press enter to stop the server..."); + + if (System.in.read() != -1) { + server.stop(); + } + else { + log.warn("Console input is not available. In order to stop the server, you need to cancel the process manually."); + } + } + catch (final Exception e) { // checkstyle-disable-line IllegalCatch + throw Exceptions.sneakyThrow(e); + } + }).start(); + + server.join(); + } + catch (final Throwable e) { // checkstyle-disable-line IllegalCatch + if (e instanceof Exception) { + log.warn(((Exception) e).getMessage()); + System.exit(1); + } + else { + throw Exceptions.sneakyThrow(e); + } + } + } + +} diff --git a/metafix-web/src/main/webapp/index.html b/metafix-web/src/main/webapp/index.html new file mode 100644 index 000000000..88eea1961 --- /dev/null +++ b/metafix-web/src/main/webapp/index.html @@ -0,0 +1,93 @@ + + + + + Fix Web Editor + + + + + + + +
+
+ +

Fix Web Editor + | Load sample + | Process

+
+
+
+
+
Data:
+
+
+
+
+
Flux:
+
+
+
+
+
+
+
+
+
+
+
+ +
+
+
+
+ + diff --git a/metafix-web/src/main/webapp/style.css b/metafix-web/src/main/webapp/style.css new file mode 100644 index 000000000..54b518589 --- /dev/null +++ b/metafix-web/src/main/webapp/style.css @@ -0,0 +1,83 @@ +body { + width: 100%; + height: 100%; + overflow: hidden; + font: 16px Helvetica,sans-serif; +} + +a { + color: #22a; + text-decoration: none; +} + +a:hover { + text-decoration: underline; +} + +.container { + display: block; + position: absolute; + top: 0; + bottom: 0; + left: 0; + right: 0; + margin: 20px; +} + +.header { + display: block; + position: absolute; + background-color: #e8e8e8; + top: 0; + left: 0; + right: 0; + height: 60px; + padding: 10px; +} + +.content { + display: block; + position: absolute; + top: 90px; + bottom: 0; + left: 0; + width: 100%; +} + +#xtext-editor { + display: block; + position: absolute; + top: 10em; + bottom: 0; + left: 0; + right: 0; + padding: 4px; + border: 1px solid #aaa; + width: 50%; + height: 70%; +} + +#data { + width: 100%; + margin-bottom: 5px; +} + +#flux { + width: 100%; +} + +.row { + display: flex; +} + +.col { + flex: 50%; +} + +#result { + padding: 15px; + margin: 15px; + white-space: pre-wrap; + text-align: left; + width: 100%; +} diff --git a/metafix/.launch/Generate Fix (fix) Language Infrastructure.launch b/metafix/.launch/Generate Fix (fix) Language Infrastructure.launch new file mode 100644 index 000000000..ef1daee71 --- /dev/null +++ b/metafix/.launch/Generate Fix (fix) Language Infrastructure.launch @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/metafix/build.gradle b/metafix/build.gradle new file mode 100644 index 000000000..41406d44a --- /dev/null +++ b/metafix/build.gradle @@ -0,0 +1,137 @@ +plugins { + id 'maven-publish' + id 'me.champeau.jmh' version '0.7.2' +} + +def passSystemProperties = { + System.properties.each { k, v -> + if (k.startsWith(group + '.')) it[k] = v + } +} + +dependencies { + implementation "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" + implementation "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}" + implementation "com.google.guava:guava:${versions.guava}" + implementation "org.apache.jena:jena-arq:${versions.jena}" + implementation "org.apache.jena:jena-core:${versions.jena}" + implementation "org.eclipse.emf:org.eclipse.emf.ecore:${versions.xtext}" // Workaround for hbz/lobid-resources#1462 + implementation "org.eclipse.xtext:org.eclipse.xtext.xbase:${versions.xtext}" + implementation "org.eclipse.xtext:org.eclipse.xtext:${versions.xtext}" + implementation "org.slf4j:slf4j-api:${versions.slf4j}" + + testImplementation "com.github.tomakehurst:wiremock-jre8:${versions.wiremock}" + testImplementation "org.eclipse.xtext:org.eclipse.xtext.testing:${versions.xtext}" + testImplementation "org.eclipse.xtext:org.eclipse.xtext.xbase.testing:${versions.xtext}" + testImplementation "org.junit.jupiter:junit-jupiter-api:${versions.junit_jupiter}" + testImplementation "org.junit.platform:junit-platform-launcher:${versions.junit_platform}" + testImplementation "org.simplify4u:slf4j2-mock:${versions.slf4j_mock}" + + testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:${versions.junit_jupiter}" + + runtimeOnly "org.slf4j:slf4j-simple:${versions.slf4j}" + + implementation project(':metafacture-commons') + implementation project(':metafacture-flowcontrol') + implementation project(':metafacture-formatting') + implementation project(':metafacture-framework') + implementation project(':metafacture-io') + implementation project(':metafacture-javaintegration') + implementation project(':metafacture-mangling') + implementation project(':metafacture-triples') + implementation project(':metamorph') + + testImplementation "nl.jqno.equalsverifier:equalsverifier:${versions.equalsverifier}" + testImplementation "org.mockito:mockito-inline:${versions.mockito}" + testImplementation "org.mockito:mockito-junit-jupiter:${versions.mockito}" +} + +apply from: "${rootDir}/gradle/metafix.gradle" + +configurations { + mwe2 { + extendsFrom implementation + + dependencies { + implementation "org.slf4j:slf4j-simple:${versions.slf4j}" + } + } +} + +dependencies { + mwe2 'org.eclipse.emf:org.eclipse.emf.mwe2.launch' + mwe2 "org.eclipse.xtext:org.eclipse.xtext.common.types:${versions.xtext}" + mwe2 "org.eclipse.xtext:org.eclipse.xtext.xtext.generator:${versions.xtext}" + mwe2 'org.eclipse.xtext:xtext-antlr-generator' + + jmh project(':metafacture-io') + jmh project(':metafacture-json') +} + +jmhJar { + // SLF4J: Class path contains multiple SLF4J bindings. + exclude '/org/slf4j/impl/StaticLoggerBinder.class' +} + +test { + useJUnitPlatform() + + testLogging { + showStandardStreams = true + exceptionFormat = 'full' + events 'SKIPPED' + } + + passSystemProperties(systemProperties) + + maxParallelForks = Runtime.runtime.availableProcessors().intdiv(2) ?: 1 +} + +task integrationTest(type: Exec, group: 'Verification') { + executable './integrationTest.sh' + + if (project.hasProperty('args')) { + args project.getProperty('args').split() + } + + if (project.hasProperty('profile')) { + environment.METAFIX_INTEGRATION_TEST_PROFILE = project.getProperty('profile') + } + + environment.METAFIX_DISABLE_TO_DO = System.getProperty('org.metafacture.metafix.disableToDo') + environment.METAFIX_KEEP_TEMP = System.getProperty('org.metafacture.metafix.keepTemp') +} + +task validateFixFile(type: JavaExec, group: 'Verification') { + mainClass = 'org.metafacture.metafix.FixStandaloneSetup' + classpath = sourceSets.main.runtimeClasspath +} + +def mwe2File = 'src/main/java/org/metafacture/metafix/GenerateFix.mwe2' +def xtextFile = 'src/main/java/org/metafacture/metafix/Fix.xtext' + +task validateXtextLanguage(type: JavaExec, group: 'Verification') { + mainClass = 'org.metafacture.metafix.validation.XtextValidator' + classpath = sourceSets.main.runtimeClasspath + args += xtextFile +} + +task generateXtextLanguage(type: JavaExec) { + mainClass = 'org.eclipse.emf.mwe2.launch.runtime.Mwe2Launcher' + classpath = configurations.mwe2 + inputs.file mwe2File + inputs.file xtextFile + outputs.dir 'src/main/xtext-gen' + args += mwe2File + args += '-p' + args += "rootPath=/${projectDir}/.." +} + +processResources.dependsOn(generateXtextLanguage) +generateXtext.dependsOn(generateXtextLanguage) +compileJava.dependsOn(generateXtextLanguage) +clean.dependsOn(cleanGenerateXtextLanguage) +check.dependsOn(validateXtextLanguage) +check.dependsOn(integrationTest) + +eclipse.classpath.plusConfigurations += [configurations.mwe2] diff --git a/metafix/integrationTest.sh b/metafix/integrationTest.sh new file mode 100755 index 000000000..898313916 --- /dev/null +++ b/metafix/integrationTest.sh @@ -0,0 +1,303 @@ +#! /bin/bash + +set -e + +cd "$(dirname "$(readlink -f "$0")")" + +metafix_file=test.flux +catmandu_file=test.cmd + +fix_file=test.fix +todo_file=todo.txt + +input_glob=input.* +expected_glob=expected.* +expected_errors_extension=err + +metafix_output_glob=output-metafix.* +catmandu_output_glob=output-catmandu.* + +root_directory="$PWD" +data_directory="$root_directory/src/test/resources/org/metafacture/metafix/integration" +gradle_command="$root_directory/../gradlew" + +function parse_boolean() { + [ "${1,,}" == true ] +} + +parse_boolean "$METAFIX_DISABLE_TO_DO" && disable_todo=1 || disable_todo= +parse_boolean "$METAFIX_INTEGRATION_TEST_PROFILE" && noprofile= || noprofile=no +parse_boolean "$METAFIX_KEEP_TEMP" && keep_temp=1 || keep_temp= +parse_boolean "$CI" && ci=1 || ci= + +[ -t 1 -a -x /usr/bin/colordiff ] && colordiff=colordiff || colordiff=cat + +function _tput() { + tput -T "${TERM:-dumb}" "$@" || true +} + +color_error=$(_tput setaf 1) # red +color_failed=$(_tput setaf 1) # red +color_failure=$(_tput bold)$(_tput setaf 1) # bold red +color_info=$(_tput setaf 5) # purple +color_invalid=$(_tput setaf 6) # cyan +color_passed=$(_tput setaf 2) # green +color_reset=$(_tput sgr0) # reset +color_skipped=$(_tput setaf 3) # yellow +color_success=$(_tput bold)$(_tput setaf 2) # bold green +color_test=$(_tput bold) # bold + +declare -A tests + +failed=0 +invalid=0 +passed=0 +skipped=0 + +current_file= + +function log() { + echo "$@" +} + +function warn() { + echo "$@" >&2 +} + +function die() { + [ $# -gt 0 ] && warn "$@" + exit 2 +} + +function rm_temp() { + [ -n "$keep_temp" ] || rm -f "$@" +} + +function run_metafix() { + local file=$1; shift + $gradle_command --console=plain -p "$root_directory" :metafacture-runner:run --args="$file" -P${noprofile}profile="${file%.*}" $@ +} + +function run_catmandu() { + : +} + +nanosecond_length=9 +nanosecond_offset=-$nanosecond_length + +function current_time() { + date +%s%N +} + +function elapsed_time() { + local elapsed_time=$(($(current_time) - $1)) seconds=0 milliseconds + + if [ ${#elapsed_time} -gt $nanosecond_length ]; then + seconds=${elapsed_time:0:$nanosecond_offset} + else + elapsed_time=$(printf "%0${nanosecond_length}d" "$elapsed_time") + fi + + [ "$seconds" -lt 60 ] && milliseconds=".${elapsed_time:$nanosecond_offset:3}" + + echo " ($(date "+%-Hh %-Mm %-S${milliseconds}s" -ud "@$seconds" | sed 's/^\(0[hm] \)*//'))" +} + +function get_file() { + local test=$1 type=$2 reason; shift 2 + + if [ $# -ne 1 ]; then + reason="Ambiguous $type files: $*" + elif [ ! -r "$1" ]; then + reason="No $type file: $1" + elif [ "$type" != "output" ] && [ ! -s "$1" ]; then + reason="Empty $type file: $1" + else + current_file=$1 + return 0 + fi + + log "$color_test$test$color_reset: ${color_invalid}INVALID$color_reset ($reason)" + + ((invalid++)) || true + + return 1 +} + +function log_file() { + if [ -s "$1" ]; then + log "$2$1" + + if [ "$3" -ne 0 -a -n "$ci" ]; then + cat "$1" + fi + else + rm_temp "$1" + fi +} + +function command_info() { + log " ${color_info}${1^} command exit status$color_reset: $2" + + log_file "$3" " ${color_info}${1^} command output$color_reset: " "$2" + log_file "$4" " ${color_info}${1^} command error$color_reset: " "$2" + + log +} + +function skip_test() { + if [ -r "$2" ]; then + local message="$color_test$1$color_reset: ${color_skipped}SKIPPED$color_reset" reason=$(head -1 "$2") + + [ -n "$reason" ] && message+=" ($reason)" + log "$message" + + ((skipped++)) || true + + return 0; + else + return 1; + fi +} + +function test_passed() { + if [ -r "$2" ]; then + log "$color_test$1$color_reset: ${color_failed}FAILED$color_reset (Marked as \"to do\", but passed.)" + + ((failed++)) || true + else + if parse_boolean "$METAFIX_LOG_PASSED"; then + log "$color_test$1$color_reset: ${color_passed}PASSED$color_reset$3" + fi + + ((passed++)) || true + fi +} + +function test_failed() { + if ! skip_test "$1" "$2"; then + log "$color_test$1$color_reset: $color_failed$4$color_reset$3" + + if [ $# -ge 13 ]; then + log " Fix: $9" + log " Input: ${10}" + log " Expected: ${11}" + log " Output: ${12}" + log " Diff: ${13}" + + [ -s "${13}" ] && $colordiff <"${13}" || rm_temp "${13}" + fi + + command_info "$5" "$6" "$7" "$8" + + ((failed++)) || true + fi +} + +function run_tests() { + local test matched=1\ + test_directory test_fix test_input test_expected test_todo\ + metafix_command_output metafix_command_error metafix_start_time\ + metafix_exit_status metafix_output metafix_diff metafix_elapsed_time + + cd "$data_directory" + + for test in $(find */ -type f -path "$1/$metafix_file" -printf "%h\n"); do + matched=0 + + [ -n "${tests[$test]}" ] && continue || tests[$test]=1 + + test_directory="$PWD/$test" + + get_file "$test" Fix "$test_directory"/$fix_file || { log; continue; } + test_fix=$current_file + + get_file "$test" input "$test_directory"/$input_glob || { log; continue; } + test_input=$current_file + + get_file "$test" expected "$test_directory"/$expected_glob || { log; continue; } + test_expected=$current_file + + test_todo="$test_directory/$todo_file" + + if [ -z "$disable_todo" ] || ! skip_test "$test" "$test_todo"; then + # TODO: catmandu (optional) + + metafix_command_output="$test_directory/metafix.out" + metafix_command_error="$test_directory/metafix.err" + metafix_command_args="$test_directory/metafix.args" + + metafix_start_time=$(current_time) + + run_metafix "$test_directory/$metafix_file" $(cat "$metafix_command_args" 2>/dev/null || true) >"$metafix_command_output" 2>"$metafix_command_error" + metafix_exit_status=$? + + metafix_elapsed_time=$(elapsed_time "$metafix_start_time") + + if [ "$metafix_exit_status" -eq 0 ]; then + if get_file "$test" output "$test_directory"/$metafix_output_glob; then + metafix_output=$current_file + metafix_diff="$test_directory/metafix.diff" + + if diff -u "$test_expected" "$metafix_output" >"$metafix_diff"; then + test_passed "$test" "$test_todo" "$metafix_elapsed_time" + + rm_temp "$metafix_diff" "$metafix_command_output" "$metafix_command_error" + else + test_failed "$test" "$test_todo" "$metafix_elapsed_time" FAILED\ + metafix "$metafix_exit_status" "$metafix_command_output" "$metafix_command_error"\ + "$test_fix" "$test_input" "$test_expected" "$metafix_output" "$metafix_diff" + fi + else + command_info metafix "$metafix_exit_status" "$metafix_command_output" "$metafix_command_error" + fi + elif [ "${test_expected##*.}" == "$expected_errors_extension" ]; then + get_file "$test" error "$metafix_command_error" || { log; continue; } + + while read -r pattern; do + if ! grep -qE "$pattern" "$metafix_command_error"; then + test_failed "$test" "$test_todo" " (Pattern not found: $pattern)" FAILED\ + metafix "$metafix_exit_status" "$metafix_command_output" "$metafix_command_error" + + continue 2 + fi + done <"$test_expected" + + test_passed "$test" "$test_todo" "$metafix_elapsed_time" + else + test_failed "$test" "$test_todo" "$metafix_elapsed_time" ERROR\ + metafix "$metafix_exit_status" "$metafix_command_output" "$metafix_command_error" + fi + fi + done + + cd - >/dev/null + + return $matched +} + +start_time=$(current_time) + +if [ $# -eq 0 ]; then + run_tests '*' || true +else + for pattern in "$@"; do + run_tests "$pattern" || warn "No tests matching pattern: $pattern" + done +fi + +[ ${#tests[@]} -gt 0 ] || die "No tests found: $data_directory" + +summary="${color_passed}$passed passed$color_reset" +summary+=", ${color_failed}$failed failed$color_reset" +summary+=", ${color_skipped}$skipped skipped$color_reset" +summary+=", ${color_invalid}$invalid invalid$color_reset" +summary+=$(elapsed_time "$start_time") + +if [ $failed -gt 0 -o $invalid -gt 0 ]; then + log "${color_failure}FAILURE$color_reset: $summary" + exit 1 +else + log "${color_success}SUCCESS$color_reset: $summary" + exit 0 +fi diff --git a/metafix/src/jmh/java/org/metafacture/metafix/AbstractBenchmark.java b/metafix/src/jmh/java/org/metafacture/metafix/AbstractBenchmark.java new file mode 100644 index 000000000..6b65f7099 --- /dev/null +++ b/metafix/src/jmh/java/org/metafacture/metafix/AbstractBenchmark.java @@ -0,0 +1,67 @@ +/* + * Copyright 2022 hbz NRW + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix; + +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +//import org.openjdk.jmh.profile.GCProfiler; +//import org.openjdk.jmh.profile.StackProfiler; +//import org.openjdk.jmh.runner.Runner; +//import org.openjdk.jmh.runner.RunnerException; +//import org.openjdk.jmh.runner.options.Options; +//import org.openjdk.jmh.runner.options.OptionsBuilder; + +import java.util.concurrent.TimeUnit; + +@Fork(2) +@Warmup(iterations = 2) +@Measurement(iterations = 4) // checkstyle-disable-line MagicNumber +@BenchmarkMode(Mode.Throughput) // AverageTime +@OutputTimeUnit(TimeUnit.SECONDS) +@State(Scope.Thread) +public abstract class AbstractBenchmark { + + public AbstractBenchmark() { + } + + @Benchmark + public void benchmark() { + workload(); + } + + protected abstract void workload(); + + /* + public static void main(final String[] args) throws RunnerException { + final Options opt = new OptionsBuilder() + .include(MetafixBenchmark.class.getSimpleName()) + .addProfiler(StackProfiler.class) + //.addProfiler(GCProfiler.class) + .build(); + + new Runner(opt).run(); + } + */ + +} diff --git a/metafix/src/jmh/java/org/metafacture/metafix/BaselineBenchmark.java b/metafix/src/jmh/java/org/metafacture/metafix/BaselineBenchmark.java new file mode 100644 index 000000000..6c0313e5a --- /dev/null +++ b/metafix/src/jmh/java/org/metafacture/metafix/BaselineBenchmark.java @@ -0,0 +1,34 @@ +/* + * Copyright 2022 hbz NRW + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix; + +import org.openjdk.jmh.annotations.OutputTimeUnit; + +import java.util.concurrent.TimeUnit; + +@OutputTimeUnit(TimeUnit.MICROSECONDS) +public class BaselineBenchmark extends AbstractBenchmark { + + public BaselineBenchmark() { + } + + @Override + protected void workload() { + // this method was intentionally left blank. + } + +} diff --git a/metafix/src/jmh/java/org/metafacture/metafix/FixParseBenchmark.java b/metafix/src/jmh/java/org/metafacture/metafix/FixParseBenchmark.java new file mode 100644 index 000000000..c513b70c1 --- /dev/null +++ b/metafix/src/jmh/java/org/metafacture/metafix/FixParseBenchmark.java @@ -0,0 +1,48 @@ +/* + * Copyright 2022 hbz NRW + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix; + +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Setup; + +public class FixParseBenchmark extends AbstractBenchmark { + + protected static final String BASE = "src/jmh/resources/org/metafacture/metafix"; + + private static final String FIXES = BASE + "/fixes/%s" + Metafix.FIX_EXTENSION; + + protected String fixFile; // checkstyle-disable-line VisibilityModifier + + @Param({ // checkstyle-disable-line AnnotationUseStyle + "nothing" + }) + private String fixDef; + + public FixParseBenchmark() { + } + + @Setup + public void setup() { + fixFile = String.format(FIXES, fixDef); + } + + @Override + protected void workload() { + FixStandaloneSetup.parseFix(fixFile); + } + +} diff --git a/metafix/src/jmh/java/org/metafacture/metafix/MetafixBenchmark.java b/metafix/src/jmh/java/org/metafacture/metafix/MetafixBenchmark.java new file mode 100644 index 000000000..4f7024c3e --- /dev/null +++ b/metafix/src/jmh/java/org/metafacture/metafix/MetafixBenchmark.java @@ -0,0 +1,86 @@ +/* + * Copyright 2022 hbz NRW + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix; + +import org.metafacture.framework.helpers.DefaultStreamReceiver; +import org.metafacture.io.FileOpener; +import org.metafacture.io.ObjectStdoutWriter; +import org.metafacture.io.RecordReader; +import org.metafacture.json.JsonDecoder; +import org.metafacture.json.JsonEncoder; + +import org.openjdk.jmh.annotations.Param; + +import java.io.IOException; +import java.io.UncheckedIOException; + +public class MetafixBenchmark extends FixParseBenchmark { // checkstyle-disable-line ClassDataAbstractionCoupling + + // TODO: Need to inject system properties into JMHTask's JavaExec process. + //private static final boolean DEBUG_OUTPUT = Boolean.parseBoolean(System.getProperty("org.metafacture.metafix.debugBenchmarkOutput")); + private static final boolean DEBUG_OUTPUT = false; + + private static final String INPUT = BASE + "/input/%s.json"; + + private FileOpener fileOpener; + private String inputFile; + + @Param({ // checkstyle-disable-line AnnotationUseStyle + "empty" + }) + private String input; + + public MetafixBenchmark() { + } + + @Override + public void setup() { + super.setup(); + + inputFile = String.format(INPUT, input); + + final Metafix metafix; + try { + metafix = new Metafix(fixFile); + } + catch (final IOException e) { + throw new UncheckedIOException(e); + } + + if (DEBUG_OUTPUT) { + metafix + .setReceiver(new JsonEncoder()) + .setReceiver(new ObjectStdoutWriter()); + } + else { + metafix + .setReceiver(new DefaultStreamReceiver()); + } + + fileOpener = new FileOpener(); + fileOpener + .setReceiver(new RecordReader()) + .setReceiver(new JsonDecoder()) + .setReceiver(metafix); + } + + @Override + protected void workload() { + fileOpener.process(inputFile); + } + +} diff --git a/metafix/src/jmh/resources/org/metafacture/metafix/fixes/nothing.fix b/metafix/src/jmh/resources/org/metafacture/metafix/fixes/nothing.fix new file mode 100644 index 000000000..174b9a02e --- /dev/null +++ b/metafix/src/jmh/resources/org/metafacture/metafix/fixes/nothing.fix @@ -0,0 +1 @@ +nothing() diff --git a/metafix/src/jmh/resources/org/metafacture/metafix/input/empty.json b/metafix/src/jmh/resources/org/metafacture/metafix/input/empty.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/metafix/src/jmh/resources/org/metafacture/metafix/input/empty.json @@ -0,0 +1 @@ +{} diff --git a/metafix/src/jmh/resources/plugin.properties b/metafix/src/jmh/resources/plugin.properties new file mode 100644 index 000000000..2a7935572 --- /dev/null +++ b/metafix/src/jmh/resources/plugin.properties @@ -0,0 +1 @@ +_UI_DiagnosticRoot_diagnostic=_UI_DiagnosticRoot_diagnostic diff --git a/metafix/src/main/java/org/metafacture/metafix/FindFixPaths.java b/metafix/src/main/java/org/metafacture/metafix/FindFixPaths.java new file mode 100644 index 000000000..df46d3afc --- /dev/null +++ b/metafix/src/main/java/org/metafacture/metafix/FindFixPaths.java @@ -0,0 +1,106 @@ +/* + * Copyright 2024 Tobias Bülte, hbz + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix; + +import org.metafacture.formatting.ObjectTemplate; +import org.metafacture.framework.FluxCommand; +import org.metafacture.framework.MetafactureException; +import org.metafacture.framework.ObjectReceiver; +import org.metafacture.framework.StreamReceiver; +import org.metafacture.framework.annotations.Description; +import org.metafacture.framework.annotations.In; +import org.metafacture.framework.annotations.Out; +import org.metafacture.framework.helpers.DefaultStreamPipe; +import org.metafacture.mangling.StreamFlattener; +import org.metafacture.triples.StreamToTriples; +import org.metafacture.triples.TripleFilter; + +import java.io.IOException; + +/** + * Provide a user-friendly way to finds all paths that have values that match + * the given pattern. + * + * @author Tobias Bülte + */ +@Description("Finds all paths that have values that match the given pattern. Allows for regex. These paths can be used in a Fix to address fields.") +@In(StreamReceiver.class) +@Out(String.class) +@FluxCommand("find-fix-paths") + +public class FindFixPaths extends DefaultStreamPipe> { + private final Metafix fix; + private String objectPattern; + + public FindFixPaths(final String objectPattern) { + this.objectPattern = objectPattern; + try { + this.fix = new Metafix("nothing()"); + this.fix.setRepeatedFieldsToEntities(true); + } + catch (final IOException e) { + throw new MetafactureException(e); + } + } + + @Override + protected void onSetReceiver() { + final TripleFilter tripleFilter = new TripleFilter(); + tripleFilter.setObjectPattern(objectPattern); + fix + .setReceiver(new StreamFlattener()) + .setReceiver(new StreamToTriples()) + .setReceiver(tripleFilter) + .setReceiver(new ObjectTemplate<>("${p}\t|\t${o}")) + .setReceiver(getReceiver()); + } + + @Override + public void startRecord(final String identifier) { + fix.startRecord(identifier); + } + + @Override + public void endRecord() { + fix.endRecord(); + } + + @Override + public void startEntity(final String name) { + fix.startEntity(name); + } + + @Override + public void endEntity() { + fix.endEntity(); + } + + @Override + public void literal(final String name, final String value) { + fix.literal(name, value); + } + + @Override + protected void onCloseStream() { + fix.closeStream(); + } + + @Override + protected void onResetStream() { + fix.resetStream(); + } +} diff --git a/metafix/src/main/java/org/metafacture/metafix/Fix.xtext b/metafix/src/main/java/org/metafacture/metafix/Fix.xtext new file mode 100644 index 000000000..1b486e8f0 --- /dev/null +++ b/metafix/src/main/java/org/metafacture/metafix/Fix.xtext @@ -0,0 +1,67 @@ +grammar org.metafacture.metafix.Fix + with org.eclipse.xtext.xbase.Xbase + +generate fix "http://www.metafacture.org/metafix/Fix" + +Fix: + elements += Expression* +; + +Expression: + Do | + Unless | + If | + MethodCall | + {Expression} SL_COMMENT +; + +Unless: + 'unless' name = QualifiedName '(' ( params += (QualifiedName|STRING) ( ',' params += (QualifiedName|STRING) )* ','? )? ( options = Options )? ')' + elements += Expression* + 'end' +; + +If: + 'if' name = QualifiedName '(' ( params += (QualifiedName|STRING) ( ',' params += (QualifiedName|STRING) )* ','? )? ( options = Options )? ')' + elements += Expression* + elseIf += ElsIf* + else = Else? + 'end' +; + +ElsIf: + 'elsif' name = QualifiedName '(' ( params += (QualifiedName|STRING) ( ',' params += (QualifiedName|STRING) )* ','? )? ( options = Options )? ')' + elements += Expression* +; + +Else: + {Else} 'else' + elements += Expression* +; + +Do: + 'do' name = QualifiedName '(' ( params += (QualifiedName|STRING) ( ',' params += (QualifiedName|STRING) )* ','? )? ( options = Options )? ')' + elements += Expression* + 'end' +; + +MethodCall: + name = QualifiedName '(' ( params += (QualifiedName|STRING) ( ',' params += (QualifiedName|STRING) )* ','? )? ( options = Options )? ')' +; + +Options: + keys += (QualifiedName|STRING) ':' values += (QualifiedName|STRING) ( ',' keys += (QualifiedName|STRING) ':' values += (QualifiedName|STRING) )* +; + +terminal fragment ESCAPED_CHAR: + '\\' ('n'|'t'|'r'|'\\'); + +@Override +terminal STRING: + '"' ( ESCAPED_CHAR | !('\\'|'"') )* '"' | + "'" ( ESCAPED_CHAR | !('\\'|"'") )* "'"; + +@Override +terminal SL_COMMENT: + '#' -> ( '\r'? '\n' ) +; diff --git a/metafix/src/main/java/org/metafacture/metafix/FixBind.java b/metafix/src/main/java/org/metafacture/metafix/FixBind.java new file mode 100644 index 000000000..69ddd77bc --- /dev/null +++ b/metafix/src/main/java/org/metafacture/metafix/FixBind.java @@ -0,0 +1,110 @@ +/* + * Copyright 2022 hbz NRW + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix; + +import org.metafacture.metafix.api.FixContext; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public enum FixBind implements FixContext { + + list { + @Override + public void execute(final Metafix metafix, final Record record, final List params, final Map options, final RecordTransformer recordTransformer) { + final String scopeVariable = options.get("var"); + Value.asList(record.get(options.get("path")), a -> { + for (int i = 0; i < a.size(); ++i) { + final Value value = a.get(i); + + // with var -> keep full record in scope, add the var: + if (scopeVariable != null) { + record.put(scopeVariable, value, false); + recordTransformer.transform(record); + record.remove(scopeVariable); + } + // w/o var -> use the currently bound value as the record: + else { + final int index = i; + + value.matchType() + .ifHash(h -> { + final Record scopeRecord = new Record(); + scopeRecord.addAll(h); + + recordTransformer.transform(scopeRecord); + a.set(index, new Value(scopeRecord)); + }) + // TODO: bind to arrays (if that makes sense) and strings (access with '.') + .orElseThrow(); + } + } + }); + } + }, + + list_as { + @Override + public void execute(final Metafix metafix, final Record record, final List params, final Map options, final RecordTransformer recordTransformer) { + final Map lists = new HashMap<>(); + options.forEach((k, v) -> Value.asList(record.get(v), a -> lists.put(k, a))); + + final int size = lists.values().stream().mapToInt(a -> a.size()).max().orElse(0); + for (int i = 0; i < size; ++i) { + final int index = i; + + lists.forEach((k, v) -> { + final Value value = index < v.size() ? v.get(index) : null; + + if (value != null) { + record.put(k, value); + } + else { + record.remove(k); + } + }); + + recordTransformer.transform(record); + } + + lists.keySet().forEach(record::remove); + } + }, + + once { + private final Map> executed = new HashMap<>(); + + @Override + public void execute(final Metafix metafix, final Record record, final List params, final Map options, final RecordTransformer recordTransformer) { + if (executed.computeIfAbsent(metafix, k -> new HashSet<>()).add(params.isEmpty() ? null : params.get(0))) { + recordTransformer.transform(record); + } + } + }, + + put_macro { + @Override + public void execute(final Metafix metafix, final Record record, final List params, final Map options, final RecordTransformer recordTransformer) { + recordTransformer.setVars(options); + metafix.putMacro(params.get(0), recordTransformer); + } + } + +} diff --git a/metafix/src/main/java/org/metafacture/metafix/FixConditional.java b/metafix/src/main/java/org/metafacture/metafix/FixConditional.java new file mode 100644 index 000000000..b41165b84 --- /dev/null +++ b/metafix/src/main/java/org/metafacture/metafix/FixConditional.java @@ -0,0 +1,187 @@ +/* + * Copyright 2021 Fabian Steeg, hbz + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix; + +import org.metafacture.metafix.api.FixPredicate; + +import java.util.List; +import java.util.Map; + +public enum FixConditional implements FixPredicate { + + all_contain { + @Override + public boolean test(final Metafix metafix, final Record record, final List params, final Map options) { + return testConditional(record, params, ALL, CONTAINS); + } + }, + any_contain { + @Override + public boolean test(final Metafix metafix, final Record record, final List params, final Map options) { + return testConditional(record, params, ANY, CONTAINS); + } + }, + none_contain { + @Override + public boolean test(final Metafix metafix, final Record record, final List params, final Map options) { + return !any_contain.test(metafix, record, params, options); + } + }, + str_contain { + @Override + public boolean test(final Metafix metafix, final Record record, final List params, final Map options) { + return testConditional(params, CONTAINS); + } + }, + + all_equal { + @Override + public boolean test(final Metafix metafix, final Record record, final List params, final Map options) { + return testConditional(record, params, ALL, EQUALS); + } + }, + any_equal { + @Override + public boolean test(final Metafix metafix, final Record record, final List params, final Map options) { + return testConditional(record, params, ANY, EQUALS); + } + }, + none_equal { + @Override + public boolean test(final Metafix metafix, final Record record, final List params, final Map options) { + return !any_equal.test(metafix, record, params, options); + } + }, + str_equal { + @Override + public boolean test(final Metafix metafix, final Record record, final List params, final Map options) { + return testConditional(params, EQUALS); + } + }, + + exists { + @Override + public boolean test(final Metafix metafix, final Record record, final List params, final Map options) { + return record.containsPath(params.get(0)); + } + }, + + in { + @Override + public boolean test(final Metafix metafix, final Record record, final List params, final Map options) { + final Value value1 = record.get(params.get(0)); + final Value value2 = record.get(params.get(1)); + + return value1 != null && value2 != null && value1.extractType((m, c) -> m + .ifArray(a1 -> value2.matchType() + .ifArray(a2 -> c.accept(a1.equals(a2))) + .orElse(v -> c.accept(false)) + ) + .ifHash(h1 -> value2.matchType() + .ifHash(h2 -> c.accept(h1.equals(h2))) + .orElse(v -> c.accept(false)) + ) + .ifString(s1 -> value2.matchType() + .ifArray(a2 -> c.accept(a2.stream().anyMatch(value1::equals))) + .ifHash(h2 -> c.accept(h2.containsField(s1))) + .ifString(s2 -> c.accept(s1.equals(s2))) + ) + ); + } + }, + is_contained_in { + @Override + public boolean test(final Metafix metafix, final Record record, final List params, final Map options) { + return in.test(metafix, record, params, options); + } + }, + + is_array { + @Override + public boolean test(final Metafix metafix, final Record record, final List params, final Map options) { + return testConditional(record, params, Value::isArray); + } + }, + is_empty { + @Override + public boolean test(final Metafix metafix, final Record record, final List params, final Map options) { + return testConditional(record, params, IS_EMPTY); + } + }, + is_false { + @Override + public boolean test(final Metafix metafix, final Record record, final List params, final Map options) { + return testStringConditional(record, params, IS_FALSE); // TODO: strict=false + } + }, + is_hash { + @Override + public boolean test(final Metafix metafix, final Record record, final List params, final Map options) { + return is_object.test(metafix, record, params, options); + } + }, + is_number { + @Override + public boolean test(final Metafix metafix, final Record record, final List params, final Map options) { + return testStringConditional(record, params, IS_NUMBER); + } + }, + is_object { + @Override + public boolean test(final Metafix metafix, final Record record, final List params, final Map options) { + return testConditional(record, params, Value::isHash); + } + }, + is_string { + @Override + public boolean test(final Metafix metafix, final Record record, final List params, final Map options) { + return testConditional(record, params, Value::isString) && !is_number.test(metafix, record, params, options); + } + }, + is_true { + @Override + public boolean test(final Metafix metafix, final Record record, final List params, final Map options) { + return testStringConditional(record, params, IS_TRUE); // TODO: strict=false + } + }, + + all_match { + @Override + public boolean test(final Metafix metafix, final Record record, final List params, final Map options) { + return testConditional(record, params, ALL, MATCHES); + } + }, + any_match { + @Override + public boolean test(final Metafix metafix, final Record record, final List params, final Map options) { + return testConditional(record, params, ANY, MATCHES); + } + }, + none_match { + @Override + public boolean test(final Metafix metafix, final Record record, final List params, final Map options) { + return !any_match.test(metafix, record, params, options); + } + }, + str_match { + @Override + public boolean test(final Metafix metafix, final Record record, final List params, final Map options) { + return testConditional(params, MATCHES); + } + } + +} diff --git a/metafix/src/main/java/org/metafacture/metafix/FixExecutionException.java b/metafix/src/main/java/org/metafacture/metafix/FixExecutionException.java new file mode 100644 index 000000000..bb20f522a --- /dev/null +++ b/metafix/src/main/java/org/metafacture/metafix/FixExecutionException.java @@ -0,0 +1,37 @@ +/* + * Copyright 2022 hbz NRW + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix; + +import org.metafacture.framework.MetafactureException; + +/** + * Indicates dynamic (i.e., data-dependent) issues during Fix execution that + * should be subject to {@link Metafix.Strictness strictness} handling. + * + * @see FixProcessException + */ +public class FixExecutionException extends MetafactureException { + + public FixExecutionException(final String message) { + super(message); + } + + public FixExecutionException(final String message, final Throwable cause) { + super(message, cause); + } + +} diff --git a/metafix/src/main/java/org/metafacture/metafix/FixMethod.java b/metafix/src/main/java/org/metafacture/metafix/FixMethod.java new file mode 100644 index 000000000..b06352338 --- /dev/null +++ b/metafix/src/main/java/org/metafacture/metafix/FixMethod.java @@ -0,0 +1,750 @@ +/* + * Copyright 2021 Fabian Steeg, hbz + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix; + +import org.metafacture.metafix.api.FixFunction; +import org.metafacture.metafix.maps.RdfMap; +import org.metafacture.metamorph.api.Maps; +import org.metafacture.metamorph.functions.ISBN; +import org.metafacture.metamorph.functions.Timestamp; +import org.metafacture.metamorph.functions.URLEncode; +import org.metafacture.metamorph.maps.FileMap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Base64; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.atomic.LongAdder; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.function.UnaryOperator; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +public enum FixMethod implements FixFunction { // checkstyle-disable-line ClassDataAbstractionCoupling|ClassFanOutComplexity + + // SCRIPT-LEVEL METHODS: + + include { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final String includeFile = params.get(0); + + if (!Metafix.isFixFile(includeFile)) { + throw new IllegalArgumentException("Not a Fix file: " + includeFile); + } + + // TODO: Catmandu load path + final String includePath = metafix.resolvePath(includeFile); + + metafix.getRecordTransformer(includePath).transform(record, options); + } + }, + log { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + // does not support Catmandu log level option FATAL + + final String level = options.getOrDefault("level", "INFO"); + final Consumer consumer; + + switch (level) { + case "DEBUG": + consumer = LOG::debug; + break; + case "ERROR": + consumer = LOG::error; + break; + case "INFO": + consumer = LOG::info; + break; + case "WARN": + consumer = LOG::warn; + break; + default: + throw new IllegalArgumentException("Unsupported log level: " + level); + } + + consumer.accept(params.get(0)); + } + }, + nothing { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + // do nothing + } + }, + put_filemap { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final String fileName = params.get(0); + final FileMap fileMap = new FileMap(); + + fileMap.setSeparator(options.getOrDefault(FILEMAP_SEPARATOR_OPTION, FILEMAP_DEFAULT_SEPARATOR)); + fileMap.setFile(metafix.resolvePath(fileName)); + + withOption(options, "allow_empty_values", fileMap::setAllowEmptyValues, this::getBoolean); + withOption(options, "compression", fileMap::setCompression); + withOption(options, "decompress_concatenated", fileMap::setDecompressConcatenated, this::getBoolean); + withOption(options, "encoding", fileMap::setEncoding); + withOption(options, "expected_columns", fileMap::setExpectedColumns, this::getInteger); + withOption(options, "ignore_pattern", fileMap::setIgnorePattern); + withOption(options, "key_column", fileMap::setKeyColumn, this::getInteger); + withOption(options, "value_column", fileMap::setValueColumn, this::getInteger); + + metafix.putMap(params.size() > 1 ? params.get(1) : fileName, fileMap); + } + }, + put_map { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + metafix.putMap(params.get(0), options); + } + }, + put_rdfmap { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final String fileName = params.get(0); + final RdfMap rdfMap = new RdfMap(); + + rdfMap.setResource(fileName, metafix::resolvePath); + + withOption(options, RdfMap.TARGET, rdfMap::setTarget); + withOption(options, RdfMap.TARGET_LANGUAGE, rdfMap::setTargetLanguage); + withOption(options, RdfMap.SELECT, rdfMap::setSelect); + withOption(options, Maps.DEFAULT_MAP_KEY, rdfMap::setDefault); + + metafix.putMap(params.size() > 1 ? params.get(1) : fileName, rdfMap); + } + }, + put_var { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + metafix.getVars().put(params.get(0), params.get(1)); + } + }, + put_vars { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + metafix.getVars().putAll(options); + } + }, + to_var { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final Value value = record.get(params.get(0)); + metafix.getVars().put(params.get(1), Value.isNull(value) ? options.getOrDefault(DEFAULT_OPTION, "") : value.asString()); + } + }, + + // RECORD-LEVEL METHODS: + + add_array { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final String field = params.get(0); + final Value newValue = newArray(params.subList(1, params.size()).stream().map(Value::new)); + record.set(field, newValue); + newValue.asArray().forEach(value -> value.withPathSet(newValue.getPath() + "." + value.getPath())); + } + }, + add_field { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + record.set(params.get(0), new Value(params.get(1))); + } + }, + add_hash { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final String field = params.get(0); + final Value newValue = Value.newHash(h -> options.forEach((f, v) -> h.put(f, new Value(v)))); + record.set(field, newValue); + } + }, + array { // array-from-hash + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final String field = params.get(0); + + record.getList(field, a -> a.forEach(v -> v.matchType().ifHash(h -> { + record.remove(field); + + h.forEach((subField, value) -> { + record.addNested(field, new Value(subField)); + record.addNested(field, value.withPathSet(null)); + }); + }))); + } + }, + call_macro { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final String macroName = params.get(0); + final RecordTransformer recordTransformer = metafix.getMacro(macroName); + + if (recordTransformer != null) { + recordTransformer.transform(record, options); + } + else { + throw new IllegalArgumentException("Macro '" + macroName + "' undefined!"); + } + } + }, + copy_field { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final String oldName = params.get(0); + final String newName = params.get(1); + + final Value oldValue = record.get(oldName); + if (!Value.isNull(oldValue)) { + oldValue.matchType() + .ifArray(a -> { + record.remove(newName); + a.forEach(v -> record.addNested(newName, v.withPathSet(null))); + }) + .orElse(v -> record.set(newName, v.withPathSet(null))); + } + } + }, + format { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final String field = params.get(0); + + record.getList(field, oldValues -> { + final String newValue = String.format(params.get(1), oldValues.stream().toArray()); + record.replace(field, new Value(Arrays.asList(new Value(newValue)))); + }); + } + }, + hash { // hash-from-array + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final String field = params.get(0); + + record.getList(field, a -> record.put(field, Value.newHash(h -> { + for (int i = 1; i < a.size(); i = i + 2) { + h.put(a.get(i - 1).asString(), a.get(i)); + } + }))); + } + }, + move_field { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + FixMethod.copy_field.apply(metafix, record, params, options); + record.remove(params.get(0)); + } + }, + parse_text { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final String field = params.get(0); + + record.getList(field, a -> a.forEach(v -> { + final Pattern p = Pattern.compile(params.get(1)); + final Matcher m = p.matcher(v.asString()); + if (m.matches()) { + record.remove(field); + + /** + * {@code Pattern.namedGroups()} not available as API, + * see https://stackoverflow.com/a/65012527. + * + * Assumptions: 1. Named groups are not escaped/quoted; + * 2. Named groups are not mixed with unnamed groups. + */ + final Matcher groupMatcher = NAMED_GROUP_PATTERN.matcher(p.pattern()); + final Value value = Value.newHash(h -> { + while (groupMatcher.find()) { + final String group = groupMatcher.group(1); + h.put(group, new Value(m.group(group))); + } + }); + + if (!value.asHash().isEmpty()) { + record.addNested(field, value); + } + else { + for (int i = 1; i <= m.groupCount(); i = i + 1) { + record.addNested(field, new Value(m.group(i))); + } + } + } + })); + } + }, + paste { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final String joinChar = options.get("join_char"); + final Value newValue = new Value(params.subList(1, params.size()).stream() + .filter(f -> literalString(f) || record.get(f) != null) + .map(f -> literalString(f) ? new Value(f.substring(1)) : Value.asList(record.get(f), null).asArray().get(0)) + .map(Value::asString).collect(Collectors.joining(joinChar != null ? joinChar : " "))); + record.set(params.get(0), newValue); + } + + private boolean literalString(final String s) { + return s.startsWith("~"); + } + }, + print_record { + private final Map scopedCounter = new HashMap<>(); + + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final boolean internal = getBoolean(options, "internal"); + final boolean pretty = getBoolean(options, "pretty"); + + if (!params.isEmpty()) { + options.put("prefix", params.get(0)); + } + + withWriter(metafix, record, options, scopedCounter, c -> { + if (internal) { + if (pretty) { + record.forEach((f, v) -> c.accept(f + "=" + v)); + } + else { + c.accept(record.toString()); + } + } + else { + try { + c.accept(record.toJson(pretty)); + } + catch (final IOException e) { + // Log a warning? Print string representation instead? + } + } + }); + } + }, + random { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final String field = params.get(0); + final int max = getInteger(params, 1); + record.set(field, new Value(String.valueOf(RANDOM.nextInt(max)))); + } + }, + reject { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + record.setReject(true); + } + }, + remove_field { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + params.forEach(p -> record.remove(p)); + } + }, + rename { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final String search = params.get(1); + final String replace = params.get(2); + + final UnaryOperator operator = s -> s.replaceAll(search, replace); + + record.transform(params.get(0), (m, c) -> m + .ifArray(a -> c.accept(renameArray(a, operator))) + .ifHash(h -> c.accept(renameHash(h, operator))) + .orElseThrow() + ); + } + + private Value renameArray(final Value.Array array, final UnaryOperator operator) { + return Value.newArray(a -> array.forEach(v -> a.add(renameValue(v, operator)))); + } + + private Value renameHash(final Value.Hash hash, final UnaryOperator operator) { + return Value.newHash(h -> hash.forEach((f, v) -> h.put(operator.apply(f), renameValue(v, operator)))); + } + + private Value renameValue(final Value value, final UnaryOperator operator) { + return value.extractType((m, c) -> m + .ifArray(a -> c.accept(renameArray(a, operator))) + .ifHash(h -> c.accept(renameHash(h, operator))) + .orElse(c) + ); + } + }, + retain { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + record.retainFields(params); + } + }, + set_array { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + add_array.apply(metafix, record, params, options); + } + }, + set_field { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + add_field.apply(metafix, record, params, options); + } + }, + set_hash { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + add_hash.apply(metafix, record, params, options); + } + }, + timestamp { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final String field = params.get(0); + final Timestamp timestamp = new Timestamp(); + + withOption(options, "format", timestamp::setFormat); + withOption(options, "language", timestamp::setLanguage); + withOption(options, "timezone", timestamp::setTimezone); + + record.set(field, new Value(timestamp.process(null))); + } + }, + vacuum { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + record.removeEmptyValues(); + } + }, + + // FIELD-LEVEL METHODS: + + // TODO SPEC: switch to morph-style named params in general? + + append { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final String value = params.get(1); + record.transform(params.get(0), s -> s + value); + } + }, + capitalize { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + record.transform(params.get(0), s -> s.substring(0, 1).toUpperCase() + s.substring(1)); + } + }, + count { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + record.transform(params.get(0), (m, c) -> m + .ifArray(a -> c.accept(new Value(a.size()))) + .ifHash(h -> c.accept(new Value(h.size()))) + ); + } + }, + downcase { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + record.transform(params.get(0), s -> s.toLowerCase()); + } + }, + filter { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final Pattern search = Pattern.compile(params.get(1)); + final boolean invert = getBoolean(options, "invert"); + + final Predicate predicate = s -> search.matcher(s.asString()).find(); + + record.transform(params.get(0), (m, c) -> m + .ifArray(a -> c.accept(newArray(a.stream().filter(invert ? predicate.negate() : predicate)))) + ); + } + }, + flatten { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + record.transform(params.get(0), (m, c) -> m + .ifArray(a -> c.accept(newArray(flatten(a.stream())))) + ); + } + }, + from_json { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final String errorString = options.get(ERROR_STRING_OPTION); + final JsonValue.Parser parser = new JsonValue.Parser(); + + record.transform(params.get(0), (m, c) -> m + .ifString(s -> { + try { + c.accept(parser.parse(s)); + } + catch (final IOException e) { + c.accept(errorString != null ? new Value(errorString) : null); + } + }) + ); + } + }, + index { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final String search = params.get(1); + record.transform(params.get(0), s -> String.valueOf(s.indexOf(search))); // TODO: multiple + } + }, + isbn { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final ISBN isbn = new ISBN(); + + withOption(options, ERROR_STRING_OPTION, isbn::setErrorString); + withOption(options, "to", isbn::setTo); + withOption(options, "verify_check_digit", isbn::setVerifyCheckDigit); + + record.transform(params.get(0), isbn::process); + } + }, + join_field { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final String joinChar = params.size() > 1 ? params.get(1) : ""; + record.transform(params.get(0), (m, c) -> m + .ifArray(a -> c.accept(new Value(a.stream().map(Value::asString).collect(Collectors.joining(joinChar))))) + ); + } + }, + lookup { + private final Map scopedCounter = new HashMap<>(); + + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final Map map; + + if (params.size() <= 1) { + map = options; + } + else { + final String mapName = params.get(1); + + if (!metafix.getMapNames().contains(mapName)) { + if (mapName.contains(".") || mapName.contains(File.separator)) { + put_filemap.apply(metafix, record, Arrays.asList(mapName), options); + } + else { + // Probably an unknown internal map? Log a warning? + } + } + + map = metafix.getMap(mapName); + } + + final String defaultValue = options.getOrDefault("default", map.get(Maps.DEFAULT_MAP_KEY)); + final boolean delete = getBoolean(options, "delete"); + final boolean printUnknown = getBoolean(options, "print_unknown"); + + final Consumer> consumer = c -> record.transform(params.get(0), oldValue -> { + final String newValue = map.get(oldValue); + if (newValue != null) { + return newValue; + } + else { + if (c != null) { + c.accept(oldValue); + } + + return defaultValue != null ? defaultValue : delete ? null : oldValue; + } + }); + + if (printUnknown) { + options.putIfAbsent("append", "true"); + withWriter(metafix, record, options, scopedCounter, consumer); + } + else { + consumer.accept(null); + } + } + }, + prepend { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final String value = params.get(1); + record.transform(params.get(0), s -> value + s); + } + }, + replace_all { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final String search = params.get(1); + final String replace = params.get(2); + + record.transform(params.get(0), s -> s.replaceAll(search, replace)); + } + }, + reverse { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + record.transform(params.get(0), (m, c) -> m + .ifArray(a -> { + final List list = a.stream().collect(Collectors.toList()); + Collections.reverse(list); + c.accept(new Value(list)); + }) + .ifString(s -> c.accept(new Value(new StringBuilder(s).reverse().toString()))) + ); + } + }, + sort_field { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final boolean numeric = getBoolean(options, "numeric"); + final boolean reverse = getBoolean(options, "reverse"); + final boolean uniq = getBoolean(options, "uniq"); + + final Function function = Value::asString; + final Comparator comparator = numeric ? + Comparator.comparing(function.andThen(Integer::parseInt)) : Comparator.comparing(function); + + record.transform(params.get(0), (m, c) -> m + .ifArray(a -> c.accept(new Value((uniq ? unique(a.stream()) : a.stream()) + .sorted(reverse ? comparator.reversed() : comparator).collect(Collectors.toList())))) + ); + } + }, + split_field { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final String splitChar = params.size() > 1 ? params.get(1) : "\\s+"; + final Pattern splitPattern = Pattern.compile(splitChar); + + final Function splitFunction = s -> + newArray(Arrays.stream(splitPattern.split(s)).map(Value::new)); + + record.transform(params.get(0), (m, c) -> m + .ifArray(a -> c.accept(newArray(a.stream().map(Value::asString).map(splitFunction)))) + .ifHash(h -> c.accept(Value.newHash(n -> h.forEach((f, w) -> n.put(f, splitFunction.apply(w.asString())))))) + .ifString(s -> c.accept(splitFunction.apply(s))) + ); + } + }, + substring { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final int offset = getInteger(params, 1); + final Integer end = params.size() > 2 ? offset + getInteger(params, 2) : null; + // TODO: final String replacement = params.size() > 3 ? params.get(3) : null; + + record.transform(params.get(0), s -> { + final int length = s.length(); + return offset > length ? s : end == null || end > length ? s.substring(offset) : s.substring(offset, end); + }); + } + }, + sum { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + record.transform(params.get(0), (m, c) -> m + .ifArray(a -> c.accept(new Value(a.stream().map(Value::asString).mapToInt(Integer::parseInt).sum()))) + ); + } + }, + to_base64 { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final boolean urlSafe = getBoolean(options, "url_safe"); + final Base64.Encoder encoder = urlSafe ? Base64.getUrlEncoder() : Base64.getEncoder(); + + record.transform(params.get(0), s -> encoder.encodeToString(s.getBytes())); + } + }, + to_json { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final String errorString = options.get(ERROR_STRING_OPTION); + final boolean pretty = getBoolean(options, "pretty"); + + record.transform(params.get(0), (m, c) -> m + .orElse(v -> { + try { + c.accept(new Value(v.toJson(pretty))); + } + catch (final IOException e) { + c.accept(errorString != null ? new Value(errorString) : null); + } + }) + ); + } + }, + trim { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + record.transform(params.get(0), String::trim); + } + }, + uniq { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + record.transform(params.get(0), (m, c) -> m + .ifArray(a -> c.accept(newArray(unique(a.stream())))) + ); + } + }, + upcase { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + record.transform(params.get(0), s -> s.toUpperCase()); + } + }, + uri_encode { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final URLEncode urlEncoder = new URLEncode(); + withOption(options, "safe_chars", urlEncoder::setSafeChars); + withOption(options, "plus_for_space", urlEncoder::setPlusForSpace, this::getBoolean); + + record.transform(params.get(0), urlEncoder::process); + } + }; + + private static final Pattern NAMED_GROUP_PATTERN = Pattern.compile("\\(\\?<(.+?)>"); + + private static final String FILEMAP_SEPARATOR_OPTION = "sep_char"; + private static final String FILEMAP_DEFAULT_SEPARATOR = ","; + + private static final String DEFAULT_OPTION = "default"; + private static final String ERROR_STRING_OPTION = "error_string"; + + private static final Random RANDOM = new Random(); + + private static final Logger LOG = LoggerFactory.getLogger(FixMethod.class); + +} diff --git a/metafix/src/main/java/org/metafacture/metafix/FixParseException.java b/metafix/src/main/java/org/metafacture/metafix/FixParseException.java new file mode 100644 index 000000000..366a58ff1 --- /dev/null +++ b/metafix/src/main/java/org/metafacture/metafix/FixParseException.java @@ -0,0 +1,31 @@ +/* + * Copyright 2022 hbz NRW + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix; + +import org.metafacture.framework.MetafactureException; + +public class FixParseException extends MetafactureException { + + public FixParseException(final String message) { + super(message); + } + + public FixParseException(final String message, final Throwable cause) { + super(message, cause); + } + +} diff --git a/metafix/src/main/java/org/metafacture/metafix/FixPath.java b/metafix/src/main/java/org/metafacture/metafix/FixPath.java new file mode 100644 index 000000000..7b5846eeb --- /dev/null +++ b/metafix/src/main/java/org/metafacture/metafix/FixPath.java @@ -0,0 +1,376 @@ +/* + * Copyright 2022 Fabian Steeg, hbz NRW + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix; + +import org.metafacture.metafix.Value.Array; +import org.metafacture.metafix.Value.Hash; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +/** + * Our goal here is something like https://metacpan.org/pod/Catmandu::Path::simple + * + * With all get/set/update/create/delete logic collected here. + * + * @author Fabian Steeg (fsteeg) + * + */ +/*package-private*/ class FixPath { + + private static final String ASTERISK = "*"; + private String[] path; + + /*package-private*/ FixPath(final String path) { + this(Value.split(path)); + } + + private FixPath(final String[] path) { + this.path = path; + } + + /*package-private*/ Value findIn(final Hash hash) { + return findIn(hash, false); + } + + /*package-private*/ Value findIn(final Hash hash, final boolean enforceStringValue) { + final String currentSegment = path[0]; + final FixPath remainingPath = new FixPath(tail(path)); + if (currentSegment.equals(ASTERISK) && remainingPath.size() > 0) { + // TODO: search in all elements of hash? + return remainingPath.findIn(hash, enforceStringValue); + } + final Value value = hash.get(currentSegment, enforceStringValue && path.length == 1); + return value == null || path.length == 1 ? value : value.extractType((m, c) -> m + .ifArray(a -> c.accept(remainingPath.findIn(a))) + .ifHash(h -> c.accept(remainingPath.findIn(h, enforceStringValue))) + .orElseThrow() + ); + } + + /*package-private*/ Value findIn(final Array array) { + final Value result; + + if (path.length == 0) { + result = new Value(array); + } + else { + final String currentSegment = path[0]; + if (currentSegment.equals(ASTERISK)) { + result = Value.newArray(resultArray -> array.forEach(v -> { + final Value findInValue = findInValue(v, tail(path)); + if (findInValue != null) { + findInValue.matchType() + // flatten result arrays (use Value#path for structure) + .ifArray(a -> a.forEach(t -> resultArray.add(t, false))) + .orElse(c -> resultArray.add(findInValue, false)); + } + })); + } + else if (isReference(currentSegment)) { + final Value referencedValue = getReferencedValue(array, currentSegment, null); + if (referencedValue != null) { + result = findInValue(referencedValue, tail(path)); + } + else { + result = null; + } + } + // TODO: WDCD? copy_field('your.name','author[].name'), where name is an array + else { + result = Value.newArray(a -> array.forEach(v -> a.add(findInValue(v, path)))); + } + } + + return result; + } + + private Value findInValue(final Value value, final String[] p) { + // TODO: move impl into enum elements, here call only value.find + return p.length == 0 ? value : value == null ? null : value.extractType((m, c) -> m + .ifArray(a -> c.accept(new FixPath(p).findIn(a))) + .ifHash(h -> c.accept(new FixPath(p).findIn(h))) + .orElse(c) + ); + } + + @Override + public String toString() { + return String.join(".", path); + } + + /*package-private*/ int size() { + return path.length; + } + + // TODO: this is still very much work in progress, I think we should + // try to replace this with consistent usage of Value#getPath + // (e.g. take care of handling repeated fields and their paths) + + /*package-private*/ FixPath to(final Value value, final int i) { + final FixPath result; + + // One *, no matching path: replace with index of current result + if (countAsterisks() == 1 && !matches(value.getPath())) { + result = new FixPath(replaceInPath(ASTERISK, i)); + } + // Multiple * or wildcards, value has a path: use the value's path + else if (value.getPath() != null && hasWildcard()) { + result = new FixPath(value.getPath()); + } + else { + result = this; + } + + return result; + } + + private boolean matches(final String thatPath) { + return thatPath != null && thatPath.replaceAll("\\.\\d+\\.", ".*.").equals(String.join(".", this.path)); + } + + private String[] replaceInPath(final String find, final int i) { + return Arrays.asList(path).stream().map(s -> s.equals(find) ? String.valueOf(i + 1) : s).toArray(String[]::new); + } + + private boolean hasWildcard() { + return Arrays.asList(path).stream().filter(s -> s.contains("*") || s.contains("?") || s.contains("|") || s.matches(".*?\\[.+?\\].*?")).findAny().isPresent(); + } + + private long countAsterisks() { + return Arrays.asList(path).stream().filter(s -> s.equals(ASTERISK)).count(); + } + + /*package-private*/ enum InsertMode { + + REPLACE { + @Override + void apply(final Hash hash, final String field, final Value value) { + hash.put(field, value); + } + + @Override + void apply(final Array array, final String field, final Value value) { + try { + final ReservedField reservedField = ReservedField.fromString(field); + if (reservedField != null) { + switch (reservedField) { + case $prepend: + array.add(0, value); + break; + case $append: + array.add(value); + break; + case $first: + array.set(0, value); + break; + case $last: + array.set(array.size() - 1, value); + break; + default: + break; + } + } + else { + array.set(Integer.valueOf(field) - 1, value); + } + } + catch (final NumberFormatException e) { + throw new IllegalStateException("Expected Hash, got Array", e); + } + } + }, + APPEND { + @Override + void apply(final Hash hash, final String field, final Value value) { + hash.add(field, value); + } + + @Override + void apply(final Array array, final String field, final Value value) { + array.add(value); + } + }; + + abstract void apply(Hash hash, String field, Value value); + + abstract void apply(Array array, String field, Value newValue); + + } + + /*package-private*/ void removeNestedFrom(final Array array) { + if (path.length >= 1 && path[0].equals(ASTERISK)) { + array.removeAll(); + } + else if (path.length >= 1 && Value.isNumber(path[0])) { + final int index = Integer.parseInt(path[0]) - 1; // TODO: 0-based Catmandu vs. 1-based Metafacture + if (index >= 0 && index < array.size()) { + if (path.length == 1) { + array.remove(index); + } + else { + removeNestedFrom(array.get(index)); + } + } + } + } + + /*package-private*/ void removeNestedFrom(final Hash hash) { + final String field = path[0]; + + if (path.length == 1) { + hash.remove(field); + } + else if (hash.containsField(field)) { + removeNestedFrom(hash.get(field)); + } + } + + private void removeNestedFrom(final Value value) { + // TODO: impl and call just value.remove + if (value != null) { + value.matchType() + .ifArray(a -> new FixPath(tail(path)).removeNestedFrom(a)) + .ifHash(h -> new FixPath(tail(path)).removeNestedFrom(h)) + .orElseThrow(); + } + } + + private Value insertInto(final Array array, final InsertMode mode, final Value newValue) { + // basic idea: reuse findIn logic here? setIn(findIn(array), newValue) + final String field = path[0]; + + if (path.length == 1) { + mode.apply(array, field, newValue); + } + else { + if (ASTERISK.equals(field)) { + array.forEach(value -> insertInto(value, mode, newValue.copy(), field, tail(path))); + } + else if (isReference(field)) { + insertInto(getReferencedValue(array, field, newValue.getPath()), mode, newValue, field, tail(path)); + } + } + + return new Value(array); + } + + /*package-private*/ Value insertInto(final Hash hash, final InsertMode mode, final Value newValue) { + // basic idea: reuse findIn logic here? setIn(findIn(hash), newValue) + final String field = path[0]; + + if (path.length == 1) { + mode.apply(hash, field, newValue); + } + else { + final String[] tail = tail(path); + insertInto(getContainerValue(hash, field, newValue.getPath(), tail[0]), mode, newValue, field, tail); + } + + return new Value(hash); + } + + private Value insertInto(final Value value, final InsertMode mode, final Value newValue, final String field, final String[] tail) { + if (value != null) { + final FixPath fixPath = new FixPath(tail); + newValue.withPathSet(value.getPath()); + return value.extractType((m, c) -> m + .ifArray(a -> c.accept(fixPath.insertInto(a, mode, newValue))) + .ifHash(h -> c.accept(fixPath.insertInto(h, mode, newValue))) + .orElseThrow()); + } + else { + throw new IllegalArgumentException("Can't find: " + field + " in: " + value); + } + } + + private Value getContainerValue(final Hash hash, final String field, final String newPath, final String nextField) { + Value result = hash.get(field); + final boolean isAddingToArray = nextField.equals(ReservedField.$prepend.name()) || nextField.equals(ReservedField.$append.name()); + if (result == null) { + result = (isAddingToArray ? Value.newArray() : Value.newHash()).withPathSet(newPath); + hash.put(field, result); + } + else { + if (isAddingToArray && result.isString()) { + final Value value = result; + result = Value.newArray(a -> a.add(value)); + hash.put(field, result); + } + } + return result; + } + + private String[] tail(final String[] fields) { + return Arrays.copyOfRange(fields, 1, fields.length); + } + + private enum ReservedField { + $prepend, $append, $first, $last; + + private static final Map STRING_TO_ENUM = new HashMap<>(); + static { + for (final ReservedField f : values()) { + STRING_TO_ENUM.put(f.toString(), f); + } + } + + static ReservedField fromString(final String string) { + return STRING_TO_ENUM.get(string); + } + } + + private boolean isReference(final String field) { + return ReservedField.fromString(field) != null || Value.isNumber(field); + } + + // TODO replace switch, extract to method on array? + private Value getReferencedValue(final Array array, final String field, final String p) { + Value referencedValue = null; + + if (Value.isNumber(field)) { + final int index = Integer.valueOf(field) - 1; + return 0 <= index && index < array.size() ? array.get(index) : null; + } + + final ReservedField reservedField = ReservedField.fromString(field); + if (reservedField != null) { + switch (reservedField) { + case $first: + referencedValue = getReferencedValue(array, "1", p); + break; + case $last: + referencedValue = getReferencedValue(array, String.valueOf(array.size()), p); + break; + case $append: + referencedValue = Value.newHash().withPathSet(p); // TODO: append non-hash? + array.add(referencedValue); + break; + case $prepend: + referencedValue = Value.newHash().withPathSet(p); + array.add(0, referencedValue); + break; + default: + break; + } + } + + return referencedValue; + } + +} diff --git a/metafix/src/main/java/org/metafacture/metafix/FixProcessException.java b/metafix/src/main/java/org/metafacture/metafix/FixProcessException.java new file mode 100644 index 000000000..849f445c7 --- /dev/null +++ b/metafix/src/main/java/org/metafacture/metafix/FixProcessException.java @@ -0,0 +1,37 @@ +/* + * Copyright 2022 hbz NRW + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix; + +import org.metafacture.framework.MetafactureException; + +/** + * Indicates static (i.e., data-independent) issues with the usage of Fix + * expressions. + * + * @see FixExecutionException + */ +public class FixProcessException extends MetafactureException { + + public FixProcessException(final String message) { + super(message); + } + + public FixProcessException(final String message, final Throwable cause) { + super(message, cause); + } + +} diff --git a/metafix/src/main/java/org/metafacture/metafix/FixRuntimeModule.java b/metafix/src/main/java/org/metafacture/metafix/FixRuntimeModule.java new file mode 100644 index 000000000..59aaca061 --- /dev/null +++ b/metafix/src/main/java/org/metafacture/metafix/FixRuntimeModule.java @@ -0,0 +1,17 @@ +package org.metafacture.metafix; + +import org.metafacture.metafix.interpreter.FixInterpreter; + +/** + * Use this class to register components to be used at runtime / without the Equinox extension registry. + */ +public class FixRuntimeModule extends AbstractFixRuntimeModule { + + public FixRuntimeModule() { + } + + public Class bindFixInterpreter() { + return FixInterpreter.class; + } + +} diff --git a/metafix/src/main/java/org/metafacture/metafix/FixStandaloneSetup.java b/metafix/src/main/java/org/metafacture/metafix/FixStandaloneSetup.java new file mode 100644 index 000000000..cbe9b86f5 --- /dev/null +++ b/metafix/src/main/java/org/metafacture/metafix/FixStandaloneSetup.java @@ -0,0 +1,55 @@ +package org.metafacture.metafix; + +import org.metafacture.metafix.fix.Fix; +import org.metafacture.metafix.validation.XtextValidator; + +import com.google.common.io.CharStreams; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.Reader; +import java.io.UncheckedIOException; + +/** + * Initialization support for running Xtext languages without Equinox extension registry. + */ +public class FixStandaloneSetup extends FixStandaloneSetupGenerated { + + public FixStandaloneSetup() { + } + + public static void main(final String[] args) { + if (args != null && args.length == 1) { + System.exit(XtextValidator.validate(args[0], new FixStandaloneSetup()) ? 0 : 1); + } + + throw new IllegalArgumentException(String.format("Usage: %s ", FixStandaloneSetup.class.getName())); + } + + public static Fix parseFix(final String path) { + return (Fix) XtextValidator.getValidatedResource(path, new FixStandaloneSetup()).getContents().get(0); + } + + public static Fix parseFix(final Reader fixDef) { + try { + return parseFix(absPathToTempFile(fixDef, ".fix")); + } + catch (final IOException e) { + throw new UncheckedIOException(e); + } + } + + public static String absPathToTempFile(final Reader fixDef, final String suffix) throws IOException { + // TODO: avoid temp file creation + final File file = File.createTempFile("metafix", suffix); + file.deleteOnExit(); + + try (FileWriter out = new FileWriter(file)) { + CharStreams.copy(fixDef, out); + } + + return file.getAbsolutePath(); + } + +} diff --git a/metafix/src/main/java/org/metafacture/metafix/GenerateFix.mwe2 b/metafix/src/main/java/org/metafacture/metafix/GenerateFix.mwe2 new file mode 100644 index 000000000..2ba03cbd7 --- /dev/null +++ b/metafix/src/main/java/org/metafacture/metafix/GenerateFix.mwe2 @@ -0,0 +1,55 @@ +module org.metafacture.metafix.GenerateFix + +import org.eclipse.xtext.xtext.generator.* +import org.eclipse.xtext.xtext.generator.model.project.* + +Workflow { + component = XtextGenerator { + configuration = { + project = StandardProjectConfig { + baseName = "metafix" + rootPath = ".." + runtimeTest = { + enabled = true + } + genericIde = { + name = "metafix-ide" + } + web = { + name = "metafix-web" + enabled = true + } + mavenLayout = true + } + code = { + encoding = "UTF-8" + lineDelimiter = "\n" + fileHeader = "/*\n * generated by Xtext \${version}\n */" + preferXtendStubs = false + } + } + language = StandardLanguage { + name = "org.metafacture.metafix.Fix" + fileExtensions = "fix" + serializer = { + generateStub = false + } + validator = { + // Generates checks for @Deprecated grammar annotations, an IssueProvider and a corresponding PropertyPage + generateDeprecationValidation = true + } + junitSupport = { + junitVersion = "5" + generateStub = false + } + webSupport = { + framework = "ace" + generateServlet = false + generateJettyLauncher = false + } + xbaseSupport = { + generateXtendInferrer = false + } + } + } +} diff --git a/metafix/src/main/java/org/metafacture/metafix/JsonValue.java b/metafix/src/main/java/org/metafacture/metafix/JsonValue.java new file mode 100644 index 000000000..d01a74ae7 --- /dev/null +++ b/metafix/src/main/java/org/metafacture/metafix/JsonValue.java @@ -0,0 +1,87 @@ +/* + * Copyright 2022 hbz NRW + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.SerializableString; +import com.fasterxml.jackson.core.util.DefaultPrettyPrinter; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.IOException; +import java.io.StringWriter; +import java.io.UncheckedIOException; + +// TODO: Utilize JsonDecoder/JsonEncoder instead? + +@FunctionalInterface +public interface JsonValue { + + void toJson(JsonGenerator jsonGenerator); + + default String toJson() throws IOException { + return toJson(false); + } + + default String toJson(final boolean prettyPrinting) throws IOException { + final StringWriter writer = new StringWriter(); + final JsonGenerator jsonGenerator = new JsonFactory().createGenerator(writer); + jsonGenerator.setPrettyPrinter(prettyPrinting ? new DefaultPrettyPrinter((SerializableString) null) : null); + + try { + toJson(jsonGenerator); + } + catch (final UncheckedIOException e) { + throw e.getCause(); + } + + jsonGenerator.flush(); + + return writer.toString(); + } + + class Parser { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + public Parser() { + } + + public Value parse(final String source) throws IOException { + return parse(MAPPER.readTree(source)); + } + + private Value parse(final JsonNode node) { + final Value value; + + if (node.isObject()) { + value = Value.newHash(h -> node.fields().forEachRemaining(e -> h.put(e.getKey(), parse(e.getValue())))); + } + else if (node.isArray()) { + value = Value.newArray(a -> node.elements().forEachRemaining(v -> a.add(parse(v)))); + } + else { + value = new Value(node.textValue()); + } + + return value; + } + + } + +} diff --git a/metafix/src/main/java/org/metafacture/metafix/ListFixPaths.java b/metafix/src/main/java/org/metafacture/metafix/ListFixPaths.java new file mode 100644 index 000000000..891dd4cf5 --- /dev/null +++ b/metafix/src/main/java/org/metafacture/metafix/ListFixPaths.java @@ -0,0 +1,52 @@ +/* + * Copyright 2023 Fabian Steeg, hbz + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix; + +import org.metafacture.framework.FluxCommand; +import org.metafacture.framework.StreamReceiver; +import org.metafacture.framework.annotations.Description; +import org.metafacture.framework.annotations.In; +import org.metafacture.framework.annotations.Out; +import org.metafacture.triples.AbstractTripleSort.Compare; + +/** + * Provide a user-friendly way to list all paths available for processing in fix (see also {@link ListFixValues}). + * + * @author Fabian Steeg + */ +@Description("Lists all paths found in the input records. These paths can be used in a Fix to address fields. Options: " + + "`count` (output occurence frequency of each path, sorted by highest frequency first; default: `true`), " + + "`template` (for formatting the internal triple structure; default: `${o}\t|\t${s}` if count is true, else `${s}`)" + + "`index` (output individual repeated subfields and array elements with index numbers instead of '*'; default: `false`)") +@In(StreamReceiver.class) +@Out(String.class) +@FluxCommand("list-fix-paths") +public class ListFixPaths extends MetafixStreamAnalyzer { + + public ListFixPaths() { + super("nothing()", Compare.PREDICATE); + setIndex(false); + } + + public void setIndex(final boolean index) { + getFix().setEntityMemberName(index ? Metafix.DEFAULT_ENTITY_MEMBER_NAME : "*"); + } + + public boolean getIndex() { + return getFix().getEntityMemberName().equals(Metafix.DEFAULT_ENTITY_MEMBER_NAME); + } +} diff --git a/metafix/src/main/java/org/metafacture/metafix/ListFixValues.java b/metafix/src/main/java/org/metafacture/metafix/ListFixValues.java new file mode 100644 index 000000000..bb56b590e --- /dev/null +++ b/metafix/src/main/java/org/metafacture/metafix/ListFixValues.java @@ -0,0 +1,49 @@ +/* + * Copyright 2023 Fabian Steeg, hbz + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix; + +import org.metafacture.framework.FluxCommand; +import org.metafacture.framework.StreamReceiver; +import org.metafacture.framework.annotations.Description; +import org.metafacture.framework.annotations.In; +import org.metafacture.framework.annotations.Out; +import org.metafacture.triples.AbstractTripleSort.Compare; + +/** + * Provide a user-friendly way to list all values for a given path (see {@link ListFixPaths}). + * + * @author Fabian Steeg + */ +@Description("Lists all values found for the given path. The paths can be found using fix-list-paths. Options: " + + "`count` (output occurence frequency of each value, sorted by highest frequency first; default: `true`)" + + "`template` (for formatting the internal triple structure; default: `${o}\t|\t${s}` if count is true, else `${s}`)") +@In(StreamReceiver.class) +@Out(String.class) +@FluxCommand("list-fix-values") +public class ListFixValues extends MetafixStreamAnalyzer { + + public ListFixValues(final String path) { + super(fix(path), Compare.OBJECT); + } + + private static String fix(final String path) { + return + "copy_field(\"" + path + "\",\"value.$append\")\n" + + "retain(\"value\")"; + } + +} diff --git a/metafix/src/main/java/org/metafacture/metafix/Metafix.java b/metafix/src/main/java/org/metafacture/metafix/Metafix.java new file mode 100644 index 000000000..478d6268e --- /dev/null +++ b/metafix/src/main/java/org/metafacture/metafix/Metafix.java @@ -0,0 +1,506 @@ +/* + * Copyright 2013, 2023 Deutsche Nationalbibliothek and others + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//TODO: move all classes here to fix package + +package org.metafacture.metafix; + +import org.metafacture.framework.FluxCommand; +import org.metafacture.framework.MetafactureException; +import org.metafacture.framework.StandardEventNames; +import org.metafacture.framework.StreamPipe; +import org.metafacture.framework.StreamReceiver; +import org.metafacture.framework.annotations.Description; +import org.metafacture.framework.annotations.In; +import org.metafacture.framework.annotations.Out; +import org.metafacture.framework.helpers.DefaultStreamReceiver; +import org.metafacture.mangling.StreamFlattener; +import org.metafacture.metafix.fix.Expression; +import org.metafacture.metamorph.api.Maps; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.Closeable; +import java.io.IOException; +import java.io.Reader; +import java.io.StringReader; +import java.io.UncheckedIOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Deque; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.function.BiConsumer; + +/** + * Transforms a data stream sent via the {@link StreamReceiver} interface. Uses + * {@link RecordTransformer} to transform records based on a Fix DSL description. + * + * @author Markus Michael Geipel (Metamorph) + * @author Christoph Böhme (Metamorph) + * @author Fabian Steeg (Metafix) + */ +@Description("Applies a fix transformation to the event stream, given as the path to a fix file or the fixes themselves.") // checkstyle-disable-line ClassDataAbstractionCoupling|ClassFanOutComplexity +@In(StreamReceiver.class) +@Out(StreamReceiver.class) +@FluxCommand("fix") +public class Metafix implements StreamPipe, Maps { + public static final String ARRAY_MARKER = "[]"; + public static final String FIX_EXTENSION = ".fix"; + public static final String VAR_END = "]"; + public static final String VAR_START = "$["; + + public static final Strictness DEFAULT_STRICTNESS = Strictness.PROCESS; + public static final String DEFAULT_ENTITY_MEMBER_NAME = "%d"; + + public static final Map NO_VARS = Collections.emptyMap(); + + public static final int MAX_ENTITY_COUNT = Integer.getInteger("org.metafacture.metafix.maxEntityCount", -1); + + private static final Logger LOG = LoggerFactory.getLogger(Metafix.class); + + private static final String ENTITIES_NOT_BALANCED = "Entity starts and ends are not balanced"; + + private final Deque entityCountStack = new LinkedList<>(); + private final List resources = new ArrayList<>(); + private final List expressions = new ArrayList<>(); + private final Map> maps = new HashMap<>(); + private final Map fixCache = new HashMap<>(); + private final Map macros = new HashMap<>(); + private final Map pathCache = new HashMap<>(); + private final Map vars = new HashMap<>(); + private final RecordTransformer recordTransformer; + private final StreamFlattener flattener = new StreamFlattener(); + + private List entities = new ArrayList<>(); + private Record currentRecord = new Record(); + private StreamReceiver outputStreamReceiver; + private Strictness strictness = DEFAULT_STRICTNESS; + private String fixFile; + private String recordIdentifier; + private String entityMemberName = DEFAULT_ENTITY_MEMBER_NAME; + private boolean repeatedFieldsToEntities; + private boolean strictnessHandlesProcessExceptions; + private int entityCount; + + public Metafix() { + this(NO_VARS); + } + + public Metafix(final Map newVars) { + init(newVars); + recordTransformer = null; + } + + public Metafix(final String fixDef) throws IOException { + this(fixDef, NO_VARS); + } + + public Metafix(final String fixDef, final Map vars) throws IOException { + init(vars); + + if (isFixFile(fixDef)) { + fixFile = fixDef; + recordTransformer = getRecordTransformer(fixDef); + } + else { + try (Reader reader = new StringReader(fixDef)) { + recordTransformer = getRecordTransformer(reader); + } + } + } + + public Metafix(final Reader fixDef) { + this(fixDef, NO_VARS); + } + + public Metafix(final Reader fixDef, final Map vars) { + init(vars); + recordTransformer = getRecordTransformer(fixDef); + } + + private void init(final Map newVars) { + flattener.setReceiver(new DefaultStreamReceiver() { + + @Override + public void literal(final String name, final String value) { + final String[] split = Value.split(name); + addValue(split[split.length - 1], new Value(value)); + // TODO use full path here to insert only once? + // new FixPath(name).insertInto(currentRecord, InsertMode.APPEND, new Value(value)); + } + + }); + + vars.putAll(newVars); + } + + /*package-private*/ static boolean isFixFile(final String fixDef) { + return fixDef.endsWith(FIX_EXTENSION); + } + + public String resolvePath(final String path) { + return pathCache.computeIfAbsent(path, this::resolvePathInternal); + } + + private String resolvePathInternal(final String path) { + final String resolvedPath; + + if (isValidUrl(path)) { + resolvedPath = path; + LOG.debug("Resolved path: url = '{}'", resolvedPath); + } + else { + final Path basePath; + + if (path.startsWith(".")) { + if (fixFile != null) { + basePath = getPath(fixFile).getParent(); + } + else { + throw new IllegalArgumentException("Cannot resolve relative path: " + path); + } + } + else { + basePath = getPath(""); + } + + resolvedPath = basePath.resolve(path).normalize().toString(); + LOG.debug("Resolved path: base = '{}', path = '{}', result = '{}'", basePath, path, resolvedPath); + } + + return resolvedPath; + } + + private boolean isValidUrl(final String url) { + try { + new URL(url); + return true; + } + catch (final MalformedURLException e) { + return false; + } + } + + private Path getPath(final String path) { + return Paths.get(path).toAbsolutePath().normalize(); + } + + public RecordTransformer getRecordTransformer(final String fixDef) { + return fixCache.computeIfAbsent(fixDef, k -> new RecordTransformer(this, FixStandaloneSetup.parseFix(k))); + } + + private RecordTransformer getRecordTransformer(final Reader fixDef) { + return new RecordTransformer(this, FixStandaloneSetup.parseFix(fixDef)); + } + + public void putMacro(final String name, final RecordTransformer macro) { + macros.put(name, macro); + } + + public RecordTransformer getMacro(final String name) { + final RecordTransformer macro = macros.get(name); + + if (macro != null) { + macro.setParentExceptionMessageFrom(recordTransformer); + } + + return macro; + } + + public List getExpressions() { + return expressions; + } + + @Override + public void startRecord(final String identifier) { + currentRecord = new Record(); + currentRecord.putVirtualField(StandardEventNames.ID, new Value(identifier)); + LOG.debug("Start record: {}", identifier); + flattener.startRecord(identifier); + entityCountStack.clear(); + entityCount = 0; + entityCountStack.add(entityCount); + recordIdentifier = identifier; + entities = new ArrayList<>(); + } + + @Override + public void endRecord() { + entityCountStack.removeLast(); + if (!entityCountStack.isEmpty()) { + throw new IllegalStateException(ENTITIES_NOT_BALANCED); + } + flattener.endRecord(); + LOG.debug("End record, walking Fix: {}", currentRecord); + recordTransformer.transform(currentRecord); + if (!currentRecord.getReject()) { + outputStreamReceiver.startRecord(recordIdentifier); + LOG.debug("Sending results to {}", outputStreamReceiver); + currentRecord.forEach(this::emit); + outputStreamReceiver.endRecord(); + } + } + + private void emit(final String field, final Value value) { + Value.asList(value, array -> { + final boolean isMulti = repeatedFieldsToEntities && array.size() > 1 || isArrayName(field); + if (isMulti) { + outputStreamReceiver.startEntity(field); + } + + for (int i = 0; i < array.size(); ++i) { + final Value currentValue = array.get(i); + final String fieldName = isMulti ? String.format(entityMemberName, i + 1) : field; + + currentValue.matchType() + .ifArray(a -> emit(isMulti ? fieldName + ARRAY_MARKER : fieldName, currentValue)) + .ifHash(h -> { + outputStreamReceiver.startEntity(fieldName); + h.forEach(this::emit); + outputStreamReceiver.endEntity(); + }) + .ifString(s -> outputStreamReceiver.literal(fieldName, s)); + } + + if (isMulti) { + outputStreamReceiver.endEntity(); + } + }); + } + + private boolean isArrayName(final String name) { + return name.endsWith(ARRAY_MARKER); + } + + private void addValue(final String name, final Value value) { + final int index = entityCountStack.peek() - 1; + if (index < 0 || entities.size() <= index) { + currentRecord.add(name, value); + } + else { + final Value entity = entities.get(index); + value.withPathSet(entity.getPath()); + entity.matchType() + .ifArray(a -> a.add(value)) + .ifHash(h -> h.add(name, value)) + .orElseThrow(); + } + } + + @Override + public void startEntity(final String name) { + if (name == null) { + throw new IllegalArgumentException("Entity name must not be null."); + } + + ++entityCount; + if (maxEntityCountExceeded()) { + LOG.debug("Maximum number of entities exceeded: {}/{}", entityCount, MAX_ENTITY_COUNT); + return; + } + + final Value value = isArrayName(name) ? Value.newArray() : Value.newHash(); + addValue(name, value); + entities.add(value); + + entityCountStack.push(entityCount); + flattener.startEntity(name); + } + + @Override + public void endEntity() { + if (maxEntityCountExceeded()) { + return; + } + + entityCountStack.pop(); + flattener.endEntity(); + } + + @Override + public void literal(final String name, final String value) { + if (maxEntityCountExceeded()) { + return; + } + + LOG.debug("Putting '{}': '{}'", name, value); + flattener.literal(name, value); + } + + @Override + public void resetStream() { + outputStreamReceiver.resetStream(); + } + + @Override + public void closeStream() { + for (final Closeable closeable : resources) { + try { + closeable.close(); + } + catch (final IOException e) { + throw new UncheckedIOException(e); + } + } + + outputStreamReceiver.closeStream(); + } + + /** + * @param streamReceiver the outputHandler to set + */ + @Override + public R setReceiver(final R streamReceiver) { + if (streamReceiver == null) { + throw new IllegalArgumentException("'streamReceiver' must not be null"); + } + outputStreamReceiver = streamReceiver; + return streamReceiver; + } + + public StreamReceiver getStreamReceiver() { + return outputStreamReceiver; + } + + public Map getVars() { + return vars; + } + + public Record getCurrentRecord() { + return currentRecord; + } + + @Override + public Collection getMapNames() { + return Collections.unmodifiableSet(maps.keySet()); + } + + @Override + public Map getMap(final String mapName) { + return maps.getOrDefault(mapName, Collections.emptyMap()); + } + + @Override + public String getValue(final String mapName, final String key) { + final Map map = getMap(mapName); + return map.containsKey(key) ? map.get(key) : map.get(Maps.DEFAULT_MAP_KEY); + } + + @Override + public Map putMap(final String mapName, final Map map) { + if (map instanceof Closeable) { + resources.add((Closeable) map); + } + + return maps.put(mapName, map); + } + + @Override + public String putValue(final String mapName, final String key, final String value) { + return maps.computeIfAbsent(mapName, k -> new HashMap<>()).put(key, value); + } + + public void setStrictness(final Strictness strictness) { + this.strictness = strictness != null ? strictness : DEFAULT_STRICTNESS; + } + + public Strictness getStrictness() { + return strictness; + } + + public void setStrictnessHandlesProcessExceptions(final boolean strictnessHandlesProcessExceptions) { + this.strictnessHandlesProcessExceptions = strictnessHandlesProcessExceptions; + } + + public boolean getStrictnessHandlesProcessExceptions() { + return strictnessHandlesProcessExceptions; + } + + public void setRepeatedFieldsToEntities(final boolean repeatedFieldsToEntities) { + this.repeatedFieldsToEntities = repeatedFieldsToEntities; + } + + public boolean getRepeatedFieldsToEntities() { + return repeatedFieldsToEntities; + } + + public void setEntityMemberName(final String entityMemberName) { + this.entityMemberName = entityMemberName; + } + + public String getEntityMemberName() { + return entityMemberName; + } + + private boolean maxEntityCountExceeded() { + return MAX_ENTITY_COUNT >= 0 && entityCount > MAX_ENTITY_COUNT; + } + + public enum Strictness { + + /** + * Aborts process by throwing an exception. + */ + PROCESS { + @Override + protected void handleInternal(final MetafactureException exception, final Record record) { + throw exception; + } + }, + + /** + * Ignores (skips) record and logs an error. + */ + RECORD { + @Override + protected void handleInternal(final MetafactureException exception, final Record record) { + log(exception, LOG::error); + record.setReject(true); // TODO: Skip remaining expressions? + } + }, + + /** + * Ignores (skips) expression and logs a warning. + */ + EXPRESSION { + @Override + protected void handleInternal(final MetafactureException exception, final Record record) { + log(exception, LOG::warn); + } + }; + + public void handle(final MetafactureException exception, final Record record) { + LOG.info("Current record: {}", record); + handleInternal(exception, record); + } + + protected abstract void handleInternal(MetafactureException exception, Record record); + + protected void log(final MetafactureException exception, final BiConsumer logger) { + logger.accept(exception.getMessage(), exception.getCause()); + } + + } +} diff --git a/metafix/src/main/java/org/metafacture/metafix/MetafixStreamAnalyzer.java b/metafix/src/main/java/org/metafacture/metafix/MetafixStreamAnalyzer.java new file mode 100644 index 000000000..e7fa9fdfa --- /dev/null +++ b/metafix/src/main/java/org/metafacture/metafix/MetafixStreamAnalyzer.java @@ -0,0 +1,139 @@ +/* + * Copyright 2023 Fabian Steeg, hbz + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix; + +import org.metafacture.formatting.ObjectTemplate; +import org.metafacture.framework.MetafactureException; +import org.metafacture.framework.ObjectReceiver; +import org.metafacture.framework.helpers.DefaultStreamPipe; +import org.metafacture.mangling.StreamFlattener; +import org.metafacture.triples.AbstractTripleSort.Compare; +import org.metafacture.triples.AbstractTripleSort.Order; +import org.metafacture.triples.StreamToTriples; +import org.metafacture.triples.TripleCount; +import org.metafacture.triples.TripleSort; + +import java.io.IOException; + +/** + * Superclass for Metafix-based analyzer modules based on triples (see {@link org.metafacture.framework.objects.Triple}). + * + * @author Fabian Steeg + */ +/* package-private */ class MetafixStreamAnalyzer extends DefaultStreamPipe> { + + private static final String DEFAULT_COUNTED_TEMPLATE = "${o}\t|\t${s}"; + private static final String DEFAULT_UNCOUNTED_TEMPLATE = "${s}"; + + private final Metafix fix; + private boolean count = true; + private final Compare countBy; + private String template; + + /* package-private */ MetafixStreamAnalyzer(final String fix, final Compare countBy) { + try { + this.fix = new Metafix(fix); + this.fix.setRepeatedFieldsToEntities(true); + } + catch (final IOException e) { + throw new MetafactureException(e); + } + this.countBy = countBy; + } + + @Override + protected void onSetReceiver() { + template = template != null ? template : count ? DEFAULT_COUNTED_TEMPLATE : DEFAULT_UNCOUNTED_TEMPLATE; + fix + .setReceiver(new StreamFlattener()) + .setReceiver(new StreamToTriples()) + .setReceiver(tripleCount()) + .setReceiver(tripleSort()) + .setReceiver(new ObjectTemplate<>(template)) + .setReceiver(getReceiver()); + } + + private TripleCount tripleCount() { + final TripleCount tripleCount = new TripleCount(); + tripleCount.setCountBy(countBy); + return tripleCount; + } + + private TripleSort tripleSort() { + final TripleSort tripleSort = new TripleSort(); + tripleSort.setNumeric(count); + tripleSort.setBy(count ? Compare.OBJECT : Compare.SUBJECT); + tripleSort.setOrder(count ? Order.DECREASING : Order.INCREASING); + return tripleSort; + } + + @Override + public void startRecord(final String identifier) { + fix.startRecord(identifier); + } + + @Override + public void endRecord() { + fix.endRecord(); + } + + @Override + public void startEntity(final String name) { + fix.startEntity(name); + } + + @Override + public void endEntity() { + fix.endEntity(); + } + + @Override + public void literal(final String name, final String value) { + fix.literal(name, value); + } + + @Override + protected void onCloseStream() { + fix.closeStream(); + } + + @Override + protected void onResetStream() { + fix.resetStream(); + } + + public void setCount(final boolean count) { + this.count = count; + } + + public boolean getCount() { + return this.count; + } + + public void setTemplate(final String template) { + this.template = template; + } + + public String getTemplate() { + return this.template; + } + + /* package-private */ Metafix getFix() { + return this.fix; + } + +} diff --git a/metafix/src/main/java/org/metafacture/metafix/Record.java b/metafix/src/main/java/org/metafacture/metafix/Record.java new file mode 100644 index 000000000..adadd6fc2 --- /dev/null +++ b/metafix/src/main/java/org/metafacture/metafix/Record.java @@ -0,0 +1,238 @@ +/* + * Copyright 2021 hbz NRW + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix; + +import org.metafacture.metafix.FixPath.InsertMode; +import org.metafacture.metafix.Value.TypeMatcher; + +import java.util.Collection; +import java.util.Deque; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.Map; +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import java.util.function.UnaryOperator; + +/** + * Represents a metadata record, i.e., a {@link Value.Hash Hash} of fields + * and values. + */ +public class Record extends Value.Hash { + + private final Map virtualFields = new LinkedHashMap<>(); + + private boolean reject; + + /** + * Creates an empty instance of {@link Record}. + */ + public Record() { + } + + /** + * Returns a shallow clone of this record. + * + * @return a new record pre-populated with all entries from this record + */ + public Record shallowClone() { + final Record clone = new Record(); + + clone.setReject(reject); + forEach(clone::put); + virtualFields.forEach(clone::putVirtualField); + + return clone; + } + + /** + * Flags whether this record should be rejected. + * + * @param reject true if this record should not be emitted, false otherwise + */ + public void setReject(final boolean reject) { + this.reject = reject; + } + + /** + * Checks whether this record should be rejected. + * + * @return true if this record should not be emitted, false otherwise + */ + public boolean getReject() { + return reject; + } + + /** + * Checks whether this record contains the virtual field. + * + * @param field the field name + * @return true if this record contains the virtual field, false otherwise + */ + public boolean containsVirtualField(final String field) { + return virtualFields.containsKey(field); + } + + /** + * Adds a virtual field/value pair to this record, provided it's not + * {@link Value#isNull(Value) null}. Virtual fields can be + * {@link #get(String) accessed} like regular metadata fields, but aren't + * {@link #forEach(BiConsumer) emitted} by default. + * + * @param field the field name + * @param value the metadata value + * + * @see #retainFields(Collection) + */ + public void putVirtualField(final String field, final Value value) { + if (!Value.isNull(value)) { + virtualFields.put(field, value); + } + } + + @Override + public String toString() { + // TODO: Improve string representation? Include reject status, virtual fields, etc.? + return super.toString(); + } + + /** + * Retrieves the field value from this record. Falls back to retrieving the + * virtual field if the field name is not already + * {@link #containsField(String) present}. + * + * @param field the field name + * @return the metadata value + */ + @Override + public Value get(final String field) { + final Value result; + if (containsField(field)) { + result = super.get(field); + } + else { + final FixPath fixPath = new FixPath(field); + if (fixPath.size() > 1) { + result = fixPath.findIn(this); + } + else { + result = virtualFields.get(field); + } + } + return result; + } + + /** + * {@link #put(String, Value) Adds} a field/value pair to this record. Turns + * virtual fields into regular metadata fields if they're not already + * {@link #containsField(String) present}. + * + * @param field the field name + * @param newValue the new metadata value + */ + @Override + public void add(final String field, final Value newValue) { + if (containsField(field)) { + super.add(field, newValue); + } + else { + put(field, newValue); + } + } + + public void addNested(final String field, final Value newValue) { + new FixPath(field).insertInto(this, InsertMode.APPEND, newValue); + } + + /** + * Sets a field/value pair to this record, replacing + * any previous association of the field with a value. + * + * @param field the field name + * @param newValue the new metadata value + */ + public void set(final String field, final Value newValue) { + final FixPath fixPath = new FixPath(field); + fixPath.insertInto(this, InsertMode.REPLACE, newValue); + } + + /** + * Retains only the given field/value pairs in this record. Turns + * virtual fields into regular metadata fields if they're not already + * {@link #containsField(String) present}. + * + * @param fields the field names + */ + @Override + public void retainFields(final Collection fields) { + virtualFields.keySet().retainAll(fields); + + virtualFields.forEach((f, v) -> { + if (!containsField(f)) { + put(f, v); + } + }); + + super.retainFields(fields); + } + + /** + * Transform this record by applying the given operator to all matching values for the given field. + * + * @param field The field + * @param operator The operator + */ + public void transform(final String field, final UnaryOperator operator) { + final FixPath findPath = new FixPath(field); + final Value found = findPath.findIn(this, true); + Value.asList(found, results -> { + final Deque toDelete = new LinkedList<>(); + for (int i = 0; i < results.size(); ++i) { + final Value oldValue = results.get(i); + final FixPath insertPath = findPath.to(oldValue, i); + final String newString = operator.apply(oldValue.asString()); + if (newString == null) { + toDelete.addFirst(insertPath); + } + else { + insertPath.insertInto(this, InsertMode.REPLACE, new Value(newString)); + } + } + toDelete.forEach(path -> path.removeNestedFrom(this)); + }); + } + + /** + * Transform this record by consuming all matching values for the given field with the given consumer. + * + * @param field The field + * @param consumer The consumer + */ + public void transform(final String field, final BiConsumer> consumer) { + final FixPath path = new FixPath(field); + final Value oldValue = path.findIn(this); + + if (oldValue != null) { + final Value newValue = oldValue.extractType(consumer); + + if (newValue != null) { + path.insertInto(this, InsertMode.REPLACE, newValue); + } + } + } + +} diff --git a/metafix/src/main/java/org/metafacture/metafix/RecordTransformer.java b/metafix/src/main/java/org/metafacture/metafix/RecordTransformer.java new file mode 100644 index 000000000..5f118efdb --- /dev/null +++ b/metafix/src/main/java/org/metafacture/metafix/RecordTransformer.java @@ -0,0 +1,400 @@ +/* + * Copyright 2021 Fabian Steeg, hbz + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix; + +import org.metafacture.commons.StringUtil; +import org.metafacture.commons.reflection.ReflectionUtil; +import org.metafacture.framework.MetafactureException; +import org.metafacture.metafix.api.FixContext; +import org.metafacture.metafix.api.FixFunction; +import org.metafacture.metafix.api.FixPredicate; +import org.metafacture.metafix.fix.Do; +import org.metafacture.metafix.fix.ElsIf; +import org.metafacture.metafix.fix.Else; +import org.metafacture.metafix.fix.Expression; +import org.metafacture.metafix.fix.Fix; +import org.metafacture.metafix.fix.If; +import org.metafacture.metafix.fix.MethodCall; +import org.metafacture.metafix.fix.Unless; + +import org.eclipse.emf.ecore.EObject; +import org.eclipse.emf.ecore.resource.Resource; +import org.eclipse.xtext.nodemodel.INode; +import org.eclipse.xtext.nodemodel.util.NodeModelUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +// TODO: Utilize org.metafacture.commons.types.ScopedHashMap for vars instead? + +/** + * Transform a record using a {@link Fix}. + * + * @author Fabian Steeg + * + */ +public class RecordTransformer { // checkstyle-disable-line ClassFanOutComplexity + + private static final Logger LOG = LoggerFactory.getLogger(RecordTransformer.class); + + private final List> consumers = new LinkedList<>(); + private final List> vars = new ArrayList<>(Collections.nCopies(Vars.values().length, null)); + private final Metafix metafix; + private final RecordTransformer parent; + + private String parentExceptionMessage; + private Supplier currentMessageSupplier; + + private enum Vars { + GLOBAL, STATIC, DYNAMIC + } + + /*package-private*/ RecordTransformer(final Metafix metafix, final Fix fix) { + this(metafix, fix.getElements(), null); + setVars(Vars.GLOBAL, metafix.getVars()); + } + + private RecordTransformer(final Metafix metafix, final List expressions, final RecordTransformer parent) { + this.metafix = metafix; + this.parent = parent; + + expressions.forEach(e -> { + final Params params = new Params(e.getParams(), this); + final Options options = new Options(e.getOptions(), this); + + if (e instanceof Do) { + processDo((Do) e, params, options); + } + else if (e instanceof If) { + processIf((If) e, params, options); + } + else if (e instanceof Unless) { + processUnless((Unless) e, params, options); + } + else if (e instanceof MethodCall) { + processFunction((MethodCall) e, params, options); + } + else { + throw new FixProcessException(executionExceptionMessage(e)); + } + }); + } + + private RecordTransformer childTransformer(final List expressions) { + return new RecordTransformer(metafix, expressions, this); + } + + public void transform(final Record record, final Map dynamicVars) { + final Map oldDynamicVars = setVars(Vars.DYNAMIC, dynamicVars); + + try { + transform(record); + } + finally { + setVars(Vars.DYNAMIC, oldDynamicVars); + } + } + + public void transform(final Record record) { + consumers.forEach(consumer -> { + final MetafactureException exception = tryRun(() -> consumer.accept(record)); + + if (exception != null) { + metafix.getStrictness().handle(exception, record); + } + }); + } + + private void processDo(final Do expression, final Params params, final Options options) { + processFix(() -> executionExceptionMessage(expression), () -> { + final FixContext context = getInstance(expression.getName(), FixContext.class, FixBind::valueOf); + final RecordTransformer recordTransformer = childTransformer(expression.getElements()); + + return record -> context.execute(metafix, record, params.resolve(), options.resolve(), recordTransformer); + }); + } + + private void processIf(final If ifExpression, final Params ifParams, final Options ifOptions) { + final List elseIfExpressions = ifExpression.getElseIf(); + final Else elseExpression = ifExpression.getElse(); + + final List> elseIfMessageSuppliers = mapList(elseIfExpressions, e -> () -> executionExceptionMessage(e, e.eResource())); + final Supplier elseMessageSupplier = () -> executionExceptionMessage(elseExpression, elseExpression.eResource()); + + processFix(() -> executionExceptionMessage(ifExpression, ifExpression.eResource()), () -> { + final FixPredicate ifPredicate = getInstance(ifExpression.getName(), FixPredicate.class, FixConditional::valueOf); + final RecordTransformer ifTransformer = childTransformer(ifExpression.getElements()); + + final List elseIfPredicates = mapList(elseIfExpressions, e -> getInstance(e.getName(), FixPredicate.class, FixConditional::valueOf)); + final List elseIfParamsList = mapList(elseIfExpressions, e -> new Params(e.getParams(), this)); + final List elseIfOptionsList = mapList(elseIfExpressions, e -> new Options(e.getOptions(), this)); + final List elseIfTransformers = mapList(elseIfExpressions, e -> childTransformer(e.getElements())); + + final RecordTransformer elseTransformer = elseExpression != null ? childTransformer(elseExpression.getElements()) : null; + + return record -> { + if (ifPredicate.test(metafix, record, ifParams.resolve(), ifOptions.resolve())) { + ifTransformer.transform(record); + } + else { + for (int i = 0; i < elseIfExpressions.size(); ++i) { + currentMessageSupplier = elseIfMessageSuppliers.get(i); + + final ElsIf elseIfExpression = elseIfExpressions.get(i); + + final FixPredicate elseIfPredicate = elseIfPredicates.get(i); + final Params elseIfParams = elseIfParamsList.get(i); + final Options elseIfOptions = elseIfOptionsList.get(i); + final RecordTransformer elseIfTransformer = elseIfTransformers.get(i); + + if (elseIfPredicate.test(metafix, record, elseIfParams.resolve(), elseIfOptions.resolve())) { + elseIfTransformer.transform(record); + return; + } + } + + if (elseExpression != null) { + currentMessageSupplier = elseMessageSupplier; + elseTransformer.transform(record); + } + } + }; + }); + } + + private void processUnless(final Unless expression, final Params params, final Options options) { + processFix(() -> executionExceptionMessage(expression, expression.eResource()), () -> { + final FixPredicate predicate = getInstance(expression.getName(), FixPredicate.class, FixConditional::valueOf); + final RecordTransformer recordTransformer = childTransformer(expression.getElements()); + + return record -> { + if (!predicate.test(metafix, record, params.resolve(), options.resolve())) { + recordTransformer.transform(record); + } + }; + }); + } + + private void processFunction(final MethodCall expression, final Params params, final Options options) { + processFix(() -> executionExceptionMessage(expression), () -> { + final FixFunction function = getInstance(expression.getName(), FixFunction.class, FixMethod::valueOf); + return record -> function.apply(metafix, record, params.resolve(), options.resolve()); + }); + } + + private T getInstance(final String name, final Class baseType, final Function enumFunction) { + return name.contains(".") ? ReflectionUtil.loadClass(name, baseType).newInstance() : enumFunction.apply(name); + } + + private List mapList(final List list, final Function function) { + return list.stream().map(function).collect(Collectors.toList()); + } + + private void processFix(final Supplier messageSupplier, final Supplier> consumerSupplier) { + currentMessageSupplier = messageSupplier; + + final MetafactureException exception = tryRun(() -> { + final Consumer consumer = consumerSupplier.get(); + + consumers.add(record -> { + currentMessageSupplier = messageSupplier; + consumer.accept(record); + }); + }); + + if (exception != null) { + throw exception; + } + } + + private MetafactureException tryRun(final Runnable runnable) { // checkstyle-disable-line ReturnCount + try { + runnable.run(); + } + catch (final FixProcessException e) { + throw e; // TODO: Add nesting information? + } + catch (final FixExecutionException e) { + return e; // TODO: Add nesting information? + } + catch (final IllegalStateException | NumberFormatException e) { + return new FixExecutionException(getCurrentExceptionMessage(), e); + } + catch (final RuntimeException e) { // checkstyle-disable-line IllegalCatch + final MetafactureException exception = new FixProcessException(getCurrentExceptionMessage(), e); + + if (metafix.getStrictnessHandlesProcessExceptions()) { + return exception; + } + else { + throw exception; + } + } + + return null; + } + + private String getCurrentExceptionMessage() { + final StringBuilder sb = new StringBuilder(); + + if (parentExceptionMessage != null) { + sb.append(parentExceptionMessage); + sb.append(" -> "); + } + + sb.append(currentMessageSupplier.get()); + + return sb.toString(); + } + + /*package-private*/ void setParentExceptionMessageFrom(final RecordTransformer parentTransformer) { + parentExceptionMessage = parentTransformer != null && parentTransformer.currentMessageSupplier != null ? + parentTransformer.currentMessageSupplier.get() : null; + } + + private String executionExceptionMessage(final Expression expression) { + return executionExceptionMessage(expression, expression.eResource()); + } + + private String executionExceptionMessage(final EObject object, final Resource resource) { + final INode node = NodeModelUtils.getNode(object); + + return String.format("Error while executing Fix expression (at %s, line %d): %s", + resource.getURI(), node.getStartLine(), NodeModelUtils.getTokenText(node)); + } + + /*package-private*/ void setVars(final Map staticVars) { + setVars(Vars.STATIC, staticVars); + } + + private Map setVars(final Vars type, final Map newVars) { + return vars.set(type.ordinal(), newVars); + } + + private Map getVars() { + final Map mergedVars = parent != null ? parent.getVars() : new HashMap<>(); + vars.stream().filter(v -> v != null).forEach(mergedVars::putAll); + return mergedVars; + } + + private abstract static class AbstractResolvable { + + protected boolean isResolvable(final String value) { + return value != null && value.contains(Metafix.VAR_START); + } + + protected String resolveVars(final String value, final Map vars) { + return value == null ? null : StringUtil.format(value, Metafix.VAR_START, Metafix.VAR_END, false, vars); + } + + protected abstract T resolve(); + + } + + private static class Params extends AbstractResolvable> { + + private final List list; + private final RecordTransformer recordTransformer; + private final boolean resolve; + + private Params(final List list, final RecordTransformer recordTransformer) { + this.list = list; + this.recordTransformer = recordTransformer; + + resolve = list.stream().anyMatch(this::isResolvable); + } + + @Override + protected List resolve() { + if (resolve) { + final List resolvedList = new ArrayList<>(list.size()); + final Map vars = recordTransformer.getVars(); + + for (final String entry : list) { + resolvedList.add(resolveVars(entry, vars)); + } + + return resolvedList; + } + else { + return list; + } + } + + } + + private static class Options extends AbstractResolvable> { + + private final Map map = new LinkedHashMap<>(); + private final RecordTransformer recordTransformer; + private final boolean resolve; + + private Options(final org.metafacture.metafix.fix.Options options, final RecordTransformer recordTransformer) { + this.recordTransformer = recordTransformer; + + boolean resolveTemp = false; + + if (options != null) { + final List keys = options.getKeys(); + final List values = options.getValues(); + + for (int i = 0; i < keys.size(); ++i) { + final String key = keys.get(i); + final String value = values.get(i); + + map.put(key, value); + + if (!resolveTemp && (isResolvable(key) || isResolvable(value))) { + resolveTemp = true; + } + } + } + + resolve = resolveTemp; + } + + @Override + protected Map resolve() { + if (resolve) { + final Map resolvedMap = new LinkedHashMap<>(map.size()); + final Map vars = recordTransformer.getVars(); + + for (final Map.Entry entry : map.entrySet()) { + resolvedMap.put(resolveVars(entry.getKey(), vars), resolveVars(entry.getValue(), vars)); + } + + return resolvedMap; + } + else { + return map; + } + } + + } + +} diff --git a/metafix/src/main/java/org/metafacture/metafix/Value.java b/metafix/src/main/java/org/metafacture/metafix/Value.java new file mode 100644 index 000000000..25d544966 --- /dev/null +++ b/metafix/src/main/java/org/metafacture/metafix/Value.java @@ -0,0 +1,923 @@ +/* + * Copyright 2021 hbz NRW + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix; + +import org.metafacture.commons.tries.SimpleRegexTrie; +import org.metafacture.commons.tries.WildcardTrie; + +import com.fasterxml.jackson.core.JsonGenerator; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.ConcurrentModificationException; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +/** + * Represents a record value, i.e., either an {@link Array}, a {@link Hash}, + * or a {@link String}. + */ +public class Value implements JsonValue { // checkstyle-disable-line ClassDataAbstractionCoupling + + private static final String FIELD_PATH_SEPARATOR = "\\."; + + private final Array array; + private final Hash hash; + private final String string; + + private final Type type; + + private String path; + + private Value(final Type type, final Array array, final Hash hash, final String string) { + final boolean hasValue = array != null || hash != null || string != null; + + if (type == null) { + if (hasValue) { + throw new IllegalArgumentException("Value without type"); + } + } + else { + if (!hasValue) { + throw new IllegalArgumentException("Type without value"); + } + } + + this.type = type; + this.array = array; + this.hash = hash; + this.string = string; + } + + public Value(final Array array) { + this(array != null ? Type.Array : null, array, null, null); + } + + public Value(final List array) { + this(array != null ? new Array() : null); + + if (array != null) { + array.forEach(this.array::add); + } + } + + public Value(final Hash hash) { + this(hash != null ? Type.Hash : null, null, hash, null); + } + + public Value(final Map hash) { + this(hash != null ? new Hash() : null); + + if (hash != null) { + hash.forEach(this.hash::put); + } + } + + public Value(final String string) { + this(string != null ? Type.String : null, null, null, string); + } + + public Value(final int integer) { + this(String.valueOf(integer)); + } + + public static Value newArray() { + return newArray(null); + } + + public static Value newArray(final Consumer consumer) { + final Array array = new Array(); + + if (consumer != null) { + consumer.accept(array); + } + + return new Value(array); + } + + public static Value newHash() { + return newHash(null); + } + + public static Value newHash(final Consumer consumer) { + final Hash hash = new Hash(); + + if (consumer != null) { + consumer.accept(hash); + } + + return new Value(hash); + } + + public boolean isArray() { + return isType(Type.Array); + } + + public boolean isHash() { + return isType(Type.Hash); + } + + public boolean isString() { + return isType(Type.String); + } + + private boolean isType(final Type targetType) { + return type == targetType; + } + + public boolean isNull() { + return isType(null); + } + + public static boolean isNull(final Value value) { + return value == null || value.isNull(); + } + + /*package-private*/ static boolean isNumber(final String s) { + return s.matches("\\d+"); + } + + public Array asArray() { + return extractType((m, c) -> m.ifArray(c).orElseThrow()); + } + + public Hash asHash() { + return extractType((m, c) -> m.ifHash(c).orElseThrow()); + } + + public String asString() { + return extractType((m, c) -> m.ifString(c).orElseThrow()); + } + + public static Value asList(final Value value, final Consumer consumer) { + return isNull(value) ? null : value.asList(consumer); + } + + public Value asList(final Consumer consumer) { + if (isArray()) { + if (consumer != null) { + consumer.accept(asArray()); + } + + return this; + } + else { + return newArray(a -> { + a.add(this); + + if (consumer != null) { + consumer.accept(a); + } + }); + } + } + + public TypeMatcher matchType() { + return new TypeMatcher(this); + } + + public T extractType(final BiConsumer> consumer) { + final AtomicReference result = new AtomicReference<>(); + consumer.accept(matchType(), result::set); + return result.get(); + } + + @Override + public final boolean equals(final Object object) { + if (object == this) { + return true; + } + + if (!(object instanceof Value)) { + return false; + } + + final Value other = (Value) object; + return Objects.equals(type, other.type) && + Objects.equals(array, other.array) && + Objects.equals(hash, other.hash) && + Objects.equals(string, other.string); + } + + @Override + public final int hashCode() { + return Objects.hashCode(type) + + Objects.hashCode(array) + + Objects.hashCode(hash) + + Objects.hashCode(string); + } + + @Override + public String toString() { + return isNull() ? null : extractType((m, c) -> m + .ifArray(a -> c.accept(a.toString())) + .ifHash(h -> c.accept(h.toString())) + .ifString(c) + .orElseThrow() + ); + } + + @Override + public void toJson(final JsonGenerator jsonGenerator) { + if (isNull()) { + try { + jsonGenerator.writeNull(); + } + catch (final IOException e) { + throw new UncheckedIOException(e); + } + } + else { + matchType() + .ifArray(a -> a.toJson(jsonGenerator)) + .ifHash(h -> h.toJson(jsonGenerator)) + .ifString(s -> { + try { + jsonGenerator.writeString(s); + } + catch (final IOException e) { + throw new UncheckedIOException(e); + } + }) + .orElseThrow(); + } + } + + /*package-private*/ static String[] split(final String fieldPath) { + return fieldPath.split(FIELD_PATH_SEPARATOR); + } + + public String getPath() { + return path; + } + + /*package-private*/ Value withPathSet(final String p) { + this.path = p; + return this; + } + + private Value withPathAppend(final int i) { + return withPathAppend(String.valueOf(i)); + } + + private Value withPathAppend(final String field) { + return withPathSet(path == null || path.isEmpty() ? field : path + "." + field); + } + + /*package-private*/ Value copy() { + return extractType((m, c) -> m + .ifArray(oldArray -> c.accept(Value.newArray(newArray -> oldArray.forEach(v -> newArray.add(v))))) + .ifHash(oldHash -> c.accept(Value.newHash(newHash -> oldHash.forEach((k, v) -> newHash.put(k, v))))) + .ifString(s -> c.accept(new Value(s))) + .orElseThrow()); + } + + private void retainFields(final Collection fields) { + matchType() + .ifArray(a -> a.retainFields(fields)) + .ifHash(h -> h.retainFields(fields)) + .orElseThrow(); + } + + enum Type { + Array, + Hash, + String + } + + public static class TypeMatcher { + + private final Set expected = EnumSet.noneOf(Type.class); + private final Value value; + + private TypeMatcher(final Value value) { + this.value = value; + } + + public TypeMatcher ifArray(final Consumer consumer) { + return match(Type.Array, consumer, value.array); + } + + public TypeMatcher ifHash(final Consumer consumer) { + return match(Type.Hash, consumer, value.hash); + } + + public TypeMatcher ifString(final Consumer consumer) { + return match(Type.String, consumer, value.string); + } + + public void orElse(final Consumer consumer) { + if (!expected.contains(value.type)) { + consumer.accept(value); + } + } + + public void orElseThrow() { + orElse(v -> { + final String types = expected.stream().map(Type::name).collect(Collectors.joining(" or ")); + throw new IllegalStateException("Expected " + types + ", got " + value.type); + }); + } + + private TypeMatcher match(final Type type, final Consumer consumer, final T rawValue) { + if (expected.add(type)) { + if (value.isType(type)) { + consumer.accept(rawValue); + } + + return this; + } + else { + throw new IllegalStateException("Already expecting " + type); + } + } + + } + + private abstract static class AbstractValueType implements JsonValue { + + protected static final Predicate REMOVE_EMPTY_VALUES = v -> + v.extractType((m, c) -> m + .ifArray(a -> { + a.removeEmptyValues(); + c.accept(a.isEmpty()); + }) + .ifHash(h -> { + h.removeEmptyValues(); + c.accept(h.isEmpty()); + }) + // TODO: Catmandu considers whitespace-only strings empty (`$v !~ /\S/`) + .ifString(s -> c.accept(s.isEmpty())) + .orElseThrow() + ); + + @Override + public abstract boolean equals(Object object); + + @Override + public abstract int hashCode(); + + @Override + public abstract String toString(); + + @Override + public abstract void toJson(JsonGenerator jsonGenerator); + + protected Map> retainFields(final Collection fields, final Function> function) { + final Map> retainFields = new HashMap<>(); + + fields.forEach(p -> { + final String[] parts = p.split(FIELD_PATH_SEPARATOR, 2); + + function.apply(parts[0]).forEach(f -> { + final Collection retainNested = retainFields.computeIfAbsent(f, k -> new HashSet<>()); + if (parts.length > 1) { + retainNested.add(parts[1]); + } + }); + }); + + return retainFields; + } + + } + + /** + * Represents an array of metadata values. + */ + public static class Array extends AbstractValueType { + + private final List list = new ArrayList<>(); + + /** + * Creates an empty instance of {@link Array}. + */ + private Array() { + } + + public void add(final Value value) { + add(value, true); + } + + /* package-private */ void add(final Value value, final boolean appendToPath) { + add(list.size(), value, appendToPath); + } + + /* package-private */ void add(final int index, final Value value) { + add(index, value, true); + } + + /* package-private */ void add(final int index, final Value value, final boolean appendToPath) { + if (!isNull(value)) { + list.add(index, appendToPath ? value.withPathAppend(index + 1) : value); + updateIndexesInPathsAfter(index); + } + } + + private void updateIndexesInPathsAfter(final int start) { + for (int index = start + 1; index < list.size(); index = index + 1) { + final Value value = list.get(index); + value.withPathSet(value.getPath().replaceAll("\\d+$", String.valueOf(index + 1))); + } + } + + public boolean isEmpty() { + return list.isEmpty(); + } + + public int size() { + return list.size(); + } + + public Value get(final int index) { + return list.get(index); + } + + public Stream stream() { + return list.stream(); + } + + private IntStream indexes() { + return IntStream.range(0, size()); + } + + private void removeEmptyValues() { + list.removeIf(REMOVE_EMPTY_VALUES); + } + + private void retainFields(final Collection fields) { + final Map> retainFields = retainFields(fields, this::findFields); + + final int max = size() - 1; + indexes().map(i -> max - i).forEach(i -> { + final Collection retainNested = retainFields.get(i); + if (retainNested == null) { + remove(i); + } + else if (!retainNested.isEmpty()) { + get(i).retainFields(retainNested); + } + }); + } + + private Set findFields(final String pattern) { + final Set fieldSet = new LinkedHashSet<>(); + + if ("*".equals(pattern)) { + indexes().forEach(fieldSet::add); + } + else { + final int index; + + switch (pattern) { + case "$first": + index = 0; + break; + case "$last": + index = size() - 1; + break; + default: + index = Integer.parseInt(pattern) - 1; // TODO: 0-based Catmandu vs. 1-based Metafacture + } + + if (index >= 0 && index < size()) { + fieldSet.add(index); + } + } + + return fieldSet; + } + + public void forEach(final Consumer consumer) { + list.forEach(consumer); + } + + @Override + public final boolean equals(final Object object) { + if (object == this) { + return true; + } + + if (!(object instanceof Array)) { + return false; + } + + final Array other = (Array) object; + return Objects.equals(list, other.list); + } + + @Override + public final int hashCode() { + return Objects.hashCode(list); + } + + @Override + public String toString() { + return list.toString(); + } + + @Override + public void toJson(final JsonGenerator jsonGenerator) { + try { + jsonGenerator.writeStartArray(); + forEach(v -> v.toJson(jsonGenerator)); + jsonGenerator.writeEndArray(); + } + catch (final IOException e) { + throw new UncheckedIOException(e); + } + } + + public void remove(final int index) { + list.remove(index); + } + + /*package-private*/ void set(final int index, final Value value) { + list.set(index, value.withPathAppend(index + 1)); + } + + /*package-private*/ void removeIf(final Predicate predicate) { + list.removeIf(predicate); + } + + /*package-private*/ void removeAll() { + list.clear(); + } + + } + + /** + * Represents a hash of metadata fields and values. + */ + public static class Hash extends AbstractValueType { + + // NOTE: Keep in sync with `WildcardTrie`/`SimpleRegexTrie` implementation in metafacture-core. + private static final Pattern ALTERNATION_PATTERN = Pattern.compile(WildcardTrie.OR_STRING, Pattern.LITERAL); + private static final ThreadLocal PATTERN_MATCHER = ThreadLocal.withInitial(() -> Pattern.compile("[*?]|\\[[^\\]]").matcher("")); + + private static final ThreadLocal> PREFIX_CACHE = ThreadLocal.withInitial(HashMap::new); + + private static final ThreadLocal>> TRIE_CACHE = ThreadLocal.withInitial(HashMap::new); + private static final ThreadLocal> TRIE = ThreadLocal.withInitial(SimpleRegexTrie::new); + + private final Map> trieCache = TRIE_CACHE.get(); + private final Map prefixCache = PREFIX_CACHE.get(); + private final Map map = new LinkedHashMap<>(); + private final Matcher patternMatcher = PATTERN_MATCHER.get(); + private final SimpleRegexTrie trie = TRIE.get(); + + /** + * Creates an empty instance of {@link Hash}. + */ + protected Hash() { + } + + /** + * Checks whether this hash contains the metadata field. + * + * @param field the field name + * @return true if this hash contains the metadata field, false otherwise + */ + public boolean containsField(final String field) { + return !findFields(field).isEmpty(); + } + + public boolean containsPath(final String fieldPath) { + final String[] path = split(fieldPath); + final String field = path[0]; + + final boolean containsField = containsField(field); + final boolean containsPath; + + if (containsField && path.length > 1) { + final Value value; + + try { + value = new FixPath(fieldPath).findIn(this); + } + catch (final IllegalStateException e) { + return false; + } + + containsPath = !isNull(value); + } + else { + containsPath = containsField; + } + + return containsPath; + } + + /** + * Checks whether this hash is empty. + * + * @return true if this hash is empty, false otherwise + */ + public boolean isEmpty() { + return map.isEmpty(); + } + + /** + * Gets the number of field/value pairs in this hash. + * + * @return the number of field/value pairs in this hash + */ + public int size() { + return map.size(); + } + + /** + * Adds a field/value pair to this hash, provided it's not {@link #isNull(Value) null}. + * + * @param field the field name + * @param value the metadata value + */ + public void put(final String field, final Value value) { + put(field, value, true); + } + + /*package-private*/ void put(final String field, final Value value, final boolean appendToPath) { + if (!isNull(value)) { + map.put(field, appendToPath ? value.withPathAppend(field) : value); + } + } + + /** + * {@link #put(String, Value) Replaces} a field/value pair in this hash, + * provided the field name is already {@link #containsField(String) present}. + * + * @param field the field name + * @param value the metadata value + */ + public void replace(final String field, final Value value) { + if (containsField(field)) { + put(field, value); + } + } + + /** + * Retrieves the field value from this hash. + * + * @param field the field name + * @return the metadata value + */ + public Value get(final String field) { + return get(field, false); + } + + /*package-private*/ Value get(final String field, final boolean enforceStringValue) { // TODO use Type.String etc.? + // TODO: special treatment (only) for exact matches? + final Set set = findFields(field); + + return set.isEmpty() ? null : set.size() == 1 ? getField(set.iterator().next(), enforceStringValue) : + newArray(a -> set.forEach(f -> getField(f, enforceStringValue).matchType() + .ifArray(b -> b.forEach(t -> a.add(t, false))) + .orElse(t -> a.add(t, false)) + )); + } + + public Value getField(final String field) { + return map.get(field); + } + + private Value getField(final String field, final boolean enforceStringValue) { + final Value value = getField(field); + + if (enforceStringValue) { + value.asString(); + } + + return value; + } + + public Value getList(final String field, final Consumer consumer) { + return asList(get(field), consumer); + } + + public void addAll(final String field, final List values) { + values.forEach(value -> add(field, new Value(value))); + } + + public void addAll(final Hash hash) { + hash.forEach(this::add); + } + + /** + * {@link #put(String, Value) Adds} a field/value pair to this hash, + * potentially merging with an existing value. + * + * @param field the field name + * @param newValue the new metadata value + */ + public void add(final String field, final Value newValue) { + final Value oldValue = new FixPath(field).findIn(this); + + if (oldValue == null) { + put(field, newValue); + } + else { + final String basePath = oldValue.getPath(); + if (!oldValue.isArray()) { // repeated field: convert single val to first in array + oldValue.withPathAppend(1); + } + + put(field, oldValue.asList(oldVals -> newValue.asList(newVals -> + newVals.forEach(newVal -> oldVals.add(newVal.withPathSet(basePath)))))); + } + } + + /** + * Removes the given field/value pair from this hash. + * + * @param field the field name + */ + public void remove(final String field) { + final FixPath fixPath = new FixPath(field); + + if (fixPath.size() > 1) { + fixPath.removeNestedFrom(this); + } + else { + modifyFields(field, this::removeField); + } + } + + public void removeField(final String field) { + map.remove(field); + } + + /** + * Retains only the given field/value pairs in this hash. + * + * @param fields the field names + */ + public void retainFields(final Collection fields) { + final Map> retainFields = retainFields(fields, this::findFields); + + map.keySet().retainAll(retainFields.keySet()); + + retainFields.forEach((k, v) -> { + if (!v.isEmpty()) { + getField(k).retainFields(v); + } + }); + } + + /** + * Recursively removes all field/value pairs from this hash whose value is empty. + */ + public void removeEmptyValues() { + map.values().removeIf(REMOVE_EMPTY_VALUES); + } + + /** + * Iterates over all field/value pairs in this hash. + * + * @param consumer the action to be performed for each field/value pair + */ + public void forEach(final BiConsumer consumer) { + map.forEach(consumer); + } + + @Override + public final boolean equals(final Object object) { + if (object == this) { + return true; + } + + if (!(object instanceof Hash)) { + return false; + } + + final Hash other = (Hash) object; + return Objects.equals(map, other.map); + } + + @Override + public final int hashCode() { + return Objects.hashCode(map); + } + + @Override + public String toString() { + return map.toString(); + } + + @Override + public void toJson(final JsonGenerator jsonGenerator) { + try { + jsonGenerator.writeStartObject(); + + forEach((f, v) -> { + try { + jsonGenerator.writeFieldName(f); + } + catch (final IOException e) { + throw new UncheckedIOException(e); + } + + v.toJson(jsonGenerator); + }); + + jsonGenerator.writeEndObject(); + } + catch (final IOException e) { + throw new UncheckedIOException(e); + } + } + + /** + * Avoids {@link ConcurrentModificationException} when modifying the hash based on matched fields. + * + * @param pattern the field name pattern + * @param consumer the action to be performed for each value + */ + /*package-private*/ void modifyFields(final String pattern, final Consumer consumer) { + findFields(pattern).forEach(consumer); + } + + private Set findFields(final String pattern) { + final Set fieldSet = new LinkedHashSet<>(); + + for (final String term : ALTERNATION_PATTERN.split(pattern)) { + findFields(term, fieldSet); + } + + return fieldSet; + } + + private void findFields(final String pattern, final Set fieldSet) { + if (!prefixCache.containsKey(pattern)) { + final Matcher matcher = patternMatcher.reset(pattern); + + if (matcher.find()) { + trie.put(pattern, pattern); + trieCache.put(pattern, new HashMap<>()); + + prefixCache.put(pattern, pattern.substring(0, matcher.start())); + } + else { + prefixCache.put(pattern, null); + } + } + + final String prefix = prefixCache.get(pattern); + + if (prefix != null) { + final Map fieldCache = trieCache.get(pattern); + + for (final String field : map.keySet()) { + if (!fieldCache.containsKey(field)) { + final boolean matches = field.startsWith(prefix) && trie.get(field).contains(pattern); + fieldCache.put(field, matches); + + if (matches) { + fieldSet.add(field); + } + } + else if (fieldCache.get(field)) { + fieldSet.add(field); + } + } + } + else if (map.containsKey(pattern)) { + fieldSet.add(pattern); + } + } + + } + +} diff --git a/metafix/src/main/java/org/metafacture/metafix/api/FixContext.java b/metafix/src/main/java/org/metafacture/metafix/api/FixContext.java new file mode 100644 index 000000000..afebc021c --- /dev/null +++ b/metafix/src/main/java/org/metafacture/metafix/api/FixContext.java @@ -0,0 +1,31 @@ +/* + * Copyright 2022 hbz NRW + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix.api; + +import org.metafacture.metafix.Metafix; +import org.metafacture.metafix.Record; +import org.metafacture.metafix.RecordTransformer; + +import java.util.List; +import java.util.Map; + +@FunctionalInterface +public interface FixContext { + + void execute(Metafix metafix, Record record, List params, Map options, RecordTransformer recordTransformer); + +} diff --git a/metafix/src/main/java/org/metafacture/metafix/api/FixFunction.java b/metafix/src/main/java/org/metafacture/metafix/api/FixFunction.java new file mode 100644 index 000000000..458a02335 --- /dev/null +++ b/metafix/src/main/java/org/metafacture/metafix/api/FixFunction.java @@ -0,0 +1,110 @@ +/* + * Copyright 2022 hbz NRW + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix.api; + +import org.metafacture.framework.StandardEventNames; +import org.metafacture.io.ObjectWriter; +import org.metafacture.metafix.Metafix; +import org.metafacture.metafix.Record; +import org.metafacture.metafix.Value; + +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.LongAdder; +import java.util.function.BiFunction; +import java.util.function.Consumer; +import java.util.function.UnaryOperator; +import java.util.stream.Stream; + +@FunctionalInterface +public interface FixFunction { + + void apply(Metafix metafix, Record record, List params, Map options); + + default void withOption(final Map options, final String key, final Consumer consumer) { + withOption(options, key, consumer, Map::get); + } + + default void withOption(final Map options, final String key, final Consumer consumer, final BiFunction, String, T> function) { + if (options.containsKey(key)) { + consumer.accept(function.apply(options, key)); + } + } + + default void withWriter(final Map options, final UnaryOperator operator, final Consumer> consumer) { + final String destination = options.getOrDefault("destination", ObjectWriter.STDOUT); + final ObjectWriter writer = new ObjectWriter<>(operator != null ? operator.apply(destination) : destination); + + withOption(options, "append", writer::setAppendIfFileExists, this::getBoolean); + withOption(options, "compression", writer::setCompression); + withOption(options, "encoding", writer::setEncoding); + withOption(options, "footer", writer::setFooter); + withOption(options, "header", writer::setHeader); + withOption(options, "separator", writer::setSeparator); + + try { + consumer.accept(writer); + } + finally { + writer.closeStream(); + } + } + + default void withWriter(final Metafix metafix, final Record record, final Map options, final Map scopedCounter, final Consumer> consumer) { + final Value idValue = record.get(options.getOrDefault("id", StandardEventNames.ID)); + + final LongAdder counter = scopedCounter.computeIfAbsent(metafix, k -> new LongAdder()); + counter.increment(); + + final UnaryOperator formatter = s -> String.format(s, + counter.sum(), Value.isNull(idValue) ? "" : idValue.toString()); + + final String prefix = formatter.apply(options.getOrDefault("prefix", "")); + withWriter(options, formatter, w -> consumer.accept(s -> w.process(prefix + s))); + } + + default boolean getBoolean(final Map options, final String key) { + return Boolean.parseBoolean(options.get(key)); + } + + default int getInteger(final Map options, final String key) { + return Integer.parseInt(options.get(key)); + } + + default int getInteger(final List params, final int index) { + return Integer.parseInt(params.get(index)); + } + + default Value newArray(final Stream stream) { + return Value.newArray(a -> stream.forEach(a::add)); + } + + default Stream unique(final Stream stream) { + final Set set = new HashSet<>(); + return stream.filter(set::add); + } + + default Stream flatten(final Stream stream) { + return stream.flatMap(v -> v.extractType((m, c) -> m + .ifArray(a -> c.accept(flatten(a.stream()))) + .orElse(w -> c.accept(Stream.of(w))) + )); + } + +} diff --git a/metafix/src/main/java/org/metafacture/metafix/api/FixPredicate.java b/metafix/src/main/java/org/metafacture/metafix/api/FixPredicate.java new file mode 100644 index 000000000..201ee920f --- /dev/null +++ b/metafix/src/main/java/org/metafacture/metafix/api/FixPredicate.java @@ -0,0 +1,91 @@ +/* + * Copyright 2022 hbz NRW + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix.api; + +import org.metafacture.metafix.Metafix; +import org.metafacture.metafix.Record; +import org.metafacture.metafix.Value; + +import java.math.BigDecimal; +import java.util.List; +import java.util.Map; +import java.util.function.BiPredicate; +import java.util.function.Predicate; +import java.util.stream.Stream; + +@FunctionalInterface +public interface FixPredicate { + + BiPredicate, Predicate> ALL = Stream::allMatch; + BiPredicate, Predicate> ANY = Stream::anyMatch; + + BiPredicate CONTAINS = String::contains; + BiPredicate EQUALS = String::equals; + BiPredicate MATCHES = String::matches; + + Predicate IS_TRUE = s -> "true".equals(s) || "1".equals(s); + Predicate IS_FALSE = s -> "false".equals(s) || "0".equals(s); + + Predicate IS_NUMBER = s -> { + try { + new BigDecimal(s); + return true; + } + catch (final NumberFormatException e) { + return false; + } + }; + + Predicate IS_EMPTY = v -> v.extractType((m, c) -> m + .ifArray(a -> c.accept(a.isEmpty())) + .ifHash(h -> c.accept(h.isEmpty())) + // TODO: Catmandu considers whitespace-only strings empty (`$v !~ /\S/`) + .ifString(s -> c.accept(s.isEmpty())) + ); + + boolean test(Metafix metafix, Record record, List params, Map options); + + default boolean testConditional(final Record record, final List params, final BiPredicate, Predicate> qualifier, final BiPredicate conditional) { + final String field = params.get(0); + final String string = params.get(1); + + final Value value = record.get(field); + return value != null && qualifier.test(value.asList(null).asArray().stream(), v -> v.extractType((m, c) -> m + .ifString(s -> c.accept(conditional.test(s, string))) + .orElse(w -> c.accept(false)) + )); + } + + default boolean testConditional(final Record record, final List params, final Predicate conditional) { + final String field = params.get(0); + + final Value value = record.get(field); + return value != null && conditional.test(value); + } + + default boolean testConditional(final List params, final BiPredicate conditional) { + return conditional.test(params.get(0), params.get(1)); + } + + default boolean testStringConditional(final Record record, final List params, final Predicate conditional) { + return testConditional(record, params, v -> v.extractType((m, c) -> m + .ifString(s -> c.accept(conditional.test(s))) + .orElse(w -> c.accept(false)) + )); + } + +} diff --git a/metafix/src/main/java/org/metafacture/metafix/interpreter/FixInterpreter.java b/metafix/src/main/java/org/metafacture/metafix/interpreter/FixInterpreter.java new file mode 100644 index 000000000..fd6717906 --- /dev/null +++ b/metafix/src/main/java/org/metafacture/metafix/interpreter/FixInterpreter.java @@ -0,0 +1,96 @@ +package org.metafacture.metafix.interpreter; + +import org.metafacture.metafix.Metafix; +import org.metafacture.metafix.fix.Do; +import org.metafacture.metafix.fix.ElsIf; +import org.metafacture.metafix.fix.Else; +import org.metafacture.metafix.fix.Expression; +import org.metafacture.metafix.fix.Fix; +import org.metafacture.metafix.fix.If; +import org.metafacture.metafix.fix.MethodCall; + +import org.eclipse.emf.ecore.EObject; +import org.eclipse.xtext.xbase.interpreter.impl.XbaseInterpreter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; + +public class FixInterpreter extends XbaseInterpreter { + + private static final Logger LOG = LoggerFactory.getLogger(FixInterpreter.class); + + private Metafix metafix; + + public FixInterpreter() { + } + + public void run(final Metafix metafixParam, final EObject program) { + if (metafixParam != null && program != null) { + metafix = metafixParam; + + if (program instanceof Fix) { + for (final Expression expression : ((Fix) program).getElements()) { + process(expression); + } + } + } + } + + private void process(final Expression expression) { // checkstyle-disable-line CyclomaticComplexity|NPathComplexity + metafix.getExpressions().add(expression); + + boolean matched = false; + + if (expression instanceof If) { + matched = true; + + final If ifExpression = (If) expression; + LOG.debug("if: " + ifExpression); + + for (final Expression element : ifExpression.getElements()) { + process(element); + } + + final List elseIfExpressions = ifExpression.getElseIf(); + for (final ElsIf elseIfExpression : elseIfExpressions) { + LOG.debug("else if: " + elseIfExpression); + + for (final Expression element : elseIfExpression.getElements()) { + process(element); + } + } + + final Else elseExpression = ifExpression.getElse(); + if (elseExpression != null) { + LOG.debug("else: " + elseExpression); + + for (final Expression element : elseExpression.getElements()) { + process(element); + } + } + } + + if (!matched) { + if (expression instanceof Do) { + matched = true; + + final Do doExpression = (Do) expression; + LOG.debug("do: " + doExpression); + + for (final Expression element : doExpression.getElements()) { + process(element); + } + } + } + + if (!matched) { + if (expression instanceof MethodCall) { + matched = true; + LOG.debug("method call: " + expression); + // TODO + } + } + } + +} diff --git a/metafix/src/main/java/org/metafacture/metafix/jvmmodel/FixJvmModelInferrer.java b/metafix/src/main/java/org/metafacture/metafix/jvmmodel/FixJvmModelInferrer.java new file mode 100644 index 000000000..ae08a5cf8 --- /dev/null +++ b/metafix/src/main/java/org/metafacture/metafix/jvmmodel/FixJvmModelInferrer.java @@ -0,0 +1,46 @@ +package org.metafacture.metafix.jvmmodel; + +import org.metafacture.metafix.Metafix; +import org.metafacture.metafix.fix.Fix; + +import org.eclipse.emf.ecore.EObject; +import org.eclipse.xtext.common.types.JvmGenericType; +import org.eclipse.xtext.naming.IQualifiedNameProvider; +import org.eclipse.xtext.xbase.jvmmodel.AbstractModelInferrer; +import org.eclipse.xtext.xbase.jvmmodel.IJvmDeclaredTypeAcceptor; +import org.eclipse.xtext.xbase.jvmmodel.JvmTypesBuilder; +import org.eclipse.xtext.xbase.lib.Extension; + +import java.util.Arrays; +import javax.inject.Inject; + +public class FixJvmModelInferrer extends AbstractModelInferrer { + + @Inject + @Extension + private JvmTypesBuilder jvmTypesBuilder; + + @Inject + @Extension + private IQualifiedNameProvider iQualifiedNameProvider; + + public FixJvmModelInferrer() { + } + + public void infer(final EObject fix, final IJvmDeclaredTypeAcceptor acceptor, final boolean isPrelinkingPhase) { + if (fix == null) { + throw new IllegalArgumentException("Unhandled parameter types: " + + Arrays.asList(fix, acceptor, isPrelinkingPhase).toString()); + } + + infer(fix instanceof Fix ? (Fix) fix : fix, acceptor, isPrelinkingPhase); + } + + private void infer(final Fix fix, final IJvmDeclaredTypeAcceptor acceptor, final boolean isPrelinkingPhase) { + acceptor.accept(jvmTypesBuilder.toClass(fix, iQualifiedNameProvider.getFullyQualifiedName(fix)), (JvmGenericType it) -> { + jvmTypesBuilder.setDocumentation(it, jvmTypesBuilder.getDocumentation(fix)); + jvmTypesBuilder.operator_add(it.getSuperTypes(), _typeReferenceBuilder.typeRef(Metafix.class)); + }); + } + +} diff --git a/metafix/src/main/java/org/metafacture/metafix/maps/RdfMap.java b/metafix/src/main/java/org/metafacture/metafix/maps/RdfMap.java new file mode 100644 index 000000000..4e7109f2d --- /dev/null +++ b/metafix/src/main/java/org/metafacture/metafix/maps/RdfMap.java @@ -0,0 +1,401 @@ +/* + * Copyright 2022 hbz + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix.maps; + +import org.metafacture.metafix.FixExecutionException; +import org.metafacture.metamorph.api.Maps; +import org.metafacture.metamorph.api.helpers.AbstractReadOnlyMap; + +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.Property; +import org.apache.jena.rdf.model.RDFNode; +import org.apache.jena.rdf.model.ResIterator; +import org.apache.jena.rdf.model.Resource; +import org.apache.jena.rdf.model.ResourceFactory; +import org.apache.jena.rdf.model.Statement; +import org.apache.jena.rdf.model.StmtIterator; +import org.apache.jena.riot.RDFDataMgr; +import org.apache.jena.shared.PropertyNotFoundException; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLConnection; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.function.UnaryOperator; + +/** + * Provides a dynamically built {@link Map} based on an RDF resource. Can be one file or a comma separated list of RDF + * files or HTTP(S) URIs. Redirections of HTTP(S) URIs are followed. + * The resources are supposed to be UTF-8 encoded. + *

+ * + * Important: When using a list of files make sure to set the proper separator. All lines that are not + * split in two parts by the separator are ignored! + * + * @author Markus Michael Geipel + * @author Pascal Christoph (dr0i) + * + * @see org.metafacture.metamorph.maps.FileMap + */ +public final class RdfMap extends AbstractReadOnlyMap { + + public static final String SELECT = "select"; + public static final String TARGET = "target"; + public static final String TARGET_LANGUAGE = "select_language"; + + private static final int MAX_REDIRECTIONS = 10; + + private static final int MIN_HTTP_STATUS_CODE = 299; + private static final int MAX_HTTP_STATUS_CODE = 400; + + private final ArrayList filenames = new ArrayList<>(); + private final Map map = new HashMap<>(); + + private Model model; + private Select select = Select.DEFAULT; + private String target; + private String targetLanguage = ""; + private boolean isUninitialized = true; + + /** + * Creates an instance of {@link RdfMap}. + */ + public RdfMap() { + setDefault(null); + } + + private boolean isURI(final String name) { + return name.toLowerCase().startsWith("http"); + } + + private void init() { + filenames.forEach(this::loadFile); + + if (!isURI(target)) { + final String[] nsPrefixAndProperty = target.split(":"); + target = nsPrefixAndProperty.length == 2 ? model.getNsPrefixURI(nsPrefixAndProperty[0]) + nsPrefixAndProperty[1] : nsPrefixAndProperty[0]; + } + + isUninitialized = false; + } + + /** + * Sets a comma separated list of files which provides the {@link Model}. + * + * @param files a comma separated list of files + */ + public void setFiles(final String files) { + Collections.addAll(filenames, files.split("\\s*,\\s*")); + } + + /** + * Sets a file which provides the {@link Model}. + * + * @param file the file + */ + public void setResource(final String file) { + filenames.add(file); + } + + /** + * Sets a file or URL which provides the {@link Model}. + * + * @param file the file or URL + * @param operator an operator to apply to the file + */ + public void setResource(final String file, final UnaryOperator operator) { + setResource(isURI(file) ? file : operator.apply(file)); + } + + private void loadFile(final String file) { + try { + final String uri = isURI(file) ? read(file) : file; + + if (model == null) { + model = RDFDataMgr.loadModel(uri); + } + else { + RDFDataMgr.read(model, uri); + } + } + catch (final IOException e) { + throw new FixExecutionException("Error while loading RDF file: " + file, e); + } + } + + /** + * Builds a Map dynamically by querying an RDF model based on a key and a targeted Property + * (to be set in {@link RdfMap#setTarget(String)}) and an optional language tag (to be set in + * {@link RdfMap#setTargetLanguage}). + *
+ * The Map acts as a cache. + *

+ * To minimize the need of parameters three different querying modes are gone through. If one fails, the next one is + * tried: + *

+ * 1. Get the value of the targeted Property of a Subject + *
+ * 2. Get the Subject matching a targeted Property value + *
+ * 3. Get the value of a Property using the value of a targeted Property + * + * @param key the Property value, or a Subject, to be looked up + */ + @Override + public String get(final Object key) { + final String resourceName = key.toString(); + String result = null; + + if (map.containsKey(resourceName)) { + result = map.get(resourceName); + } + else { + if (isUninitialized) { + init(); + } + + final Resource resource = ResourceFactory.createResource(resourceName); + final Property targetProperty = ResourceFactory.createProperty(target); + + try { + if (select.equals(Select.SUBJECT)) { + result = getSubjectUsingPropertyAndLiteral(resourceName, targetProperty); + } + else { + // 1. try to get LITERAL using SUBJECT and PROPERTY + if (!targetLanguage.isEmpty()) { + result = model.getRequiredProperty(resource, targetProperty, targetLanguage).getString(); + } + else { + result = model.getRequiredProperty(resource, targetProperty).getString(); + } + } + } + catch (final PropertyNotFoundException | NullPointerException | NoSuchElementException e) { + // 2. try to get SUBJECT using PROPERTY and LITERAL + if (select.equals(Select.DEFAULT)) { + result = getSubjectUsingPropertyAndLiteral(resourceName, targetProperty); + } + // 3. try to get LITERAL of PREDICATE A using PREDICATE B + if (!select.equals(Select.SUBJECT)) { + if (result == null) { + result = getLiteralOfPredicateUsingOtherPredicate(resourceName, targetProperty); + } + } + } + + map.put(resourceName, result); + } + + return result; + } + + private String getLiteralOfPredicateUsingOtherPredicate(final String resourceName, final Property targetProperty) { + final ResIterator iter = model.listSubjectsWithProperty(targetProperty); + String result = map.get(Maps.DEFAULT_MAP_KEY); + + while (iter.hasNext()) { + final Resource resource = iter.nextResource(); + final StmtIterator iterProp = resource.listProperties(targetProperty); + + while (iterProp.hasNext()) { + final Statement stmt = iterProp.nextStatement(); + + if (stmt.getObject().asLiteral().getString().equals(resourceName)) { + final StmtIterator subIterProp = resource.listProperties(targetProperty); + + while (subIterProp.hasNext()) { + final Statement subStmt = subIterProp.nextStatement(); + + if (subStmt.getLanguage().equals(targetLanguage) && !subStmt.getString().equals(resourceName)) { + result = subStmt.getString(); + } + } + } + } + } + + return result; + } + + private String getSubjectUsingPropertyAndLiteral(final String resourceName, final Property targetProperty) { + final ResIterator iter = model.listSubjectsWithProperty(targetProperty); + String result = map.get(Maps.DEFAULT_MAP_KEY); + + while (iter.hasNext()) { + final Resource resource = iter.nextResource(); + final StmtIterator stmtIterator = resource.listProperties(targetProperty); + + while (stmtIterator.hasNext()) { + final RDFNode node = stmtIterator.next().getObject(); + + if (!targetLanguage.isEmpty()) { + if (node.asLiteral().toString().equals(resourceName + "@" + targetLanguage)) { + result = resource.getURI(); + break; + } + } + else { + if (node.asLiteral().getString().equals(resourceName)) { + result = resource.getURI(); + break; + } + } + } + } + + return result; + } + + /** + * Gets the language of the target Property which is queried in the RDF. Valid values are defined by BCP47. + * + * @return the targeted language + */ + public String getTargetLanguage() { + return targetLanguage; + } + + /** + * Sets the language of the target Property which is queried in the RDF. Valid values are defined by BCP47. + *
+ * Setting the language of the target Property is optional. + * + * @param targetLanguage the language of the target Property to be queried + */ + public void setTargetLanguage(final String targetLanguage) { + this.targetLanguage = targetLanguage; + } + + /** + * Gets the target Property which is queried in the RDF. Namespaces are allowed. + * + * @return the target Property to be queried + */ + public String getTarget() { + return target; + } + + /** + * Sets the target Property which is queried in the RDF. Namespaces are allowed. + *
+ * Setting a target Property is mandatory. + * + * @param target the Property to be queried + */ + public void setTarget(final String target) { + this.target = target; + } + + /** + * Gets whether the Subject or the Object or a mixture of both should be retrieved in the RDF. + *
+ * Setting "select" is optional. + * + * @return the selected position to be retrieved + **/ + public String getSelect() { + return select.toString(); + } + + /** + * Sets whether the Subject or the Object or a mixture of both should be retrieved in the RDF. + *
+ * Setting "select" is optional. + * Defaults to retrieve both: tries to get "objects" and as a fallback "subjects". + * + * @param position the position to be retrieved. Can be "subject" or "object". + */ + public void setSelect(final String position) { + if (Select.SUBJECT.name().equalsIgnoreCase(position)) { + select = Select.SUBJECT; + } + else if (Select.OBJECT.name().equalsIgnoreCase(position)) { + select = Select.OBJECT; + } + else { + throw new FixExecutionException("Couldn't set parameter - use 'subject' or 'object' as value"); + } + } + + /** + * Sets the default value returned if the key couldn't be found. + *
+ * Default value: {@link Maps#DEFAULT_MAP_KEY} + * + * @param defaultValue the default value returned + */ + public void setDefault(final String defaultValue) { + map.put(Maps.DEFAULT_MAP_KEY, defaultValue); + } + + /** + * Gets a redirected URL, if any redirection takes place. Adapted predated code from org.apache.jena.rdfxml.xmlinput.JenaReader. + *

+ * Note: Using newer Jena version (needs java 11) this method would be obsolete. + * + * @param url the URL to resolve + * @return the (redirected) URL + * @throws IOException if any IO error occurs + */ + private String read(final String url) throws IOException { + String connectionURL = url; + + int count = 0; + URLConnection conn; + + while (true) { + final URLConnection conn2 = new URL(connectionURL).openConnection(); + if (!(conn2 instanceof HttpURLConnection)) { + conn = conn2; + break; + } + + count += 1; + if (count > MAX_REDIRECTIONS) { + throw new IOException("Too many redirects followed for " + url); + } + + final HttpURLConnection httpURLConnection = (HttpURLConnection) conn2; + conn2.setRequestProperty("accept", "*/*"); + + final int statusCode = httpURLConnection.getResponseCode(); + if (statusCode <= MIN_HTTP_STATUS_CODE || statusCode >= MAX_HTTP_STATUS_CODE) { + conn = conn2; + break; + } + + // Redirect + connectionURL = conn2.getHeaderField("Location"); + if (connectionURL == null || url.equals(connectionURL)) { + throw new IOException("Failed to follow redirects for " + url); + } + } + + return conn.getURL().toString(); + } + + private enum Select { + SUBJECT, OBJECT, DEFAULT + } + +} diff --git a/metafix/src/main/java/org/metafacture/metafix/scoping/FixScopeProvider.java b/metafix/src/main/java/org/metafacture/metafix/scoping/FixScopeProvider.java new file mode 100644 index 000000000..14f0a75a2 --- /dev/null +++ b/metafix/src/main/java/org/metafacture/metafix/scoping/FixScopeProvider.java @@ -0,0 +1,14 @@ +package org.metafacture.metafix.scoping; + +/** + * This class contains custom scoping description. + * + * See https://www.eclipse.org/Xtext/documentation/303_runtime_concepts.html#scoping + * on how and when to use it. + */ +public class FixScopeProvider extends AbstractFixScopeProvider { + + public FixScopeProvider() { + } + +} diff --git a/metafix/src/main/java/org/metafacture/metafix/validation/FixValidator.java b/metafix/src/main/java/org/metafacture/metafix/validation/FixValidator.java new file mode 100644 index 000000000..600112f12 --- /dev/null +++ b/metafix/src/main/java/org/metafacture/metafix/validation/FixValidator.java @@ -0,0 +1,13 @@ +package org.metafacture.metafix.validation; + +/** + * This class contains custom validation rules. + * + * See https://www.eclipse.org/Xtext/documentation/303_runtime_concepts.html#validation + */ +public class FixValidator extends AbstractFixValidator { + + public FixValidator() { + } + +} diff --git a/metafix/src/main/java/org/metafacture/metafix/validation/XtextValidator.java b/metafix/src/main/java/org/metafacture/metafix/validation/XtextValidator.java new file mode 100644 index 000000000..5427598e4 --- /dev/null +++ b/metafix/src/main/java/org/metafacture/metafix/validation/XtextValidator.java @@ -0,0 +1,89 @@ +package org.metafacture.metafix.validation; + +import org.metafacture.metafix.FixParseException; + +import org.eclipse.emf.common.util.URI; +import org.eclipse.xtext.ISetup; +import org.eclipse.xtext.XtextStandaloneSetup; +import org.eclipse.xtext.resource.XtextResource; +import org.eclipse.xtext.resource.XtextResourceSet; +import org.eclipse.xtext.util.CancelIndicator; +import org.eclipse.xtext.validation.CheckMode; +import org.eclipse.xtext.validation.Issue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +public class XtextValidator { + + private static final Logger LOG = LoggerFactory.getLogger(XtextValidator.class); + + private XtextValidator() { + throw new IllegalAccessError("Utility class"); + } + + private static boolean validate(final XtextResource resource, final ISetup setup) { + final List issues = resource.getResourceServiceProvider() + .getResourceValidator().validate(resource, CheckMode.ALL, CancelIndicator.NullImpl); + + final int count = issues.size(); + + if (count > 0) { + LOG.warn("The {} file '{}' has {} issue{}:", + resourceType(setup), resource.getURI().toFileString(), count, count > 1 ? "s" : ""); + + issues.forEach(i -> LOG.warn("- {}: {} ({}:{})", + i.getSeverity(), i.getMessage(), i.getLineNumber(), i.getColumn())); + + return false; + } + + return true; + } + + public static boolean validate(final String path, final ISetup setup) { + return validate(getResource(path, setup), setup); + } + + private static XtextResource getResource(final String path, final ISetup setup) { + final File file = new File(path); + String absolutePath; + + try { + absolutePath = file.getCanonicalPath(); + } + catch (final IOException e) { + absolutePath = file.getAbsolutePath(); + } + + return (XtextResource) setup.createInjectorAndDoEMFRegistration() + .getInstance(XtextResourceSet.class).getResource(URI.createFileURI(absolutePath), true); + } + + public static XtextResource getValidatedResource(final String path, final ISetup setup) { + final XtextResource resource = getResource(path, setup); + + if (validate(resource, setup)) { + return resource; + } + else { + throw new FixParseException("Invalid " + resourceType(setup) + " resource: " + path); + } + } + + private static String resourceType(final ISetup setup) { + return setup.getClass().getSimpleName(); + } + + public static void main(final String[] args) { + if (args != null && args.length == 1) { + System.exit(validate(args[0], new XtextStandaloneSetup()) ? 0 : 1); + } + + throw new IllegalArgumentException(String.format("Usage: %s ", XtextValidator.class.getName())); + } + +} diff --git a/metafix/src/main/resources/flux-commands.properties b/metafix/src/main/resources/flux-commands.properties new file mode 100644 index 000000000..33cb50c27 --- /dev/null +++ b/metafix/src/main/resources/flux-commands.properties @@ -0,0 +1,18 @@ +# Copyright 2020 Fabian Steeg, hbz +# +# Licensed under the Apache License, Version 2.0 the "License"; +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +fix org.metafacture.metafix.Metafix +list-fix-paths org.metafacture.metafix.ListFixPaths +list-fix-values org.metafacture.metafix.ListFixValues +find-fix-paths org.metafacture.metafix.FindFixPaths diff --git a/metafix/src/test/java/org/metafacture/metafix/ArrayValueTest.java b/metafix/src/test/java/org/metafacture/metafix/ArrayValueTest.java new file mode 100644 index 000000000..b59c427a2 --- /dev/null +++ b/metafix/src/test/java/org/metafacture/metafix/ArrayValueTest.java @@ -0,0 +1,34 @@ +/* + * Copyright 2022 hbz NRW + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix; + +import nl.jqno.equalsverifier.EqualsVerifier; +import org.junit.jupiter.api.Test; + +public class ArrayValueTest { + + public ArrayValueTest() { + } + + @Test + public void shouldSatisfyEqualsContract() { + EqualsVerifier.forClass(Value.Array.class) + .withPrefabValues(Value.class, Value.newArray(), Value.newHash()) + .verify(); + } + +} diff --git a/metafix/src/test/java/org/metafacture/metafix/FindFixPathsTest.java b/metafix/src/test/java/org/metafacture/metafix/FindFixPathsTest.java new file mode 100644 index 000000000..2a871cae2 --- /dev/null +++ b/metafix/src/test/java/org/metafacture/metafix/FindFixPathsTest.java @@ -0,0 +1,83 @@ +/* + * Copyright 2024 Tobias Bülte, hbz + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix; + +import org.metafacture.framework.ObjectReceiver; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InOrder; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.exceptions.base.MockitoAssertionError; +import org.mockito.junit.jupiter.MockitoExtension; + +/** + * Tests for class {@link FindFixPaths}. + * + * @author Tobias Bülte + * + */ +@ExtendWith(MockitoExtension.class) +public final class FindFixPathsTest { + + private final FindFixPaths finder = new FindFixPaths(".*ETL.*"); + + @Mock + private ObjectReceiver receiver; + + public FindFixPathsTest() { + } + + @Test + public void testShouldFindPaths() { + verify( + "a\t|\tAn ETL test", + "c.2\t|\tETL what?"); + } + + private void processRecord() { + finder.setReceiver(receiver); + finder.startRecord(""); + finder.literal("a", "An ETL test"); + finder.literal("b", ""); + finder.literal("b", "Dummi"); + finder.literal("b", "Dog"); + finder.literal("c", ""); + finder.literal("c", "ETL what?"); + finder.endRecord(); + finder.closeStream(); + } + + private void verify(final String... result) throws MockitoAssertionError { + processRecord(); + try { + final InOrder ordered = Mockito.inOrder(receiver); + for (final String r : result) { + ordered.verify(receiver).process(r); + } + ordered.verify(receiver, Mockito.times(2)).closeStream(); + ordered.verifyNoMoreInteractions(); + Mockito.verifyNoMoreInteractions(receiver); + } + catch (final MockitoAssertionError e) { + System.out.println(Mockito.mockingDetails(receiver).printInvocations()); + throw e; + } + } + +} diff --git a/metafix/src/test/java/org/metafacture/metafix/FixParsingTest.java b/metafix/src/test/java/org/metafacture/metafix/FixParsingTest.java new file mode 100644 index 000000000..cc2607b0f --- /dev/null +++ b/metafix/src/test/java/org/metafacture/metafix/FixParsingTest.java @@ -0,0 +1,116 @@ +package org.metafacture.metafix; + +import org.metafacture.metafix.fix.Fix; +import org.metafacture.metafix.tests.FixInjectorProvider; + +import com.google.inject.Inject; +import org.eclipse.emf.common.util.EList; +import org.eclipse.emf.ecore.resource.Resource; +import org.eclipse.xtext.testing.InjectWith; +import org.eclipse.xtext.testing.extensions.InjectionExtension; +import org.eclipse.xtext.testing.util.ParseHelper; +import org.eclipse.xtext.xbase.lib.IterableExtensions; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +@ExtendWith(InjectionExtension.class) +@InjectWith(FixInjectorProvider.class) +public class FixParsingTest { + + private static final String LITERAL_END = "end"; + + @Inject + private ParseHelper parseHelper; + + public FixParsingTest() { + } + + @Test + public void shouldParseSimple() throws Exception { + parse( + "map(a,b)" + ); + } + + @Test + public void shouldParseQuotedStrings() throws Exception { + parse( + "add_field(hello,'world')", + "add_field(hello,\"world\")", + "add_field(hello,\"w-o:r l/d\")", + "add_field(hello,'\tw\n\torld')", + "add_field(hello,'\\tw\\n\\torld')", + "add_field(hello,'\"world\"')" + ); + } + + @Test + public void shouldParseMappings() throws Exception { + parse( + "# simple field name mappings", + "", + "map(a,b)", + "", + "# nested field structure", + "", + "map(e1)", + "map(e1.e2)", + "map(e1.e2.d)", + "", + "# pass-through for unmapped fields", + "", + "map(_else)", + "" + ); + } + + @Test + public void shouldParseTransformations() throws Exception { + parse( + "# FIX is a macro-language for data transformations", + "", + "# Simple fixes", + "", + "add_field(hello,\"world\")", + "remove_field(my.deep.nested.junk)", + "copy_field(stats,output.$append)", + "", + "# Conditionals", + "", + "if exists(error)", + "\tset_field(is_valid, no)", + "\tlog(error)", + "elsif exists(warning)", + "\tset_field(is_valid, yes)", + "\tlog(warning)", + "else", + "\tset_field(is_valid, yes)", + LITERAL_END, + "", + "# Loops", + "", + "do list(path)", + "\tadd_field(foo,bar)", + LITERAL_END, + "", + "# Nested expressions", + "", + "do marc_each()", + "\tif marc_has(f700)", + "\t\tmarc_map(f700a,authors.$append)", + "\t" + LITERAL_END, + LITERAL_END, + "" + ); + } + + private void parse(final String... fix) throws Exception { + final Fix result = parseHelper.parse(String.join("\n", fix)); + Assertions.assertNotNull(result); + + final EList errors = result.eResource().getErrors(); + Assertions.assertTrue(errors.isEmpty(), "Unexpected errors: " + IterableExtensions.join(errors, ", ")); + } + +} diff --git a/metafix/src/test/java/org/metafacture/metafix/HashValueTest.java b/metafix/src/test/java/org/metafacture/metafix/HashValueTest.java new file mode 100644 index 000000000..433a3fad3 --- /dev/null +++ b/metafix/src/test/java/org/metafacture/metafix/HashValueTest.java @@ -0,0 +1,433 @@ +/* + * Copyright 2021 hbz NRW + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix; + +import org.metafacture.commons.tries.SimpleRegexTrie; + +import nl.jqno.equalsverifier.EqualsVerifier; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; + +public class HashValueTest { + + private static final String FIELD = "field"; + private static final String OTHER_FIELD = "other field"; + private static final String ALTERNATE_FIELD = FIELD.replace("e", "i"); + + private static final String FIELD_CHARACTER_CLASS = FIELD.replace("e", "[aeiou]"); + private static final String FIELD_ALTERNATION = FIELD + "|" + ALTERNATE_FIELD; + private static final String FIELD_WILDCARD = FIELD.replace("e", "?"); + + private static final Value VALUE = new Value("value"); + private static final Value OTHER_VALUE = new Value("other value"); + + public HashValueTest() { + } + + @Test + public void shouldSatisfyEqualsContract() { + EqualsVerifier.forClass(Value.Hash.class) + .withPrefabValues(Value.class, Value.newArray(), Value.newHash()) + .withPrefabValues(SimpleRegexTrie.class, new SimpleRegexTrie(), new SimpleRegexTrie()) + .withIgnoredFields("patternMatcher", "prefixCache", "trie", "trieCache") + .verify(); + } + + @Test + public void shouldNotContainMissingField() { + final Value.Hash hash = newHash(); + Assertions.assertFalse(hash.containsField(FIELD)); + } + + @Test + public void shouldContainExistingField() { + final Value.Hash hash = newHash(); + hash.put(FIELD, VALUE); + + Assertions.assertTrue(hash.containsField(FIELD)); + } + + @Test + public void shouldNotContainNullValue() { + final Value.Hash hash = newHash(); + hash.put(FIELD, null); + + Assertions.assertFalse(hash.containsField(FIELD)); + } + + @Test + public void shouldContainCharacterClassField() { + final Value.Hash hash = newHash(); + hash.put(FIELD, VALUE); + + Assertions.assertTrue(hash.containsField(FIELD_CHARACTER_CLASS)); + } + + @Test + public void shouldContainAlternationField() { + final Value.Hash hash = newHash(); + hash.put(FIELD, VALUE); + + Assertions.assertTrue(hash.containsField(FIELD_ALTERNATION)); + } + + @Test + public void shouldContainWildcardField() { + final Value.Hash hash = newHash(); + hash.put(FIELD, VALUE); + + Assertions.assertTrue(hash.containsField(FIELD_WILDCARD)); + } + + @Test + public void shouldBeEmptyByDefault() { + final Value.Hash hash = newHash(); + Assertions.assertTrue(hash.isEmpty()); + } + + @Test + public void shouldNotBeEmptyAfterAddingValue() { + final Value.Hash hash = newHash(); + hash.put(FIELD, VALUE); + + Assertions.assertFalse(hash.isEmpty()); + } + + @Test + public void shouldNotAddNullValue() { + final Value.Hash hash = newHash(); + hash.put(FIELD, null); + + Assertions.assertTrue(hash.isEmpty()); + } + + @Test + public void shouldGetSizeOfDefaultMapping() { + final Value.Hash hash = newHash(); + Assertions.assertEquals(0, hash.size()); + } + + @Test + public void shouldGetSizeAfterAddingValues() { + final Value.Hash hash = newHash(); + hash.put(FIELD, VALUE); + hash.put(OTHER_FIELD, OTHER_VALUE); + + Assertions.assertEquals(2, hash.size()); + } + + @Test + public void shouldNotGetMissingField() { + final Value.Hash hash = newHash(); + Assertions.assertNull(hash.get(FIELD)); + } + + @Test + public void shouldGetExistingField() { + final Value.Hash hash = newHash(); + hash.put(FIELD, VALUE); + + Assertions.assertEquals(VALUE, hash.get(FIELD)); + } + + @Test + public void shouldGetSingleCharacterClassField() { + final Value.Hash hash = newHash(); + hash.put(FIELD, VALUE); + + Assertions.assertEquals(VALUE, hash.get(FIELD_CHARACTER_CLASS)); + } + + @Test + public void shouldGetSingleAlternationField() { + final Value.Hash hash = newHash(); + hash.put(FIELD, VALUE); + + Assertions.assertEquals(VALUE, hash.get(FIELD_ALTERNATION)); + } + + @Test + public void shouldGetSingleWildcardField() { + final Value.Hash hash = newHash(); + hash.put(FIELD, VALUE); + + Assertions.assertEquals(VALUE, hash.get(FIELD_WILDCARD)); + } + + @Test + public void shouldGetMultipleCharacterClassFields() { + final Value.Hash hash = newHash(); + hash.put(FIELD, VALUE); + hash.put(ALTERNATE_FIELD, OTHER_VALUE); + + assertArray(hash.get(FIELD_CHARACTER_CLASS)); + } + + @Test + public void shouldGetMultipleAlternationFields() { + final Value.Hash hash = newHash(); + hash.put(FIELD, VALUE); + hash.put(ALTERNATE_FIELD, OTHER_VALUE); + + assertArray(hash.get(FIELD_ALTERNATION)); + } + + @Test + public void shouldGetMultipleWildcardFields() { + final Value.Hash hash = newHash(); + hash.put(FIELD, VALUE); + hash.put(ALTERNATE_FIELD, OTHER_VALUE); + + assertArray(hash.get(FIELD_WILDCARD)); + } + + @Test + public void shouldNotReplaceMissingField() { + final Value.Hash hash = newHash(); + hash.replace(FIELD, VALUE); + + Assertions.assertNull(hash.get(FIELD)); + Assertions.assertFalse(hash.containsField(FIELD)); + } + + @Test + public void shouldReplaceExistingField() { + final Value.Hash hash = newHash(); + hash.put(FIELD, VALUE); + hash.replace(FIELD, OTHER_VALUE); + + Assertions.assertEquals(OTHER_VALUE, hash.get(FIELD)); + } + + @Test + public void shouldNotReplaceExistingFieldWithNullValue() { + final Value.Hash hash = newHash(); + hash.put(FIELD, VALUE); + hash.replace(FIELD, (Value) null); + + Assertions.assertEquals(VALUE, hash.get(FIELD)); + } + + @Test + public void shouldRemoveMissingField() { + final Value.Hash hash = newHash(); + hash.remove(FIELD); + + Assertions.assertNull(hash.get(FIELD)); + Assertions.assertFalse(hash.containsField(FIELD)); + } + + @Test + public void shouldRemoveExistingField() { + final Value.Hash hash = newHash(); + hash.put(FIELD, VALUE); + hash.remove(FIELD); + + Assertions.assertNull(hash.get(FIELD)); + Assertions.assertFalse(hash.containsField(FIELD)); + } + + @Test + public void shouldRemoveCharacterClassFields() { + final Value.Hash hash = newHash(); + hash.put(FIELD, VALUE); + hash.put(ALTERNATE_FIELD, VALUE); + hash.remove(FIELD_CHARACTER_CLASS); + + Assertions.assertTrue(hash.isEmpty()); + } + + @Test + public void shouldRemoveAlternationFields() { + final Value.Hash hash = newHash(); + hash.put(FIELD, VALUE); + hash.put(ALTERNATE_FIELD, VALUE); + hash.remove(FIELD_ALTERNATION); + + Assertions.assertTrue(hash.isEmpty()); + } + + @Test + public void shouldRemoveWildcardFields() { + final Value.Hash hash = newHash(); + hash.put(FIELD, VALUE); + hash.put(ALTERNATE_FIELD, VALUE); + hash.remove(FIELD_WILDCARD); + + Assertions.assertTrue(hash.isEmpty()); + } + + @Test + public void shouldNotContainRemovedField() { + final Value.Hash hash = newHash(); + Assertions.assertFalse(hash.containsField(FIELD)); + + hash.put(FIELD, VALUE); + Assertions.assertTrue(hash.containsField(FIELD)); + + hash.remove(FIELD); + Assertions.assertFalse(hash.containsField(FIELD)); + } + + @Test + public void shouldRetainFields() { + final Value.Hash hash = newHash(); + hash.put(FIELD, VALUE); + hash.put(OTHER_FIELD, OTHER_VALUE); + + hash.retainFields(Arrays.asList(FIELD)); + + Assertions.assertTrue(hash.containsField(FIELD)); + Assertions.assertFalse(hash.containsField(OTHER_FIELD)); + } + + @Test + public void shouldRetainNoFields() { + final Value.Hash hash = newHash(); + hash.put(FIELD, VALUE); + + hash.retainFields(Arrays.asList()); + + Assertions.assertTrue(hash.isEmpty()); + } + + @Test + public void shouldNotRetainMissingFields() { + final Value.Hash hash = newHash(); + hash.put(FIELD, VALUE); + + hash.retainFields(Arrays.asList(FIELD, OTHER_FIELD)); + + Assertions.assertTrue(hash.containsField(FIELD)); + Assertions.assertFalse(hash.containsField(OTHER_FIELD)); + } + + @Test + public void shouldRetainCharacterClassFields() { + final Value.Hash hash = newHash(); + hash.put(FIELD, VALUE); + hash.put(OTHER_FIELD, OTHER_VALUE); + + hash.retainFields(Arrays.asList(FIELD_CHARACTER_CLASS)); + + Assertions.assertTrue(hash.containsField(FIELD)); + Assertions.assertFalse(hash.containsField(OTHER_FIELD)); + } + + @Test + public void shouldRetainAlternationFields() { + final Value.Hash hash = newHash(); + hash.put(FIELD, VALUE); + hash.put(OTHER_FIELD, OTHER_VALUE); + + hash.retainFields(Arrays.asList(FIELD_ALTERNATION)); + + Assertions.assertTrue(hash.containsField(FIELD)); + Assertions.assertFalse(hash.containsField(OTHER_FIELD)); + } + + @Test + public void shouldRetainWildcardFields() { + final Value.Hash hash = newHash(); + hash.put(FIELD, VALUE); + hash.put(OTHER_FIELD, OTHER_VALUE); + + hash.retainFields(Arrays.asList(FIELD_WILDCARD)); + + Assertions.assertTrue(hash.containsField(FIELD)); + Assertions.assertFalse(hash.containsField(OTHER_FIELD)); + } + + @Test + public void shouldRemoveEmptyValues() { + final Value.Hash hash = newHash(); + hash.put(FIELD, VALUE); + hash.put(OTHER_FIELD, new Value("")); + + hash.removeEmptyValues(); + + Assertions.assertTrue(hash.containsField(FIELD)); + Assertions.assertFalse(hash.containsField(OTHER_FIELD)); + } + + @Test + public void shouldIterateOverFieldValuePairs() { + final Value emptyValue = new Value(""); + final Value specialValue = new Value("1"); + + final Value.Hash hash = newHash(); + hash.put(FIELD, VALUE); + hash.put(OTHER_FIELD, OTHER_VALUE); + hash.put("empty field", emptyValue); + hash.put("_special field", specialValue); + + MetafixTestHelpers.assertEmittedFields(hash, + Arrays.asList(FIELD, OTHER_FIELD, "empty field", "_special field"), + Arrays.asList(VALUE, OTHER_VALUE, emptyValue, specialValue) + ); + } + + private void shouldFindArray(final String field) { + final Value.Hash hash = newHash(); + hash.put(FIELD, Value.newArray(a -> a.add(VALUE))); + + Assertions.assertEquals(VALUE, new FixPath(String.join(".", FIELD, field)).findIn(hash)); + } + + @Test + public void shouldFindArrayIndex() { + shouldFindArray("1"); + } + + @Test + public void shouldFindArrayWildcard() { + shouldFindArray("$last"); + } + + private void shouldFindArraySubfield(final String field) { + final Value.Hash hash = newHash(); + hash.put(FIELD, Value.newArray(a -> a.add(Value.newHash(h -> h.put(OTHER_FIELD, OTHER_VALUE))))); + + Assertions.assertEquals(OTHER_VALUE, new FixPath(String.join(".", FIELD, field, OTHER_FIELD)).findIn(hash)); + } + + @Test + public void shouldFindArrayIndexSubfield() { + shouldFindArraySubfield("1"); + } + + @Test + public void shouldFindArrayWildcardSubfield() { + shouldFindArraySubfield("$last"); + } + + private Value.Hash newHash() { + return Value.newHash().asHash(); + } + + private void assertArray(final Value result) { + Assertions.assertTrue(result.isArray()); + + final Value.Array array = result.asArray(); + Assertions.assertEquals(2, array.size()); + Assertions.assertEquals(VALUE, array.get(0)); + Assertions.assertEquals(OTHER_VALUE, array.get(1)); + } + +} diff --git a/metafix/src/test/java/org/metafacture/metafix/InterpreterTest.java b/metafix/src/test/java/org/metafacture/metafix/InterpreterTest.java new file mode 100644 index 000000000..afc3ac24e --- /dev/null +++ b/metafix/src/test/java/org/metafacture/metafix/InterpreterTest.java @@ -0,0 +1,58 @@ +package org.metafacture.metafix; + +import org.metafacture.metafix.fix.Fix; +import org.metafacture.metafix.interpreter.FixInterpreter; +import org.metafacture.metafix.tests.FixInjectorProvider; + +import com.google.inject.Inject; +import org.eclipse.xtext.testing.InjectWith; +import org.eclipse.xtext.testing.extensions.InjectionExtension; +import org.eclipse.xtext.testing.util.ParseHelper; +import org.eclipse.xtext.xbase.lib.Extension; +import org.junit.Assert; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +@ExtendWith(InjectionExtension.class) +@InjectWith(FixInjectorProvider.class) +public class InterpreterTest { + + @Inject + @Extension + private FixInterpreter fixInterpreter; + + @Inject + @Extension + private ParseHelper parseHelper; + + public InterpreterTest() { + } + + @Test + public void shouldInterpretSimple() throws Exception { + interpret(1, + "add_field(hello,world)" + ); + } + + @Test + public void shouldInterpretNested() throws Exception { + interpret(3, + "do marc_each()", + "\tif marc_has(f700)", + "\t\tmarc_map(f700a,authors.$append)", + "\tend", + "end", + "" + ); + } + + private void interpret(final int expressions, final String... fix) throws Exception { + final Metafix metafix = new Metafix(); + Assert.assertTrue(metafix.getExpressions().isEmpty()); + + fixInterpreter.run(metafix, parseHelper.parse(String.join("\n", fix))); + Assert.assertEquals("metafix expressions: " + metafix.getExpressions(), expressions, metafix.getExpressions().size()); + } + +} diff --git a/metafix/src/test/java/org/metafacture/metafix/ListFixPathsTest.java b/metafix/src/test/java/org/metafacture/metafix/ListFixPathsTest.java new file mode 100644 index 000000000..886abf888 --- /dev/null +++ b/metafix/src/test/java/org/metafacture/metafix/ListFixPathsTest.java @@ -0,0 +1,126 @@ +/* + * Copyright 2023 Fabian Steeg, hbz + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix; + +import org.metafacture.framework.ObjectReceiver; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InOrder; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.exceptions.base.MockitoAssertionError; +import org.mockito.junit.jupiter.MockitoExtension; + +/** + * Tests for class {@link ListFixPaths}. + * + * @author Fabian Steeg + * + */ +@ExtendWith(MockitoExtension.class) +public final class ListFixPathsTest { + + private final ListFixPaths lister = new ListFixPaths(); + + @Mock + private ObjectReceiver receiver; + + public ListFixPathsTest() { + } + + @Test + public void testShouldListPaths() { + verify( + "3\t|\tb.*", + "2\t|\tc.*", + "1\t|\ta"); + } + + @Test + public void testShouldListPathsNoCount() { + lister.setCount(false); + verify( + "a", + "b.*", + "c.*"); + } + + @Test + public void testShouldListPathsUseIndex() { + lister.setIndex(true); + verify( + "1\t|\ta", + "1\t|\tb.1", + "1\t|\tb.2", + "1\t|\tb.3", + "1\t|\tc.1", + "1\t|\tc.2"); + } + + @Test + public void testShouldListPathsNoCountUseIndex() { + lister.setCount(false); + lister.setIndex(true); + verify( + "a", + "b.1", + "b.2", + "b.3", + "c.1", + "c.2"); + } + + @Test + public void testShouldListPathsSortedByFrequency() { + verify( + "3\t|\tb.*", + "2\t|\tc.*", + "1\t|\ta"); + } + + private void processRecord() { + lister.setReceiver(receiver); + lister.startRecord(""); + lister.literal("a", ""); + lister.literal("b", ""); + lister.literal("b", ""); + lister.literal("b", ""); + lister.literal("c", ""); + lister.literal("c", ""); + lister.endRecord(); + lister.closeStream(); + } + + private void verify(final String... result) throws MockitoAssertionError { + processRecord(); + try { + final InOrder ordered = Mockito.inOrder(receiver); + for (final String r : result) { + ordered.verify(receiver).process(r); + } + ordered.verify(receiver, Mockito.times(2)).closeStream(); + ordered.verifyNoMoreInteractions(); + Mockito.verifyNoMoreInteractions(receiver); + } + catch (final MockitoAssertionError e) { + System.out.println(Mockito.mockingDetails(receiver).printInvocations()); + throw e; + } + } + +} diff --git a/metafix/src/test/java/org/metafacture/metafix/ListFixValuesTest.java b/metafix/src/test/java/org/metafacture/metafix/ListFixValuesTest.java new file mode 100644 index 000000000..60f94fe8e --- /dev/null +++ b/metafix/src/test/java/org/metafacture/metafix/ListFixValuesTest.java @@ -0,0 +1,140 @@ +/* + * Copyright 2023 Fabian Steeg, hbz + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix; + +import org.metafacture.framework.ObjectReceiver; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InOrder; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.exceptions.base.MockitoAssertionError; +import org.mockito.junit.jupiter.MockitoExtension; + +/** + * Tests for class {@link ListFixValues}. + * + * @author Fabian Steeg + * + */ +@ExtendWith(MockitoExtension.class) +public final class ListFixValuesTest { + + private ListFixValues lister; + + @Mock + private ObjectReceiver receiver; + + public ListFixValuesTest() { + } + + @Test + public void testShouldListPathsSimple() { + lister = new ListFixValues("a"); + verify("2\t|\taA"); + } + + @Test + public void testShouldListPathsWildcard() { + lister = new ListFixValues("a.*"); + verify("2\t|\taA"); + } + + @Test + public void testShouldListPathsIndex() { + lister = new ListFixValues("a.1"); + verify("1\t|\taA"); + } + + @Test + public void testShouldListPathsTwoValues() { + lister = new ListFixValues("b"); + verify( + "2\t|\tbB", + "1\t|\tbA"); + } + + @Test + public void testShouldListPathsTwoValuesWildcard() { + lister = new ListFixValues("b.*"); + verify( + "2\t|\tbB", + "1\t|\tbA"); + } + + @Test + public void testShouldListPathsTwoValuesIndex() { + lister = new ListFixValues("b.1"); + verify( + "1\t|\tbB"); + } + + @Test + public void testShouldListPathsSortCount() { + lister = new ListFixValues("c"); + verify( + "3\t|\tcC", + "1\t|\tcA", + "1\t|\tcB"); + } + + @Test + public void testShouldListPathsDontCount() { + lister = new ListFixValues("c"); + lister.setCount(false); + verify( + "cA", + "cB", + "cC"); + } + + private void processRecord() { + lister.setReceiver(receiver); + lister.startRecord(""); + lister.literal("a", "aA"); + lister.literal("a", "aA"); + lister.literal("b", "bB"); + lister.literal("b", "bB"); + lister.literal("b", "bA"); + lister.literal("c", "cC"); + lister.literal("c", "cB"); + lister.literal("c", "cC"); + lister.literal("c", "cA"); + lister.literal("c", "cC"); + lister.endRecord(); + lister.closeStream(); + } + + private void verify(final String... result) throws MockitoAssertionError { + processRecord(); + try { + final InOrder ordered = Mockito.inOrder(receiver); + for (final String r : result) { + ordered.verify(receiver).process(r); + } + ordered.verify(receiver, Mockito.times(2)).closeStream(); + ordered.verifyNoMoreInteractions(); + Mockito.verifyNoMoreInteractions(receiver); + } + catch (final MockitoAssertionError e) { + System.out.println(Mockito.mockingDetails(receiver).printInvocations()); + throw e; + } + } + +} diff --git a/metafix/src/test/java/org/metafacture/metafix/MetafixBindTest.java b/metafix/src/test/java/org/metafacture/metafix/MetafixBindTest.java new file mode 100644 index 000000000..889179169 --- /dev/null +++ b/metafix/src/test/java/org/metafacture/metafix/MetafixBindTest.java @@ -0,0 +1,1165 @@ +/* + * Copyright 2021 Fabian Steeg, hbz + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix; + +import org.metafacture.framework.StreamReceiver; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.Arrays; + +/** + * Test Metafix binds. + * + * @author Fabian Steeg + */ +@ExtendWith(MockitoExtension.class) +@ExtendWith(MetafixToDo.Extension.class) +public class MetafixBindTest { + + @Mock + private StreamReceiver streamReceiver; + + public MetafixBindTest() { + } + + @Test + public void doList() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "do list('path': 'name', 'var': 'n')", + " upcase('n')", + " trim('n')", + " copy_field('n', 'author')", + "end", + "remove_field('name')"), + i -> { + i.startRecord("1"); + i.literal("name", " A University"); + i.literal("name", "Max "); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("author", "MAX"); + o.get().endRecord(); + }); + } + + @Test + public void doListExplicitAppend() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('author')", + "do list('path': 'name', 'var': 'n')", + " upcase('n')", + " trim('n')", + " copy_field('n', 'author.$append')", + "end", + "remove_field('name')"), + i -> { + i.startRecord("1"); + i.literal("name", " A University"); + i.literal("name", "Max "); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("author", "A UNIVERSITY"); + o.get().literal("author", "MAX"); + o.get().endRecord(); + }); + } + + @Test + public void doListFullRecordInScope() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "do list('path': 'name', 'var': 'n')", + " if any_equal('type','book')", + " paste('title','~Book:','n')", + " else", + " paste('title','~Journal:','n')", + " end", + "end", + "retain('title')"), + i -> { + i.startRecord("1"); + i.literal("type", "book"); + i.literal("name", "A book"); + i.endRecord(); + i.startRecord("2"); + i.literal("type", "journal"); + i.literal("name", "A journal"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("title", "Book: A book"); + o.get().endRecord(); + o.get().startRecord("2"); + o.get().literal("title", "Journal: A journal"); + o.get().endRecord(); + }); + } + + @Test + public void bindingScopeWithVar() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "do list('path':'foo','var':'loop')", + " copy_field('test','loop.baz')", + " copy_field('loop.bar','loop.qux')", + "end" + ), + i -> { + i.startRecord("1"); + i.startEntity("foo"); + i.literal("bar", "1"); + i.endEntity(); + i.startEntity("foo"); + i.literal("bar", "2"); + i.endEntity(); + i.literal("test", "42"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("foo"); + o.get().literal("bar", "1"); + o.get().literal("baz", "42"); + o.get().literal("qux", "1"); + o.get().endEntity(); + o.get().startEntity("foo"); + o.get().literal("bar", "2"); + o.get().literal("baz", "42"); + o.get().literal("qux", "2"); + o.get().endEntity(); + o.get().literal("test", "42"); + o.get().endRecord(); + } + ); + } + + @Test + public void bindingScopeWithoutVar() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "do list('path':'foo')", + " copy_field('test','baz')", + " copy_field('bar','qux')", + "end" + ), + i -> { + i.startRecord("1"); + i.startEntity("foo"); + i.literal("bar", "1"); + i.endEntity(); + i.startEntity("foo"); + i.literal("bar", "2"); + i.endEntity(); + i.literal("test", "42"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("foo"); + o.get().literal("bar", "1"); + o.get().literal("qux", "1"); + o.get().endEntity(); + o.get().startEntity("foo"); + o.get().literal("bar", "2"); + o.get().literal("qux", "2"); + o.get().endEntity(); + o.get().literal("test", "42"); + o.get().endRecord(); + } + ); + } + + @Test + public void doListPathWithDots() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "do list('path': 'some.name', 'var': 'n')", + " upcase('n')", + " trim('n')", + " copy_field('n', 'author')", + "end", + "remove_field('some')"), + i -> { + i.startRecord("1"); + i.startEntity("some"); + i.literal("name", " A University"); + i.literal("name", "Max "); + i.endEntity(); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("author", "MAX"); + o.get().endRecord(); + }); + } + + @Test + public void doListPathWithDotsExplicitAppend() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('author')", + "do list('path': 'some.name', 'var': 'n')", + " upcase('n')", + " trim('n')", + " copy_field('n', 'author.$append')", + "end", + "remove_field('some')"), + i -> { + i.startRecord("1"); + i.startEntity("some"); + i.literal("name", " A University"); + i.literal("name", "Max "); + i.endEntity(); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("author", "A UNIVERSITY"); + o.get().literal("author", "MAX"); + o.get().endRecord(); + }); + } + + @Test + public void doListWithAppendAndLast() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('author[]')", + "do list('path': 'creator', 'var': 'c')", + " copy_field('c.name', 'author[].$append.name')", + " add_field('author[].$last.type', 'Default')", + "end", + "remove_field('creator')"), + i -> { + i.startRecord("1"); + i.startEntity("creator"); + i.literal("name", "A University"); + i.endEntity(); + i.startEntity("creator"); + i.literal("name", "Max"); + i.endEntity(); + i.endRecord(); + }, (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("author[]"); + o.get().startEntity("1"); + o.get().literal("name", "A University"); + o.get().literal("type", "Default"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("name", "Max"); + o.get().literal("type", "Default"); + f.apply(2).endEntity(); + o.get().endRecord(); + }); + } + + @Test + public void doListEntitesToLiterals() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "do list('path': 'creator', 'var': 'c')", + " upcase('c.name')", + " trim('c.name')", + " copy_field('c.name', 'author')", + "end", + "remove_field('creator')"), + i -> { + i.startRecord("1"); + i.startEntity("creator"); + i.literal("name", " A University"); + i.endEntity(); + i.startEntity("creator"); + i.literal("name", "Max "); + i.endEntity(); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("author", "MAX"); + o.get().endRecord(); + }); + } + + @Test + public void doListEntitesToLiteralsExplicitAppend() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('author')", + "do list('path': 'creator', 'var': 'c')", + " upcase('c.name')", + " trim('c.name')", + " copy_field('c.name', 'author.$append')", + "end", + "remove_field('creator')"), + i -> { + i.startRecord("1"); + i.startEntity("creator"); + i.literal("name", " A University"); + i.endEntity(); + i.startEntity("creator"); + i.literal("name", "Max "); + i.endEntity(); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("author", "A UNIVERSITY"); + o.get().literal("author", "MAX"); + o.get().endRecord(); + }); + } + + @Test + public void doListEntitesToEntities() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('author[]')", + "do list('path': 'creator', 'var': 'c')", + " copy_field('c.name', 'author[].$append.name')", + " if all_contain('c.name', 'University')", + " add_field('author[].$last.type', 'Organization')", + " else", + " add_field('author[].$last.type', 'Person')", //", + " end", + "end", + "remove_field('creator')"), + i -> { + i.startRecord("1"); + i.startEntity("creator"); + i.literal("name", "A University"); + i.endEntity(); + i.startEntity("creator"); + i.literal("name", "Max"); + i.endEntity(); + i.endRecord(); + }, (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("author[]"); + o.get().startEntity("1"); + o.get().literal("name", "A University"); + o.get().literal("type", "Organization"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("name", "Max"); + o.get().literal("type", "Person"); + f.apply(2).endEntity(); + o.get().endRecord(); + }); + } + + @Test + public void wildcardForNestedEntities() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('author[]')", + "do list('path': 'creator', 'var': 'c')", + " if any_match('c.role.*.roleTerm.*.value','aut|cre')", + " copy_field('c.name', 'author[].$append.name')", + " end", + "end", + "remove_field('creator')"), + i -> { + i.startRecord("1"); + i.startEntity("creator"); + i.literal("name", "A University"); + i.startEntity("role"); + i.startEntity("roleTerm"); + i.literal("value", "aut"); + i.endEntity(); + i.startEntity("roleTerm"); + i.literal("value", "tau"); + i.endEntity(); + i.endEntity(); + i.endEntity(); + i.startEntity("creator"); + i.literal("name", "Max"); + i.startEntity("role"); + i.startEntity("roleTerm"); + i.literal("value", "cre"); + i.endEntity(); + i.startEntity("roleTerm"); + i.literal("value", "rec"); + i.endEntity(); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("author[]"); + o.get().startEntity("1"); + o.get().literal("name", "A University"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("name", "Max"); + f.apply(2).endEntity(); + o.get().endRecord(); + }); + } + + public void doListIndexedArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "do list('path': 'name[]', 'var': 'n')", + " copy_field('n', 'author')", + "end", + "remove_field('name[]')"), + i -> { + i.startRecord("1"); + i.startEntity("name[]"); + i.literal("1", "A University"); + i.literal("2", "Max"); + i.endEntity(); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().startEntity("author"); + o.get().literal("1", "A University"); + o.get().literal("2", "Max"); + o.get().endEntity(); + o.get().endRecord(); + }); + } + + @Test + public void doListIndexedArrayToArrayOfObjects() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('author[]')", + "do list('path': 'name[]', 'var': 'n')", + " copy_field('n', 'author[].$append.name')", + "end", + "remove_field('name[]')"), + i -> { + i.startRecord("1"); + i.startEntity("name[]"); + i.literal("1", "A University"); + i.literal("2", "Max"); + i.endEntity(); + i.endRecord(); + }, (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("author[]"); + o.get().startEntity("1"); + o.get().literal("name", "A University"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("name", "Max"); + f.apply(2).endEntity(); + o.get().endRecord(); + }); + } + + @Test + public void doListIndexedArrayOfObjects() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "do list('path': 'name[]', 'var': 'n')", + " copy_field('n.name', 'author')", + "end", + "remove_field('name[]')"), + i -> { + i.startRecord("1"); + i.startEntity("name[]"); + i.startEntity("1"); + i.literal("name", "A University"); + i.endEntity(); + i.startEntity("2"); + i.literal("name", "Max"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("author", "Max"); + o.get().endRecord(); + }); + } + + @Test + public void doListIndexedArrayOfObjectsExplicitAppend() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('author')", + "do list('path': 'name[]', 'var': 'n')", + " copy_field('n.name', 'author.$append')", + "end", + "remove_field('name[]')"), + i -> { + i.startRecord("1"); + i.startEntity("name[]"); + i.startEntity("1"); + i.literal("name", "A University"); + i.endEntity(); + i.startEntity("2"); + i.literal("name", "Max"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("author", "A University"); + o.get().literal("author", "Max"); + o.get().endRecord(); + }); + } + + @Test + public void doListIndexedArrayOfObjectsToArrayOfObjects() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('author[]')", + "do list('path': 'name[]', 'var': 'n')", + " copy_field('n.name', 'author[].$append.name')", + "end", + "remove_field('name[]')"), + i -> { + i.startRecord("1"); + i.startEntity("name[]"); + i.startEntity("1"); + i.literal("name", "A University"); + i.endEntity(); + i.startEntity("2"); + i.literal("name", "Max"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("author[]"); + o.get().startEntity("1"); + o.get().literal("name", "A University"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("name", "Max"); + f.apply(2).endEntity(); + o.get().endRecord(); + }); + } + + @Test + @MetafixToDo("implement Fix-style binds with collectors?") + public void ifInCollector() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "do entity('author')", + " map('name')", + " if all_contain('name', 'University')", + " add_field('type', 'Organization')", + " end", + "end"), + i -> { + i.startRecord("1"); + i.literal("name", "A University"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().startEntity("author"); + o.get().literal("type", "Organization"); + o.get().literal("name", "A University"); + o.get().endEntity(); + o.get().endRecord(); + }); + } + + @Test + @MetafixToDo("implement Fix-style binds with collectors?") + public void ifInCollectorMultiRecords() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "do entity('author')", + " map('name')", + " if all_contain('name', 'University')", + " add_field('type', 'Organization')", + " end", + "end"), + i -> { + i.startRecord("1"); + i.literal("name", "Max"); + i.endRecord(); + + i.startRecord("2"); + i.literal("name", "A University"); + i.endRecord(); + + i.startRecord("3"); + i.literal("name", "Mary"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().startEntity("author"); + o.get().literal("name", "Max"); + o.get().endEntity(); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().startEntity("author"); + o.get().literal("type", "Organization"); + o.get().literal("name", "A University"); + o.get().endEntity(); + o.get().endRecord(); + + o.get().startRecord("3"); + o.get().startEntity("author"); + o.get().literal("name", "Mary"); + o.get().endEntity(); + o.get().endRecord(); + }); + } + + @Test + @MetafixToDo("implement Fix-style binds with collectors?") + public void ifInCollectorChoose() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "do choose(flushWith: 'record')", + " if all_contain('name', 'University')", + " add_field('type', 'Organization')", + " end", + "end"), + i -> { + i.startRecord("1"); + i.literal("name", "Max University"); + i.endRecord(); + + i.startRecord("2"); + i.literal("name", "Max Musterman"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("type", "Organization"); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().endRecord(); + }); + } + + @Test + @MetafixToDo("implement Fix-style binds with collectors?") + public void ifInCollectorCombine() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "do combine(name: 'fullName', value: '${first} ${last}')", + " if all_contain('author.type', 'Person')", + " map('author.first', 'first')", + " map('author.last', 'last')", + " end", + "end"), + i -> { + i.startRecord("1"); + i.startEntity("author"); + i.literal("type", "Organization"); + i.endEntity(); + i.endRecord(); + + i.startRecord("2"); + i.startEntity("author"); + i.literal("first", "Max"); + i.literal("last", "Musterman"); + i.literal("type", "DifferentiatedPerson"); + i.endEntity(); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().literal("fullName", "Max Musterman"); + o.get().endRecord(); + }); + } + + private void shouldIterateOverList(final String path, final int expectedCount) { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('trace')", + "do list(path: '" + path + "', 'var': '$i')", + " add_field('trace.$append', 'true')", + "end", + "retain('trace')" + ), + i -> { + i.startRecord("1"); + i.literal("name", "Mary"); + i.literal("name", "University"); + i.literal("nome", "Max"); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + f.apply(expectedCount).literal("trace", "true"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldIterateOverList() { + shouldIterateOverList("name", 2); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/119 + public void shouldIterateOverListWithCharacterClass() { + shouldIterateOverList("n[ao]me", 3); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/119 + public void shouldIterateOverListWithAlternation() { + shouldIterateOverList("name|nome", 3); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/119 + public void shouldIterateOverListWithWildcard() { + shouldIterateOverList("n?me", 3); + } + + private void shouldIterateOverListOfHashes(final String path, final int expectedCount) { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('trace')", + "do list(path: '" + path + "', 'var': '$i')", + " add_field('trace.$append', 'true')", + "end", + "retain('trace')" + ), + i -> { + i.startRecord("1"); + i.startEntity("name"); + i.literal("value", "Mary"); + i.endEntity(); + i.startEntity("name"); + i.literal("value", "University"); + i.endEntity(); + i.startEntity("nome"); + i.literal("value", "Max"); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + f.apply(expectedCount).literal("trace", "true"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldIterateOverListOfHashes() { + shouldIterateOverListOfHashes("name.value", 2); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/119 + public void shouldIterateOverListOfHashesWithCharacterClass() { + shouldIterateOverListOfHashes("n[ao]me.value", 3); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/119 + @MetafixToDo("See https://github.com/metafacture/metafacture-fix/issues/143") + public void shouldIterateOverListOfHashesWithAlternation() { + shouldIterateOverListOfHashes("name.value|nome.value", 3); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/119 + public void shouldIterateOverListOfHashesWithWildcard() { + shouldIterateOverListOfHashes("n?me.value", 3); + } + + @Test // checkstyle-disable-line JavaNCSS + // See https://github.com/metafacture/metafacture-fix/issues/119 + public void shouldPerformComplexOperationWithPathWildcard() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('coll[]')", + "do list(path: 'feld?', 'var': '$i')", + " add_field('coll[].$append.feldtest', 'true')", + " copy_field('$i.a.value', 'coll[].$last.a')", + "end", + "retain('coll[]')" + ), + i -> { + i.startRecord("1"); + i.startEntity("feldA"); + i.startEntity("a"); + i.literal("value", "Dog"); + i.endEntity(); + i.endEntity(); + i.startEntity("feldA"); + i.startEntity("a"); + i.literal("value", "Ape"); + i.endEntity(); + i.endEntity(); + i.startEntity("feldA"); + i.startEntity("a"); + i.literal("value", "Giraffe"); + i.endEntity(); + i.endEntity(); + i.startEntity("feldB"); + i.startEntity("a"); + i.literal("value", "Crocodile"); + i.endEntity(); + i.endEntity(); + i.startEntity("feldB"); + i.startEntity("a"); + i.literal("value", "Human"); + i.endEntity(); + i.endEntity(); + i.startEntity("feldB"); + i.startEntity("a"); + i.literal("value", "Bird"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("coll[]"); + o.get().startEntity("1"); + o.get().literal("feldtest", "true"); + o.get().literal("a", "Dog"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("feldtest", "true"); + o.get().literal("a", "Ape"); + o.get().endEntity(); + o.get().startEntity("3"); + o.get().literal("feldtest", "true"); + o.get().literal("a", "Giraffe"); + o.get().endEntity(); + o.get().startEntity("4"); + o.get().literal("feldtest", "true"); + o.get().literal("a", "Crocodile"); + o.get().endEntity(); + o.get().startEntity("5"); + o.get().literal("feldtest", "true"); + o.get().literal("a", "Human"); + o.get().endEntity(); + o.get().startEntity("6"); + o.get().literal("feldtest", "true"); + o.get().literal("a", "Bird"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldDoListAsWithSingleList() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('sourceOrga[]')", + "do list_as(orgId: 'ccm:university[]')", + " copy_field(orgId, 'sourceOrga[].$append.id')", + "end" + ), + i -> { + i.startRecord("1"); + i.startEntity("ccm:university[]"); + i.literal("1", "https://ror.org/0304hq317"); + i.literal("2", "https://ror.org/014nnvj65"); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("ccm:university[]"); + o.get().literal("1", "https://ror.org/0304hq317"); + o.get().literal("2", "https://ror.org/014nnvj65"); + o.get().endEntity(); + o.get().startEntity("sourceOrga[]"); + o.get().startEntity("1"); + o.get().literal("id", "https://ror.org/0304hq317"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("id", "https://ror.org/014nnvj65"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldDoListAsWithMultipleLists() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('sourceOrga[]')", + "do list_as(orgId: 'ccm:university[]', orgName: 'ccm:university_DISPLAYNAME[]', orgLoc: 'ccm:university_LOCATION[]')", + " copy_field(orgId, 'sourceOrga[].$append.id')", + " copy_field(orgName, 'sourceOrga[].$last.name')", + " copy_field(orgLoc, 'sourceOrga[].$last.location')", + "end" + ), + i -> { + i.startRecord("1"); + i.startEntity("ccm:university[]"); + i.literal("1", "https://ror.org/0304hq317"); + i.literal("2", "https://ror.org/014nnvj65"); + i.endEntity(); + i.startEntity("ccm:university_DISPLAYNAME[]"); + i.literal("1", "Gottfried Wilhelm Leibniz Universität Hannover"); + i.literal("2", "Technische Hochschule Köln"); + i.endEntity(); + i.startEntity("ccm:university_LOCATION[]"); + i.literal("1", "Hannover"); + i.literal("2", "Köln"); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("ccm:university[]"); + o.get().literal("1", "https://ror.org/0304hq317"); + o.get().literal("2", "https://ror.org/014nnvj65"); + o.get().endEntity(); + o.get().startEntity("ccm:university_DISPLAYNAME[]"); + o.get().literal("1", "Gottfried Wilhelm Leibniz Universität Hannover"); + o.get().literal("2", "Technische Hochschule Köln"); + o.get().endEntity(); + o.get().startEntity("ccm:university_LOCATION[]"); + o.get().literal("1", "Hannover"); + o.get().literal("2", "Köln"); + o.get().endEntity(); + o.get().startEntity("sourceOrga[]"); + o.get().startEntity("1"); + o.get().literal("id", "https://ror.org/0304hq317"); + o.get().literal("name", "Gottfried Wilhelm Leibniz Universität Hannover"); + o.get().literal("location", "Hannover"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("id", "https://ror.org/014nnvj65"); + o.get().literal("name", "Technische Hochschule Köln"); + o.get().literal("location", "Köln"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test // checkstyle-disable-line JavaNCSS + public void shouldDoListAsWithMultipleListsOfDifferentSizes() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('sourceOrga[]')", + "do list_as(orgId: 'ccm:university[]', orgName: 'ccm:university_DISPLAYNAME[]', orgLoc: 'ccm:university_LOCATION[]')", + " set_hash('sourceOrga[].$append')", + " copy_field(orgId, 'sourceOrga[].$last.id')", + " copy_field(orgName, 'sourceOrga[].$last.name')", + " copy_field(orgLoc, 'sourceOrga[].$last.location')", + "end" + ), + i -> { + i.startRecord("1"); + i.startEntity("ccm:university[]"); + i.literal("1", "https://ror.org/0304hq317"); + i.literal("2", "https://ror.org/014nnvj65"); + i.endEntity(); + i.startEntity("ccm:university_DISPLAYNAME[]"); + i.literal("1", "Gottfried Wilhelm Leibniz Universität Hannover"); + i.literal("2", "Technische Hochschule Köln"); + i.literal("3", "Universität zu Köln"); + i.literal("4", "Stadtbibliothek Köln"); + i.endEntity(); + i.startEntity("ccm:university_LOCATION[]"); + i.literal("1", "Hannover"); + i.literal("2", "Köln"); + i.literal("3", "Köln"); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("ccm:university[]"); + o.get().literal("1", "https://ror.org/0304hq317"); + o.get().literal("2", "https://ror.org/014nnvj65"); + o.get().endEntity(); + o.get().startEntity("ccm:university_DISPLAYNAME[]"); + o.get().literal("1", "Gottfried Wilhelm Leibniz Universität Hannover"); + o.get().literal("2", "Technische Hochschule Köln"); + o.get().literal("3", "Universität zu Köln"); + o.get().literal("4", "Stadtbibliothek Köln"); + o.get().endEntity(); + o.get().startEntity("ccm:university_LOCATION[]"); + o.get().literal("1", "Hannover"); + o.get().literal("2", "Köln"); + o.get().literal("3", "Köln"); + o.get().endEntity(); + o.get().startEntity("sourceOrga[]"); + o.get().startEntity("1"); + o.get().literal("id", "https://ror.org/0304hq317"); + o.get().literal("name", "Gottfried Wilhelm Leibniz Universität Hannover"); + o.get().literal("location", "Hannover"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("id", "https://ror.org/014nnvj65"); + o.get().literal("name", "Technische Hochschule Köln"); + o.get().literal("location", "Köln"); + o.get().endEntity(); + o.get().startEntity("3"); + o.get().literal("name", "Universität zu Köln"); + o.get().literal("location", "Köln"); + o.get().endEntity(); + o.get().startEntity("4"); + o.get().literal("name", "Stadtbibliothek Köln"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldExecuteOnlyOnce() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "do once()", + " add_field(executed, 'true')", + "end", + "do once()", + " add_field(never_executed, 'true')", + "end" + ), + i -> { + i.startRecord("1"); + i.endRecord(); + i.startRecord("2"); + i.endRecord(); + i.startRecord("3"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("executed", "true"); + o.get().endRecord(); + o.get().startRecord("2"); + o.get().endRecord(); + o.get().startRecord("3"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldExecuteOnlyOncePerFixInstance() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "do once()", + " add_field(executed, 'true')", + "end", + "include('src/test/resources/org/metafacture/metafix/fixes/once.fix')" + ), + i -> { + i.startRecord("1"); + i.endRecord(); + i.startRecord("2"); + i.endRecord(); + i.startRecord("3"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("executed", "true"); + o.get().literal("included", "true"); + o.get().endRecord(); + o.get().startRecord("2"); + o.get().literal("included", "true"); + o.get().endRecord(); + o.get().startRecord("3"); + o.get().literal("included", "true"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldExecuteOnlyOncePerIdentifier() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "do once()", + " add_field(executed, 'true')", + "end", + "do once()", + " add_field(never_executed, 'true')", + "end", + "do once('setup')", + " add_field(setup, 'true')", + "end", + "do once('setup')", + " add_field(already_setup, 'true')", + "end" + ), + i -> { + i.startRecord("1"); + i.endRecord(); + i.startRecord("2"); + i.endRecord(); + i.startRecord("3"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("executed", "true"); + o.get().literal("setup", "true"); + o.get().endRecord(); + o.get().startRecord("2"); + o.get().endRecord(); + o.get().startRecord("3"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldExecuteOnlyOnceConditionally() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if exists(execute)", + " do once()", + " add_field(executed, 'false')", + " end", + " do once('setup')", + " add_field(setup, 'true')", + " end", + "else", + " do once()", + " add_field(already_executed, 'true')", + " end", + "end" + ), + i -> { + i.startRecord("1"); + i.endRecord(); + i.startRecord("2"); + i.literal("execute", "too late"); + i.endRecord(); + i.startRecord("3"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("already_executed", "true"); + o.get().endRecord(); + o.get().startRecord("2"); + o.get().literal("execute", "too late"); + o.get().literal("setup", "true"); + o.get().endRecord(); + o.get().startRecord("3"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldExecuteCustomJavaContext() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "do org.metafacture.metafix.util.TestContext(test, data: '42')", + " add_field(title,'marc')", + "end" + ), + i -> { + i.startRecord("1"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("BEFORE", "test"); + o.get().literal("title", "marc"); + o.get().literal("AFTER", "42"); + o.get().endRecord(); + } + ); + } + +} diff --git a/metafix/src/test/java/org/metafacture/metafix/MetafixIfTest.java b/metafix/src/test/java/org/metafacture/metafix/MetafixIfTest.java new file mode 100644 index 000000000..5b8000dcc --- /dev/null +++ b/metafix/src/test/java/org/metafacture/metafix/MetafixIfTest.java @@ -0,0 +1,2549 @@ +/* + * Copyright 2021 Fabian Steeg, hbz + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix; + +import org.metafacture.framework.StreamReceiver; + +import com.google.common.collect.ImmutableMap; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.Arrays; + +/** + * Test Metafix `if` conditionals. + * + * @author Fabian Steeg + */ +@ExtendWith(MockitoExtension.class) // checkstyle-disable-line JavaNCSS +public class MetafixIfTest { + + @Mock + private StreamReceiver streamReceiver; + + public MetafixIfTest() { + } + + @Test + public void ifAny() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if any_contain('name', 'University')", + " add_field('type', 'Organization')", + "end"), + i -> { + i.startRecord("1"); + i.literal("name", "Mary"); + i.literal("name", "A University"); + i.endRecord(); + + i.startRecord("2"); + i.literal("name", "Mary"); + i.literal("name", "Max"); + i.endRecord(); + + i.startRecord("3"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("name", "Mary"); + o.get().literal("name", "A University"); + o.get().literal("type", "Organization"); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().literal("name", "Mary"); + o.get().literal("name", "Max"); + o.get().endRecord(); + + o.get().startRecord("3"); + o.get().endRecord(); + }); + } + + @Test + public void ifAll() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if all_contain('name', 'University')", + " add_field('type', 'Organization')", + "end"), + i -> { + i.startRecord("1"); + i.literal("name", "Mary"); + i.literal("name", "A University"); + i.endRecord(); + + i.startRecord("2"); + i.literal("name", "Great University"); + i.literal("name", "A University"); + i.endRecord(); + + i.startRecord("3"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("name", "Mary"); + o.get().literal("name", "A University"); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().literal("name", "Great University"); + o.get().literal("name", "A University"); + o.get().literal("type", "Organization"); + o.get().endRecord(); + + o.get().startRecord("3"); + o.get().endRecord(); + }); + } + + @Test + public void ifNone() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if none_contain('author.name', 'University')", + " add_field('type', 'Person')", + "end" + ), + i -> { + i.startRecord("1"); + i.startEntity("author"); + i.literal("name", "Mary"); + i.literal("name", "A University"); + i.endEntity(); + i.endRecord(); + + i.startRecord("2"); + i.startEntity("author"); + i.literal("name", "Max"); + i.literal("name", "Mary"); + i.endEntity(); + i.endRecord(); + + i.startRecord("3"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("author"); + o.get().literal("name", "Mary"); + o.get().literal("name", "A University"); + o.get().endEntity(); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().startEntity("author"); + o.get().literal("name", "Max"); + o.get().literal("name", "Mary"); + o.get().endEntity(); + o.get().literal("type", "Person"); + o.get().endRecord(); + + o.get().startRecord("3"); + o.get().literal("type", "Person"); + o.get().endRecord(); + } + ); + } + + @Test + public void ifEqual() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if all_equal('name', 'University')", + " add_field('type', 'Organization')", + "end"), + i -> { + i.startRecord("1"); + i.literal("name", "University"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("name", "University"); + o.get().literal("type", "Organization"); + o.get().endRecord(); + }); + } + + @Test + public void ifContainMoveField() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if all_contain('name', 'University')", + " move_field('name', 'orgName')", + "end"), + i -> { + i.startRecord("1"); + i.literal("name", "Max"); + i.endRecord(); + + i.startRecord("2"); + i.literal("name", "A University"); + i.endRecord(); + + i.startRecord("3"); + i.literal("name", "Mary"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("name", "Max"); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().literal("orgName", "A University"); + o.get().endRecord(); + + o.get().startRecord("3"); + o.get().literal("name", "Mary"); + o.get().endRecord(); + }); + } + + @Test + public void moveAndAddIfContain() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "move_field('name', 'author.name')", + "if all_contain('author.name', 'University')", + " add_field('author.type', 'Organization')", + "end"), + i -> { + i.startRecord("1"); + i.literal("name", "A University"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().startEntity("author"); + o.get().literal("name", "A University"); + o.get().literal("type", "Organization"); + o.get().endEntity(); + o.get().endRecord(); + }); + } + + @Test + public void ifContainMultipleAddField() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if all_contain('name', 'University')", + " add_field('type', 'Organization')", + " add_field('comment', 'type was guessed')", + "end"), + i -> { + i.startRecord("1"); + i.literal("name", "A University"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("name", "A University"); + o.get().literal("type", "Organization"); + o.get().literal("comment", "type was guessed"); + o.get().endRecord(); + }); + } + + @Test + public void ifAnyMatch() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if any_match('name', '.*University.*')", + " add_field('type', 'Organization')", + "end"), + i -> { + i.startRecord("1"); + i.literal("name", "Max"); + i.endRecord(); + + i.startRecord("2"); + i.literal("name", "Some University"); + i.literal("name", "Filibandrina"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("name", "Max"); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().literal("name", "Some University"); + o.get().literal("name", "Filibandrina"); + o.get().literal("type", "Organization"); + o.get().endRecord(); + }); + } + + @Test + public void ifAllMatch() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if all_match('name', '.*University.*')", + " add_field('type', 'Organization')", + "end"), + i -> { + i.startRecord("1"); + i.literal("name", "Max"); + i.literal("name", "A University"); + i.endRecord(); + + i.startRecord("2"); + i.literal("name", "Some University"); + i.literal("name", "University Filibandrina"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("name", "Max"); + o.get().literal("name", "A University"); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().literal("name", "Some University"); + o.get().literal("name", "University Filibandrina"); + o.get().literal("type", "Organization"); + o.get().endRecord(); + }); + } + + @Test + public void ifAnyMatchNested() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if any_match('author.name.label', '.*University.*')", + " add_field('author.type', 'Organization')", + "end"), + i -> { + i.startRecord("1"); + i.startEntity("author"); + i.startEntity("name"); + i.literal("label", "Max"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + + i.startRecord("2"); + i.startEntity("author"); + i.startEntity("name"); + i.literal("label", "Some University"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("author"); + o.get().startEntity("name"); + o.get().literal("label", "Max"); + f.apply(2).endEntity(); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().startEntity("author"); + o.get().startEntity("name"); + o.get().literal("label", "Some University"); + o.get().endEntity(); + o.get().literal("type", "Organization"); + o.get().endEntity(); + o.get().endRecord(); + }); + } + + @Test + public void shouldNotImplicitlyMatchNestedField() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if any_match('author.name', '.*University.*')", + " add_field('author.type', 'Organization')", + "end" + ), + i -> { + i.startRecord("1"); + i.startEntity("author"); + i.literal("name", "A University"); + i.endEntity(); + i.endRecord(); + + i.startRecord("2"); + i.startEntity("author"); + i.startEntity("name"); + i.literal("label", "Some University"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("author"); + o.get().literal("name", "A University"); + o.get().literal("type", "Organization"); + o.get().endEntity(); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().startEntity("author"); + o.get().startEntity("name"); + o.get().literal("label", "Some University"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void ifAnyMatchFirstRecord() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if any_match('name', '.*University.*')", + " add_field('type', 'Organization')", + "end"), + i -> { + i.startRecord("1"); + i.literal("name", "Some University"); + i.endRecord(); + + i.startRecord("2"); + i.literal("name", "Max"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("name", "Some University"); + o.get().literal("type", "Organization"); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().literal("name", "Max"); + o.get().endRecord(); + }); + } + + @Test + public void ifAnyMatchLastRecord() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if any_match('name', '.*University.*')", + " add_field('type', 'Organization')", + "end"), + i -> { + i.startRecord("1"); + i.literal("name", "Max"); + i.endRecord(); + + i.startRecord("2"); + i.literal("name", "Some University"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("name", "Max"); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().literal("name", "Some University"); + o.get().literal("type", "Organization"); + o.get().endRecord(); + }); + } + + @Test + public void unlessAnyMatch() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "unless any_match('name', '.*University.*')", + " add_field('type', 'Person')", + "end"), + i -> { + i.startRecord("1"); + i.literal("name", "Max"); + i.endRecord(); + + i.startRecord("2"); + i.literal("name", "Some University"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("name", "Max"); + o.get().literal("type", "Person"); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().literal("name", "Some University"); + o.get().endRecord(); + }); + } + + @Test + public void ifAnyMatchElse() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if any_match('name', '.*University.*')", + " add_field('type', 'Organization')", + "else", + " add_field('type', 'Person')", + "end"), + i -> { + i.startRecord("1"); + i.literal("name", "Max"); + i.endRecord(); + + i.startRecord("2"); + i.literal("name", "Some University"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("name", "Max"); + o.get().literal("type", "Person"); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().literal("name", "Some University"); + o.get().literal("type", "Organization"); + o.get().endRecord(); + }); + } + + @Test + public void ifAnyMatchElsif() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if any_match('name', '.*University.*')", + " add_field('type', 'Organization')", + "elsif any_match('name', '[^ ]* [^ ]*')", + " add_field('type', 'Person')", + "else", + " add_field('type', 'Unknown')", + "end"), + i -> { + i.startRecord("1"); + i.literal("name", "Max Power"); + i.endRecord(); + + i.startRecord("2"); + i.literal("name", "Some University"); + i.endRecord(); + + i.startRecord("3"); + i.literal("name", "Filibandrina"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("name", "Max Power"); + o.get().literal("type", "Person"); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().literal("name", "Some University"); + o.get().literal("type", "Organization"); + o.get().endRecord(); + + o.get().startRecord("3"); + o.get().literal("name", "Filibandrina"); + o.get().literal("type", "Unknown"); + o.get().endRecord(); + }); + } + + @Test + public void ifAnyMatchMultipleElsif() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if any_match('name', '.*University.*')", + " add_field('type', 'Organization')", + "elsif any_match('name', 'Mary')", + " add_field('type', 'Person1')", + "elsif any_match('name', 'Max')", + " add_field('type', 'Person2')", + "elsif any_match('name', '[^ ]* [^ ]*')", + " add_field('type', 'Person')", + "else", + " add_field('type', 'Unknown')", + "end"), + i -> { + i.startRecord("1"); + i.literal("name", "Mary Power"); + i.literal("name", "Max Power"); + i.endRecord(); + + i.startRecord("2"); + i.literal("name", "Some University"); + i.endRecord(); + + i.startRecord("3"); + i.literal("name", "Filibandrina"); + i.endRecord(); + + i.startRecord("4"); + i.literal("name", "Mary"); + i.literal("name", "Max Power"); + i.endRecord(); + + i.startRecord("5"); + i.literal("name", "Max"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("name", "Mary Power"); + o.get().literal("name", "Max Power"); + o.get().literal("type", "Person"); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().literal("name", "Some University"); + o.get().literal("type", "Organization"); + o.get().endRecord(); + + o.get().startRecord("3"); + o.get().literal("name", "Filibandrina"); + o.get().literal("type", "Unknown"); + o.get().endRecord(); + + o.get().startRecord("4"); + o.get().literal("name", "Mary"); + o.get().literal("name", "Max Power"); + o.get().literal("type", "Person1"); + o.get().endRecord(); + + o.get().startRecord("5"); + o.get().literal("name", "Max"); + o.get().literal("type", "Person2"); + o.get().endRecord(); + }); + } + + private void shouldEqualAny(final String path) { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if any_equal('" + path + "', 'University')", + " add_field('type', 'Organization')", + "end" + ), + i -> { + i.startRecord("1"); + i.literal("name", "Mary"); + i.literal("name", "University"); + i.literal("nome", "Max"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("name", "Mary"); + o.get().literal("name", "University"); + o.get().literal("nome", "Max"); + o.get().literal("type", "Organization"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldEqualAny() { + shouldEqualAny("name"); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/115 + public void shouldEqualAnyCharacterClass() { + shouldEqualAny("n[ao]me"); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/115 + public void shouldEqualAnyAlternation() { + shouldEqualAny("name|nome"); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/115 + public void shouldEqualAnyWildcard() { + shouldEqualAny("n?me"); + } + + private void shouldEqualAnyNested(final String path) { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if any_equal('" + path + "', 'University')", + " add_field('data.type', 'Organization')", + "end" + ), + i -> { + i.startRecord("1"); + i.startEntity("data"); + i.literal("name", "Mary"); + i.literal("name", "University"); + i.literal("nome", "Max"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("data"); + o.get().literal("name", "Mary"); + o.get().literal("name", "University"); + o.get().literal("nome", "Max"); + o.get().literal("type", "Organization"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldEqualAnyNested() { + shouldEqualAnyNested("data.name"); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/115 + public void shouldEqualAnyNestedCharacterClass() { + shouldEqualAnyNested("data.n[ao]me"); + } + + @Test + public void shouldEqualAnyNestedAlternation() { + shouldEqualAnyNested("data.name|data.nome"); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/115 + public void shouldEqualAnyNestedWildcard() { + shouldEqualAnyNested("data.n?me"); + } + + private void shouldEqualAnyListBind(final String path) { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "do list(path: 'data', ^var: '$i')", + " if any_equal('" + path + "', 'University')", + " add_field('$i.type', 'Organization')", + " end", + "end" + ), + i -> { + i.startRecord("1"); + i.startEntity("data"); + i.literal("name", "Mary"); + i.literal("name", "University"); + i.literal("nome", "Max"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("data"); + o.get().literal("name", "Mary"); + o.get().literal("name", "University"); + o.get().literal("nome", "Max"); + o.get().literal("type", "Organization"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldEqualAnyListBind() { + shouldEqualAnyListBind("$i.name"); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/115 + public void shouldEqualAnyListBindCharacterClass() { + shouldEqualAnyListBind("$i.n[ao]me"); + } + + @Test + public void shouldEqualAnyListBindAlternation() { + shouldEqualAnyListBind("$i.name|$i.nome"); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/115 + public void shouldEqualAnyListBindWildcard() { + shouldEqualAnyListBind("$i.n?me"); + } + + @Test + public void shouldContainImmediateField() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if exists('name')", + " add_field('type', 'Organization')", + "end" + ), + i -> { + i.startRecord("1"); + i.literal("name", "Mary"); + i.literal("name", "A University"); + i.endRecord(); + + i.startRecord("2"); + i.literal("name", "Mary"); + i.literal("nome", "Max"); + i.endRecord(); + + i.startRecord("3"); + i.literal("nome", "Max"); + i.endRecord(); + + i.startRecord("4"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("name", "Mary"); + o.get().literal("name", "A University"); + o.get().literal("type", "Organization"); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().literal("name", "Mary"); + o.get().literal("nome", "Max"); + o.get().literal("type", "Organization"); + o.get().endRecord(); + + o.get().startRecord("3"); + o.get().literal("nome", "Max"); + o.get().endRecord(); + + o.get().startRecord("4"); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/117 + public void shouldContainNestedField() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if exists('data.name')", + " add_field('type', 'Organization')", + "end" + ), + i -> { + i.startRecord("1"); + i.startEntity("data"); + i.literal("name", "Mary"); + i.literal("name", "A University"); + i.endEntity(); + i.endRecord(); + + i.startRecord("2"); + i.startEntity("data"); + i.literal("name", "Mary"); + i.literal("nome", "Max"); + i.endEntity(); + i.endRecord(); + + i.startRecord("3"); + i.literal("data", "Mary"); + i.literal("name", "Max"); + i.endRecord(); + + i.startRecord("4"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("data"); + o.get().literal("name", "Mary"); + o.get().literal("name", "A University"); + o.get().endEntity(); + o.get().literal("type", "Organization"); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().startEntity("data"); + o.get().literal("name", "Mary"); + o.get().literal("nome", "Max"); + o.get().endEntity(); + o.get().literal("type", "Organization"); + o.get().endRecord(); + + o.get().startRecord("3"); + o.get().literal("data", "Mary"); + o.get().literal("name", "Max"); + o.get().endRecord(); + + o.get().startRecord("4"); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/117 + public void shouldContainNestedArrayField() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if exists('data[].*.name')", + " add_field('type', 'Organization')", + "end" + ), + i -> { + i.startRecord("1"); + i.startEntity("data[]"); + i.startEntity("1"); + i.literal("nome", "Mary"); + i.endEntity(); + i.startEntity("2"); + i.literal("name", "A University"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + + i.startRecord("2"); + i.startEntity("data[]"); + i.literal("1", "Mary"); + i.literal("2", "Max"); + i.endEntity(); + i.endRecord(); + + i.startRecord("3"); + i.literal("data", "Mary"); + i.literal("name", "Max"); + i.endRecord(); + + i.startRecord("4"); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("data[]"); + o.get().startEntity("1"); + o.get().literal("nome", "Mary"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("name", "A University"); + f.apply(2).endEntity(); + o.get().literal("type", "Organization"); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().startEntity("data[]"); + o.get().literal("1", "Mary"); + o.get().literal("2", "Max"); + o.get().endEntity(); + o.get().literal("type", "Organization"); // FIXME: `data[].*.name` must not return `[Mary, Max]` + o.get().endRecord(); + + o.get().startRecord("3"); + o.get().literal("data", "Mary"); + o.get().literal("name", "Max"); + o.get().endRecord(); + + o.get().startRecord("4"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldResolveVariablesInIf() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if any_contain('name', 'Uni$[var]sity')", + " add_field('type', 'Organization')", + "end" + ), + ImmutableMap.of("var", "ver"), + i -> { + i.startRecord("1"); + i.literal("name", "Mary"); + i.literal("name", "A University"); + i.endRecord(); + + i.startRecord("2"); + i.literal("name", "Mary"); + i.literal("name", "Max"); + i.endRecord(); + + i.startRecord("3"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("name", "Mary"); + o.get().literal("name", "A University"); + o.get().literal("type", "Organization"); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().literal("name", "Mary"); + o.get().literal("name", "Max"); + o.get().endRecord(); + + o.get().startRecord("3"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldResolveVariablesInElsIf() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if any_match('name', '.*University.*')", + " add_field('type', 'Organization')", + "elsif any_match('$[var]e', '[^ ]* [^ ]*')", + " add_field('type', 'Person')", + "else", + " add_field('type', 'Unknown')", + "end" + ), + ImmutableMap.of("var", "nam"), + i -> { + i.startRecord("1"); + i.literal("name", "Max Power"); + i.endRecord(); + + i.startRecord("2"); + i.literal("name", "Some University"); + i.endRecord(); + + i.startRecord("3"); + i.literal("name", "Filibandrina"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("name", "Max Power"); + o.get().literal("type", "Person"); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().literal("name", "Some University"); + o.get().literal("type", "Organization"); + o.get().endRecord(); + + o.get().startRecord("3"); + o.get().literal("name", "Filibandrina"); + o.get().literal("type", "Unknown"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldResolveVariablesInUnless() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "unless any_match('name', '.*Uni$[var]sity.*')", + " add_field('type', 'Person')", + "end" + ), + ImmutableMap.of("var", "ver"), + i -> { + i.startRecord("1"); + i.literal("name", "Max"); + i.endRecord(); + + i.startRecord("2"); + i.literal("name", "Some University"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("name", "Max"); + o.get().literal("type", "Person"); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().literal("name", "Some University"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldApplyCustomJavaPredicate() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if org.metafacture.metafix.util.TestPredicate(name, Test)", + " add_field('type', 'TEST')", + "end" + ), + i -> { + i.startRecord("1"); + i.literal("name", "Max"); + i.endRecord(); + + i.startRecord("2"); + i.literal("name", "Test"); + i.endRecord(); + + i.startRecord("3"); + i.literal("test", "Some University"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("name", "Max"); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().literal("name", "Test"); + o.get().literal("type", "TEST"); + o.get().endRecord(); + + o.get().startRecord("3"); + o.get().literal("test", "Some University"); + o.get().literal("type", "TEST"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldIncludeLocationAndTextInProcessExceptionInConditional() { + final String text1 = "elsif exists()"; + final String text2 = "nothing()"; + final String message = "Error while executing Fix expression (at FILE, line 3): " + text1 + " " + text2; + + MetafixTestHelpers.assertProcessException(/*BasicIndexOutOfBoundsException.class*/null, s -> s.replaceAll("file:/.+?\\.fix", "FILE"), message, () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if exists('animal')", + "nothing()", + text1, + text2, + "end" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals"); + i.literal("1", "dog"); + i.literal("2", "cat"); + i.literal("3", "zebra"); + i.endEntity(); + i.endRecord(); + }, + o -> { + } + ) + ); + } + + @Test + public void shouldIncludeLocationAndTextInProcessExceptionInBody() { + final String text = "add_field()"; + final String message = "Error while executing Fix expression (at FILE, line 4): " + text; + + MetafixTestHelpers.assertProcessException(/*BasicIndexOutOfBoundsException.class*/null, s -> s.replaceAll("file:/.+?\\.fix", "FILE"), message, () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if exists('animal')", + "nothing()", + "elsif exists('animals')", + text, + "end" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals"); + i.literal("1", "dog"); + i.literal("2", "cat"); + i.literal("3", "zebra"); + i.endEntity(); + i.endRecord(); + }, + o -> { + } + ) + ); + } + + @Test + public void ifOnNonExistingIndexInArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if exists('animals[].2')", + " copy_field('animals[].2', 'animals2')", + "end" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.literal("1", "dog"); + i.literal("2", "elefant"); + i.endEntity(); + i.endRecord(); + i.startRecord("2"); + i.startEntity("animals[]"); + i.literal("1", "dog"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("animals[]"); + o.get().literal("1", "dog"); + o.get().literal("2", "elefant"); + o.get().endEntity(); + o.get().literal("animals2", "elefant"); + o.get().endRecord(); + o.get().startRecord("2"); + o.get().startEntity("animals[]"); + o.get().literal("1", "dog"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void ifOnNonExistingIndexInRepeatedField() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if exists('animal.2')", + " copy_field('animal.2', 'animal2')", + "end" + ), + i -> { + i.startRecord("1"); + i.literal("animal", "dog"); + i.literal("animal", "elefant"); + i.endRecord(); + i.startRecord("2"); + i.literal("animal", "dog"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("animal", "dog"); + o.get().literal("animal", "elefant"); + o.get().literal("animal2", "elefant"); + o.get().endRecord(); + o.get().startRecord("2"); + o.get().literal("animal", "dog"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldContainStringInString() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if in(foo,bar)", + " add_field(forty_two,ok)", + "end" + ), + i -> { + i.startRecord("1"); + i.literal("foo", "42"); + i.literal("bar", "42"); + i.endRecord(); + i.startRecord("2"); + i.literal("foo", "42"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("foo", "42"); + o.get().literal("bar", "42"); + o.get().literal("forty_two", "ok"); + o.get().endRecord(); + o.get().startRecord("2"); + o.get().literal("foo", "42"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldContainStringInStringAlias() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if is_contained_in(foo,bar)", + " add_field(forty_two,ok)", + "end" + ), + i -> { + i.startRecord("1"); + i.literal("foo", "42"); + i.literal("bar", "42"); + i.endRecord(); + i.startRecord("2"); + i.literal("foo", "42"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("foo", "42"); + o.get().literal("bar", "42"); + o.get().literal("forty_two", "ok"); + o.get().endRecord(); + o.get().startRecord("2"); + o.get().literal("foo", "42"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldContainStringInArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if in(foo,bar)", + " add_field(test,ok)", + "end" + ), + i -> { + i.startRecord("1"); + i.literal("foo", "1"); + i.literal("bar", "1"); + i.literal("bar", "2"); + i.literal("bar", "3"); + i.endRecord(); + i.startRecord("2"); + i.literal("foo", "42"); + i.literal("bar", "1"); + i.literal("bar", "2"); + i.literal("bar", "3"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("foo", "1"); + o.get().literal("bar", "1"); + o.get().literal("bar", "2"); + o.get().literal("bar", "3"); + o.get().literal("test", "ok"); + o.get().endRecord(); + o.get().startRecord("2"); + o.get().literal("foo", "42"); + o.get().literal("bar", "1"); + o.get().literal("bar", "2"); + o.get().literal("bar", "3"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldContainStringInHash() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if in(foo,bar)", + " add_field(test,ok)", + "end" + ), + i -> { + i.startRecord("1"); + i.literal("foo", "name"); + i.startEntity("bar"); + i.literal("name", "Patrick"); + i.endEntity(); + i.endRecord(); + i.startRecord("2"); + i.literal("foo", "name"); + i.startEntity("bar"); + i.startEntity("deep"); + i.literal("name", "Nicolas"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().literal("foo", "name"); + o.get().startEntity("bar"); + o.get().literal("name", "Patrick"); + o.get().endEntity(); + o.get().literal("test", "ok"); + o.get().endRecord(); + o.get().startRecord("2"); + o.get().literal("foo", "name"); + o.get().startEntity("bar"); + o.get().startEntity("deep"); + o.get().literal("name", "Nicolas"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldContainArrayInArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if in(foo,bar)", + " add_field(test,ok)", + "end" + ), + i -> { + i.startRecord("1"); + i.literal("foo", "1"); + i.literal("foo", "2"); + i.literal("bar", "1"); + i.literal("bar", "2"); + i.endRecord(); + i.startRecord("2"); + i.literal("foo", "1"); + i.literal("foo", "2"); + i.literal("bar", "1"); + i.literal("bar", "2"); + i.literal("bar", "3"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("foo", "1"); + o.get().literal("foo", "2"); + o.get().literal("bar", "1"); + o.get().literal("bar", "2"); + o.get().literal("test", "ok"); + o.get().endRecord(); + o.get().startRecord("2"); + o.get().literal("foo", "1"); + o.get().literal("foo", "2"); + o.get().literal("bar", "1"); + o.get().literal("bar", "2"); + o.get().literal("bar", "3"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldContainHashInHash() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if in(foo,bar)", + " add_field(test,ok)", + "end" + ), + i -> { + i.startRecord("1"); + i.startEntity("foo"); + i.literal("a", "b"); + i.endEntity(); + i.startEntity("bar"); + i.literal("a", "b"); + i.endEntity(); + i.endRecord(); + i.startRecord("2"); + i.startEntity("foo"); + i.literal("a", "b"); + i.endEntity(); + i.startEntity("bar"); + i.literal("a", "b"); + i.literal("c", "d"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("foo"); + o.get().literal("a", "b"); + o.get().endEntity(); + o.get().startEntity("bar"); + o.get().literal("a", "b"); + o.get().endEntity(); + o.get().literal("test", "ok"); + o.get().endRecord(); + o.get().startRecord("2"); + o.get().startEntity("foo"); + o.get().literal("a", "b"); + o.get().endEntity(); + o.get().startEntity("bar"); + o.get().literal("a", "b"); + o.get().literal("c", "d"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldNotContainArrayInString() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "unless in(bar,foo)", + " add_field(test,ok)", + "end" + ), + i -> { + i.startRecord("1"); + i.literal("foo", "1"); + i.literal("bar", "1"); + i.literal("bar", "2"); + i.literal("bar", "3"); + i.endRecord(); + i.startRecord("2"); + i.literal("foo", "42"); + i.literal("bar", "1"); + i.literal("bar", "2"); + i.literal("bar", "3"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("foo", "1"); + o.get().literal("bar", "1"); + o.get().literal("bar", "2"); + o.get().literal("bar", "3"); + o.get().literal("test", "ok"); + o.get().endRecord(); + o.get().startRecord("2"); + o.get().literal("foo", "42"); + o.get().literal("bar", "1"); + o.get().literal("bar", "2"); + o.get().literal("bar", "3"); + o.get().literal("test", "ok"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldNotContainHashInString() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "unless in(bar,foo)", + " add_field(test,ok)", + "end" + ), + i -> { + i.startRecord("1"); + i.literal("foo", "name"); + i.startEntity("bar"); + i.literal("name", "Patrick"); + i.endEntity(); + i.endRecord(); + i.startRecord("2"); + i.literal("foo", "name"); + i.startEntity("bar"); + i.startEntity("deep"); + i.literal("name", "Nicolas"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().literal("foo", "name"); + o.get().startEntity("bar"); + o.get().literal("name", "Patrick"); + o.get().endEntity(); + o.get().literal("test", "ok"); + o.get().endRecord(); + o.get().startRecord("2"); + o.get().literal("foo", "name"); + o.get().startEntity("bar"); + o.get().startEntity("deep"); + o.get().literal("name", "Nicolas"); + f.apply(2).endEntity(); + o.get().literal("test", "ok"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldReportArrayAsArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if is_array(foo)", + " add_field(test,ok)", + "end" + ), + i -> { + i.startRecord("1"); + i.literal("foo", "bar"); + i.literal("foo", "1"); + i.endRecord(); + i.startRecord("2"); + i.literal("foo", "1"); + i.endRecord(); + i.startRecord("3"); + i.startEntity("foo"); + i.literal("foo", "bar"); + i.endEntity(); + i.endRecord(); + i.startRecord("4"); + i.startEntity("foo"); + i.literal("foo", "bar"); + i.endEntity(); + i.startEntity("foo"); + i.endEntity(); + i.endRecord(); + i.startRecord("5"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("foo", "bar"); + o.get().literal("foo", "1"); + o.get().literal("test", "ok"); + o.get().endRecord(); + o.get().startRecord("2"); + o.get().literal("foo", "1"); + o.get().endRecord(); + o.get().startRecord("3"); + o.get().startEntity("foo"); + o.get().literal("foo", "bar"); + o.get().endEntity(); + o.get().endRecord(); + o.get().startRecord("4"); + o.get().startEntity("foo"); + o.get().literal("foo", "bar"); + o.get().endEntity(); + o.get().startEntity("foo"); + o.get().endEntity(); + o.get().literal("test", "ok"); + o.get().endRecord(); + o.get().startRecord("5"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldReportArrayEntityAsArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if is_array('foo[]')", + " add_field(test,ok)", + "end" + ), + i -> { + i.startRecord("1"); + i.startEntity("foo[]"); + i.literal("1", "bar"); + i.endEntity(); + i.endRecord(); + i.startRecord("2"); + i.startEntity("foo[]"); + i.endEntity(); + i.endRecord(); + i.startRecord("3"); + i.startEntity("foo"); + i.endEntity(); + i.startEntity("foo"); + i.endEntity(); + i.endRecord(); + i.startRecord("4"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("foo[]"); + o.get().literal("1", "bar"); + o.get().endEntity(); + o.get().literal("test", "ok"); + o.get().endRecord(); + o.get().startRecord("2"); + o.get().startEntity("foo[]"); + o.get().endEntity(); + o.get().literal("test", "ok"); + o.get().endRecord(); + o.get().startRecord("3"); + o.get().startEntity("foo"); + o.get().endEntity(); + o.get().startEntity("foo"); + o.get().endEntity(); + o.get().endRecord(); + o.get().startRecord("4"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldReportEmptyArrayAsArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array(foo)", + "if is_array(foo)", + " add_field(test,ok)", + "end" + ), + i -> { + i.startRecord("1"); + i.literal("foo", "bar"); + i.literal("foo", "1"); + i.endRecord(); + i.startRecord("2"); + i.literal("foo", "1"); + i.endRecord(); + i.startRecord("3"); + i.startEntity("foo"); + i.literal("foo", "bar"); + i.endEntity(); + i.endRecord(); + i.startRecord("4"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("test", "ok"); + o.get().endRecord(); + o.get().startRecord("2"); + o.get().literal("test", "ok"); + o.get().endRecord(); + o.get().startRecord("3"); + o.get().literal("test", "ok"); + o.get().endRecord(); + o.get().startRecord("4"); + o.get().literal("test", "ok"); + o.get().endRecord(); + } + ); + } + + @Test // checkstyle-disable-line JavaNCSS + public void shouldReportEmptyValueAsEmpty() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if is_empty(foo)", + " add_field(test,ok)", + "end" + ), + i -> { + i.startRecord("1"); + i.literal("foo", ""); + i.endRecord(); + i.startRecord("2"); + i.literal("foo", "bar"); + i.endRecord(); + i.startRecord("3"); + i.literal("foo", " "); + i.endRecord(); + i.startRecord("4"); + i.literal("foo", "bar"); + i.literal("foo", ""); + i.endRecord(); + i.startRecord("5"); + i.startEntity("foo"); + i.endEntity(); + i.endRecord(); + i.startRecord("6"); + i.startEntity("foo"); + i.literal("foo", ""); + i.endEntity(); + i.endRecord(); + i.startRecord("7"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("foo", ""); + o.get().literal("test", "ok"); + o.get().endRecord(); + o.get().startRecord("2"); + o.get().literal("foo", "bar"); + o.get().endRecord(); + o.get().startRecord("3"); + o.get().literal("foo", " "); + //o.get().literal("test", "ok"); + o.get().endRecord(); + o.get().startRecord("4"); + o.get().literal("foo", "bar"); + o.get().literal("foo", ""); + o.get().endRecord(); + o.get().startRecord("5"); + o.get().startEntity("foo"); + o.get().endEntity(); + o.get().literal("test", "ok"); + o.get().endRecord(); + o.get().startRecord("6"); + o.get().startEntity("foo"); + o.get().literal("foo", ""); + o.get().endEntity(); + o.get().endRecord(); + o.get().startRecord("7"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldReportEmptyArrayEntityAsEmpty() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if is_empty('foo[]')", + " add_field(test,ok)", + "end" + ), + i -> { + i.startRecord("1"); + i.startEntity("foo[]"); + i.endEntity(); + i.endRecord(); + i.startRecord("2"); + i.startEntity("foo[]"); + i.literal("1", "bar"); + i.endEntity(); + i.endRecord(); + i.startRecord("3"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("foo[]"); + o.get().endEntity(); + o.get().literal("test", "ok"); + o.get().endRecord(); + o.get().startRecord("2"); + o.get().startEntity("foo[]"); + o.get().literal("1", "bar"); + o.get().endEntity(); + o.get().endRecord(); + o.get().startRecord("3"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldReportEmptyArrayAsEmpty() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array(foo)", + "if is_empty(foo)", + " add_field(test,ok)", + "end" + ), + i -> { + i.startRecord("1"); + i.literal("foo", "bar"); + i.literal("foo", "1"); + i.endRecord(); + i.startRecord("2"); + i.literal("foo", "1"); + i.endRecord(); + i.startRecord("3"); + i.startEntity("foo"); + i.literal("foo", "bar"); + i.endEntity(); + i.endRecord(); + i.startRecord("4"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("test", "ok"); + o.get().endRecord(); + o.get().startRecord("2"); + o.get().literal("test", "ok"); + o.get().endRecord(); + o.get().startRecord("3"); + o.get().literal("test", "ok"); + o.get().endRecord(); + o.get().startRecord("4"); + o.get().literal("test", "ok"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldReportFalseStringAsFalse() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if is_false(foo)", + " add_field(test,ok)", + "end" + ), + i -> { + i.startRecord("1"); + i.literal("foo", "false"); + i.endRecord(); + i.startRecord("2"); + i.literal("foo", "true"); + i.endRecord(); + i.startRecord("3"); + i.literal("foo", "bar"); + i.endRecord(); + i.startRecord("4"); + i.startEntity("foo"); + i.literal("foo", "false"); + i.endEntity(); + i.endRecord(); + i.startRecord("5"); + i.literal("foo", "true"); + i.literal("foo", "false"); + i.endRecord(); + i.startRecord("6"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("foo", "false"); + o.get().literal("test", "ok"); + o.get().endRecord(); + o.get().startRecord("2"); + o.get().literal("foo", "true"); + o.get().endRecord(); + o.get().startRecord("3"); + o.get().literal("foo", "bar"); + o.get().endRecord(); + o.get().startRecord("4"); + o.get().startEntity("foo"); + o.get().literal("foo", "false"); + o.get().endEntity(); + o.get().endRecord(); + o.get().startRecord("5"); + o.get().literal("foo", "true"); + o.get().literal("foo", "false"); + o.get().endRecord(); + o.get().startRecord("6"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldReportFalseNumberAsFalse() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if is_false(foo)", + " add_field(test,ok)", + "end" + ), + i -> { + i.startRecord("1"); + i.literal("foo", "0"); + i.endRecord(); + i.startRecord("2"); + i.literal("foo", "1"); + i.endRecord(); + i.startRecord("3"); + i.literal("foo", "42"); + i.endRecord(); + i.startRecord("4"); + i.startEntity("foo"); + i.literal("foo", "0"); + i.endEntity(); + i.endRecord(); + i.startRecord("5"); + i.literal("foo", "1"); + i.literal("foo", "0"); + i.endRecord(); + i.startRecord("6"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("foo", "0"); + o.get().literal("test", "ok"); + o.get().endRecord(); + o.get().startRecord("2"); + o.get().literal("foo", "1"); + o.get().endRecord(); + o.get().startRecord("3"); + o.get().literal("foo", "42"); + o.get().endRecord(); + o.get().startRecord("4"); + o.get().startEntity("foo"); + o.get().literal("foo", "0"); + o.get().endEntity(); + o.get().endRecord(); + o.get().startRecord("5"); + o.get().literal("foo", "1"); + o.get().literal("foo", "0"); + o.get().endRecord(); + o.get().startRecord("6"); + o.get().endRecord(); + } + ); + } + + @Test // checkstyle-disable-line JavaNCSS + public void shouldReportHashAsHash() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if is_hash(foo)", + " add_field(test,ok)", + "end" + ), + i -> { + i.startRecord("1"); + i.literal("foo", "bar"); + i.literal("foo", "1"); + i.endRecord(); + i.startRecord("2"); + i.literal("foo", "1"); + i.endRecord(); + i.startRecord("3"); + i.startEntity("foo"); + i.literal("foo", "bar"); + i.endEntity(); + i.endRecord(); + i.startRecord("4"); + i.startEntity("foo"); + i.endEntity(); + i.endRecord(); + i.startRecord("5"); + i.startEntity("foo"); + i.literal("foo", "bar"); + i.endEntity(); + i.startEntity("foo"); + i.endEntity(); + i.endRecord(); + i.startRecord("6"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("foo", "bar"); + o.get().literal("foo", "1"); + o.get().endRecord(); + o.get().startRecord("2"); + o.get().literal("foo", "1"); + o.get().endRecord(); + o.get().startRecord("3"); + o.get().startEntity("foo"); + o.get().literal("foo", "bar"); + o.get().endEntity(); + o.get().literal("test", "ok"); + o.get().endRecord(); + o.get().startRecord("4"); + o.get().startEntity("foo"); + o.get().endEntity(); + o.get().literal("test", "ok"); + o.get().endRecord(); + o.get().startRecord("5"); + o.get().startEntity("foo"); + o.get().literal("foo", "bar"); + o.get().endEntity(); + o.get().startEntity("foo"); + o.get().endEntity(); + o.get().endRecord(); + o.get().startRecord("6"); + o.get().endRecord(); + } + ); + } + + @Test // checkstyle-disable-line JavaNCSS + public void shouldReportNumberStringAsNumber() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if is_number(foo)", + " add_field(test,ok)", + "end" + ), + i -> { + i.startRecord("1"); + i.literal("foo", "1"); + i.endRecord(); + i.startRecord("2"); + i.literal("foo", "0"); + i.endRecord(); + i.startRecord("3"); + i.literal("foo", "-1"); + i.endRecord(); + i.startRecord("4"); + i.literal("foo", "1.1"); + i.endRecord(); + i.startRecord("5"); + i.literal("foo", "-1.1"); + i.endRecord(); + i.startRecord("6"); + i.literal("foo", "1.1x"); + i.endRecord(); + i.startRecord("7"); + i.startEntity("foo"); + i.literal("foo", "1"); + i.endEntity(); + i.endRecord(); + i.startRecord("8"); + i.literal("foo", "1"); + i.literal("foo", "0"); + i.endRecord(); + i.startRecord("9"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("foo", "1"); + o.get().literal("test", "ok"); + o.get().endRecord(); + o.get().startRecord("2"); + o.get().literal("foo", "0"); + o.get().literal("test", "ok"); + o.get().endRecord(); + o.get().startRecord("3"); + o.get().literal("foo", "-1"); + o.get().literal("test", "ok"); + o.get().endRecord(); + o.get().startRecord("4"); + o.get().literal("foo", "1.1"); + o.get().literal("test", "ok"); + o.get().endRecord(); + o.get().startRecord("5"); + o.get().literal("foo", "-1.1"); + o.get().literal("test", "ok"); + o.get().endRecord(); + o.get().startRecord("6"); + o.get().literal("foo", "1.1x"); + o.get().endRecord(); + o.get().startRecord("7"); + o.get().startEntity("foo"); + o.get().literal("foo", "1"); + o.get().endEntity(); + o.get().endRecord(); + o.get().startRecord("8"); + o.get().literal("foo", "1"); + o.get().literal("foo", "0"); + o.get().endRecord(); + o.get().startRecord("9"); + o.get().endRecord(); + } + ); + } + + @Test // checkstyle-disable-line JavaNCSS + public void shouldReportHashAsObject() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if is_object(foo)", + " add_field(test,ok)", + "end" + ), + i -> { + i.startRecord("1"); + i.literal("foo", "bar"); + i.literal("foo", "1"); + i.endRecord(); + i.startRecord("2"); + i.literal("foo", "1"); + i.endRecord(); + i.startRecord("3"); + i.startEntity("foo"); + i.literal("foo", "bar"); + i.endEntity(); + i.endRecord(); + i.startRecord("4"); + i.startEntity("foo"); + i.endEntity(); + i.endRecord(); + i.startRecord("5"); + i.startEntity("foo"); + i.literal("foo", "bar"); + i.endEntity(); + i.startEntity("foo"); + i.endEntity(); + i.endRecord(); + i.startRecord("6"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("foo", "bar"); + o.get().literal("foo", "1"); + o.get().endRecord(); + o.get().startRecord("2"); + o.get().literal("foo", "1"); + o.get().endRecord(); + o.get().startRecord("3"); + o.get().startEntity("foo"); + o.get().literal("foo", "bar"); + o.get().endEntity(); + o.get().literal("test", "ok"); + o.get().endRecord(); + o.get().startRecord("4"); + o.get().startEntity("foo"); + o.get().endEntity(); + o.get().literal("test", "ok"); + o.get().endRecord(); + o.get().startRecord("5"); + o.get().startEntity("foo"); + o.get().literal("foo", "bar"); + o.get().endEntity(); + o.get().startEntity("foo"); + o.get().endEntity(); + o.get().endRecord(); + o.get().startRecord("6"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldReportEmptyHashAsObject() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_hash(foo)", + "if is_object(foo)", + " add_field(test,ok)", + "end" + ), + i -> { + i.startRecord("1"); + i.literal("foo", "bar"); + i.literal("foo", "1"); + i.endRecord(); + i.startRecord("2"); + i.literal("foo", "1"); + i.endRecord(); + i.startRecord("3"); + i.startEntity("foo"); + i.literal("foo", "bar"); + i.endEntity(); + i.endRecord(); + i.startRecord("4"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("foo"); + o.get().endEntity(); + o.get().literal("test", "ok"); + o.get().endRecord(); + o.get().startRecord("2"); + o.get().startEntity("foo"); + o.get().endEntity(); + o.get().literal("test", "ok"); + o.get().endRecord(); + o.get().startRecord("3"); + o.get().startEntity("foo"); + o.get().endEntity(); + o.get().literal("test", "ok"); + o.get().endRecord(); + o.get().startRecord("4"); + o.get().startEntity("foo"); + o.get().endEntity(); + o.get().literal("test", "ok"); + o.get().endRecord(); + } + ); + } + + @Test // checkstyle-disable-line JavaNCSS + public void shouldReportStringAsString() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if is_string(foo)", + " add_field(test,ok)", + "end" + ), + i -> { + i.startRecord("1"); + i.literal("foo", "1"); + i.endRecord(); + i.startRecord("2"); + i.literal("foo", "0"); + i.endRecord(); + i.startRecord("3"); + i.literal("foo", "-1"); + i.endRecord(); + i.startRecord("4"); + i.literal("foo", "1.1"); + i.endRecord(); + i.startRecord("5"); + i.literal("foo", "-1.1"); + i.endRecord(); + i.startRecord("6"); + i.literal("foo", "1.1x"); + i.endRecord(); + i.startRecord("7"); + i.literal("foo", "bar"); + i.endRecord(); + i.startRecord("8"); + i.literal("foo", ""); + i.endRecord(); + i.startRecord("9"); + i.startEntity("foo"); + i.literal("foo", "bar"); + i.endEntity(); + i.endRecord(); + i.startRecord("10"); + i.literal("foo", "bar"); + i.literal("foo", "0"); + i.endRecord(); + i.startRecord("11"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("foo", "1"); + o.get().endRecord(); + o.get().startRecord("2"); + o.get().literal("foo", "0"); + o.get().endRecord(); + o.get().startRecord("3"); + o.get().literal("foo", "-1"); + o.get().endRecord(); + o.get().startRecord("4"); + o.get().literal("foo", "1.1"); + o.get().endRecord(); + o.get().startRecord("5"); + o.get().literal("foo", "-1.1"); + o.get().endRecord(); + o.get().startRecord("6"); + o.get().literal("foo", "1.1x"); + o.get().literal("test", "ok"); + o.get().endRecord(); + o.get().startRecord("7"); + o.get().literal("foo", "bar"); + o.get().literal("test", "ok"); + o.get().endRecord(); + o.get().startRecord("8"); + o.get().literal("foo", ""); + o.get().literal("test", "ok"); + o.get().endRecord(); + o.get().startRecord("9"); + o.get().startEntity("foo"); + o.get().literal("foo", "bar"); + o.get().endEntity(); + o.get().endRecord(); + o.get().startRecord("10"); + o.get().literal("foo", "bar"); + o.get().literal("foo", "0"); + o.get().endRecord(); + o.get().startRecord("11"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldReportTrueStringAsTrue() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if is_true(foo)", + " add_field(test,ok)", + "end" + ), + i -> { + i.startRecord("1"); + i.literal("foo", "true"); + i.endRecord(); + i.startRecord("2"); + i.literal("foo", "false"); + i.endRecord(); + i.startRecord("3"); + i.literal("foo", "bar"); + i.endRecord(); + i.startRecord("4"); + i.startEntity("foo"); + i.literal("foo", "true"); + i.endEntity(); + i.endRecord(); + i.startRecord("5"); + i.literal("foo", "true"); + i.literal("foo", "false"); + i.endRecord(); + i.startRecord("6"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("foo", "true"); + o.get().literal("test", "ok"); + o.get().endRecord(); + o.get().startRecord("2"); + o.get().literal("foo", "false"); + o.get().endRecord(); + o.get().startRecord("3"); + o.get().literal("foo", "bar"); + o.get().endRecord(); + o.get().startRecord("4"); + o.get().startEntity("foo"); + o.get().literal("foo", "true"); + o.get().endEntity(); + o.get().endRecord(); + o.get().startRecord("5"); + o.get().literal("foo", "true"); + o.get().literal("foo", "false"); + o.get().endRecord(); + o.get().startRecord("6"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldReportTrueNumberAsTrue() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if is_true(foo)", + " add_field(test,ok)", + "end" + ), + i -> { + i.startRecord("1"); + i.literal("foo", "1"); + i.endRecord(); + i.startRecord("2"); + i.literal("foo", "0"); + i.endRecord(); + i.startRecord("3"); + i.literal("foo", "42"); + i.endRecord(); + i.startRecord("4"); + i.startEntity("foo"); + i.literal("foo", "1"); + i.endEntity(); + i.endRecord(); + i.startRecord("5"); + i.literal("foo", "1"); + i.literal("foo", "0"); + i.endRecord(); + i.startRecord("6"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("foo", "1"); + o.get().literal("test", "ok"); + o.get().endRecord(); + o.get().startRecord("2"); + o.get().literal("foo", "0"); + o.get().endRecord(); + o.get().startRecord("3"); + o.get().literal("foo", "42"); + o.get().endRecord(); + o.get().startRecord("4"); + o.get().startEntity("foo"); + o.get().literal("foo", "1"); + o.get().endEntity(); + o.get().endRecord(); + o.get().startRecord("5"); + o.get().literal("foo", "1"); + o.get().literal("foo", "0"); + o.get().endRecord(); + o.get().startRecord("6"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldContainString() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if str_contain('name', 'a$[var]')", + " add_field('type', 'Organization')", + "end" + ), + ImmutableMap.of("var", "me"), + i -> { + i.startRecord("1"); + i.literal("name", "Mame"); + i.literal("name", "A University"); + i.endRecord(); + + i.startRecord("2"); + i.literal("name", "Mame"); + i.literal("name", "Max"); + i.endRecord(); + + i.startRecord("3"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("name", "Mame"); + o.get().literal("name", "A University"); + o.get().literal("type", "Organization"); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().literal("name", "Mame"); + o.get().literal("name", "Max"); + o.get().literal("type", "Organization"); + o.get().endRecord(); + + o.get().startRecord("3"); + o.get().literal("type", "Organization"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldEqualString() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if str_equal('name', 'na$[var]')", + " add_field('type', 'Organization')", + "end" + ), + ImmutableMap.of("var", "me"), + i -> { + i.startRecord("1"); + i.literal("name", "Mame"); + i.literal("name", "A University"); + i.endRecord(); + + i.startRecord("2"); + i.literal("name", "Mame"); + i.literal("name", "Max"); + i.endRecord(); + + i.startRecord("3"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("name", "Mame"); + o.get().literal("name", "A University"); + o.get().literal("type", "Organization"); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().literal("name", "Mame"); + o.get().literal("name", "Max"); + o.get().literal("type", "Organization"); + o.get().endRecord(); + + o.get().startRecord("3"); + o.get().literal("type", "Organization"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldMatchString() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if str_match('name', '.*a$[var]')", + " add_field('type', 'Organization')", + "end" + ), + ImmutableMap.of("var", "me"), + i -> { + i.startRecord("1"); + i.literal("name", "Mame"); + i.literal("name", "A University"); + i.endRecord(); + + i.startRecord("2"); + i.literal("name", "Mame"); + i.literal("name", "Max"); + i.endRecord(); + + i.startRecord("3"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("name", "Mame"); + o.get().literal("name", "A University"); + o.get().literal("type", "Organization"); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().literal("name", "Mame"); + o.get().literal("name", "Max"); + o.get().literal("type", "Organization"); + o.get().endRecord(); + + o.get().startRecord("3"); + o.get().literal("type", "Organization"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldTestMacroVariable() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('type')", + "do put_macro('test')", + " if str_contain('name', 'a$[var]')", + " add_field('type.$append', 'Organization: $[var]')", + " end", + "end", + "call_macro('test', 'var': 'm')", + "call_macro('test', 'var': 'me')", + "call_macro('test', 'var': 'mee')" + ), + i -> { + i.startRecord("1"); + i.literal("name", "Mame"); + i.literal("name", "A University"); + i.endRecord(); + + i.startRecord("2"); + i.literal("name", "Mame"); + i.literal("name", "Max"); + i.endRecord(); + + i.startRecord("3"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("name", "Mame"); + o.get().literal("name", "A University"); + o.get().literal("type", "Organization: m"); + o.get().literal("type", "Organization: me"); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().literal("name", "Mame"); + o.get().literal("name", "Max"); + o.get().literal("type", "Organization: m"); + o.get().literal("type", "Organization: me"); + o.get().endRecord(); + + o.get().startRecord("3"); + o.get().literal("type", "Organization: m"); + o.get().literal("type", "Organization: me"); + o.get().endRecord(); + } + ); + } + +} diff --git a/metafix/src/test/java/org/metafacture/metafix/MetafixLookupTest.java b/metafix/src/test/java/org/metafacture/metafix/MetafixLookupTest.java new file mode 100644 index 000000000..d527fb80c --- /dev/null +++ b/metafix/src/test/java/org/metafacture/metafix/MetafixLookupTest.java @@ -0,0 +1,1478 @@ +/* + * Copyright 2021 Fabian Steeg, hbz + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix; + +import org.metafacture.framework.StreamReceiver; +import org.metafacture.metamorph.api.MorphExecutionException; + +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.client.WireMock; +import com.github.tomakehurst.wiremock.core.WireMockConfiguration; +import com.github.tomakehurst.wiremock.junit.WireMockRule; +import com.github.tomakehurst.wiremock.matching.UrlPattern; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.Arrays; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * Tests Metafix lookup. Following the cheat sheet examples at + * https://github.com/LibreCat/Catmandu/wiki/Fixes-Cheat-Sheet + * + * @author Fabian Steeg + */ +@ExtendWith(MockitoExtension.class) +public class MetafixLookupTest { + private static final String CSV_MAP = "src/test/resources/org/metafacture/metafix/maps/test.csv"; + private static final String CSV_PATH = "/maps/test.csv"; + private static final String CSV_URL = "%s" + CSV_PATH; + private static final String RDF_MAP = "src/test/resources/org/metafacture/metafix/maps/test.ttl"; + private static final String HCRT_RDF_MAP = "src/test/resources/org/metafacture/metafix/maps/hcrt.ttl"; + private static final String RDF_PATH = "/maps/rpb.ttl"; + private static final String RDF_URL = "%s" + RDF_PATH; + private static final String TSV_MAP = "src/test/resources/org/metafacture/metafix/maps/test.tsv"; + private static final String LOOKUP = "lookup('title.*',"; + + private static final WireMockServer WIRE_MOCK_SERVER = new WireMockRule(WireMockConfiguration.wireMockConfig() + .jettyAcceptors(Runtime.getRuntime().availableProcessors()) + .dynamicPort()); + + @Mock + private StreamReceiver streamReceiver; + + public MetafixLookupTest() { + } + + @BeforeAll + public static void setStubForWireMock() { + WIRE_MOCK_SERVER.start(); + + // stubs for RDF + final UrlPattern rdfUrlPattern = WireMock.urlPathEqualTo(RDF_PATH); + final String redirectToUrl = "/redirect" + RDF_PATH; + final UrlPattern urlPatternRedirectToUrl = WireMock.urlPathEqualTo(redirectToUrl); + WIRE_MOCK_SERVER.stubFor(WireMock.get(rdfUrlPattern) + .willReturn(WireMock.temporaryRedirect(redirectToUrl))); + final String rdfResponseBody = loadFile(RDF_PATH); + WIRE_MOCK_SERVER.stubFor(WireMock.get(urlPatternRedirectToUrl) + .willReturn(WireMock.aResponse() + .withHeader("Content-Type", "text/turtle") + .withBody(rdfResponseBody))); + + // stub for CSV + final UrlPattern csvUrlPattern = WireMock.urlPathEqualTo(CSV_PATH); + final String csvResponseBody = loadFile(CSV_PATH); + WIRE_MOCK_SERVER.stubFor(WireMock.get(csvUrlPattern).willReturn(WireMock.aResponse().withBody(csvResponseBody))); + } + + private static String loadFile(final String path) { + return new BufferedReader(new InputStreamReader( + Objects.requireNonNull(MetafixLookupTest.class.getResourceAsStream("." + path)))).lines().collect(Collectors.joining("\n")); + } + + @AfterAll + public static void tearDownWireMock() { + WIRE_MOCK_SERVER.stop(); + } + + @Test + public void inline() { + assertMap( + LOOKUP + " Aloha: Alohaeha, 'Moin': 'Moin zäme', __default: Tach)" + ); + } + + @Test + public void inlineMultilineIndent() { + assertMap( + LOOKUP, + " Aloha: Alohaeha,", + " Moin: 'Moin zäme',", + " __default: Tach)" + ); + } + + @Test + public void inlineDotNotationNested() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "lookup('data.title.*', Aloha: Alohaeha, 'Moin': 'Moin zäme', __default: Tach)" + ), + i -> { + i.startRecord("1"); + i.startEntity("data"); + i.literal("title", "Aloha"); + i.literal("title", "Moin"); + i.literal("title", "Hey"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("data"); + o.get().literal("title", "Alohaeha"); + o.get().literal("title", "Moin zäme"); + o.get().literal("title", "Tach"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldLookupInternalArrayWithAsterisk() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('title', 'Aloha')", + LOOKUP + " Aloha: Alohaeha)" + ), + i -> { + i.startRecord("1"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "Alohaeha"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldLookupDeduplicatedInternalArrayWithAsterisk() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('title', 'Aloha', 'Aloha')", + "uniq('title')", + LOOKUP + " Aloha: Alohaeha)" + ), + i -> { + i.startRecord("1"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "Alohaeha"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldNotLookupCopiedInternalArrayWithAsterisk() { + MetafixTestHelpers.assertExecutionException(IllegalStateException.class, "Expected Array or Hash, got String", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('data', 'Aloha')", + "set_array('title')", + "copy_field('data', 'title')", + LOOKUP + " Aloha: Alohaeha)" + ), + i -> { + i.startRecord("1"); + i.endRecord(); + }, + o -> { + } + ) + ); + } + + @Test + public void shouldLookupCopiedInternalArrayWithAsteriskExplicitAppend() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('data', 'Aloha')", + "set_array('title')", + "copy_field('data', 'title.$append')", + LOOKUP + " Aloha: Alohaeha)" + ), + i -> { + i.startRecord("1"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("data", "Aloha"); + o.get().literal("title", "Alohaeha"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldNotLookupCopiedDeduplicatedInternalArrayWithAsterisk() { + MetafixTestHelpers.assertExecutionException(IllegalStateException.class, "Expected Array or Hash, got String", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('data', 'Aloha', 'Aloha')", + "uniq('data')", + "set_array('title')", + "copy_field('data', 'title')", + LOOKUP + " Aloha: Alohaeha)" + ), + i -> { + i.startRecord("1"); + i.endRecord(); + }, + o -> { + } + ) + ); + } + + @Test + public void shouldLookupCopiedDeduplicatedInternalArrayWithAsteriskExplicitAppend() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('data', 'Aloha', 'Aloha')", + "uniq('data')", + "set_array('title')", + "copy_field('data', 'title.$append')", + LOOKUP + " Aloha: Alohaeha)" + ), + i -> { + i.startRecord("1"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("data", "Aloha"); + o.get().literal("title", "Alohaeha"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldLookupCopiedExternalArrayWithAsteriskExplicitAppend() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('title')", + "copy_field('data', 'title.$append')", + LOOKUP + " Aloha: Alohaeha)" + ), + i -> { + i.startRecord("1"); + i.literal("data", "Aloha"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("data", "Aloha"); + o.get().literal("title", "Alohaeha"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldLookupCopiedDeduplicatedExternalArrayWithAsteriskExplicitAppend() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "uniq('data')", + "set_array('title')", + "copy_field('data', 'title.$append')", + LOOKUP + " Aloha: Alohaeha)" + ), + i -> { + i.startRecord("1"); + i.literal("data", "Aloha"); + i.literal("data", "Aloha"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("data", "Aloha"); + o.get().literal("title", "Alohaeha"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldNotLookupMovedDeduplicatedExternalArrayWithAsterisk() { + MetafixTestHelpers.assertExecutionException(IllegalStateException.class, "Expected Array or Hash, got String", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "uniq('data')", + "set_array('title')", + "move_field('data', 'title')", + LOOKUP + " Aloha: Alohaeha)" + ), + i -> { + i.startRecord("1"); + i.literal("data", "Aloha"); + i.literal("data", "Aloha"); + i.endRecord(); + }, + o -> { + } + ) + ); + } + + @Test + public void shouldLookupMovedDeduplicatedExternalArrayWithAsteriskExplicitAppend() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "uniq('data')", + "set_array('title')", + "move_field('data', 'title.$append')", + LOOKUP + " Aloha: Alohaeha)" + ), + i -> { + i.startRecord("1"); + i.literal("data", "Aloha"); + i.literal("data", "Aloha"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "Alohaeha"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldNotLookupMovedExternalArrayWithAsterisk() { + MetafixTestHelpers.assertExecutionException(IllegalStateException.class, "Expected Array or Hash, got String", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('title')", + "move_field('data', 'title')", + LOOKUP + " Aloha: Alohaeha)" + ), + i -> { + i.startRecord("1"); + i.literal("data", "Aloha"); + i.endRecord(); + }, + o -> { + } + ) + ); + } + + @Test + public void shouldLookupMovedExternalArrayWithAsteriskExplicitAppend() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('title')", + "move_field('data', 'title.$append')", + LOOKUP + " Aloha: Alohaeha)" + ), + i -> { + i.startRecord("1"); + i.literal("data", "Aloha"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "Alohaeha"); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/121 + public void shouldLookupArraySubFieldWithAsterisk() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "lookup('animals[].*.Aanimal', '" + TSV_MAP + "', 'sep_char': '\t')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.startEntity("1"); + i.literal("name", "Jake"); + i.literal("Aanimal", "Aloha"); + i.endEntity(); + i.startEntity("2"); + i.literal("name", "Blacky"); + i.literal("Aanimal", "Hey"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("animals[]"); + o.get().startEntity("1"); + o.get().literal("name", "Jake"); + o.get().literal("Aanimal", "Alohaeha"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("name", "Blacky"); + o.get().literal("Aanimal", "Tach"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void csv() { + assertMap( + LOOKUP + " '" + CSV_MAP + "')" + ); + } + + @Test + public void csvViaUrl() { + final String baseUrl = WIRE_MOCK_SERVER.baseUrl(); + final String mockedCsvUrl = String.format(CSV_URL, baseUrl); + assertMap( + LOOKUP + " '" + mockedCsvUrl + "')" + ); + } + + @Test + public void tsv() { + assertMap( + LOOKUP + " '" + TSV_MAP + "', sep_char:'\t')" + ); + } + + @Test + public void shouldLookupInSeparateInternalMap() { + assertMap( + "put_map('testMap', Aloha: Alohaeha, 'Moin': 'Moin zäme', __default: Tach)", + LOOKUP + " 'testMap')" + ); + } + + @Test + public void shouldUseDefaultOptionFromLookupOption() { + assertMap( + "put_map('testMap', Aloha: Alohaeha, 'Moin': 'Moin zäme')", + LOOKUP + " 'testMap', 'default': 'Tach')" + ); + } + + @Test + public void shouldLookupInSeparateExternalFileMap() { + assertMap( + "put_filemap('" + CSV_MAP + "')", + LOOKUP + " '" + CSV_MAP + "')" + ); + } + + @Test + public void shouldNotLookupInRelativeExternalFileMapFromInlineScript() { + final String mapFile = "../maps/test.csv"; + + MetafixTestHelpers.assertProcessException(IllegalArgumentException.class, "Cannot resolve relative path: " + mapFile, () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + LOOKUP + " '" + mapFile + "')" + ), + i -> { + i.startRecord(""); + i.endRecord(); + }, + o -> { + } + ) + ); + } + + @Test + public void shouldLookupInRelativeExternalFileMapFromExternalScript() { + assertMap( + "src/test/resources/org/metafacture/metafix/fixes/filemap_lookup.fix" + ); + } + + @Test + public void shouldLookupInSeparateExternalFileMapWithName() { + assertMap( + "put_filemap('" + CSV_MAP + "', 'testMap')", + LOOKUP + " 'testMap')" + ); + } + + @Test + public void shouldLookupInSeparateExternalFileMapWithOptions() { + assertMap( + "put_filemap('" + TSV_MAP + "', sep_char: '\t')", + LOOKUP + " '" + TSV_MAP + "')" + ); + } + + @Test + public void shouldLookupInSeparateExternalFileMapWithNameAndOptions() { + assertMap( + "put_filemap('" + TSV_MAP + "', 'testMap', sep_char: '\t')", + LOOKUP + " 'testMap')" + ); + } + + @Test + public void shouldDefineMultipleSeparateMaps() { + assertMap( + "put_map('testMap', Aloha: Alohaeha, 'Moin': 'Moin zäme', __default: Tach)", + "put_map('testMap2', __default: Hi)", + LOOKUP + " 'testMap')" + ); + } + + @Test + public void shouldOverwriteExistingSeparateMap() { + assertMap( + "put_map('testMap', __default: Hi)", + "put_filemap('" + CSV_MAP + "', 'testMap')", + LOOKUP + " 'testMap')" + ); + } + + @Test + public void shouldIgnoreOptionsOnLookupInSeparateInternalMap() { + assertMap( + "put_map('testMap', Aloha: Alohaeha, 'Moin': 'Moin zäme', __default: Tach)", + LOOKUP + " 'testMap', __default: Hi)" + ); + } + + @Test + public void shouldIgnoreOptionsOnLookupInSeparateExternalFileMap() { + assertMap( + "put_filemap('" + CSV_MAP + "')", + LOOKUP + " '" + CSV_MAP + "', sep_char: '\t')" + ); + } + + @Test + public void shouldNotLookupInExternalFileMapWithWrongOptions() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + LOOKUP + " '" + CSV_MAP + "', sep_char:'\t')" + ), + i -> { + i.startRecord("1"); + i.literal("title", "Aloha"); + i.literal("title", "Moin"); + i.literal("title", "Hey"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "Aloha"); + o.get().literal("title", "Moin"); + o.get().literal("title", "Hey"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldDeleteLookupInExternalFileMapWithWrongOptions() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + LOOKUP + " '" + CSV_MAP + "', sep_char: '\t', delete: 'true')" + ), + i -> { + i.startRecord("1"); + i.literal("title", "Aloha"); + i.literal("title", "Moin"); + i.literal("title", "Hey"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/149 + public void shouldKeepOriginalValueIfNotFoundAndNoDefault() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "lookup('title.*', Aloha: 'Alohaeha', 'Moin': 'Moin zäme')" + ), + i -> { + i.startRecord("1"); + i.literal("title", "Aloha"); + i.literal("title", "Moin"); + i.literal("title", "Yo"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "Alohaeha"); + o.get().literal("title", "Moin zäme"); + o.get().literal("title", "Yo"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldUseDefaultValueIfNotFound() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "lookup('title.*', Aloha: 'Alohaeha', 'Moin': 'Moin zäme', __default: Tach)" + ), + i -> { + i.startRecord("1"); + i.literal("title", "Aloha"); + i.literal("title", "Moin"); + i.literal("title", "Yo"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "Alohaeha"); + o.get().literal("title", "Moin zäme"); + o.get().literal("title", "Tach"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldUseDefaultOptionValueIfNotFound() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "put_map('testMap', Aloha: Alohaeha, 'Moin': 'Moin zäme')", + "lookup('title.*', 'testMap', 'default': 'Tach')" + ), + i -> { + i.startRecord("1"); + i.literal("title", "Aloha"); + i.literal("title", "Moin"); + i.literal("title", "Yo"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "Alohaeha"); + o.get().literal("title", "Moin zäme"); + o.get().literal("title", "Tach"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldPreferDefaultOptionValueOverDefaultMapValue() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "put_map('testMap', Aloha: Alohaeha, 'Moin': 'Moin zäme', __default: Tach)", + "lookup('title.*', 'testMap', 'default': 'Hi')" + ), + i -> { + i.startRecord("1"); + i.literal("title", "Aloha"); + i.literal("title", "Moin"); + i.literal("title", "Yo"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "Alohaeha"); + o.get().literal("title", "Moin zäme"); + o.get().literal("title", "Hi"); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/149 + public void shouldDeleteNonFoundLookupOnDemand() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "lookup('title.*', Aloha: Alohaeha, 'Moin': 'Moin zäme', delete: 'true')" + ), + i -> { + i.startRecord("1"); + i.literal("title", "Aloha"); + i.literal("title", "Moin"); + i.literal("title", "Yo"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "Alohaeha"); + o.get().literal("title", "Moin zäme"); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/149 + public void shouldNotDeleteNonFoundLookupExplicitly() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "lookup('title.*', Aloha: Alohaeha, 'Moin': 'Moin zäme', delete: 'false')" + ), + i -> { + i.startRecord("1"); + i.literal("title", "Aloha"); + i.literal("title", "Moin"); + i.literal("title", "Yo"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "Alohaeha"); + o.get().literal("title", "Moin zäme"); + o.get().literal("title", "Yo"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldLookupAfterKeepingUnsuccessfulLookup() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "lookup('title.*', Aloha: Alohaeha, 'Moin': 'Moin zäme')" + ), + i -> { + i.startRecord("1"); + i.literal("title", "Aloha"); + i.literal("title", "Yo"); + i.literal("title", "Moin"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "Alohaeha"); + o.get().literal("title", "Yo"); + o.get().literal("title", "Moin zäme"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldLookupAfterDeletingUnsuccessfulLookup() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "lookup('title.*', Aloha: Alohaeha, 'Moin': 'Moin zäme', delete: 'true')" + ), + i -> { + i.startRecord("1"); + i.literal("title", "Aloha"); + i.literal("title", "Yo"); + i.literal("title", "Moin"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "Alohaeha"); + o.get().literal("title", "Moin zäme"); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/149 + public void shouldDeleteNonFoundLookupOnDemandNonRepeatedField() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "lookup('title', Aloha: Alohaeha, 'Moin': 'Moin zäme', delete: 'true')" + ), + i -> { + i.startRecord("1"); + i.literal("title", "Aloha"); + i.endRecord(); + i.startRecord("2"); + i.literal("title", "Moin"); + i.endRecord(); + i.startRecord("3"); + i.literal("title", "Yo"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "Alohaeha"); + o.get().endRecord(); + o.get().startRecord("2"); + o.get().literal("title", "Moin zäme"); + o.get().endRecord(); + o.get().startRecord("3"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldDeleteNonFoundLookupOnDemandAndVacuum() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "put_map('testMap', Aloha: Alohaeha, 'Moin': 'Moin zäme')", + "lookup('title.*', testMap, delete: 'true')", + "vacuum()" + ), + i -> { + i.startRecord("1"); + i.literal("title", "Aloha"); + i.literal("title", "Moin"); + i.literal("title", "Yo"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "Alohaeha"); + o.get().literal("title", "Moin zäme"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldDeleteNonFoundLookupOnDemandAndMove() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "put_map('testMap', Aloha: Alohaeha, 'Moin': 'Moin zäme')", + "lookup('title.*', testMap, delete: 'true')", + "move_field('title','t')" + ), + i -> { + i.startRecord("1"); + i.literal("title", "Aloha"); + i.literal("title", "Moin"); + i.literal("title", "Yo"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("t", "Alohaeha"); + o.get().literal("t", "Moin zäme"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldIgnoreOptionsOnSubsequentLookupInExternalFileMap() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + LOOKUP + " '" + CSV_MAP + "')", + LOOKUP + " '" + CSV_MAP + "', sep_char: '\t')" + ), + i -> { + i.startRecord("1"); + i.literal("title", "Aloha"); + i.literal("title", "Moin"); + i.literal("title", "Hey"); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + f.apply(3).literal("title", "Tach"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldNotLookupInUnknownInternalMap() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + LOOKUP + " 'testMap')" + ), + i -> { + i.startRecord("1"); + i.literal("title", "Aloha"); + i.literal("title", "Moin"); + i.literal("title", "Hey"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "Aloha"); + o.get().literal("title", "Moin"); + o.get().literal("title", "Hey"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldDeleteLookupInUnknownInternalMap() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + LOOKUP + " 'testMap', delete: 'true')" + ), + i -> { + i.startRecord("1"); + i.literal("title", "Aloha"); + i.literal("title", "Moin"); + i.literal("title", "Hey"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().endRecord(); + } + ); + } + + @Test // checkstyle-disable-line JavaNCSS + public void shouldLookupInNestedArrays() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "put_map('rswk-indicator', s: 'SubjectHeading')", + "lookup('subject[].*.componentList[].*.type[].*', 'rswk-indicator')" + ), + i -> { + i.startRecord("1"); + i.startEntity("subject[]"); + i.startEntity("1"); + i.startEntity("componentList[]"); + i.startEntity("1"); + i.startEntity("type[]"); + i.literal("1", "s"); + i.endEntity(); + i.endEntity(); + i.startEntity("2"); + i.startEntity("type[]"); + i.literal("1", "s"); + i.endEntity(); + i.endEntity(); + i.startEntity("3"); + i.startEntity("type[]"); + i.literal("1", "s"); + i.endEntity(); + i.endEntity(); + i.startEntity("4"); + i.startEntity("type[]"); + i.literal("1", "s"); + i.endEntity(); + i.endEntity(); + i.startEntity("5"); + i.startEntity("type[]"); + i.literal("1", "s"); + i.endEntity(); + i.endEntity(); + i.endEntity(); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("subject[]"); + o.get().startEntity("1"); + o.get().startEntity("componentList[]"); + o.get().startEntity("1"); + o.get().startEntity("type[]"); + o.get().literal("1", "SubjectHeading"); + f.apply(2).endEntity(); + o.get().startEntity("2"); + o.get().startEntity("type[]"); + o.get().literal("1", "SubjectHeading"); + f.apply(2).endEntity(); + o.get().startEntity("3"); + o.get().startEntity("type[]"); + o.get().literal("1", "SubjectHeading"); + f.apply(2).endEntity(); + o.get().startEntity("4"); + o.get().startEntity("type[]"); + o.get().literal("1", "SubjectHeading"); + f.apply(2).endEntity(); + o.get().startEntity("5"); + o.get().startEntity("type[]"); + o.get().literal("1", "SubjectHeading"); + f.apply(5).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldLookupInCopiedNestedArraysCreatedWithAppend() { + shouldLookupInCopiedNestedArraysCreatedWith("$append"); + } + + @Test + public void shouldLookupInCopiedNestedArraysCreatedWithPrepend() { + shouldLookupInCopiedNestedArraysCreatedWith("$prepend"); + } + + private void shouldLookupInCopiedNestedArraysCreatedWith(final String reservedField) { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "put_map('rswk-indicator', s: 'SubjectHeading')", + "set_array('subject[]')", + "set_array('subject[]." + reservedField + ".componentList[]')", + "set_array('subject[].$last.componentList[]." + reservedField + ".type[]')", + "do list(path: 'D', 'var': '$i')", + " copy_field('$i', 'subject[].$last.componentList[].$last.type[]." + reservedField + "')", + "end", + "lookup('subject[].*.componentList[].*.type[].*', 'rswk-indicator')", + "retain('subject[]')" + ), + i -> { + i.startRecord("1"); + i.literal("D", "s"); + i.literal("D", "s"); + i.literal("D", "s"); + i.literal("D", "s"); + i.literal("D", "s"); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("subject[]"); + o.get().startEntity("1"); + o.get().startEntity("componentList[]"); + o.get().startEntity("1"); + o.get().startEntity("type[]"); + o.get().literal("1", "SubjectHeading"); + o.get().literal("2", "SubjectHeading"); + o.get().literal("3", "SubjectHeading"); + o.get().literal("4", "SubjectHeading"); + o.get().literal("5", "SubjectHeading"); + f.apply(5).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldFailLookupInUnknownExternalMap() { + MetafixTestHelpers.assertProcessException(MorphExecutionException.class, s -> s.replaceAll(": /.+?/testMap\\.csv", ": FILE"), "File not found: FILE", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + LOOKUP + " 'testMap.csv')" + ), + i -> { + i.startRecord("1"); + i.literal("title", "Aloha"); + i.literal("title", "Moin"); + i.literal("title", "Hey"); + i.endRecord(); + }, + o -> { + } + ) + ); + } + + private void shouldPrintUnknown(final String args, final String defaultValue, final String expected) { + MetafixTestHelpers.assertStdout(expected, () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + LOOKUP + " Aloha: Alohaeha, 'Moin': 'Moin zäme'" + args + ", print_unknown: 'true')" + ), + i -> { + i.startRecord("rec1"); + i.literal("name", "moe"); + i.literal("title", "Aloha"); + i.literal("title", "Moin"); + i.literal("title", "Hey"); + i.literal("title", "you"); + i.literal("title", "there"); + i.endRecord(); + + i.startRecord("rec2"); + i.literal("name", "joe"); + i.literal("title", "Aloha"); + i.literal("title", "you"); + i.literal("title", "too"); + i.endRecord(); + }, + (o, f) -> { + final boolean delete = "__delete".equals(defaultValue); + + o.get().startRecord("rec1"); + o.get().literal("name", "moe"); + o.get().literal("title", "Alohaeha"); + o.get().literal("title", "Moin zäme"); + + if (defaultValue == null) { + o.get().literal("title", "Hey"); + o.get().literal("title", "you"); + o.get().literal("title", "there"); + } + else if (!delete) { + f.apply(3).literal("title", defaultValue); + } + + o.get().endRecord(); + + o.get().startRecord("rec2"); + o.get().literal("name", "joe"); + o.get().literal("title", "Alohaeha"); + + if (defaultValue == null) { + o.get().literal("title", "you"); + o.get().literal("title", "too"); + } + else if (!delete) { + f.apply(2).literal("title", defaultValue); + } + + o.get().endRecord(); + } + ) + ); + } + + @Test + public void shouldPrintUnknown() { + shouldPrintUnknown("", null, "Hey\nyou\nthere\nyou\ntoo\n"); + } + + @Test + public void shouldPrintUnknownWithDefault() { + shouldPrintUnknown(", __default: Tach", "Tach", "Hey\nyou\nthere\nyou\ntoo\n"); + } + + @Test + public void shouldPrintUnknownWithDelete() { + shouldPrintUnknown(", delete: 'true'", "__delete", "Hey\nyou\nthere\nyou\ntoo\n"); + } + + @Test + public void shouldPrintUnknownWithPrefix() { + shouldPrintUnknown(", prefix: '<%d:%s>'", null, "<1:rec1>Hey\n<1:rec1>you\n<1:rec1>there\n<2:rec2>you\n<2:rec2>too\n"); + } + + @Test + public void shouldPrintUnknownWithPrefixAndIdField() { + shouldPrintUnknown(", prefix: '<%d:%s>', id: 'name'", null, "<1:moe>Hey\n<1:moe>you\n<1:moe>there\n<2:joe>you\n<2:joe>too\n"); + } + + @Test + public void shouldPrintUnknownWithHeader() { + shouldPrintUnknown(", header: '<%d:%s>'", null, "<%d:%s>Hey\nyou\nthere\n<%d:%s>you\ntoo\n"); + } + + @Test + public void shouldPrintUnknownWithFooter() { + shouldPrintUnknown(", footer: '<%d:%s>'", null, "Hey\nyou\nthere<%d:%s>you\ntoo<%d:%s>"); + } + + @Test + public void shouldPrintUnknownWithSeparator() { + shouldPrintUnknown(", separator: '<%d:%s>'", null, "Hey<%d:%s>you<%d:%s>there\nyou<%d:%s>too\n"); + } + + @Test + public void shouldPrintUnknownToFile() throws IOException { + MetafixTestHelpers.assertTempFile("Hey\nyou\nthere\nyou\ntoo\n", p -> shouldPrintUnknown(", destination: '" + p + "'", null, "")); + } + + @Test + public void shouldPrintUnknownToFileWithoutAppend() throws IOException { + MetafixTestHelpers.assertTempFile("you\ntoo\n", p -> shouldPrintUnknown(", destination: '" + p + "', append: 'false'", null, "")); + } + + @Test + public void shouldLookupInSeparateExternalRdfFileMapWithName() { + assertRdfMap( + "put_rdfmap('" + RDF_MAP + "', 'testMapSkosNotation', target: 'skos:notation')", + "lookup('notation', 'testMapSkosNotation')" + ); + } + + @Test + public void shouldLookupInSeparateExternalRdfFileMapWithDifferentTargets() { + assertRdfMapWithDifferentTargets( + "put_rdfmap('" + RDF_MAP + "', 'testRdfMapSkosNotation', target: 'skos:notation')", + "put_rdfmap('" + RDF_MAP + "', 'testRdfMapCreated', target: 'created', __default: '__default')", + "lookup('notation', 'testRdfMapSkosNotation')", + "lookup('created', 'testRdfMapCreated')" + ); + } + + @Test + public void shouldLookupInExternalRdfUseDefinedDefaultValueIfNotFound() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "put_rdfmap('" + RDF_MAP + "', 'rdfmap', target: 'created', __default: '0000-01-01')", + "lookup('created', 'rdfmap')" + ), + i -> { + i.startRecord("1"); + i.literal("created", "https://w3id.org/kim/hochschulfaechersystematik/n4"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("created", "0000-01-01"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldLookupInExternalRdfUseDefaultValueIfNotFound() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "put_rdfmap('" + RDF_MAP + "', 'rdfmap', target: 'created', __default: '__default')", + "lookup('created', 'rdfmap')" + ), + i -> { + i.startRecord("1"); + i.literal("created", "https://w3id.org/kim/hochschulfaechersystematik/n4"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("created", "__default"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldLookupInExternalRdfMapGetObjectOfSubjectWithTargetedPredicate() { + assertRdfMap( + "put_rdfmap('" + RDF_MAP + "', 'rdfmap', target: 'skos:notation')", + "lookup('notation', 'rdfmap')" + ); + } + + @Test + public void shouldExplicitLookupRdfUrlWithRedirection() { + final String baseUrl = WIRE_MOCK_SERVER.baseUrl(); + final String mockedRdfUrl = String.format(RDF_URL, baseUrl); + + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "put_rdfmap('" + mockedRdfUrl + "', 'testMapSkosNotation', target: 'skos:prefLabel')", + "lookup('prefLabel', 'testMapSkosNotation')" + ), + i -> { + i.startRecord("1"); + i.literal("prefLabel", "http://purl.org/lobid/rpb#n882022"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("prefLabel", "Presserecht"); + o.get().endRecord(); + } + ); + } + + @Test // Scenario 1 + public void shouldLookupInExternalRdfMapGetObjectOfSubjectWithTargetedPredicateOfSpecificLanguage() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('prefLabel', 'https://w3id.org/kim/hochschulfaechersystematik/n4')", + "put_rdfmap('" + RDF_MAP + "', 'rdfmap', target: 'skos:prefLabel', select_language: 'de')", + "lookup('prefLabel.*', 'rdfmap')" + ), + i -> { + i.startRecord("1"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("prefLabel", "Mathematik, Naturwissenschaften"); + o.get().endRecord(); + } + ); + } + + @Test // Scenario 2 + public void shouldLookupInExternalRdfMapGetSubjectWithTargetedPredicateOfSpecificLanguage() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('id', 'Mathematics, Natural Sciences')", + "put_rdfmap('" + RDF_MAP + "', 'rdfmap', target: 'skos:prefLabel', select_language: 'en')", + "lookup('id.*', 'rdfmap')" + ), + i -> { + i.startRecord("1"); + i.literal("prefLabel", "Mathematics, Natural Science"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("prefLabel", "Mathematics, Natural Science"); + o.get().literal("id", "https://w3id.org/kim/hochschulfaechersystematik/n4"); + o.get().endRecord(); + } + ); + } + + @Test // Scenario lookupRdfPropertyToProperty + public void shouldLookupInExternalRdfMapGetPropertyOfSpecificLanguageWithTargetedPredicate() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "put_rdfmap('" + HCRT_RDF_MAP + "', 'rdfmap', target: 'skos:prefLabel', select_language: 'en')", + "lookup('a', 'rdfmap')" + ), + i -> { + i.startRecord("1"); + i.literal("a", "Softwareanwendung"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("a", "Software Application"); + o.get().endRecord(); + } + ); + } + + @Test // Scenario lookupRdfPropertyToSubject + public void shouldLookupInExternalRdfMapGetSubjectOfPropertyWithTargetedPredicate() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "put_rdfmap('" + HCRT_RDF_MAP + "', 'rdfmap', target: 'skos:prefLabel')", + "lookup('a', 'rdfmap')" + ), + i -> { + i.startRecord("1"); + i.literal("a", "Softwareanwendung"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("a", "https://w3id.org/kim/hcrt/application"); + o.get().endRecord(); + } + ); + } + + @Test + public void lookupRdfDefinedPropertyToSubjectNonDefault() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "put_rdfmap('" + HCRT_RDF_MAP + "', 'rdfmap', target: 'skos:prefLabel', select_language: 'de')", + "lookup('a', 'rdfmap')" + ), + i -> { + i.startRecord("1"); + i.literal("name", "Jake"); + i.literal("a", "Softwareanwendung"); + i.endRecord(); + i.startRecord("2"); + i.literal("name", "Noone"); + i.literal("a", "cat"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("name", "Jake"); + o.get().literal("a", "https://w3id.org/kim/hcrt/application"); + o.get().endRecord(); + o.get().startRecord("2"); + o.get().literal("name", "Noone"); + o.get().literal("a", "cat"); + o.get().endRecord(); + } + ); + } + + @Test // Scenario 3 + public void shouldLookupInExternalRdfMapGetObjectWithTargetedPredicateOfSpecificLanguageUsingNamespace() { + shouldLookupInExternalRdfMapGetObjectWithTargetedPredicateOfSpecificLanguage("skos:prefLabel"); + } + + @Test // Scenario 3 without namespace + public void shouldLookupInExternalRdfMapGetObjectWithTargetedPredicateOfSpecificLanguageWithoutNamespace() { + shouldLookupInExternalRdfMapGetObjectWithTargetedPredicateOfSpecificLanguage("http://www.w3.org/2004/02/skos/core#prefLabel"); + } + + @Test + public void shouldLookupRdfDefinedPropertyToSubject() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "put_rdfmap('" + HCRT_RDF_MAP + "', 'rdfmap', target: 'skos:prefLabel', select_language: 'de', select: 'subject')", + "lookup('a', 'rdfmap')" + ), + i -> { + i.startRecord("1"); + i.literal("name", "Jake"); + i.literal("a", "Softwareanwendung"); + i.endRecord(); + i.startRecord("2"); + i.literal("name", "Blacky"); + i.literal("a", "Nachschlagewerk"); + i.endRecord(); + i.startRecord("3"); + i.literal("name", "Noone"); + i.literal("a", "cat"); + i.endRecord(); + i.startRecord("4"); + i.literal("name", "Noone_2"); + i.literal("a", "Assessment"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("name", "Jake"); + o.get().literal("a", "https://w3id.org/kim/hcrt/application"); + o.get().endRecord(); + o.get().startRecord("2"); + o.get().literal("name", "Blacky"); + o.get().literal("a", "https://w3id.org/kim/hcrt/index"); + o.get().endRecord(); + o.get().startRecord("3"); + o.get().literal("name", "Noone"); + o.get().literal("a", "cat"); + o.get().endRecord(); + o.get().startRecord("4"); + o.get().literal("name", "Noone_2"); + o.get().literal("a", "Assessment"); + o.get().endRecord(); + } + ); + } + + private void shouldLookupInExternalRdfMapGetObjectWithTargetedPredicateOfSpecificLanguage(final String target) { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('prefLabel', 'Mathematics, Natural Sciences')", + "put_rdfmap('" + RDF_MAP + "', 'rdfmap', target: '" + target + "', select_language: 'de')", + "lookup('prefLabel.*', 'rdfmap')" + ), + i -> { + i.startRecord("1"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("prefLabel", "Mathematik, Naturwissenschaften"); + o.get().endRecord(); + } + ); + } + + private void assertMap(final String... fixDef) { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(fixDef), + i -> { + i.startRecord("1"); + i.literal("title", "Aloha"); + i.literal("title", "Moin"); + i.literal("title", "Hey"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "Alohaeha"); + o.get().literal("title", "Moin zäme"); + o.get().literal("title", "Tach"); + o.get().endRecord(); + } + ); + } + + private void assertRdfMap(final String... fixDef) { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(fixDef), + i -> { + i.startRecord("1"); + i.literal("notation", "https://w3id.org/kim/hochschulfaechersystematik/n4"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("notation", "4"); + o.get().endRecord(); + } + ); + } + + private void assertRdfMapWithDifferentTargets(final String... fixDef) { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(fixDef), + i -> { + i.startRecord("1"); + i.literal("notation", "https://w3id.org/kim/hochschulfaechersystematik/n4"); + i.literal("created", "https://w3id.org/kim/hochschulfaechersystematik/n4"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("notation", "4"); + o.get().literal("created", "__default"); + o.get().endRecord(); + } + ); + } + +} diff --git a/metafix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java b/metafix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java new file mode 100644 index 000000000..c198e8a23 --- /dev/null +++ b/metafix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java @@ -0,0 +1,4239 @@ +/* + * Copyright 2021 Fabian Steeg, hbz + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix; // checkstyle-disable-line JavaNCSS + +import org.metafacture.framework.StreamReceiver; + +import com.google.common.collect.ImmutableMap; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.Arrays; + +/** + * Tests Metafix field level methods. Following the cheat sheet examples at + * https://github.com/LibreCat/Catmandu/wiki/Fixes-Cheat-Sheet + * + * @author Fabian Steeg + */ +@ExtendWith(MockitoExtension.class) // checkstyle-disable-line JavaNCSS +@ExtendWith(MetafixToDo.Extension.class) +public class MetafixMethodTest { + + private static final String YOUTUBE_URL = "https://www.youtube.com/watch?v=daLgsPSvD9A"; + + @Mock + private StreamReceiver streamReceiver; + + public MetafixMethodTest() { + } + + @Test + public void shouldUpcaseString() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "upcase('title')" + ), + i -> { + i.startRecord("1"); + i.literal("title", "marc"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "MARC"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldUpcaseDotNotationNested() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "upcase('data.title')" + ), + i -> { + i.startRecord("1"); + i.startEntity("data"); + i.literal("title", "marc"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("data"); + o.get().literal("title", "MARC"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldDowncaseString() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "downcase('title')" + ), + i -> { + i.startRecord("1"); + i.literal("title", "MARC"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "marc"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldDowncaseStringsInArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "downcase('title.*')" + ), + i -> { + i.startRecord("1"); + i.literal("title", "MARC"); + i.literal("title", "Json"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "marc"); + o.get().literal("title", "json"); + o.get().endRecord(); + } + ); + } + + @Test + @MetafixToDo("NumberFormatException: For input string: '*'") + public void shouldFilterArrayWithAsteriskAtEnd() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "filter('title[].*', 'faust')" + ), + i -> { + i.startRecord("1"); + i.startEntity("title[]"); + i.startEntity("1[]"); + i.literal("1", "faust"); + i.endEntity(); + i.startEntity("2[]"); + i.literal("1", "räuber"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("title[]"); + o.get().startEntity("1[]"); + o.get().literal("1", "faust"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldCapitalizeString() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "capitalize('title')" + ), + i -> { + i.startRecord("1"); + i.literal("title", "marc"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "Marc"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldCapitalizeStringsInArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "capitalize('title.*')" + ), + i -> { + i.startRecord("1"); + i.literal("title", "marc"); + i.literal("title", "json"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "Marc"); + o.get().literal("title", "Json"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldNotCapitalizeArray() { + MetafixTestHelpers.assertExecutionException(IllegalStateException.class, "Expected String, got Array", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "capitalize('title')" + ), + i -> { + i.startRecord("1"); + i.literal("title", "marc"); + i.literal("title", "json"); + i.endRecord(); + }, + o -> { + } + ) + ); + } + + @Test + public void shouldGetSubstringOfString() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "substring('rel', '5', '3')" + ), + i -> { + i.startRecord("1"); + i.literal("rel", "grandson"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("rel", "son"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldGetSubstringOfStringWithoutLength() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "substring('rel', '5')" + ), + i -> { + i.startRecord("1"); + i.literal("rel", "grandson"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("rel", "son"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldGetSubstringOfTruncatedString() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "substring('rel', '5', '6')" + ), + i -> { + i.startRecord("1"); + i.literal("rel", "grandson"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("rel", "son"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldNotGetSubstringOutsideOfString() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "substring('rel', '9', '3')" + ), + i -> { + i.startRecord("1"); + i.literal("rel", "grandson"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("rel", "grandson"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldGetSubstringWithVar() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "substring('rel', '0', '$[length]')" + ), + ImmutableMap.of("length", "5"), + i -> { + i.startRecord("1"); + i.literal("rel", "grandson"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("rel", "grand"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldTrimStringSingle() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "trim('title')" + ), + i -> { + i.startRecord("1"); + i.literal("title", " marc "); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "marc"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldNotTrimRepeatedField() { + MetafixTestHelpers.assertExecutionException(IllegalStateException.class, "Expected String, got Array", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "trim('data.title')" + ), + i -> { + i.startRecord("1"); + i.startEntity("data"); + i.literal("title", " marc "); + i.literal("title", " json "); + i.endEntity(); + i.endRecord(); + }, + o -> { + } + ) + ); + } + + @Test + public void shouldNotTrimIndexedArray() { + MetafixTestHelpers.assertExecutionException(IllegalStateException.class, "Expected String, got Array", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "trim('data.title[]')" + ), + i -> { + i.startRecord("1"); + i.startEntity("data"); + i.startEntity("title[]"); + i.literal("1", " marc "); + i.literal("2", " json "); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + o -> { + } + ) + ); + } + + @Test + public void shouldNotTrimHash() { + MetafixTestHelpers.assertExecutionException(IllegalStateException.class, "Expected String, got Hash", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "trim('data.title')" + ), + i -> { + i.startRecord("1"); + i.startEntity("data"); + i.startEntity("title"); + i.literal("1", " marc "); + i.literal("2", " json "); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + o -> { + } + ) + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/pull/133 + public void shouldNotTrimStringInImplicitArrayOfHashes() { + MetafixTestHelpers.assertExecutionException(IllegalStateException.class, "Expected Hash, got Array", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "trim('data.title')" + ), + i -> { + i.startRecord("1"); + i.startEntity("data"); + i.literal("title", " marc "); + i.endEntity(); + i.startEntity("data"); + i.literal("title", " json "); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("data"); + o.get().literal("title", " marc "); + o.get().endEntity(); + o.get().startEntity("data"); + o.get().literal("title", " json "); + o.get().endEntity(); + o.get().endRecord(); + } + ) + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/pull/133 + public void shouldTrimStringInExplicitArrayOfHashes() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "trim('data.*.title')" + ), + i -> { + i.startRecord("1"); + i.startEntity("data"); + i.literal("title", " marc "); + i.endEntity(); + i.startEntity("data"); + i.literal("title", " json "); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("data"); + o.get().literal("title", "marc"); + o.get().endEntity(); + o.get().startEntity("data"); + o.get().literal("title", "json"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void format() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "format(number,'%-5s: %s')"), // TODO actual number formatting with JSON-equiv record + i -> { + i.startRecord("1"); + i.literal("number", "41"); + i.literal("number", "15"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("number", "41 : 15"); + o.get().endRecord(); + }); + } + + @Test + public void parseText() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "parse_text(date, '(\\\\d{4})-(\\\\d{2})-(\\\\d{2})')"), + i -> { + i.startRecord("1"); + i.literal("date", "2015-03-07"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("date", "2015"); + o.get().literal("date", "03"); + o.get().literal("date", "07"); + o.get().endRecord(); + }); + } + + @Test + public void parseTextNamedGroups() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "parse_text(date, '(?\\\\d{4})-(?\\\\d{2})-(?\\\\d{2})')"), + i -> { + i.startRecord("1"); + i.literal("date", "2015-03-07"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().startEntity("date"); + o.get().literal("year", "2015"); + o.get().literal("month", "03"); + o.get().literal("day", "07"); + o.get().endEntity(); + o.get().endRecord(); + }); + } + + @Test + public void parseTextNestedNamedGroups() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "parse_text(data, '.(?.(?.).(?.).).(?.).')"), + i -> { + i.startRecord("1"); + i.literal("data", "abcdefghi"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().startEntity("data"); + o.get().literal("outer1", "bcdef"); + o.get().literal("inner1", "c"); + o.get().literal("inner2", "e"); + o.get().literal("outer2", "h"); + o.get().endEntity(); + o.get().endRecord(); + }); + } + + @Test + public void parseTextMixedGroups() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "parse_text(data, '(?.)(.)(?.)')" + ), + i -> { + i.startRecord("1"); + i.literal("data", "abc"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("data"); + o.get().literal("a", "a"); + o.get().literal("c", "c"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void parseTextEscapedGroups() { + MetafixTestHelpers.assertProcessException(IllegalArgumentException.class, "No group with name ", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "parse_text(data, '(?.)(.)\\\\(?.\\\\)')" + ), + i -> { + i.startRecord("1"); + i.literal("data", "ab(c)"); + i.endRecord(); + }, + o -> { + } + ) + ); + } + + @Test + public void parseTextQuotedGroups() { + MetafixTestHelpers.assertProcessException(IllegalArgumentException.class, "No group with name ", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "parse_text(data, '(?.)(.)\\\\Q(?.)\\\\E')" + ), + i -> { + i.startRecord("1"); + i.literal("data", "ab(?.)"); + i.endRecord(); + }, + o -> { + } + ) + ); + } + + @Test + public void alternation() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "trim('title-1|title-2')"), + i -> { + i.startRecord("1"); + i.endRecord(); + + i.startRecord("2"); + i.literal("title-1", " marc "); + i.literal("title-2", " json "); + i.endRecord(); + + i.startRecord("3"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().literal("title-1", "marc"); + o.get().literal("title-2", "json"); + o.get().endRecord(); + + o.get().startRecord("3"); + o.get().endRecord(); + }); + } + + @Test + public void wildcardSingleChar() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "trim('title-?')"), + i -> { + i.startRecord("1"); + i.endRecord(); + + i.startRecord("2"); + i.literal("title-1", " marc "); + i.literal("title-2", " json "); + i.endRecord(); + + i.startRecord("3"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().literal("title-1", "marc"); + o.get().literal("title-2", "json"); + o.get().endRecord(); + + o.get().startRecord("3"); + o.get().endRecord(); + }); + } + + @Test + public void wildcardMultiChar() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "trim('title*')"), + i -> { + i.startRecord("1"); + i.endRecord(); + + i.startRecord("2"); + i.literal("title-1", " marc "); + i.literal("title-2", " json "); + i.endRecord(); + + i.startRecord("3"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().literal("title-1", "marc"); + o.get().literal("title-2", "json"); + o.get().endRecord(); + + o.get().startRecord("3"); + o.get().endRecord(); + }); + } + + @Test + public void wildcardNestedPartialSingle() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "trim('work.title-?')"), + i -> { + i.startRecord("1"); + i.endRecord(); + + i.startRecord("2"); + i.startEntity("work"); + i.literal("title-1", " marc "); + i.literal("title-2", " json "); + i.endEntity(); + i.endRecord(); + + i.startRecord("3"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().startEntity("work"); + o.get().literal("title-1", "marc"); + o.get().literal("title-2", "json"); + o.get().endEntity(); + o.get().endRecord(); + + o.get().startRecord("3"); + o.get().endRecord(); + }); + } + + @Test + public void wildcardNestedPartialMulti() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "trim('work.title*')"), + i -> { + i.startRecord("1"); + i.endRecord(); + + i.startRecord("2"); + i.startEntity("work"); + i.literal("title-1", " marc "); + i.literal("title-2", " json "); + i.endEntity(); + i.endRecord(); + + i.startRecord("3"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().startEntity("work"); + o.get().literal("title-1", "marc"); + o.get().literal("title-2", "json"); + o.get().endEntity(); + o.get().endRecord(); + + o.get().startRecord("3"); + o.get().endRecord(); + }); + } + + @Test + @MetafixToDo("See https://github.com/metafacture/metafacture-fix/issues/121") + public void wildcardFullFieldNonIndex() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "trim('*')"), + i -> { + i.startRecord("1"); + i.endRecord(); + + i.startRecord("2"); + i.literal("title-1", " marc "); + i.literal("title-2", " json "); + i.endRecord(); + + i.startRecord("3"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().literal("title-1", "marc"); + o.get().literal("title-2", "json"); + o.get().endRecord(); + + o.get().startRecord("3"); + o.get().endRecord(); + }); + } + + @Test + @MetafixToDo("See https://github.com/metafacture/metafacture-fix/issues/121") + public void wildcardNestedFullFieldNonIndex() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "trim('work.*')"), + i -> { + i.startRecord("1"); + i.endRecord(); + + i.startRecord("2"); + i.startEntity("work"); + i.literal("title-1", " marc "); + i.literal("title-2", " json "); + i.endEntity(); + i.endRecord(); + + i.startRecord("3"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().startEntity("work"); + o.get().literal("title-1", "marc"); + o.get().literal("title-2", "json"); + o.get().endEntity(); + o.get().endRecord(); + + o.get().startRecord("3"); + o.get().endRecord(); + }); + } + + @Test + public void characterClass() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "trim('title-[12]')"), + i -> { + i.startRecord("1"); + i.endRecord(); + + i.startRecord("2"); + i.literal("title-1", " marc "); + i.literal("title-2", " json "); + i.endRecord(); + + i.startRecord("3"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().literal("title-1", "marc"); + o.get().literal("title-2", "json"); + o.get().endRecord(); + + o.get().startRecord("3"); + o.get().endRecord(); + }); + } + + @Test + public void shouldDoNothing() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "nothing()" + ), + i -> { + i.startRecord("1"); + i.literal("title", "marc"); + i.literal("title", "json"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "marc"); + o.get().literal("title", "json"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldAppendValue() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "append(title, ' ?!')" + ), + i -> { + i.startRecord("1"); + i.literal("title", "metafix"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "metafix ?!"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldAppendValueInHash() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "append(animols.name, ' boss')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animols"); + i.literal("name", "bird"); + i.literal("type", "TEST"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("animols"); + o.get().literal("name", "bird boss"); + o.get().literal("type", "TEST"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldAppendValueInArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "append('animals[].1', ' is cool')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.literal("1", "dog"); + i.literal("2", "cat"); + i.literal("3", "zebra"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("animals[]"); + o.get().literal("1", "dog is cool"); + o.get().literal("2", "cat"); + o.get().literal("3", "zebra"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/100 + public void shouldAppendValueInEntireArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "append('animals[].*', ' is cool')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.literal("1", "dog"); + i.literal("2", "cat"); + i.literal("3", "zebra"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("animals[]"); + o.get().literal("1", "dog is cool"); + o.get().literal("2", "cat is cool"); + o.get().literal("3", "zebra is cool"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldNotAppendValueToArray() { + MetafixTestHelpers.assertExecutionException(IllegalStateException.class, "Expected String, got Array", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "append('animals[]', 'another one')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.literal("1", "dog"); + i.literal("2", "cat"); + i.literal("3", "zebra"); + i.endEntity(); + i.endRecord(); + }, + o -> { + } + ) + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/100 + public void shouldNotAppendValueToHash() { + MetafixTestHelpers.assertExecutionException(IllegalStateException.class, "Expected String, got Hash", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "append('animals', ' is cool')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals"); + i.literal("1", "dog"); + i.literal("2", "cat"); + i.literal("3", "zebra"); + i.endEntity(); + i.endRecord(); + }, + o -> { + } + ) + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/123 + public void shouldIncludeLocationAndTextInExecutionException() { + final String text = "append('animals', ' is cool')"; + final String message = "Error while executing Fix expression (at FILE, line 2): " + text; + + MetafixTestHelpers.assertThrows(FixExecutionException.class, s -> s.replaceAll("file:/.+?\\.fix", "FILE"), message, () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "# comment", + text, + "nothing()" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals"); + i.literal("1", "dog"); + i.literal("2", "cat"); + i.literal("3", "zebra"); + i.endEntity(); + i.endRecord(); + }, + o -> { + } + ) + ); + } + + @Test + public void shouldCountNumberOfValuesInArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "count(numbers)" + ), + i -> { + i.startRecord("1"); + i.literal("numbers", "41"); + i.literal("numbers", "42"); + i.literal("numbers", "6"); + i.literal("numbers", "6"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("numbers", "4"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldCountNumberOfValuesInHash() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "count(person)" + ), + i -> { + i.startRecord("1"); + i.startEntity("person"); + i.literal("name", "François"); + i.literal("age", "12"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("person", "2"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldFilterArrayValues() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "filter(animals, '[Cc]at')" + ), + i -> { + i.startRecord("1"); + i.literal("animals", "Lion"); + i.literal("animals", "Cat"); + i.literal("animals", "Tiger"); + i.literal("animals", "Bobcat"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("animals", "Cat"); + o.get().literal("animals", "Bobcat"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldFilterArrayValuesInverted() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "filter(animals, '[Cc]at', invert: 'true')" + ), + i -> { + i.startRecord("1"); + i.literal("animals", "Lion"); + i.literal("animals", "Cat"); + i.literal("animals", "Tiger"); + i.literal("animals", "Bobcat"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("animals", "Lion"); + o.get().literal("animals", "Tiger"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldConvertNullFromJson() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "from_json(test)" + ), + i -> { + i.startRecord("1"); + i.literal("test", "null"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("test", "null"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldConvertStringFromJson() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "from_json(test)" + ), + i -> { + i.startRecord("1"); + i.literal("test", "\"eeny meeny miny moe\""); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("test", "eeny meeny miny moe"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldConvertArrayFromJson() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "from_json(test)" + ), + i -> { + i.startRecord("1"); + i.literal("test", "[\"eeny\",\"meeny\",\"miny\",\"moe\"]"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("test", "eeny"); + o.get().literal("test", "meeny"); + o.get().literal("test", "miny"); + o.get().literal("test", "moe"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldConvertNestedArrayFromJson() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "from_json(test)" + ), + i -> { + i.startRecord("1"); + i.literal("test", "{\"zoo[]\":[{\"animals[]\":[[\"ant\",\"dog\"],\"cat\",[\"fish\",[\"zebra\",\"horse\"],\"hippo\"],\"giraffe\"]}]}"); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("test"); + o.get().startEntity("zoo[]"); + o.get().startEntity("1"); + o.get().startEntity("animals[]"); + o.get().startEntity("1[]"); + o.get().literal("1", "ant"); + o.get().literal("2", "dog"); + o.get().endEntity(); + o.get().literal("2", "cat"); + o.get().startEntity("3[]"); + o.get().literal("1", "fish"); + o.get().startEntity("2[]"); + o.get().literal("1", "zebra"); + o.get().literal("2", "horse"); + o.get().endEntity(); + o.get().literal("3", "hippo"); + o.get().endEntity(); + o.get().literal("4", "giraffe"); + f.apply(4).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldConvertHashFromJson() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "from_json(test)" + ), + i -> { + i.startRecord("1"); + i.literal("test", "{\"a\":[\"eeny\",\"meeny\"],\"b\":\"miny\",\"c\":{\"d\":\"moe\"}}"); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("test"); + o.get().literal("a", "eeny"); + o.get().literal("a", "meeny"); + o.get().literal("b", "miny"); + o.get().startEntity("c"); + o.get().literal("d", "moe"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldConvertHashFromPrettyJson() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "from_json(test)" + ), + i -> { + i.startRecord("1"); + i.literal("test", + "{\n" + + " \"a\" : [ \"eeny\", \"meeny\" ],\n" + + " \"b\" : \"miny\",\n" + + " \"c\" : {\n" + + " \"d\" : \"moe\"\n" + + " }\n" + + "}" + ); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("test"); + o.get().literal("a", "eeny"); + o.get().literal("a", "meeny"); + o.get().literal("b", "miny"); + o.get().startEntity("c"); + o.get().literal("d", "moe"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldNotConvertInvalidObjectFromJson() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "from_json(test)" + ), + i -> { + i.startRecord("1"); + i.literal("test", "eeny meeny miny moe"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("test", "eeny meeny miny moe"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldConvertInvalidObjectFromJsonWithErrorString() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "from_json(test, error_string: 'invalid')" + ), + i -> { + i.startRecord("1"); + i.literal("test", "eeny meeny miny moe"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("test", "invalid"); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/100 + public void shouldFilterArrayObjectValues() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "filter('animals[]', '[Cc]at')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.literal("1", "Lion"); + i.literal("2", "Cat"); + i.literal("3", "Tiger"); + i.literal("4", "Bobcat"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("animals[]"); + o.get().literal("1", "Cat"); + o.get().literal("2", "Bobcat"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldFlattenFlatArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "flatten('flat')" + ), + i -> { + i.startRecord("1"); + i.literal("flat", "1"); + i.literal("flat", "2"); + i.literal("flat", "3"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("flat", "1"); + o.get().literal("flat", "2"); + o.get().literal("flat", "3"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldFlattenFlatHash() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "flatten('flat')" + ), + i -> { + i.startRecord("1"); + i.startEntity("flat"); + i.literal("a", "1"); + i.literal("a", "2"); + i.literal("a", "3"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("flat"); + o.get().literal("a", "1"); + o.get().literal("a", "2"); + o.get().literal("a", "3"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldFlattenFlatString() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "flatten('flat')" + ), + i -> { + i.startRecord("1"); + i.literal("flat", "1"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("flat", "1"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldFlattenNestedArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "flatten('deep[]')" + ), + i -> { + i.startRecord("1"); + i.startEntity("deep[]"); + i.literal("1", "1"); + i.startEntity("2[]"); + i.literal("1", "2"); + i.literal("2", "3"); + i.endEntity(); + i.startEntity("3[]"); + i.startEntity("1[]"); + i.literal("1", "4"); + i.literal("2", "5"); + i.endEntity(); + i.literal("2", "6"); + i.endEntity(); + i.literal("4", "7"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("deep[]"); + o.get().literal("1", "1"); + o.get().literal("2", "2"); + o.get().literal("3", "3"); + o.get().literal("4", "4"); + o.get().literal("5", "5"); + o.get().literal("6", "6"); + o.get().literal("7", "7"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldFlattenNestedArrayWithHashes() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "flatten('deep[]')" + ), + i -> { + i.startRecord("1"); + i.startEntity("deep[]"); + i.literal("1", "1"); + i.startEntity("2"); + i.literal("a", "2"); + i.literal("a", "3"); + i.endEntity(); + i.startEntity("3[]"); + i.startEntity("1"); + i.literal("a", "4"); + i.literal("a", "5"); + i.endEntity(); + i.literal("2", "6"); + i.endEntity(); + i.literal("4", "7"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("deep[]"); + o.get().literal("1", "1"); + o.get().startEntity("2"); + o.get().literal("a", "2"); + o.get().literal("a", "3"); + o.get().endEntity(); + o.get().startEntity("3"); + o.get().literal("a", "4"); + o.get().literal("a", "5"); + o.get().endEntity(); + o.get().literal("4", "6"); + o.get().literal("5", "7"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldGetFirstIndexOfSubstring() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "index(animal, 'n')" + ), + i -> { + i.startRecord("1"); + i.literal("animal", "bunny"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("animal", "2"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldGetIndexOfSubstring() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "index(title, 't')" + ), + i -> { + i.startRecord("1"); + i.literal("title", "metafix"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "2"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldCleanseISBN10() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "isbn(test)" + ), + i -> { + i.startRecord("1"); + i.literal("test", "ISBN: 1-.93.3-988-31-2 EUro 17.70"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("test", "1933988312"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldConvertISBN10ToISBN13() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "isbn(test, to: 'isbn13')" + ), + i -> { + i.startRecord("1"); + i.literal("test", "1933988312"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("test", "9781933988313"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldConvertISBN13ToISBN10() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "isbn(test, to: 'isbn10')" + ), + i -> { + i.startRecord("1"); + i.literal("test", "9781933988313"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("test", "1933988312"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldNotProcessInvalidISBN10() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "isbn(test, verify_check_digit: 'true')" + ), + i -> { + i.startRecord("1"); + i.literal("test", "1933988313"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldSetInvalidISBN10ToErrorString() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "isbn(test, verify_check_digit: 'true', error_string: 'invalid')" + ), + i -> { + i.startRecord("1"); + i.literal("test", "1933988313"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("test", "invalid"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldJoinArrayField() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "join_field(numbers, '/')" + ), + i -> { + i.startRecord("1"); + i.literal("numbers", "6"); + i.literal("numbers", "42"); + i.literal("numbers", "41"); + i.literal("numbers", "6"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("numbers", "6/42/41/6"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldJoinSingleField() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "join_field(number, '/')" + ), + i -> { + i.startRecord("1"); + i.literal("number", "6"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("number", "6"); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/100 + public void shouldJoinArrayObjectField() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "join_field('animals[]', ',')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.literal("1", "dog"); + i.literal("2", "cat"); + i.literal("3", "zebra"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("animals[]"); + o.get().literal("1", "dog,cat,zebra"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldPrependValue() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "prepend(title, 'I love ')" + ), + i -> { + i.startRecord("1"); + i.literal("title", "metafix"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "I love metafix"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldPrependValueInHash() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "prepend(animols.name, 'nice ')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animols"); + i.literal("name", "bird"); + i.literal("type", "TEST"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("animols"); + o.get().literal("name", "nice bird"); + o.get().literal("type", "TEST"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldPrependValueInArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "prepend('animals[].1', 'cool ')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.literal("1", "dog"); + i.literal("2", "cat"); + i.literal("3", "zebra"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("animals[]"); + o.get().literal("1", "cool dog"); + o.get().literal("2", "cat"); + o.get().literal("3", "zebra"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/100 + public void shouldPrependValueInEntireArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "prepend('animals[].*', 'cool ')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.literal("1", "dog"); + i.literal("2", "cat"); + i.literal("3", "zebra"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("animals[]"); + o.get().literal("1", "cool dog"); + o.get().literal("2", "cool cat"); + o.get().literal("3", "cool zebra"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/121 + public void shouldPrependValueInNestedArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "prepend('nestedTest[].*.test[].*', 'Number ')" + ), + i -> { + i.startRecord("1"); + i.startEntity("nestedTest[]"); + i.startEntity("1"); + i.startEntity("test[]"); + i.literal("1", "One"); + i.literal("2", "Two"); + i.literal("3", "Three"); + i.endEntity(); + i.endEntity(); + i.startEntity("2"); + i.startEntity("test[]"); + i.literal("1", "4"); + i.literal("2", "5"); + i.literal("3", "6"); + i.endEntity(); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("nestedTest[]"); + o.get().startEntity("1"); + o.get().startEntity("test[]"); + o.get().literal("1", "Number One"); + o.get().literal("2", "Number Two"); + o.get().literal("3", "Number Three"); + f.apply(2).endEntity(); + o.get().startEntity("2"); + o.get().startEntity("test[]"); + o.get().literal("1", "Number 4"); + o.get().literal("2", "Number 5"); + o.get().literal("3", "Number 6"); + f.apply(3).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/121 + public void shouldPrependValueInArraySubField() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "prepend('coll[].*.a', 'HELLO ')" + ), + i -> { + i.startRecord("1"); + i.startEntity("coll[]"); + i.startEntity("1"); + i.literal("a", "Dog"); + i.literal("b", "Dog"); + i.endEntity(); + i.startEntity("2"); + i.literal("a", "Ape"); + i.literal("b", "Ape"); + i.endEntity(); + i.startEntity("3"); + i.literal("a", "Giraffe"); + i.endEntity(); + i.startEntity("4"); + i.literal("a", "Crocodile"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("coll[]"); + o.get().startEntity("1"); + o.get().literal("a", "HELLO Dog"); + o.get().literal("b", "Dog"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("a", "HELLO Ape"); + o.get().literal("b", "Ape"); + o.get().endEntity(); + o.get().startEntity("3"); + o.get().literal("a", "HELLO Giraffe"); + o.get().endEntity(); + o.get().startEntity("4"); + o.get().literal("a", "HELLO Crocodile"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/100 + public void shouldNotPrependValueToArray() { + MetafixTestHelpers.assertExecutionException(IllegalStateException.class, "Expected String, got Array", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "prepend('animals[]', 'the first one')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.literal("1", "dog"); + i.literal("2", "cat"); + i.literal("3", "zebra"); + i.endEntity(); + i.endRecord(); + }, + o -> { + } + ) + ); + } + + @Test + public void shouldNotPrependValueToArrayWithWildcard() { + MetafixTestHelpers.assertExecutionException(IllegalStateException.class, "Expected String, got Array", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "prepend('animal?[]', 'the first one')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.literal("1", "dog"); + i.literal("2", "cat"); + i.literal("3", "zebra"); + i.endEntity(); + i.endRecord(); + }, + o -> { + } + ) + ); + } + + @Test + public void shouldNotPrependValueToArrayWithWildcardNested() { + MetafixTestHelpers.assertExecutionException(IllegalStateException.class, "Expected String, got Array", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "prepend('some.animal?[]', 'the first one')" + ), + i -> { + i.startRecord("1"); + i.startEntity("some"); + i.startEntity("animals[]"); + i.literal("1", "dog"); + i.literal("2", "cat"); + i.literal("3", "zebra"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + o -> { + } + ) + ); + } + + @Test + public void shouldReplaceAllRegexes() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "replace_all(title, '[aei]', 'X')" + ), + i -> { + i.startRecord("1"); + i.literal("title", "metafix"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "mXtXfXx"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldReplaceAllRegexesWithMatch() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "replace_all(title, '[aei]', 'X$0Y')" + ), + i -> { + i.startRecord("1"); + i.literal("title", "metafix"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "mXeYtXaYfXiYx"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldReplaceAllRegexesWithGroupNumber() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "replace_all(title, '([aei])', 'X$1Y')" + ), + i -> { + i.startRecord("1"); + i.literal("title", "metafix"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "mXeYtXaYfXiYx"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldReplaceAllRegexesWithGroupName() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "replace_all(title, '(?[aei])', 'X${letter}Y')" + ), + i -> { + i.startRecord("1"); + i.literal("title", "metafix"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "mXeYtXaYfXiYx"); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/100 + public void shouldReplaceAllRegexesInArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "replace_all('animals[].*', a, QR)" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.literal("1", "dog"); + i.literal("2", "cat"); + i.literal("3", "zebra"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("animals[]"); + o.get().literal("1", "dog"); + o.get().literal("2", "cQRt"); + o.get().literal("3", "zebrQR"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/121 + public void shouldReplaceAllRegexesInArraySubField() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "replace_all('coll[].*.a', 'o', '__')" + ), + i -> { + i.startRecord("1"); + i.startEntity("coll[]"); + i.startEntity("1"); + i.literal("a", "Dog"); + i.literal("b", "Dog"); + i.endEntity(); + i.startEntity("2"); + i.literal("a", "Ape"); + i.literal("b", "Ape"); + i.endEntity(); + i.startEntity("3"); + i.literal("a", "Giraffe"); + i.endEntity(); + i.startEntity("4"); + i.literal("a", "Crocodile"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("coll[]"); + o.get().startEntity("1"); + o.get().literal("a", "D__g"); + o.get().literal("b", "Dog"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("a", "Ape"); + o.get().literal("b", "Ape"); + o.get().endEntity(); + o.get().startEntity("3"); + o.get().literal("a", "Giraffe"); + o.get().endEntity(); + o.get().startEntity("4"); + o.get().literal("a", "Cr__c__dile"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/255 + public void shouldFailToReplaceAllInRepeatedSubfieldOfObjectWithAsterisk() { + MetafixTestHelpers.assertProcessException(IllegalArgumentException.class, "Can't find: 2 in: null", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "replace_all('predecessor[].*.label', 'Vorg. ---> ', '')" + ), + i -> { + i.startRecord("1"); + i.startEntity("predecessor[]"); + i.startEntity("1"); + i.literal("label", "Früher u.d.T."); + i.literal("label", "Gewaltprävention in Schule und Jugendhilfe"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + o -> { + } + ) + ); + } + + @Test + public void shouldNotInsertOptionalArraySubFieldWithAsteriskInReplaceAll() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "replace_all('coll[].*.b', 'x', 'y')" + ), + i -> { + i.startRecord("1"); + i.startEntity("coll[]"); + i.startEntity("1"); + i.literal("a", "Dog"); + i.endEntity(); + i.startEntity("2"); + i.literal("b", "Ape"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("coll[]"); + o.get().startEntity("1"); + o.get().literal("a", "Dog"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("b", "Ape"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldNotInsertOptionalRepeatedHashSubFieldWithAsteriskInReplaceAll() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "replace_all('coll.*.b', 'x', 'y')" + ), + i -> { + i.startRecord("1"); + i.startEntity("coll"); + i.literal("a", "Dog"); + i.endEntity(); + i.startEntity("coll"); + i.literal("b", "Ape"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("coll"); + o.get().literal("a", "Dog"); + o.get().endEntity(); + o.get().startEntity("coll"); + o.get().literal("b", "Ape"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void replaceAllInOptionalSubfieldInArrayOfObjectsWithAsterisk() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "replace_all('RSWK[].*.subjectGenre', '[.]$', '')" + ), + i -> { + i.startRecord("1"); + i.startEntity("RSWK[]"); + i.startEntity("1"); + i.literal("subjectTopicName", "Nonprofit organizations"); + i.endEntity(); + i.startEntity("2"); + i.literal("subjectTopicName", "Nonprofit organizations"); + i.literal("subjectGenre", "Case studies."); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("RSWK[]"); + o.get().startEntity("1"); + o.get().literal("subjectTopicName", "Nonprofit organizations"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("subjectTopicName", "Nonprofit organizations"); + o.get().literal("subjectGenre", "Case studies"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void inDoBindCopyFieldWithVarInSourceAndTarget() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('RSWK[]')", + "do list(path: '650??', 'var': '$i')", + " copy_field('$i.a', 'RSWK[].$append.subjectTopicName')", + " copy_field('$i.v', 'RSWK[].$last.subjectGenre')", + "end", + "retain('RSWK[]')" + ), + i -> { + i.startRecord("1"); + i.startEntity("650 "); + i.literal("a", "Nonprofit organizations"); + i.literal("x", "Management."); + i.endEntity(); + i.startEntity("650 "); + i.literal("a", "Nonprofit organizations"); + i.literal("x", "Management"); + i.literal("v", "Case studies."); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("RSWK[]"); + o.get().startEntity("1"); + o.get().literal("subjectTopicName", "Nonprofit organizations"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("subjectTopicName", "Nonprofit organizations"); + o.get().literal("subjectGenre", "Case studies."); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + private void replaceAllWithWildcardAfterCopyFieldWithVarInSourceAndTarget(final boolean explicitArray) { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + explicitArray ? "set_array('RSWK[]')" : "", + "do list(path: '650??', 'var': '$i')", + " copy_field('$i.a', 'RSWK[].$append.subjectTopicName')", + " copy_field('$i.v', 'RSWK[].$last.subjectGenre')", + "end", + "replace_all('RSWK[].*.subjectGenre', '[.]$', '')", + "retain('RSWK[]')" + ), + i -> { + i.startRecord("1"); + i.startEntity("650 "); + i.literal("a", "Nonprofit organizations"); + i.literal("x", "Management."); + i.endEntity(); + i.startEntity("650 "); + i.literal("a", "Nonprofit organizations"); + i.literal("x", "Management"); + i.literal("v", "Case studies."); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("RSWK[]"); + o.get().startEntity("1"); + o.get().literal("subjectTopicName", "Nonprofit organizations"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("subjectTopicName", "Nonprofit organizations"); + o.get().literal("subjectGenre", "Case studies"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void replaceAllWithWildcardAfterCopyFieldWithVarInSourceAndTarget() { + replaceAllWithWildcardAfterCopyFieldWithVarInSourceAndTarget(true); + } + + @Test + public void replaceAllWithWildcardAfterCopyFieldWithVarInSourceAndTargetImplicitArray() { + replaceAllWithWildcardAfterCopyFieldWithVarInSourceAndTarget(false); + } + + @Test + public void multipleReplaceAllWithWildcardAfterCopyFieldWithVarInSourceAndTarget() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('RSWK[]')", + "do list(path: '650??', 'var': '$i')", + " copy_field('$i.a', 'RSWK[].$append.subjectTopicName')", + " copy_field('$i.v', 'RSWK[].$last.subjectGenre')", + "end", + "replace_all('RSWK[].*.subjectGenre', '[.]$', '')", + "replace_all('RSWK[].*.subjectGenre', '[.]$', '')", + "retain('RSWK[]')" + ), + i -> { + i.startRecord("1"); + i.startEntity("650 "); + i.literal("a", "Nonprofit organizations"); + i.literal("x", "Management."); + i.endEntity(); + i.startEntity("650 "); + i.literal("a", "Nonprofit organizations"); + i.literal("x", "Management"); + i.literal("v", "Case studies."); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("RSWK[]"); + o.get().startEntity("1"); + o.get().literal("subjectTopicName", "Nonprofit organizations"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("subjectTopicName", "Nonprofit organizations"); + o.get().literal("subjectGenre", "Case studies"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void copyFieldToSubfieldOfArrayOfObjectsWithIndexImplicitAppend() { + MetafixTestHelpers.assertProcessException(IllegalArgumentException.class, "Can't find: 1 in: null", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('test[]')", + "copy_field('key', 'test[].1.field')" + ), + i -> { + i.startRecord("1"); + i.literal("key", "value"); + i.endRecord(); + }, + o -> { + } + ) + ); + } + + @Test + public void copyFieldToSubfieldOfArrayOfObjectsWithExplicitAppend() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('test[]')", + "copy_field('key', 'test[].$append.field')" + ), + i -> { + i.startRecord("1"); + i.literal("key", "value"); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().literal("key", "value"); + o.get().startEntity("test[]"); + o.get().startEntity("1"); + o.get().literal("field", "value"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void copyFieldToSubfieldOfArrayOfStringsWithIndexImplicitAppend() { + MetafixTestHelpers.assertProcessException(IndexOutOfBoundsException.class, "Index 0 out of bounds for length 0", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('test[]')", + "copy_field('key', 'test[].1')" + ), + i -> { + i.startRecord("1"); + i.literal("key", "value"); + i.endRecord(); + }, + o -> { + } + ) + ); + } + + @Test + public void copyFieldToSubfieldOfArrayOfStringsWithExplicitAppend() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('test[]')", + "copy_field('key', 'test[].$append')" + ), + i -> { + i.startRecord("1"); + i.literal("key", "value"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("key", "value"); + o.get().startEntity("test[]"); + o.get().literal("1", "value"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + // We currently fail on unresolved references, see MetafixRecordTests#assertThrowsOnEmptyArray + public void addFieldIntoArrayOfObjectsWithLastWildcardMissingError() { + MetafixTestHelpers.assertProcessException(IllegalArgumentException.class, "Can't find: $last in: null", () -> { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "add_field('animals[].$last.key', 'value')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("animals[]"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + }); + } + + @Test + public void addFieldIntoArrayOfObjectsWithLastWildcardAllEmpty() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if exists('animals[].$last')", + " add_field('animals[].$last.key', 'value')", + "end" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("animals[]"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void addFieldIntoArrayOfObjectsWithLastWildcardLastEmpty() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if exists('animals[].$last')", + " add_field('animals[].$last.key', 'value')", + "end" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.startEntity("1"); + i.literal("name", "Jake"); + i.literal("type", "dog"); + i.endEntity(); + i.startEntity("2"); + i.literal("name", "Blacky"); + i.literal("type", "bird"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + i.startRecord("2"); + i.startEntity("animals[]"); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("animals[]"); + o.get().startEntity("1"); + o.get().literal("name", "Jake"); + o.get().literal("type", "dog"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("name", "Blacky"); + o.get().literal("type", "bird"); + o.get().literal("key", "value"); + f.apply(2).endEntity(); + o.get().endRecord(); + o.get().startRecord("2"); + o.get().startEntity("animals[]"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldReplaceAllRegexesInCopiedArraySubField() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('coll[]')", + "copy_field('a', 'coll[].$append.a')", + "replace_all('coll[].*.a', 'o', '__')" + ), + i -> { + i.startRecord("1"); + i.literal("a", "Dog"); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().literal("a", "Dog"); + o.get().startEntity("coll[]"); + o.get().startEntity("1"); + o.get().literal("a", "D__g"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldReplaceAllRegexesInMovedArraySubField() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('coll[]')", + "move_field('a', 'coll[].$append.a')", + "replace_all('coll[].*.a', 'o', '__')" + ), + i -> { + i.startRecord("1"); + i.literal("a", "Dog"); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("coll[]"); + o.get().startEntity("1"); + o.get().literal("a", "D__g"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldReplaceAllRegexesInCopiedArraySubFieldOriginal() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('coll[]')", + "copy_field('a', 'coll[].$append.a')", + "replace_all('a', 'o', '__')" + ), + i -> { + i.startRecord("1"); + i.literal("a", "Dog"); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().literal("a", "D__g"); + o.get().startEntity("coll[]"); + o.get().startEntity("1"); + o.get().literal("a", "Dog"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldReplaceAllRegexesInListCopiedArraySubField() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('coll[]')", + "do list(path: 'a', 'var': '$i')", + " copy_field('$i', 'coll[].$append.a')", + "end", + "replace_all('coll[].*.a', 'o', '__')" + ), + i -> { + i.startRecord("1"); + i.literal("a", "Dog"); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().literal("a", "Dog"); + o.get().startEntity("coll[]"); + o.get().startEntity("1"); + o.get().literal("a", "D__g"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldCopyBindVarWithDollarAfterLookup() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('@coll[]')", + "do list(path: 'a', 'var': '$i')", + " lookup('$i.name')", + " copy_field('$i.name', '@coll[].$append')", + "end", + "remove_field('a')" + ), + i -> { + i.startRecord("1"); + i.startEntity("a"); + i.literal("name", "Dog"); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("@coll[]"); + o.get().literal("1", "Dog"); + f.apply(1).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldCopyToFieldWithIndexAndReservedFieldName() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('a[].*.test[]', 'test')", + "copy_field('some', 'a[].1.test[].$append')", + "remove_field('some')" + ), + i -> { + i.startRecord("1"); + i.literal("some", "thing"); + i.startEntity("a[]"); + i.startEntity("1"); + i.literal("name", "Dog"); + i.endEntity(); + i.startEntity("2"); + i.literal("name", "Cat"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("a[]"); + o.get().startEntity("1"); + o.get().literal("name", "Dog"); + o.get().startEntity("test[]"); + o.get().literal("1", "test"); + o.get().literal("2", "thing"); + f.apply(2).endEntity(); + o.get().startEntity("2"); + o.get().literal("name", "Cat"); + o.get().startEntity("test[]"); + o.get().literal("1", "test"); + f.apply(3).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldCopyToFieldWithTwoReservedFieldNames() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('a[].*.test[]', 'test')", + "copy_field('some', 'a[].$first.test[].$append')", + "remove_field('some')" + ), + i -> { + i.startRecord("1"); + i.literal("some", "thing"); + i.startEntity("a[]"); + i.startEntity("1"); + i.literal("name", "Dog"); + i.endEntity(); + i.startEntity("2"); + i.literal("name", "Cat"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("a[]"); + o.get().startEntity("1"); + o.get().literal("name", "Dog"); + o.get().startEntity("test[]"); + o.get().literal("1", "test"); + o.get().literal("2", "thing"); + f.apply(2).endEntity(); + o.get().startEntity("2"); + o.get().literal("name", "Cat"); + o.get().startEntity("test[]"); + o.get().literal("1", "test"); + f.apply(3).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldMoveFieldToPathWithIndexAndReservedField() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "move_field('b', 'names[].1.$append')" + ), + i -> { + i.startRecord("1"); + i.literal("b", "b"); + i.startEntity("names[]"); + i.startEntity("1[]"); + i.literal("1", "a"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("names[]"); + o.get().startEntity("1[]"); + o.get().literal("1", "a"); + o.get().literal("2", "b"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldMoveFieldToPathWithTwoReservedFields() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "move_field('b', 'names[].$first.$append')" + ), + i -> { + i.startRecord("1"); + i.literal("b", "b"); + i.startEntity("names[]"); + i.startEntity("1[]"); + i.literal("1", "a"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("names[]"); + o.get().startEntity("1[]"); + o.get().literal("1", "a"); + o.get().literal("2", "b"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/121 + public void shouldReplaceAllRegexesInNestedArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "replace_all('nestedTest[].*.test[].*', 'o', '__')" + ), + i -> { + i.startRecord("1"); + i.startEntity("nestedTest[]"); + i.startEntity("1"); + i.startEntity("test[]"); + i.literal("1", "One"); + i.literal("2", "Two"); + i.literal("3", "Three"); + i.endEntity(); + i.endEntity(); + i.startEntity("2"); + i.startEntity("test[]"); + i.literal("1", "4"); + i.literal("2", "5"); + i.literal("3", "6"); + i.endEntity(); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("nestedTest[]"); + o.get().startEntity("1"); + o.get().startEntity("test[]"); + o.get().literal("1", "One"); + o.get().literal("2", "Tw__"); + o.get().literal("3", "Three"); + f.apply(2).endEntity(); + o.get().startEntity("2"); + o.get().startEntity("test[]"); + o.get().literal("1", "4"); + o.get().literal("2", "5"); + o.get().literal("3", "6"); + f.apply(3).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void copyFieldWithoutReplaceAllString() { + copyFieldWithReplaceAllString(false); + } + + @Test + public void copyFieldWithReplaceAllString() { + copyFieldWithReplaceAllString(true); + } + + private void copyFieldWithReplaceAllString(final boolean replaceAll) { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "do list(path: 'contribution[]', 'var': '$i')", + " copy_field('$i.label', '$i.agent.altLabel')", + "end", + "do list(path: 'subject[]', 'var': '$i')", + " copy_field('$i.label', '$i.altLabel')", + "end", + replaceAll ? "replace_all('contribution[].*.agent.altLabel', 't', '')" : "", + replaceAll ? "replace_all('subject[].*.altLabel', 't', '')" : "" + ), + i -> { + i.startRecord("1"); + i.startEntity("contribution[]"); + i.startEntity("1"); + i.literal("label", "contribution"); + i.endEntity(); + i.endEntity(); + i.startEntity("subject[]"); + i.startEntity("1"); + i.literal("label", "subject"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("contribution[]"); + o.get().startEntity("1"); + o.get().literal("label", "contribution"); + o.get().startEntity("agent"); + o.get().literal("altLabel", replaceAll ? "conribuion" : "contribution"); + f.apply(3).endEntity(); + o.get().startEntity("subject[]"); + o.get().startEntity("1"); + o.get().literal("label", "subject"); + o.get().literal("altLabel", replaceAll ? "subjec" : "subject"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void addFieldWithoutReplaceAllArray() { + addFieldWithReplaceAllArray(false); + } + + @Test + public void addFieldWithReplaceAllArray() { + addFieldWithReplaceAllArray(true); + } + + private void addFieldWithReplaceAllArray(final boolean replaceAll) { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "do list(path: 'contribution[]', 'var': '$i')", + " set_array('$i.agent.altLabel[]')", + " add_field('$i.agent.altLabel[].$append', 'contribution')", + "end", + "do list(path: 'subject[]', 'var': '$i')", + " set_array('$i.altLabel[]')", + " add_field('$i.altLabel[].$append', 'subject')", + "end", + replaceAll ? "replace_all('contribution[].*.agent.altLabel[].*', 't', '')" : "", + replaceAll ? "replace_all('subject[].*.altLabel[].*', 't', '')" : "" + ), + i -> { + i.startRecord("1"); + i.startEntity("contribution[]"); + i.startEntity("1"); + i.literal("label", "contribution"); + i.endEntity(); + i.endEntity(); + i.startEntity("subject[]"); + i.startEntity("1"); + i.literal("label", "subject"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("contribution[]"); + o.get().startEntity("1"); + o.get().literal("label", "contribution"); + o.get().startEntity("agent"); + o.get().startEntity("altLabel[]"); + o.get().literal("1", replaceAll ? "conribuion" : "contribution"); + f.apply(4).endEntity(); + o.get().startEntity("subject[]"); + o.get().startEntity("1"); + o.get().literal("label", "subject"); + o.get().startEntity("altLabel[]"); + o.get().literal("1", replaceAll ? "subjec" : "subject"); + f.apply(3).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void setArrayWithoutReplaceAll() { + setArrayWithReplaceAll(false); + } + + @Test + public void setArrayWithReplaceAll() { + setArrayWithReplaceAll(true); + } + + private void setArrayWithReplaceAll(final boolean replaceAll) { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "do list(path: 'contribution[]', 'var': '$i')", + " set_array('$i.agent.altLabel[]', 'contribution')", + "end", + "do list(path: 'subject[]', 'var': '$i')", + " set_array('$i.altLabel[]', 'subject')", + "end", + replaceAll ? "replace_all('contribution[].*.agent.altLabel[].*', 't', '')" : "", + replaceAll ? "replace_all('subject[].*.altLabel[].*', 't', '')" : "" + ), + i -> { + i.startRecord("1"); + i.startEntity("contribution[]"); + i.startEntity("1"); + i.literal("label", "contribution"); + i.endEntity(); + i.endEntity(); + i.startEntity("subject[]"); + i.startEntity("1"); + i.literal("label", "subject"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("contribution[]"); + o.get().startEntity("1"); + o.get().literal("label", "contribution"); + o.get().startEntity("agent"); + o.get().startEntity("altLabel[]"); + o.get().literal("1", replaceAll ? "conribuion" : "contribution"); + f.apply(4).endEntity(); + o.get().startEntity("subject[]"); + o.get().startEntity("1"); + o.get().literal("label", "subject"); + o.get().startEntity("altLabel[]"); + o.get().literal("1", replaceAll ? "subjec" : "subject"); + f.apply(3).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void copyFieldWithoutReplaceAllArray() { + copyFieldWithReplaceAllArray(false); + } + + @Test + public void copyFieldWithReplaceAllArray() { + copyFieldWithReplaceAllArray(true); + } + + private void copyFieldWithReplaceAllArray(final boolean replaceAll) { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "do list(path: 'contribution[]', 'var': '$i')", + " set_array('$i.agent.altLabel[]')", + " copy_field('$i.label', '$i.agent.altLabel[].$append')", + "end", + "do list(path: 'subject[]', 'var': '$i')", + " set_array('$i.altLabel[]')", + " copy_field('$i.label', '$i.altLabel[].$append')", + "end", + replaceAll ? "replace_all('contribution[].*.agent.altLabel[].*', 't', '')" : "", + replaceAll ? "replace_all('subject[].*.altLabel[].*', 't', '')" : "" + ), + i -> { + i.startRecord("1"); + i.startEntity("contribution[]"); + i.startEntity("1"); + i.literal("label", "contribution"); + i.endEntity(); + i.endEntity(); + i.startEntity("subject[]"); + i.startEntity("1"); + i.literal("label", "subject"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("contribution[]"); + o.get().startEntity("1"); + o.get().literal("label", "contribution"); + o.get().startEntity("agent"); + o.get().startEntity("altLabel[]"); + o.get().literal("1", replaceAll ? "conribuion" : "contribution"); + f.apply(4).endEntity(); + o.get().startEntity("subject[]"); + o.get().startEntity("1"); + o.get().literal("label", "subject"); + o.get().startEntity("altLabel[]"); + o.get().literal("1", replaceAll ? "subjec" : "subject"); + f.apply(3).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void pasteWithoutReplaceAll() { + pasteWithReplaceAll(false); + } + + @Test + public void pasteWithReplaceAll() { + pasteWithReplaceAll(true); + } + + private void pasteWithReplaceAll(final boolean replaceAll) { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "do list(path: 'contribution[]', 'var': '$i')", + " set_array('$i.agent.altLabel[]')", + " paste('$i.agent.altLabel[].$append', '$i.label', '~!')", + "end", + "do list(path: 'subject[]', 'var': '$i')", + " set_array('$i.altLabel[]')", + " paste('$i.altLabel[].$append', '$i.label', '~!')", + "end", + replaceAll ? "replace_all('contribution[].*.agent.altLabel[].*', ' !', '')" : "", + replaceAll ? "replace_all('subject[].*.altLabel[].*', ' !', '')" : "" + ), + i -> { + i.startRecord("1"); + i.startEntity("contribution[]"); + i.startEntity("1"); + i.literal("label", "contribution"); + i.endEntity(); + i.endEntity(); + i.startEntity("subject[]"); + i.startEntity("1"); + i.literal("label", "subject"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("contribution[]"); + o.get().startEntity("1"); + o.get().literal("label", "contribution"); + o.get().startEntity("agent"); + o.get().startEntity("altLabel[]"); + o.get().literal("1", replaceAll ? "contribution" : "contribution !"); + f.apply(4).endEntity(); + o.get().startEntity("subject[]"); + o.get().startEntity("1"); + o.get().literal("label", "subject"); + o.get().startEntity("altLabel[]"); + o.get().literal("1", replaceAll ? "subject" : "subject !"); + f.apply(3).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldReplaceAllRegexesInArrayByIndex() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "replace_all('names.2', 'a', 'X')" + ), + i -> { + i.startRecord("1"); + i.literal("names", "Max"); + i.literal("names", "Jake"); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().literal("names", "Max"); + o.get().literal("names", "JXke"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldReplaceAllRegexesInArrayByArrayWildcard() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "replace_all('names.$last', 'a', 'X')" + ), + i -> { + i.startRecord("1"); + i.literal("names", "Max"); + i.literal("names", "Jake"); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().literal("names", "Max"); + o.get().literal("names", "JXke"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldReplaceAllRegexesInArraySubFieldByIndex() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "replace_all('names[].2.name', 'a', 'X')" + ), + i -> { + i.startRecord("1"); + i.startEntity("names[]"); + i.startEntity("1"); + i.literal("name", "Max"); + i.endEntity(); + i.startEntity("2"); + i.literal("name", "Jake"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("names[]"); + o.get().startEntity("1"); + o.get().literal("name", "Max"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("name", "JXke"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldReplaceAllRegexesInArraySubFieldByArrayWildcard() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "replace_all('names[].$last.name', 'a', 'X')" + ), + i -> { + i.startRecord("1"); + i.startEntity("names[]"); + i.startEntity("1"); + i.literal("name", "Max"); + i.endEntity(); + i.startEntity("2"); + i.literal("name", "Jake"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("names[]"); + o.get().startEntity("1"); + o.get().literal("name", "Max"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("name", "JXke"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldReverseString() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "reverse(title)" + ), + i -> { + i.startRecord("1"); + i.literal("title", "metafix"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "xifatem"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldReverseArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "reverse(title)" + ), + i -> { + i.startRecord("1"); + i.literal("title", "marc"); + i.literal("title", "json"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "json"); + o.get().literal("title", "marc"); + o.get().endRecord(); + } + ); + } + + @Test + @MetafixToDo("See https://github.com/metafacture/metafacture-fix/issues/121") + public void shouldReverseArrayOfStringsWithAsterisk() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "reverse('test[].*')" + ), + i -> { + i.startRecord("1"); + i.startEntity("test[]"); + i.literal("1", "One"); + i.literal("2", "Two"); + i.literal("3", "Three"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("test[]"); + o.get().literal("1", "enO"); + o.get().literal("2", "owT"); + o.get().literal("3", "eerhT"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + @MetafixToDo("java.lang.ArrayIndexOutOfBoundsException: 0; see https://github.com/metafacture/metafacture-fix/issues/121") + public void shouldReverseArrayOfHashesWithAsterisk() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "reverse('ANIMALS[].*')" + ), + i -> { + i.startRecord("1"); + i.startEntity("ANIMALS[]"); + i.startEntity("1"); + i.literal("Aanimal", "dog"); + i.literal("name", "Jake"); + i.endEntity(); + i.startEntity("2"); + i.literal("Aanimal", "parrot"); + i.literal("name", "Blacky"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("ANIMALS[]"); + o.get().startEntity("1"); + o.get().literal("Aanimal", "parrot"); + o.get().literal("name", "Blacky"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("Aanimal", "dog"); + o.get().literal("name", "Jake"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldSortField() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "sort_field(numbers)" + ), + i -> { + i.startRecord("1"); + i.literal("numbers", "6"); + i.literal("numbers", "42"); + i.literal("numbers", "41"); + i.literal("numbers", "6"); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().literal("numbers", "41"); + o.get().literal("numbers", "42"); + f.apply(2).literal("numbers", "6"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldSortFieldNumerically() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "sort_field(numbers, numeric: 'true')" + ), + i -> { + i.startRecord("1"); + i.literal("numbers", "6"); + i.literal("numbers", "42"); + i.literal("numbers", "41"); + i.literal("numbers", "6"); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + f.apply(2).literal("numbers", "6"); + o.get().literal("numbers", "41"); + o.get().literal("numbers", "42"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldFailToSortNumericallyWithInvalidNumber() { + MetafixTestHelpers.assertExecutionException(NumberFormatException.class, "For input string: \"x\"", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "sort_field(numbers, numeric: 'true')" + ), + i -> { + i.startRecord("1"); + i.literal("numbers", "6"); + i.literal("numbers", "42"); + i.literal("numbers", "x"); + i.literal("numbers", "6"); + i.endRecord(); + }, + o -> { + } + ) + ); + } + + @Test + public void shouldSortFieldInReverse() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "sort_field(numbers, reverse: 'true')" + ), + i -> { + i.startRecord("1"); + i.literal("numbers", "6"); + i.literal("numbers", "42"); + i.literal("numbers", "41"); + i.literal("numbers", "6"); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + f.apply(2).literal("numbers", "6"); + o.get().literal("numbers", "42"); + o.get().literal("numbers", "41"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldSortFieldNumericallyInReverse() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "sort_field(numbers, numeric: 'true', reverse: 'true')" + ), + i -> { + i.startRecord("1"); + i.literal("numbers", "6"); + i.literal("numbers", "42"); + i.literal("numbers", "41"); + i.literal("numbers", "6"); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().literal("numbers", "42"); + o.get().literal("numbers", "41"); + f.apply(2).literal("numbers", "6"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldSortFieldAndRemoveDuplicates() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "sort_field(numbers, uniq: 'true')" + ), + i -> { + i.startRecord("1"); + i.literal("numbers", "6"); + i.literal("numbers", "42"); + i.literal("numbers", "41"); + i.literal("numbers", "6"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("numbers", "41"); + o.get().literal("numbers", "42"); + o.get().literal("numbers", "6"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldSortArrayFieldWithAsterisk() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "sort_field('OTHERS[].*.dnimals[]')" + ), + i -> { + i.startRecord("1"); + i.startEntity("OTHERS[]"); + i.startEntity("1"); + i.literal("name", "Jake"); + i.startEntity("dnimals[]"); + i.literal("1", "dog"); + i.literal("2", "zebra"); + i.literal("3", "cat"); + i.endEntity(); + i.startEntity("dumbers[]"); + i.literal("1", "7"); + i.literal("2", "2"); + i.literal("3", "1"); + i.literal("4", "10"); + i.endEntity(); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("OTHERS[]"); + o.get().startEntity("1"); + o.get().literal("name", "Jake"); + o.get().startEntity("dnimals[]"); + o.get().literal("1", "cat"); + o.get().literal("2", "dog"); + o.get().literal("3", "zebra"); + o.get().endEntity(); + o.get().startEntity("dumbers[]"); + o.get().literal("1", "7"); + o.get().literal("2", "2"); + o.get().literal("3", "1"); + o.get().literal("4", "10"); + f.apply(3).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldSplitStringField() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "split_field(date, '-')" + ), + i -> { + i.startRecord("1"); + i.literal("date", "1918-17-16"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("date", "1918"); + o.get().literal("date", "17"); + o.get().literal("date", "16"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldSplitArrayField() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "split_field(date, '-')" + ), + i -> { + i.startRecord("1"); + i.literal("date", "1918-17-16"); + i.literal("date", "2021-22-23"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("date", "1918"); + o.get().literal("date", "17"); + o.get().literal("date", "16"); + o.get().literal("date", "2021"); + o.get().literal("date", "22"); + o.get().literal("date", "23"); + o.get().endRecord(); + } + ); + } + + @Test + @MetafixToDo("Arrays in arrays need to be preserved. See disabled isArray in FixPath#appendIn.") + public void moveToNestedArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "move_field('date[]', 'd[]')" + ), + i -> { + i.startRecord("1"); + i.startEntity("date[]"); + i.startEntity("1[]"); + i.literal("1", "1918"); + i.literal("2", "17"); + i.literal("3", "16"); + i.endEntity(); + i.startEntity("2[]"); + i.literal("1", "2021"); + i.literal("2", "22"); + i.literal("3", "23"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("d[]"); + o.get().startEntity("1[]"); + o.get().literal("1", "1918"); + o.get().literal("2", "17"); + o.get().literal("3", "16"); + o.get().endEntity(); + o.get().startEntity("2[]"); + o.get().literal("1", "2021"); + o.get().literal("2", "22"); + o.get().literal("3", "23"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldSplitMarkedArrayFieldIntoArrayOfArrays() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "split_field('date[]', '-')" + ), + i -> { + i.startRecord("1"); + i.literal("date[]", "1918-17-16"); + i.literal("date[]", "2021-22-23"); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("date[]"); + o.get().startEntity("1[]"); + o.get().literal("1", "1918"); + o.get().literal("2", "17"); + o.get().literal("3", "16"); + o.get().endEntity(); + o.get().startEntity("2[]"); + o.get().literal("1", "2021"); + o.get().literal("2", "22"); + o.get().literal("3", "23"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldSplitHashField() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "split_field(date, '-')" + ), + i -> { + i.startRecord("1"); + i.startEntity("date"); + i.literal("a", "1918-17-16"); + i.literal("b", "2021-22-23"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("date"); + o.get().literal("a", "1918"); + o.get().literal("a", "17"); + o.get().literal("a", "16"); + o.get().literal("b", "2021"); + o.get().literal("b", "22"); + o.get().literal("b", "23"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldSplitNestedField() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "split_field('others[].*.tools', '--')" + ), + i -> { + i.startRecord("1"); + i.startEntity("others[]"); + i.startEntity("1"); + i.literal("tools", "hammer--saw--bow"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("others[]"); + o.get().startEntity("1"); + o.get().literal("tools", "hammer"); + o.get().literal("tools", "saw"); + o.get().literal("tools", "bow"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldSumNumbers() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "sum(numbers)" + ), + i -> { + i.startRecord("1"); + i.literal("numbers", "41"); + i.literal("numbers", "42"); + i.literal("numbers", "6"); + i.literal("numbers", "6"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("numbers", "95"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldSumArrayFieldWithAsterisk() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "sum('OTHERS[].*.dumbers[]')" + ), + i -> { + i.startRecord("1"); + i.startEntity("OTHERS[]"); + i.startEntity("1"); + i.literal("name", "Jake"); + i.startEntity("dnimals[]"); + i.literal("1", "dog"); + i.literal("2", "zebra"); + i.literal("3", "cat"); + i.endEntity(); + i.startEntity("dumbers[]"); + i.literal("1", "7"); + i.literal("2", "2"); + i.literal("3", "1"); + i.literal("4", "10"); + i.endEntity(); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("OTHERS[]"); + o.get().startEntity("1"); + o.get().literal("name", "Jake"); + o.get().startEntity("dnimals[]"); + o.get().literal("1", "dog"); + o.get().literal("2", "zebra"); + o.get().literal("3", "cat"); + o.get().endEntity(); + o.get().startEntity("dumbers[]"); + o.get().literal("1", "20"); + f.apply(3).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldConvertStringToJson() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "to_json(test)" + ), + i -> { + i.startRecord("1"); + i.literal("test", "eeny meeny miny moe"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("test", "\"eeny meeny miny moe\""); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldConvertArrayToJson() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "to_json(test)" + ), + i -> { + i.startRecord("1"); + i.literal("test", "eeny"); + i.literal("test", "meeny"); + i.literal("test", "miny"); + i.literal("test", "moe"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("test", "[\"eeny\",\"meeny\",\"miny\",\"moe\"]"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldConvertNestedArrayToJson() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "to_json(test)" + ), + i -> { + i.startRecord("1"); + i.startEntity("test"); + i.startEntity("zoo[]"); + i.startEntity("1"); + i.startEntity("animals[]"); + i.startEntity("1[]"); + i.literal("1", "ant"); + i.literal("2", "dog"); + i.endEntity(); + i.literal("2", "cat"); + i.startEntity("3[]"); + i.literal("1", "fish"); + i.startEntity("2[]"); + i.literal("1", "zebra"); + i.literal("2", "horse"); + i.endEntity(); + i.literal("3", "hippo"); + i.endEntity(); + i.literal("4", "giraffe"); + i.endEntity(); + i.endEntity(); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("test", "{\"zoo[]\":[{\"animals[]\":[[\"ant\",\"dog\"],\"cat\",[\"fish\",[\"zebra\",\"horse\"],\"hippo\"],\"giraffe\"]}]}"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldConvertHashToJson() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "to_json(test)" + ), + i -> { + i.startRecord("1"); + i.startEntity("test"); + i.literal("a", "eeny"); + i.literal("a", "meeny"); + i.literal("b", "miny"); + i.startEntity("c"); + i.literal("d", "moe"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("test", "{\"a\":[\"eeny\",\"meeny\"],\"b\":\"miny\",\"c\":{\"d\":\"moe\"}}"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldConvertHashToPrettyJson() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "to_json(test, pretty: 'true')" + ), + i -> { + i.startRecord("1"); + i.startEntity("test"); + i.literal("a", "eeny"); + i.literal("a", "meeny"); + i.literal("b", "miny"); + i.startEntity("c"); + i.literal("d", "moe"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("test", + "{\n" + + " \"a\" : [ \"eeny\", \"meeny\" ],\n" + + " \"b\" : \"miny\",\n" + + " \"c\" : {\n" + + " \"d\" : \"moe\"\n" + + " }\n" + + "}" + ); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldRemoveDuplicateStrings() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "uniq(numbers)" + ), + i -> { + i.startRecord("1"); + i.literal("numbers", "41"); + i.literal("numbers", "42"); + i.literal("numbers", "6"); + i.literal("numbers", "6"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("numbers", "41"); + o.get().literal("numbers", "42"); + o.get().literal("numbers", "6"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldRemoveDuplicateArraysAtDifferentPath() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "uniq('arrays[]')" + ), + i -> { + i.startRecord("1"); + i.startEntity("arrays[]"); + i.startEntity("1"); + i.literal("number", "41"); + i.literal("number", "23"); + i.endEntity(); + i.startEntity("2"); + i.literal("number", "42"); + i.literal("number", "23"); + i.endEntity(); + i.startEntity("3"); + i.literal("number", "6"); + i.literal("number", "23"); + i.endEntity(); + i.startEntity("4"); + i.literal("number", "6"); + i.literal("number", "23"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("arrays[]"); + o.get().startEntity("1"); + o.get().literal("number", "41"); + o.get().literal("number", "23"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("number", "42"); + o.get().literal("number", "23"); + o.get().endEntity(); + o.get().startEntity("3"); + o.get().literal("number", "6"); + o.get().literal("number", "23"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldRemoveDuplicateArrays() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "uniq('arrays[]')" + ), + i -> { + i.startRecord("1"); + i.startEntity("arrays[]"); + i.startEntity("1"); + i.literal("number", "41"); + i.literal("number", "23"); + i.endEntity(); + i.startEntity("2"); + i.literal("number", "42"); + i.literal("number", "23"); + i.endEntity(); + i.startEntity("3"); + i.literal("number", "6"); + i.literal("number", "23"); + i.endEntity(); + i.startEntity("3"); + i.literal("number", "6"); + i.literal("number", "23"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("arrays[]"); + o.get().startEntity("1"); + o.get().literal("number", "41"); + o.get().literal("number", "23"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("number", "42"); + o.get().literal("number", "23"); + o.get().endEntity(); + o.get().startEntity("3"); + o.get().literal("number", "6"); + o.get().literal("number", "23"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldRemoveDuplicateHashesAtDifferentPath() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "uniq('hashes[]')" + ), + i -> { + i.startRecord("1"); + i.startEntity("hashes[]"); + i.startEntity("1"); + i.literal("number", "41"); + i.endEntity(); + i.startEntity("2"); + i.literal("number", "42"); + i.endEntity(); + i.startEntity("3"); + i.literal("number", "6"); + i.endEntity(); + i.startEntity("4"); + i.literal("number", "6"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("hashes[]"); + o.get().startEntity("1"); + o.get().literal("number", "41"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("number", "42"); + o.get().endEntity(); + o.get().startEntity("3"); + o.get().literal("number", "6"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldRemoveDuplicateHashes() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "uniq('hashes[]')" + ), + i -> { + i.startRecord("1"); + i.startEntity("hashes[]"); + i.startEntity("1"); + i.literal("number", "41"); + i.endEntity(); + i.startEntity("2"); + i.literal("number", "42"); + i.endEntity(); + i.startEntity("3"); + i.literal("number", "6"); + i.endEntity(); + i.startEntity("3"); + i.literal("number", "6"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("hashes[]"); + o.get().startEntity("1"); + o.get().literal("number", "41"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("number", "42"); + o.get().endEntity(); + o.get().startEntity("3"); + o.get().literal("number", "6"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldApplyCustomJavaFunction() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "org.metafacture.metafix.util.TestFunction(data, foo: '42', bar: 'baz')" + ), + i -> { + i.startRecord("1"); + i.literal("title", "marc"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "marc"); + o.get().literal("test", "DATA"); + o.get().literal("foo", "42"); + o.get().literal("bar", "baz"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldUriEncodeString() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "uri_encode('title')" + ), + i -> { + i.startRecord("1"); + i.literal("title", "café"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "caf%C3%A9"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldUriEncodePathSegment() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "uri_encode('id')" + ), + i -> { + i.startRecord("1"); + i.literal("id", "/DE-A96:% (3)#!"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("id", "%2FDE-A96%3A%25+%283%29%23%21"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldUriEncodePathSegmentWithoutPlusForSpace() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "uri_encode('id', plus_for_space:'false')" + ), + i -> { + i.startRecord("1"); + i.literal("id", "/DE-A96:% (3)#!"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("id", "%2FDE-A96%3A%25%20%283%29%23%21"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldUriEncodePathSegmentWithoutSafeChars() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "uri_encode('id', safe_chars:'')" + ), + i -> { + i.startRecord("1"); + i.literal("id", "/DE-A96:% (3)#!"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("id", "%2FDE%2DA96%3A%25+%283%29%23%21"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldTransformStringToBase64() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "to_base64('data.title')" + ), + i -> { + i.startRecord("1"); + i.startEntity("data"); + i.literal("title", "this-is-a-test"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("data"); + o.get().literal("title", "dGhpcy1pcy1hLXRlc3Q="); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldTransformUrlSafeToBase64() { + urlToBase64(",url_safe:'true'", "aHR0cHM6Ly93d3cueW91dHViZS5jb20vd2F0Y2g_dj1kYUxnc1BTdkQ5QQ=="); + } + + @Test + public void shouldTransformNotUrlSafeToBase64AsDefault() { + urlToBase64("", "aHR0cHM6Ly93d3cueW91dHViZS5jb20vd2F0Y2g/dj1kYUxnc1BTdkQ5QQ=="); + } + + private void urlToBase64(final String option, final String expected) { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "to_base64('data.title'" + option + ")" + ), + i -> { + i.startRecord("1"); + i.startEntity("data"); + i.literal("title", YOUTUBE_URL); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("data"); + o.get().literal("title", expected); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test // checkstyle-disable-line JavaNCSS + public void shouldCreateVariableFromLiteralValue() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "to_var('data.title', 'testVar')", + "add_field('testResult', 'This is a $[testVar]')" + ), + i -> { + i.startRecord("1"); + i.startEntity("data"); + i.literal("title", "test"); + i.endEntity(); + i.endRecord(); + i.startRecord("2"); + i.endRecord(); + i.startRecord("3"); + i.startEntity("data"); + i.literal("title", "final-test"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("data"); + o.get().literal("title", "test"); + o.get().endEntity(); + o.get().literal("testResult", "This is a test"); + o.get().endRecord(); + o.get().startRecord("2"); + o.get().literal("testResult", "This is a "); + o.get().endRecord(); + o.get().startRecord("3"); + o.get().startEntity("data"); + o.get().literal("title", "final-test"); + o.get().endEntity(); + o.get().literal("testResult", "This is a final-test"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldCreateVariableFromLiteralValueWithDefault() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "to_var('data.title', 'testVar', 'default': 'n/a')", + "add_field('testResult', 'This is a $[testVar]')" + ), + i -> { + i.startRecord("1"); + i.startEntity("data"); + i.literal("title", "test"); + i.endEntity(); + i.endRecord(); + i.startRecord("2"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("data"); + o.get().literal("title", "test"); + o.get().endEntity(); + o.get().literal("testResult", "This is a test"); + o.get().endRecord(); + o.get().startRecord("2"); + o.get().literal("testResult", "This is a n/a"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldNotCreateVariableFromArrayValue() { + MetafixTestHelpers.assertExecutionException(IllegalStateException.class, "Expected String, got Array", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "to_var('data.title', 'testVar')" + ), + i -> { + i.startRecord("1"); + i.startEntity("data"); + i.literal("title", "test1"); + i.literal("title", "test2"); + i.endEntity(); + i.endRecord(); + }, + o -> { + } + ) + ); + } + + @Test + public void shouldNotCreateVariableFromHashValue() { + MetafixTestHelpers.assertExecutionException(IllegalStateException.class, "Expected String, got Hash", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "to_var('data.title', 'testVar')" + ), + i -> { + i.startRecord("1"); + i.startEntity("data"); + i.startEntity("title"); + i.literal("key", "value"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + o -> { + } + ) + ); + } + +} diff --git a/metafix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java b/metafix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java new file mode 100644 index 000000000..b8c13b873 --- /dev/null +++ b/metafix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java @@ -0,0 +1,3715 @@ +/* + * Copyright 2021, 2023 Fabian Steeg, hbz + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix; // checkstyle-disable-line JavaNCSS + +import org.metafacture.framework.StreamReceiver; +import org.metafacture.metamorph.api.MorphBuildException; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentMatchers; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.io.IOException; +import java.util.Arrays; +import java.util.function.Consumer; +import java.util.function.Supplier; + +/** + * Tests Metafix record level methods. Following the cheat sheet + * examples at https://github.com/LibreCat/Catmandu/wiki/Fixes-Cheat-Sheet + * + * @author Fabian Steeg + */ +@ExtendWith(MockitoExtension.class) // checkstyle-disable-line JavaNCSS +@ExtendWith(MetafixToDo.Extension.class) +public class MetafixRecordTest { + + @Mock + private StreamReceiver streamReceiver; + + public MetafixRecordTest() { + } + + @Test + public void entitiesPassThrough() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "nothing()"), + i -> { + i.startRecord("1"); + i.startEntity("deep"); + i.startEntity("nested"); + i.literal("field", "value"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("deep"); + o.get().startEntity("nested"); + o.get().literal("field", "value"); + f.apply(2).endEntity(); + o.get().endRecord(); + }); + } + + @Test + public void shouldNotEmitVirtualFieldsByDefault() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "nothing()" + ), + i -> { + i.startRecord("1"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldEmitVirtualFieldsWhenRetained() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "retain('_id')" + ), + i -> { + i.startRecord("1"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("_id", "1"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldEmitVirtualFieldsWhenCopied() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "copy_field('_id', id)" + ), + i -> { + i.startRecord("1"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("id", "1"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldEmitVirtualFieldsWhenAdded() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "add_field('_id', 'id')" + ), + i -> { + i.startRecord("1"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("_id", "id"); + o.get().endRecord(); + } + ); + } + + @Test + public void entitiesPassThroughRepeatNestedEntity() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "nothing()"), + i -> { + i.startRecord("1"); + i.startEntity("deep"); + i.startEntity("nested"); + i.literal("field", "value1"); + i.endEntity(); + i.startEntity("nested"); + i.literal("field", "value2"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("deep"); + o.get().startEntity("nested"); + o.get().literal("field", "value1"); + o.get().endEntity(); + o.get().startEntity("nested"); + o.get().literal("field", "value2"); + f.apply(2).endEntity(); + o.get().endRecord(); + }); + } + + @Test + public void setEmpty() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_field('my.nested.name','patrick')", + "set_field('your.nested.name','nicolas')"), + i -> { + i.startRecord("1"); + i.endRecord(); + }, (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("my"); + o.get().startEntity("nested"); + o.get().literal("name", "patrick"); + f.apply(2).endEntity(); + o.get().startEntity("your"); + o.get().startEntity("nested"); + o.get().literal("name", "nicolas"); + f.apply(2).endEntity(); + o.get().endRecord(); + }); + } + + @Test + public void setExisting() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_field('my.nested.name','patrick')", + "set_field('your.nested.name','nicolas')"), + i -> { + i.startRecord("1"); + i.startEntity("my"); + i.startEntity("nested"); + i.literal("name", "max"); + i.endEntity(); + i.endEntity(); + i.startEntity("your"); + i.startEntity("nested"); + i.literal("name", "mo"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("my"); + o.get().startEntity("nested"); + o.get().literal("name", "patrick"); + f.apply(2).endEntity(); + o.get().startEntity("your"); + o.get().startEntity("nested"); + o.get().literal("name", "nicolas"); + f.apply(2).endEntity(); + o.get().endRecord(); + }); + } + + @Test + public void add() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "add_field('my.name','patrick')", + "add_field('my.name','nicolas')" + ), + i -> { + i.startRecord("1"); + i.endRecord(); + + i.startRecord("2"); + i.startEntity("my"); + i.literal("name", "max"); + i.endEntity(); + i.endRecord(); + + i.startRecord("3"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("my"); + o.get().literal("name", "nicolas"); + o.get().endEntity(); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().startEntity("my"); + o.get().literal("name", "nicolas"); + o.get().endEntity(); + o.get().endRecord(); + + o.get().startRecord("3"); + o.get().startEntity("my"); + o.get().literal("name", "nicolas"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void addWithAppendInNewArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('my.name')", + "add_field('my.name.$append','patrick')", + "add_field('my.name.$append','nicolas')" + ), + i -> { + i.startRecord("1"); + i.endRecord(); + + i.startRecord("2"); + i.startEntity("my"); + i.literal("name", "max"); + i.endEntity(); + i.endRecord(); + + i.startRecord("3"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("my"); + o.get().literal("name", "patrick"); + o.get().literal("name", "nicolas"); + o.get().endEntity(); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().startEntity("my"); + o.get().literal("name", "patrick"); + o.get().literal("name", "nicolas"); + o.get().endEntity(); + o.get().endRecord(); + + o.get().startRecord("3"); + o.get().startEntity("my"); + o.get().literal("name", "patrick"); + o.get().literal("name", "nicolas"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void addWithAppendInImplicitArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "add_field('my.name.$append','patrick')", + "add_field('my.name.$append','nicolas')" + ), + i -> { + i.startRecord("1"); + i.endRecord(); + + i.startRecord("2"); + i.startEntity("my"); + i.literal("name", "max"); + i.endEntity(); + i.endRecord(); + + i.startRecord("3"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("my"); + o.get().literal("name", "patrick"); + o.get().literal("name", "nicolas"); + o.get().endEntity(); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().startEntity("my"); + o.get().literal("name", "max"); + o.get().literal("name", "patrick"); + o.get().literal("name", "nicolas"); + o.get().endEntity(); + o.get().endRecord(); + + o.get().startRecord("3"); + o.get().startEntity("my"); + o.get().literal("name", "patrick"); + o.get().literal("name", "nicolas"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void addWithPrependInImplicitArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "add_field('my.name.$prepend','patrick')", + "add_field('my.name.$prepend','nicolas')" + ), + i -> { + i.startRecord("1"); + i.endRecord(); + + i.startRecord("2"); + i.startEntity("my"); + i.literal("name", "max"); + i.endEntity(); + i.endRecord(); + + i.startRecord("3"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("my"); + o.get().literal("name", "nicolas"); + o.get().literal("name", "patrick"); + o.get().endEntity(); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().startEntity("my"); + o.get().literal("name", "nicolas"); + o.get().literal("name", "patrick"); + o.get().literal("name", "max"); + o.get().endEntity(); + o.get().endRecord(); + + o.get().startRecord("3"); + o.get().startEntity("my"); + o.get().literal("name", "nicolas"); + o.get().literal("name", "patrick"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void addWithLastInNonArray() { + MetafixTestHelpers.assertExecutionException(IllegalStateException.class, "Expected Array or Hash, got String", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "add_field('my.name.$last','patrick')" + ), + i -> { + i.startRecord("1"); + i.startEntity("my"); + i.literal("name", "max"); + i.endEntity(); + i.endRecord(); + }, + o -> { + } + ) + ); + } + + @Test + public void addWithAppendInArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "add_field('names.$append','patrick')" + ), + i -> { + i.startRecord("1"); + i.literal("names", "max"); + i.literal("names", "mo"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("names", "max"); + o.get().literal("names", "mo"); + o.get().literal("names", "patrick"); + o.get().endRecord(); + } + ); + } + + @Test + public void addWithAppendInHash() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "add_field('author.names.$append','patrick')" + ), + i -> { + i.startRecord("1"); + i.startEntity("author"); + i.literal("names", "max"); + i.literal("names", "mo"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("author"); + o.get().literal("names", "max"); + o.get().literal("names", "mo"); + o.get().literal("names", "patrick"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void addWithAppendInArrayWithSubfieldFromRepeatedField() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "add_field('authors.$append.name','patrick')" + ), + i -> { + i.startRecord("1"); + i.startEntity("authors"); + i.literal("name", "max"); + i.endEntity(); + i.startEntity("authors"); + i.literal("name", "mo"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("authors"); + o.get().literal("name", "max"); + o.get().endEntity(); + o.get().startEntity("authors"); + o.get().literal("name", "mo"); + o.get().endEntity(); + o.get().startEntity("authors"); + o.get().literal("name", "patrick"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void addWithAppendInArrayWithSubfieldFromIndexedArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "add_field('authors[].$append.name','patrick')"), + i -> { + i.startRecord("1"); + i.startEntity("authors[]"); + i.startEntity("1"); + i.literal("name", "max"); + i.endEntity(); + i.startEntity("2"); + i.literal("name", "mo"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("authors[]"); + o.get().startEntity("1"); + o.get().literal("name", "max"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("name", "mo"); + o.get().endEntity(); + o.get().startEntity("3"); + o.get().literal("name", "patrick"); + f.apply(2).endEntity(); + o.get().endRecord(); + }); + } + + @Test + public void simpleAppendWithArrayOfStrings() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "add_field('animals[].$append', 'duck')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.literal("1", "cat"); + i.literal("2", "dog"); + i.literal("3", "fox"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("animals[]"); + o.get().literal("1", "cat"); + o.get().literal("2", "dog"); + o.get().literal("3", "fox"); + o.get().literal("4", "duck"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void complexAppendWithArrayOfStrings() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "copy_field('others', 'animals[].$append')", + "move_field('fictional', 'animals[].$append')", + "add_field('animals[].$append', 'earthworm')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.literal("1", "dog"); + i.literal("2", "cat"); + i.endEntity(); + i.literal("others", "human"); + i.literal("fictional", "unicorn"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("animals[]"); + o.get().literal("1", "dog"); + o.get().literal("2", "cat"); + o.get().literal("3", "human"); + o.get().literal("4", "unicorn"); + o.get().literal("5", "earthworm"); + o.get().endEntity(); + o.get().literal("others", "human"); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/92 + public void complexAppendWithArrayOfObjects() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "copy_field('others', 'animals[].$append')", + "move_field('fictional', 'animals[].$append')", + "add_field('animals[].$append.animal', 'earthworm')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.startEntity("1"); + i.literal("animal", "dog"); + i.endEntity(); + i.startEntity("2"); + i.literal("animal", "cat"); + i.endEntity(); + i.endEntity(); + i.startEntity("others"); + i.literal("animal", "human"); + i.endEntity(); + i.startEntity("fictional"); + i.literal("animal", "unicorn"); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("animals[]"); + o.get().startEntity("1"); + o.get().literal("animal", "dog"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("animal", "cat"); + o.get().endEntity(); + o.get().startEntity("3"); + o.get().literal("animal", "human"); + o.get().endEntity(); + o.get().startEntity("4"); + o.get().literal("animal", "unicorn"); + o.get().endEntity(); + o.get().startEntity("5"); + o.get().literal("animal", "earthworm"); + f.apply(2).endEntity(); + o.get().startEntity("others"); + o.get().literal("animal", "human"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void appendWithWildcard() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('stringimals[]')", + "copy_field('?nimal', 'stringimals[].$append')" + ), + i -> { + i.startRecord("1"); + i.literal("animal", "dog"); + i.literal("bnimal", "cat"); + i.literal("cnimal", "zebra"); + i.literal("dnimol", "bunny"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("animal", "dog"); + o.get().literal("bnimal", "cat"); + o.get().literal("cnimal", "zebra"); + o.get().literal("dnimol", "bunny"); + o.get().startEntity("stringimals[]"); + o.get().literal("1", "dog"); + o.get().literal("2", "cat"); + o.get().literal("3", "zebra"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/99 + public void simpleCopyWithWildcard() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "copy_field('?nimal', 'animal')" + ), + i -> { + i.startRecord("1"); + i.literal("animal", "dog"); + i.endRecord(); + i.startRecord("2"); + i.literal("bnimal", "cat"); + i.endRecord(); + i.startRecord("3"); + i.literal("cnimal", "zebra"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("animal", "dog"); + o.get().endRecord(); + o.get().startRecord("2"); + o.get().literal("bnimal", "cat"); + o.get().literal("animal", "cat"); + o.get().endRecord(); + o.get().startRecord("3"); + o.get().literal("cnimal", "zebra"); + o.get().literal("animal", "zebra"); + o.get().endRecord(); + } + ); + } + + @Test + public void appendWithMultipleWildcards() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('stringimals[]')", + "copy_field('?ni??l', 'stringimals[].$append')" + ), + i -> { + i.startRecord("1"); + i.literal("animal", "dog"); + i.literal("bnimal", "cat"); + i.literal("cnimal", "zebra"); + i.literal("dnimol", "bunny"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("animal", "dog"); + o.get().literal("bnimal", "cat"); + o.get().literal("cnimal", "zebra"); + o.get().literal("dnimol", "bunny"); + o.get().startEntity("stringimals[]"); + o.get().literal("1", "dog"); + o.get().literal("2", "cat"); + o.get().literal("3", "zebra"); + o.get().literal("4", "bunny"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void appendWithAsteriksWildcard() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('stringimals[]')", + "copy_field('*al', 'stringimals[].$append')" + ), + i -> { + i.startRecord("1"); + i.literal("animal", "dog"); + i.literal("bnimal", "cat"); + i.literal("cnimal", "zebra"); + i.literal("dnimol", "bunny"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("animal", "dog"); + o.get().literal("bnimal", "cat"); + o.get().literal("cnimal", "zebra"); + o.get().literal("dnimol", "bunny"); + o.get().startEntity("stringimals[]"); + o.get().literal("1", "dog"); + o.get().literal("2", "cat"); + o.get().literal("3", "zebra"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void appendWithBracketWildcard() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('stringimals[]')", + "copy_field('[ac]nimal', 'stringimals[].$append')" + ), + i -> { + i.startRecord("1"); + i.literal("animal", "dog"); + i.literal("bnimal", "cat"); + i.literal("cnimal", "zebra"); + i.literal("dnimol", "bunny"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("animal", "dog"); + o.get().literal("bnimal", "cat"); + o.get().literal("cnimal", "zebra"); + o.get().literal("dnimol", "bunny"); + o.get().startEntity("stringimals[]"); + o.get().literal("1", "dog"); + o.get().literal("2", "zebra"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/89 + public void appendWithAsteriksWildcardAtTheEnd() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('stringimals[]')", + "copy_field('ani*', 'stringimals[].$append')" + ), + i -> { + i.startRecord("1"); + i.literal("animals", "dog"); + i.literal("animals", "cat"); + i.literal("animals", "zebra"); + i.literal("animal", "bunny"); + i.startEntity("animols"); + i.literal("name", "bird"); + i.literal("type", "TEST"); + i.endEntity(); + i.literal("ANIMALS", "dragon and unicorn"); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().literal("animals", "dog"); + o.get().literal("animals", "cat"); + o.get().literal("animals", "zebra"); + o.get().literal("animal", "bunny"); + o.get().startEntity("animols"); + o.get().literal("name", "bird"); + o.get().literal("type", "TEST"); + o.get().endEntity(); + o.get().literal("ANIMALS", "dragon and unicorn"); + o.get().startEntity("stringimals[]"); + o.get().literal("1", "dog"); + o.get().literal("2", "cat"); + o.get().literal("3", "zebra"); + o.get().literal("4", "bunny"); + o.get().startEntity("5"); + o.get().literal("name", "bird"); + o.get().literal("type", "TEST"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + @MetafixToDo("See https://github.com/metafacture/metafacture-fix/pull/113") + public void shouldCopyArrayFieldWithoutAsterisk() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('TEST_TWO[]')", + "copy_field('test[]', 'TEST_TWO[].$append')" + ), + i -> { + i.startRecord("1"); + i.startEntity("test[]"); + i.literal("1", "One"); + i.literal("2", "Two"); + i.literal("3", "Three"); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("test[]"); + o.get().literal("1", "One"); + o.get().literal("2", "Two"); + o.get().literal("3", "Three"); + o.get().endEntity(); + o.get().startEntity("TEST_TWO[]"); + o.get().startEntity("1"); + o.get().literal("1", "One"); + o.get().literal("2", "Two"); + o.get().literal("3", "Three"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + @MetafixToDo("See https://github.com/metafacture/metafacture-fix/issues/113") + public void copyFieldArrayOfObjectsAndListNewArrayOfObjectsAndMoveSubfield() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "copy_field('author[]','creator[]')", + "do list(path:'creator[]')", + " move_field('name','label')", + "end", + "retain('creator[]')"), + i -> { + i.startRecord("1"); + i.startEntity("author[]"); + i.startEntity("1"); + i.literal("name", "A University"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("creator[]"); + o.get().startEntity("1"); + o.get().literal("label", "A University"); + f.apply(2).endEntity(); + o.get().endRecord(); + }); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/121 + public void shouldCopyArrayFieldWithAsterisk() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('TEST_TWO[]')", + "copy_field('test[].*', 'TEST_TWO[].$append')" + ), + i -> { + i.startRecord("1"); + i.startEntity("test[]"); + i.literal("1", "One"); + i.literal("2", "Two"); + i.literal("3", "Three"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("test[]"); + o.get().literal("1", "One"); + o.get().literal("2", "Two"); + o.get().literal("3", "Three"); + o.get().endEntity(); + o.get().startEntity("TEST_TWO[]"); + o.get().literal("1", "One"); + o.get().literal("2", "Two"); + o.get().literal("3", "Three"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/121 + public void shouldCopyNestedArrayFieldWithAsterisk() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('TEST_4[]')", + "copy_field('nestedTest[].*.test[].*', 'TEST_4[].$append')" + ), + i -> { + i.startRecord("1"); + i.startEntity("nestedTest[]"); + i.startEntity("1"); + i.startEntity("test[]"); + i.literal("1", "One"); + i.literal("2", "Two"); + i.literal("3", "Three"); + i.endEntity(); + i.endEntity(); + i.startEntity("2"); + i.startEntity("test[]"); + i.literal("1", "4"); + i.literal("2", "5"); + i.literal("3", "6"); + i.endEntity(); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("nestedTest[]"); + o.get().startEntity("1"); + o.get().startEntity("test[]"); + o.get().literal("1", "One"); + o.get().literal("2", "Two"); + o.get().literal("3", "Three"); + f.apply(2).endEntity(); + o.get().startEntity("2"); + o.get().startEntity("test[]"); + o.get().literal("1", "4"); + o.get().literal("2", "5"); + o.get().literal("3", "6"); + f.apply(3).endEntity(); + o.get().startEntity("TEST_4[]"); + o.get().literal("1", "One"); + o.get().literal("2", "Two"); + o.get().literal("3", "Three"); + o.get().literal("4", "4"); + o.get().literal("5", "5"); + o.get().literal("6", "6"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/121 + public void shouldCopyArraySubFieldWithAsterisk() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('TEST_5[]')", + "copy_field('coll[].*.b', 'TEST_5[].$append')" + ), + i -> { + i.startRecord("1"); + i.startEntity("coll[]"); + i.startEntity("1"); + i.literal("a", "Dog"); + i.literal("b", "Dog"); + i.endEntity(); + i.startEntity("2"); + i.literal("a", "Ape"); + i.literal("b", "Ape"); + i.endEntity(); + i.startEntity("3"); + i.literal("a", "Giraffe"); + i.endEntity(); + i.startEntity("4"); + i.literal("a", "Crocodile"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("coll[]"); + o.get().startEntity("1"); + o.get().literal("a", "Dog"); + o.get().literal("b", "Dog"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("a", "Ape"); + o.get().literal("b", "Ape"); + o.get().endEntity(); + o.get().startEntity("3"); + o.get().literal("a", "Giraffe"); + o.get().endEntity(); + o.get().startEntity("4"); + o.get().literal("a", "Crocodile"); + f.apply(2).endEntity(); + o.get().startEntity("TEST_5[]"); + o.get().literal("1", "Dog"); + o.get().literal("2", "Ape"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void addFieldToFirstObjectInRepeatedFields() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "add_field('animals.$first.kind','nice')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals"); + i.literal("name", "dog"); + i.endEntity(); + i.startEntity("animals"); + i.literal("name", "cat"); + i.endEntity(); + i.startEntity("animals"); + i.literal("name", "fox"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("animals"); + o.get().literal("name", "dog"); + o.get().literal("kind", "nice"); + o.get().endEntity(); + o.get().startEntity("animals"); + o.get().literal("name", "cat"); + o.get().endEntity(); + o.get().startEntity("animals"); + o.get().literal("name", "fox"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void addFieldToLastObjectInRepeatedFields() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "add_field('animals.$last.kind','nice')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals"); + i.literal("name", "dog"); + i.endEntity(); + i.startEntity("animals"); + i.literal("name", "cat"); + i.endEntity(); + i.startEntity("animals"); + i.literal("name", "fox"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("animals"); + o.get().literal("name", "dog"); + o.get().endEntity(); + o.get().startEntity("animals"); + o.get().literal("name", "cat"); + o.get().endEntity(); + o.get().startEntity("animals"); + o.get().literal("name", "fox"); + o.get().literal("kind", "nice"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void addFieldToObjectByIndexInRepeatedFields() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "add_field('animals.2.kind','nice')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals"); + i.literal("name", "dog"); + i.endEntity(); + i.startEntity("animals"); + i.literal("name", "cat"); + i.endEntity(); + i.startEntity("animals"); + i.literal("name", "fox"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("animals"); + o.get().literal("name", "dog"); + o.get().endEntity(); + o.get().startEntity("animals"); + o.get().literal("name", "cat"); + o.get().literal("kind", "nice"); + o.get().endEntity(); + o.get().startEntity("animals"); + o.get().literal("name", "fox"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void addFieldToFirstObjectInIndexedArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "add_field('animals[].$first.kind','nice')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.startEntity("1"); + i.literal("name", "dog"); + i.endEntity(); + i.startEntity("2"); + i.literal("name", "cat"); + i.endEntity(); + i.startEntity("3"); + i.literal("name", "fox"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("animals[]"); + o.get().startEntity("1"); + o.get().literal("name", "dog"); + o.get().literal("kind", "nice"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("name", "cat"); + o.get().endEntity(); + o.get().startEntity("3"); + o.get().literal("name", "fox"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void addFieldToLastObjectInIndexedArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "add_field('animals[].$last.kind','nice')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.startEntity("1"); + i.literal("name", "dog"); + i.endEntity(); + i.startEntity("2"); + i.literal("name", "cat"); + i.endEntity(); + i.startEntity("3"); + i.literal("name", "fox"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("animals[]"); + o.get().startEntity("1"); + o.get().literal("name", "dog"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("name", "cat"); + o.get().endEntity(); + o.get().startEntity("3"); + o.get().literal("name", "fox"); + o.get().literal("kind", "nice"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void addFieldToObjectByIndexInIndexedArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "add_field('animals[].2.kind','nice')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.startEntity("1"); + i.literal("name", "dog"); + i.endEntity(); + i.startEntity("2"); + i.literal("name", "cat"); + i.endEntity(); + i.startEntity("3"); + i.literal("name", "fox"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("animals[]"); + o.get().startEntity("1"); + o.get().literal("name", "dog"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("name", "cat"); + o.get().literal("kind", "nice"); + o.get().endEntity(); + o.get().startEntity("3"); + o.get().literal("name", "fox"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void addFieldToFirstObjectMissing() { + assertThrowsOnEmptyArray("$first"); + } + + @Test + public void addFieldToLastObjectMissing() { + assertThrowsOnEmptyArray("$last"); + } + + @Test + public void addFieldToObjectByIndexMissing() { + assertThrowsOnEmptyArray("2"); + } + + private void assertThrowsOnEmptyArray(final String index) { + MetafixTestHelpers.assertProcessException(IllegalArgumentException.class, "Can't find: " + index + " in: null", () -> { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "add_field('animals[]." + index + ".kind','nice')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.endEntity(); + i.endRecord(); + }, + o -> { + } + ); + }); + } + + @Test + public void shouldAddArraySubFieldWithAsterisk() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "add_field('coll[].*.c', 'test')" + ), + i -> { + i.startRecord("1"); + i.startEntity("coll[]"); + i.startEntity("1"); + i.literal("a", "Dog"); + i.literal("b", "Dog"); + i.endEntity(); + i.startEntity("2"); + i.literal("a", "Ape"); + i.literal("b", "Ape"); + i.endEntity(); + i.startEntity("3"); + i.literal("a", "Giraffe"); + i.endEntity(); + i.startEntity("4"); + i.literal("a", "Crocodile"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("coll[]"); + o.get().startEntity("1"); + o.get().literal("a", "Dog"); + o.get().literal("b", "Dog"); + o.get().literal("c", "test"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("a", "Ape"); + o.get().literal("b", "Ape"); + o.get().literal("c", "test"); + o.get().endEntity(); + o.get().startEntity("3"); + o.get().literal("a", "Giraffe"); + o.get().literal("c", "test"); + o.get().endEntity(); + o.get().startEntity("4"); + o.get().literal("a", "Crocodile"); + o.get().literal("c", "test"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void move() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "move_field('my.name','your.name')", + "move_field('missing','whatever')"), + i -> { + i.startRecord("1"); + i.endRecord(); + + i.startRecord("2"); + i.startEntity("my"); + i.literal("name", "max"); + i.endEntity(); + i.endRecord(); + + i.startRecord("3"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().startEntity("my"); + o.get().endEntity(); + o.get().startEntity("your"); + o.get().literal("name", "max"); + o.get().endEntity(); + o.get().endRecord(); + + o.get().startRecord("3"); + o.get().endRecord(); + }); + } + + @Test + public void copy() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "copy_field('your.name','your.name2')"), + i -> { + i.startRecord("1"); + i.endRecord(); + + i.startRecord("2"); + i.startEntity("your"); + i.literal("name", "max"); + i.endEntity(); + i.endRecord(); + + i.startRecord("3"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().startEntity("your"); + o.get().literal("name", "max"); + o.get().literal("name2", "max"); + o.get().endEntity(); + o.get().endRecord(); + + o.get().startRecord("3"); + o.get().endRecord(); + }); + } + + @Test + public void copyIntoArrayOfStrings() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + // "set_array('author')", <- results in separate objects/entities here + "copy_field('your.name','author.name[]')", + "remove_field('your')"), + i -> { + i.startRecord("1"); + i.startEntity("your"); + i.literal("name", "maxi-mi"); + i.literal("name", "maxi-ma"); + i.endEntity(); + i.endRecord(); + }, (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("author"); + o.get().startEntity("name[]"); + o.get().literal("1", "maxi-mi"); + o.get().literal("2", "maxi-ma"); + f.apply(2).endEntity(); + o.get().endRecord(); + }); + } + + @Test + public void copyArrayOfStrings() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "copy_field('your','author')", + "remove_field('your')" + ), + i -> { + i.startRecord("1"); + i.startEntity("your"); + i.literal("name", "maxi-mi"); + i.literal("name", "maxi-ma"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("author"); + o.get().literal("name", "maxi-mi"); + o.get().literal("name", "maxi-ma"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void renameArrayOfStrings() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "move_field('your','author')" + ), + i -> { + i.startRecord("1"); + i.startEntity("your"); + i.literal("name", "maxi-mi"); + i.literal("name", "maxi-ma"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("author"); + o.get().literal("name", "maxi-mi"); + o.get().literal("name", "maxi-ma"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void copyArrayOfHashes() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "copy_field('author', 'authors[]')", + "remove_field('author')"), + i -> { + i.startRecord("1"); + i.startEntity("author"); + i.literal("name", "max"); + i.endEntity(); + i.startEntity("author"); + i.literal("name", "mo"); + i.endEntity(); + i.endRecord(); + }, (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("authors[]"); + o.get().startEntity("1"); + o.get().literal("name", "max"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("name", "mo"); + f.apply(2).endEntity(); + o.get().endRecord(); + }); + } + + @Test + public void renameArrayOfHashes() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "move_field('author', 'authors[]')"), + i -> { + i.startRecord("1"); + i.startEntity("author"); + i.literal("name", "max"); + i.endEntity(); + i.startEntity("author"); + i.literal("name", "mo"); + i.endEntity(); + i.endRecord(); + }, (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("authors[]"); + o.get().startEntity("1"); + o.get().literal("name", "max"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("name", "mo"); + f.apply(2).endEntity(); + o.get().endRecord(); + }); + } + + @Test + public void copyIntoImplicitArrayAppend() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "copy_field('your.name','author[].$append.name')", + "remove_field('your')"), + i -> { + i.startRecord("1"); + i.startEntity("your"); + i.literal("name", "max"); + i.endEntity(); + i.startEntity("your"); + i.literal("name", "mo"); + i.endEntity(); + i.endRecord(); + }, (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("author[]"); + o.get().startEntity("1"); + o.get().literal("name", "max"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("name", "mo"); + f.apply(2).endEntity(); + o.get().endRecord(); + }); + } + + @Test + public void copyIntoImplicitArrayPrepend() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "copy_field('your.name','author[].$prepend.name')", + "remove_field('your')"), + i -> { + i.startRecord("1"); + i.startEntity("your"); + i.literal("name", "max"); + i.endEntity(); + i.startEntity("your"); + i.literal("name", "mo"); + i.endEntity(); + i.endRecord(); + }, (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("author[]"); + o.get().startEntity("1"); + o.get().literal("name", "mo"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("name", "max"); + f.apply(2).endEntity(); + o.get().endRecord(); + }); + } + + @Test + public void copyIntoExplicitArrayAppend() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('author[]')", + "copy_field('your.name','author[].$append.name')", + "remove_field('your')"), + i -> { + i.startRecord("1"); + i.startEntity("your"); + i.literal("name", "max"); + i.endEntity(); + i.startEntity("your"); + i.literal("name", "mo"); + i.endEntity(); + i.endRecord(); + }, (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("author[]"); + o.get().startEntity("1"); + o.get().literal("name", "max"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("name", "mo"); + f.apply(2).endEntity(); + o.get().endRecord(); + }); + } + + @Test + public void copyIntoArrayTopLevel() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('author[]')", + "copy_field('your.name', 'author[]')", + "remove_field('your')"), + i -> { + i.startRecord("1"); + i.startEntity("your"); + i.literal("name", "maxi-mi"); + i.literal("name", "maxi-ma"); + i.endEntity(); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().startEntity("author[]"); + o.get().literal("1", "maxi-mi"); + o.get().literal("2", "maxi-ma"); + o.get().endEntity(); + o.get().endRecord(); + }); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/106 + public void shouldCopyMarkedArrayOfStringsIntoUnmarkedArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "copy_field('animal_string_Array[]', 'animals_repeated_SimpleField')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animal_string_Array[]"); + i.literal("1", "dog"); + i.literal("2", "elefant"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("animal_string_Array[]"); + o.get().literal("1", "dog"); + o.get().literal("2", "elefant"); + o.get().endEntity(); + o.get().literal("animals_repeated_SimpleField", "dog"); + o.get().literal("animals_repeated_SimpleField", "elefant"); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/106 + public void shouldCopyMarkedArrayOfHashesIntoUnmarkedArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "copy_field('animal_object_Array[]', 'animals_repeated_ObjectField')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animal_object_Array[]"); + i.startEntity("1"); + i.literal("name", "dog"); + i.endEntity(); + i.startEntity("2"); + i.literal("name", "elefant"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("animal_object_Array[]"); + o.get().startEntity("1"); + o.get().literal("name", "dog"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("name", "elefant"); + f.apply(2).endEntity(); + o.get().startEntity("animals_repeated_ObjectField"); + o.get().literal("name", "dog"); + o.get().endEntity(); + o.get().startEntity("animals_repeated_ObjectField"); + o.get().literal("name", "elefant"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/106 + public void shouldCopyMarkedArrayOfHashesIntoMarkedArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "copy_field('animal_object_Array[]', 'test_animal_object_Array[]')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animal_object_Array[]"); + i.startEntity("1"); + i.literal("name", "dog"); + i.endEntity(); + i.startEntity("2"); + i.literal("name", "elefant"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("animal_object_Array[]"); + o.get().startEntity("1"); + o.get().literal("name", "dog"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("name", "elefant"); + f.apply(2).endEntity(); + o.get().startEntity("test_animal_object_Array[]"); + o.get().startEntity("1"); + o.get().literal("name", "dog"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("name", "elefant"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void removeLiteral() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "remove_field('your.name')"), + i -> { + i.startRecord("1"); + i.endRecord(); + + i.startRecord("2"); + i.startEntity("your"); + i.literal("name", "max"); + i.endEntity(); + i.endRecord(); + + i.startRecord("3"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().startEntity("your"); + o.get().endEntity(); + o.get().endRecord(); + + o.get().startRecord("3"); + o.get().endRecord(); + }); + } + + @Test + public void removeLiteralAndEntity() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "remove_field('your.name')", + "remove_field('your')"), + i -> { + i.startRecord("1"); + i.endRecord(); + + i.startRecord("2"); + i.startEntity("your"); + i.literal("name", "max"); + i.endEntity(); + i.endRecord(); + + i.startRecord("3"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().endRecord(); + + o.get().startRecord("3"); + o.get().endRecord(); + }); + } + + @Test + public void removeEntity() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "remove_field('your')"), + i -> { + i.startRecord("1"); + i.endRecord(); + + i.startRecord("2"); + i.startEntity("your"); + i.literal("name", "max"); + i.endEntity(); + i.endRecord(); + + i.startRecord("3"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().endRecord(); + + o.get().startRecord("3"); + o.get().endRecord(); + }); + } + + @Test + public void removeArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "remove_field('name')"), + i -> { + i.startRecord("1"); + i.literal("name", "max"); + i.literal("name", "mo"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().endRecord(); + }); + } + + @Test + public void removeArrayElementsByWildcard() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "remove_field('name.*')"), + i -> { + i.startRecord("1"); + i.literal("name", "max"); + i.literal("name", "mo"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().endRecord(); + }); + } + + @Test + public void setArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('foo[]','a','b','c')"), + i -> { + i.startRecord("1"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().startEntity("foo[]"); + o.get().literal("1", "a"); + o.get().literal("2", "b"); + o.get().literal("3", "c"); + o.get().endEntity(); + o.get().endRecord(); + }); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/111 + public void setArrayReplaceExisting() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('foo[]','a','b','c')"), + i -> { + i.startRecord("1"); + i.startEntity("foo[]"); + i.literal("1", "A"); + i.literal("2", "B"); + i.literal("3", "C"); + i.endEntity(); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().startEntity("foo[]"); + o.get().literal("1", "a"); + o.get().literal("2", "b"); + o.get().literal("3", "c"); + o.get().endEntity(); + o.get().endRecord(); + }); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/130 + public void setArrayInArrayWithAsterisk() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('foo[].*.test[]', 'test')"), + i -> { + i.startRecord("1"); + i.startEntity("foo[]"); + i.startEntity("1"); + i.literal("id", "A"); + i.endEntity(); + i.startEntity("2"); + i.literal("id", "B"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("foo[]"); + o.get().startEntity("1"); + o.get().literal("id", "A"); + o.get().startEntity("test[]"); + o.get().literal("1", "test"); + f.apply(2).endEntity(); + o.get().startEntity("2"); + o.get().literal("id", "B"); + o.get().startEntity("test[]"); + o.get().literal("1", "test"); + f.apply(3).endEntity(); + o.get().endRecord(); + }); + } + + @Test + public void setHash() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_hash('foo','a': 'b','c': 'd')"), + i -> { + i.startRecord("1"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().startEntity("foo"); + o.get().literal("a", "b"); + o.get().literal("c", "d"); + o.get().endEntity(); + o.get().endRecord(); + }); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/111 + public void setHashReplaceExisting() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_hash('foo','a': 'b','c': 'd')"), + i -> { + i.startRecord("1"); + i.startEntity("foo"); + i.literal("a", "B"); + i.literal("c", "D"); + i.endEntity(); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().startEntity("foo"); + o.get().literal("a", "b"); + o.get().literal("c", "d"); + o.get().endEntity(); + o.get().endRecord(); + }); + } + + @Test + public void paste() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "paste('my.string','m.n.z','m.n.a','m.n.b','m.n.c','m.n.d','m.n.e')", + "remove_field('m')"), + i -> { + i.startRecord("1"); + i.startEntity("m"); + i.startEntity("n"); + i.literal("a", "eeny"); + i.literal("b", "meeny"); + i.literal("c", "miny"); + i.literal("d", "moe"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().startEntity("my"); + o.get().literal("string", "eeny meeny miny moe"); + o.get().endEntity(); + o.get().endRecord(); + }); + } + + @Test + public void pasteWithCustomSep() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "paste('my.string','a','b','c','d','join_char': ', ')", + "remove_field('a','b','c','d')"), + i -> { + i.startRecord("1"); + i.literal("a", "eeny"); + i.literal("b", "meeny"); + i.literal("c", "miny"); + i.literal("d", "moe"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().startEntity("my"); + o.get().literal("string", "eeny, meeny, miny, moe"); + o.get().endEntity(); + o.get().endRecord(); + }); + } + + @Test + public void pasteWithLiteralStrings() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "paste('my.string','~Hi','a','~how are you?')", + "remove_field('a','b','c','d')"), + i -> { + i.startRecord("1"); + i.literal("a", "eeny"); + i.literal("b", "meeny"); + i.literal("c", "miny"); + i.literal("d", "moe"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().startEntity("my"); + o.get().literal("string", "Hi eeny how are you?"); + o.get().endEntity(); + o.get().endRecord(); + }); + } + + private void shouldPrintRecord(final String before, final String args, final String after, final Consumer> consumer, final String expected) { + MetafixTestHelpers.assertStdout(expected, () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + before, + "print_record(" + args + ")", + after + ), + i -> { + i.startRecord("rec1"); + i.literal("a", "eeny"); + i.literal("a", "meeny"); + i.startEntity("c"); + i.literal("d", "moe"); + i.endEntity(); + i.endRecord(); + }, o -> { + o.get().startRecord("rec1"); + o.get().literal("a", "eeny"); + o.get().literal("a", "meeny"); + o.get().startEntity("c"); + o.get().literal("d", "moe"); + o.get().endEntity(); + + if (consumer != null) { + consumer.accept(o); + } + + o.get().endRecord(); + } + ) + ); + } + + @Test + public void shouldPrintRecord() { + shouldPrintRecord("", "", "", null, + "{\"a\":[\"eeny\",\"meeny\"],\"c\":{\"d\":\"moe\"}}\n"); + } + + @Test + public void shouldPrintRecordWithPrefix() { + shouldPrintRecord("", "'<%d:%s>'", "", null, + "<1:rec1>{\"a\":[\"eeny\",\"meeny\"],\"c\":{\"d\":\"moe\"}}\n"); + } + + @Test + public void shouldPrintRecordWithPrefixAndIdField() { + shouldPrintRecord("", "'<%d:%s>', id: 'c.d'", "", null, + "<1:moe>{\"a\":[\"eeny\",\"meeny\"],\"c\":{\"d\":\"moe\"}}\n"); + } + + @Test + public void shouldPrintRecordWithHeader() { + shouldPrintRecord("", "header: '<%d:%s>'", "", null, + "<%d:%s>{\"a\":[\"eeny\",\"meeny\"],\"c\":{\"d\":\"moe\"}}\n"); + } + + @Test + public void shouldPrintRecordWithFooter() { + shouldPrintRecord("", "footer: '<%d:%s>'", "", null, + "{\"a\":[\"eeny\",\"meeny\"],\"c\":{\"d\":\"moe\"}}<%d:%s>"); + } + + @Test + public void shouldPrintRecordWithPrettyPrinting() { + shouldPrintRecord("", "pretty: 'true'", "", null, + "{\n" + + " \"a\" : [ \"eeny\", \"meeny\" ],\n" + + " \"c\" : {\n" + + " \"d\" : \"moe\"\n" + + " }\n" + + "}\n" + ); + } + + @Test + public void shouldPrintRecordAfterTransformation() { + shouldPrintRecord("add_field('x', '23')", "", "", + o -> o.get().literal("x", "23"), + "{\"a\":[\"eeny\",\"meeny\"],\"c\":{\"d\":\"moe\"},\"x\":\"23\"}\n"); + } + + @Test + public void shouldPrintRecordBeforeTransformation() { + shouldPrintRecord("", "", "add_field('x', '23')", + o -> o.get().literal("x", "23"), + "{\"a\":[\"eeny\",\"meeny\"],\"c\":{\"d\":\"moe\"}}\n"); + } + + @Test + public void shouldPrintRecordToFile() throws IOException { + MetafixTestHelpers.assertTempFile( + "{\"a\":[\"eeny\",\"meeny\"],\"c\":{\"d\":\"moe\"}}\n", + p -> shouldPrintRecord("", "destination: '" + p + "'", "", null, "")); + } + + @Test + public void hashFromArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('foo','a','b','c','d')", + "hash('foo')"), + i -> { + i.startRecord("1"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().startEntity("foo"); + o.get().literal("a", "b"); + o.get().literal("c", "d"); + o.get().endEntity(); + o.get().endRecord(); + }); + } + + @Test + public void arrayFromHash() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_hash('foo','a': 'b','c': 'd')", + "array('foo')"), + i -> { + i.startRecord("1"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("foo", "a"); + o.get().literal("foo", "b"); + o.get().literal("foo", "c"); + o.get().literal("foo", "d"); + o.get().endRecord(); + }); + } + + @Test + public void shouldCallMacro() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('test')", + "do put_macro('test')", + " add_field('test.$append', '42')", + "end", + "call_macro('test')", + "call_macro('test')" + ), + i -> { + i.startRecord("1"); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + f.apply(2).literal("test", "42"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldNotCallUnknownMacro() { + MetafixTestHelpers.assertProcessException(IllegalArgumentException.class, "Macro 'test' undefined!", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "call_macro('test')", + "call_macro('test')" + ), + i -> { + i.startRecord("1"); + i.endRecord(); + }, + o -> { + } + ) + ); + } + + @Test + public void shouldCallMacroWithVariables() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('test')", + "put_vars(a: '1', b: '2')", // global variables + "do put_macro('test', b: '22', c: '33')", // "static" local variables + " add_field('test.$append', '$[a]-$[b]-$[c]-$[d]')", + "end", + "call_macro('test', c: '333', d: '444')", // "dynamic" local variables + "call_macro('test', b: '555', d: '666')", + "add_field('vars', '$[a]-$[b]')" + ), + i -> { + i.startRecord("1"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("test", "1-22-333-444"); + o.get().literal("test", "1-555-33-666"); + o.get().literal("vars", "1-2"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldCallMacroWithVariablesPassedToNestedBinds() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "put_vars(a: '1', b: '2')", // global variables + "do put_macro('test', b: '22', c: '33')", // "static" local variables + " do once()", + " add_field('test', '$[a]-$[b]-$[c]-$[d]')", + " end", + "end", + "call_macro('test', c: '333', d: '444')", // "dynamic" local variables + "call_macro('test', b: '555', d: '666')", + "add_field('vars', '$[a]-$[b]')" + ), + i -> { + i.startRecord("1"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("test", "1-22-333-444"); + o.get().literal("vars", "1-2"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldCallMacroWithVariablesPassedToNestedConditionals() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "put_vars(a: '1', b: '2')", // global variables + "do put_macro('test', b: '22', c: '33')", // "static" local variables + " if any_equal('cond', '$[d]')", + " add_field('test', '$[a]-$[b]-$[c]-$[d]')", + " end", + "end", + "call_macro('test', c: '333', d: '444')", // "dynamic" local variables + "call_macro('test', b: '555', d: '666')", + "add_field('vars', '$[a]-$[b]')" + ), + i -> { + i.startRecord("1"); + i.literal("cond", "666"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("cond", "666"); + o.get().literal("test", "1-555-33-666"); + o.get().literal("vars", "1-2"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldNotLeakVariablesFromMacro() { + MetafixTestHelpers.assertProcessException(IllegalArgumentException.class, "Variable 'c' was not assigned!\nAssigned variables:\n{a=1, b=2}", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "put_vars(a: '1', b: '2')", // global variables + "do put_macro('test', b: '22', c: '33')", // "static" local variables + "end", + "call_macro('test', c: '333', d: '444')", // "dynamic" local variables + "call_macro('test', b: '555', d: '666')", + "add_field('test', '$[a]-$[b]-$[c]-$[d]')" + ), + i -> { + i.startRecord("1"); + i.endRecord(); + }, + o -> { + } + ) + ); + } + + @Test + public void shouldCallNestedMacro() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "do put_macro('test1', c: '23')", + " add_field('test$[a]', '42')", + " call_macro('test2', b: '$[b]', c: '$[c]')", + "end", + "do put_macro('test2')", + " add_field('test$[b]', '$[c]')", + "end", + "call_macro('test1', a: '1', b: '2')", + "call_macro('test1', a: '3', b: '4')" + ), + i -> { + i.startRecord("1"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("test1", "42"); + o.get().literal("test2", "23"); + o.get().literal("test3", "42"); + o.get().literal("test4", "23"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldIncludeCallerLocationAndTextForExceptionInMacro() { + final String format = "Error while executing Fix expression (at FILE, line %d): %s"; + + final String text1 = "call_macro('test')"; + final String text2 = "append('animals', ' is cool')"; + final String message = String.format(format, 4, text1) + " -> " + String.format(format, 2, text2); + + MetafixTestHelpers.assertThrows(FixExecutionException.class, s -> s.replaceAll("file:/.+?\\.fix", "FILE"), message, () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "do put_macro('test')", + text2, + "end", + text1 + ), + i -> { + i.startRecord("1"); + i.startEntity("animals"); + i.literal("1", "dog"); + i.literal("2", "cat"); + i.literal("3", "zebra"); + i.endEntity(); + i.endRecord(); + }, + o -> { + } + ) + ); + } + + @Test + public void shouldIncludeCallerLocationAndTextForExceptionInIncludedMacro() { + final String format = "Error while executing Fix expression (at FILE, line %d): %s"; + + final String text1 = "call_macro('test')"; + final String text2 = "append('animals', ' is cool')"; + final String message = String.format(format, 2, text1) + " -> " + String.format(format, 2, text2); + + MetafixTestHelpers.assertThrows(FixExecutionException.class, s -> s.replaceAll("file:/.+?\\.fix", "FILE"), message, () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "include('src/test/resources/org/metafacture/metafix/fixes/macro.fix')", + text1 + ), + i -> { + i.startRecord("1"); + i.startEntity("animals"); + i.literal("1", "dog"); + i.literal("2", "cat"); + i.literal("3", "zebra"); + i.endEntity(); + i.endRecord(); + }, + o -> { + } + ) + ); + } + + @Test + public void reject() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if exists('_metadata.error')", + " reject()", + "end"), + i -> { + i.startRecord("1"); + i.startEntity("_metadata"); + i.literal("error", "details"); + i.endEntity(); + i.endRecord(); + + i.startRecord("2"); + i.endRecord(); + }, o -> { + o.get().startRecord("2"); + o.get().endRecord(); + }); + } + + @Test + @MetafixToDo("Is set_array with $append something we need/want? WDCD?") + public void appendArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('nums[]', '1')", + "set_array('nums[].$append', '2', '3')"), + i -> { + i.startRecord("1"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().startEntity("nums[]"); + o.get().literal("1", "1"); + o.get().literal("2", "2"); + o.get().literal("3", "3"); + o.get().endEntity(); + o.get().endRecord(); + }); + } + + @Test + public void mixedArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('@context[]', 'https://w3id.org/kim/lrmi-profile/draft/context.jsonld')", + "set_hash('@context[].$append', '@language': 'de')"), + i -> { + i.startRecord("1"); + i.endRecord(); + }, (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("@context[]"); + o.get().literal("1", "https://w3id.org/kim/lrmi-profile/draft/context.jsonld"); + o.get().startEntity("2"); + o.get().literal("@language", "de"); + f.apply(2).endEntity(); + o.get().endRecord(); + }); + } + + @Test + public void retain() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "retain('1','3')"), + i -> { + i.startRecord("1"); + i.literal("1", "one"); + i.literal("2", "two"); + i.literal("3", "tre"); + i.literal("4", "for"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("1", "one"); + o.get().literal("3", "tre"); + o.get().endRecord(); + }); + } + + @Test // checkstyle-disable-line JavaNCSS + public void retainNested() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "retain('a.b.c','a.[cd].b','b[].2.c','c[].*.a','c[].2.b')" + ), + i -> { + i.startRecord("1"); + i.startEntity("a"); + i.startEntity("a"); + i.literal("b", "1"); + i.literal("c", "2"); + i.endEntity(); + i.startEntity("b"); + i.literal("c", "1"); + i.literal("d", "2"); + i.endEntity(); + i.startEntity("c"); + i.literal("b", "1"); + i.literal("c", "2"); + i.endEntity(); + i.endEntity(); + i.startEntity("b[]"); + i.startEntity("1"); + i.literal("a", "1"); + i.literal("b", "2"); + i.endEntity(); + i.startEntity("2"); + i.literal("a", "1"); + i.literal("b", "2"); + i.literal("c", "3"); + i.endEntity(); + i.startEntity("3"); + i.literal("c", "4"); + i.endEntity(); + i.endEntity(); + i.startEntity("c[]"); + i.startEntity("1"); + i.literal("a", "1"); + i.literal("b", "2"); + i.endEntity(); + i.startEntity("2"); + i.literal("a", "1"); + i.literal("b", "2"); + i.literal("c", "3"); + i.endEntity(); + i.startEntity("3"); + i.literal("c", "4"); + i.endEntity(); + i.endEntity(); + i.startEntity("d"); + i.literal("e", "5"); + i.endEntity(); + i.literal("e", "6"); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("a"); + o.get().startEntity("b"); + o.get().literal("c", "1"); + o.get().endEntity(); + o.get().startEntity("c"); + o.get().literal("b", "1"); + f.apply(2).endEntity(); + o.get().startEntity("b[]"); + o.get().startEntity("1"); + o.get().literal("c", "3"); + f.apply(2).endEntity(); + o.get().startEntity("c[]"); + o.get().startEntity("1"); + o.get().literal("a", "1"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("a", "1"); + o.get().literal("b", "2"); + o.get().endEntity(); + o.get().startEntity("3"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void retainNestedReservedFields() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "retain('b[].$first.b','c[].$last')" + ), + i -> { + i.startRecord("1"); + i.startEntity("b[]"); + i.startEntity("1"); + i.literal("a", "1"); + i.literal("b", "2"); + i.endEntity(); + i.startEntity("2"); + i.literal("a", "1"); + i.literal("b", "2"); + i.literal("c", "3"); + i.endEntity(); + i.startEntity("3"); + i.literal("c", "4"); + i.endEntity(); + i.endEntity(); + i.startEntity("c[]"); + i.startEntity("1"); + i.literal("a", "1"); + i.literal("b", "2"); + i.endEntity(); + i.startEntity("2"); + i.literal("a", "1"); + i.literal("b", "2"); + i.literal("c", "3"); + i.endEntity(); + i.startEntity("3"); + i.literal("c", "4"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("b[]"); + o.get().startEntity("1"); + o.get().literal("b", "2"); + f.apply(2).endEntity(); + o.get().startEntity("c[]"); + o.get().startEntity("1"); + o.get().literal("c", "4"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldDeleteEmptyArrays() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "vacuum()" + ), + i -> { + i.startRecord("1"); + i.startEntity("1[]"); + i.literal("1", "one"); + i.endEntity(); + i.startEntity("2[]"); + i.endEntity(); + i.startEntity("3[]"); + i.literal("1", "tre"); + i.endEntity(); + i.startEntity("4[]"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("1[]"); + o.get().literal("1", "one"); + o.get().endEntity(); + o.get().startEntity("3[]"); + o.get().literal("1", "tre"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldDeleteEmptyHashes() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "vacuum()" + ), + i -> { + i.startRecord("1"); + i.startEntity("1"); + i.literal("1", "one"); + i.endEntity(); + i.startEntity("2"); + i.endEntity(); + i.startEntity("3"); + i.literal("1", "tre"); + i.endEntity(); + i.startEntity("4"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("1"); + o.get().literal("1", "one"); + o.get().endEntity(); + o.get().startEntity("3"); + o.get().literal("1", "tre"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldDeleteEmptyStrings() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "vacuum()" + ), + i -> { + i.startRecord("1"); + i.literal("1", "one"); + i.literal("2", ""); + i.literal("3", "tre"); + i.literal("4", ""); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("1", "one"); + o.get().literal("3", "tre"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldDeleteEmptyNestedArrays() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "vacuum()" + ), + i -> { + i.startRecord("1"); + i.startEntity("arrays"); + i.startEntity("1[]"); + i.literal("1", "one"); + i.endEntity(); + i.startEntity("2[]"); + i.endEntity(); + i.startEntity("3[]"); + i.literal("1", "tre"); + i.endEntity(); + i.startEntity("4[]"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("arrays"); + o.get().startEntity("1[]"); + o.get().literal("1", "one"); + o.get().endEntity(); + o.get().startEntity("3[]"); + o.get().literal("1", "tre"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldDeleteEmptyNestedHashes() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "vacuum()" + ), + i -> { + i.startRecord("1"); + i.startEntity("hashes"); + i.startEntity("1"); + i.literal("1", "one"); + i.endEntity(); + i.startEntity("2"); + i.endEntity(); + i.startEntity("3"); + i.literal("1", "tre"); + i.endEntity(); + i.startEntity("4"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("hashes"); + o.get().startEntity("1"); + o.get().literal("1", "one"); + o.get().endEntity(); + o.get().startEntity("3"); + o.get().literal("1", "tre"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldDeleteEmptyNestedStrings() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "vacuum()" + ), + i -> { + i.startRecord("1"); + i.startEntity("nested"); + i.literal("1", "one"); + i.literal("2", ""); + i.literal("3", "tre"); + i.literal("4", ""); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("nested"); + o.get().literal("1", "one"); + o.get().literal("3", "tre"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldDeleteEmptyDeeplyNestedArrays() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "vacuum()" + ), + i -> { + i.startRecord("1"); + i.startEntity("arrays[]"); + i.endEntity(); + i.startEntity("hashes"); + i.startEntity("foo[]"); + i.endEntity(); + i.endEntity(); + i.literal("me", "1"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("me", "1"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldDeleteEmptyArraysInArrays() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "vacuum()" + ), + i -> { + i.startRecord("1"); + i.startEntity("arrays[]"); + i.startEntity("1[]"); + i.endEntity(); + i.startEntity("2[]"); + i.literal("1", ":-P yuck"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("arrays[]"); + o.get().startEntity("1[]"); + o.get().literal("1", ":-P yuck"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void nulls() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "retain('1','2','3')"), + i -> { + i.startRecord("1"); + i.literal("1", "one"); + i.literal("2", ""); + i.literal("3", null); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("1", "one"); + o.get().literal("2", ""); + o.get().endRecord(); + }); + } + + @Test + public void repeatToArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "nothing()" + ), + i -> { + i.startRecord("1"); + i.literal("name", "max"); + i.literal("name", "mo"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("name", "max"); + o.get().literal("name", "mo"); + o.get().endRecord(); + } + ); + } + + @Test + public void accessArrayByIndex() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "upcase('name.2')" + ), + i -> { + i.startRecord("1"); + i.literal("name", "max"); + i.literal("name", "mo"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("name", "max"); + o.get().literal("name", "MO"); + o.get().endRecord(); + } + ); + } + + @Test + public void transformSingleField() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "upcase('name')" + ), + i -> { + i.startRecord("1"); + i.literal("name", "max"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("name", "MAX"); + o.get().endRecord(); + } + ); + } + + @Test + @MetafixToDo("Same name, is replaced. Repeated fields to array?") + public void transformRepeatedField() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "upcase('name')" + ), + i -> { + i.startRecord("1"); + i.literal("name", "max"); + i.literal("name", "mo"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("name", "MAX"); + o.get().literal("name", "MO"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldEmitNestedArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "nothing()" + ), + i -> { + i.startRecord("1"); + i.startEntity("test"); + i.startEntity("zoo[]"); + i.startEntity("1"); + i.startEntity("animals[]"); + i.startEntity("1[]"); + i.literal("1", "ant"); + i.literal("2", "dog"); + i.endEntity(); + i.literal("2", "cat"); + i.startEntity("3[]"); + i.literal("1", "fish"); + i.startEntity("2[]"); + i.literal("1", "zebra"); + i.literal("2", "horse"); + i.endEntity(); + i.literal("3", "hippo"); + i.endEntity(); + i.literal("4", "giraffe"); + i.endEntity(); + i.endEntity(); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("test"); + o.get().startEntity("zoo[]"); + o.get().startEntity("1"); + o.get().startEntity("animals[]"); + o.get().startEntity("1[]"); + o.get().literal("1", "ant"); + o.get().literal("2", "dog"); + o.get().endEntity(); + o.get().literal("2", "cat"); + o.get().startEntity("3[]"); + o.get().literal("1", "fish"); + o.get().startEntity("2[]"); + o.get().literal("1", "zebra"); + o.get().literal("2", "horse"); + o.get().endEntity(); + o.get().literal("3", "hippo"); + o.get().endEntity(); + o.get().literal("4", "giraffe"); + f.apply(4).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void emitEntityForRepeatedField() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "nothing()" + ), + i -> { + i.setRepeatedFieldsToEntities(true); + i.startRecord("1"); + i.literal("name", "max"); + i.literal("name", "mo"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("name"); + o.get().literal("1", "max"); + o.get().literal("2", "mo"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void dontEmitEntityForSingleField() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "nothing()" + ), + i -> { + i.setRepeatedFieldsToEntities(true); + i.startRecord("1"); + i.literal("name", "max"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("name", "max"); + o.get().endRecord(); + } + ); + } + + @Test + public void setRepeatedFieldsToEntitiesAndSetEntityMemberName() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "nothing()" + ), + i -> { + i.setRepeatedFieldsToEntities(true); + i.setEntityMemberName("*"); + + i.startRecord("1"); + i.literal("name", "max"); + i.literal("name", "mo"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("name"); + o.get().literal("*", "max"); + o.get().literal("*", "mo"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void setRepeatedFieldsToEntitiesAndSetEntityMemberNameWithNumericalSubfield() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "nothing()" + ), + i -> { + i.setRepeatedFieldsToEntities(true); + i.setEntityMemberName("*"); + + i.startRecord("1"); + i.startEntity("1001 "); + i.literal("1", "max"); + i.literal("1", "mo"); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("1001 "); + o.get().startEntity("1"); + o.get().literal("*", "max"); + o.get().literal("*", "mo"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void setRepeatedFieldsToEntitiesWithNumericalSubfields() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "nothing()" + ), + i -> { + i.setRepeatedFieldsToEntities(true); + + i.startRecord("1"); + i.startEntity("1001 "); + i.literal("1", "max"); + i.literal("1", "mo"); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("1001 "); + o.get().startEntity("1"); + o.get().literal("1", "max"); + o.get().literal("2", "mo"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void setEntityMemberNameNoArrayMarkerOrEntity() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "nothing()" + ), + i -> { + i.setEntityMemberName("*"); // no arrays or entities, no effect + + i.startRecord("1"); + i.startEntity("1001 "); + i.literal("1", "max"); + i.literal("1", "mo"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("1001 "); + o.get().literal("1", "max"); + o.get().literal("1", "mo"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void setEntityMemberNameWithArrayMarker() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "nothing()" + ), + i -> { + i.setEntityMemberName("*"); + + i.startRecord("1"); + i.startEntity("1001 []"); + i.literal("1", "max"); + i.literal("1", "mo"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("1001 []"); + o.get().literal("*", "max"); + o.get().literal("*", "mo"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + public void shouldNotAccessArrayImplicitly() { + MetafixTestHelpers.assertExecutionException(IllegalStateException.class, "Expected String, got Array", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "upcase('name')" + ), + i -> { + i.startRecord("1"); + i.literal("name", "max"); + i.literal("name", "mo"); + i.endRecord(); + }, + o -> { + } + ) + ); + } + + @Test + public void shouldAccessArrayByWildcard() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "upcase('name.*')" + ), + i -> { + i.startRecord("1"); + i.literal("name", "max"); + i.literal("name", "mo"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("name", "MAX"); + o.get().literal("name", "MO"); + o.get().endRecord(); + } + ); + } + + @Test + public void repeatToArrayOfObjects() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "nothing()" + ), + i -> { + i.startRecord("1"); + i.startEntity("author"); + i.literal("name", "max"); + i.endEntity(); + i.startEntity("author"); + i.literal("name", "mo"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("author"); + o.get().literal("name", "max"); + o.get().endEntity(); + o.get().startEntity("author"); + o.get().literal("name", "mo"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void accessArrayOfObjectsByIndex() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "upcase('author.2.name')" + ), + i -> { + i.startRecord("1"); + i.startEntity("author"); + i.literal("name", "max"); + i.endEntity(); + i.startEntity("author"); + i.literal("name", "mo"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("author"); + o.get().literal("name", "max"); + o.get().endEntity(); + o.get().startEntity("author"); + o.get().literal("name", "MO"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void accessArrayOfObjectsByWildcard() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "upcase('author.*.name')" + ), + i -> { + i.startRecord("1"); + i.startEntity("author"); + i.literal("name", "max"); + i.endEntity(); + i.startEntity("author"); + i.literal("name", "mo"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("author"); + o.get().literal("name", "MAX"); + o.get().endEntity(); + o.get().startEntity("author"); + o.get().literal("name", "MO"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void accessArrayOfObjectsByDoListBind() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "do list('path':'author','var':'a')", + " upcase('a.name')", + "end" + ), + i -> { + i.startRecord("1"); + i.startEntity("author"); + i.literal("name", "max"); + i.endEntity(); + i.startEntity("author"); + i.literal("name", "mo"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("author"); + o.get().literal("name", "MAX"); + o.get().endEntity(); + o.get().startEntity("author"); + o.get().literal("name", "MO"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldAddRandomNumber() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "random(test, '100')" + ), + i -> { + i.startRecord("1"); + i.literal("title", "marc"); + i.literal("title", "json"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "marc"); + o.get().literal("title", "json"); + o.get().literal(ArgumentMatchers.eq("test"), ArgumentMatchers.argThat(i -> Integer.parseInt(i) < 100)); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/100 + public void shouldReplaceExistingValueWithRandomNumber() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "random(others, '100')" + ), + i -> { + i.startRecord("1"); + i.literal("others", "human"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal(ArgumentMatchers.eq("others"), ArgumentMatchers.argThat(i -> Integer.parseInt(i) < 100)); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldAddRandomNumberToMarkedArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "random('animals[].$append', '100')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.literal("1", "cat"); + i.literal("2", "dog"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("animals[]"); + o.get().literal("1", "cat"); + o.get().literal("2", "dog"); + o.get().literal(ArgumentMatchers.eq("3"), ArgumentMatchers.argThat(i -> Integer.parseInt(i) < 100)); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldAddObjectWithRandomNumberToMarkedArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('bnimals[]')", + "random('bnimals[].$append.number', '100')" + ), + i -> { + i.startRecord("1"); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("bnimals[]"); + o.get().startEntity("1"); + o.get().literal(ArgumentMatchers.eq("number"), ArgumentMatchers.argThat(i -> Integer.parseInt(i) < 100)); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldAddRandomNumberToUnmarkedArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "random('animals.$append', '100')" + ), + i -> { + i.startRecord("1"); + i.literal("animals", "cat"); + i.literal("animals", "dog"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("animals", "cat"); + o.get().literal("animals", "dog"); + o.get().literal(ArgumentMatchers.eq("animals"), ArgumentMatchers.argThat(i -> Integer.parseInt(i) < 100)); + o.get().endRecord(); + } + ); + } + + @Test + @MetafixToDo("See https://github.com/metafacture/metafacture-fix/issues/100") + public void shouldNotAppendRandomNumberToHash() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "random('animals.$append', '100')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals"); + i.literal("1", "cat"); + i.literal("2", "dog"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("animals"); + o.get().literal("1", "cat"); + o.get().literal("2", "dog"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldRenameFieldsInHash() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "rename(your, '[ae]', X)" + ), + i -> { + i.startRecord("1"); + i.startEntity("your"); + i.literal("name", "nicolas"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("your"); + o.get().literal("nXmX", "nicolas"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/100 + public void shouldRecursivelyRenameFieldsInHash() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "rename(others, ani, QR)" + ), + i -> { + i.startRecord("1"); + i.startEntity("others"); + i.literal("animal", "human"); + i.literal("canister", "metall"); + i.startEntity("area"); + i.literal("ani", "test"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("others"); + o.get().literal("QRmal", "human"); + o.get().literal("cQRster", "metall"); + o.get().startEntity("area"); + o.get().literal("QR", "test"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/100 + public void shouldRecursivelyRenameFieldsInArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "rename('animals[]', ani, XY)" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.startEntity("1"); + i.literal("animal", "dog"); + i.endEntity(); + i.startEntity("2"); + i.literal("animal", "cat"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("animals[]"); + o.get().startEntity("1"); + o.get().literal("XYmal", "dog"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("XYmal", "cat"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + @MetafixToDo("java.lang.ArrayIndexOutOfBoundsException: 0; see https://github.com/metafacture/metafacture-fix/issues/100") + public void shouldRenameAllFieldsInHash() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "rename('.', ani, XY)" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals"); + i.literal("animal", "dog"); + i.literal("animal", "cat"); + i.endEntity(); + i.startEntity("others"); + i.literal("animal", "human"); + i.literal("canister", "metall"); + i.startEntity("area"); + i.literal("ani", "test"); + i.endEntity(); + i.endEntity(); + i.startEntity("fictional"); + i.literal("animal", "unicorn"); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("XYmals"); + o.get().literal("XYmal", "dog"); + o.get().literal("XYmal", "cat"); + o.get().endEntity(); + o.get().startEntity("others"); + o.get().literal("XYmal", "human"); + o.get().literal("cXYster", "metall"); + o.get().startEntity("area"); + o.get().literal("XY", "test"); + f.apply(2).endEntity(); + o.get().startEntity("fictional"); + o.get().literal("XYmal", "unicorn"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + @MetafixToDo("java.lang.ArrayIndexOutOfBoundsException: 0; see https://github.com/metafacture/metafacture-fix/issues/121") + public void shouldRenameArrayFieldWithAsterisk() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "rename('OTHERS[].*', 'd', 'XY')" + ), + i -> { + i.startRecord("1"); + i.startEntity("OTHERS[]"); + i.startEntity("1"); + i.literal("name", "Jake"); + i.startEntity("dnimals[]"); + i.literal("1", "dog"); + i.literal("2", "zebra"); + i.literal("3", "cat"); + i.endEntity(); + i.startEntity("dumbers[]"); + i.literal("1", "7"); + i.literal("2", "2"); + i.literal("3", "1"); + i.literal("4", "10"); + i.endEntity(); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("OTHERS[]"); + o.get().startEntity("1"); + o.get().literal("name", "Jake"); + o.get().startEntity("XYnimals[]"); + o.get().literal("1", "dog"); + o.get().literal("2", "zebra"); + o.get().literal("3", "cat"); + o.get().endEntity(); + o.get().startEntity("XYumbers[]"); + o.get().literal("1", "7"); + o.get().literal("2", "2"); + o.get().literal("3", "1"); + o.get().literal("4", "10"); + f.apply(3).endEntity(); + o.get().endRecord(); + } + ); + } + + private void shouldNotAddTimestamp(final String options, final String message) { + MetafixTestHelpers.assertProcessException(MorphBuildException.class, message, () -> shouldAddTimestamp(options, "")); + } + + private void shouldAddTimestamp(final String options, final String pattern) { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "timestamp(test" + options + ")" + ), + i -> { + i.startRecord("1"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal(ArgumentMatchers.eq("test"), ArgumentMatchers.matches(pattern)); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldAddTimestamp() { + shouldAddTimestamp("", "\\d+"); + } + + @Test + public void shouldAddTimestampWithFormat() { + shouldAddTimestamp(", format: 'yyyy-MM-dd'", "\\d{4}-\\d{2}-\\d{2}"); + } + + @Test + public void shouldNotAddTimestampWithUnsupportedFormat() { + shouldNotAddTimestamp(", format: \"'\"", "The date/time format ''' is not supported. "); + } + + @Test + public void shouldAddTimestampWithLanguage() { + shouldAddTimestamp(", format: 'yyyy-MM-dd G', language: 'pl'", "\\d{4}-\\d{2}-\\d{2} n\\.e\\."); + } + + @Test + public void shouldNotAddTimestampWithUnsupportedLanguage() { + shouldNotAddTimestamp(", language: '--'", "Language '--' not supported."); + } + + @Test + public void shouldAddTimestampWithTimezone() { + shouldAddTimestamp(", format: 'yyyy-MM-dd Z', timezone: 'UTC'", "\\d{4}-\\d{2}-\\d{2} \\+0000"); + } + + @Test + @MetafixToDo("See https://github.com/metafacture/metafacture-fix/pull/170") + public void shouldNotSplitLiteralName() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "nothing()" + ), + i -> { + i.startRecord("1"); + i.literal("123. ", "foo"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("123. ", "foo"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldNotSplitEntityName() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "nothing()" + ), + i -> { + i.startRecord("1"); + i.startEntity("123. "); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("123. "); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + +} diff --git a/metafix/src/test/java/org/metafacture/metafix/MetafixScriptTest.java b/metafix/src/test/java/org/metafacture/metafix/MetafixScriptTest.java new file mode 100644 index 000000000..2141ed055 --- /dev/null +++ b/metafix/src/test/java/org/metafacture/metafix/MetafixScriptTest.java @@ -0,0 +1,620 @@ +/* + * Copyright 2021 hbz NRW + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix; + +import org.metafacture.framework.StreamReceiver; +import org.metafacture.framework.helpers.DefaultStreamReceiver; + +import com.google.common.collect.ImmutableMap; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; +import org.slf4j.Logger; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Map; +import java.util.function.Consumer; +import java.util.function.Supplier; +import java.util.regex.Pattern; + +/** + * Tests Metafix script level methods. + */ +@ExtendWith(MockitoExtension.class) +public class MetafixScriptTest { + + private static final String MAP_NAME = "testMap"; + + private static final String CSV_MAP = "src/test/resources/org/metafacture/metafix/maps/test.csv"; + private static final String TSV_MAP = "src/test/resources/org/metafacture/metafix/maps/test.tsv"; + + @Mock(name = "org.metafacture.metafix.FixMethod") + private Logger fixMethodLogger; + + @Mock + private StreamReceiver streamReceiver; + + public MetafixScriptTest() { + } + + @Test + public void shouldPutSingleVariable() { + assertVar("put_var('varName', 'value')", + null, + ImmutableMap.of("varName", "value")); + } + + @Test + public void shouldPutMultipleVariables() { + assertVar("put_var('varName', 'value')\nput_var('varName2', 'value2')", + null, + ImmutableMap.of("varName", "value", "varName2", "value2")); + } + + @Test + public void shouldPutMultipleVariablesFromMap() { + assertVar("put_vars(varName: 'value', varName2: 'value2')", + null, + ImmutableMap.of("varName", "value", "varName2", "value2")); + } + + @Test + public void shouldResolveVariablesInSingleVariable() { + assertVar("put_var('varName', 'value$[var]')", + ImmutableMap.of("var", "1"), + ImmutableMap.of("varName", "value1")); + } + + @Test + public void shouldResolveVariablesInMultipleVariables() { + assertVar("put_var('varName', 'value$[var]')\nput_var('$[varName]Var', 'value2')", + ImmutableMap.of("var", "1"), + ImmutableMap.of("varName", "value1", "value1Var", "value2")); + } + + @Test + public void shouldResolveVariablesInOptionsKeys() { + assertVar("put_vars('varName$[var]': 'value')", + ImmutableMap.of("var", "1"), + ImmutableMap.of("varName1", "value")); + } + + @Test + public void shouldResolveVariablesInOptionsValues() { + assertVar("put_vars('varName': 'value$[var]')", + ImmutableMap.of("var", "1"), + ImmutableMap.of("varName", "value1")); + } + + @Test + public void shouldResolveVariablesInOptionsFromPreviousMap() { + assertVar("put_vars('varName': 'value$[var]')\nput_vars('$[varName]Var': 'value2')", + ImmutableMap.of("var", "1"), + ImmutableMap.of("varName", "value1", "value1Var", "value2")); + } + + @Test + public void shouldNotResolveVariablesInOptionsFromCurrentMap() { + MetafixTestHelpers.assertProcessException(IllegalArgumentException.class, "Variable 'varName' was not assigned!\nAssigned variables:\n{var=1}", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "put_vars(varName: 'value$[var]', '$[varName]Var': 'value2')" + ), + ImmutableMap.of("var", "1"), + i -> { + i.startRecord(""); + i.endRecord(); + }, + o -> { + } + ) + ); + } + + @Test + public void shouldPutEmptyInternalMap() { + assertMap("put_map('" + MAP_NAME + "')", MAP_NAME); + } + + @Test + public void shouldPutMultipleInternalMaps() { + assertFix("put_map('" + MAP_NAME + "')\nput_map('" + MAP_NAME + "2')", f -> { + assertMap(f, MAP_NAME); + assertMap(f, MAP_NAME + "2"); + }); + } + + @Test + public void shouldPutInternalMapWithOptions() { + assertMap("put_map('" + MAP_NAME + "', k1: 'v1', k2: 'v2')", MAP_NAME); + } + + @Test + public void shouldPutExternalFileMap() { + assertMap("put_filemap('" + CSV_MAP + "')", CSV_MAP); + } + + @Test + public void shouldNotPutRelativeExternalFileMapFromInlineScript() { + final String mapFile = "../maps/test.csv"; + + MetafixTestHelpers.assertProcessException(IllegalArgumentException.class, "Cannot resolve relative path: " + mapFile, () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "put_filemap('" + mapFile + "')" + ), + i -> { + i.startRecord(""); + i.endRecord(); + }, + o -> { + } + ) + ); + } + + @Test + public void shouldPutRelativeExternalFileMapFromExternalScript() { + assertMap("src/test/resources/org/metafacture/metafix/fixes/filemap.fix", MAP_NAME); + } + + @Test + public void shouldPutMultipleExternalFileMaps() { + assertFix("put_filemap('" + CSV_MAP + "')\nput_filemap('" + TSV_MAP + "')", f -> { + assertMap(f, CSV_MAP); + assertMap(f, TSV_MAP); + }); + } + + @Test + public void shouldPutExternalFileMapWithName() { + assertMap("put_filemap('" + CSV_MAP + "', '" + MAP_NAME + "')", MAP_NAME); + } + + @Test + public void shouldPutExternalFileMapWithOptions() { + assertMap("put_filemap('" + TSV_MAP + "', sep_char: '\t')", TSV_MAP); + } + + @Test + public void shouldPutExternalFileMapWithNameAndOptions() { + assertMap("put_filemap('" + TSV_MAP + "', '" + MAP_NAME + "', sep_char: '\t')", MAP_NAME); + } + + @Test + public void shouldLog() { + assertFix("log('test')", f -> { + Mockito.verify(fixMethodLogger).info("test"); + Mockito.verifyNoMoreInteractions(fixMethodLogger); + }); + } + + @Test + public void shouldLogWithLevel() { + assertFix("log('test', level: 'DEBUG')", f -> { + Mockito.verify(fixMethodLogger).debug("test"); + Mockito.verifyNoMoreInteractions(fixMethodLogger); + }); + } + + @Test + public void shouldNotLogWithUnsupportedLevel() { + MetafixTestHelpers.assertProcessException(IllegalArgumentException.class, "Unsupported log level: FATAL", () -> + assertFix("log('test', level: 'FATAL')", f -> { }) + ); + } + + @Test + public void shouldDoNothing() { + assertFix("nothing()", f -> { }); + } + + @Test + public void shouldIncludeFixFile() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('trace')", + "add_field('trace.$append', 'before include')", + "include('src/test/resources/org/metafacture/metafix/fixes/base.fix')", + "add_field('trace.$append', 'after include')" + ), + i -> { + i.startRecord("1"); + i.literal("record", "1"); + i.endRecord(); + + i.startRecord("2"); + i.literal("record", "2"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("record", "1"); + o.get().literal("trace", "before include"); + o.get().literal("trace", "base 1"); + o.get().literal("trace", "after include"); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().literal("record", "2"); + o.get().literal("trace", "before include"); + o.get().literal("trace", "base 2"); + o.get().literal("trace", "after include"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldIncludeFixFileWithVariables() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "include('src/test/resources/org/metafacture/metafix/fixes/vars.fix', a: '1', b: '23')", + "include('src/test/resources/org/metafacture/metafix/fixes/vars.fix', a: '2', b: '42')" + ), + i -> { + i.startRecord("1"); + i.endRecord(); + + i.startRecord("2"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("test1", "1-23"); + o.get().literal("test2", "1-42"); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().literal("test1", "2-23"); + o.get().literal("test2", "2-42"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldNotLeakVariablesFromIncludingFixFile() { + MetafixTestHelpers.assertProcessException(IllegalArgumentException.class, "Variable 'a' was not assigned!\nAssigned variables:\n{}", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "include('src/test/resources/org/metafacture/metafix/fixes/vars.fix', a: '1', b: '23')", + "add_field('vars', '$[a]-$[b]')" + ), + i -> { + i.startRecord("1"); + i.endRecord(); + }, + o -> { + } + ) + ); + } + + @Test + public void shouldIncludeFixFileInBind() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('trace')", + "add_field('trace.$append', 'before bind')", + "do list(path: 'data', 'var': '$i')", + " paste('trace.$append', '~before include', '$i')", + " include('src/test/resources/org/metafacture/metafix/fixes/var.fix')", + " paste('trace.$append', '~after include', '$i')", + "end", + "add_field('trace.$append', 'after bind')" + ), + i -> { + i.startRecord("1"); + i.literal("record", "1"); + i.literal("data", "marc"); + i.literal("data", "json"); + i.endRecord(); + + i.startRecord("2"); + i.literal("record", "2"); + i.literal("data", "test"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("record", "1"); + o.get().literal("data", "marc"); + o.get().literal("data", "json"); + o.get().literal("trace", "before bind"); + o.get().literal("trace", "before include marc"); + o.get().literal("trace", "marc 1"); + o.get().literal("trace", "after include MARC"); + o.get().literal("trace", "before include json"); + o.get().literal("trace", "json 1"); + o.get().literal("trace", "after include JSON"); + o.get().literal("trace", "after bind"); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().literal("record", "2"); + o.get().literal("data", "test"); + o.get().literal("trace", "before bind"); + o.get().literal("trace", "before include test"); + o.get().literal("trace", "test 2"); + o.get().literal("trace", "after include TEST"); + o.get().literal("trace", "after bind"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldNotIncludeRelativeFixFileFromInlineScript() { + MetafixTestHelpers.assertProcessException(IllegalArgumentException.class, "Cannot resolve relative path: ./base.fix", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "include('src/test/resources/org/metafacture/metafix/fixes/include.fix')" + ), + i -> { + i.startRecord(""); + i.endRecord(); + }, + o -> { + } + ) + ); + } + + @Test + public void shouldIncludeFixFileFromExternalScript() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "src/test/resources/org/metafacture/metafix/fixes/include.fix" + ), + i -> { + i.startRecord("1"); + i.literal("record", "1"); + i.endRecord(); + + i.startRecord("2"); + i.literal("record", "2"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("record", "1"); + o.get().literal("trace", "before include"); + o.get().literal("trace", "base 1"); + o.get().literal("trace", "after include"); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().literal("record", "2"); + o.get().literal("trace", "before include"); + o.get().literal("trace", "base 2"); + o.get().literal("trace", "after include"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldIncludeNestedFixFileFromExternalScript() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "src/test/resources/org/metafacture/metafix/fixes/nested.fix" + ), + i -> { + i.startRecord("1"); + i.literal("record", "1"); + i.endRecord(); + + i.startRecord("2"); + i.literal("record", "2"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("record", "1"); + o.get().literal("trace", "before nested"); + o.get().literal("trace", "before include"); + o.get().literal("trace", "base 1"); + o.get().literal("trace", "after include"); + o.get().literal("trace", "after nested"); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().literal("record", "2"); + o.get().literal("trace", "before nested"); + o.get().literal("trace", "before include"); + o.get().literal("trace", "base 2"); + o.get().literal("trace", "after include"); + o.get().literal("trace", "after nested"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldIncludeLocationAndTextInExecutionException() { + final String fixFile = "src/test/resources/org/metafacture/metafix/fixes/error.fix"; + final String message = "Error while executing Fix expression (at FILE, line 2): append(\"animals\", \" is cool\")"; + + MetafixTestHelpers.assertThrows(FixExecutionException.class, s -> s.replaceAll("file:/.+?" + Pattern.quote(fixFile), "FILE"), message, () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(fixFile), + i -> { + i.startRecord("1"); + i.startEntity("animals"); + i.literal("1", "dog"); + i.literal("2", "cat"); + i.literal("3", "zebra"); + i.endEntity(); + i.endRecord(); + }, + o -> { + } + ) + ); + } + + private void assertStrictness(final Metafix.Strictness strictness, final String fixDef, final boolean stubLogging, final Consumer in, final Consumer> out) { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "add_field('before', '')", + fixDef, + "add_field('after', '')" + ), + i -> { + if (in != null) { + in.accept(i); + } + + final Metafix.Strictness strictnessSpy = Mockito.spy(strictness); + i.setStrictness(strictnessSpy); + + if (stubLogging) { + Mockito.doNothing().when(strictnessSpy).log(Mockito.any(), Mockito.any()); + } + + i.startRecord("1"); + i.literal("data", "foo"); + i.endRecord(); + + i.startRecord("2"); + i.literal("data", "foo"); + i.literal("data", "bar"); + i.endRecord(); + + i.startRecord("3"); + i.literal("data", "bar"); + i.endRecord(); + }, + out + ); + + // TODO: Test logging statements + } + + private void assertStrictness(final Metafix.Strictness strictness, final boolean stubLogging, final Consumer> out) { + assertStrictness(strictness, "upcase('data')", stubLogging, null, out); + } + + @Test + public void shouldSkipExpressionOnExecutionException() { + assertStrictness(Metafix.Strictness.EXPRESSION, true, o -> { + o.get().startRecord("1"); + o.get().literal("data", "FOO"); + o.get().literal("before", ""); + o.get().literal("after", ""); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().literal("data", "foo"); + o.get().literal("data", "bar"); + o.get().literal("before", ""); + o.get().literal("after", ""); + o.get().endRecord(); + + o.get().startRecord("3"); + o.get().literal("data", "BAR"); + o.get().literal("before", ""); + o.get().literal("after", ""); + o.get().endRecord(); + }); + } + + @Test + public void shouldSkipRecordOnExecutionException() { + assertStrictness(Metafix.Strictness.RECORD, true, o -> { + o.get().startRecord("1"); + o.get().literal("data", "FOO"); + o.get().literal("before", ""); + o.get().literal("after", ""); + o.get().endRecord(); + + o.get().startRecord("3"); + o.get().literal("data", "BAR"); + o.get().literal("before", ""); + o.get().literal("after", ""); + o.get().endRecord(); + }); + } + + @Test + public void shouldAbortProcessOnExecutionException() { + MetafixTestHelpers.assertExecutionException(IllegalStateException.class, "Expected String, got Array", () -> + assertStrictness(Metafix.Strictness.PROCESS, false, o -> { + }) + ); + } + + @Test + public void shouldAbortProcessOnProcessException() { + MetafixTestHelpers.assertProcessException(IllegalArgumentException.class, "No enum constant org.metafacture.metafix.FixMethod.foo", () -> + assertStrictness(Metafix.Strictness.EXPRESSION, "foo()", false, null, o -> { + }) + ); + } + + @Test + public void shouldOptionallySkipExpressionOnProcessException() { + assertStrictness(Metafix.Strictness.EXPRESSION, "upcase()", true, i -> i.setStrictnessHandlesProcessExceptions(true), o -> { + o.get().startRecord("1"); + o.get().literal("data", "foo"); + o.get().literal("before", ""); + o.get().literal("after", ""); + o.get().endRecord(); + + o.get().startRecord("2"); + o.get().literal("data", "foo"); + o.get().literal("data", "bar"); + o.get().literal("before", ""); + o.get().literal("after", ""); + o.get().endRecord(); + + o.get().startRecord("3"); + o.get().literal("data", "bar"); + o.get().literal("before", ""); + o.get().literal("after", ""); + o.get().endRecord(); + }); + } + + private void assertVar(final String fixDef, final Map vars, final Map result) { + assertFix(fixDef, vars, f -> result.forEach((k, v) -> Assertions.assertEquals(v, f.getVars().get(k)))); + } + + private void assertMap(final String fixDef, final String mapName) { + assertFix(fixDef, f -> assertMap(f, mapName)); + } + + private void assertMap(final Metafix metafix, final String mapName) { + Assertions.assertTrue(metafix.getMapNames().contains(mapName)); + Assertions.assertNotNull(metafix.getMap(mapName)); + } + + private void assertFix(final String fixDef, final Consumer consumer) { + assertFix(fixDef, null, consumer); + } + + private void assertFix(final String fixDef, final Map vars, final Consumer consumer) { + try { + final Metafix metafix = vars == null ? new Metafix(fixDef) : new Metafix(fixDef, vars); + + // Prepare and trigger script execution + metafix.setReceiver(new DefaultStreamReceiver()); + metafix.startRecord(""); + metafix.endRecord(); + + consumer.accept(metafix); + } + catch (final IOException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/metafix/src/test/java/org/metafacture/metafix/MetafixSelectorTest.java b/metafix/src/test/java/org/metafacture/metafix/MetafixSelectorTest.java new file mode 100644 index 000000000..e914572d7 --- /dev/null +++ b/metafix/src/test/java/org/metafacture/metafix/MetafixSelectorTest.java @@ -0,0 +1,86 @@ +/* + * Copyright 2021 Fabian Steeg, hbz + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix; + +import org.metafacture.framework.StreamReceiver; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.Arrays; + +/** + * Test Metafix selectors. + * + * @author Fabian Steeg + * + */ +@ExtendWith(MockitoExtension.class) +@ExtendWith(MetafixToDo.Extension.class) +public final class MetafixSelectorTest { + + @Mock + private StreamReceiver streamReceiver; + + public MetafixSelectorTest() { } + + @Test + @MetafixToDo("Support reject() with condition") + public void reject() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "reject exists(error)"), + i -> { + i.startRecord("1"); + i.literal("error", "details"); + i.endRecord(); + + i.startRecord("2"); + i.endRecord(); + }, o -> { + o.get().startRecord("2"); + o.get().literal("name", "Mary"); + o.get().endRecord(); + }); + } + + @Test + public void rejectWithExplicitConditional() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if exists(error)", + " reject()", + "end" + ), + i -> { + i.startRecord("1"); + i.literal("error", "details"); + i.endRecord(); + + i.startRecord("2"); + i.literal("name", "Mary"); + i.endRecord(); + }, + o -> { + o.get().startRecord("2"); + o.get().literal("name", "Mary"); + o.get().endRecord(); + } + ); + } + +} diff --git a/metafix/src/test/java/org/metafacture/metafix/MetafixTest.java b/metafix/src/test/java/org/metafacture/metafix/MetafixTest.java new file mode 100644 index 000000000..5ddbe182b --- /dev/null +++ b/metafix/src/test/java/org/metafacture/metafix/MetafixTest.java @@ -0,0 +1,169 @@ +/* + * Copyright 2021 hbz NRW + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix; + +import org.metafacture.metamorph.api.Maps; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * Tests Metafix API methods. + */ +public class MetafixTest { + + private static final String MAP_NAME = "testMap"; + private static final String KEY = "outName"; + private static final String VALUE = "testValue"; + + public MetafixTest() { + } + + @Test + public void shouldPutVar() { + final Metafix metafix = new Metafix(); + metafix.getVars().put(KEY, VALUE); + + Assertions.assertEquals(VALUE, metafix.getVars().get(KEY)); + } + + @Test + public void shouldPutVarWithMutableMap() { + final Map map = new HashMap<>(); + map.put(KEY, VALUE); + + final Metafix metafix = new Metafix(map); + metafix.getVars().put(KEY + "2", VALUE + "2"); + + Assertions.assertEquals(VALUE, metafix.getVars().get(KEY)); + Assertions.assertEquals(VALUE + "2", metafix.getVars().get(KEY + "2")); + } + + @Test + public void shouldPutVarWithImmutableMap() { + final Map map = new HashMap<>(); + map.put(KEY, VALUE); + + final Metafix metafix = new Metafix(Collections.unmodifiableMap(map)); + metafix.getVars().put(KEY + "2", VALUE + "2"); + + Assertions.assertEquals(VALUE, metafix.getVars().get(KEY)); + Assertions.assertEquals(VALUE + "2", metafix.getVars().get(KEY + "2")); + } + + @Test + public void shouldGetMapNames() { + final Metafix metafix = new Metafix(); + metafix.putMap(MAP_NAME, new HashMap<>()); + metafix.putMap(MAP_NAME + "2", new HashMap<>()); + + final Collection actualNames = metafix.getMapNames(); + final Collection expectedNames = Arrays.asList(MAP_NAME, MAP_NAME + "2"); + + Assertions.assertTrue(actualNames.containsAll(expectedNames), "missing names"); + Assertions.assertTrue(expectedNames.containsAll(actualNames), "unexpected names"); + } + + @Test + public void shouldGetMap() { + final Map map = new HashMap<>(); + + final Metafix metafix = new Metafix(); + metafix.putMap(MAP_NAME, map); + + Assertions.assertSame(map, metafix.getMap(MAP_NAME)); + } + + @Test + public void shouldGetEmptyUnknownMap() { + final Metafix metafix = new Metafix(); + Assertions.assertEquals(new HashMap<>(), metafix.getMap(MAP_NAME)); + } + + @Test + public void shouldPutValueIntoNewMap() { + final Metafix metafix = new Metafix(); + metafix.putValue(MAP_NAME, KEY, VALUE); + + Assertions.assertNotNull(metafix.getMap(MAP_NAME)); + Assertions.assertEquals(VALUE, metafix.getValue(MAP_NAME, KEY)); + } + + @Test + public void shouldPutValueIntoExistingMap() { + final Map map = new HashMap<>(); + + final Metafix metafix = new Metafix(); + metafix.putMap(MAP_NAME, map); + metafix.putValue(MAP_NAME, KEY, VALUE); + + Assertions.assertEquals(VALUE, map.get(KEY)); + Assertions.assertEquals(VALUE, metafix.getValue(MAP_NAME, KEY)); + } + + @Test + public void shouldGetValueFromMap() { + final Map map = new HashMap<>(); + map.put(KEY, VALUE); + + final Metafix metafix = new Metafix(); + metafix.putMap(MAP_NAME, map); + + Assertions.assertEquals(VALUE, metafix.getValue(MAP_NAME, KEY)); + } + + @Test + public void shouldNotGetValueFromUnknownMap() { + final Metafix metafix = new Metafix(); + Assertions.assertNull(metafix.getValue(MAP_NAME, KEY)); + } + + @Test + public void shouldNotGetValueForUnknownKey() { + final Metafix metafix = new Metafix(); + metafix.putMap(MAP_NAME, new HashMap<>()); + + Assertions.assertNull(metafix.getValue(MAP_NAME, KEY)); + } + + @Test + public void shouldGetDefaultValueForUnknownKey() { + final Map map = new HashMap<>(); + map.put(Maps.DEFAULT_MAP_KEY, VALUE); + + final Metafix metafix = new Metafix(); + metafix.putMap(MAP_NAME, map); + + Assertions.assertEquals(VALUE, metafix.getValue(MAP_NAME, KEY)); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/79 + public void shouldThrowExceptionForInvalidFixFile() { + final String fixFile = "src/test/resources/org/metafacture/metafix/fixes/invalid.fix"; + MetafixTestHelpers.assertThrows(FixParseException.class, "Invalid FixStandaloneSetup resource: " + fixFile, () -> new Metafix(fixFile)); + + // TODO: Test logging statements + } + +} diff --git a/metafix/src/test/java/org/metafacture/metafix/MetafixTestHelpers.java b/metafix/src/test/java/org/metafacture/metafix/MetafixTestHelpers.java new file mode 100644 index 000000000..ce5f7a5b8 --- /dev/null +++ b/metafix/src/test/java/org/metafacture/metafix/MetafixTestHelpers.java @@ -0,0 +1,173 @@ +/* + * Copyright 2020, 2021 hbz NRW + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix; + +import org.metafacture.framework.StreamReceiver; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.function.Executable; +import org.mockito.InOrder; +import org.mockito.Mockito; +import org.mockito.exceptions.base.MockitoAssertionError; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.PrintStream; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import java.util.function.IntFunction; +import java.util.function.Supplier; +import java.util.function.UnaryOperator; + +/** + * Helper functions for Metafix tests. + * + * @author Jens Wille (metamorph.TestHelpers) + * @author Fabian Steeg (MetafixTestHelpers) + */ +public final class MetafixTestHelpers { + + private MetafixTestHelpers() { + } + + public static void assertExecutionException(final Class expectedClass, final String expectedMessage, final Executable executable) { + assertThrows(FixExecutionException.class, expectedClass, expectedMessage, executable, UnaryOperator.identity()); + } + + public static void assertProcessException(final Class expectedClass, final String expectedMessage, final Executable executable) { + assertProcessException(expectedClass, UnaryOperator.identity(), expectedMessage, executable); + } + + public static void assertProcessException(final Class expectedClass, final UnaryOperator operator, final String expectedMessage, final Executable executable) { + assertThrows(FixProcessException.class, expectedClass, expectedMessage, executable, operator); + } + + public static void assertThrows(final Class expectedClass, final String expectedMessage, final Executable executable) { + assertThrows(expectedClass, null, expectedMessage, executable, UnaryOperator.identity()); + } + + public static void assertThrows(final Class expectedClass, final UnaryOperator operator, final String expectedMessage, final Executable executable) { + assertThrows(expectedClass, null, expectedMessage, executable, operator); + } + + private static void assertThrows(final Class exceptionClass, final Class expectedClass, final String expectedMessage, final Executable executable, final UnaryOperator operator) { + final Throwable thrownException = Assertions.assertThrows(exceptionClass, executable); + final Throwable actualException; + + if (expectedClass != null) { + actualException = thrownException.getCause(); + Assertions.assertSame(expectedClass, actualException.getClass()); + } + else { + actualException = thrownException; + } + + Assertions.assertEquals(expectedMessage, operator.apply(actualException.getMessage())); + } + + public static void assertStdout(final String expected, final Runnable runnable) { + final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + final PrintStream originalStdout = System.out; + + try (PrintStream printStream = new PrintStream(outputStream)) { + System.setOut(printStream); + runnable.run(); + } + finally { + System.setOut(originalStdout); + } + + Assertions.assertEquals(expected, outputStream.toString()); + } + + public static void assertTempFile(final String expected, final Consumer consumer) throws IOException { + final File tempFile = File.createTempFile("metafixTestHelpers", ""); + tempFile.deleteOnExit(); + + consumer.accept(tempFile.getPath()); + Assertions.assertEquals(expected, new String(Files.readAllBytes(tempFile.toPath()))); + } + + public static void assertFix(final StreamReceiver receiver, final List fixDef, final Consumer in, + final Consumer> out) { + assertFix(receiver, fixDef, in, (s, f) -> out.accept(s), Metafix.NO_VARS); + } + + public static void assertFix(final StreamReceiver receiver, final List fixDef, final Consumer in, + final BiConsumer, IntFunction> out) { + assertFix(receiver, fixDef, in, out, Metafix.NO_VARS); + } + + public static void assertFix(final StreamReceiver receiver, final List fixDef, final Map vars, + final Consumer in, final Consumer> out) { + assertFix(receiver, fixDef, in, (s, f) -> out.accept(s), vars); + } + + public static void assertFix(final StreamReceiver receiver, final List fixDef, final Map vars, + final Consumer in, final BiConsumer, IntFunction> out) { + assertFix(receiver, fixDef, in, out, vars); + } + + private static void assertFix(final StreamReceiver receiver, final List fixLines, final Consumer in, + final BiConsumer, IntFunction> out, final Map vars) { + final String fixString = String.join("\n", fixLines); + final Metafix metafix = fix(receiver, fixString, vars); + final InOrder ordered = Mockito.inOrder(receiver); + try { + in.accept(metafix); + out.accept(() -> ordered.verify(receiver), i -> ordered.verify(receiver, Mockito.times(i))); + ordered.verifyNoMoreInteractions(); + Mockito.verifyNoMoreInteractions(receiver); + } + catch (final MockitoAssertionError e) { + System.out.println("\nFix string: " + fixString); + System.out.println(Mockito.mockingDetails(receiver).printInvocations()); + throw e; + } + } + + private static Metafix fix(final StreamReceiver receiver, final String fix, final Map vars) { + Metafix metafix = null; + try { + metafix = new Metafix(fix, vars); + metafix.setReceiver(receiver); + } + catch (final IOException e) { + e.printStackTrace(); + } + return metafix; + } + + public static void assertEmittedFields(final Value.Hash hash, final List expectedFields, final List expectedValues) { + final List actualFields = new ArrayList<>(); + final List actualValues = new ArrayList<>(); + + hash.forEach((f, v) -> { + actualFields.add(f); + actualValues.add(v); + }); + + Assertions.assertEquals(expectedFields, actualFields); + Assertions.assertEquals(expectedValues, actualValues); + } + +} diff --git a/metafix/src/test/java/org/metafacture/metafix/MetafixToDo.java b/metafix/src/test/java/org/metafacture/metafix/MetafixToDo.java new file mode 100644 index 000000000..121f32b6a --- /dev/null +++ b/metafix/src/test/java/org/metafacture/metafix/MetafixToDo.java @@ -0,0 +1,123 @@ +package org.metafacture.metafix; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.AfterAllCallback; +import org.junit.jupiter.api.extension.ConditionEvaluationResult; +import org.junit.jupiter.api.extension.ExecutionCondition; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.InvocationInterceptor; +import org.junit.jupiter.api.extension.ReflectiveInvocationContext; +import org.opentest4j.TestAbortedException; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import java.lang.reflect.Method; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Consumer; + +@Target(ElementType.METHOD) +@Retention(RetentionPolicy.RUNTIME) +@ExtendWith(MetafixToDo.Handler.class) +public @interface MetafixToDo { + + String value(); + + class Extension implements AfterAllCallback, InvocationInterceptor { + + private static final boolean DISABLE_TO_DO = Boolean.parseBoolean(System.getProperty("org.metafacture.metafix.disableToDo")); + + private boolean annotationPresent; + + private Extension() { + } + + @Override + public void afterAll(final ExtensionContext context) { + if (!annotationPresent) { + for (final Method method : context.getTestClass().get().getDeclaredMethods()) { + if (method.isAnnotationPresent(Test.class) && method.isAnnotationPresent(MetafixToDo.class)) { + return; + } + } + + Assertions.fail("Unused extension (no annotations present): " + Handler.EXTENSION_NAME); + } + } + + @Override + public void interceptTestMethod(final InvocationInterceptor.Invocation invocation, final ReflectiveInvocationContext invocationContext, final ExtensionContext extensionContext) throws Throwable { + if (DISABLE_TO_DO) { + handleAnnotation(invocationContext, a -> { + throw new TestAbortedException(a.value()); + }); + + invocation.proceed(); + return; + } + + try { + invocation.proceed(); + } + catch (final Throwable e) { // checkstyle-disable-line IllegalCatch + handleAnnotation(invocationContext, a -> { + throw new TestAbortedException(a.value(), e); + }); + + throw e; + } + + handleAnnotation(invocationContext, a -> Assertions.fail("Marked as " + a + ", but passed.")); + } + + private void handleAnnotation(final ReflectiveInvocationContext invocationContext, final Consumer consumer) { + final MetafixToDo annotation = invocationContext.getExecutable().getAnnotation(MetafixToDo.class); + + if (annotation != null) { + annotationPresent = true; + consumer.accept(annotation); + } + } + + } + + class Handler implements ExecutionCondition { + + private static final Class EXTENSION_CLASS = Extension.class; + private static final String EXTENSION_NAME = EXTENSION_CLASS.getTypeName(); + + private static final Map, Boolean> EXTENSION_PRESENT = new HashMap<>(); + + private Handler() { + } + + @Override + public ConditionEvaluationResult evaluateExecutionCondition(final ExtensionContext context) { + final boolean extensionPresent = EXTENSION_PRESENT.computeIfAbsent(context.getTestClass().get(), k -> { + for (final ExtendWith annotation : k.getAnnotationsByType(ExtendWith.class)) { + for (final Class extensionClass : annotation.value()) { + if (extensionClass.isAssignableFrom(EXTENSION_CLASS)) { + return true; + } + } + } + + return false; + }); + + if (extensionPresent) { + return ConditionEvaluationResult.enabled("Extension present: " + EXTENSION_NAME); + } + else { + Assertions.fail("Extension missing: " + EXTENSION_NAME); + return null; // not reached + } + } + + } + +} diff --git a/metafix/src/test/java/org/metafacture/metafix/RecordTest.java b/metafix/src/test/java/org/metafacture/metafix/RecordTest.java new file mode 100644 index 000000000..a70701e0f --- /dev/null +++ b/metafix/src/test/java/org/metafacture/metafix/RecordTest.java @@ -0,0 +1,366 @@ +/* + * Copyright 2021 hbz NRW + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.concurrent.atomic.LongAdder; +import java.util.function.Consumer; + +public class RecordTest { + + private static final String FIELD = "field"; + private static final String OTHER_FIELD = "other field"; + + private static final Value VALUE = new Value("value"); + private static final Value OTHER_VALUE = new Value("other value"); + + public RecordTest() { + } + + @Test + public void shouldCreateShallowCloneFromEmptyRecord() { + final Record record = new Record(); + final Record clone = record.shallowClone(); + + Assertions.assertNotSame(record, clone); + } + + @Test + public void shouldCreateShallowCloneFromNonEmptyRecord() { + final Record record = new Record(); + record.put(FIELD, VALUE); + + final Record clone = record.shallowClone(); + + Assertions.assertNotSame(record, clone); + Assertions.assertSame(record.get(FIELD), clone.get(FIELD)); + } + + @Test + public void shouldNotModifyTopLevelFromShallowClone() { + final Record record = new Record(); + + final Record clone = record.shallowClone(); + clone.put(FIELD, VALUE); + + Assertions.assertNotSame(record, clone); + Assertions.assertNull(record.get(FIELD)); + Assertions.assertNotNull(clone.get(FIELD)); + } + + @Test + public void shouldNotModifyOverwrittenValueFromShallowClone() { + final Record record = new Record(); + record.put(FIELD, VALUE); + + final Record clone = record.shallowClone(); + clone.put(FIELD, OTHER_VALUE); + + Assertions.assertNotSame(record, clone); + Assertions.assertEquals(VALUE, record.get(FIELD)); + Assertions.assertEquals(OTHER_VALUE, clone.get(FIELD)); + } + + @Test + public void shouldModifySubLevelFromShallowClone() { + final Record record = new Record(); + record.put(FIELD, Value.newArray(a -> a.add(VALUE))); + + final Record clone = record.shallowClone(); + clone.get(FIELD).asArray().add(OTHER_VALUE); + + Assertions.assertNotSame(record, clone); + Assertions.assertSame(record.get(FIELD), clone.get(FIELD)); + + final Value.Array array = clone.get(FIELD).asArray(); + Assertions.assertEquals(OTHER_VALUE, array.get(1)); + } + + @Test + public void shouldEmitRecordByDefault() { + final Record record = new Record(); + Assertions.assertFalse(record.getReject()); + } + + @Test + public void shouldNotEmitRecordIfRejected() { + final Record record = new Record(); + record.setReject(true); + + Assertions.assertTrue(record.getReject()); + } + + @Test + public void shouldEmitCloneOfDefaultRecord() { + final Record record = new Record(); + final Record clone = record.shallowClone(); + + Assertions.assertFalse(clone.getReject()); + } + + @Test + public void shouldNotEmitCloneOfRejectedRecord() { + final Record record = new Record(); + record.setReject(true); + + final Record clone = record.shallowClone(); + + Assertions.assertTrue(clone.getReject()); + } + + @Test + public void shouldNotEmitCloneOfDefaultRecordIfRejected() { + final Record record = new Record(); + + final Record clone = record.shallowClone(); + clone.setReject(true); + + Assertions.assertFalse(record.getReject()); + Assertions.assertTrue(clone.getReject()); + } + + @Test + public void shouldNotContainMissingVirtualField() { + final Record record = new Record(); + Assertions.assertFalse(record.containsVirtualField(FIELD)); + } + + @Test + public void shouldContainExistingVirtualField() { + final Record record = new Record(); + record.putVirtualField(FIELD, VALUE); + + Assertions.assertTrue(record.containsVirtualField(FIELD)); + } + + @Test + public void shouldNotContainVirtualFieldWithNullValue() { + final Record record = new Record(); + record.putVirtualField(FIELD, null); + + Assertions.assertFalse(record.containsVirtualField(FIELD)); + } + + @Test + public void shouldGetVirtualField() { + final Record record = new Record(); + record.putVirtualField(FIELD, VALUE); + + Assertions.assertEquals(VALUE, record.get(FIELD)); + } + + @Test + public void shouldGetRegularFieldInsteadOfVirtualField() { + final Record record = new Record(); + record.putVirtualField(FIELD, VALUE); + + record.put(FIELD, OTHER_VALUE); + + Assertions.assertEquals(OTHER_VALUE, record.get(FIELD)); + } + + @Test + public void shouldNotEmitVirtualFieldsByDefault() { + final Record record = new Record(); + record.putVirtualField(FIELD, VALUE); + + MetafixTestHelpers.assertEmittedFields(record, Arrays.asList(), Arrays.asList()); + } + + @Test + public void shouldEmitVirtualFieldsWhenRetained() { + final Record record = new Record(); + record.putVirtualField(FIELD, VALUE); + + record.retainFields(Arrays.asList(FIELD)); + + MetafixTestHelpers.assertEmittedFields(record, Arrays.asList(FIELD), Arrays.asList(VALUE)); + } + + @Test + public void shouldEmitVirtualFieldsWhenCopied() { + final Record record = new Record(); + record.putVirtualField(FIELD, VALUE); + + record.put(FIELD, record.get(FIELD)); + + MetafixTestHelpers.assertEmittedFields(record, Arrays.asList(FIELD), Arrays.asList(VALUE)); + } + + @Test + public void shouldEmitVirtualFieldsWhenAdded() { + final Record record = new Record(); + record.putVirtualField(FIELD, VALUE); + + record.put(FIELD, OTHER_VALUE); + + MetafixTestHelpers.assertEmittedFields(record, Arrays.asList(FIELD), Arrays.asList(OTHER_VALUE)); + } + + @Test + public void shouldCreateShallowCloneFromRecordWithVirtualFields() { + final Record record = new Record(); + record.putVirtualField(FIELD, VALUE); + + final Record clone = record.shallowClone(); + + Assertions.assertNotSame(record, clone); + Assertions.assertSame(record.get(FIELD), clone.get(FIELD)); + } + + @Test + public void shouldNotModifyTopLevelFromShallowCloneWithVirtualFields() { + final Record record = new Record(); + + final Record clone = record.shallowClone(); + clone.putVirtualField(FIELD, VALUE); + + Assertions.assertNotSame(record, clone); + Assertions.assertNull(record.get(FIELD)); + Assertions.assertNotNull(clone.get(FIELD)); + } + + @Test + public void shouldNotModifyOverwrittenValueFromShallowCloneWithVirtualFields() { + final Record record = new Record(); + record.putVirtualField(FIELD, VALUE); + + final Record clone = record.shallowClone(); + clone.putVirtualField(FIELD, OTHER_VALUE); + + Assertions.assertNotSame(record, clone); + Assertions.assertEquals(VALUE, record.get(FIELD)); + Assertions.assertEquals(OTHER_VALUE, clone.get(FIELD)); + } + + @Test + public void shouldModifySubLevelFromShallowCloneWithVirtualFields() { + final Record record = new Record(); + record.putVirtualField(FIELD, Value.newArray(a -> a.add(VALUE))); + + final Record clone = record.shallowClone(); + clone.get(FIELD).asArray().add(OTHER_VALUE); + + Assertions.assertNotSame(record, clone); + Assertions.assertSame(record.get(FIELD), clone.get(FIELD)); + + final Value.Array array = clone.get(FIELD).asArray(); + Assertions.assertEquals(OTHER_VALUE, array.get(1)); + } + + @Test + public void shouldNotEmitVirtualFieldsFromShallowClone() { + final Record record = new Record(); + record.putVirtualField(FIELD, VALUE); + + final Record clone = record.shallowClone(); + MetafixTestHelpers.assertEmittedFields(clone, Arrays.asList(), Arrays.asList()); + } + + private void shouldFindArrayIndexSubfield(final Consumer consumer) { + final Record record = new Record(); + record.put(FIELD, Value.newArray(a -> a.add(Value.newHash(h -> { + h.put(FIELD, VALUE); + h.put(OTHER_FIELD, OTHER_VALUE); + })))); + + if (consumer != null) { + consumer.accept(record); + } + + Assertions.assertEquals(VALUE, record.get(String.join(".", FIELD, "1", FIELD))); + } + + @Test + public void shouldFindArrayIndexSubfield() { + shouldFindArrayIndexSubfield(null); + } + + @Test + public void shouldFindArrayIndexSubfieldAfterTransformingOtherSubfield() { + shouldFindArrayIndexSubfield(record -> { + final String path = String.join(".", FIELD, "1", OTHER_FIELD); + final String value = "transformed value"; + + record.transform(path, s -> value); + Assertions.assertEquals(new Value(value), record.get(path)); + }); + } + + @Test + public void shouldFindArrayIndexSubfieldAfterTransformingOtherSubfieldToNull() { + shouldFindArrayIndexSubfield(record -> { + final String path = String.join(".", FIELD, "1", OTHER_FIELD); + final String value = null; + + record.transform(path, s -> value); + Assertions.assertEquals(value, record.get(path)); + }); + } + + @Test + public void shouldPreserveOrderWhenTransformingArraySubfields() { + final Record record = new Record(); + record.put(FIELD, Value.newArray(a -> { + for (int i = 0; i < 3; ++i) { + a.add(Value.newHash(h -> { + h.put(FIELD, VALUE); + h.put(OTHER_FIELD, OTHER_VALUE); + })); + } + })); + + final String path = String.join(".", FIELD, "%s", OTHER_FIELD); + + final LongAdder counter = new LongAdder(); + record.transform(String.format(path, "*"), s -> { + counter.increment(); + return s + counter; + }); + + Assertions.assertEquals(new Value(OTHER_VALUE + "1"), record.get(String.format(path, 1))); + Assertions.assertEquals(new Value(OTHER_VALUE + "2"), record.get(String.format(path, 2))); + Assertions.assertEquals(new Value(OTHER_VALUE + "3"), record.get(String.format(path, 3))); + } + + @Test + public void shouldPreserveOrderWhenTransformingOptionalArraySubfield() { + final Record record = new Record(); + final String path = String.join(".", FIELD, "%s", OTHER_FIELD); + record.put(FIELD, Value.newArray(a -> { + a.add(Value.newHash(h -> { + h.put(FIELD, VALUE); + })); + a.add(Value.newHash(h -> { + h.put(FIELD, VALUE); + // For proper transformation, we have to explicitly set the path (as in Metafix.java): + h.put(OTHER_FIELD, OTHER_VALUE.withPathSet(String.format(path, a.size() + 1)), false); + })); + })); + + final String value = "transformed value"; + + record.transform(String.format(path, "*"), s -> value); + + Assertions.assertEquals(null, record.get(String.format(path, 1))); + Assertions.assertEquals(new Value(value), record.get(String.format(path, 2))); + } + +} diff --git a/metafix/src/test/java/org/metafacture/metafix/ValueTest.java b/metafix/src/test/java/org/metafacture/metafix/ValueTest.java new file mode 100644 index 000000000..ceac6f2cb --- /dev/null +++ b/metafix/src/test/java/org/metafacture/metafix/ValueTest.java @@ -0,0 +1,36 @@ +/* + * Copyright 2022 hbz NRW + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix; + +import nl.jqno.equalsverifier.EqualsVerifier; +import org.junit.jupiter.api.Test; + +public class ValueTest { + + public ValueTest() { + } + + @Test + public void shouldSatisfyEqualsContract() { + EqualsVerifier.forClass(Value.class) + .withPrefabValues(Value.class, Value.newArray(), Value.newHash()) + .withPrefabValues(Value.Hash.class, Value.newHash().asHash(), Value.newHash(h -> h.put("k", new Value("v"))).asHash()) + .withIgnoredFields("path") + .verify(); + } + +} diff --git a/metafix/src/test/java/org/metafacture/metafix/util/TestContext.java b/metafix/src/test/java/org/metafacture/metafix/util/TestContext.java new file mode 100644 index 000000000..9ef9a9c0d --- /dev/null +++ b/metafix/src/test/java/org/metafacture/metafix/util/TestContext.java @@ -0,0 +1,40 @@ +/* + * Copyright 2022 hbz NRW + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix.util; + +import org.metafacture.metafix.Metafix; +import org.metafacture.metafix.Record; +import org.metafacture.metafix.RecordTransformer; +import org.metafacture.metafix.Value; +import org.metafacture.metafix.api.FixContext; + +import java.util.List; +import java.util.Map; + +public class TestContext implements FixContext { + + public TestContext() { + } + + @Override + public void execute(final Metafix metafix, final Record record, final List params, final Map options, final RecordTransformer recordTransformer) { + record.add("BEFORE", new Value(params.get(0))); + recordTransformer.transform(record); + record.add("AFTER", new Value(options.get("data"))); + } + +} diff --git a/metafix/src/test/java/org/metafacture/metafix/util/TestFunction.java b/metafix/src/test/java/org/metafacture/metafix/util/TestFunction.java new file mode 100644 index 000000000..5db0d7f17 --- /dev/null +++ b/metafix/src/test/java/org/metafacture/metafix/util/TestFunction.java @@ -0,0 +1,42 @@ +/* + * Copyright 2022 hbz NRW + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix.util; + +import org.metafacture.metafix.FixMethod; +import org.metafacture.metafix.Metafix; +import org.metafacture.metafix.Record; +import org.metafacture.metafix.Value; +import org.metafacture.metafix.api.FixFunction; + +import java.util.List; +import java.util.Map; + +public class TestFunction implements FixFunction { + + public TestFunction() { + } + + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + params.add(params.get(0).toUpperCase()); + params.set(0, "test"); + FixMethod.add_field.apply(metafix, record, params, options); + + options.forEach((k, v) -> record.set(k, new Value(v))); + } + +} diff --git a/metafix/src/test/java/org/metafacture/metafix/util/TestPredicate.java b/metafix/src/test/java/org/metafacture/metafix/util/TestPredicate.java new file mode 100644 index 000000000..94d179aaa --- /dev/null +++ b/metafix/src/test/java/org/metafacture/metafix/util/TestPredicate.java @@ -0,0 +1,38 @@ +/* + * Copyright 2022 hbz NRW + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix.util; + +import org.metafacture.metafix.FixConditional; +import org.metafacture.metafix.Metafix; +import org.metafacture.metafix.Record; +import org.metafacture.metafix.api.FixPredicate; + +import java.util.List; +import java.util.Map; + +public class TestPredicate implements FixPredicate { + + public TestPredicate() { + } + + @Override + public boolean test(final Metafix metafix, final Record record, final List params, final Map options) { + return !FixConditional.exists.test(metafix, record, params, options) || + FixConditional.any_equal.test(metafix, record, params, options); + } + +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/fixes/base.fix b/metafix/src/test/resources/org/metafacture/metafix/fixes/base.fix new file mode 100644 index 000000000..a2793347f --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/fixes/base.fix @@ -0,0 +1 @@ +paste("trace.$append", "~base", "_id") diff --git a/metafix/src/test/resources/org/metafacture/metafix/fixes/error.fix b/metafix/src/test/resources/org/metafacture/metafix/fixes/error.fix new file mode 100644 index 000000000..55a9a1a56 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/fixes/error.fix @@ -0,0 +1,3 @@ +# comment +append("animals", " is cool") +nothing() diff --git a/metafix/src/test/resources/org/metafacture/metafix/fixes/filemap.fix b/metafix/src/test/resources/org/metafacture/metafix/fixes/filemap.fix new file mode 100644 index 000000000..2084b6a21 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/fixes/filemap.fix @@ -0,0 +1 @@ +put_filemap("../maps/test.csv", "testMap") diff --git a/metafix/src/test/resources/org/metafacture/metafix/fixes/filemap_lookup.fix b/metafix/src/test/resources/org/metafacture/metafix/fixes/filemap_lookup.fix new file mode 100644 index 000000000..4a47502e0 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/fixes/filemap_lookup.fix @@ -0,0 +1 @@ +lookup("title.*", "../maps/test.csv") diff --git a/metafix/src/test/resources/org/metafacture/metafix/fixes/include.fix b/metafix/src/test/resources/org/metafacture/metafix/fixes/include.fix new file mode 100644 index 000000000..9f8f6d39d --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/fixes/include.fix @@ -0,0 +1,6 @@ +unless exists("trace") + set_array("trace") +end +add_field("trace.$append", "before include") +include("./base.fix") +add_field("trace.$append", "after include") diff --git a/metafix/src/test/resources/org/metafacture/metafix/fixes/invalid.fix b/metafix/src/test/resources/org/metafacture/metafix/fixes/invalid.fix new file mode 100644 index 000000000..75db11443 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/fixes/invalid.fix @@ -0,0 +1 @@ +lookup("license.id", ,"data/maps/duepublico-licenses.tsv","sep_char":"\t") diff --git a/metafix/src/test/resources/org/metafacture/metafix/fixes/macro.fix b/metafix/src/test/resources/org/metafacture/metafix/fixes/macro.fix new file mode 100644 index 000000000..9086d9ef5 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/fixes/macro.fix @@ -0,0 +1,3 @@ +do put_macro("test") + append('animals', ' is cool') +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/fixes/nested.fix b/metafix/src/test/resources/org/metafacture/metafix/fixes/nested.fix new file mode 100644 index 000000000..33c9ede03 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/fixes/nested.fix @@ -0,0 +1,4 @@ +set_array("trace") +add_field("trace.$append", "before nested") +include("src/test/resources/org/metafacture/metafix/fixes/include.fix") +add_field("trace.$append", "after nested") diff --git a/metafix/src/test/resources/org/metafacture/metafix/fixes/once.fix b/metafix/src/test/resources/org/metafacture/metafix/fixes/once.fix new file mode 100644 index 000000000..8e4ec1206 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/fixes/once.fix @@ -0,0 +1,5 @@ +add_field(included, "true") + +do once() + add_field(also_executed, "false") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/fixes/var.fix b/metafix/src/test/resources/org/metafacture/metafix/fixes/var.fix new file mode 100644 index 000000000..a0ae0c133 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/fixes/var.fix @@ -0,0 +1,2 @@ +paste("trace.$append", "$i", "_id") +upcase("$i") diff --git a/metafix/src/test/resources/org/metafacture/metafix/fixes/vars.fix b/metafix/src/test/resources/org/metafacture/metafix/fixes/vars.fix new file mode 100644 index 000000000..19c9f274b --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/fixes/vars.fix @@ -0,0 +1 @@ +paste("test$[a]", "_id", "~$[b]", join_char: "-") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listArrayOfObjectsAndMoveSubfield/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listArrayOfObjectsAndMoveSubfield/expected.json new file mode 100644 index 000000000..66f295c28 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listArrayOfObjectsAndMoveSubfield/expected.json @@ -0,0 +1,24 @@ +{ + "author" : [ { + "name" : "RUVIVAL Team", + "type" : "Person" + }, { + "name" : "Samuel Duval", + "type" : "Person" + }, { + "name" : "Berenice Lopez Mendez", + "type" : "Person" + }, { + "name" : "Lukas Schreiner", + "type" : "Person" + }, { + "name" : "Isidora Vrbavac", + "type" : "Person" + } ] +} +{ + "author" : [ { + "name" : "Jürgen Böhner", + "type" : "Person" + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listArrayOfObjectsAndMoveSubfield/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listArrayOfObjectsAndMoveSubfield/input.json new file mode 100644 index 000000000..847b571a1 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listArrayOfObjectsAndMoveSubfield/input.json @@ -0,0 +1,32 @@ +{ + "author": [ + { + "name": "RUVIVAL Team", + "@type": "Person" + }, + { + "name": "Samuel Duval", + "@type": "Person" + }, + { + "name": "Berenice Lopez Mendez", + "@type": "Person" + }, + { + "name": "Lukas Schreiner", + "@type": "Person" + }, + { + "name": "Isidora Vrbavac", + "@type": "Person" + } + ] +} +{ + "author": [ + { + "@type": "Person", + "name": "Jürgen Böhner" + } + ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listArrayOfObjectsAndMoveSubfield/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listArrayOfObjectsAndMoveSubfield/test.fix new file mode 100644 index 000000000..43280fc09 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listArrayOfObjectsAndMoveSubfield/test.fix @@ -0,0 +1,3 @@ +do list(path:"author[]") + move_field("@type","type") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listArrayOfObjectsAndMoveSubfield/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listArrayOfObjectsAndMoveSubfield/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listArrayOfObjectsAndMoveSubfield/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listArrayOfObjectsWithVariableAndPathWildcards/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listArrayOfObjectsWithVariableAndPathWildcards/expected.json new file mode 100644 index 000000000..f982c1e18 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listArrayOfObjectsWithVariableAndPathWildcards/expected.json @@ -0,0 +1,23 @@ +{ + "arrayOfObjects_1" : [ { + "name" : "red", + "type" : "color" + }, { + "name" : "green", + "type" : "color" + }, { + "name" : "blue", + "type" : "color" + } ], + "arrayOfObjects_2" : [ { + "name" : "purple", + "type" : "color" + }, { + "name" : "orange", + "type" : "color" + }, { + "name" : "yellow", + "type" : "color" + } ], + "test" : [ "red", "green", "blue", "purple", "orange", "yellow" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listArrayOfObjectsWithVariableAndPathWildcards/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listArrayOfObjectsWithVariableAndPathWildcards/input.json new file mode 100644 index 000000000..e5c6533a3 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listArrayOfObjectsWithVariableAndPathWildcards/input.json @@ -0,0 +1,23 @@ +{ + "arrayOfObjects_1" : [ { + "name": "red", + "type" : "color" + }, { + "name":"green", + "type" : "color" + }, { + "name":"blue", + "type" : "color" + } ], + "arrayOfObjects_2" : [ { + "name": "purple", + "type" : "color" + }, { + "name":"orange", + "type" : "color" + }, { + "name":"yellow", + "type" : "color" + } ], + "test" : [ ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listArrayOfObjectsWithVariableAndPathWildcards/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listArrayOfObjectsWithVariableAndPathWildcards/test.fix new file mode 100644 index 000000000..93e06d79f --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listArrayOfObjectsWithVariableAndPathWildcards/test.fix @@ -0,0 +1,3 @@ +do list(path: "arrayOfObjects??[]", "var": "$i") + copy_field("$i.name", "test[].$append") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listArrayOfObjectsWithVariableAndPathWildcards/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listArrayOfObjectsWithVariableAndPathWildcards/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listArrayOfObjectsWithVariableAndPathWildcards/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listArrayOfObjectsWithVariableUnlessExistsWithOutsideReferenceAndCopy_field/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listArrayOfObjectsWithVariableUnlessExistsWithOutsideReferenceAndCopy_field/expected.json new file mode 100644 index 000000000..650336ca4 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listArrayOfObjectsWithVariableUnlessExistsWithOutsideReferenceAndCopy_field/expected.json @@ -0,0 +1,15 @@ +{ + "arrayOfObjects" : [ { + "name" : "red", + "type" : "color" + }, { + "name" : "green", + "type" : "color" + }, { + "name" : "blue", + "type" : "color" + } ], + "test" : { + "key" : "red" + } +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listArrayOfObjectsWithVariableUnlessExistsWithOutsideReferenceAndCopy_field/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listArrayOfObjectsWithVariableUnlessExistsWithOutsideReferenceAndCopy_field/input.json new file mode 100644 index 000000000..963e2b411 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listArrayOfObjectsWithVariableUnlessExistsWithOutsideReferenceAndCopy_field/input.json @@ -0,0 +1,12 @@ +{ + "arrayOfObjects" : [ { + "name": "red", + "type" : "color" + }, { + "name":"green", + "type" : "color" + }, { + "name":"blue", + "type" : "color" + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listArrayOfObjectsWithVariableUnlessExistsWithOutsideReferenceAndCopy_field/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listArrayOfObjectsWithVariableUnlessExistsWithOutsideReferenceAndCopy_field/test.fix new file mode 100644 index 000000000..e1556ba25 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listArrayOfObjectsWithVariableUnlessExistsWithOutsideReferenceAndCopy_field/test.fix @@ -0,0 +1,5 @@ +do list(path: "arrayOfObjects[]", "var": "$i") + unless exists("test.key") + copy_field("$i.name", "test.key") + end +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listArrayOfObjectsWithVariableUnlessExistsWithOutsideReferenceAndCopy_field/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listArrayOfObjectsWithVariableUnlessExistsWithOutsideReferenceAndCopy_field/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listArrayOfObjectsWithVariableUnlessExistsWithOutsideReferenceAndCopy_field/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listAsSimple/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listAsSimple/expected.json new file mode 100644 index 000000000..1a0ba8330 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listAsSimple/expected.json @@ -0,0 +1,33 @@ +{ + "ccm:university" : [ "https://ror.org/0304hq317", "https://ror.org/014nnvj65" ], + "ccm:university_DISPLAYNAME" : [ "Gottfried Wilhelm Leibniz Universität Hannover", "Technische Hochschule Köln" ], + "sourceOrganization" : [ { + "id" : "https://ror.org/0304hq317", + "name" : "Gottfried Wilhelm Leibniz Universität Hannover" + }, { + "id" : "https://ror.org/014nnvj65", + "name" : "Technische Hochschule Köln" + } ] +} +{ + "ccm:university" : [ "https://ror.org/02k7v4d05", "https://ror.org/04cvxnb49", "https://ror.org/024d6js02" ], + "ccm:university_DISPLAYNAME" : [ "Universität Bern", "Johann Wolfgang Goethe-Universität Frankfurt am Main", "Karls-Universität Prag" ], + "sourceOrganization" : [ { + "id" : "https://ror.org/02k7v4d05", + "name" : "Universität Bern" + }, { + "id" : "https://ror.org/04cvxnb49", + "name" : "Johann Wolfgang Goethe-Universität Frankfurt am Main" + }, { + "id" : "https://ror.org/024d6js02", + "name" : "Karls-Universität Prag" + } ] +} +{ + "ccm:university" : [ "https://ror.org/04tsk2644" ], + "ccm:university_DISPLAYNAME" : [ "Ruhr-Universität Bochum" ], + "sourceOrganization" : [ { + "id" : "https://ror.org/04tsk2644", + "name" : "Ruhr-Universität Bochum" + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listAsSimple/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listAsSimple/input.json new file mode 100644 index 000000000..47cb98d7d --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listAsSimple/input.json @@ -0,0 +1,12 @@ +{ + "ccm:university" : [ "https://ror.org/0304hq317", "https://ror.org/014nnvj65" ], + "ccm:university_DISPLAYNAME" : [ "Gottfried Wilhelm Leibniz Universität Hannover", "Technische Hochschule Köln" ] +} +{ + "ccm:university" : [ "https://ror.org/02k7v4d05", "https://ror.org/04cvxnb49", "https://ror.org/024d6js02" ], + "ccm:university_DISPLAYNAME" : [ "Universität Bern", "Johann Wolfgang Goethe-Universität Frankfurt am Main", "Karls-Universität Prag" ] +} +{ + "ccm:university" : [ "https://ror.org/04tsk2644" ], + "ccm:university_DISPLAYNAME" : [ "Ruhr-Universität Bochum" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listAsSimple/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listAsSimple/test.fix new file mode 100644 index 000000000..ce9bf245a --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listAsSimple/test.fix @@ -0,0 +1,4 @@ +do list_as(orgId: "ccm:university[]", orgName: "ccm:university_DISPLAYNAME[]") + copy_field(orgId, "sourceOrganization[].$append.id") + copy_field(orgName, "sourceOrganization[].$last.name") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listAsSimple/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listAsSimple/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listAsSimple/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfObjectsWithVariableAndCopy_fieldListValueToAnotherArrayOfStrings/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfObjectsWithVariableAndCopy_fieldListValueToAnotherArrayOfStrings/expected.json new file mode 100644 index 000000000..cc1e4b91c --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfObjectsWithVariableAndCopy_fieldListValueToAnotherArrayOfStrings/expected.json @@ -0,0 +1,13 @@ +{ + "arrayOfObjects" : [ { + "name" : "red", + "type" : "color" + }, { + "name" : "green", + "type" : "color" + }, { + "name" : "blue", + "type" : "color" + } ], + "array" : [ "red", "green", "blue" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfObjectsWithVariableAndCopy_fieldListValueToAnotherArrayOfStrings/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfObjectsWithVariableAndCopy_fieldListValueToAnotherArrayOfStrings/input.json new file mode 100644 index 000000000..c36657049 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfObjectsWithVariableAndCopy_fieldListValueToAnotherArrayOfStrings/input.json @@ -0,0 +1,13 @@ +{ + "arrayOfObjects" : [ { + "name": "red", + "type" : "color" + }, { + "name":"green", + "type" : "color" + }, { + "name":"blue", + "type" : "color" + } ], + "array" : [ ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfObjectsWithVariableAndCopy_fieldListValueToAnotherArrayOfStrings/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfObjectsWithVariableAndCopy_fieldListValueToAnotherArrayOfStrings/test.fix new file mode 100644 index 000000000..8021bf185 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfObjectsWithVariableAndCopy_fieldListValueToAnotherArrayOfStrings/test.fix @@ -0,0 +1,3 @@ +do list(path: "arrayOfObjects[]", "var": "$i") + copy_field("$i.name", "array[].$append") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfObjectsWithVariableAndCopy_fieldListValueToAnotherArrayOfStrings/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfObjectsWithVariableAndCopy_fieldListValueToAnotherArrayOfStrings/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfObjectsWithVariableAndCopy_fieldListValueToAnotherArrayOfStrings/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfObjectsWithVariableAndCopy_fieldListValueToAnotherNestedArrayOfStrings/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfObjectsWithVariableAndCopy_fieldListValueToAnotherNestedArrayOfStrings/expected.json new file mode 100644 index 000000000..5a63ebfb5 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfObjectsWithVariableAndCopy_fieldListValueToAnotherNestedArrayOfStrings/expected.json @@ -0,0 +1,15 @@ +{ + "arrayOfObjects" : [ { + "name" : "red", + "type" : "color" + }, { + "name" : "green", + "type" : "color" + }, { + "name" : "blue", + "type" : "color" + } ], + "test" : { + "array" : [ "red", "green", "blue" ] + } +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfObjectsWithVariableAndCopy_fieldListValueToAnotherNestedArrayOfStrings/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfObjectsWithVariableAndCopy_fieldListValueToAnotherNestedArrayOfStrings/input.json new file mode 100644 index 000000000..5bd10ca2e --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfObjectsWithVariableAndCopy_fieldListValueToAnotherNestedArrayOfStrings/input.json @@ -0,0 +1,13 @@ +{ + "arrayOfObjects" : [ { + "name": "red", + "type" : "color" + }, { + "name":"green", + "type" : "color" + }, { + "name":"blue", + "type" : "color" + } ], + "test" : { "array" : [ ] } +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfObjectsWithVariableAndCopy_fieldListValueToAnotherNestedArrayOfStrings/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfObjectsWithVariableAndCopy_fieldListValueToAnotherNestedArrayOfStrings/test.fix new file mode 100644 index 000000000..6ba0c5012 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfObjectsWithVariableAndCopy_fieldListValueToAnotherNestedArrayOfStrings/test.fix @@ -0,0 +1,3 @@ +do list(path: "arrayOfObjects[]", "var": "$i") + copy_field("$i.name", "test.array[].$append") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfObjectsWithVariableAndCopy_fieldListValueToAnotherNestedArrayOfStrings/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfObjectsWithVariableAndCopy_fieldListValueToAnotherNestedArrayOfStrings/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfObjectsWithVariableAndCopy_fieldListValueToAnotherNestedArrayOfStrings/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfStringsWithVariableAndAppend/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfStringsWithVariableAndAppend/expected.json new file mode 100644 index 000000000..52327acbe --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfStringsWithVariableAndAppend/expected.json @@ -0,0 +1,3 @@ +{ + "colors" : [ "Red is a nice color", "Green is a nice color", "Blue is a nice color" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfStringsWithVariableAndAppend/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfStringsWithVariableAndAppend/input.json new file mode 100644 index 000000000..b70a3f5fd --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfStringsWithVariableAndAppend/input.json @@ -0,0 +1,3 @@ +{ + "colors" : [ "red", "green", "blue" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfStringsWithVariableAndAppend/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfStringsWithVariableAndAppend/test.fix new file mode 100644 index 000000000..cda909170 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfStringsWithVariableAndAppend/test.fix @@ -0,0 +1,3 @@ +do list(path:"colors[]", "var": "$i") + append("$i", " is a nice color") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfStringsWithVariableAndAppend/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfStringsWithVariableAndAppend/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfStringsWithVariableAndAppend/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfStringsWithVariableAndAppend/todo.txt b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfStringsWithVariableAndAppend/todo.txt new file mode 100644 index 000000000..9dd034d9d --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfStringsWithVariableAndAppend/todo.txt @@ -0,0 +1 @@ +See issue #189 diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfStringsWithVariableAndCopy_fieldListValueToTheOutside/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfStringsWithVariableAndCopy_fieldListValueToTheOutside/expected.json new file mode 100644 index 000000000..67da01f97 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfStringsWithVariableAndCopy_fieldListValueToTheOutside/expected.json @@ -0,0 +1,4 @@ +{ + "colors" : [ "red", "green", "blue" ], + "test" : [ "red", "green", "blue" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfStringsWithVariableAndCopy_fieldListValueToTheOutside/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfStringsWithVariableAndCopy_fieldListValueToTheOutside/input.json new file mode 100644 index 000000000..388948c64 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfStringsWithVariableAndCopy_fieldListValueToTheOutside/input.json @@ -0,0 +1,4 @@ +{ + "colors" : [ "red", "green", "blue" ], + "test" : [ ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfStringsWithVariableAndCopy_fieldListValueToTheOutside/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfStringsWithVariableAndCopy_fieldListValueToTheOutside/test.fix new file mode 100644 index 000000000..b42b81940 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfStringsWithVariableAndCopy_fieldListValueToTheOutside/test.fix @@ -0,0 +1,3 @@ +do list(path:"colors[]", "var": "$i") + copy_field("$i", "test[].$append") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfStringsWithVariableAndCopy_fieldListValueToTheOutside/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfStringsWithVariableAndCopy_fieldListValueToTheOutside/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfStringsWithVariableAndCopy_fieldListValueToTheOutside/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfStringsWithoutVariableAndAppend/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfStringsWithoutVariableAndAppend/expected.json new file mode 100644 index 000000000..52327acbe --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfStringsWithoutVariableAndAppend/expected.json @@ -0,0 +1,3 @@ +{ + "colors" : [ "Red is a nice color", "Green is a nice color", "Blue is a nice color" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfStringsWithoutVariableAndAppend/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfStringsWithoutVariableAndAppend/input.json new file mode 100644 index 000000000..b70a3f5fd --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfStringsWithoutVariableAndAppend/input.json @@ -0,0 +1,3 @@ +{ + "colors" : [ "red", "green", "blue" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfStringsWithoutVariableAndAppend/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfStringsWithoutVariableAndAppend/test.fix new file mode 100644 index 000000000..9cdf212bd --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfStringsWithoutVariableAndAppend/test.fix @@ -0,0 +1,4 @@ +do list(path:"colors[]") + upcase(".") + append(".", " is a nice color") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfStringsWithoutVariableAndAppend/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfStringsWithoutVariableAndAppend/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfStringsWithoutVariableAndAppend/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfStringsWithoutVariableAndAppend/todo.txt b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfStringsWithoutVariableAndAppend/todo.txt new file mode 100644 index 000000000..efdfa1745 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/listSimpleArrayOfStringsWithoutVariableAndAppend/todo.txt @@ -0,0 +1 @@ +See issue #191 diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/onceSimple/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/onceSimple/expected.json new file mode 100644 index 000000000..3ed02adf3 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/onceSimple/expected.json @@ -0,0 +1,22 @@ +{ + "author" : [ { + "name" : "RUVIVAL Team", + "type" : "Person" + }, { + "name" : "Samuel Duval", + "type" : "Person" + } ], + "@type" : "test" +} +{ + "author" : [ { + "name" : "Jürgen Böhner", + "type" : "Person" + } ] +} +{ + "author" : [ { + "name" : "Peter Müller", + "type" : "Person" + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/onceSimple/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/onceSimple/input.json new file mode 100644 index 000000000..61c639503 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/onceSimple/input.json @@ -0,0 +1,23 @@ +{ + "author" : [ { + "name" : "RUVIVAL Team", + "type" : "Person" + }, { + "name" : "Samuel Duval", + "type" : "Person" + } ] +} +{ + "author" : [ { + "name" : "Jürgen Böhner", + "type" : "Person" + } ] +} +{ + "author" : [ { + "name" : "Peter Müller", + "type" : "Person" + } ] +} + + diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/onceSimple/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/onceSimple/test.fix new file mode 100644 index 000000000..16da405f6 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/onceSimple/test.fix @@ -0,0 +1,3 @@ +do once() + add_field("@type","test") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/onceSimple/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/onceSimple/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/bind/fromJson/toJson/onceSimple/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_containArrayOfStrings/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_containArrayOfStrings/expected.json new file mode 100644 index 000000000..1146d70d3 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_containArrayOfStrings/expected.json @@ -0,0 +1,14 @@ +{ + "type" : [ "dog", "dog", "dog" ], + "test" : "test" +} +{ + "type" : [ "video", "dog", "dog" ], + "test" : "test" +} +{ + "type" : [ "book", "pc" ] +} +{ + "type" : [ "human", "cat" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_containArrayOfStrings/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_containArrayOfStrings/input.json new file mode 100644 index 000000000..b3bf99adf --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_containArrayOfStrings/input.json @@ -0,0 +1,12 @@ +{ + "type" : [ "dog", "dog", "dog" ] +} +{ + "type" : [ "video", "dog", "dog" ] +} +{ + "type" : [ "book", "pc" ] +} +{ + "type" : [ "human", "cat" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_containArrayOfStrings/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_containArrayOfStrings/test.fix new file mode 100644 index 000000000..8825e3d50 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_containArrayOfStrings/test.fix @@ -0,0 +1,3 @@ +if all_contain("type[]", "o") + add_field("test", "test") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_containArrayOfStrings/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_containArrayOfStrings/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_containArrayOfStrings/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_containSimple/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_containSimple/expected.json new file mode 100644 index 000000000..b6c3e2c62 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_containSimple/expected.json @@ -0,0 +1,14 @@ +{ + "type" : "dog" +} +{ + "type" : "video", + "test" : "test" +} +{ + "type" : "book" +} +{ + "type" : "stone", + "test" : "test" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_containSimple/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_containSimple/input.json new file mode 100644 index 000000000..456749e6b --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_containSimple/input.json @@ -0,0 +1,12 @@ +{ + "type" : "dog" +} +{ + "type" : "video" +} +{ + "type" : "book" +} +{ + "type" : "stone" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_containSimple/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_containSimple/test.fix new file mode 100644 index 000000000..38dddaab1 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_containSimple/test.fix @@ -0,0 +1,3 @@ +if all_contain("type", "e") + add_field("test", "test") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_containSimple/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_containSimple/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_containSimple/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_equalArrayOfStrings/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_equalArrayOfStrings/expected.json new file mode 100644 index 000000000..7e7c2711e --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_equalArrayOfStrings/expected.json @@ -0,0 +1,13 @@ +{ + "type" : [ "dog", "dog", "dog" ], + "test" : "test" +} +{ + "type" : [ "video", "dog", "dog" ] +} +{ + "type" : [ "book", "dog", "dog" ] +} +{ + "type" : [ "stone", "cat" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_equalArrayOfStrings/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_equalArrayOfStrings/input.json new file mode 100644 index 000000000..44f9a90f4 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_equalArrayOfStrings/input.json @@ -0,0 +1,13 @@ +{ + "type" : [ "dog", "dog", "dog" ] +} +{ + "type" : [ "video", "dog", "dog" ] +} +{ + "type" : [ "book", "dog", "dog" ] +} +{ + "type" : [ "stone", "cat" ] +} + diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_equalArrayOfStrings/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_equalArrayOfStrings/test.fix new file mode 100644 index 000000000..2c9ba564b --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_equalArrayOfStrings/test.fix @@ -0,0 +1,3 @@ +if all_equal("type[]", "dog") + add_field("test", "test") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_equalArrayOfStrings/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_equalArrayOfStrings/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_equalArrayOfStrings/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_equalSimple/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_equalSimple/expected.json new file mode 100644 index 000000000..d7587388f --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_equalSimple/expected.json @@ -0,0 +1,13 @@ +{ + "type" : "dog" +} +{ + "type" : "video" +} +{ + "type" : "book", + "test" : "test" +} +{ + "type" : "stone" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_equalSimple/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_equalSimple/input.json new file mode 100644 index 000000000..3b0365e84 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_equalSimple/input.json @@ -0,0 +1,13 @@ +{ + "type" : "dog" +} +{ + "type" : "video" +} +{ + "type" : "book" +} +{ + "type" : "stone" +} + diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_equalSimple/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_equalSimple/test.fix new file mode 100644 index 000000000..19c264684 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_equalSimple/test.fix @@ -0,0 +1,3 @@ +if all_equal("type", "book") + add_field("test", "test") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_equalSimple/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_equalSimple/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_equalSimple/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_matchArrayOfStrings/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_matchArrayOfStrings/expected.json new file mode 100644 index 000000000..329c7811a --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_matchArrayOfStrings/expected.json @@ -0,0 +1,13 @@ +{ + "type" : [ "dog", "dog", "dog" ], + "test" : "test" +} +{ + "type" : [ "video", "dog", "dog" ] +} +{ + "type" : [ "book", "pc" ] +} +{ + "type" : [ "stone", "cat" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_matchArrayOfStrings/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_matchArrayOfStrings/input.json new file mode 100644 index 000000000..94ebe864e --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_matchArrayOfStrings/input.json @@ -0,0 +1,12 @@ +{ + "type" : [ "dog", "dog", "dog" ] +} +{ + "type" : [ "video", "dog", "dog" ] +} +{ + "type" : [ "book", "pc" ] +} +{ + "type" : [ "stone", "cat" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_matchArrayOfStrings/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_matchArrayOfStrings/test.fix new file mode 100644 index 000000000..6d6fd67f1 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_matchArrayOfStrings/test.fix @@ -0,0 +1,3 @@ +if all_match("type[]", "\\S{3}") + add_field("test", "test") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_matchArrayOfStrings/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_matchArrayOfStrings/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_matchArrayOfStrings/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_matchSimple/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_matchSimple/expected.json new file mode 100644 index 000000000..b6c3e2c62 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_matchSimple/expected.json @@ -0,0 +1,14 @@ +{ + "type" : "dog" +} +{ + "type" : "video", + "test" : "test" +} +{ + "type" : "book" +} +{ + "type" : "stone", + "test" : "test" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_matchSimple/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_matchSimple/input.json new file mode 100644 index 000000000..3b0365e84 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_matchSimple/input.json @@ -0,0 +1,13 @@ +{ + "type" : "dog" +} +{ + "type" : "video" +} +{ + "type" : "book" +} +{ + "type" : "stone" +} + diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_matchSimple/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_matchSimple/test.fix new file mode 100644 index 000000000..9c5edd6e9 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_matchSimple/test.fix @@ -0,0 +1,3 @@ +if all_match("type", "\\S{5}") + add_field("test", "test") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_matchSimple/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_matchSimple/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAll_matchSimple/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_containArrayOfStrings/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_containArrayOfStrings/expected.json new file mode 100644 index 000000000..e83154260 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_containArrayOfStrings/expected.json @@ -0,0 +1,15 @@ +{ + "type" : [ "dog", "dog", "dog" ], + "test" : "test" +} +{ + "type" : [ "video", "dog", "dog" ], + "test" : "test" +} +{ + "type" : [ "book", "pc" ], + "test" : "test" +} +{ + "type" : [ "human", "cat" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_containArrayOfStrings/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_containArrayOfStrings/input.json new file mode 100644 index 000000000..b3bf99adf --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_containArrayOfStrings/input.json @@ -0,0 +1,12 @@ +{ + "type" : [ "dog", "dog", "dog" ] +} +{ + "type" : [ "video", "dog", "dog" ] +} +{ + "type" : [ "book", "pc" ] +} +{ + "type" : [ "human", "cat" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_containArrayOfStrings/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_containArrayOfStrings/test.fix new file mode 100644 index 000000000..78a6c28de --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_containArrayOfStrings/test.fix @@ -0,0 +1,3 @@ +if any_contain("type[]", "o") + add_field("test", "test") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_containArrayOfStrings/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_containArrayOfStrings/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_containArrayOfStrings/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_containSimple/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_containSimple/expected.json new file mode 100644 index 000000000..b6c3e2c62 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_containSimple/expected.json @@ -0,0 +1,14 @@ +{ + "type" : "dog" +} +{ + "type" : "video", + "test" : "test" +} +{ + "type" : "book" +} +{ + "type" : "stone", + "test" : "test" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_containSimple/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_containSimple/input.json new file mode 100644 index 000000000..3b0365e84 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_containSimple/input.json @@ -0,0 +1,13 @@ +{ + "type" : "dog" +} +{ + "type" : "video" +} +{ + "type" : "book" +} +{ + "type" : "stone" +} + diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_containSimple/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_containSimple/test.fix new file mode 100644 index 000000000..1fd10f05d --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_containSimple/test.fix @@ -0,0 +1,3 @@ +if any_contain("type", "e") + add_field("test", "test") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_containSimple/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_containSimple/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_containSimple/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_equalArrayOfStrings/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_equalArrayOfStrings/expected.json new file mode 100644 index 000000000..46fb05664 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_equalArrayOfStrings/expected.json @@ -0,0 +1,15 @@ +{ + "type" : [ "dog", "dog", "dog" ], + "test" : "test" +} +{ + "type" : [ "video", "dog", "dog" ], + "test" : "test" +} +{ + "type" : [ "book", "dog", "dog" ], + "test" : "test" +} +{ + "type" : [ "stone", "cat" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_equalArrayOfStrings/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_equalArrayOfStrings/input.json new file mode 100644 index 000000000..44f9a90f4 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_equalArrayOfStrings/input.json @@ -0,0 +1,13 @@ +{ + "type" : [ "dog", "dog", "dog" ] +} +{ + "type" : [ "video", "dog", "dog" ] +} +{ + "type" : [ "book", "dog", "dog" ] +} +{ + "type" : [ "stone", "cat" ] +} + diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_equalArrayOfStrings/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_equalArrayOfStrings/test.fix new file mode 100644 index 000000000..47d7cc4ac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_equalArrayOfStrings/test.fix @@ -0,0 +1,3 @@ +if any_equal("type[]", "dog") + add_field("test", "test") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_equalArrayOfStrings/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_equalArrayOfStrings/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_equalArrayOfStrings/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_equalSimple/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_equalSimple/expected.json new file mode 100644 index 000000000..d7587388f --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_equalSimple/expected.json @@ -0,0 +1,13 @@ +{ + "type" : "dog" +} +{ + "type" : "video" +} +{ + "type" : "book", + "test" : "test" +} +{ + "type" : "stone" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_equalSimple/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_equalSimple/input.json new file mode 100644 index 000000000..3b0365e84 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_equalSimple/input.json @@ -0,0 +1,13 @@ +{ + "type" : "dog" +} +{ + "type" : "video" +} +{ + "type" : "book" +} +{ + "type" : "stone" +} + diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_equalSimple/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_equalSimple/test.fix new file mode 100644 index 000000000..da59206a8 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_equalSimple/test.fix @@ -0,0 +1,3 @@ +if any_equal("type", "book") + add_field("test", "test") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_equalSimple/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_equalSimple/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_equalSimple/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_matchArrayOfStrings/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_matchArrayOfStrings/expected.json new file mode 100644 index 000000000..7d2117afc --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_matchArrayOfStrings/expected.json @@ -0,0 +1,15 @@ +{ + "type" : [ "dog", "dog", "dog" ], + "test" : "test" +} +{ + "type" : [ "video", "dog", "dog" ], + "test" : "test" +} +{ + "type" : [ "book", "pc" ] +} +{ + "type" : [ "stone", "cat" ], + "test" : "test" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_matchArrayOfStrings/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_matchArrayOfStrings/input.json new file mode 100644 index 000000000..94ebe864e --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_matchArrayOfStrings/input.json @@ -0,0 +1,12 @@ +{ + "type" : [ "dog", "dog", "dog" ] +} +{ + "type" : [ "video", "dog", "dog" ] +} +{ + "type" : [ "book", "pc" ] +} +{ + "type" : [ "stone", "cat" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_matchArrayOfStrings/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_matchArrayOfStrings/test.fix new file mode 100644 index 000000000..e6b026619 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_matchArrayOfStrings/test.fix @@ -0,0 +1,3 @@ +if any_match("type[]", "\\S{3}") + add_field("test", "test") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_matchArrayOfStrings/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_matchArrayOfStrings/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_matchArrayOfStrings/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_matchSimple/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_matchSimple/expected.json new file mode 100644 index 000000000..b6c3e2c62 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_matchSimple/expected.json @@ -0,0 +1,14 @@ +{ + "type" : "dog" +} +{ + "type" : "video", + "test" : "test" +} +{ + "type" : "book" +} +{ + "type" : "stone", + "test" : "test" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_matchSimple/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_matchSimple/input.json new file mode 100644 index 000000000..3b0365e84 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_matchSimple/input.json @@ -0,0 +1,13 @@ +{ + "type" : "dog" +} +{ + "type" : "video" +} +{ + "type" : "book" +} +{ + "type" : "stone" +} + diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_matchSimple/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_matchSimple/test.fix new file mode 100644 index 000000000..388b374d6 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_matchSimple/test.fix @@ -0,0 +1,3 @@ +if any_match("type", "\\S{5}") + add_field("test", "test") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_matchSimple/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_matchSimple/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifAny_matchSimple/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifExistsSimple/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifExistsSimple/expected.json new file mode 100644 index 000000000..95c74b365 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifExistsSimple/expected.json @@ -0,0 +1,7 @@ +{ + "type" : "dog", + "test" : "test" +} +{ + "not_type" : "video" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifExistsSimple/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifExistsSimple/input.json new file mode 100644 index 000000000..a2f4346aa --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifExistsSimple/input.json @@ -0,0 +1,6 @@ +{ + "type" : "dog" +} +{ + "not_type" : "video" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifExistsSimple/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifExistsSimple/test.fix new file mode 100644 index 000000000..9b631a5e0 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifExistsSimple/test.fix @@ -0,0 +1,3 @@ +if exists("type") + add_field("test", "test") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifExistsSimple/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifExistsSimple/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifExistsSimple/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifInArrayOfStrings/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifInArrayOfStrings/expected.json new file mode 100644 index 000000000..6aed08349 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifInArrayOfStrings/expected.json @@ -0,0 +1,9 @@ +{ + "test" : "cat", + "animals" : [ "cat", "dog", "zebra" ], + "this" : "works" +} +{ + "test" : "cat", + "animal" : [ "parrot", "dog", "zebra" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifInArrayOfStrings/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifInArrayOfStrings/input.json new file mode 100644 index 000000000..1257bed1c --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifInArrayOfStrings/input.json @@ -0,0 +1,8 @@ +{ + "test" : "cat", + "animals" : [ "cat", "dog", "zebra" ] +} +{ + "test" : "cat", + "animal" : [ "parrot", "dog", "zebra" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifInArrayOfStrings/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifInArrayOfStrings/test.fix new file mode 100644 index 000000000..87b3fd6c1 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifInArrayOfStrings/test.fix @@ -0,0 +1,3 @@ +if in("test","animals[]") + add_field("this","works") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifInArrayOfStrings/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifInArrayOfStrings/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifInArrayOfStrings/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifInSimpleField/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifInSimpleField/expected.json new file mode 100644 index 000000000..c2aba9485 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifInSimpleField/expected.json @@ -0,0 +1,9 @@ +{ + "test" : "cat", + "animal" : "cat", + "this" : "works" +} +{ + "test" : "cat", + "animal" : "dog" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifInSimpleField/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifInSimpleField/input.json new file mode 100644 index 000000000..bd8edcdf6 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifInSimpleField/input.json @@ -0,0 +1,8 @@ +{ + "test" : "cat", + "animal" : "cat" +} +{ + "test" : "cat", + "animal" : "dog" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifInSimpleField/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifInSimpleField/test.fix new file mode 100644 index 000000000..207e71bd8 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifInSimpleField/test.fix @@ -0,0 +1,3 @@ +if in("animal","test") + add_field("this","works") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifInSimpleField/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifInSimpleField/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifInSimpleField/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsArraySimple/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsArraySimple/expected.json new file mode 100644 index 000000000..05719d37e --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsArraySimple/expected.json @@ -0,0 +1,12 @@ +{ + "elementA" : "test", + "elementA" : "Another Test", + "elementB" : "Hello" +} +{ + "elementA" : "test" +} +{ + "elementA" : [ "Test", "Another Test" ], + "elementB" : "Hello" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsArraySimple/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsArraySimple/input.json new file mode 100644 index 000000000..951c13c3b --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsArraySimple/input.json @@ -0,0 +1,11 @@ +{ + "elementA" : "test", + "elementA" : "Another Test" +} +{ + "elementA" : "test" +} +{ + "elementA" : [ "Test", "Another Test" ] +} + diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsArraySimple/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsArraySimple/test.fix new file mode 100644 index 000000000..8159ff029 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsArraySimple/test.fix @@ -0,0 +1,3 @@ +if is_array("elementA|elementA[]") + add_field("elementB", "Hello") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsArraySimple/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsArraySimple/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsArraySimple/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsObjectSimple/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsObjectSimple/expected.json new file mode 100644 index 000000000..82d9dc3ba --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsObjectSimple/expected.json @@ -0,0 +1,13 @@ +{ + "elementA" : { + "test" : "element" + }, + "elementB" : "Hello", + "elementC" : "World" +} +{ + "elementA" : "test" +} +{ + "elementA" : [ "Test", "Another Test" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsObjectSimple/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsObjectSimple/input.json new file mode 100644 index 000000000..a2edfc1c7 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsObjectSimple/input.json @@ -0,0 +1,12 @@ +{ + "elementA" : { + "test" : "element" + } +} +{ + "elementA" : "test" +} +{ + "elementA" : [ "Test", "Another Test" ] +} + diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsObjectSimple/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsObjectSimple/test.fix new file mode 100644 index 000000000..983b07f9d --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsObjectSimple/test.fix @@ -0,0 +1,7 @@ +if is_object("elementA|elementA[]") + add_field("elementB", "Hello") +end + +if is_hash("elementA|elementA[]") + add_field("elementC", "World") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsObjectSimple/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsObjectSimple/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsObjectSimple/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsStringSimple/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsStringSimple/expected.json new file mode 100644 index 000000000..0fc52545b --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsStringSimple/expected.json @@ -0,0 +1,12 @@ +{ + "elementA" : { + "test" : "element" + } +} +{ + "elementA" : "test", + "elementB" : "Hello" +} +{ + "elementA" : [ "Test", "Another Test" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsStringSimple/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsStringSimple/input.json new file mode 100644 index 000000000..a2edfc1c7 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsStringSimple/input.json @@ -0,0 +1,12 @@ +{ + "elementA" : { + "test" : "element" + } +} +{ + "elementA" : "test" +} +{ + "elementA" : [ "Test", "Another Test" ] +} + diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsStringSimple/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsStringSimple/test.fix new file mode 100644 index 000000000..9949663c6 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsStringSimple/test.fix @@ -0,0 +1,3 @@ +if is_string("elementA|elementA[]") + add_field("elementB", "Hello") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsStringSimple/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsStringSimple/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsStringSimple/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsStringsButNumbers/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsStringsButNumbers/expected.json new file mode 100644 index 000000000..a02550ea6 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsStringsButNumbers/expected.json @@ -0,0 +1,4 @@ +{ + "elementA" : "123", + "elementB" : "Hello" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsStringsButNumbers/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsStringsButNumbers/input.json new file mode 100644 index 000000000..6b81e2450 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsStringsButNumbers/input.json @@ -0,0 +1,5 @@ +{ + "elementA" : "123" +} + + diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsStringsButNumbers/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsStringsButNumbers/test.fix new file mode 100644 index 000000000..2ee95c1d8 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsStringsButNumbers/test.fix @@ -0,0 +1,3 @@ +if is_string("elementA") + add_field("elementB", "Hello") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsStringsButNumbers/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsStringsButNumbers/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsStringsButNumbers/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsStringsButNumbers/todo.txt b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsStringsButNumbers/todo.txt new file mode 100644 index 000000000..cca72167c --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifIsStringsButNumbers/todo.txt @@ -0,0 +1 @@ +See issue LibreCat/Catmandu#393 diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifWithElse/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifWithElse/expected.json new file mode 100644 index 000000000..85d2c0237 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifWithElse/expected.json @@ -0,0 +1,30 @@ +{ + "name" : "red", + "type" : "color", + "test" : "false" +} +{ + "name" : "green", + "type" : "color", + "test" : "false" +} +{ + "name" : "blue", + "type" : "color", + "test" : "true" +} +{ + "name" : "purple", + "type" : "color", + "test" : "false" +} +{ + "name" : "orange", + "type" : "color", + "test" : "false" +} +{ + "name" : "yellow", + "type" : "color", + "test" : "false" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifWithElse/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifWithElse/input.json new file mode 100644 index 000000000..0ca0c305c --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifWithElse/input.json @@ -0,0 +1,24 @@ +{ + "name": "red", + "type": "color" +} +{ + "name": "green", + "type": "color" +} +{ + "name": "blue", + "type": "color" +} +{ + "name": "purple", + "type": "color" +} +{ + "name": "orange", + "type": "color" +} +{ + "name": "yellow", + "type": "color" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifWithElse/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifWithElse/test.fix new file mode 100644 index 000000000..c5cd3c5dc --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifWithElse/test.fix @@ -0,0 +1,5 @@ +if all_equal("name", "blue") + add_field("test", "true") +else + add_field("test", "false") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifWithElse/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifWithElse/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifWithElse/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifWithElsifAndElse/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifWithElsifAndElse/expected.json new file mode 100644 index 000000000..294155bef --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifWithElsifAndElse/expected.json @@ -0,0 +1,30 @@ +{ + "name" : "red", + "type" : "color", + "test_2" : "true" +} +{ + "name" : "green", + "type" : "color", + "test" : "false" +} +{ + "name" : "blue", + "type" : "color", + "test" : "true" +} +{ + "name" : "purple", + "type" : "color", + "test" : "false" +} +{ + "name" : "orange", + "type" : "color", + "test" : "false" +} +{ + "name" : "yellow", + "type" : "color", + "test" : "false" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifWithElsifAndElse/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifWithElsifAndElse/input.json new file mode 100644 index 000000000..0ca0c305c --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifWithElsifAndElse/input.json @@ -0,0 +1,24 @@ +{ + "name": "red", + "type": "color" +} +{ + "name": "green", + "type": "color" +} +{ + "name": "blue", + "type": "color" +} +{ + "name": "purple", + "type": "color" +} +{ + "name": "orange", + "type": "color" +} +{ + "name": "yellow", + "type": "color" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifWithElsifAndElse/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifWithElsifAndElse/test.fix new file mode 100644 index 000000000..a3f2ee5e8 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifWithElsifAndElse/test.fix @@ -0,0 +1,7 @@ +if all_equal("name", "blue") + add_field("test", "true") +elsif all_equal("name", "red") + add_field("test_2", "true") +else + add_field("test", "false") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifWithElsifAndElse/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifWithElsifAndElse/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifWithElsifAndElse/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifWithMultipleElsifAndElse/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifWithMultipleElsifAndElse/expected.json new file mode 100644 index 000000000..d92d2ff31 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifWithMultipleElsifAndElse/expected.json @@ -0,0 +1,30 @@ +{ + "name" : "red", + "type" : "color", + "test_2" : "true" +} +{ + "name" : "green", + "type" : "color", + "test_4" : "true" +} +{ + "name" : "blue", + "type" : "color", + "test" : "true" +} +{ + "name" : "purple", + "type" : "color", + "test" : "false" +} +{ + "name" : "orange", + "type" : "color", + "test" : "false" +} +{ + "name" : "yellow", + "type" : "color", + "test_3" : "true" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifWithMultipleElsifAndElse/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifWithMultipleElsifAndElse/input.json new file mode 100644 index 000000000..0ca0c305c --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifWithMultipleElsifAndElse/input.json @@ -0,0 +1,24 @@ +{ + "name": "red", + "type": "color" +} +{ + "name": "green", + "type": "color" +} +{ + "name": "blue", + "type": "color" +} +{ + "name": "purple", + "type": "color" +} +{ + "name": "orange", + "type": "color" +} +{ + "name": "yellow", + "type": "color" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifWithMultipleElsifAndElse/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifWithMultipleElsifAndElse/test.fix new file mode 100644 index 000000000..710a60ffc --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifWithMultipleElsifAndElse/test.fix @@ -0,0 +1,11 @@ +if all_equal("name", "blue") + add_field("test", "true") +elsif all_equal("name", "red") + add_field("test_2", "true") +elsif all_equal("name", "yellow") + add_field("test_3", "true") +elsif all_equal("name", "green") + add_field("test_4", "true") +else + add_field("test", "false") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifWithMultipleElsifAndElse/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifWithMultipleElsifAndElse/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/ifWithMultipleElsifAndElse/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_containArrayOfStrings/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_containArrayOfStrings/expected.json new file mode 100644 index 000000000..9b4938fd1 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_containArrayOfStrings/expected.json @@ -0,0 +1,14 @@ +{ + "type" : [ "dog", "dog", "dog" ] +} +{ + "type" : [ "video", "dog", "dog" ] +} +{ + "type" : [ "book", "pc" ], + "test" : "test" +} +{ + "type" : [ "human", "cat" ], + "test" : "test" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_containArrayOfStrings/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_containArrayOfStrings/input.json new file mode 100644 index 000000000..b3bf99adf --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_containArrayOfStrings/input.json @@ -0,0 +1,12 @@ +{ + "type" : [ "dog", "dog", "dog" ] +} +{ + "type" : [ "video", "dog", "dog" ] +} +{ + "type" : [ "book", "pc" ] +} +{ + "type" : [ "human", "cat" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_containArrayOfStrings/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_containArrayOfStrings/test.fix new file mode 100644 index 000000000..e9b116762 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_containArrayOfStrings/test.fix @@ -0,0 +1,3 @@ +unless all_contain("type[]", "o") + add_field("test", "test") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_containArrayOfStrings/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_containArrayOfStrings/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_containArrayOfStrings/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_containSimple/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_containSimple/expected.json new file mode 100644 index 000000000..a9dfc678e --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_containSimple/expected.json @@ -0,0 +1,14 @@ +{ + "type" : "dog", + "test" : "test" +} +{ + "type" : "video" +} +{ + "type" : "book", + "test" : "test" +} +{ + "type" : "stone" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_containSimple/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_containSimple/input.json new file mode 100644 index 000000000..456749e6b --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_containSimple/input.json @@ -0,0 +1,12 @@ +{ + "type" : "dog" +} +{ + "type" : "video" +} +{ + "type" : "book" +} +{ + "type" : "stone" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_containSimple/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_containSimple/test.fix new file mode 100644 index 000000000..9aee0063c --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_containSimple/test.fix @@ -0,0 +1,3 @@ +unless all_contain("type", "e") + add_field("test", "test") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_containSimple/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_containSimple/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_containSimple/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_equalArrayOfStrings/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_equalArrayOfStrings/expected.json new file mode 100644 index 000000000..357da237a --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_equalArrayOfStrings/expected.json @@ -0,0 +1,15 @@ +{ + "type" : [ "dog", "dog", "dog" ] +} +{ + "type" : [ "video", "dog", "dog" ], + "test" : "test" +} +{ + "type" : [ "book", "dog", "dog" ], + "test" : "test" +} +{ + "type" : [ "stone", "cat" ], + "test" : "test" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_equalArrayOfStrings/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_equalArrayOfStrings/input.json new file mode 100644 index 000000000..44f9a90f4 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_equalArrayOfStrings/input.json @@ -0,0 +1,13 @@ +{ + "type" : [ "dog", "dog", "dog" ] +} +{ + "type" : [ "video", "dog", "dog" ] +} +{ + "type" : [ "book", "dog", "dog" ] +} +{ + "type" : [ "stone", "cat" ] +} + diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_equalArrayOfStrings/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_equalArrayOfStrings/test.fix new file mode 100644 index 000000000..72a945e35 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_equalArrayOfStrings/test.fix @@ -0,0 +1,3 @@ +unless all_equal("type[]", "dog") + add_field("test", "test") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_equalArrayOfStrings/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_equalArrayOfStrings/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_equalArrayOfStrings/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_equalSimple/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_equalSimple/expected.json new file mode 100644 index 000000000..adf602150 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_equalSimple/expected.json @@ -0,0 +1,15 @@ +{ + "type" : "dog", + "test" : "test" +} +{ + "type" : "video", + "test" : "test" +} +{ + "type" : "book" +} +{ + "type" : "stone", + "test" : "test" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_equalSimple/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_equalSimple/input.json new file mode 100644 index 000000000..3b0365e84 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_equalSimple/input.json @@ -0,0 +1,13 @@ +{ + "type" : "dog" +} +{ + "type" : "video" +} +{ + "type" : "book" +} +{ + "type" : "stone" +} + diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_equalSimple/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_equalSimple/test.fix new file mode 100644 index 000000000..15184797f --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_equalSimple/test.fix @@ -0,0 +1,3 @@ +unless all_equal("type", "book") + add_field("test", "test") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_equalSimple/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_equalSimple/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_equalSimple/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_matchArrayOfStrings/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_matchArrayOfStrings/expected.json new file mode 100644 index 000000000..9f75db565 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_matchArrayOfStrings/expected.json @@ -0,0 +1,15 @@ +{ + "type" : [ "dog", "dog", "dog" ] +} +{ + "type" : [ "video", "dog", "dog" ], + "test" : "test" +} +{ + "type" : [ "book", "pc" ], + "test" : "test" +} +{ + "type" : [ "stone", "cat" ], + "test" : "test" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_matchArrayOfStrings/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_matchArrayOfStrings/input.json new file mode 100644 index 000000000..94ebe864e --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_matchArrayOfStrings/input.json @@ -0,0 +1,12 @@ +{ + "type" : [ "dog", "dog", "dog" ] +} +{ + "type" : [ "video", "dog", "dog" ] +} +{ + "type" : [ "book", "pc" ] +} +{ + "type" : [ "stone", "cat" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_matchArrayOfStrings/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_matchArrayOfStrings/test.fix new file mode 100644 index 000000000..4c1d79ba3 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_matchArrayOfStrings/test.fix @@ -0,0 +1,3 @@ +unless all_match("type[]", "\\S{3}") + add_field("test", "test") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_matchArrayOfStrings/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_matchArrayOfStrings/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_matchArrayOfStrings/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_matchSimple/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_matchSimple/expected.json new file mode 100644 index 000000000..a9dfc678e --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_matchSimple/expected.json @@ -0,0 +1,14 @@ +{ + "type" : "dog", + "test" : "test" +} +{ + "type" : "video" +} +{ + "type" : "book", + "test" : "test" +} +{ + "type" : "stone" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_matchSimple/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_matchSimple/input.json new file mode 100644 index 000000000..3b0365e84 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_matchSimple/input.json @@ -0,0 +1,13 @@ +{ + "type" : "dog" +} +{ + "type" : "video" +} +{ + "type" : "book" +} +{ + "type" : "stone" +} + diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_matchSimple/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_matchSimple/test.fix new file mode 100644 index 000000000..27a35c577 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_matchSimple/test.fix @@ -0,0 +1,3 @@ +unless all_match("type", "\\S{5}") + add_field("test", "test") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_matchSimple/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_matchSimple/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAll_matchSimple/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_containArrayOfStrings/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_containArrayOfStrings/expected.json new file mode 100644 index 000000000..421b47af4 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_containArrayOfStrings/expected.json @@ -0,0 +1,13 @@ +{ + "type" : [ "dog", "dog", "dog" ] +} +{ + "type" : [ "video", "dog", "dog" ] +} +{ + "type" : [ "book", "pc" ] +} +{ + "type" : [ "human", "cat" ], + "test" : "test" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_containArrayOfStrings/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_containArrayOfStrings/input.json new file mode 100644 index 000000000..b3bf99adf --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_containArrayOfStrings/input.json @@ -0,0 +1,12 @@ +{ + "type" : [ "dog", "dog", "dog" ] +} +{ + "type" : [ "video", "dog", "dog" ] +} +{ + "type" : [ "book", "pc" ] +} +{ + "type" : [ "human", "cat" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_containArrayOfStrings/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_containArrayOfStrings/test.fix new file mode 100644 index 000000000..050369082 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_containArrayOfStrings/test.fix @@ -0,0 +1,3 @@ +unless any_contain("type[]", "o") + add_field("test", "test") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_containArrayOfStrings/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_containArrayOfStrings/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_containArrayOfStrings/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_containSimple/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_containSimple/expected.json new file mode 100644 index 000000000..a9dfc678e --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_containSimple/expected.json @@ -0,0 +1,14 @@ +{ + "type" : "dog", + "test" : "test" +} +{ + "type" : "video" +} +{ + "type" : "book", + "test" : "test" +} +{ + "type" : "stone" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_containSimple/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_containSimple/input.json new file mode 100644 index 000000000..456749e6b --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_containSimple/input.json @@ -0,0 +1,12 @@ +{ + "type" : "dog" +} +{ + "type" : "video" +} +{ + "type" : "book" +} +{ + "type" : "stone" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_containSimple/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_containSimple/test.fix new file mode 100644 index 000000000..67503aae3 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_containSimple/test.fix @@ -0,0 +1,3 @@ +unless any_contain("type", "e") + add_field("test", "test") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_containSimple/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_containSimple/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_containSimple/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_equalArrayOfStrings/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_equalArrayOfStrings/expected.json new file mode 100644 index 000000000..2e4058968 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_equalArrayOfStrings/expected.json @@ -0,0 +1,13 @@ +{ + "type" : [ "dog", "dog", "dog" ] +} +{ + "type" : [ "video", "dog", "dog" ] +} +{ + "type" : [ "book", "dog", "dog" ] +} +{ + "type" : [ "stone", "cat" ], + "test" : "test" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_equalArrayOfStrings/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_equalArrayOfStrings/input.json new file mode 100644 index 000000000..9d3027f5f --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_equalArrayOfStrings/input.json @@ -0,0 +1,12 @@ +{ + "type" : [ "dog", "dog", "dog" ] +} +{ + "type" : [ "video", "dog", "dog" ] +} +{ + "type" : [ "book", "dog", "dog" ] +} +{ + "type" : [ "stone", "cat" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_equalArrayOfStrings/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_equalArrayOfStrings/test.fix new file mode 100644 index 000000000..a6b584093 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_equalArrayOfStrings/test.fix @@ -0,0 +1,3 @@ +unless any_equal("type[]", "dog") + add_field("test", "test") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_equalArrayOfStrings/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_equalArrayOfStrings/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_equalArrayOfStrings/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_equalSimple/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_equalSimple/expected.json new file mode 100644 index 000000000..adf602150 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_equalSimple/expected.json @@ -0,0 +1,15 @@ +{ + "type" : "dog", + "test" : "test" +} +{ + "type" : "video", + "test" : "test" +} +{ + "type" : "book" +} +{ + "type" : "stone", + "test" : "test" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_equalSimple/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_equalSimple/input.json new file mode 100644 index 000000000..456749e6b --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_equalSimple/input.json @@ -0,0 +1,12 @@ +{ + "type" : "dog" +} +{ + "type" : "video" +} +{ + "type" : "book" +} +{ + "type" : "stone" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_equalSimple/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_equalSimple/test.fix new file mode 100644 index 000000000..e8608f684 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_equalSimple/test.fix @@ -0,0 +1,3 @@ +unless any_equal("type", "book") + add_field("test", "test") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_equalSimple/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_equalSimple/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_equalSimple/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_matchArrayOfStrings/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_matchArrayOfStrings/expected.json new file mode 100644 index 000000000..396182480 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_matchArrayOfStrings/expected.json @@ -0,0 +1,13 @@ +{ + "type" : [ "dog", "dog", "dog" ] +} +{ + "type" : [ "video", "dog", "dog" ] +} +{ + "type" : [ "book", "pc" ], + "test" : "test" +} +{ + "type" : [ "stone", "cat" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_matchArrayOfStrings/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_matchArrayOfStrings/input.json new file mode 100644 index 000000000..94ebe864e --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_matchArrayOfStrings/input.json @@ -0,0 +1,12 @@ +{ + "type" : [ "dog", "dog", "dog" ] +} +{ + "type" : [ "video", "dog", "dog" ] +} +{ + "type" : [ "book", "pc" ] +} +{ + "type" : [ "stone", "cat" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_matchArrayOfStrings/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_matchArrayOfStrings/test.fix new file mode 100644 index 000000000..537f0bb6e --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_matchArrayOfStrings/test.fix @@ -0,0 +1,3 @@ +unless any_match("type[]", "\\S{3}") + add_field("test", "test") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_matchArrayOfStrings/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_matchArrayOfStrings/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_matchArrayOfStrings/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_matchSimple/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_matchSimple/expected.json new file mode 100644 index 000000000..a9dfc678e --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_matchSimple/expected.json @@ -0,0 +1,14 @@ +{ + "type" : "dog", + "test" : "test" +} +{ + "type" : "video" +} +{ + "type" : "book", + "test" : "test" +} +{ + "type" : "stone" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_matchSimple/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_matchSimple/input.json new file mode 100644 index 000000000..3b0365e84 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_matchSimple/input.json @@ -0,0 +1,13 @@ +{ + "type" : "dog" +} +{ + "type" : "video" +} +{ + "type" : "book" +} +{ + "type" : "stone" +} + diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_matchSimple/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_matchSimple/test.fix new file mode 100644 index 000000000..27a35c577 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_matchSimple/test.fix @@ -0,0 +1,3 @@ +unless all_match("type", "\\S{5}") + add_field("test", "test") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_matchSimple/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_matchSimple/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessAny_matchSimple/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessExistsSimple/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessExistsSimple/expected.json new file mode 100644 index 000000000..4c08a71cf --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessExistsSimple/expected.json @@ -0,0 +1,7 @@ +{ + "type" : "dog" +} +{ + "not_type" : "video", + "test" : "test" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessExistsSimple/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessExistsSimple/input.json new file mode 100644 index 000000000..a2f4346aa --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessExistsSimple/input.json @@ -0,0 +1,6 @@ +{ + "type" : "dog" +} +{ + "not_type" : "video" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessExistsSimple/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessExistsSimple/test.fix new file mode 100644 index 000000000..946a6a942 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessExistsSimple/test.fix @@ -0,0 +1,3 @@ +unless exists("type") + add_field("test", "test") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessExistsSimple/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessExistsSimple/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessExistsSimple/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessExists_createArrayOfStringsAndCopyField/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessExists_createArrayOfStringsAndCopyField/expected.json new file mode 100644 index 000000000..4b2d62713 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessExists_createArrayOfStringsAndCopyField/expected.json @@ -0,0 +1,11 @@ +{ + "animals" : [ "dog", "parrot", "shark" ] +} +{ + "animals" : "human", + "animals" : [ "human" ] +} +{ + "animals" : "dog", + "animals" : [ "dog" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessExists_createArrayOfStringsAndCopyField/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessExists_createArrayOfStringsAndCopyField/input.json new file mode 100644 index 000000000..872462ef3 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessExists_createArrayOfStringsAndCopyField/input.json @@ -0,0 +1,9 @@ +{ + "animals" : [ "dog", "parrot", "shark" ] +} +{ + "animals" : "human" +} +{ + "animals" : "dog" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessExists_createArrayOfStringsAndCopyField/mapfile.tsv b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessExists_createArrayOfStringsAndCopyField/mapfile.tsv new file mode 100644 index 000000000..e7f9756ae --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessExists_createArrayOfStringsAndCopyField/mapfile.tsv @@ -0,0 +1,6 @@ +dog mammal +cat mammal +parrot bird +shark fish +dragon fictional animal +unicorn fictional animal diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessExists_createArrayOfStringsAndCopyField/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessExists_createArrayOfStringsAndCopyField/test.fix new file mode 100644 index 000000000..1543d2d99 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessExists_createArrayOfStringsAndCopyField/test.fix @@ -0,0 +1,3 @@ +unless exists("animals[]") + copy_field("animals","animals[].$append") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessExists_createArrayOfStringsAndCopyField/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessExists_createArrayOfStringsAndCopyField/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessExists_createArrayOfStringsAndCopyField/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessInArrayOfStrings/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessInArrayOfStrings/expected.json new file mode 100644 index 000000000..6ec5ef6d8 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessInArrayOfStrings/expected.json @@ -0,0 +1,9 @@ +{ + "test" : "cat", + "animals" : [ "cat", "dog", "zebra" ] +} +{ + "test" : "cat", + "animal" : [ "parrot", "dog", "zebra" ], + "this" : "works" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessInArrayOfStrings/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessInArrayOfStrings/input.json new file mode 100644 index 000000000..1257bed1c --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessInArrayOfStrings/input.json @@ -0,0 +1,8 @@ +{ + "test" : "cat", + "animals" : [ "cat", "dog", "zebra" ] +} +{ + "test" : "cat", + "animal" : [ "parrot", "dog", "zebra" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessInArrayOfStrings/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessInArrayOfStrings/test.fix new file mode 100644 index 000000000..cd7c0a275 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessInArrayOfStrings/test.fix @@ -0,0 +1,3 @@ +unless in("test","animals[]") + add_field("this","works") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessInArrayOfStrings/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessInArrayOfStrings/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessInArrayOfStrings/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessInSimpleField/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessInSimpleField/expected.json new file mode 100644 index 000000000..2643413a1 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessInSimpleField/expected.json @@ -0,0 +1,9 @@ +{ + "test" : "cat", + "animal" : "cat" +} +{ + "test" : "cat", + "animal" : "dog", + "this" : "works" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessInSimpleField/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessInSimpleField/input.json new file mode 100644 index 000000000..bd8edcdf6 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessInSimpleField/input.json @@ -0,0 +1,8 @@ +{ + "test" : "cat", + "animal" : "cat" +} +{ + "test" : "cat", + "animal" : "dog" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessInSimpleField/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessInSimpleField/test.fix new file mode 100644 index 000000000..b2d6d2155 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessInSimpleField/test.fix @@ -0,0 +1,3 @@ +unless in("animal","test") + add_field("this","works") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessInSimpleField/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessInSimpleField/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessInSimpleField/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessIsArraySimple/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessIsArraySimple/expected.json new file mode 100644 index 000000000..12f4ddc82 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessIsArraySimple/expected.json @@ -0,0 +1,11 @@ +{ + "elementA" : "test", + "elementA" : "Another Test" +} +{ + "elementA" : "test", + "elementB" : "Hello" +} +{ + "elementA" : [ "Test", "Another Test" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessIsArraySimple/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessIsArraySimple/input.json new file mode 100644 index 000000000..951c13c3b --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessIsArraySimple/input.json @@ -0,0 +1,11 @@ +{ + "elementA" : "test", + "elementA" : "Another Test" +} +{ + "elementA" : "test" +} +{ + "elementA" : [ "Test", "Another Test" ] +} + diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessIsArraySimple/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessIsArraySimple/test.fix new file mode 100644 index 000000000..f047da659 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessIsArraySimple/test.fix @@ -0,0 +1,3 @@ +unless is_array("elementA|elementA[]") + add_field("elementB", "Hello") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessIsArraySimple/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessIsArraySimple/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessIsArraySimple/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessIsObjectSimple/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessIsObjectSimple/expected.json new file mode 100644 index 000000000..52ff922a1 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessIsObjectSimple/expected.json @@ -0,0 +1,15 @@ +{ + "elementA" : { + "test" : "element" + } +} +{ + "elementA" : "test", + "elementB" : "Hello", + "elementC" : "World" +} +{ + "elementA" : [ "Test", "Another Test" ], + "elementB" : "Hello", + "elementC" : "World" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessIsObjectSimple/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessIsObjectSimple/input.json new file mode 100644 index 000000000..a2edfc1c7 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessIsObjectSimple/input.json @@ -0,0 +1,12 @@ +{ + "elementA" : { + "test" : "element" + } +} +{ + "elementA" : "test" +} +{ + "elementA" : [ "Test", "Another Test" ] +} + diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessIsObjectSimple/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessIsObjectSimple/test.fix new file mode 100644 index 000000000..ced4195e9 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessIsObjectSimple/test.fix @@ -0,0 +1,7 @@ +unless is_object("elementA|elementA[]") + add_field("elementB", "Hello") +end + +unless is_hash("elementA|elementA[]") + add_field("elementC", "World") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessIsObjectSimple/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessIsObjectSimple/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessIsObjectSimple/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessIsStringSimple/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessIsStringSimple/expected.json new file mode 100644 index 000000000..6d1441e43 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessIsStringSimple/expected.json @@ -0,0 +1,13 @@ +{ + "elementA" : { + "test" : "element" + }, + "elementB" : "Hello" +} +{ + "elementA" : "test" +} +{ + "elementA" : [ "Test", "Another Test" ], + "elementB" : "Hello" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessIsStringSimple/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessIsStringSimple/input.json new file mode 100644 index 000000000..a2edfc1c7 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessIsStringSimple/input.json @@ -0,0 +1,12 @@ +{ + "elementA" : { + "test" : "element" + } +} +{ + "elementA" : "test" +} +{ + "elementA" : [ "Test", "Another Test" ] +} + diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessIsStringSimple/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessIsStringSimple/test.fix new file mode 100644 index 000000000..da7f8bdd9 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessIsStringSimple/test.fix @@ -0,0 +1,3 @@ +unless is_string("elementA|elementA[]") + add_field("elementB", "Hello") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessIsStringSimple/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessIsStringSimple/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromJson/toJson/unlessIsStringSimple/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromXml/toJson/ifIsArraySimple/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromXml/toJson/ifIsArraySimple/expected.json new file mode 100644 index 000000000..8746a8e7e --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromXml/toJson/ifIsArraySimple/expected.json @@ -0,0 +1,8 @@ +{ + "123" : "Test" +} +{ + "123" : "Test", + "123" : "Another Test", + "test" : "Hello" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromXml/toJson/ifIsArraySimple/input.xml b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromXml/toJson/ifIsArraySimple/input.xml new file mode 100644 index 000000000..e818d4d11 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromXml/toJson/ifIsArraySimple/input.xml @@ -0,0 +1,10 @@ + + + + Test + + + Test + Another Test + + diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromXml/toJson/ifIsArraySimple/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromXml/toJson/ifIsArraySimple/test.fix new file mode 100644 index 000000000..e1dc4406c --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromXml/toJson/ifIsArraySimple/test.fix @@ -0,0 +1,3 @@ +if is_array("123") + add_field("test", "Hello") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromXml/toJson/ifIsArraySimple/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromXml/toJson/ifIsArraySimple/test.flux new file mode 100644 index 000000000..1d8c3df8f --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/conditional/fromXml/toJson/ifIsArraySimple/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.xml" +|open-file +|decode-xml +|handle-marcxml +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfObjects/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfObjects/expected.json new file mode 100644 index 000000000..8e371b205 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfObjects/expected.json @@ -0,0 +1,9 @@ +{ + "animals" : [ { + "name" : "Jake", + "type" : "mammal" + }, { + "name" : "Blacky", + "type" : "bird" + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfObjects/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfObjects/input.json new file mode 100644 index 000000000..387053945 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfObjects/input.json @@ -0,0 +1,9 @@ +{ + "animals": [{ + "name": "Jake", + "type": "dog" + },{ + "name": "Blacky", + "type": "parrot" + }] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfObjects/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfObjects/test.fix new file mode 100644 index 000000000..7c6137656 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfObjects/test.fix @@ -0,0 +1,9 @@ +put_map("map", + "dog":"mammal", + "cat":"mammal", + "parrot":"bird", + "shark":"fish", + "dragon":"fictional animal", + "unicorn":"fictional animal") + +lookup("animals[].*.type", "map") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfObjects/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfObjects/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfObjects/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfObjectsWithListBind/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfObjectsWithListBind/expected.json new file mode 100644 index 000000000..8e371b205 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfObjectsWithListBind/expected.json @@ -0,0 +1,9 @@ +{ + "animals" : [ { + "name" : "Jake", + "type" : "mammal" + }, { + "name" : "Blacky", + "type" : "bird" + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfObjectsWithListBind/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfObjectsWithListBind/input.json new file mode 100644 index 000000000..387053945 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfObjectsWithListBind/input.json @@ -0,0 +1,9 @@ +{ + "animals": [{ + "name": "Jake", + "type": "dog" + },{ + "name": "Blacky", + "type": "parrot" + }] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfObjectsWithListBind/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfObjectsWithListBind/test.fix new file mode 100644 index 000000000..9866ec52b --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfObjectsWithListBind/test.fix @@ -0,0 +1,11 @@ +put_map("map", + "dog":"mammal", + "cat":"mammal", + "parrot":"bird", + "shark":"fish", + "dragon":"fictional animal", + "unicorn":"fictional animal") + +do list(path: "animals[]") + lookup("type", "map") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfObjectsWithListBind/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfObjectsWithListBind/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfObjectsWithListBind/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStrings/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStrings/expected.json new file mode 100644 index 000000000..392ddb0bb --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStrings/expected.json @@ -0,0 +1,3 @@ +{ + "animals" : [ "mammal", "bird", "fish" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStrings/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStrings/input.json new file mode 100644 index 000000000..e65dda098 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStrings/input.json @@ -0,0 +1,3 @@ +{ + "animals" : [ "dog", "parrot", "shark" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStrings/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStrings/test.fix new file mode 100644 index 000000000..7935a9342 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStrings/test.fix @@ -0,0 +1,9 @@ +put_map("map", + "dog":"mammal", + "cat":"mammal", + "parrot":"bird", + "shark":"fish", + "dragon":"fictional animal", + "unicorn":"fictional animal") + +lookup("animals[].*", "map") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStrings/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStrings/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStrings/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsDeleteNotMatchingValues/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsDeleteNotMatchingValues/expected.json new file mode 100644 index 000000000..6a4aff11d --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsDeleteNotMatchingValues/expected.json @@ -0,0 +1,15 @@ +{ + "animals" : [ "mammal", "bird", "fish" ] +} +{ + "animals" : [ "bird", "fish" ] +} +{ + "animals" : [ "mammal" ] +} +{ + "animals" : [ "mammal", "fish" ] +} +{ + "animals" : [ ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsDeleteNotMatchingValues/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsDeleteNotMatchingValues/input.json new file mode 100644 index 000000000..ca6c661f4 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsDeleteNotMatchingValues/input.json @@ -0,0 +1,15 @@ +{ + "animals" : [ "dog", "parrot", "shark" ] +} +{ + "animals" : [ "human", "parrot", "shark" ] +} +{ + "animals" : [ "dog" ] +} +{ + "animals" : [ "dog", "human", "shark" ] +} +{ + "animals" : [ "human" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsDeleteNotMatchingValues/mapfile.tsv b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsDeleteNotMatchingValues/mapfile.tsv new file mode 100644 index 000000000..e7f9756ae --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsDeleteNotMatchingValues/mapfile.tsv @@ -0,0 +1,6 @@ +dog mammal +cat mammal +parrot bird +shark fish +dragon fictional animal +unicorn fictional animal diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsDeleteNotMatchingValues/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsDeleteNotMatchingValues/test.fix new file mode 100644 index 000000000..a069a35ea --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsDeleteNotMatchingValues/test.fix @@ -0,0 +1 @@ +lookup("animals[].*", "./mapfile.tsv", "sep_char":"\t", delete:"true") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsDeleteNotMatchingValues/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsDeleteNotMatchingValues/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsDeleteNotMatchingValues/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsWithNotMatchingValues/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsWithNotMatchingValues/expected.json new file mode 100644 index 000000000..1020784f3 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsWithNotMatchingValues/expected.json @@ -0,0 +1,15 @@ +{ + "animals" : [ "mammal", "bird", "fish" ] +} +{ + "animals" : [ "human", "bird", "fish" ] +} +{ + "animals" : [ "mammal" ] +} +{ + "animals" : [ "mammal", "human", "fish" ] +} +{ + "animals" : [ "human" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsWithNotMatchingValues/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsWithNotMatchingValues/input.json new file mode 100644 index 000000000..ca6c661f4 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsWithNotMatchingValues/input.json @@ -0,0 +1,15 @@ +{ + "animals" : [ "dog", "parrot", "shark" ] +} +{ + "animals" : [ "human", "parrot", "shark" ] +} +{ + "animals" : [ "dog" ] +} +{ + "animals" : [ "dog", "human", "shark" ] +} +{ + "animals" : [ "human" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsWithNotMatchingValues/mapfile.tsv b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsWithNotMatchingValues/mapfile.tsv new file mode 100644 index 000000000..e7f9756ae --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsWithNotMatchingValues/mapfile.tsv @@ -0,0 +1,6 @@ +dog mammal +cat mammal +parrot bird +shark fish +dragon fictional animal +unicorn fictional animal diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsWithNotMatchingValues/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsWithNotMatchingValues/test.fix new file mode 100644 index 000000000..1b453ed2b --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsWithNotMatchingValues/test.fix @@ -0,0 +1 @@ +lookup("animals[].*", "./mapfile.tsv", "sep_char":"\t") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsWithNotMatchingValues/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsWithNotMatchingValues/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsWithNotMatchingValues/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsWithNotMatchingValues_complex/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsWithNotMatchingValues_complex/expected.json new file mode 100644 index 000000000..978c63d97 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsWithNotMatchingValues_complex/expected.json @@ -0,0 +1,11 @@ +{ + "animals" : [ "mammal", "bird", "fish" ] +} +{ + "animals" : "human", + "animals" : [ "human" ] +} +{ + "animals" : "dog", + "animals" : [ "mammal" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsWithNotMatchingValues_complex/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsWithNotMatchingValues_complex/input.json new file mode 100644 index 000000000..872462ef3 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsWithNotMatchingValues_complex/input.json @@ -0,0 +1,9 @@ +{ + "animals" : [ "dog", "parrot", "shark" ] +} +{ + "animals" : "human" +} +{ + "animals" : "dog" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsWithNotMatchingValues_complex/mapfile.tsv b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsWithNotMatchingValues_complex/mapfile.tsv new file mode 100644 index 000000000..e7f9756ae --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsWithNotMatchingValues_complex/mapfile.tsv @@ -0,0 +1,6 @@ +dog mammal +cat mammal +parrot bird +shark fish +dragon fictional animal +unicorn fictional animal diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsWithNotMatchingValues_complex/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsWithNotMatchingValues_complex/test.fix new file mode 100644 index 000000000..5d5828afc --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsWithNotMatchingValues_complex/test.fix @@ -0,0 +1,5 @@ +unless exists("animals[]") + copy_field("animals","animals[].$append") +end + +lookup("animals[].*", "./mapfile.tsv", "sep_char":"\t") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsWithNotMatchingValues_complex/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsWithNotMatchingValues_complex/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInArrayOfStringsWithNotMatchingValues_complex/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInDeeplyNestedArrayOfObjects/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInDeeplyNestedArrayOfObjects/expected.json new file mode 100644 index 000000000..321419185 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInDeeplyNestedArrayOfObjects/expected.json @@ -0,0 +1,52 @@ +{ + "subject" : [ { + "componentList" : [ { + "gndIdentifier" : "(DE-588)4131620-4", + "id" : "(DE-588)4131620-4", + "label" : "Schulhof", + "source" : { + "id" : "https://d-nb.info/gnd/7749153-1", + "label" : "Gemeinsame Normdatei (GND)" + }, + "type" : [ "SubjectHeading" ] + }, { + "gndIdentifier" : "(DE-588)4116546-9", + "id" : "(DE-588)4116546-9", + "label" : "Sozialraum", + "source" : { + "id" : "https://d-nb.info/gnd/7749153-1", + "label" : "Gemeinsame Normdatei (GND)" + }, + "type" : [ "SubjectHeading" ] + }, { + "gndIdentifier" : "(DE-588)4680202-2", + "id" : "(DE-588)4680202-2", + "label" : "Informelles Lernen", + "source" : { + "id" : "https://d-nb.info/gnd/7749153-1", + "label" : "Gemeinsame Normdatei (GND)" + }, + "type" : [ "SubjectHeading" ] + }, { + "gndIdentifier" : "(DE-588)4155947-2", + "id" : "(DE-588)4155947-2", + "label" : "Ganztagsschule", + "source" : { + "id" : "https://d-nb.info/gnd/7749153-1", + "label" : "Gemeinsame Normdatei (GND)" + }, + "type" : [ "SubjectHeading" ] + }, { + "gndIdentifier" : "(DE-588)4130615-6", + "id" : "(DE-588)4130615-6", + "label" : "Sportpädagogik", + "source" : { + "id" : "https://d-nb.info/gnd/7749153-1", + "label" : "Gemeinsame Normdatei (GND)" + }, + "type" : [ "SubjectHeading" ] + } ], + "label" : [ ], + "type" : [ "ComplexSubject" ] + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInDeeplyNestedArrayOfObjects/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInDeeplyNestedArrayOfObjects/input.json new file mode 100644 index 000000000..0ad354ed6 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInDeeplyNestedArrayOfObjects/input.json @@ -0,0 +1,52 @@ +{ + "subject" : [ { + "componentList" : [ { + "gndIdentifier" : "(DE-588)4131620-4", + "id" : "(DE-588)4131620-4", + "label" : "Schulhof", + "source" : { + "id" : "https://d-nb.info/gnd/7749153-1", + "label" : "Gemeinsame Normdatei (GND)" + }, + "type" : [ "s" ] + }, { + "gndIdentifier" : "(DE-588)4116546-9", + "id" : "(DE-588)4116546-9", + "label" : "Sozialraum", + "source" : { + "id" : "https://d-nb.info/gnd/7749153-1", + "label" : "Gemeinsame Normdatei (GND)" + }, + "type" : [ "s" ] + }, { + "gndIdentifier" : "(DE-588)4680202-2", + "id" : "(DE-588)4680202-2", + "label" : "Informelles Lernen", + "source" : { + "id" : "https://d-nb.info/gnd/7749153-1", + "label" : "Gemeinsame Normdatei (GND)" + }, + "type" : [ "s" ] + }, { + "gndIdentifier" : "(DE-588)4155947-2", + "id" : "(DE-588)4155947-2", + "label" : "Ganztagsschule", + "source" : { + "id" : "https://d-nb.info/gnd/7749153-1", + "label" : "Gemeinsame Normdatei (GND)" + }, + "type" : [ "s" ] + }, { + "gndIdentifier" : "(DE-588)4130615-6", + "id" : "(DE-588)4130615-6", + "label" : "Sportpädagogik", + "source" : { + "id" : "https://d-nb.info/gnd/7749153-1", + "label" : "Gemeinsame Normdatei (GND)" + }, + "type" : [ "s" ] + } ], + "label" : [ ], + "type" : [ "ComplexSubject" ] + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInDeeplyNestedArrayOfObjects/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInDeeplyNestedArrayOfObjects/test.fix new file mode 100644 index 000000000..942ed6840 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInDeeplyNestedArrayOfObjects/test.fix @@ -0,0 +1,2 @@ +put_map("rswk-indicator", s: "SubjectHeading") +lookup("subject[].*.componentList[].*.type[].*", "rswk-indicator") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInDeeplyNestedArrayOfObjects/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInDeeplyNestedArrayOfObjects/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInDeeplyNestedArrayOfObjects/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects/expected.json new file mode 100644 index 000000000..ea680b41d --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects/expected.json @@ -0,0 +1,13 @@ +{ + "animals" : [ { + "name" : "Jake", + "classification" : { + "type" : "mammal" + } + }, { + "name" : "Blacky", + "classification" : { + "type" : "bird" + } + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects/input.json new file mode 100644 index 000000000..a747b8008 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects/input.json @@ -0,0 +1,13 @@ +{ + "animals": [{ + "name": "Jake", + "classification": { + "type": "dog" + } + },{ + "name": "Blacky", + "classification": { + "type": "parrot" + } + }] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects/mapfile.tsv b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects/mapfile.tsv new file mode 100644 index 000000000..e7f9756ae --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects/mapfile.tsv @@ -0,0 +1,6 @@ +dog mammal +cat mammal +parrot bird +shark fish +dragon fictional animal +unicorn fictional animal diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects/test.fix new file mode 100644 index 000000000..43b779389 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects/test.fix @@ -0,0 +1 @@ +lookup("animals[].*.classification.type", "./mapfile.tsv", "sep_char":"\t") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind/expected.json new file mode 100644 index 000000000..ea680b41d --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind/expected.json @@ -0,0 +1,13 @@ +{ + "animals" : [ { + "name" : "Jake", + "classification" : { + "type" : "mammal" + } + }, { + "name" : "Blacky", + "classification" : { + "type" : "bird" + } + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind/input.json new file mode 100644 index 000000000..d4b2b42de --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind/input.json @@ -0,0 +1,13 @@ +{ + "animals": [{ + "name": "Jake", + "classification": { + "type": "dog" + } + },{ + "name": "Blacky", + "classification": { + "type": "parrot" + } + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind/mapfile.tsv b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind/mapfile.tsv new file mode 100644 index 000000000..e7f9756ae --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind/mapfile.tsv @@ -0,0 +1,6 @@ +dog mammal +cat mammal +parrot bird +shark fish +dragon fictional animal +unicorn fictional animal diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind/test.fix new file mode 100644 index 000000000..3205409f7 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind/test.fix @@ -0,0 +1,3 @@ +do list(path: "animals[]") + lookup("classification.type", "./mapfile.tsv", "sep_char":"\t") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_complex/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_complex/expected.json new file mode 100644 index 000000000..371da87fc --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_complex/expected.json @@ -0,0 +1,14 @@ +{ + "test" : [ "dog", "parrot" ], + "animals" : [ { + "animal" : "dog", + "classification" : { + "type" : "mammal" + } + }, { + "animal" : "parrot", + "classification" : { + "type" : "bird" + } + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_complex/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_complex/input.json new file mode 100644 index 000000000..8376431bb --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_complex/input.json @@ -0,0 +1,3 @@ +{ + "test" : [ "dog", "parrot" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_complex/mapfile.tsv b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_complex/mapfile.tsv new file mode 100644 index 000000000..e7f9756ae --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_complex/mapfile.tsv @@ -0,0 +1,6 @@ +dog mammal +cat mammal +parrot bird +shark fish +dragon fictional animal +unicorn fictional animal diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_complex/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_complex/test.fix new file mode 100644 index 000000000..cb5c1e6d5 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_complex/test.fix @@ -0,0 +1,8 @@ +do list(path: "test[]", "var": "$f") + copy_field("$f", "animals[].$append.animal") + copy_field("$f", "animals[].$last.classification.type") +end + +do list(path: "animals[]") + lookup("classification.type", "./mapfile.tsv", "sep_char":"\t") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_complex/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_complex/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_complex/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_complex_withNotMatchingValue/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_complex_withNotMatchingValue/expected.json new file mode 100644 index 000000000..2faa8ce26 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_complex_withNotMatchingValue/expected.json @@ -0,0 +1,19 @@ +{ + "test" : [ "dog", "parrot", "human" ], + "animals" : [ { + "animal" : "dog", + "classification" : { + "type" : "mammal" + } + }, { + "animal" : "parrot", + "classification" : { + "type" : "bird" + } + }, { + "animal" : "human", + "classification" : { + "type" : "human" + } + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_complex_withNotMatchingValue/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_complex_withNotMatchingValue/input.json new file mode 100644 index 000000000..e6d1520c2 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_complex_withNotMatchingValue/input.json @@ -0,0 +1,3 @@ +{ + "test" : [ "dog", "parrot", "human" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_complex_withNotMatchingValue/mapfile.tsv b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_complex_withNotMatchingValue/mapfile.tsv new file mode 100644 index 000000000..e7f9756ae --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_complex_withNotMatchingValue/mapfile.tsv @@ -0,0 +1,6 @@ +dog mammal +cat mammal +parrot bird +shark fish +dragon fictional animal +unicorn fictional animal diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_complex_withNotMatchingValue/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_complex_withNotMatchingValue/test.fix new file mode 100644 index 000000000..cb5c1e6d5 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_complex_withNotMatchingValue/test.fix @@ -0,0 +1,8 @@ +do list(path: "test[]", "var": "$f") + copy_field("$f", "animals[].$append.animal") + copy_field("$f", "animals[].$last.classification.type") +end + +do list(path: "animals[]") + lookup("classification.type", "./mapfile.tsv", "sep_char":"\t") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_complex_withNotMatchingValue/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_complex_withNotMatchingValue/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_complex_withNotMatchingValue/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_withNotMatchingValue/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_withNotMatchingValue/expected.json new file mode 100644 index 000000000..774ee6438 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_withNotMatchingValue/expected.json @@ -0,0 +1,18 @@ +{ + "animals" : [ { + "name" : "Jake", + "classification" : { + "type" : "mammal" + } + }, { + "name" : "Blacky", + "classification" : { + "type" : "bird" + } + }, { + "name" : "Jens", + "classification" : { + "type" : "human" + } + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_withNotMatchingValue/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_withNotMatchingValue/input.json new file mode 100644 index 000000000..23344a1c1 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_withNotMatchingValue/input.json @@ -0,0 +1,18 @@ +{ + "animals": [{ + "name": "Jake", + "classification": { + "type": "dog" + } + },{ + "name": "Blacky", + "classification": { + "type": "parrot" + } + }, { + "name": "Jens", + "classification": { + "type": "human" + } + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_withNotMatchingValue/mapfile.tsv b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_withNotMatchingValue/mapfile.tsv new file mode 100644 index 000000000..e7f9756ae --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_withNotMatchingValue/mapfile.tsv @@ -0,0 +1,6 @@ +dog mammal +cat mammal +parrot bird +shark fish +dragon fictional animal +unicorn fictional animal diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_withNotMatchingValue/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_withNotMatchingValue/test.fix new file mode 100644 index 000000000..3205409f7 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_withNotMatchingValue/test.fix @@ -0,0 +1,3 @@ +do list(path: "animals[]") + lookup("classification.type", "./mapfile.tsv", "sep_char":"\t") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_withNotMatchingValue/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_withNotMatchingValue/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjectsWithListBind_withNotMatchingValue/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex/expected.json new file mode 100644 index 000000000..371da87fc --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex/expected.json @@ -0,0 +1,14 @@ +{ + "test" : [ "dog", "parrot" ], + "animals" : [ { + "animal" : "dog", + "classification" : { + "type" : "mammal" + } + }, { + "animal" : "parrot", + "classification" : { + "type" : "bird" + } + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex/input.json new file mode 100644 index 000000000..8376431bb --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex/input.json @@ -0,0 +1,3 @@ +{ + "test" : [ "dog", "parrot" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex/mapfile.tsv b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex/mapfile.tsv new file mode 100644 index 000000000..e7f9756ae --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex/mapfile.tsv @@ -0,0 +1,6 @@ +dog mammal +cat mammal +parrot bird +shark fish +dragon fictional animal +unicorn fictional animal diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex/test.fix new file mode 100644 index 000000000..153503b1a --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex/test.fix @@ -0,0 +1,6 @@ +do list(path: "test[]", "var": "$f") + copy_field("$f", "animals[].$append.animal") + copy_field("$f", "animals[].$last.classification.type") +end + +lookup("animals[].*.classification.type", "./mapfile.tsv", "sep_char":"\t") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_2/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_2/expected.json new file mode 100644 index 000000000..915515797 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_2/expected.json @@ -0,0 +1,22 @@ +{ + "metadata" : { + "real" : { + "test" : [ { + "animal" : "dog" + }, { + "animal" : "parrot" + } ] + } + }, + "animals" : [ { + "animal" : "dog", + "classification" : { + "type" : "mammal" + } + }, { + "animal" : "parrot", + "classification" : { + "type" : "bird" + } + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_2/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_2/input.json new file mode 100644 index 000000000..d5752887b --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_2/input.json @@ -0,0 +1,9 @@ +{ + "metadata" : { + "real" : { + "test" : [ { + "animal" : "Jake" + }, { + "animal" : "Blacky" + } ] } } +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_2/mapfile.tsv b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_2/mapfile.tsv new file mode 100644 index 000000000..e7f9756ae --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_2/mapfile.tsv @@ -0,0 +1,6 @@ +dog mammal +cat mammal +parrot bird +shark fish +dragon fictional animal +unicorn fictional animal diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_2/mapfile_2.tsv b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_2/mapfile_2.tsv new file mode 100644 index 000000000..d0b2b7658 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_2/mapfile_2.tsv @@ -0,0 +1,2 @@ +Jake dog +Blacky parrot diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_2/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_2/test.fix new file mode 100644 index 000000000..f1742a015 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_2/test.fix @@ -0,0 +1,7 @@ +do list(path: "metadata.real.test[]", "var": "$f") + lookup("$f.animal", "./mapfile_2.tsv", "sep_char":"\t") + copy_field("$f.animal", "animals[].$append.animal") + copy_field("$f.animal", "animals[].$last.classification.type") +end + +lookup("animals[].*.classification.type", "./mapfile.tsv", "sep_char":"\t") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_2/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_2/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_2/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_2_withNotMatchingValue/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_2_withNotMatchingValue/expected.json new file mode 100644 index 000000000..083a4d647 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_2_withNotMatchingValue/expected.json @@ -0,0 +1,29 @@ +{ + "metadata" : { + "real" : { + "test" : [ { + "animal" : "mammal" + }, { + "animal" : "bird" + }, { + "animal" : "human" + } ] + } + }, + "animals" : [ { + "animal" : "dog", + "classification" : { + "type" : "mammal" + } + }, { + "animal" : "parrot", + "classification" : { + "type" : "bird" + } + }, { + "animal" : "human", + "classification" : { + "type" : "human" + } + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_2_withNotMatchingValue/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_2_withNotMatchingValue/input.json new file mode 100644 index 000000000..7b5bdc21a --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_2_withNotMatchingValue/input.json @@ -0,0 +1,9 @@ +{ "metadata" : { + "real" : + { + "test" : [ { "animal" : "Jake" }, + { "animal" :"Blacky"}, + { "animal" : "Jens"} ] + } +} +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_2_withNotMatchingValue/mapfile.tsv b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_2_withNotMatchingValue/mapfile.tsv new file mode 100644 index 000000000..e7f9756ae --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_2_withNotMatchingValue/mapfile.tsv @@ -0,0 +1,6 @@ +dog mammal +cat mammal +parrot bird +shark fish +dragon fictional animal +unicorn fictional animal diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_2_withNotMatchingValue/mapfile_2.tsv b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_2_withNotMatchingValue/mapfile_2.tsv new file mode 100644 index 000000000..d0b2b7658 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_2_withNotMatchingValue/mapfile_2.tsv @@ -0,0 +1,2 @@ +Jake dog +Blacky parrot diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_2_withNotMatchingValue/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_2_withNotMatchingValue/test.fix new file mode 100644 index 000000000..f1742a015 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_2_withNotMatchingValue/test.fix @@ -0,0 +1,7 @@ +do list(path: "metadata.real.test[]", "var": "$f") + lookup("$f.animal", "./mapfile_2.tsv", "sep_char":"\t") + copy_field("$f.animal", "animals[].$append.animal") + copy_field("$f.animal", "animals[].$last.classification.type") +end + +lookup("animals[].*.classification.type", "./mapfile.tsv", "sep_char":"\t") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_2_withNotMatchingValue/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_2_withNotMatchingValue/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_2_withNotMatchingValue/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_2_withNotMatchingValue/todo.txt b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_2_withNotMatchingValue/todo.txt new file mode 100644 index 000000000..f5546343c --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_2_withNotMatchingValue/todo.txt @@ -0,0 +1 @@ +See issue #149 diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_withNotMatchingValue/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_withNotMatchingValue/expected.json new file mode 100644 index 000000000..2faa8ce26 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_withNotMatchingValue/expected.json @@ -0,0 +1,19 @@ +{ + "test" : [ "dog", "parrot", "human" ], + "animals" : [ { + "animal" : "dog", + "classification" : { + "type" : "mammal" + } + }, { + "animal" : "parrot", + "classification" : { + "type" : "bird" + } + }, { + "animal" : "human", + "classification" : { + "type" : "human" + } + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_withNotMatchingValue/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_withNotMatchingValue/input.json new file mode 100644 index 000000000..e6d1520c2 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_withNotMatchingValue/input.json @@ -0,0 +1,3 @@ +{ + "test" : [ "dog", "parrot", "human" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_withNotMatchingValue/mapfile.tsv b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_withNotMatchingValue/mapfile.tsv new file mode 100644 index 000000000..e7f9756ae --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_withNotMatchingValue/mapfile.tsv @@ -0,0 +1,6 @@ +dog mammal +cat mammal +parrot bird +shark fish +dragon fictional animal +unicorn fictional animal diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_withNotMatchingValue/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_withNotMatchingValue/test.fix new file mode 100644 index 000000000..153503b1a --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_withNotMatchingValue/test.fix @@ -0,0 +1,6 @@ +do list(path: "test[]", "var": "$f") + copy_field("$f", "animals[].$append.animal") + copy_field("$f", "animals[].$last.classification.type") +end + +lookup("animals[].*.classification.type", "./mapfile.tsv", "sep_char":"\t") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_withNotMatchingValue/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_withNotMatchingValue/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupInSubfieldInArrayOfObjects_complex_withNotMatchingValue/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectToLanguageVariantOfTheSameObject/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectToLanguageVariantOfTheSameObject/expected.json new file mode 100644 index 000000000..cd580b37c --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectToLanguageVariantOfTheSameObject/expected.json @@ -0,0 +1,16 @@ +{ + "name" : "Jake", + "a" : "Software Application" +} +{ + "name" : "Blacky", + "a" : "Reference Work" +} +{ + "name" : "Noone", + "a" : "cat" +} +{ + "name" : "Noone_2", + "a" : "Grafik" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectToLanguageVariantOfTheSameObject/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectToLanguageVariantOfTheSameObject/input.json new file mode 100644 index 000000000..805bc00da --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectToLanguageVariantOfTheSameObject/input.json @@ -0,0 +1,16 @@ +{ + "name" : "Jake", + "a" : "Softwareanwendung" +} +{ + "name" : "Blacky", + "a" : "Nachschlagewerk" +} +{ + "name" : "Noone", + "a" : "cat" +} +{ + "name" : "Noone_2", + "a" : "Grafik" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectToLanguageVariantOfTheSameObject/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectToLanguageVariantOfTheSameObject/test.fix new file mode 100644 index 000000000..1894a379e --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectToLanguageVariantOfTheSameObject/test.fix @@ -0,0 +1,2 @@ +put_rdfmap("../../../../../maps/hcrt.ttl", "rdfmap", target: "http://www.w3.org/2004/02/skos/core#prefLabel", select_language: "en") +lookup("a", "rdfmap") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectToLanguageVariantOfTheSameObject/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectToLanguageVariantOfTheSameObject/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectToLanguageVariantOfTheSameObject/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectToOtherObjectWithSpecificLanguage/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectToOtherObjectWithSpecificLanguage/expected.json new file mode 100644 index 000000000..3f3f94ed6 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectToOtherObjectWithSpecificLanguage/expected.json @@ -0,0 +1,16 @@ +{ + "name" : "Jake", + "a" : "Abbildung" +} +{ + "name" : "Blacky", + "a" : "Audio" +} +{ + "name" : "Noone", + "a" : "cat" +} +{ + "name" : "Noone_2", + "a" : "Übung" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectToOtherObjectWithSpecificLanguage/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectToOtherObjectWithSpecificLanguage/input.json new file mode 100644 index 000000000..ead932dda --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectToOtherObjectWithSpecificLanguage/input.json @@ -0,0 +1,16 @@ +{ + "name" : "Jake", + "a" : "Bild" +} +{ + "name" : "Blacky", + "a" : "Tonaufnahme" +} +{ + "name" : "Noone", + "a" : "cat" +} +{ + "name" : "Noone_2", + "a" : "Übung" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectToOtherObjectWithSpecificLanguage/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectToOtherObjectWithSpecificLanguage/test.fix new file mode 100644 index 000000000..7f2021ce0 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectToOtherObjectWithSpecificLanguage/test.fix @@ -0,0 +1,2 @@ +put_rdfmap("../../../../../maps/hcrt.ttl", "rdfmap", target: "http://www.w3.org/2004/02/skos/core#altLabel", target_language: "de", select: "object", returnPredicate:"http://www.w3.org/2004/02/skos/core#prefLabel", return_language: "de") +lookup("a", "rdfmap") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectToOtherObjectWithSpecificLanguage/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectToOtherObjectWithSpecificLanguage/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectToOtherObjectWithSpecificLanguage/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectToOtherObjectWithSpecificLanguage/todo.txt b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectToOtherObjectWithSpecificLanguage/todo.txt new file mode 100644 index 000000000..58059898f --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectToOtherObjectWithSpecificLanguage/todo.txt @@ -0,0 +1 @@ +Lookup a specific object of predicate return other object in specific language for the same subject. diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectToSubject/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectToSubject/expected.json new file mode 100644 index 000000000..2d5a35560 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectToSubject/expected.json @@ -0,0 +1,12 @@ +{ + "name" : "Jake", + "a" : "https://w3id.org/kim/hcrt/application" +} +{ + "name" : "Blacky", + "a" : "https://w3id.org/kim/hcrt/index" +} +{ + "name" : "Noone", + "a" : "cat" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectToSubject/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectToSubject/input.json new file mode 100644 index 000000000..138442e8e --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectToSubject/input.json @@ -0,0 +1,12 @@ +{ + "name" : "Jake", + "a" : "Softwareanwendung" +} +{ + "name" : "Blacky", + "a" : "Nachschlagewerk" +} +{ + "name" : "Noone", + "a" : "cat" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectToSubject/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectToSubject/test.fix new file mode 100644 index 000000000..e2c358f02 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectToSubject/test.fix @@ -0,0 +1,2 @@ +put_rdfmap("../../../../../maps/hcrt.ttl", "rdfmap", target: "skos:prefLabel") +lookup("a", "rdfmap") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectToSubject/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectToSubject/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectToSubject/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectWithSpecificLanguageToSubject/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectWithSpecificLanguageToSubject/expected.json new file mode 100644 index 000000000..fc18b370f --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectWithSpecificLanguageToSubject/expected.json @@ -0,0 +1,16 @@ +{ + "name" : "Jake", + "a" : "https://w3id.org/kim/hcrt/application" +} +{ + "name" : "Blacky", + "a" : "https://w3id.org/kim/hcrt/index" +} +{ + "name" : "Noone", + "a" : "cat" +} +{ + "name" : "Noone_2", + "a" : "Assessment" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectWithSpecificLanguageToSubject/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectWithSpecificLanguageToSubject/input.json new file mode 100644 index 000000000..1bc7685ca --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectWithSpecificLanguageToSubject/input.json @@ -0,0 +1,16 @@ +{ + "name" : "Jake", + "a" : "Softwareanwendung" +} +{ + "name" : "Blacky", + "a" : "Nachschlagewerk" +} +{ + "name" : "Noone", + "a" : "cat" +} +{ + "name" : "Noone_2", + "a" : "Assessment" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectWithSpecificLanguageToSubject/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectWithSpecificLanguageToSubject/test.fix new file mode 100644 index 000000000..406226342 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectWithSpecificLanguageToSubject/test.fix @@ -0,0 +1,2 @@ +put_rdfmap("../../../../../maps/hcrt.ttl", "rdfmap", target: "http://www.w3.org/2004/02/skos/core#prefLabel", select_language: "de", select: "subject") +lookup("a", "rdfmap") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectWithSpecificLanguageToSubject/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectWithSpecificLanguageToSubject/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfObjectWithSpecificLanguageToSubject/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfPropertyToProperty/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfPropertyToProperty/expected.json new file mode 100644 index 000000000..fd909c9a3 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfPropertyToProperty/expected.json @@ -0,0 +1,12 @@ +{ + "name" : "Jake", + "a" : "Software Application" +} +{ + "name" : "Blacky", + "a" : "Reference Work" +} +{ + "name" : "Noone", + "a" : "cat" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfPropertyToProperty/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfPropertyToProperty/input.json new file mode 100644 index 000000000..138442e8e --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfPropertyToProperty/input.json @@ -0,0 +1,12 @@ +{ + "name" : "Jake", + "a" : "Softwareanwendung" +} +{ + "name" : "Blacky", + "a" : "Nachschlagewerk" +} +{ + "name" : "Noone", + "a" : "cat" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfPropertyToProperty/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfPropertyToProperty/test.fix new file mode 100644 index 000000000..e8ca947a4 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfPropertyToProperty/test.fix @@ -0,0 +1,2 @@ +put_rdfmap("../../../../../maps/hcrt.ttl", "rdfmap", target: "http://www.w3.org/2004/02/skos/core#prefLabel", select_language: 'en') +lookup("a", "rdfmap") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfPropertyToProperty/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfPropertyToProperty/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfPropertyToProperty/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfSubjectToObjectWithSpecificLanguage/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfSubjectToObjectWithSpecificLanguage/expected.json new file mode 100644 index 000000000..138442e8e --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfSubjectToObjectWithSpecificLanguage/expected.json @@ -0,0 +1,12 @@ +{ + "name" : "Jake", + "a" : "Softwareanwendung" +} +{ + "name" : "Blacky", + "a" : "Nachschlagewerk" +} +{ + "name" : "Noone", + "a" : "cat" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfSubjectToObjectWithSpecificLanguage/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfSubjectToObjectWithSpecificLanguage/input.json new file mode 100644 index 000000000..129725a51 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfSubjectToObjectWithSpecificLanguage/input.json @@ -0,0 +1,12 @@ +{ + "name": "Jake", + "a": "https://w3id.org/kim/hcrt/application" +} +{ + "name": "Blacky", + "a": "https://w3id.org/kim/hcrt/index" +} +{ + "name": "Noone", + "a": "cat" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfSubjectToObjectWithSpecificLanguage/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfSubjectToObjectWithSpecificLanguage/test.fix new file mode 100644 index 000000000..7e0ca30a4 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfSubjectToObjectWithSpecificLanguage/test.fix @@ -0,0 +1,3 @@ +# Lookup replaces with default value by default. Should be an option. +put_rdfmap("../../../../../maps/hcrt.ttl", "rdfmap", target: "http://www.w3.org/2004/02/skos/core#prefLabel", select_language: "de") +lookup("a", "rdfmap") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfSubjectToObjectWithSpecificLanguage/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfSubjectToObjectWithSpecificLanguage/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupRdfSubjectToObjectWithSpecificLanguage/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithExternalTsvFile/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithExternalTsvFile/expected.json new file mode 100644 index 000000000..3180af96b --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithExternalTsvFile/expected.json @@ -0,0 +1,8 @@ +{ + "name" : "Jake", + "type" : "mammal" +} +{ + "name" : "Blacky", + "type" : "bird" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithExternalTsvFile/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithExternalTsvFile/input.json new file mode 100644 index 000000000..3cf528254 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithExternalTsvFile/input.json @@ -0,0 +1,8 @@ +{ + "name": "Jake", + "type": "dog" +} +{ + "name": "Blacky", + "type": "parrot" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithExternalTsvFile/mapfile.tsv b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithExternalTsvFile/mapfile.tsv new file mode 100644 index 000000000..e7f9756ae --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithExternalTsvFile/mapfile.tsv @@ -0,0 +1,6 @@ +dog mammal +cat mammal +parrot bird +shark fish +dragon fictional animal +unicorn fictional animal diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithExternalTsvFile/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithExternalTsvFile/test.fix new file mode 100644 index 000000000..f66f23e42 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithExternalTsvFile/test.fix @@ -0,0 +1 @@ +lookup("type", "./mapfile.tsv", "sep_char":"\t") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithExternalTsvFile/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithExternalTsvFile/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithExternalTsvFile/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithInternalMap/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithInternalMap/expected.json new file mode 100644 index 000000000..3180af96b --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithInternalMap/expected.json @@ -0,0 +1,8 @@ +{ + "name" : "Jake", + "type" : "mammal" +} +{ + "name" : "Blacky", + "type" : "bird" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithInternalMap/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithInternalMap/input.json new file mode 100644 index 000000000..a02a380e7 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithInternalMap/input.json @@ -0,0 +1,8 @@ +{ + "name" : "Jake", + "type" : "dog" +} +{ + "name" : "Blacky", + "type" : "parrot" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithInternalMap/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithInternalMap/test.fix new file mode 100644 index 000000000..796c09b30 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithInternalMap/test.fix @@ -0,0 +1,9 @@ +put_map("map", + "dog":"mammal", + "cat":"mammal", + "parrot":"bird", + "shark":"fish", + "dragon":"fictional animal", + "unicorn":"fictional animal") + +lookup("type", "map") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithInternalMap/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithInternalMap/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithInternalMap/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithInternalMapDeleteNotMatchingLiteral/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithInternalMapDeleteNotMatchingLiteral/expected.json new file mode 100644 index 000000000..b39dc6fee --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithInternalMapDeleteNotMatchingLiteral/expected.json @@ -0,0 +1,11 @@ +{ + "name" : "Jake", + "type" : "mammal" +} +{ + "name" : "Blacky", + "type" : "bird" +} +{ + "name" : "Jens" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithInternalMapDeleteNotMatchingLiteral/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithInternalMapDeleteNotMatchingLiteral/input.json new file mode 100644 index 000000000..d4fdcd5d9 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithInternalMapDeleteNotMatchingLiteral/input.json @@ -0,0 +1,13 @@ +{ + "name" : "Jake", + "type" : "dog" +} +{ + "name" : "Blacky", + "type" : "parrot" +} +{ + "name" : "Jens", + "type" : "human" +} + diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithInternalMapDeleteNotMatchingLiteral/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithInternalMapDeleteNotMatchingLiteral/test.fix new file mode 100644 index 000000000..6d2df6943 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithInternalMapDeleteNotMatchingLiteral/test.fix @@ -0,0 +1,9 @@ +put_map("map", + "dog":"mammal", + "cat":"mammal", + "parrot":"bird", + "shark":"fish", + "dragon":"fictional animal", + "unicorn":"fictional animal") + +lookup("type", "map", delete: "true") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithInternalMapDeleteNotMatchingLiteral/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithInternalMapDeleteNotMatchingLiteral/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithInternalMapDeleteNotMatchingLiteral/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithInternalMapWithoutMatch/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithInternalMapWithoutMatch/expected.json new file mode 100644 index 000000000..45350351c --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithInternalMapWithoutMatch/expected.json @@ -0,0 +1,12 @@ +{ + "name" : "Jake", + "type" : "mammal" +} +{ + "name" : "Blacky", + "type" : "bird" +} +{ + "name" : "Jens", + "type" : "human" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithInternalMapWithoutMatch/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithInternalMapWithoutMatch/input.json new file mode 100644 index 000000000..d4fdcd5d9 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithInternalMapWithoutMatch/input.json @@ -0,0 +1,13 @@ +{ + "name" : "Jake", + "type" : "dog" +} +{ + "name" : "Blacky", + "type" : "parrot" +} +{ + "name" : "Jens", + "type" : "human" +} + diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithInternalMapWithoutMatch/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithInternalMapWithoutMatch/test.fix new file mode 100644 index 000000000..796c09b30 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithInternalMapWithoutMatch/test.fix @@ -0,0 +1,9 @@ +put_map("map", + "dog":"mammal", + "cat":"mammal", + "parrot":"bird", + "shark":"fish", + "dragon":"fictional animal", + "unicorn":"fictional animal") + +lookup("type", "map") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithInternalMapWithoutMatch/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithInternalMapWithoutMatch/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithInternalMapWithoutMatch/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithMultiColumnFile/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithMultiColumnFile/expected.json new file mode 100644 index 000000000..0a8b60103 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithMultiColumnFile/expected.json @@ -0,0 +1,8 @@ +{ + "name" : "RVK (Regensburger Verbundklassifikation)", + "id" : "https://d-nb.info/gnd/4449787-8" +} +{ + "name" : "ZDB-Systematik", + "id" : "http://bartoc.org/en/node/18915" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithMultiColumnFile/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithMultiColumnFile/input.json new file mode 100644 index 000000000..9899d65d6 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithMultiColumnFile/input.json @@ -0,0 +1,6 @@ +{ + "name": "rvk" +} +{ + "name": "zdbs" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithMultiColumnFile/mapfile.tsv b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithMultiColumnFile/mapfile.tsv new file mode 100644 index 000000000..5dd3ad557 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithMultiColumnFile/mapfile.tsv @@ -0,0 +1,3 @@ +rvk RVK (Regensburger Verbundklassifikation) https://d-nb.info/gnd/4449787-8 +udc UDC (Universal Decimal Classification) https://d-nb.info/gnd/4114037-0 +zdbs ZDB-Systematik http://bartoc.org/en/node/18915 diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithMultiColumnFile/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithMultiColumnFile/test.fix new file mode 100644 index 000000000..605b8a4a0 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithMultiColumnFile/test.fix @@ -0,0 +1,6 @@ +put_filemap("./mapfile.tsv", "idLookup", sep_char:"\t",key_column:"1",value_column:"2",expected_columns:"3") +put_filemap("./mapfile.tsv", "nameLookup", sep_char:"\t",expected_columns:"-1") + +lookup("name", "nameLookup") +copy_field("name","id") +lookup("id", "idLookup") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithMultiColumnFile/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithMultiColumnFile/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromJson/toJson/lookupSimpleWithMultiColumnFile/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromXml/toJson/lookupInDeeplyNestedArrayOfObjects/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromXml/toJson/lookupInDeeplyNestedArrayOfObjects/expected.json new file mode 100644 index 000000000..593c42cbf --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromXml/toJson/lookupInDeeplyNestedArrayOfObjects/expected.json @@ -0,0 +1,51 @@ +{ + "subject" : [ { + "type" : [ "ComplexSubject" ], + "componentList" : [ { + "type" : [ "SubjectHeading" ], + "label" : "Schulhof", + "source" : { + "label" : "Gemeinsame Normdatei (GND)", + "id" : "https://d-nb.info/gnd/7749153-1" + }, + "id" : "(DE-588)4131620-4", + "gndIdentifier" : "(DE-588)4131620-4" + }, { + "type" : [ "SubjectHeading" ], + "label" : "Sozialraum", + "source" : { + "label" : "Gemeinsame Normdatei (GND)", + "id" : "https://d-nb.info/gnd/7749153-1" + }, + "id" : "(DE-588)4116546-9", + "gndIdentifier" : "(DE-588)4116546-9" + }, { + "type" : [ "SubjectHeading" ], + "label" : "Informelles Lernen", + "source" : { + "label" : "Gemeinsame Normdatei (GND)", + "id" : "https://d-nb.info/gnd/7749153-1" + }, + "id" : "(DE-588)4680202-2", + "gndIdentifier" : "(DE-588)4680202-2" + }, { + "type" : [ "SubjectHeading" ], + "label" : "Ganztagsschule", + "source" : { + "label" : "Gemeinsame Normdatei (GND)", + "id" : "https://d-nb.info/gnd/7749153-1" + }, + "id" : "(DE-588)4155947-2", + "gndIdentifier" : "(DE-588)4155947-2" + }, { + "type" : [ "SubjectHeading" ], + "label" : "Sportpädagogik", + "source" : { + "label" : "Gemeinsame Normdatei (GND)", + "id" : "https://d-nb.info/gnd/7749153-1" + }, + "id" : "(DE-588)4130615-6", + "gndIdentifier" : "(DE-588)4130615-6" + } ] + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromXml/toJson/lookupInDeeplyNestedArrayOfObjects/input.xml b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromXml/toJson/lookupInDeeplyNestedArrayOfObjects/input.xml new file mode 100644 index 000000000..5f2ac0657 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromXml/toJson/lookupInDeeplyNestedArrayOfObjects/input.xml @@ -0,0 +1,45 @@ + + + + + Schulhof + s + (DE-588)4131620-4 + (uri) https://portal.dnb.de/opac.htm?method=simpleSearch&cqlMode=true&query=idn=041316207 + (uri) http://viaf.org/viaf/sourceID/DNB|041316207 + GND-041316207 + + + Sozialraum + s + (DE-588)4116546-9 + (uri) https://portal.dnb.de/opac.htm?method=simpleSearch&cqlMode=true&query=idn=041165462 + (uri) http://viaf.org/viaf/sourceID/DNB|041165462 + GND-041165462 + + + Informelles Lernen + s + (DE-588)4680202-2 + (uri) https://portal.dnb.de/opac.htm?method=simpleSearch&cqlMode=true&query=idn=964127970 + (uri) http://viaf.org/viaf/sourceID/DNB|964127970 + GND-964127970 + + + Ganztagsschule + s + (DE-588)4155947-2 + (uri) https://portal.dnb.de/opac.htm?method=simpleSearch&cqlMode=true&query=idn=041559479 + (uri) http://viaf.org/viaf/sourceID/DNB|041559479 + GND-041559479 + + + Sportpädagogik + s + (DE-588)4130615-6 + (uri) https://portal.dnb.de/opac.htm?method=simpleSearch&cqlMode=true&query=idn=041306155 + (uri) http://viaf.org/viaf/sourceID/DNB|041306155 + GND-041306155 + + + diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromXml/toJson/lookupInDeeplyNestedArrayOfObjects/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromXml/toJson/lookupInDeeplyNestedArrayOfObjects/test.fix new file mode 100644 index 000000000..6a9c42c5a --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromXml/toJson/lookupInDeeplyNestedArrayOfObjects/test.fix @@ -0,0 +1,28 @@ +put_map("rswk-indicator", s: "SubjectHeading") + +if exists("6890?") + set_array("subject[].$append.type[]", "ComplexSubject") + set_array("subject[].$last.label") + set_array("subject[].$last.componentList[]") + do list(path: "6890?", "var": "$i") + set_array("subject[].$last.componentList[].$append.type[]") + do list(path: "$i.D", "var": "$k") + copy_field("$k", "subject[].$last.componentList[].$last.type[].$append") + end + copy_field("$i.a", "subject[].$last.componentList[].$last.label") + do list(path: "$i.0", "var": "$j") + if any_match("$j", "^\\(DE-588\\)(.*)$") + add_field("subject[].$last.componentList[].$last.source.label", "Gemeinsame Normdatei (GND)") + add_field("subject[].$last.componentList[].$last.source.id", "https://d-nb.info/gnd/7749153-1") + copy_field("$j", "subject[].$last.componentList[].$last.id") + replace_all("subject[].$last.componentList[].$last..id", "^\\(DE-588\\)(.*)$", "http://d-nb.info/gnd/$1") + copy_field("$j", "subject[].$last.componentList[].$last.gndIdentifier") + replace_all("subject[].$last.componentList[].$last..gndIdentifier", "^\\(DE-588\\)(.*)$", "$1") + end + end + end +end + +lookup("subject[].*.componentList[].*.type[].*", "rswk-indicator") + +retain("subject[]") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromXml/toJson/lookupInDeeplyNestedArrayOfObjects/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromXml/toJson/lookupInDeeplyNestedArrayOfObjects/test.flux new file mode 100644 index 000000000..1d8c3df8f --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/lookup/fromXml/toJson/lookupInDeeplyNestedArrayOfObjects/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.xml" +|open-file +|decode-xml +|handle-marcxml +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjects/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjects/expected.json new file mode 100644 index 000000000..683225a33 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjects/expected.json @@ -0,0 +1,16 @@ +{ + "animals" : [ { + "name" : "Jake", + "type" : "dog" + }, { + "name" : "Blacky", + "type" : "bird is big" + } ], + "animals_2" : [ { + "name" : "Jake", + "type" : "dog is cool" + }, { + "name" : "Blacky", + "type" : "bird is cool" + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjects/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjects/input.json new file mode 100644 index 000000000..ba66dd7b9 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjects/input.json @@ -0,0 +1,22 @@ +{ + "animals": [ + { + "name": "Jake", + "type": "dog" + }, + { + "name": "Blacky", + "type": "bird" + } + ], + "animals_2": [ + { + "name": "Jake", + "type": "dog" + }, + { + "name": "Blacky", + "type": "bird" + } + ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjects/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjects/test.fix new file mode 100644 index 000000000..c51d06201 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjects/test.fix @@ -0,0 +1,2 @@ +append("animals[].2.type", " is big") +append("animals_2[].*.type", " is cool") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjects/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjects/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjects/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjectsWithAsteriskArrayWildcard/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjectsWithAsteriskArrayWildcard/expected.json new file mode 100644 index 000000000..9ba117c17 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjectsWithAsteriskArrayWildcard/expected.json @@ -0,0 +1,15 @@ +{ + "animals" : [ { + "name" : "Jake", + "type" : "dog is cool" + }, { + "name" : "Blacky", + "type" : "bird is cool" + } ] +} +{ + "animals" : [ { + "name" : "Test", + "type" : "test is cool" + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjectsWithAsteriskArrayWildcard/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjectsWithAsteriskArrayWildcard/input.json new file mode 100644 index 000000000..0a5a29be9 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjectsWithAsteriskArrayWildcard/input.json @@ -0,0 +1,15 @@ +{ + "animals": [ { + "name": "Jake", + "type": "dog" + }, { + "name": "Blacky", + "type": "bird" + } ] +} +{ + "animals": [ { + "name": "Test", + "type": "test" + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjectsWithAsteriskArrayWildcard/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjectsWithAsteriskArrayWildcard/test.fix new file mode 100644 index 000000000..437d32d1c --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjectsWithAsteriskArrayWildcard/test.fix @@ -0,0 +1 @@ +append("animals[].*.type", " is cool") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjectsWithAsteriskArrayWildcard/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjectsWithAsteriskArrayWildcard/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjectsWithAsteriskArrayWildcard/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjectsWithFirstArrayWildcard/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjectsWithFirstArrayWildcard/expected.json new file mode 100644 index 000000000..5756c5225 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjectsWithFirstArrayWildcard/expected.json @@ -0,0 +1,9 @@ +{ + "animals" : [ { + "name" : "Jake", + "type" : "dog is cool" + }, { + "name" : "Blacky", + "type" : "bird" + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjectsWithFirstArrayWildcard/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjectsWithFirstArrayWildcard/input.json new file mode 100644 index 000000000..30e83b94e --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjectsWithFirstArrayWildcard/input.json @@ -0,0 +1,12 @@ +{ + "animals": [ + { + "name": "Jake", + "type": "dog" + }, + { + "name": "Blacky", + "type": "bird" + } + ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjectsWithFirstArrayWildcard/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjectsWithFirstArrayWildcard/test.fix new file mode 100644 index 000000000..f05251ab9 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjectsWithFirstArrayWildcard/test.fix @@ -0,0 +1 @@ +append("animals[].$first.type", " is cool") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjectsWithFirstArrayWildcard/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjectsWithFirstArrayWildcard/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjectsWithFirstArrayWildcard/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjectsWithLastArrayWildcard/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjectsWithLastArrayWildcard/expected.json new file mode 100644 index 000000000..d847b8b9f --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjectsWithLastArrayWildcard/expected.json @@ -0,0 +1,9 @@ +{ + "animals" : [ { + "name" : "Jake", + "type" : "dog" + }, { + "name" : "Blacky", + "type" : "bird is cool" + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjectsWithLastArrayWildcard/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjectsWithLastArrayWildcard/input.json new file mode 100644 index 000000000..30e83b94e --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjectsWithLastArrayWildcard/input.json @@ -0,0 +1,12 @@ +{ + "animals": [ + { + "name": "Jake", + "type": "dog" + }, + { + "name": "Blacky", + "type": "bird" + } + ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjectsWithLastArrayWildcard/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjectsWithLastArrayWildcard/test.fix new file mode 100644 index 000000000..385fbd9b0 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjectsWithLastArrayWildcard/test.fix @@ -0,0 +1 @@ +append("animals[].$last.type", " is cool") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjectsWithLastArrayWildcard/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjectsWithLastArrayWildcard/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfObjectsWithLastArrayWildcard/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfStrings/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfStrings/expected.json new file mode 100644 index 000000000..25135982b --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfStrings/expected.json @@ -0,0 +1,4 @@ +{ + "animals" : [ "dog", "cat is big", "zebra" ], + "animals_2" : [ "dog is cool", "cat is cool", "zebra is cool" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfStrings/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfStrings/input.json new file mode 100644 index 000000000..948fff188 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfStrings/input.json @@ -0,0 +1,4 @@ +{ + "animals": ["dog", "cat", "zebra"], + "animals_2": ["dog", "cat", "zebra"] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfStrings/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfStrings/test.fix new file mode 100644 index 000000000..fef5b2d10 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfStrings/test.fix @@ -0,0 +1,3 @@ +append("animals[].2", " is big") + +append("animals_2[].*", " is cool") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfStrings/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfStrings/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendArrayOfStrings/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendSimple/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendSimple/expected.json new file mode 100644 index 000000000..3e7fb2e84 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendSimple/expected.json @@ -0,0 +1,7 @@ +{ + "animal" : "bunny is cool", + "animal_2" : { + "name" : "bird boss", + "type" : "TEST" + } +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendSimple/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendSimple/input.json new file mode 100644 index 000000000..0c72e14b8 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendSimple/input.json @@ -0,0 +1,7 @@ +{ + "animal": "bunny", + "animal_2": { + "name": "bird", + "type": "TEST" + } +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendSimple/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendSimple/test.fix new file mode 100644 index 000000000..d6f8d7496 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendSimple/test.fix @@ -0,0 +1,2 @@ +append("animal", " is cool") +append("animal_2.name", " boss") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendSimple/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendSimple/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/appendSimple/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/countArrayOfObjects/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/countArrayOfObjects/expected.json new file mode 100644 index 000000000..dd1187e64 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/countArrayOfObjects/expected.json @@ -0,0 +1,3 @@ +{ + "array" : [ "4" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/countArrayOfObjects/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/countArrayOfObjects/input.json new file mode 100644 index 000000000..83ddc67f3 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/countArrayOfObjects/input.json @@ -0,0 +1,11 @@ +{ + "array" : [ { + "key" : "value" + }, { + "key_2" : "more_value" + }, { + "key_3" : "even_more_value" + }, { + "key_4" : "a_lot_more_value" + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/countArrayOfObjects/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/countArrayOfObjects/test.fix new file mode 100644 index 000000000..79e211b40 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/countArrayOfObjects/test.fix @@ -0,0 +1,2 @@ +count("array[]") + diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/countArrayOfObjects/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/countArrayOfObjects/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/countArrayOfObjects/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/countArrayOfStrings/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/countArrayOfStrings/expected.json new file mode 100644 index 000000000..184a76025 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/countArrayOfStrings/expected.json @@ -0,0 +1,3 @@ +{ + "animals" : [ "3" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/countArrayOfStrings/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/countArrayOfStrings/input.json new file mode 100644 index 000000000..12733202d --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/countArrayOfStrings/input.json @@ -0,0 +1,3 @@ +{ + "animals" : [ "dog", "cat", "zebra" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/countArrayOfStrings/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/countArrayOfStrings/test.fix new file mode 100644 index 000000000..ca013daf3 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/countArrayOfStrings/test.fix @@ -0,0 +1,2 @@ +count("animals[]") + diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/countArrayOfStrings/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/countArrayOfStrings/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/countArrayOfStrings/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/countObject/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/countObject/expected.json new file mode 100644 index 000000000..2090063d3 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/countObject/expected.json @@ -0,0 +1,3 @@ +{ + "animals" : "4" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/countObject/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/countObject/input.json new file mode 100644 index 000000000..0c4e63f58 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/countObject/input.json @@ -0,0 +1,8 @@ +{ + "animals": { + "name": "bird", + "type": "TEST", + "test": "test", + "bla": "bla" + } +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/countObject/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/countObject/test.fix new file mode 100644 index 000000000..fbc3591d0 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/countObject/test.fix @@ -0,0 +1 @@ +count("animals") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/countObject/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/countObject/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/countObject/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/filterArrayOfstrings/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/filterArrayOfstrings/expected.json new file mode 100644 index 000000000..8e32c07f8 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/filterArrayOfstrings/expected.json @@ -0,0 +1,5 @@ +{ + "animals" : [ "zebra", "Zebra" ], + "investigators" : [ "Justus", "Bob" ], + "tools" : [ "saw", "bow" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/filterArrayOfstrings/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/filterArrayOfstrings/input.json new file mode 100644 index 000000000..7b1055b16 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/filterArrayOfstrings/input.json @@ -0,0 +1,5 @@ +{ + "animals" : [ "dog", "cat", "zebra", "Zebra" ], + "investigators" : [ "Justus", "Peter", "Bob" ], + "tools" : [ "hammer", "saw", "bow" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/filterArrayOfstrings/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/filterArrayOfstrings/test.fix new file mode 100644 index 000000000..e7082c7a3 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/filterArrayOfstrings/test.fix @@ -0,0 +1,3 @@ +filter("animals[]", "[Zz]ebra") +filter("investigators[]", "Justus|Bob") +filter("tools[]", ".*w") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/filterArrayOfstrings/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/filterArrayOfstrings/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/filterArrayOfstrings/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/flatten/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/flatten/expected.json new file mode 100644 index 000000000..ffe5820e2 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/flatten/expected.json @@ -0,0 +1,3 @@ +{ + "animals" : [ "ant", "dog", "cat", "fish", "zebra", "horse", "hippo", "giraffe" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/flatten/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/flatten/input.json new file mode 100644 index 000000000..c12e2dd05 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/flatten/input.json @@ -0,0 +1,3 @@ +{ + "animals" : [ [ "ant", "dog" ], "cat", [ "fish", [ "zebra", "horse" ], "hippo" ], "giraffe" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/flatten/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/flatten/test.fix new file mode 100644 index 000000000..71860260a --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/flatten/test.fix @@ -0,0 +1 @@ +flatten("animals[]") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/flatten/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/flatten/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/flatten/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/flattenNested/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/flattenNested/expected.json new file mode 100644 index 000000000..d4d0c7269 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/flattenNested/expected.json @@ -0,0 +1,5 @@ +{ + "zoo" : [ { + "animals" : [ "ant", "dog", "cat", "fish", "zebra", "horse", "hippo", "giraffe" ] + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/flattenNested/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/flattenNested/input.json new file mode 100644 index 000000000..3a93cea86 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/flattenNested/input.json @@ -0,0 +1,3 @@ +{ + "zoo": [ {"animals" : [ [ "ant", "dog" ], "cat", [ "fish", [ "zebra", "horse" ], "hippo" ], "giraffe"] } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/flattenNested/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/flattenNested/test.fix new file mode 100644 index 000000000..9bd196609 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/flattenNested/test.fix @@ -0,0 +1 @@ +flatten("zoo[].*.animals[]") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/flattenNested/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/flattenNested/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/flattenNested/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/indexSimple/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/indexSimple/expected.json new file mode 100644 index 000000000..192689e18 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/indexSimple/expected.json @@ -0,0 +1,3 @@ +{ + "animal" : "2" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/indexSimple/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/indexSimple/input.json new file mode 100644 index 000000000..c7dbeca2a --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/indexSimple/input.json @@ -0,0 +1,3 @@ +{ + "animal": "canary" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/indexSimple/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/indexSimple/test.fix new file mode 100644 index 000000000..3585635d7 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/indexSimple/test.fix @@ -0,0 +1 @@ +index("animal", "n") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/indexSimple/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/indexSimple/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/indexSimple/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/isbn/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/isbn/expected.json new file mode 100644 index 000000000..6b4f257c1 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/isbn/expected.json @@ -0,0 +1,6 @@ +{ + "test_1" : "9783894018109", + "test_2" : "ungültig", + "test_3" : "9783442151479", + "test_4" : "3821808187" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/isbn/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/isbn/input.json new file mode 100644 index 000000000..a9b95cb08 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/isbn/input.json @@ -0,0 +1,6 @@ +{ + "test_1" : "978-3-89401-810-9", + "test_2" : "978-3-89401-810-9", + "test_3" : "3442151473", + "test_4" : "978-3-8218-0818-5" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/isbn/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/isbn/test.fix new file mode 100644 index 000000000..6784ad9ec --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/isbn/test.fix @@ -0,0 +1,4 @@ +isbn("test_1",to:"clean") +isbn("test_2",to:"clean",verify_check_digit:"true",error_string:"ungültig") +isbn("test_3",to:"ISBN13",verify_check_digit:"true",error_string:"ungültig") +isbn("test_4",to:"ISBN10",verify_check_digit:"true",error_string:"ungültig") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/isbn/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/isbn/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/isbn/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/join_fieldSimpleAndChangeToArray/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/join_fieldSimpleAndChangeToArray/expected.json new file mode 100644 index 000000000..00f64362b --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/join_fieldSimpleAndChangeToArray/expected.json @@ -0,0 +1,3 @@ +{ + "investigators" : "Justus?Peter?Bob" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/join_fieldSimpleAndChangeToArray/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/join_fieldSimpleAndChangeToArray/input.json new file mode 100644 index 000000000..6004ff7c1 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/join_fieldSimpleAndChangeToArray/input.json @@ -0,0 +1,3 @@ +{ + "investigators" : [ "Justus", "Peter", "Bob" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/join_fieldSimpleAndChangeToArray/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/join_fieldSimpleAndChangeToArray/test.fix new file mode 100644 index 000000000..fdaf9966b --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/join_fieldSimpleAndChangeToArray/test.fix @@ -0,0 +1,2 @@ +join_field("investigators[]","?") +move_field("investigators[]","investigators") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/join_fieldSimpleAndChangeToArray/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/join_fieldSimpleAndChangeToArray/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/join_fieldSimpleAndChangeToArray/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/prependArrayOfObjectsWithAsteriskArrayWildcard/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/prependArrayOfObjectsWithAsteriskArrayWildcard/expected.json new file mode 100644 index 000000000..5947823d5 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/prependArrayOfObjectsWithAsteriskArrayWildcard/expected.json @@ -0,0 +1,15 @@ +{ + "animals" : [ { + "name" : "Jake", + "type" : "Big dog" + }, { + "name" : "Blacky", + "type" : "Big bird" + } ] +} +{ + "animals" : [ { + "name" : "Test", + "type" : "Big test" + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/prependArrayOfObjectsWithAsteriskArrayWildcard/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/prependArrayOfObjectsWithAsteriskArrayWildcard/input.json new file mode 100644 index 000000000..54d9218b7 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/prependArrayOfObjectsWithAsteriskArrayWildcard/input.json @@ -0,0 +1,20 @@ +{ + "animals": [ + { + "name": "Jake", + "type": "dog" + }, + { + "name": "Blacky", + "type": "bird" + } + ] +} +{ + "animals": [ + { + "name": "Test", + "type": "test" + } + ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/prependArrayOfObjectsWithAsteriskArrayWildcard/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/prependArrayOfObjectsWithAsteriskArrayWildcard/test.fix new file mode 100644 index 000000000..0e5cdbcc4 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/prependArrayOfObjectsWithAsteriskArrayWildcard/test.fix @@ -0,0 +1 @@ +prepend("animals[].*.type", "Big ") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/prependArrayOfObjectsWithAsteriskArrayWildcard/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/prependArrayOfObjectsWithAsteriskArrayWildcard/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/prependArrayOfObjectsWithAsteriskArrayWildcard/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/prependArrayOfObjectsWithFirstArrayWildcard/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/prependArrayOfObjectsWithFirstArrayWildcard/expected.json new file mode 100644 index 000000000..e6406073e --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/prependArrayOfObjectsWithFirstArrayWildcard/expected.json @@ -0,0 +1,9 @@ +{ + "animals" : [ { + "name" : "Jake", + "type" : "Big dog" + }, { + "name" : "Blacky", + "type" : "bird" + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/prependArrayOfObjectsWithFirstArrayWildcard/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/prependArrayOfObjectsWithFirstArrayWildcard/input.json new file mode 100644 index 000000000..30e83b94e --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/prependArrayOfObjectsWithFirstArrayWildcard/input.json @@ -0,0 +1,12 @@ +{ + "animals": [ + { + "name": "Jake", + "type": "dog" + }, + { + "name": "Blacky", + "type": "bird" + } + ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/prependArrayOfObjectsWithFirstArrayWildcard/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/prependArrayOfObjectsWithFirstArrayWildcard/test.fix new file mode 100644 index 000000000..93f038948 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/prependArrayOfObjectsWithFirstArrayWildcard/test.fix @@ -0,0 +1 @@ +prepend("animals[].$first.type", "Big ") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/prependArrayOfObjectsWithFirstArrayWildcard/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/prependArrayOfObjectsWithFirstArrayWildcard/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/prependArrayOfObjectsWithFirstArrayWildcard/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/prependArrayOfObjectsWithLastArrayWildcard/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/prependArrayOfObjectsWithLastArrayWildcard/expected.json new file mode 100644 index 000000000..6b354c7c4 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/prependArrayOfObjectsWithLastArrayWildcard/expected.json @@ -0,0 +1,9 @@ +{ + "animals" : [ { + "name" : "Jake", + "type" : "dog" + }, { + "name" : "Blacky", + "type" : "Big bird" + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/prependArrayOfObjectsWithLastArrayWildcard/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/prependArrayOfObjectsWithLastArrayWildcard/input.json new file mode 100644 index 000000000..30e83b94e --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/prependArrayOfObjectsWithLastArrayWildcard/input.json @@ -0,0 +1,12 @@ +{ + "animals": [ + { + "name": "Jake", + "type": "dog" + }, + { + "name": "Blacky", + "type": "bird" + } + ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/prependArrayOfObjectsWithLastArrayWildcard/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/prependArrayOfObjectsWithLastArrayWildcard/test.fix new file mode 100644 index 000000000..dc09536a3 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/prependArrayOfObjectsWithLastArrayWildcard/test.fix @@ -0,0 +1 @@ +prepend("animals[].$last.type", "Big ") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/prependArrayOfObjectsWithLastArrayWildcard/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/prependArrayOfObjectsWithLastArrayWildcard/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/prependArrayOfObjectsWithLastArrayWildcard/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInArrayOfStringsWithAsterisk/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInArrayOfStringsWithAsterisk/expected.json new file mode 100644 index 000000000..3784e98df --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInArrayOfStringsWithAsterisk/expected.json @@ -0,0 +1,3 @@ +{ + "key" : [ "value", "m__re_value", "even_m__re_value" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInArrayOfStringsWithAsterisk/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInArrayOfStringsWithAsterisk/input.json new file mode 100644 index 000000000..160261df2 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInArrayOfStringsWithAsterisk/input.json @@ -0,0 +1,3 @@ +{ + "key" : [ "value", "more_value", "even_more_value" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInArrayOfStringsWithAsterisk/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInArrayOfStringsWithAsterisk/test.fix new file mode 100644 index 000000000..1643a90a8 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInArrayOfStringsWithAsterisk/test.fix @@ -0,0 +1 @@ +replace_all("key[].*", "o", "__") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInArrayOfStringsWithAsterisk/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInArrayOfStringsWithAsterisk/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInArrayOfStringsWithAsterisk/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInArrayOfStringsWithAsterisk_complex/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInArrayOfStringsWithAsterisk_complex/expected.json new file mode 100644 index 000000000..d1e0167ca --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInArrayOfStringsWithAsterisk_complex/expected.json @@ -0,0 +1,16 @@ +{ + "nested" : { + "object" : { + "inLanguage" : "de-DE" + } + }, + "inLanguage" : [ "de" ] +} +{ + "nested" : { + "object" : { + "inLanguage" : "en-EN" + } + }, + "inLanguage" : [ "en" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInArrayOfStringsWithAsterisk_complex/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInArrayOfStringsWithAsterisk_complex/input.json new file mode 100644 index 000000000..b1e3aa67a --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInArrayOfStringsWithAsterisk_complex/input.json @@ -0,0 +1,14 @@ +{ + "nested" : { + "object": { + "inLanguage": "de-DE" + } + } +} +{ + "nested" : { + "object": { + "inLanguage": "en-EN" + } + } +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInArrayOfStringsWithAsterisk_complex/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInArrayOfStringsWithAsterisk_complex/test.fix new file mode 100644 index 000000000..563025639 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInArrayOfStringsWithAsterisk_complex/test.fix @@ -0,0 +1,2 @@ +copy_field("nested.object.inLanguage", "inLanguage[].$append") +replace_all("inLanguage[].*", '-.*', '') diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInArrayOfStringsWithAsterisk_complex/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInArrayOfStringsWithAsterisk_complex/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInArrayOfStringsWithAsterisk_complex/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInArrayOfStringsWithDoList/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInArrayOfStringsWithDoList/expected.json new file mode 100644 index 000000000..3784e98df --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInArrayOfStringsWithDoList/expected.json @@ -0,0 +1,3 @@ +{ + "key" : [ "value", "m__re_value", "even_m__re_value" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInArrayOfStringsWithDoList/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInArrayOfStringsWithDoList/input.json new file mode 100644 index 000000000..160261df2 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInArrayOfStringsWithDoList/input.json @@ -0,0 +1,3 @@ +{ + "key" : [ "value", "more_value", "even_more_value" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInArrayOfStringsWithDoList/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInArrayOfStringsWithDoList/test.fix new file mode 100644 index 000000000..a8e33ca3a --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInArrayOfStringsWithDoList/test.fix @@ -0,0 +1,3 @@ +do list(path: "key[]", "var": "$i") + replace_all("$i", "o", "__") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInArrayOfStringsWithDoList/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInArrayOfStringsWithDoList/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInArrayOfStringsWithDoList/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInArrayOfStringsWithDoList/todo.txt b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInArrayOfStringsWithDoList/todo.txt new file mode 100644 index 000000000..9dd034d9d --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInArrayOfStringsWithDoList/todo.txt @@ -0,0 +1 @@ +See issue #189 diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInPastedSubfieldOfArrayOfObjectsWithAsteriskWithGrouping/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInPastedSubfieldOfArrayOfObjectsWithAsteriskWithGrouping/expected.json new file mode 100644 index 000000000..1dd0f7991 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInPastedSubfieldOfArrayOfObjectsWithAsteriskWithGrouping/expected.json @@ -0,0 +1,16 @@ +{ + "GST" : [ { + "a" : "TestA1" + }, { + "a" : "TestB1", + "b" : "TestB2" + }, { + "a" : "TestC1", + "b" : "TestC2", + "c" : "TestC3" + } ], + "subject" : [ { + "name" : "Test", + "altLabel" : [ "TestA1", "TestB1 ( TestB2 )", "TestC1 ( TestC2 TestC3 )" ] + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInPastedSubfieldOfArrayOfObjectsWithAsteriskWithGrouping/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInPastedSubfieldOfArrayOfObjectsWithAsteriskWithGrouping/input.json new file mode 100644 index 000000000..37aa9694c --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInPastedSubfieldOfArrayOfObjectsWithAsteriskWithGrouping/input.json @@ -0,0 +1,14 @@ +{ + "GST" : [ + { + "a" : "TestA1" + }, { + "a" : "TestB1", + "b" : "TestB2" + }, { + "a" : "TestC1", + "b" : "TestC2", + "c" : "TestC3" + } + ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInPastedSubfieldOfArrayOfObjectsWithAsteriskWithGrouping/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInPastedSubfieldOfArrayOfObjectsWithAsteriskWithGrouping/test.fix new file mode 100644 index 000000000..f2acb0d1b --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInPastedSubfieldOfArrayOfObjectsWithAsteriskWithGrouping/test.fix @@ -0,0 +1,7 @@ +add_field("subject[].$append.name","Test") + +do list(path:"GST[]", "var": "$i") # Other altLabels have a "," character between $a and $b. + paste("subject[].$last.altLabel[].$append", "$i.a", "~(", "$i.b", "$i.c", "~)") +end + +replace_all("subject[].*.altLabel[].*"," | \\( \\)","") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInPastedSubfieldOfArrayOfObjectsWithAsteriskWithGrouping/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInPastedSubfieldOfArrayOfObjectsWithAsteriskWithGrouping/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInPastedSubfieldOfArrayOfObjectsWithAsteriskWithGrouping/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInPastedSubfieldOfArrayOfObjectsWithAsteriskWithGrouping_2/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInPastedSubfieldOfArrayOfObjectsWithAsteriskWithGrouping_2/expected.json new file mode 100644 index 000000000..f5f9c4ca6 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInPastedSubfieldOfArrayOfObjectsWithAsteriskWithGrouping_2/expected.json @@ -0,0 +1,18 @@ +{ + "GST" : [ { + "a" : "TestA1" + }, { + "a" : "TestB1", + "b" : "TestB2" + }, { + "a" : "TestC1", + "b" : "TestC2", + "c" : "TestC3" + } ], + "subject" : [ { + "name" : "Test", + "agent" : { + "altLabel" : [ "TestA1", "TestB1 ( TestB2 )", "TestC1 ( TestC2 TestC3 )" ] + } + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInPastedSubfieldOfArrayOfObjectsWithAsteriskWithGrouping_2/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInPastedSubfieldOfArrayOfObjectsWithAsteriskWithGrouping_2/input.json new file mode 100644 index 000000000..37aa9694c --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInPastedSubfieldOfArrayOfObjectsWithAsteriskWithGrouping_2/input.json @@ -0,0 +1,14 @@ +{ + "GST" : [ + { + "a" : "TestA1" + }, { + "a" : "TestB1", + "b" : "TestB2" + }, { + "a" : "TestC1", + "b" : "TestC2", + "c" : "TestC3" + } + ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInPastedSubfieldOfArrayOfObjectsWithAsteriskWithGrouping_2/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInPastedSubfieldOfArrayOfObjectsWithAsteriskWithGrouping_2/test.fix new file mode 100644 index 000000000..7df342166 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInPastedSubfieldOfArrayOfObjectsWithAsteriskWithGrouping_2/test.fix @@ -0,0 +1,7 @@ +add_field("subject[].$append.name","Test") + +do list(path:"GST[]", "var": "$i") # Other altLabels have a "," character between $a and $b. + paste("subject[].$last.agent.altLabel[].$append", "$i.a", "~(", "$i.b", "$i.c", "~)") +end + +replace_all("subject[].*.agent.altLabel[].*"," | \\( \\)","") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInPastedSubfieldOfArrayOfObjectsWithAsteriskWithGrouping_2/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInPastedSubfieldOfArrayOfObjectsWithAsteriskWithGrouping_2/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInPastedSubfieldOfArrayOfObjectsWithAsteriskWithGrouping_2/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk/expected.json new file mode 100644 index 000000000..13edff839 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk/expected.json @@ -0,0 +1,12 @@ +{ + "key" : [ { + "word" : "value", + "foo" : "bar" + }, { + "word" : "m__re_value", + "foo" : "bar" + }, { + "word" : "even_m__re_value", + "foo" : "bar" + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk/input.json new file mode 100644 index 000000000..aeba8bf7d --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk/input.json @@ -0,0 +1,12 @@ +{ + "key" : [ { + "word" : "value", + "foo" : "bar" + }, { + "word" : "more_value", + "foo" : "bar" + }, { + "word" : "even_more_value", + "foo" : "bar" + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk/test.fix new file mode 100644 index 000000000..4e8f6c17f --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk/test.fix @@ -0,0 +1 @@ +replace_all("key[].*.word", "o", "__") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsteriskWithGrouping/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsteriskWithGrouping/expected.json new file mode 100644 index 000000000..6b51192d6 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsteriskWithGrouping/expected.json @@ -0,0 +1,15 @@ +{ + "creator" : [ { + "id" : "test", + "name" : "Max Musterfrau" + }, { + "id" : "test", + "name" : "Uni Köln" + }, { + "id" : "test", + "name" : "Bugs Bunny" + }, { + "id" : "test", + "name" : "Carmen Santiego" + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsteriskWithGrouping/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsteriskWithGrouping/input.json new file mode 100644 index 000000000..5d7c0a59a --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsteriskWithGrouping/input.json @@ -0,0 +1,17 @@ +{ + "creator" : [ + { + "id" : "test", + "name": "Musterfrau, Max" + }, { + "id" : "test", + "name": "Uni Köln" + }, { + "id" : "test", + "name": "Bugs Bunny" + }, { + "id" : "test", + "name": "Santiego, Carmen" + } + ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsteriskWithGrouping/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsteriskWithGrouping/test.fix new file mode 100644 index 000000000..43274127c --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsteriskWithGrouping/test.fix @@ -0,0 +1 @@ +replace_all("creator[].*.name","(.+), (.+)", "$2 $1") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsteriskWithGrouping/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsteriskWithGrouping/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsteriskWithGrouping/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex/expected.json new file mode 100644 index 000000000..c438b4f04 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex/expected.json @@ -0,0 +1,27 @@ +{ + "dateModified" : "2020-01-01T13:05:29+00:00", + "datePublished" : "2018-12-21T12:05:29+00:00", + "test" : [ { + "id" : "test", + "dateModified" : "2020-01-01", + "dateCreated" : "2018-12-21" + } ] +} +{ + "dateModified" : "2016-08-10T22:05:29+00:00", + "datePublished" : "2013-12-21T12:05:29+00:00", + "test" : [ { + "id" : "test", + "dateModified" : "2016-08-10", + "dateCreated" : "2013-12-21" + } ] +} +{ + "dateModified" : "2010-08-29T12:05:29+00:00", + "datePublished" : "2000-01-21T12:05:29+00:00", + "test" : [ { + "id" : "test", + "dateModified" : "2010-08-29", + "dateCreated" : "2000-01-21" + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex/input.json new file mode 100644 index 000000000..373ae8541 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex/input.json @@ -0,0 +1,26 @@ +{ + "dateModified": "2020-01-01T13:05:29+00:00", + "datePublished": "2018-12-21T12:05:29+00:00", + "test" : [ + { + "id" : "test" + } ] +} +{ + "dateModified": "2016-08-10T22:05:29+00:00", + "datePublished": "2013-12-21T12:05:29+00:00", + "test" : [ + { + "id" : "test" + } ] +} +{ + "dateModified": "2010-08-29T12:05:29+00:00", + "datePublished": "2000-01-21T12:05:29+00:00", + "test" : [ + { + "id" : "test" + } ] +} + + diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex/test.fix new file mode 100644 index 000000000..42fd79f37 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex/test.fix @@ -0,0 +1,5 @@ +copy_field("dateModified", "test[].$last.dateModified") +replace_all("test[].*.dateModified", 'T.*', '') + +copy_field("datePublished", "test[].$last.dateCreated") +replace_all("test[].*.dateCreated", 'T.*', '') diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex_2/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex_2/expected.json new file mode 100644 index 000000000..aeac93c01 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex_2/expected.json @@ -0,0 +1,30 @@ +{ + "id" : "test", + "dateModified" : "2020-01-01T13:05:29+00:00", + "datePublished" : "2018-12-21T12:05:29+00:00", + "test" : [ { + "id" : "test", + "dateModified" : "2020-01-01", + "dateCreated" : "2018-12-21" + } ] +} +{ + "id" : "test", + "dateModified" : "2016-08-10T22:05:29+00:00", + "datePublished" : "2013-12-21T12:05:29+00:00", + "test" : [ { + "id" : "test", + "dateModified" : "2016-08-10", + "dateCreated" : "2013-12-21" + } ] +} +{ + "id" : "test", + "dateModified" : "2010-08-29T12:05:29+00:00", + "datePublished" : "2000-01-21T12:05:29+00:00", + "test" : [ { + "id" : "test", + "dateModified" : "2010-08-29", + "dateCreated" : "2000-01-21" + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex_2/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex_2/input.json new file mode 100644 index 000000000..b714f5858 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex_2/input.json @@ -0,0 +1,17 @@ +{ + "id" : "test", + "dateModified": "2020-01-01T13:05:29+00:00", + "datePublished": "2018-12-21T12:05:29+00:00" +} +{ + "id" : "test", + "dateModified": "2016-08-10T22:05:29+00:00", + "datePublished": "2013-12-21T12:05:29+00:00" +} +{ + "id" : "test", + "dateModified": "2010-08-29T12:05:29+00:00", + "datePublished": "2000-01-21T12:05:29+00:00" +} + + diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex_2/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex_2/test.fix new file mode 100644 index 000000000..de729ffce --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex_2/test.fix @@ -0,0 +1,7 @@ +copy_field("id", "test[].$append.id") + +copy_field("dateModified", "test[].$last.dateModified") +replace_all("test[].*.dateModified", 'T.*', '') + +copy_field("datePublished", "test[].$last.dateCreated") +replace_all("test[].*.dateCreated", 'T.*', '') diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex_2/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex_2/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex_2/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex_3/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex_3/expected.json new file mode 100644 index 000000000..dc6175aa6 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex_3/expected.json @@ -0,0 +1,33 @@ +{ + "id" : "test", + "dateModified" : "2020-01-01T13:05:29+00:00", + "datePublished" : "2018-12-21T12:05:29+00:00", + "dateCreated" : "2018-12-21", + "test" : [ { + "id" : "test", + "dateModified" : "2020-01-01", + "dateCreated" : "2018-12-21" + } ] +} +{ + "id" : "test", + "dateModified" : "2016-08-10T22:05:29+00:00", + "datePublished" : "2013-12-21T12:05:29+00:00", + "dateCreated" : "2013-12-21", + "test" : [ { + "id" : "test", + "dateModified" : "2016-08-10", + "dateCreated" : "2013-12-21" + } ] +} +{ + "id" : "test", + "dateModified" : "2010-08-29T12:05:29+00:00", + "datePublished" : "2000-01-21T12:05:29+00:00", + "dateCreated" : "2000-01-21", + "test" : [ { + "id" : "test", + "dateModified" : "2010-08-29", + "dateCreated" : "2000-01-21" + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex_3/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex_3/input.json new file mode 100644 index 000000000..b714f5858 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex_3/input.json @@ -0,0 +1,17 @@ +{ + "id" : "test", + "dateModified": "2020-01-01T13:05:29+00:00", + "datePublished": "2018-12-21T12:05:29+00:00" +} +{ + "id" : "test", + "dateModified": "2016-08-10T22:05:29+00:00", + "datePublished": "2013-12-21T12:05:29+00:00" +} +{ + "id" : "test", + "dateModified": "2010-08-29T12:05:29+00:00", + "datePublished": "2000-01-21T12:05:29+00:00" +} + + diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex_3/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex_3/test.fix new file mode 100644 index 000000000..447807d2e --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex_3/test.fix @@ -0,0 +1,10 @@ +copy_field("datePublished", "dateCreated") +replace_all("dateCreated", 'T.*', '') + +copy_field("id", "test[].$append.id") + +copy_field("dateModified", "test[].$last.dateModified") +replace_all("test[].*.dateModified", 'T.*', '') + +copy_field("datePublished", "test[].$last.dateCreated") +replace_all("test[].*.dateCreated", 'T.*', '') diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex_3/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex_3/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex_3/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex_4/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex_4/expected.json new file mode 100644 index 000000000..0c57c529e --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex_4/expected.json @@ -0,0 +1,24 @@ +{ + "id" : "test", + "node" : { + "dateModified" : "2019-07-23T09:25:58Z", + "datePublished" : "2017-11-13T16:11:36Z" + }, + "test" : [ { + "id" : "test", + "dateModified" : "2019-07-23", + "dateCreated" : "2017-11-13" + } ] +} +{ + "id" : "test", + "node" : { + "dateModified" : "2019-07-23T09:26:01Z", + "datePublished" : "2017-11-13T15:36:34Z" + }, + "test" : [ { + "id" : "test", + "dateModified" : "2019-07-23", + "dateCreated" : "2017-11-13" + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex_4/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex_4/input.json new file mode 100644 index 000000000..1082848ce --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex_4/input.json @@ -0,0 +1,17 @@ +{ + "id" : "test", + "node" : { + "dateModified" : "2019-07-23T09:25:58Z", + "datePublished" : "2017-11-13T16:11:36Z" + } +} +{ + "id" : "test", + "node" : { + "dateModified" : "2019-07-23T09:26:01Z", + "datePublished" : "2017-11-13T15:36:34Z" + } +} + + + diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex_4/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex_4/test.fix new file mode 100644 index 000000000..f5341a851 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex_4/test.fix @@ -0,0 +1,7 @@ +copy_field("id", "test[].$append.id") + +copy_field("node.dateModified", "test[].$last.dateModified") +replace_all("test[].*.dateModified", 'T.+Z', '') + +copy_field("node.datePublished", "test[].$last.dateCreated") +replace_all("test[].*.dateCreated", 'T.+Z', '') diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex_4/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex_4/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithAsterisk_complex_4/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithLastWildcard/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithLastWildcard/expected.json new file mode 100644 index 000000000..03e39e178 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithLastWildcard/expected.json @@ -0,0 +1,12 @@ +{ + "key" : [ { + "word" : "moo_value", + "foo" : "bar" + }, { + "word" : "more_value", + "foo" : "bar" + }, { + "word" : "even_m__re_value", + "foo" : "bar" + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithLastWildcard/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithLastWildcard/input.json new file mode 100644 index 000000000..73cd505a3 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithLastWildcard/input.json @@ -0,0 +1,12 @@ +{ + "key" : [ { + "word" : "moo_value", + "foo" : "bar" + }, { + "word" : "more_value", + "foo" : "bar" + }, { + "word" : "even_more_value", + "foo" : "bar" + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithLastWildcard/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithLastWildcard/test.fix new file mode 100644 index 000000000..6831bd773 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithLastWildcard/test.fix @@ -0,0 +1 @@ +replace_all("key[].$last.word", "o", "__") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithLastWildcard/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithLastWildcard/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allInSubfieldOfArrayOfObjectsWithLastWildcard/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allSimple/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allSimple/expected.json new file mode 100644 index 000000000..ca88cd1ca --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allSimple/expected.json @@ -0,0 +1,3 @@ +{ + "key" : "b__ard" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allSimple/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allSimple/input.json new file mode 100644 index 000000000..cae33cb0c --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allSimple/input.json @@ -0,0 +1,3 @@ +{ + "key" : "board" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allSimple/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allSimple/test.fix new file mode 100644 index 000000000..1c2c099e0 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allSimple/test.fix @@ -0,0 +1 @@ +replace_all("key", "o", "__") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allSimple/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allSimple/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/replace_allSimple/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseArrayOfStrings/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseArrayOfStrings/expected.json new file mode 100644 index 000000000..516ffb503 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseArrayOfStrings/expected.json @@ -0,0 +1,3 @@ +{ + "animals" : [ "zebra", "cat", "dog" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseArrayOfStrings/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseArrayOfStrings/input.json new file mode 100644 index 000000000..12733202d --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseArrayOfStrings/input.json @@ -0,0 +1,3 @@ +{ + "animals" : [ "dog", "cat", "zebra" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseArrayOfStrings/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseArrayOfStrings/test.fix new file mode 100644 index 000000000..1b418b269 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseArrayOfStrings/test.fix @@ -0,0 +1 @@ +reverse("animals[]") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseArrayOfStrings/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseArrayOfStrings/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseArrayOfStrings/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseSimpleField/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseSimpleField/expected.json new file mode 100644 index 000000000..2749dc0ae --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseSimpleField/expected.json @@ -0,0 +1,7 @@ +{ + "animal" : "ynnub", + "animal_2" : { + "name" : "bird", + "type" : "TSET" + } +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseSimpleField/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseSimpleField/input.json new file mode 100644 index 000000000..0c72e14b8 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseSimpleField/input.json @@ -0,0 +1,7 @@ +{ + "animal": "bunny", + "animal_2": { + "name": "bird", + "type": "TEST" + } +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseSimpleField/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseSimpleField/test.fix new file mode 100644 index 000000000..a52822548 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseSimpleField/test.fix @@ -0,0 +1,2 @@ +reverse("animal") +reverse("animal_2.type") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseSimpleField/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseSimpleField/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseSimpleField/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseSubfieldInArrayOfObjects/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseSubfieldInArrayOfObjects/expected.json new file mode 100644 index 000000000..c71bf85ed --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseSubfieldInArrayOfObjects/expected.json @@ -0,0 +1,12 @@ +{ + "test" : [ { + "key" : "1_eulav", + "test" : "test_1" + }, { + "key" : "2_eulav", + "test" : "test_2" + }, { + "key" : "3_eulav", + "test" : "test_3" + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseSubfieldInArrayOfObjects/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseSubfieldInArrayOfObjects/input.json new file mode 100644 index 000000000..84d0da50f --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseSubfieldInArrayOfObjects/input.json @@ -0,0 +1,12 @@ +{ + "test": [{ + "key": "value_1", + "test": "test_1" + },{ + "key": "value_2", + "test": "test_2" + },{ + "key": "value_3", + "test": "test_3" + }] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseSubfieldInArrayOfObjects/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseSubfieldInArrayOfObjects/test.fix new file mode 100644 index 000000000..f3bfc0bbc --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseSubfieldInArrayOfObjects/test.fix @@ -0,0 +1 @@ +reverse("test[].*.key") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseSubfieldInArrayOfObjects/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseSubfieldInArrayOfObjects/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseSubfieldInArrayOfObjects/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseSubfieldInArrayOfObjects/todo.txt b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseSubfieldInArrayOfObjects/todo.txt new file mode 100644 index 000000000..0b9661d26 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseSubfieldInArrayOfObjects/todo.txt @@ -0,0 +1 @@ +See issue #121 diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseValuesInArrayOfStrings/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseValuesInArrayOfStrings/expected.json new file mode 100644 index 000000000..9e7b81482 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseValuesInArrayOfStrings/expected.json @@ -0,0 +1,3 @@ +{ + "animals" : [ "god", "tac", "arbez" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseValuesInArrayOfStrings/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseValuesInArrayOfStrings/input.json new file mode 100644 index 000000000..0218f89c9 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseValuesInArrayOfStrings/input.json @@ -0,0 +1,3 @@ +{ + "animals": [ "dog", "cat", "zebra" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseValuesInArrayOfStrings/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseValuesInArrayOfStrings/test.fix new file mode 100644 index 000000000..5c655bc7f --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseValuesInArrayOfStrings/test.fix @@ -0,0 +1 @@ +reverse("animals[].*") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseValuesInArrayOfStrings/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseValuesInArrayOfStrings/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseValuesInArrayOfStrings/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseValuesInArrayOfStrings/todo.txt b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseValuesInArrayOfStrings/todo.txt new file mode 100644 index 000000000..0b9661d26 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/reverseValuesInArrayOfStrings/todo.txt @@ -0,0 +1 @@ +See issue #121 diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sort_fieldInArrayOfObjectsWithAsterisk/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sort_fieldInArrayOfObjectsWithAsterisk/expected.json new file mode 100644 index 000000000..daad4ce41 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sort_fieldInArrayOfObjectsWithAsterisk/expected.json @@ -0,0 +1,11 @@ +{ + "adventureDuo" : [ { + "name" : "Jake", + "animal" : "dog", + "tools" : [ "arms", "magic", "shapeShifting" ] + }, { + "name" : "Finn", + "animal" : "human", + "tools" : [ "anotherSword", "arm", "sword" ] + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sort_fieldInArrayOfObjectsWithAsterisk/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sort_fieldInArrayOfObjectsWithAsterisk/input.json new file mode 100644 index 000000000..9ae00b1cb --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sort_fieldInArrayOfObjectsWithAsterisk/input.json @@ -0,0 +1,11 @@ +{ + "adventureDuo": [ { + "name" : "Jake", + "animal" : "dog", + "tools" : [ "magic", "arms", "shapeShifting" ] + }, { + "name" : "Finn", + "animal" : "human", + "tools" : [ "sword", "anotherSword", "arm" ] + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sort_fieldInArrayOfObjectsWithAsterisk/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sort_fieldInArrayOfObjectsWithAsterisk/test.fix new file mode 100644 index 000000000..a18704c68 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sort_fieldInArrayOfObjectsWithAsterisk/test.fix @@ -0,0 +1,2 @@ +sort_field("adventureDuo[].*.tools[]") + diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sort_fieldInArrayOfObjectsWithAsterisk/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sort_fieldInArrayOfObjectsWithAsterisk/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sort_fieldInArrayOfObjectsWithAsterisk/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sort_fieldInArrayOfObjectsWithAsterisk/todo.txt b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sort_fieldInArrayOfObjectsWithAsterisk/todo.txt new file mode 100644 index 000000000..0b9661d26 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sort_fieldInArrayOfObjectsWithAsterisk/todo.txt @@ -0,0 +1 @@ +See issue #121 diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sort_fieldSimple/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sort_fieldSimple/expected.json new file mode 100644 index 000000000..0b6279cc8 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sort_fieldSimple/expected.json @@ -0,0 +1,9 @@ +{ + "animals" : [ "cat", "cat", "dog", "zebra" ], + "animals_2" : [ "zebra", "dog", "cat", "cat" ], + "animals_3" : [ "cat", "dog", "zebra" ], + "numbers" : [ "1", "10", "2", "7", "7" ], + "numbers_2" : [ "1", "2", "7", "7", "10" ], + "numbers_3" : [ "10", "7", "7", "2", "1" ], + "numbers_4" : [ "1", "2", "7", "10" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sort_fieldSimple/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sort_fieldSimple/input.json new file mode 100644 index 000000000..4d31b867a --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sort_fieldSimple/input.json @@ -0,0 +1,9 @@ +{ + "animals" : [ "dog", "cat", "cat", "zebra" ], + "animals_2" : [ "dog", "cat", "cat", "zebra" ], + "animals_3" : [ "dog", "cat", "cat", "zebra" ], + "numbers" : [ "7", "2", "7", "1", "10" ], + "numbers_2" : [ "7", "2", "7", "1", "10" ], + "numbers_3" : [ "7", "2", "7", "1", "10" ], + "numbers_4" : [ "7", "2", "7", "1", "10" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sort_fieldSimple/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sort_fieldSimple/test.fix new file mode 100644 index 000000000..1ae49f39e --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sort_fieldSimple/test.fix @@ -0,0 +1,7 @@ +sort_field("animals[]") +sort_field("numbers[]") +sort_field("numbers_2[]",numeric:"true") +sort_field("animals_2[]",reverse:"true") +sort_field("numbers_3[]", numeric:"true",reverse:"true") +sort_field("animals_3[]", uniq:"true") +sort_field("numbers_4[]",numeric:"true", uniq:"true") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sort_fieldSimple/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sort_fieldSimple/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sort_fieldSimple/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSimple+move/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSimple+move/expected.json new file mode 100644 index 000000000..8b9250cc2 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSimple+move/expected.json @@ -0,0 +1,3 @@ +{ + "keywords" : [ "Ingenieurwissenschaften", "Technik", " Medizin", " angewandte Wissenschaften", "Naturwissenschaften", "Naturwissenschaften und Mathematik", "RUVIVAL Toolbox", "RUVIVAL", "Regenwassernutzung", "Indigenes Wissen" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSimple+move/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSimple+move/input.json new file mode 100644 index 000000000..28ba80712 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSimple+move/input.json @@ -0,0 +1,3 @@ +{ + "keywords": "Ingenieurwissenschaften,Technik, Medizin, angewandte Wissenschaften,Naturwissenschaften,Naturwissenschaften und Mathematik,RUVIVAL Toolbox,RUVIVAL,Regenwassernutzung,Indigenes Wissen" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSimple+move/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSimple+move/test.fix new file mode 100644 index 000000000..fa9b77805 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSimple+move/test.fix @@ -0,0 +1,3 @@ +# ---------------keywords---------------------------------- +split_field("keywords",",") +move_field("keywords","keywords[]") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSimple+move/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSimple+move/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSimple+move/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSimple+trim+move/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSimple+trim+move/expected.json new file mode 100644 index 000000000..3261ada43 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSimple+trim+move/expected.json @@ -0,0 +1,3 @@ +{ + "keywords" : [ "Ingenieurwissenschaften", "Technik", "Medizin", "angewandte Wissenschaften", "Naturwissenschaften", "Naturwissenschaften und Mathematik", "RUVIVAL Toolbox", "RUVIVAL", "Regenwassernutzung", "Indigenes Wissen" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSimple+trim+move/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSimple+trim+move/input.json new file mode 100644 index 000000000..28ba80712 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSimple+trim+move/input.json @@ -0,0 +1,3 @@ +{ + "keywords": "Ingenieurwissenschaften,Technik, Medizin, angewandte Wissenschaften,Naturwissenschaften,Naturwissenschaften und Mathematik,RUVIVAL Toolbox,RUVIVAL,Regenwassernutzung,Indigenes Wissen" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSimple+trim+move/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSimple+trim+move/test.fix new file mode 100644 index 000000000..8ae7b5203 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSimple+trim+move/test.fix @@ -0,0 +1,4 @@ +# ---------------keywords---------------------------------- +split_field("keywords",",") +trim("keywords.*") +move_field("keywords","keywords[]") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSimple+trim+move/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSimple+trim+move/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSimple+trim+move/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSimple/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSimple/expected.json new file mode 100644 index 000000000..4efa30ad2 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSimple/expected.json @@ -0,0 +1,5 @@ +{ + "tools" : "magic", + "tools" : "arms", + "tools" : "shapeShifting" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSimple/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSimple/input.json new file mode 100644 index 000000000..3ef275adf --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSimple/input.json @@ -0,0 +1,3 @@ +{ + "tools" : "magic--arms--shapeShifting" +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSimple/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSimple/test.fix new file mode 100644 index 000000000..d443afdf0 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSimple/test.fix @@ -0,0 +1 @@ +split_field("tools", "--") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSimple/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSimple/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSimple/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSubfieldInArrayOfObjectsWithAsterisk/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSubfieldInArrayOfObjectsWithAsterisk/expected.json new file mode 100644 index 000000000..b9e1e27b5 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSubfieldInArrayOfObjectsWithAsterisk/expected.json @@ -0,0 +1,15 @@ +{ + "adventureDuo" : [ { + "name" : "Jake", + "animal" : "dog", + "tools" : "magic", + "tools" : "arms", + "tools" : "shapeShifting" + }, { + "name" : "Finn", + "animal" : "human", + "tools" : "sword", + "tools" : "anotherSword", + "tools" : "arm" + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSubfieldInArrayOfObjectsWithAsterisk/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSubfieldInArrayOfObjectsWithAsterisk/input.json new file mode 100644 index 000000000..1e14a8384 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSubfieldInArrayOfObjectsWithAsterisk/input.json @@ -0,0 +1,11 @@ +{ + "adventureDuo": [ { + "name" : "Jake", + "animal" : "dog", + "tools" : "magic--arms--shapeShifting" + }, { + "name" : "Finn", + "animal" : "human", + "tools" : "sword--anotherSword--arm" + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSubfieldInArrayOfObjectsWithAsterisk/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSubfieldInArrayOfObjectsWithAsterisk/test.fix new file mode 100644 index 000000000..d50fde614 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSubfieldInArrayOfObjectsWithAsterisk/test.fix @@ -0,0 +1 @@ +split_field("adventureDuo[].*.tools", "--") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSubfieldInArrayOfObjectsWithAsterisk/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSubfieldInArrayOfObjectsWithAsterisk/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSubfieldInArrayOfObjectsWithAsterisk/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSubfieldInArrayOfObjectsWithAsterisk/todo.txt b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSubfieldInArrayOfObjectsWithAsterisk/todo.txt new file mode 100644 index 000000000..0b9661d26 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSubfieldInArrayOfObjectsWithAsterisk/todo.txt @@ -0,0 +1 @@ +See issue #121 diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSubfieldInObject/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSubfieldInObject/expected.json new file mode 100644 index 000000000..0a69d6b74 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSubfieldInObject/expected.json @@ -0,0 +1,7 @@ +{ + "people" : { + "investigators" : "Justus", + "investigators" : "Peter", + "investigators" : "Bob" + } +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSubfieldInObject/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSubfieldInObject/input.json new file mode 100644 index 000000000..00b7a05a9 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSubfieldInObject/input.json @@ -0,0 +1,5 @@ +{ + "people" : { + "investigators" : "Justus?Peter?Bob" + } +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSubfieldInObject/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSubfieldInObject/test.fix new file mode 100644 index 000000000..9775183cf --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSubfieldInObject/test.fix @@ -0,0 +1 @@ +split_field("people.investigators","\\?") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSubfieldInObject/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSubfieldInObject/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/split_fieldSubfieldInObject/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/substringJustStart/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/substringJustStart/expected.json new file mode 100644 index 000000000..c981f36ff --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/substringJustStart/expected.json @@ -0,0 +1 @@ +{"key":"cat"} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/substringJustStart/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/substringJustStart/input.json new file mode 100644 index 000000000..378024549 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/substringJustStart/input.json @@ -0,0 +1 @@ +{"key":"tacocat"} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/substringJustStart/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/substringJustStart/test.fix new file mode 100644 index 000000000..3460d70b5 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/substringJustStart/test.fix @@ -0,0 +1 @@ +substring("key", "4") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/substringJustStart/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/substringJustStart/test.flux new file mode 100644 index 000000000..30a8181bf --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/substringJustStart/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/substringStartLength/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/substringStartLength/expected.json new file mode 100644 index 000000000..a12a00264 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/substringStartLength/expected.json @@ -0,0 +1 @@ +{"key":"taco"} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/substringStartLength/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/substringStartLength/input.json new file mode 100644 index 000000000..378024549 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/substringStartLength/input.json @@ -0,0 +1 @@ +{"key":"tacocat"} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/substringStartLength/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/substringStartLength/test.fix new file mode 100644 index 000000000..0812d0ff8 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/substringStartLength/test.fix @@ -0,0 +1 @@ +substring("key", "0", "4") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/substringStartLength/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/substringStartLength/test.flux new file mode 100644 index 000000000..30a8181bf --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/substringStartLength/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sumSimple/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sumSimple/expected.json new file mode 100644 index 000000000..ce6d5321f --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sumSimple/expected.json @@ -0,0 +1,4 @@ +{ + "numbers" : [ "27" ], + "numbers_2" : [ "10" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sumSimple/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sumSimple/input.json new file mode 100644 index 000000000..3abb6ce35 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sumSimple/input.json @@ -0,0 +1,4 @@ +{ + "numbers" : [ "7", "2", "7", "1", "10" ], + "numbers_2" : [ "1", "2", "3", "4"] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sumSimple/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sumSimple/test.fix new file mode 100644 index 000000000..b3ff41e57 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sumSimple/test.fix @@ -0,0 +1,3 @@ +sum("numbers[]") +sum("numbers_2[]") + diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sumSimple/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sumSimple/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sumSimple/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sumSubfieldInArrayOfObjects/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sumSubfieldInArrayOfObjects/expected.json new file mode 100644 index 000000000..117b4a1ed --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sumSubfieldInArrayOfObjects/expected.json @@ -0,0 +1,7 @@ +{ + "objects" : [ { + "numbers" : [ "27" ] + }, { + "numbers" : [ "10" ] + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sumSubfieldInArrayOfObjects/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sumSubfieldInArrayOfObjects/input.json new file mode 100644 index 000000000..c31cf0b1d --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sumSubfieldInArrayOfObjects/input.json @@ -0,0 +1,7 @@ +{ + "objects" : [ { + "numbers" : [ "7", "2", "7", "1", "10" ] + }, { + "numbers" : [ "1", "2", "3", "4"] + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sumSubfieldInArrayOfObjects/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sumSubfieldInArrayOfObjects/test.fix new file mode 100644 index 000000000..f134b293b --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sumSubfieldInArrayOfObjects/test.fix @@ -0,0 +1,2 @@ +sum("objects[].*.numbers[]") + diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sumSubfieldInArrayOfObjects/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sumSubfieldInArrayOfObjects/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sumSubfieldInArrayOfObjects/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sumSubfieldInArrayOfObjects/todo.txt b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sumSubfieldInArrayOfObjects/todo.txt new file mode 100644 index 000000000..0b9661d26 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/sumSubfieldInArrayOfObjects/todo.txt @@ -0,0 +1 @@ +See issue #121 diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/trimArrayOfStrings/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/trimArrayOfStrings/expected.json new file mode 100644 index 000000000..3261ada43 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/trimArrayOfStrings/expected.json @@ -0,0 +1,3 @@ +{ + "keywords" : [ "Ingenieurwissenschaften", "Technik", "Medizin", "angewandte Wissenschaften", "Naturwissenschaften", "Naturwissenschaften und Mathematik", "RUVIVAL Toolbox", "RUVIVAL", "Regenwassernutzung", "Indigenes Wissen" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/trimArrayOfStrings/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/trimArrayOfStrings/input.json new file mode 100644 index 000000000..8b9250cc2 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/trimArrayOfStrings/input.json @@ -0,0 +1,3 @@ +{ + "keywords" : [ "Ingenieurwissenschaften", "Technik", " Medizin", " angewandte Wissenschaften", "Naturwissenschaften", "Naturwissenschaften und Mathematik", "RUVIVAL Toolbox", "RUVIVAL", "Regenwassernutzung", "Indigenes Wissen" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/trimArrayOfStrings/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/trimArrayOfStrings/test.fix new file mode 100644 index 000000000..502eb6f99 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/trimArrayOfStrings/test.fix @@ -0,0 +1 @@ +trim("keywords[].*") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/trimArrayOfStrings/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/trimArrayOfStrings/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/trimArrayOfStrings/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/uniqMultipleArraysWithAsterisk/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/uniqMultipleArraysWithAsterisk/expected.json new file mode 100644 index 000000000..b63e161e8 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/uniqMultipleArraysWithAsterisk/expected.json @@ -0,0 +1,7 @@ +{ + "subject": [ { + "keywords" : [ "ETL", "ELT", "Extract", "Load", "Transform", "Data Quality" ] + },{ + "keywords" : [ "dog", "zebra", "cat" ] + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/uniqMultipleArraysWithAsterisk/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/uniqMultipleArraysWithAsterisk/input.json new file mode 100644 index 000000000..40b03536a --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/uniqMultipleArraysWithAsterisk/input.json @@ -0,0 +1,7 @@ +{ + "subject": [ { + "keywords" : [ "ETL", "ELT", "Extract", "Load", "Transform", "Data Quality", "Extract", "Load", "Transform", "Transform" ] + },{ + "keywords" : [ "dog", "dog", "zebra", "cat", "cat" ] + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/uniqMultipleArraysWithAsterisk/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/uniqMultipleArraysWithAsterisk/test.fix new file mode 100644 index 000000000..b223aecb0 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/uniqMultipleArraysWithAsterisk/test.fix @@ -0,0 +1 @@ +uniq("subject[].*.keywords[]") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/uniqMultipleArraysWithAsterisk/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/uniqMultipleArraysWithAsterisk/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/uniqMultipleArraysWithAsterisk/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/uniqMultipleArraysWithAsterisk/todo.txt b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/uniqMultipleArraysWithAsterisk/todo.txt new file mode 100644 index 000000000..ccaae036c --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/uniqMultipleArraysWithAsterisk/todo.txt @@ -0,0 +1 @@ +See issue #261 diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/uniqSimple/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/uniqSimple/expected.json new file mode 100644 index 000000000..9f7bc5d62 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/uniqSimple/expected.json @@ -0,0 +1,3 @@ +{ + "keywords" : [ "ETL", "ELT", "Extract", "Load", "Transform", "Data Quality" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/uniqSimple/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/uniqSimple/input.json new file mode 100644 index 000000000..ecdd9913f --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/uniqSimple/input.json @@ -0,0 +1,3 @@ +{ + "keywords" : [ "ETL", "ELT", "Extract", "Load", "Transform", "Data Quality", "Extract", "Load", "Transform", "Transform" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/uniqSimple/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/uniqSimple/test.fix new file mode 100644 index 000000000..9ffbc477a --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/uniqSimple/test.fix @@ -0,0 +1 @@ +uniq("keywords[]") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/uniqSimple/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/uniqSimple/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/uniqSimple/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/upcaseInArrayOfStrings/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/upcaseInArrayOfStrings/expected.json new file mode 100644 index 000000000..1f88137e2 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/upcaseInArrayOfStrings/expected.json @@ -0,0 +1,3 @@ +{ + "key" : [ "VALUE", "MORE_VALUE", "EVEN_MORE_VALUE" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/upcaseInArrayOfStrings/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/upcaseInArrayOfStrings/input.json new file mode 100644 index 000000000..160261df2 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/upcaseInArrayOfStrings/input.json @@ -0,0 +1,3 @@ +{ + "key" : [ "value", "more_value", "even_more_value" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/upcaseInArrayOfStrings/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/upcaseInArrayOfStrings/test.fix new file mode 100644 index 000000000..dcb7f1094 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/upcaseInArrayOfStrings/test.fix @@ -0,0 +1 @@ +upcase("key[].*") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/upcaseInArrayOfStrings/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/upcaseInArrayOfStrings/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/upcaseInArrayOfStrings/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/upcaseInArrayOfStringsWithDoList/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/upcaseInArrayOfStringsWithDoList/expected.json new file mode 100644 index 000000000..dd85ba7c3 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/upcaseInArrayOfStringsWithDoList/expected.json @@ -0,0 +1,3 @@ +{ + "key" : [ "VALUE", "MORE VALUE", "EVEN MORE VALUE" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/upcaseInArrayOfStringsWithDoList/input.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/upcaseInArrayOfStringsWithDoList/input.json new file mode 100644 index 000000000..160261df2 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/upcaseInArrayOfStringsWithDoList/input.json @@ -0,0 +1,3 @@ +{ + "key" : [ "value", "more_value", "even_more_value" ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/upcaseInArrayOfStringsWithDoList/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/upcaseInArrayOfStringsWithDoList/test.fix new file mode 100644 index 000000000..b7ed93ba4 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/upcaseInArrayOfStringsWithDoList/test.fix @@ -0,0 +1,3 @@ +do list(path: "key[]", "var": "$i") + upcase("$i") +end diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/upcaseInArrayOfStringsWithDoList/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/upcaseInArrayOfStringsWithDoList/test.flux new file mode 100644 index 000000000..7c3575fac --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/upcaseInArrayOfStringsWithDoList/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.json" +|open-file +|as-records +|decode-json +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/upcaseInArrayOfStringsWithDoList/todo.txt b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/upcaseInArrayOfStringsWithDoList/todo.txt new file mode 100644 index 000000000..9dd034d9d --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/upcaseInArrayOfStringsWithDoList/todo.txt @@ -0,0 +1 @@ +See issue #189 diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromXml/toJson/replace_allInOptionalSubfieldOfRepeatedObjectsWithAsterisk/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromXml/toJson/replace_allInOptionalSubfieldOfRepeatedObjectsWithAsterisk/expected.json new file mode 100644 index 000000000..3b010d5fe --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromXml/toJson/replace_allInOptionalSubfieldOfRepeatedObjectsWithAsterisk/expected.json @@ -0,0 +1,8 @@ +{ + "RSWK" : [ { + "subjectTopicName" : "Nonprofit organizations" + }, { + "subjectTopicName" : "Nonprofit organizations", + "subjectGenre" : "Case studies" + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromXml/toJson/replace_allInOptionalSubfieldOfRepeatedObjectsWithAsterisk/input.xml b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromXml/toJson/replace_allInOptionalSubfieldOfRepeatedObjectsWithAsterisk/input.xml new file mode 100644 index 000000000..78fd0b839 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromXml/toJson/replace_allInOptionalSubfieldOfRepeatedObjectsWithAsterisk/input.xml @@ -0,0 +1,14 @@ + + + + + Nonprofit organizations + Management. + + + Nonprofit organizations + Management + Case studies. + + + diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromXml/toJson/replace_allInOptionalSubfieldOfRepeatedObjectsWithAsterisk/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromXml/toJson/replace_allInOptionalSubfieldOfRepeatedObjectsWithAsterisk/test.fix new file mode 100644 index 000000000..9c1c1c447 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromXml/toJson/replace_allInOptionalSubfieldOfRepeatedObjectsWithAsterisk/test.fix @@ -0,0 +1,8 @@ +set_array("RSWK[]") +do list(path: "650??", "var": "$i") + copy_field("$i.a", "RSWK[].$append.subjectTopicName") + copy_field("$i.v", "RSWK[].$last.subjectGenre") +end +replace_all("RSWK[].*.subjectGenre", "[.]$", "") + +retain("RSWK[]") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromXml/toJson/replace_allInOptionalSubfieldOfRepeatedObjectsWithAsterisk/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromXml/toJson/replace_allInOptionalSubfieldOfRepeatedObjectsWithAsterisk/test.flux new file mode 100644 index 000000000..1d8c3df8f --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromXml/toJson/replace_allInOptionalSubfieldOfRepeatedObjectsWithAsterisk/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.xml" +|open-file +|decode-xml +|handle-marcxml +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromXml/toJson/replace_allInOptionalSubfieldOfRepeatedObjectsWithAsteriskImplicitArray/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromXml/toJson/replace_allInOptionalSubfieldOfRepeatedObjectsWithAsteriskImplicitArray/expected.json new file mode 100644 index 000000000..4faa8410b --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromXml/toJson/replace_allInOptionalSubfieldOfRepeatedObjectsWithAsteriskImplicitArray/expected.json @@ -0,0 +1,8 @@ +{ + "rswk" : [ { + "subjectTopicName" : "Nonprofit organizations" + }, { + "subjectTopicName" : "Nonprofit organizations", + "subjectGenre" : "Case studies" + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromXml/toJson/replace_allInOptionalSubfieldOfRepeatedObjectsWithAsteriskImplicitArray/input.xml b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromXml/toJson/replace_allInOptionalSubfieldOfRepeatedObjectsWithAsteriskImplicitArray/input.xml new file mode 100644 index 000000000..78fd0b839 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromXml/toJson/replace_allInOptionalSubfieldOfRepeatedObjectsWithAsteriskImplicitArray/input.xml @@ -0,0 +1,14 @@ + + + + + Nonprofit organizations + Management. + + + Nonprofit organizations + Management + Case studies. + + + diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromXml/toJson/replace_allInOptionalSubfieldOfRepeatedObjectsWithAsteriskImplicitArray/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromXml/toJson/replace_allInOptionalSubfieldOfRepeatedObjectsWithAsteriskImplicitArray/test.fix new file mode 100644 index 000000000..4a1d14252 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromXml/toJson/replace_allInOptionalSubfieldOfRepeatedObjectsWithAsteriskImplicitArray/test.fix @@ -0,0 +1,7 @@ +do list(path: "650??", "var": "$i") + copy_field("$i.a", "rswk[].$append.subjectTopicName") + copy_field("$i.v", "rswk[].$last.subjectGenre") +end +replace_all("rswk[].*.subjectGenre", "[.]$", "") + +retain("rswk[]") diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromXml/toJson/replace_allInOptionalSubfieldOfRepeatedObjectsWithAsteriskImplicitArray/test.flux b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromXml/toJson/replace_allInOptionalSubfieldOfRepeatedObjectsWithAsteriskImplicitArray/test.flux new file mode 100644 index 000000000..1d8c3df8f --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromXml/toJson/replace_allInOptionalSubfieldOfRepeatedObjectsWithAsteriskImplicitArray/test.flux @@ -0,0 +1,8 @@ +FLUX_DIR + "input.xml" +|open-file +|decode-xml +|handle-marcxml +|fix(FLUX_DIR + "test.fix") +|encode-json(prettyPrinting="true") +|write(FLUX_DIR + "output-metafix.json") +; diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromXml/toJson/replace_toUpper/expected.json b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromXml/toJson/replace_toUpper/expected.json new file mode 100644 index 000000000..d8a8d167d --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromXml/toJson/replace_toUpper/expected.json @@ -0,0 +1,8 @@ +{ + "contribution" : [ { + "agent" : { + "label" : "COLEMAN, Julie", + "label" : "KAY, Christian J." + } + } ] +} diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromXml/toJson/replace_toUpper/input.xml b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromXml/toJson/replace_toUpper/input.xml new file mode 100644 index 000000000..5cb5dcd5e --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromXml/toJson/replace_toUpper/input.xml @@ -0,0 +1,11 @@ + + + + + COLEMAN, Julie + Editor + KAY, Christian J. + NS + + + diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromXml/toJson/replace_toUpper/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromXml/toJson/replace_toUpper/test.fix new file mode 100644 index 000000000..6bc9cbcdc --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromXml/toJson/replace_toUpper/test.fix @@ -0,0 +1,7 @@ +do list(path: "700[01] ", "var": "$i") + set_hash("contribution[].$append.agent") + copy_field("$i.a", "contribution[].$last.agent.label") +end + +replace_all("contribution[].*.agent.label", "(? + + + + COLEMAN, Julie + Editor + KAY, Christian J. + NS + + + diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromXml/toJson/replace_toUpperStrictnessHandlesProcessExceptions/test.fix b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromXml/toJson/replace_toUpperStrictnessHandlesProcessExceptions/test.fix new file mode 100644 index 000000000..6bc9cbcdc --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromXml/toJson/replace_toUpperStrictnessHandlesProcessExceptions/test.fix @@ -0,0 +1,7 @@ +do list(path: "700[01] ", "var": "$i") + set_hash("contribution[].$append.agent") + copy_field("$i.a", "contribution[].$last.agent.label") +end + +replace_all("contribution[].*.agent.label", "(? . +@prefix dct: . +@prefix skos: . +@prefix vann: . + + + a skos:Concept ; + skos:prefLabel "Software Application"@en, "Softwareanwendung"@de, "Програмне забезпечення"@uk ; + skos:topConceptOf . + + + a skos:Concept ; + skos:altLabel "Assessment"@de ; + skos:prefLabel "Assessment"@en, "Lernkontrolle"@de, "Оцінювання"@uk ; + skos:scopeNote "unter anderem (Selbst-)Tests"@de ; + skos:topConceptOf . + + + a skos:Concept ; + skos:altLabel "Tonaufnahme"@de, "Звукозапис"@uk ; + skos:prefLabel "Audio"@de, "Audio Recording"@en, "Аудіозапис"@uk ; + skos:topConceptOf . + + + a skos:Concept ; + skos:prefLabel "Case Study"@en, "Fallstudie"@de, "Приклад"@uk ; + skos:topConceptOf . + + + a skos:Concept ; + skos:prefLabel "Course"@en, "Kurs"@de, "Курс"@uk ; + skos:topConceptOf . + + + a skos:Concept ; + skos:prefLabel "Data"@en, "Daten"@de, "Дані"@uk ; + skos:scopeNote "amongst others raw and example data"@en, "unter anderem Roh- oder Beispieldaten"@de, "включно з необробленими даними та зразками"@uk ; + skos:topConceptOf . + + + a skos:Concept ; + skos:altLabel "Grafik"@de, "Графіка"@uk ; + skos:prefLabel "Diagram"@en, "Diagramm"@de, "Діаграма"@uk ; + skos:topConceptOf . + + + a skos:Concept ; + skos:prefLabel "Drill and Practice"@en, "Übung"@de, "Практика"@uk ; + skos:topConceptOf . + + + a skos:Concept ; + skos:prefLabel "Game"@en, "Lernspiel"@de, "Навчальна гра"@uk ; + skos:topConceptOf . + + + a skos:Concept ; + skos:prefLabel "Experiment"@de, "Experiment"@en, "Експеримент"@uk ; + skos:topConceptOf . + + + a skos:Concept ; + skos:altLabel "Bild"@de, "Зображення"@uk ; + skos:prefLabel "Abbildung"@de, "Image"@en, "Ілюстрація"@uk ; + skos:scopeNote "Fotos, Grafiken und sonstige Bilder"@de, "photos, graphics, and other images"@en, "Фотографії, графіки та інші зображення"@uk ; + skos:topConceptOf . + + + a skos:Concept ; + skos:prefLabel "Nachschlagewerk"@de, "Reference Work"@en, "Довідник"@uk ; + skos:scopeNote "e.g.glossary, encyclopedia, dictionary"@en, "zum Beispiel Glossar, Enzyklopädie, Lexikon"@de, "наприклад, глосарій, енциклопедія, словник"@uk ; + skos:topConceptOf . + + + a skos:Concept ; + skos:prefLabel "Lesson Plan"@en, "Unterrichtsplanung"@de, "План уроку"@uk ; + skos:topConceptOf . + + + a skos:Concept ; + skos:prefLabel "Karte"@de, "Map"@en, "Мапа"@uk ; + skos:topConceptOf . + + + a skos:Concept ; + skos:prefLabel "Other"@en, "Sonstiges"@de, "Різне"@uk ; + skos:topConceptOf . + + + a skos:Concept ; + skos:prefLabel "Portal"@de, "Web Portal"@en, "Портал"@uk ; + skos:topConceptOf . + + + a skos:Concept ; + skos:prefLabel "Fragebogen"@de, "Questionnaire"@en, "Анкета"@uk ; + skos:scopeNote "also research assignment and WebQuest"@en, "auch Rechercheauftrag und WebQuest"@de, "також дослідницьке завдання та веб-квест"@uk ; + skos:topConceptOf . + + + dct:description "Eine Wertelliste für Typen von Lernressourcen (Learning Resource Type), entstanden im Kontext des Metadatenschemas \"LOM for Higher Education OER Repositories\" (https://w3id.org/dini-ag-kim/hs-oer-lom-profil/latest/)."@de ; + dct:issued "2020-02-07" ; + dct:license ; + dct:publisher ; + dct:title "Higher Education Resource Types"@en, "Hochschulcampus Ressourcentypen"@de, "Типи ресурсів вищої освіти"@uk ; + vann:preferredNamespacePrefix "hcrt" ; + vann:preferredNamespaceUri "https://w3id.org/kim/hcrt/" ; + a skos:ConceptScheme ; + skos:hasTopConcept , , , , , , , , , , , , , , , , , , , , , , , , , . + + + a skos:Concept ; + skos:prefLabel "Script"@en, "Skript"@de, "Запис"@uk ; + skos:scopeNote "for example lecture notes"@en, "zum Beispiel Vorlesungsskript"@de, "наприклад конспект лекцій"@uk ; + skos:topConceptOf . + + + a skos:Concept ; + skos:prefLabel "Musiknoten"@de, "Sheet Music"@en, "Ноти"@uk ; + skos:topConceptOf . + + + a skos:Concept ; + skos:prefLabel "Simulation"@de, "Simulation"@en, "Симуляція"@uk ; + skos:topConceptOf . + + + a skos:Concept ; + skos:prefLabel "Presentation"@en, "Präsentation"@de, "Презентація"@uk ; + skos:topConceptOf . + + + a skos:Concept ; + skos:prefLabel "Text"@en, "Textdokument"@de, "Текстовий документ"@uk ; + skos:scopeNote "for example article, essay"@en, "zum Beispiel Artikel, Aufsatz, Abhandlung"@de, "наприклад, стаття, есе, трактат"@uk ; + skos:topConceptOf . + + + a skos:Concept ; + skos:prefLabel "Lehrbuch"@de, "Textbook"@en, "Підручник"@uk ; + skos:topConceptOf . + + + a skos:Concept ; + skos:prefLabel "Video"@de, "Video"@en, "Відео"@uk ; + skos:topConceptOf . + + + a skos:Concept ; + skos:prefLabel "Web Page"@en, "Webseite"@de, "Веб-сайт"@uk ; + skos:topConceptOf . + + + a skos:Concept ; + skos:prefLabel "Arbeitsmaterial"@de, "Worksheet"@en, "Робочий матеріал"@uk ; + skos:scopeNote "zum Beispiel Arbeitsblatt"@de, "наприклад робочий аркуш"@uk ; + skos:topConceptOf . + diff --git a/metafix/src/test/resources/org/metafacture/metafix/maps/rpb.ttl b/metafix/src/test/resources/org/metafacture/metafix/maps/rpb.ttl new file mode 100644 index 000000000..129240c6e --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/maps/rpb.ttl @@ -0,0 +1,6819 @@ +@prefix rdf: . +@prefix dct: . +@prefix skos: . +@prefix vann: . + + + a skos:ConceptScheme ; + dct:title "Systematik der Rheinland-Pfälzischen Bibliographie"@de , "Classification scheme for the bibliography of Rhineland-Palatinate"@en ; + dct:license ; + dct:description "This classification was created for use in the bibliography of . The transformation to SKOS was carried out by Felix Ostrowski for the hbz." ; + dct:issued "2014-01-28" ; + dct:publisher ; + vann:preferredNamespaceUri "http://purl.org/lobid/rpb#" ; + vann:preferredNamespacePrefix "rpb" ; + skos:hasTopConcept , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , . + + + a skos:Concept ; + skos:narrower , , , , , ; + skos:notation "rpb100000" ; + skos:prefLabel "Landeskunde"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb100100" ; + skos:prefLabel "Landeskunde allgemein"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb101000" ; + skos:prefLabel "Bibliografie"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , ; + skos:notation "rpb102000" ; + skos:prefLabel "Landesbeschreibung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb102040" ; + skos:prefLabel "Kreisbeschreibung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb102042" ; + skos:prefLabel "Verbandsgemeindebeschreibung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb102045" ; + skos:prefLabel "Regionenbeschreibung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb102050" ; + skos:prefLabel "Ortsbeschreibung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb102060" ; + skos:prefLabel "Reisebericht"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb102070" ; + skos:prefLabel "Wandern / Führer"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb105000" ; + skos:prefLabel "Heimatpflege"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb106000" ; + skos:prefLabel "Heimatverein"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb109000" ; + skos:prefLabel "Biografie"@de . + + + a skos:Concept ; + skos:narrower , , , ; + skos:notation "rpb120000" ; + skos:prefLabel "Kartografie. Geodäsie"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb120100" ; + skos:prefLabel "Kartografie. Geodäsie allgemein"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , ; + skos:notation "rpb122000" ; + skos:prefLabel "Kartografie"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb122030" ; + skos:prefLabel "Kartenauswertung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb122050" ; + skos:prefLabel "Computerkartografie"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb122070" ; + skos:prefLabel "Luftbildauswertung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb124000" ; + skos:prefLabel "Geodäsie"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb126000" ; + skos:prefLabel "Karte"@de . + + + a skos:Concept ; + skos:narrower , , , , , , , , , ; + skos:notation "rpb140000" ; + skos:prefLabel "Geowissenschaften"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb140100" ; + skos:prefLabel "Geowissenschaften allgemein"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , ; + skos:notation "rpb141000" ; + skos:prefLabel "Geophysik"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb141020" ; + skos:prefLabel "Erdmagnetismus"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb141030" ; + skos:prefLabel "Schwere"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb141040" ; + skos:prefLabel "Erdbeben"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , ; + skos:notation "rpb141200" ; + skos:prefLabel "Geologie"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb141220" ; + skos:prefLabel "Tektonik"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb141230" ; + skos:prefLabel "Ingenieurgeologie"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb141240" ; + skos:prefLabel "Stratigraphie"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , ; + skos:notation "rpb141400" ; + skos:prefLabel "Mineralogie. Gesteinskunde"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb141420" ; + skos:prefLabel "Mineralogie"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb141430" ; + skos:prefLabel "Gesteinskunde"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb141440" ; + skos:prefLabel "Geochemie"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb141450" ; + skos:prefLabel "Lagerstättenkunde"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb141460" ; + skos:prefLabel "Mineralquelle. Thermalquelle"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , ; + skos:notation "rpb141600" ; + skos:prefLabel "Paläontologie"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , ; + skos:notation "rpb141620" ; + skos:prefLabel "Paläobotanik"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb141622" ; + skos:prefLabel "Fossile Sporenpflanzen"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb141624" ; + skos:prefLabel "Fossile Samenpflanzen"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , ; + skos:notation "rpb141630" ; + skos:prefLabel "Paläozoologie"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb141632" ; + skos:prefLabel "Fossile Wirbellose"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb141634" ; + skos:prefLabel "Fossile Wirbeltiere"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb141640" ; + skos:prefLabel "Mikropaläontologie"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , ; + skos:notation "rpb142000" ; + skos:prefLabel "Bodenkunde"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb142020" ; + skos:prefLabel "Bodenentwicklung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb142030" ; + skos:prefLabel "Bodenphysik"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb142040" ; + skos:prefLabel "Bodenbiologie"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb142050" ; + skos:prefLabel "Bodenmechanik"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb142060" ; + skos:prefLabel "Bodenchemie"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , , , ; + skos:notation "rpb142100" ; + skos:prefLabel "Geomorphologie"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb142110" ; + skos:prefLabel "Relief / Geografie"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb142120" ; + skos:prefLabel "Abtragung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb142130" ; + skos:prefLabel "Klimamorphologie"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb142140" ; + skos:prefLabel "Glazialmorphologie. Periglazialgeomorphologie"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb142150" ; + skos:prefLabel "Vulkanismus"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb142170" ; + skos:prefLabel "Karst"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb142180" ; + skos:prefLabel "Höhle"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb142190" ; + skos:prefLabel "Angewandte Geomorphologie"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , , , , ; + skos:notation "rpb142300" ; + skos:prefLabel "Wasser"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb142310" ; + skos:prefLabel "Wasserrecht"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , ; + skos:notation "rpb142320" ; + skos:prefLabel "Fließgewässer"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb142323" ; + skos:prefLabel "Hochwasser"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb142325" ; + skos:prefLabel "Niedrigwasser"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb142330" ; + skos:prefLabel "See"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb142340" ; + skos:prefLabel "Moor"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower ; + skos:notation "rpb142350" ; + skos:prefLabel "Hydrogeologie"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb142352" ; + skos:prefLabel "Bodenwasser"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb142360" ; + skos:prefLabel "Wasserhaushalt"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , ; + skos:notation "rpb142370" ; + skos:prefLabel "Wasserwirtschaft"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb142372" ; + skos:prefLabel "Trinkwasserversorgung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb142375" ; + skos:prefLabel "Brauchwasser"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower ; + skos:notation "rpb142380" ; + skos:prefLabel "Hydroökologie"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb142382" ; + skos:prefLabel "Wasseranalyse"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb142390" ; + skos:prefLabel "Wasserstatistik"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , , ; + skos:notation "rpb142500" ; + skos:prefLabel "Klima"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , ; + skos:notation "rpb142520" ; + skos:prefLabel "Klimaelement"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb142522" ; + skos:prefLabel "Sonnenstrahlung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb142523" ; + skos:prefLabel "Lufttemperatur"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb142524" ; + skos:prefLabel "Luftfeuchtigkeit"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb142525" ; + skos:prefLabel "Luftdruck"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb142526" ; + skos:prefLabel "Luftbewegung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb142527" ; + skos:prefLabel "Luftelektrizität"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb142530" ; + skos:prefLabel "Klimatologie"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb142540" ; + skos:prefLabel "Wetterdienst"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb142541" ; + skos:prefLabel "Wetterlage"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb142550" ; + skos:prefLabel "Klimafaktor"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb142560" ; + skos:prefLabel "Klimaschwankung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb142570" ; + skos:prefLabel "Paläoklimatologie"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb146000" ; + skos:prefLabel "Geoökologie"@de . + + + a skos:Concept ; + skos:narrower , , , , ; + skos:notation "rpb160000" ; + skos:prefLabel "Biowissenschaften"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb160100" ; + skos:prefLabel "Biowissenschaften allgemein"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , ; + skos:notation "rpb161000" ; + skos:prefLabel "Biologie"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb161030" ; + skos:prefLabel "Ökologie"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb161040" ; + skos:prefLabel "Biozönose"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb161050" ; + skos:prefLabel "Mikrobiologie"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , ; + skos:notation "rpb162000" ; + skos:prefLabel "Pflanzen"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb162030" ; + skos:prefLabel "Kryptogamen"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb162040" ; + skos:prefLabel "Samenpflanzen"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , , , ; + skos:notation "rpb163000" ; + skos:prefLabel "Tiere"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb163020" ; + skos:prefLabel "Wirbellose"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb163030" ; + skos:prefLabel "Insekten"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb163040" ; + skos:prefLabel "Wirbeltiere"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb163050" ; + skos:prefLabel "Fische"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb163060" ; + skos:prefLabel "Lurche"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb163070" ; + skos:prefLabel "Reptilien"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb163080" ; + skos:prefLabel "Vögel"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb163090" ; + skos:prefLabel "Säugetiere"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , ; + skos:notation "rpb164000" ; + skos:prefLabel "Humanbiologie"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb164020" ; + skos:prefLabel "Menschenrasse"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb164030" ; + skos:prefLabel "Skelettfund"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb164040" ; + skos:prefLabel "Paläoanthropologie"@de . + + + a skos:Concept ; + skos:narrower , , , , , , , , , , ; + skos:notation "rpb200000" ; + skos:prefLabel "Historische Hilfswissenschaften"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb200100" ; + skos:prefLabel "Historische Hilfswissenschaften allgemein"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb202000" ; + skos:prefLabel "Chronologie"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb202500" ; + skos:prefLabel "Metrologie"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb203000" ; + skos:prefLabel "Urkundenlehre"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb203500" ; + skos:prefLabel "Paläografie"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb204000" ; + skos:prefLabel "Siegelkunde"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , ; + skos:notation "rpb205000" ; + skos:prefLabel "Numismatik"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb205020" ; + skos:prefLabel "Münze"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb205030" ; + skos:prefLabel "Papiergeld"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb206000" ; + skos:prefLabel "Epigraphik"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower ; + skos:notation "rpb207000" ; + skos:prefLabel "Genealogie"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb207020" ; + skos:prefLabel "Familie"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower ; + skos:notation "rpb208000" ; + skos:prefLabel "Heraldik"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb208500" ; + skos:prefLabel "Flagge"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb209000" ; + skos:prefLabel "Orden / Ehrenzeichen"@de . + + + a skos:Concept ; + skos:narrower , , ; + skos:notation "rpb210000" ; + skos:prefLabel "Archiv. Museum"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb210100" ; + skos:prefLabel "Archiv. Museum allgemein"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , , , , , ; + skos:notation "rpb213000" ; + skos:prefLabel "Archiv"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb213010" ; + skos:prefLabel "Bundesarchiv Koblenz"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb213020" ; + skos:prefLabel "Staatsarchiv"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , ; + skos:notation "rpb213030" ; + skos:prefLabel "Kreisarchiv. Gemeindearchiv"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb213031" ; + skos:prefLabel "Kreisarchiv"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb213033" ; + skos:prefLabel "Gemeindearchiv"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb213040" ; + skos:prefLabel "Kirchenarchiv"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , ; + skos:notation "rpb213050" ; + skos:prefLabel "Archiv für Literatur, Kunst und Wissenschaft"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb213052" ; + skos:prefLabel "Literaturarchiv"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb213053" ; + skos:prefLabel "Kunstarchiv"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb213054" ; + skos:prefLabel "Wissenschaftsarchiv"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb213060" ; + skos:prefLabel "Parteiarchiv"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , ; + skos:notation "rpb213070" ; + skos:prefLabel "Medienarchiv"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb213072" ; + skos:prefLabel "Rundfunkarchiv"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb213073" ; + skos:prefLabel "Pressearchiv"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb213074" ; + skos:prefLabel "Filmarchiv"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower ; + skos:notation "rpb213080" ; + skos:prefLabel "Wirtschaftsarchiv"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb213082" ; + skos:prefLabel "Firmenarchiv"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb213090" ; + skos:prefLabel "Familienarchiv"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb213092" ; + skos:prefLabel "Sonstige Archive"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , ; + skos:notation "rpb217000" ; + skos:prefLabel "Museum"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb217010" ; + skos:prefLabel "Museumspädagogik"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb217020" ; + skos:prefLabel "Naturkundemuseum"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb217030" ; + skos:prefLabel "Technisches Museum"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb217040" ; + skos:prefLabel "Historisches Museum. Heimatmuseum"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb217050" ; + skos:prefLabel "Volkskundemuseum"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb217090" ; + skos:prefLabel "Sonstige Museen"@de . + + + a skos:Concept ; + skos:narrower , , , , , , , , , ; + skos:notation "rpb220000" ; + skos:prefLabel "Vor- und Frühgeschichte. Archäologie"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb220100" ; + skos:prefLabel "Vor- und Frühgeschichte. Archäologie allgemein"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb220500" ; + skos:prefLabel "Archäologie"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb221000" ; + skos:prefLabel "Vor- und Frühgeschichte"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb222000" ; + skos:prefLabel "Paläolithikum. Mesolithikum"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , ; + skos:notation "rpb223000" ; + skos:prefLabel "Neolithikum"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb223020" ; + skos:prefLabel "Bandkeramik"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb223030" ; + skos:prefLabel "Rössener Kultur"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb223040" ; + skos:prefLabel "Michelsberger Kultur"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb223050" ; + skos:prefLabel "Megalithkultur"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb223060" ; + skos:prefLabel "Schnurkeramik"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb223070" ; + skos:prefLabel "Glockenbecherkultur"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , ; + skos:notation "rpb224000" ; + skos:prefLabel "Bronzezeit"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb224050" ; + skos:prefLabel "Hügelgräberkultur"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb224060" ; + skos:prefLabel "Urnenfelderkultur"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , ; + skos:notation "rpb225000" ; + skos:prefLabel "Vorrömische Eisenzeit"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb225020" ; + skos:prefLabel "Hallstattkultur"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb225025" ; + skos:prefLabel "Hunsrück-Eifel-Kultur"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb225030" ; + skos:prefLabel "Latène-Zeit"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb225040" ; + skos:prefLabel "Kelten"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb225050" ; + skos:prefLabel "Germanen"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , ; + skos:notation "rpb226000" ; + skos:prefLabel "Römerzeit"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb226020" ; + skos:prefLabel "Kelten"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb226030" ; + skos:prefLabel "Römer"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb226040" ; + skos:prefLabel "Germanen"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb226050" ; + skos:prefLabel "Franken "@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , ; + skos:notation "rpb227000" ; + skos:prefLabel "Völkerwanderung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb227020" ; + skos:prefLabel "Römer"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb227030" ; + skos:prefLabel "Germanen"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb227040" ; + skos:prefLabel "Franken "@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , ; + skos:notation "rpb228000" ; + skos:prefLabel "Mittelalterliche Archäologie. Neuzeitliche Archäologie"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb228020" ; + skos:prefLabel "Bestattung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb228030" ; + skos:prefLabel "Funde"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb228080" ; + skos:prefLabel "Industriearchäologie"@de . + + + a skos:Concept ; + skos:narrower , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , ; + skos:notation "rpb240000" ; + skos:prefLabel "Geschichte"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb240100" ; + skos:prefLabel "Quelle"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb240200" ; + skos:prefLabel "Geschichtsschreibung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb240300" ; + skos:prefLabel "Regionalgeschichte"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb240350" ; + skos:prefLabel "Verbandsgemeindegeschichte"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb240400" ; + skos:prefLabel "Ortsgeschichte"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb240500" ; + skos:prefLabel "Historische Ausstellung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb240600" ; + skos:prefLabel "Gedenkstätte"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb241050" ; + skos:prefLabel "Rheinland-Pfalz (gesamt) / Geschichte Anfänge-1945"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb241100" ; + skos:prefLabel "Rheinland / Geschichte Anfänge-1945"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb241200" ; + skos:prefLabel "Pfalz / Geschichte Anfänge-1945"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb241400" ; + skos:prefLabel "Rheinhessen / Geschichte Anfänge-1945"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb242100" ; + skos:prefLabel "Kurpfalz"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb242200" ; + skos:prefLabel "Hochstift Trier"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb242300" ; + skos:prefLabel "Hochstift Mainz"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb242500" ; + skos:prefLabel "Hochstift Speyer"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb242600" ; + skos:prefLabel "Hochstift Worms"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , ; + skos:notation "rpb242700" ; + skos:prefLabel "Reichsterritorium. Reichsbeziehung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb242720" ; + skos:prefLabel "Reichstag"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb242730" ; + skos:prefLabel "Reichsrecht"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , ; + skos:notation "rpb242740" ; + skos:prefLabel "Reichsgut"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb242743" ; + skos:prefLabel "Königshof"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb242744" ; + skos:prefLabel "Königspfalz"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb242746" ; + skos:prefLabel "Reichsabtei"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb242747" ; + skos:prefLabel "Reichsstadt"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb242760" ; + skos:prefLabel "Reichsritterschaft"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , , , , , , , , , , , , , ; + skos:notation "rpb242800" ; + skos:prefLabel "Kleinere Territorien und Teile auswärtiger Territorien"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb242810" ; + skos:prefLabel "Luxemburg"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb242812" ; + skos:prefLabel "Hochstift Köln"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb242814" ; + skos:prefLabel "Grafschaft Homburg, Saar"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb242816" ; + skos:prefLabel "Grafschaft Saarbrücken"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb242818" ; + skos:prefLabel "Staat Leiningen"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb242830" ; + skos:prefLabel "Grafschaft Virneburg"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb242832" ; + skos:prefLabel "Grafschaft Manderscheid"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb242834" ; + skos:prefLabel "Herzogtum Arenberg"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb242836" ; + skos:prefLabel "Fürstentum Löwenstein-Wertheim"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb242850" ; + skos:prefLabel "Grafschaft Wied"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb242852" ; + skos:prefLabel "Fürstentum Isenburg. Grafschaft Isenburg"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb242854" ; + skos:prefLabel "Grafschaft Sayn"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb242856" ; + skos:prefLabel "Grafschaft Katzenelnbogen"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb242870" ; + skos:prefLabel "Grafschaft Veldenz"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb242872" ; + skos:prefLabel "Grafschaft Sponheim"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb242874" ; + skos:prefLabel "Wild- und Rheingrafen"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb242876" ; + skos:prefLabel "Pfalz-Simmern"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb242878" ; + skos:prefLabel "Hanau-Lichtenberg"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb243100" ; + skos:prefLabel "Pfalz-Zweibrücken"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb243400" ; + skos:prefLabel "Staat Nassau"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , ; + skos:notation "rpb245100" ; + skos:prefLabel "Französische Besetzung / 1792-1815"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb245120" ; + skos:prefLabel "Mainzer Republik"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb245130" ; + skos:prefLabel "Wälderdepartement"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb245140" ; + skos:prefLabel "Rhein-Mosel-Département"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb245160" ; + skos:prefLabel "Département Donnersberg"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb245180" ; + skos:prefLabel "Saardepartement"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb245300" ; + skos:prefLabel "Rheinbund / 1806-1813"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb246100" ; + skos:prefLabel "Rheinprovinz"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb246200" ; + skos:prefLabel "Hessen-Darmstadt"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb246300" ; + skos:prefLabel "Hessen-Nassau"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb246500" ; + skos:prefLabel "Bayerische Pfalz"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , ; + skos:notation "rpb246700" ; + skos:prefLabel "Kleinere Territorien des 19. Jahrhunderts"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb246720" ; + skos:prefLabel "Fürstentum Birkenfeld"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb246740" ; + skos:prefLabel "Fürstentum Lichtenberg"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb246760" ; + skos:prefLabel "Oberamt Meisenheim"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb248000" ; + skos:prefLabel "Rheinland-Pfalz / Geschichte 1945-1947"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb248100" ; + skos:prefLabel "Rheinland / Geschichte 1945-"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb248300" ; + skos:prefLabel "Rheinhessen / Geschichte 1945-"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb248500" ; + skos:prefLabel "Pfalz / Geschichte 1945-"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb249100" ; + skos:prefLabel "Rheinland-Pfalz / Geschichte 1947-"@de . + + + a skos:Concept ; + skos:narrower , , ; + skos:notation "rpb260000" ; + skos:prefLabel "Militär- und Wehrwesen"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb260100" ; + skos:prefLabel "Militär- und Wehrwesen allgemein"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , ; + skos:notation "rpb262000" ; + skos:prefLabel "Militär"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , ; + skos:notation "rpb262010" ; + skos:prefLabel "Militärbau"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb262011" ; + skos:prefLabel "Burg"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb262012" ; + skos:prefLabel "Festung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb262014" ; + skos:prefLabel "Schanze"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb262016" ; + skos:prefLabel "Standort"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , ; + skos:notation "rpb262020" ; + skos:prefLabel "Militär / Ausrüstung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb262022" ; + skos:prefLabel "Kriegswaffe"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb262023" ; + skos:prefLabel "Rüstung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb262024" ; + skos:prefLabel "Uniform"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb262025" ; + skos:prefLabel "Feldzeichen"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb262026" ; + skos:prefLabel "Orden / Ehrenzeichen"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , , ; + skos:notation "rpb263000" ; + skos:prefLabel "Wehrwesen"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb263010" ; + skos:prefLabel "Bürgerwehr"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb263020" ; + skos:prefLabel "Militär / Geschichte Anfänge-1918"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb263030" ; + skos:prefLabel "Deutschland / Reichswehr"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb263040" ; + skos:prefLabel "Deutschland / Wehrmacht"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb263050" ; + skos:prefLabel "Ausländisches Militär"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb263060" ; + skos:prefLabel "Deutschland / Bundeswehr"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb263090" ; + skos:prefLabel "Soldat"@de . + + + a skos:Concept ; + skos:narrower , , , , , ; + skos:notation "rpb400000" ; + skos:prefLabel "Staat. Politik"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb400100" ; + skos:prefLabel "Staat. Politik allgemein"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , ; + skos:notation "rpb402000" ; + skos:prefLabel "Staatsgrundlage"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb402020" ; + skos:prefLabel "Staatsrecht"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb402030" ; + skos:prefLabel "Staatsangehörigkeit"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb402040" ; + skos:prefLabel "Staatsgebiet"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb402050" ; + skos:prefLabel "Hoheitsrecht"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb402060" ; + skos:prefLabel "Staatsform"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb402070" ; + skos:prefLabel "Föderalismus"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , ; + skos:notation "rpb402300" ; + skos:prefLabel "Verfassung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb402330" ; + skos:prefLabel "Verfassung / Geschichte"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower ; + skos:notation "rpb402340" ; + skos:prefLabel "Grundrecht"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb402344" ; + skos:prefLabel "Datenschutz"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb402350" ; + skos:prefLabel "Gewaltenteilung"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , ; + skos:notation "rpb404000" ; + skos:prefLabel "Politisches System"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb404030" ; + skos:prefLabel "Regierungsbildung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb404040" ; + skos:prefLabel "Regierungschef"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb404050" ; + skos:prefLabel "Regierung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb404060" ; + skos:prefLabel "Ministerium"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb404070" ; + skos:prefLabel "Regierungspolitik"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb404090" ; + skos:prefLabel "Landespartnerschaft"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , ; + skos:notation "rpb404300" ; + skos:prefLabel "Parlament"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb404320" ; + skos:prefLabel "Parlament / Geschichte"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb404340" ; + skos:prefLabel "Regierungspartei"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb404350" ; + skos:prefLabel "Opposition"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , ; + skos:notation "rpb404360" ; + skos:prefLabel "Parlamentsorganisation"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb404361" ; + skos:prefLabel "Parlamentsausschuss"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb404365" ; + skos:prefLabel "Ombudsmann"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb404380" ; + skos:prefLabel "Abgeordneter"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , , , ; + skos:notation "rpb406000" ; + skos:prefLabel "Politische Willensbildung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb406020" ; + skos:prefLabel "Öffentlichkeitsarbeit"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb406030" ; + skos:prefLabel "Politische Bildung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb406040" ; + skos:prefLabel "Öffentliche Meinung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb406060" ; + skos:prefLabel "Politische Bewegung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb406080" ; + skos:prefLabel "Parteistiftung"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , ; + skos:notation "rpb406100" ; + skos:prefLabel "Partei. Politikerin. Politiker"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb406120" ; + skos:prefLabel "Partei"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb406130" ; + skos:prefLabel "Politikerin. Politiker"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , ; + skos:notation "rpb406200" ; + skos:prefLabel "Politische Gruppe"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb406230" ; + skos:prefLabel "Politischer Verein"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb406250" ; + skos:prefLabel "Bürgerinitiative"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb406270" ; + skos:prefLabel "Projektgruppe"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , , , , ; + skos:notation "rpb406300" ; + skos:prefLabel "Wahl"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb406310" ; + skos:prefLabel "Wahlrecht"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb406320" ; + skos:prefLabel "Wahlkreiseinteilung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb406330" ; + skos:prefLabel "Volksabstimmung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb406340" ; + skos:prefLabel "Europawahl"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb406350" ; + skos:prefLabel "Bundestagswahl"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb406360" ; + skos:prefLabel "Landtagswahl"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb406370" ; + skos:prefLabel "Kreistag / Wahl"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb406380" ; + skos:prefLabel "Bezirkstag / Wahl"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb406390" ; + skos:prefLabel "Kommunalwahl"@de . + + + a skos:Concept ; + skos:narrower , , , , , , , , , ; + skos:notation "rpb420000" ; + skos:prefLabel "Verwaltung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb420100" ; + skos:prefLabel "Verwaltung allgemein"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb421000" ; + skos:prefLabel "Verwaltungsrecht"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb421400" ; + skos:prefLabel "Verwaltungskontrolle"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb422000" ; + skos:prefLabel "Verwaltung / Geschichte"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , ; + skos:notation "rpb423000" ; + skos:prefLabel "Allgemeine Verwaltung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb423020" ; + skos:prefLabel "Verwaltung / Struktur"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb423030" ; + skos:prefLabel "Behörde / Organisation"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb423050" ; + skos:prefLabel "Behörde / Datenverarbeitung"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , ; + skos:notation "rpb423400" ; + skos:prefLabel "Verwaltungsreform"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb423420" ; + skos:prefLabel "Länderneugliederung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb423430" ; + skos:prefLabel "Gebietsreform"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb423440" ; + skos:prefLabel "Funktionalreform"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , ; + skos:notation "rpb423600" ; + skos:prefLabel "Öffentlicher Dienst"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb423610" ; + skos:prefLabel "Dienstrecht"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb423620" ; + skos:prefLabel "Personalwesen"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb423640" ; + skos:prefLabel "Beamter"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , ; + skos:notation "rpb424000" ; + skos:prefLabel "Öffentlicher Haushalt"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , ; + skos:notation "rpb424020" ; + skos:prefLabel "Finanzverwaltung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb424021" ; + skos:prefLabel "Finanzamt"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb424022" ; + skos:prefLabel "Finanzausgleich"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb424024" ; + skos:prefLabel "Finanzreform"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb424026" ; + skos:prefLabel "Öffentliche Beschaffung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb424028" ; + skos:prefLabel "Rechnungshof"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb424030" ; + skos:prefLabel "Abgabe"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower ; + skos:notation "rpb424040" ; + skos:prefLabel "Steuer"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb424042" ; + skos:prefLabel "Steuerberatung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb424050" ; + skos:prefLabel "Gebühr"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb424060" ; + skos:prefLabel "Zoll"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , ; + skos:notation "rpb425000" ; + skos:prefLabel "Bezirksverwaltung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb425020" ; + skos:prefLabel "Regierungspräsident"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb425030" ; + skos:prefLabel "Regierungsbezirk Koblenz"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb425040" ; + skos:prefLabel "Regierungsbezirk Trier"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower ; + skos:notation "rpb425050" ; + skos:prefLabel "Regierungsbezirk Rheinhessen-Pfalz"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb425052" ; + skos:prefLabel "Bezirksverband Pfalz"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , ; + skos:notation "rpb425200" ; + skos:prefLabel "Kreisverwaltung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb425220" ; + skos:prefLabel "Kreisrecht"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb425230" ; + skos:prefLabel "Kreisparlament"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb425290" ; + skos:prefLabel "Kreispartnerschaft"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , , ; + skos:notation "rpb426000" ; + skos:prefLabel "Gemeindeverwaltung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb426020" ; + skos:prefLabel "Ortsrecht"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb426030" ; + skos:prefLabel "Gemeinderat"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb426040" ; + skos:prefLabel "Kommunale Selbstverwaltung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb426050" ; + skos:prefLabel "Auftragsverwaltung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb426060" ; + skos:prefLabel "Kommunaler Spitzenverband"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb426070" ; + skos:prefLabel "Verbandsgemeinde"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb426090" ; + skos:prefLabel "Kommunale Partnerschaft"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , ; + skos:notation "rpb427000" ; + skos:prefLabel "Obere Landesbehörde"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb427010" ; + skos:prefLabel "Rheinland-Pfalz / Struktur- und Genehmigungsdirektion Nord"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb427020" ; + skos:prefLabel "Rheinland-Pfalz / Struktur- und Genehmigungsdirektion Süd"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb427030" ; + skos:prefLabel "Rheinland-Pfalz / Aufsichts- und Dienstleistungsdirektion"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb427040" ; + skos:prefLabel "Rheinland-Pfalz / Landesuntersuchungsamt"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , , ; + skos:notation "rpb428000" ; + skos:prefLabel "Sicherheit und Ordnung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb428020" ; + skos:prefLabel "Ordnungsrecht"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , ; + skos:notation "rpb428030" ; + skos:prefLabel "Polizei"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb428032" ; + skos:prefLabel "Polizeiorganisation"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb428033" ; + skos:prefLabel "Polizei / Ausrüstung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb428040" ; + skos:prefLabel "Zivilschutz"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , ; + skos:notation "rpb428050" ; + skos:prefLabel "Staatsschutz"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb428052" ; + skos:prefLabel "Grenzschutz"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb428054" ; + skos:prefLabel "Verfassungsschutz"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb428060" ; + skos:prefLabel "Personenstandswesen"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , ; + skos:notation "rpb428070" ; + skos:prefLabel "Sicherheitsbehörde"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb428072" ; + skos:prefLabel "Technischer Überwachungsverein"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb428074" ; + skos:prefLabel "Feuerwehr"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb428076" ; + skos:prefLabel "Rettungswesen"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb428078" ; + skos:prefLabel "Katastrophe. Unfall"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb428080" ; + skos:prefLabel "Friedhofsordnung"@de . + + + a skos:Concept ; + skos:narrower , , , , , , ; + skos:notation "rpb440000" ; + skos:prefLabel "Recht"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb440100" ; + skos:prefLabel "Recht allgemein"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower ; + skos:notation "rpb442000" ; + skos:prefLabel "Recht / Geschichte"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb442050" ; + skos:prefLabel "Recht / Geschichte / Quelle"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb443000" ; + skos:prefLabel "Privatrecht"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , ; + skos:notation "rpb444000" ; + skos:prefLabel "Strafrecht"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb444030" ; + skos:prefLabel "Strafvollzug"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb444050" ; + skos:prefLabel "Kriminalität"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb444070" ; + skos:prefLabel "Kriminalfall"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , ; + skos:notation "rpb446000" ; + skos:prefLabel "Gerichtsbarkeit"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb446010" ; + skos:prefLabel "Verfassungsgerichtsbarkeit"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , ; + skos:notation "rpb446030" ; + skos:prefLabel "Ordentliche Gerichtsbarkeit"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb446032" ; + skos:prefLabel "Strafgerichtsbarkeit"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb446034" ; + skos:prefLabel "Zivilgerichtsbarkeit"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb446050" ; + skos:prefLabel "Verwaltungsgerichtsbarkeit"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb446060" ; + skos:prefLabel "Finanzgerichtsbarkeit"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb446070" ; + skos:prefLabel "Arbeitsgerichtsbarkeit"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb446080" ; + skos:prefLabel "Sozialgerichtsbarkeit"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb447000" ; + skos:prefLabel "Rechtsprechung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb448000" ; + skos:prefLabel "Rechtspflege"@de . + + + a skos:Concept ; + skos:narrower , , , , , ; + skos:notation "rpb500000" ; + skos:prefLabel "Bevölkerung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb500100" ; + skos:prefLabel "Bevölkerung allgemein"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb502000" ; + skos:prefLabel "Bevölkerung / Geschichte"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , , , ; + skos:notation "rpb503000" ; + skos:prefLabel "Bevölkerungsstruktur"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb503020" ; + skos:prefLabel "Altersstruktur"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb503030" ; + skos:prefLabel "Geschlechtsverhältnis"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb503040" ; + skos:prefLabel "Volkszählung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb503050" ; + skos:prefLabel "Einwohnerverzeichnis"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb503060" ; + skos:prefLabel "Haushaltsgrösse"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb503070" ; + skos:prefLabel "Wohnstandard"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb503080" ; + skos:prefLabel "Einkommensstatistik"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb503090" ; + skos:prefLabel "Denomination / Religion"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , ; + skos:notation "rpb503200" ; + skos:prefLabel "Bevölkerungsentwicklung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb503220" ; + skos:prefLabel "Bevölkerungspolitik"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb503230" ; + skos:prefLabel "Geburt / Statistik"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb503240" ; + skos:prefLabel "Heirat / Statistik"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb503250" ; + skos:prefLabel "Tod / Statistik"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb503260" ; + skos:prefLabel "Bevölkerungsentwicklung / Prognose"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , ; + skos:notation "rpb503400" ; + skos:prefLabel "Migration"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower ; + skos:notation "rpb503420" ; + skos:prefLabel "Binnenwanderung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb503424" ; + skos:prefLabel "Berufspendlerin. Berufspendler"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb503430" ; + skos:prefLabel "Auswanderung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb503440" ; + skos:prefLabel "Einwanderung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb503600" ; + skos:prefLabel "Bevölkerungsdichte"@de . + + + a skos:Concept ; + skos:narrower , , , , , , ; + skos:notation "rpb520000" ; + skos:prefLabel "Sozialwesen"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb520100" ; + skos:prefLabel "Sozialwesen allgemein"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb521000" ; + skos:prefLabel "Sozialpolitik"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb521200" ; + skos:prefLabel "Sozialrecht"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , ; + skos:notation "rpb521400" ; + skos:prefLabel "Sozialversicherung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb521410" ; + skos:prefLabel "Krankenversicherung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb521420" ; + skos:prefLabel "Unfallversicherung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb521430" ; + skos:prefLabel "Rentenversicherung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb521450" ; + skos:prefLabel "Arbeitslosenversicherung"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , ; + skos:notation "rpb522000" ; + skos:prefLabel "Sozialhilfe / Kostenträger"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb522010" ; + skos:prefLabel "Öffentlicher Träger"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb522020" ; + skos:prefLabel "Kirchlicher Träger"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb522030" ; + skos:prefLabel "Privater Träger"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb522050" ; + skos:prefLabel "Karitative Stiftung"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , , , , ; + skos:notation "rpb523000" ; + skos:prefLabel "Sozialhilfe"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb523010" ; + skos:prefLabel "Arbeitslosenunterstützung"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower ; + skos:notation "rpb523030" ; + skos:prefLabel "Fürsorge"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb523035" ; + skos:prefLabel "Obdachlosenhilfe"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb523040" ; + skos:prefLabel "Unfallfürsorge"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb523050" ; + skos:prefLabel "Krankenfürsorge"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb523060" ; + skos:prefLabel "Behindertenhilfe"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb523070" ; + skos:prefLabel "Kriegsopferfürsorge"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb523075" ; + skos:prefLabel "Gefangenenfürsorge"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb523080" ; + skos:prefLabel "Waisenfürsorge"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb523090" ; + skos:prefLabel "Altenhilfe"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , ; + skos:notation "rpb524000" ; + skos:prefLabel "Sozialpädagogik"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb524010" ; + skos:prefLabel "Jugendhilfe"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb524050" ; + skos:prefLabel "Bewährungshilfe"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb524060" ; + skos:prefLabel "Erziehungsberatung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb524070" ; + skos:prefLabel "Familienfürsorge"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb524080" ; + skos:prefLabel "Berufsbetreuung"@de . + + + a skos:Concept ; + skos:narrower , , , , , ; + skos:notation "rpb530000" ; + skos:prefLabel "Gesundheitswesen"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb530100" ; + skos:prefLabel "Gesundheitswesen allgemein"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , ; + skos:notation "rpb532000" ; + skos:prefLabel "Medizin"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb532010" ; + skos:prefLabel "Medizin / Geschichte"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb532030" ; + skos:prefLabel "Medizinische Versorgung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb532050" ; + skos:prefLabel "Ärztin. Arzt. Heilberuf"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb532070" ; + skos:prefLabel "Gesundheitsvorsorge"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb533000" ; + skos:prefLabel "Apotheke"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , ; + skos:notation "rpb534000" ; + skos:prefLabel "Krankenversorgung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb534010" ; + skos:prefLabel "Krankenpflege"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb534020" ; + skos:prefLabel "Krankenpflege / Beruf"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb534030" ; + skos:prefLabel "Krankenhaus"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb534050" ; + skos:prefLabel "Sanatorium"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb534060" ; + skos:prefLabel "Suchtkrankenhilfe"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb534080" ; + skos:prefLabel "Psychiatrie"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb535000" ; + skos:prefLabel "Hygiene"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb537000" ; + skos:prefLabel "Veterinärwesen"@de . + + + a skos:Concept ; + skos:narrower , , , , , , , , , , , , , , , , , , , , , , , ; + skos:notation "rpb540000" ; + skos:prefLabel "Wirtschaft"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb540100" ; + skos:prefLabel "Wirtschaft allgemein"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb541000" ; + skos:prefLabel "Wirtschaftspolitik"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb541500" ; + skos:prefLabel "Wirtschaftsrecht"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb542000" ; + skos:prefLabel "Wirtschaft / Geschichte"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , ; + skos:notation "rpb543000" ; + skos:prefLabel "Wirtschaftsstruktur"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb543010" ; + skos:prefLabel "Wirtschaftsförderung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb543040" ; + skos:prefLabel "Bruttoinlandsprodukt"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb543060" ; + skos:prefLabel "Außenwirtschaft"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb543080" ; + skos:prefLabel "Wirtschaftsstatistik"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb543200" ; + skos:prefLabel "Wirtschaftsverfassung"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , ; + skos:notation "rpb543400" ; + skos:prefLabel "Wirtschaftsverband"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb543430" ; + skos:prefLabel "Industrie- und Handelskammer"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb543440" ; + skos:prefLabel "Verbraucherverband"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb543460" ; + skos:prefLabel "Arbeitgeberverband"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb543480" ; + skos:prefLabel "Gewerkschaft"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , ; + skos:notation "rpb543600" ; + skos:prefLabel "Arbeitsmarkt"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb543620" ; + skos:prefLabel "Unternehmerin. Unternehmer"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb543630" ; + skos:prefLabel "Arbeitnehmerin. Arbeitnehmer"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb543640" ; + skos:prefLabel "Freier Beruf"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb543660" ; + skos:prefLabel "Lohn"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb543680" ; + skos:prefLabel "Arbeitslosigkeit"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb543800" ; + skos:prefLabel "Marketing"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , , ; + skos:notation "rpb544000" ; + skos:prefLabel "Landwirtschaft"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb544010" ; + skos:prefLabel "Agrarpolitik"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb544020" ; + skos:prefLabel "Landwirtschaft / Geschichte"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb544030" ; + skos:prefLabel "Bauernhof / Geschichte"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb544040" ; + skos:prefLabel "Flurform"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , ; + skos:notation "rpb544050" ; + skos:prefLabel "Landwirtschaft / Beruf"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb544052" ; + skos:prefLabel "Bäuerin. Bauer"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb544054" ; + skos:prefLabel "Landfrau"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb544056" ; + skos:prefLabel "Landarbeiter"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb544060" ; + skos:prefLabel "Landwirtschaftliche Betriebslehre"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb544090" ; + skos:prefLabel "Landwirtschaftsgenossenschaft"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , ; + skos:notation "rpb544200" ; + skos:prefLabel "Agrarproduktion"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb544220" ; + skos:prefLabel "Ackerbau"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb544230" ; + skos:prefLabel "Grünlandwirtschaft"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower ; + skos:notation "rpb544240" ; + skos:prefLabel "Gartenbau"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb544245" ; + skos:prefLabel "Baumschule"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb544250" ; + skos:prefLabel "Obstbau"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb544280" ; + skos:prefLabel "Tierzucht"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb544300" ; + skos:prefLabel "Weinbau"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , ; + skos:notation "rpb544310" ; + skos:prefLabel "Weinbau / Geschichte"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , ; + skos:notation "rpb544320" ; + skos:prefLabel "Weinbaugebiet"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb544322" ; + skos:prefLabel "Großlage"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb544325" ; + skos:prefLabel "Weingut"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , ; + skos:notation "rpb544340" ; + skos:prefLabel "Weinherstellung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb544341" ; + skos:prefLabel "Schaumweinherstellung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb544344" ; + skos:prefLabel "Branntweinherstellung"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower ; + skos:notation "rpb544360" ; + skos:prefLabel "Weinhandel"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb544361" ; + skos:prefLabel "Schaumwein / Handel"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , ; + skos:notation "rpb544400" ; + skos:prefLabel "Forstwirtschaft"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb544410" ; + skos:prefLabel "Forstrecht"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb544420" ; + skos:prefLabel "Forstwirtschaft / Geschichte"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb544440" ; + skos:prefLabel "Forstproduktion"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb544450" ; + skos:prefLabel "Baumart"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb544460" ; + skos:prefLabel "Forst"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb544490" ; + skos:prefLabel "Försterin. Förster"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , ; + skos:notation "rpb544600" ; + skos:prefLabel "Jagd. Fischfang"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb544610" ; + skos:prefLabel "Jagd"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb544620" ; + skos:prefLabel "Jagdrecht"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb544640" ; + skos:prefLabel "Wild"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb544650" ; + skos:prefLabel "Fischfang"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb544660" ; + skos:prefLabel "Fischereirecht"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb544670" ; + skos:prefLabel "Fischzucht"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , , , ; + skos:notation "rpb545000" ; + skos:prefLabel "Bergbau"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb545010" ; + skos:prefLabel "Bergrecht"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb545020" ; + skos:prefLabel "Bergbau / Geschichte"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb545030" ; + skos:prefLabel "Kohlenbergbau"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb545040" ; + skos:prefLabel "Gesteinsabbau"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb545050" ; + skos:prefLabel "Erzbergbau"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb545060" ; + skos:prefLabel "Salzbergbau. Kalibergbau"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb545070" ; + skos:prefLabel "Erdöl"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb545080" ; + skos:prefLabel "Erdgas"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , ; + skos:notation "rpb546000" ; + skos:prefLabel "Energiewirtschaft"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , ; + skos:notation "rpb546020" ; + skos:prefLabel "Elektrizitätsversorgung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb546022" ; + skos:prefLabel "Wasserkraftwerk"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb546024" ; + skos:prefLabel "Wärmekraftwerk"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb546026" ; + skos:prefLabel "Kernkraftwerk"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb546028" ; + skos:prefLabel "Alternative Energiequelle"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb546040" ; + skos:prefLabel "Gasversorgung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb546060" ; + skos:prefLabel "Verbundwirtschaft"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , ; + skos:notation "rpb547000" ; + skos:prefLabel "Handwerk"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb547020" ; + skos:prefLabel "Handwerk / Geschichte"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower ; + skos:notation "rpb547030" ; + skos:prefLabel "Handwerk / Beruf"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb547035" ; + skos:prefLabel "Handwerksbetrieb"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , ; + skos:notation "rpb547040" ; + skos:prefLabel "Handwerksorganisation"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb547042" ; + skos:prefLabel "Handwerkskammer"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb547044" ; + skos:prefLabel "Innung"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , ; + skos:notation "rpb547400" ; + skos:prefLabel "Industrie"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb547420" ; + skos:prefLabel "Industrie / Geschichte"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb547440" ; + skos:prefLabel "Industriezweig"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb547460" ; + skos:prefLabel "Industriebetrieb"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb547600" ; + skos:prefLabel "Technik. Technologie"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , , ; + skos:notation "rpb548000" ; + skos:prefLabel "Handel"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb548010" ; + skos:prefLabel "Handelsrecht"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb548020" ; + skos:prefLabel "Handel / Geschichte"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb548030" ; + skos:prefLabel "Handelsform"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower ; + skos:notation "rpb548040" ; + skos:prefLabel "Messe / Wirtschaft"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb548045" ; + skos:prefLabel "Markt"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb548050" ; + skos:prefLabel "Firma"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb548060" ; + skos:prefLabel "Handelsgut"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb548090" ; + skos:prefLabel "Handelsgenossenschaft"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower ; + skos:notation "rpb548200" ; + skos:prefLabel "Dienstleistungssektor"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb548250" ; + skos:prefLabel "Dienstleistungsbetrieb"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb548300" ; + skos:prefLabel "Öffentliches Unternehmen"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb548400" ; + skos:prefLabel "Bank"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb548600" ; + skos:prefLabel "Versicherung"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , ; + skos:notation "rpb548800" ; + skos:prefLabel "Tourismus"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb548820" ; + skos:prefLabel "Kurort"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb548830" ; + skos:prefLabel "Naherholung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb548840" ; + skos:prefLabel "Gastgewerbe"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb548870" ; + skos:prefLabel "Jugendherberge"@de . + + + a skos:Concept ; + skos:narrower , , , , , , , , ; + skos:notation "rpb550000" ; + skos:prefLabel "Verkehr"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb550100" ; + skos:prefLabel "Verkehr allgemein"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb551000" ; + skos:prefLabel "Verkehrsrecht"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb552000" ; + skos:prefLabel "Verkehr / Geschichte"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb553000" ; + skos:prefLabel "Straßenverkehr"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb554000" ; + skos:prefLabel "Öffentlicher Personennahverkehr"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower ; + skos:notation "rpb555000" ; + skos:prefLabel "Eisenbahn"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb555020" ; + skos:prefLabel "Eisenbahn / Geschichte"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb556000" ; + skos:prefLabel "Luftverkehr"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , ; + skos:notation "rpb557000" ; + skos:prefLabel "Schifffahrt"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb557020" ; + skos:prefLabel "Schifffahrt / Geschichte"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb557040" ; + skos:prefLabel "Kanal"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb557060" ; + skos:prefLabel "Hafen"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , ; + skos:notation "rpb558000" ; + skos:prefLabel "Post. Fernmeldewesen"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb558020" ; + skos:prefLabel "Post / Geschichte"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb558040" ; + skos:prefLabel "Briefmarke"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb558050" ; + skos:prefLabel "Fernmeldewesen"@de . + + + a skos:Concept ; + skos:narrower , , , , , , ; + skos:notation "rpb560000" ; + skos:prefLabel "Siedlung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb560100" ; + skos:prefLabel "Siedlung allgemein"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , ; + skos:notation "rpb562000" ; + skos:prefLabel "Siedlung / Geschichte"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb562040" ; + skos:prefLabel "Wüstung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb562060" ; + skos:prefLabel "Allmende"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb562300" ; + skos:prefLabel "Kulturlandschaft"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb562600" ; + skos:prefLabel "Siedlungsraum"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , ; + skos:notation "rpb563000" ; + skos:prefLabel "Siedlungsform"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb563020" ; + skos:prefLabel "Industriestadt"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb563030" ; + skos:prefLabel "Marktzentrum"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb563040" ; + skos:prefLabel "Wohnsiedlung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb563050" ; + skos:prefLabel "Gewerbegebiet"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , ; + skos:notation "rpb564000" ; + skos:prefLabel "Ländliche Siedlung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb564040" ; + skos:prefLabel "Ländliche Siedlungsform"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb564050" ; + skos:prefLabel "Dorf / Topographie"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb564080" ; + skos:prefLabel "Dorferneuerung"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , , ; + skos:notation "rpb566000" ; + skos:prefLabel "Stadt"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb566010" ; + skos:prefLabel "Stadttyp"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb566020" ; + skos:prefLabel "Städtische Siedlungsform"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , ; + skos:notation "rpb566040" ; + skos:prefLabel "Stadtgliederung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb566041" ; + skos:prefLabel "Stadtkern"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb566042" ; + skos:prefLabel "Vorort"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb566043" ; + skos:prefLabel "Geschäftsviertel"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb566045" ; + skos:prefLabel "Stadtteil"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb566046" ; + skos:prefLabel "Stadt / Erholungsgebiet"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , ; + skos:notation "rpb566050" ; + skos:prefLabel "Stadt / Topographie"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb566051" ; + skos:prefLabel "Platz"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb566052" ; + skos:prefLabel "Straße"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb566053" ; + skos:prefLabel "Öffentliches Gebäude"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb566060" ; + skos:prefLabel "Verstädterung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb566070" ; + skos:prefLabel "Ballungsraum"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb566080" ; + skos:prefLabel "Stadt / Umland"@de . + + + a skos:Concept ; + skos:narrower , , ; + skos:notation "rpb570000" ; + skos:prefLabel "Raumordnung und Städtebau"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb570100" ; + skos:prefLabel "Raumordnung und Städtebau allgemein"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , ; + skos:notation "rpb572000" ; + skos:prefLabel "Raumordnung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb572020" ; + skos:prefLabel "Landesplanung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb572030" ; + skos:prefLabel "Regionalplanung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb572040" ; + skos:prefLabel "Kreisplanung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb572050" ; + skos:prefLabel "Landschaftsplanung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb572060" ; + skos:prefLabel "Gemeindeplanung"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , ; + skos:notation "rpb574000" ; + skos:prefLabel "Bauwesen"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb574020" ; + skos:prefLabel "Bau- und Bodenrecht"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb574030" ; + skos:prefLabel "Städtebau"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb574040" ; + skos:prefLabel "Straßenbau"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb574050" ; + skos:prefLabel "Wohnungsbau"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb574060" ; + skos:prefLabel "Wohnungswirtschaft"@de . + + + a skos:Concept ; + skos:narrower , , , ; + skos:notation "rpb580000" ; + skos:prefLabel "Umwelt- und Naturschutz"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb580100" ; + skos:prefLabel "Umwelt- und Naturschutz allgemein"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , , ; + skos:notation "rpb582000" ; + skos:prefLabel "Umweltschutz"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb582010" ; + skos:prefLabel "Klimaschutz"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb582020" ; + skos:prefLabel "Bodenschutz"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb582030" ; + skos:prefLabel "Luftverschmutzung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb582040" ; + skos:prefLabel "Lärmbelastung"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , ; + skos:notation "rpb582050" ; + skos:prefLabel "Gewässerschutz"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb582052" ; + skos:prefLabel "Abwasser"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb582054" ; + skos:prefLabel "Kanalisation"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb582060" ; + skos:prefLabel "Abfallbeseitigung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb582070" ; + skos:prefLabel "Strahlenbelastung"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , , ; + skos:notation "rpb584000" ; + skos:prefLabel "Naturschutz. Landschaftspflege"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb584040" ; + skos:prefLabel "Naturdenkmal"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb584050" ; + skos:prefLabel "Pflanzen / Artenschutz"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb584060" ; + skos:prefLabel "Tierschutz"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb584070" ; + skos:prefLabel "Naturschutzgebiet"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb584075" ; + skos:prefLabel "Nationalpark"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb584080" ; + skos:prefLabel "Naturpark"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb584090" ; + skos:prefLabel "Landschaftsentwicklung"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , ; + skos:notation "rpb586000" ; + skos:prefLabel "Grünanlage"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb586020" ; + skos:prefLabel "Park"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb586030" ; + skos:prefLabel "Botanischer Garten"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb586040" ; + skos:prefLabel "Zoologischer Garten"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb586080" ; + skos:prefLabel "Gartenbauausstellung"@de . + + + a skos:Concept ; + skos:narrower , , , , , , , ; + skos:notation "rpb610000" ; + skos:prefLabel "Kirche"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb610100" ; + skos:prefLabel "Kirche allgemein"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , , , , , ; + skos:notation "rpb611000" ; + skos:prefLabel "Katholische Kirche"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb611010" ; + skos:prefLabel "Kirchengeschichte"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb611020" ; + skos:prefLabel "Kirchenrecht"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb611025" ; + skos:prefLabel "Kirchenverwaltung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb611030" ; + skos:prefLabel "Kirchengemeinde"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , ; + skos:notation "rpb611040" ; + skos:prefLabel "Kirchliches Leben"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb611042" ; + skos:prefLabel "Seelsorge"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb611043" ; + skos:prefLabel "Laienamt"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb611044" ; + skos:prefLabel "Kirchlicher Verein"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb611045" ; + skos:prefLabel "Kirchliche Stiftung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb611050" ; + skos:prefLabel "Geistlicher. Ordensleute"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb611060" ; + skos:prefLabel "Kloster. Stift"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb611070" ; + skos:prefLabel "Orden"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , ; + skos:notation "rpb611080" ; + skos:prefLabel "Gottesdienst"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb611082" ; + skos:prefLabel "Sakrament"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb611083" ; + skos:prefLabel "Kirchenfest"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb611084" ; + skos:prefLabel "Prozession"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb611085" ; + skos:prefLabel "Wallfahrt"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb611090" ; + skos:prefLabel "Heiligenverehrung"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , ; + skos:notation "rpb612000" ; + skos:prefLabel "Reformation"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower ; + skos:notation "rpb612020" ; + skos:prefLabel "Calvinismus"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb612022" ; + skos:prefLabel "Zwinglianer"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower ; + skos:notation "rpb612030" ; + skos:prefLabel "Täufer"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb612032" ; + skos:prefLabel "Mennoniten"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb612040" ; + skos:prefLabel "Schwärmer / Theologie"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb612050" ; + skos:prefLabel "Reformator"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb612070" ; + skos:prefLabel "Glaubensflüchtling"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb612080" ; + skos:prefLabel "Hugenotten"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb612500" ; + skos:prefLabel "Gegenreformation"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , , ; + skos:notation "rpb613000" ; + skos:prefLabel "Evangelische Kirche"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb613010" ; + skos:prefLabel "Kirchengeschichte"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb613020" ; + skos:prefLabel "Kirchenrecht"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb613025" ; + skos:prefLabel "Kirchenverwaltung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb613030" ; + skos:prefLabel "Kirchengemeinde"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , ; + skos:notation "rpb613040" ; + skos:prefLabel "Kirchliches Leben"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb613042" ; + skos:prefLabel "Seelsorge"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb613043" ; + skos:prefLabel "Diakonie"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb613044" ; + skos:prefLabel "Kirchlicher Verein"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb613045" ; + skos:prefLabel "Kirchliche Stiftung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb613050" ; + skos:prefLabel "Pfarrerin. Pfarrer. Geistliche"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , ; + skos:notation "rpb613070" ; + skos:prefLabel "Gottesdienst"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb613072" ; + skos:prefLabel "Sakrament"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb613073" ; + skos:prefLabel "Kirchenfest"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb615000" ; + skos:prefLabel "Sonstige christliche Religionsgemeinschaften"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb616000" ; + skos:prefLabel "Ökumene"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , ; + skos:notation "rpb617000" ; + skos:prefLabel "Bestattung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb617030" ; + skos:prefLabel "Friedhof"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb617050" ; + skos:prefLabel "Grabmal"@de . + + + a skos:Concept ; + skos:narrower , , ; + skos:notation "rpb630000" ; + skos:prefLabel "Juden"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb630100" ; + skos:prefLabel "Juden allgemein"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb631000" ; + skos:prefLabel "Judentum"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower ; + skos:notation "rpb632000" ; + skos:prefLabel "Juden / Geschichte"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb632050" ; + skos:prefLabel "Judenverfolgung"@de . + + + a skos:Concept ; + skos:narrower , ; + skos:notation "rpb650000" ; + skos:prefLabel "Nichtchristliche Religion"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , ; + skos:notation "rpb651000" ; + skos:prefLabel "Nichtchristliche Religion allgemein"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb651020" ; + skos:prefLabel "Islam"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb651030" ; + skos:prefLabel "Buddhismus"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower ; + skos:notation "rpb655000" ; + skos:prefLabel "Weltanschauungsgemeinschaft"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb655030" ; + skos:prefLabel "Freimaurer"@de . + + + a skos:Concept ; + skos:narrower , , , , , , , , , , ; + skos:notation "rpb700000" ; + skos:prefLabel "Volkskunde"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb700100" ; + skos:prefLabel "Volkskunde allgemein"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb702000" ; + skos:prefLabel "Alltag"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , ; + skos:notation "rpb704000" ; + skos:prefLabel "Brauch"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb704020" ; + skos:prefLabel "Brauch / Alltag"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , ; + skos:notation "rpb704040" ; + skos:prefLabel "Brauch / Jahreslauf"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb704042" ; + skos:prefLabel "Winter / Brauch. Weihnachten. Neujahr"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb704043" ; + skos:prefLabel "Karneval"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb704044" ; + skos:prefLabel "Frühling / Brauch. Ostern"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb704045" ; + skos:prefLabel "Mai / Brauch. Pfingsten. Sommer / Brauch"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb704046" ; + skos:prefLabel "Erntedankfest"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb704047" ; + skos:prefLabel "Kirchweih"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , ; + skos:notation "rpb704060" ; + skos:prefLabel "Lebenslauf / Brauch"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb704061" ; + skos:prefLabel "Geburt"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb704063" ; + skos:prefLabel "Liebe / Brauch. Verlobung. Hochzeit"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb704064" ; + skos:prefLabel "Tod / Brauch. Bestattung"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower ; + skos:notation "rpb704080" ; + skos:prefLabel "Beruf / Brauch"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb704082" ; + skos:prefLabel "Handwerk / Brauch"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb704090" ; + skos:prefLabel "Brauchtumspflege / Verein"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , , ; + skos:notation "rpb704200" ; + skos:prefLabel "Volkswissen"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb704220" ; + skos:prefLabel "Volksmedizin"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb704230" ; + skos:prefLabel "Ethnobotanik"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb704240" ; + skos:prefLabel "Volkszoologie"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb704250" ; + skos:prefLabel "Wetter / Bauernregel"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb704260" ; + skos:prefLabel "Astrologie"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb704270" ; + skos:prefLabel "Wahrsagen"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb704280" ; + skos:prefLabel "Alchemie"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb704400" ; + skos:prefLabel "Rechtliche Volkskunde"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower ; + skos:notation "rpb704600" ; + skos:prefLabel "Religiöse Volkskunde"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower ; + skos:notation "rpb704630" ; + skos:prefLabel "Volksfrömmigkeit"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb704634" ; + skos:prefLabel "Gegenstände der Volksfrömmigkeit"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb704700" ; + skos:prefLabel "Volksglaube"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , , ; + skos:notation "rpb706000" ; + skos:prefLabel "Volksliteratur"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb706020" ; + skos:prefLabel "Volksbuch"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb706040" ; + skos:prefLabel "Märchen. Sage. Legende"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower ; + skos:notation "rpb706050" ; + skos:prefLabel "Schwank"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb706052" ; + skos:prefLabel "Witz"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb706060" ; + skos:prefLabel "Sprichwort"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb706070" ; + skos:prefLabel "Inschrift"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb706080" ; + skos:prefLabel "Rätsel"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb706090" ; + skos:prefLabel "Mundartliteratur"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , ; + skos:notation "rpb706200" ; + skos:prefLabel "Volksmusik. Volkstanz"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb706210" ; + skos:prefLabel "Volksmusik"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb706230" ; + skos:prefLabel "Volkslied"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb706240" ; + skos:prefLabel "Volkstanz"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , ; + skos:notation "rpb708000" ; + skos:prefLabel "Sachkulturforschung"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , ; + skos:notation "rpb708020" ; + skos:prefLabel "Hausform"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb708024" ; + skos:prefLabel "Bürgerhaus"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb708026" ; + skos:prefLabel "Bauernhaus"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb708028" ; + skos:prefLabel "Landwirtschaftliches Gebäude"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb708029" ; + skos:prefLabel "Mühle"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb708030" ; + skos:prefLabel "Hausrat"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb708040" ; + skos:prefLabel "Gerät"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb708050" ; + skos:prefLabel "Spielzeug"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb708060" ; + skos:prefLabel "Nahrung"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , , ; + skos:notation "rpb708200" ; + skos:prefLabel "Volkskunst"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb708220" ; + skos:prefLabel "Holzbearbeitung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb708230" ; + skos:prefLabel "Malerei"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb708250" ; + skos:prefLabel "Keramik"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb708260" ; + skos:prefLabel "Steinbearbeitung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb708270" ; + skos:prefLabel "Metallbearbeitung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb708280" ; + skos:prefLabel "Glas"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb708290" ; + skos:prefLabel "Textilien"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , ; + skos:notation "rpb708400" ; + skos:prefLabel "Kleidung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb708420" ; + skos:prefLabel "Kostümkunde"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb708430" ; + skos:prefLabel "Tracht"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb708450" ; + skos:prefLabel "Schmuck"@de . + + + a skos:Concept ; + skos:narrower , , , , ; + skos:notation "rpb720000" ; + skos:prefLabel "Gesellschaft"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb720100" ; + skos:prefLabel "Gesellschaft allgemein"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb722000" ; + skos:prefLabel "Sozialgeschichte"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , ; + skos:notation "rpb724000" ; + skos:prefLabel "Sozialstruktur"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb724020" ; + skos:prefLabel "Kind"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb724030" ; + skos:prefLabel "Jugend"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb724040" ; + skos:prefLabel "Frau"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb724050" ; + skos:prefLabel "Mann"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb724060" ; + skos:prefLabel "Alter"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb724070" ; + skos:prefLabel "Original / Person"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , , , ; + skos:notation "rpb725000" ; + skos:prefLabel "Gruppe"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb725010" ; + skos:prefLabel "Familie"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb725020" ; + skos:prefLabel "Nachbarschaft"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb725030" ; + skos:prefLabel "Dorfgemeinschaft"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb725040" ; + skos:prefLabel "Alternativbewegung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb725050" ; + skos:prefLabel "Interessenverband"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb725060" ; + skos:prefLabel "Randgruppe"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb725070" ; + skos:prefLabel "Ausländerin. Ausländer"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb725080" ; + skos:prefLabel "Behinderter Mensch"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb726000" ; + skos:prefLabel "Sozialer Wandel"@de . + + + a skos:Concept ; + skos:narrower , , , , , , ; + skos:notation "rpb730000" ; + skos:prefLabel "Kultur und Freizeit"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb730100" ; + skos:prefLabel "Kultur und Freizeit allgemein"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb731000" ; + skos:prefLabel "Kulturpolitik"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb732000" ; + skos:prefLabel "Kulturgeschichte"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , ; + skos:notation "rpb733000" ; + skos:prefLabel "Kulturleben"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb733030" ; + skos:prefLabel "Kulturveranstaltung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb733050" ; + skos:prefLabel "Kulturpreis"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb733070" ; + skos:prefLabel "Kulturverein"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb734000" ; + skos:prefLabel "Feier. Fest"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , ; + skos:notation "rpb735000" ; + skos:prefLabel "Freizeit und Erholung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb735020" ; + skos:prefLabel "Freizeiteinrichtung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb735040" ; + skos:prefLabel "Freizeitgestaltung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb735090" ; + skos:prefLabel "Verein"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , ; + skos:notation "rpb736000" ; + skos:prefLabel "Sport und Spiel"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb736030" ; + skos:prefLabel "Sportart"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb736050" ; + skos:prefLabel "Sportverein"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb736080" ; + skos:prefLabel "Sportlerin. Sportler"@de . + + + a skos:Concept ; + skos:narrower , , , , , , ; + skos:notation "rpb740000" ; + skos:prefLabel "Sprache"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb740100" ; + skos:prefLabel "Sprache allgemein"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb742000" ; + skos:prefLabel "Sprache / Geschichte"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb743000" ; + skos:prefLabel "Grammatik"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb744000" ; + skos:prefLabel "Mundart"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb745000" ; + skos:prefLabel "Sprachgeographie"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , ; + skos:notation "rpb746000" ; + skos:prefLabel "Namenkunde"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb746020" ; + skos:prefLabel "Personenname"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , ; + skos:notation "rpb746030" ; + skos:prefLabel "Hausname. Hofname"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb746032" ; + skos:prefLabel "Hausname"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb746034" ; + skos:prefLabel "Hofname"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , ; + skos:notation "rpb746040" ; + skos:prefLabel "Geographischer Name"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb746042" ; + skos:prefLabel "Ortsname"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb746043" ; + skos:prefLabel "Straßenname"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb746044" ; + skos:prefLabel "Flurname"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb746045" ; + skos:prefLabel "Bergname"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb746046" ; + skos:prefLabel "Gewässername"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb746050" ; + skos:prefLabel "Tiername"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb746060" ; + skos:prefLabel "Pflanzenname"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , ; + skos:notation "rpb747000" ; + skos:prefLabel "Soziolinguistik"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb747010" ; + skos:prefLabel "Umgangssprache"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb747020" ; + skos:prefLabel "Geheimsprache"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb747030" ; + skos:prefLabel "Fachsprache"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb747040" ; + skos:prefLabel "Spracherwerb"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb747050" ; + skos:prefLabel "Sprachunterricht"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb747060" ; + skos:prefLabel "Sprachpflege"@de . + + + a skos:Concept ; + skos:narrower , , , , , , ; + skos:notation "rpb760000" ; + skos:prefLabel "Literatur"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb760100" ; + skos:prefLabel "Literatur allgemein"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , ; + skos:notation "rpb761000" ; + skos:prefLabel "Literaturwissenschaft"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb761020" ; + skos:prefLabel "Literaturgeographie"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb761050" ; + skos:prefLabel "Stoff / Literatur. Motiv / Literatur"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb762000" ; + skos:prefLabel "Literatur / Geschichte"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , ; + skos:notation "rpb766000" ; + skos:prefLabel "Literatursoziologie"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb766030" ; + skos:prefLabel "Leserin. Leser"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb766050" ; + skos:prefLabel "Literaturförderung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb766060" ; + skos:prefLabel "Literaturausstellung"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower ; + skos:notation "rpb767000" ; + skos:prefLabel "Literarischer Text"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb767040" ; + skos:prefLabel "Anthologie"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , ; + skos:notation "rpb768000" ; + skos:prefLabel "Schriftstellerin. Schriftsteller"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb768010" ; + skos:prefLabel "Schriftstellerin. Schriftsteller / Primärliteratur"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb768030" ; + skos:prefLabel "Schriftstellerin. Schriftsteller / Sekundärliteratur"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , ; + skos:notation "rpb769000" ; + skos:prefLabel "Literaturgattung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb769020" ; + skos:prefLabel "Lyrik"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb769030" ; + skos:prefLabel "Drama"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb769040" ; + skos:prefLabel "Epik"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb769050" ; + skos:prefLabel "Kurzform"@de . + + + a skos:Concept ; + skos:narrower , , , , , , , ; + skos:notation "rpb780000" ; + skos:prefLabel "Bildung. Erziehung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb780100" ; + skos:prefLabel "Bildung. Erziehung allgemein"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower ; + skos:notation "rpb781000" ; + skos:prefLabel "Bildungspolitik"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb781040" ; + skos:prefLabel "Ausbildungsförderung"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower ; + skos:notation "rpb782000" ; + skos:prefLabel "Erziehung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb782500" ; + skos:prefLabel "Vorschulerziehung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb783000" ; + skos:prefLabel "Schule / Geschichte"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , , , , , , , , , , , , , , , , , ; + skos:notation "rpb784000" ; + skos:prefLabel "Allgemein bildende Schule"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , ; + skos:notation "rpb784010" ; + skos:prefLabel "Schulpolitik"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb784012" ; + skos:prefLabel "Schulreform"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb784014" ; + skos:prefLabel "Schulversuch"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb784016" ; + skos:prefLabel "Privatschule"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb784020" ; + skos:prefLabel "Schulrecht"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb784025" ; + skos:prefLabel "Elternarbeit"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb784030" ; + skos:prefLabel "Schulverwaltung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb784035" ; + skos:prefLabel "Schulbau"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb784040" ; + skos:prefLabel "Schulgliederung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb784041" ; + skos:prefLabel "Grundschule"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb784042" ; + skos:prefLabel "Hauptschule"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb784043" ; + skos:prefLabel "Realschule"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb784033" ; + skos:prefLabel "Realschule Plus"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb784044" ; + skos:prefLabel "Gymnasium"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb784045" ; + skos:prefLabel "Gesamtschule"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb784046" ; + skos:prefLabel "Kollegschule"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb784047" ; + skos:prefLabel "Sonderschule"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb784048" ; + skos:prefLabel "Regionale Schule"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb784049" ; + skos:prefLabel "Duale Oberschule"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , ; + skos:notation "rpb784050" ; + skos:prefLabel "Schulstufe"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb784052" ; + skos:prefLabel "Primarstufe"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb784053" ; + skos:prefLabel "Sekundarstufe 1"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb784054" ; + skos:prefLabel "Sekundarstufe 2"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , ; + skos:notation "rpb784060" ; + skos:prefLabel "Lehrerin. Lehrer"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb784062" ; + skos:prefLabel "Lehrerbildung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb784064" ; + skos:prefLabel "Lehrerfortbildung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb784066" ; + skos:prefLabel "Lehrerin. Lehrer / Besoldung"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , ; + skos:notation "rpb784070" ; + skos:prefLabel "Schülerin. Schüler"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb784072" ; + skos:prefLabel "Schülermitverwaltung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb784074" ; + skos:prefLabel "Schülertransport"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb784076" ; + skos:prefLabel "Schulverpflegung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb784078" ; + skos:prefLabel "Schüleraustausch"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb784080" ; + skos:prefLabel "Unterricht"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb784090" ; + skos:prefLabel "Schulleben"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , , ; + skos:notation "rpb785000" ; + skos:prefLabel "Berufsbildende Schule"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb785020" ; + skos:prefLabel "Berufsschule"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb785030" ; + skos:prefLabel "Berufsfachschule"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb785040" ; + skos:prefLabel "Berufsaufbauschule"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb785050" ; + skos:prefLabel "Fachoberschule"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb785060" ; + skos:prefLabel "Berufliches Gymnasium"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb785070" ; + skos:prefLabel "Fachschule"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb785080" ; + skos:prefLabel "Nichtstaatliche Berufsschule"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb785500" ; + skos:prefLabel "Berufsausbildung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb786000" ; + skos:prefLabel "Außerschulische Bildung"@de . + + + a skos:Concept ; + skos:narrower , , , , , , , , ; + skos:notation "rpb790000" ; + skos:prefLabel "Hochschule. Wissenschaft"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb790100" ; + skos:prefLabel "Hochschule. Wissenschaft allgemein"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb791000" ; + skos:prefLabel "Hochschulrecht"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb792000" ; + skos:prefLabel "Hochschulpolitik"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower ; + skos:notation "rpb794000" ; + skos:prefLabel "Hochschule"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb794010" ; + skos:prefLabel "Einzelne Hochschule"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb796000" ; + skos:prefLabel "Fachhochschule"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb797000" ; + skos:prefLabel "Hochschullehrerin. Hochschullehrer. Wissenschaft"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb798000" ; + skos:prefLabel "Studentin. Student"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb798200" ; + skos:prefLabel "Außeruniversitäre Forschung"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower ; + skos:notation "rpb799000" ; + skos:prefLabel "Wissenschaftsförderung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb799200" ; + skos:prefLabel "Wissenschaftliche Gesellschaft"@de . + + + a skos:Concept ; + skos:narrower , , , , ; + skos:notation "rpb800000" ; + skos:prefLabel "Darstellende Kunst"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb800100" ; + skos:prefLabel "Darstellende Kunst allgemein"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , , ; + skos:notation "rpb802000" ; + skos:prefLabel "Theater"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb802020" ; + skos:prefLabel "Theater / Geschichte"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb802030" ; + skos:prefLabel "Volksschauspiel"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb802040" ; + skos:prefLabel "Laienspiel"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb802050" ; + skos:prefLabel "Kindertheater. Jugendtheater"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb802060" ; + skos:prefLabel "Einzelne Theater"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb802070" ; + skos:prefLabel "Inszenierung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb802080" ; + skos:prefLabel "Schauspielerin. Schauspieler"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb803000" ; + skos:prefLabel "Ballett"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower ; + skos:notation "rpb804000" ; + skos:prefLabel "Film"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb804020" ; + skos:prefLabel "Filmtheater"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower ; + skos:notation "rpb806000" ; + skos:prefLabel "Kleinkunst"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb806020" ; + skos:prefLabel "Kabarett"@de . + + + a skos:Concept ; + skos:narrower , , , , , , , , , , , , ; + skos:notation "rpb820000" ; + skos:prefLabel "Musik"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb820100" ; + skos:prefLabel "Musik allgemein"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb821000" ; + skos:prefLabel "Musikwissenschaft"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb822000" ; + skos:prefLabel "Musikerin. Musiker / Ausbildung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb822400" ; + skos:prefLabel "Musikförderung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb822500" ; + skos:prefLabel "Musikpreis"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb823000" ; + skos:prefLabel "Musik / Geschichte"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , ; + skos:notation "rpb824000" ; + skos:prefLabel "Musikerin. Musiker"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb824020" ; + skos:prefLabel "Instrumentalmusikerin. Instrumentalmusiker"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb824040" ; + skos:prefLabel "Komponistin. Komponist"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb824060" ; + skos:prefLabel "Dirigentin. Dirigent"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb824080" ; + skos:prefLabel "Sängerin. Sänger"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb824500" ; + skos:prefLabel "Chor"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb825000" ; + skos:prefLabel "Orchester"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower ; + skos:notation "rpb826000" ; + skos:prefLabel "Konzert. Musikveranstaltung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb826020" ; + skos:prefLabel "Einzelne Aufführungen"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower ; + skos:notation "rpb827000" ; + skos:prefLabel "Kirchenmusik"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb827010" ; + skos:prefLabel "Kirchenmusik / Aufführung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb828000" ; + skos:prefLabel "Musikinstrument"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb829000" ; + skos:prefLabel "Laienmusik"@de . + + + a skos:Concept ; + skos:narrower , , , , , , , , , , , , ; + skos:notation "rpb840000" ; + skos:prefLabel "Kunst. Architektur"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb840100" ; + skos:prefLabel "Kunst. Architektur allgemein"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , , ; + skos:notation "rpb841000" ; + skos:prefLabel "Kunst"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb841020" ; + skos:prefLabel "Kunststudium"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb841040" ; + skos:prefLabel "Kunstmuseum. Kunstsammlung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb841050" ; + skos:prefLabel "Kunstgalerie"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb841060" ; + skos:prefLabel "Kunstausstellung"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , ; + skos:notation "rpb841070" ; + skos:prefLabel "Künstlerin.Künstler"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb841072" ; + skos:prefLabel "Bildhauerin. Bildhauer"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb841074" ; + skos:prefLabel "Malerin. Maler"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb841076" ; + skos:prefLabel "Zeichnerin. Zeichner"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb841078" ; + skos:prefLabel "Grafikerin. Grafiker"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb841079" ; + skos:prefLabel "Fotografin. Fotograf"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb841080" ; + skos:prefLabel "Künstlervereinigung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb841090" ; + skos:prefLabel "Kunstförderung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb842000" ; + skos:prefLabel "Kunst / Geschichte"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , ; + skos:notation "rpb843000" ; + skos:prefLabel "Architektur"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb843010" ; + skos:prefLabel "Baukonstruktion. Bautechnik"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb843020" ; + skos:prefLabel "Öffentliches Gebäude"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , ; + skos:notation "rpb843040" ; + skos:prefLabel "Büro-, Geschäfts- und Industriebauten"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb843042" ; + skos:prefLabel "Bürohaus"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb843044" ; + skos:prefLabel "Geschäftshaus"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb843046" ; + skos:prefLabel "Industriebau"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb843050" ; + skos:prefLabel "Bauernhaus"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb843060" ; + skos:prefLabel "Wohnhaus"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb843090" ; + skos:prefLabel "Architektin. Architekt"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , , , ; + skos:notation "rpb844000" ; + skos:prefLabel "Baudenkmal. Kunstwerk"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb844010" ; + skos:prefLabel "Kunst / Inventar"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb844020" ; + skos:prefLabel "Baustil"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb844030" ; + skos:prefLabel "Sakralbau"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb844040" ; + skos:prefLabel "Burg. Schloss"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb844050" ; + skos:prefLabel "Profanarchitektur"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb844060" ; + skos:prefLabel "Park. Garten"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb844070" ; + skos:prefLabel "Brunnen"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb844080" ; + skos:prefLabel "Denkmal"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb844200" ; + skos:prefLabel "Denkmalpflege. Denkmalschutz"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb844500" ; + skos:prefLabel "Städtebau"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower ; + skos:notation "rpb845000" ; + skos:prefLabel "Plastik"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb845020" ; + skos:prefLabel "Plastik / Einzelne Objekte"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower ; + skos:notation "rpb846000" ; + skos:prefLabel "Malerei"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb846020" ; + skos:prefLabel "Malerei / Einzelne Objekte"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb847000" ; + skos:prefLabel "Zeichnung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb847500" ; + skos:prefLabel "Druckgrafik"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb848000" ; + skos:prefLabel "Fotografie"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , , , , , ; + skos:notation "rpb849000" ; + skos:prefLabel "Kunsthandwerk"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb849020" ; + skos:prefLabel "Goldschmiedekunst. Silberschmiedekunst. Schmuck"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb849030" ; + skos:prefLabel "Eisenguss"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb849032" ; + skos:prefLabel "Bronzeguss"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb849034" ; + skos:prefLabel "Zinnguss"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb849040" ; + skos:prefLabel "Keramik"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , ; + skos:notation "rpb849050" ; + skos:prefLabel "Porzellan. Glas. Fayence"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb849052" ; + skos:prefLabel "Porzellan"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb849054" ; + skos:prefLabel "Fayence"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb849056" ; + skos:prefLabel "Glas"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb849060" ; + skos:prefLabel "Hausrat"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb849070" ; + skos:prefLabel "Möbel"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , ; + skos:notation "rpb849080" ; + skos:prefLabel "Textilien. Teppich. Tapete"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb849082" ; + skos:prefLabel "Textilien"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb849084" ; + skos:prefLabel "Teppich"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb849086" ; + skos:prefLabel "Tapete"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb849090" ; + skos:prefLabel "Uhr. Musikinstrument"@de . + + + a skos:Concept ; + skos:narrower , , ; + skos:notation "rpb860000" ; + skos:prefLabel "Buch. Bibliothek"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb860100" ; + skos:prefLabel "Buch. Bibliothek allgemein"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , ; + skos:notation "rpb861000" ; + skos:prefLabel "Buch"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb861010" ; + skos:prefLabel "Schrift / Geschichte"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb861020" ; + skos:prefLabel "Handschrift"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb861030" ; + skos:prefLabel "Buch / Geschichte"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb861040" ; + skos:prefLabel "Buchdruck"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , ; + skos:notation "rpb861050" ; + skos:prefLabel "Verlag"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb861052" ; + skos:prefLabel "Urheberrecht"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb861054" ; + skos:prefLabel "Verlagsrecht"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb861056" ; + skos:prefLabel "Verlegerin. Verleger"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb861060" ; + skos:prefLabel "Buchhandel"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , , , , , ; + skos:notation "rpb865000" ; + skos:prefLabel "Bibliothek"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb865010" ; + skos:prefLabel "Bibliotheksrecht"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb865020" ; + skos:prefLabel "Bibliothek / Geschichte"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb865030" ; + skos:prefLabel "Bibliotheksplanung"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb865040" ; + skos:prefLabel "Wissenschaftliche Bibliothek"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , ; + skos:notation "rpb865050" ; + skos:prefLabel "Öffentliche Bibliothek"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb865052" ; + skos:prefLabel "Stadtbibliothek"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb865054" ; + skos:prefLabel "Kommunale Bibliothek"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb865060" ; + skos:prefLabel "Spezialbibliothek"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb865070" ; + skos:prefLabel "Schulbibliothek"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb865080" ; + skos:prefLabel "Privatbibliothek"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb865090" ; + skos:prefLabel "Bibliothekarin. Bibliothekar"@de . + + + a skos:Concept ; + skos:narrower , , ; + skos:notation "rpb880000" ; + skos:prefLabel "Publizistik. Information. Dokumentation"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb880100" ; + skos:prefLabel "Publizistik. Information. Dokumentation allgemein"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , , , , ; + skos:notation "rpb882000" ; + skos:prefLabel "Publizistik"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower , ; + skos:notation "rpb882020" ; + skos:prefLabel "Presse"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb882022" ; + skos:prefLabel "Presserecht"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb882026" ; + skos:prefLabel "Zeitung. Zeitschrift"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb882030" ; + skos:prefLabel "Audiovisuelle Medien"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb882040" ; + skos:prefLabel "Hörfunk"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb882060" ; + skos:prefLabel "Fernsehen"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb882090" ; + skos:prefLabel "Journalistin. Journalist"@de . + + + a skos:Concept ; + skos:broader ; + skos:narrower ; + skos:notation "rpb884000" ; + skos:prefLabel "Information und Dokumentation"@de . + + + a skos:Concept ; + skos:broader ; + skos:notation "rpb884020" ; + skos:prefLabel "Internet"@de . diff --git a/metafix/src/test/resources/org/metafacture/metafix/maps/test.csv b/metafix/src/test/resources/org/metafacture/metafix/maps/test.csv new file mode 100644 index 000000000..1c4d3d8ce --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/maps/test.csv @@ -0,0 +1,3 @@ +Aloha,Alohaeha +Moin,Moin zäme +__default,Tach diff --git a/metafix/src/test/resources/org/metafacture/metafix/maps/test.tsv b/metafix/src/test/resources/org/metafacture/metafix/maps/test.tsv new file mode 100644 index 000000000..df72e2ba2 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/maps/test.tsv @@ -0,0 +1,3 @@ +Aloha Alohaeha +Moin Moin zäme +__default Tach diff --git a/metafix/src/test/resources/org/metafacture/metafix/maps/test.ttl b/metafix/src/test/resources/org/metafacture/metafix/maps/test.ttl new file mode 100644 index 000000000..6229e9ea4 --- /dev/null +++ b/metafix/src/test/resources/org/metafacture/metafix/maps/test.ttl @@ -0,0 +1,12 @@ +@base . +@prefix dct: . +@prefix skos: . +@prefix schema: . +@prefix vann: . + + a skos:Concept ; + skos:prefLabel "Mathematik, Naturwissenschaften"@de, "Mathematics, Natural Sciences"@en ; + skos:narrower , , , , , , , ; + skos:notation "4" ; + skos:topConceptOf . + diff --git a/metafix/src/test/resources/simplelogger.properties b/metafix/src/test/resources/simplelogger.properties new file mode 100644 index 000000000..873eb3107 --- /dev/null +++ b/metafix/src/test/resources/simplelogger.properties @@ -0,0 +1,5 @@ +# SLF4J's SimpleLogger configuration file +# +# See https://www.slf4j.org/api/org/slf4j/impl/SimpleLogger.html + +org.slf4j.simpleLogger.defaultLogLevel=warn diff --git a/misc/vim/indent/metafacture-fix.vim b/misc/vim/indent/metafacture-fix.vim new file mode 100644 index 000000000..12f466d30 --- /dev/null +++ b/misc/vim/indent/metafacture-fix.vim @@ -0,0 +1,38 @@ +" Only load this indent file when no other was loaded yet. +if exists("b:did_indent") + finish +endif +let b:did_indent = 1 + +setlocal indentexpr=GetMetafactureFixIndent() +setlocal indentkeys+=0=do,0=if,0=unless,0=elsif,0=else,0=end + +let b:undo_indent = "setlocal indentexpr< indentkeys<" + +if exists("*GetMetafactureFixIndent") + finish +endif + +function! GetMetafactureFixIndent() + let lnum = prevnonblank(v:lnum - 1) + + if lnum == 0 + return 0 + endif + + if synIDattr(synID(v:lnum, 1, 1), "name") =~ '^fixComment*' + return indent(v:lnum) + endif + + let new_indent = indent(lnum) + + if getline(lnum) =~ '^\s*\(\(do\|if\|unless\|elsif\)\>\|else$\)\|($' + let new_indent += shiftwidth() + endif + + if getline(v:lnum) =~ '^\s*\(elsif\>\|\(else\|end\|)\)$\)' + let new_indent -= shiftwidth() + endif + + return new_indent +endfunction diff --git a/misc/vim/syntax/metafacture-fix.vim b/misc/vim/syntax/metafacture-fix.vim new file mode 100644 index 000000000..9113c9c9f --- /dev/null +++ b/misc/vim/syntax/metafacture-fix.vim @@ -0,0 +1,38 @@ +if exists("b:current_syntax") + finish +endif + +syn keyword fixBind bind do doset +syn keyword fixConditional else elsif if unless +syn keyword fixKeyword block end +syn keyword fixOperator and or +syn keyword fixPreProc use +syn keyword fixSelector reject select +syn match fixBareString /\v[^[:space:]\\,;:=>()"'\$*]+/ +syn match fixComment /\v(#|\/\/).*$/ +syn match fixFunction /\v([a-z][_0-9a-zA-Z]*\.)*[a-zA-Z][_0-9a-zA-Z]*\s*\(/me=e-1,he=e-1 +syn match fixOperator /\v(\&\&|\|\|)/ +syn match fixWildcard /\v\$(append|first|last|prepend)>/ +syn match fixWildcard /\v(^\s*)\@<=\*/ +syn region fixCommentRegion start="\v\/\*" skip=/\v\\./ end="\v\*\/" +syn region fixDoubleQuotedString start=/\v"/ skip=/\v\\./ end=/\v"/ +syn region fixIfBlock start="if" end="end" fold transparent +syn region fixSingleQuotedString start=/\v'/ skip=/\v\\./ end=/\v'/ + +syn sync ccomment fixCommentRegion + +hi link fixBareString String +hi link fixBind Keyword +hi link fixComment Comment +hi link fixCommentRegion Comment +hi link fixConditional Keyword +hi link fixDoubleQuotedString String +hi link fixFunction Function +hi link fixKeyword Keyword +hi link fixOperator Operator +hi link fixPreProc PreProc +hi link fixSelector Keyword +hi link fixSingleQuotedString String +hi link fixWildcard Special + +let b:current_syntax = "metafacture-fix" diff --git a/misc/vim/syntax/syntax-sample.fix b/misc/vim/syntax/syntax-sample.fix new file mode 100644 index 000000000..ebe91dcad --- /dev/null +++ b/misc/vim/syntax/syntax-sample.fix @@ -0,0 +1,48 @@ +# vim:ft=metafacture-fix + +# Fix is a macro-language for data transformations + +use "strict" + +# Simple fixes + +add_field(hello, world) # end-of-line comment +remove_field(my.deep.*.junk.$last) +copy_field(stats.*, output.$append) + +# Conditionals + +if exists(error) || bla() + set_field(is_valid, no) + log(error) +elsif exists(warning) && blub() + set_field(is_valid, yes) + log(warning) +else + set_field(is_valid, yes) + en ds +end + +# Loops + +do list(path) + add_field(foo, bar) and foo.bar(key: "val=$1") +end + +# Nested expressions + +do marc_each() + if marc_has(f700) + marc_map(f700a, authors.$append.bla, 'bla') + end +end + +// single-line comment +copy_field(foo, /* in-line comment */ bar) + +/* +multi-line comment +with a twist \*/ +*/ + +reject exists(error.field) diff --git a/settings.gradle b/settings.gradle index 0eb4448f1..851033b7f 100644 --- a/settings.gradle +++ b/settings.gradle @@ -46,6 +46,9 @@ include ':metafacture-linkeddata' include ':metafacture-flux' include ':metafacture-runner' include ':metafacture-yaml' +include ':metafix' +include ':metafix-ide' +include ':metafix-web' include ':metamorph-api' include ':metamorph' include ':metamorph-test'