Skip to content
This repository has been archived by the owner on Jan 27, 2025. It is now read-only.

Commit

Permalink
Add skos lookup (metafacture/metafacture-core#415)
Browse files Browse the repository at this point in the history 
Works like fix function 'lookup', also using a Map. The Map is build dynamically
querying an RDF model.

- set version of Java in github action to 11
  • Loading branch information
@dr0i
dr0i committed Jun 23, 2022
1 parent e35f821 commit c930c58
Show file tree
Hide file tree
Showing 9 changed files with 463 additions and 49 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/build.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,10 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Set up JDK 1.8
- name: Set up JDK 11
uses: actions/setup-java@v1
with:
java-version: 1.8
java-version: 11
- name: Grant execute permission for gradlew
run: chmod +x gradlew
- name: Build with Gradle
Expand Down
9 changes: 9 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -448,6 +448,15 @@ lookup("<sourceField>", "<mapFile>", sep_char: ”,”)
lookup("<sourceField>", "<mapName>")
lookup("<sourceField>", "<mapName>", default: "NA")
```
#### `lookup_rdf`

Looks up matching values in an RDF resource and replaces the field value with this match. A file as well as an HTTP(S) resource can be used.

```perl
lookup_rdf("<sourceField>", "<rdfResource>", target: "<RDF Property>")
lookup_rdf("<sourceField>", "<rdfResource>", target: "<RDF Property>", targetLanguage: "<RDF language tag>")
lookup_rdf("<sourceField>", "<rdfResource>", target: "<RDF Property>", __default: "NA")
```

#### `replace_all`

Expand Down
1 change: 1 addition & 0 deletions build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ subprojects {
'ace': '1.3.3',
'equalsverifier': '3.8.2',
'jetty': '9.4.14.v20181114',
'jena': '4.5.0',
'jquery': '3.3.1-1',
'junit_jupiter': '5.8.2',
'junit_platform': '1.4.2',
Expand Down
2 changes: 2 additions & 0 deletions metafix/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ def passSystemProperties = {
}

dependencies {
implementation "org.apache.jena:jena-core:${versions.jena}"
implementation "org.apache.jena:jena-arq:${versions.jena}"
implementation "org.eclipse.xtext:org.eclipse.xtext:${versions.xtext}"
implementation "org.eclipse.xtext:org.eclipse.xtext.xbase:${versions.xtext}"
implementation "com.google.guava:guava:${versions.guava}"
Expand Down
135 changes: 88 additions & 47 deletions metafix/src/main/java/org/metafacture/metafix/FixMethod.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package org.metafacture.metafix;

import org.metafacture.metafix.api.FixFunction;
import org.metafacture.metafix.maps.RdfMap;
import org.metafacture.metamorph.api.Maps;
import org.metafacture.metamorph.maps.FileMap;

Expand Down Expand Up @@ -77,6 +78,24 @@ public void apply(final Metafix metafix, final Record record, final List<String>
metafix.putMap(params.get(0), options);
}
},
put_rdfmap {
@Override
public void apply(final Metafix metafix, final Record record, final List<String> params, final Map<String, String> options) {
final String fileName = params.get(0);
final RdfMap rdf = new RdfMap();
rdf.setFile(metafix.resolvePath(fileName));
if (options.containsKey("targetLanguage")) {
rdf.setTargetLanguage(options.get("targetLanguage"));
}
if (options.containsKey("target")) {
rdf.setTarget(options.get("target"));
}
if (options.containsKey(Maps.DEFAULT_MAP_KEY)) {
rdf.put(Maps.DEFAULT_MAP_KEY, defaultValue);
}
metafix.putMap(params.size() > 1 ? params.get(1) : fileName, rdf);
}
},
put_var {
@Override
public void apply(final Metafix metafix, final Record record, final List<String> params, final Map<String, String> options) {
Expand All @@ -99,6 +118,7 @@ public void apply(final Metafix metafix, final Record record, final List<String>
}
},
array { // array-from-hash

@Override
public void apply(final Metafix metafix, final Record record, final List<String> params, final Map<String, String> options) {
final String field = params.get(0);
Expand Down Expand Up @@ -136,6 +156,7 @@ public void apply(final Metafix metafix, final Record record, final List<String>
}
},
hash { // hash-from-array

@Override
public void apply(final Metafix metafix, final Record record, final List<String> params, final Map<String, String> options) {
final String field = params.get(0);
Expand Down Expand Up @@ -197,9 +218,9 @@ public void apply(final Metafix metafix, final Record record, final List<String>
public void apply(final Metafix metafix, final Record record, final List<String> params, final Map<String, String> options) {
final String joinChar = options.get("join_char");
final Value newValue = new Value(params.subList(1, params.size()).stream()
.filter(f -> literalString(f) || record.get(f) != null)
.map(f -> literalString(f) ? new Value(f.substring(1)) : Value.asList(record.get(f), null).asArray().get(0))
.map(Value::asString).collect(Collectors.joining(joinChar != null ? joinChar : " ")));
.filter(f -> literalString(f) || record.get(f) != null)
.map(f -> literalString(f) ? new Value(f.substring(1)) : Value.asList(record.get(f), null).asArray().get(0))
.map(Value::asString).collect(Collectors.joining(joinChar != null ? joinChar : " ")));
record.set(params.get(0), newValue);
}

Expand Down Expand Up @@ -236,9 +257,9 @@ public void apply(final Metafix metafix, final Record record, final List<String>
final UnaryOperator<String> operator = s -> s.replaceAll(search, replace);

record.transform(params.get(0), (m, c) -> m
.ifArray(a -> c.accept(renameArray(a, operator)))
.ifHash(h -> c.accept(renameHash(h, operator)))
.orElseThrow()
.ifArray(a -> c.accept(renameArray(a, operator)))
.ifHash(h -> c.accept(renameHash(h, operator)))
.orElseThrow()
);
}

Expand All @@ -252,9 +273,9 @@ private Value renameHash(final Value.Hash hash, final UnaryOperator<String> oper

private Value renameValue(final Value value, final UnaryOperator<String> operator) {
return value.extractType((m, c) -> m
.ifArray(a -> c.accept(renameArray(a, operator)))
.ifHash(h -> c.accept(renameHash(h, operator)))
.orElse(c)
.ifArray(a -> c.accept(renameArray(a, operator)))
.ifHash(h -> c.accept(renameHash(h, operator)))
.orElse(c)
);
}
},
Expand Down Expand Up @@ -314,8 +335,8 @@ public void apply(final Metafix metafix, final Record record, final List<String>
@Override
public void apply(final Metafix metafix, final Record record, final List<String> params, final Map<String, String> options) {
record.transform(params.get(0), (m, c) -> m
.ifArray(a -> c.accept(new Value(a.size())))
.ifHash(h -> c.accept(new Value(h.size())))
.ifArray(a -> c.accept(new Value(a.size())))
.ifHash(h -> c.accept(new Value(h.size())))
);
}
},
Expand All @@ -334,7 +355,7 @@ public void apply(final Metafix metafix, final Record record, final List<String>
final Predicate<Value> predicate = s -> search.matcher(s.asString()).find();

record.transform(params.get(0), (m, c) -> m
.ifArray(a -> c.accept(newArray(a.stream().filter(invert ? predicate.negate() : predicate))))
.ifArray(a -> c.accept(newArray(a.stream().filter(invert ? predicate.negate() : predicate))))
);
}
},
Expand All @@ -350,40 +371,30 @@ public void apply(final Metafix metafix, final Record record, final List<String>
public void apply(final Metafix metafix, final Record record, final List<String> params, final Map<String, String> options) {
final String joinChar = params.size() > 1 ? params.get(1) : "";
record.transform(params.get(0), (m, c) -> m
.ifArray(a -> c.accept(new Value(a.stream().map(Value::asString).collect(Collectors.joining(joinChar)))))
.ifArray(a -> c.accept(new Value(a.stream().map(Value::asString).collect(Collectors.joining(joinChar)))))
);
}
},
lookup {
@Override
public void apply(final Metafix metafix, final Record record, final List<String> params, final Map<String, String> options) {
final Map<String, String> map;

if (params.size() <= 1) {
map = options;
}
else {
final String mapName = params.get(1);

if (!metafix.getMapNames().contains(mapName)) {
if (mapName.contains(".") || mapName.contains(File.separator)) {
put_filemap.apply(metafix, record, Arrays.asList(mapName), options);
}
else {
// Probably an unknown internal map? Log a warning?
}
}

map = metafix.getMap(mapName);
}

final String defaultValue = map.get(Maps.DEFAULT_MAP_KEY); // TODO: Catmandu uses 'default'
final Map<String, String> map = extracted(metafix, record, params, options, KIND_OF_FILEMAP);
record.transform(params.get(0), oldValue -> {
final String newValue = map.getOrDefault(oldValue, defaultValue);
return newValue != null ? newValue : getBoolean(options, "delete") ? null : oldValue;
});
}
},
lookup_rdf {
@Override
public void apply(final Metafix metafix, final Record record, final List<String> params, final Map<String, String> options) {
final Map<String, String> map = extracted(metafix, record, params, options, KIND_OF_RDFMAP);
record.transform(params.get(0), oldValue -> {
final String newValue = map.get(oldValue);
return newValue != null ? newValue : getBoolean(options, "delete") ? null : oldValue;
});
}
},
prepend {
@Override
public void apply(final Metafix metafix, final Record record, final List<String> params, final Map<String, String> options) {
Expand All @@ -404,12 +415,12 @@ public void apply(final Metafix metafix, final Record record, final List<String>
@Override
public void apply(final Metafix metafix, final Record record, final List<String> params, final Map<String, String> options) {
record.transform(params.get(0), (m, c) -> m
.ifArray(a -> {
final List<Value> list = a.stream().collect(Collectors.toList());
Collections.reverse(list);
c.accept(new Value(list));
})
.ifString(s -> c.accept(new Value(new StringBuilder(s).reverse().toString())))
.ifArray(a -> {
final List<Value> list = a.stream().collect(Collectors.toList());
Collections.reverse(list);
c.accept(new Value(list));
})
.ifString(s -> c.accept(new Value(new StringBuilder(s).reverse().toString())))
);
}
},
Expand All @@ -425,8 +436,8 @@ public void apply(final Metafix metafix, final Record record, final List<String>
Comparator.comparing(function.andThen(Integer::parseInt)) : Comparator.comparing(function);

record.transform(params.get(0), (m, c) -> m
.ifArray(a -> c.accept(new Value((uniq ? unique(a.stream()) : a.stream())
.sorted(reverse ? comparator.reversed() : comparator).collect(Collectors.toList()))))
.ifArray(a -> c.accept(new Value((uniq ? unique(a.stream()) : a.stream())
.sorted(reverse ? comparator.reversed() : comparator).collect(Collectors.toList()))))
);
}
},
Expand All @@ -440,9 +451,9 @@ public void apply(final Metafix metafix, final Record record, final List<String>
newArray(Arrays.stream(splitPattern.split(s)).map(Value::new));

record.transform(params.get(0), (m, c) -> m
.ifArray(a -> c.accept(newArray(a.stream().map(Value::asString).map(splitFunction))))
.ifHash(h -> c.accept(Value.newHash(n -> h.forEach((f, w) -> n.put(f, splitFunction.apply(w.asString()))))))
.ifString(s -> c.accept(splitFunction.apply(s)))
.ifArray(a -> c.accept(newArray(a.stream().map(Value::asString).map(splitFunction))))
.ifHash(h -> c.accept(Value.newHash(n -> h.forEach((f, w) -> n.put(f, splitFunction.apply(w.asString()))))))
.ifString(s -> c.accept(splitFunction.apply(s)))
);
}
},
Expand All @@ -463,7 +474,7 @@ public void apply(final Metafix metafix, final Record record, final List<String>
@Override
public void apply(final Metafix metafix, final Record record, final List<String> params, final Map<String, String> options) {
record.transform(params.get(0), (m, c) -> m
.ifArray(a -> c.accept(new Value(a.stream().map(Value::asString).mapToInt(Integer::parseInt).sum())))
.ifArray(a -> c.accept(new Value(a.stream().map(Value::asString).mapToInt(Integer::parseInt).sum())))
);
}
},
Expand All @@ -477,7 +488,7 @@ public void apply(final Metafix metafix, final Record record, final List<String>
@Override
public void apply(final Metafix metafix, final Record record, final List<String> params, final Map<String, String> options) {
record.transform(params.get(0), (m, c) -> m
.ifArray(a -> c.accept(newArray(unique(a.stream()))))
.ifArray(a -> c.accept(newArray(unique(a.stream()))))
);
}
},
Expand All @@ -488,11 +499,41 @@ public void apply(final Metafix metafix, final Record record, final List<String>
}
};

public static final String KIND_OF_RDFMAP = "rdfmap";
public static final String KIND_OF_FILEMAP = "filemap";
private static final Pattern NAMED_GROUP_PATTERN = Pattern.compile("\\(\\?<(.+?)>");

private static final String FILEMAP_SEPARATOR_OPTION = "sep_char";
private static final String FILEMAP_DEFAULT_SEPARATOR = ",";

private static final Random RANDOM = new Random();
private static String defaultValue;

private static Map<String, String> extracted(final Metafix metafix, final Record record, final List<String> params, final Map<String, String> options, final String kindOfMap) {
final Map<String, String> map;
if (params.size() <= 1) {
map = options;
}
else {
final String mapName = params.get(1);

if (!metafix.getMapNames().contains(mapName)) {
if (mapName.contains(".") || mapName.contains(File.separator)) {
if (kindOfMap.equals(KIND_OF_FILEMAP)) {
put_filemap.apply(metafix, record, Arrays.asList(mapName), options);
}
if (kindOfMap.equals(KIND_OF_RDFMAP)) {
put_rdfmap.apply(metafix, record, Arrays.asList(mapName), options);
}
}
else {
// Probably an unknown internal map? Log a warning?
}
}
map = metafix.getMap(mapName);
}
defaultValue = map.get(Maps.DEFAULT_MAP_KEY); // TODO: Catmandu uses 'default'
return map;
}

}
1 change: 1 addition & 0 deletions metafix/src/main/java/org/metafacture/metafix/FixPath.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -307,6 +307,7 @@ private enum ReservedField {
$append, $first, $last;

private static final Map<String, ReservedField> STRING_TO_ENUM = new HashMap<>();

static {
for (final ReservedField f : values()) {
STRING_TO_ENUM.put(f.toString(), f);
Expand Down
Loading

0 comments on commit c930c58

Please sign in to comment.