From e5eac41afde9c87fdae6e0956b8aa61b75b21ad8 Mon Sep 17 00:00:00 2001 From: Martin Raifer Date: Fri, 14 May 2021 11:37:25 +0200 Subject: [PATCH 01/34] extract interfaces for OSMContribution & OSMEntitySnapshot --- .../ohsome/oshdb/api/db/OSHDBDatabase.java | 2 +- .../ohsome/oshdb/api/db/OSHDBIgnite.java | 2 +- .../heigit/ohsome/oshdb/api/db/OSHDBJdbc.java | 2 +- .../api/mapreducer/GeometrySplitter.java | 10 +- .../oshdb/api/mapreducer/MapAggregator.java | 20 +-- .../oshdb/api/mapreducer/MapReducer.java | 101 ++++++----- .../api/mapreducer/OSMContributionView.java | 4 +- .../api/mapreducer/OSMEntitySnapshotView.java | 4 +- .../oshdb/api/mapreducer/backend/Kernels.java | 22 +-- .../backend/MapReducerIgniteAffinityCall.java | 6 +- .../backend/MapReducerIgniteLocalPeek.java | 6 +- .../backend/MapReducerIgniteScanQuery.java | 6 +- .../mapreducer/backend/MapReducerJdbc.java | 2 +- .../backend/MapReducerJdbcMultithread.java | 6 +- .../backend/MapReducerJdbcSinglethread.java | 6 +- .../oshdb/api/object/OSHDBMapReducible.java | 7 - ...ribution.java => OSMContributionImpl.java} | 162 +++--------------- ...apshot.java => OSMEntitySnapshotImpl.java} | 55 ++---- .../ohsome/oshdb/api/tests/TestCollect.java | 2 +- .../oshdb/api/tests/TestFlatMapAggregate.java | 2 +- .../TestFlatMapAggregateGroupedByEntity.java | 2 +- .../oshdb/api/tests/TestFlatMapReduce.java | 2 +- .../TestFlatMapReduceGroupedByEntity.java | 4 +- .../ohsome/oshdb/api/tests/TestForEach.java | 2 +- .../tests/TestHelpersOSMContributionView.java | 2 +- .../TestHelpersOSMEntitySnapshotView.java | 2 +- .../oshdb/api/tests/TestLambdaFilter.java | 2 +- .../api/tests/TestMapAggregateByGeometry.java | 4 +- .../api/tests/TestMapAggregateByIndex.java | 4 +- .../tests/TestMapAggregateByTimestamp.java | 4 +- .../ohsome/oshdb/api/tests/TestMapReduce.java | 4 +- .../oshdb/api/tests/TestOSHDBFilter.java | 4 +- ...stOSMContributionGetContributorUserId.java | 23 +-- .../oshdb/api/tests/TestOSMDataFilters.java | 2 +- .../ohsome/oshdb/api/tests/TestQuantiles.java | 2 +- .../ohsome/oshdb/api/tests/TestStream.java | 2 +- .../util/mappable/OSHDBMapReducible.java | 6 + .../oshdb/util/mappable/OSMContribution.java | 155 +++++++++++++++++ .../util/mappable/OSMEntitySnapshot.java | 51 ++++++ .../oshdb/util/mappable/package-info.java | 4 + 40 files changed, 400 insertions(+), 308 deletions(-) delete mode 100644 oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/object/OSHDBMapReducible.java rename oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/object/{OSMContribution.java => OSMContributionImpl.java} (52%) rename oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/object/{OSMEntitySnapshot.java => OSMEntitySnapshotImpl.java} (57%) create mode 100644 oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/mappable/OSHDBMapReducible.java create mode 100644 oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/mappable/OSMContribution.java create mode 100644 oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/mappable/OSMEntitySnapshot.java create mode 100644 oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/mappable/package-info.java diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBDatabase.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBDatabase.java index 99df566df..5f0933dfd 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBDatabase.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBDatabase.java @@ -3,8 +3,8 @@ import java.util.OptionalLong; import org.heigit.ohsome.oshdb.OSHDB; import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer; -import org.heigit.ohsome.oshdb.api.object.OSHDBMapReducible; import org.heigit.ohsome.oshdb.util.exceptions.OSHDBTimeoutException; +import org.heigit.ohsome.oshdb.util.mappable.OSHDBMapReducible; /** * OSHDB database backend connector. diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBIgnite.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBIgnite.java index fe68b0c4d..d05e25b57 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBIgnite.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBIgnite.java @@ -13,10 +13,10 @@ import org.heigit.ohsome.oshdb.api.mapreducer.backend.MapReducerIgniteAffinityCall; import org.heigit.ohsome.oshdb.api.mapreducer.backend.MapReducerIgniteLocalPeek; import org.heigit.ohsome.oshdb.api.mapreducer.backend.MapReducerIgniteScanQuery; -import org.heigit.ohsome.oshdb.api.object.OSHDBMapReducible; import org.heigit.ohsome.oshdb.osm.OSMType; import org.heigit.ohsome.oshdb.util.TableNames; import org.heigit.ohsome.oshdb.util.exceptions.OSHDBTableNotFoundException; +import org.heigit.ohsome.oshdb.util.mappable.OSHDBMapReducible; /** * OSHDB database backend connector to a Ignite system. diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBJdbc.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBJdbc.java index ee89ffd27..ee2dead6a 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBJdbc.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/db/OSHDBJdbc.java @@ -15,10 +15,10 @@ import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer; import org.heigit.ohsome.oshdb.api.mapreducer.backend.MapReducerJdbcMultithread; import org.heigit.ohsome.oshdb.api.mapreducer.backend.MapReducerJdbcSinglethread; -import org.heigit.ohsome.oshdb.api.object.OSHDBMapReducible; import org.heigit.ohsome.oshdb.osm.OSMType; import org.heigit.ohsome.oshdb.util.TableNames; import org.heigit.ohsome.oshdb.util.exceptions.OSHDBTableNotFoundException; +import org.heigit.ohsome.oshdb.util.mappable.OSHDBMapReducible; /** * OSHDB database backend connector to a JDBC database file. diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/GeometrySplitter.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/GeometrySplitter.java index d841b73aa..b5a8d9fe4 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/GeometrySplitter.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/GeometrySplitter.java @@ -14,14 +14,16 @@ import java.util.stream.Stream; import org.heigit.ohsome.oshdb.OSHDBBoundable; import org.heigit.ohsome.oshdb.OSHDBBoundingBox; -import org.heigit.ohsome.oshdb.api.object.OSMContribution; -import org.heigit.ohsome.oshdb.api.object.OSMEntitySnapshot; +import org.heigit.ohsome.oshdb.api.object.OSMContributionImpl; +import org.heigit.ohsome.oshdb.api.object.OSMEntitySnapshotImpl; import org.heigit.ohsome.oshdb.util.celliterator.ContributionType; import org.heigit.ohsome.oshdb.util.celliterator.LazyEvaluatedObject; import org.heigit.ohsome.oshdb.util.geometry.OSHDBGeometryBuilder; import org.heigit.ohsome.oshdb.util.geometry.fip.FastBboxInPolygon; import org.heigit.ohsome.oshdb.util.geometry.fip.FastBboxOutsidePolygon; import org.heigit.ohsome.oshdb.util.geometry.fip.FastPolygonOperations; +import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; +import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; import org.locationtech.jts.geom.Envelope; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.GeometryFactory; @@ -124,7 +126,7 @@ public Map splitOSMEntitySnapshot(OSMEntitySnapshot data) // not actually intersecting -> skip return Stream.empty(); } else { - return Stream.of(new IndexData<>(index, new OSMEntitySnapshot(data, + return Stream.of(new IndexData<>(index, new OSMEntitySnapshotImpl(data, new LazyEvaluatedObject<>(() -> faultTolerantIntersection(snapshotGeometry, poop)) ))); @@ -203,7 +205,7 @@ public Map splitOSMContribution(OSMContribution data) { // not actually intersecting -> skip return Stream.empty(); } else { - return Stream.of(new IndexData<>(index, new OSMContribution(data, + return Stream.of(new IndexData<>(index, new OSMContributionImpl(data, new LazyEvaluatedObject<>(() -> faultTolerantIntersection(contributionGeometryBefore, poop)), new LazyEvaluatedObject<>(() -> diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapAggregator.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapAggregator.java index 1220b4cc1..fd6791a0f 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapAggregator.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapAggregator.java @@ -29,9 +29,8 @@ import org.heigit.ohsome.oshdb.api.generic.OSHDBCombinedIndex; import org.heigit.ohsome.oshdb.api.generic.WeightedValue; import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer.Grouping; -import org.heigit.ohsome.oshdb.api.object.OSHDBMapReducible; -import org.heigit.ohsome.oshdb.api.object.OSMContribution; -import org.heigit.ohsome.oshdb.api.object.OSMEntitySnapshot; +import org.heigit.ohsome.oshdb.api.object.OSMContributionImpl; +import org.heigit.ohsome.oshdb.api.object.OSMEntitySnapshotImpl; import org.heigit.ohsome.oshdb.filter.FilterExpression; import org.heigit.ohsome.oshdb.osm.OSMType; import org.heigit.ohsome.oshdb.util.exceptions.OSHDBInvalidTimestampException; @@ -42,6 +41,7 @@ import org.heigit.ohsome.oshdb.util.function.SerializableFunction; import org.heigit.ohsome.oshdb.util.function.SerializablePredicate; import org.heigit.ohsome.oshdb.util.function.SerializableSupplier; +import org.heigit.ohsome.oshdb.util.mappable.OSHDBMapReducible; import org.heigit.ohsome.oshdb.util.tagtranslator.OSMTagInterface; import org.jetbrains.annotations.Contract; import org.locationtech.jts.geom.Geometry; @@ -220,17 +220,15 @@ MapAggregator, X> aggregateByGeometry(Map geometr ); } else { MapAggregator, ? extends OSHDBMapReducible> ret; - if (this.mapReducer.forClass.equals(OSMContribution.class)) { - ret = this.flatMap(x -> gs.splitOSMContribution((OSMContribution) x).entrySet()) + if (mapReducer.isContributionViewQuery()) { + ret = this.flatMap(x -> gs.splitOSMContribution((OSMContributionImpl) x).entrySet()) .aggregateBy(Entry::getKey, geometries.keySet()).map(Entry::getValue); - } else if (this.mapReducer.forClass.equals(OSMEntitySnapshot.class)) { - ret = this.flatMap(x -> gs.splitOSMEntitySnapshot((OSMEntitySnapshot) x).entrySet()) + } else if (mapReducer.isOSMEntitySnapshotQuery()) { + ret = this.flatMap(x -> gs.splitOSMEntitySnapshot((OSMEntitySnapshotImpl) x).entrySet()) .aggregateBy(Entry::getKey, geometries.keySet()).map(Entry::getValue); } else { - throw new UnsupportedOperationException( - "aggregateByGeometry not implemented for objects of type: " - + this.mapReducer.forClass.toString() - ); + throw new UnsupportedOperationException(String.format( + MapReducer.UNIMPLEMENTED_DATA_VIEW, this.mapReducer.viewClass)); } @SuppressWarnings("unchecked") // no mapper functions have been applied -> the type is still X MapAggregator, X> result = diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapReducer.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapReducer.java index 3193650f9..f420638d9 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapReducer.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapReducer.java @@ -33,9 +33,8 @@ import org.heigit.ohsome.oshdb.api.db.OSHDBJdbc; import org.heigit.ohsome.oshdb.api.generic.NumberUtils; import org.heigit.ohsome.oshdb.api.generic.WeightedValue; -import org.heigit.ohsome.oshdb.api.object.OSHDBMapReducible; -import org.heigit.ohsome.oshdb.api.object.OSMContribution; -import org.heigit.ohsome.oshdb.api.object.OSMEntitySnapshot; +import org.heigit.ohsome.oshdb.api.object.OSMContributionImpl; +import org.heigit.ohsome.oshdb.api.object.OSMEntitySnapshotImpl; import org.heigit.ohsome.oshdb.filter.AndOperator; import org.heigit.ohsome.oshdb.filter.Filter; import org.heigit.ohsome.oshdb.filter.FilterExpression; @@ -63,6 +62,9 @@ import org.heigit.ohsome.oshdb.util.function.SerializableSupplier; import org.heigit.ohsome.oshdb.util.geometry.Geo; import org.heigit.ohsome.oshdb.util.geometry.OSHDBGeometryBuilder; +import org.heigit.ohsome.oshdb.util.mappable.OSHDBMapReducible; +import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; +import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; import org.heigit.ohsome.oshdb.util.taginterpreter.DefaultTagInterpreter; import org.heigit.ohsome.oshdb.util.taginterpreter.TagInterpreter; import org.heigit.ohsome.oshdb.util.tagtranslator.OSMTag; @@ -123,22 +125,23 @@ public abstract class MapReducer implements MapAggregatable, X>, X>, Serializable { private static final Logger LOG = LoggerFactory.getLogger(MapReducer.class); - public static final String TAG_KEY_NOT_FOUND = + protected static final String TAG_KEY_NOT_FOUND = "Tag key {} not found. No data will match this filter."; - public static final String TAG_NOT_FOUND = + protected static final String TAG_NOT_FOUND = "Tag {}={} not found. No data will match this filter."; - public static final String EMPTY_TAG_LIST = + protected static final String EMPTY_TAG_LIST = "Empty tag value list. No data will match this filter."; - public static final String UNIMPLEMENTED_DATA_VIEW = "Unimplemented data view: %s"; - public static final String UNSUPPORTED_GROUPING = "Unsupported grouping: %s"; + protected static final String UNIMPLEMENTED_DATA_VIEW = "Unimplemented data view: %s"; + protected static final String UNSUPPORTED_GROUPING = "Unsupported grouping: %s"; protected transient OSHDBDatabase oshdb; protected transient OSHDBJdbc keytables; protected Long timeout = null; - // internal state - Class forClass; + /** the class representing the used OSHDB view: either {@link OSMContribution} or + * {@link OSMEntitySnapshot}. */ + Class viewClass; enum Grouping { NONE, BY_ID @@ -172,9 +175,9 @@ public boolean isCancelable() { // basic constructor - protected MapReducer(OSHDBDatabase oshdb, Class forClass) { + protected MapReducer(OSHDBDatabase oshdb, Class viewClass) { this.oshdb = oshdb; - this.forClass = forClass; + this.viewClass = viewClass; } // copy constructor @@ -182,7 +185,7 @@ protected MapReducer(MapReducer obj) { this.oshdb = obj.oshdb; this.keytables = obj.keytables; - this.forClass = obj.forClass; + this.viewClass = obj.viewClass; this.grouping = obj.grouping; this.tagTranslator = obj.tagTranslator; @@ -354,7 +357,7 @@ public MapReducer timestamps( */ @Contract(pure = true) public MapReducer timestamps(String isoDate) { - if (this.forClass.equals(OSMContribution.class)) { + if (isContributionViewQuery()) { LOG.warn("OSMContributionView requires two or more timestamps, but only one was supplied."); } return this.timestamps(isoDate, isoDate, new String[] {}); @@ -790,12 +793,12 @@ public MapReducer filter(FilterExpression f) { ret.mappers.clear(); if (this.grouping == Grouping.NONE) { // no grouping -> directly filter using the geometries of the snapshot / contribution - if (ret.forClass.equals(OSMEntitySnapshot.class)) { + if (isOSMEntitySnapshotQuery()) { ret = ret.filter(x -> { OSMEntitySnapshot s = (OSMEntitySnapshot) x; return f.applyOSMGeometry(s.getEntity(), s::getGeometry); }); - } else if (ret.forClass.equals(OSMContribution.class)) { + } else if (isContributionViewQuery()) { ret = ret.filter(x -> { OSMContribution c = (OSMContribution) x; if (c.is(ContributionType.CREATION)) { @@ -810,7 +813,7 @@ public MapReducer filter(FilterExpression f) { } } else if (this.grouping == Grouping.BY_ID) { // grouping by entity -> filter each list entry individually - if (ret.forClass.equals(OSMEntitySnapshot.class)) { + if (isOSMEntitySnapshotQuery()) { @SuppressWarnings("unchecked") MapReducer filteredListMapper = (MapReducer) ret.map(x -> (Collection) x) .map(snapshots -> snapshots.stream() @@ -818,7 +821,7 @@ public MapReducer filter(FilterExpression f) { .collect(Collectors.toCollection(ArrayList::new))) .filter(snapshots -> !snapshots.isEmpty()); ret = filteredListMapper; - } else if (ret.forClass.equals(OSMContribution.class)) { + } else if (isContributionViewQuery()) { @SuppressWarnings("unchecked") MapReducer filteredListMapper = (MapReducer) ret.map(x -> (Collection) x) .map(contributions -> contributions.stream() @@ -964,10 +967,10 @@ public MapAggregator aggregateByTimestamp() // by timestamp indexing function -> for some views we need to match the input data to the list SerializableFunction indexer; - if (this.forClass.equals(OSMContribution.class)) { + if (isContributionViewQuery()) { final TreeSet timestamps = new TreeSet<>(this.tstamps.get()); indexer = data -> timestamps.floor(((OSMContribution) data).getTimestamp()); - } else if (this.forClass.equals(OSMEntitySnapshot.class)) { + } else if (isOSMEntitySnapshotQuery()) { indexer = data -> ((OSMEntitySnapshot) data).getTimestamp(); } else { throw new UnsupportedOperationException( @@ -1062,15 +1065,15 @@ MapAggregator aggregateByGeometry(Map geometries) "please call aggregateByGeometry before setting any map or flatMap functions"); } else { MapAggregator ret; - if (this.forClass.equals(OSMContribution.class)) { - ret = this.flatMap(x -> gs.splitOSMContribution((OSMContribution) x).entrySet()) + if (isContributionViewQuery()) { + ret = this.flatMap(x -> gs.splitOSMContribution((OSMContributionImpl) x).entrySet()) .aggregateBy(Entry::getKey, geometries.keySet()).map(Entry::getValue); - } else if (this.forClass.equals(OSMEntitySnapshot.class)) { - ret = this.flatMap(x -> gs.splitOSMEntitySnapshot((OSMEntitySnapshot) x).entrySet()) + } else if (isOSMEntitySnapshotQuery()) { + ret = this.flatMap(x -> gs.splitOSMEntitySnapshot((OSMEntitySnapshotImpl) x).entrySet()) .aggregateBy(Entry::getKey, geometries.keySet()).map(Entry::getValue); } else { - throw new UnsupportedOperationException( - "aggregateByGeometry not implemented for objects of type: " + this.forClass); + throw new UnsupportedOperationException(String.format( + UNIMPLEMENTED_DATA_VIEW, this.viewClass)); } @SuppressWarnings("unchecked") // no mapper functions have been applied so the type is still X MapAggregator result = (MapAggregator) ret; @@ -1136,7 +1139,7 @@ public S reduce( case NONE: if (this.mappers.stream().noneMatch(MapFunction::isFlatMapper)) { final SerializableFunction mapper = this.getMapper(); - if (this.forClass.equals(OSMContribution.class)) { + if (isContributionViewQuery()) { @SuppressWarnings("Convert2MethodRef") // having just `mapper::apply` here is problematic, see https://github.com/GIScience/oshdb/pull/37 final SerializableFunction contributionMapper = @@ -1147,7 +1150,7 @@ public S reduce( accumulator, combiner ); - } else if (this.forClass.equals(OSMEntitySnapshot.class)) { + } else if (isOSMEntitySnapshotQuery()) { @SuppressWarnings("Convert2MethodRef") // having just `mapper::apply` here is problematic, see https://github.com/GIScience/oshdb/pull/37 final SerializableFunction snapshotMapper = @@ -1160,11 +1163,11 @@ public S reduce( ); } else { throw new UnsupportedOperationException(String.format( - UNIMPLEMENTED_DATA_VIEW, this.forClass)); + UNIMPLEMENTED_DATA_VIEW, this.viewClass)); } } else { final SerializableFunction> flatMapper = this.getFlatMapper(); - if (this.forClass.equals(OSMContribution.class)) { + if (isContributionViewQuery()) { return this.flatMapReduceCellsOSMContributionGroupedById( (List inputList) -> { List outputList = new LinkedList<>(); @@ -1173,7 +1176,7 @@ public S reduce( .forEach(data -> Iterables.addAll(outputList, data)); return outputList; }, identitySupplier, accumulator, combiner); - } else if (this.forClass.equals(OSMEntitySnapshot.class)) { + } else if (isOSMEntitySnapshotQuery()) { return this.flatMapReduceCellsOSMEntitySnapshotGroupedById( (List inputList) -> { List outputList = new LinkedList<>(); @@ -1184,7 +1187,7 @@ public S reduce( }, identitySupplier, accumulator, combiner); } else { throw new UnsupportedOperationException(String.format( - UNIMPLEMENTED_DATA_VIEW, this.forClass)); + UNIMPLEMENTED_DATA_VIEW, this.viewClass)); } } case BY_ID: @@ -1197,7 +1200,7 @@ public S reduce( } else { flatMapper = this.getFlatMapper(); } - if (this.forClass.equals(OSMContribution.class)) { + if (isContributionViewQuery()) { @SuppressWarnings("Convert2MethodRef") // having just `flatMapper::apply` here is problematic, see https://github.com/GIScience/oshdb/pull/37 final SerializableFunction, Iterable> contributionFlatMapper = @@ -1208,7 +1211,7 @@ public S reduce( accumulator, combiner ); - } else if (this.forClass.equals(OSMEntitySnapshot.class)) { + } else if (isOSMEntitySnapshotQuery()) { @SuppressWarnings("Convert2MethodRef") // having just `flatMapper::apply` here is problematic, see https://github.com/GIScience/oshdb/pull/37 final SerializableFunction, Iterable> snapshotFlatMapper = @@ -1221,7 +1224,7 @@ public S reduce( ); } else { throw new UnsupportedOperationException(String.format( - UNIMPLEMENTED_DATA_VIEW, this.forClass)); + UNIMPLEMENTED_DATA_VIEW, this.viewClass)); } default: throw new UnsupportedOperationException(String.format( @@ -1652,13 +1655,13 @@ private Stream streamInternal() throws Exception { case NONE: if (this.mappers.stream().noneMatch(MapFunction::isFlatMapper)) { final SerializableFunction mapper = this.getMapper(); - if (this.forClass.equals(OSMContribution.class)) { + if (isContributionViewQuery()) { @SuppressWarnings("Convert2MethodRef") // having just `mapper::apply` here is problematic, see https://github.com/GIScience/oshdb/pull/37 final SerializableFunction contributionMapper = data -> mapper.apply(data); return this.mapStreamCellsOSMContribution(contributionMapper); - } else if (this.forClass.equals(OSMEntitySnapshot.class)) { + } else if (isOSMEntitySnapshotQuery()) { @SuppressWarnings("Convert2MethodRef") // having just `mapper::apply` here is problematic, see https://github.com/GIScience/oshdb/pull/37 final SerializableFunction snapshotMapper = @@ -1666,11 +1669,11 @@ private Stream streamInternal() throws Exception { return this.mapStreamCellsOSMEntitySnapshot(snapshotMapper); } else { throw new UnsupportedOperationException(String.format( - UNIMPLEMENTED_DATA_VIEW, this.forClass)); + UNIMPLEMENTED_DATA_VIEW, this.viewClass)); } } else { final SerializableFunction> flatMapper = this.getFlatMapper(); - if (this.forClass.equals(OSMContribution.class)) { + if (isContributionViewQuery()) { return this.flatMapStreamCellsOSMContributionGroupedById( (List inputList) -> { List outputList = new LinkedList<>(); @@ -1679,7 +1682,7 @@ private Stream streamInternal() throws Exception { .forEach(data -> Iterables.addAll(outputList, data)); return outputList; }); - } else if (this.forClass.equals(OSMEntitySnapshot.class)) { + } else if (isOSMEntitySnapshotQuery()) { return this.flatMapStreamCellsOSMEntitySnapshotGroupedById( (List inputList) -> { List outputList = new LinkedList<>(); @@ -1690,7 +1693,7 @@ private Stream streamInternal() throws Exception { }); } else { throw new UnsupportedOperationException(String.format( - UNIMPLEMENTED_DATA_VIEW, this.forClass)); + UNIMPLEMENTED_DATA_VIEW, this.viewClass)); } } case BY_ID: @@ -1703,13 +1706,13 @@ private Stream streamInternal() throws Exception { } else { flatMapper = this.getFlatMapper(); } - if (this.forClass.equals(OSMContribution.class)) { + if (isContributionViewQuery()) { @SuppressWarnings("Convert2MethodRef") // having just `mapper::apply` here is problematic, see https://github.com/GIScience/oshdb/pull/37 final SerializableFunction, Iterable> contributionFlatMapper = data -> flatMapper.apply(data); return this.flatMapStreamCellsOSMContributionGroupedById(contributionFlatMapper); - } else if (this.forClass.equals(OSMEntitySnapshot.class)) { + } else if (isOSMEntitySnapshotQuery()) { @SuppressWarnings("Convert2MethodRef") // having just `mapper::apply` here is problematic, see https://github.com/GIScience/oshdb/pull/37 final SerializableFunction, Iterable> snapshotFlatMapper = @@ -1717,7 +1720,7 @@ private Stream streamInternal() throws Exception { return this.flatMapStreamCellsOSMEntitySnapshotGroupedById(snapshotFlatMapper); } else { throw new UnsupportedOperationException(String.format( - UNIMPLEMENTED_DATA_VIEW, this.forClass)); + UNIMPLEMENTED_DATA_VIEW, this.viewClass)); } default: throw new UnsupportedOperationException(String.format( @@ -1951,6 +1954,14 @@ protected abstract S flatMapReduceCellsOSMEntitySnapshotGroupedById( // Some helper methods for internal use in the mapReduce functions // ----------------------------------------------------------------------------------------------- + protected boolean isContributionViewQuery() { + return OSMContribution.class.isAssignableFrom(this.viewClass); + } + + protected boolean isOSMEntitySnapshotQuery() { + return OSMEntitySnapshot.class.isAssignableFrom(this.viewClass); + } + protected TagInterpreter getTagInterpreter() throws ParseException, IOException { if (this.tagInterpreter == null) { this.tagInterpreter = new DefaultTagInterpreter(this.getTagTranslator()); @@ -2070,7 +2081,7 @@ private SerializableFunction> getFlatMapper() { // gets list of timestamps to use for zerofilling Collection getZerofillTimestamps() { - if (this.forClass.equals(OSMEntitySnapshot.class)) { + if (isOSMEntitySnapshotQuery()) { return this.tstamps.get(); } else { SortedSet result = new TreeSet<>(this.tstamps.get()); diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/OSMContributionView.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/OSMContributionView.java index a16e69cda..ae2084fe4 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/OSMContributionView.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/OSMContributionView.java @@ -1,7 +1,7 @@ package org.heigit.ohsome.oshdb.api.mapreducer; import org.heigit.ohsome.oshdb.api.db.OSHDBDatabase; -import org.heigit.ohsome.oshdb.api.object.OSMContribution; +import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; /** * Returns all modifications to OSM elements within a given time period. @@ -10,6 +10,6 @@ public class OSMContributionView { private OSMContributionView() {} public static MapReducer on(OSHDBDatabase oshdb) { - return oshdb.createMapReducer(OSMContribution.class); + return oshdb.createMapReducer(OSMContribution.class); } } diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/OSMEntitySnapshotView.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/OSMEntitySnapshotView.java index 738a92b87..89b36d56e 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/OSMEntitySnapshotView.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/OSMEntitySnapshotView.java @@ -1,7 +1,7 @@ package org.heigit.ohsome.oshdb.api.mapreducer; import org.heigit.ohsome.oshdb.api.db.OSHDBDatabase; -import org.heigit.ohsome.oshdb.api.object.OSMEntitySnapshot; +import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; /** * Returns the state of OSM elements at specific given points in time. @@ -10,6 +10,6 @@ public class OSMEntitySnapshotView { private OSMEntitySnapshotView() {} public static MapReducer on(OSHDBDatabase oshdb) { - return oshdb.createMapReducer(OSMEntitySnapshot.class); + return oshdb.createMapReducer(OSMEntitySnapshot.class); } } diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/Kernels.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/Kernels.java index dc2bce770..7b7c33d8e 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/Kernels.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/Kernels.java @@ -8,13 +8,15 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Stream; import javax.annotation.Nonnull; -import org.heigit.ohsome.oshdb.api.object.OSMContribution; -import org.heigit.ohsome.oshdb.api.object.OSMEntitySnapshot; +import org.heigit.ohsome.oshdb.api.object.OSMContributionImpl; +import org.heigit.ohsome.oshdb.api.object.OSMEntitySnapshotImpl; import org.heigit.ohsome.oshdb.grid.GridOSHEntity; import org.heigit.ohsome.oshdb.util.celliterator.CellIterator; import org.heigit.ohsome.oshdb.util.function.SerializableBiFunction; import org.heigit.ohsome.oshdb.util.function.SerializableFunction; import org.heigit.ohsome.oshdb.util.function.SerializableSupplier; +import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; +import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; class Kernels implements Serializable { interface CellProcessor extends SerializableBiFunction {} @@ -57,7 +59,7 @@ static CellProcessor getOSMContributionCellReducer( cellIterator.iterateByContribution(oshEntityCell) .filter(ignored -> process.isActive()) .forEach(contribution -> { - OSMContribution osmContribution = new OSMContribution(contribution); + OSMContribution osmContribution = new OSMContributionImpl(contribution); accInternal.set(accumulator.apply(accInternal.get(), mapper.apply(osmContribution))); }); return accInternal.get(); @@ -87,7 +89,7 @@ static CellProcessor getOSMContributionGroupingCellReducer( cellIterator.iterateByContribution(oshEntityCell) .filter(ignored -> process.isActive()) .forEach(contribution -> { - OSMContribution thisContribution = new OSMContribution(contribution); + OSMContribution thisContribution = new OSMContributionImpl(contribution); if (contributions.size() > 0 && thisContribution.getEntityAfter().getId() != contributions .get(contributions.size() - 1).getEntityAfter().getId()) { @@ -131,7 +133,7 @@ static CellProcessor getOSMEntitySnapshotCellReducer( cellIterator.iterateByTimestamps(oshEntityCell) .filter(ignored -> process.isActive()) .forEach(data -> { - OSMEntitySnapshot snapshot = new OSMEntitySnapshot(data); + OSMEntitySnapshot snapshot = new OSMEntitySnapshotImpl(data); // immediately fold the result accInternal.set(accumulator.apply(accInternal.get(), mapper.apply(snapshot))); }); @@ -162,7 +164,7 @@ static CellProcessor getOSMEntitySnapshotGroupingCellReducer( cellIterator.iterateByTimestamps(oshEntityCell) .filter(ignored -> process.isActive()) .forEach(data -> { - OSMEntitySnapshot thisSnapshot = new OSMEntitySnapshot(data); + OSMEntitySnapshot thisSnapshot = new OSMEntitySnapshotImpl(data); if (osmEntitySnapshots.size() > 0 && thisSnapshot.getEntity().getId() != osmEntitySnapshots .get(osmEntitySnapshots.size() - 1).getEntity().getId()) { @@ -202,7 +204,7 @@ static CellProcessor> getOSMContributionCellStreamer( // iterate over the history of all OSM objects in the current cell return cellIterator.iterateByContribution(oshEntityCell) .filter(ignored -> process.isActive()) - .map(OSMContribution::new) + .map(OSMContributionImpl::new) .map(mapper); }; } @@ -225,7 +227,7 @@ static CellProcessor> getOSMContributionGroupingCellStreamer( List result = new LinkedList<>(); cellIterator.iterateByContribution(oshEntityCell) .filter(ignored -> process.isActive()) - .map(OSMContribution::new) + .map(OSMContributionImpl::new) .forEach(contribution -> { if (contributions.size() > 0 && contribution.getEntityAfter().getId() != contributions.get(contributions.size() - 1).getEntityAfter().getId()) { @@ -259,7 +261,7 @@ static CellProcessor> getOSMEntitySnapshotCellStreamer( // iterate over the history of all OSM objects in the current cell return cellIterator.iterateByTimestamps(oshEntityCell) .filter(ignored -> process.isActive()) - .map(OSMEntitySnapshot::new) + .map(OSMEntitySnapshotImpl::new) .map(mapper); }; } @@ -282,7 +284,7 @@ static CellProcessor> getOSMEntitySnapshotGroupingCellStreamer( List result = new LinkedList<>(); cellIterator.iterateByTimestamps(oshEntityCell) .filter(ignored -> process.isActive()) - .map(OSMEntitySnapshot::new) + .map(OSMEntitySnapshotImpl::new) .forEach(contribution -> { if (snapshots.size() > 0 && contribution.getEntity().getId() != snapshots.get(snapshots.size() - 1).getEntity().getId()) { diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerIgniteAffinityCall.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerIgniteAffinityCall.java index fd2b0fde4..bcbd18dc2 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerIgniteAffinityCall.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerIgniteAffinityCall.java @@ -34,9 +34,6 @@ import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer; import org.heigit.ohsome.oshdb.api.mapreducer.backend.Kernels.CancelableProcessStatus; import org.heigit.ohsome.oshdb.api.mapreducer.backend.Kernels.CellProcessor; -import org.heigit.ohsome.oshdb.api.object.OSHDBMapReducible; -import org.heigit.ohsome.oshdb.api.object.OSMContribution; -import org.heigit.ohsome.oshdb.api.object.OSMEntitySnapshot; import org.heigit.ohsome.oshdb.grid.GridOSHEntity; import org.heigit.ohsome.oshdb.index.XYGridTree.CellIdRange; import org.heigit.ohsome.oshdb.osm.OSMType; @@ -48,6 +45,9 @@ import org.heigit.ohsome.oshdb.util.function.SerializableBinaryOperator; import org.heigit.ohsome.oshdb.util.function.SerializableFunction; import org.heigit.ohsome.oshdb.util.function.SerializableSupplier; +import org.heigit.ohsome.oshdb.util.mappable.OSHDBMapReducible; +import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; +import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; import org.jetbrains.annotations.NotNull; import org.json.simple.parser.ParseException; diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerIgniteLocalPeek.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerIgniteLocalPeek.java index 18b4d4dbb..a4e24304d 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerIgniteLocalPeek.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerIgniteLocalPeek.java @@ -27,9 +27,6 @@ import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer; import org.heigit.ohsome.oshdb.api.mapreducer.backend.Kernels.CellProcessor; import org.heigit.ohsome.oshdb.api.mapreducer.backend.OSHDBIgniteMapReduceComputeTask.CancelableIgniteMapReduceJob; -import org.heigit.ohsome.oshdb.api.object.OSHDBMapReducible; -import org.heigit.ohsome.oshdb.api.object.OSMContribution; -import org.heigit.ohsome.oshdb.api.object.OSMEntitySnapshot; import org.heigit.ohsome.oshdb.grid.GridOSHEntity; import org.heigit.ohsome.oshdb.index.XYGridTree.CellIdRange; import org.heigit.ohsome.oshdb.util.CellId; @@ -42,6 +39,9 @@ import org.heigit.ohsome.oshdb.util.function.SerializableBinaryOperator; import org.heigit.ohsome.oshdb.util.function.SerializableFunction; import org.heigit.ohsome.oshdb.util.function.SerializableSupplier; +import org.heigit.ohsome.oshdb.util.mappable.OSHDBMapReducible; +import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; +import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; import org.heigit.ohsome.oshdb.util.taginterpreter.TagInterpreter; import org.jetbrains.annotations.NotNull; import org.locationtech.jts.geom.Geometry; diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerIgniteScanQuery.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerIgniteScanQuery.java index 2e03bba1b..ac69019a1 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerIgniteScanQuery.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerIgniteScanQuery.java @@ -33,9 +33,6 @@ import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer; import org.heigit.ohsome.oshdb.api.mapreducer.backend.Kernels.CellProcessor; import org.heigit.ohsome.oshdb.api.mapreducer.backend.OSHDBIgniteMapReduceComputeTask.CancelableIgniteMapReduceJob; -import org.heigit.ohsome.oshdb.api.object.OSHDBMapReducible; -import org.heigit.ohsome.oshdb.api.object.OSMContribution; -import org.heigit.ohsome.oshdb.api.object.OSMEntitySnapshot; import org.heigit.ohsome.oshdb.grid.GridOSHEntity; import org.heigit.ohsome.oshdb.index.XYGridTree.CellIdRange; import org.heigit.ohsome.oshdb.osm.OSMType; @@ -49,6 +46,9 @@ import org.heigit.ohsome.oshdb.util.function.SerializableBinaryOperator; import org.heigit.ohsome.oshdb.util.function.SerializableFunction; import org.heigit.ohsome.oshdb.util.function.SerializableSupplier; +import org.heigit.ohsome.oshdb.util.mappable.OSHDBMapReducible; +import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; +import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; import org.heigit.ohsome.oshdb.util.taginterpreter.TagInterpreter; import org.jetbrains.annotations.NotNull; import org.locationtech.jts.geom.Geometry; diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerJdbc.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerJdbc.java index 60dc09da8..872e33368 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerJdbc.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerJdbc.java @@ -17,11 +17,11 @@ import org.heigit.ohsome.oshdb.api.db.OSHDBJdbc; import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer; import org.heigit.ohsome.oshdb.api.mapreducer.backend.Kernels.CancelableProcessStatus; -import org.heigit.ohsome.oshdb.api.object.OSHDBMapReducible; import org.heigit.ohsome.oshdb.grid.GridOSHEntity; import org.heigit.ohsome.oshdb.index.XYGridTree.CellIdRange; import org.heigit.ohsome.oshdb.util.TableNames; import org.heigit.ohsome.oshdb.util.exceptions.OSHDBTimeoutException; +import org.heigit.ohsome.oshdb.util.mappable.OSHDBMapReducible; abstract class MapReducerJdbc extends MapReducer implements CancelableProcessStatus { diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerJdbcMultithread.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerJdbcMultithread.java index b20f6d7aa..6f75856c3 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerJdbcMultithread.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerJdbcMultithread.java @@ -8,15 +8,15 @@ import org.heigit.ohsome.oshdb.api.db.OSHDBDatabase; import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer; import org.heigit.ohsome.oshdb.api.mapreducer.backend.Kernels.CellProcessor; -import org.heigit.ohsome.oshdb.api.object.OSHDBMapReducible; -import org.heigit.ohsome.oshdb.api.object.OSMContribution; -import org.heigit.ohsome.oshdb.api.object.OSMEntitySnapshot; import org.heigit.ohsome.oshdb.index.XYGridTree.CellIdRange; import org.heigit.ohsome.oshdb.util.celliterator.CellIterator; import org.heigit.ohsome.oshdb.util.function.SerializableBiFunction; import org.heigit.ohsome.oshdb.util.function.SerializableBinaryOperator; import org.heigit.ohsome.oshdb.util.function.SerializableFunction; import org.heigit.ohsome.oshdb.util.function.SerializableSupplier; +import org.heigit.ohsome.oshdb.util.mappable.OSHDBMapReducible; +import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; +import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; import org.jetbrains.annotations.NotNull; import org.json.simple.parser.ParseException; diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerJdbcSinglethread.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerJdbcSinglethread.java index 9cce1b3ec..c8dcc1fd1 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerJdbcSinglethread.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/backend/MapReducerJdbcSinglethread.java @@ -9,9 +9,6 @@ import org.heigit.ohsome.oshdb.api.db.OSHDBDatabase; import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer; import org.heigit.ohsome.oshdb.api.mapreducer.backend.Kernels.CellProcessor; -import org.heigit.ohsome.oshdb.api.object.OSHDBMapReducible; -import org.heigit.ohsome.oshdb.api.object.OSMContribution; -import org.heigit.ohsome.oshdb.api.object.OSMEntitySnapshot; import org.heigit.ohsome.oshdb.grid.GridOSHEntity; import org.heigit.ohsome.oshdb.index.XYGridTree.CellIdRange; import org.heigit.ohsome.oshdb.util.celliterator.CellIterator; @@ -19,6 +16,9 @@ import org.heigit.ohsome.oshdb.util.function.SerializableBinaryOperator; import org.heigit.ohsome.oshdb.util.function.SerializableFunction; import org.heigit.ohsome.oshdb.util.function.SerializableSupplier; +import org.heigit.ohsome.oshdb.util.mappable.OSHDBMapReducible; +import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; +import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; import org.jetbrains.annotations.NotNull; import org.json.simple.parser.ParseException; diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/object/OSHDBMapReducible.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/object/OSHDBMapReducible.java deleted file mode 100644 index b942afe28..000000000 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/object/OSHDBMapReducible.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.heigit.ohsome.oshdb.api.object; - -/** - * Marks a class as possible data type of an OSHDB-MapReducer. - */ -public interface OSHDBMapReducible { -} diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/object/OSMContribution.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/object/OSMContributionImpl.java similarity index 52% rename from oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/object/OSMContribution.java rename to oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/object/OSMContributionImpl.java index 5f87eb72c..4f4100b11 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/object/OSMContribution.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/object/OSMContributionImpl.java @@ -11,11 +11,13 @@ import org.heigit.ohsome.oshdb.osm.OSMWay; import org.heigit.ohsome.oshdb.util.celliterator.CellIterator.IterateAllEntry; import org.heigit.ohsome.oshdb.util.celliterator.ContributionType; +import org.heigit.ohsome.oshdb.util.celliterator.LazyEvaluatedContributionTypes; import org.heigit.ohsome.oshdb.util.celliterator.LazyEvaluatedObject; +import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; import org.locationtech.jts.geom.Geometry; /** - * Holds information about a single modification ("contribution") of a single entity in database. + * Information about a single modification ("contribution") of a single OSM object. * *

* It holds the information about: @@ -28,17 +30,17 @@ * modification of a geometry, altering of the tag list, etc.) * */ -public class OSMContribution implements OSHDBMapReducible, Comparable { +public class OSMContributionImpl implements OSMContribution { private final IterateAllEntry data; - public OSMContribution(IterateAllEntry data) { + public OSMContributionImpl(IterateAllEntry data) { this.data = data; } /** * Creates a copy of the given contribution object with an updated before/after geometry. */ - public OSMContribution( + public OSMContributionImpl( OSMContribution other, Geometry reclippedGeometryBefore, Geometry reclippedGeometryAfter @@ -52,178 +54,77 @@ public OSMContribution( /** * Creates a copy of the given contribution object with an updated before/after geometry. */ - public OSMContribution( + public OSMContributionImpl( OSMContribution other, LazyEvaluatedObject reclippedGeometryBefore, LazyEvaluatedObject reclippedGeometryAfter ) { this.data = new IterateAllEntry( - other.data.timestamp, - other.data.osmEntity, - other.data.previousOsmEntity, - other.data.oshEntity, + other.getTimestamp(), + other.getEntityAfter(), + other.getEntityBefore(), + other.getOSHEntity(), reclippedGeometryAfter, reclippedGeometryBefore, - other.data.unclippedGeometry, - other.data.unclippedPreviousGeometry, - other.data.activities, - other.data.changeset + new LazyEvaluatedObject<>(other::getGeometryUnclippedAfter), + new LazyEvaluatedObject<>(other::getGeometryUnclippedBefore), + new LazyEvaluatedContributionTypes(other::is), + other.getChangesetId() ); } - /** - * Returns the timestamp at which this data modification has happened. - * - * @return the modification timestamp as a OSHDBTimestamp object - */ + @Override public OSHDBTimestamp getTimestamp() { return data.timestamp; } - /** - * Returns the geometry of the entity before this modification clipped to the requested area of - * interest. May be `null` if this is an entity creation. - * - * @return a JTS Geometry object representing the entity's state before the modification (clipped - * to the respective area of interest) - */ + @Override public Geometry getGeometryBefore() { return data.previousGeometry.get(); } - /** - * Returns the geometry of the entity before this modification. This is the full (unclipped) - * geometry of the entity. May be `null` if this is an entity creation. - * - * @return a JTS Geometry object representing the entity's state before the modification (not - * clipped to the respective area of interest) - */ + @Override public Geometry getGeometryUnclippedBefore() { return data.unclippedPreviousGeometry.get(); } - /** - * Returns the geometry of the entity after this modification clipped to the requested area of - * interest. May be `null` if this is an entity deletion. - * - * @return a JTS Geometry object representing the entity's state after the modification (clipped - * to the respective area of interest) - */ + @Override public Geometry getGeometryAfter() { return data.geometry.get(); } - /** - * Returns the geometry of the entity after this modification. This is the full (unclipped) - * geometry of the entity. May be `null` if this is an entity deletion. - * - * @return a JTS Geometry object representing the entity's state after the modification (not - * clipped to the respective area of interest) - */ + @Override public Geometry getGeometryUnclippedAfter() { return data.unclippedGeometry.get(); } - /** - * Returns the entity object in its state before this modification. - * Is `null` if this is a entity creation. - * - * @return the entity object as it was before this modification - */ + @Override public OSMEntity getEntityBefore() { return data.previousOsmEntity; } - /** - * Returns the entity object in its state after this modification. - * - *

- * If this is a entity deletion, the returned entity will have the visible flag set to false: - * `entity.getEntityAfter().isVisible == false` - *

- * - * @return the entity object as it was after this modification - */ + @Override public OSMEntity getEntityAfter() { return data.osmEntity; } - /** - * The (parent) osh entity of the osm entities involved in this contribution. - * - * @return the OSHEntity object of this contribution - */ + @Override public OSHEntity getOSHEntity() { return data.oshEntity; } - /** - * Checks if this contribution is of the given contribution type. - * - *

- * It can be one or more of: - *

- *
    - *
  • CREATION
    • - *
  • DELETION
    • - *
  • TAG_CHANGE
    • - *
  • GEOMETRY_CHANGE
    • - *
- * - *

- * If this is a entity creation or deletion, the other flags are not set (even though one might - * argue that a just created object clearly has a different geometry than before, for example). - *

- * - * @return a set of modification type this contribution made on the underlying data - */ + @Override public boolean is(ContributionType contributionType) { return data.activities.contains(contributionType); } - /** - * Determined the type of modification this contribution has made. - * - *

- * Can be one or more of: - *

- *
    - *
  • CREATION
    • - *
  • DELETION
    • - *
  • TAG_CHANGE
    • - *
  • GEOMETRY_CHANGE
    • - *
- * - *

- * If this is a entity creation or deletion, the other flags are not set (even though one might - * argue that a just created object clearly has a different geometry than before, for example). - *

- * - * @return a set of modification type this contribution made on the underlying data - */ + @Override public EnumSet getContributionTypes() { return data.activities.get(); } - /** - * Returns the user id of the osm contributor responsible for this data modification. - * - *

- * This user id can be different from what one gets by calling `.getEntityAfter().getUserId()` - * if the contribution is a pure geometry change (i.e. the entity itself has not ben modified, - * but one or more of its child entities): - *

- * - *

- * If the entity is a way or relation, and in a contribution *"only"* the geometry has been - * changed, we can't find out the respective contributor's user id only by looking at the - * entity alone – instead, we need to iterate over all the entity's children to find the actual - * contributor's user id. - *

- * - * @return returns the user id of the contributor who made this modification - */ + @Override public int getContributorUserId() { // todo: optimizable if done directly in CellIterator?? OSMEntity entity = this.getEntityAfter(); @@ -272,20 +173,11 @@ public int getContributorUserId() { return userId; } - /** - * Returns the osm changeset id of the contribution. - * - * @return the id of the osm changeset represented by the current contribution object - */ + @Override public long getChangesetId() { return data.changeset; } - /** - * {@inheritDoc} - * - *

Note: this class has a natural ordering that is inconsistent with equals.

- */ @Override public int compareTo(@Nonnull OSMContribution other) { return ComparisonChain.start() diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/object/OSMEntitySnapshot.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/object/OSMEntitySnapshotImpl.java similarity index 57% rename from oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/object/OSMEntitySnapshot.java rename to oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/object/OSMEntitySnapshotImpl.java index b64fe73fa..21ca9157e 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/object/OSMEntitySnapshot.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/object/OSMEntitySnapshotImpl.java @@ -7,89 +7,66 @@ import org.heigit.ohsome.oshdb.osm.OSMEntity; import org.heigit.ohsome.oshdb.util.celliterator.CellIterator.IterateByTimestampEntry; import org.heigit.ohsome.oshdb.util.celliterator.LazyEvaluatedObject; +import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; import org.locationtech.jts.geom.Geometry; /** - * Stores information about a single data entity at a specific time "snapshot". + * Information about a single OSM object at a specific point in time ("snapshot"). * *

Alongside the entity and the timestamp, also the entity's geometry is provided.

*/ -public class OSMEntitySnapshot implements OSHDBMapReducible, Comparable { +public class OSMEntitySnapshotImpl implements + org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot { private final IterateByTimestampEntry data; - public OSMEntitySnapshot(IterateByTimestampEntry data) { + public OSMEntitySnapshotImpl(IterateByTimestampEntry data) { this.data = data; } /** * Creates a copy of the given entity snapshot object with an updated geometry. */ - public OSMEntitySnapshot(OSMEntitySnapshot other, Geometry reclippedGeometry) { + public OSMEntitySnapshotImpl(OSMEntitySnapshot other, Geometry reclippedGeometry) { this(other, new LazyEvaluatedObject<>(reclippedGeometry)); } /** * Creates a copy of the given entity snapshot object with an updated geometry. */ - public OSMEntitySnapshot( + public OSMEntitySnapshotImpl( OSMEntitySnapshot other, LazyEvaluatedObject reclippedGeometry ) { this.data = new IterateByTimestampEntry( - other.data.timestamp, - other.data.osmEntity, - other.data.oshEntity, + other.getTimestamp(), + other.getEntity(), + other.getOSHEntity(), reclippedGeometry, - other.data.unclippedGeometry + new LazyEvaluatedObject<>(other::getGeometryUnclipped) ); } - /** - * The timestamp for which the snapshot of this data entity has been obtained. - * - * @return snapshot timestamp as an OSHDBTimestamp object - */ + @Override public OSHDBTimestamp getTimestamp() { return data.timestamp; } - /** - * The geometry of this entity at the snapshot's timestamp clipped to the requested area of - * interest. - * - * @return the geometry as a JTS Geometry - */ + @Override public Geometry getGeometry() { return data.geometry.get(); } - /** - * The geometry of this entity at the snapshot's timestamp. This is the full (unclipped) geometry - * of the osm entity. - * - * @return the unclipped geometry of the osm entity snapshot as a JTS Geometry - */ + @Override public Geometry getGeometryUnclipped() { return data.unclippedGeometry.get(); } - /** - * The entity for which the snapshot has been obtained. - * - *

This is the (not deleted) version of a OSHEntity that was valid at the provided snapshot - * timestamp.

- * - * @return the OSMEntity object of this snapshot - */ + @Override public OSMEntity getEntity() { return data.osmEntity; } - /** - * The (parent) osh entity of the osm entity for which the snapshot has been obtained. - * - * @return the OSHEntity object corresponding to this snapshot - */ + @Override public OSHEntity getOSHEntity() { return data.oshEntity; } diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestCollect.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestCollect.java index dd4712593..dce304a3c 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestCollect.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestCollect.java @@ -13,8 +13,8 @@ import org.heigit.ohsome.oshdb.api.db.OSHDBH2; import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer; import org.heigit.ohsome.oshdb.api.mapreducer.OSMContributionView; -import org.heigit.ohsome.oshdb.api.object.OSMContribution; import org.heigit.ohsome.oshdb.osm.OSMType; +import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; import org.heigit.ohsome.oshdb.util.time.OSHDBTimestamps; import org.junit.Test; diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestFlatMapAggregate.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestFlatMapAggregate.java index cebaeca0c..dd55d085c 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestFlatMapAggregate.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestFlatMapAggregate.java @@ -15,8 +15,8 @@ import org.heigit.ohsome.oshdb.api.db.OSHDBH2; import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer; import org.heigit.ohsome.oshdb.api.mapreducer.OSMContributionView; -import org.heigit.ohsome.oshdb.api.object.OSMContribution; import org.heigit.ohsome.oshdb.osm.OSMType; +import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; import org.heigit.ohsome.oshdb.util.time.OSHDBTimestamps; import org.junit.Test; diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestFlatMapAggregateGroupedByEntity.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestFlatMapAggregateGroupedByEntity.java index 42c09ce81..340d1c848 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestFlatMapAggregateGroupedByEntity.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestFlatMapAggregateGroupedByEntity.java @@ -12,9 +12,9 @@ import org.heigit.ohsome.oshdb.api.db.OSHDBH2; import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer; import org.heigit.ohsome.oshdb.api.mapreducer.OSMContributionView; -import org.heigit.ohsome.oshdb.api.object.OSMContribution; import org.heigit.ohsome.oshdb.osm.OSMType; import org.heigit.ohsome.oshdb.util.celliterator.ContributionType; +import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; import org.heigit.ohsome.oshdb.util.time.OSHDBTimestamps; import org.junit.Test; diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestFlatMapReduce.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestFlatMapReduce.java index 7a050bb92..367031557 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestFlatMapReduce.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestFlatMapReduce.java @@ -15,8 +15,8 @@ import org.heigit.ohsome.oshdb.api.db.OSHDBH2; import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer; import org.heigit.ohsome.oshdb.api.mapreducer.OSMContributionView; -import org.heigit.ohsome.oshdb.api.object.OSMContribution; import org.heigit.ohsome.oshdb.osm.OSMType; +import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; import org.heigit.ohsome.oshdb.util.time.OSHDBTimestamps; import org.junit.Test; diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestFlatMapReduceGroupedByEntity.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestFlatMapReduceGroupedByEntity.java index a43f497ab..517289554 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestFlatMapReduceGroupedByEntity.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestFlatMapReduceGroupedByEntity.java @@ -9,10 +9,10 @@ import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer; import org.heigit.ohsome.oshdb.api.mapreducer.OSMContributionView; import org.heigit.ohsome.oshdb.api.mapreducer.OSMEntitySnapshotView; -import org.heigit.ohsome.oshdb.api.object.OSMContribution; -import org.heigit.ohsome.oshdb.api.object.OSMEntitySnapshot; import org.heigit.ohsome.oshdb.osm.OSMType; import org.heigit.ohsome.oshdb.util.celliterator.ContributionType; +import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; +import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; import org.heigit.ohsome.oshdb.util.time.OSHDBTimestamps; import org.junit.Test; diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestForEach.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestForEach.java index fd5266d1e..42970000d 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestForEach.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestForEach.java @@ -8,8 +8,8 @@ import org.heigit.ohsome.oshdb.api.db.OSHDBH2; import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer; import org.heigit.ohsome.oshdb.api.mapreducer.OSMContributionView; -import org.heigit.ohsome.oshdb.api.object.OSMContribution; import org.heigit.ohsome.oshdb.osm.OSMType; +import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; import org.heigit.ohsome.oshdb.util.time.OSHDBTimestamps; import org.junit.Test; diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestHelpersOSMContributionView.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestHelpersOSMContributionView.java index 78b77c313..bb7e8a389 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestHelpersOSMContributionView.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestHelpersOSMContributionView.java @@ -13,9 +13,9 @@ import org.heigit.ohsome.oshdb.api.generic.WeightedValue; import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer; import org.heigit.ohsome.oshdb.api.mapreducer.OSMContributionView; -import org.heigit.ohsome.oshdb.api.object.OSMContribution; import org.heigit.ohsome.oshdb.osm.OSMType; import org.heigit.ohsome.oshdb.util.celliterator.ContributionType; +import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; import org.heigit.ohsome.oshdb.util.time.OSHDBTimestamps; import org.junit.Test; diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestHelpersOSMEntitySnapshotView.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestHelpersOSMEntitySnapshotView.java index 6904b27ff..0c4e7fd2d 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestHelpersOSMEntitySnapshotView.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestHelpersOSMEntitySnapshotView.java @@ -11,8 +11,8 @@ import org.heigit.ohsome.oshdb.api.generic.WeightedValue; import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer; import org.heigit.ohsome.oshdb.api.mapreducer.OSMEntitySnapshotView; -import org.heigit.ohsome.oshdb.api.object.OSMEntitySnapshot; import org.heigit.ohsome.oshdb.osm.OSMType; +import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; import org.heigit.ohsome.oshdb.util.time.OSHDBTimestamps; import org.junit.Test; diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestLambdaFilter.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestLambdaFilter.java index a83f0ca1f..afa026e83 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestLambdaFilter.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestLambdaFilter.java @@ -9,9 +9,9 @@ import org.heigit.ohsome.oshdb.api.db.OSHDBH2; import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer; import org.heigit.ohsome.oshdb.api.mapreducer.OSMContributionView; -import org.heigit.ohsome.oshdb.api.object.OSMContribution; import org.heigit.ohsome.oshdb.osm.OSMType; import org.heigit.ohsome.oshdb.util.celliterator.ContributionType; +import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; import org.heigit.ohsome.oshdb.util.time.OSHDBTimestamps; import org.junit.Test; diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapAggregateByGeometry.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapAggregateByGeometry.java index 2e1fddecd..93abc32a0 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapAggregateByGeometry.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapAggregateByGeometry.java @@ -17,11 +17,11 @@ import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer; import org.heigit.ohsome.oshdb.api.mapreducer.OSMContributionView; import org.heigit.ohsome.oshdb.api.mapreducer.OSMEntitySnapshotView; -import org.heigit.ohsome.oshdb.api.object.OSMContribution; -import org.heigit.ohsome.oshdb.api.object.OSMEntitySnapshot; import org.heigit.ohsome.oshdb.osm.OSMType; import org.heigit.ohsome.oshdb.util.geometry.Geo; import org.heigit.ohsome.oshdb.util.geometry.OSHDBGeometryBuilder; +import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; +import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; import org.heigit.ohsome.oshdb.util.time.OSHDBTimestamps; import org.junit.Test; import org.locationtech.jts.geom.Polygon; diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapAggregateByIndex.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapAggregateByIndex.java index bced580ba..db5020387 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapAggregateByIndex.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapAggregateByIndex.java @@ -13,9 +13,9 @@ import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer; import org.heigit.ohsome.oshdb.api.mapreducer.OSMContributionView; import org.heigit.ohsome.oshdb.api.mapreducer.OSMEntitySnapshotView; -import org.heigit.ohsome.oshdb.api.object.OSMContribution; -import org.heigit.ohsome.oshdb.api.object.OSMEntitySnapshot; import org.heigit.ohsome.oshdb.osm.OSMType; +import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; +import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; import org.heigit.ohsome.oshdb.util.time.OSHDBTimestamps; import org.junit.Test; diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapAggregateByTimestamp.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapAggregateByTimestamp.java index b19d3c25e..be96b23af 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapAggregateByTimestamp.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapAggregateByTimestamp.java @@ -13,10 +13,10 @@ import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer; import org.heigit.ohsome.oshdb.api.mapreducer.OSMContributionView; import org.heigit.ohsome.oshdb.api.mapreducer.OSMEntitySnapshotView; -import org.heigit.ohsome.oshdb.api.object.OSMContribution; -import org.heigit.ohsome.oshdb.api.object.OSMEntitySnapshot; import org.heigit.ohsome.oshdb.osm.OSMType; import org.heigit.ohsome.oshdb.util.exceptions.OSHDBInvalidTimestampException; +import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; +import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; import org.heigit.ohsome.oshdb.util.time.OSHDBTimestamps; import org.junit.Test; diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduce.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduce.java index b06c74de8..12d6cf913 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduce.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestMapReduce.java @@ -11,11 +11,11 @@ import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer; import org.heigit.ohsome.oshdb.api.mapreducer.OSMContributionView; import org.heigit.ohsome.oshdb.api.mapreducer.OSMEntitySnapshotView; -import org.heigit.ohsome.oshdb.api.object.OSMContribution; -import org.heigit.ohsome.oshdb.api.object.OSMEntitySnapshot; import org.heigit.ohsome.oshdb.osm.OSMType; import org.heigit.ohsome.oshdb.util.exceptions.OSHDBTimeoutException; import org.heigit.ohsome.oshdb.util.function.SerializableFunction; +import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; +import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; import org.heigit.ohsome.oshdb.util.time.OSHDBTimestamps; import org.junit.Test; diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestOSHDBFilter.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestOSHDBFilter.java index e8942eb58..7d91a3928 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestOSHDBFilter.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestOSHDBFilter.java @@ -10,10 +10,10 @@ import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer; import org.heigit.ohsome.oshdb.api.mapreducer.OSMContributionView; import org.heigit.ohsome.oshdb.api.mapreducer.OSMEntitySnapshotView; -import org.heigit.ohsome.oshdb.api.object.OSMContribution; -import org.heigit.ohsome.oshdb.api.object.OSMEntitySnapshot; import org.heigit.ohsome.oshdb.filter.FilterParser; import org.heigit.ohsome.oshdb.osm.OSMType; +import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; +import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; import org.junit.Test; diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestOSMContributionGetContributorUserId.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestOSMContributionGetContributorUserId.java index 3b0a61a99..916cebfe7 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestOSMContributionGetContributorUserId.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestOSMContributionGetContributorUserId.java @@ -7,7 +7,7 @@ import java.util.EnumSet; import java.util.List; import org.heigit.ohsome.oshdb.OSHDBTimestamp; -import org.heigit.ohsome.oshdb.api.object.OSMContribution; +import org.heigit.ohsome.oshdb.api.object.OSMContributionImpl; import org.heigit.ohsome.oshdb.impl.osh.OSHNodeImpl; import org.heigit.ohsome.oshdb.impl.osh.OSHWayImpl; import org.heigit.ohsome.oshdb.osh.OSHEntity; @@ -19,6 +19,7 @@ import org.heigit.ohsome.oshdb.util.celliterator.CellIterator.IterateAllEntry; import org.heigit.ohsome.oshdb.util.celliterator.ContributionType; import org.heigit.ohsome.oshdb.util.celliterator.LazyEvaluatedContributionTypes; +import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; import org.junit.Test; /** @@ -35,7 +36,7 @@ public TestOSMContributionGetContributorUserId() throws Exception { @Test public void node() throws Exception { // timestamp match - OSMContribution c = new OSMContribution(new IterateAllEntry( + OSMContribution c = new OSMContributionImpl(new IterateAllEntry( new OSHDBTimestamp(123), new OSMNode(1L, 1, 123L, 1L, 7, new int[]{}, 0, 0), null, dummyOshEntity, @@ -45,7 +46,7 @@ public void node() throws Exception { )); assertEquals(7, c.getContributorUserId()); // contribution type match - c = new OSMContribution(new IterateAllEntry( + c = new OSMContributionImpl(new IterateAllEntry( new OSHDBTimestamp(123), new OSMNode(1L, 1, 122L, 1L, 7, new int[] {}, 0, 0), null, dummyOshEntity, @@ -54,7 +55,7 @@ public void node() throws Exception { 1L )); assertEquals(7, c.getContributorUserId()); - c = new OSMContribution(new IterateAllEntry( + c = new OSMContributionImpl(new IterateAllEntry( new OSHDBTimestamp(123), new OSMNode(1L, 2, 122L, 2L, 7, new int[] { 3, 4 }, 0, 0), new OSMNode(1L, 1, 121L, 1L, 6, new int[] { 1, 2 }, 0, 0), @@ -64,7 +65,7 @@ public void node() throws Exception { 2L )); assertEquals(7, c.getContributorUserId()); - c = new OSMContribution(new IterateAllEntry( + c = new OSMContributionImpl(new IterateAllEntry( new OSHDBTimestamp(123), // negative version == isVisible = false new OSMNode(1L, -2, 122L, 2L, 7, new int[] {}, 0, 0), @@ -76,7 +77,7 @@ public void node() throws Exception { )); // non-match assertEquals(7, c.getContributorUserId()); - c = new OSMContribution(new IterateAllEntry( + c = new OSMContributionImpl(new IterateAllEntry( new OSHDBTimestamp(123), new OSMNode(1L, 1, 122L, 1L, 7, new int[] {}, 0, 0), new OSMNode(1L, 1, 122L, 1L, 7, new int[] {}, 0, 0), @@ -90,7 +91,7 @@ public void node() throws Exception { @Test public void wayDirect() throws Exception { - OSMContribution c = new OSMContribution(new IterateAllEntry( + OSMContribution c = new OSMContributionImpl(new IterateAllEntry( new OSHDBTimestamp(123), new OSMWay(1L, 1, 123L, 1L, 7, new int[] {}, new OSMMember[] {}), null, dummyOshEntity, @@ -112,7 +113,7 @@ public void wayIndirect() throws Exception { 1L, 1, 122L, 1L, 1, new int[] {}, new OSMMember[] { new OSMMember(3, OSMType.NODE, 0, OSHNodeImpl.build(versions)) }); - OSMContribution c = new OSMContribution(new IterateAllEntry( + OSMContribution c = new OSMContributionImpl(new IterateAllEntry( new OSHDBTimestamp(123), entity, entity, dummyOshEntity, @@ -125,7 +126,7 @@ public void wayIndirect() throws Exception { @Test public void relationDirect() throws Exception { - OSMContribution c = new OSMContribution(new IterateAllEntry( + OSMContribution c = new OSMContributionImpl(new IterateAllEntry( new OSHDBTimestamp(123), new OSMRelation(1L, 1, 123L, 1L, 7, new int[] {}, new OSMMember[] {}), null, @@ -154,7 +155,7 @@ public void relationIndirectWay() throws Exception { 1L, 1, 122L, 1L, 1, new int[] {}, new OSMMember[] { new OSMMember(3, OSMType.WAY, 0, OSHWayImpl.build(versions, Collections.emptyList())) }); - OSMContribution c = new OSMContribution(new IterateAllEntry( + OSMContribution c = new OSMContributionImpl(new IterateAllEntry( new OSHDBTimestamp(123), entity, entity, dummyOshEntity, @@ -184,7 +185,7 @@ public void relationIndirectWayNode() throws Exception { OSHWayImpl.build(versions, Collections.singletonList(OSHNodeImpl.build(nodeVersions))) ) }); - OSMContribution c = new OSMContribution(new IterateAllEntry( + OSMContribution c = new OSMContributionImpl(new IterateAllEntry( new OSHDBTimestamp(123), entity, entity, dummyOshEntity, diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestOSMDataFilters.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestOSMDataFilters.java index dee75f7df..ef4080967 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestOSMDataFilters.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestOSMDataFilters.java @@ -17,9 +17,9 @@ import org.heigit.ohsome.oshdb.api.db.OSHDBH2; import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer; import org.heigit.ohsome.oshdb.api.mapreducer.OSMEntitySnapshotView; -import org.heigit.ohsome.oshdb.api.object.OSMEntitySnapshot; import org.heigit.ohsome.oshdb.osm.OSMType; import org.heigit.ohsome.oshdb.util.geometry.OSHDBGeometryBuilder; +import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; import org.heigit.ohsome.oshdb.util.tagtranslator.OSMTag; import org.heigit.ohsome.oshdb.util.tagtranslator.OSMTagKey; import org.heigit.ohsome.oshdb.util.time.OSHDBTimestamps; diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestQuantiles.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestQuantiles.java index b09b3c28c..02df6d3cd 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestQuantiles.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestQuantiles.java @@ -14,8 +14,8 @@ import org.heigit.ohsome.oshdb.api.mapreducer.MapAggregator; import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer; import org.heigit.ohsome.oshdb.api.mapreducer.OSMEntitySnapshotView; -import org.heigit.ohsome.oshdb.api.object.OSMEntitySnapshot; import org.heigit.ohsome.oshdb.osm.OSMType; +import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; import org.heigit.ohsome.oshdb.util.time.OSHDBTimestamps; import org.junit.Test; diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestStream.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestStream.java index 58a444cd9..42b49b1eb 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestStream.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestStream.java @@ -8,8 +8,8 @@ import org.heigit.ohsome.oshdb.api.db.OSHDBH2; import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer; import org.heigit.ohsome.oshdb.api.mapreducer.OSMContributionView; -import org.heigit.ohsome.oshdb.api.object.OSMContribution; import org.heigit.ohsome.oshdb.osm.OSMType; +import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; import org.heigit.ohsome.oshdb.util.time.OSHDBTimestamps; import org.junit.Test; diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/mappable/OSHDBMapReducible.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/mappable/OSHDBMapReducible.java new file mode 100644 index 000000000..c141f49e1 --- /dev/null +++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/mappable/OSHDBMapReducible.java @@ -0,0 +1,6 @@ +package org.heigit.ohsome.oshdb.util.mappable; + +/** + * Marks a class as possible data type of an OSHDB-MapReducer. + */ +public interface OSHDBMapReducible {} diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/mappable/OSMContribution.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/mappable/OSMContribution.java new file mode 100644 index 000000000..ab0389285 --- /dev/null +++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/mappable/OSMContribution.java @@ -0,0 +1,155 @@ +package org.heigit.ohsome.oshdb.util.mappable; + +import java.util.EnumSet; +import org.heigit.ohsome.oshdb.OSHDBTimestamp; +import org.heigit.ohsome.oshdb.osh.OSHEntity; +import org.heigit.ohsome.oshdb.osm.OSMEntity; +import org.heigit.ohsome.oshdb.util.celliterator.ContributionType; +import org.locationtech.jts.geom.Geometry; + +/** + * A modification ("contribution") of a single OSM object. + */ +public interface OSMContribution extends OSHDBMapReducible, Comparable { + + /** + * Returns the timestamp at which this data modification has happened. + * + * @return the modification timestamp as a OSHDBTimestamp object + */ + OSHDBTimestamp getTimestamp(); + + /** + * Returns the geometry of the entity before this modification clipped to the requested area of + * interest. May be `null` if this is an entity creation. + * + * @return a JTS Geometry object representing the entity's state before the modification (clipped + * to the respective area of interest) + */ + Geometry getGeometryBefore(); + + /** + * Returns the geometry of the entity before this modification. This is the full (unclipped) + * geometry of the entity. May be `null` if this is an entity creation. + * + * @return a JTS Geometry object representing the entity's state before the modification (not + * clipped to the respective area of interest) + */ + Geometry getGeometryUnclippedBefore(); + + /** + * Returns the geometry of the entity after this modification clipped to the requested area of + * interest. May be `null` if this is an entity deletion. + * + * @return a JTS Geometry object representing the entity's state after the modification (clipped + * to the respective area of interest) + */ + Geometry getGeometryAfter(); + + /** + * Returns the geometry of the entity after this modification. This is the full (unclipped) + * geometry of the entity. May be `null` if this is an entity deletion. + * + * @return a JTS Geometry object representing the entity's state after the modification (not + * clipped to the respective area of interest) + */ + Geometry getGeometryUnclippedAfter(); + + /** + * Returns the entity object in its state before this modification. + * Is `null` if this is a entity creation. + * + * @return the entity object as it was before this modification + */ + OSMEntity getEntityBefore(); + + /** + * Returns the entity object in its state after this modification. + * + *

+ * If this is a entity deletion, the returned entity will have the visible flag set to false: + * `entity.getEntityAfter().isVisible == false` + *

+ * + * @return the entity object as it was after this modification + */ + OSMEntity getEntityAfter(); + + /** + * The (parent) osh entity of the osm entities involved in this contribution. + * + * @return the OSHEntity object of this contribution + */ + OSHEntity getOSHEntity(); + + /** + * Checks if this contribution is of the given contribution type. + * + *

+ * It can be one or more of: + *

+ *
    + *
  • CREATION
    • + *
  • DELETION
    • + *
  • TAG_CHANGE
    • + *
  • GEOMETRY_CHANGE
    • + *
+ * + *

+ * If this is a entity creation or deletion, the other flags are not set (even though one might + * argue that a just created object clearly has a different geometry than before, for example). + *

+ * + * @return a set of modification type this contribution made on the underlying data + */ + boolean is(ContributionType contributionType); + + /** + * Determined the type of modification this contribution has made. + * + *

+ * Can be one or more of: + *

+ *
    + *
  • CREATION
    • + *
  • DELETION
    • + *
  • TAG_CHANGE
    • + *
  • GEOMETRY_CHANGE
    • + *
+ * + *

+ * If this is a entity creation or deletion, the other flags are not set (even though one might + * argue that a just created object clearly has a different geometry than before, for example). + *

+ * + * @return a set of modification type this contribution made on the underlying data + */ + EnumSet getContributionTypes(); + + /** + * Returns the user id of the osm contributor responsible for this data modification. + * + *

+ * This user id can be different from what one gets by calling `.getEntityAfter().getUserId()` + * if the contribution is a pure geometry change (i.e. the entity itself has not ben modified, + * but one or more of its child entities): + *

+ * + *

+ * If the entity is a way or relation, and in a contribution *"only"* the geometry has been + * changed, we can't find out the respective contributor's user id only by looking at the + * entity alone – instead, we need to iterate over all the entity's children to find the actual + * contributor's user id. + *

+ * + * @return returns the user id of the contributor who made this modification + */ + int getContributorUserId(); + + /** + * Returns the osm changeset id of the contribution. + * + * @return the id of the osm changeset represented by the current contribution object + */ + long getChangesetId(); +} diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/mappable/OSMEntitySnapshot.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/mappable/OSMEntitySnapshot.java new file mode 100644 index 000000000..8048c0d98 --- /dev/null +++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/mappable/OSMEntitySnapshot.java @@ -0,0 +1,51 @@ +package org.heigit.ohsome.oshdb.util.mappable; + +import org.heigit.ohsome.oshdb.OSHDBTimestamp; +import org.heigit.ohsome.oshdb.osh.OSHEntity; +import org.heigit.ohsome.oshdb.osm.OSMEntity; +import org.locationtech.jts.geom.Geometry; + +/** + * Information about a single OSM object at a specific point in time ("snapshot"). + */ +public interface OSMEntitySnapshot extends OSHDBMapReducible, Comparable { + /** + * The timestamp for which the snapshot of this data entity has been obtained. + * + * @return snapshot timestamp as an OSHDBTimestamp object + */ + OSHDBTimestamp getTimestamp(); + + /** + * The geometry of this entity at the snapshot's timestamp clipped to the requested area of + * interest. + * + * @return the geometry as a JTS Geometry + */ + Geometry getGeometry(); + + /** + * The geometry of this entity at the snapshot's timestamp. This is the full (unclipped) geometry + * of the osm entity. + * + * @return the unclipped geometry of the osm entity snapshot as a JTS Geometry + */ + Geometry getGeometryUnclipped(); + + /** + * The entity for which the snapshot has been obtained. + * + *

This is the (not deleted) version of a OSHEntity that was valid at the provided snapshot + * timestamp.

+ * + * @return the OSMEntity object of this snapshot + */ + OSMEntity getEntity(); + + /** + * The (parent) osh entity of the osm entity for which the snapshot has been obtained. + * + * @return the OSHEntity object corresponding to this snapshot + */ + OSHEntity getOSHEntity(); +} diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/mappable/package-info.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/mappable/package-info.java new file mode 100644 index 000000000..43c4f4f4e --- /dev/null +++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/mappable/package-info.java @@ -0,0 +1,4 @@ +/** + * Interfaces for "mappable" objects in the oshdb API. + */ +package org.heigit.ohsome.oshdb.util.mappable; From e93f91226c487d305195d1e87ea18eab2acbc5a6 Mon Sep 17 00:00:00 2001 From: Martin Raifer Date: Fri, 14 May 2021 12:44:33 +0200 Subject: [PATCH 02/34] prepare methods for filtering snapshots/contributions directly --- .../ohsome/oshdb/filter/FilterExpression.java | 30 ++++ .../ohsome/oshdb/filter/NegatableFilter.java | 12 ++ .../ohsome/oshdb/filter/ApplyOSHTest.java | 2 +- .../filter/ApplyOSMContributionTest.java | 128 ++++++++++++++++++ .../filter/ApplyOSMEntitySnapshotTest.java | 67 +++++++++ .../oshdb/filter/ApplyOSMGeometryTest.java | 2 +- .../ohsome/oshdb/filter/ApplyOSMTest.java | 2 +- 7 files changed, 240 insertions(+), 3 deletions(-) create mode 100644 oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMContributionTest.java create mode 100644 oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMEntitySnapshotTest.java diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/FilterExpression.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/FilterExpression.java index 3b1bfe138..2e239c7f3 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/FilterExpression.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/FilterExpression.java @@ -9,6 +9,8 @@ import java.util.function.Supplier; import org.heigit.ohsome.oshdb.osh.OSHEntity; import org.heigit.ohsome.oshdb.osm.OSMEntity; +import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; +import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; import org.jetbrains.annotations.Contract; import org.locationtech.jts.geom.Geometry; @@ -40,6 +42,7 @@ public interface FilterExpression extends Serializable { */ @Contract(pure = true) default boolean applyOSH(OSHEntity entity) { + // (potentially slow) default implementation tests every version individually return Streams.stream(entity.getVersions()).anyMatch(this::applyOSM); } @@ -55,6 +58,7 @@ default boolean applyOSH(OSHEntity entity) { */ @Contract(pure = true) default boolean applyOSMGeometry(OSMEntity entity, Supplier geometrySupplier) { + // dummy implementation for basic filters: ignores the geometry, just looks at the OSM entity return applyOSM(entity); } @@ -73,6 +77,32 @@ default boolean applyOSMGeometry(OSMEntity entity, Geometry geometry) { return applyOSMGeometry(entity, () -> geometry); } + /** + * Apply a filter to a snapshot ({@link OSMEntitySnapshot}) of an OSM entity. + * + * @param snapshot a snapshot of the OSM entity to check + * @return true if the the OSM entity snapshot fulfills the specified filter, otherwise false. + */ + @Contract(pure = true) + default boolean applyOSMEntitySnapshot(OSMEntitySnapshot snapshot) { + return applyOSMGeometry(snapshot.getEntity(), snapshot::getGeometry); + } + + /** + * Apply a filter to a contribution ({@link OSMEntitySnapshot}) to an OSM entity. + * + *

A contribution matches the given filter if either the state of the OSM entity before the + * modification or the state of it after the modification matches the filter.

+ * + * @param contribution a modification of the OSM entity to check + * @return true if the the OSM contribution fulfills the specified filter, otherwise false. + */ + @Contract(pure = true) + default boolean applyOSMContribution(OSMContribution contribution) { + return applyOSMGeometry(contribution.getEntityAfter(), contribution::getGeometryAfter) + || applyOSMGeometry(contribution.getEntityBefore(), contribution::getGeometryBefore); + } + /** * Returns the opposite of the current filter expression. * diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/NegatableFilter.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/NegatableFilter.java index 17f56fd9f..a2e270b7e 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/NegatableFilter.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/NegatableFilter.java @@ -4,6 +4,8 @@ import javax.annotation.Nonnull; import org.heigit.ohsome.oshdb.osh.OSHEntity; import org.heigit.ohsome.oshdb.osm.OSMEntity; +import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; +import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; import org.locationtech.jts.geom.Geometry; /** @@ -44,6 +46,16 @@ public boolean applyOSMGeometry(OSMEntity entity, Supplier geometrySup return this.filter.applyOSMGeometry(entity, geometrySupplier) ^ this.negated; } + @Override + public boolean applyOSMEntitySnapshot(OSMEntitySnapshot snapshot) { + return this.filter.applyOSMEntitySnapshot(snapshot) ^ this.negated; + } + + @Override + public boolean applyOSMContribution(OSMContribution contribution) { + return this.filter.applyOSMContribution(contribution) ^ this.negated; + } + @Override public NegatableFilter negate() { return new NegatableFilter(this.filter, !this.negated); diff --git a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSHTest.java b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSHTest.java index 537e3ece8..be793e6de 100644 --- a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSHTest.java +++ b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSHTest.java @@ -7,7 +7,7 @@ import org.junit.Test; /** - * Tests the parsing of filters and the application to OSM entities. + * Tests the application of filters to OSH entities. */ public class ApplyOSHTest extends FilterTest { diff --git a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMContributionTest.java b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMContributionTest.java new file mode 100644 index 000000000..ea6425195 --- /dev/null +++ b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMContributionTest.java @@ -0,0 +1,128 @@ +package org.heigit.ohsome.oshdb.filter; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.EnumSet; +import java.util.Set; +import org.heigit.ohsome.oshdb.OSHDBTimestamp; +import org.heigit.ohsome.oshdb.osh.OSHEntity; +import org.heigit.ohsome.oshdb.osm.OSMEntity; +import org.heigit.ohsome.oshdb.util.celliterator.ContributionType; +import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; +import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; +import org.jetbrains.annotations.NotNull; +import org.junit.Test; +import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.geom.GeometryFactory; + +/** + * Tests the parsing of filters and the application to OSM entity snapshots. + */ +public class ApplyOSMContributionTest extends FilterTest { + private final GeometryFactory gf = new GeometryFactory(); + + private class TestOSMContribution implements OSMContribution { + public static final String UNSUPPORTED_IN_TEST = "not supported for TestOSMEntitySnapshot"; + private final OSMEntity entityBefore; + private final Geometry geometryBefore; + private final OSMEntity entityAfter; + private final Geometry geometryAfter; + private final long changsetId; + private final int contributorUserId; + private final Set contributionTypes; + + public TestOSMContribution( + OSMEntity entityBefore, Geometry geometryBefore, + OSMEntity entityAfter, Geometry geometryAfter, + long changesetId, int contributorUserId, Set contributionTypes) { + this.entityBefore = entityBefore; + this.geometryBefore = geometryBefore; + this.entityAfter = entityAfter; + this.geometryAfter = geometryAfter; + this.changsetId = changesetId; + this.contributorUserId = contributorUserId; + this.contributionTypes = contributionTypes; + } + + @Override + public OSHDBTimestamp getTimestamp() { + throw new UnsupportedOperationException(UNSUPPORTED_IN_TEST); + } + + @Override + public Geometry getGeometryBefore() { + return this.geometryBefore; + } + + @Override + public Geometry getGeometryUnclippedBefore() { + return this.geometryBefore; + } + + @Override + public Geometry getGeometryAfter() { + return this.geometryAfter; + } + + @Override + public Geometry getGeometryUnclippedAfter() { + return this.geometryAfter; + } + + @Override + public OSMEntity getEntityBefore() { + return this.entityBefore; + } + + @Override + public OSMEntity getEntityAfter() { + return this.entityAfter; + } + + @Override + public OSHEntity getOSHEntity() { + throw new UnsupportedOperationException(UNSUPPORTED_IN_TEST); + } + + @Override + public boolean is(ContributionType contributionType) { + return this.contributionTypes.contains(contributionType); + } + + @Override + public EnumSet getContributionTypes() { + return EnumSet.copyOf(this.contributionTypes); + } + + @Override + public int getContributorUserId() { + return this.contributorUserId; + } + + @Override + public long getChangesetId() { + return this.changsetId; + } + + @Override + public int compareTo(@NotNull OSMContribution contribution) { + throw new UnsupportedOperationException(UNSUPPORTED_IN_TEST); + } + } + + @Test + public void testBasicFallback() { + FilterExpression expression = parser.parse("geometry:point"); + final Geometry point = gf.createPoint(); + final Geometry line = gf.createLineString(); + final OSMEntity entity = createTestOSMEntityNode(); + final Set noType = EnumSet.noneOf(ContributionType.class); + assertTrue(expression.applyOSMContribution(new TestOSMContribution( + entity, point, entity, line, 1L, 2, noType + ))); + assertFalse(expression.applyOSMContribution(new TestOSMContribution( + entity, line, entity, line, 1L, 2, noType + ))); + } +} diff --git a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMEntitySnapshotTest.java b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMEntitySnapshotTest.java new file mode 100644 index 000000000..15af862d6 --- /dev/null +++ b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMEntitySnapshotTest.java @@ -0,0 +1,67 @@ +package org.heigit.ohsome.oshdb.filter; + +import static org.junit.Assert.assertTrue; + +import org.heigit.ohsome.oshdb.OSHDBTimestamp; +import org.heigit.ohsome.oshdb.osh.OSHEntity; +import org.heigit.ohsome.oshdb.osm.OSMEntity; +import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; +import org.jetbrains.annotations.NotNull; +import org.junit.Test; +import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.geom.GeometryFactory; + +/** + * Tests the parsing of filters and the application to OSM entity snapshots. + */ +public class ApplyOSMEntitySnapshotTest extends FilterTest { + private final GeometryFactory gf = new GeometryFactory(); + + private class TestOSMEntitySnapshot implements OSMEntitySnapshot { + public static final String UNSUPPORTED_IN_TEST = "not supported for TestOSMEntitySnapshot"; + private final OSMEntity entity; + private final Geometry geometry; + + public TestOSMEntitySnapshot(OSMEntity entity, Geometry geometry) { + this.entity = entity; + this.geometry = geometry; + } + + @Override + public OSHDBTimestamp getTimestamp() { + throw new UnsupportedOperationException(UNSUPPORTED_IN_TEST); + } + + @Override + public Geometry getGeometry() { + return this.geometry; + } + + @Override + public Geometry getGeometryUnclipped() { + return this.geometry; + } + + @Override + public OSMEntity getEntity() { + return this.entity; + } + + @Override + public OSHEntity getOSHEntity() { + throw new UnsupportedOperationException(UNSUPPORTED_IN_TEST); + } + + @Override + public int compareTo(@NotNull OSMEntitySnapshot snapshot) { + throw new UnsupportedOperationException(UNSUPPORTED_IN_TEST); + } + } + + @Test + public void testBasicFallback() { + FilterExpression expression = parser.parse("geometry:point"); + assertTrue(expression.applyOSMEntitySnapshot(new TestOSMEntitySnapshot( + createTestOSMEntityNode(), gf.createPoint()))); + } +} diff --git a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMGeometryTest.java b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMGeometryTest.java index 1bcfc48c9..0ae18a7e8 100644 --- a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMGeometryTest.java +++ b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMGeometryTest.java @@ -11,7 +11,7 @@ import org.locationtech.jts.geom.GeometryFactory; /** - * Tests the parsing of filters and the application to OSM entities. + * Tests the application of filters to OSM geometries. */ public class ApplyOSMGeometryTest extends FilterTest { private final GeometryFactory gf = new GeometryFactory(); diff --git a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMTest.java b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMTest.java index 871f7f887..65b09ad06 100644 --- a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMTest.java +++ b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMTest.java @@ -7,7 +7,7 @@ import org.junit.Test; /** - * Tests the parsing of filters and the application to OSM entities. + * Tests the application of filters to OSM entities. */ public class ApplyOSMTest extends FilterTest { @Test From 71445dbcfd15e7216dea03f25a8d4494b11e957a Mon Sep 17 00:00:00 2001 From: Martin Raifer Date: Fri, 14 May 2021 15:45:38 +0200 Subject: [PATCH 03/34] prepare changeset/contibutor filters --- .../oshdb/filter/ChangesetIdFilterEquals.java | 48 +++++++++++++++++++ .../filter/ContributorUserIdFilterEquals.java | 48 +++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ChangesetIdFilterEquals.java create mode 100644 oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ContributorUserIdFilterEquals.java diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ChangesetIdFilterEquals.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ChangesetIdFilterEquals.java new file mode 100644 index 000000000..57a8c1074 --- /dev/null +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ChangesetIdFilterEquals.java @@ -0,0 +1,48 @@ +package org.heigit.ohsome.oshdb.filter; + +import org.heigit.ohsome.oshdb.osm.OSMEntity; +import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; +import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; +import org.jetbrains.annotations.Contract; + +/** + * A filter which selects OSM contributions by a changeset id. + */ +public class ChangesetIdFilterEquals extends NegatableFilter { + final long changesetId; + + ChangesetIdFilterEquals(long changesetId) { + super(new FilterInternal() { + @Override + public boolean applyOSM(OSMEntity entity) { + return true; + } + + @Override + public boolean applyOSMContribution(OSMContribution contribution) { + return contribution.getChangesetId() == changesetId; + } + + @Override + public boolean applyOSMEntitySnapshot(OSMEntitySnapshot ignored) { + throw new IllegalStateException("changeset filter is not applicable to entity snapshots"); + } + }); + this.changesetId = changesetId; + } + + /** + * Returns the OSM type of this filter. + * + * @return the OSM type of this filter. + */ + @Contract(pure = true) + public long getChangesetId() { + return this.changesetId; + } + + @Override + public String toString() { + return "changeset:" + this.changesetId; + } +} diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ContributorUserIdFilterEquals.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ContributorUserIdFilterEquals.java new file mode 100644 index 000000000..0c9d05fe8 --- /dev/null +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ContributorUserIdFilterEquals.java @@ -0,0 +1,48 @@ +package org.heigit.ohsome.oshdb.filter; + +import org.heigit.ohsome.oshdb.osm.OSMEntity; +import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; +import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; +import org.jetbrains.annotations.Contract; + +/** + * A filter which selects OSM contributions by a user id. + */ +public class ContributorUserIdFilterEquals extends NegatableFilter { + final long userId; + + ContributorUserIdFilterEquals(long userId) { + super(new FilterInternal() { + @Override + public boolean applyOSM(OSMEntity entity) { + return true; + } + + @Override + public boolean applyOSMContribution(OSMContribution contribution) { + return contribution.getContributorUserId() == userId; + } + + @Override + public boolean applyOSMEntitySnapshot(OSMEntitySnapshot ignored) { + throw new IllegalStateException("contributor filter is not applicable to entity snapshots"); + } + }); + this.userId = userId; + } + + /** + * Returns the OSM type of this filter. + * + * @return the OSM type of this filter. + */ + @Contract(pure = true) + public long getUserId() { + return this.userId; + } + + @Override + public String toString() { + return "contributor:" + this.userId; + } +} From fa30c535f306f3ffabcabb6f448e3697f5272564 Mon Sep 17 00:00:00 2001 From: Martin Raifer Date: Fri, 14 May 2021 16:50:30 +0200 Subject: [PATCH 04/34] parse contributor and changeset filters --- .../ohsome/oshdb/filter/FilterParser.java | 28 ++++++++++++++++++- .../heigit/ohsome/oshdb/filter/ParseTest.java | 25 +++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/FilterParser.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/FilterParser.java index 7cb7de32e..93b6a2b26 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/FilterParser.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/FilterParser.java @@ -37,6 +37,17 @@ public class FilterParser { * respective OSHDB counterparts. */ public FilterParser(TagTranslator tt) { + this(tt, false); + } + + /** + * Creates a new parser for OSM entity filters. + * + * @param tt A tagtranslator object, used to transform OSM tags (e.g. "building=yes") to their + * respective OSHDB counterparts. + * @param allowContributorFilters if true enables filtering by contributor/user id. + */ + public FilterParser(TagTranslator tt, boolean allowContributorFilters) { final Parser whitespace = Scanners.WHITESPACES.skipMany(); final Parser keystr = Patterns.regex("[a-zA-Z_0-9:-]+") @@ -84,6 +95,8 @@ public FilterParser(TagTranslator tt) { .map(ignored -> "*"); final Parser area = Patterns.string("area").toScanner("area"); final Parser length = Patterns.string("length").toScanner("length"); + final Parser changeset = Patterns.string("changeset").toScanner("changeset"); + final Parser contributor = Patterns.string("contributor").toScanner("contributor"); final Parser tagFilter = Parsers.sequence( string, @@ -200,6 +213,17 @@ public FilterParser(TagTranslator tt) { geometryFilterArea, geometryFilterLength); + final Parser changesetIdFilter = Parsers.sequence( + changeset, colon, number + ).map(ChangesetIdFilterEquals::new); + Parser contributorUserIdFilter = Parsers.sequence( + contributor, colon, number + ).map(ContributorUserIdFilterEquals::new); + if (!allowContributorFilters) { + contributorUserIdFilter = contributorUserIdFilter.followedBy( + Parsers.fail("contributor user id filter not enabled")); + } + final Parser filter = Parsers.or( tagFilter, multiTagFilter, @@ -210,7 +234,9 @@ public FilterParser(TagTranslator tt) { rangeIdFilter, typeFilter, geometryTypeFilter, - geometryFilter); + geometryFilter, + changesetIdFilter, + contributorUserIdFilter); final Parser parensStart = Patterns.string("(").toScanner("(").followedBy(whitespace); final Parser parensEnd = whitespace.followedBy(Patterns.string(")").toScanner(")")); diff --git a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ParseTest.java b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ParseTest.java index 0ef31d2a9..db1bcafc9 100644 --- a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ParseTest.java +++ b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ParseTest.java @@ -2,6 +2,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.util.Arrays; import java.util.Collections; @@ -10,6 +11,7 @@ import org.heigit.ohsome.oshdb.filter.GeometryTypeFilter.GeometryType; import org.heigit.ohsome.oshdb.osm.OSMType; import org.heigit.ohsome.oshdb.util.OSHDBTagKey; +import org.jparsec.error.ParserException; import org.junit.Test; /** @@ -324,4 +326,27 @@ public void testGeometryFilterLength() { FilterExpression expression = parser.parse("length:(1..10)"); assertTrue(expression instanceof GeometryFilterLength); } + + @Test + public void testChangesetIdFilter() { + FilterExpression expression = parser.parse("changeset:42"); + assertTrue(expression instanceof ChangesetIdFilterEquals); + assertEquals(42, ((ChangesetIdFilterEquals) expression).getChangesetId()); + assertEquals("changeset:42", expression.toString()); + } + + @Test + public void testContributorIdFilterEnabled() { + FilterParser parser = new FilterParser(tagTranslator, true); + FilterExpression expression = parser.parse("contributor:1" /* Steve <3 */); + assertTrue(expression instanceof ContributorUserIdFilterEquals); + assertEquals(1, ((ContributorUserIdFilterEquals) expression).getUserId()); + assertEquals("contributor:1", expression.toString()); + } + + @Test(expected = ParserException.class) + public void testContributorIdFilterNotEnabled() { + FilterExpression expression = parser.parse("contributor:0"); + fail(); + } } From 05c5d1d52cc553ccf9194a266b15c3a943ced6f4 Mon Sep 17 00:00:00 2001 From: Martin Raifer Date: Fri, 14 May 2021 18:29:39 +0200 Subject: [PATCH 05/34] only check valid entity states for creations/deletions --- .../heigit/ohsome/oshdb/filter/FilterExpression.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/FilterExpression.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/FilterExpression.java index 2e239c7f3..21dff13d1 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/FilterExpression.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/FilterExpression.java @@ -9,6 +9,7 @@ import java.util.function.Supplier; import org.heigit.ohsome.oshdb.osh.OSHEntity; import org.heigit.ohsome.oshdb.osm.OSMEntity; +import org.heigit.ohsome.oshdb.util.celliterator.ContributionType; import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; import org.jetbrains.annotations.Contract; @@ -99,8 +100,14 @@ default boolean applyOSMEntitySnapshot(OSMEntitySnapshot snapshot) { */ @Contract(pure = true) default boolean applyOSMContribution(OSMContribution contribution) { - return applyOSMGeometry(contribution.getEntityAfter(), contribution::getGeometryAfter) - || applyOSMGeometry(contribution.getEntityBefore(), contribution::getGeometryBefore); + if (contribution.is(ContributionType.CREATION)) { + return applyOSMGeometry(contribution.getEntityAfter(), contribution::getGeometryAfter); + } else if (contribution.is(ContributionType.DELETION)) { + return applyOSMGeometry(contribution.getEntityBefore(), contribution::getGeometryBefore); + } else { + return applyOSMGeometry(contribution.getEntityBefore(), contribution::getGeometryBefore) + || applyOSMGeometry(contribution.getEntityAfter(), contribution::getGeometryAfter); + } } /** From 4ea9a24bbd057ea5bf34d5763de7a84748a58fbb Mon Sep 17 00:00:00 2001 From: Martin Raifer Date: Fri, 14 May 2021 18:30:02 +0200 Subject: [PATCH 06/34] directly use new "high level" filter methods in mapreducer --- .../oshdb/api/mapreducer/MapReducer.java | 24 ++++--------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapReducer.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapReducer.java index f420638d9..6b6506ba7 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapReducer.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapReducer.java @@ -796,19 +796,12 @@ public MapReducer filter(FilterExpression f) { if (isOSMEntitySnapshotQuery()) { ret = ret.filter(x -> { OSMEntitySnapshot s = (OSMEntitySnapshot) x; - return f.applyOSMGeometry(s.getEntity(), s::getGeometry); + return f.applyOSMEntitySnapshot(s); }); } else if (isContributionViewQuery()) { ret = ret.filter(x -> { OSMContribution c = (OSMContribution) x; - if (c.is(ContributionType.CREATION)) { - return f.applyOSMGeometry(c.getEntityAfter(), c::getGeometryAfter); - } else if (c.is(ContributionType.DELETION)) { - return f.applyOSMGeometry(c.getEntityBefore(), c::getGeometryBefore); - } else { - return f.applyOSMGeometry(c.getEntityBefore(), c::getGeometryBefore) - || f.applyOSMGeometry(c.getEntityAfter(), c::getGeometryAfter); - } + return f.applyOSMContribution(c); }); } } else if (this.grouping == Grouping.BY_ID) { @@ -817,7 +810,7 @@ public MapReducer filter(FilterExpression f) { @SuppressWarnings("unchecked") MapReducer filteredListMapper = (MapReducer) ret.map(x -> (Collection) x) .map(snapshots -> snapshots.stream() - .filter(s -> f.applyOSMGeometry(s.getEntity(), s::getGeometry)) + .filter(f::applyOSMEntitySnapshot) .collect(Collectors.toCollection(ArrayList::new))) .filter(snapshots -> !snapshots.isEmpty()); ret = filteredListMapper; @@ -825,16 +818,7 @@ public MapReducer filter(FilterExpression f) { @SuppressWarnings("unchecked") MapReducer filteredListMapper = (MapReducer) ret.map(x -> (Collection) x) .map(contributions -> contributions.stream() - .filter(c -> { - if (c.is(ContributionType.CREATION)) { - return f.applyOSMGeometry(c.getEntityAfter(), c::getGeometryAfter); - } else if (c.is(ContributionType.DELETION)) { - return f.applyOSMGeometry(c.getEntityBefore(), c::getGeometryBefore); - } else { - return f.applyOSMGeometry(c.getEntityBefore(), c::getGeometryBefore) - || f.applyOSMGeometry(c.getEntityAfter(), c::getGeometryAfter); - } - }) + .filter(f::applyOSMContribution) .collect(Collectors.toCollection(ArrayList::new))) .filter(contributions -> !contributions.isEmpty()); ret = filteredListMapper; From 263e3c6b45cf8d797e3b9853ddaa91357dcc9573 Mon Sep 17 00:00:00 2001 From: Martin Raifer Date: Tue, 18 May 2021 17:20:12 +0200 Subject: [PATCH 07/34] add to changelog --- CHANGELOG.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a062ac67..5cb8d2123 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,12 +6,16 @@ Changelog ### breaking changes * reorganize java packages, moving them from `org/heigit/bigspatialdata` to `org/heigit/ohsome` -* integrate [ohsome-filter](https://gitlab.gistools.geog.uni-heidelberg.de/giscience/big-data/ohsome/libs/ohsome-filter) module fully into this repository, renaming it to `oshdb-filter` ([#306]) +* integrate [ohsome-filter](https://gitlab.gistools.geog.uni-heidelberg.de/giscience/big-data/ohsome/libs/ohsome-filter) module fully into this repository, renaming it to `oshdb-filter` ([#306]), and enhance filtering capabilities ([#380]) * rename and move submodules of `oshdb-tool` ([#384]) * rename some classes, methods and enum constants; move some classes/interfaces ([#369], [#374]) > See the _upgrading from 0.6_ section below for instructions how to update your code according to these breaking changes. +### new features + +* enhance functionality of oshdb-filter: add new `changeset: ` and (optional) `contributor: ` filters ([#380]) + ### performance improvements * replace an unnecessarily used Map with a more lightweight implementation using a List. ([#352]) @@ -60,6 +64,7 @@ Changelog [#369]: https://github.com/GIScience/oshdb/pull/369 [#374]: https://github.com/GIScience/oshdb/pull/374 [#375]: https://github.com/GIScience/oshdb/pull/375 +[#384]: https://github.com/GIScience/oshdb/pull/380 [#384]: https://github.com/GIScience/oshdb/pull/384 From 3c980f234db6cc32b7fb866c82d790cdc770f8d2 Mon Sep 17 00:00:00 2001 From: Martin Raifer Date: Tue, 18 May 2021 17:36:02 +0200 Subject: [PATCH 08/34] enhance unit tests * check all possible cases for the "fallback" case for contribution view queries * add tests for changeset id and contributor user id filters --- .../filter/ApplyOSMContributionTest.java | 44 +++++++++++++++---- .../filter/ApplyOSMEntitySnapshotTest.java | 7 ++- 2 files changed, 41 insertions(+), 10 deletions(-) diff --git a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMContributionTest.java b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMContributionTest.java index ea6425195..178bfcc8c 100644 --- a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMContributionTest.java +++ b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMContributionTest.java @@ -21,9 +21,13 @@ */ public class ApplyOSMContributionTest extends FilterTest { private final GeometryFactory gf = new GeometryFactory(); + private final Geometry point = gf.createPoint(); + private final Geometry line = gf.createLineString(); + private final OSMEntity entity = createTestOSMEntityNode(); + private static final Set NO_TYPE = EnumSet.noneOf(ContributionType.class); - private class TestOSMContribution implements OSMContribution { - public static final String UNSUPPORTED_IN_TEST = "not supported for TestOSMEntitySnapshot"; + private static class TestOSMContribution implements OSMContribution { + private static final String UNSUPPORTED_IN_TEST = "not supported for TestOSMEntitySnapshot"; private final OSMEntity entityBefore; private final Geometry geometryBefore; private final OSMEntity entityAfter; @@ -114,15 +118,39 @@ public int compareTo(@NotNull OSMContribution contribution) { @Test public void testBasicFallback() { FilterExpression expression = parser.parse("geometry:point"); - final Geometry point = gf.createPoint(); - final Geometry line = gf.createLineString(); - final OSMEntity entity = createTestOSMEntityNode(); - final Set noType = EnumSet.noneOf(ContributionType.class); + // expect true if either the "before" state … assertTrue(expression.applyOSMContribution(new TestOSMContribution( - entity, point, entity, line, 1L, 2, noType + entity, point, entity, line, 1L, 2, NO_TYPE + ))); + // … or the "after" state matches the filter + assertTrue(expression.applyOSMContribution(new TestOSMContribution( + entity, line, entity, point, 1L, 2, NO_TYPE + ))); + // neither the before nor after geometries match the filter + assertFalse(expression.applyOSMContribution(new TestOSMContribution( + entity, line, entity, line, 1L, 2, NO_TYPE + ))); + } + + @Test + public void testChangesetId() { + FilterExpression expression = parser.parse("changeset:42"); + assertFalse(expression.applyOSMContribution(new TestOSMContribution( + entity, point, entity, point, 1L, 1, NO_TYPE + ))); + assertTrue(expression.applyOSMContribution(new TestOSMContribution( + entity, point, entity, point, 42L, 2, NO_TYPE + ))); + } + + @Test + public void testContributorUserId() { + FilterExpression expression = (new FilterParser(tagTranslator, true)).parse("contributor:1"); + assertTrue(expression.applyOSMContribution(new TestOSMContribution( + entity, point, entity, point, 1L, 1, NO_TYPE ))); assertFalse(expression.applyOSMContribution(new TestOSMContribution( - entity, line, entity, line, 1L, 2, noType + entity, point, entity, point, 1L, 2, NO_TYPE ))); } } diff --git a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMEntitySnapshotTest.java b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMEntitySnapshotTest.java index 15af862d6..e51d4629e 100644 --- a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMEntitySnapshotTest.java +++ b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMEntitySnapshotTest.java @@ -1,5 +1,6 @@ package org.heigit.ohsome.oshdb.filter; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import org.heigit.ohsome.oshdb.OSHDBTimestamp; @@ -17,8 +18,8 @@ public class ApplyOSMEntitySnapshotTest extends FilterTest { private final GeometryFactory gf = new GeometryFactory(); - private class TestOSMEntitySnapshot implements OSMEntitySnapshot { - public static final String UNSUPPORTED_IN_TEST = "not supported for TestOSMEntitySnapshot"; + private static class TestOSMEntitySnapshot implements OSMEntitySnapshot { + private static final String UNSUPPORTED_IN_TEST = "not supported for TestOSMEntitySnapshot"; private final OSMEntity entity; private final Geometry geometry; @@ -63,5 +64,7 @@ public void testBasicFallback() { FilterExpression expression = parser.parse("geometry:point"); assertTrue(expression.applyOSMEntitySnapshot(new TestOSMEntitySnapshot( createTestOSMEntityNode(), gf.createPoint()))); + assertFalse(expression.applyOSMEntitySnapshot(new TestOSMEntitySnapshot( + createTestOSMEntityNode(), gf.createLineString()))); } } From 7a3092e282df77ad4dc5eb0daabfe65476b8cf61 Mon Sep 17 00:00:00 2001 From: Martin Raifer Date: Tue, 18 May 2021 18:10:30 +0200 Subject: [PATCH 09/34] drop unnecessary (sometimes wrongly copy-pasted) javadoc for simple getters --- .../heigit/ohsome/oshdb/filter/ChangesetIdFilterEquals.java | 5 ----- .../java/org/heigit/ohsome/oshdb/filter/ConstantFilter.java | 6 +----- .../ohsome/oshdb/filter/ContributorUserIdFilterEquals.java | 5 ----- .../java/org/heigit/ohsome/oshdb/filter/IdFilterEquals.java | 5 ----- .../org/heigit/ohsome/oshdb/filter/IdFilterNotEquals.java | 5 ----- .../java/org/heigit/ohsome/oshdb/filter/TypeFilter.java | 5 ----- 6 files changed, 1 insertion(+), 30 deletions(-) diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ChangesetIdFilterEquals.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ChangesetIdFilterEquals.java index 57a8c1074..d2d701191 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ChangesetIdFilterEquals.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ChangesetIdFilterEquals.java @@ -31,11 +31,6 @@ public boolean applyOSMEntitySnapshot(OSMEntitySnapshot ignored) { this.changesetId = changesetId; } - /** - * Returns the OSM type of this filter. - * - * @return the OSM type of this filter. - */ @Contract(pure = true) public long getChangesetId() { return this.changesetId; diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ConstantFilter.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ConstantFilter.java index 9e13c38bc..dec3f6434 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ConstantFilter.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ConstantFilter.java @@ -14,11 +14,7 @@ public class ConstantFilter implements Filter { this.state = state; } - /** - * Returns the state of this filter. - * - * @return the state of this filter. - */ + /** Returns the true/false state of this filter. */ @Contract(pure = true) public boolean getState() { return this.state; diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ContributorUserIdFilterEquals.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ContributorUserIdFilterEquals.java index 0c9d05fe8..c97035ac9 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ContributorUserIdFilterEquals.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ContributorUserIdFilterEquals.java @@ -31,11 +31,6 @@ public boolean applyOSMEntitySnapshot(OSMEntitySnapshot ignored) { this.userId = userId; } - /** - * Returns the OSM type of this filter. - * - * @return the OSM type of this filter. - */ @Contract(pure = true) public long getUserId() { return this.userId; diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/IdFilterEquals.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/IdFilterEquals.java index a9882a40f..8c3752fa4 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/IdFilterEquals.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/IdFilterEquals.java @@ -14,11 +14,6 @@ public class IdFilterEquals implements Filter { this.id = id; } - /** - * Returns the OSM type of this filter. - * - * @return the OSM type of this filter. - */ @Contract(pure = true) public long getId() { return this.id; diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/IdFilterNotEquals.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/IdFilterNotEquals.java index b8ed414f3..23ebb6b58 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/IdFilterNotEquals.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/IdFilterNotEquals.java @@ -14,11 +14,6 @@ public class IdFilterNotEquals implements Filter { this.id = id; } - /** - * Returns the OSM type of this filter. - * - * @return the OSM type of this filter. - */ @Contract(pure = true) public long getId() { return this.id; diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/TypeFilter.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/TypeFilter.java index dda8f51ab..ec098ac31 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/TypeFilter.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/TypeFilter.java @@ -18,11 +18,6 @@ public class TypeFilter implements Filter { this.type = type; } - /** - * Returns the OSM type of this filter. - * - * @return the OSM type of this filter. - */ @Contract(pure = true) public OSMType getType() { return this.type; From 9b0c4260fb9a8ea96c4cdfa894997fef4b76b7d8 Mon Sep 17 00:00:00 2001 From: Martin Raifer Date: Tue, 18 May 2021 18:14:45 +0200 Subject: [PATCH 10/34] add test that changeset/contributor filters fails on snapshot view queries --- .../filter/ApplyOSMEntitySnapshotTest.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMEntitySnapshotTest.java b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMEntitySnapshotTest.java index e51d4629e..427a8867c 100644 --- a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMEntitySnapshotTest.java +++ b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMEntitySnapshotTest.java @@ -2,6 +2,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import org.heigit.ohsome.oshdb.OSHDBTimestamp; import org.heigit.ohsome.oshdb.osh.OSHEntity; @@ -67,4 +68,22 @@ public void testBasicFallback() { assertFalse(expression.applyOSMEntitySnapshot(new TestOSMEntitySnapshot( createTestOSMEntityNode(), gf.createLineString()))); } + + @Test(expected = IllegalStateException.class) + @SuppressWarnings("ResultOfMethodCallIgnored") + public void testChangesetId() { + FilterExpression expression = parser.parse("changeset:42"); + expression.applyOSMEntitySnapshot( + new TestOSMEntitySnapshot(createTestOSMEntityNode(), gf.createPoint())); + fail(); + } + + @Test(expected = IllegalStateException.class) + @SuppressWarnings("ResultOfMethodCallIgnored") + public void testContributorUserId() { + FilterExpression expression = (new FilterParser(tagTranslator, true)).parse("contributor:1"); + expression.applyOSMEntitySnapshot( + new TestOSMEntitySnapshot(createTestOSMEntityNode(), gf.createPoint())); + fail(); + } } From 56ccb92aa132d78b5cf1961bc9d6f9a945bf0a21 Mon Sep 17 00:00:00 2001 From: Martin Raifer Date: Tue, 18 May 2021 18:30:10 +0200 Subject: [PATCH 11/34] test that contributor/changeset filter let all entities through --- .../org/heigit/ohsome/oshdb/filter/ApplyOSMTest.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMTest.java b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMTest.java index 65b09ad06..8d05ae0ef 100644 --- a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMTest.java +++ b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMTest.java @@ -200,4 +200,16 @@ public void testGeometryFilterLength() throws IOException { FilterExpression expression = parser.parse("length:(1..2)"); assertTrue(expression.applyOSM(createTestOSMEntityWay(new long[] {}))); } + + @Test + public void testChangesetId() throws IOException { + FilterExpression expression = parser.parse("changeset:42"); + assertTrue(expression.applyOSM(createTestOSMEntityNode())); + } + + @Test + public void testContributorUserId() throws IOException { + FilterExpression expression = (new FilterParser(tagTranslator, true)).parse("contributor:1"); + assertTrue(expression.applyOSM(createTestOSMEntityNode())); + } } From 7b49ef7a18e4bfe213db21748d840b904b491b4d Mon Sep 17 00:00:00 2001 From: Martin Raifer Date: Wed, 19 May 2021 17:07:00 +0200 Subject: [PATCH 12/34] add new filters to oshdb-filter documentation --- oshdb-filter/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/oshdb-filter/README.md b/oshdb-filter/README.md index c8e1aa8c3..15dcededd 100644 --- a/oshdb-filter/README.md +++ b/oshdb-filter/README.md @@ -73,6 +73,8 @@ Filters are defined in textual form. A filter expression can be composed out of | `geometry:geom-type` | matches anything which has a geometry of the given type (_point_, _line_, _polygon_, or _other_) | `geometry:polygon` | | `area:(from..to-range)` | matches all features with an area that falls into the given range/interval given as two numbers in decimal or scientific notation separated by `..`. The values are interpreted as square meters (`m²`). The lower or upper limit of the interval may be omitted to select features having an area up to or starting from the given value, respectively. | `area:(123.4..1E6)` | | `length:(from..to-range)` | matches all features with a length that falls into the given range/interval given as two numbers in decimal or scientific notation separated by `..`. The values are interpreted as meters (`m`). The lower or upper limit of the interval may be omitted to select features having an area up to or starting from the given value, respectively. | `length:(100..)` | +| `changeset:id` | matches OSM contributions performed in the given OSM changeset. Can only be used in queries using the `OSMContributionView`. | `changeset:42` | +| `contributor:uid` | matches OSM contributions performed by a given OSM user (specified by their `uid`). Can only be used in queries using the `OSMContributionView`. | `contributor:1` | ### Operators From 7caf5e0d832e170fd7057fb2e1b3f9419c197629 Mon Sep 17 00:00:00 2001 From: Martin Raifer Date: Thu, 20 May 2021 15:56:32 +0200 Subject: [PATCH 13/34] use interface instead of impl. class; consistent naming of helpers --- .../api/mapreducer/GeometrySplitter.java | 9 ++- .../oshdb/api/mapreducer/MapAggregator.java | 13 ++--- .../oshdb/api/mapreducer/MapReducer.java | 55 +++++++++---------- 3 files changed, 36 insertions(+), 41 deletions(-) diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/GeometrySplitter.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/GeometrySplitter.java index b5a8d9fe4..05bc3ce6e 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/GeometrySplitter.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/GeometrySplitter.java @@ -24,7 +24,6 @@ import org.heigit.ohsome.oshdb.util.geometry.fip.FastPolygonOperations; import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; -import org.locationtech.jts.geom.Envelope; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.GeometryFactory; import org.locationtech.jts.geom.Polygonal; @@ -104,7 +103,7 @@ public Map splitOSMEntitySnapshot(OSMEntitySnapshot data) } // now we can check against the actual contribution geometry - Geometry snapshotGeometry = data.getGeometry(); + var snapshotGeometry = data.getGeometry(); OSHDBBoundingBox snapshotBbox = OSHDBGeometryBuilder.boundingBoxOf( snapshotGeometry.getEnvelopeInternal() ); @@ -168,8 +167,8 @@ public Map splitOSMContribution(OSMContribution data) { } // now we can check against the actual contribution geometry - Geometry contributionGeometryBefore = data.getGeometryBefore(); - Geometry contributionGeometryAfter = data.getGeometryAfter(); + var contributionGeometryBefore = data.getGeometryBefore(); + var contributionGeometryAfter = data.getGeometryAfter(); OSHDBBoundingBox contributionGeometryBbox; if (data.is(ContributionType.CREATION)) { contributionGeometryBbox = OSHDBGeometryBuilder.boundingBoxOf( @@ -180,7 +179,7 @@ public Map splitOSMContribution(OSMContribution data) { contributionGeometryBefore.getEnvelopeInternal() ); } else { - Envelope env = contributionGeometryBefore.getEnvelopeInternal(); + var env = contributionGeometryBefore.getEnvelopeInternal(); env.expandToInclude(contributionGeometryAfter.getEnvelopeInternal()); contributionGeometryBbox = OSHDBGeometryBuilder.boundingBoxOf(env); } diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapAggregator.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapAggregator.java index fd6791a0f..baf732f35 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapAggregator.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapAggregator.java @@ -29,8 +29,6 @@ import org.heigit.ohsome.oshdb.api.generic.OSHDBCombinedIndex; import org.heigit.ohsome.oshdb.api.generic.WeightedValue; import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer.Grouping; -import org.heigit.ohsome.oshdb.api.object.OSMContributionImpl; -import org.heigit.ohsome.oshdb.api.object.OSMEntitySnapshotImpl; import org.heigit.ohsome.oshdb.filter.FilterExpression; import org.heigit.ohsome.oshdb.osm.OSMType; import org.heigit.ohsome.oshdb.util.exceptions.OSHDBInvalidTimestampException; @@ -42,12 +40,13 @@ import org.heigit.ohsome.oshdb.util.function.SerializablePredicate; import org.heigit.ohsome.oshdb.util.function.SerializableSupplier; import org.heigit.ohsome.oshdb.util.mappable.OSHDBMapReducible; +import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; +import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; import org.heigit.ohsome.oshdb.util.tagtranslator.OSMTagInterface; import org.jetbrains.annotations.Contract; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.Polygonal; - /** * A MapReducer with built-in aggregation by an arbitrary index. * @@ -220,11 +219,11 @@ MapAggregator, X> aggregateByGeometry(Map geometr ); } else { MapAggregator, ? extends OSHDBMapReducible> ret; - if (mapReducer.isContributionViewQuery()) { - ret = this.flatMap(x -> gs.splitOSMContribution((OSMContributionImpl) x).entrySet()) + if (mapReducer.isOSMContributionViewQuery()) { + ret = this.flatMap(x -> gs.splitOSMContribution((OSMContribution) x).entrySet()) .aggregateBy(Entry::getKey, geometries.keySet()).map(Entry::getValue); - } else if (mapReducer.isOSMEntitySnapshotQuery()) { - ret = this.flatMap(x -> gs.splitOSMEntitySnapshot((OSMEntitySnapshotImpl) x).entrySet()) + } else if (mapReducer.isOSMEntitySnapshotViewQuery()) { + ret = this.flatMap(x -> gs.splitOSMEntitySnapshot((OSMEntitySnapshot) x).entrySet()) .aggregateBy(Entry::getKey, geometries.keySet()).map(Entry::getValue); } else { throw new UnsupportedOperationException(String.format( diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapReducer.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapReducer.java index 6b6506ba7..ebcf66dcc 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapReducer.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapReducer.java @@ -33,8 +33,6 @@ import org.heigit.ohsome.oshdb.api.db.OSHDBJdbc; import org.heigit.ohsome.oshdb.api.generic.NumberUtils; import org.heigit.ohsome.oshdb.api.generic.WeightedValue; -import org.heigit.ohsome.oshdb.api.object.OSMContributionImpl; -import org.heigit.ohsome.oshdb.api.object.OSMEntitySnapshotImpl; import org.heigit.ohsome.oshdb.filter.AndOperator; import org.heigit.ohsome.oshdb.filter.Filter; import org.heigit.ohsome.oshdb.filter.FilterExpression; @@ -49,7 +47,6 @@ import org.heigit.ohsome.oshdb.osm.OSMEntity; import org.heigit.ohsome.oshdb.osm.OSMType; import org.heigit.ohsome.oshdb.util.OSHDBTagKey; -import org.heigit.ohsome.oshdb.util.celliterator.ContributionType; import org.heigit.ohsome.oshdb.util.exceptions.OSHDBInvalidTimestampException; import org.heigit.ohsome.oshdb.util.exceptions.OSHDBKeytablesNotFoundException; import org.heigit.ohsome.oshdb.util.function.OSHEntityFilter; @@ -357,7 +354,7 @@ public MapReducer timestamps( */ @Contract(pure = true) public MapReducer timestamps(String isoDate) { - if (isContributionViewQuery()) { + if (isOSMContributionViewQuery()) { LOG.warn("OSMContributionView requires two or more timestamps, but only one was supplied."); } return this.timestamps(isoDate, isoDate, new String[] {}); @@ -793,12 +790,12 @@ public MapReducer filter(FilterExpression f) { ret.mappers.clear(); if (this.grouping == Grouping.NONE) { // no grouping -> directly filter using the geometries of the snapshot / contribution - if (isOSMEntitySnapshotQuery()) { + if (isOSMEntitySnapshotViewQuery()) { ret = ret.filter(x -> { OSMEntitySnapshot s = (OSMEntitySnapshot) x; return f.applyOSMEntitySnapshot(s); }); - } else if (isContributionViewQuery()) { + } else if (isOSMContributionViewQuery()) { ret = ret.filter(x -> { OSMContribution c = (OSMContribution) x; return f.applyOSMContribution(c); @@ -806,7 +803,7 @@ public MapReducer filter(FilterExpression f) { } } else if (this.grouping == Grouping.BY_ID) { // grouping by entity -> filter each list entry individually - if (isOSMEntitySnapshotQuery()) { + if (isOSMEntitySnapshotViewQuery()) { @SuppressWarnings("unchecked") MapReducer filteredListMapper = (MapReducer) ret.map(x -> (Collection) x) .map(snapshots -> snapshots.stream() @@ -814,7 +811,7 @@ public MapReducer filter(FilterExpression f) { .collect(Collectors.toCollection(ArrayList::new))) .filter(snapshots -> !snapshots.isEmpty()); ret = filteredListMapper; - } else if (isContributionViewQuery()) { + } else if (isOSMContributionViewQuery()) { @SuppressWarnings("unchecked") MapReducer filteredListMapper = (MapReducer) ret.map(x -> (Collection) x) .map(contributions -> contributions.stream() @@ -951,10 +948,10 @@ public MapAggregator aggregateByTimestamp() // by timestamp indexing function -> for some views we need to match the input data to the list SerializableFunction indexer; - if (isContributionViewQuery()) { + if (isOSMContributionViewQuery()) { final TreeSet timestamps = new TreeSet<>(this.tstamps.get()); indexer = data -> timestamps.floor(((OSMContribution) data).getTimestamp()); - } else if (isOSMEntitySnapshotQuery()) { + } else if (isOSMEntitySnapshotViewQuery()) { indexer = data -> ((OSMEntitySnapshot) data).getTimestamp(); } else { throw new UnsupportedOperationException( @@ -1049,11 +1046,11 @@ MapAggregator aggregateByGeometry(Map geometries) "please call aggregateByGeometry before setting any map or flatMap functions"); } else { MapAggregator ret; - if (isContributionViewQuery()) { - ret = this.flatMap(x -> gs.splitOSMContribution((OSMContributionImpl) x).entrySet()) + if (isOSMContributionViewQuery()) { + ret = this.flatMap(x -> gs.splitOSMContribution((OSMContribution) x).entrySet()) .aggregateBy(Entry::getKey, geometries.keySet()).map(Entry::getValue); - } else if (isOSMEntitySnapshotQuery()) { - ret = this.flatMap(x -> gs.splitOSMEntitySnapshot((OSMEntitySnapshotImpl) x).entrySet()) + } else if (isOSMEntitySnapshotViewQuery()) { + ret = this.flatMap(x -> gs.splitOSMEntitySnapshot((OSMEntitySnapshot) x).entrySet()) .aggregateBy(Entry::getKey, geometries.keySet()).map(Entry::getValue); } else { throw new UnsupportedOperationException(String.format( @@ -1123,7 +1120,7 @@ public S reduce( case NONE: if (this.mappers.stream().noneMatch(MapFunction::isFlatMapper)) { final SerializableFunction mapper = this.getMapper(); - if (isContributionViewQuery()) { + if (isOSMContributionViewQuery()) { @SuppressWarnings("Convert2MethodRef") // having just `mapper::apply` here is problematic, see https://github.com/GIScience/oshdb/pull/37 final SerializableFunction contributionMapper = @@ -1134,7 +1131,7 @@ public S reduce( accumulator, combiner ); - } else if (isOSMEntitySnapshotQuery()) { + } else if (isOSMEntitySnapshotViewQuery()) { @SuppressWarnings("Convert2MethodRef") // having just `mapper::apply` here is problematic, see https://github.com/GIScience/oshdb/pull/37 final SerializableFunction snapshotMapper = @@ -1151,7 +1148,7 @@ public S reduce( } } else { final SerializableFunction> flatMapper = this.getFlatMapper(); - if (isContributionViewQuery()) { + if (isOSMContributionViewQuery()) { return this.flatMapReduceCellsOSMContributionGroupedById( (List inputList) -> { List outputList = new LinkedList<>(); @@ -1160,7 +1157,7 @@ public S reduce( .forEach(data -> Iterables.addAll(outputList, data)); return outputList; }, identitySupplier, accumulator, combiner); - } else if (isOSMEntitySnapshotQuery()) { + } else if (isOSMEntitySnapshotViewQuery()) { return this.flatMapReduceCellsOSMEntitySnapshotGroupedById( (List inputList) -> { List outputList = new LinkedList<>(); @@ -1184,7 +1181,7 @@ public S reduce( } else { flatMapper = this.getFlatMapper(); } - if (isContributionViewQuery()) { + if (isOSMContributionViewQuery()) { @SuppressWarnings("Convert2MethodRef") // having just `flatMapper::apply` here is problematic, see https://github.com/GIScience/oshdb/pull/37 final SerializableFunction, Iterable> contributionFlatMapper = @@ -1195,7 +1192,7 @@ public S reduce( accumulator, combiner ); - } else if (isOSMEntitySnapshotQuery()) { + } else if (isOSMEntitySnapshotViewQuery()) { @SuppressWarnings("Convert2MethodRef") // having just `flatMapper::apply` here is problematic, see https://github.com/GIScience/oshdb/pull/37 final SerializableFunction, Iterable> snapshotFlatMapper = @@ -1639,13 +1636,13 @@ private Stream streamInternal() throws Exception { case NONE: if (this.mappers.stream().noneMatch(MapFunction::isFlatMapper)) { final SerializableFunction mapper = this.getMapper(); - if (isContributionViewQuery()) { + if (isOSMContributionViewQuery()) { @SuppressWarnings("Convert2MethodRef") // having just `mapper::apply` here is problematic, see https://github.com/GIScience/oshdb/pull/37 final SerializableFunction contributionMapper = data -> mapper.apply(data); return this.mapStreamCellsOSMContribution(contributionMapper); - } else if (isOSMEntitySnapshotQuery()) { + } else if (isOSMEntitySnapshotViewQuery()) { @SuppressWarnings("Convert2MethodRef") // having just `mapper::apply` here is problematic, see https://github.com/GIScience/oshdb/pull/37 final SerializableFunction snapshotMapper = @@ -1657,7 +1654,7 @@ private Stream streamInternal() throws Exception { } } else { final SerializableFunction> flatMapper = this.getFlatMapper(); - if (isContributionViewQuery()) { + if (isOSMContributionViewQuery()) { return this.flatMapStreamCellsOSMContributionGroupedById( (List inputList) -> { List outputList = new LinkedList<>(); @@ -1666,7 +1663,7 @@ private Stream streamInternal() throws Exception { .forEach(data -> Iterables.addAll(outputList, data)); return outputList; }); - } else if (isOSMEntitySnapshotQuery()) { + } else if (isOSMEntitySnapshotViewQuery()) { return this.flatMapStreamCellsOSMEntitySnapshotGroupedById( (List inputList) -> { List outputList = new LinkedList<>(); @@ -1690,13 +1687,13 @@ private Stream streamInternal() throws Exception { } else { flatMapper = this.getFlatMapper(); } - if (isContributionViewQuery()) { + if (isOSMContributionViewQuery()) { @SuppressWarnings("Convert2MethodRef") // having just `mapper::apply` here is problematic, see https://github.com/GIScience/oshdb/pull/37 final SerializableFunction, Iterable> contributionFlatMapper = data -> flatMapper.apply(data); return this.flatMapStreamCellsOSMContributionGroupedById(contributionFlatMapper); - } else if (isOSMEntitySnapshotQuery()) { + } else if (isOSMEntitySnapshotViewQuery()) { @SuppressWarnings("Convert2MethodRef") // having just `mapper::apply` here is problematic, see https://github.com/GIScience/oshdb/pull/37 final SerializableFunction, Iterable> snapshotFlatMapper = @@ -1938,11 +1935,11 @@ protected abstract S flatMapReduceCellsOSMEntitySnapshotGroupedById( // Some helper methods for internal use in the mapReduce functions // ----------------------------------------------------------------------------------------------- - protected boolean isContributionViewQuery() { + protected boolean isOSMContributionViewQuery() { return OSMContribution.class.isAssignableFrom(this.viewClass); } - protected boolean isOSMEntitySnapshotQuery() { + protected boolean isOSMEntitySnapshotViewQuery() { return OSMEntitySnapshot.class.isAssignableFrom(this.viewClass); } @@ -2065,7 +2062,7 @@ private SerializableFunction> getFlatMapper() { // gets list of timestamps to use for zerofilling Collection getZerofillTimestamps() { - if (isOSMEntitySnapshotQuery()) { + if (isOSMEntitySnapshotViewQuery()) { return this.tstamps.get(); } else { SortedSet result = new TreeSet<>(this.tstamps.get()); From 10bec30baaa64fb96c60b5decd497855a18aca08 Mon Sep 17 00:00:00 2001 From: Martin Raifer Date: Thu, 20 May 2021 16:09:42 +0200 Subject: [PATCH 14/34] replace potentially slow default implementation of applyOSH see https://github.com/GIScience/oshdb/pull/380#discussion_r636091584 --- .../heigit/ohsome/oshdb/filter/FilterExpression.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/FilterExpression.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/FilterExpression.java index 21dff13d1..0acc3d38a 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/FilterExpression.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/FilterExpression.java @@ -35,16 +35,16 @@ public interface FilterExpression extends Serializable { /** * Apply the filter to an OSH entity. * - *

Must return the same as oshEntity.getVersions().….anyMatch(applyOSM).

+ *

Must be compatible with the result of {@link #applyOSM}, e.g. that it must not return false + * when oshEntity.getVersions().….anyMatch(applyOSM) would evaluate to true.

* * @param entity the OSH entity to check. - * @return true if the at least one of the OSH entity's versions fulfills the specified filter, - * false otherwise. + * @return false if the filter knows that none of the versions of the OSH entity can fulfill the + * specified filter, true otherwise. */ @Contract(pure = true) default boolean applyOSH(OSHEntity entity) { - // (potentially slow) default implementation tests every version individually - return Streams.stream(entity.getVersions()).anyMatch(this::applyOSM); + return true; } /** From df1fb11fa5b705cf9be617169e48733251f2deec Mon Sep 17 00:00:00 2001 From: Martin Raifer Date: Thu, 20 May 2021 18:51:07 +0200 Subject: [PATCH 15/34] f --- .../org/heigit/ohsome/oshdb/filter/TagFilterNotEquals.java | 7 +++++++ .../heigit/ohsome/oshdb/filter/TagFilterNotEqualsAny.java | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/TagFilterNotEquals.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/TagFilterNotEquals.java index 7ca8335b9..4f9664a57 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/TagFilterNotEquals.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/TagFilterNotEquals.java @@ -1,6 +1,8 @@ package org.heigit.ohsome.oshdb.filter; +import com.google.common.collect.Streams; import org.heigit.ohsome.oshdb.OSHDBTag; +import org.heigit.ohsome.oshdb.osh.OSHEntity; import org.heigit.ohsome.oshdb.osm.OSMEntity; /** @@ -18,6 +20,11 @@ public OSHDBTag getTag() { return this.tag; } + @Override + public boolean applyOSH(OSHEntity entity) { + return Streams.stream(entity.getVersions()).anyMatch(this::applyOSM); + } + @Override public boolean applyOSM(OSMEntity entity) { return !entity.hasTagValue(tag.getKey(), tag.getValue()); diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/TagFilterNotEqualsAny.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/TagFilterNotEqualsAny.java index 468d6422e..821a30ff6 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/TagFilterNotEqualsAny.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/TagFilterNotEqualsAny.java @@ -1,5 +1,7 @@ package org.heigit.ohsome.oshdb.filter; +import com.google.common.collect.Streams; +import org.heigit.ohsome.oshdb.osh.OSHEntity; import org.heigit.ohsome.oshdb.osm.OSMEntity; import org.heigit.ohsome.oshdb.util.OSHDBTagKey; @@ -18,6 +20,11 @@ public OSHDBTagKey getTag() { return this.tag; } + @Override + public boolean applyOSH(OSHEntity entity) { + return Streams.stream(entity.getVersions()).anyMatch(this::applyOSM); + } + @Override public boolean applyOSM(OSMEntity entity) { return !entity.hasTagKey(tag.toInt()); From 78dcceefcc1b6424e0588e350f0d2f499d9bff14 Mon Sep 17 00:00:00 2001 From: Martin Raifer Date: Thu, 20 May 2021 16:43:42 +0200 Subject: [PATCH 16/34] implement applyOSH pre-filtering to changeset/contributor filters --- .../oshdb/filter/ChangesetIdFilterEquals.java | 6 +++ .../filter/ContributorUserIdFilterEquals.java | 6 +++ .../ohsome/oshdb/filter/NegatableFilter.java | 43 +++++++++++++++++++ 3 files changed, 55 insertions(+) diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ChangesetIdFilterEquals.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ChangesetIdFilterEquals.java index d2d701191..313ba6d47 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ChangesetIdFilterEquals.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ChangesetIdFilterEquals.java @@ -1,5 +1,6 @@ package org.heigit.ohsome.oshdb.filter; +import org.heigit.ohsome.oshdb.osh.OSHEntity; import org.heigit.ohsome.oshdb.osm.OSMEntity; import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; @@ -13,6 +14,11 @@ public class ChangesetIdFilterEquals extends NegatableFilter { ChangesetIdFilterEquals(long changesetId) { super(new FilterInternal() { + @Override + public boolean applyOSH(OSHEntity entity) { + return applyToOSHEntityRecursively(entity, v -> v.getChangesetId() == changesetId); + } + @Override public boolean applyOSM(OSMEntity entity) { return true; diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ContributorUserIdFilterEquals.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ContributorUserIdFilterEquals.java index c97035ac9..b5e820b7c 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ContributorUserIdFilterEquals.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ContributorUserIdFilterEquals.java @@ -1,5 +1,6 @@ package org.heigit.ohsome.oshdb.filter; +import org.heigit.ohsome.oshdb.osh.OSHEntity; import org.heigit.ohsome.oshdb.osm.OSMEntity; import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; @@ -13,6 +14,11 @@ public class ContributorUserIdFilterEquals extends NegatableFilter { ContributorUserIdFilterEquals(long userId) { super(new FilterInternal() { + @Override + public boolean applyOSH(OSHEntity entity) { + return applyToOSHEntityRecursively(entity, v -> v.getUserId() == userId); + } + @Override public boolean applyOSM(OSMEntity entity) { return true; diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/NegatableFilter.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/NegatableFilter.java index a2e270b7e..0a79e87d6 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/NegatableFilter.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/NegatableFilter.java @@ -1,6 +1,10 @@ package org.heigit.ohsome.oshdb.filter; +import com.google.common.collect.Streams; +import java.io.IOException; +import java.util.function.Predicate; import java.util.function.Supplier; +import java.util.stream.Stream; import javax.annotation.Nonnull; import org.heigit.ohsome.oshdb.osh.OSHEntity; import org.heigit.ohsome.oshdb.osm.OSMEntity; @@ -65,4 +69,43 @@ public NegatableFilter negate() { public String toString() { return (this.negated ? "not " : "") + this.filter.toString(); } + + /** + * Helper method to test a predicate on all versions of an OSH entity, including its + * references/members and references of members. + * + * @param entity the OSH entity to test. + * @param predicate the predicate to apply to each version of the entity + * @return true if any of the versions of the entity or its referenced child entities matches the + * given predicate. + */ + protected static boolean applyToOSHEntityRecursively(OSHEntity entity, Predicate predicate) { + try { + switch (entity.getType()) { + case NODE: + return Streams.stream(entity.getVersions()).anyMatch(predicate); + case WAY: + return Streams.concat( + Streams.stream(entity.getVersions()), + entity.getNodes().stream().flatMap(n -> Streams.stream(n.getVersions())) + ).anyMatch(predicate); + case RELATION: + default: + return Streams.concat( + Streams.stream(entity.getVersions()), + entity.getNodes().stream().flatMap(n -> Streams.stream(n.getVersions())), + entity.getWays().stream().flatMap(w -> Streams.stream(w.getVersions())), + entity.getWays().stream().flatMap(w -> { + try { + return w.getNodes().stream().flatMap(wn -> Streams.stream(wn.getVersions())); + } catch(IOException ignored) { + return Stream.empty(); + } + }) + ).anyMatch(predicate); + } + } catch(IOException ignored) { + return true; + } + } } \ No newline at end of file From 628a82aa15df9f0ee3394d075db7d77c72208410 Mon Sep 17 00:00:00 2001 From: Martin Raifer Date: Thu, 20 May 2021 18:30:43 +0200 Subject: [PATCH 17/34] implement changeset/contributor filtering by list & range --- .../oshdb/filter/ChangesetIdFilterEquals.java | 10 +-- .../filter/ChangesetIdFilterEqualsAnyOf.java | 57 +++++++++++++ .../oshdb/filter/ChangesetIdFilterRange.java | 40 +++++++++ .../filter/ContributorUserIdFilterEquals.java | 10 +-- .../ContributorUserIdFilterEqualsAnyOf.java | 57 +++++++++++++ .../filter/ContributorUserIdFilterRange.java | 40 +++++++++ .../heigit/ohsome/oshdb/filter/Filter.java | 2 +- .../ohsome/oshdb/filter/FilterExpression.java | 1 - .../ohsome/oshdb/filter/FilterParser.java | 48 ++++++++--- .../ohsome/oshdb/filter/GeometryFilter.java | 4 +- .../ohsome/oshdb/filter/IdFilterRange.java | 28 +------ .../heigit/ohsome/oshdb/filter/IdRange.java | 42 ++++++++++ .../ohsome/oshdb/filter/NegatableFilter.java | 82 ++++++++++--------- .../ohsome/oshdb/filter/NegateTest.java | 2 +- .../heigit/ohsome/oshdb/filter/ParseTest.java | 39 ++++++++- 15 files changed, 368 insertions(+), 94 deletions(-) create mode 100644 oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ChangesetIdFilterEqualsAnyOf.java create mode 100644 oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ChangesetIdFilterRange.java create mode 100644 oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ContributorUserIdFilterEqualsAnyOf.java create mode 100644 oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ContributorUserIdFilterRange.java create mode 100644 oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/IdRange.java diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ChangesetIdFilterEquals.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ChangesetIdFilterEquals.java index 313ba6d47..3b6b9a8df 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ChangesetIdFilterEquals.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ChangesetIdFilterEquals.java @@ -33,6 +33,11 @@ public boolean applyOSMContribution(OSMContribution contribution) { public boolean applyOSMEntitySnapshot(OSMEntitySnapshot ignored) { throw new IllegalStateException("changeset filter is not applicable to entity snapshots"); } + + @Override + public String toString() { + return "changeset:" + changesetId; + } }); this.changesetId = changesetId; } @@ -41,9 +46,4 @@ public boolean applyOSMEntitySnapshot(OSMEntitySnapshot ignored) { public long getChangesetId() { return this.changesetId; } - - @Override - public String toString() { - return "changeset:" + this.changesetId; - } } diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ChangesetIdFilterEqualsAnyOf.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ChangesetIdFilterEqualsAnyOf.java new file mode 100644 index 000000000..0c5ae8d58 --- /dev/null +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ChangesetIdFilterEqualsAnyOf.java @@ -0,0 +1,57 @@ +package org.heigit.ohsome.oshdb.filter; + +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import org.heigit.ohsome.oshdb.osh.OSHEntity; +import org.heigit.ohsome.oshdb.osm.OSMEntity; +import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; +import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; +import org.jetbrains.annotations.Contract; + +/** + * A filter which selects OSM contributions by matching to a list of changeset ids. + */ +public class ChangesetIdFilterEqualsAnyOf extends NegatableFilter { + private final Collection changesetIdList; + + ChangesetIdFilterEqualsAnyOf(@Nonnull Collection changesetIdList) { + super(new FilterInternal() { + private final Set changesetIds = new HashSet<>(changesetIdList); + + @Override + public boolean applyOSH(OSHEntity entity) { + return applyToOSHEntityRecursively(entity, v -> changesetIds.contains(v.getChangesetId())); + } + + @Override + public boolean applyOSM(OSMEntity entity) { + return true; + } + + @Override + public boolean applyOSMContribution(OSMContribution contribution) { + return changesetIds.contains(contribution.getChangesetId()); + } + + @Override + public boolean applyOSMEntitySnapshot(OSMEntitySnapshot ignored) { + throw new IllegalStateException("changeset filter is not applicable to entity snapshots"); + } + + @Override + public String toString() { + return "changeset:in(" + changesetIdList.stream().map(String::valueOf) + .collect(Collectors.joining(",")) + ")"; + } + }); + this.changesetIdList = changesetIdList; + } + + @Contract(pure = true) + public Collection getChangesetIdList() { + return this.changesetIdList; + } +} diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ChangesetIdFilterRange.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ChangesetIdFilterRange.java new file mode 100644 index 000000000..8e5b60789 --- /dev/null +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ChangesetIdFilterRange.java @@ -0,0 +1,40 @@ +package org.heigit.ohsome.oshdb.filter; + +import org.heigit.ohsome.oshdb.osh.OSHEntity; +import org.heigit.ohsome.oshdb.osm.OSMEntity; +import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; +import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; + +/** + * A filter which selects OSM contributions by matching to a range of changeset ids. + */ +public class ChangesetIdFilterRange extends NegatableFilter { + ChangesetIdFilterRange(IdRange changesetIdRange) { + super(new FilterInternal() { + @Override + public boolean applyOSH(OSHEntity entity) { + return applyToOSHEntityRecursively(entity, v -> changesetIdRange.test(v.getChangesetId())); + } + + @Override + public boolean applyOSM(OSMEntity entity) { + return true; + } + + @Override + public boolean applyOSMContribution(OSMContribution contribution) { + return changesetIdRange.test(contribution.getChangesetId()); + } + + @Override + public boolean applyOSMEntitySnapshot(OSMEntitySnapshot ignored) { + throw new IllegalStateException("changeset filter is not applicable to entity snapshots"); + } + + @Override + public String toString() { + return "changeset:in-range" + changesetIdRange; + } + }); + } +} diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ContributorUserIdFilterEquals.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ContributorUserIdFilterEquals.java index b5e820b7c..6b8d8c0d1 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ContributorUserIdFilterEquals.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ContributorUserIdFilterEquals.java @@ -33,6 +33,11 @@ public boolean applyOSMContribution(OSMContribution contribution) { public boolean applyOSMEntitySnapshot(OSMEntitySnapshot ignored) { throw new IllegalStateException("contributor filter is not applicable to entity snapshots"); } + + @Override + public String toString() { + return "contributor:" + userId; + } }); this.userId = userId; } @@ -41,9 +46,4 @@ public boolean applyOSMEntitySnapshot(OSMEntitySnapshot ignored) { public long getUserId() { return this.userId; } - - @Override - public String toString() { - return "contributor:" + this.userId; - } } diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ContributorUserIdFilterEqualsAnyOf.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ContributorUserIdFilterEqualsAnyOf.java new file mode 100644 index 000000000..ac6fe6e5a --- /dev/null +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ContributorUserIdFilterEqualsAnyOf.java @@ -0,0 +1,57 @@ +package org.heigit.ohsome.oshdb.filter; + +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import org.heigit.ohsome.oshdb.osh.OSHEntity; +import org.heigit.ohsome.oshdb.osm.OSMEntity; +import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; +import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; +import org.jetbrains.annotations.Contract; + +/** + * A filter which selects OSM contributions by matching to a list of contributor user ids. + */ +public class ContributorUserIdFilterEqualsAnyOf extends NegatableFilter { + private final Collection contributorUserIdList; + + ContributorUserIdFilterEqualsAnyOf(@Nonnull Collection contributorUserIdList) { + super(new FilterInternal() { + private final Set contributorUserIds = new HashSet<>(contributorUserIdList); + + @Override + public boolean applyOSH(OSHEntity entity) { + return applyToOSHEntityRecursively(entity, v -> contributorUserIds.contains(v.getUserId())); + } + + @Override + public boolean applyOSM(OSMEntity entity) { + return true; + } + + @Override + public boolean applyOSMContribution(OSMContribution contribution) { + return contributorUserIds.contains(contribution.getContributorUserId()); + } + + @Override + public boolean applyOSMEntitySnapshot(OSMEntitySnapshot ignored) { + throw new IllegalStateException("contributor filter is not applicable to entity snapshots"); + } + + @Override + public String toString() { + return "contributor:in(" + contributorUserIdList.stream().map(String::valueOf) + .collect(Collectors.joining(",")) + ")"; + } + }); + this.contributorUserIdList = contributorUserIdList; + } + + @Contract(pure = true) + public Collection getContributorUserIdList() { + return this.contributorUserIdList; + } +} diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ContributorUserIdFilterRange.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ContributorUserIdFilterRange.java new file mode 100644 index 000000000..04f5006e3 --- /dev/null +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ContributorUserIdFilterRange.java @@ -0,0 +1,40 @@ +package org.heigit.ohsome.oshdb.filter; + +import org.heigit.ohsome.oshdb.osh.OSHEntity; +import org.heigit.ohsome.oshdb.osm.OSMEntity; +import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; +import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; + +/** + * A filter which selects OSM contributions by a range of contributor user ids. + */ +public class ContributorUserIdFilterRange extends NegatableFilter { + ContributorUserIdFilterRange(IdRange contributorUserIdRange) { + super(new FilterInternal() { + @Override + public boolean applyOSH(OSHEntity entity) { + return applyToOSHEntityRecursively(entity, v -> contributorUserIdRange.test(v.getUserId())); + } + + @Override + public boolean applyOSM(OSMEntity entity) { + return true; + } + + @Override + public boolean applyOSMContribution(OSMContribution contribution) { + return contributorUserIdRange.test(contribution.getContributorUserId()); + } + + @Override + public boolean applyOSMEntitySnapshot(OSMEntitySnapshot ignored) { + throw new IllegalStateException("contributor filter is not applicable to entity snapshots"); + } + + @Override + public String toString() { + return "contributor:in-range" + contributorUserIdRange; + } + }); + } +} diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/Filter.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/Filter.java index 7e0669456..b53355e0d 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/Filter.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/Filter.java @@ -41,7 +41,7 @@ static Filter byOSHEntity(OSHEntityFilter oshCallback) { * @return a filter object which filters using the given predicate */ static Filter byOSMEntity(OSMEntityFilter osmCallback) { - return new NegatableFilter(osmCallback::test); + return by(ignored -> true, osmCallback); } /** diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/FilterExpression.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/FilterExpression.java index 0acc3d38a..d034e8692 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/FilterExpression.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/FilterExpression.java @@ -1,6 +1,5 @@ package org.heigit.ohsome.oshdb.filter; -import com.google.common.collect.Streams; import java.io.Serializable; import java.util.ArrayList; import java.util.Collections; diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/FilterParser.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/FilterParser.java index 93b6a2b26..df0875107 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/FilterParser.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/FilterParser.java @@ -2,6 +2,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; import org.heigit.ohsome.oshdb.OSHDBTag; import org.heigit.ohsome.oshdb.filter.GeometryFilter.ValueRange; import org.heigit.ohsome.oshdb.filter.GeometryTypeFilter.GeometryType; @@ -22,7 +23,7 @@ /** * A parser for OSM entity filters. * - *

Such filters can select OSM entites by their tags, their type or other attributes. Filters + *

Such filters can select OSM entities by their tags, their type or other attributes. Filters * can contain boolean operators (and/or/not) and parentheses can be used.

* *

Example: "type:way and highway=residential and not (lit=yes or lit=automatic)"

@@ -166,19 +167,21 @@ public FilterParser(TagTranslator tt, boolean allowContributorFilters) { final Parser dotdot = whitespace .followedBy(Patterns.string("..").toScanner("DOT-DOT (..)")) .followedBy(whitespace); - final Parser rangeIdFilter = Parsers.sequence( - id, - colon, + final Parser range = Parsers.sequence( Scanners.isChar('('), whitespace, Parsers.or( Parsers.sequence(number, dotdot, number, - (min, ignored, max) -> new IdFilterRange.IdRange(min, max)), + (min, ignored, max) -> new IdRange(min, max)), number.followedBy(dotdot).map( - min -> new IdFilterRange.IdRange(min, Long.MAX_VALUE)), + min -> new IdRange(min, Long.MAX_VALUE)), Parsers.sequence(dotdot, number).map( - max -> new IdFilterRange.IdRange(Long.MIN_VALUE, max)) - ).followedBy(whitespace).followedBy(Scanners.isChar(')'))) + max -> new IdRange(Long.MIN_VALUE, max)) + ).followedBy(whitespace).followedBy(Scanners.isChar(')'))); + final Parser rangeIdFilter = Parsers.sequence( + id, + colon, + range) .map(IdFilterRange::new); final Parser typeFilter = Parsers.sequence( type, @@ -213,15 +216,34 @@ public FilterParser(TagTranslator tt, boolean allowContributorFilters) { geometryFilterArea, geometryFilterLength); + // changeset id filters final Parser changesetIdFilter = Parsers.sequence( changeset, colon, number ).map(ChangesetIdFilterEquals::new); + final Parser multiChangesetIdFilter = Parsers.sequence( + changeset, colon, numberSequence + ).map(ChangesetIdFilterEqualsAnyOf::new); + final Parser rangeChangesetIdFilter = Parsers.sequence( + changeset, colon, range + ).map(ChangesetIdFilterRange::new); + // contributor user id filters Parser contributorUserIdFilter = Parsers.sequence( contributor, colon, number ).map(ContributorUserIdFilterEquals::new); + Parser multiContributorUserIdFilter = Parsers.sequence( + contributor, colon, numberSequence + ).map(ids -> ids.stream().map(Number::intValue).collect(Collectors.toList()) + ).map(ContributorUserIdFilterEqualsAnyOf::new); + Parser rangeContributorUserIdFilter = Parsers.sequence( + contributor, colon, range + ).map(ContributorUserIdFilterRange::new); if (!allowContributorFilters) { - contributorUserIdFilter = contributorUserIdFilter.followedBy( - Parsers.fail("contributor user id filter not enabled")); + final var contributorFilterDisabled = Parsers.fail("contributor user id filter not enabled"); + contributorUserIdFilter = contributorUserIdFilter.followedBy(contributorFilterDisabled); + multiContributorUserIdFilter = + multiContributorUserIdFilter.followedBy(contributorFilterDisabled); + rangeContributorUserIdFilter = + rangeContributorUserIdFilter.followedBy(contributorFilterDisabled); } final Parser filter = Parsers.or( @@ -236,7 +258,11 @@ public FilterParser(TagTranslator tt, boolean allowContributorFilters) { geometryTypeFilter, geometryFilter, changesetIdFilter, - contributorUserIdFilter); + multiChangesetIdFilter, + rangeChangesetIdFilter, + contributorUserIdFilter, + multiContributorUserIdFilter, + rangeContributorUserIdFilter); final Parser parensStart = Patterns.string("(").toScanner("(").followedBy(whitespace); final Parser parensEnd = whitespace.followedBy(Patterns.string(")").toScanner(")")); diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/GeometryFilter.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/GeometryFilter.java index dac3d34ec..3040b2263 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/GeometryFilter.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/GeometryFilter.java @@ -58,8 +58,8 @@ public String toString() { } } - interface RangedFilter extends FilterInternal { - ValueRange getRange(); + abstract static class RangedFilter extends FilterInternal { + public abstract ValueRange getRange(); } protected GeometryFilter( diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/IdFilterRange.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/IdFilterRange.java index d3a4673c3..7ff4184dd 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/IdFilterRange.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/IdFilterRange.java @@ -1,6 +1,5 @@ package org.heigit.ohsome.oshdb.filter; -import java.io.Serializable; import javax.annotation.Nonnull; import org.heigit.ohsome.oshdb.osh.OSHEntity; import org.heigit.ohsome.oshdb.osm.OSMEntity; @@ -9,31 +8,6 @@ * A filter which executes a "id [not] in range" check. */ public class IdFilterRange extends NegatableFilter { - static class IdRange implements Serializable { - private final long fromId; - private final long toId; - - IdRange(long fromId, long toId) { - if (toId < fromId) { - long buffer = toId; - toId = fromId; - fromId = buffer; - } - this.fromId = fromId; - this.toId = toId; - } - - private boolean test(long id) { - return id >= fromId && id <= toId; - } - - public String toString() { - return (fromId == Long.MIN_VALUE ? "" : fromId) - + ".." - + (toId == Long.MAX_VALUE ? "" : toId); - } - } - IdFilterRange(@Nonnull IdRange range) { super(new FilterInternal() { @Override @@ -48,7 +22,7 @@ public boolean applyOSM(OSMEntity entity) { @Override public String toString() { - return "id:in-range" + range.toString(); + return "id:in-range" + range; } }); } diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/IdRange.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/IdRange.java new file mode 100644 index 000000000..a2a60f4cf --- /dev/null +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/IdRange.java @@ -0,0 +1,42 @@ +package org.heigit.ohsome.oshdb.filter; + +import java.io.Serializable; + +/** + * Helper class to handle ranges of ids (incl. user ids, changeset ids, etc.). + * + *

The range's limits are tested inclusively: the range (10..12) would match the values 10, + * 11 and 12, but not 9 or 13 for example.

+ */ +class IdRange implements Serializable { + + private final long fromId; + private final long toId; + + /** + * Creates a new id range. + * + * @param fromId lower limit of the range. + * @param toId upper limit of the range. + */ + IdRange(long fromId, long toId) { + if (toId < fromId) { + long buffer = toId; + toId = fromId; + fromId = buffer; + } + this.fromId = fromId; + this.toId = toId; + } + + /** Checks if the given id falls into the id range. */ + public boolean test(long id) { + return id >= fromId && id <= toId; + } + + public String toString() { + return (fromId == Long.MIN_VALUE ? "" : fromId) + + ".." + + (toId == Long.MAX_VALUE ? "" : toId); + } +} diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/NegatableFilter.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/NegatableFilter.java index 0a79e87d6..f5a1c905f 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/NegatableFilter.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/NegatableFilter.java @@ -16,11 +16,51 @@ * A filter which implements the negate method using a boolean flag. */ class NegatableFilter implements Filter { - interface FilterInternal extends Filter { + abstract static class FilterInternal implements Filter { @Override - default FilterExpression negate() { + public FilterExpression negate() { throw new IllegalStateException("Invalid call of inner negate() on a negatable filter"); } + + /** + * Helper method to test a predicate on all versions of an OSH entity, including its + * references/members and references of members. + * + * @param entity the OSH entity to test. + * @param predicate the predicate to apply to each version of the entity + * @return true if any of the versions of the entity or its referenced child entities matches + * the given predicate. + */ + protected static boolean applyToOSHEntityRecursively( + OSHEntity entity, Predicate predicate) { + try { + switch (entity.getType()) { + case NODE: + return Streams.stream(entity.getVersions()).anyMatch(predicate); + case WAY: + return Streams.concat( + Streams.stream(entity.getVersions()), + entity.getNodes().stream().flatMap(n -> Streams.stream(n.getVersions())) + ).anyMatch(predicate); + case RELATION: + default: + return Streams.concat( + Streams.stream(entity.getVersions()), + entity.getNodes().stream().flatMap(n -> Streams.stream(n.getVersions())), + entity.getWays().stream().flatMap(w -> Streams.stream(w.getVersions())), + entity.getWays().stream().flatMap(w -> { + try { + return w.getNodes().stream().flatMap(wn -> Streams.stream(wn.getVersions())); + } catch (IOException ignored) { + return Stream.empty(); + } + }) + ).anyMatch(predicate); + } + } catch (IOException ignored) { + return true; + } + } } private final boolean negated; @@ -70,42 +110,4 @@ public String toString() { return (this.negated ? "not " : "") + this.filter.toString(); } - /** - * Helper method to test a predicate on all versions of an OSH entity, including its - * references/members and references of members. - * - * @param entity the OSH entity to test. - * @param predicate the predicate to apply to each version of the entity - * @return true if any of the versions of the entity or its referenced child entities matches the - * given predicate. - */ - protected static boolean applyToOSHEntityRecursively(OSHEntity entity, Predicate predicate) { - try { - switch (entity.getType()) { - case NODE: - return Streams.stream(entity.getVersions()).anyMatch(predicate); - case WAY: - return Streams.concat( - Streams.stream(entity.getVersions()), - entity.getNodes().stream().flatMap(n -> Streams.stream(n.getVersions())) - ).anyMatch(predicate); - case RELATION: - default: - return Streams.concat( - Streams.stream(entity.getVersions()), - entity.getNodes().stream().flatMap(n -> Streams.stream(n.getVersions())), - entity.getWays().stream().flatMap(w -> Streams.stream(w.getVersions())), - entity.getWays().stream().flatMap(w -> { - try { - return w.getNodes().stream().flatMap(wn -> Streams.stream(wn.getVersions())); - } catch(IOException ignored) { - return Stream.empty(); - } - }) - ).anyMatch(predicate); - } - } catch(IOException ignored) { - return true; - } - } } \ No newline at end of file diff --git a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/NegateTest.java b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/NegateTest.java index 4fd55bfd1..fcd74f7d3 100644 --- a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/NegateTest.java +++ b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/NegateTest.java @@ -111,7 +111,7 @@ public void testIdEqualsAnyOfFilter() { @Test public void testIdInRangeFilter() { - FilterExpression expression = new IdFilterRange(new IdFilterRange.IdRange(1, 3)); + FilterExpression expression = new IdFilterRange(new IdRange(1, 3)); FilterExpression negation = expression.negate(); OSMEntity testEntity = createTestOSMEntityNode(); assertNotEquals(expression.applyOSM(testEntity), negation.applyOSM(testEntity)); diff --git a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ParseTest.java b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ParseTest.java index db1bcafc9..4a33a5186 100644 --- a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ParseTest.java +++ b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ParseTest.java @@ -4,9 +4,11 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.EnumSet; +import java.util.List; import org.heigit.ohsome.oshdb.OSHDBTag; import org.heigit.ohsome.oshdb.filter.GeometryTypeFilter.GeometryType; import org.heigit.ohsome.oshdb.osm.OSMType; @@ -335,6 +337,22 @@ public void testChangesetIdFilter() { assertEquals("changeset:42", expression.toString()); } + @Test + public void testChangesetIdListFilter() { + FilterExpression expression = parser.parse("changeset:(1,2,3)"); + assertTrue(expression instanceof ChangesetIdFilterEqualsAnyOf); + assertEquals(List.of(1L, 2L, 3L), new ArrayList<>( + ((ChangesetIdFilterEqualsAnyOf) expression).getChangesetIdList())); + assertEquals("changeset:in(1,2,3)", expression.toString()); + } + + @Test + public void testChangesetIdRangeFilter() { + FilterExpression expression = parser.parse("changeset:(10..12)"); + assertTrue(expression instanceof ChangesetIdFilterRange); + assertEquals("changeset:in-range10..12", expression.toString()); + } + @Test public void testContributorIdFilterEnabled() { FilterParser parser = new FilterParser(tagTranslator, true); @@ -344,9 +362,28 @@ public void testContributorIdFilterEnabled() { assertEquals("contributor:1", expression.toString()); } + @Test + public void testContributorUserIdListFilter() { + FilterParser parser = new FilterParser(tagTranslator, true); + FilterExpression expression = parser.parse("contributor:(1,2,3)"); + assertTrue(expression instanceof ContributorUserIdFilterEqualsAnyOf); + assertEquals(List.of(1, 2, 3), new ArrayList<>( + ((ContributorUserIdFilterEqualsAnyOf) expression).getContributorUserIdList())); + assertEquals("contributor:in(1,2,3)", expression.toString()); + } + + @Test + public void testContributorUserIdRangeFilter() { + FilterParser parser = new FilterParser(tagTranslator, true); + FilterExpression expression = parser.parse("contributor:(10..12)"); + assertTrue(expression instanceof ContributorUserIdFilterRange); + assertEquals("contributor:in-range10..12", expression.toString()); + } + @Test(expected = ParserException.class) + @SuppressWarnings("ResultOfMethodCallIgnored") public void testContributorIdFilterNotEnabled() { - FilterExpression expression = parser.parse("contributor:0"); + parser.parse("contributor:0"); fail(); } } From 23067fbd9e7ddd789def57424fb1663c0fc48203 Mon Sep 17 00:00:00 2001 From: Martin Raifer Date: Fri, 21 May 2021 14:34:42 +0200 Subject: [PATCH 18/34] improve tests for contribution based filters --- .../ohsome/oshdb/filter/ApplyOSHTest.java | 81 +++++++++++++++++++ .../filter/ApplyOSMContributionTest.java | 45 ++++++++--- .../filter/ApplyOSMEntitySnapshotTest.java | 38 +++++++-- .../ohsome/oshdb/filter/ApplyOSMTest.java | 37 +++++++-- .../ohsome/oshdb/filter/FilterTest.java | 36 +++++++-- 5 files changed, 205 insertions(+), 32 deletions(-) diff --git a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSHTest.java b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSHTest.java index be793e6de..63d1b181d 100644 --- a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSHTest.java +++ b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSHTest.java @@ -4,6 +4,11 @@ import static org.junit.Assert.assertTrue; import java.io.IOException; +import org.heigit.ohsome.oshdb.osh.OSHNode; +import org.heigit.ohsome.oshdb.osh.OSHWay; +import org.heigit.ohsome.oshdb.osm.OSMNode; +import org.heigit.ohsome.oshdb.osm.OSMRelation; +import org.heigit.ohsome.oshdb.osm.OSMWay; import org.junit.Test; /** @@ -329,4 +334,80 @@ public void testGeometryFilterLength() throws IOException { FilterExpression expression = parser.parse("length:(1..2)"); assertTrue(expression.applyOSH(createTestOSHEntityWay(createTestOSMEntityWay(new long[] {})))); } + + private void testOSHEntityWithMetadata(FilterExpression expression) throws IOException { + // a node + assertTrue(expression.applyOSH(createTestOSHEntityNode( + createTestOSMEntityNode(1, 1), + createTestOSMEntityNode(42, 4), + createTestOSMEntityNode(100, 10) + ))); + assertFalse(expression.applyOSH(createTestOSHEntityNode( + createTestOSMEntityNode(1, 1), + createTestOSMEntityNode(100, 10) + ))); + // a way + assertTrue(expression.applyOSH(createTestOSHEntityWay(new OSMWay[] { + createTestOSMEntityWay(42, 4, new long[] {}) + }, new OSHNode[] {}))); + assertTrue(expression.applyOSH(createTestOSHEntityWay(new OSMWay[] { + createTestOSMEntityWay(1, 1, new long[] {}) + }, new OSHNode[] { createTestOSHEntityNode( + createTestOSMEntityNode(42, 4) + )}))); + // a relation + assertTrue(expression.applyOSH(createTestOSHEntityRelation(new OSMRelation[] { + createTestOSMEntityRelation(42, 4) + }, new OSHNode[] {}, new OSHWay[] {}))); + assertTrue(expression.applyOSH(createTestOSHEntityRelation(new OSMRelation[] { + createTestOSMEntityRelation(1, 1) + }, new OSHNode[] { createTestOSHEntityNode( + createTestOSMEntityNode(42, 4) + )}, new OSHWay[] {}))); + assertTrue(expression.applyOSH(createTestOSHEntityRelation(new OSMRelation[] { + createTestOSMEntityRelation(1, 1) + }, new OSHNode[] {}, new OSHWay[] { createTestOSHEntityWay( + createTestOSMEntityWay(42, 4, new long[] {}) + )}))); + assertTrue(expression.applyOSH(createTestOSHEntityRelation(new OSMRelation[] { + createTestOSMEntityRelation(1, 1) + }, new OSHNode[] {createTestOSHEntityNode( + createTestOSMEntityNode(42, 4) + )}, new OSHWay[] { createTestOSHEntityWay( + createTestOSMEntityWay(1, 1, new long[] {}) + )}))); + } + + @Test + public void testChangesetId() throws IOException { + testOSHEntityWithMetadata(parser.parse("changeset:42")); + } + + @Test + public void testChangesetIdList() throws IOException { + testOSHEntityWithMetadata(parser.parse("changeset:(41,42,43)")); + } + + @Test + public void testChangesetIdRange() throws IOException { + testOSHEntityWithMetadata(parser.parse("changeset:(41..43)")); + } + + @Test + public void testContributorUserId() throws IOException { + var parser = new FilterParser(tagTranslator, true); + testOSHEntityWithMetadata(parser.parse("contributor:4")); + } + + @Test + public void testContributorUserIdList() throws IOException { + var parser = new FilterParser(tagTranslator, true); + testOSHEntityWithMetadata(parser.parse("contributor:(3,4,5)")); + } + + @Test + public void testContributorUserIdRange() throws IOException { + var parser = new FilterParser(tagTranslator, true); + testOSHEntityWithMetadata(parser.parse("contributor:(3..5)")); + } } diff --git a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMContributionTest.java b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMContributionTest.java index 178bfcc8c..7ac7dff7e 100644 --- a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMContributionTest.java +++ b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMContributionTest.java @@ -10,7 +10,6 @@ import org.heigit.ohsome.oshdb.osm.OSMEntity; import org.heigit.ohsome.oshdb.util.celliterator.ContributionType; import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; -import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; import org.jetbrains.annotations.NotNull; import org.junit.Test; import org.locationtech.jts.geom.Geometry; @@ -132,25 +131,45 @@ public void testBasicFallback() { ))); } - @Test - public void testChangesetId() { - FilterExpression expression = parser.parse("changeset:42"); + private void testContribution(FilterExpression expression) { assertFalse(expression.applyOSMContribution(new TestOSMContribution( - entity, point, entity, point, 1L, 1, NO_TYPE + entity, point, entity, point, 1L, 10, NO_TYPE ))); assertTrue(expression.applyOSMContribution(new TestOSMContribution( - entity, point, entity, point, 42L, 2, NO_TYPE + entity, point, entity, point, 42L, 1, NO_TYPE ))); } + @Test + public void testChangesetId() { + testContribution(parser.parse("changeset:42")); + } + + @Test + public void testChangesetIdList() { + testContribution(parser.parse("changeset:(41,42,43)")); + } + + @Test + public void testChangesetIdRange() { + testContribution(parser.parse("changeset:(41..43)")); + } + @Test public void testContributorUserId() { - FilterExpression expression = (new FilterParser(tagTranslator, true)).parse("contributor:1"); - assertTrue(expression.applyOSMContribution(new TestOSMContribution( - entity, point, entity, point, 1L, 1, NO_TYPE - ))); - assertFalse(expression.applyOSMContribution(new TestOSMContribution( - entity, point, entity, point, 1L, 2, NO_TYPE - ))); + FilterParser parser = new FilterParser(tagTranslator, true); + testContribution(parser.parse("contributor:1")); + } + + @Test + public void testContributorUserIdList() { + FilterParser parser = new FilterParser(tagTranslator, true); + testContribution(parser.parse("contributor:(1,2,3)")); + } + + @Test + public void testContributorUserIdRange() { + FilterParser parser = new FilterParser(tagTranslator, true); + testContribution(parser.parse("contributor:(1..2)")); } } diff --git a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMEntitySnapshotTest.java b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMEntitySnapshotTest.java index 427a8867c..8edc5427f 100644 --- a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMEntitySnapshotTest.java +++ b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMEntitySnapshotTest.java @@ -69,21 +69,43 @@ public void testBasicFallback() { createTestOSMEntityNode(), gf.createLineString()))); } - @Test(expected = IllegalStateException.class) @SuppressWarnings("ResultOfMethodCallIgnored") - public void testChangesetId() { - FilterExpression expression = parser.parse("changeset:42"); + private void testFailWithSnapshot(FilterExpression expression) { expression.applyOSMEntitySnapshot( new TestOSMEntitySnapshot(createTestOSMEntityNode(), gf.createPoint())); fail(); } @Test(expected = IllegalStateException.class) - @SuppressWarnings("ResultOfMethodCallIgnored") + public void testChangesetId() { + testFailWithSnapshot(parser.parse("changeset:42")); + } + + @Test(expected = IllegalStateException.class) + public void testChangesetIdList() { + testFailWithSnapshot(parser.parse("changeset:(41,42,43)")); + } + + @Test(expected = IllegalStateException.class) + public void testChangesetIdRange() { + testFailWithSnapshot(parser.parse("changeset:(41..43)")); + } + + @Test(expected = IllegalStateException.class) public void testContributorUserId() { - FilterExpression expression = (new FilterParser(tagTranslator, true)).parse("contributor:1"); - expression.applyOSMEntitySnapshot( - new TestOSMEntitySnapshot(createTestOSMEntityNode(), gf.createPoint())); - fail(); + FilterParser parser = new FilterParser(tagTranslator, true); + testFailWithSnapshot(parser.parse("contributor:1")); + } + + @Test(expected = IllegalStateException.class) + public void testContributorUserIdList() { + FilterParser parser = new FilterParser(tagTranslator, true); + testFailWithSnapshot(parser.parse("contributor:(1,2,3)")); + } + + @Test(expected = IllegalStateException.class) + public void testContributorUserIdRange() { + FilterParser parser = new FilterParser(tagTranslator, true); + testFailWithSnapshot(parser.parse("contributor:(1..3)")); } } diff --git a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMTest.java b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMTest.java index 8d05ae0ef..e68537452 100644 --- a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMTest.java +++ b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMTest.java @@ -190,26 +190,53 @@ public void testConstant() { } @Test - public void testGeometryFilterArea() throws IOException { + public void testGeometryFilterArea() { FilterExpression expression = parser.parse("area:(1..2)"); assertTrue(expression.applyOSM(createTestOSMEntityWay(new long[] {}))); } @Test - public void testGeometryFilterLength() throws IOException { + public void testGeometryFilterLength() { FilterExpression expression = parser.parse("length:(1..2)"); assertTrue(expression.applyOSM(createTestOSMEntityWay(new long[] {}))); } @Test - public void testChangesetId() throws IOException { + public void testChangesetId() { FilterExpression expression = parser.parse("changeset:42"); assertTrue(expression.applyOSM(createTestOSMEntityNode())); } @Test - public void testContributorUserId() throws IOException { - FilterExpression expression = (new FilterParser(tagTranslator, true)).parse("contributor:1"); + public void testChangesetIdList() { + FilterExpression expression = parser.parse("changeset:(1,2,3)"); + assertTrue(expression.applyOSM(createTestOSMEntityNode())); + } + + @Test + public void testChangesetIdRange() { + FilterExpression expression = parser.parse("changeset:(10..12)"); + assertTrue(expression.applyOSM(createTestOSMEntityNode())); + } + + @Test + public void testContributorUserId() { + var parser = new FilterParser(tagTranslator, true); + FilterExpression expression = parser.parse("contributor:1"); + assertTrue(expression.applyOSM(createTestOSMEntityNode())); + } + + @Test + public void testContributorUserIdList() { + var parser = new FilterParser(tagTranslator, true); + FilterExpression expression = parser.parse("contributor:(1,2,3)"); + assertTrue(expression.applyOSM(createTestOSMEntityNode())); + } + + @Test + public void testContributorUserIdRange() { + var parser = new FilterParser(tagTranslator, true); + FilterExpression expression = parser.parse("contributor:(10..12)"); assertTrue(expression.applyOSM(createTestOSMEntityNode())); } } diff --git a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/FilterTest.java b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/FilterTest.java index 2ea12d474..027e6c33f 100644 --- a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/FilterTest.java +++ b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/FilterTest.java @@ -56,19 +56,33 @@ protected int[] createTestTags(String... keyValues) { } protected OSMNode createTestOSMEntityNode(String... keyValues) { - return new OSMNode(1, 1, 0L, 1, 1, createTestTags(keyValues), 0, 0); + return createTestOSMEntityNode(1, 1, keyValues); + } + + protected OSMNode createTestOSMEntityNode(long changesetId, int userId, String... keyValues) { + return new OSMNode(1, 1, 0L, changesetId, userId, createTestTags(keyValues), 0, 0); } protected OSMWay createTestOSMEntityWay(long[] nodeIds, String... keyValues) { + return createTestOSMEntityWay(1, 1, nodeIds, keyValues); + } + + protected OSMWay createTestOSMEntityWay( + long changesetId, int userId, long[] nodeIds, String... keyValues) { OSMMember[] refs = new OSMMember[nodeIds.length]; for (int i = 0; i < refs.length; i++) { refs[i] = new OSMMember(nodeIds[i], OSMType.NODE, 0); } - return new OSMWay(1, 1, 0L, 1, 1, createTestTags(keyValues), refs); + return new OSMWay(1, 1, 0L, changesetId, userId, createTestTags(keyValues), refs); } protected OSMRelation createTestOSMEntityRelation(String... keyValues) { - return new OSMRelation(1, 1, 0L, 1, 1, createTestTags(keyValues), + return createTestOSMEntityRelation(1, 1, keyValues); + } + + protected OSMRelation createTestOSMEntityRelation( + long changesetId, int userId, String... keyValues) { + return new OSMRelation(1, 1, 0L, changesetId, userId, createTestTags(keyValues), new OSMMember[] {}); } @@ -77,11 +91,21 @@ protected OSHNode createTestOSHEntityNode(OSMNode... versions) throws IOExceptio } protected OSHWay createTestOSHEntityWay(OSMWay...versions) throws IOException { - return OSHWayImpl.build(Arrays.asList(versions), Collections.emptyList()); + return createTestOSHEntityWay(versions, new OSHNode[] {}); + } + + protected OSHWay createTestOSHEntityWay( + OSMWay[] versions, OSHNode[] referencedNodes) throws IOException { + return OSHWayImpl.build(Arrays.asList(versions), Arrays.asList(referencedNodes)); } protected OSHRelation createTestOSHEntityRelation(OSMRelation... versions) throws IOException { - return OSHRelationImpl.build(Arrays.asList(versions), Collections.emptyList(), - Collections.emptyList()); + return createTestOSHEntityRelation(versions, new OSHNode[] {}, new OSHWay[] {}); + } + + protected OSHRelation createTestOSHEntityRelation( + OSMRelation[] versions, OSHNode[] nodes, OSHWay[] ways) throws IOException { + return OSHRelationImpl.build(Arrays.asList(versions), Arrays.asList(nodes), + Arrays.asList(ways)); } } From 7fbba7edad9176a154c585948e568663d12f54ba Mon Sep 17 00:00:00 2001 From: Martin Raifer Date: Tue, 25 May 2021 17:14:14 +0200 Subject: [PATCH 19/34] remove now redundant reimplementation of applyOSH --- .../java/org/heigit/ohsome/oshdb/filter/GeometryFilter.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/GeometryFilter.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/GeometryFilter.java index 3040b2263..da70f76dd 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/GeometryFilter.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/GeometryFilter.java @@ -67,11 +67,6 @@ protected GeometryFilter( GeometryMetricEvaluator metricEvaluator ) { super(new RangedFilter() { - @Override - public boolean applyOSH(OSHEntity entity) { - return true; - } - @Override public boolean applyOSM(OSMEntity entity) { return true; From e194364d194c47d3ac4198ae15c9c18edd49c893 Mon Sep 17 00:00:00 2001 From: Martin Raifer Date: Tue, 25 May 2021 17:28:37 +0200 Subject: [PATCH 20/34] add documentation for range/list versions of changeset/uid filters --- oshdb-filter/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/oshdb-filter/README.md b/oshdb-filter/README.md index 15dcededd..18c0b2815 100644 --- a/oshdb-filter/README.md +++ b/oshdb-filter/README.md @@ -74,7 +74,11 @@ Filters are defined in textual form. A filter expression can be composed out of | `area:(from..to-range)` | matches all features with an area that falls into the given range/interval given as two numbers in decimal or scientific notation separated by `..`. The values are interpreted as square meters (`m²`). The lower or upper limit of the interval may be omitted to select features having an area up to or starting from the given value, respectively. | `area:(123.4..1E6)` | | `length:(from..to-range)` | matches all features with a length that falls into the given range/interval given as two numbers in decimal or scientific notation separated by `..`. The values are interpreted as meters (`m`). The lower or upper limit of the interval may be omitted to select features having an area up to or starting from the given value, respectively. | `length:(100..)` | | `changeset:id` | matches OSM contributions performed in the given OSM changeset. Can only be used in queries using the `OSMContributionView`. | `changeset:42` | +| `changeset:(list,of,ids)` | matches OSM contributions performed in one of the given OSM changeset ids. Can only be used in queries using the `OSMContributionView`. | `changeset:(1,42,100)` | +| `changeset:(from..to-range)` | matches OSM contributions performed in an OSM changeset falling into the given range of changeset ids. Can only be used in queries using the `OSMContributionView`. | `changeset:(100..200)` | | `contributor:uid` | matches OSM contributions performed by a given OSM user (specified by their `uid`). Can only be used in queries using the `OSMContributionView`. | `contributor:1` | +| `contributor:(list,of,uids)` | matches OSM contributions performed by a given OSM user (specified as a list of `uid`s). Can only be used in queries using the `OSMContributionView`. | `contributor:(1,107037)` | +| `contributor:(from..to-range)` | matches OSM contributions performed by a given OSM user (specified as a range of `uid`s). Can only be used in queries using the `OSMContributionView`. | `contributor:(32..63)` | ### Operators From ed5da8f74a45bdd922b54a28ee0909887c933789 Mon Sep 17 00:00:00 2001 From: Martin Raifer Date: Thu, 27 May 2021 15:42:59 +0200 Subject: [PATCH 21/34] update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5cb8d2123..6035cb287 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ Changelog ### new features -* enhance functionality of oshdb-filter: add new `changeset: ` and (optional) `contributor: ` filters ([#380]) +* enhance functionality of oshdb-filter: add new `changeset: ` and (optional) `contributor: ` filters ([#380]) ### performance improvements From 31a6b411f3b0ab08e41cd4997a56853157d422a1 Mon Sep 17 00:00:00 2001 From: Martin Raifer Date: Thu, 27 May 2021 16:01:13 +0200 Subject: [PATCH 22/34] simplify min/max swapping Co-authored-by: Rafael Troilo --- .../java/org/heigit/ohsome/oshdb/filter/IdRange.java | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/IdRange.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/IdRange.java index a2a60f4cf..4efc8b4f5 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/IdRange.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/IdRange.java @@ -20,13 +20,8 @@ class IdRange implements Serializable { * @param toId upper limit of the range. */ IdRange(long fromId, long toId) { - if (toId < fromId) { - long buffer = toId; - toId = fromId; - fromId = buffer; - } - this.fromId = fromId; - this.toId = toId; + this.fromId = Math.min(fromId, toId); + this.toId = Math.max(fromId, toId); } /** Checks if the given id falls into the id range. */ From fbfc3adf67ab3f47494b0fdffc71111e60ba2677 Mon Sep 17 00:00:00 2001 From: Martin Raifer Date: Fri, 28 May 2021 18:45:32 +0200 Subject: [PATCH 23/34] properly implement negateable filters --- .../oshdb/filter/ChangesetIdFilterEquals.java | 5 ++ .../filter/ChangesetIdFilterEqualsAnyOf.java | 6 --- .../oshdb/filter/ChangesetIdFilterRange.java | 6 --- .../filter/ContributorUserIdFilterEquals.java | 6 --- .../ContributorUserIdFilterEqualsAnyOf.java | 6 --- .../filter/ContributorUserIdFilterRange.java | 6 --- .../heigit/ohsome/oshdb/filter/Filter.java | 46 +++++++++++++++---- .../ohsome/oshdb/filter/FilterExpression.java | 20 ++++---- .../ohsome/oshdb/filter/GeometryFilter.java | 11 ++++- .../oshdb/filter/IdFilterEqualsAnyOf.java | 14 +++++- .../ohsome/oshdb/filter/IdFilterRange.java | 10 ++++ .../ohsome/oshdb/filter/NegatableFilter.java | 42 +++++++++++++++-- .../ohsome/oshdb/filter/ApplyOSHTest.java | 1 - 13 files changed, 124 insertions(+), 55 deletions(-) diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ChangesetIdFilterEquals.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ChangesetIdFilterEquals.java index 3b6b9a8df..8f242aa53 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ChangesetIdFilterEquals.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ChangesetIdFilterEquals.java @@ -24,6 +24,11 @@ public boolean applyOSM(OSMEntity entity) { return true; } + @Override + boolean applyOSMNegated(OSMEntity entity) { + return true; + } + @Override public boolean applyOSMContribution(OSMContribution contribution) { return contribution.getChangesetId() == changesetId; diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ChangesetIdFilterEqualsAnyOf.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ChangesetIdFilterEqualsAnyOf.java index 0c5ae8d58..76feb9201 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ChangesetIdFilterEqualsAnyOf.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ChangesetIdFilterEqualsAnyOf.java @@ -6,7 +6,6 @@ import java.util.stream.Collectors; import javax.annotation.Nonnull; import org.heigit.ohsome.oshdb.osh.OSHEntity; -import org.heigit.ohsome.oshdb.osm.OSMEntity; import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; import org.jetbrains.annotations.Contract; @@ -26,11 +25,6 @@ public boolean applyOSH(OSHEntity entity) { return applyToOSHEntityRecursively(entity, v -> changesetIds.contains(v.getChangesetId())); } - @Override - public boolean applyOSM(OSMEntity entity) { - return true; - } - @Override public boolean applyOSMContribution(OSMContribution contribution) { return changesetIds.contains(contribution.getChangesetId()); diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ChangesetIdFilterRange.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ChangesetIdFilterRange.java index 8e5b60789..9b0cfb4fc 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ChangesetIdFilterRange.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ChangesetIdFilterRange.java @@ -1,7 +1,6 @@ package org.heigit.ohsome.oshdb.filter; import org.heigit.ohsome.oshdb.osh.OSHEntity; -import org.heigit.ohsome.oshdb.osm.OSMEntity; import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; @@ -16,11 +15,6 @@ public boolean applyOSH(OSHEntity entity) { return applyToOSHEntityRecursively(entity, v -> changesetIdRange.test(v.getChangesetId())); } - @Override - public boolean applyOSM(OSMEntity entity) { - return true; - } - @Override public boolean applyOSMContribution(OSMContribution contribution) { return changesetIdRange.test(contribution.getChangesetId()); diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ContributorUserIdFilterEquals.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ContributorUserIdFilterEquals.java index 6b8d8c0d1..d0fb99201 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ContributorUserIdFilterEquals.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ContributorUserIdFilterEquals.java @@ -1,7 +1,6 @@ package org.heigit.ohsome.oshdb.filter; import org.heigit.ohsome.oshdb.osh.OSHEntity; -import org.heigit.ohsome.oshdb.osm.OSMEntity; import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; import org.jetbrains.annotations.Contract; @@ -19,11 +18,6 @@ public boolean applyOSH(OSHEntity entity) { return applyToOSHEntityRecursively(entity, v -> v.getUserId() == userId); } - @Override - public boolean applyOSM(OSMEntity entity) { - return true; - } - @Override public boolean applyOSMContribution(OSMContribution contribution) { return contribution.getContributorUserId() == userId; diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ContributorUserIdFilterEqualsAnyOf.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ContributorUserIdFilterEqualsAnyOf.java index ac6fe6e5a..2b7ccc626 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ContributorUserIdFilterEqualsAnyOf.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ContributorUserIdFilterEqualsAnyOf.java @@ -6,7 +6,6 @@ import java.util.stream.Collectors; import javax.annotation.Nonnull; import org.heigit.ohsome.oshdb.osh.OSHEntity; -import org.heigit.ohsome.oshdb.osm.OSMEntity; import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; import org.jetbrains.annotations.Contract; @@ -26,11 +25,6 @@ public boolean applyOSH(OSHEntity entity) { return applyToOSHEntityRecursively(entity, v -> contributorUserIds.contains(v.getUserId())); } - @Override - public boolean applyOSM(OSMEntity entity) { - return true; - } - @Override public boolean applyOSMContribution(OSMContribution contribution) { return contributorUserIds.contains(contribution.getContributorUserId()); diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ContributorUserIdFilterRange.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ContributorUserIdFilterRange.java index 04f5006e3..7238a49dd 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ContributorUserIdFilterRange.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ContributorUserIdFilterRange.java @@ -1,7 +1,6 @@ package org.heigit.ohsome.oshdb.filter; import org.heigit.ohsome.oshdb.osh.OSHEntity; -import org.heigit.ohsome.oshdb.osm.OSMEntity; import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; @@ -16,11 +15,6 @@ public boolean applyOSH(OSHEntity entity) { return applyToOSHEntityRecursively(entity, v -> contributorUserIdRange.test(v.getUserId())); } - @Override - public boolean applyOSM(OSMEntity entity) { - return true; - } - @Override public boolean applyOSMContribution(OSMContribution contribution) { return contributorUserIdRange.test(contribution.getContributorUserId()); diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/Filter.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/Filter.java index b53355e0d..cb09018ba 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/Filter.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/Filter.java @@ -41,11 +41,21 @@ static Filter byOSHEntity(OSHEntityFilter oshCallback) { * @return a filter object which filters using the given predicate */ static Filter byOSMEntity(OSMEntityFilter osmCallback) { - return by(ignored -> true, osmCallback); + return new NegatableFilter(new FilterInternal() { + @Override + public boolean applyOSM(OSMEntity entity) { + return osmCallback.test(entity); + } + + @Override + boolean applyOSMNegated(OSMEntity entity) { + return !osmCallback.test(entity); + } + }); } /** - * Constructs a simple filter based on two predicates. + * Constructs a custom filter based on OSH & OSM predicates. * *

The callbacks are called for each OSH/OSM entity and decide whether the OSH/OSM object * should be kept (by returning true) or discarded (by returning false).

@@ -59,11 +69,26 @@ static Filter byOSMEntity(OSMEntityFilter osmCallback) { static Filter by( OSHEntityFilter oshCallback, OSMEntityFilter osmCallback) { - return by(oshCallback, osmCallback, (ignored, ignored2) -> true); + return new NegatableFilter(new FilterInternal() { + @Override + public boolean applyOSH(OSHEntity entity) { + return oshCallback.test(entity); + } + + @Override + public boolean applyOSM(OSMEntity entity) { + return osmCallback.test(entity); + } + + @Override + boolean applyOSMNegated(OSMEntity entity) { + return !osmCallback.test(entity); + } + }); } /** - * Constructs a simple filter based on two predicates and a geometry test. + * Constructs a custom filter based on OSH & OSM predicates and a geometry test. * *

The callbacks are called for each OSM feature and decide whether the feature * should be kept (by returning true) or discarded (by returning false).

@@ -85,19 +110,24 @@ static Filter by( ) { return new NegatableFilter(new FilterInternal() { @Override - public boolean applyOSM(OSMEntity entity) { - return osmCallback.test(entity); + public boolean applyOSH(OSHEntity entity) { + return oshCallback.test(entity); } @Override - public boolean applyOSH(OSHEntity entity) { - return oshCallback.test(entity); + public boolean applyOSM(OSMEntity entity) { + return osmCallback.test(entity); } @Override public boolean applyOSMGeometry(OSMEntity entity, Supplier geometrySupplier) { return geomCallback.test(entity, geometrySupplier); } + + @Override + boolean applyOSMGeometryNegated(OSMEntity entity, Supplier geometrySupplier) { + return !geomCallback.test(entity, geometrySupplier); + } }); } } diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/FilterExpression.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/FilterExpression.java index d034e8692..4c46cc83c 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/FilterExpression.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/FilterExpression.java @@ -21,16 +21,6 @@ * of boolean operators, parentheses, tag filters and/or other filters.

*/ public interface FilterExpression extends Serializable { - - /** - * Apply the filter to an OSM entity. - * - * @param entity the OSM entity to check. - * @return true if the entity fulfills the specified filter, otherwise false. - */ - @Contract(pure = true) - boolean applyOSM(OSMEntity entity); - /** * Apply the filter to an OSH entity. * @@ -43,9 +33,19 @@ public interface FilterExpression extends Serializable { */ @Contract(pure = true) default boolean applyOSH(OSHEntity entity) { + // dummy implementation for basic filters: pass all OSH entities -> check its versions instead return true; } + /** + * Apply the filter to an OSM entity. + * + * @param entity the OSM entity to check. + * @return true if the entity fulfills the specified filter, otherwise false. + */ + @Contract(pure = true) + boolean applyOSM(OSMEntity entity); + /** * Apply the filter to an "OSM feature" (i.e. an entity with a geometry). * diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/GeometryFilter.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/GeometryFilter.java index da70f76dd..01460c483 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/GeometryFilter.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/GeometryFilter.java @@ -4,7 +4,6 @@ import java.util.function.Supplier; import java.util.function.ToDoubleFunction; import javax.annotation.Nonnull; -import org.heigit.ohsome.oshdb.osh.OSHEntity; import org.heigit.ohsome.oshdb.osm.OSMEntity; import org.locationtech.jts.geom.Geometry; @@ -72,11 +71,21 @@ public boolean applyOSM(OSMEntity entity) { return true; } + @Override + boolean applyOSMNegated(OSMEntity entity) { + return true; + } + @Override public boolean applyOSMGeometry(OSMEntity entity, Supplier geometrySupplier) { return valueRange.test(metricEvaluator.applyAsDouble(geometrySupplier.get())); } + @Override + boolean applyOSMGeometryNegated(OSMEntity entity, Supplier geometrySupplier) { + return !applyOSMGeometry(entity, geometrySupplier); + } + @Override public String toString() { return metricEvaluator.toString() + ":" + valueRange.toString(); diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/IdFilterEqualsAnyOf.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/IdFilterEqualsAnyOf.java index e45b5eb84..9904b9025 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/IdFilterEqualsAnyOf.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/IdFilterEqualsAnyOf.java @@ -17,15 +17,25 @@ public class IdFilterEqualsAnyOf extends NegatableFilter { private final Set ids = new HashSet<>(idList); @Override - public boolean applyOSM(OSMEntity entity) { + public boolean applyOSH(OSHEntity entity) { return this.ids.contains(entity.getId()); } @Override - public boolean applyOSH(OSHEntity entity) { + boolean applyOSHNegated(OSHEntity entity) { + return !this.applyOSH(entity); + } + + @Override + public boolean applyOSM(OSMEntity entity) { return this.ids.contains(entity.getId()); } + @Override + boolean applyOSMNegated(OSMEntity entity) { + return !this.applyOSM(entity); + } + @Override public String toString() { return "id:in" + this.ids.stream().map(String::valueOf).collect(Collectors.joining(",")); diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/IdFilterRange.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/IdFilterRange.java index 7ff4184dd..ac332e888 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/IdFilterRange.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/IdFilterRange.java @@ -15,11 +15,21 @@ public boolean applyOSH(OSHEntity entity) { return range.test(entity.getId()); } + @Override + boolean applyOSHNegated(OSHEntity entity) { + return !this.applyOSH(entity); + } + @Override public boolean applyOSM(OSMEntity entity) { return range.test(entity.getId()); } + @Override + boolean applyOSMNegated(OSMEntity entity) { + return !this.applyOSM(entity); + } + @Override public String toString() { return "id:in-range" + range; diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/NegatableFilter.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/NegatableFilter.java index f5a1c905f..23c30f34f 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/NegatableFilter.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/NegatableFilter.java @@ -10,6 +10,7 @@ import org.heigit.ohsome.oshdb.osm.OSMEntity; import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; +import org.jetbrains.annotations.Contract; import org.locationtech.jts.geom.Geometry; /** @@ -22,6 +23,35 @@ public FilterExpression negate() { throw new IllegalStateException("Invalid call of inner negate() on a negatable filter"); } + @Override + public boolean applyOSH(OSHEntity entity) { + return true; + } + + /** Inverse of {@link FilterExpression#applyOSH(OSHEntity)} */ + @Contract(pure = true) + boolean applyOSHNegated(OSHEntity entity) { + return true; + } + + @Override + public boolean applyOSM(OSMEntity entity) { + return true; + } + + /** Inverse of {@link FilterExpression#applyOSM(OSMEntity)} */ + @Contract(pure = true) + boolean applyOSMNegated(OSMEntity entity) { + return true; + } + + /** Inverse of {@link FilterExpression#applyOSMGeometry(OSMEntity, Supplier)}. */ + @Contract(pure = true) + boolean applyOSMGeometryNegated(OSMEntity entity, Supplier geometrySupplier) { + // dummy implementation for basic filters: ignores the geometry, just looks at the OSM entity + return applyOSMNegated(entity); + } + /** * Helper method to test a predicate on all versions of an OSH entity, including its * references/members and references of members. @@ -77,17 +107,23 @@ protected NegatableFilter(@Nonnull FilterInternal filter) { @Override public boolean applyOSH(OSHEntity entity) { - return this.filter.applyOSH(entity) ^ this.negated; + return negated + ? this.filter.applyOSHNegated(entity) + : this.filter.applyOSH(entity); } @Override public boolean applyOSM(OSMEntity entity) { - return this.filter.applyOSM(entity) ^ this.negated; + return negated + ? this.filter.applyOSMNegated(entity) + : this.filter.applyOSM(entity); } @Override public boolean applyOSMGeometry(OSMEntity entity, Supplier geometrySupplier) { - return this.filter.applyOSMGeometry(entity, geometrySupplier) ^ this.negated; + return negated + ? this.filter.applyOSMGeometryNegated(entity, geometrySupplier) + : this.filter.applyOSMGeometry(entity, geometrySupplier); } @Override diff --git a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSHTest.java b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSHTest.java index 63d1b181d..17573efc4 100644 --- a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSHTest.java +++ b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSHTest.java @@ -6,7 +6,6 @@ import java.io.IOException; import org.heigit.ohsome.oshdb.osh.OSHNode; import org.heigit.ohsome.oshdb.osh.OSHWay; -import org.heigit.ohsome.oshdb.osm.OSMNode; import org.heigit.ohsome.oshdb.osm.OSMRelation; import org.heigit.ohsome.oshdb.osm.OSMWay; import org.junit.Test; From b51d75f0850ab9d87c043ad587bce0987bf8d7a6 Mon Sep 17 00:00:00 2001 From: Martin Raifer Date: Tue, 1 Jun 2021 11:10:43 +0200 Subject: [PATCH 24/34] make `byOSHEntity` filters a bit sharper --- .../org/heigit/ohsome/oshdb/filter/Filter.java | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/Filter.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/Filter.java index cb09018ba..9425d2fb6 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/Filter.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/Filter.java @@ -26,7 +26,22 @@ public interface Filter extends FilterExpression { * @return a filter object which filters using the given predicate */ static Filter byOSHEntity(OSHEntityFilter oshCallback) { - return by(oshCallback, ignored -> true); + return new NegatableFilter(new FilterInternal() { + @Override + public boolean applyOSM(OSMEntity entity) { + return true; + } + + @Override + public boolean applyOSH(OSHEntity entity) { + return oshCallback.test(entity); + } + + @Override + boolean applyOSHNegated(OSHEntity entity) { + return !oshCallback.test(entity); + } + }); } /** From 9f8fecf36cb35be8e1ef7f21b1c7ec887a24e961 Mon Sep 17 00:00:00 2001 From: Martin Raifer Date: Wed, 2 Jun 2021 12:03:12 +0200 Subject: [PATCH 25/34] drop `Filter.by` helpers Reason: They did not really fulfil a specific purpose because one can already "and" combine OSH+OSM filters by calling `.filter` twice, and the geometry callback filter basically just replicates the `.filter(Predicate)` anyway. Also the given documentation and examples were not in line with the actual behaviour when one would have negated the filter, so it in the end just confusing to use. --- CHANGELOG.md | 2 +- .../heigit/ohsome/oshdb/filter/Filter.java | 79 ------------------- .../ohsome/oshdb/filter/ApplyOSMTest.java | 1 - .../ohsome/oshdb/filter/FilterByTest.java | 31 +------- 4 files changed, 3 insertions(+), 110 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6035cb287..ebee22f0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ Changelog ### breaking changes * reorganize java packages, moving them from `org/heigit/bigspatialdata` to `org/heigit/ohsome` -* integrate [ohsome-filter](https://gitlab.gistools.geog.uni-heidelberg.de/giscience/big-data/ohsome/libs/ohsome-filter) module fully into this repository, renaming it to `oshdb-filter` ([#306]), and enhance filtering capabilities ([#380]) +* integrate [ohsome-filter](https://gitlab.gistools.geog.uni-heidelberg.de/giscience/big-data/ohsome/libs/ohsome-filter) module fully into this repository, renaming it to `oshdb-filter` ([#306]) * rename and move submodules of `oshdb-tool` ([#384]) * rename some classes, methods and enum constants; move some classes/interfaces ([#369], [#374]) diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/Filter.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/Filter.java index 9425d2fb6..9fbf9f314 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/Filter.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/Filter.java @@ -1,13 +1,10 @@ package org.heigit.ohsome.oshdb.filter; -import java.util.function.Supplier; import org.heigit.ohsome.oshdb.filter.NegatableFilter.FilterInternal; import org.heigit.ohsome.oshdb.osh.OSHEntity; import org.heigit.ohsome.oshdb.osm.OSMEntity; import org.heigit.ohsome.oshdb.util.function.OSHEntityFilter; import org.heigit.ohsome.oshdb.util.function.OSMEntityFilter; -import org.heigit.ohsome.oshdb.util.function.SerializableBiPredicate; -import org.locationtech.jts.geom.Geometry; /** * A filter condition which can be applied to an OSM entity. @@ -69,80 +66,4 @@ boolean applyOSMNegated(OSMEntity entity) { }); } - /** - * Constructs a custom filter based on OSH & OSM predicates. - * - *

The callbacks are called for each OSH/OSM entity and decide whether the OSH/OSM object - * should be kept (by returning true) or discarded (by returning false).

- * - *

Example: `Filter.by(osh -> osh.getId() == 42, osm -> osm.getVersion() == 1);`

- * - * @param oshCallback predicate which tests osh entities - * @param osmCallback predicate which tests osm entities - * @return a filter object which filters using the given predicates - */ - static Filter by( - OSHEntityFilter oshCallback, - OSMEntityFilter osmCallback) { - return new NegatableFilter(new FilterInternal() { - @Override - public boolean applyOSH(OSHEntity entity) { - return oshCallback.test(entity); - } - - @Override - public boolean applyOSM(OSMEntity entity) { - return osmCallback.test(entity); - } - - @Override - boolean applyOSMNegated(OSMEntity entity) { - return !osmCallback.test(entity); - } - }); - } - - /** - * Constructs a custom filter based on OSH & OSM predicates and a geometry test. - * - *

The callbacks are called for each OSM feature and decide whether the feature - * should be kept (by returning true) or discarded (by returning false).

- * - *

Example: `Filter.by(osh -> osh.getId() == 42, osm -> osm.getVersion() == 1, - * (osm, geometrySupplier) -> geometrySupplier.get() instanceOf Polygon);`

- * - * @param oshCallback predicate which tests osh entities - * @param osmCallback predicate which tests osm entities - * @param geomCallback predicate which tests osm geometries, alongside the geometry (given as a - * supplier method), the entity itself is given also to be able to perform - * filtering on the whole "OSM Feature" (metadata + tags + geometry). - * @return a filter object which filters using the given predicates - */ - static Filter by( - OSHEntityFilter oshCallback, - OSMEntityFilter osmCallback, - SerializableBiPredicate> geomCallback - ) { - return new NegatableFilter(new FilterInternal() { - @Override - public boolean applyOSH(OSHEntity entity) { - return oshCallback.test(entity); - } - - @Override - public boolean applyOSM(OSMEntity entity) { - return osmCallback.test(entity); - } - - @Override - public boolean applyOSMGeometry(OSMEntity entity, Supplier geometrySupplier) { - return geomCallback.test(entity, geometrySupplier); - } - - @Override - boolean applyOSMGeometryNegated(OSMEntity entity, Supplier geometrySupplier) { - return !geomCallback.test(entity, geometrySupplier); - } - }); - } } diff --git a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMTest.java b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMTest.java index e68537452..e4dd56cf0 100644 --- a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMTest.java +++ b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMTest.java @@ -3,7 +3,6 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import java.io.IOException; import org.junit.Test; /** diff --git a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/FilterByTest.java b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/FilterByTest.java index 14ef26e45..544fb9b78 100644 --- a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/FilterByTest.java +++ b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/FilterByTest.java @@ -7,11 +7,7 @@ import org.heigit.ohsome.oshdb.osh.OSHNode; import org.heigit.ohsome.oshdb.osm.OSMNode; import org.junit.Test; -import org.locationtech.jts.geom.Coordinate; -import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.GeometryFactory; -import org.locationtech.jts.geom.Point; -import org.locationtech.jts.geom.Polygon; /** * Tests helper methods for creating filters from lambda functions. @@ -29,7 +25,7 @@ public void testFilterByOSMEntity() { FilterExpression expression; expression = Filter.byOSMEntity(e -> e.getVersion() == 1); assertTrue(expression.applyOSH(testOSHEntity) && expression.applyOSM(testOSMEntity)); - expression = Filter.byOSMEntity(e -> e.getVersion() != 1); + expression = expression.negate(); assertFalse(expression.applyOSH(testOSHEntity) && expression.applyOSM(testOSMEntity)); } @@ -38,30 +34,7 @@ public void testFilterByOSHEntity() { FilterExpression expression; expression = Filter.byOSHEntity(e -> e.getId() == 1); assertTrue(expression.applyOSH(testOSHEntity) && expression.applyOSM(testOSMEntity)); - expression = Filter.byOSHEntity(e -> e.getId() != 1); + expression = expression.negate(); assertFalse(expression.applyOSH(testOSHEntity) && expression.applyOSM(testOSMEntity)); } - - @Test - public void testFilterByCombined() { - FilterExpression expression; - expression = Filter.by(e -> e.getId() == 1, e -> e.getVersion() == 1); - assertTrue(expression.applyOSH(testOSHEntity) && expression.applyOSM(testOSMEntity)); - expression = Filter.by(e -> e.getId() != 1, e -> e.getVersion() == 1); - assertFalse(expression.applyOSH(testOSHEntity) && expression.applyOSM(testOSMEntity)); - expression = Filter.by(e -> e.getId() == 1, e -> e.getVersion() != 1); - assertFalse(expression.applyOSH(testOSHEntity) && expression.applyOSM(testOSMEntity)); - } - - @Test - public void testFilterByCombinedWithGeom() { - final Geometry emptyPoint = gf.createPoint(new Coordinate(0, 0)); - FilterExpression expression; - expression = Filter.by(e -> true, e -> true, (e, g) -> g.get() instanceof Point); - assertTrue(expression.applyOSMGeometry(testOSMEntity, () -> emptyPoint)); - assertTrue(expression.applyOSMGeometry(testOSMEntity, emptyPoint)); - expression = Filter.by(e -> true, e -> true, (e, g) -> g.get() instanceof Polygon); - assertFalse(expression.applyOSMGeometry(testOSMEntity, () -> emptyPoint)); - assertFalse(expression.applyOSMGeometry(testOSMEntity, emptyPoint)); - } } From a34b9734df47f2daa4d9aa4be4f91f9a58204880 Mon Sep 17 00:00:00 2001 From: Martin Raifer Date: Fri, 4 Jun 2021 15:26:02 +0200 Subject: [PATCH 26/34] remove unused variable and import --- .../test/java/org/heigit/ohsome/oshdb/filter/FilterByTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/FilterByTest.java b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/FilterByTest.java index 544fb9b78..dbc866d16 100644 --- a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/FilterByTest.java +++ b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/FilterByTest.java @@ -7,13 +7,11 @@ import org.heigit.ohsome.oshdb.osh.OSHNode; import org.heigit.ohsome.oshdb.osm.OSMNode; import org.junit.Test; -import org.locationtech.jts.geom.GeometryFactory; /** * Tests helper methods for creating filters from lambda functions. */ public class FilterByTest extends FilterTest { - private final GeometryFactory gf = new GeometryFactory(); private final OSMNode testOSMEntity = createTestOSMEntityNode(); private final OSHNode testOSHEntity = createTestOSHEntityNode(testOSMEntity); From 674445eb18e322d50125b03f7a907ebf1a5dd466 Mon Sep 17 00:00:00 2001 From: Martin Raifer Date: Fri, 4 Jun 2021 15:31:55 +0200 Subject: [PATCH 27/34] also test referenced nodes of way-members of a relation --- .../java/org/heigit/ohsome/oshdb/filter/ApplyOSHTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSHTest.java b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSHTest.java index 17573efc4..79200fb2a 100644 --- a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSHTest.java +++ b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSHTest.java @@ -370,11 +370,11 @@ private void testOSHEntityWithMetadata(FilterExpression expression) throws IOExc )}))); assertTrue(expression.applyOSH(createTestOSHEntityRelation(new OSMRelation[] { createTestOSMEntityRelation(1, 1) + }, new OSHNode[] {}, new OSHWay[] { createTestOSHEntityWay(new OSMWay[]{ + createTestOSMEntityWay(1, 1, new long[]{1}) }, new OSHNode[] {createTestOSHEntityNode( createTestOSMEntityNode(42, 4) - )}, new OSHWay[] { createTestOSHEntityWay( - createTestOSMEntityWay(1, 1, new long[] {}) - )}))); + )})}))); } @Test From d1301938cfdcf4f8090383022116c81ac0693cb7 Mon Sep 17 00:00:00 2001 From: Martin Raifer Date: Fri, 4 Jun 2021 15:46:12 +0200 Subject: [PATCH 28/34] explicitly test negated versions of geometry filters --- .../ohsome/oshdb/filter/GeometryFilter.java | 5 ----- .../ohsome/oshdb/filter/ApplyOSMGeometryTest.java | 15 +++++++++++++++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/GeometryFilter.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/GeometryFilter.java index 01460c483..5ef5b921a 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/GeometryFilter.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/GeometryFilter.java @@ -71,11 +71,6 @@ public boolean applyOSM(OSMEntity entity) { return true; } - @Override - boolean applyOSMNegated(OSMEntity entity) { - return true; - } - @Override public boolean applyOSMGeometry(OSMEntity entity, Supplier geometrySupplier) { return valueRange.test(metricEvaluator.applyAsDouble(geometrySupplier.get())); diff --git a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMGeometryTest.java b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMGeometryTest.java index 0ae18a7e8..867a76d79 100644 --- a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMGeometryTest.java +++ b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMGeometryTest.java @@ -20,6 +20,8 @@ public class ApplyOSMGeometryTest extends FilterTest { public void testGeometryTypeFilterPoint() { FilterExpression expression = parser.parse("geometry:point"); assertTrue(expression.applyOSMGeometry(createTestOSMEntityNode(), gf.createPoint())); + // negated + assertFalse(expression.negate().applyOSMGeometry(createTestOSMEntityNode(), gf.createPoint())); } @Test @@ -113,6 +115,11 @@ public void testGeometryFilterArea() { // approx 4.9m² OSHDBGeometryBuilder.getGeometry(new OSHDBBoundingBox(0, 0, 2E-5, 2E-5)) )); + // negated + assertTrue(expression.negate().applyOSMGeometry(entity, + // approx 0.3m² + OSHDBGeometryBuilder.getGeometry(new OSHDBBoundingBox(0, 0, 5E-6, 5E-6)) + )); } @Test @@ -140,5 +147,13 @@ public void testGeometryFilterLength() { new Coordinate(2E-5, 0) }) )); + // negated + assertTrue(expression.negate().applyOSMGeometry(entity, + // approx 0.6m + gf.createLineString(new Coordinate[] { + new Coordinate(0, 0), + new Coordinate(5E-6, 0) + }) + )); } } From 495ec0d14232e33634d9fda7cb101217a1b2d440 Mon Sep 17 00:00:00 2001 From: Martin Raifer Date: Fri, 4 Jun 2021 16:15:23 +0200 Subject: [PATCH 29/34] drop unnecessarily overridden method --- .../java/org/heigit/ohsome/oshdb/filter/NegatableFilter.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/NegatableFilter.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/NegatableFilter.java index 23c30f34f..0867dc986 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/NegatableFilter.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/NegatableFilter.java @@ -23,11 +23,6 @@ public FilterExpression negate() { throw new IllegalStateException("Invalid call of inner negate() on a negatable filter"); } - @Override - public boolean applyOSH(OSHEntity entity) { - return true; - } - /** Inverse of {@link FilterExpression#applyOSH(OSHEntity)} */ @Contract(pure = true) boolean applyOSHNegated(OSHEntity entity) { From 974620c1d4ecb86aa708332131a46ab73d1f661a Mon Sep 17 00:00:00 2001 From: Martin Raifer Date: Fri, 4 Jun 2021 16:47:53 +0200 Subject: [PATCH 30/34] don't crash on user-supplied filter classes during optimization --- .../oshdb/api/mapreducer/MapReducer.java | 6 +++++- .../oshdb/api/tests/TestOSHDBFilter.java | 19 +++++++++++++++++++ .../ohsome/oshdb/filter/FilterExpression.java | 7 ++++--- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapReducer.java b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapReducer.java index ebcf66dcc..f8e71d00b 100644 --- a/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapReducer.java +++ b/oshdb-api/src/main/java/org/heigit/ohsome/oshdb/api/mapreducer/MapReducer.java @@ -2170,7 +2170,11 @@ private MapReducer optimizeFilters(MapReducer mapRed, FilterExpression // basic optimizations mapRed = optimizeFilters0(mapRed, filter); // more advanced optimizations that rely on analyzing the DNF of a filter expression - mapRed = optimizeFilters1(mapRed, filter); + try { + mapRed = optimizeFilters1(mapRed, filter); + } catch (IllegalStateException ignored) { + // if a filter cannot be normalized -> just don't perform this optimization step + } return mapRed; } diff --git a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestOSHDBFilter.java b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestOSHDBFilter.java index 7d91a3928..480a18ed4 100644 --- a/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestOSHDBFilter.java +++ b/oshdb-api/src/test/java/org/heigit/ohsome/oshdb/api/tests/TestOSHDBFilter.java @@ -10,7 +10,9 @@ import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer; import org.heigit.ohsome.oshdb.api.mapreducer.OSMContributionView; import org.heigit.ohsome.oshdb.api.mapreducer.OSMEntitySnapshotView; +import org.heigit.ohsome.oshdb.filter.FilterExpression; import org.heigit.ohsome.oshdb.filter.FilterParser; +import org.heigit.ohsome.oshdb.osm.OSMEntity; import org.heigit.ohsome.oshdb.osm.OSMType; import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; @@ -133,4 +135,21 @@ public void testFilterNonExistentTag() throws Exception { fail("should not crash on non-existent tags"); } } + + @Test + public void testFilterNotCrashDuringNormalize() throws Exception { + var mr = createMapReducerOSMContribution(); + mr = mr.filter(new FilterExpression() { + @Override + public boolean applyOSM(OSMEntity entity) { + return false; + } + + @Override + public FilterExpression negate() { + throw new RuntimeException("not implemented"); + } + }); + assertEquals(0, (long) mr.count()); + } } diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/FilterExpression.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/FilterExpression.java index 4c46cc83c..0317c43db 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/FilterExpression.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/FilterExpression.java @@ -123,6 +123,9 @@ default boolean applyOSMContribution(OSMContribution contribution) { *

for example: A∧(B∨C) ⇔ (A∧B)∨(A∧C)

* * @return a disjunction of conjunctions of filter expressions: A∧B∧… ∨ C∧D∧… ∨ … + * @throws IllegalStateException if the filter cannot be normalized (all filters provided by the + * oshdb-filter module are normalizable, but this can occur for + * user defined filter expressions) */ @Contract(pure = true) default List> normalize() { @@ -150,9 +153,7 @@ default List> normalize() { combined.addAll(exp2); return combined; } else { - String error = "unsupported state during filter normalization"; - assert false : error; - throw new IllegalStateException(error); + throw new IllegalStateException("unsupported state during filter normalization"); } } } From 18a822a30f07e1a157ee5fcaecc6d4cc8e75a74d Mon Sep 17 00:00:00 2001 From: Martin Raifer Date: Fri, 4 Jun 2021 16:56:29 +0200 Subject: [PATCH 31/34] add test for the default applyOSMNegated --- .../heigit/ohsome/oshdb/filter/ChangesetIdFilterEquals.java | 5 ----- .../java/org/heigit/ohsome/oshdb/filter/ApplyOSMTest.java | 2 ++ 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ChangesetIdFilterEquals.java b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ChangesetIdFilterEquals.java index 8f242aa53..3b6b9a8df 100644 --- a/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ChangesetIdFilterEquals.java +++ b/oshdb-filter/src/main/java/org/heigit/ohsome/oshdb/filter/ChangesetIdFilterEquals.java @@ -24,11 +24,6 @@ public boolean applyOSM(OSMEntity entity) { return true; } - @Override - boolean applyOSMNegated(OSMEntity entity) { - return true; - } - @Override public boolean applyOSMContribution(OSMContribution contribution) { return contribution.getChangesetId() == changesetId; diff --git a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMTest.java b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMTest.java index e4dd56cf0..6f8369036 100644 --- a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMTest.java +++ b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMTest.java @@ -204,6 +204,7 @@ public void testGeometryFilterLength() { public void testChangesetId() { FilterExpression expression = parser.parse("changeset:42"); assertTrue(expression.applyOSM(createTestOSMEntityNode())); + assertTrue(expression.negate().applyOSM(createTestOSMEntityNode())); } @Test @@ -223,6 +224,7 @@ public void testContributorUserId() { var parser = new FilterParser(tagTranslator, true); FilterExpression expression = parser.parse("contributor:1"); assertTrue(expression.applyOSM(createTestOSMEntityNode())); + assertTrue(expression.negate().applyOSM(createTestOSMEntityNode())); } @Test From 53eacccb20d5bd49f76df1b07ffd158849251c45 Mon Sep 17 00:00:00 2001 From: Martin Raifer Date: Fri, 4 Jun 2021 17:27:05 +0200 Subject: [PATCH 32/34] also test "fallback" implementation in negatable filter --- .../ohsome/oshdb/filter/ApplyOSMContributionTest.java | 10 ++++++++++ .../oshdb/filter/ApplyOSMEntitySnapshotTest.java | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMContributionTest.java b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMContributionTest.java index 7ac7dff7e..3e87e3d03 100644 --- a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMContributionTest.java +++ b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMContributionTest.java @@ -131,6 +131,16 @@ public void testBasicFallback() { ))); } + @Test + public void testNegatableFilter() { + FilterExpression expression = parser.parse("id:(1,2)"); + var testObject = new TestOSMContribution( + entity, point, entity, point, 1L, 2, NO_TYPE + ); + assertTrue(expression.applyOSMContribution(testObject)); + assertFalse(expression.negate().applyOSMContribution(testObject)); + } + private void testContribution(FilterExpression expression) { assertFalse(expression.applyOSMContribution(new TestOSMContribution( entity, point, entity, point, 1L, 10, NO_TYPE diff --git a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMEntitySnapshotTest.java b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMEntitySnapshotTest.java index 8edc5427f..ac0a03783 100644 --- a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMEntitySnapshotTest.java +++ b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMEntitySnapshotTest.java @@ -67,6 +67,16 @@ public void testBasicFallback() { createTestOSMEntityNode(), gf.createPoint()))); assertFalse(expression.applyOSMEntitySnapshot(new TestOSMEntitySnapshot( createTestOSMEntityNode(), gf.createLineString()))); + assertFalse(expression.negate().applyOSMEntitySnapshot(new TestOSMEntitySnapshot( + createTestOSMEntityNode(), gf.createPoint()))); + } + + @Test + public void testNegatableFilter() { + FilterExpression expression = parser.parse("id:(1,2)"); + var testObject = new TestOSMEntitySnapshot(createTestOSMEntityNode(), gf.createPoint()); + assertTrue(expression.applyOSMEntitySnapshot(testObject)); + assertFalse(expression.negate().applyOSMEntitySnapshot(testObject)); } @SuppressWarnings("ResultOfMethodCallIgnored") From a5b388801c2d2aea80d650d64f6dcede898e69a7 Mon Sep 17 00:00:00 2001 From: Martin Raifer Date: Fri, 4 Jun 2021 17:41:01 +0200 Subject: [PATCH 33/34] add test for (negated) applyOSMGeometry fallback --- .../ohsome/oshdb/filter/FilterByTest.java | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/FilterByTest.java b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/FilterByTest.java index dbc866d16..d047b2a81 100644 --- a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/FilterByTest.java +++ b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/FilterByTest.java @@ -7,11 +7,13 @@ import org.heigit.ohsome.oshdb.osh.OSHNode; import org.heigit.ohsome.oshdb.osm.OSMNode; import org.junit.Test; +import org.locationtech.jts.geom.GeometryFactory; /** * Tests helper methods for creating filters from lambda functions. */ public class FilterByTest extends FilterTest { + private final GeometryFactory gf = new GeometryFactory(); private final OSMNode testOSMEntity = createTestOSMEntityNode(); private final OSHNode testOSHEntity = createTestOSHEntityNode(testOSMEntity); @@ -19,20 +21,27 @@ public FilterByTest() throws IOException { } @Test - public void testFilterByOSMEntity() { + public void testFilterByOSHEntity() { FilterExpression expression; - expression = Filter.byOSMEntity(e -> e.getVersion() == 1); + expression = Filter.byOSHEntity(e -> e.getId() == 1); assertTrue(expression.applyOSH(testOSHEntity) && expression.applyOSM(testOSMEntity)); expression = expression.negate(); assertFalse(expression.applyOSH(testOSHEntity) && expression.applyOSM(testOSMEntity)); } @Test - public void testFilterByOSHEntity() { + public void testFilterByOSMEntity() { FilterExpression expression; - expression = Filter.byOSHEntity(e -> e.getId() == 1); + expression = Filter.byOSMEntity(e -> e.getVersion() == 1); assertTrue(expression.applyOSH(testOSHEntity) && expression.applyOSM(testOSMEntity)); expression = expression.negate(); assertFalse(expression.applyOSH(testOSHEntity) && expression.applyOSM(testOSMEntity)); } + + @Test + public void testFilterByOSMEntityApplyGeometryFallback() { + FilterExpression expression = Filter.byOSMEntity(e -> e.getVersion() == 1); + assertTrue(expression.applyOSMGeometry(testOSMEntity, gf.createPoint())); + assertFalse(expression.negate().applyOSMGeometry(testOSMEntity, gf.createPoint())); + } } From 35d199a06a73749e69fd7f89540987540fbf06d5 Mon Sep 17 00:00:00 2001 From: Martin Raifer Date: Fri, 11 Jun 2021 17:51:21 +0200 Subject: [PATCH 34/34] add additional negated test Co-authored-by: Rafael Troilo --- .../org/heigit/ohsome/oshdb/filter/ApplyOSMGeometryTest.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMGeometryTest.java b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMGeometryTest.java index 867a76d79..1eb0caee1 100644 --- a/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMGeometryTest.java +++ b/oshdb-filter/src/test/java/org/heigit/ohsome/oshdb/filter/ApplyOSMGeometryTest.java @@ -116,6 +116,10 @@ public void testGeometryFilterArea() { OSHDBGeometryBuilder.getGeometry(new OSHDBBoundingBox(0, 0, 2E-5, 2E-5)) )); // negated + assertFalse(expression.negate().applyOSMGeometry(entity, + // approx 1.2m² + OSHDBGeometryBuilder.getGeometry(new OSHDBBoundingBox(0, 0, 1E-5, 1E-5)) + )); assertTrue(expression.negate().applyOSMGeometry(entity, // approx 0.3m² OSHDBGeometryBuilder.getGeometry(new OSHDBBoundingBox(0, 0, 5E-6, 5E-6))