Merge branch 'simpleFeatureTypes-with-groupByBoundary' into 'master'

refactor feature type filters

See merge request giscience/big-data/ohsome/ohsome-api!28

src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/executor/ElementsRequestExecutor.java

@@ -175,7 +175,9 @@ public static void executeElements(ElementsGeometry elemGeom, HttpServletRequest
   public static void executeElementsFullHistory(ElementsGeometry elemGeom,
       HttpServletRequest servletRequest, HttpServletResponse servletResponse) throws Exception {
     InputProcessor inputProcessor = new InputProcessor(servletRequest, false, false);
+    inputProcessor.getProcessingData().setIsFullHistory(true);
     InputProcessor snapshotInputProcessor = new InputProcessor(servletRequest, true, false);
+    snapshotInputProcessor.getProcessingData().setIsFullHistory(true);
     MapReducer<OSMEntitySnapshot> mapRedSnapshot = null;
     MapReducer<OSMContribution> mapRedContribution = null;
     if (DbConnData.db instanceof OSHDBIgnite) {
@@ -213,14 +215,17 @@ public static void executeElementsFullHistory(ElementsGeometry elemGeom,
     ExecutionUtils exeUtils = new ExecutionUtils(processingData);
     RemoteTagTranslator mapTagTranslator = DbConnData.mapTagTranslator;
     inputProcessor.processPropertiesParam();
+    InputProcessingUtils utils = inputProcessor.getUtils();
     final boolean includeTags = inputProcessor.includeTags();
     final boolean includeOSMMetadata = inputProcessor.includeOSMMetadata();
     final boolean unclippedGeometries = inputProcessor.isUnclipped();
+    final Set<SimpleFeatureType> simpleFeatureTypes = processingData.getSimpleFeatureTypes();
     String startTimestamp = ISODateTimeParser.parseISODateTime(requestParameters.getTime()[0])
         .format(DateTimeFormatter.ISO_DATE_TIME);
     String endTimestamp = ISODateTimeParser.parseISODateTime(requestParameters.getTime()[1])
         .format(DateTimeFormatter.ISO_DATE_TIME);
-    contributionPreResult = mapRedContribution.groupByEntity().flatMap(contributions -> {
+    MapReducer<List<OSMContribution>> mapRedContributions = mapRedContribution.groupByEntity();
+    contributionPreResult = mapRedContributions.flatMap(contributions -> {
       List<Feature> output = new LinkedList<>();
       Map<String, Object> properties;
       Geometry currentGeom = null;
@@ -248,8 +253,11 @@ public static void executeElementsFullHistory(ElementsGeometry elemGeom,
           properties.put("@validFrom", validFrom);
           properties.put("@validTo", validTo);
           if (!currentGeom.isEmpty()) {
-            output.add(exeUtils.createOSMFeature(currentEntity, currentGeom, properties, keysInt,
-                includeTags, includeOSMMetadata, elemGeom, mapTagTranslator.get()));
+            if (!processingData.containsSimpleFeatureTypes()
+                || utils.checkGeometryOnSimpleFeatures(currentGeom, simpleFeatureTypes)) {
+              output.add(exeUtils.createOSMFeature(currentEntity, currentGeom, properties, keysInt,
+                  includeTags, includeOSMMetadata, elemGeom, mapTagTranslator.get()));
+            }
           }
         }
         skipNext = false;
@@ -273,8 +281,11 @@ public static void executeElementsFullHistory(ElementsGeometry elemGeom,
         properties.put("@validFrom", validFrom);
         properties.put("@validTo", validTo);
         if (!currentGeom.isEmpty()) {
-          output.add(exeUtils.createOSMFeature(currentEntity, currentGeom, properties, keysInt,
-              includeTags, includeOSMMetadata, elemGeom, mapTagTranslator.get()));
+          if (!processingData.containsSimpleFeatureTypes()
+              || utils.checkGeometryOnSimpleFeatures(currentGeom, simpleFeatureTypes)) {
+            output.add(exeUtils.createOSMFeature(currentEntity, currentGeom, properties, keysInt,
+                includeTags, includeOSMMetadata, elemGeom, mapTagTranslator.get()));
+          }
         }
       }
       return output;
@@ -285,7 +296,7 @@ public static void executeElementsFullHistory(ElementsGeometry elemGeom,
         .filter(snapshots -> snapshots.get(0).getGeometry() == snapshots.get(1).getGeometry()
             && snapshots.get(0).getEntity().getVersion() == snapshots.get(1).getEntity()
                 .getVersion())
-        .map(snapshots -> snapshots.get(0)).map(snapshot -> {
+        .map(snapshots -> snapshots.get(0)).flatMap(snapshot -> {
           Map<String, Object> properties = new TreeMap<>();
           OSMEntity entity = snapshot.getEntity();
           if (includeOSMMetadata) {
@@ -296,8 +307,13 @@ public static void executeElementsFullHistory(ElementsGeometry elemGeom,
               TimestampFormatter.getInstance().isoDateTime(snapshot.getTimestamp()));
           properties.put("@validFrom", startTimestamp);
           properties.put("@validTo", endTimestamp);
-          return exeUtils.createOSMFeature(entity, geom, properties, keysInt, includeTags,
-              includeOSMMetadata, elemGeom, mapTagTranslator.get());
+          if (!processingData.containsSimpleFeatureTypes()
+              || utils.checkGeometryOnSimpleFeatures(geom, simpleFeatureTypes)) {
+            return Collections.singletonList(exeUtils.createOSMFeature(entity, geom, properties,
+                keysInt, includeTags, includeOSMMetadata, elemGeom, mapTagTranslator.get()));
+          } else {
+            return Collections.emptyList();
+          }
         });
     Stream<Feature> contributionStream = contributionPreResult.stream().filter(Objects::nonNull);
     Stream<Feature> snapshotStream = snapshotPreResult.stream().filter(Objects::nonNull);
@@ -414,8 +430,7 @@ public static Response executeCountLengthPerimeterArea(RequestResource requestRe
    * @throws Exception thrown by
    *         {@link org.heigit.bigspatialdata.ohsome.ohsomeapi.inputprocessing.InputProcessor#processParameters()
    *         processParameters} and
-   *         {@link org.heigit.bigspatialdata.ohsome.ohsomeapi.executor.ExecutionUtils#computeCountLengthPerimeterAreaGbB(RequestResource, BoundaryType, MapReducer)
-   *         computeCountLengthPerimeterAreaGbB}
+   *         {@link org.heigit.bigspatialdata.ohsome.ohsomeapi.executor.ExecutionUtils#computeCountLengthPerimeterAreaGbB(RequestResource, BoundaryType, MapReducer, InputProcessingUtils)}
    */
   public static Response executeCountLengthPerimeterAreaGroupByBoundary(
       RequestResource requestResource, HttpServletRequest servletRequest,
@@ -424,17 +439,18 @@ public static Response executeCountLengthPerimeterAreaGroupByBoundary(
     SortedMap<OSHDBCombinedIndex<OSHDBTimestamp, Integer>, ? extends Number> result;
     MapReducer<OSMEntitySnapshot> mapRed;
     InputProcessor inputProcessor = new InputProcessor(servletRequest, isSnapshot, isDensity);
+    inputProcessor.getProcessingData().setIsGroupByBoundary(true);
     ProcessingData processingData = inputProcessor.getProcessingData();
     RequestParameters requestParameters = processingData.getRequestParameters();
     ExecutionUtils exeUtils = new ExecutionUtils(processingData);
     mapRed = inputProcessor.processParameters();
+    InputProcessingUtils utils = inputProcessor.getUtils();
     result = exeUtils.computeCountLengthPerimeterAreaGbB(requestResource,
-        processingData.getBoundaryType(), mapRed);
+        processingData.getBoundaryType(), mapRed, utils);
     SortedMap<Integer, ? extends SortedMap<OSHDBTimestamp, ? extends Number>> groupByResult;
     groupByResult = ExecutionUtils.nest(result);
     GroupByResult[] resultSet = new GroupByResult[groupByResult.size()];
     Object groupByName;
-    InputProcessingUtils utils = inputProcessor.getUtils();
     Object[] boundaryIds = utils.getBoundaryIds();
     int count = 0;
     ArrayList<Geometry> boundaries = new ArrayList<>(processingData.getBoundaryList());
@@ -486,13 +502,13 @@ public static Response executeCountLengthPerimeterAreaGroupByBoundary(
    *         {@link org.heigit.bigspatialdata.ohsome.ohsomeapi.inputprocessing.InputProcessor#processParameters()
    *         processParameters}
    */
-  @SuppressWarnings({"unchecked"}) // intentionally as check for P on Polygonal is already performed
   public static <P extends Geometry & Polygonal> Response executeCountLengthPerimeterAreaGroupByBoundaryGroupByTag(
       RequestResource requestResource, HttpServletRequest servletRequest,
       HttpServletResponse servletResponse, boolean isSnapshot, boolean isDensity) throws Exception {
     final long startTime = System.currentTimeMillis();
     MapReducer<OSMEntitySnapshot> mapRed = null;
     InputProcessor inputProcessor = new InputProcessor(servletRequest, isSnapshot, isDensity);
+    inputProcessor.getProcessingData().setIsGroupByBoundary(true);
     String[] groupByKey = inputProcessor.splitParamOnComma(
         inputProcessor.createEmptyArrayIfNull(servletRequest.getParameterValues("groupByKey")));
     if (groupByKey.length != 1) {
@@ -515,10 +531,16 @@ public static <P extends Geometry & Polygonal> Response executeCountLengthPerime
       }
     }
     ArrayList<Geometry> arrGeoms = new ArrayList<>(processingData.getBoundaryList());
+    @SuppressWarnings("unchecked") // intentionally as check for P on Polygonal is already performed
     Map<Integer, P> geoms = IntStream.range(0, arrGeoms.size()).boxed()
         .collect(Collectors.toMap(idx -> idx, idx -> (P) arrGeoms.get(idx)));
+    InputProcessingUtils utils = inputProcessor.getUtils();
+    MapAggregator<Integer, OSMEntitySnapshot> mapAgg = mapRed.aggregateByGeometry(geoms);
+    if (processingData.containsSimpleFeatureTypes()) {
+      mapAgg = utils.filterOnSimpleFeatures(mapAgg, processingData);
+    }
     MapAggregator<OSHDBCombinedIndex<OSHDBCombinedIndex<Integer, Pair<Integer, Integer>>, OSHDBTimestamp>, Geometry> preResult =
-        mapRed.aggregateByGeometry(geoms)
+        mapAgg
             .map(f -> exeUtils.mapSnapshotToTags(keysInt, valuesInt, f))
             .aggregateBy(Pair::getKey, zeroFill).map(Pair::getValue)
             .aggregateByTimestamp(OSMEntitySnapshot::getTimestamp).map(x -> x.getGeometry());
@@ -527,7 +549,6 @@ public static <P extends Geometry & Polygonal> Response executeCountLengthPerime
     SortedMap<OSHDBCombinedIndex<Integer, Pair<Integer, Integer>>, ? extends SortedMap<OSHDBTimestamp, ? extends Number>> groupByResult =
         OSHDBCombinedIndex.nest(result);
     GroupByResult[] resultSet = new GroupByResult[groupByResult.entrySet().size()];
-    InputProcessingUtils utils = inputProcessor.getUtils();
     Object[] boundaryIds = utils.getBoundaryIds();
     int count = 0;
     ArrayList<Geometry> boundaries = new ArrayList<>(processingData.getBoundaryList());
@@ -1008,7 +1029,6 @@ public static Response executeCountLengthPerimeterAreaShareRatio(RequestResource
    *         {@link org.heigit.bigspatialdata.oshdb.api.mapreducer.MapAggregator#count() count}, or
    *         {@link org.heigit.bigspatialdata.oshdb.api.mapreducer.MapAggregator#sum() sum}
    */
-  @SuppressWarnings({"unchecked"}) // intentionally as check for P on Polygonal is already performed
   public static <P extends Geometry & Polygonal> Response executeCountLengthPerimeterAreaShareRatioGroupByBoundary(
       RequestResource requestResource, HttpServletRequest servletRequest,
       HttpServletResponse servletResponse, boolean isSnapshot, boolean isDensity, boolean isShare)
@@ -1018,6 +1038,7 @@ public static <P extends Geometry & Polygonal> Response executeCountLengthPerime
         null;
     MapReducer<OSMEntitySnapshot> mapRed = null;
     InputProcessor inputProcessor = new InputProcessor(servletRequest, isSnapshot, isDensity);
+    inputProcessor.getProcessingData().setIsGroupByBoundary(true);
     inputProcessor.getProcessingData().setIsShareRatio(true);
     mapRed = inputProcessor.processParameters();
     ProcessingData processingData = inputProcessor.getProcessingData();
@@ -1099,6 +1120,7 @@ public static <P extends Geometry & Polygonal> Response executeCountLengthPerime
     }
     MapAggregator<OSHDBCombinedIndex<OSHDBCombinedIndex<OSHDBTimestamp, Integer>, MatchType>, Geometry> preResult =
         null;
+    @SuppressWarnings({"unchecked"}) // intentionally as check for P on Polygonal is already performed
     Map<Integer, P> geoms = arrGeoms.stream()
         .collect(Collectors.toMap(geom -> arrGeoms.indexOf(geom), geom -> (P) geom));
     ExecutionUtils exeUtils = new ExecutionUtils(processingData);

src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/executor/ExecutionUtils.java

@@ -36,6 +36,7 @@
 import org.heigit.bigspatialdata.ohsome.ohsomeapi.exception.BadRequestException;
 import org.heigit.bigspatialdata.ohsome.ohsomeapi.exception.ExceptionMessages;
 import org.heigit.bigspatialdata.ohsome.ohsomeapi.inputprocessing.BoundaryType;
+import org.heigit.bigspatialdata.ohsome.ohsomeapi.inputprocessing.InputProcessingUtils;
 import org.heigit.bigspatialdata.ohsome.ohsomeapi.inputprocessing.InputProcessor;
 import org.heigit.bigspatialdata.ohsome.ohsomeapi.inputprocessing.ProcessingData;
 import org.heigit.bigspatialdata.ohsome.ohsomeapi.inputprocessing.SimpleFeatureType;
@@ -435,18 +436,23 @@ public org.wololo.geojson.Feature createOSMFeature(OSMEntity entity, Geometry ge
   }
 
   /** Computes the result for the /count|length|perimeter|area/groupBy/boundary resources. */
-  @SuppressWarnings({"unchecked"}) // intentionally as check for P on Polygonal is already performed
   public <P extends Geometry & Polygonal> SortedMap<OSHDBCombinedIndex<OSHDBTimestamp, Integer>, ? extends Number> computeCountLengthPerimeterAreaGbB(
       RequestResource requestResource, BoundaryType boundaryType,
-      MapReducer<OSMEntitySnapshot> mapRed) throws Exception {
+      MapReducer<OSMEntitySnapshot> mapRed, InputProcessingUtils utils) throws Exception {
     if (boundaryType == BoundaryType.NOBOUNDARY) {
       throw new BadRequestException(ExceptionMessages.NO_BOUNDARY);
     }
     MapAggregator<OSHDBCombinedIndex<OSHDBTimestamp, Integer>, Geometry> preResult;
     ArrayList<Geometry> arrGeoms = new ArrayList<>(processingData.getBoundaryList());
+    @SuppressWarnings("unchecked") // intentionally as check for P on Polygonal is already performed
     Map<Integer, P> geoms = IntStream.range(0, arrGeoms.size()).boxed()
         .collect(Collectors.toMap(idx -> idx, idx -> (P) arrGeoms.get(idx)));
-    preResult = mapRed.aggregateByTimestamp().aggregateByGeometry(geoms).map(x -> x.getGeometry());
+    MapAggregator<OSHDBCombinedIndex<OSHDBTimestamp, Integer>, OSMEntitySnapshot> mapAgg =
+        mapRed.aggregateByTimestamp().aggregateByGeometry(geoms);
+    if (processingData.containsSimpleFeatureTypes()) {
+      mapAgg = utils.filterOnSimpleFeatures(mapAgg, processingData);
+    }
+    preResult = mapAgg.map(OSMEntitySnapshot::getGeometry);
     switch (requestResource) {
       case COUNT:
         return preResult.count();

src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/executor/UsersRequestExecutor.java

@@ -32,6 +32,7 @@
 import org.heigit.bigspatialdata.ohsome.ohsomeapi.output.dataaggregationresponse.users.UsersResult;
 import org.heigit.bigspatialdata.oshdb.api.generic.OSHDBCombinedIndex;
 import org.heigit.bigspatialdata.oshdb.api.generic.function.SerializableFunction;
+import org.heigit.bigspatialdata.oshdb.api.mapreducer.MapAggregator;
 import org.heigit.bigspatialdata.oshdb.api.mapreducer.MapReducer;
 import org.heigit.bigspatialdata.oshdb.api.object.OSMContribution;
 import org.heigit.bigspatialdata.oshdb.osm.OSMType;
@@ -292,21 +293,26 @@ public static <P extends Geometry & Polygonal> Response executeCountGroupByBound
     SortedMap<OSHDBCombinedIndex<OSHDBTimestamp, Integer>, Integer> result = null;
     MapReducer<OSMContribution> mapRed = null;
     InputProcessor inputProcessor = new InputProcessor(servletRequest, false, isDensity);
+    inputProcessor.getProcessingData().setIsGroupByBoundary(true);
     mapRed = inputProcessor.processParameters();
     ProcessingData processingData = inputProcessor.getProcessingData();
     RequestParameters requestParameters = processingData.getRequestParameters();
     ArrayList<Geometry> arrGeoms = processingData.getBoundaryList();
     @SuppressWarnings("unchecked") // intentionally as check for P on Polygonal is already performed
     Map<Integer, P> geoms = IntStream.range(0, arrGeoms.size()).boxed()
         .collect(Collectors.toMap(idx -> idx, idx -> (P) arrGeoms.get(idx)));
-    result = mapRed.aggregateByTimestamp().aggregateByGeometry(geoms)
-        .map(OSMContribution::getContributorUserId).countUniq();
+    InputProcessingUtils utils = inputProcessor.getUtils();
+    MapAggregator<OSHDBCombinedIndex<OSHDBTimestamp, Integer>, OSMContribution> mapAgg
+        = mapRed.aggregateByTimestamp().aggregateByGeometry(geoms);
+    if (processingData.containsSimpleFeatureTypes()) {
+      mapAgg = utils.filterOnSimpleFeatures(mapAgg, processingData);
+    }
+    result = mapAgg.map(OSMContribution::getContributorUserId).countUniq();
     SortedMap<Integer, SortedMap<OSHDBTimestamp, Integer>> groupByResult;
     groupByResult = ExecutionUtils.nest(result);
     GroupByResult[] resultSet = new GroupByResult[groupByResult.size()];
     int count = 0;
     ExecutionUtils exeUtils = new ExecutionUtils(processingData);
-    InputProcessingUtils utils = inputProcessor.getUtils();
     Object[] boundaryIds = utils.getBoundaryIds();
     for (Entry<Integer, SortedMap<OSHDBTimestamp, Integer>> entry : groupByResult.entrySet()) {
       UsersResult[] results = exeUtils.fillUsersResult(entry.getValue(),