Merge branch 'extend-types-parameter' into 'master'

Extend types parameter

See merge request giscience/big-data/ohsome/ohsome-api!25

src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/exception/ExceptionMessages.java

@@ -34,7 +34,8 @@ public class ExceptionMessages {
       "The given query is too large in respect to the given timeout. Please use a smaller region "
           + "and/or coarser time period.";
   public static final String PROPERTIES_PARAM =
-      "The properties parameter can only contain the values 'tags' and/or 'metadata'.";
+      "The properties parameter can only contain the values 'tags' and/or 'metadata' and/or "
+      + "'unclipped'.";
   public static final String SHOWMETADATA_PARAM = "The showMetadata parameter can only contain the "
       + "values 'true', 'yes', 'false', or 'no'.";
   public static final String TIME_FORMAT = "The provided time parameter is not ISO-8601 conform.";
@@ -45,6 +46,8 @@ public class ExceptionMessages {
   public static final String TIMEOUT = "The given timeout is too long. It has to be shorter than ";
   public static final String TIMEOUT_FORMAT = "The given timeout does not fit to its format. Please"
       + " give one value in seconds and use a point as the decimal delimiter, if needed.";
+  public static final String TYPES_PARAM = "Parameter 'types' (and 'types2') can only have 'node' "
+      + "and/or 'way' and/or 'relation' OR 'point' and/or 'line' and/or 'polygon' and/or 'other'";
 
   private ExceptionMessages() {
     throw new IllegalStateException("Utility class");

src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/executor/ElementsRequestExecutor.java

@@ -214,6 +214,7 @@ public static void executeElementsFullHistory(ElementsGeometry elemGeom,
     inputProcessor.processPropertiesParam();
     final boolean includeTags = inputProcessor.includeTags();
     final boolean includeOSMMetadata = inputProcessor.includeOSMMetadata();
+    final boolean unclippedGeometries = inputProcessor.isUnclipped();
     String startTimestamp = ISODateTimeParser.parseISODateTime(requestParameters.getTime()[0])
         .format(DateTimeFormatter.ISO_DATE_TIME);
     String endTimestamp = ISODateTimeParser.parseISODateTime(requestParameters.getTime()[1])
@@ -233,15 +234,16 @@ public static void executeElementsFullHistory(ElementsGeometry elemGeom,
       } else {
         // if not "creation": take "before" as starting "row" (geom, tags), valid_from = t_start
         currentEntity = contributions.get(0).getEntityBefore();
-        currentGeom = contributions.get(0).getGeometryBefore();
+        currentGeom = exeUtils.getGeometry(contributions.get(0), unclippedGeometries, true);
         validFrom = startTimestamp;
-      }
+      }    
       // then for each contribution:
       for (OSMContribution contribution : contributions) {
         // set valid_to of previous row, add to output list (output.add(…))
         validTo = TimestampFormatter.getInstance().isoDateTime(contribution.getTimestamp());
         if (!skipNext) {
           properties = new TreeMap<>();
+          properties = exeUtils.addContribType(contribution, properties, includeOSMMetadata);
           properties.put("@validFrom", validFrom);
           properties.put("@validTo", validTo);
           if (!currentGeom.isEmpty()) {
@@ -256,15 +258,17 @@ public static void executeElementsFullHistory(ElementsGeometry elemGeom,
         } else {
           // else: take "after" as next row
           currentEntity = contribution.getEntityAfter();
-          currentGeom = contribution.getGeometryAfter();
+          currentGeom = exeUtils.getGeometry(contribution, unclippedGeometries, false);
           validFrom = TimestampFormatter.getInstance().isoDateTime(contribution.getTimestamp());
         }
       }
       // after loop:
-      if (!contributions.get(contributions.size() - 1).is(ContributionType.DELETION)) {
+      OSMContribution lastContribution = contributions.get(contributions.size() - 1);
+      if (!lastContribution.is(ContributionType.DELETION)) {
         // if last contribution was not "deletion": set valid_to = t_end, add row to output list
         validTo = endTimestamp;
         properties = new TreeMap<>();
+        properties = exeUtils.addContribType(lastContribution, properties, includeOSMMetadata);
         properties.put("@validFrom", validFrom);
         properties.put("@validTo", validTo);
         if (!currentGeom.isEmpty()) {
@@ -275,6 +279,7 @@ public static void executeElementsFullHistory(ElementsGeometry elemGeom,
       return output;
     });
     MapReducer<Feature> snapshotPreResult = null;
+    // handles cases where valid_from = t_start, valid_to = t_end
     snapshotPreResult = mapRedSnapshot.groupByEntity().filter(snapshots -> snapshots.size() == 2)
         .filter(snapshots -> snapshots.get(0).getGeometry() == snapshots.get(1).getGeometry()
             && snapshots.get(0).getEntity().getVersion() == snapshots.get(1).getEntity()
@@ -292,7 +297,7 @@ public static void executeElementsFullHistory(ElementsGeometry elemGeom,
           properties.put("@validTo", endTimestamp);
           return exeUtils.createOSMFeature(entity, geom, properties, keysInt, includeTags,
               includeOSMMetadata, elemGeom, mapTagTranslator.get());
-        }); // valid_from = t_start, valid_to = t_end
+        });
     Stream<Feature> contributionStream = contributionPreResult.stream().filter(Objects::nonNull);
     Stream<Feature> snapshotStream = snapshotPreResult.stream().filter(Objects::nonNull);
     Metadata metadata = null;
@@ -884,7 +889,7 @@ public static Response executeCountLengthPerimeterAreaShareRatio(RequestResource
     EnumSet<OSMType> osmTypes1 =
         (EnumSet<OSMType>) inputProcessor.getProcessingData().getOsmTypes();
     if (!isShare) {
-      inputProcessor.defineOSMTypes(servletRequest.getParameterValues("types2"));
+      inputProcessor.defineTypes(servletRequest.getParameterValues("types2"), mapRed);
     }
     EnumSet<OSMType> osmTypes2 =
         (EnumSet<OSMType>) inputProcessor.getProcessingData().getOsmTypes();
@@ -1041,7 +1046,7 @@ public static <P extends Geometry & Polygonal> Response executeCountLengthPerime
     }
     EnumSet<OSMType> osmTypes1 = (EnumSet<OSMType>) processingData.getOsmTypes();
     if (!isShare) {
-      inputProcessor.defineOSMTypes(servletRequest.getParameterValues("types2"));
+      inputProcessor.defineTypes(servletRequest.getParameterValues("types2"), mapRed);
     }
     EnumSet<OSMType> osmTypes2 =
         (EnumSet<OSMType>) inputProcessor.getProcessingData().getOsmTypes();

src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/executor/ExecutionUtils.java

@@ -132,8 +132,8 @@ public static DecimalFormat defineDecimalFormat(String format) {
    * Caches the given mapper value in the user data of the <code>Geometry</code> object.
    * 
    * @param geom <code>Geometry</code> of an OSMEntitySnapshot object
-   * @param mapper arbitrary function that returns a time-independent value from a snapshot object, for
-   *        example lenght, area, perimeter
+   * @param mapper arbitrary function that returns a time-independent value from a snapshot object,
+   *        for example lenght, area, perimeter
    * @return evaluated mapper function or cached value stored in the user data of the
    *         <code>Geometry</code> object
    */
@@ -145,7 +145,7 @@ public static Double cacheInUserData(Geometry geom, SerializableSupplier<Double>
   }
 
   /**
-   * Adapted helper function, which works like 
+   * Adapted helper function, which works like
    * {@link org.heigit.bigspatialdata.oshdb.api.generic.OSHDBCombinedIndex#nest(Map) nest} but has
    * switched &lt;U&gt; and &lt;V&gt; parameters.
    *
@@ -822,6 +822,38 @@ public Response createShareGroupByBoundaryResponse(Object[] boundaryIds, String[
         groupByResultSet);
   }
 
+  /** Adds the respective contribution type(s) to the properties if includeMetadata=true. */
+  public Map<String, Object> addContribType(OSMContribution contribution,
+      Map<String, Object> properties, boolean includeMetadata) {
+    if (includeMetadata) {
+      properties.put("@contributionTypes", contribution.getContributionTypes());
+    }
+    return properties;
+  }
+
+  /**
+   * Extracts and returns a geometry out of the given contribution. The boolean values specify if it
+   * should be clipped/unclipped and if the geometry before/after a contribution should be taken.
+   */
+  public Geometry getGeometry(OSMContribution contribution, boolean unclippedGeometries,
+      boolean before) {
+    Geometry geom = null;
+    if (unclippedGeometries) {
+      if (before) {
+        geom = contribution.getGeometryUnclippedBefore();
+      } else {
+        geom = contribution.getGeometryUnclippedAfter();
+      }
+    } else {
+      if (before) {
+        geom = contribution.getGeometryBefore();
+      } else {
+        geom = contribution.getGeometryAfter();
+      }
+    }
+    return geom;
+  }
+
   /**
    * Creates the csv response for /elements/_/groupBy requests.
    * 

src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/inputprocessing/InputProcessingUtils.java

@@ -6,14 +6,26 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Set;
 import org.heigit.bigspatialdata.ohsome.ohsomeapi.exception.BadRequestException;
 import org.heigit.bigspatialdata.ohsome.ohsomeapi.exception.ExceptionMessages;
 import org.heigit.bigspatialdata.ohsome.ohsomeapi.exception.NotFoundException;
+import org.heigit.bigspatialdata.ohsome.ohsomeapi.oshdb.DbConnData;
 import org.heigit.bigspatialdata.ohsome.ohsomeapi.oshdb.ExtractMetadata;
+import org.heigit.bigspatialdata.oshdb.api.generic.function.SerializablePredicate;
+import org.heigit.bigspatialdata.oshdb.api.mapreducer.MapReducer;
+import org.heigit.bigspatialdata.oshdb.api.object.OSHDBMapReducible;
+import org.heigit.bigspatialdata.oshdb.api.object.OSMEntitySnapshot;
+import org.heigit.bigspatialdata.oshdb.osm.OSMType;
+import org.heigit.bigspatialdata.oshdb.util.OSHDBTag;
+import org.heigit.bigspatialdata.oshdb.util.tagtranslator.TagTranslator;
 import org.heigit.bigspatialdata.oshdb.util.time.ISODateTimeParser;
 import org.heigit.bigspatialdata.oshdb.util.time.OSHDBTimestamps;
 import org.heigit.bigspatialdata.oshdb.util.time.TimestampFormatter;
 import org.locationtech.jts.geom.Geometry;
+import org.locationtech.jts.geom.Lineal;
+import org.locationtech.jts.geom.Polygonal;
+import org.locationtech.jts.geom.Puntal;
 
 /** Holds utility methods that are used by the input processing and executor classes. */
 public class InputProcessingUtils {
@@ -288,6 +300,107 @@ public String[] extractIsoTime(String time) throws BadRequestException, NotFound
     return timeVals;
   }
 
+  /** Sorts the given timestamps from oldest to newest. */
+  public String[] sortTimestamps(String[] timestamps) throws BadRequestException {
+    List<String> timeStringList = new ArrayList<>();
+    for (String timestamp : timestamps) {
+      try {
+        ZonedDateTime zdt = ISODateTimeParser.parseISODateTime(timestamp);
+        checkTemporalExtend(zdt.format(DateTimeFormatter.ISO_DATE_TIME));
+        timeStringList.add(zdt.format(DateTimeFormatter.ISO_DATE_TIME));
+      } catch (Exception e) {
+        throw new BadRequestException(ExceptionMessages.TIME_FORMAT);
+      }
+    }
+    Collections.sort(timeStringList);
+    return timeStringList.toArray(timestamps);
+  }
+
+  /** Checks the given custom boundary id. At the moment only used if output format = csv. */
+  public void checkCustomBoundaryId(String id) {
+    if (id.contains(";")) {
+      throw new BadRequestException("The given custom ids cannot contain semicolons, "
+          + "if you want to use csv as output format.");
+    }
+  }
+
+  /**
+   * Checks if the given geometry is within the underlying data-polygon. Returns also true if no
+   * data-polygon is given.
+   * 
+   * @param geom <code>Geometry</code>, which is tested against the data-polygon
+   * @return <code>true</code> - if inside <br>
+   *         <code>false</code> - if not inside
+   */
+  public boolean isWithin(Geometry geom) {
+    if (ExtractMetadata.dataPoly != null) {
+      return geom.within(ExtractMetadata.dataPoly);
+    }
+    return true;
+  }
+
+  /** Checks if the given String is one of the simple feature types (point, line, polygon). */
+  public boolean isSimpleFeatureType(String type) {
+    return "point".equalsIgnoreCase(type) || "line".equalsIgnoreCase(type)
+        || "polygon".equalsIgnoreCase(type) || "other".equalsIgnoreCase(type);
+  }
+
+  /**
+   * Applies an entity filter using only planar relations (relations with an area) on the given
+   * MapReducer object. It uses the tags "type=multipolygon" and "type=boundary".
+   */
+  public <T extends OSHDBMapReducible> MapReducer<T> filterOnPlanarRelations(MapReducer<T> mapRed) {
+    // further filtering to not look at all relations
+    TagTranslator tt = DbConnData.tagTranslator;
+    OSHDBTag typeMultipolygon = tt.getOSHDBTagOf("type", "multipolygon");
+    OSHDBTag typeBoundary = tt.getOSHDBTagOf("type", "boundary");
+    mapRed.osmEntityFilter(entity -> {
+      return !entity.getType().equals(OSMType.RELATION)
+          || entity.hasTagValue(typeMultipolygon.getKey(), typeMultipolygon.getValue())
+          || entity.hasTagValue(typeBoundary.getKey(), typeBoundary.getValue());
+    });
+    return mapRed;
+  }
+
+  /**
+   * Applies respective Puntal|Lineal|Polygonal filter(s) on features of the given MapReducer.
+   * 
+   * @return MapReducer with filtered geometries
+   */
+  @SuppressWarnings("unchecked") // unchecked to allow cast of (MapReducer<T>) to mapRed
+  public <T extends OSHDBMapReducible> MapReducer<T> filterOnSimpleFeatures(MapReducer<T> mapRed,
+      ProcessingData processingData) {
+    MapReducer<OSMEntitySnapshot> mapReducer = (MapReducer<OSMEntitySnapshot>) mapRed;
+    Set<SimpleFeatureType> simpleFeatureTypes = processingData.getSimpleFeatureTypes();
+    boolean containsPoint = false;
+    boolean containsLine = false;
+    boolean containsPolygon = false;
+    boolean containsOther = false;
+    for (SimpleFeatureType type : simpleFeatureTypes) {
+      if (type.equals(SimpleFeatureType.POINT)) {
+        containsPoint = true;
+      } else if (type.equals(SimpleFeatureType.LINE)) {
+        containsLine = true;
+      } else if (type.equals(SimpleFeatureType.POLYGON)) {
+        containsPolygon = true;
+      } else if (type.equals(SimpleFeatureType.OTHER)) {
+        containsOther = true;
+      }
+    }
+    final boolean hasPoly = containsPolygon;
+    final boolean hasPoint = containsPoint;
+    final boolean hasLine = containsLine;
+    final boolean hasOther = containsOther;
+    return (MapReducer<T>) mapReducer
+        .filter((SerializablePredicate<OSMEntitySnapshot>) predicate -> {
+          return (hasPoly && predicate.getGeometry() instanceof Polygonal)
+              || (hasPoint && predicate.getGeometry() instanceof Puntal)
+              || (hasLine && predicate.getGeometry() instanceof Lineal)
+              || (hasOther && "GeometryCollection"
+                  .equalsIgnoreCase(predicate.getGeometry().getGeometryType()));
+        });
+  }
+
   /**
    * Checks the provided time info on its temporal extent.
    * 
@@ -356,45 +469,6 @@ protected void checkPeriodOnIsoConformity(String period) throws BadRequestExcept
     }
   }
 
-  /** Sorts the given timestamps from oldest to newest. */
-  public String[] sortTimestamps(String[] timestamps) throws BadRequestException {
-    List<String> timeStringList = new ArrayList<>();
-    for (String timestamp : timestamps) {
-      try {
-        ZonedDateTime zdt = ISODateTimeParser.parseISODateTime(timestamp);
-        checkTemporalExtend(zdt.format(DateTimeFormatter.ISO_DATE_TIME));
-        timeStringList.add(zdt.format(DateTimeFormatter.ISO_DATE_TIME));
-      } catch (Exception e) {
-        throw new BadRequestException(ExceptionMessages.TIME_FORMAT);
-      }
-    }
-    Collections.sort(timeStringList);
-    return timeStringList.toArray(timestamps);
-  }
-
-  /** Checks the given custom boundary id. At the moment only used if output format = csv. */
-  public void checkCustomBoundaryId(String id) {
-    if (id.contains(";")) {
-      throw new BadRequestException("The given custom ids cannot contain semicolons, "
-          + "if you want to use csv as output format.");
-    }
-  }
-
-  /**
-   * Checks if the given geometry is within the underlying data-polygon. Returns also true if no
-   * data-polygon is given.
-   * 
-   * @param geom <code>Geometry</code>, which is tested against the data-polygon
-   * @return <code>true</code> - if inside <br>
-   *         <code>false</code> - if not inside
-   */
-  public boolean isWithin(Geometry geom) {
-    if (ExtractMetadata.dataPoly != null) {
-      return geom.within(ExtractMetadata.dataPoly);
-    }
-    return true;
-  }
-
   /**
    * Splits the given boundary parameter (bboxes, bcircles, or bpolys) on '|' to seperate the
    * different bounding objects.