upgade to OSHDB version 0.7.0

CHANGELOG.md

@@ -10,11 +10,14 @@ Changelog
 
 ### Other Changes
 
+* upgrade to OSHDB version 0.7.0 – note that oshdb database files from the previous version are not compatible with this version anymore, you have to either [redownload](https://downloads.ohsome.org/OSHDB/v0.7/) or [recreate](https://github.com/GIScience/oshdb/blob/0.7/oshdb-etl/README.md) them from scratch ([#222])
 * adapt error message for contributions extration endpoint in case of wrong `time` parameter value ([#208])
 
 [#201]: https://github.com/GIScience/ohsome-api/issues/201
 [#208]: https://github.com/GIScience/ohsome-api/issues/208
 [#214]: https://github.com/GIScience/ohsome-api/issues/214
+[#222]: https://github.com/GIScience/ohsome-api/pull/222
+
 
 ## 1.5.0
 

Jenkinsfile

@@ -11,7 +11,7 @@ pipeline {
     LATEST_COMMIT_ID = sh(returnStdout: true, script: 'git describe --tags --long  --always').trim()
 
     // START CUSTOM ohsome API
-    MAVEN_TEST_OPTIONS = '-Dport_get=8081 -Dport_post=8082 -Dport_data=8083 -DdbFilePathProperty="--database.db=/opt/data/heidelberg.oshdb"'
+    MAVEN_TEST_OPTIONS = '-Dport_get=8081 -Dport_post=8082 -Dport_data=8083 -DdbFilePathProperty="--database.db=/opt/data/heidelberg-v0.7.oshdb"'
     // END CUSTOM ohsome API
     SNAPSHOT_BRANCH_REGEX = /(^master$)/
     RELEASE_REGEX = /^([0-9]+(\.[0-9]+)*)(-(RC|beta-|alpha-)[0-9]+)?$/

README.md

@@ -129,7 +129,7 @@ To run the tests locally, you need the following:
     * -Dport.get starts data-aggregation + metadata tests using GET requests
     * -Dport.post starts data-aggregation tests using POST requests
     * -Dport.data starts data-extraction tests using GET and POST requests
-2. [heidelberg.oshdb](https://downloads.ohsome.org/OSHDB/v0.6/europe/germany/baden-wuerttemberg/heidelberg_68900_2020-07-23.oshdb.mv.db) file (or any other, which includes the data from Heidelberg)
+2. [heidelberg.oshdb](https://downloads.ohsome.org/OSHDB/v0.7/europe/germany/baden-wuerttemberg/heidelberg.oshdb.mv.db) file (or any other, which includes the data from Heidelberg)
 3. maven command: `mvn -Dport_get=8081 -Dport_post=8082 -Dport_data=8083 -DdbFilePathProperty="--database.db=<path-to-your-heidelberg.oshdb-file>" test`
 
 *Note:* 

pom.xml

@@ -35,7 +35,7 @@
     <jts2geojson.version>0.13.0</jts2geojson.version>
     <mavenjar.version>3.2.0</mavenjar.version>
     <opencsv.version>4.0</opencsv.version>
-    <oshdb.version>0.6.4</oshdb.version>
+    <oshdb.version>0.7.0</oshdb.version>
     <projectlombok.version>1.18.16</projectlombok.version>
     <sonar.sources>src/main/lombok</sonar.sources>
     <springboot.version>2.0.3.RELEASE</springboot.version>
@@ -93,6 +93,21 @@
         </exclusion>
       </exclusions>
     </dependency>
+    <dependency>
+      <groupId>org.heigit.ohsome</groupId>
+      <artifactId>oshdb-api-ignite</artifactId>
+      <version>${oshdb.version}</version>
+      <exclusions>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-simple</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
     <!-- needed for creating a custom error reporting class -->
     <dependency>
       <groupId>com.fasterxml.jackson.datatype</groupId>

src/main/lombok/org/heigit/ohsome/ohsomeapi/Application.java

@@ -5,17 +5,17 @@
 import java.sql.DriverManager;
 import java.sql.SQLException;
 import java.util.Properties;
-import org.heigit.bigspatialdata.oshdb.api.db.OSHDBH2;
-import org.heigit.bigspatialdata.oshdb.api.db.OSHDBIgnite;
-import org.heigit.bigspatialdata.oshdb.api.db.OSHDBJdbc;
-import org.heigit.bigspatialdata.oshdb.util.exceptions.OSHDBKeytablesNotFoundException;
-import org.heigit.bigspatialdata.oshdb.util.tagtranslator.TagTranslator;
 import org.heigit.ohsome.ohsomeapi.exception.DatabaseAccessException;
 import org.heigit.ohsome.ohsomeapi.exception.ExceptionMessages;
 import org.heigit.ohsome.ohsomeapi.inputprocessing.ProcessingData;
 import org.heigit.ohsome.ohsomeapi.oshdb.DbConnData;
 import org.heigit.ohsome.ohsomeapi.oshdb.RemoteTagTranslator;
 import org.heigit.ohsome.ohsomeapi.utils.RequestUtils;
+import org.heigit.ohsome.oshdb.api.db.OSHDBH2;
+import org.heigit.ohsome.oshdb.api.db.OSHDBIgnite;
+import org.heigit.ohsome.oshdb.api.db.OSHDBJdbc;
+import org.heigit.ohsome.oshdb.util.exceptions.OSHDBKeytablesNotFoundException;
+import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator;
 import org.springframework.boot.ApplicationArguments;
 import org.springframework.boot.ApplicationRunner;
 import org.springframework.boot.DefaultApplicationArguments;
@@ -72,12 +72,12 @@ public static void main(String[] args) {
    * @throws RuntimeException if a class with a specific name could not be found, or if the database
    *         parameter is not defined
    * @throws DatabaseAccessException if the access to keytables or database is not possible
-   * @throws SQLException thrown by {@link org.heigit.bigspatialdata.oshdb.api.db.OSHDBH2
+   * @throws SQLException thrown by {@link org.heigit.ohsome.oshdb.api.db.OSHDBH2
    *         #OSHDBH2(String) OSHDBH2}
-   * @throws ClassNotFoundException thrown by {@link org.heigit.bigspatialdata.oshdb.api.db.OSHDBH2
+   * @throws ClassNotFoundException thrown by {@link org.heigit.ohsome.oshdb.api.db.OSHDBH2
    *         #OSHDBH2(String) OSHDBH2}
    * @throws OSHDBKeytablesNotFoundException thrown by {@link
-   *         org.heigit.bigspatialdata.oshdb.util.tagtranslator.TagTranslator
+   *         org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator
    *         #TagTranslator(java.sql.Connection) TagTranslator}
    * @throws IOException thrown by {@link org.heigit.ohsome.ohsomeapi.utils.RequestUtils
    *         #extractOSHDBMetadata() extractOSHDBMetadata}

src/main/lombok/org/heigit/ohsome/ohsomeapi/exception/CustomizedExceptionHandler.java

@@ -3,8 +3,8 @@
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
 import javax.servlet.http.HttpServletRequest;
-import org.heigit.bigspatialdata.oshdb.util.exceptions.OSHDBTimeoutException;
 import org.heigit.ohsome.ohsomeapi.utils.RequestUtils;
+import org.heigit.ohsome.oshdb.util.exceptions.OSHDBTimeoutException;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.ControllerAdvice;

src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/AggregateRequestExecutor.java

@@ -18,15 +18,6 @@
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 import org.apache.commons.lang3.tuple.ImmutablePair;
-import org.heigit.bigspatialdata.oshdb.api.generic.OSHDBCombinedIndex;
-import org.heigit.bigspatialdata.oshdb.api.generic.function.SerializableFunction;
-import org.heigit.bigspatialdata.oshdb.api.mapreducer.MapAggregator;
-import org.heigit.bigspatialdata.oshdb.api.mapreducer.MapReducer;
-import org.heigit.bigspatialdata.oshdb.api.object.OSMEntitySnapshot;
-import org.heigit.bigspatialdata.oshdb.util.OSHDBTimestamp;
-import org.heigit.bigspatialdata.oshdb.util.geometry.Geo;
-import org.heigit.bigspatialdata.oshdb.util.time.TimestampFormatter;
-import org.heigit.ohsome.filter.FilterExpression;
 import org.heigit.ohsome.ohsomeapi.Application;
 import org.heigit.ohsome.ohsomeapi.exception.BadRequestException;
 import org.heigit.ohsome.ohsomeapi.exception.ExceptionMessages;
@@ -45,6 +36,15 @@
 import org.heigit.ohsome.ohsomeapi.output.groupby.GroupByResult;
 import org.heigit.ohsome.ohsomeapi.output.ratio.RatioResult;
 import org.heigit.ohsome.ohsomeapi.utils.RequestUtils;
+import org.heigit.ohsome.oshdb.OSHDBTimestamp;
+import org.heigit.ohsome.oshdb.api.generic.OSHDBCombinedIndex;
+import org.heigit.ohsome.oshdb.api.mapreducer.MapAggregator;
+import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer;
+import org.heigit.ohsome.oshdb.filter.FilterExpression;
+import org.heigit.ohsome.oshdb.util.function.SerializableFunction;
+import org.heigit.ohsome.oshdb.util.geometry.Geo;
+import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot;
+import org.heigit.ohsome.oshdb.util.time.TimestampFormatter;
 import org.locationtech.jts.geom.Geometry;
 import org.locationtech.jts.geom.Polygonal;
 
@@ -74,8 +74,8 @@ public AggregateRequestExecutor(RequestResource requestResource,
    *         PERIMETER, and AREA are permitted here
    * @throws Exception thrown by {@link org.heigit.ohsome.ohsomeapi.inputprocessing.InputProcessor
    *         #processParameters() processParameters},
-   *         {@link org.heigit.bigspatialdata.oshdb.api.mapreducer.MapAggregator#count() count}, or
-   *         {@link org.heigit.bigspatialdata.oshdb.api.mapreducer.MapAggregator#sum() sum}
+   *         {@link org.heigit.ohsome.oshdb.api.mapreducer.MapAggregator#count() count}, or
+   *         {@link org.heigit.ohsome.oshdb.api.mapreducer.MapAggregator#sum() sum}
    */
   public Response aggregate() throws Exception {
     final SortedMap<OSHDBTimestamp, ? extends Number> result;
@@ -333,10 +333,10 @@ private ElementsResult[] fillElementsResult(SortedMap<OSHDBTimestamp, ? extends
    * Computes the result for the /count|length|perimeter|area/groupBy/boundary resources.
    *
    * @throws BadRequestException if a boundary parameter is not defined.
-   * @throws Exception thrown by {@link org.heigit.bigspatialdata.oshdb.api.mapreducer.MapAggregator
-   *         #count() count}, or
-   *         {@link org.heigit.bigspatialdata.oshdb.api.mapreducer.MapAggregator
-   *         #sum(SerializableFunction) sum}
+   * @throws Exception thrown by
+   *         {@link org.heigit.ohsome.oshdb.api.mapreducer.MapAggregator#count() count}, or
+   *         {@link org.heigit.ohsome.oshdb.api.mapreducer.MapAggregator#sum(SerializableFunction)
+   *         sum}
    */
   private <P extends Geometry & Polygonal> SortedMap<OSHDBCombinedIndex<OSHDBTimestamp, Integer>,
         ? extends Number> computeCountLengthPerimeterAreaGbB(RequestResource requestResource,

src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/ContributionsExecutor.java

@@ -6,11 +6,6 @@
 import java.util.SortedMap;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
-import org.heigit.bigspatialdata.oshdb.api.mapreducer.MapReducer;
-import org.heigit.bigspatialdata.oshdb.api.object.OSMContribution;
-import org.heigit.bigspatialdata.oshdb.util.OSHDBTimestamp;
-import org.heigit.bigspatialdata.oshdb.util.celliterator.ContributionType;
-import org.heigit.ohsome.filter.FilterExpression;
 import org.heigit.ohsome.ohsomeapi.Application;
 import org.heigit.ohsome.ohsomeapi.exception.BadRequestException;
 import org.heigit.ohsome.ohsomeapi.inputprocessing.InputProcessor;
@@ -21,6 +16,11 @@
 import org.heigit.ohsome.ohsomeapi.output.Metadata;
 import org.heigit.ohsome.ohsomeapi.output.Response;
 import org.heigit.ohsome.ohsomeapi.output.contributions.ContributionsResult;
+import org.heigit.ohsome.oshdb.OSHDBTimestamp;
+import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer;
+import org.heigit.ohsome.oshdb.filter.FilterExpression;
+import org.heigit.ohsome.oshdb.util.celliterator.ContributionType;
+import org.heigit.ohsome.oshdb.util.mappable.OSMContribution;
 import org.locationtech.jts.geom.Geometry;
 
 /**
@@ -62,7 +62,7 @@ public ContributionsExecutor(HttpServletRequest servletRequest,
    *         processParameters},
    *         {@link org.heigit.ohsome.ohsomeapi.inputprocessing.InputProcessor
    *         #processParameters(ComputeMode) processParameters} and
-   *         {@link org.heigit.bigspatialdata.oshdb.api.mapreducer.MapAggregator#count() count}
+   *         {@link org.heigit.ohsome.oshdb.api.mapreducer.MapAggregator#count() count}
    * @throws UnsupportedOperationException thrown by
    *         {@link org.heigit.ohsome.ohsomeapi.executor.ContributionsExecutor
    *         #usersCount(MapReducer) usersCount} and
@@ -117,10 +117,9 @@ public Response count(boolean isUsersRequest, boolean isContributionsLatestCount
    * @param mapRed a MapReducer of OSM contributions
    * @return SortedMap with counts of users aggregated by timestamp
    * @throws Exception thrown by
-   *         {@link org.heigit.bigspatialdata.oshdb.api.mapreducer.MapAggregator #countUniq()
-   *         countUniq}
+   *         {@link org.heigit.ohsome.oshdb.api.mapreducer.MapAggregator #countUniq() countUniq}
    * @throws UnsupportedOperationException thrown by
-   *         {@link org.heigit.bigspatialdata.oshdb.api.mapreducer.MapReducer#aggregateByTimestamp()
+   *         {@link org.heigit.ohsome.oshdb.api.mapreducer.MapReducer#aggregateByTimestamp()
    *         aggregateByTimeStamp}
    */
   private SortedMap<OSHDBTimestamp, Integer> usersCount(MapReducer<OSMContribution> mapRed)
@@ -136,9 +135,9 @@ private SortedMap<OSHDBTimestamp, Integer> usersCount(MapReducer<OSMContribution
    * @param isContributionsLatest the boolean value relative to the endpoint /contributions/latest
    * @return SortedMap with counts of contributions aggregated by timestamp
    * @throws Exception thrown by
-   *         {@link org.heigit.bigspatialdata.oshdb.api.mapreducer.MapAggregator #count() count}
+   *         {@link org.heigit.ohsome.oshdb.api.mapreducer.MapAggregator #count() count}
    * @throws UnsupportedOperationException thrown by
-   *         {@link org.heigit.bigspatialdata.oshdb.api.mapreducer.MapReducer#aggregateByTimestamp()
+   *         {@link org.heigit.ohsome.oshdb.api.mapreducer.MapReducer#aggregateByTimestamp()
    *         aggregateByTimeStamp}
    */
   private SortedMap<OSHDBTimestamp, Integer> contributionsCount(MapReducer<OSMContribution> mapRed,

src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/DataExtractionTransformer.java

@@ -8,15 +8,15 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeMap;
-import org.heigit.bigspatialdata.oshdb.api.object.OSMContribution;
-import org.heigit.bigspatialdata.oshdb.api.object.OSMEntitySnapshot;
-import org.heigit.bigspatialdata.oshdb.osm.OSMEntity;
-import org.heigit.bigspatialdata.oshdb.util.celliterator.ContributionType;
-import org.heigit.bigspatialdata.oshdb.util.time.TimestampFormatter;
-import org.heigit.ohsome.filter.FilterExpression;
 import org.heigit.ohsome.ohsomeapi.controller.dataextraction.elements.ElementsGeometry;
 import org.heigit.ohsome.ohsomeapi.inputprocessing.InputProcessingUtils;
 import org.heigit.ohsome.ohsomeapi.inputprocessing.SimpleFeatureType;
+import org.heigit.ohsome.oshdb.filter.FilterExpression;
+import org.heigit.ohsome.oshdb.osm.OSMEntity;
+import org.heigit.ohsome.oshdb.util.celliterator.ContributionType;
+import org.heigit.ohsome.oshdb.util.mappable.OSMContribution;
+import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot;
+import org.heigit.ohsome.oshdb.util.time.TimestampFormatter;
 import org.locationtech.jts.geom.Geometry;
 import org.wololo.geojson.Feature;
 

src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/DataRequestExecutor.java

@@ -9,14 +9,6 @@
 import java.util.stream.Stream;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
-import org.heigit.bigspatialdata.oshdb.api.db.OSHDBIgnite;
-import org.heigit.bigspatialdata.oshdb.api.db.OSHDBIgnite.ComputeMode;
-import org.heigit.bigspatialdata.oshdb.api.mapreducer.MapReducer;
-import org.heigit.bigspatialdata.oshdb.api.object.OSMContribution;
-import org.heigit.bigspatialdata.oshdb.api.object.OSMEntitySnapshot;
-import org.heigit.bigspatialdata.oshdb.util.tagtranslator.TagTranslator;
-import org.heigit.bigspatialdata.oshdb.util.time.IsoDateTimeParser;
-import org.heigit.ohsome.filter.FilterExpression;
 import org.heigit.ohsome.ohsomeapi.Application;
 import org.heigit.ohsome.ohsomeapi.controller.dataextraction.elements.ElementsGeometry;
 import org.heigit.ohsome.ohsomeapi.exception.BadRequestException;
@@ -28,6 +20,14 @@
 import org.heigit.ohsome.ohsomeapi.oshdb.DbConnData;
 import org.heigit.ohsome.ohsomeapi.output.ExtractionResponse;
 import org.heigit.ohsome.ohsomeapi.output.Metadata;
+import org.heigit.ohsome.oshdb.api.db.OSHDBIgnite;
+import org.heigit.ohsome.oshdb.api.db.OSHDBIgnite.ComputeMode;
+import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer;
+import org.heigit.ohsome.oshdb.filter.FilterExpression;
+import org.heigit.ohsome.oshdb.util.mappable.OSMContribution;
+import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot;
+import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator;
+import org.heigit.ohsome.oshdb.util.time.IsoDateTimeParser;
 import org.wololo.geojson.Feature;
 
 /** Holds executor methods for the following endpoints: /elementsFullHistory, /contributions. */
@@ -52,9 +52,9 @@ public DataRequestExecutor(RequestResource requestResource, ElementsGeometry ele
    *
    * @throws Exception thrown by {@link org.heigit.ohsome.ohsomeapi.inputprocessing.InputProcessor
    *         #processParameters() processParameters},
-   *         {@link org.heigit.bigspatialdata.oshdb.util.time.IsoDateTimeParser
+   *         {@link org.heigit.ohsome.oshdb.util.time.IsoDateTimeParser
    *         #parseIsoDateTime(String) parseIsoDateTime},
-   *         {@link org.heigit.bigspatialdata.oshdb.api.mapreducer.MapReducer#stream() stream}, or
+   *         {@link org.heigit.ohsome.oshdb.api.mapreducer.MapReducer#stream() stream}, or
    *         {@link org.heigit.ohsome.ohsomeapi.executor.ExecutionUtils
    *         #streamResponse(HttpServletResponse, ExtractionResponse, Stream)
    *         streamElementsResponse}
@@ -69,8 +69,8 @@ public void extract() throws Exception {
       // on ignite: Use AffinityCall backend, which is the only one properly supporting streaming
       // of result data, without buffering the whole result in memory before returning the result.
       // This allows to write data out to the client via a chunked HTTP response.
-      mapRedSnapshot = snapshotInputProcessor.processParameters(ComputeMode.AffinityCall);
-      mapRedContribution = inputProcessor.processParameters(ComputeMode.AffinityCall);
+      mapRedSnapshot = snapshotInputProcessor.processParameters(ComputeMode.AFFINITY_CALL);
+      mapRedContribution = inputProcessor.processParameters(ComputeMode.AFFINITY_CALL);
     } else {
       mapRedSnapshot = snapshotInputProcessor.processParameters();
       mapRedContribution = inputProcessor.processParameters();