From b329f6a8bf80d15ad34548583361b725769c22c9 Mon Sep 17 00:00:00 2001 From: Martin Raifer Date: Tue, 14 Jan 2020 17:41:06 +0100 Subject: [PATCH 1/5] delay tag translation when using remote tag translation with the affinitycall backend, each cell got it's own tagtranslator object and connection. This made the tagtranslator caches ineffective, causing a bottleneck and overloaded the keytables database. doing the translation of tags on the client also has the benefit of minimizing the network traffic inside the cluster nodes --- .../executor/ElementsRequestExecutor.java | 13 +++--- .../ohsomeapi/executor/ExecutionUtils.java | 40 ++++++++++++++----- 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/executor/ElementsRequestExecutor.java b/src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/executor/ElementsRequestExecutor.java index 9c4b9164..a8478589 100644 --- a/src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/executor/ElementsRequestExecutor.java +++ b/src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/executor/ElementsRequestExecutor.java @@ -33,7 +33,6 @@ import org.heigit.bigspatialdata.ohsome.ohsomeapi.inputprocessing.SimpleFeatureType; import org.heigit.bigspatialdata.ohsome.ohsomeapi.oshdb.DbConnData; import org.heigit.bigspatialdata.ohsome.ohsomeapi.oshdb.ExtractMetadata; -import org.heigit.bigspatialdata.ohsome.ohsomeapi.oshdb.RemoteTagTranslator; import org.heigit.bigspatialdata.ohsome.ohsomeapi.output.Description; import org.heigit.bigspatialdata.ohsome.ohsomeapi.output.dataaggregationresponse.Attribution; import org.heigit.bigspatialdata.ohsome.ohsomeapi.output.dataaggregationresponse.DefaultAggregationResponse; @@ -120,7 +119,6 @@ public static void executeElements(ElementsGeometry elemGeom, HttpServletRequest } final MapReducer preResult; ExecutionUtils exeUtils = new ExecutionUtils(processingData); - RemoteTagTranslator mapTagTranslator = DbConnData.mapTagTranslator; preResult = mapRed.map(snapshot -> { Map properties = new TreeMap<>(); if (includeOSMMetadata) { @@ -134,7 +132,7 @@ public static void executeElements(ElementsGeometry elemGeom, HttpServletRequest geom = snapshot.getGeometryUnclipped(); } return exeUtils.createOSMFeature(snapshot.getEntity(), geom, properties, keysInt, includeTags, - includeOSMMetadata, elemGeom, mapTagTranslator.get()); + includeOSMMetadata, elemGeom); }).filter(Objects::nonNull); Metadata metadata = null; if (processingData.isShowMetadata()) { @@ -164,7 +162,7 @@ public static void executeElements(ElementsGeometry elemGeom, HttpServletRequest * {@link org.heigit.bigspatialdata.oshdb.util.time.ISODateTimeParser#parseISODateTime(String) * parseISODateTime}, * {@link org.heigit.bigspatialdata.oshdb.api.mapreducer.MapReducer#stream() stream}, or - * {@link org.heigit.bigspatialdata.ohsome.ohsomeapi.executor.ExecutionUtils#streamElementsResponse(HttpServletResponse, DataResponse, boolean, Stream, Stream) + * {@link #streamElementsResponse(HttpServletResponse, DataResponse, boolean, Stream * streamElementsResponse} */ public static void executeElementsFullHistory(ElementsGeometry elemGeom, @@ -202,7 +200,6 @@ public static void executeElementsFullHistory(ElementsGeometry elemGeom, } MapReducer contributionPreResult = null; ExecutionUtils exeUtils = new ExecutionUtils(processingData); - RemoteTagTranslator mapTagTranslator = DbConnData.mapTagTranslator; inputProcessor.processPropertiesParam(); InputProcessingUtils utils = inputProcessor.getUtils(); final boolean includeTags = inputProcessor.includeTags(); @@ -246,7 +243,7 @@ public static void executeElementsFullHistory(ElementsGeometry elemGeom, if (!processingData.containsSimpleFeatureTypes() || utils.checkGeometryOnSimpleFeatures(currentGeom, simpleFeatureTypes)) { output.add(exeUtils.createOSMFeature(currentEntity, currentGeom, properties, keysInt, - includeTags, includeOSMMetadata, elemGeom, mapTagTranslator.get())); + includeTags, includeOSMMetadata, elemGeom)); } } } @@ -275,7 +272,7 @@ public static void executeElementsFullHistory(ElementsGeometry elemGeom, if (!processingData.containsSimpleFeatureTypes() || utils.checkGeometryOnSimpleFeatures(currentGeom, simpleFeatureTypes)) { output.add(exeUtils.createOSMFeature(currentEntity, currentGeom, properties, keysInt, - includeTags, includeOSMMetadata, elemGeom, mapTagTranslator.get())); + includeTags, includeOSMMetadata, elemGeom)); } } } @@ -304,7 +301,7 @@ public static void executeElementsFullHistory(ElementsGeometry elemGeom, if (!processingData.containsSimpleFeatureTypes() || utils.checkGeometryOnSimpleFeatures(geom, simpleFeatureTypes)) { return Collections.singletonList(exeUtils.createOSMFeature(entity, geom, properties, - keysInt, includeTags, includeOSMMetadata, elemGeom, mapTagTranslator.get())); + keysInt, includeTags, includeOSMMetadata, elemGeom)); } else { return Collections.emptyList(); } diff --git a/src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/executor/ExecutionUtils.java b/src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/executor/ExecutionUtils.java index 8a130eaf..fdbd8972 100644 --- a/src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/executor/ExecutionUtils.java +++ b/src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/executor/ExecutionUtils.java @@ -46,7 +46,9 @@ import org.heigit.bigspatialdata.ohsome.ohsomeapi.inputprocessing.InputProcessor; import org.heigit.bigspatialdata.ohsome.ohsomeapi.inputprocessing.ProcessingData; import org.heigit.bigspatialdata.ohsome.ohsomeapi.inputprocessing.SimpleFeatureType; +import org.heigit.bigspatialdata.ohsome.ohsomeapi.oshdb.DbConnData; import org.heigit.bigspatialdata.ohsome.ohsomeapi.oshdb.ExtractMetadata; +import org.heigit.bigspatialdata.ohsome.ohsomeapi.oshdb.RemoteTagTranslator; import org.heigit.bigspatialdata.ohsome.ohsomeapi.output.Description; import org.heigit.bigspatialdata.ohsome.ohsomeapi.output.dataaggregationresponse.Attribution; import org.heigit.bigspatialdata.ohsome.ohsomeapi.output.dataaggregationresponse.Metadata; @@ -79,6 +81,7 @@ import org.heigit.bigspatialdata.oshdb.util.OSHDBTag; import org.heigit.bigspatialdata.oshdb.util.OSHDBTimestamp; import org.heigit.bigspatialdata.oshdb.util.celliterator.ContributionType; +import org.heigit.bigspatialdata.oshdb.util.exceptions.OSHDBKeytablesNotFoundException; import org.heigit.bigspatialdata.oshdb.util.geometry.Geo; import org.heigit.bigspatialdata.oshdb.util.geometry.OSHDBGeometryBuilder; import org.heigit.bigspatialdata.oshdb.util.tagtranslator.OSMTag; @@ -218,8 +221,8 @@ public static & Serializable, V extends Comparable snapshotStream, - Stream contributionStream) throws Exception { - + Stream contributionStream + ) throws Exception { JsonFactory jsonFactory = new JsonFactory(); ByteArrayOutputStream tempStream = new ByteArrayOutputStream(); @@ -407,15 +410,16 @@ public List createCsvTopComments(String url, String text, String apiVe /** Creates the Feature objects in the OSM data response. */ public org.wololo.geojson.Feature createOSMFeature(OSMEntity entity, Geometry geometry, Map properties, int[] keysInt, boolean includeTags, - boolean includeOSMMetadata, ElementsGeometry elemGeom, TagTranslator tt) { + boolean includeOSMMetadata, ElementsGeometry elemGeom) { if (geometry.isEmpty()) { // skip invalid geometries (e.g. ways with 0 nodes) return null; } if (includeTags) { for (OSHDBTag oshdbTag : entity.getTags()) { - OSMTag tag = tt.getOSMTagOf(oshdbTag); - properties.put(tag.getKey(), tag.getValue()); + //OSMTag tag = tt.getOSMTagOf(oshdbTag); + //properties.put(tag.getKey(), tag.getValue()); + properties.put(String.valueOf(oshdbTag.getKey()), oshdbTag); } } else if (keysInt.length != 0) { int[] tags = entity.getRawTags(); @@ -424,8 +428,10 @@ public org.wololo.geojson.Feature createOSMFeature(OSMEntity entity, Geometry ge int tagValueId = tags[i + 1]; for (int key : keysInt) { if (tagKeyId == key) { - OSMTag tag = tt.getOSMTagOf(tagKeyId, tagValueId); - properties.put(tag.getKey(), tag.getValue()); + //OSMTag tag = tt.getOSMTagOf(tagKeyId, tagValueId); + //properties.put(tag.getKey(), tag.getValue()); + properties.put(String.valueOf(tagKeyId), new OSHDBTag(tagKeyId, tagValueId)); + break; } } } @@ -1089,13 +1095,29 @@ private ImmutablePair, List> createCsvResponseForUsersGro /** Fills the given stream with output data using multiple parallel threads. */ private void writeStreamResponse(ThreadLocal outputJsonGen, Stream stream, ThreadLocal outputBuffers, - final ServletOutputStream outputStream) - throws ExecutionException, InterruptedException, IOException { + final ServletOutputStream outputStream + ) throws ExecutionException, InterruptedException, IOException, OSHDBKeytablesNotFoundException { + TagTranslator tt = new TagTranslator(DbConnData.keytables.getConnection()); ReentrantLock lock = new ReentrantLock(); AtomicBoolean errored = new AtomicBoolean(false); ForkJoinPool threadPool = new ForkJoinPool(ProcessingData.getNumberOfDataExtractionThreads()); try { threadPool.submit(() -> stream.parallel().map(data -> { + // 0. resolve tags + Map tags = data.getProperties(); + List keysToDelete = new LinkedList<>(); + List tagsToAdd = new LinkedList<>(); + for (Entry tag : tags.entrySet()) { + String key = tag.getKey(); + if (key.charAt(0) != '@') { + keysToDelete.add(key); + tagsToAdd.add(tt.getOSMTagOf((OSHDBTag) tag.getValue())); + } + } + tags.keySet().removeAll(keysToDelete); + for (OSMTag tag : tagsToAdd) { + tags.put(tag.getKey(), tag.getValue()); + } // 1. convert features to geojson try { outputBuffers.get().reset(); From ccf42a5f03aa0f6ba7f1010157b4f8d89ddd8c4e Mon Sep 17 00:00:00 2001 From: Martin Raifer Date: Wed, 15 Jan 2020 16:57:32 +0100 Subject: [PATCH 2/5] use a db connection pool for tag translation in data extraction --- pom.xml | 6 +++++ .../ohsome/ohsomeapi/Application.java | 8 ++++++ .../ohsomeapi/executor/ExecutionUtils.java | 25 ++++++++++++++++--- .../ohsome/ohsomeapi/oshdb/DbConnData.java | 5 +++- 4 files changed, 40 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index ea7aa7c8..6b1c8ae2 100644 --- a/pom.xml +++ b/pom.xml @@ -146,6 +146,12 @@ commons-csv ${apachecommons-csv.version} + + + com.zaxxer + HikariCP + 3.4.2 + diff --git a/src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/Application.java b/src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/Application.java index d97260c4..e15e208f 100644 --- a/src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/Application.java +++ b/src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/Application.java @@ -1,5 +1,7 @@ package org.heigit.bigspatialdata.ohsome.ohsomeapi; +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; import java.sql.DriverManager; import java.sql.SQLException; import org.heigit.bigspatialdata.ohsome.ohsomeapi.inputprocessing.ProcessingData; @@ -98,6 +100,12 @@ public static void preRun(ApplicationArguments args) throws Exception { throw new RuntimeException(e); } }); + HikariConfig hikariConfig = new HikariConfig(); + hikariConfig.setJdbcUrl(jdbcParam[1]); + hikariConfig.setUsername(jdbcParam[2]); + hikariConfig.setPassword(jdbcParam[3]); + hikariConfig.setMaximumPoolSize(numberOfDataExtractionThreads); + DbConnData.keytablesDbPoolConfig = hikariConfig; break; case "database.multithreading": if (args.getOptionValues(paramName).get(0).equalsIgnoreCase("false")) { diff --git a/src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/executor/ExecutionUtils.java b/src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/executor/ExecutionUtils.java index fdbd8972..05ded2bb 100644 --- a/src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/executor/ExecutionUtils.java +++ b/src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/executor/ExecutionUtils.java @@ -9,10 +9,12 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import com.opencsv.CSVWriter; +import com.zaxxer.hikari.HikariDataSource; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.Serializable; import java.nio.charset.StandardCharsets; +import java.sql.SQLException; import java.text.DecimalFormat; import java.text.DecimalFormatSymbols; import java.util.ArrayList; @@ -1096,8 +1098,22 @@ private ImmutablePair, List> createCsvResponseForUsersGro private void writeStreamResponse(ThreadLocal outputJsonGen, Stream stream, ThreadLocal outputBuffers, final ServletOutputStream outputStream - ) throws ExecutionException, InterruptedException, IOException, OSHDBKeytablesNotFoundException { - TagTranslator tt = new TagTranslator(DbConnData.keytables.getConnection()); + ) throws ExecutionException, InterruptedException, IOException { + ThreadLocal tts; + HikariDataSource keytablesConnectionPool; + if (DbConnData.keytablesDbPoolConfig != null) { + keytablesConnectionPool = new HikariDataSource(DbConnData.keytablesDbPoolConfig); + tts = ThreadLocal.withInitial(() -> { + try { + return new TagTranslator(keytablesConnectionPool.getConnection()); + } catch (OSHDBKeytablesNotFoundException | SQLException e) { + throw new RuntimeException(e); + } + }); + } else { + keytablesConnectionPool = null; + tts = ThreadLocal.withInitial(() -> DbConnData.tagTranslator); + } ReentrantLock lock = new ReentrantLock(); AtomicBoolean errored = new AtomicBoolean(false); ForkJoinPool threadPool = new ForkJoinPool(ProcessingData.getNumberOfDataExtractionThreads()); @@ -1111,7 +1127,7 @@ private void writeStreamResponse(ThreadLocal outputJsonGen, String key = tag.getKey(); if (key.charAt(0) != '@') { keysToDelete.add(key); - tagsToAdd.add(tt.getOSMTagOf((OSHDBTag) tag.getValue())); + tagsToAdd.add(tts.get().getOSMTagOf((OSHDBTag) tag.getValue())); } } tags.keySet().removeAll(keysToDelete); @@ -1155,6 +1171,9 @@ private void writeStreamResponse(ThreadLocal outputJsonGen, } finally { threadPool.shutdown(); outputStream.flush(); + if (keytablesConnectionPool != null) { + keytablesConnectionPool.close(); + } } } diff --git a/src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/oshdb/DbConnData.java b/src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/oshdb/DbConnData.java index fd89f7a7..d48a1b60 100644 --- a/src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/oshdb/DbConnData.java +++ b/src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/oshdb/DbConnData.java @@ -1,5 +1,7 @@ package org.heigit.bigspatialdata.ohsome.ohsomeapi.oshdb; +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; import org.heigit.bigspatialdata.oshdb.api.db.OSHDBDatabase; import org.heigit.bigspatialdata.oshdb.api.db.OSHDBJdbc; import org.heigit.bigspatialdata.oshdb.util.tagtranslator.TagTranslator; @@ -11,7 +13,8 @@ public class DbConnData { public static OSHDBJdbc keytables = null; public static TagTranslator tagTranslator = null; public static RemoteTagTranslator mapTagTranslator = null; - + public static HikariConfig keytablesDbPoolConfig = null; + private DbConnData() { throw new IllegalStateException("Utility class"); } From 57f3cbf6c74c45822dbbc4fe385cb51d2158987e Mon Sep 17 00:00:00 2001 From: Fabian Kowatsch Date: Mon, 20 Jan 2020 16:03:44 +0100 Subject: [PATCH 3/5] some minor code cleanup --- .../executor/ElementsRequestExecutor.java | 2 +- .../ohsomeapi/executor/ExecutionUtils.java | 20 ++++++------------- .../ohsome/ohsomeapi/oshdb/DbConnData.java | 1 - 3 files changed, 7 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/executor/ElementsRequestExecutor.java b/src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/executor/ElementsRequestExecutor.java index a8478589..e841fd5f 100644 --- a/src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/executor/ElementsRequestExecutor.java +++ b/src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/executor/ElementsRequestExecutor.java @@ -162,7 +162,7 @@ public static void executeElements(ElementsGeometry elemGeom, HttpServletRequest * {@link org.heigit.bigspatialdata.oshdb.util.time.ISODateTimeParser#parseISODateTime(String) * parseISODateTime}, * {@link org.heigit.bigspatialdata.oshdb.api.mapreducer.MapReducer#stream() stream}, or - * {@link #streamElementsResponse(HttpServletResponse, DataResponse, boolean, Stream + * {@link #streamElementsResponse(HttpServletResponse, DataResponse, boolean, Stream) * streamElementsResponse} */ public static void executeElementsFullHistory(ElementsGeometry elemGeom, diff --git a/src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/executor/ExecutionUtils.java b/src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/executor/ExecutionUtils.java index 05ded2bb..606922da 100644 --- a/src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/executor/ExecutionUtils.java +++ b/src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/executor/ExecutionUtils.java @@ -50,7 +50,6 @@ import org.heigit.bigspatialdata.ohsome.ohsomeapi.inputprocessing.SimpleFeatureType; import org.heigit.bigspatialdata.ohsome.ohsomeapi.oshdb.DbConnData; import org.heigit.bigspatialdata.ohsome.ohsomeapi.oshdb.ExtractMetadata; -import org.heigit.bigspatialdata.ohsome.ohsomeapi.oshdb.RemoteTagTranslator; import org.heigit.bigspatialdata.ohsome.ohsomeapi.output.Description; import org.heigit.bigspatialdata.ohsome.ohsomeapi.output.dataaggregationresponse.Attribution; import org.heigit.bigspatialdata.ohsome.ohsomeapi.output.dataaggregationresponse.Metadata; @@ -223,8 +222,7 @@ public static & Serializable, V extends Comparable snapshotStream, - Stream contributionStream - ) throws Exception { + Stream contributionStream) throws Exception { JsonFactory jsonFactory = new JsonFactory(); ByteArrayOutputStream tempStream = new ByteArrayOutputStream(); @@ -250,12 +248,10 @@ public void writeIndentation(JsonGenerator g, int level) throws IOException { super.writeIndentation(g, level + 1); } }; - DefaultPrettyPrinter printer = new DefaultPrettyPrinter("") - .withArrayIndenter(indenter) - .withObjectIndenter(indenter); + DefaultPrettyPrinter printer = + new DefaultPrettyPrinter("").withArrayIndenter(indenter).withObjectIndenter(indenter); return jsonFactory.createGenerator(outputBuffers.get(), JsonEncoding.UTF8) - .setCodec(objMapper) - .setPrettyPrinter(printer); + .setCodec(objMapper).setPrettyPrinter(printer); } catch (IOException e) { throw new RuntimeException(e); } @@ -419,8 +415,6 @@ public org.wololo.geojson.Feature createOSMFeature(OSMEntity entity, Geometry ge } if (includeTags) { for (OSHDBTag oshdbTag : entity.getTags()) { - //OSMTag tag = tt.getOSMTagOf(oshdbTag); - //properties.put(tag.getKey(), tag.getValue()); properties.put(String.valueOf(oshdbTag.getKey()), oshdbTag); } } else if (keysInt.length != 0) { @@ -430,8 +424,6 @@ public org.wololo.geojson.Feature createOSMFeature(OSMEntity entity, Geometry ge int tagValueId = tags[i + 1]; for (int key : keysInt) { if (tagKeyId == key) { - //OSMTag tag = tt.getOSMTagOf(tagKeyId, tagValueId); - //properties.put(tag.getKey(), tag.getValue()); properties.put(String.valueOf(tagKeyId), new OSHDBTag(tagKeyId, tagValueId)); break; } @@ -1097,8 +1089,8 @@ private ImmutablePair, List> createCsvResponseForUsersGro /** Fills the given stream with output data using multiple parallel threads. */ private void writeStreamResponse(ThreadLocal outputJsonGen, Stream stream, ThreadLocal outputBuffers, - final ServletOutputStream outputStream - ) throws ExecutionException, InterruptedException, IOException { + final ServletOutputStream outputStream) + throws ExecutionException, InterruptedException, IOException { ThreadLocal tts; HikariDataSource keytablesConnectionPool; if (DbConnData.keytablesDbPoolConfig != null) { diff --git a/src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/oshdb/DbConnData.java b/src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/oshdb/DbConnData.java index d48a1b60..1c5c6c34 100644 --- a/src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/oshdb/DbConnData.java +++ b/src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/oshdb/DbConnData.java @@ -1,7 +1,6 @@ package org.heigit.bigspatialdata.ohsome.ohsomeapi.oshdb; import com.zaxxer.hikari.HikariConfig; -import com.zaxxer.hikari.HikariDataSource; import org.heigit.bigspatialdata.oshdb.api.db.OSHDBDatabase; import org.heigit.bigspatialdata.oshdb.api.db.OSHDBJdbc; import org.heigit.bigspatialdata.oshdb.util.tagtranslator.TagTranslator; From 273b6ae8f535d224d45c48f454363dddd79e595e Mon Sep 17 00:00:00 2001 From: Martin Raifer Date: Tue, 21 Jan 2020 11:53:15 +0100 Subject: [PATCH 4/5] properly shut down application context after each test step --- .../bigspatialdata/ohsome/ohsomeapi/Application.java | 9 +++++++-- .../ohsomeapi/controller/ElementsControllerTest.java | 8 ++++++++ .../ohsome/ohsomeapi/controller/GetControllerTest.java | 8 ++++++++ .../ohsome/ohsomeapi/controller/PostControllerTest.java | 8 ++++++++ 4 files changed, 31 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/Application.java b/src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/Application.java index e15e208f..9f313a63 100644 --- a/src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/Application.java +++ b/src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/Application.java @@ -1,9 +1,9 @@ package org.heigit.bigspatialdata.ohsome.ohsomeapi; import com.zaxxer.hikari.HikariConfig; -import com.zaxxer.hikari.HikariDataSource; import java.sql.DriverManager; import java.sql.SQLException; +import org.springframework.context.ApplicationContext; import org.heigit.bigspatialdata.ohsome.ohsomeapi.inputprocessing.ProcessingData; import org.heigit.bigspatialdata.ohsome.ohsomeapi.oshdb.DbConnData; import org.heigit.bigspatialdata.ohsome.ohsomeapi.oshdb.RemoteTagTranslator; @@ -33,6 +33,11 @@ public class Application implements ApplicationRunner { public static final int DEFAULT_NUMBER_OF_CLUSTER_NODES = 24; public static final int DEFAULT_NUMBER_OF_DATA_EXTRACTION_THREADS = 40; + private static ApplicationContext context; + public static ApplicationContext getApplicationContext() { + return context; + } + /** Main method to run this SpringBootApplication. */ public static void main(String[] args) { if (args == null || args.length == 0) { @@ -43,7 +48,7 @@ public static void main(String[] args) { } try { preRun(new DefaultApplicationArguments(args)); - SpringApplication.run(Application.class, args); + context = SpringApplication.run(Application.class, args); } catch (Exception e) { e.printStackTrace(); System.exit(1); diff --git a/src/test/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/controller/ElementsControllerTest.java b/src/test/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/controller/ElementsControllerTest.java index 6ffa381a..e57f796b 100644 --- a/src/test/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/controller/ElementsControllerTest.java +++ b/src/test/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/controller/ElementsControllerTest.java @@ -10,8 +10,10 @@ import java.util.Spliterators; import java.util.stream.StreamSupport; import org.heigit.bigspatialdata.ohsome.ohsomeapi.Application; +import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; +import org.springframework.boot.SpringApplication; import org.springframework.boot.test.web.client.TestRestTemplate; import org.springframework.http.ResponseEntity; import org.springframework.util.LinkedMultiValueMap; @@ -36,6 +38,12 @@ public static void applicationMainStartup() { Application.main(params.toArray(new String[0])); } + /** Stops this application context. */ + @AfterClass + public static void applicationMainShutdown() { + SpringApplication.exit(Application.getApplicationContext(), () -> 0); + } + /* * ./elements/geometry tests */ diff --git a/src/test/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/controller/GetControllerTest.java b/src/test/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/controller/GetControllerTest.java index bff808af..f32ace34 100644 --- a/src/test/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/controller/GetControllerTest.java +++ b/src/test/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/controller/GetControllerTest.java @@ -14,8 +14,10 @@ import java.util.stream.StreamSupport; import org.apache.commons.csv.CSVRecord; import org.heigit.bigspatialdata.ohsome.ohsomeapi.Application; +import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; +import org.springframework.boot.SpringApplication; import org.springframework.boot.test.web.client.TestRestTemplate; import org.springframework.http.ResponseEntity; @@ -38,6 +40,12 @@ public static void applicationMainStartup() { Application.main(params.toArray(new String[0])); } + /** Stops this application context. */ + @AfterClass + public static void applicationMainShutdown() { + SpringApplication.exit(Application.getApplicationContext(), () -> 0); + } + /** Method to get response body as String */ private String getResponseBody(String urlParams) { TestRestTemplate restTemplate = new TestRestTemplate(); diff --git a/src/test/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/controller/PostControllerTest.java b/src/test/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/controller/PostControllerTest.java index 3877c2e4..76ce9b95 100644 --- a/src/test/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/controller/PostControllerTest.java +++ b/src/test/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/controller/PostControllerTest.java @@ -13,8 +13,10 @@ import java.util.stream.StreamSupport; import org.apache.commons.csv.CSVRecord; import org.heigit.bigspatialdata.ohsome.ohsomeapi.Application; +import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; +import org.springframework.boot.SpringApplication; import org.springframework.boot.test.web.client.TestRestTemplate; import org.springframework.http.ResponseEntity; import org.springframework.util.LinkedMultiValueMap; @@ -38,6 +40,12 @@ public static void applicationMainStartup() { Application.main(params.toArray(new String[0])); } + /** Stops this application context. */ + @AfterClass + public static void applicationMainShutdown() { + SpringApplication.exit(Application.getApplicationContext(), () -> 0); + } + /* * /elements/count tests */ From 5788af605d1474251046d40f14a4bb92a3091177 Mon Sep 17 00:00:00 2001 From: Martin Raifer Date: Tue, 21 Jan 2020 12:00:03 +0100 Subject: [PATCH 5/5] fix javadoc reference --- .../ohsome/ohsomeapi/executor/ElementsRequestExecutor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/executor/ElementsRequestExecutor.java b/src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/executor/ElementsRequestExecutor.java index e841fd5f..2e00e1f2 100644 --- a/src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/executor/ElementsRequestExecutor.java +++ b/src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/executor/ElementsRequestExecutor.java @@ -162,7 +162,7 @@ public static void executeElements(ElementsGeometry elemGeom, HttpServletRequest * {@link org.heigit.bigspatialdata.oshdb.util.time.ISODateTimeParser#parseISODateTime(String) * parseISODateTime}, * {@link org.heigit.bigspatialdata.oshdb.api.mapreducer.MapReducer#stream() stream}, or - * {@link #streamElementsResponse(HttpServletResponse, DataResponse, boolean, Stream) + * {@link org.heigit.bigspatialdata.ohsome.ohsomeapi.executor.ExecutionUtils#streamElementsResponse(HttpServletResponse, DataResponse, boolean, Stream, Stream) * streamElementsResponse} */ public static void executeElementsFullHistory(ElementsGeometry elemGeom,