From ae1b09656d49d622569a5fb5b60d1eff247b74b4 Mon Sep 17 00:00:00 2001 From: Johannes Visintini Date: Tue, 10 Nov 2020 18:01:24 +0100 Subject: [PATCH 1/2] Extract flatMap lambda functions from DataRequestExecutor --- .../executor/DataRequestExecutor.java | 163 ++------------ .../ohsomeapi/executor/FlatMapExecutor.java | 205 ++++++++++++++++++ 2 files changed, 220 insertions(+), 148 deletions(-) create mode 100644 src/main/java/org/heigit/ohsome/ohsomeapi/executor/FlatMapExecutor.java diff --git a/src/main/java/org/heigit/ohsome/ohsomeapi/executor/DataRequestExecutor.java b/src/main/java/org/heigit/ohsome/ohsomeapi/executor/DataRequestExecutor.java index c6cdbe09..1497da1a 100644 --- a/src/main/java/org/heigit/ohsome/ohsomeapi/executor/DataRequestExecutor.java +++ b/src/main/java/org/heigit/ohsome/ohsomeapi/executor/DataRequestExecutor.java @@ -2,13 +2,10 @@ import java.time.format.DateTimeFormatter; import java.util.Collections; -import java.util.LinkedList; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.TreeMap; import java.util.stream.Stream; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -17,11 +14,8 @@ import org.heigit.bigspatialdata.oshdb.api.mapreducer.MapReducer; import org.heigit.bigspatialdata.oshdb.api.object.OSMContribution; import org.heigit.bigspatialdata.oshdb.api.object.OSMEntitySnapshot; -import org.heigit.bigspatialdata.oshdb.osm.OSMEntity; -import org.heigit.bigspatialdata.oshdb.util.celliterator.ContributionType; import org.heigit.bigspatialdata.oshdb.util.tagtranslator.TagTranslator; import org.heigit.bigspatialdata.oshdb.util.time.ISODateTimeParser; -import org.heigit.bigspatialdata.oshdb.util.time.TimestampFormatter; import org.heigit.ohsome.filter.FilterExpression; import org.heigit.ohsome.ohsomeapi.Application; import org.heigit.ohsome.ohsomeapi.controller.rawdata.ElementsGeometry; @@ -34,7 +28,6 @@ import org.heigit.ohsome.ohsomeapi.oshdb.DbConnData; import org.heigit.ohsome.ohsomeapi.output.dataaggregationresponse.Metadata; import org.heigit.ohsome.ohsomeapi.output.rawdataresponse.DataResponse; -import org.locationtech.jts.geom.Geometry; import org.wololo.geojson.Feature; /** Holds executor methods for the following endpoints: /elementsFullHistory, /contributions. */ @@ -106,9 +99,7 @@ public void extract() throws Exception { final boolean isContributionsLatestEndpoint = requestResource.equals(RequestResource.CONTRIBUTIONSLATEST); final boolean isContributionsEndpoint = - (isContributionsLatestEndpoint || requestResource.equals(RequestResource.CONTRIBUTIONS)) - ? true - : false; + isContributionsLatestEndpoint || requestResource.equals(RequestResource.CONTRIBUTIONS); final Set simpleFeatureTypes = processingData.getSimpleFeatureTypes(); Optional filter = processingData.getFilterExpression(); final boolean requiresGeometryTypeCheck = @@ -119,103 +110,14 @@ public void extract() throws Exception { String endTimestamp = ISODateTimeParser.parseISODateTime(requestParameters.getTime()[1]) .format(DateTimeFormatter.ISO_DATE_TIME); MapReducer> mapRedContributions = mapRedContribution.groupByEntity(); - MapReducer contributionPreResult = mapRedContributions.flatMap(contributions -> { - List output = new LinkedList<>(); - Map properties; - Geometry currentGeom = null; - OSMEntity currentEntity = null; - String validFrom = null; - String validTo; - boolean skipNext = false; - if (!isContributionsLatestEndpoint) { - // first contribution: - OSMContribution firstContribution = contributions.get(0); - if (firstContribution.is(ContributionType.CREATION) && !isContributionsEndpoint) { - skipNext = true; - } else { - // if not "creation": take "before" as starting "row" (geom, tags), valid_from = t_start - currentEntity = firstContribution.getEntityBefore(); - currentGeom = exeUtils.getGeometry(firstContribution, clipGeometries, true); - validFrom = startTimestamp; - } - // then for each contribution: - for (int i = 0; i < contributions.size(); i++) { - if (i == contributions.size() - 1 && isContributionsEndpoint) { - // end the loop when last contribution is reached as it gets added later on - break; - } - OSMContribution contribution = contributions.get(i); - if (isContributionsEndpoint) { - currentEntity = contribution.getEntityAfter(); - currentGeom = exeUtils.getGeometry(contribution, clipGeometries, false); - validFrom = TimestampFormatter.getInstance().isoDateTime(contribution.getTimestamp()); - } - // set valid_to of previous row - validTo = TimestampFormatter.getInstance().isoDateTime(contribution.getTimestamp()); - if (!skipNext && currentGeom != null && !currentGeom.isEmpty()) { - boolean addToOutput = addEntityToOutput(processingData, utils, simpleFeatureTypes, - requiresGeometryTypeCheck, filterExpression, currentGeom, currentEntity); - if (addToOutput) { - properties = new TreeMap<>(); - if (!isContributionsEndpoint) { - properties.put("@validFrom", validFrom); - properties.put("@validTo", validTo); - } else { - properties.put("@timestamp", validTo); - } - output.add(exeUtils.createOSMFeature(currentEntity, currentGeom, properties, keysInt, - includeTags, includeOSMMetadata, isContributionsEndpoint, elementsGeometry, - contribution.getContributionTypes())); - } - } - skipNext = false; - if (contribution.is(ContributionType.DELETION)) { - // if deletion: skip output of next row - skipNext = true; - } else if (!isContributionsEndpoint) { - // else: take "after" as next row - currentEntity = contribution.getEntityAfter(); - currentGeom = exeUtils.getGeometry(contribution, clipGeometries, false); - validFrom = TimestampFormatter.getInstance().isoDateTime(contribution.getTimestamp()); - } - } - } - // after loop: - OSMContribution lastContribution = contributions.get(contributions.size() - 1); - currentGeom = exeUtils.getGeometry(lastContribution, clipGeometries, false); - currentEntity = lastContribution.getEntityAfter(); - if (!lastContribution.is(ContributionType.DELETION)) { - // if last contribution was not "deletion": set valid_to = t_end - validTo = endTimestamp; - properties = new TreeMap<>(); - if (!isContributionsEndpoint) { - properties.put("@validFrom", validFrom); - properties.put("@validTo", validTo); - } else { - properties.put("@timestamp", - TimestampFormatter.getInstance().isoDateTime(lastContribution.getTimestamp())); - } - if (!currentGeom.isEmpty()) { - boolean addToOutput = addEntityToOutput(processingData, utils, simpleFeatureTypes, - requiresGeometryTypeCheck, filterExpression, currentGeom, currentEntity); - if (addToOutput) { - output.add(exeUtils.createOSMFeature(currentEntity, currentGeom, properties, keysInt, - includeTags, includeOSMMetadata, isContributionsEndpoint, elementsGeometry, - lastContribution.getContributionTypes())); - } - } - } else if (isContributionsEndpoint) { - // adds the deletion feature for a /contributions request - currentGeom = exeUtils.getGeometry(lastContribution, clipGeometries, true); - properties = new TreeMap<>(); - properties.put("@timestamp", - TimestampFormatter.getInstance().isoDateTime(lastContribution.getTimestamp())); - output.add(exeUtils.createOSMFeature(currentEntity, currentGeom, properties, keysInt, false, - includeOSMMetadata, isContributionsEndpoint, elementsGeometry, - lastContribution.getContributionTypes())); - } - return output; - }).filter(Objects::nonNull); + final boolean isContainingSimpleFeatureTypes = processingData.isContainingSimpleFeatureTypes(); + FlatMapExecutor flatMapExecutor = new FlatMapExecutor(isContributionsLatestEndpoint, + isContributionsEndpoint, exeUtils, clipGeometries, startTimestamp, utils, + simpleFeatureTypes, requiresGeometryTypeCheck, filterExpression, keysInt, includeTags, + includeOSMMetadata, elementsGeometry, endTimestamp, isContainingSimpleFeatureTypes); + MapReducer contributionPreResult = mapRedContributions + .flatMap(flatMapExecutor::buildChangedFeatures) + .filter(Objects::nonNull); Metadata metadata = null; if (processingData.isShowMetadata()) { metadata = new Metadata(null, requestResource.getDescription(), @@ -226,34 +128,15 @@ public void extract() throws Exception { MapReducer snapshotPreResult = null; if (!isContributionsEndpoint) { // handles cases where valid_from = t_start, valid_to = t_end; i.e. non-modified data - snapshotPreResult = mapRedSnapshot.groupByEntity().filter(snapshots -> snapshots.size() == 2) + snapshotPreResult = mapRedSnapshot + .groupByEntity() + .filter(snapshots -> snapshots.size() == 2) .filter(snapshots -> snapshots.get(0).getGeometry() == snapshots.get(1).getGeometry() && snapshots.get(0).getEntity().getVersion() == snapshots.get(1).getEntity() .getVersion()) - .map(snapshots -> snapshots.get(0)).flatMap(snapshot -> { - Map properties = new TreeMap<>(); - OSMEntity entity = snapshot.getEntity(); - if (includeOSMMetadata) { - properties.put("@lastEdit", entity.getTimestamp().toString()); - } - Geometry geom = snapshot.getGeometry(); - if (!clipGeometries) { - geom = snapshot.getGeometryUnclipped(); - } - properties.put("@snapshotTimestamp", - TimestampFormatter.getInstance().isoDateTime(snapshot.getTimestamp())); - properties.put("@validFrom", startTimestamp); - properties.put("@validTo", endTimestamp); - boolean addToOutput = addEntityToOutput(processingData, utils, simpleFeatureTypes, - requiresGeometryTypeCheck, filterExpression, geom, entity); - if (addToOutput) { - return Collections.singletonList( - exeUtils.createOSMFeature(entity, geom, properties, keysInt, includeTags, - includeOSMMetadata, isContributionsEndpoint, elementsGeometry, null)); - } else { - return Collections.emptyList(); - } - }).filter(Objects::nonNull); + .map(snapshots -> snapshots.get(0)) + .flatMap(flatMapExecutor::buildUnchangedFeatures) + .filter(Objects::nonNull); } try ( Stream snapshotStream = @@ -263,20 +146,4 @@ public void extract() throws Exception { Stream.concat(contributionStream, snapshotStream)); } } - - /** Checks whether the given entity should be added to the output (true) or not (false). */ - private boolean addEntityToOutput(ProcessingData processingData, InputProcessingUtils utils, - final Set simpleFeatureTypes, final boolean requiresGeometryTypeCheck, - FilterExpression filterExpression, Geometry currentGeom, OSMEntity currentEntity) { - boolean addToOutput; - if (processingData.isContainingSimpleFeatureTypes()) { - addToOutput = utils.checkGeometryOnSimpleFeatures(currentGeom, simpleFeatureTypes); - } else if (requiresGeometryTypeCheck) { - addToOutput = filterExpression.applyOSMGeometry(currentEntity, currentGeom); - } else { - addToOutput = true; - } - return addToOutput; - } - } diff --git a/src/main/java/org/heigit/ohsome/ohsomeapi/executor/FlatMapExecutor.java b/src/main/java/org/heigit/ohsome/ohsomeapi/executor/FlatMapExecutor.java new file mode 100644 index 00000000..a8204207 --- /dev/null +++ b/src/main/java/org/heigit/ohsome/ohsomeapi/executor/FlatMapExecutor.java @@ -0,0 +1,205 @@ +package org.heigit.ohsome.ohsomeapi.executor; + +import java.io.Serializable; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import org.heigit.bigspatialdata.oshdb.api.object.OSMContribution; +import org.heigit.bigspatialdata.oshdb.api.object.OSMEntitySnapshot; +import org.heigit.bigspatialdata.oshdb.osm.OSMEntity; +import org.heigit.bigspatialdata.oshdb.util.celliterator.ContributionType; +import org.heigit.bigspatialdata.oshdb.util.time.TimestampFormatter; +import org.heigit.ohsome.filter.FilterExpression; +import org.heigit.ohsome.ohsomeapi.controller.rawdata.ElementsGeometry; +import org.heigit.ohsome.ohsomeapi.inputprocessing.InputProcessingUtils; +import org.heigit.ohsome.ohsomeapi.inputprocessing.SimpleFeatureType; +import org.locationtech.jts.geom.Geometry; +import org.wololo.geojson.Feature; + +public class FlatMapExecutor implements Serializable { + + private final boolean isContributionsLatestEndpoint; + private final boolean isContributionsEndpoint; + private final ExecutionUtils exeUtils; + private final boolean clipGeometries; + private final String startTimestamp; + private final InputProcessingUtils utils; + private final Set simpleFeatureTypes; + private final boolean requiresGeometryTypeCheck; + private final FilterExpression filterExpression; + private final int[] keysInt; + private final boolean includeTags; + private final boolean includeOSMMetadata; + private final ElementsGeometry elementsGeometry; + private final String endTimestamp; + private final boolean isContainingSimpleFeatureTypes; + + public FlatMapExecutor(boolean isContributionsLatestEndpoint, boolean isContributionsEndpoint, + ExecutionUtils exeUtils, boolean clipGeometries, String startTimestamp, + InputProcessingUtils utils, + Set simpleFeatureTypes, boolean requiresGeometryTypeCheck, + FilterExpression filterExpression, int[] keysInt, boolean includeTags, + boolean includeOSMMetadata, + ElementsGeometry elementsGeometry, String endTimestamp, + boolean isContainingSimpleFeatureTypes) { + this.isContributionsLatestEndpoint = isContributionsLatestEndpoint; + this.isContributionsEndpoint = isContributionsEndpoint; + this.exeUtils = exeUtils; + this.clipGeometries = clipGeometries; + this.startTimestamp = startTimestamp; + this.utils = utils; + this.simpleFeatureTypes = simpleFeatureTypes; + this.requiresGeometryTypeCheck = requiresGeometryTypeCheck; + this.filterExpression = filterExpression; + this.keysInt = keysInt; + this.includeTags = includeTags; + this.includeOSMMetadata = includeOSMMetadata; + this.elementsGeometry = elementsGeometry; + this.endTimestamp = endTimestamp; + this.isContainingSimpleFeatureTypes = isContainingSimpleFeatureTypes; + } + + public List buildChangedFeatures(List contributions) { + List output = new LinkedList<>(); + Map properties; + Geometry currentGeom = null; + OSMEntity currentEntity = null; + String validFrom = null; + String validTo; + boolean skipNext = false; + if (!isContributionsLatestEndpoint) { + // first contribution: + OSMContribution firstContribution = contributions.get(0); + if (firstContribution.is(ContributionType.CREATION) && !isContributionsEndpoint) { + skipNext = true; + } else { + // if not "creation": take "before" as starting "row" (geom, tags), valid_from = t_start + currentEntity = firstContribution.getEntityBefore(); + currentGeom = exeUtils.getGeometry(firstContribution, clipGeometries, true); + validFrom = startTimestamp; + } + // then for each contribution: + for (int i = 0; i < contributions.size(); i++) { + if (i == contributions.size() - 1 && isContributionsEndpoint) { + // end the loop when last contribution is reached as it gets added later on + break; + } + OSMContribution contribution = contributions.get(i); + if (isContributionsEndpoint) { + currentEntity = contribution.getEntityAfter(); + currentGeom = exeUtils.getGeometry(contribution, clipGeometries, false); + validFrom = TimestampFormatter.getInstance().isoDateTime(contribution.getTimestamp()); + } + // set valid_to of previous row + validTo = TimestampFormatter.getInstance().isoDateTime(contribution.getTimestamp()); + if (!skipNext && currentGeom != null && !currentGeom.isEmpty()) { + boolean addToOutput = addEntityToOutput(isContainingSimpleFeatureTypes, utils, + simpleFeatureTypes, requiresGeometryTypeCheck, filterExpression, currentGeom, + currentEntity); + if (addToOutput) { + properties = new TreeMap<>(); + if (!isContributionsEndpoint) { + properties.put("@validFrom", validFrom); + properties.put("@validTo", validTo); + } else { + properties.put("@timestamp", validTo); + } + output.add(exeUtils.createOSMFeature(currentEntity, currentGeom, properties, keysInt, + includeTags, includeOSMMetadata, isContributionsEndpoint, elementsGeometry, + contribution.getContributionTypes())); + } + } + skipNext = false; + if (contribution.is(ContributionType.DELETION)) { + // if deletion: skip output of next row + skipNext = true; + } else if (!isContributionsEndpoint) { + // else: take "after" as next row + currentEntity = contribution.getEntityAfter(); + currentGeom = exeUtils.getGeometry(contribution, clipGeometries, false); + validFrom = TimestampFormatter.getInstance().isoDateTime(contribution.getTimestamp()); + } + } + } + // after loop: + OSMContribution lastContribution = contributions.get(contributions.size() - 1); + currentGeom = exeUtils.getGeometry(lastContribution, clipGeometries, false); + currentEntity = lastContribution.getEntityAfter(); + if (!lastContribution.is(ContributionType.DELETION)) { + // if last contribution was not "deletion": set valid_to = t_end + validTo = endTimestamp; + properties = new TreeMap<>(); + if (!isContributionsEndpoint) { + properties.put("@validFrom", validFrom); + properties.put("@validTo", validTo); + } else { + properties.put("@timestamp", + TimestampFormatter.getInstance().isoDateTime(lastContribution.getTimestamp())); + } + if (!currentGeom.isEmpty()) { + boolean addToOutput = addEntityToOutput(isContainingSimpleFeatureTypes, utils, + simpleFeatureTypes, requiresGeometryTypeCheck, filterExpression, currentGeom, + currentEntity); + if (addToOutput) { + output.add(exeUtils.createOSMFeature(currentEntity, currentGeom, properties, keysInt, + includeTags, includeOSMMetadata, isContributionsEndpoint, elementsGeometry, + lastContribution.getContributionTypes())); + } + } + } else if (isContributionsEndpoint) { + // adds the deletion feature for a /contributions request + currentGeom = exeUtils.getGeometry(lastContribution, clipGeometries, true); + properties = new TreeMap<>(); + properties.put("@timestamp", + TimestampFormatter.getInstance().isoDateTime(lastContribution.getTimestamp())); + output.add(exeUtils.createOSMFeature(currentEntity, currentGeom, properties, keysInt, false, + includeOSMMetadata, isContributionsEndpoint, elementsGeometry, + lastContribution.getContributionTypes())); + } + return output; + } + + public List buildUnchangedFeatures(OSMEntitySnapshot snapshot) { + Map properties = new TreeMap<>(); + OSMEntity entity = snapshot.getEntity(); + if (includeOSMMetadata) { + properties.put("@lastEdit", entity.getTimestamp().toString()); + } + Geometry geom = snapshot.getGeometry(); + if (!clipGeometries) { + geom = snapshot.getGeometryUnclipped(); + } + properties.put("@snapshotTimestamp", + TimestampFormatter.getInstance().isoDateTime(snapshot.getTimestamp())); + properties.put("@validFrom", startTimestamp); + properties.put("@validTo", endTimestamp); + boolean addToOutput = addEntityToOutput(isContainingSimpleFeatureTypes, utils, + simpleFeatureTypes, requiresGeometryTypeCheck, filterExpression, geom, entity); + if (addToOutput) { + return Collections.singletonList( + exeUtils.createOSMFeature(entity, geom, properties, keysInt, includeTags, + includeOSMMetadata, isContributionsEndpoint, elementsGeometry, null)); + } else { + return Collections.emptyList(); + } + } + + /** Checks whether the given entity should be added to the output (true) or not (false). */ + public static boolean addEntityToOutput(boolean isContainingSimpleFeatureTypes, + InputProcessingUtils utils, + final Set simpleFeatureTypes, final boolean requiresGeometryTypeCheck, + FilterExpression filterExpression, Geometry currentGeom, OSMEntity currentEntity) { + boolean addToOutput; + if (isContainingSimpleFeatureTypes) { + addToOutput = utils.checkGeometryOnSimpleFeatures(currentGeom, simpleFeatureTypes); + } else if (requiresGeometryTypeCheck) { + addToOutput = filterExpression.applyOSMGeometry(currentEntity, currentGeom); + } else { + addToOutput = true; + } + return addToOutput; + } +} From 3d8be75e6c16a9f5b7403b604e6c5793661934ec Mon Sep 17 00:00:00 2001 From: Johannes Visintini Date: Tue, 10 Nov 2020 18:07:18 +0100 Subject: [PATCH 2/2] add changelog for 1.2.1 --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e03f9567..14de2784 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ ## current master +## 1.2.1 + +### Bug Fixes + +* fixing bug Ignite trying to load non-serializable objects within lambda functions ([#75](https://github.com/GIScience/ohsome-api/pull/75)) ## 1.2.0