Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Various minor code cleanups #520

Merged
merged 5 commits into from
Dec 23, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 14 additions & 14 deletions src/main/java/de/komoot/photon/PhotonDoc.java
Original file line number Diff line number Diff line change
Expand Up @@ -18,21 +18,21 @@
@Getter
@Slf4j
public class PhotonDoc {
final private long placeId;
final private String osmType;
final private long osmId;
final private String tagKey;
final private String tagValue;
final private Map<String, String> name;
private final long placeId;
private final String osmType;
private final long osmId;
private final String tagKey;
private final String tagValue;
private final Map<String, String> name;
private String postcode;
final private Map<String, String> address;
final private Map<String, String> extratags;
final private Envelope bbox;
final private long parentPlaceId; // 0 if unset
final private double importance;
final private CountryCode countryCode;
final private long linkedPlaceId; // 0 if unset
final private int rankAddress;
private final Map<String, String> address;
private final Map<String, String> extratags;
private final Envelope bbox;
private final long parentPlaceId; // 0 if unset
private final double importance;
private final CountryCode countryCode;
private final long linkedPlaceId; // 0 if unset
private final int rankAddress;

private Map<AddressType, Map<String, String>> addressParts = new EnumMap<>(AddressType.class);
private Set<Map<String, String>> context = new HashSet<>();
Expand Down
11 changes: 1 addition & 10 deletions src/main/java/de/komoot/photon/elasticsearch/Importer.java
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,6 @@
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.Requests;
import org.elasticsearch.search.builder.SearchSourceBuilder;

import java.io.IOException;

Expand All @@ -20,8 +18,6 @@
public class Importer implements de.komoot.photon.Importer {
private int documentCount = 0;

private final String indexName = "photon";
private final String indexType = "place";
private final Client esClient;
private BulkRequestBuilder bulkRequest;
private final String[] languages;
Expand All @@ -35,7 +31,7 @@ public Importer(Client esClient, String languages) {
@Override
public void add(PhotonDoc doc) {
try {
this.bulkRequest.add(this.esClient.prepareIndex(indexName, indexType).
this.bulkRequest.add(this.esClient.prepareIndex(PhotonIndex.NAME, PhotonIndex.TYPE).
setSource(Utils.convert(doc, languages)).setId(doc.getUid()));
} catch (IOException e) {
log.error("could not bulk add document " + doc.getUid(), e);
Expand All @@ -62,9 +58,4 @@ public void finish() {
this.saveDocuments();
this.documentCount = 0;
}

public long count() {
return this.esClient.search(Requests.searchRequest(indexName).types(indexType).source(SearchSourceBuilder.searchSource().size(0))).actionGet().getHits()
.getTotalHits();
}
}
11 changes: 11 additions & 0 deletions src/main/java/de/komoot/photon/elasticsearch/PhotonIndex.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
package de.komoot.photon.elasticsearch;

/**
* Constants describing the index used for the photon database.
*/
public class PhotonIndex {
private PhotonIndex() {}

public static final String NAME = "photon";
public static final String TYPE = "place";
}
16 changes: 8 additions & 8 deletions src/main/java/de/komoot/photon/elasticsearch/Server.java
Original file line number Diff line number Diff line change
Expand Up @@ -183,29 +183,29 @@ public void recreateIndex() throws IOException {
final Client client = this.getClient();
final InputStream mappings = Thread.currentThread().getContextClassLoader()
.getResourceAsStream("mappings.json");
final InputStream index_settings = Thread.currentThread().getContextClassLoader()
final InputStream indexSettings = Thread.currentThread().getContextClassLoader()
.getResourceAsStream("index_settings.json");
final Charset utf8_charset = Charset.forName("utf-8");
final Charset utf8Charset = Charset.forName("utf-8");

String mappingsString = IOUtils.toString(mappings, utf8_charset);
String mappingsString = IOUtils.toString(mappings, utf8Charset);
JSONObject mappingsJSON = new JSONObject(mappingsString);

// add all langs to the mapping
mappingsJSON = addLangsToMapping(mappingsJSON);

JSONObject settings = new JSONObject(IOUtils.toString(index_settings, utf8_charset));
JSONObject settings = new JSONObject(IOUtils.toString(indexSettings, utf8Charset));
if (shards != null) {
settings.put("index", new JSONObject("{ \"number_of_shards\":" + shards + " }"));
}
client.admin().indices().prepareCreate("photon").setSettings(settings.toString(), XContentType.JSON).execute().actionGet();
;
client.admin().indices().preparePutMapping("photon").setType("place").setSource(mappingsJSON.toString(), XContentType.JSON).execute().actionGet();
client.admin().indices().prepareCreate(PhotonIndex.NAME).setSettings(settings.toString(), XContentType.JSON).execute().actionGet();

client.admin().indices().preparePutMapping(PhotonIndex.NAME).setType(PhotonIndex.TYPE).setSource(mappingsJSON.toString(), XContentType.JSON).execute().actionGet();
log.info("mapping created: " + mappingsJSON.toString());
}

public void deleteIndex() {
try {
this.getClient().admin().indices().prepareDelete("photon").execute().actionGet();
this.getClient().admin().indices().prepareDelete(PhotonIndex.NAME).execute().actionGet();
} catch (IndexNotFoundException e) {
// ignore
}
Expand Down
8 changes: 4 additions & 4 deletions src/main/java/de/komoot/photon/elasticsearch/Updater.java
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ public void finish() {

@Override
public void updateOrCreate(PhotonDoc updatedDoc) {
final boolean exists = this.esClient.get(this.esClient.prepareGet("photon", "place", String.valueOf(updatedDoc.getPlaceId())).request()).actionGet().isExists();
final boolean exists = this.esClient.get(this.esClient.prepareGet(PhotonIndex.NAME, PhotonIndex.TYPE, String.valueOf(updatedDoc.getPlaceId())).request()).actionGet().isExists();
if (exists) {
this.update(updatedDoc);
} else {
Expand All @@ -42,22 +42,22 @@ public void updateOrCreate(PhotonDoc updatedDoc) {

public void create(PhotonDoc doc) {
try {
this.bulkRequest.add(this.esClient.prepareIndex("photon", "place").setSource(Utils.convert(doc, this.languages)).setId(String.valueOf(doc.getPlaceId())));
this.bulkRequest.add(this.esClient.prepareIndex(PhotonIndex.NAME, PhotonIndex.TYPE).setSource(Utils.convert(doc, this.languages)).setId(String.valueOf(doc.getPlaceId())));
} catch (IOException e) {
log.error(String.format("creation of new doc [%s] failed", doc), e);
}
}

public void update(PhotonDoc doc) {
try {
this.bulkRequest.add(this.esClient.prepareUpdate("photon", "place", String.valueOf(doc.getPlaceId())).setDoc(Utils.convert(doc, this.languages)));
this.bulkRequest.add(this.esClient.prepareUpdate(PhotonIndex.NAME, PhotonIndex.TYPE, String.valueOf(doc.getPlaceId())).setDoc(Utils.convert(doc, this.languages)));
} catch (IOException e) {
log.error(String.format("update of new doc [%s] failed", doc), e);
}
}

public void delete(Long id) {
this.bulkRequest.add(this.esClient.prepareDelete("photon", "place", String.valueOf(id)));
this.bulkRequest.add(this.esClient.prepareDelete(PhotonIndex.NAME, PhotonIndex.TYPE, String.valueOf(id)));
}

private void updateDocuments() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
import de.komoot.photon.Updater;
import de.komoot.photon.nominatim.model.UpdateRow;
import org.apache.commons.dbcp2.BasicDataSource;
import org.postgis.jts.JtsWrapper;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowMapper;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ public Map<String, String> getMap(ResultSet rs, String columnName) throws SQLExc
public Geometry extractGeometry(ResultSet rs, String columnName) throws SQLException {
JtsGeometry geom = (JtsGeometry) rs.getObject(columnName);
if (geom == null) {
//info("no geometry found in column " + columnName);
return null;
}
return geom.getGeometry();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
* Created by Holger Bruch on 10/13/2018.
*/
public class LocationParamConverter {
private final static GeometryFactory geometryFactory = new GeometryFactory(new PrecisionModel(), 4326);
private static final GeometryFactory geometryFactory = new GeometryFactory(new PrecisionModel(), 4326);
private boolean mandatory;

public LocationParamConverter(boolean mandatory) {
Expand Down
34 changes: 17 additions & 17 deletions src/main/java/de/komoot/photon/query/PhotonQueryBuilder.java
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,9 @@
* Created by Sachin Dole on 2/12/2015.
*/
public class PhotonQueryBuilder {
private FunctionScoreQueryBuilder m_finalQueryWithoutTagFilterBuilder;
private FunctionScoreQueryBuilder finalQueryWithoutTagFilterBuilder;

private BoolQueryBuilder m_queryBuilderForTopLevelFilter;
private BoolQueryBuilder queryBuilderForTopLevelFilter;

private State state;

Expand All @@ -50,11 +50,11 @@ public class PhotonQueryBuilder {

private GeoBoundingBoxQueryBuilder bboxQueryBuilder;

private BoolQueryBuilder m_finalQueryBuilder;
private BoolQueryBuilder finalQueryBuilder;

protected ArrayList<FilterFunctionBuilder> m_alFilterFunction4QueryBuilder = new ArrayList<>(1);
protected ArrayList<FilterFunctionBuilder> alFilterFunction4QueryBuilder = new ArrayList<>(1);

protected QueryBuilder m_query4QueryBuilder;
protected QueryBuilder query4QueryBuilder;


private PhotonQueryBuilder(String query, String language) {
Expand All @@ -65,7 +65,7 @@ private PhotonQueryBuilder(String query, String language) {
.analyzer("search_ngram").minimumShouldMatch("100%");

// @formatter:off
m_query4QueryBuilder = QueryBuilders.boolQuery()
query4QueryBuilder = QueryBuilders.boolQuery()
.must(QueryBuilders.boolQuery().should(defaultMatchQueryBuilder).should(languageMatchQueryBuilder)
.minimumShouldMatch("1"))
.should(QueryBuilders.matchQuery(String.format("name.%s.raw", language), query).boost(200)
Expand All @@ -79,13 +79,13 @@ private PhotonQueryBuilder(String query, String language) {
ScriptScoreFunctionBuilder functionBuilder4QueryBuilder =
ScoreFunctionBuilders.scriptFunction(new Script(ScriptType.INLINE, "painless", strCode, new HashMap<String, Object>()));

m_alFilterFunction4QueryBuilder.add(new FilterFunctionBuilder(functionBuilder4QueryBuilder));
alFilterFunction4QueryBuilder.add(new FilterFunctionBuilder(functionBuilder4QueryBuilder));

m_finalQueryWithoutTagFilterBuilder = new FunctionScoreQueryBuilder(m_query4QueryBuilder, m_alFilterFunction4QueryBuilder.toArray(new FilterFunctionBuilder[0]))
finalQueryWithoutTagFilterBuilder = new FunctionScoreQueryBuilder(query4QueryBuilder, alFilterFunction4QueryBuilder.toArray(new FilterFunctionBuilder[0]))
.boostMode(CombineFunction.MULTIPLY).scoreMode(ScoreMode.MULTIPLY);

// @formatter:off
m_queryBuilderForTopLevelFilter = QueryBuilders.boolQuery()
queryBuilderForTopLevelFilter = QueryBuilders.boolQuery()
.should(QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery("housenumber")))
.should(QueryBuilders.matchQuery("housenumber", query).analyzer("standard"))
.should(QueryBuilders.existsQuery(String.format("name.%s.raw", language)));
Expand Down Expand Up @@ -117,9 +117,9 @@ public PhotonQueryBuilder withLocationBias(Point point, double scale) {
"double score = 0.1 + " + scale + " / (1.0 + dist * 0.001 / 10.0); " +
"score";
ScriptScoreFunctionBuilder builder = ScoreFunctionBuilders.scriptFunction(new Script(ScriptType.INLINE, "painless", strCode, params));
m_alFilterFunction4QueryBuilder.add(new FilterFunctionBuilder(builder));
m_finalQueryWithoutTagFilterBuilder =
new FunctionScoreQueryBuilder(m_query4QueryBuilder, m_alFilterFunction4QueryBuilder.toArray(new FilterFunctionBuilder[0]))
alFilterFunction4QueryBuilder.add(new FilterFunctionBuilder(builder));
finalQueryWithoutTagFilterBuilder =
new FunctionScoreQueryBuilder(query4QueryBuilder, alFilterFunction4QueryBuilder.toArray(new FilterFunctionBuilder[0]))
.boostMode(CombineFunction.MULTIPLY);
return this;
}
Expand Down Expand Up @@ -286,25 +286,25 @@ public PhotonQueryBuilder withLenientMatch() {
* have no effect.
*/
public QueryBuilder buildQuery() {
if (state.equals(State.FINISHED)) return m_finalQueryBuilder;
if (state.equals(State.FINISHED)) return finalQueryBuilder;

m_finalQueryBuilder = QueryBuilders.boolQuery().must(m_finalQueryWithoutTagFilterBuilder).filter(m_queryBuilderForTopLevelFilter);
finalQueryBuilder = QueryBuilders.boolQuery().must(finalQueryWithoutTagFilterBuilder).filter(queryBuilderForTopLevelFilter);

if (state.equals(State.FILTERED)) {
BoolQueryBuilder tagFilters = QueryBuilders.boolQuery();
if (orQueryBuilderForIncludeTagFiltering != null)
tagFilters.must(orQueryBuilderForIncludeTagFiltering);
if (andQueryBuilderForExcludeTagFiltering != null)
tagFilters.must(andQueryBuilderForExcludeTagFiltering);
m_finalQueryBuilder.filter(tagFilters);
finalQueryBuilder.filter(tagFilters);
}

if (bboxQueryBuilder != null)
m_queryBuilderForTopLevelFilter.filter(bboxQueryBuilder);
queryBuilderForTopLevelFilter.filter(bboxQueryBuilder);

state = State.FINISHED;

return m_finalQueryBuilder;
return finalQueryBuilder;
}


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,10 @@
*/
public class PhotonRequestFactory {
private final RequestLanguageResolver languageResolver;
private final static LocationParamConverter optionalLocationParamConverter = new LocationParamConverter(false);
private static final LocationParamConverter optionalLocationParamConverter = new LocationParamConverter(false);
private final BoundingBoxParamConverter bboxParamConverter;

protected static HashSet<String> m_hsRequestQueryParams = new HashSet<>(Arrays.asList("lang", "q", "lon", "lat",
private static final HashSet<String> REQUEST_QUERY_PARAMS = new HashSet<>(Arrays.asList("lang", "q", "lon", "lat",
"limit", "osm_tag", "location_bias_scale", "bbox", "debug"));

public PhotonRequestFactory(List<String> supportedLanguages, String defaultLanguage) {
Expand All @@ -30,8 +30,8 @@ public PhotonRequest create(Request webRequest) throws BadRequestException {


for (String queryParam : webRequest.queryParams())
if (!m_hsRequestQueryParams.contains(queryParam))
throw new BadRequestException(400, "unknown query parameter '" + queryParam + "'. Allowed parameters are: " + m_hsRequestQueryParams);
if (!REQUEST_QUERY_PARAMS.contains(queryParam))
throw new BadRequestException(400, "unknown query parameter '" + queryParam + "'. Allowed parameters are: " + REQUEST_QUERY_PARAMS);

String language = languageResolver.resolveRequestedLanguage(webRequest);

Expand Down
13 changes: 6 additions & 7 deletions src/main/java/de/komoot/photon/query/ReverseRequestFactory.java
Original file line number Diff line number Diff line change
Expand Up @@ -12,18 +12,18 @@
*/
public class ReverseRequestFactory {
private final RequestLanguageResolver languageResolver;
private final static LocationParamConverter mandatoryLocationParamConverter = new LocationParamConverter(true);
private static final LocationParamConverter mandatoryLocationParamConverter = new LocationParamConverter(true);

protected static HashSet<String> m_hsRequestQueryParams = new HashSet<>(Arrays.asList("lang", "lon", "lat", "radius", "query_string_filter", "distance_sort", "limit"));
private static final HashSet<String> REQUEST_QUERY_PARAMS = new HashSet<>(Arrays.asList("lang", "lon", "lat", "radius", "query_string_filter", "distance_sort", "limit"));

public ReverseRequestFactory(List<String> supportedLanguages, String defaultLanguage) {
this.languageResolver = new RequestLanguageResolver(supportedLanguages, defaultLanguage);
}

public <R extends ReverseRequest> R create(Request webRequest) throws BadRequestException {
public ReverseRequest create(Request webRequest) throws BadRequestException {
for (String queryParam : webRequest.queryParams()) {
if (!m_hsRequestQueryParams.contains(queryParam))
throw new BadRequestException(400, "unknown query parameter '" + queryParam + "'. Allowed parameters are: " + m_hsRequestQueryParams);
if (!REQUEST_QUERY_PARAMS.contains(queryParam))
throw new BadRequestException(400, "unknown query parameter '" + queryParam + "'. Allowed parameters are: " + REQUEST_QUERY_PARAMS);
}

String language = languageResolver.resolveRequestedLanguage(webRequest);
Expand Down Expand Up @@ -70,7 +70,6 @@ public <R extends ReverseRequest> R create(Request webRequest) throws BadRequest
}

String queryStringFilter = webRequest.queryParams("query_string_filter");
ReverseRequest reverseRequest = new ReverseRequest(location, language, radius, queryStringFilter, limit, locationDistanceSort);
return (R) reverseRequest;
return new ReverseRequest(location, language, radius, queryStringFilter, limit, locationDistanceSort);
}
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package de.komoot.photon.searcher;

import de.komoot.photon.elasticsearch.PhotonIndex;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.client.Client;
Expand All @@ -19,7 +20,7 @@ public BaseElasticsearchSearcher(Client client) {

public SearchResponse search(QueryBuilder queryBuilder, Integer limit) {
TimeValue timeout = TimeValue.timeValueSeconds(7);
return client.prepareSearch("photon").
return client.prepareSearch(PhotonIndex.NAME).
setSearchType(SearchType.QUERY_AND_FETCH).
setQuery(queryBuilder).
setSize(limit).
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package de.komoot.photon.searcher;

import com.vividsolutions.jts.geom.Point;
import de.komoot.photon.elasticsearch.PhotonIndex;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
Expand All @@ -25,7 +26,7 @@ public SearchResponse search(QueryBuilder queryBuilder, Integer limit, Point loc
Boolean locationDistanceSort) {
TimeValue timeout = TimeValue.timeValueSeconds(7);

SearchRequestBuilder builder = client.prepareSearch("photon").setSearchType(SearchType.QUERY_AND_FETCH)
SearchRequestBuilder builder = client.prepareSearch(PhotonIndex.NAME).setSearchType(SearchType.QUERY_AND_FETCH)
.setQuery(queryBuilder).setSize(limit).setTimeout(timeout);

if (locationDistanceSort)
Expand Down
6 changes: 3 additions & 3 deletions src/main/java/de/komoot/photon/utils/ConvertToJson.java
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@
*/
@Slf4j
public class ConvertToJson {
private final static String[] KEYS_LANG_UNSPEC = {Constants.OSM_ID, Constants.OSM_VALUE, Constants.OSM_KEY, Constants.POSTCODE, Constants.HOUSENUMBER, Constants.COUNTRYCODE, Constants.OSM_TYPE};
private final static String[] KEYS_LANG_SPEC = {Constants.NAME, Constants.COUNTRY, Constants.CITY, Constants.DISTRICT, Constants.LOCALITY, Constants.STREET, Constants.STATE, Constants.COUNTY};
private final static String[] NAME_PRECEDENCE = {"default", "housename", "int", "loc", "reg", "alt", "old"};
private static final String[] KEYS_LANG_UNSPEC = {Constants.OSM_ID, Constants.OSM_VALUE, Constants.OSM_KEY, Constants.POSTCODE, Constants.HOUSENUMBER, Constants.COUNTRYCODE, Constants.OSM_TYPE};
private static final String[] KEYS_LANG_SPEC = {Constants.NAME, Constants.COUNTRY, Constants.CITY, Constants.DISTRICT, Constants.LOCALITY, Constants.STREET, Constants.STATE, Constants.COUNTY};
private static final String[] NAME_PRECEDENCE = {"default", "housename", "int", "loc", "reg", "alt", "old"};
private final String lang;

public ConvertToJson(String lang) {
Expand Down
Loading