From b8807b4007f9f7c45411bae6ea4d1dc102b1c8ce Mon Sep 17 00:00:00 2001 From: Alex Albu Date: Tue, 21 Jul 2020 14:33:44 -0400 Subject: [PATCH] Ignore ES indexes without mappings --- .../elasticsearch/ElasticsearchMetadata.java | 11 +++- .../client/ElasticsearchClient.java | 32 +++++++--- .../BaseElasticsearchSmokeTest.java | 58 +++++++++++++++++++ 3 files changed, 90 insertions(+), 11 deletions(-) diff --git a/presto-elasticsearch/src/main/java/io/prestosql/elasticsearch/ElasticsearchMetadata.java b/presto-elasticsearch/src/main/java/io/prestosql/elasticsearch/ElasticsearchMetadata.java index 7d81af5d981d..a400a8d4aaff 100644 --- a/presto-elasticsearch/src/main/java/io/prestosql/elasticsearch/ElasticsearchMetadata.java +++ b/presto-elasticsearch/src/main/java/io/prestosql/elasticsearch/ElasticsearchMetadata.java @@ -17,6 +17,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.io.BaseEncoding; import io.airlift.json.ObjectMapperProvider; import io.prestosql.elasticsearch.client.ElasticsearchClient; @@ -53,6 +54,7 @@ import java.util.Map; import java.util.Optional; import java.util.OptionalLong; +import java.util.Set; import java.util.stream.Collectors; import static com.google.common.collect.ImmutableMap.toImmutableMap; @@ -339,13 +341,16 @@ public List listTables(ConnectorSession session, Optional result = ImmutableList.builder(); + Set indexes = ImmutableSet.copyOf(client.getIndexes()); - client.getIndexes().stream() + indexes.stream() .map(index -> new SchemaTableName(this.schemaName, index)) .forEach(result::add); - client.getAliases().stream() - .map(index -> new SchemaTableName(this.schemaName, index)) + client.getAliases().entrySet().stream() + .filter(entry -> indexes.contains(entry.getKey())) + .flatMap(entry -> entry.getValue().stream() + .map(alias -> new SchemaTableName(this.schemaName, alias))) .forEach(result::add); return result.build(); diff --git a/presto-elasticsearch/src/main/java/io/prestosql/elasticsearch/client/ElasticsearchClient.java b/presto-elasticsearch/src/main/java/io/prestosql/elasticsearch/client/ElasticsearchClient.java index 534eef149c97..1cb1ef68040f 100644 --- a/presto-elasticsearch/src/main/java/io/prestosql/elasticsearch/client/ElasticsearchClient.java +++ b/presto-elasticsearch/src/main/java/io/prestosql/elasticsearch/client/ElasticsearchClient.java @@ -447,12 +447,22 @@ private int shardPreference(SearchShardsResponse.Shard left, SearchShardsRespons public List getIndexes() { - return doRequest("/_cat/indices?h=index&format=json&s=index:asc", body -> { + return doRequest("/_cat/indices?h=index,docs.count,docs.deleted&format=json&s=index:asc", body -> { try { ImmutableList.Builder result = ImmutableList.builder(); JsonNode root = OBJECT_MAPPER.readTree(body); for (int i = 0; i < root.size(); i++) { - result.add(root.get(i).get("index").asText()); + String index = root.get(i).get("index").asText(); + // make sure the index has mappings we can use to derive the schema + int docsCount = root.get(i).get("docs.count").asInt(); + int deletedDocsCount = root.get(i).get("docs.deleted").asInt(); + if (docsCount == 0 && deletedDocsCount == 0) { + // without documents, the index won't have any dynamic mappings, but maybe there are some explicit ones + if (getIndexMetadata(index).getSchema().getFields().isEmpty()) { + continue; + } + } + result.add(index); } return result.build(); } @@ -462,18 +472,21 @@ public List getIndexes() }); } - public List getAliases() + public Map> getAliases() { return doRequest("/_aliases", body -> { try { - ImmutableList.Builder result = ImmutableList.builder(); + ImmutableMap.Builder> result = ImmutableMap.builder(); JsonNode root = OBJECT_MAPPER.readTree(body); - Iterator elements = root.elements(); + Iterator> elements = root.fields(); while (elements.hasNext()) { - JsonNode element = elements.next(); - JsonNode aliases = element.get("aliases"); - result.addAll(aliases.fieldNames()); + Map.Entry element = elements.next(); + JsonNode aliases = element.getValue().get("aliases"); + Iterator aliasNames = aliases.fieldNames(); + if (aliasNames.hasNext()) { + result.put(element.getKey(), ImmutableList.copyOf(aliasNames)); + } } return result.build(); } @@ -493,6 +506,9 @@ public IndexMetadata getIndexMetadata(String index) .elements().next() .get("mappings"); + if (!mappings.elements().hasNext()) { + return new IndexMetadata(new IndexMetadata.ObjectType(ImmutableList.of())); + } if (!mappings.has("properties")) { // Older versions of ElasticSearch supported multiple "type" mappings // for a given index. Newer versions support only one and don't diff --git a/presto-elasticsearch/src/test/java/io/prestosql/elasticsearch/BaseElasticsearchSmokeTest.java b/presto-elasticsearch/src/test/java/io/prestosql/elasticsearch/BaseElasticsearchSmokeTest.java index afca7a21c82c..a6bdcefb7a4b 100644 --- a/presto-elasticsearch/src/test/java/io/prestosql/elasticsearch/BaseElasticsearchSmokeTest.java +++ b/presto-elasticsearch/src/test/java/io/prestosql/elasticsearch/BaseElasticsearchSmokeTest.java @@ -43,6 +43,8 @@ import static java.lang.String.format; import static java.nio.charset.StandardCharsets.UTF_8; import static org.assertj.core.api.Assertions.assertThat; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; public abstract class BaseElasticsearchSmokeTest extends AbstractTestIntegrationSmokeTest @@ -818,6 +820,56 @@ public void testPassthroughQuery() "Elasticsearch query for 'orders' is not valid JSON"); } + @Test + public void testEmptyIndexWithMappings() + throws IOException + { + String indexName = "test_empty_index_with_mappings"; + + @Language("JSON") + String mappings = "" + + "{" + + " \"properties\": { " + + " \"dummy_column\": { \"type\": \"long\" }" + + " }" + + "}"; + + createIndex(indexName, mappings); + + assertQuery(format("SELECT column_name FROM information_schema.columns WHERE table_name = '%s'", indexName), "VALUES ('dummy_column')"); + assertTrue(computeActual("SHOW TABLES").getOnlyColumnAsSet().contains(indexName)); + assertQueryReturnsEmptyResult("SELECT * FROM " + indexName); + } + + @Test + public void testEmptyIndexNoMappings() + throws IOException + { + String indexName = "test_empty_index"; + + createIndex(indexName); + assertTableDoesNotExist(indexName); + } + + @Test + public void testEmptyAliasNoMappings() + throws IOException + { + String indexName = "test_empty_index_for_alias"; + String aliasName = "test_empty_alias"; + + createIndex(indexName); + addAlias(indexName, aliasName); + assertTableDoesNotExist(aliasName); + } + + private void assertTableDoesNotExist(String name) + { + assertQueryReturnsEmptyResult(format("SELECT * FROM information_schema.columns WHERE table_name = '%s'", name)); + assertFalse(computeActual("SHOW TABLES").getOnlyColumnAsSet().contains(name)); + assertQueryFails("SELECT * FROM " + name, ".*Table 'elasticsearch.tpch." + name + "' does not exist"); + } + protected abstract String indexEndpoint(String index, String docId); private void index(String index, Map document) @@ -840,6 +892,12 @@ private void addAlias(String index, String alias) protected abstract String indexMapping(@Language("JSON") String properties); + private void createIndex(String indexName) + throws IOException + { + client.getLowLevelClient().performRequest("PUT", "/" + indexName); + } + private void createIndex(String indexName, @Language("JSON") String properties) throws IOException {