diff --git a/graylog2-rest-client/src/main/java/org/graylog2/restclient/models/Index.java b/graylog2-rest-client/src/main/java/org/graylog2/restclient/models/Index.java index 4756f83a7415..94f1dc6bd10c 100644 --- a/graylog2-rest-client/src/main/java/org/graylog2/restclient/models/Index.java +++ b/graylog2-rest-client/src/main/java/org/graylog2/restclient/models/Index.java @@ -27,7 +27,6 @@ import org.graylog2.restclient.models.api.responses.system.indices.ShardRoutingResponse; import org.graylog2.restroutes.generated.routes; import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -200,30 +199,28 @@ public ShardMeterResponse getRefreshMeter() { public static class Range { - private final DateTime starts; - private final boolean providesCalculationInfo; - - private long calculationTookMs = 0; - private DateTime calculatedAt = null; + private final DateTime begin; + private final DateTime end; + private final long calculationTookMs; + private final DateTime calculatedAt; public Range(IndexRangeSummary ir) { - this.starts = new DateTime(ir.start(), DateTimeZone.UTC); - - if (ir.calculatedAt() != null && ir.calculationTookMs() >= 0) { - this.providesCalculationInfo = true; + this.begin = ir.begin(); + this.end = ir.end(); this.calculationTookMs = ir.calculationTookMs(); - this.calculatedAt = new DateTime(ir.calculatedAt(), DateTimeZone.UTC); - } else { - this.providesCalculationInfo = false; - } + this.calculatedAt = ir.calculatedAt(); + } + + public DateTime getBegin() { + return begin; } - public DateTime getStarts() { - return starts; + public DateTime getEnd() { + return end; } public boolean isProvidesCalculationInfo() { - return providesCalculationInfo; + return true; } public long getCalculationTookMs() { diff --git a/graylog2-rest-models/src/main/java/org/graylog2/rest/models/system/indexer/responses/IndexRangeSummary.java b/graylog2-rest-models/src/main/java/org/graylog2/rest/models/system/indexer/responses/IndexRangeSummary.java index 72d35a77be14..6cd257832a2d 100644 --- a/graylog2-rest-models/src/main/java/org/graylog2/rest/models/system/indexer/responses/IndexRangeSummary.java +++ b/graylog2-rest-models/src/main/java/org/graylog2/rest/models/system/indexer/responses/IndexRangeSummary.java @@ -27,23 +27,27 @@ @AutoValue @JsonAutoDetect public abstract class IndexRangeSummary { - @JsonProperty("index") + @JsonProperty("index_name") public abstract String indexName(); + @JsonProperty("begin") + public abstract DateTime begin(); + + @JsonProperty("end") + public abstract DateTime end(); + @Nullable @JsonProperty("calculated_at") public abstract DateTime calculatedAt(); - @JsonProperty("starts") - public abstract DateTime start(); - - @JsonProperty("calculation_took_ms") + @JsonProperty("took_ms") public abstract int calculationTookMs(); @JsonCreator - public static IndexRangeSummary create(@JsonProperty("index") String indexName, + public static IndexRangeSummary create(@JsonProperty("index_name") String indexName, + @JsonProperty("begin") DateTime begin, + @JsonProperty("end") DateTime end, @Nullable @JsonProperty("calculated_at") DateTime calculatedAt, - @JsonProperty("starts") DateTime start, - @JsonProperty("calculation_took_ms") int calculationTookMs) { - return new AutoValue_IndexRangeSummary(indexName, calculatedAt, start, calculationTookMs); + @JsonProperty("took_ms") int calculationTookMs) { + return new AutoValue_IndexRangeSummary(indexName, begin, end, calculatedAt, calculationTookMs); } } diff --git a/graylog2-server/src/main/java/org/graylog2/bindings/PersistenceServicesBindings.java b/graylog2-server/src/main/java/org/graylog2/bindings/PersistenceServicesBindings.java index 2456443778ee..4bd682aed024 100644 --- a/graylog2-server/src/main/java/org/graylog2/bindings/PersistenceServicesBindings.java +++ b/graylog2-server/src/main/java/org/graylog2/bindings/PersistenceServicesBindings.java @@ -17,20 +17,20 @@ package org.graylog2.bindings; import com.google.inject.AbstractModule; -import org.graylog2.collectors.CollectorService; -import org.graylog2.collectors.CollectorServiceImpl; import org.graylog2.alerts.AlertService; import org.graylog2.alerts.AlertServiceImpl; import org.graylog2.cluster.NodeService; import org.graylog2.cluster.NodeServiceImpl; +import org.graylog2.collectors.CollectorService; +import org.graylog2.collectors.CollectorServiceImpl; import org.graylog2.dashboards.DashboardService; import org.graylog2.dashboards.DashboardServiceImpl; import org.graylog2.indexer.IndexFailureService; import org.graylog2.indexer.IndexFailureServiceImpl; import org.graylog2.indexer.PersistedDeadLetterService; import org.graylog2.indexer.PersistedDeadLetterServiceImpl; +import org.graylog2.indexer.ranges.EsIndexRangeService; import org.graylog2.indexer.ranges.IndexRangeService; -import org.graylog2.indexer.ranges.IndexRangeServiceImpl; import org.graylog2.inputs.InputService; import org.graylog2.inputs.InputServiceImpl; import org.graylog2.notifications.NotificationService; @@ -52,9 +52,6 @@ import org.graylog2.system.activities.SystemMessageServiceImpl; import org.graylog2.users.UserServiceImpl; -/** - * @author Dennis Oelkers - */ public class PersistenceServicesBindings extends AbstractModule { @Override protected void configure() { @@ -65,7 +62,7 @@ protected void configure() { bind(PersistedDeadLetterService.class).to(PersistedDeadLetterServiceImpl.class); bind(IndexFailureService.class).to(IndexFailureServiceImpl.class); bind(NodeService.class).to(NodeServiceImpl.class); - bind(IndexRangeService.class).to(IndexRangeServiceImpl.class); + bind(IndexRangeService.class).to(EsIndexRangeService.class); bind(InputService.class).to(InputServiceImpl.class); bind(StreamRuleService.class).to(StreamRuleServiceImpl.class); bind(UserService.class).to(UserServiceImpl.class); diff --git a/graylog2-server/src/main/java/org/graylog2/bindings/ServerObjectMapperModule.java b/graylog2-server/src/main/java/org/graylog2/bindings/ServerObjectMapperModule.java index a4729b34e9fc..9738dacb547a 100644 --- a/graylog2-server/src/main/java/org/graylog2/bindings/ServerObjectMapperModule.java +++ b/graylog2-server/src/main/java/org/graylog2/bindings/ServerObjectMapperModule.java @@ -17,18 +17,12 @@ package org.graylog2.bindings; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.module.SimpleModule; -import org.graylog2.database.ObjectIdSerializer; +import org.graylog2.bindings.providers.ServerObjectMapperProvider; import org.graylog2.shared.bindings.ObjectMapperModule; public class ServerObjectMapperModule extends ObjectMapperModule { - @Override - protected ObjectMapper makeObjectMapper() { - final ObjectMapper objectMapper = super.makeObjectMapper(); - - objectMapper.registerModule(new SimpleModule().addSerializer(new ObjectIdSerializer())); - - return objectMapper; + protected void configure() { + bind(ObjectMapper.class).toProvider(ServerObjectMapperProvider.class).asEagerSingleton(); } } diff --git a/graylog2-server/src/main/java/org/graylog2/bindings/providers/ServerObjectMapperProvider.java b/graylog2-server/src/main/java/org/graylog2/bindings/providers/ServerObjectMapperProvider.java new file mode 100644 index 000000000000..c51e4bc22135 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog2/bindings/providers/ServerObjectMapperProvider.java @@ -0,0 +1,33 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog2.bindings.providers; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.module.SimpleModule; +import org.graylog2.database.ObjectIdSerializer; +import org.graylog2.shared.bindings.providers.ObjectMapperProvider; + +import javax.inject.Provider; +import javax.inject.Singleton; + +@Singleton +public class ServerObjectMapperProvider extends ObjectMapperProvider implements Provider { + public ServerObjectMapperProvider() { + super(); + this.objectMapper.registerModule(new SimpleModule().addSerializer(new ObjectIdSerializer())); + } +} diff --git a/graylog2-server/src/main/java/org/graylog2/indexer/IndexHelper.java b/graylog2-server/src/main/java/org/graylog2/indexer/IndexHelper.java index fa364919c18f..61689a251ec2 100644 --- a/graylog2-server/src/main/java/org/graylog2/indexer/IndexHelper.java +++ b/graylog2-server/src/main/java/org/graylog2/indexer/IndexHelper.java @@ -29,7 +29,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Comparator; import java.util.List; import java.util.Set; @@ -91,8 +90,8 @@ public static Set determineAffectedIndices(IndexRangeService indexRangeS TimeRange range) { Set indices = Sets.newHashSet(); - for (IndexRange indexRange : indexRangeService.getFrom((int) (range.getFrom().getMillis() / 1000))) { - indices.add(indexRange.getIndexName()); + for (IndexRange indexRange : indexRangeService.find(range.getFrom(), range.getTo())) { + indices.add(indexRange.indexName()); } // Always include the most recent index in some cases. @@ -107,14 +106,9 @@ public static Set determineAffectedIndices(IndexRangeService indexRangeS public static Set determineAffectedIndicesWithRanges(IndexRangeService indexRangeService, Deflector deflector, TimeRange range) { - Set indices = Sets.newTreeSet(new Comparator() { - @Override - public int compare(IndexRange o1, IndexRange o2) { - return o2.getStart().compareTo(o1.getStart()); - } - }); + Set indices = Sets.newTreeSet(IndexRange.COMPARATOR); - for (IndexRange indexRange : indexRangeService.getFrom((int) (range.getFrom().getMillis() / 1000))) { + for (IndexRange indexRange : indexRangeService.find(range.getFrom(), range.getTo())) { indices.add(indexRange); } diff --git a/graylog2-server/src/main/java/org/graylog2/indexer/Mapping.java b/graylog2-server/src/main/java/org/graylog2/indexer/IndexMapping.java similarity index 54% rename from graylog2-server/src/main/java/org/graylog2/indexer/Mapping.java rename to graylog2-server/src/main/java/org/graylog2/indexer/IndexMapping.java index ca01d287baeb..4f4c73810730 100644 --- a/graylog2-server/src/main/java/org/graylog2/indexer/Mapping.java +++ b/graylog2-server/src/main/java/org/graylog2/indexer/IndexMapping.java @@ -18,48 +18,91 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; import org.elasticsearch.client.Client; -import org.graylog2.indexer.messages.Messages; import org.graylog2.plugin.Tools; +import javax.inject.Inject; +import javax.inject.Singleton; import java.io.Serializable; import java.util.List; import java.util.Map; +import static com.google.common.base.Preconditions.checkNotNull; /** * Representing the message type mapping in ElasticSearch. This is giving ES more * information about what the fields look like and how it should analyze them. */ -public class Mapping { +@Singleton +public class IndexMapping { + public static final String TYPE_MESSAGE = "message"; + public static final String TYPE_INDEX_RANGE = "index_range"; - public static PutMappingRequest getPutMappingRequest(final Client client, - final String index, - final String analyzer, - boolean storeTimestampsAsDocValues) { - final PutMappingRequestBuilder builder = client.admin().indices().preparePutMapping(index); - builder.setType(Messages.TYPE); + private final Client client; - final Map mapping = ImmutableMap.of( + @Inject + public IndexMapping(Client client) { + this.client = checkNotNull(client); + } + + public ActionFuture createMapping(final String index, final String type, final Map mapping) { + return client.admin().indices().putMapping(mappingRequest(index, type, mapping)); + } + + private PutMappingRequest mappingRequest(final String index, final String type, final Map mapping) { + return client.admin().indices().preparePutMapping(index) + .setType(type) + .setSource(ImmutableMap.of(type, mapping)) + .request(); + } + + public Map metaMapping() { + final ImmutableMap stringProperty = ImmutableMap.of( + "type", "string", + "index", "not_analyzed", + "doc_values", true); + final ImmutableMap dateProperty = ImmutableMap.of( + "type", "date", + "format", "date_time", + "index", "not_analyzed", + "doc_values", true); + final ImmutableMap intProperty = ImmutableMap.of( + "type", "integer", + "index", "no", + "doc_values", true); + final Map properties = ImmutableMap.of( + "index_name", stringProperty, + "begin", dateProperty, + "end", dateProperty, + "calculated_at", dateProperty, + "took_ms", intProperty + ); + + return ImmutableMap.of( + "properties", properties, + "_source", enabled(), + "_timestamp", ImmutableMap.of( + "enabled", true, + "format", "date_time")); + } + + public Map messageMapping(final String analyzer, boolean storeTimestampsAsDocValues) { + return ImmutableMap.of( "properties", partFieldProperties(analyzer, storeTimestampsAsDocValues), "dynamic_templates", partDefaultAllInDynamicTemplate(), // Compress source field "_source", enabledAndCompressed(), // Enable purging by TTL "_ttl", enabled()); - - final Map> completeMapping = ImmutableMap.of(Messages.TYPE, mapping); - - builder.setSource(completeMapping); - return builder.request(); } /* * Disable analyzing for every field by default. */ - private static List>> partDefaultAllInDynamicTemplate() { + private List>> partDefaultAllInDynamicTemplate() { final Map notAnalyzed = ImmutableMap.of("index", "not_analyzed"); final Map defaultAll = ImmutableMap.of( // Match all @@ -74,8 +117,8 @@ private static List>> partDefaultAllInDynamicTem /* * Enable analyzing for some fields again. Like for message and full_message. */ - private static Map> partFieldProperties(String analyzer, - boolean storeTimestampsAsDocValues) { + private Map> partFieldProperties(String analyzer, + boolean storeTimestampsAsDocValues) { return ImmutableMap.of( "message", analyzedString(analyzer), "full_message", analyzedString(analyzer), @@ -86,33 +129,32 @@ private static List>> partDefaultAllInDynamicTem "source", analyzedString("analyzer_keyword")); } - private static Map analyzedString(String analyzer) { + private Map analyzedString(String analyzer) { return ImmutableMap.of( "index", "analyzed", "type", "string", "analyzer", analyzer); } - private static Map typeTimeWithMillis(boolean storeTimestampsAsDocValues) { - final ImmutableMap.Builder builder = ImmutableMap.builder(); - builder.put("type", "date") + private Map typeTimeWithMillis(boolean storeTimestampsAsDocValues) { + final ImmutableMap.Builder builder = ImmutableMap.builder() + .put("type", "date") .put("format", Tools.ES_DATE_FORMAT); if (storeTimestampsAsDocValues) { builder.put("doc_values", true); } + return builder.build(); } - private static Map enabled() { + private Map enabled() { return ImmutableMap.of("enabled", true); } - - private static Map enabledAndCompressed() { + private Map enabledAndCompressed() { return ImmutableMap.of( "enabled", true, "compress", true); } - } \ No newline at end of file diff --git a/graylog2-server/src/main/java/org/graylog2/indexer/indices/Indices.java b/graylog2-server/src/main/java/org/graylog2/indexer/indices/Indices.java index ce59607b7b6a..2b4f48ac47b0 100644 --- a/graylog2-server/src/main/java/org/graylog2/indexer/indices/Indices.java +++ b/graylog2-server/src/main/java/org/graylog2/indexer/indices/Indices.java @@ -17,7 +17,6 @@ package org.graylog2.indexer.indices; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Maps; import com.google.common.collect.Sets; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionFuture; @@ -30,14 +29,15 @@ import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse; import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; -import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest; import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; import org.elasticsearch.action.admin.indices.flush.FlushRequest; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; import org.elasticsearch.action.admin.indices.optimize.OptimizeRequest; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.action.admin.indices.stats.IndexStats; import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest; @@ -45,7 +45,6 @@ import org.elasticsearch.action.admin.indices.stats.ShardStats; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.count.CountRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; @@ -59,13 +58,13 @@ import org.elasticsearch.common.collect.UnmodifiableIterator; import org.elasticsearch.common.hppc.cursors.ObjectObjectCursor; import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.SearchHit; import org.graylog2.configuration.ElasticsearchConfiguration; +import org.graylog2.indexer.IndexMapping; import org.graylog2.indexer.IndexNotFoundException; -import org.graylog2.indexer.Mapping; -import org.graylog2.indexer.messages.Messages; import org.graylog2.plugin.indexer.retention.IndexManagement; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -87,11 +86,13 @@ public class Indices implements IndexManagement { private final Client c; private final ElasticsearchConfiguration configuration; + private final IndexMapping indexMapping; @Inject - public Indices(Client client, ElasticsearchConfiguration configuration) { + public Indices(Client client, ElasticsearchConfiguration configuration, IndexMapping indexMapping) { this.c = client; this.configuration = configuration; + this.indexMapping = indexMapping; } public void move(String source, String target) { @@ -161,20 +162,6 @@ public Map getAll() { return isr.actionGet().getIndices(); } - public long getTotalNumberOfMessages() { - return c.count(new CountRequest(allIndicesAlias())).actionGet().getCount(); - } - - public long getTotalSize() { - return c.admin().indices().stats( - new IndicesStatsRequest().indices(allIndicesAlias())) - .actionGet() - .getTotal() - .getStore() - .getSize() - .getMb(); - } - public String allIndicesAlias() { return configuration.getIndexPrefix() + "_*"; } @@ -200,35 +187,28 @@ public String aliasTarget(String alias) { } public boolean create(String indexName) { - Map settings = Maps.newHashMap(); - settings.put("number_of_shards", configuration.getShards()); - settings.put("number_of_replicas", configuration.getReplicas()); - Map keywordLowercase = Maps.newHashMap(); - keywordLowercase.put("tokenizer", "keyword"); - keywordLowercase.put("filter", "lowercase"); - settings.put("index.analysis.analyzer.analyzer_keyword", keywordLowercase); - - CreateIndexRequest cir = new CreateIndexRequest(indexName); - cir.settings(settings); - - final ActionFuture createFuture = c.admin().indices().create(cir); - final boolean acknowledged = createFuture.actionGet().isAcknowledged(); - if (!acknowledged) { + final Map keywordLowercase = ImmutableMap.of( + "tokenizer", "keyword", + "filter", "lowercase"); + final Map settings = ImmutableMap.of( + "number_of_shards", configuration.getShards(), + "number_of_replicas", configuration.getReplicas(), + "index.analysis.analyzer.analyzer_keyword", keywordLowercase); + + final CreateIndexRequest cir = c.admin().indices().prepareCreate(indexName).setSettings(settings).request(); + if (!c.admin().indices().create(cir).actionGet().isAcknowledged()) { return false; } - final PutMappingRequest mappingRequest = Mapping.getPutMappingRequest(c, indexName, configuration.getAnalyzer(), - configuration.isStoreTimestampsAsDocValues()); - return c.admin().indices().putMapping(mappingRequest).actionGet().isAcknowledged(); - } - public ImmutableMap getMetadata() { - Map metaData = Maps.newHashMap(); - - for (ObjectObjectCursor next : c.admin().cluster().state(new ClusterStateRequest()).actionGet().getState().getMetaData().indices()) { - metaData.put(next.key, next.value); - } + final Map messageMapping = indexMapping.messageMapping(configuration.getAnalyzer(), + configuration.isStoreTimestampsAsDocValues()); + final PutMappingResponse messageMappingResponse = + indexMapping.createMapping(indexName, IndexMapping.TYPE_MESSAGE, messageMapping).actionGet(); + final Map metaMapping = indexMapping.metaMapping(); + final PutMappingResponse metaMappingResponse = + indexMapping.createMapping(indexName, IndexMapping.TYPE_INDEX_RANGE, metaMapping).actionGet(); - return ImmutableMap.copyOf(metaData); + return messageMappingResponse.isAcknowledged() && metaMappingResponse.isAcknowledged(); } public Set getAllMessageFields() { @@ -239,7 +219,7 @@ public Set getAllMessageFields() { for (ObjectObjectCursor m : cs.getMetaData().indices()) { try { - MappingMetaData mmd = m.value.mapping(Messages.TYPE); + MappingMetaData mmd = m.value.mapping(IndexMapping.TYPE_MESSAGE); if (mmd == null) { // There is no mapping if there are no messages in the index. continue; @@ -262,7 +242,7 @@ private IndexRequestBuilder manualIndexRequest(String index, Map b.setId(id); b.setSource(doc); b.setOpType(IndexRequest.OpType.INDEX); - b.setType(Messages.TYPE); + b.setType(IndexMapping.TYPE_MESSAGE); b.setConsistencyLevel(WriteConsistencyLevel.ONE); return b; @@ -279,6 +259,26 @@ public void setReadOnly(String index) { c.admin().indices().updateSettings(new UpdateSettingsRequest(index).settings(sb.build())).actionGet(); } + public boolean isReadOnly(String index) { + final GetSettingsRequest request = c.admin().indices().prepareGetSettings(index).request(); + final GetSettingsResponse response = c.admin().indices().getSettings(request).actionGet(); + + return response.getIndexToSettings().get(index).getAsBoolean("index.blocks.write", false); + } + + public void setReadWrite(String index) { + Settings settings = ImmutableSettings.builder() + .put("index.blocks.write", false) + .put("index.blocks.read", false) + .put("index.blocks.metadata", false) + .build(); + + final UpdateSettingsRequest request = c.admin().indices().prepareUpdateSettings(index) + .setSettings(settings) + .request(); + c.admin().indices().updateSettings(request).actionGet(); + } + public void flush(String index) { FlushRequest flush = new FlushRequest(index); flush.force(true); // Just flushes. Even if it is not necessary. diff --git a/graylog2-server/src/main/java/org/graylog2/indexer/messages/Messages.java b/graylog2-server/src/main/java/org/graylog2/indexer/messages/Messages.java index db8540a94737..e26447e256ba 100644 --- a/graylog2-server/src/main/java/org/graylog2/indexer/messages/Messages.java +++ b/graylog2-server/src/main/java/org/graylog2/indexer/messages/Messages.java @@ -35,6 +35,7 @@ import org.graylog2.configuration.ElasticsearchConfiguration; import org.graylog2.indexer.DeadLetter; import org.graylog2.indexer.Deflector; +import org.graylog2.indexer.IndexMapping; import org.graylog2.indexer.results.ResultMessage; import org.graylog2.plugin.Message; import org.slf4j.Logger; @@ -48,7 +49,6 @@ @Singleton public class Messages { - public static final String TYPE = "message"; private static final Logger log = LoggerFactory.getLogger(Messages.class); private final Client c; @@ -147,7 +147,7 @@ private IndexRequestBuilder buildIndexRequest(String index, Map b.setIndex(index); b.setContentType(XContentType.JSON); b.setOpType(IndexRequest.OpType.INDEX); - b.setType(TYPE); + b.setType(IndexMapping.TYPE_MESSAGE); b.setConsistencyLevel(WriteConsistencyLevel.ONE); return b; diff --git a/graylog2-server/src/main/java/org/graylog2/indexer/ranges/CreateNewSingleIndexRangeJob.java b/graylog2-server/src/main/java/org/graylog2/indexer/ranges/CreateNewSingleIndexRangeJob.java index 1fe6efc52bc2..22aa44dc9521 100644 --- a/graylog2-server/src/main/java/org/graylog2/indexer/ranges/CreateNewSingleIndexRangeJob.java +++ b/graylog2-server/src/main/java/org/graylog2/indexer/ranges/CreateNewSingleIndexRangeJob.java @@ -19,15 +19,10 @@ import com.google.inject.assistedinject.Assisted; import com.google.inject.assistedinject.AssistedInject; import org.graylog2.indexer.Deflector; -import org.graylog2.indexer.EmptyIndexException; -import org.graylog2.indexer.searches.Searches; -import org.graylog2.plugin.database.ValidationException; import org.graylog2.shared.system.activities.ActivityWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Map; - import static com.google.common.base.Preconditions.checkNotNull; public class CreateNewSingleIndexRangeJob extends RebuildIndexRangesJob { @@ -41,10 +36,9 @@ public interface Factory { @AssistedInject public CreateNewSingleIndexRangeJob(@Assisted Deflector deflector, @Assisted String indexName, - Searches searches, ActivityWriter activityWriter, IndexRangeService indexRangeService) { - super(deflector, searches, activityWriter, indexRangeService); + super(deflector, activityWriter, indexRangeService); this.indexName = checkNotNull(indexName); } @@ -62,14 +56,11 @@ public String getInfo() { public void execute() { LOG.info("Calculating ranges for index {}.", indexName); try { - final IndexRange indexRange = indexRangeService.create(calculateRange(indexName)); - indexRangeService.destroy(indexName); + final IndexRange indexRange = indexRangeService.calculateRange(indexName); indexRangeService.save(indexRange); LOG.info("Created ranges for index {}.", indexName); - } catch (ValidationException e) { - LOG.error("Unable to save index range for index {}: {}", indexName, e); } catch (Exception e) { - LOG.error("Exception during index range calculation for index {}: ", indexName, e); + LOG.error("Exception during index range calculation for index " + indexName, e); } } diff --git a/graylog2-server/src/main/java/org/graylog2/indexer/ranges/EsIndexRangeService.java b/graylog2-server/src/main/java/org/graylog2/indexer/ranges/EsIndexRangeService.java new file mode 100644 index 000000000000..839902d8dcec --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog2/indexer/ranges/EsIndexRangeService.java @@ -0,0 +1,258 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog2.indexer.ranges; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Stopwatch; +import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableSortedSet; +import com.google.common.primitives.Ints; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.NoShardAvailableActionException; +import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.action.get.GetRequestBuilder; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.client.Client; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.FilterBuilders; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.indices.IndexMissingException; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.bucket.filter.Filter; +import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.stats.Stats; +import org.graylog2.database.NotFoundException; +import org.graylog2.indexer.IndexMapping; +import org.graylog2.indexer.indices.Indices; +import org.graylog2.indexer.searches.TimestampStats; +import org.graylog2.plugin.Tools; +import org.graylog2.shared.system.activities.ActivityWriter; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; +import javax.inject.Inject; +import java.util.Map; +import java.util.SortedSet; +import java.util.concurrent.TimeUnit; + +public class EsIndexRangeService implements IndexRangeService { + private static final Logger LOG = LoggerFactory.getLogger(EsIndexRangeService.class); + + private final Client client; + private final ActivityWriter activityWriter; + private final ObjectMapper objectMapper; + private final Indices indices; + + @Inject + public EsIndexRangeService(Client client, ActivityWriter activityWriter, ObjectMapper objectMapper, Indices indices) { + this.client = client; + this.activityWriter = activityWriter; + this.objectMapper = objectMapper; + this.indices = indices; + } + + @Override + @Nullable + public IndexRange get(String index) throws NotFoundException { + final GetRequest request = new GetRequestBuilder(client, index) + .setType(IndexMapping.TYPE_INDEX_RANGE) + .setId(index) + .request(); + + final GetResponse r; + try { + r = client.get(request).actionGet(); + } catch (IndexMissingException | NoShardAvailableActionException e) { + throw new NotFoundException(e); + } + + if (!r.isExists()) { + throw new NotFoundException("Index [" + index + "] not found."); + } + + return parseSource(r.getIndex(), r.getSource()); + } + + @Nullable + private IndexRange parseSource(String index, Map fields) { + try { + return IndexRange.create( + index, + parseFromDateString((String) fields.get("begin")), + parseFromDateString((String) fields.get("end")), + parseFromDateString((String) fields.get("calculated_at")), + (int) fields.get("took_ms") + ); + } catch (Exception e) { + return null; + } + } + + private DateTime parseFromDateString(String s) { + return DateTime.parse(s); + } + + @Override + public SortedSet find(DateTime begin, DateTime end) { + final RangeQueryBuilder beginRangeQuery = QueryBuilders.rangeQuery("begin").gte(begin.getMillis()); + final RangeQueryBuilder endRangeQuery = QueryBuilders.rangeQuery("end").lte(end.getMillis()); + final BoolQueryBuilder completeRangeQuery = QueryBuilders.boolQuery() + .must(beginRangeQuery) + .must(endRangeQuery); + final SearchRequest request = client.prepareSearch() + .setTypes(IndexMapping.TYPE_INDEX_RANGE) + .setIndices(indices.allIndicesAlias()) + .setQuery(completeRangeQuery) + .request(); + + final SearchResponse response = client.search(request).actionGet(); + final ImmutableSortedSet.Builder indexRanges = ImmutableSortedSet.orderedBy(IndexRange.COMPARATOR); + for (SearchHit searchHit : response.getHits()) { + final IndexRange indexRange = parseSource(searchHit.getIndex(), searchHit.getSource()); + if (indexRange != null) { + indexRanges.add(indexRange); + } + } + + return indexRanges.build(); + } + + @Override + public SortedSet findAll() { + final SearchRequest request = client.prepareSearch() + .setTypes(IndexMapping.TYPE_INDEX_RANGE) + .setIndices(indices.allIndicesAlias()) + .setQuery(QueryBuilders.matchAllQuery()) + .request(); + + final SearchResponse response = client.search(request).actionGet(); + final ImmutableSortedSet.Builder indexRanges = ImmutableSortedSet.orderedBy(IndexRange.COMPARATOR); + for (SearchHit searchHit : response.getHits()) { + final IndexRange indexRange = parseSource(searchHit.getIndex(), searchHit.getSource()); + if (indexRange != null) { + indexRanges.add(indexRange); + } + } + + return indexRanges.build(); + } + + @Override + public IndexRange calculateRange(String index) { + final Stopwatch sw = Stopwatch.createStarted(); + final DateTime now = DateTime.now(DateTimeZone.UTC); + final TimestampStats stats = timestampStatsOfIndex(index); + final int duration = Ints.saturatedCast(sw.stop().elapsed(TimeUnit.MILLISECONDS)); + + LOG.info("Calculated range of [{}] in [{}ms].", index, duration); + return IndexRange.create(index, stats.min(), stats.max(), now, duration); + } + + /** + * Calculate stats (min, max, avg) about the message timestamps in the given index. + * + * @param index Name of the index to query. + * @return the timestamp stats in the given index, or {@code null} if they couldn't be calculated. + * @see org.elasticsearch.search.aggregations.metrics.stats.Stats + */ + @VisibleForTesting + protected TimestampStats timestampStatsOfIndex(String index) { + final FilterAggregationBuilder builder = AggregationBuilders.filter("agg") + .filter(FilterBuilders.existsFilter("timestamp")) + .subAggregation(AggregationBuilders.stats("ts_stats").field("timestamp")); + final SearchRequestBuilder srb = client.prepareSearch() + .setIndices(index) + .setSearchType(SearchType.COUNT) + .addAggregation(builder); + + final SearchResponse response; + try { + response = client.search(srb.request()).actionGet(); + } catch (IndexMissingException e) { + throw e; + } catch (ElasticsearchException e) { + LOG.error("Error while calculating timestamp stats in index <" + index + ">", e); + throw new IndexMissingException(new Index(index)); + } + + final Filter f = response.getAggregations().get("agg"); + if (f.getDocCount() == 0L) { + LOG.debug("No documents with attribute \"timestamp\" found in index <{}>", index); + return TimestampStats.EMPTY; + } + + final Stats stats = f.getAggregations().get("ts_stats"); + final DateTimeFormatter formatter = DateTimeFormat.forPattern(Tools.ES_DATE_FORMAT).withZoneUTC(); + final DateTime min = formatter.parseDateTime(stats.getMinAsString()); + final DateTime max = formatter.parseDateTime(stats.getMaxAsString()); + final DateTime avg = formatter.parseDateTime(stats.getAvgAsString()); + + return TimestampStats.create(min, max, avg); + } + + @Override + public void save(IndexRange indexRange) { + final byte[] source; + try { + source = objectMapper.writeValueAsBytes(indexRange); + } catch (JsonProcessingException e) { + throw Throwables.propagate(e); + } + + final String indexName = indexRange.indexName(); + final boolean readOnly = indices.isReadOnly(indexName); + + if (readOnly) { + indices.setReadWrite(indexName); + } + + final IndexRequest request = client.prepareIndex() + .setIndex(indexName) + .setType(IndexMapping.TYPE_INDEX_RANGE) + .setId(indexName) + .setRefresh(true) + .setSource(source) + .request(); + final IndexResponse response = client.index(request).actionGet(); + + if (readOnly) { + indices.setReadOnly(indexName); + } + + if (response.isCreated()) { + LOG.debug("Successfully saved index range: {}", indexRange); + } else { + LOG.debug("Successfully updated index range: {}", indexRange); + } + } +} \ No newline at end of file diff --git a/graylog2-server/src/main/java/org/graylog2/indexer/ranges/IndexRange.java b/graylog2-server/src/main/java/org/graylog2/indexer/ranges/IndexRange.java index 13361d3a4313..2cb38d94b19e 100644 --- a/graylog2-server/src/main/java/org/graylog2/indexer/ranges/IndexRange.java +++ b/graylog2-server/src/main/java/org/graylog2/indexer/ranges/IndexRange.java @@ -16,18 +16,38 @@ */ package org.graylog2.indexer.ranges; -import org.graylog2.plugin.database.Persisted; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.auto.value.AutoValue; import org.joda.time.DateTime; -/** - * @author Dennis Oelkers - */ -public interface IndexRange extends Persisted { - String getIndexName(); +import java.util.Comparator; + +@AutoValue +@JsonAutoDetect +public abstract class IndexRange { + public static final Comparator COMPARATOR = new IndexRangeComparator(); + + @JsonProperty + public abstract String indexName(); + + @JsonProperty + public abstract DateTime begin(); + + @JsonProperty + public abstract DateTime end(); - DateTime getCalculatedAt(); + @JsonProperty + public abstract DateTime calculatedAt(); - DateTime getStart(); + @JsonProperty("took_ms") + public abstract int calculationDuration(); - int getCalculationTookMs(); -} + public static IndexRange create(String indexName, + DateTime begin, + DateTime end, + DateTime calculatedAt, + int calculationDuration) { + return new AutoValue_IndexRange(indexName, begin, end, calculatedAt, calculationDuration); + } +} \ No newline at end of file diff --git a/graylog2-server/src/main/java/org/graylog2/indexer/ranges/IndexRangeComparator.java b/graylog2-server/src/main/java/org/graylog2/indexer/ranges/IndexRangeComparator.java index b99c02f9ac08..3d4be717c9e0 100644 --- a/graylog2-server/src/main/java/org/graylog2/indexer/ranges/IndexRangeComparator.java +++ b/graylog2-server/src/main/java/org/graylog2/indexer/ranges/IndexRangeComparator.java @@ -24,6 +24,6 @@ public class IndexRangeComparator implements Comparator { */ @Override public int compare(IndexRange o1, IndexRange o2) { - return o2.getStart().compareTo(o1.getStart()); + return o2.end().compareTo(o1.end()); } } diff --git a/graylog2-server/src/main/java/org/graylog2/indexer/ranges/IndexRangeImpl.java b/graylog2-server/src/main/java/org/graylog2/indexer/ranges/IndexRangeImpl.java deleted file mode 100644 index 2de00035decd..000000000000 --- a/graylog2-server/src/main/java/org/graylog2/indexer/ranges/IndexRangeImpl.java +++ /dev/null @@ -1,107 +0,0 @@ -/** - * This file is part of Graylog. - * - * Graylog is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Graylog is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Graylog. If not, see . - */ -package org.graylog2.indexer.ranges; - -import com.fasterxml.jackson.annotation.JsonValue; -import com.google.common.collect.Maps; -import org.bson.types.ObjectId; -import org.graylog2.database.CollectionName; -import org.graylog2.database.PersistedImpl; -import org.graylog2.plugin.database.validators.Validator; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -@CollectionName("index_ranges") -public class IndexRangeImpl extends PersistedImpl implements IndexRange { - - private static final Logger LOG = LoggerFactory.getLogger(IndexRangeImpl.class); - - public IndexRangeImpl(Map fields) { - super(fields); - } - - protected IndexRangeImpl(ObjectId id, Map fields) { - super(id, fields); - } - - @Override - public String getIndexName() { - return (String) fields.get("index"); - } - - @Override - public DateTime getCalculatedAt() { - if (fields.containsKey("calculated_at")) { - int ts = (Integer) fields.get("calculated_at"); - long unixMs = ts * 1000L; - return new DateTime(unixMs, DateTimeZone.UTC); - } else { - return null; - } - } - - @Override - public DateTime getStart() { - int ts = (Integer) fields.get("start"); - long unixMs = ts * 1000L; - return new DateTime(unixMs, DateTimeZone.UTC); - } - - @Override - public int getCalculationTookMs() { - if (fields.containsKey("took_ms")) { - return (Integer) fields.get("took_ms"); - } else { - return -1; - } - } - - @Override - public Map getValidations() { - return Collections.emptyMap(); - } - - @Override - public Map getEmbeddedValidations(String key) { - return Collections.emptyMap(); - } - - @JsonValue - public Map asMap() { - HashMap fields = Maps.newHashMap(); - fields.put("index", getIndexName()); - fields.put("starts", getStart()); - // Calculated at and the calculation time in ms are not always set, depending on how/why the entry was created. - DateTime calculatedAt = getCalculatedAt(); - if (calculatedAt != null) { - fields.put("calculated_at", calculatedAt); - } - - int calculationTookMs = getCalculationTookMs(); - if (calculationTookMs >= 0) { - fields.put("calculation_took_ms", calculationTookMs); - } - return fields; - } - -} \ No newline at end of file diff --git a/graylog2-server/src/main/java/org/graylog2/indexer/ranges/IndexRangeService.java b/graylog2-server/src/main/java/org/graylog2/indexer/ranges/IndexRangeService.java index b8e0ed5e993c..2cc63692dd3c 100644 --- a/graylog2-server/src/main/java/org/graylog2/indexer/ranges/IndexRangeService.java +++ b/graylog2-server/src/main/java/org/graylog2/indexer/ranges/IndexRangeService.java @@ -17,22 +17,18 @@ package org.graylog2.indexer.ranges; import org.graylog2.database.NotFoundException; -import org.graylog2.plugin.database.PersistedService; +import org.joda.time.DateTime; -import java.util.List; -import java.util.Map; +import java.util.SortedSet; -/** - * @author Dennis Oelkers - */ -public interface IndexRangeService extends PersistedService { +public interface IndexRangeService { IndexRange get(String index) throws NotFoundException; - List getFrom(int timestamp); + SortedSet find(DateTime begin, DateTime end); - void destroy(String index); + SortedSet findAll(); - IndexRange create(Map range); + void save(IndexRange indexRange); - void destroyAll(); + IndexRange calculateRange(String index); } diff --git a/graylog2-server/src/main/java/org/graylog2/indexer/ranges/IndexRangeServiceImpl.java b/graylog2-server/src/main/java/org/graylog2/indexer/ranges/IndexRangeServiceImpl.java deleted file mode 100644 index 79a73e73f75d..000000000000 --- a/graylog2-server/src/main/java/org/graylog2/indexer/ranges/IndexRangeServiceImpl.java +++ /dev/null @@ -1,98 +0,0 @@ -/** - * This file is part of Graylog. - * - * Graylog is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Graylog is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Graylog. If not, see . - */ -package org.graylog2.indexer.ranges; - -import com.google.common.collect.Lists; -import com.mongodb.BasicDBObject; -import com.mongodb.DBObject; -import org.bson.types.ObjectId; -import org.graylog2.database.MongoConnection; -import org.graylog2.database.NotFoundException; -import org.graylog2.database.PersistedServiceImpl; -import org.graylog2.shared.system.activities.Activity; -import org.graylog2.shared.system.activities.ActivityWriter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.inject.Inject; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.Map; - -public class IndexRangeServiceImpl extends PersistedServiceImpl implements IndexRangeService { - private static final Logger LOG = LoggerFactory.getLogger(IndexRangeServiceImpl.class); - private static final Comparator COMPARATOR = new IndexRangeComparator(); - - private final ActivityWriter activityWriter; - - @Inject - public IndexRangeServiceImpl(MongoConnection mongoConnection, ActivityWriter activityWriter) { - super(mongoConnection); - this.activityWriter = activityWriter; - } - - @Override - public IndexRange get(String index) throws NotFoundException { - DBObject dbo = findOne(IndexRangeImpl.class, new BasicDBObject("index", index)); - - if (dbo == null) - throw new NotFoundException("Index " + index + " not found."); - - return new IndexRangeImpl((ObjectId) dbo.get("_id"), dbo.toMap()); - } - - @Override - public List getFrom(int timestamp) { - List ranges = Lists.newArrayList(); - - BasicDBObject query = new BasicDBObject(); - query.put("start", new BasicDBObject("$gte", timestamp)); - - for (DBObject dbo : query(IndexRangeImpl.class, query)) { - ranges.add(new IndexRangeImpl((ObjectId) dbo.get("_id"), dbo.toMap())); - } - - Collections.sort(ranges, COMPARATOR); - - return ranges; - } - - @Override - public void destroy(String index) { - try { - final IndexRange range = get(index); - destroy(range); - } catch (NotFoundException e) { - return; - } - - String x = "Removed range meta-information of [" + index + "]"; - LOG.info(x); - activityWriter.write(new Activity(x, IndexRangeImpl.class)); - } - - @Override - public IndexRange create(Map range) { - return new IndexRangeImpl(range); - } - - @Override - public void destroyAll() { - destroyAll(IndexRangeImpl.class); - } -} \ No newline at end of file diff --git a/graylog2-server/src/main/java/org/graylog2/indexer/ranges/RebuildIndexRangesJob.java b/graylog2-server/src/main/java/org/graylog2/indexer/ranges/RebuildIndexRangesJob.java index e86c0d259152..541d51632d63 100644 --- a/graylog2-server/src/main/java/org/graylog2/indexer/ranges/RebuildIndexRangesJob.java +++ b/graylog2-server/src/main/java/org/graylog2/indexer/ranges/RebuildIndexRangesJob.java @@ -17,52 +17,38 @@ package org.graylog2.indexer.ranges; import com.google.common.base.Stopwatch; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; -import com.google.common.primitives.Ints; import com.google.inject.assistedinject.Assisted; import com.google.inject.assistedinject.AssistedInject; import org.graylog2.indexer.Deflector; -import org.graylog2.indexer.searches.Searches; -import org.graylog2.plugin.Tools; import org.graylog2.shared.system.activities.Activity; import org.graylog2.shared.system.activities.ActivityWriter; import org.graylog2.system.jobs.SystemJob; -import org.joda.time.DateTime; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.List; -import java.util.Map; import java.util.concurrent.TimeUnit; -import static com.google.common.base.MoreObjects.firstNonNull; - public class RebuildIndexRangesJob extends SystemJob { public interface Factory { RebuildIndexRangesJob create(Deflector deflector); } private static final Logger LOG = LoggerFactory.getLogger(RebuildIndexRangesJob.class); + private static final int MAX_CONCURRENCY = 1; - public static final int MAX_CONCURRENCY = 1; - - private boolean cancelRequested = false; - private int indicesToCalculate = 0; - private int indicesCalculated = 0; + private volatile boolean cancelRequested = false; + private volatile int indicesToCalculate = 0; + private volatile int indicesCalculated = 0; protected final Deflector deflector; - private final Searches searches; private final ActivityWriter activityWriter; protected final IndexRangeService indexRangeService; @AssistedInject public RebuildIndexRangesJob(@Assisted Deflector deflector, - Searches searches, ActivityWriter activityWriter, IndexRangeService indexRangeService) { this.deflector = deflector; - this.searches = searches; this.activityWriter = activityWriter; this.indexRangeService = indexRangeService; } @@ -89,7 +75,6 @@ public String getDescription() { @Override public void execute() { - List> ranges = Lists.newArrayList(); info("Re-calculating index ranges."); String[] indices = deflector.getAllDeflectorIndexNames(); @@ -108,7 +93,9 @@ public void execute() { } try { - ranges.add(calculateRange(index)); + final IndexRange indexRange = indexRangeService.calculateRange(index); + indexRangeService.save(indexRange); + LOG.debug("Created ranges for index {}: {}", index, indexRange); } catch (Exception e) { LOG.info("Could not calculate range of index [" + index + "]. Skipping.", e); } finally { @@ -116,34 +103,9 @@ public void execute() { } } - // Now that all is calculated we can replace the whole collection at once. - updateCollection(ranges); - info("Done calculating index ranges for " + indices.length + " indices. Took " + sw.stop().elapsed(TimeUnit.MILLISECONDS) + "ms."); } - protected Map calculateRange(String index) { - final Stopwatch x = Stopwatch.createStarted(); - final DateTime timestamp = firstNonNull(searches.findNewestMessageTimestampOfIndex(index), Tools.iso8601()); - final int rangeEnd = Ints.saturatedCast(timestamp.getMillis() / 1000L); - final int took = Ints.saturatedCast(x.stop().elapsed(TimeUnit.MILLISECONDS)); - - LOG.info("Calculated range of [{}] in [{}ms].", index, took); - return ImmutableMap.of( - "index", index, - "start", rangeEnd, // FIXME The name of the attribute is massively misleading and should be rectified some time - "calculated_at", Tools.getUTCTimestamp(), - "took_ms", took); - } - - private void updateCollection(List> ranges) { - indexRangeService.destroyAll(); - for (Map range : ranges) { - IndexRange indexRange = indexRangeService.create(range); - indexRangeService.saveWithoutValidation(indexRange); - } - } - protected void info(String what) { LOG.info(what); activityWriter.write(new Activity(what, RebuildIndexRangesJob.class)); diff --git a/graylog2-server/src/main/java/org/graylog2/indexer/searches/Searches.java b/graylog2-server/src/main/java/org/graylog2/indexer/searches/Searches.java index e6b5819bf245..cab8ee07db56 100644 --- a/graylog2-server/src/main/java/org/graylog2/indexer/searches/Searches.java +++ b/graylog2-server/src/main/java/org/graylog2/indexer/searches/Searches.java @@ -43,7 +43,6 @@ import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.metrics.max.Max; import org.elasticsearch.search.aggregations.metrics.min.Min; -import org.elasticsearch.search.aggregations.metrics.stats.Stats; import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStats; import org.elasticsearch.search.sort.SortOrder; import org.graylog2.Configuration; @@ -246,7 +245,7 @@ public SearchResult search(SearchesConfig config) { Set indexNames = Sets.newHashSet(); for (IndexRange index : indices) { - indexNames.add(index.getIndexName()); + indexNames.add(index.indexName()); } SearchRequest request = searchRequest(config, indexNames).request(); @@ -493,140 +492,6 @@ public HistogramResult fieldHistogram(String query, String field, DateHistogramI r.getTook()); } - /** - * WARNING: The name of that method is wrong and it returns the newest message of an index! - */ - public SearchHit firstOfIndex(String index) { - return oneOfIndex(index, matchAllQuery(), SortOrder.DESC); - } - - /** - * Find the oldest message timestamp in the given index. - * - * @param index Name of the index to query. - * @return the oldest message timestamp in the given index, or {@code null} if it couldn't be found. - */ - public DateTime findOldestMessageTimestampOfIndex(String index) { - final FilterAggregationBuilder builder = AggregationBuilders.filter("agg") - .filter(FilterBuilders.existsFilter("timestamp")) - .subAggregation(AggregationBuilders.min("ts_min").field("timestamp")); - final SearchRequestBuilder srb = c.prepareSearch() - .setIndices(index) - .setSearchType(SearchType.COUNT) - .addAggregation(builder); - - final SearchResponse response; - try { - response = c.search(srb.request()).actionGet(); - } catch (IndexMissingException e) { - throw e; - } catch (ElasticsearchException e) { - LOG.error("Error while calculating oldest timestamp in index <" + index + ">", e); - return null; - } - esRequestTimer.update(response.getTookInMillis(), TimeUnit.MILLISECONDS); - - final Filter f = response.getAggregations().get("agg"); - if (f.getDocCount() == 0L) { - LOG.debug("No documents with attribute \"timestamp\" found in index <{}>", index); - return null; - } - - final Min min = f.getAggregations().get("ts_min"); - final String minTimeStamp = min.getValueAsString(); - final DateTimeFormatter formatter = DateTimeFormat.forPattern(Tools.ES_DATE_FORMAT).withZoneUTC(); - - return formatter.parseDateTime(minTimeStamp); - } - - /** - * Calculate stats (min, max, avg) about the message timestamps in the given index. - * - * @param index Name of the index to query. - * @return the timestamp stats in the given index, or {@code null} if they couldn't be calculated. - * @see org.elasticsearch.search.aggregations.metrics.stats.Stats - */ - public TimestampStats timestampStatsOfIndex(String index) { - final FilterAggregationBuilder builder = AggregationBuilders.filter("agg") - .filter(FilterBuilders.existsFilter("timestamp")) - .subAggregation(AggregationBuilders.stats("ts_stats").field("timestamp")); - final SearchRequestBuilder srb = c.prepareSearch() - .setIndices(index) - .setSearchType(SearchType.COUNT) - .addAggregation(builder); - - final SearchResponse response; - try { - response = c.search(srb.request()).actionGet(); - } catch (IndexMissingException e) { - throw e; - } catch (ElasticsearchException e) { - LOG.error("Error while calculating timestamp stats in index <" + index + ">", e); - return null; - } - esRequestTimer.update(response.getTookInMillis(), TimeUnit.MILLISECONDS); - - final Filter f = response.getAggregations().get("agg"); - if (f.getDocCount() == 0L) { - LOG.debug("No documents with attribute \"timestamp\" found in index <{}>", index); - return null; - } - - final Stats stats = f.getAggregations().get("ts_stats"); - final DateTimeFormatter formatter = DateTimeFormat.forPattern(Tools.ES_DATE_FORMAT).withZoneUTC(); - final DateTime min = formatter.parseDateTime(stats.getMinAsString()); - final DateTime max = formatter.parseDateTime(stats.getMaxAsString()); - final DateTime avg = formatter.parseDateTime(stats.getAvgAsString()); - - return TimestampStats.create(min, max, avg); - } - - /** - * WARNING: The name of that method is wrong and it returns the oldest message of an index! - */ - public SearchHit lastOfIndex(String index) { - return oneOfIndex(index, matchAllQuery(), SortOrder.ASC); - } - - /** - * Find the newest message timestamp in the given index. - * - * @param index Name of the index to query. - * @return the youngest message timestamp in the given index, or {@code null} if it couldn't be found. - */ - public DateTime findNewestMessageTimestampOfIndex(String index) { - final FilterAggregationBuilder builder = AggregationBuilders.filter("agg") - .filter(FilterBuilders.existsFilter("timestamp")) - .subAggregation(AggregationBuilders.max("ts_max").field("timestamp")); - final SearchRequestBuilder srb = c.prepareSearch() - .setIndices(index) - .setSearchType(SearchType.COUNT) - .addAggregation(builder); - - final SearchResponse response; - try { - response = c.search(srb.request()).actionGet(); - } catch (IndexMissingException e) { - throw e; - } catch (ElasticsearchException e) { - LOG.error("Error while calculating newest timestamp in index <" + index + ">", e); - return null; - } - esRequestTimer.update(response.getTookInMillis(), TimeUnit.MILLISECONDS); - - final Filter f = response.getAggregations().get("agg"); - if (f.getDocCount() == 0L) { - LOG.debug("No documents with attribute \"timestamp\" found in index <{}>", index); - return null; - } - - final Max max = f.getAggregations().get("ts_max"); - final String maxTimeStamp = max.getValueAsString(); - final DateTimeFormatter formatter = DateTimeFormat.forPattern(Tools.ES_DATE_FORMAT).withZoneUTC(); - - return formatter.parseDateTime(maxTimeStamp); - } - private SearchRequestBuilder searchRequest(SearchesConfig config, Set indices) { final SearchRequestBuilder request; @@ -773,4 +638,4 @@ public FieldTypeException(Throwable e) { super(e); } } -} +} \ No newline at end of file diff --git a/graylog2-server/src/main/java/org/graylog2/indexer/searches/TimestampStats.java b/graylog2-server/src/main/java/org/graylog2/indexer/searches/TimestampStats.java index 321ba6fe5c13..cc3bcfff457d 100644 --- a/graylog2-server/src/main/java/org/graylog2/indexer/searches/TimestampStats.java +++ b/graylog2-server/src/main/java/org/graylog2/indexer/searches/TimestampStats.java @@ -18,9 +18,13 @@ import com.google.auto.value.AutoValue; import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; @AutoValue public abstract class TimestampStats { + public static final TimestampStats EMPTY = + create(new DateTime(0L, DateTimeZone.UTC), new DateTime(0L, DateTimeZone.UTC), new DateTime(0L, DateTimeZone.UTC)); + public abstract DateTime min(); public abstract DateTime max(); diff --git a/graylog2-server/src/main/java/org/graylog2/rest/resources/search/SearchResource.java b/graylog2-server/src/main/java/org/graylog2/rest/resources/search/SearchResource.java index 9a93590a8e67..f0618ace64a0 100644 --- a/graylog2-server/src/main/java/org/graylog2/rest/resources/search/SearchResource.java +++ b/graylog2-server/src/main/java/org/graylog2/rest/resources/search/SearchResource.java @@ -154,7 +154,12 @@ protected Set indexRangeListToValueList(Set index final Set result = Sets.newHashSetWithExpectedSize(indexRanges.size()); for (IndexRange indexRange : indexRanges) { - result.add(IndexRangeSummary.create(indexRange.getIndexName(), indexRange.getCalculatedAt(), indexRange.getStart(), indexRange.getCalculationTookMs())); + result.add(IndexRangeSummary.create( + indexRange.indexName(), + indexRange.begin(), + indexRange.end(), + indexRange.calculatedAt(), + indexRange.calculationDuration())); } return result; diff --git a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/IndexRangesResource.java b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/IndexRangesResource.java index fb26e934ef36..3af8a40b6061 100644 --- a/graylog2-server/src/main/java/org/graylog2/rest/resources/system/IndexRangesResource.java +++ b/graylog2-server/src/main/java/org/graylog2/rest/resources/system/IndexRangesResource.java @@ -75,8 +75,8 @@ public IndexRangesResource(IndexRangeService indexRangeService, @Produces(MediaType.APPLICATION_JSON) public IndexRangesResponse list() { final List ranges = Lists.newArrayList(); - for (IndexRange range : indexRangeService.getFrom(0)) { - if (!isPermitted(RestPermissions.INDEXRANGES_READ, range.getIndexName())) { + for (IndexRange range : indexRangeService.findAll()) { + if (!isPermitted(RestPermissions.INDEXRANGES_READ, range.indexName())) { continue; } ranges.add(range); diff --git a/graylog2-server/src/test/java/org/graylog2/alerts/types/FieldContentValueAlertConditionTest.java b/graylog2-server/src/test/java/org/graylog2/alerts/types/FieldContentValueAlertConditionTest.java index a7bef34e5bb0..fda501397ddc 100644 --- a/graylog2-server/src/test/java/org/graylog2/alerts/types/FieldContentValueAlertConditionTest.java +++ b/graylog2-server/src/test/java/org/graylog2/alerts/types/FieldContentValueAlertConditionTest.java @@ -26,7 +26,6 @@ import org.graylog2.alerts.AbstractAlertCondition; import org.graylog2.alerts.AlertConditionTest; import org.graylog2.indexer.ranges.IndexRange; -import org.graylog2.indexer.ranges.IndexRangeImpl; import org.graylog2.indexer.results.SearchResult; import org.graylog2.indexer.searches.Searches; import org.graylog2.indexer.searches.Sorting; @@ -35,6 +34,7 @@ import org.graylog2.plugin.alarms.AlertCondition; import org.graylog2.plugin.streams.Stream; import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; import org.junit.Test; import java.util.Collections; @@ -44,7 +44,10 @@ import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertNotNull; -import static org.mockito.Matchers.*; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyInt; +import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; @@ -73,11 +76,9 @@ public void testRunMatchingMessagesInStream() throws Exception { when(searchHit.getIndex()).thenReturn("graylog_test"); when(searchHits.iterator()).thenReturn(Iterators.singletonIterator(searchHit)); - final HashMap fields = Maps.newHashMap(); - fields.put("index", "graylog_test"); - fields.put("started_at", DateTime.now().minusDays(1).getMillis()); - - final Set indexRanges = Sets.newHashSet(new IndexRangeImpl(fields)); + final DateTime now = DateTime.now(DateTimeZone.UTC); + final IndexRange indexRange = IndexRange.create("graylog_test", now.minusDays(1), now, now, 0); + final Set indexRanges = Sets.newHashSet(indexRange); final SearchResult searchResult = spy(new SearchResult(searchHits, indexRanges, "message:something", @@ -107,11 +108,9 @@ public void testRunNoMatchingMessages() throws Exception { final SearchHits searchHits = mock(SearchHits.class); when(searchHits.iterator()).thenReturn(Collections.emptyIterator()); - final HashMap fields = Maps.newHashMap(); - fields.put("index", "graylog_test"); - fields.put("started_at", DateTime.now().minusDays(1).getMillis()); - - final Set indexRanges = Sets.newHashSet(new IndexRangeImpl(fields)); + final DateTime now = DateTime.now(DateTimeZone.UTC); + final IndexRange indexRange = IndexRange.create("graylog_test", now.minusDays(1), now, now, 0); + final Set indexRanges = Sets.newHashSet(indexRange); final SearchResult searchResult = spy(new SearchResult(searchHits, indexRanges, "message:something", diff --git a/graylog2-server/src/test/java/org/graylog2/indexer/indices/IndicesTest.java b/graylog2-server/src/test/java/org/graylog2/indexer/indices/IndicesTest.java index 5603335fc377..826beeedc9f9 100644 --- a/graylog2-server/src/test/java/org/graylog2/indexer/indices/IndicesTest.java +++ b/graylog2-server/src/test/java/org/graylog2/indexer/indices/IndicesTest.java @@ -20,8 +20,6 @@ import com.lordofthejars.nosqlunit.core.LoadStrategyEnum; import com.lordofthejars.nosqlunit.elasticsearch.ElasticsearchRule; import com.lordofthejars.nosqlunit.elasticsearch.EmbeddedElasticsearch; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; @@ -31,7 +29,8 @@ import org.elasticsearch.client.Client; import org.elasticsearch.client.IndicesAdminClient; import org.graylog2.configuration.ElasticsearchConfiguration; -import org.graylog2.indexer.searches.SearchesTest; +import org.graylog2.indexer.IndexMapping; +import org.graylog2.indexer.nosqlunit.IndexCreatingLoadStrategyFactory; import org.junit.Before; import org.junit.ClassRule; import org.junit.Rule; @@ -69,12 +68,12 @@ public String getIndexPrefix() { public IndicesTest() { this.elasticsearchRule = newElasticsearchRule().defaultEmbeddedElasticsearch(); - this.elasticsearchRule.setLoadStrategyFactory(new SearchesTest.IndexCreatingLoadStrategyFactory(Collections.singleton(INDEX_NAME))); + this.elasticsearchRule.setLoadStrategyFactory(new IndexCreatingLoadStrategyFactory(Collections.singleton(INDEX_NAME))); } @Before public void setUp() throws Exception { - indices = new Indices(client, CONFIG); + indices = new Indices(client, CONFIG, new IndexMapping(client)); } @Test diff --git a/graylog2-server/src/test/java/org/graylog2/indexer/nosqlunit/IndexCreatingDatabaseOperation.java b/graylog2-server/src/test/java/org/graylog2/indexer/nosqlunit/IndexCreatingDatabaseOperation.java new file mode 100644 index 000000000000..af1f8ace5d38 --- /dev/null +++ b/graylog2-server/src/test/java/org/graylog2/indexer/nosqlunit/IndexCreatingDatabaseOperation.java @@ -0,0 +1,77 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog2.indexer.nosqlunit; + +import com.google.common.collect.ImmutableSet; +import com.lordofthejars.nosqlunit.core.DatabaseOperation; +import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.IndicesAdminClient; +import org.graylog2.configuration.ElasticsearchConfiguration; +import org.graylog2.indexer.IndexMapping; +import org.graylog2.indexer.indices.Indices; + +import java.io.InputStream; +import java.util.Set; + +public class IndexCreatingDatabaseOperation implements DatabaseOperation { + private final DatabaseOperation databaseOperation; + private final Client client; + private final Set indexes; + + public IndexCreatingDatabaseOperation(DatabaseOperation databaseOperation, Set indexes) { + this.databaseOperation = databaseOperation; + this.client = databaseOperation.connectionManager(); + this.indexes = ImmutableSet.copyOf(indexes); + } + + @Override + public void insert(InputStream dataScript) { + final IndicesAdminClient indicesAdminClient = client.admin().indices(); + for (String index : indexes) { + IndicesExistsResponse indicesExistsResponse = indicesAdminClient.prepareExists(index) + .execute() + .actionGet(); + + if (indicesExistsResponse.isExists()) { + client.admin().indices().prepareDelete(index).execute().actionGet(); + } + + Indices indices = new Indices(client, new ElasticsearchConfiguration(), new IndexMapping(client)); + if (!indices.create(index)) { + throw new IllegalStateException("Couldn't create index " + index); + } + } + + databaseOperation.insert(dataScript); + } + + @Override + public void deleteAll() { + databaseOperation.deleteAll(); + } + + @Override + public boolean databaseIs(InputStream expectedData) { + return databaseOperation.databaseIs(expectedData); + } + + @Override + public Client connectionManager() { + return databaseOperation.connectionManager(); + } +} diff --git a/graylog2-server/src/test/java/org/graylog2/indexer/nosqlunit/IndexCreatingLoadStrategyFactory.java b/graylog2-server/src/test/java/org/graylog2/indexer/nosqlunit/IndexCreatingLoadStrategyFactory.java new file mode 100644 index 000000000000..a00d463a3448 --- /dev/null +++ b/graylog2-server/src/test/java/org/graylog2/indexer/nosqlunit/IndexCreatingLoadStrategyFactory.java @@ -0,0 +1,44 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog2.indexer.nosqlunit; + +import com.google.common.collect.ImmutableSet; +import com.lordofthejars.nosqlunit.core.DatabaseOperation; +import com.lordofthejars.nosqlunit.core.LoadStrategyEnum; +import com.lordofthejars.nosqlunit.core.LoadStrategyFactory; +import com.lordofthejars.nosqlunit.core.LoadStrategyOperation; +import com.lordofthejars.nosqlunit.core.ReflectionLoadStrategyFactory; + +import java.util.Set; + +public class IndexCreatingLoadStrategyFactory implements LoadStrategyFactory { + private final LoadStrategyFactory loadStrategyFactory; + private final Set indexNames; + + public IndexCreatingLoadStrategyFactory(Set indexNames) { + this.loadStrategyFactory = new ReflectionLoadStrategyFactory(); + this.indexNames = ImmutableSet.copyOf(indexNames); + } + + @Override + @SuppressWarnings("unchecked") + public LoadStrategyOperation getLoadStrategyInstance(LoadStrategyEnum loadStrategyEnum, DatabaseOperation databaseOperation) { + return loadStrategyFactory.getLoadStrategyInstance( + loadStrategyEnum, + new IndexCreatingDatabaseOperation(databaseOperation, indexNames)); + } +} diff --git a/graylog2-server/src/test/java/org/graylog2/indexer/ranges/EsIndexRangeServiceTest.java b/graylog2-server/src/test/java/org/graylog2/indexer/ranges/EsIndexRangeServiceTest.java new file mode 100644 index 000000000000..51fe9d98875f --- /dev/null +++ b/graylog2-server/src/test/java/org/graylog2/indexer/ranges/EsIndexRangeServiceTest.java @@ -0,0 +1,255 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog2.indexer.ranges; + +import com.google.common.collect.ImmutableSet; +import com.lordofthejars.nosqlunit.annotation.UsingDataSet; +import com.lordofthejars.nosqlunit.core.LoadStrategyEnum; +import com.lordofthejars.nosqlunit.elasticsearch.ElasticsearchRule; +import com.lordofthejars.nosqlunit.elasticsearch.EmbeddedElasticsearch; +import org.assertj.jodatime.api.Assertions; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.indices.IndexMissingException; +import org.graylog2.configuration.ElasticsearchConfiguration; +import org.graylog2.database.NotFoundException; +import org.graylog2.indexer.IndexMapping; +import org.graylog2.indexer.indices.Indices; +import org.graylog2.indexer.nosqlunit.IndexCreatingLoadStrategyFactory; +import org.graylog2.indexer.searches.TimestampStats; +import org.graylog2.shared.bindings.providers.ObjectMapperProvider; +import org.graylog2.shared.system.activities.NullActivityWriter; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.runners.MockitoJUnitRunner; + +import javax.inject.Inject; +import java.util.Set; + +import static com.lordofthejars.nosqlunit.elasticsearch.ElasticsearchRule.ElasticsearchRuleBuilder.newElasticsearchRule; +import static com.lordofthejars.nosqlunit.elasticsearch.EmbeddedElasticsearch.EmbeddedElasticsearchRuleBuilder.newEmbeddedElasticsearchRule; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assume.assumeTrue; + +@RunWith(MockitoJUnitRunner.class) +public class EsIndexRangeServiceTest { + @ClassRule + public static final EmbeddedElasticsearch EMBEDDED_ELASTICSEARCH = newEmbeddedElasticsearchRule() + .settings(ImmutableSettings.settingsBuilder().put("action.auto_create_index", false).build()) + .build(); + private static final ImmutableSet INDEX_NAMES = ImmutableSet.of("graylog", "graylog_1", "graylog_2", "ignored"); + private static final ElasticsearchConfiguration ELASTICSEARCH_CONFIGURATION = new ElasticsearchConfiguration() { + @Override + public String getIndexPrefix() { + return "graylog"; + } + }; + + @Rule + public ElasticsearchRule elasticsearchRule; + + @Inject + private Client client; + private Indices indices; + + private EsIndexRangeService indexRangeService; + + public EsIndexRangeServiceTest() { + this.elasticsearchRule = newElasticsearchRule().defaultEmbeddedElasticsearch(); + this.elasticsearchRule.setLoadStrategyFactory(new IndexCreatingLoadStrategyFactory(INDEX_NAMES)); + } + + @Before + public void setUp() throws Exception { + final NullActivityWriter activityWriter = new NullActivityWriter(); + indices = new Indices(client, ELASTICSEARCH_CONFIGURATION, new IndexMapping(client)); + indexRangeService = new EsIndexRangeService(client, activityWriter, new ObjectMapperProvider().get(), indices); + } + + @Test + @UsingDataSet(locations = "EsIndexRangeServiceTest.json", loadStrategy = LoadStrategyEnum.CLEAN_INSERT) + public void getReturnsExistingIndexRange() throws Exception { + IndexRange indexRange = indexRangeService.get("graylog_1"); + + assertThat(indexRange.indexName()).isEqualTo("graylog_1"); + assertThat(indexRange.begin()).isEqualTo(new DateTime(2015, 1, 1, 0, 0, DateTimeZone.UTC)); + assertThat(indexRange.end()).isEqualTo(new DateTime(2015, 1, 2, 0, 0, DateTimeZone.UTC)); + assertThat(indexRange.calculatedAt()).isEqualTo(new DateTime(2015, 1, 1, 0, 0, DateTimeZone.UTC)); + assertThat(indexRange.calculationDuration()).isEqualTo(23); + } + + @Test(expected = NotFoundException.class) + public void getThrowsNotFoundException() throws Exception { + indexRangeService.get("does-not-exist"); + } + + @Test + @UsingDataSet(locations = "EsIndexRangeServiceTest.json", loadStrategy = LoadStrategyEnum.CLEAN_INSERT) + public void findReturnsIndexRangesWithinGivenRange() throws Exception { + final DateTime begin = new DateTime(2015, 1, 1, 0, 0, DateTimeZone.UTC); + final DateTime end = new DateTime(2015, 1, 2, 0, 0, DateTimeZone.UTC); + Set indexRanges = indexRangeService.find(begin, end); + + assertThat(indexRanges).hasSize(1); + } + + @Test + @UsingDataSet(locations = "EsIndexRangeServiceTest.json", loadStrategy = LoadStrategyEnum.CLEAN_INSERT) + public void findReturnsNothingBeforeBegin() throws Exception { + final DateTime begin = new DateTime(2016, 1, 1, 0, 0, DateTimeZone.UTC); + final DateTime end = new DateTime(2016, 1, 2, 0, 0, DateTimeZone.UTC); + Set indexRanges = indexRangeService.find(begin, end); + + assertThat(indexRanges).isEmpty(); + } + + @Test + @UsingDataSet(locations = "EsIndexRangeServiceTest.json", loadStrategy = LoadStrategyEnum.CLEAN_INSERT) + public void findAllReturnsAllIndexRanges() throws Exception { + assertThat(indexRangeService.findAll()).hasSize(2); + } + + @Test + @UsingDataSet(locations = "EsIndexRangeServiceTest.json", loadStrategy = LoadStrategyEnum.CLEAN_INSERT) + public void calculateRangeReturnsIndexRange() throws Exception { + final String index = "graylog"; + final DateTime min = new DateTime(2015, 1, 1, 1, 0, DateTimeZone.UTC); + final DateTime max = new DateTime(2015, 1, 1, 5, 0, DateTimeZone.UTC); + final IndexRange indexRange = indexRangeService.calculateRange(index); + + assertThat(indexRange.indexName()).isEqualTo(index); + assertThat(indexRange.begin()).isEqualTo(min); + assertThat(indexRange.end()).isEqualTo(max); + Assertions.assertThat(indexRange.calculatedAt()).isEqualToIgnoringHours(DateTime.now(DateTimeZone.UTC)); + } + + @Test + @UsingDataSet(locations = "EsIndexRangeServiceTest-EmptyIndex.json", loadStrategy = LoadStrategyEnum.CLEAN_INSERT) + public void testCalculateRangeWithEmptyIndex() throws Exception { + final String index = "graylog"; + final IndexRange range = indexRangeService.calculateRange(index); + + assertThat(range).isNotNull(); + assertThat(range.indexName()).isEqualTo(index); + assertThat(range.begin()).isEqualTo(new DateTime(0L, DateTimeZone.UTC)); + assertThat(range.end()).isEqualTo(new DateTime(0L, DateTimeZone.UTC)); + } + + @Test(expected = IndexMissingException.class) + public void testCalculateRangeWithNonExistingIndex() throws Exception { + indexRangeService.calculateRange("does-not-exist"); + } + + @Test + @UsingDataSet(loadStrategy = LoadStrategyEnum.DELETE_ALL) + public void savePersistsIndexRange() throws Exception { + final String indexName = "graylog"; + final DateTime begin = new DateTime(2015, 1, 1, 0, 0, DateTimeZone.UTC); + final DateTime end = new DateTime(2015, 1, 2, 0, 0, DateTimeZone.UTC); + final DateTime now = DateTime.now(DateTimeZone.UTC); + final IndexRange indexRange = IndexRange.create(indexName, begin, end, now, 42); + + indexRangeService.save(indexRange); + + final IndexRange result = indexRangeService.get(indexName); + assertThat(result.indexName()).isEqualTo(indexName); + assertThat(result.begin()).isEqualTo(begin); + assertThat(result.end()).isEqualTo(end); + assertThat(result.calculatedAt()).isEqualTo(now); + assertThat(result.calculationDuration()).isEqualTo(42); + } + + @Test + @UsingDataSet(locations = "EsIndexRangeServiceTest.json", loadStrategy = LoadStrategyEnum.CLEAN_INSERT) + public void savePersistsIndexRangeInReadOnlyIndex() throws Exception { + final String indexName = "graylog_read_only"; + final DateTime begin = new DateTime(2015, 1, 1, 0, 0, DateTimeZone.UTC); + final DateTime end = new DateTime(2015, 1, 2, 0, 0, DateTimeZone.UTC); + final DateTime now = DateTime.now(DateTimeZone.UTC); + final IndexRange indexRange = IndexRange.create(indexName, begin, end, now, 42); + + try { + indices.create(indexName); + indices.setReadOnly(indexName); + assumeTrue(indices.isReadOnly(indexName)); + indexRangeService.save(indexRange); + + assertThat(indices.isReadOnly(indexName)).isTrue(); + + final IndexRange result = indexRangeService.get(indexName); + assertThat(result.indexName()).isEqualTo(indexName); + assertThat(result.begin()).isEqualTo(begin); + assertThat(result.end()).isEqualTo(end); + assertThat(result.calculatedAt()).isEqualTo(now); + assertThat(result.calculationDuration()).isEqualTo(42); + } finally { + indices.delete(indexName); + } + } + + @Test + @UsingDataSet(loadStrategy = LoadStrategyEnum.DELETE_ALL) + public void saveOverwritesExistingIndexRange() throws Exception { + final String indexName = "graylog"; + final DateTime begin = new DateTime(2015, 1, 1, 0, 0, DateTimeZone.UTC); + final DateTime end = new DateTime(2015, 1, 2, 0, 0, DateTimeZone.UTC); + final DateTime now = DateTime.now(DateTimeZone.UTC); + final IndexRange indexRangeBefore = IndexRange.create(indexName, begin, end, now, 1); + final IndexRange indexRangeAfter = IndexRange.create(indexName, begin, end, now, 2); + + indexRangeService.save(indexRangeBefore); + + final IndexRange before = indexRangeService.get(indexName); + assertThat(before.calculationDuration()).isEqualTo(1); + + indexRangeService.save(indexRangeAfter); + + final IndexRange after = indexRangeService.get(indexName); + assertThat(after.calculationDuration()).isEqualTo(2); + } + + + @Test + @UsingDataSet(loadStrategy = LoadStrategyEnum.CLEAN_INSERT) + public void testTimestampStatsOfIndex() throws Exception { + TimestampStats stats = indexRangeService.timestampStatsOfIndex("graylog"); + + assertThat(stats.min()).isEqualTo(new DateTime(2015, 1, 1, 1, 0, DateTimeZone.UTC)); + assertThat(stats.max()).isEqualTo(new DateTime(2015, 1, 1, 5, 0, DateTimeZone.UTC)); + assertThat(stats.avg()).isEqualTo(new DateTime(2015, 1, 1, 3, 0, DateTimeZone.UTC)); + } + + @Test + @UsingDataSet(locations = "EsIndexRangeServiceTest-EmptyIndex.json", loadStrategy = LoadStrategyEnum.CLEAN_INSERT) + public void testTimestampStatsOfIndexWithEmptyIndex() throws Exception { + TimestampStats stats = indexRangeService.timestampStatsOfIndex("graylog"); + + assertThat(stats.min()).isEqualTo(new DateTime(0L, DateTimeZone.UTC)); + assertThat(stats.max()).isEqualTo(new DateTime(0L, DateTimeZone.UTC)); + assertThat(stats.avg()).isEqualTo(new DateTime(0L, DateTimeZone.UTC)); + } + + @Test(expected = IndexMissingException.class) + public void testTimestampStatsOfIndexWithNonExistingIndex() throws Exception { + indexRangeService.timestampStatsOfIndex("does-not-exist"); + } +} \ No newline at end of file diff --git a/graylog2-server/src/test/java/org/graylog2/indexer/ranges/IndexRangeServiceImplTest.java b/graylog2-server/src/test/java/org/graylog2/indexer/ranges/IndexRangeServiceImplTest.java deleted file mode 100644 index 6e1edfe314a8..000000000000 --- a/graylog2-server/src/test/java/org/graylog2/indexer/ranges/IndexRangeServiceImplTest.java +++ /dev/null @@ -1,136 +0,0 @@ -/** - * This file is part of Graylog. - * - * Graylog is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Graylog is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Graylog. If not, see . - */ -package org.graylog2.indexer.ranges; - -import com.google.common.collect.ImmutableMap; -import com.google.common.primitives.Ints; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.core.LoadStrategyEnum; -import com.lordofthejars.nosqlunit.mongodb.InMemoryMongoDb; -import org.graylog2.database.MongoConnectionRule; -import org.graylog2.database.NotFoundException; -import org.graylog2.shared.system.activities.NullActivityWriter; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; - -import java.util.List; - -import static com.lordofthejars.nosqlunit.mongodb.InMemoryMongoDb.InMemoryMongoRuleBuilder.newInMemoryMongoDbRule; -import static org.assertj.core.api.Assertions.assertThat; - -public class IndexRangeServiceImplTest { - @ClassRule - public static final InMemoryMongoDb IN_MEMORY_MONGO_DB = newInMemoryMongoDbRule().build(); - - @Rule - public MongoConnectionRule mongoRule = MongoConnectionRule.build("test"); - - private IndexRangeServiceImpl indexRangeService; - - @Before - public void setUp() throws Exception { - indexRangeService = new IndexRangeServiceImpl(mongoRule.getMongoConnection(), new NullActivityWriter()); - } - - @Test - @UsingDataSet(locations = "IndexRangeServiceImplTest.json", loadStrategy = LoadStrategyEnum.CLEAN_INSERT) - public void getReturnsExistingIndexRange() throws Exception { - IndexRange indexRange = indexRangeService.get("graylog_1"); - - assertThat(indexRange.getIndexName()).isEqualTo("graylog_1"); - assertThat(indexRange.getStart()).isEqualTo(new DateTime(2015, 1, 1, 0, 0, DateTimeZone.UTC)); - assertThat(indexRange.getCalculatedAt()).isEqualTo(new DateTime(2015, 1, 1, 0, 0, DateTimeZone.UTC)); - assertThat(indexRange.getCalculationTookMs()).isEqualTo(23); - } - - @Test(expected = NotFoundException.class) - public void getThrowsNotFoundException() throws Exception { - indexRangeService.get("does-not-exist"); - } - - @Test - @UsingDataSet(locations = "IndexRangeServiceImplTest.json", loadStrategy = LoadStrategyEnum.CLEAN_INSERT) - public void getFromReturnsIndexRangesAfterTimestamp() throws Exception { - final long millis = new DateTime(2015, 1, 1, 0, 0, DateTimeZone.UTC).getMillis(); - List indexRanges = indexRangeService.getFrom(Ints.saturatedCast(millis / 1000L)); - - assertThat(indexRanges) - .hasSize(2) - .isSortedAccordingTo(new IndexRangeComparator()); - } - - @Test - @UsingDataSet(locations = "IndexRangeServiceImplTest.json", loadStrategy = LoadStrategyEnum.CLEAN_INSERT) - public void getFromReturnsNothingBeforeTimestamp() throws Exception { - final long millis = new DateTime(2016, 1, 1, 0, 0, DateTimeZone.UTC).getMillis(); - List indexRanges = indexRangeService.getFrom(Ints.saturatedCast(millis / 1000L)); - - assertThat(indexRanges).isEmpty(); - } - - @Test - @UsingDataSet(locations = "IndexRangeServiceImplTest.json", loadStrategy = LoadStrategyEnum.CLEAN_INSERT) - public void destroyRemovesIndexRange() throws Exception { - indexRangeService.destroy("graylog_1"); - - List indexRanges = indexRangeService.getFrom(0); - - assertThat(indexRanges).hasSize(1); - assertThat(indexRanges.get(0).getIndexName()).isEqualTo("graylog_2"); - } - - @Test - @UsingDataSet(locations = "IndexRangeServiceImplTest.json", loadStrategy = LoadStrategyEnum.CLEAN_INSERT) - public void destroyRemovesIgnoresNonExistingIndexRange() throws Exception { - indexRangeService.destroy("does-not-exist"); - - final long millis = new DateTime(2015, 1, 1, 0, 0, DateTimeZone.UTC).getMillis(); - List indexRanges = indexRangeService.getFrom(Ints.saturatedCast(millis / 1000L)); - - assertThat(indexRanges).hasSize(2); - } - - @Test - public void createReturnsIndexRange() throws Exception { - final DateTime dateTime = new DateTime(2015, 1, 1, 0, 0, DateTimeZone.UTC); - final int timestamp = Ints.saturatedCast(dateTime.getMillis() / 1000L); - IndexRange indexRange = indexRangeService.create(ImmutableMap.of( - "index", "graylog_3", - "start", timestamp, - "calculated_at", timestamp, - "took_ms", 42 - ) - ); - - assertThat(indexRange.getIndexName()).isEqualTo("graylog_3"); - assertThat(indexRange.getStart()).isEqualTo(dateTime); - assertThat(indexRange.getCalculatedAt()).isEqualTo(dateTime); - assertThat(indexRange.getCalculationTookMs()).isEqualTo(42); - } - - @Test - @UsingDataSet(locations = "IndexRangeServiceImplTest.json", loadStrategy = LoadStrategyEnum.CLEAN_INSERT) - public void destroyAllKillsAllIndexRanges() throws Exception { - indexRangeService.destroyAll(); - - assertThat(indexRangeService.getFrom(0)).isEmpty(); - } -} \ No newline at end of file diff --git a/graylog2-server/src/test/java/org/graylog2/indexer/ranges/IndexRangeTest.java b/graylog2-server/src/test/java/org/graylog2/indexer/ranges/IndexRangeTest.java new file mode 100644 index 000000000000..9badcb6881d9 --- /dev/null +++ b/graylog2-server/src/test/java/org/graylog2/indexer/ranges/IndexRangeTest.java @@ -0,0 +1,65 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog2.indexer.ranges; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.jayway.jsonpath.Configuration; +import com.jayway.jsonpath.JsonPath; +import org.graylog2.bindings.providers.ServerObjectMapperProvider; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class IndexRangeTest { + @Test + public void testCreate() throws Exception { + String indexName = "test"; + DateTime begin = new DateTime(2015, 1, 1, 0, 0, DateTimeZone.UTC); + DateTime end = new DateTime(2015, 2, 1, 0, 0, DateTimeZone.UTC); + DateTime calculatedAt = new DateTime(2015, 2, 1, 0, 0, DateTimeZone.UTC); + int calculationDuration = 42; + IndexRange indexRange = IndexRange.create(indexName, begin, end, calculatedAt, calculationDuration); + + assertThat(indexRange.indexName()).isEqualTo(indexName); + assertThat(indexRange.begin()).isEqualTo(begin); + assertThat(indexRange.end()).isEqualTo(end); + assertThat(indexRange.calculatedAt()).isEqualTo(calculatedAt); + assertThat(indexRange.calculationDuration()).isEqualTo(calculationDuration); + } + + @Test + public void testJsonMapping() throws Exception { + String indexName = "test"; + DateTime begin = new DateTime(2015, 1, 1, 0, 0, DateTimeZone.UTC); + DateTime end = new DateTime(2015, 2, 1, 0, 0, DateTimeZone.UTC); + DateTime calculatedAt = new DateTime(2015, 2, 1, 0, 0, DateTimeZone.UTC); + int calculationDuration = 42; + IndexRange indexRange = IndexRange.create(indexName, begin, end, calculatedAt, calculationDuration); + + ObjectMapper objectMapper = new ServerObjectMapperProvider().get(); + String json = objectMapper.writeValueAsString(indexRange); + Object document = Configuration.defaultConfiguration().jsonProvider().parse(json); + + assertThat(JsonPath.read(document, "$.index_name")).isEqualTo(indexName); + assertThat(JsonPath.read(document, "$.begin")).asString().isEqualTo(begin.toString()); + assertThat(JsonPath.read(document, "$.end")).isEqualTo(end.toString()); + assertThat(JsonPath.read(document, "$.calculated_at")).isEqualTo(calculatedAt.toString()); + assertThat(JsonPath.read(document, "$.took_ms")).isEqualTo(calculationDuration); + } +} \ No newline at end of file diff --git a/graylog2-server/src/test/java/org/graylog2/indexer/ranges/RebuildIndexRangesJobTest.java b/graylog2-server/src/test/java/org/graylog2/indexer/ranges/RebuildIndexRangesJobTest.java deleted file mode 100644 index ce56dda4f7ab..000000000000 --- a/graylog2-server/src/test/java/org/graylog2/indexer/ranges/RebuildIndexRangesJobTest.java +++ /dev/null @@ -1,104 +0,0 @@ -/** - * This file is part of Graylog. - * - * Graylog is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Graylog is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Graylog. If not, see . - */ -package org.graylog2.indexer.ranges; - -import com.codahale.metrics.MetricRegistry; -import com.google.common.primitives.Ints; -import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.core.LoadStrategyEnum; -import com.lordofthejars.nosqlunit.elasticsearch.ElasticsearchRule; -import com.lordofthejars.nosqlunit.elasticsearch.EmbeddedElasticsearch; -import org.elasticsearch.client.Client; -import org.elasticsearch.indices.IndexMissingException; -import org.graylog2.Configuration; -import org.graylog2.indexer.Deflector; -import org.graylog2.indexer.searches.Searches; -import org.graylog2.indexer.searches.SearchesTest; -import org.graylog2.shared.system.activities.NullActivityWriter; -import org.joda.time.DateTime; -import org.joda.time.DateTimeUtils; -import org.joda.time.DateTimeZone; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mock; -import org.mockito.runners.MockitoJUnitRunner; - -import javax.inject.Inject; -import java.util.Collections; -import java.util.Map; - -import static com.lordofthejars.nosqlunit.elasticsearch.ElasticsearchRule.ElasticsearchRuleBuilder.newElasticsearchRule; -import static com.lordofthejars.nosqlunit.elasticsearch.EmbeddedElasticsearch.EmbeddedElasticsearchRuleBuilder.newEmbeddedElasticsearchRule; -import static org.assertj.core.api.Assertions.assertThat; - -@RunWith(MockitoJUnitRunner.class) -public class RebuildIndexRangesJobTest { - private static final String INDEX_NAME = "graylog"; - @ClassRule - public static final EmbeddedElasticsearch EMBEDDED_ELASTICSEARCH = newEmbeddedElasticsearchRule().build(); - @Rule - public ElasticsearchRule elasticsearchRule; - - @Mock - private Deflector deflector; - @Inject - private Client client; - @Mock - private IndexRangeService indexRangeService; - private RebuildIndexRangesJob rebuildIndexRangesJob; - - public RebuildIndexRangesJobTest() { - this.elasticsearchRule = newElasticsearchRule().defaultEmbeddedElasticsearch(); - this.elasticsearchRule.setLoadStrategyFactory(new SearchesTest.IndexCreatingLoadStrategyFactory(Collections.singleton(INDEX_NAME))); - } - - @Before - public void setUp() throws Exception { - final MetricRegistry metricRegistry = new MetricRegistry(); - final Searches searches = new Searches(new Configuration(), deflector, indexRangeService, client, metricRegistry); - rebuildIndexRangesJob = new RebuildIndexRangesJob(deflector, searches, new NullActivityWriter(), indexRangeService); - } - - @Test - @UsingDataSet(loadStrategy = LoadStrategyEnum.CLEAN_INSERT) - public void testCalculateRange() throws Exception { - final Map range = rebuildIndexRangesJob.calculateRange(INDEX_NAME); - - assertThat(range).isNotNull(); - assertThat(range.get("index")).isEqualTo(INDEX_NAME); - assertThat(range.get("start")).isEqualTo(Ints.saturatedCast(new DateTime(2015, 1, 1, 12, 0, DateTimeZone.UTC).getMillis() / 1000L)); - } - - @Test - @UsingDataSet(locations = "RebuildIndexRangesJobTest-EmptyIndex.json", loadStrategy = LoadStrategyEnum.CLEAN_INSERT) - public void testCalculateRangeWithEmptyIndex() throws Exception { - DateTimeUtils.setCurrentMillisFixed(new DateTime(2014, 1, 1, 0, 0, DateTimeZone.UTC).getMillis()); - final Map range = rebuildIndexRangesJob.calculateRange(INDEX_NAME); - - assertThat(range).isNotNull(); - assertThat(range.get("index")).isEqualTo(INDEX_NAME); - assertThat(range.get("start")).isEqualTo(Ints.saturatedCast(new DateTime(2014, 1, 1, 0, 0, DateTimeZone.UTC).getMillis() / 1000L)); - } - - @Test(expected = IndexMissingException.class) - public void testCalculateRangeWithNonExistingIndex() throws Exception { - rebuildIndexRangesJob.calculateRange("does-not-exist"); - } -} \ No newline at end of file diff --git a/graylog2-server/src/test/java/org/graylog2/indexer/searches/SearchesTest.java b/graylog2-server/src/test/java/org/graylog2/indexer/searches/SearchesTest.java index ffc12327e222..8709cdb1ae94 100644 --- a/graylog2-server/src/test/java/org/graylog2/indexer/searches/SearchesTest.java +++ b/graylog2-server/src/test/java/org/graylog2/indexer/searches/SearchesTest.java @@ -19,25 +19,19 @@ import com.codahale.metrics.Histogram; import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.Timer; -import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSortedSet; import com.lordofthejars.nosqlunit.annotation.UsingDataSet; -import com.lordofthejars.nosqlunit.core.DatabaseOperation; import com.lordofthejars.nosqlunit.core.LoadStrategyEnum; -import com.lordofthejars.nosqlunit.core.LoadStrategyFactory; -import com.lordofthejars.nosqlunit.core.LoadStrategyOperation; -import com.lordofthejars.nosqlunit.core.ReflectionLoadStrategyFactory; import com.lordofthejars.nosqlunit.elasticsearch.ElasticsearchRule; import com.lordofthejars.nosqlunit.elasticsearch.EmbeddedElasticsearch; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; import org.elasticsearch.client.Client; -import org.elasticsearch.client.IndicesAdminClient; import org.elasticsearch.indices.IndexMissingException; import org.elasticsearch.search.SearchHit; import org.graylog2.Configuration; -import org.graylog2.configuration.ElasticsearchConfiguration; import org.graylog2.indexer.Deflector; -import org.graylog2.indexer.indices.Indices; +import org.graylog2.indexer.nosqlunit.IndexCreatingLoadStrategyFactory; import org.graylog2.indexer.ranges.IndexRange; +import org.graylog2.indexer.ranges.IndexRangeComparator; import org.graylog2.indexer.ranges.IndexRangeService; import org.graylog2.indexer.results.CountResult; import org.graylog2.indexer.results.FieldStatsResult; @@ -45,7 +39,6 @@ import org.graylog2.indexer.results.TermsResult; import org.graylog2.indexer.results.TermsStatsResult; import org.graylog2.indexer.searches.timeranges.AbsoluteRange; -import org.graylog2.plugin.database.validators.Validator; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.junit.Before; @@ -53,20 +46,18 @@ import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; import javax.inject.Inject; -import java.io.InputStream; import java.util.Collections; -import java.util.List; import java.util.Map; -import java.util.Set; +import java.util.SortedSet; import static com.lordofthejars.nosqlunit.elasticsearch.ElasticsearchRule.ElasticsearchRuleBuilder.newElasticsearchRule; import static com.lordofthejars.nosqlunit.elasticsearch.EmbeddedElasticsearch.EmbeddedElasticsearchRuleBuilder.newEmbeddedElasticsearchRule; import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Matchers.anyInt; -import static org.mockito.Mockito.mock; +import static org.mockito.Matchers.any; import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) @@ -80,55 +71,39 @@ public class SearchesTest { public ElasticsearchRule elasticsearchRule; private static final String INDEX_NAME = "graylog"; - private static final List INDEX_RANGES = Collections.singletonList(new IndexRange() { - @Override - public String getIndexName() { - return INDEX_NAME; - } - - @Override - public DateTime getCalculatedAt() { - return DateTime.now(); - } - - @Override - public DateTime getStart() { - return new DateTime(2015, 1, 1, 0, 0, DateTimeZone.UTC); - } - - @Override - public int getCalculationTookMs() { - return 0; - } - - @Override - public String getId() { - return "id"; - } - - @Override - public Map getFields() { - return Collections.emptyMap(); - } - - @Override - public Map getValidations() { - return Collections.emptyMap(); - } - - @Override - public Map getEmbeddedValidations(String key) { - return Collections.emptyMap(); - } - - @Override - public Map asMap() { - return Collections.emptyMap(); - } - }); - - private final Deflector deflector = mock(Deflector.class); - private final IndexRangeService indexRangeService = mock(IndexRangeService.class); + private static final SortedSet INDEX_RANGES = ImmutableSortedSet + .orderedBy(new IndexRangeComparator()) + .add(new IndexRange() { + @Override + public String indexName() { + return INDEX_NAME; + } + + @Override + public DateTime calculatedAt() { + return DateTime.now(); + } + + @Override + public DateTime end() { + return new DateTime(2015, 1, 1, 0, 0, DateTimeZone.UTC); + } + + @Override + public int calculationDuration() { + return 0; + } + + @Override + public DateTime begin() { + return new DateTime(0L, DateTimeZone.UTC); + } + }).build(); + + @Mock + private Deflector deflector; + @Mock + private IndexRangeService indexRangeService; private MetricRegistry metricRegistry; private Searches searches; @@ -143,7 +118,7 @@ public SearchesTest() { @Before public void setUp() throws Exception { - when(indexRangeService.getFrom(anyInt())).thenReturn(INDEX_RANGES); + when(indexRangeService.find(any(DateTime.class), any(DateTime.class))).thenReturn(INDEX_RANGES); metricRegistry = new MetricRegistry(); searches = new Searches(new Configuration(), deflector, indexRangeService, client, metricRegistry); } @@ -350,153 +325,4 @@ public void fieldHistogramRecordsMetrics() throws Exception { assertThat(histogram.getCount()).isEqualTo(1L); assertThat(histogram.getSnapshot().getValues()).containsExactly(86400L); } - - @Test - @UsingDataSet(loadStrategy = LoadStrategyEnum.CLEAN_INSERT) - public void testFirstOfIndex() throws Exception { - SearchHit searchHit = searches.firstOfIndex("graylog"); - - assertThat(searchHit.getSource()).containsKey("timestamp"); - assertThat(searchHit.getSource().get("timestamp")).isEqualTo("2015-01-01 05:00:00.000"); - } - - @Test - @UsingDataSet(loadStrategy = LoadStrategyEnum.CLEAN_INSERT) - public void testLastOfIndex() throws Exception { - SearchHit searchHit = searches.lastOfIndex("graylog"); - - assertThat(searchHit.getSource()).containsKey("timestamp"); - assertThat(searchHit.getSource().get("timestamp")).isEqualTo("2015-01-01 01:00:00.000"); - } - - @Test - @UsingDataSet(loadStrategy = LoadStrategyEnum.CLEAN_INSERT) - public void testFindNewestMessageTimestampOfIndex() throws Exception { - DateTime dateTime = searches.findNewestMessageTimestampOfIndex("graylog"); - - assertThat(dateTime).isEqualTo(new DateTime(2015, 1, 1, 5, 0, DateTimeZone.UTC)); - } - - @Test - @UsingDataSet(locations = "SearchesTest-EmptyIndex.json", loadStrategy = LoadStrategyEnum.CLEAN_INSERT) - public void testFindNewestMessageTimestampOfIndexWithEmptyIndex() throws Exception { - DateTime dateTime = searches.findNewestMessageTimestampOfIndex("graylog"); - - assertThat(dateTime).isNull(); - } - - @Test(expected = IndexMissingException.class) - public void testFindNewestMessageTimestampOfIndexWithNonExistingIndex() throws Exception { - searches.findNewestMessageTimestampOfIndex("does-not-exist"); - } - - @Test - @UsingDataSet(loadStrategy = LoadStrategyEnum.CLEAN_INSERT) - public void testFindOldestMessageTimestampOfIndex() throws Exception { - DateTime dateTime = searches.findOldestMessageTimestampOfIndex("graylog"); - - assertThat(dateTime).isEqualTo(new DateTime(2015, 1, 1, 1, 0, DateTimeZone.UTC)); - } - - @Test - @UsingDataSet(locations = "SearchesTest-EmptyIndex.json", loadStrategy = LoadStrategyEnum.CLEAN_INSERT) - public void testFindOldestMessageTimestampOfIndexWithEmptyIndex() throws Exception { - DateTime dateTime = searches.findOldestMessageTimestampOfIndex("graylog"); - - assertThat(dateTime).isNull(); - } - - @Test(expected = IndexMissingException.class) - public void testFindOldestMessageTimestampOfIndexWithNonExistingIndex() throws Exception { - searches.findOldestMessageTimestampOfIndex("does-not-exist"); - } - - @Test - @UsingDataSet(loadStrategy = LoadStrategyEnum.CLEAN_INSERT) - public void testTimestampStatsOfIndex() throws Exception { - TimestampStats stats = searches.timestampStatsOfIndex("graylog"); - - assertThat(stats.min()).isEqualTo(new DateTime(2015, 1, 1, 1, 0, DateTimeZone.UTC)); - assertThat(stats.max()).isEqualTo(new DateTime(2015, 1, 1, 5, 0, DateTimeZone.UTC)); - assertThat(stats.avg()).isEqualTo(new DateTime(2015, 1, 1, 3, 0, DateTimeZone.UTC)); - } - - @Test - @UsingDataSet(locations = "SearchesTest-EmptyIndex.json", loadStrategy = LoadStrategyEnum.CLEAN_INSERT) - public void testTimestampStatsOfIndexWithEmptyIndex() throws Exception { - TimestampStats stats = searches.timestampStatsOfIndex("graylog"); - - assertThat(stats).isNull(); - } - - @Test(expected = IndexMissingException.class) - public void testTimestampStatsOfIndexWithNonExistingIndex() throws Exception { - searches.timestampStatsOfIndex("does-not-exist"); - } - - public static class IndexCreatingLoadStrategyFactory implements LoadStrategyFactory { - private final LoadStrategyFactory loadStrategyFactory; - private final Set indexNames; - - public IndexCreatingLoadStrategyFactory(Set indexNames) { - this.loadStrategyFactory = new ReflectionLoadStrategyFactory(); - this.indexNames = ImmutableSet.copyOf(indexNames); - } - - @Override - public LoadStrategyOperation getLoadStrategyInstance(LoadStrategyEnum loadStrategyEnum, DatabaseOperation databaseOperation) { - return loadStrategyFactory.getLoadStrategyInstance( - loadStrategyEnum, - new IndexCreatingDatabaseOperation(databaseOperation, indexNames)); - } - } - - public static class IndexCreatingDatabaseOperation implements DatabaseOperation { - private final DatabaseOperation databaseOperation; - private final Client client; - private final Set indexes; - - public IndexCreatingDatabaseOperation(DatabaseOperation databaseOperation, Set indexes) { - this.databaseOperation = databaseOperation; - this.client = databaseOperation.connectionManager(); - this.indexes = ImmutableSet.copyOf(indexes); - } - - @Override - public void insert(InputStream dataScript) { - final IndicesAdminClient indicesAdminClient = client.admin().indices(); - final String[] indexNames = indexes.toArray(new String[indexes.size()]); - IndicesExistsResponse indicesExistsResponse = indicesAdminClient.prepareExists(indexNames) - .execute() - .actionGet(); - - if (indicesExistsResponse.isExists()) { - client.admin().indices().prepareDelete(indexNames).execute().actionGet(); - } - - Indices indices = new Indices(client, new ElasticsearchConfiguration()); - for (String index : indexes) { - if (!indices.create(index)) { - throw new IllegalStateException("Couldn't create index " + index); - } - } - - databaseOperation.insert(dataScript); - } - - @Override - public void deleteAll() { - databaseOperation.deleteAll(); - } - - @Override - public boolean databaseIs(InputStream expectedData) { - return databaseOperation.databaseIs(expectedData); - } - - @Override - public Client connectionManager() { - return databaseOperation.connectionManager(); - } - } } diff --git a/graylog2-server/src/test/java/org/graylog2/indexer/searches/TimestampStatsTest.java b/graylog2-server/src/test/java/org/graylog2/indexer/searches/TimestampStatsTest.java new file mode 100644 index 000000000000..ab8516c45767 --- /dev/null +++ b/graylog2-server/src/test/java/org/graylog2/indexer/searches/TimestampStatsTest.java @@ -0,0 +1,44 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog2.indexer.searches; + +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.junit.Test; + +import static org.assertj.jodatime.api.Assertions.assertThat; + +public class TimestampStatsTest { + @Test + public void testCreate() throws Exception { + DateTime min = new DateTime(2015, 1, 1, 0, 0, DateTimeZone.UTC); + DateTime max = new DateTime(2015, 1, 3, 0, 0, DateTimeZone.UTC); + DateTime avg = new DateTime(2015, 1, 2, 0, 0, DateTimeZone.UTC); + TimestampStats timestampStats = TimestampStats.create(min, max, avg); + + assertThat(timestampStats.min()).isEqualTo(min); + assertThat(timestampStats.max()).isEqualTo(max); + assertThat(timestampStats.avg()).isEqualTo(avg); + } + + @Test + public void testEmptyInstance() throws Exception { + assertThat(TimestampStats.EMPTY.min()).isEqualTo(new DateTime(0L, DateTimeZone.UTC)); + assertThat(TimestampStats.EMPTY.max()).isEqualTo(new DateTime(0L, DateTimeZone.UTC)); + assertThat(TimestampStats.EMPTY.avg()).isEqualTo(new DateTime(0L, DateTimeZone.UTC)); + } +} \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog2/indexer/searches/SearchesTest-EmptyIndex.json b/graylog2-server/src/test/resources/org/graylog2/indexer/ranges/EsIndexRangeServiceTest-EmptyIndex.json similarity index 100% rename from graylog2-server/src/test/resources/org/graylog2/indexer/searches/SearchesTest-EmptyIndex.json rename to graylog2-server/src/test/resources/org/graylog2/indexer/ranges/EsIndexRangeServiceTest-EmptyIndex.json diff --git a/graylog2-server/src/test/resources/org/graylog2/indexer/ranges/EsIndexRangeServiceTest.json b/graylog2-server/src/test/resources/org/graylog2/indexer/ranges/EsIndexRangeServiceTest.json new file mode 100644 index 000000000000..79d5b9125c49 --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog2/indexer/ranges/EsIndexRangeServiceTest.json @@ -0,0 +1,252 @@ +{ + "documents": [ + { + "document": [ + { + "index": { + "indexName": "graylog", + "indexType": "message", + "indexId": "0" + } + }, + { + "data": { + "source": "example.org", + "message": "Hi", + "timestamp": "2015-01-01 01:00:00.000" + } + } + ] + }, + { + "document": [ + { + "index": { + "indexName": "graylog", + "indexType": "message", + "indexId": "1" + } + }, + { + "data": { + "source": "example.org", + "message": "Ho", + "timestamp": "2015-01-01 02:00:00.000", + "n": 1 + } + } + ] + }, + { + "document": [ + { + "index": { + "indexName": "graylog", + "indexType": "message", + "indexId": "2" + } + }, + { + "data": { + "source": "example.org", + "message": "Hi", + "timestamp": "2015-01-01 03:00:00.000", + "n": 1 + } + } + ] + }, + { + "document": [ + { + "index": { + "indexName": "graylog", + "indexType": "message", + "indexId": "3" + } + }, + { + "data": { + "source": "example.org", + "message": "Ho", + "timestamp": "2015-01-01 04:00:00.000", + "n": 2 + } + } + ] + }, + { + "document": [ + { + "index": { + "indexName": "graylog", + "indexType": "message", + "indexId": "4" + } + }, + { + "data": { + "source": "example.org", + "message": "Hi", + "timestamp": "2015-01-01 05:00:00.000", + "n": 2 + } + } + ] + }, + { + "document": [ + { + "index": { + "indexName": "graylog", + "indexType": "message", + "indexId": "5" + } + }, + { + "data": { + "source": "example.com", + "message": "Ho", + "timestamp": "2015-01-01 01:00:00.000" + } + } + ] + }, + { + "document": [ + { + "index": { + "indexName": "graylog", + "indexType": "message", + "indexId": "6" + } + }, + { + "data": { + "source": "example.com", + "message": "Hi", + "timestamp": "2015-01-01 02:00:00.000", + "n": 3 + } + } + ] + }, + { + "document": [ + { + "index": { + "indexName": "graylog", + "indexType": "message", + "indexId": "7" + } + }, + { + "data": { + "source": "example.com", + "message": "Ho", + "timestamp": "2015-01-01 03:00:00.000", + "n": 3 + } + } + ] + }, + { + "document": [ + { + "index": { + "indexName": "graylog", + "indexType": "message", + "indexId": "8" + } + }, + { + "data": { + "source": "example.com", + "message": "Hi", + "timestamp": "2015-01-01 04:00:00.000", + "n": 3 + } + } + ] + }, + { + "document": [ + { + "index": { + "indexName": "graylog", + "indexType": "message", + "indexId": "9" + } + }, + { + "data": { + "source": "example.com", + "message": "Ho", + "timestamp": "2015-01-01 05:00:00.000", + "n": 4 + } + } + ] + }, + { + "document": [ + { + "index": { + "indexName": "graylog_1", + "indexType": "index_range", + "indexId": "graylog_1" + } + }, + { + "data": { + "_id": "graylog_1", + "calculated_at": "2015-01-01T00:00:00.000Z", + "begin": "2015-01-01T00:00:00.000Z", + "end": "2015-01-02T00:00:00.000Z", + "took_ms": 23 + } + } + ] + }, + { + "document": [ + { + "index": { + "indexName": "graylog_2", + "indexType": "index_range", + "indexId": "graylog_2" + } + }, + { + "data": { + "_id": "graylog_2", + "calculated_at": "2015-01-02T00:00:00.000Z", + "begin": "2015-01-03T00:00:00.000Z", + "end": "2015-01-03T00:00:00.000Z", + "took_ms": 42 + } + } + ] + }, + { + "document": [ + { + "index": { + "indexName": "ignored", + "indexType": "index_range", + "indexId": "ignored" + } + }, + { + "data": { + "_id": "ignored", + "calculated_at": "2015-01-01T00:00:00.000Z", + "begin": "2015-01-01T00:00:00.000Z", + "end": "2015-01-02T00:00:00.000Z", + "took_ms": 23 + } + } + ] + } + ] +} diff --git a/graylog2-shared/src/main/java/org/graylog2/shared/bindings/ObjectMapperModule.java b/graylog2-shared/src/main/java/org/graylog2/shared/bindings/ObjectMapperModule.java index 5be7a86750f3..01a2f5466939 100644 --- a/graylog2-shared/src/main/java/org/graylog2/shared/bindings/ObjectMapperModule.java +++ b/graylog2-shared/src/main/java/org/graylog2/shared/bindings/ObjectMapperModule.java @@ -16,37 +16,13 @@ */ package org.graylog2.shared.bindings; -import com.codahale.metrics.json.MetricsModule; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.PropertyNamingStrategy; -import com.fasterxml.jackson.databind.SerializationFeature; -import com.fasterxml.jackson.databind.module.SimpleModule; -import com.fasterxml.jackson.datatype.guava.GuavaModule; -import com.fasterxml.jackson.datatype.joda.JodaModule; import com.google.inject.AbstractModule; -import org.graylog2.shared.jackson.SizeSerializer; -import org.graylog2.shared.rest.RangeJsonSerializer; - -import java.util.concurrent.TimeUnit; +import org.graylog2.shared.bindings.providers.ObjectMapperProvider; public class ObjectMapperModule extends AbstractModule { - @Override protected void configure() { - final ObjectMapper objectMapper = makeObjectMapper(); - bind(ObjectMapper.class).toInstance(objectMapper); + bind(ObjectMapper.class).toProvider(ObjectMapperProvider.class).asEagerSingleton(); } - - protected ObjectMapper makeObjectMapper() { - return new ObjectMapper() - .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS) - .setPropertyNamingStrategy(new PropertyNamingStrategy.LowerCaseWithUnderscoresStrategy()) - .registerModule(new JodaModule()) - .registerModule(new GuavaModule()) - .registerModule(new MetricsModule(TimeUnit.SECONDS, TimeUnit.SECONDS, false)) - .registerModule(new SimpleModule() - .addSerializer(new RangeJsonSerializer()) - .addSerializer(new SizeSerializer())); - } - } diff --git a/graylog2-shared/src/main/java/org/graylog2/shared/bindings/providers/ObjectMapperProvider.java b/graylog2-shared/src/main/java/org/graylog2/shared/bindings/providers/ObjectMapperProvider.java new file mode 100644 index 000000000000..e9f1361c18c0 --- /dev/null +++ b/graylog2-shared/src/main/java/org/graylog2/shared/bindings/providers/ObjectMapperProvider.java @@ -0,0 +1,53 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog2.shared.bindings.providers; + +import com.codahale.metrics.json.MetricsModule; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.PropertyNamingStrategy; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.datatype.guava.GuavaModule; +import com.fasterxml.jackson.datatype.joda.JodaModule; +import org.graylog2.shared.jackson.SizeSerializer; +import org.graylog2.shared.rest.RangeJsonSerializer; + +import javax.inject.Provider; +import javax.inject.Singleton; +import java.util.concurrent.TimeUnit; + +@Singleton +public class ObjectMapperProvider implements Provider { + protected final ObjectMapper objectMapper; + + public ObjectMapperProvider() { + this.objectMapper = new ObjectMapper() + .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS) + .setPropertyNamingStrategy(new PropertyNamingStrategy.LowerCaseWithUnderscoresStrategy()) + .registerModule(new JodaModule()) + .registerModule(new GuavaModule()) + .registerModule(new MetricsModule(TimeUnit.SECONDS, TimeUnit.SECONDS, false)) + .registerModule(new SimpleModule() + .addSerializer(new RangeJsonSerializer()) + .addSerializer(new SizeSerializer())); + } + + @Override + public ObjectMapper get() { + return objectMapper; + } +} diff --git a/integration-tests/src/test/resources/integration/search/searchForExistingKeyword.json b/integration-tests/src/test/resources/integration/search/searchForExistingKeyword.json index 5a7c4b553483..c3a6c0322dcb 100644 --- a/integration-tests/src/test/resources/integration/search/searchForExistingKeyword.json +++ b/integration-tests/src/test/resources/integration/search/searchForExistingKeyword.json @@ -1,5 +1,26 @@ { "documents": [ + { + "document" : [ + { + "index" : { + "indexName": "graylog_0", + "indexType": "index_range", + "indexId": "graylog_0" + } + }, + { + "data" : { + "_id": "graylog_0", + "index_name": "graylog_0", + "begin": "2015-01-01T00:00:00.000Z", + "end": "2015-12-31T23:59:59.999Z", + "calculated_at": "2015-06-16T11:32:16.827Z", + "took_ms": 42 + } + } + ] + }, { "document" : [ {