From 6cae3b3e9efb5859f89fc25b381b1d138fa3559a Mon Sep 17 00:00:00 2001 From: palindrome <31748527+hlgp@users.noreply.github.com> Date: Mon, 2 Dec 2024 09:44:50 -0500 Subject: [PATCH 1/8] Feature/composite timestamp (#2187) * The composite timestamp allows us to encode two values in the timestamp to be used in accumulo keys. The event date will take the first (right hand most) 46 bits. The last 17 bits (except for the sign bit) will be used to encode how many days after the event date that we should base the ageoff on. If the timestamp is negative, then to calculate the values the complement is taken and then the two values are extracted. The ageoff is encoded as an age off delta which is the number of days after the event date. * Added composite timestamp tests * Add get and set Timestamp on the record container. * Refined the composite timestamp to check for bounds (#2268) * Incorporated timestamp concept into the RawRecordContainer (#2297) * Override getDate to be compatible with extending classes * Updated sharded ingest to appropriately use getTimestamp vs getDate * Updated index table to use composite timestamp * Updated edge table to use composite timestamp * Added an explicit INVALID_TIMESTAMP instead of using Long.MIN_VALUE to determine if the date is not set * Updated the global index uid aggregator to not aggregate across different timestamps * Removed the "timestampIgnored" flag on the global index uid aggregator as we never turn that off and we don't want to * Added test to ensure that keys with different timestamps do not get aggregated * Updated ageoff day delta calculation to be based on the difference between the start-of-day values for the event and age off dates. --------- Co-authored-by: hlgp Co-authored-by: Ivan Bella <347158+ivakegg@users.noreply.github.com> --- .../bin/services/datawave/bootstrap-ingest.sh | 13 +- .../bin/services/datawave/install-ingest.sh | 9 +- .../bin/services/hadoop/bootstrap.sh | 6 +- .../filter/ConfigurableAgeOffFilter.java | 4 +- .../iterators/filter/RegexFilterBase.java | 3 +- .../iterators/filter/TokenFilterBase.java | 3 +- .../filter/TokenizingFilterBase.java | 3 +- .../filter/ageoff/DataTypeAgeOffFilter.java | 5 +- .../filter/ageoff/FieldAgeOffFilter.java | 3 +- .../filter/ageoff/MaximumAgeOffFilter.java | 7 +- .../filter/ColumnVisibilityLabeledFilter.java | 3 +- .../filter/ConfigurableAgeOffFilterTest.java | 6 + .../filter/ageoff/FieldAgeOffFilterTest.java | 35 ++ .../datawave/util/CompositeTimestamp.java | 212 ++++++++++ .../datawave/util/CompositeTimestampTest.java | 371 ++++++++++++++++++ .../resources/config/shard-ingest-config.xml | 9 + .../ingest/config/RawRecordContainerImpl.java | 28 +- .../ingest/data/RawRecordContainer.java | 61 ++- .../data/config/ingest/BaseIngestHelper.java | 1 + .../reader/AbstractEventRecordReader.java | 3 +- .../handler/atom/AtomDataTypeHandler.java | 16 +- .../atom/AtomErrorDataTypeHandler.java | 4 +- .../edge/ProtobufEdgeDataTypeHandler.java | 4 +- .../handler/error/ErrorDataTypeHandler.java | 2 +- .../error/ErrorShardedDataTypeHandler.java | 2 +- .../handler/shard/ShardedDataTypeHandler.java | 71 ++-- .../ContentIndexingColumnBasedHandler.java | 5 +- ...ndedContentIndexingColumnBasedHandler.java | 13 +- .../aggregator/GlobalIndexUidAggregator.java | 59 +-- .../table/aggregator/PropogatingCombiner.java | 13 + .../config/AbstractTableConfigHelper.java | 4 + .../table/config/ShardTableConfigHelper.java | 1 + .../iterators/PropogatingIterator.java | 2 +- .../datawave/iterators/ValueCombiner.java | 8 +- .../policy/ExampleIngestPolicyEnforcer.java | 2 +- .../data/RawRecordContainerImplTest.java | 12 +- .../ingest/mapreduce/EventMapperTest.java | 4 +- .../ingest/mapreduce/SimpleRawRecord.java | 15 +- .../shard/ShardedDataTypeHandlerTest.java | 69 +++- ...ContentIndexingColumnBasedHandlerTest.java | 2 +- .../GlobalIndexUidAggregatorTest.java | 100 +++-- .../config/AbstractTableConfigHelperTest.java | 2 + .../ingest/csv/mr/input/CSVRecordReader.java | 2 +- .../json/mr/input/JsonRecordReader.java | 2 +- .../wikipedia/WikipediaDataTypeHandler.java | 2 +- ...eldIndexCountingIteratorPerVisibility.java | 5 +- .../filter/GlobalIndexDateRangeFilter.java | 3 +- .../query/iterator/filter/LoadDateFilter.java | 3 +- .../iterators/FieldIndexCountingIterator.java | 6 +- .../datawave/query/predicate/TimeFilter.java | 4 +- .../shard/FieldIndexCountQueryLogic.java | 5 +- .../DocumentTransformerSupport.java | 3 +- .../java/datawave/helpers/PrintUtility.java | 13 +- ...estLimitReturnedGroupsToHitTermGroups.java | 18 +- .../util/CommonalityTokenTestDataIngest.java | 28 ++ 55 files changed, 1097 insertions(+), 192 deletions(-) create mode 100644 warehouse/core/src/main/java/datawave/util/CompositeTimestamp.java create mode 100644 warehouse/core/src/test/java/datawave/util/CompositeTimestampTest.java diff --git a/contrib/datawave-quickstart/bin/services/datawave/bootstrap-ingest.sh b/contrib/datawave-quickstart/bin/services/datawave/bootstrap-ingest.sh index bd689ec71d1..8629602bc00 100644 --- a/contrib/datawave-quickstart/bin/services/datawave/bootstrap-ingest.sh +++ b/contrib/datawave-quickstart/bin/services/datawave/bootstrap-ingest.sh @@ -184,7 +184,7 @@ function datawaveIngestWikipedia() { [ ! -f "${wikipediaRawFile}" ] && error "File not found: ${wikipediaRawFile}" && return 1 local wikipediaHdfsFile="${DW_DATAWAVE_INGEST_HDFS_BASEDIR}/$( basename ${wikipediaRawFile} )" - local putFileCommand="hdfs dfs -copyFromLocal ${wikipediaRawFile} ${wikipediaHdfsFile}" + local putFileCommand="hdfs dfs -copyFromLocal -f ${wikipediaRawFile} ${wikipediaHdfsFile}" local inputFormat="datawave.ingest.wikipedia.WikipediaEventInputFormat" local jobCommand="${DW_DATAWAVE_INGEST_HOME}/bin/ingest/live-ingest.sh ${wikipediaHdfsFile} ${DW_DATAWAVE_INGEST_NUM_SHARDS} -inputFormat ${inputFormat} -data.name.override=wikipedia ${extraOpts}" @@ -211,7 +211,7 @@ function datawaveIngestCsv() { [ ! -f "${csvRawFile}" ] && error "File not found: ${csvRawFile}" && return 1 local csvHdfsFile="${DW_DATAWAVE_INGEST_HDFS_BASEDIR}/$( basename ${csvRawFile} )" - local putFileCommand="hdfs dfs -copyFromLocal ${csvRawFile} ${csvHdfsFile}" + local putFileCommand="hdfs dfs -copyFromLocal -f ${csvRawFile} ${csvHdfsFile}" local inputFormat="datawave.ingest.csv.mr.input.CSVFileInputFormat" local jobCommand="${DW_DATAWAVE_INGEST_HOME}/bin/ingest/live-ingest.sh ${csvHdfsFile} ${DW_DATAWAVE_INGEST_NUM_SHARDS} -inputFormat ${inputFormat} -data.name.override=mycsv ${extraOpts}" @@ -232,7 +232,7 @@ function datawaveIngestJson() { [ ! -f "${jsonRawFile}" ] && error "File not found: ${jsonRawFile}" && return 1 local jsonHdfsFile="${DW_DATAWAVE_INGEST_HDFS_BASEDIR}/$( basename ${jsonRawFile} )" - local putFileCommand="hdfs dfs -copyFromLocal ${jsonRawFile} ${jsonHdfsFile}" + local putFileCommand="hdfs dfs -copyFromLocal -f ${jsonRawFile} ${jsonHdfsFile}" local inputFormat="datawave.ingest.json.mr.input.JsonInputFormat" local jobCommand="${DW_DATAWAVE_INGEST_HOME}/bin/ingest/live-ingest.sh ${jsonHdfsFile} ${DW_DATAWAVE_INGEST_NUM_SHARDS} -inputFormat ${inputFormat} -data.name.override=myjson ${extraOpts}" @@ -347,3 +347,10 @@ function datawaveIngestTarballName() { local dwVersion="$(getDataWaveVersion)" echo "$( basename "${DW_DATAWAVE_INGEST_TARBALL/-\*-/-$dwVersion-}" )" } + +function datawaveIngestExamples() { + datawaveIngestWikipedia ${DW_DATAWAVE_INGEST_TEST_FILE_WIKI} + datawaveIngestJson ${DW_DATAWAVE_INGEST_TEST_FILE_JSON} + datawaveIngestCsv ${DW_DATAWAVE_INGEST_TEST_FILE_CSV} +} + diff --git a/contrib/datawave-quickstart/bin/services/datawave/install-ingest.sh b/contrib/datawave-quickstart/bin/services/datawave/install-ingest.sh index a2cd713d960..2ab3f06fa7b 100755 --- a/contrib/datawave-quickstart/bin/services/datawave/install-ingest.sh +++ b/contrib/datawave-quickstart/bin/services/datawave/install-ingest.sh @@ -159,13 +159,6 @@ function initializeDatawaveTables() { fi } -function ingestExampleData() { - # Ingest some canned, example data files - datawaveIngestWikipedia "${DW_DATAWAVE_INGEST_TEST_FILE_WIKI}" - datawaveIngestJson "${DW_DATAWAVE_INGEST_TEST_FILE_JSON}" - datawaveIngestCsv "${DW_DATAWAVE_INGEST_TEST_FILE_CSV}" -} - initializeDatawaveTables @@ -186,4 +179,4 @@ info "See \$DW_CLOUD_HOME/bin/services/datawave/bootstrap-ingest.sh to view/edit # Ingest raw data examples, if appropriate... -[ "${DW_REDEPLOY_IN_PROGRESS}" != true ] && [ "${DW_DATAWAVE_INGEST_TEST_SKIP}" == false ] && ingestExampleData +[ "${DW_REDEPLOY_IN_PROGRESS}" != true ] && [ "${DW_DATAWAVE_INGEST_TEST_SKIP}" == false ] && datawaveIngestExamples diff --git a/contrib/datawave-quickstart/bin/services/hadoop/bootstrap.sh b/contrib/datawave-quickstart/bin/services/hadoop/bootstrap.sh index 9f191b00402..8cec3a75726 100644 --- a/contrib/datawave-quickstart/bin/services/hadoop/bootstrap.sh +++ b/contrib/datawave-quickstart/bin/services/hadoop/bootstrap.sh @@ -45,8 +45,8 @@ dfs.replication 1" DW_HADOOP_MR_HEAPDUMP_DIR="${DW_CLOUD_DATA}/heapdumps" # mapred-site.xml (Format: {}) -DW_HADOOP_MAPRED_SITE_CONF="mapreduce.jobhistory.address http://${DW_BIND_HOST}:8020 -mapreduce.jobhistory.webapp.address http://${DW_BIND_HOST}:8021 +DW_HADOOP_MAPRED_SITE_CONF="mapreduce.jobhistory.address ${DW_BIND_HOST}:8020 +mapreduce.jobhistory.webapp.address ${DW_BIND_HOST}:8021 mapreduce.jobhistory.intermediate-done-dir ${DW_HADOOP_MR_INTER_DIR} mapreduce.jobhistory.done-dir ${DW_HADOOP_MR_DONE_DIR} mapreduce.map.memory.mb 2048 @@ -72,7 +72,7 @@ yarn.nodemanager.pmem-check-enabled false yarn.nodemanager.vmem-check-enabled false yarn.nodemanager.resource.memory-mb 6144 yarn.app.mapreduce.am.resource.mb 1024 -yarn.log.server.url http://localhost:8070/jobhistory/logs" +yarn.log.server.url http://localhost:8021/jobhistory/logs" # capacity-scheduler.xml (Format: {}) DW_HADOOP_CAPACITY_SCHEDULER_CONF="yarn.scheduler.capacity.maximum-applications 10000 diff --git a/warehouse/age-off/src/main/java/datawave/iterators/filter/ConfigurableAgeOffFilter.java b/warehouse/age-off/src/main/java/datawave/iterators/filter/ConfigurableAgeOffFilter.java index 61b8f23bb08..21f1f692bb0 100644 --- a/warehouse/age-off/src/main/java/datawave/iterators/filter/ConfigurableAgeOffFilter.java +++ b/warehouse/age-off/src/main/java/datawave/iterators/filter/ConfigurableAgeOffFilter.java @@ -35,6 +35,8 @@ import datawave.ingest.util.cache.watch.FileRuleCacheValue; import datawave.iterators.filter.ageoff.AgeOffPeriod; import datawave.iterators.filter.ageoff.AppliedRule; +import datawave.iterators.filter.ageoff.FilterRule; +import datawave.util.CompositeTimestamp; /** * This class provides a subclass of the {@code org.apache.accumulo.core.iterators.Filter} class and implements the {@code Option Describer} interface. It @@ -167,7 +169,7 @@ public boolean accept(Key k, Value v) { return true; // short circuit check - long timeStamp = k.getTimestamp(); + long timeStamp = CompositeTimestamp.getAgeOffDate(k.getTimestamp()); if (timeStamp > this.shortCircuitDateMillis) return true; diff --git a/warehouse/age-off/src/main/java/datawave/iterators/filter/RegexFilterBase.java b/warehouse/age-off/src/main/java/datawave/iterators/filter/RegexFilterBase.java index ba94dcbb953..d19a9e5f670 100644 --- a/warehouse/age-off/src/main/java/datawave/iterators/filter/RegexFilterBase.java +++ b/warehouse/age-off/src/main/java/datawave/iterators/filter/RegexFilterBase.java @@ -13,6 +13,7 @@ import datawave.iterators.filter.ageoff.AgeOffPeriod; import datawave.iterators.filter.ageoff.AppliedRule; import datawave.iterators.filter.ageoff.FilterOptions; +import datawave.util.CompositeTimestamp; /** * This class provides an abstract base class to be extended to filter based on matching a REGEX to the {@code String} object that represents some portion of a @@ -69,7 +70,7 @@ public boolean accept(AgeOffPeriod period, Key k, Value v) { String keyField = getKeyField(k, v); Matcher matcher = pattern.matcher(keyField); if (matcher.find()) { - long timeStamp = k.getTimestamp(); + long timeStamp = CompositeTimestamp.getAgeOffDate(k.getTimestamp()); dtFlag = timeStamp > period.getCutOffMilliseconds(); if (log.isTraceEnabled()) { log.trace("timeStamp = " + timeStamp); diff --git a/warehouse/age-off/src/main/java/datawave/iterators/filter/TokenFilterBase.java b/warehouse/age-off/src/main/java/datawave/iterators/filter/TokenFilterBase.java index dd963b4b557..b6a59a034da 100644 --- a/warehouse/age-off/src/main/java/datawave/iterators/filter/TokenFilterBase.java +++ b/warehouse/age-off/src/main/java/datawave/iterators/filter/TokenFilterBase.java @@ -10,6 +10,7 @@ import datawave.iterators.filter.ageoff.AgeOffPeriod; import datawave.iterators.filter.ageoff.AppliedRule; import datawave.iterators.filter.ageoff.FilterOptions; +import datawave.util.CompositeTimestamp; import datawave.util.StringUtils; /** @@ -69,7 +70,7 @@ public boolean accept(AgeOffPeriod period, Key k, Value v) { dtFlag = true; } else { if (hasToken(k, v, patternBytes)) { - long timeStamp = k.getTimestamp(); + long timeStamp = CompositeTimestamp.getAgeOffDate(k.getTimestamp()); dtFlag = timeStamp > period.getCutOffMilliseconds(); if (log.isTraceEnabled()) { log.trace("timeStamp = " + timeStamp); diff --git a/warehouse/age-off/src/main/java/datawave/iterators/filter/TokenizingFilterBase.java b/warehouse/age-off/src/main/java/datawave/iterators/filter/TokenizingFilterBase.java index b2078cd0502..edb4e219f1d 100644 --- a/warehouse/age-off/src/main/java/datawave/iterators/filter/TokenizingFilterBase.java +++ b/warehouse/age-off/src/main/java/datawave/iterators/filter/TokenizingFilterBase.java @@ -10,6 +10,7 @@ import datawave.iterators.filter.ageoff.AgeOffPeriod; import datawave.iterators.filter.ageoff.AppliedRule; import datawave.iterators.filter.ageoff.FilterOptions; +import datawave.util.CompositeTimestamp; /** * TokenizingAgeoffFilter cuts a field into tokens (splitting at a specified set of delimiters), and makes ageoff decisions based on whether or not any of the @@ -119,7 +120,7 @@ public boolean accept(AgeOffPeriod period, Key k, Value V) { cutoffTimestamp -= calculatedTTL; } ruleApplied = true; - return k.getTimestamp() > cutoffTimestamp; + return CompositeTimestamp.getAgeOffDate(k.getTimestamp()) > cutoffTimestamp; } @Override diff --git a/warehouse/age-off/src/main/java/datawave/iterators/filter/ageoff/DataTypeAgeOffFilter.java b/warehouse/age-off/src/main/java/datawave/iterators/filter/ageoff/DataTypeAgeOffFilter.java index 74963cbc2b3..a9d7ee7f54b 100644 --- a/warehouse/age-off/src/main/java/datawave/iterators/filter/ageoff/DataTypeAgeOffFilter.java +++ b/warehouse/age-off/src/main/java/datawave/iterators/filter/ageoff/DataTypeAgeOffFilter.java @@ -16,6 +16,7 @@ import com.google.common.collect.Sets; import datawave.iterators.filter.AgeOffConfigParams; +import datawave.util.CompositeTimestamp; /** * Data type age off filter. Traverses through indexed tables @@ -209,11 +210,11 @@ public boolean accept(AgeOffPeriod period, Key k, Value v) { if (dataTypeCutoff == null) { if (defaultCutoffTime >= 0) { ruleApplied = true; - accept = k.getTimestamp() > defaultCutoffTime; + accept = CompositeTimestamp.getAgeOffDate(k.getTimestamp()) > defaultCutoffTime; } } else { ruleApplied = true; - accept = k.getTimestamp() > dataTypeCutoff; + accept = CompositeTimestamp.getAgeOffDate(k.getTimestamp()) > dataTypeCutoff; } // after age-off is applied check, if we are accepting this KeyValue and this is a Scan on a dataType which only accepts on timestamp // only continue to accept the KeyValue if the timestamp for the dataType matches what is configured diff --git a/warehouse/age-off/src/main/java/datawave/iterators/filter/ageoff/FieldAgeOffFilter.java b/warehouse/age-off/src/main/java/datawave/iterators/filter/ageoff/FieldAgeOffFilter.java index 9d61166f340..d0875f2176b 100644 --- a/warehouse/age-off/src/main/java/datawave/iterators/filter/ageoff/FieldAgeOffFilter.java +++ b/warehouse/age-off/src/main/java/datawave/iterators/filter/ageoff/FieldAgeOffFilter.java @@ -17,6 +17,7 @@ import datawave.iterators.filter.AgeOffConfigParams; import datawave.iterators.filter.ColumnVisibilityOrFilter; +import datawave.util.CompositeTimestamp; /** * Field age off filter. Traverses through indexed tables and non-indexed tables. Example follows. Note that any field TTL will follow the same units specified @@ -208,7 +209,7 @@ public boolean accept(AgeOffPeriod period, Key k, Value v) { Long dataTypeCutoff = (fieldTimes.containsKey(field)) ? fieldTimes.get(field) : null; if (dataTypeCutoff != null) { ruleApplied = true; - return k.getTimestamp() > dataTypeCutoff; + return CompositeTimestamp.getAgeOffDate(k.getTimestamp()) > dataTypeCutoff; } return true; diff --git a/warehouse/age-off/src/main/java/datawave/iterators/filter/ageoff/MaximumAgeOffFilter.java b/warehouse/age-off/src/main/java/datawave/iterators/filter/ageoff/MaximumAgeOffFilter.java index 257e0333ed9..643bb59accc 100644 --- a/warehouse/age-off/src/main/java/datawave/iterators/filter/ageoff/MaximumAgeOffFilter.java +++ b/warehouse/age-off/src/main/java/datawave/iterators/filter/ageoff/MaximumAgeOffFilter.java @@ -6,6 +6,7 @@ import org.apache.log4j.Logger; import datawave.iterators.filter.AgeOffConfigParams; +import datawave.util.CompositeTimestamp; /** * Data type age off filter. Traverses through indexed tables @@ -59,7 +60,11 @@ public boolean accept(AgeOffPeriod period, Key k, Value v) { // this rule determines whether to accept / deny (ageoff) a K/V // based solely on whether a timestamp is before (older than) the cutoff for aging off - return k.getTimestamp() > period.getCutOffMilliseconds(); + if (CompositeTimestamp.getAgeOffDate(k.getTimestamp()) > period.getCutOffMilliseconds()) { + return true; + } else { + return false; + } } /** diff --git a/warehouse/age-off/src/test/java/datawave/iterators/filter/ColumnVisibilityLabeledFilter.java b/warehouse/age-off/src/test/java/datawave/iterators/filter/ColumnVisibilityLabeledFilter.java index 9101cb2fe58..214381672ce 100644 --- a/warehouse/age-off/src/test/java/datawave/iterators/filter/ColumnVisibilityLabeledFilter.java +++ b/warehouse/age-off/src/test/java/datawave/iterators/filter/ColumnVisibilityLabeledFilter.java @@ -11,6 +11,7 @@ import datawave.iterators.filter.ageoff.AgeOffPeriod; import datawave.iterators.filter.ageoff.AppliedRule; import datawave.iterators.filter.ageoff.FilterOptions; +import datawave.util.CompositeTimestamp; import datawave.util.StringUtils; /** @@ -108,7 +109,7 @@ public boolean accept(AgeOffPeriod ageOffPeriod, Key k, Value V) { cutOff -= timeToLive; } this.filterRuleApplied = true; - return k.getTimestamp() > cutOff; + return CompositeTimestamp.getAgeOffDate(k.getTimestamp()) > cutOff; } } return true; diff --git a/warehouse/age-off/src/test/java/datawave/iterators/filter/ConfigurableAgeOffFilterTest.java b/warehouse/age-off/src/test/java/datawave/iterators/filter/ConfigurableAgeOffFilterTest.java index b35f976f4bf..9290f93f0ac 100644 --- a/warehouse/age-off/src/test/java/datawave/iterators/filter/ConfigurableAgeOffFilterTest.java +++ b/warehouse/age-off/src/test/java/datawave/iterators/filter/ConfigurableAgeOffFilterTest.java @@ -28,6 +28,7 @@ import datawave.iterators.filter.ageoff.ConfigurableIteratorEnvironment; import datawave.iterators.filter.ageoff.FilterOptions; import datawave.query.iterator.SortedListKeyValueIterator; +import datawave.util.CompositeTimestamp; public class ConfigurableAgeOffFilterTest { @@ -138,6 +139,9 @@ public void testAcceptKeyValue_TtlSet() throws Exception { // copy cofigs to actual filter we are testing filter.initialize(wrapper); + long tomorrow = System.currentTimeMillis() + CompositeTimestamp.MILLIS_PER_DAY; + long compositeTS = CompositeTimestamp.getCompositeTimeStamp(daysAgo(365), tomorrow); + // brand new key should be good assertThat(filter.accept(new Key(), VALUE), is(true)); // first five will hit the ttl short circuit @@ -155,6 +159,8 @@ public void testAcceptKeyValue_TtlSet() throws Exception { assertThat(filter.accept(getKey("foo", daysAgo(8)), VALUE), is(true)); // this is really old and matches so should not be accepted assertThat(filter.accept(getKey("foo", daysAgo(365)), VALUE), is(false)); + // this is really old and matches, but has a future age off date, so should be accepted + assertThat(filter.accept(getKey("foo", compositeTS), VALUE), is(true)); } diff --git a/warehouse/age-off/src/test/java/datawave/iterators/filter/ageoff/FieldAgeOffFilterTest.java b/warehouse/age-off/src/test/java/datawave/iterators/filter/ageoff/FieldAgeOffFilterTest.java index 469887d5575..36571bdaf04 100644 --- a/warehouse/age-off/src/test/java/datawave/iterators/filter/ageoff/FieldAgeOffFilterTest.java +++ b/warehouse/age-off/src/test/java/datawave/iterators/filter/ageoff/FieldAgeOffFilterTest.java @@ -14,6 +14,7 @@ import datawave.iterators.filter.AgeOffConfigParams; import datawave.iterators.filter.AgeOffTtlUnits; +import datawave.util.CompositeTimestamp; public class FieldAgeOffFilterTest { private static final String VISIBILITY_PATTERN = "MY_VIS"; @@ -84,6 +85,7 @@ public void testIndexTrueUsesDefaultWhenFieldLacksTtl() { Key key = new Key("1234", "field_z\\x00my-uuid", "field_z\u0000value", VISIBILITY_PATTERN, tenSecondsAgo); Assert.assertFalse(ageOffFilter.accept(filterOptions.getAgeOffPeriod(System.currentTimeMillis()), key, new Value())); Assert.assertTrue(ageOffFilter.isFilterRuleApplied()); + key = new Key("1234", "field_y", "field_y\u0000value", VISIBILITY_PATTERN, tenSecondsAgo); Assert.assertFalse(ageOffFilter.accept(filterOptions.getAgeOffPeriod(System.currentTimeMillis()), key, new Value())); Assert.assertTrue(ageOffFilter.isFilterRuleApplied()); @@ -236,6 +238,39 @@ public void testIgnoresDocument() { Assert.assertFalse(ageOffFilter.isFilterRuleApplied()); } + @Test + public void testCompositeTimestamp() { + EditableAccumuloConfiguration conf = new EditableAccumuloConfiguration(DefaultConfiguration.getInstance()); + conf.put("table.custom.isindextable", "true"); + iterEnv.setConf(conf); + + long tenSecondsAgo = System.currentTimeMillis() - (10L * ONE_SEC); + long tomorrow = System.currentTimeMillis() + CompositeTimestamp.MILLIS_PER_DAY; + + long compositeTS = CompositeTimestamp.getCompositeTimeStamp(tenSecondsAgo, tomorrow); + + FieldAgeOffFilter ageOffFilter = new FieldAgeOffFilter(); + FilterOptions filterOptions = createFilterOptionsWithPattern(); + // set the default to 5 seconds + filterOptions.setTTL(5L); + filterOptions.setTTLUnits(AgeOffTtlUnits.SECONDS); + // set up ttls for field_y and field_z only, deliberately exclude the ttl for field_y + filterOptions.setOption("fields", "field_y"); + filterOptions.setOption("field_y.ttl", "2"); // 2 seconds + ageOffFilter.init(filterOptions, iterEnv); + + // age off date allows this to accept + Key key = new Key("1234", "field_y", "field_y\u0000value", VISIBILITY_PATTERN, compositeTS); + Assert.assertTrue(ageOffFilter.accept(filterOptions.getAgeOffPeriod(System.currentTimeMillis()), key, new Value())); + Assert.assertTrue(ageOffFilter.isFilterRuleApplied()); + + // vanilla date does not + key = new Key("1234", "field_y", "field_y\u0000value", VISIBILITY_PATTERN, tenSecondsAgo); + Assert.assertFalse(ageOffFilter.accept(filterOptions.getAgeOffPeriod(System.currentTimeMillis()), key, new Value())); + Assert.assertTrue(ageOffFilter.isFilterRuleApplied()); + + } + @Test public void testKeepsMatchBeforeTtl() { long oneSecondAgo = System.currentTimeMillis() - (1 * ONE_SEC); diff --git a/warehouse/core/src/main/java/datawave/util/CompositeTimestamp.java b/warehouse/core/src/main/java/datawave/util/CompositeTimestamp.java new file mode 100644 index 00000000000..023ba517ed4 --- /dev/null +++ b/warehouse/core/src/main/java/datawave/util/CompositeTimestamp.java @@ -0,0 +1,212 @@ +package datawave.util; + +import java.util.Calendar; +import java.util.Comparator; +import java.util.TimeZone; + +/** + * The composite timestamp allows us to encode two values in the timestamp to be used in accumulo keys. The event date will take the first (right hand most) 46 + * bits. The last 17 bits (except for the sign bit) will be used to encode how many days after the event date that we should base the ageoff on. If the + * timestamp is negative, then to calculate the values the complement is taken and then the two values are extracted. The ageoff is encoded as an age off delta + * which is the number of days after the event date. + */ +public class CompositeTimestamp { + + // The number of bits for the event date + private static final int allocationForEventDate = 46; + // A mask for the event date + private static final long mask = -1L >>> (8 * 8 - allocationForEventDate); + // The max age off delta in days + private static final long maxDiff = (-1L << (allocationForEventDate + 1)) >>> (allocationForEventDate + 1); + // A useful constant + public static final long MILLIS_PER_DAY = 1000 * 60 * 60 * 24; + + // A constant for an invalid timestamp + public static final long INVALID_TIMESTAMP = Long.MIN_VALUE; + // The minimum event date + public static final long MIN_EVENT_DATE = (0 - mask); // also equivalent to Long.MIN_VALUE + 1 + // the maximum event date + public static final long MAX_EVENT_DATE = mask; + + /** + * Determine if the timestamp is composite (i.e. non-zero age off delta) + * + * @param ts + * @return True if composite + */ + public static boolean isCompositeTimestamp(long ts) { + validateTimestamp(ts); + return (Math.abs(ts) >>> allocationForEventDate > 0); + } + + /** + * Get the event date portion of the timestamp + * + * @param ts + * @return the event date + */ + public static long getEventDate(long ts) { + validateTimestamp(ts); + long eventTs = (Math.abs(ts) & mask); + if (ts < 0) { + eventTs = 0 - eventTs; + } + return eventTs; + } + + /** + * Determine the age off date portion of the timestamp. This is the event date plus the ageoff delta converted to milliseconds. + * + * @param ts + * @return The age off date + */ + public static long getAgeOffDate(long ts) { + validateTimestamp(ts); + long baseTs = Math.abs(ts); + long eventTs = (baseTs & mask); + long ageOffDiff = ((baseTs >>> allocationForEventDate) * MILLIS_PER_DAY); + if (ts < 0) { + eventTs = 0 - eventTs; + } + return eventTs + ageOffDiff; + } + + /** + * Determine the age off delta porton of the timestamp. This is the number of days difference from the event date. + * + * @param ts + * @return the age off delta + */ + public static int getAgeOffDeltaDays(long ts) { + validateTimestamp(ts); + long baseTs = Math.abs(ts); + long ageOffDiff = ((baseTs >>> allocationForEventDate) * MILLIS_PER_DAY); + return (int) ageOffDiff; + } + + /** + * Calculate an age off delta based on a timezone. This will calculate the begininning of the day in the given timezone for both the event date and the age + * off date, and then will return that difference in days. + * + * @param eventDate + * @param ageOffDate + * @param tz + * @return the age off delta + */ + public static int computeAgeOffDeltaDays(long eventDate, long ageOffDate, TimeZone tz) { + validateEventDate(eventDate); + Calendar c = Calendar.getInstance(tz); + c.setTimeInMillis(eventDate); + c.set(Calendar.HOUR_OF_DAY, 0); + c.set(Calendar.MINUTE, 0); + c.set(Calendar.SECOND, 0); + c.set(Calendar.MILLISECOND, 0); + long eventDateStart = c.getTimeInMillis(); + + c.setTimeInMillis(ageOffDate); + c.set(Calendar.HOUR_OF_DAY, 0); + c.set(Calendar.MINUTE, 0); + c.set(Calendar.SECOND, 0); + c.set(Calendar.MILLISECOND, 0); + long ageOffDateStart = c.getTimeInMillis(); + + long delta = (ageOffDateStart - eventDateStart) / MILLIS_PER_DAY; + validateAgeOffDelta(delta); + return (int) delta; + } + + /** + * Get the composite timestamp using the supplied event date and age off delta in days. + * + * @param eventDate + * @param ageOffDeltaDays + * @return The composite timestamp + */ + public static long getCompositeTimestamp(long eventDate, int ageOffDeltaDays) { + validateEventDate(eventDate); + validateAgeOffDelta(ageOffDeltaDays); + long eventBase = Math.abs(eventDate); + long compositeTS = ((long) ageOffDeltaDays << allocationForEventDate) | eventBase; + if (eventDate < 0) { + compositeTS = 0 - compositeTS; + } + return compositeTS; + } + + /** + * Get the composite timestamp using the supplied eventDate and an age off delta in days based on the supplied age off date and time zone. + * + * @param eventDate + * @param ageOffDate + * @param tz + * @return The composite timestamp + */ + public static long getCompositeTimestamp(long eventDate, long ageOffDate, TimeZone tz) { + return getCompositeTimestamp(eventDate, computeAgeOffDeltaDays(eventDate, ageOffDate, tz)); + } + + /** + * Get the composite timestamp using the supplied eventDate and an age off delta in days based on the supplied age off date using the GMT timezone + * + * @param eventDate + * @param ageOffDate + * @return The composite timestamp + */ + public static long getCompositeTimeStamp(long eventDate, long ageOffDate) { + return getCompositeTimestamp(eventDate, ageOffDate, TimeZone.getTimeZone("GMT")); + } + + /** + * Return a comparator for composite timestamps. Orders firstly on the event date, and if equal then on the ageoff date. Note for values with the same event + * date, the timestamps will order naturally for positive event dates, but need to be reversed for negative event dates. This should only be an issue for + * the global index and hence this comparator can be used in that case. + * + * @return a comparison of the timestamps + */ + public static Comparator comparator() { + return (o1, o2) -> { + long eventDate1 = CompositeTimestamp.getEventDate(o1); + long eventDate2 = CompositeTimestamp.getEventDate(o2); + if (eventDate1 < eventDate2) { + return -1; + } else if (eventDate1 == eventDate2) { + long ageOffDate1 = CompositeTimestamp.getAgeOffDate(o1); + long ageOffDate2 = CompositeTimestamp.getAgeOffDate(o2); + if (ageOffDate1 < ageOffDate2) { + return -1; + } else if (ageOffDate1 == ageOffDate2) { + return 0; + } else { + return 1; + } + } else { + return 1; + } + }; + } + + private static void validateTimestamp(long ts) { + if (ts == INVALID_TIMESTAMP) { + throw new IllegalArgumentException("Invalid timestamp"); + } + } + + private static void validateEventDate(long eventDate) { + if (eventDate < MIN_EVENT_DATE) { + throw new IllegalArgumentException("Event date cannot be less than " + MIN_EVENT_DATE); + } + if (eventDate > MAX_EVENT_DATE) { + throw new IllegalArgumentException("Event Date cannot be greater than " + MAX_EVENT_DATE); + } + } + + private static void validateAgeOffDelta(long ageOffDeltaDays) { + if (ageOffDeltaDays < 0) { + throw new IllegalArgumentException("Age off date must be greater to or equal to the event date"); + } + if (ageOffDeltaDays > maxDiff) { + throw new IllegalArgumentException("Difference between event date and age off date cannot be more than " + maxDiff + " days"); + } + } + +} diff --git a/warehouse/core/src/test/java/datawave/util/CompositeTimestampTest.java b/warehouse/core/src/test/java/datawave/util/CompositeTimestampTest.java new file mode 100644 index 00000000000..33cfad12281 --- /dev/null +++ b/warehouse/core/src/test/java/datawave/util/CompositeTimestampTest.java @@ -0,0 +1,371 @@ +package datawave.util; + +import java.time.Instant; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Calendar; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import org.junit.Assert; +import org.junit.Test; + +public class CompositeTimestampTest { + + @Test + public void testConversion() { + long eventDate = Instant.from(DateTimeFormatter.ISO_INSTANT.parse("2022-10-26T01:00:00Z")).toEpochMilli(); + long ageOff = eventDate + CompositeTimestamp.MILLIS_PER_DAY; + long expectedTS = 72035490177664L; + + long compositeTS = CompositeTimestamp.getCompositeTimeStamp(eventDate, ageOff); + + Assert.assertEquals(expectedTS, compositeTS); + Assert.assertTrue(CompositeTimestamp.isCompositeTimestamp(compositeTS)); + Assert.assertFalse(CompositeTimestamp.isCompositeTimestamp(eventDate)); + Assert.assertFalse(CompositeTimestamp.isCompositeTimestamp(ageOff)); + Assert.assertEquals(CompositeTimestamp.getEventDate(compositeTS), eventDate); + Assert.assertEquals(CompositeTimestamp.getAgeOffDate(compositeTS), ageOff); + } + + @Test + public void testOldDateWithOldAgeoff() { + long eventDate = Instant.from(DateTimeFormatter.ISO_INSTANT.parse("1900-01-01T00:00:00Z")).toEpochMilli(); + long ageOff = Instant.from(DateTimeFormatter.ISO_INSTANT.parse("1950-01-01T00:00:00Z")).toEpochMilli(); + long expectedTS = -1285076215161299968L; + + long compositeTS = CompositeTimestamp.getCompositeTimeStamp(eventDate, ageOff); + + Assert.assertEquals(expectedTS, compositeTS); + Assert.assertTrue(CompositeTimestamp.isCompositeTimestamp(compositeTS)); + Assert.assertFalse(CompositeTimestamp.isCompositeTimestamp(eventDate)); + Assert.assertFalse(CompositeTimestamp.isCompositeTimestamp(ageOff)); + Assert.assertEquals(CompositeTimestamp.getEventDate(compositeTS), eventDate); + Assert.assertEquals(CompositeTimestamp.getAgeOffDate(compositeTS), ageOff); + } + + @Test + public void testOldDateWithModernAgeoff() { + long eventDate = Instant.from(DateTimeFormatter.ISO_INSTANT.parse("1960-01-01T00:00:00Z")).toEpochMilli(); + long ageOff = Instant.from(DateTimeFormatter.ISO_INSTANT.parse("2025-01-01T00:00:00Z")).toEpochMilli(); + long expectedTS = -1670695039885298688L; + + long compositeTS = CompositeTimestamp.getCompositeTimeStamp(eventDate, ageOff); + + Assert.assertEquals(expectedTS, compositeTS); + Assert.assertTrue(CompositeTimestamp.isCompositeTimestamp(compositeTS)); + Assert.assertFalse(CompositeTimestamp.isCompositeTimestamp(eventDate)); + Assert.assertFalse(CompositeTimestamp.isCompositeTimestamp(ageOff)); + Assert.assertEquals(CompositeTimestamp.getEventDate(compositeTS), eventDate); + Assert.assertEquals(CompositeTimestamp.getAgeOffDate(compositeTS), ageOff); + } + + @Test + public void testOldDateWithMaxAgeoff() { + long eventDate = Instant.from(DateTimeFormatter.ISO_INSTANT.parse("1960-01-01T00:00:00Z")).toEpochMilli(); + long ageOff = eventDate + (131071L * CompositeTimestamp.MILLIS_PER_DAY); + long expectedTS = -9223301983729798144L; + + long compositeTS = CompositeTimestamp.getCompositeTimeStamp(eventDate, ageOff); + + Assert.assertEquals(expectedTS, compositeTS); + Assert.assertTrue(CompositeTimestamp.isCompositeTimestamp(compositeTS)); + Assert.assertFalse(CompositeTimestamp.isCompositeTimestamp(eventDate)); + Assert.assertFalse(CompositeTimestamp.isCompositeTimestamp(ageOff)); + Assert.assertEquals(CompositeTimestamp.getEventDate(compositeTS), eventDate); + Assert.assertEquals(CompositeTimestamp.getAgeOffDate(compositeTS), ageOff); + } + + @Test + public void testVeryOldDateWithMaxAgeoff() { + long eventDate = Instant.from(DateTimeFormatter.ISO_INSTANT.parse("1654-01-01T00:00:00Z")).toEpochMilli(); + long ageOff = eventDate + (131071L * CompositeTimestamp.MILLIS_PER_DAY); + long expectedTS = -9223311640052998144L; + + long compositeTs1 = Long.MIN_VALUE + 1; + long eventDate1 = CompositeTimestamp.getEventDate(compositeTs1); + + long compositeTS = CompositeTimestamp.getCompositeTimeStamp(eventDate, ageOff); + + Assert.assertEquals(expectedTS, compositeTS); + Assert.assertTrue(CompositeTimestamp.isCompositeTimestamp(compositeTS)); + Assert.assertFalse(CompositeTimestamp.isCompositeTimestamp(eventDate)); + Assert.assertFalse(CompositeTimestamp.isCompositeTimestamp(ageOff)); + Assert.assertEquals(CompositeTimestamp.getEventDate(compositeTS), eventDate); + Assert.assertEquals(CompositeTimestamp.getAgeOffDate(compositeTS), ageOff); + } + + @Test + public void testEventUpperDateBound() { + long eventDate = (-1L >>> 18); + long compositeTS = CompositeTimestamp.getCompositeTimeStamp(eventDate, eventDate); + Assert.assertEquals(eventDate, compositeTS); + Assert.assertFalse(CompositeTimestamp.isCompositeTimestamp(compositeTS)); + Assert.assertEquals(CompositeTimestamp.getEventDate(compositeTS), eventDate); + Assert.assertEquals(CompositeTimestamp.getAgeOffDate(compositeTS), eventDate); + + try { + CompositeTimestamp.getCompositeTimeStamp(eventDate + 1, 0); + Assert.fail("Expected event date greater than 17 bits to fail"); + } catch (IllegalArgumentException e) { + // expected + } + } + + @Test + public void testEventLowerDateBound() { + long eventDate = (0 - (-1L >>> 18)); + long compositeTS = CompositeTimestamp.getCompositeTimeStamp(eventDate, eventDate); + Assert.assertEquals(eventDate, compositeTS); + Assert.assertFalse(CompositeTimestamp.isCompositeTimestamp(compositeTS)); + Assert.assertEquals(CompositeTimestamp.getEventDate(compositeTS), eventDate); + Assert.assertEquals(CompositeTimestamp.getAgeOffDate(compositeTS), eventDate); + + try { + CompositeTimestamp.getCompositeTimeStamp(eventDate - 1, 0); + Assert.fail("Expected event date greater than 17 bits to fail"); + } catch (IllegalArgumentException e) { + // expected + } + } + + @Test + public void testAgeOffDateBounds() { + long eventDate = (0 - (-1L >>> 18)); + long ageOffEventDays = (-1L >>> 47); + long ageOffEventDelta = ageOffEventDays * 1000 * 60 * 60 * 24; + long compositeTS = CompositeTimestamp.getCompositeTimeStamp(eventDate, eventDate + ageOffEventDelta); + Assert.assertTrue(CompositeTimestamp.isCompositeTimestamp(compositeTS)); + Assert.assertEquals(CompositeTimestamp.getEventDate(compositeTS), eventDate); + Assert.assertEquals(CompositeTimestamp.getAgeOffDate(compositeTS), eventDate + ageOffEventDelta); + + try { + CompositeTimestamp.getCompositeTimeStamp(eventDate, eventDate - CompositeTimestamp.MILLIS_PER_DAY); + Assert.fail("Expected ageoff date less than event date to fail"); + } catch (IllegalArgumentException e) { + // expected + } + + try { + CompositeTimestamp.getCompositeTimeStamp(CompositeTimestamp.MILLIS_PER_DAY, 0); + Assert.fail("Expected ageoff date less than event date to fail"); + } catch (IllegalArgumentException e) { + // expected + } + + try { + CompositeTimestamp.getCompositeTimeStamp(eventDate, eventDate + ageOffEventDelta + CompositeTimestamp.MILLIS_PER_DAY); + Assert.fail("Expected age off date greater than " + ageOffEventDays + " days from event date to fail"); + } catch (IllegalArgumentException e) { + // expected + } + } + + @Test + public void testMaxEntropy() { + long eventDate = -1L; + long ageOff = -1L; + + long compositeTS = CompositeTimestamp.getCompositeTimeStamp(eventDate, ageOff); + + Assert.assertEquals(-1L, compositeTS); + // since the ageoff is equal to the event date, this is not considered a composite timestamp + Assert.assertFalse(CompositeTimestamp.isCompositeTimestamp(compositeTS)); + Assert.assertEquals(CompositeTimestamp.getEventDate(compositeTS), eventDate); + Assert.assertEquals(CompositeTimestamp.getAgeOffDate(compositeTS), ageOff); + } + + @Test + public void testDoomsday() { + long compositeTs = Long.MAX_VALUE - 5L; + + long eventDate = CompositeTimestamp.getEventDate(compositeTs); // 4199-11-24 + long now = System.currentTimeMillis(); + + Date endGame = new Date(TimeUnit.MILLISECONDS.toMillis(eventDate)); + Assert.assertTrue("Doomsday is " + endGame + ". You have less than one year before timestamps roll over. Get cracking.", + (now + (365L * CompositeTimestamp.MILLIS_PER_DAY)) < eventDate); + + } + + @Test + public void testPostDoomsday() { + long eventDate = Instant.from(DateTimeFormatter.ISO_INSTANT.parse("4199-11-25T00:00:00Z")).toEpochMilli(); + long ageOff = Instant.from(DateTimeFormatter.ISO_INSTANT.parse("4299-06-01T00:00:00Z")).toEpochMilli(); + + try { + CompositeTimestamp.getCompositeTimeStamp(eventDate, ageOff); + Assert.fail("Expected event date to fail"); + } catch (IllegalArgumentException e) { + // expected + } + + } + + @Test + public void testMin() { + long ts = CompositeTimestamp.getCompositeTimeStamp(CompositeTimestamp.MIN_EVENT_DATE, CompositeTimestamp.MIN_EVENT_DATE); + long event = CompositeTimestamp.getEventDate(ts); + long age = CompositeTimestamp.getEventDate(ts); + Assert.assertEquals(event, age); + } + + @Test + public void testInvalid() { + try { + CompositeTimestamp.getEventDate(CompositeTimestamp.INVALID_TIMESTAMP); + Assert.fail("Invalid timestamp not detected"); + } catch (IllegalArgumentException e) { + + } + try { + CompositeTimestamp.getAgeOffDate(CompositeTimestamp.INVALID_TIMESTAMP); + Assert.fail("Invalid timestamp not detected"); + } catch (IllegalArgumentException e) { + + } + try { + CompositeTimestamp.isCompositeTimestamp(CompositeTimestamp.INVALID_TIMESTAMP); + Assert.fail("Invalid timestamp not detected"); + } catch (IllegalArgumentException e) { + + } + } + + protected boolean isOrdered(Long... times) { + List list1 = new ArrayList<>(Arrays.asList(times)); + List list2 = new ArrayList<>(Arrays.asList(times)); + Collections.sort(list2); + return list1.equals(list2); + } + + protected boolean isOrdered(Comparator comparator, Long... times) { + List list1 = new ArrayList<>(Arrays.asList(times)); + List list2 = new ArrayList<>(Arrays.asList(times)); + Collections.sort(list2, comparator); + return list1.equals(list2); + } + + @Test + public void testOrderingAgeOffDates() { + Calendar cal = Calendar.getInstance(); + long twoMonthsLater = cal.getTimeInMillis(); + cal.add(Calendar.MONTH, -1); + long aMonthLater = cal.getTimeInMillis(); + cal.add(Calendar.MONTH, -1); + long aDate = cal.getTimeInMillis(); + + // test the same positive eventdate, but different ageoff dates + long t1 = CompositeTimestamp.getCompositeTimeStamp(aDate, aDate); + long t2 = CompositeTimestamp.getCompositeTimeStamp(aDate, aMonthLater); + long t3 = CompositeTimestamp.getCompositeTimeStamp(aDate, twoMonthsLater); + + // in this case the natural ordering will be correct + Assert.assertTrue(isOrdered(t1, t2, t3)); + // and the comparator will maintain that ordering + Assert.assertTrue(isOrdered(CompositeTimestamp.comparator(), t1, t2, t3)); + + cal = Calendar.getInstance(); + cal.setTimeInMillis(0); + cal.add(Calendar.MONTH, -1); + twoMonthsLater = cal.getTimeInMillis(); + cal.add(Calendar.MONTH, -1); + aMonthLater = cal.getTimeInMillis(); + cal.add(Calendar.MONTH, -1); + aDate = cal.getTimeInMillis(); + + // test the same negative eventdate, but different ageoff dates + t1 = CompositeTimestamp.getCompositeTimeStamp(aDate, aDate); + t2 = CompositeTimestamp.getCompositeTimeStamp(aDate, aMonthLater); + t3 = CompositeTimestamp.getCompositeTimeStamp(aDate, twoMonthsLater); + + // in this case the natural ordering will be incorrect ( and in fact exactly opposite ) + Assert.assertFalse(isOrdered(t1, t2, t3)); + Assert.assertTrue(isOrdered(t2, t2, t1)); + // but the comparator will maintain the correct ordering + Assert.assertTrue(isOrdered(CompositeTimestamp.comparator(), t1, t2, t3)); + } + + @Test + public void testOrderingEventDates() { + Calendar cal = Calendar.getInstance(); + long twoMonthsLater = cal.getTimeInMillis(); + cal.add(Calendar.MONTH, -1); + long aMonthLater = cal.getTimeInMillis(); + cal.add(Calendar.MONTH, -1); + long aDate = cal.getTimeInMillis(); + + // test different event dates with the equivalent ageoff dates + long t1 = CompositeTimestamp.getCompositeTimeStamp(aDate, aDate); + long t2 = CompositeTimestamp.getCompositeTimeStamp(aMonthLater, aMonthLater); + long t3 = CompositeTimestamp.getCompositeTimeStamp(twoMonthsLater, twoMonthsLater); + + // in this case the natural ordering will be correct + Assert.assertTrue(isOrdered(t1, t2, t3)); + // and the comparator will maintain that ordering + Assert.assertTrue(isOrdered(CompositeTimestamp.comparator(), t1, t2, t3)); + + cal = Calendar.getInstance(); + cal.setTimeInMillis(0); + cal.add(Calendar.MONTH, -1); + twoMonthsLater = cal.getTimeInMillis(); + cal.add(Calendar.MONTH, -1); + aMonthLater = cal.getTimeInMillis(); + cal.add(Calendar.MONTH, -1); + aDate = cal.getTimeInMillis(); + + // test different negative event dates with the equivalent ageoff dates + t1 = CompositeTimestamp.getCompositeTimeStamp(aDate, aDate); + t2 = CompositeTimestamp.getCompositeTimeStamp(aMonthLater, aMonthLater); + t3 = CompositeTimestamp.getCompositeTimeStamp(twoMonthsLater, twoMonthsLater); + + // in this case the natural ordering will be correct + Assert.assertTrue(isOrdered(t1, t2, t3)); + // and the comparator will maintain that ordering + Assert.assertTrue(isOrdered(CompositeTimestamp.comparator(), t1, t2, t3)); + } + + @Test + public void testOrderingEventAndAgeoffDates() { + Calendar cal = Calendar.getInstance(); + long twoMonthsLater = cal.getTimeInMillis(); + cal.add(Calendar.MONTH, -1); + long aMonthLater = cal.getTimeInMillis(); + cal.add(Calendar.MONTH, -1); + long aDate = cal.getTimeInMillis(); + + // a mix of ageoff dates and event dates + long t1 = CompositeTimestamp.getCompositeTimeStamp(aDate, twoMonthsLater); + long t2 = CompositeTimestamp.getCompositeTimeStamp(aMonthLater, aMonthLater); + long t3 = CompositeTimestamp.getCompositeTimeStamp(twoMonthsLater, twoMonthsLater); + + // in this case the natural ordering will be incorrect + Assert.assertFalse(isOrdered(t1, t2, t3)); + // but the comparator will maintain the correct ordering + Assert.assertTrue(isOrdered(CompositeTimestamp.comparator(), t1, t2, t3)); + + cal = Calendar.getInstance(); + cal.setTimeInMillis(0); + cal.add(Calendar.MONTH, -1); + twoMonthsLater = cal.getTimeInMillis(); + cal.add(Calendar.MONTH, -1); + aMonthLater = cal.getTimeInMillis(); + cal.add(Calendar.MONTH, -1); + aDate = cal.getTimeInMillis(); + + // a mix of negative ageoff dates and negative event dates + t1 = CompositeTimestamp.getCompositeTimeStamp(aDate, twoMonthsLater); + t2 = CompositeTimestamp.getCompositeTimeStamp(aMonthLater, aMonthLater); + t3 = CompositeTimestamp.getCompositeTimeStamp(twoMonthsLater, twoMonthsLater); + + // in this case the natural ordering will be correct (surprisingly) + Assert.assertTrue(isOrdered(t1, t2, t3)); + // and the comparator will maintain that ordering + Assert.assertTrue(isOrdered(CompositeTimestamp.comparator(), t1, t2, t3)); + } + +} diff --git a/warehouse/ingest-configuration/src/main/resources/config/shard-ingest-config.xml b/warehouse/ingest-configuration/src/main/resources/config/shard-ingest-config.xml index 05001c62094..86150eb31d4 100644 --- a/warehouse/ingest-configuration/src/main/resources/config/shard-ingest-config.xml +++ b/warehouse/ingest-configuration/src/main/resources/config/shard-ingest-config.xml @@ -70,6 +70,15 @@ datawave.ingest.table.config.ShardTableConfigHelper + + ${table.name.shardIndex}.disable.versioning.iterator + true + + + ${table.name.shardReverseIndex}.disable.versioning.iterator + true + + markings.setup.iterator.enabled false diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/config/RawRecordContainerImpl.java b/warehouse/ingest-core/src/main/java/datawave/ingest/config/RawRecordContainerImpl.java index 8e06d2c614e..96dea3dc999 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/config/RawRecordContainerImpl.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/config/RawRecordContainerImpl.java @@ -37,6 +37,7 @@ import datawave.ingest.data.config.ingest.IgnorableErrorHelperInterface; import datawave.ingest.protobuf.RawRecordContainer.Data; import datawave.marking.MarkingFunctions; +import datawave.util.CompositeTimestamp; public class RawRecordContainerImpl implements Writable, Configurable, RawRecordContainer { @@ -61,7 +62,10 @@ public class RawRecordContainerImpl implements Writable, Configurable, RawRecord private Multimap fatalErrors = HashMultimap.create(); private Multimap ignorableErrorHelpers = HashMultimap.create(); - private long eventDate = Long.MIN_VALUE; + /** + * This is the composite date for this event + */ + private long timestamp = CompositeTimestamp.INVALID_TIMESTAMP; private Type dataType = null; private UID uid = null; private UIDBuilder uidBuilder; @@ -179,13 +183,13 @@ public void setDataType(Type dataType) { } @Override - public long getDate() { - return this.eventDate; + public long getTimestamp() { + return this.timestamp; } @Override - public void setDate(long date) { - this.eventDate = date; + public void setTimestamp(long date) { + this.timestamp = date; } @Override @@ -421,7 +425,7 @@ public boolean equals(Object other) { } RawRecordContainerImpl e = (RawRecordContainerImpl) other; EqualsBuilder equals = new EqualsBuilder(); - equals.append(this.eventDate, e.eventDate); + equals.append(this.timestamp, e.timestamp); equals.append(this.dataType, e.dataType); equals.append(this.uid, e.uid); equals.append(this.errors, e.errors); @@ -439,7 +443,7 @@ public boolean equals(Object other) { @Override public int hashCode() { - int result = (int) (eventDate ^ (eventDate >>> 32)); + int result = (int) (timestamp ^ (timestamp >>> 32)); result = 31 * result + (dataType != null ? dataType.hashCode() : 0); result = 31 * result + (uid != null ? uid.hashCode() : 0); result = 31 * result + (errors != null ? errors.hashCode() : 0); @@ -465,7 +469,7 @@ public int hashCode() { protected RawRecordContainerImpl copyInto(RawRecordContainerImpl rrci) { copyConfiguration(rrci); - rrci.eventDate = this.eventDate; + rrci.timestamp = this.timestamp; rrci.dataType = this.dataType; rrci.uid = this.uid; rrci.errors = new ConcurrentSkipListSet<>(this.errors); @@ -645,7 +649,7 @@ public void reloadConfiguration() { @Override public void write(DataOutput out) throws IOException { Data.Builder builder = Data.newBuilder(); - builder.setDate(this.eventDate); + builder.setDate(this.timestamp); if (null != this.dataType) builder.setDataType(this.dataType.typeName()); if (null != this.uid) @@ -679,7 +683,7 @@ public void readFields(DataInput in) throws IOException { in.readFully(buf); Data data = Data.parseFrom(buf); - this.eventDate = data.getDate(); + this.timestamp = data.getDate(); if (data.hasDataType()) try { this.dataType = TypeRegistry.getType(data.getDataType()); @@ -715,7 +719,7 @@ public void readFields(DataInput in) throws IOException { * Resets state for re-use. */ public void clear() { - eventDate = Long.MIN_VALUE; + timestamp = CompositeTimestamp.INVALID_TIMESTAMP; dataType = null; uid = null; errors.clear(); @@ -740,7 +744,7 @@ public void clear() { @Override public String toString() { ToStringBuilder buf = new ToStringBuilder(this); - buf.append("eventDate", this.eventDate); + buf.append("timestamp", this.timestamp); buf.append("dataType", dataType.typeName()); buf.append("uid", String.valueOf(this.uid)); buf.append("errors", errors); diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/RawRecordContainer.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/RawRecordContainer.java index dde4200bc6f..189820a4853 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/RawRecordContainer.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/RawRecordContainer.java @@ -9,6 +9,7 @@ import org.apache.accumulo.core.security.ColumnVisibility; import datawave.data.hash.UID; +import datawave.util.CompositeTimestamp; /** * Generic container used to hold raw source data. It is used in various parts of the ingest framework and is typically persisted as an "event" within DW's @@ -45,7 +46,43 @@ public interface RawRecordContainer { * * @return the date for this raw record */ - long getDate(); + default long getDate() { + return CompositeTimestamp.getEventDate(getTimestamp()); + } + + /** + * Gets the ageoff date associated with the record + * + * @return the ageoff date for this raw record + */ + default long getAgeOffDate() { + return CompositeTimestamp.getAgeOffDate(getTimestamp()); + } + + /** + * Get the composite timestamp asociated with this record (@see CompositeTimestamp) + * + * @return + */ + long getTimestamp(); + + /** + * Determine if the timestamp has been set. This avoids having to compare the timestamp with an arbitrary value + */ + default boolean isTimestampSet() { + return getTimestamp() != CompositeTimestamp.INVALID_TIMESTAMP; + } + + /** + * This is synonomis with setTimestamp(date, date) + * + * @param timestamp + * primary date to be associated with the record, a.k.a. the "event date" + */ + @Deprecated + default void setDate(long timestamp) { + setTimestamp(timestamp); + } /** * In the DW data model, this date is often referred to as "event date" and represents the primary date value for the record. At ingest time, it is @@ -56,11 +93,29 @@ public interface RawRecordContainer { * Thus, this date is typically leveraged by DW's query api as the basis for 'begin' / 'end' date ranges for user queries. However, DW also has the ability * to leverage other dates within your records at query time, if needed. So, for date filtering concerns, you're not necessarily stuck with your choice of * 'event' date in this regard + *

* - * @param date + *

+ * This date is treated as a composite timestamp which includes the age off date as well. If the ageoff date is identical to the event date (which is + * usually the case), then the event date and the timestamp will be the exactly same value. See CompositeTimestamp for more information. The getDate() + * method will only return the event date portion of this date and the getAgeOffDate() will return the ageoff portion. This method should eventually be + * deprecated and setTimestamp should be used instead. + *

+ * + * @param timestamp * primary date to be associated with the record, a.k.a. the "event date" */ - void setDate(long date); + void setTimestamp(long timestamp); + + /** + * Equivalent to setTimestamp(CompositeTimestamp.getCompositeTimestamp(eventDate, ageOffDate)); + * + * @param eventDate + * @param ageOffDate + */ + default void setTimestamp(long eventDate, long ageOffDate) { + setTimestamp(CompositeTimestamp.getCompositeTimeStamp(eventDate, ageOffDate)); + } Collection getErrors(); diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/BaseIngestHelper.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/BaseIngestHelper.java index 4be57bdf85e..c3d28d3a2d8 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/BaseIngestHelper.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/BaseIngestHelper.java @@ -47,6 +47,7 @@ * fields values from the datatypes that they represent. */ public abstract class BaseIngestHelper extends AbstractIngestHelper implements CompositeIngest, VirtualIngest { + /** * Configuration parameter to specify that data should be marked for delete on ingest. */ diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/input/reader/AbstractEventRecordReader.java b/warehouse/ingest-core/src/main/java/datawave/ingest/input/reader/AbstractEventRecordReader.java index 52e0c50e864..20083dae89e 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/input/reader/AbstractEventRecordReader.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/input/reader/AbstractEventRecordReader.java @@ -35,6 +35,7 @@ import datawave.ingest.data.config.MarkingsHelper; import datawave.policy.IngestPolicyEnforcer; import datawave.policy.Policy; +import datawave.util.CompositeTimestamp; public abstract class AbstractEventRecordReader extends RecordReader implements EventRecordReader { @@ -246,7 +247,7 @@ protected void extractEventDate(final String fieldName, final String fieldValue) } } } - if (event.getDate() == Long.MIN_VALUE) { + if (!event.isTimestampSet()) { List patterns = new ArrayList<>(formatters.size()); for (SimpleDateFormat formatter : formatters) { patterns.add(formatter.toPattern()); diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomDataTypeHandler.java index b0f22fc8539..f48dfa0a9ec 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomDataTypeHandler.java @@ -180,20 +180,20 @@ public long process(KEYIN key, RawRecordContainer event, Multimap Long.MIN_VALUE) { + if (event.isTimestampSet()) { eventDate = DateHelper.format(event.getDate()); } Text colq = safeAppend(null, eventDate); diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java index 00fa4a0c1a1..e167321c8d7 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java @@ -311,7 +311,7 @@ record = record.copy(); // ShardId 'd' DataType\0UID\0Name for document content event using Event.Writable String colq = record.getDataType().outputName() + '\0' + record.getId() + '\0' + EVENT_CONTENT_FIELD; Key k = createKey(getShardId(record), new Text(ExtendedDataTypeHandler.FULL_CONTENT_COLUMN_FAMILY), new Text(colq), getVisibility(record, null), - record.getDate(), this.helper.getDeleteMode()); + record.getTimestamp(), this.helper.getDeleteMode()); BulkIngestKey ebKey = new BulkIngestKey(getShardTableName(), k); contextWriter.write(ebKey, value, context); diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java index dc3f27c7a94..4073a14a0f4 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java @@ -47,6 +47,7 @@ import datawave.ingest.util.DiskSpaceStarvationStrategy; import datawave.marking.MarkingFunctions; import datawave.query.model.Direction; +import datawave.util.CompositeTimestamp; import datawave.util.TextUtil; /** @@ -405,6 +406,9 @@ public Multimap processBulk(KEYIN key, RawRecordContainer e if (event.fatalError()) { return null; } else { + // create an event that returns its timestamp date + IngestHelperInterface helper = getHelper(event.getDataType()); + if (isReindexEnabled) { Multimap filteredEventFields = filterByRequestedFields(eventFields); if (filteredEventFields.isEmpty()) { @@ -739,13 +743,13 @@ protected Multimap createTermIndexColumn(RawRecordContainer } } // Create a key for the masked field value with the masked visibility. - Key k = this.createIndexKey(normalizedMaskedValue.getBytes(), colf, colq, maskedVisibility, event.getDate(), false); + Key k = this.createIndexKey(normalizedMaskedValue.getBytes(), colf, colq, maskedVisibility, event.getTimestamp(), false); BulkIngestKey bkey = new BulkIngestKey(tableName, k); values.put(bkey, indexValue); } if (!StringUtils.isEmpty(fieldValue)) { // Now create a key for the unmasked value with the original visibility - Key k = this.createIndexKey(fieldValue.getBytes(), colf, colq, visibility, event.getDate(), deleteMode); + Key k = this.createIndexKey(fieldValue.getBytes(), colf, colq, visibility, event.getTimestamp(), deleteMode); BulkIngestKey bkey = new BulkIngestKey(tableName, k); values.put(bkey, indexValue); } @@ -765,7 +769,7 @@ protected Multimap createTermIndexColumn(RawRecordContainer refVisibility = maskedVisibility; } - Key k = this.createIndexKey(fieldValue.getBytes(), colf, colq, refVisibility, event.getDate(), deleteMode); + Key k = this.createIndexKey(fieldValue.getBytes(), colf, colq, refVisibility, event.getTimestamp(), deleteMode); BulkIngestKey bkey = new BulkIngestKey(tableName, k); values.put(bkey, indexValue); @@ -851,14 +855,34 @@ protected Key createKey(byte[] row, Text colf, Text colq, byte[] vis, long ts, b * @return Accumulo Key object */ protected Key createIndexKey(byte[] row, Text colf, Text colq, byte[] vis, long ts, boolean delete) { - // Truncate the timestamp to the day - long tsToDay = (ts / MS_PER_DAY) * MS_PER_DAY; - - Key k = new Key(row, 0, row.length, colf.getBytes(), 0, colf.getLength(), colq.getBytes(), 0, colq.getLength(), vis, 0, vis.length, tsToDay); + Key k = new Key(row, 0, row.length, colf.getBytes(), 0, colf.getLength(), colq.getBytes(), 0, colq.getLength(), vis, 0, vis.length, + getIndexTimestamp(ts)); k.setDeleted(delete); return k; } + /** + * trim the event date and ageoff portions of the ts to the beginning of the day + * + * @param ts + * @return the timestamp to be used for index entries + */ + public static long getIndexTimestamp(long ts) { + long tsToDay = trimToBeginningOfDay(CompositeTimestamp.getEventDate(ts)); + long ageOffToDay = trimToBeginningOfDay(CompositeTimestamp.getAgeOffDate(ts)); + return CompositeTimestamp.getCompositeTimeStamp(tsToDay, ageOffToDay); + } + + /** + * Trim ms to the beginning of the day + * + * @param date + * @return the time at the beginning of the day + */ + public static long trimToBeginningOfDay(long date) { + return (date / MS_PER_DAY) * MS_PER_DAY; + } + /** * Creates a shard column key and does *NOT* apply masking logic * @@ -933,7 +957,7 @@ protected Multimap createShardEventColumn(RawRecordContaine // Generate a key for the original, unmasked field field value if (!StringUtils.isEmpty(fieldValue)) { // One key with the original value and original visibility - Key cbKey = createKey(shardId, colf, unmaskedColq, visibility, event.getDate(), deleteMode); + Key cbKey = createKey(shardId, colf, unmaskedColq, visibility, event.getTimestamp(), deleteMode); BulkIngestKey bKey = new BulkIngestKey(this.getShardTableName(), cbKey); values.put(bKey, NULL_VALUE); @@ -955,7 +979,7 @@ protected Multimap createShardEventColumn(RawRecordContaine } // Else create one key for the field with the original value and the masked visiblity - Key cbKey = createKey(shardId, colf, unmaskedColq, refVisibility, event.getDate(), deleteMode); + Key cbKey = createKey(shardId, colf, unmaskedColq, refVisibility, event.getTimestamp(), deleteMode); BulkIngestKey bKey = new BulkIngestKey(this.getShardTableName(), cbKey); if (log.isTraceEnabled()) log.trace("Creating bulk ingest Key " + bKey); @@ -974,7 +998,7 @@ protected void createMaskedShardEventColumn(RawRecordContainer event, Text colf, TextUtil.textAppend(maskedColq, maskedFieldValue, replaceMalformedUTF8); // Another key with masked value and masked visibility - Key cbKey = createKey(shardId, colf, maskedColq, maskedVisibility, event.getDate(), deleteMode); + Key cbKey = createKey(shardId, colf, maskedColq, maskedVisibility, event.getTimestamp(), deleteMode); BulkIngestKey bKey = new BulkIngestKey(this.getShardTableName(), cbKey); values.put(bKey, NULL_VALUE); } @@ -1032,7 +1056,7 @@ protected Multimap createShardFieldIndexColumn(RawRecordCon if (null != maskedFieldHelper && maskedFieldHelper.contains(fieldName)) { // Put unmasked colq with original visibility - Key k = createKey(shardId, colf, unmaskedColq, visibility, event.getDate(), deleteMode); + Key k = createKey(shardId, colf, unmaskedColq, visibility, event.getTimestamp(), deleteMode); BulkIngestKey bKey = new BulkIngestKey(this.getShardTableName(), k); values.put(bKey, value); @@ -1044,7 +1068,7 @@ protected Multimap createShardFieldIndexColumn(RawRecordCon TextUtil.textAppend(maskedColq, event.getId().toString(), replaceMalformedUTF8); // Put masked colq with masked visibility - Key key = createKey(shardId, colf, maskedColq, maskedVisibility, event.getDate(), deleteMode); + Key key = createKey(shardId, colf, maskedColq, maskedVisibility, event.getTimestamp(), deleteMode); BulkIngestKey bulkIngestKey = new BulkIngestKey(this.getShardTableName(), key); values.put(bulkIngestKey, value); } @@ -1059,7 +1083,7 @@ protected Multimap createShardFieldIndexColumn(RawRecordCon refVisibility = maskedVisibility; } - Key k = createKey(shardId, colf, unmaskedColq, refVisibility, event.getDate(), deleteMode); + Key k = createKey(shardId, colf, unmaskedColq, refVisibility, event.getTimestamp(), deleteMode); BulkIngestKey bKey = new BulkIngestKey(this.getShardTableName(), k); values.put(bKey, value); } @@ -1110,7 +1134,7 @@ protected void createShardFieldIndexColumn(RawRecordContainer event, Multimap context, StatusReporter reporter, Text uid, byte[] visibility, byte[] shardId, byte[] rawValue) throws IOException, InterruptedException, MutationsRejectedException { - Key k = createKey(shardId, new Text(ExtendedDataTypeHandler.FULL_CONTENT_COLUMN_FAMILY), uid, visibility, event.getDate(), + Key k = createKey(shardId, new Text(ExtendedDataTypeHandler.FULL_CONTENT_COLUMN_FAMILY), uid, visibility, event.getTimestamp(), this.ingestHelper.getDeleteMode()); ByteArrayOutputStream baos = null; @@ -754,7 +754,7 @@ protected void createShardFieldIndexColumn(RawRecordContainer event, ContextWrit value = DataTypeHandler.NULL_VALUE; } - Key k = createKey(shardId, colf, colq, visibility, event.getDate(), deleteMode); + Key k = createKey(shardId, colf, colq, visibility, event.getTimestamp(), deleteMode); BulkIngestKey bKey = new BulkIngestKey(new Text(this.getShardTableName()), k); contextWriter.write(bKey, value, context); } @@ -798,8 +798,9 @@ protected void createTermFrequencyIndex(RawRecordContainer event, ContextWriter< colq.append(this.eventDataTypeName).append('\u0000').append(this.eventUid).append('\u0000').append(nfv.getIndexedFieldValue()).append('\u0000') .append(nfv.getIndexedFieldName()); - BulkIngestKey bKey = new BulkIngestKey(new Text(this.getShardTableName()), new Key(shardId, - ExtendedDataTypeHandler.TERM_FREQUENCY_COLUMN_FAMILY.getBytes(), colq.toString().getBytes(), visibility, event.getDate(), deleteMode)); + BulkIngestKey bKey = new BulkIngestKey(new Text(this.getShardTableName()), + new Key(shardId, ExtendedDataTypeHandler.TERM_FREQUENCY_COLUMN_FAMILY.getBytes(), colq.toString().getBytes(), visibility, + event.getTimestamp(), deleteMode)); contextWriter.write(bKey, value, context); } @@ -844,7 +845,7 @@ protected void createIndexColumn(RawRecordContainer event, ContextWriter getValues(SortedKeyValueIterator iterator) { + return new ValueCombiner(iterator, PartialKey.ROW_COLFAM_COLQUAL_COLVIS_TIME); + } + + @Override public Value aggregate() { Builder builder = Uid.List.newBuilder(); @@ -229,27 +238,14 @@ public Value reduce(Key key, Iterator iter) { */ private boolean processRemovalUids(Uid.List value) { for (String uid : value.getREMOVEDUIDList()) { - // Don't remove the UID if it's in the UID list since that means a newer key - // (larger timestamp value) added the UID and we don't want to undo that add. - // If timestampsIgnored is set, then we are presuming lots of collisions on - // timestamp and the order of incoming values is non-deterministic. In that case + // The order of incoming values for the same timestamp is non-deterministic. In that case // we give precedence to a removal over an add and mark this UID as removed even // if it was in the UID list. - // - // If we're not propagating changes then don't bother adding UIDs to the - // REMOVEDUID list. This happens if either we're doing a scan or a full major - // compaction. In those cases, we're guaranteed that we'll see all of the values - // for a given key and therefore there's no need to include removed UIDs in the - // output protocol buffer since the remove have already been applied. - if (timestampsIgnored) { - uids.remove(uid); - // Even if propagate is false, the removal UID should persist until all PB lists - // in the current iteration have been seen, in case a subsequent PB has the UID - // marked for removal. - uidsToRemove.add(uid); - } else if (propogate && !uids.contains(uid)) { - uidsToRemove.add(uid); - } + uids.remove(uid); + // Even if propagate is false, the removal UID should persist until all PB lists + // in the current iteration have been seen, in case a subsequent PB has the UID + // marked for removal. + uidsToRemove.add(uid); // Avoid exceeding maxUids in the uidToRemove list, even when propogating. // Check for this condition after adding each of the remove UID entries to uidsToRemove, // which also de-duplicates uids in that Set. @@ -326,18 +322,12 @@ public boolean propogateKey() { @Override public boolean validateOptions(Map options) { boolean valid = super.validateOptions(options); - if (valid) { - if (options.containsKey(TIMESTAMPS_IGNORED)) { - timestampsIgnored = Boolean.parseBoolean(options.get(TIMESTAMPS_IGNORED)); - } - } return valid; } @Override public SortedKeyValueIterator deepCopy(IteratorEnvironment env) { GlobalIndexUidAggregator copy = (GlobalIndexUidAggregator) super.deepCopy(env); - copy.timestampsIgnored = timestampsIgnored; copy.propogate = propogate; // Not copying other fields that are all cleared in the reset() method. return copy; @@ -346,12 +336,5 @@ public SortedKeyValueIterator deepCopy(IteratorEnvironment env) { @Override public void init(SortedKeyValueIterator source, Map options, IteratorEnvironment env) throws IOException { super.init(source, options, env); - if (options.containsKey(TIMESTAMPS_IGNORED)) { - timestampsIgnored = Boolean.parseBoolean(options.get(TIMESTAMPS_IGNORED)); - } - } - - public static void setTimestampsIgnoredOpt(IteratorSetting is, boolean timestampsIgnored) { - is.addOption(TIMESTAMPS_IGNORED, Boolean.toString(timestampsIgnored)); } } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/table/aggregator/PropogatingCombiner.java b/warehouse/ingest-core/src/main/java/datawave/ingest/table/aggregator/PropogatingCombiner.java index 089a5536663..79e7a60cb0e 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/table/aggregator/PropogatingCombiner.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/table/aggregator/PropogatingCombiner.java @@ -5,8 +5,11 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.iterators.Combiner; +import org.apache.accumulo.core.iterators.SortedKeyValueIterator; import org.apache.log4j.Logger; +import datawave.iterators.ValueCombiner; + /** * * Aids in determining if a value and the corresponding keys should propogate @@ -23,6 +26,16 @@ public abstract class PropogatingCombiner extends Combiner { */ protected boolean propogate = true; + /** + * Get the iterator of values to be combined + * + * @param iterator + * @return a value iterator + */ + public Iterator getValues(SortedKeyValueIterator iterator) { + return new ValueCombiner(iterator); + } + /** * Shpuld return a thread safe value. * diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/AbstractTableConfigHelper.java b/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/AbstractTableConfigHelper.java index 479a98d2a88..41d2c811d9f 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/AbstractTableConfigHelper.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/AbstractTableConfigHelper.java @@ -82,6 +82,8 @@ public static void setPropertyIfNecessary(String tableName, String propertyName, * the aggregators that should be set on {@code tableName} * @param tops * accumulo table operations helper for configuring tables + * @param config + * the configuration * @param log * a {@link Logger} for diagnostic messages * @@ -143,6 +145,8 @@ public static Map generateInitialTableProperties(Configuration co * the aggregators to check for on {@code tableName} * @param tops * accumulo table operations helper for configuring tables + * @param config + * the configuration * @return {@code true} if {@code aggregators} are configured on {@code tableName} and {@code false} if not * * @throws TableNotFoundException diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/ShardTableConfigHelper.java b/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/ShardTableConfigHelper.java index 49ba804d89f..8100a053fc9 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/ShardTableConfigHelper.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/ShardTableConfigHelper.java @@ -170,6 +170,7 @@ protected void configureShardTable(TableOperations tops) throws AccumuloExceptio // Set a text index aggregator on the "tf" (Term Frequency) column family CombinerConfiguration tfConf = new CombinerConfiguration(new Column("tf"), new IteratorSetting(10, "TF", datawave.ingest.table.aggregator.TextIndexAggregator.class.getName())); + setAggregatorConfigurationIfNecessary(tableName, Collections.singletonList(tfConf), tops, conf, log); if (markingsSetupIteratorEnabled) { diff --git a/warehouse/ingest-core/src/main/java/datawave/iterators/PropogatingIterator.java b/warehouse/ingest-core/src/main/java/datawave/iterators/PropogatingIterator.java index d7eadac85fe..91ae66d9cb9 100644 --- a/warehouse/ingest-core/src/main/java/datawave/iterators/PropogatingIterator.java +++ b/warehouse/ingest-core/src/main/java/datawave/iterators/PropogatingIterator.java @@ -150,7 +150,7 @@ private boolean aggregateRowColumn() throws IOException { // reset the state of the combiner. aggr.reset(); - aggregatedValue = aggr.reduce(keyToAggregate, new ValueCombiner(iterator)); + aggregatedValue = aggr.reduce(keyToAggregate, aggr.getValues(iterator)); // always propagate deletes if (aggr.propogateKey() || workKey.isDeleted()) { diff --git a/warehouse/ingest-core/src/main/java/datawave/iterators/ValueCombiner.java b/warehouse/ingest-core/src/main/java/datawave/iterators/ValueCombiner.java index 0168ff42681..e4653804382 100644 --- a/warehouse/ingest-core/src/main/java/datawave/iterators/ValueCombiner.java +++ b/warehouse/ingest-core/src/main/java/datawave/iterators/ValueCombiner.java @@ -18,6 +18,7 @@ public class ValueCombiner implements Iterator { Key topKey; SortedKeyValueIterator source; boolean hasNext; + PartialKey keysToCombine = PartialKey.ROW_COLFAM_COLQUAL_COLVIS; private static final Logger log = Logger.getLogger(ValueCombiner.class); @@ -28,7 +29,12 @@ public class ValueCombiner implements Iterator { * The {@code SortedKeyValueIterator} from which to read data. */ public ValueCombiner(SortedKeyValueIterator source) { + this(source, PartialKey.ROW_COLFAM_COLQUAL_COLVIS); + } + + public ValueCombiner(SortedKeyValueIterator source, PartialKey keysToCombine) { this.source = source; + this.keysToCombine = keysToCombine; topKey = new Key(source.getTopKey()); hasNext = _hasNext(); } @@ -40,7 +46,7 @@ private boolean _hasNext() { log.trace(source.getTopKey()); } - return source.hasTop() && topKey.equals(source.getTopKey(), PartialKey.ROW_COLFAM_COLQUAL_COLVIS); + return source.hasTop() && topKey.equals(source.getTopKey(), keysToCombine); } /** diff --git a/warehouse/ingest-core/src/main/java/datawave/policy/ExampleIngestPolicyEnforcer.java b/warehouse/ingest-core/src/main/java/datawave/policy/ExampleIngestPolicyEnforcer.java index c1942d3d5be..e0c01c673e6 100644 --- a/warehouse/ingest-core/src/main/java/datawave/policy/ExampleIngestPolicyEnforcer.java +++ b/warehouse/ingest-core/src/main/java/datawave/policy/ExampleIngestPolicyEnforcer.java @@ -51,7 +51,7 @@ private void validateSecurityMarkings(RawRecordContainer event) { * an event container */ private void validateEventDate(RawRecordContainer event) { - if (event.getDate() == Long.MIN_VALUE) { + if (!event.isTimestampSet()) { event.addError(RawDataErrorNames.EVENT_DATE_MISSING); log.error("Event date missing for Event in raw file: " + event.getRawFileName() + ", record: " + event.getRawRecordNumber()); } diff --git a/warehouse/ingest-core/src/test/java/datawave/ingest/data/RawRecordContainerImplTest.java b/warehouse/ingest-core/src/test/java/datawave/ingest/data/RawRecordContainerImplTest.java index cbd546f38b7..a8f191526b3 100644 --- a/warehouse/ingest-core/src/test/java/datawave/ingest/data/RawRecordContainerImplTest.java +++ b/warehouse/ingest-core/src/test/java/datawave/ingest/data/RawRecordContainerImplTest.java @@ -22,6 +22,7 @@ import datawave.ingest.config.IngestConfigurationFactory; import datawave.ingest.config.RawRecordContainerImpl; import datawave.ingest.data.config.MarkingsHelper; +import datawave.util.CompositeTimestamp; public class RawRecordContainerImplTest { @@ -377,17 +378,18 @@ public void validate() { if (getAltIds() == null || getAltIds().isEmpty()) { addError(RawDataErrorNames.UUID_MISSING); } - if (Long.MIN_VALUE == getDate()) { + if (!isTimestampSet()) { addError(RawDataErrorNames.EVENT_DATE_MISSING); + } else { + if (0L == getRawFileTimestamp()) { + setRawFileTimestamp(getDate()); + } } + if (0 == getRawRecordNumber()) { addError(RawDataErrorNames.INVALID_RECORD_NUMBER); } - if (0L == getRawFileTimestamp()) { - setRawFileTimestamp(getDate()); - } - try { initializeMarkingsHelper(); } catch (Throwable t) { diff --git a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/EventMapperTest.java b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/EventMapperTest.java index b0248906afa..39e6046c659 100644 --- a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/EventMapperTest.java +++ b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/EventMapperTest.java @@ -79,7 +79,7 @@ public void setUp() throws Exception { record = new SimpleRawRecord(); record.setRawFileTimestamp(eventTime); record.setDataType(type); - record.setDate(eventTime); + record.setTimestamp(eventTime); record.setRawFileName("/some/filename"); record.setRawData("some data".getBytes()); record.generateId(null); @@ -87,7 +87,7 @@ record = new SimpleRawRecord(); errorRecord = new SimpleRawRecord(); errorRecord.setRawFileTimestamp(0); errorRecord.setDataType(type); - errorRecord.setDate(eventTime); + errorRecord.setTimestamp(eventTime); errorRecord.setRawFileName("/some/filename"); errorRecord.setRawData("some data".getBytes()); errorRecord.generateId(null); diff --git a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/SimpleRawRecord.java b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/SimpleRawRecord.java index 10429435564..9d8180b4021 100644 --- a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/SimpleRawRecord.java +++ b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/SimpleRawRecord.java @@ -20,6 +20,7 @@ import datawave.ingest.data.RawRecordContainer; import datawave.ingest.data.Type; import datawave.ingest.data.TypeRegistry; +import datawave.util.CompositeTimestamp; /** * A POJO implementation of the {@link datawave.ingest.data.RawRecordContainer}. @@ -31,7 +32,7 @@ public class SimpleRawRecord implements RawRecordContainer, Writable { private Map securityMarkings = new TreeMap<>(); private UID id = uidBuilder.newId(); private Type dataType; - private long date = Long.MIN_VALUE; + private long timestamp = CompositeTimestamp.INVALID_TIMESTAMP; private Collection errors = new ArrayList<>(); private Collection altIds = Collections.emptyList(); private String rawFileName = ""; @@ -89,13 +90,13 @@ public void setDataType(Type dataType) { } @Override - public long getDate() { - return date; + public long getTimestamp() { + return timestamp; } @Override - public void setDate(long date) { - this.date = date; + public void setTimestamp(long timestamp) { + this.timestamp = timestamp; } @Override @@ -247,7 +248,7 @@ public void write(DataOutput dataOutput) throws IOException { id.write(dataOutput); dataOutput.writeUTF(dataType.typeName()); - dataOutput.writeLong(date); + dataOutput.writeLong(timestamp); TestWritableUtil.writeCollection(errors, dataOutput); TestWritableUtil.writeCollection(altIds, dataOutput); @@ -274,7 +275,7 @@ public void readFields(DataInput dataInput) throws IOException { id.readFields(dataInput); dataType = TypeRegistry.getType(dataInput.readUTF()); - date = dataInput.readLong(); + timestamp = dataInput.readLong(); errors = TestWritableUtil.readCollection(dataInput); altIds = TestWritableUtil.readCollection(dataInput); diff --git a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandlerTest.java b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandlerTest.java index eac3bd65cc6..8acf41dd121 100644 --- a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandlerTest.java +++ b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandlerTest.java @@ -5,8 +5,10 @@ import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.concurrent.TimeUnit; import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.TaskAttemptID; @@ -17,8 +19,7 @@ import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; -import datawave.data.type.GeometryType; -import datawave.data.type.NumberType; +import datawave.data.hash.HashUID; import datawave.ingest.config.RawRecordContainerImpl; import datawave.ingest.data.RawRecordContainer; import datawave.ingest.data.Type; @@ -27,15 +28,14 @@ import datawave.ingest.data.config.MaskedFieldHelper; import datawave.ingest.data.config.NormalizedContentInterface; import datawave.ingest.data.config.NormalizedFieldAndValue; -import datawave.ingest.data.config.ingest.BaseIngestHelper; import datawave.ingest.data.config.ingest.ContentBaseIngestHelper; import datawave.ingest.mapreduce.job.BulkIngestKey; -import datawave.ingest.mapreduce.partition.BalancedShardPartitioner; import datawave.ingest.protobuf.Uid; import datawave.ingest.table.config.ShardTableConfigHelper; import datawave.ingest.table.config.TableConfigHelper; import datawave.policy.IngestPolicyEnforcer; import datawave.query.model.Direction; +import datawave.util.CompositeTimestamp; import datawave.util.TableName; public class ShardedDataTypeHandlerTest { @@ -48,6 +48,8 @@ public class ShardedDataTypeHandlerTest { private static final String DATA_TYPE_NAME = "wkt"; private static final String INGEST_HELPER_CLASS = TestIngestHelper.class.getName(); + private static final long MS_PER_DAY = TimeUnit.DAYS.toMillis(1); + Configuration configuration; public static class TestIngestHelper extends ContentBaseIngestHelper { @@ -140,6 +142,7 @@ public void testCreateTermIndex() { record.setRawFileName("data_" + 0 + ".dat"); record.setRawRecordNumber(1); record.setRawData(entry.getBytes(StandardCharsets.UTF_8)); + record.setDate(System.currentTimeMillis()); Uid.List uid = Uid.List.newBuilder().setIGNORE(false).setCOUNT(1).addUID("d8zay2.-3pnndm.-anolok").build(); byte[] visibility = new byte[] {65, 76, 76}; @@ -160,6 +163,7 @@ public void testCreateTermReverseIndex() { record.setRawFileName("data_" + 0 + ".dat"); record.setRawRecordNumber(1); record.setRawData(entry.getBytes(StandardCharsets.UTF_8)); + record.setDate(System.currentTimeMillis()); Uid.List uid = Uid.List.newBuilder().setIGNORE(false).setCOUNT(1).addUID("d8zay2.-3pnndm.-anolok").build(); byte[] visibility = new byte[] {65, 76, 76}; @@ -189,6 +193,7 @@ public void testMaskedForward() { record.setRawFileName("data_" + 0 + ".dat"); record.setRawRecordNumber(1); record.setRawData(entry.getBytes(StandardCharsets.UTF_8)); + record.setDate(System.currentTimeMillis()); Uid.List uid = Uid.List.newBuilder().setIGNORE(false).setCOUNT(1).addUID("d8zay2.-3pnndm.-anolok").build(); byte[] visibility = new byte[] {65, 76, 76}; @@ -218,6 +223,7 @@ public void testNonMaskedReverseIndex() { record.setRawFileName("data_" + 0 + ".dat"); record.setRawRecordNumber(1); record.setRawData(entry.getBytes(StandardCharsets.UTF_8)); + record.setDate(System.currentTimeMillis()); Uid.List uid = Uid.List.newBuilder().setIGNORE(false).setCOUNT(1).addUID("d8zay2.-3pnndm.-anolok").build(); byte[] visibility = new byte[] {65, 76, 76}; @@ -246,6 +252,7 @@ public void testNonMaskedVisibility() { record.setRawFileName("data_" + 0 + ".dat"); record.setRawRecordNumber(1); record.setRawData(entry.getBytes(StandardCharsets.UTF_8)); + record.setDate(System.currentTimeMillis()); Uid.List uid = Uid.List.newBuilder().setIGNORE(false).setCOUNT(1).addUID("d8zay2.-3pnndm.-anolok").build(); byte[] visibility = new byte[] {65, 76, 76}; @@ -262,4 +269,58 @@ public void testNonMaskedVisibility() { } } + @Test + public void testAgeOffDate() { + Type dataType = new Type(DATA_TYPE_NAME, TestIngestHelper.class, null, null, 10, null); + String entry = "testingtesting"; + RawRecordContainer record = new RawRecordContainerImpl(); + record.setDataType(dataType); + record.setRawFileName("data_" + 0 + ".dat"); + record.setRawRecordNumber(1); + record.setRawData(entry.getBytes(StandardCharsets.UTF_8)); + record.setId(HashUID.builder().newId(record.getRawData())); + record.setVisibility(new ColumnVisibility("PUBLIC")); + long expectedEventDate = System.currentTimeMillis(); + record.setDate(expectedEventDate); + Multimap fields = ingestHelper.getEventFields(record); + + assertEquals(expectedEventDate, record.getDate()); + assertEquals(expectedEventDate, record.getAgeOffDate()); + assertEquals(expectedEventDate, record.getTimestamp()); + + Multimap data = handler.processBulk(null, record, fields, null); + + long expectedTimestamp = CompositeTimestamp.getCompositeTimeStamp(expectedEventDate, expectedEventDate); + long tsToDay = (expectedEventDate / MS_PER_DAY) * MS_PER_DAY; + long expectedIndexTimestamp = CompositeTimestamp.getCompositeTimeStamp(tsToDay, tsToDay); + for (BulkIngestKey key : data.keySet()) { + if (key.getTableName().toString().toUpperCase().contains("INDEX")) { + assertEquals(key.toString(), expectedIndexTimestamp, key.getKey().getTimestamp()); + } else { + assertEquals(key.toString(), expectedTimestamp, key.getKey().getTimestamp()); + } + } + + // now get an ageoff date (must be a multiple of MS_PER_DAY past the event date) + long expectedAgeOffDate = expectedEventDate + (MS_PER_DAY * 11); + expectedTimestamp = CompositeTimestamp.getCompositeTimeStamp(expectedEventDate, expectedAgeOffDate); + record.setTimestamp(expectedTimestamp); + + assertEquals(expectedEventDate, record.getDate()); + assertEquals(expectedAgeOffDate, record.getAgeOffDate()); + assertEquals(expectedTimestamp, record.getTimestamp()); + + data = handler.processBulk(null, record, fields, null); + + long ageOffToDay = (expectedAgeOffDate / MS_PER_DAY) * MS_PER_DAY; + expectedIndexTimestamp = CompositeTimestamp.getCompositeTimeStamp(tsToDay, ageOffToDay); + for (BulkIngestKey key : data.keySet()) { + if (key.getTableName().toString().toUpperCase().contains("INDEX")) { + assertEquals(key.toString(), expectedIndexTimestamp, key.getKey().getTimestamp()); + } else { + assertEquals(key.toString(), expectedTimestamp, key.getKey().getTimestamp()); + } + } + } + } diff --git a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/tokenize/ContentIndexingColumnBasedHandlerTest.java b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/tokenize/ContentIndexingColumnBasedHandlerTest.java index d4f7d5fb748..05fb07cc75d 100644 --- a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/tokenize/ContentIndexingColumnBasedHandlerTest.java +++ b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/tokenize/ContentIndexingColumnBasedHandlerTest.java @@ -155,7 +155,7 @@ private void setupMocks() { EasyMock.expect(event.getVisibility()).andReturn(colVis).anyTimes(); EasyMock.expect(event.getDataType()).andReturn(TypeRegistry.getType(TEST_TYPE)).anyTimes(); EasyMock.expect(event.getId()).andReturn(uid).anyTimes(); - EasyMock.expect(event.getDate()).andReturn(CURRENT_TIME).anyTimes(); + EasyMock.expect(event.getTimestamp()).andReturn(CURRENT_TIME).anyTimes(); EasyMock.replay(event); } catch (Exception e) { throw new RuntimeException(e); diff --git a/warehouse/ingest-core/src/test/java/datawave/ingest/table/aggregator/GlobalIndexUidAggregatorTest.java b/warehouse/ingest-core/src/test/java/datawave/ingest/table/aggregator/GlobalIndexUidAggregatorTest.java index da5c24d3dcf..226b8b7f3fd 100644 --- a/warehouse/ingest-core/src/test/java/datawave/ingest/table/aggregator/GlobalIndexUidAggregatorTest.java +++ b/warehouse/ingest-core/src/test/java/datawave/ingest/table/aggregator/GlobalIndexUidAggregatorTest.java @@ -12,25 +12,36 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import java.io.IOException; +import java.time.Instant; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.Iterator; import java.util.List; +import java.util.Set; import java.util.UUID; import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.iterators.SortedKeyValueIterator; import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.junit.Test; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import com.google.common.collect.TreeMultimap; import datawave.ingest.protobuf.Uid; import datawave.ingest.protobuf.Uid.List.Builder; +import datawave.iterators.SortedMultiMapIterator; +import datawave.util.CompositeTimestamp; public class GlobalIndexUidAggregatorTest { @@ -236,10 +247,9 @@ public void testCountWithDuplicates() throws Exception { } @Test - public void testRemoveAndReAddUUID() throws Exception { + public void testRemoveAndThenAddUUID() throws Exception { GlobalIndexUidAggregator localAgg = new GlobalIndexUidAggregator(); IteratorSetting is = new IteratorSetting(19, "test", GlobalIndexUidAggregator.class); - GlobalIndexUidAggregator.setTimestampsIgnoredOpt(is, false); GlobalIndexUidAggregator.setCombineAllColumns(is, true); localAgg.validateOptions(is.getOptions()); @@ -258,12 +268,12 @@ public void testRemoveAndReAddUUID() throws Exception { Collections.reverse(values); Value result = localAgg.reduce(new Key("key"), values.iterator()); Uid.List resultList = Uid.List.parseFrom(result.get()); - assertEquals(2, resultList.getCOUNT()); - assertEquals(2, resultList.getUIDCount()); - assertEquals(2, resultList.getUIDList().size()); - assertEquals(0, resultList.getREMOVEDUIDList().size()); + assertEquals(1, resultList.getCOUNT()); + assertEquals(1, resultList.getUIDCount()); + assertEquals(1, resultList.getUIDList().size()); + assertEquals(1, resultList.getREMOVEDUIDList().size()); assertTrue(resultList.getUIDList().contains(uuid1)); - assertTrue(resultList.getUIDList().contains(uuid2)); + assertTrue(resultList.getREMOVEDUIDList().contains(uuid2)); } @Test @@ -298,7 +308,6 @@ public void testRemoveAndReAddUUIDWithTimestampsIgnored() throws Exception { public void testNegativeCountWithPartialMajorCompaction() throws Exception { GlobalIndexUidAggregator localAgg = new GlobalIndexUidAggregator(); IteratorSetting is = new IteratorSetting(19, "test", GlobalIndexUidAggregator.class); - GlobalIndexUidAggregator.setTimestampsIgnoredOpt(is, false); GlobalIndexUidAggregator.setCombineAllColumns(is, true); localAgg.validateOptions(is.getOptions()); @@ -413,7 +422,6 @@ public void testNegativeCountWithPartialMajorCompactionAndTimestampsIgnored() th public void testRemoveAndReAddUUIDWithPartialMajorCompaction() throws Exception { GlobalIndexUidAggregator localAgg = new GlobalIndexUidAggregator(); IteratorSetting is = new IteratorSetting(19, "test", GlobalIndexUidAggregator.class); - GlobalIndexUidAggregator.setTimestampsIgnoredOpt(is, false); GlobalIndexUidAggregator.setCombineAllColumns(is, true); localAgg.validateOptions(is.getOptions()); @@ -468,12 +476,12 @@ public void testRemoveAndReAddUUIDWithPartialMajorCompaction() throws Exception result = localAgg.reduce(new Key("key"), values.iterator()); resultList = Uid.List.parseFrom(result.get()); - assertEquals(2, resultList.getCOUNT()); - assertEquals(2, resultList.getUIDCount()); - assertEquals(2, resultList.getUIDList().size()); + assertEquals(1, resultList.getCOUNT()); + assertEquals(1, resultList.getUIDCount()); + assertEquals(1, resultList.getUIDList().size()); assertTrue(resultList.getUIDList().contains(uuid1)); - assertTrue(resultList.getUIDList().contains(uuid2)); - assertEquals(0, resultList.getREMOVEDUIDList().size()); + assertFalse(resultList.getUIDList().contains(uuid2)); + assertEquals(1, resultList.getREMOVEDUIDList().size()); } @Test @@ -728,7 +736,6 @@ public void testAddDuplicateUUIDWithSeenIgnoreAndCompaction() throws Exception { public void testRemoveAndReAdd() throws Exception { GlobalIndexUidAggregator localAgg = new GlobalIndexUidAggregator(); IteratorSetting is = new IteratorSetting(19, "test", GlobalIndexUidAggregator.class); - GlobalIndexUidAggregator.setTimestampsIgnoredOpt(is, false); GlobalIndexUidAggregator.setCombineAllColumns(is, true); localAgg.validateOptions(is.getOptions()); @@ -737,20 +744,19 @@ public void testRemoveAndReAdd() throws Exception { String uuid1 = UUID.randomUUID().toString(); ArrayList values = Lists.newArrayList(); - // When we're considering timestamps, an add of a UID, followed by a removal - // of that UID, and then a re-add should result in the UID ending up in the - // UID list. + // an add of a UID, followed by a removal of that UID, and then a re-add + // should still result in the UID removed values.add(toValue(createNewUidList(uuid1))); values.add(toValue(createNewRemoveUidList(uuid1))); values.add(toValue(createNewUidList(uuid1))); Value result = localAgg.reduce(new Key("key"), values.iterator()); Uid.List resultList = Uid.List.parseFrom(result.get()); - assertEquals(1, resultList.getUIDCount()); - assertEquals(1, resultList.getUIDList().size()); - assertEquals(0, resultList.getREMOVEDUIDCount()); - assertEquals(0, resultList.getREMOVEDUIDList().size()); - assertEquals(1, resultList.getCOUNT()); + assertEquals(0, resultList.getUIDCount()); + assertEquals(0, resultList.getUIDList().size()); + assertEquals(1, resultList.getREMOVEDUIDCount()); + assertEquals(1, resultList.getREMOVEDUIDList().size()); + assertEquals(0, resultList.getCOUNT()); } @Test @@ -891,6 +897,54 @@ public void testWeKeepUidListDuringDoubleRemovals() { assertTrue(agg.propogateKey()); } + @Test + public void testCompositeTimestampsMatter() throws IOException { + TreeMultimap keyValues = TreeMultimap.create(); + + long eventDate = Instant.from(DateTimeFormatter.ISO_INSTANT.parse("1960-01-01T00:00:00Z")).toEpochMilli(); + long ageOff = eventDate + (131071L * CompositeTimestamp.MILLIS_PER_DAY); + long negativeCompositeTS = CompositeTimestamp.getCompositeTimeStamp(eventDate, ageOff); + + eventDate = Instant.from(DateTimeFormatter.ISO_INSTANT.parse("2022-10-26T01:00:00Z")).toEpochMilli(); + ageOff = eventDate + CompositeTimestamp.MILLIS_PER_DAY; + long compositeTS = CompositeTimestamp.getCompositeTimeStamp(eventDate, ageOff); + + ageOff = eventDate + (CompositeTimestamp.MILLIS_PER_DAY * 10); + long largerCompositeTS = CompositeTimestamp.getCompositeTimeStamp(eventDate, ageOff); + + List values = new ArrayList<>(); + for (int i = 0; i < 6; i++) { + String uuid = UUID.randomUUID().toString(); + values.add(toValue(createNewUidList(uuid))); + } + + Key key1 = new Key("key", "cf", "cq", "PUBLIC", negativeCompositeTS); + Key key2 = new Key("key", "cf", "cq", "PUBLIC", compositeTS); + Key key3 = new Key("key", "cf", "cq", "PUBLIC", largerCompositeTS); + + keyValues.put(key1, values.get(0)); + keyValues.put(key2, values.get(1)); + keyValues.put(key2, values.get(2)); + keyValues.put(key3, values.get(3)); + keyValues.put(key3, values.get(4)); + keyValues.put(key3, values.get(5)); + + // get an iterator of these key/value pairs (sorted) + SortedKeyValueIterator iterator = new SortedMultiMapIterator(keyValues); + iterator.seek(new Range(), Collections.emptySet(), false); + + // get a unique list of the keys (sorted + Iterator keys = keyValues.keySet().iterator(); + while (keys.hasNext()) { + Key key = keys.next(); + Set expected = Sets.newHashSet(keyValues.get(key)); + // ensure each call to getValues() returns all of the values for the next key where only the timestamp differs + Set actual = Sets.newHashSet(agg.getValues(iterator)); + assertEquals(expected, actual); + } + assertFalse(iterator.hasTop()); + } + private Value agg(List values) { agg.reset(); return agg.reduce(new Key("row"), values.iterator()); diff --git a/warehouse/ingest-core/src/test/java/datawave/ingest/table/config/AbstractTableConfigHelperTest.java b/warehouse/ingest-core/src/test/java/datawave/ingest/table/config/AbstractTableConfigHelperTest.java index dcbe818fceb..f62818acd4b 100644 --- a/warehouse/ingest-core/src/test/java/datawave/ingest/table/config/AbstractTableConfigHelperTest.java +++ b/warehouse/ingest-core/src/test/java/datawave/ingest/table/config/AbstractTableConfigHelperTest.java @@ -56,6 +56,8 @@ public static class TestAbstractTableConfigHelperImpl extends AbstractTableConfi public AbstractTableConfigHelperTest parent; + private Configuration config; + protected Logger createMockLogger() { Logger log = PowerMock.createMock(Logger.class); diff --git a/warehouse/ingest-csv/src/main/java/datawave/ingest/csv/mr/input/CSVRecordReader.java b/warehouse/ingest-csv/src/main/java/datawave/ingest/csv/mr/input/CSVRecordReader.java index dfdb4e99417..52899115f0d 100644 --- a/warehouse/ingest-csv/src/main/java/datawave/ingest/csv/mr/input/CSVRecordReader.java +++ b/warehouse/ingest-csv/src/main/java/datawave/ingest/csv/mr/input/CSVRecordReader.java @@ -202,7 +202,7 @@ else if (!StringUtils.isEmpty(this.csvHelper.getEventIdFieldName()) && fieldName } } // if we set the date with id, don't overwrite it - if (!(fieldName.equals(eventDateFieldName) && event.getDate() > Long.MIN_VALUE)) { + if (!(fieldName.equals(eventDateFieldName) && event.isTimestampSet())) { super.checkField(fieldName, fieldValue); } diff --git a/warehouse/ingest-json/src/main/java/datawave/ingest/json/mr/input/JsonRecordReader.java b/warehouse/ingest-json/src/main/java/datawave/ingest/json/mr/input/JsonRecordReader.java index 520b8f55337..73661a44fd5 100644 --- a/warehouse/ingest-json/src/main/java/datawave/ingest/json/mr/input/JsonRecordReader.java +++ b/warehouse/ingest-json/src/main/java/datawave/ingest/json/mr/input/JsonRecordReader.java @@ -229,7 +229,7 @@ public RawRecordContainer getEvent() { event.setRawData(currentJsonObj.toString().getBytes()); - if (Long.MIN_VALUE == event.getDate()) { + if (!event.isTimestampSet()) { event.setDate(System.currentTimeMillis()); } diff --git a/warehouse/ingest-wikipedia/src/main/java/datawave/ingest/wikipedia/WikipediaDataTypeHandler.java b/warehouse/ingest-wikipedia/src/main/java/datawave/ingest/wikipedia/WikipediaDataTypeHandler.java index 7acfa9ddc6e..2a8abeb05c1 100644 --- a/warehouse/ingest-wikipedia/src/main/java/datawave/ingest/wikipedia/WikipediaDataTypeHandler.java +++ b/warehouse/ingest-wikipedia/src/main/java/datawave/ingest/wikipedia/WikipediaDataTypeHandler.java @@ -352,7 +352,7 @@ protected void createShardEventColumn(RawRecordContainer event, ContextWriter key.getTimestamp()) ? maxTimeStamp : key.getTimestamp(); + this.maxTimeStamp = (this.maxTimeStamp > key.getTimestamp()) ? maxTimeStamp : CompositeTimestamp.getEventDate(key.getTimestamp()); } private boolean acceptTimestamp(Key k) { - return this.stampRange.containsLong(k.getTimestamp()); + return this.stampRange.containsLong(CompositeTimestamp.getEventDate(k.getTimestamp())); } private boolean isFieldIndexKey(Key key) { diff --git a/warehouse/query-core/src/main/java/datawave/core/iterators/filter/GlobalIndexDateRangeFilter.java b/warehouse/query-core/src/main/java/datawave/core/iterators/filter/GlobalIndexDateRangeFilter.java index 3b51a2a755e..4d1359826f6 100644 --- a/warehouse/query-core/src/main/java/datawave/core/iterators/filter/GlobalIndexDateRangeFilter.java +++ b/warehouse/query-core/src/main/java/datawave/core/iterators/filter/GlobalIndexDateRangeFilter.java @@ -15,6 +15,7 @@ import org.apache.log4j.Logger; import datawave.query.Constants; +import datawave.util.CompositeTimestamp; /** * The iterator skips entries in the global index for entries that lie outside the date range set on the BatchScanner @@ -66,7 +67,7 @@ public static void main(String[] args) { @Override public boolean accept(Key k, Value v) { - return range.containsLong(k.getTimestamp()); + return range.containsLong(CompositeTimestamp.getEventDate(k.getTimestamp())); } diff --git a/warehouse/query-core/src/main/java/datawave/query/iterator/filter/LoadDateFilter.java b/warehouse/query-core/src/main/java/datawave/query/iterator/filter/LoadDateFilter.java index 62a2630248d..5b762a4a38f 100644 --- a/warehouse/query-core/src/main/java/datawave/query/iterator/filter/LoadDateFilter.java +++ b/warehouse/query-core/src/main/java/datawave/query/iterator/filter/LoadDateFilter.java @@ -16,6 +16,7 @@ import datawave.core.iterators.ColumnRangeIterator; import datawave.edge.util.EdgeValue; +import datawave.util.CompositeTimestamp; import datawave.util.time.DateHelper; /** @@ -118,6 +119,6 @@ public boolean accept(Key k, Value V) { // event date from the timestamp value rather than the column qualifier. private String extractEventDate(Key k) { - return DateHelper.format(k.getTimestamp()); + return DateHelper.format(CompositeTimestamp.getEventDate(k.getTimestamp())); } } diff --git a/warehouse/query-core/src/main/java/datawave/query/iterators/FieldIndexCountingIterator.java b/warehouse/query-core/src/main/java/datawave/query/iterators/FieldIndexCountingIterator.java index 137cc924b0e..263dd99ee3d 100644 --- a/warehouse/query-core/src/main/java/datawave/query/iterators/FieldIndexCountingIterator.java +++ b/warehouse/query-core/src/main/java/datawave/query/iterators/FieldIndexCountingIterator.java @@ -32,6 +32,7 @@ import datawave.iterators.IteratorSettingHelper; import datawave.marking.MarkingFunctions; import datawave.query.Constants; +import datawave.util.CompositeTimestamp; import datawave.util.TextUtil; /** @@ -355,11 +356,12 @@ private void consume(Key key) { this.count += 1; // set most recent timestamp - this.maxTimeStamp = (this.maxTimeStamp > key.getTimestamp()) ? maxTimeStamp : key.getTimestamp(); + this.maxTimeStamp = (this.maxTimeStamp > CompositeTimestamp.getEventDate(key.getTimestamp())) ? maxTimeStamp + : CompositeTimestamp.getEventDate(key.getTimestamp()); } private boolean acceptTimestamp(Key k) { - return this.stampRange.containsLong(k.getTimestamp()); + return this.stampRange.containsLong(CompositeTimestamp.getEventDate(k.getTimestamp())); } private boolean isFieldIndexKey(Key key) { diff --git a/warehouse/query-core/src/main/java/datawave/query/predicate/TimeFilter.java b/warehouse/query-core/src/main/java/datawave/query/predicate/TimeFilter.java index fa11987eec8..f7094a96799 100644 --- a/warehouse/query-core/src/main/java/datawave/query/predicate/TimeFilter.java +++ b/warehouse/query-core/src/main/java/datawave/query/predicate/TimeFilter.java @@ -9,6 +9,8 @@ import com.google.common.base.Predicate; +import datawave.util.CompositeTimestamp; + /** * Excludes documents which do not fall within the given time range */ @@ -36,7 +38,7 @@ private class KeyTimeFilter implements Predicate { public boolean apply(Key input) { final long timestamp = input.getTimestamp(); - return acceptedRange.containsLong(timestamp); + return acceptedRange.containsLong(CompositeTimestamp.getEventDate(timestamp)); } } diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java index 0720f412b09..261c988c016 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java @@ -42,6 +42,7 @@ import datawave.query.tables.ShardQueryLogic; import datawave.query.transformer.FieldIndexCountQueryTransformer; import datawave.query.util.MetadataHelper; +import datawave.util.CompositeTimestamp; import datawave.util.StringUtils; import datawave.webservice.query.exception.QueryException; @@ -487,8 +488,8 @@ public Tuple(MarkingFunctions mf) { public void aggregate(Key key, Value val) { uniqueVisibilities.add(key.getColumnVisibility()); count += Long.parseLong(new String(val.get())); - if (maxTimestamp < key.getTimestamp()) { - maxTimestamp = key.getTimestamp(); + if (maxTimestamp < CompositeTimestamp.getEventDate(key.getTimestamp())) { + maxTimestamp = CompositeTimestamp.getEventDate(key.getTimestamp()); } } diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformerSupport.java b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformerSupport.java index 213c1d2cf4b..4fbd5a2db1c 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformerSupport.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformerSupport.java @@ -46,6 +46,7 @@ import datawave.query.iterator.QueryOptions; import datawave.query.iterator.profile.QuerySpan; import datawave.query.jexl.JexlASTHelper; +import datawave.util.CompositeTimestamp; import datawave.util.StringUtils; import datawave.util.time.DateHelper; import datawave.webservice.query.result.event.EventBase; @@ -386,7 +387,7 @@ protected void collectCardinalities(Document document, Key documentKey, String u log.trace("Document.getTimestamp() returned Log.MAX_VALUE - " + documentKey + " - computed dataDate from row: " + dataDate); } } else { - dataDate = new Date(timestamp); + dataDate = new Date(CompositeTimestamp.getEventDate(timestamp)); } resultCardinalityDocumentDate.addEntry(valueMap, eventId, dataType, dataDate); diff --git a/warehouse/query-core/src/test/java/datawave/helpers/PrintUtility.java b/warehouse/query-core/src/test/java/datawave/helpers/PrintUtility.java index 058d06d36e1..7d1d97ee4fb 100644 --- a/warehouse/query-core/src/test/java/datawave/helpers/PrintUtility.java +++ b/warehouse/query-core/src/test/java/datawave/helpers/PrintUtility.java @@ -23,6 +23,7 @@ import datawave.edge.util.EdgeValue; import datawave.edge.util.ExtendedHyperLogLogPlus; import datawave.ingest.protobuf.Uid; +import datawave.util.CompositeTimestamp; /** * A set of static methods for printing tables in mock Accumulo instance. @@ -61,8 +62,10 @@ public static void printTable(final AccumuloClient client, final Authorizations final Scanner scanner = client.createScanner(tableName, authorizations); for (final Entry e : scanner) { sb.append(e.getKey().toStringNoTime()); - sb.append(' '); - sb.append(dateFormat.format(new Date(e.getKey().getTimestamp()))); + sb.append(" EventDate "); + sb.append(dateFormat.format(new Date(CompositeTimestamp.getEventDate(e.getKey().getTimestamp())))); + sb.append(" AgeOffDate "); + sb.append(dateFormat.format(new Date(CompositeTimestamp.getAgeOffDate(e.getKey().getTimestamp())))); sb.append('\t'); sb.append(getPrintableValue(tableName, e.getKey(), e.getValue())); sb.append("\n"); @@ -85,8 +88,10 @@ public static void printTable(final AccumuloClient client, final Authorizations final Scanner scanner = client.createScanner(tableName, authorizations); for (final Entry e : scanner) { sb.append(e.getKey().toStringNoTime()); - sb.append(' '); - sb.append(dateFormat.format(new Date(e.getKey().getTimestamp()))); + sb.append(" EventDate "); + sb.append(dateFormat.format(new Date(CompositeTimestamp.getEventDate(e.getKey().getTimestamp())))); + sb.append(" AgeOffDate "); + sb.append(dateFormat.format(new Date(CompositeTimestamp.getAgeOffDate(e.getKey().getTimestamp())))); sb.append('\t'); sb.append(getPrintableValue(tableName, e.getKey(), e.getValue())); sb.append("\n"); diff --git a/warehouse/query-core/src/test/java/datawave/query/TestLimitReturnedGroupsToHitTermGroups.java b/warehouse/query-core/src/test/java/datawave/query/TestLimitReturnedGroupsToHitTermGroups.java index 3f105da01fd..2914592b8b3 100644 --- a/warehouse/query-core/src/test/java/datawave/query/TestLimitReturnedGroupsToHitTermGroups.java +++ b/warehouse/query-core/src/test/java/datawave/query/TestLimitReturnedGroupsToHitTermGroups.java @@ -249,7 +249,7 @@ public void testMultipleGroups() throws Exception { "CANINE.PET.12:bernese", "CAT.PET.12:himalayan", "BIRD.PET.12:cockatiel", "FISH.PET.12:swordtail", "CANINE.PET.13:shepherd", "CAT.PET.13:ragdoll", "BIRD.PET.13:lovebird", "FISH.PET.13:tetra", "CANINE.WILD.1:coyote", "CAT.WILD.1:tiger", "BIRD.WILD.1:hawk", "FISH.WILD.1:tuna", - "REPTILE.PET.1:snake", "DOG.WILD.1:coyote"); + "REPTILE.PET.1:snake", "DOG.WILD.1:coyote", "CAT.PET.50:sphynx", "CANINE.PET.50:doberman"); //@formatter:on runTestQuery(queryString, format.parse("20091231"), format.parse("20150101"), extraParameters, goodResults); @@ -267,7 +267,7 @@ public void testMultipleGroupsWithRegexAndReturnFields() throws Exception { // definitely should NOT include group 2 or 3 Set goodResults = Sets.newHashSet("CANINE.PET.0:beagle", "CANINE.PET.1:basset", "CANINE.PET.12:bernese", "CANINE.PET.13:shepherd", - "CANINE.WILD.1:coyote"); + "CANINE.WILD.1:coyote", "CANINE.PET.50:doberman"); runTestQuery(queryString, format.parse("20091231"), format.parse("20150101"), extraParameters, goodResults); } @@ -367,7 +367,7 @@ public void testMultipleIncludeGroupingFalse() throws Exception { String queryString = "filter:getAllMatches(CANINE,'.*e.*')"; Set goodResults = Sets.newHashSet("BIRD:parakeet", "BIRD:canary", "CANINE:beagle", "CANINE:coyote", "CANINE:basset", "CANINE:shepherd", - "CANINE:bernese"); + "CANINE:bernese", "CANINE:doberman"); runTestQuery(queryString, format.parse("20091231"), format.parse("20150101"), extraParameters, goodResults); } @@ -385,7 +385,8 @@ public void testLimiting() throws Exception { Set goodResults = Sets.newHashSet("BIRD:parakeet", "CANINE:beagle", "CANINE:coyote", "CANINE:basset", "CANINE:shepherd", "CANINE:bernese", "FISH:tuna", "CAT:tabby", "CAT:tom", "FISH:swordtail", "FISH:angelfish", "CAT:siamese", "FISH:goldfish", "CAT:himalayan", "CAT:leopard", "CAT:cougar", "CAT:calico", "CAT:tiger", "FISH:tetra", "FISH:mackerel", "FISH:shark", "CAT:puma", "CAT:ragdoll", "FISH:beta", - "FISH:guppy", "FISH:salmon", "REPTILE:snake", "DOG:coyote", "SIZE:20,12.5", "SIZE:90,26.5"); + + "FISH:guppy", "FISH:salmon", "REPTILE:snake", "DOG:coyote", "SIZE:20,12.5", "SIZE:90,26.5", "CAT:sphynx", "CANINE:doberman"); runTestQuery(queryString, format.parse("20091231"), format.parse("20150101"), extraParameters, goodResults); } @@ -403,7 +404,8 @@ public void testLimitingToZero() throws Exception { Set goodResults = Sets.newHashSet("CANINE:beagle", "CANINE:coyote", "CANINE:basset", "CANINE:shepherd", "CANINE:bernese", "FISH:tuna", "CAT:tabby", "CAT:tom", "FISH:swordtail", "FISH:angelfish", "CAT:siamese", "FISH:goldfish", "CAT:himalayan", "CAT:leopard", "CAT:cougar", "CAT:calico", "CAT:tiger", "FISH:tetra", "FISH:mackerel", "FISH:shark", "CAT:puma", "CAT:ragdoll", "FISH:beta", - "FISH:guppy", "FISH:salmon", "REPTILE:snake", "SIZE:20,12.5", "SIZE:90,26.5"); + + "FISH:guppy", "FISH:salmon", "REPTILE:snake", "SIZE:20,12.5", "SIZE:90,26.5", "CAT:sphynx", "CANINE:doberman"); runTestQuery(queryString, format.parse("20091231"), format.parse("20150101"), extraParameters, goodResults); } @@ -423,8 +425,9 @@ public void testLimitingWithGrouping() throws Exception { "BIRD.WILD.1:hawk", "FISH.PET.12:swordtail", "CAT.PET.13:ragdoll", "FISH.WILD.0:shark", "CAT.PET.1:calico", "FISH.PET.0:beta", "CAT.WILD.1:tiger", "FISH.PET.2:angelfish", "CAT.PET.0:tabby", "FISH.WILD.2:mackerel", "FISH.PET.13:tetra", "FISH.PET.1:goldfish", "FISH.PET.3:guppy", "CAT.PET.12:himalayan", "FISH.WILD.1:tuna", "FISH.WILD.3:salmon", "CAT.WILD.3:puma", "CAT.WILD.2:leopard", + "CAT.PET.3:siamese", "CAT.WILD.0:cougar", "CAT.PET.2:tom", "REPTILE.PET.1:snake", "DOG.WILD.1:coyote", "SIZE.CANINE.3:20,12.5", - "SIZE.CANINE.WILD.1:90,26.5"); + "SIZE.CANINE.WILD.1:90,26.5", "CAT.PET.50:sphynx", "CANINE.PET.50:doberman"); runTestQuery(queryString, format.parse("20091231"), format.parse("20150101"), extraParameters, goodResults); } @@ -444,8 +447,9 @@ public void testLimitingToZeroWithGrouping() throws Exception { "BIRD.WILD.1:hawk", "FISH.PET.12:swordtail", "CAT.PET.13:ragdoll", "FISH.WILD.0:shark", "CAT.PET.1:calico", "FISH.PET.0:beta", "CAT.WILD.1:tiger", "FISH.PET.2:angelfish", "CAT.PET.0:tabby", "FISH.WILD.2:mackerel", "FISH.PET.13:tetra", "FISH.PET.1:goldfish", "FISH.PET.3:guppy", "CAT.PET.12:himalayan", "FISH.WILD.1:tuna", "FISH.WILD.3:salmon", "CAT.WILD.3:puma", "CAT.WILD.2:leopard", + "CAT.PET.3:siamese", "CAT.WILD.0:cougar", "CAT.PET.2:tom", "REPTILE.PET.1:snake", "DOG.WILD.1:coyote", "SIZE.CANINE.3:20,12.5", - "SIZE.CANINE.WILD.1:90,26.5"); + "SIZE.CANINE.WILD.1:90,26.5", "CAT.PET.50:sphynx", "CANINE.PET.50:doberman"); runTestQuery(queryString, format.parse("20091231"), format.parse("20150101"), extraParameters, goodResults); } diff --git a/warehouse/query-core/src/test/java/datawave/query/util/CommonalityTokenTestDataIngest.java b/warehouse/query-core/src/test/java/datawave/query/util/CommonalityTokenTestDataIngest.java index 14397276cd6..6d6b3d03a6c 100644 --- a/warehouse/query-core/src/test/java/datawave/query/util/CommonalityTokenTestDataIngest.java +++ b/warehouse/query-core/src/test/java/datawave/query/util/CommonalityTokenTestDataIngest.java @@ -20,6 +20,7 @@ import datawave.data.type.Type; import datawave.ingest.protobuf.Uid; import datawave.query.QueryTestTableHelper; +import datawave.util.CompositeTimestamp; import datawave.util.TableName; /** @@ -49,6 +50,13 @@ public static void writeItAll(AccumuloClient client, WhatKindaRange range) throw Mutation mutation = null; String myUID = UID.builder().newId("MyUid".getBytes(), (Date) null).toString(); + String myUID2 = UID.builder().newId("MyUid2".getBytes(), (Date) null).toString(); + String myUID3 = UID.builder().newId("MyUid3".getBytes(), (Date) null).toString(); + + long ageOffTimestamp = 1699041441288l; + long timeStamp2 = CompositeTimestamp.getCompositeTimeStamp(timeStamp, ageOffTimestamp); + + long timeStamp3 = CompositeTimestamp.getCompositeTimeStamp(ageOffTimestamp, ageOffTimestamp); try { // write the shard table : @@ -113,6 +121,14 @@ public static void writeItAll(AccumuloClient client, WhatKindaRange range) throw mutation.put(datatype + "\u0000" + myUID, "DOG.WILD.1" + "\u0000" + "coyote", columnVisibility, timeStamp, emptyValue); mutation.put(datatype + "\u0000" + myUID, "REPTILE.PET.1" + "\u0000" + "snake", columnVisibility, timeStamp, emptyValue); + // add an event with a composite timestamp that should return + mutation.put(datatype + "\u0000" + myUID2, "CAT.PET.50" + "\u0000" + "sphynx", columnVisibility, timeStamp2, emptyValue); + mutation.put(datatype + "\u0000" + myUID2, "CANINE.PET.50" + "\u0000" + "doberman", columnVisibility, timeStamp2, emptyValue); + + // add an event with a composite timestamp that should NOT return + mutation.put(datatype + "\u0000" + myUID3, "CAT.PET.60" + "\u0000" + "manx", columnVisibility, timeStamp3, emptyValue); + mutation.put(datatype + "\u0000" + myUID3, "CANINE.PET.60" + "\u0000" + "doberman", columnVisibility, timeStamp3, emptyValue); + bw.addMutation(mutation); } finally { @@ -129,6 +145,10 @@ public static void writeItAll(AccumuloClient client, WhatKindaRange range) throw mutation = new Mutation(lcNoDiacriticsType.normalize("tabby")); mutation.put("CAT", shard + "\u0000" + datatype, columnVisibility, timeStamp, range == WhatKindaRange.SHARD ? getValueForNuthinAndYourHitsForFree() : getValueForBuilderFor(myUID)); + mutation.put("CAT", shard + "\u0000" + datatype, columnVisibility, timeStamp2, + range == WhatKindaRange.SHARD ? getValueForNuthinAndYourHitsForFree() : getValueForBuilderFor(myUID2)); + mutation.put("CAT", shard + "\u0000" + datatype, columnVisibility, timeStamp3, + range == WhatKindaRange.SHARD ? getValueForNuthinAndYourHitsForFree() : getValueForBuilderFor(myUID3)); bw.addMutation(mutation); mutation = new Mutation(lcNoDiacriticsType.normalize("calico")); mutation.put("CAT", shard + "\u0000" + datatype, columnVisibility, timeStamp, @@ -183,6 +203,10 @@ public static void writeItAll(AccumuloClient client, WhatKindaRange range) throw mutation = new Mutation(lcNoDiacriticsType.normalize("shepherd")); mutation.put("CANINE", shard + "\u0000" + datatype, columnVisibility, timeStamp, range == WhatKindaRange.SHARD ? getValueForNuthinAndYourHitsForFree() : getValueForBuilderFor(myUID)); + mutation.put("CANINE", shard + "\u0000" + datatype, columnVisibility, timeStamp2, + range == WhatKindaRange.SHARD ? getValueForNuthinAndYourHitsForFree() : getValueForBuilderFor(myUID2)); + mutation.put("CANINE", shard + "\u0000" + datatype, columnVisibility, timeStamp3, + range == WhatKindaRange.SHARD ? getValueForNuthinAndYourHitsForFree() : getValueForBuilderFor(myUID3)); bw.addMutation(mutation); mutation = new Mutation(lcNoDiacriticsType.normalize("wolf")); mutation.put("CANINE", shard + "\u0000" + datatype, columnVisibility, timeStamp, @@ -301,6 +325,8 @@ public static void writeItAll(AccumuloClient client, WhatKindaRange range) throw // cats mutation.put("fi\u0000" + "CAT", lcNoDiacriticsType.normalize("tabby") + "\u0000" + datatype + "\u0000" + myUID, columnVisibility, timeStamp, emptyValue); + mutation.put("fi\u0000" + "CAT", lcNoDiacriticsType.normalize("tabby") + "\u0000" + datatype + "\u0000" + myUID2, columnVisibility, timeStamp2, + emptyValue); mutation.put("fi\u0000" + "CAT", lcNoDiacriticsType.normalize("calico") + "\u0000" + datatype + "\u0000" + myUID, columnVisibility, timeStamp, emptyValue); mutation.put("fi\u0000" + "CAT", lcNoDiacriticsType.normalize("tom") + "\u0000" + datatype + "\u0000" + myUID, columnVisibility, timeStamp, @@ -328,6 +354,8 @@ public static void writeItAll(AccumuloClient client, WhatKindaRange range) throw emptyValue); mutation.put("fi\u0000" + "CANINE", lcNoDiacriticsType.normalize("shepherd") + "\u0000" + datatype + "\u0000" + myUID, columnVisibility, timeStamp, emptyValue); + mutation.put("fi\u0000" + "CANINE", lcNoDiacriticsType.normalize("shepherd") + "\u0000" + datatype + "\u0000" + myUID2, columnVisibility, + timeStamp2, emptyValue); mutation.put("fi\u0000" + "CANINE", lcNoDiacriticsType.normalize("wolf") + "\u0000" + datatype + "\u0000" + myUID, columnVisibility, timeStamp, emptyValue); mutation.put("fi\u0000" + "CANINE", lcNoDiacriticsType.normalize("coyote") + "\u0000" + datatype + "\u0000" + myUID, columnVisibility, timeStamp, From e0a85e8d9cb22f85bfbab0712a4c5d01732c58cb Mon Sep 17 00:00:00 2001 From: Ivan Bella <347158+ivakegg@users.noreply.github.com> Date: Fri, 29 Nov 2024 13:41:29 +0000 Subject: [PATCH 2/8] Updated metadata submodule --- core/utils/metadata-utils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/utils/metadata-utils b/core/utils/metadata-utils index 6169ce38337..7f05830ac67 160000 --- a/core/utils/metadata-utils +++ b/core/utils/metadata-utils @@ -1 +1 @@ -Subproject commit 6169ce38337a31e9ef0a68285b544f7459010ab7 +Subproject commit 7f05830ac6788d492a8e38ef99f173d20bbaef65 From ca314708067af3e6b9732c2811e7fbf975661ebd Mon Sep 17 00:00:00 2001 From: Laura Schanno Date: Fri, 29 Nov 2024 08:42:44 -0500 Subject: [PATCH 3/8] Add test for patterned model mapping (#2653) * Add a test that shows the effect of expanding a query with a patterned forward mapping. * Created by request as part of datawave-metadata-utils/pull/39. --- pom.xml | 2 +- .../transformer/QueryModelExpansionTests.java | 197 ++++++++++++++++++ .../VisibilityWiseGuysIngestWithModel.java | 4 +- 3 files changed, 200 insertions(+), 3 deletions(-) create mode 100644 warehouse/query-core/src/test/java/datawave/query/transformer/QueryModelExpansionTests.java diff --git a/pom.xml b/pom.xml index c97d35bbd5a..71403ac1cde 100644 --- a/pom.xml +++ b/pom.xml @@ -67,7 +67,7 @@ 3.0.0 4.0.1 1.0.0 - 4.0.6 + 4.0.7-patternModelMapping 3.0.0 1.0.0 4.0.7 diff --git a/warehouse/query-core/src/test/java/datawave/query/transformer/QueryModelExpansionTests.java b/warehouse/query-core/src/test/java/datawave/query/transformer/QueryModelExpansionTests.java new file mode 100644 index 00000000000..707dfcffa31 --- /dev/null +++ b/warehouse/query-core/src/test/java/datawave/query/transformer/QueryModelExpansionTests.java @@ -0,0 +1,197 @@ +package datawave.query.transformer; + +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.TimeZone; +import java.util.UUID; + +import javax.inject.Inject; + +import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.commons.jexl3.parser.ASTJexlScript; +import org.apache.log4j.Logger; +import org.jboss.arquillian.container.test.api.Deployment; +import org.jboss.arquillian.junit.Arquillian; +import org.jboss.shrinkwrap.api.ShrinkWrap; +import org.jboss.shrinkwrap.api.asset.StringAsset; +import org.jboss.shrinkwrap.api.spec.JavaArchive; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; + +import datawave.configuration.spring.SpringBean; +import datawave.core.query.configuration.GenericQueryConfiguration; +import datawave.helpers.PrintUtility; +import datawave.ingest.data.TypeRegistry; +import datawave.microservice.query.QueryImpl; +import datawave.query.QueryParameters; +import datawave.query.QueryTestTableHelper; +import datawave.query.RebuildingScannerTestHelper; +import datawave.query.function.deserializer.KryoDocumentDeserializer; +import datawave.query.jexl.JexlASTHelper; +import datawave.query.tables.ShardQueryLogic; +import datawave.query.tables.edge.DefaultEdgeEventQueryLogic; +import datawave.query.util.VisibilityWiseGuysIngestWithModel; +import datawave.test.JexlNodeAssert; +import datawave.util.TableName; +import datawave.webservice.edgedictionary.RemoteEdgeDictionary; + +/** + * Tests the expansion of queries from the query model. + */ +public abstract class QueryModelExpansionTests { + + @RunWith(Arquillian.class) + public static class ShardRange extends QueryModelExpansionTests { + + @Override + protected VisibilityWiseGuysIngestWithModel.WhatKindaRange getRange() { + return VisibilityWiseGuysIngestWithModel.WhatKindaRange.SHARD; + } + } + + @RunWith(Arquillian.class) + public static class DocumentRange extends QueryModelExpansionTests { + + @Override + protected VisibilityWiseGuysIngestWithModel.WhatKindaRange getRange() { + return VisibilityWiseGuysIngestWithModel.WhatKindaRange.DOCUMENT; + } + } + + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private static final Logger log = Logger.getLogger(QueryModelExpansionTests.class); + private static final Authorizations auths = new Authorizations("ALL", "E", "I"); + + @Inject + @SpringBean(name = "EventQuery") + protected ShardQueryLogic logic; + protected Set authSet = Collections.singleton(auths); + protected KryoDocumentDeserializer deserializer; + + private final DateFormat format = new SimpleDateFormat("yyyyMMdd"); + private final Map queryParameters = new HashMap<>(); + private Date startDate; + private Date endDate; + private String query; + private String expectedPlan; + + @Deployment + public static JavaArchive createDeployment() { + return ShrinkWrap.create(JavaArchive.class) + .addPackages(true, "org.apache.deltaspike", "io.astefanutti.metrics.cdi", "datawave.query", "org.jboss.logging", + "datawave.webservice.query.result.event") + .deleteClass(DefaultEdgeEventQueryLogic.class).deleteClass(RemoteEdgeDictionary.class) + .deleteClass(datawave.query.metrics.QueryMetricQueryLogic.class) + .addAsManifestResource(new StringAsset( + "" + "datawave.query.tables.edge.MockAlternative" + ""), + "beans.xml"); + } + + @AfterClass + public static void teardown() { + TypeRegistry.reset(); + } + + @Before + public void setup() throws ParseException { + TimeZone.setDefault(TimeZone.getTimeZone("GMT")); + + this.logic.setFullTableScanEnabled(true); + this.logic.setMaxEvaluationPipelines(1); + this.logic.setQueryExecutionForPageTimeout(300000000000000L); + this.deserializer = new KryoDocumentDeserializer(); + this.startDate = format.parse("20091231"); + this.endDate = format.parse("20150101"); + } + + @After + public void tearDown() { + queryParameters.clear(); + } + + protected abstract VisibilityWiseGuysIngestWithModel.WhatKindaRange getRange(); + + private void runTestQuery() throws Exception { + log.debug("test plan against expected plan"); + + QueryImpl settings = new QueryImpl(); + settings.setBeginDate(this.startDate); + settings.setEndDate(this.endDate); + settings.setPagesize(Integer.MAX_VALUE); + settings.setQueryAuthorizations(auths.serialize()); + settings.setQuery(this.query); + settings.setParameters(this.queryParameters); + settings.setId(UUID.randomUUID()); + + log.debug("query: " + settings.getQuery()); + log.debug("logic: " + settings.getQueryLogicName()); + + AccumuloClient client = createClient(); + GenericQueryConfiguration config = logic.initialize(client, settings, authSet); + logic.setupQuery(config); + + String plan = logic.getPlan(client, settings, authSet, true, true); + + // order of terms in planned script is arbitrary, fall back to comparing the jexl trees + ASTJexlScript plannedScript = JexlASTHelper.parseJexlQuery(plan); + ASTJexlScript expectedScript = JexlASTHelper.parseJexlQuery(this.expectedPlan); + JexlNodeAssert.assertThat(plannedScript).isEqualTo(expectedScript); + } + + private AccumuloClient createClient() throws Exception { + AccumuloClient client = new QueryTestTableHelper(getClass().getName(), log, RebuildingScannerTestHelper.TEARDOWN.NEVER, + RebuildingScannerTestHelper.INTERRUPT.NEVER).client; + VisibilityWiseGuysIngestWithModel.writeItAll(client, getRange()); + PrintUtility.printTable(client, auths, TableName.SHARD); + PrintUtility.printTable(client, auths, TableName.SHARD_INDEX); + PrintUtility.printTable(client, auths, QueryTestTableHelper.MODEL_TABLE_NAME); + return client; + } + + private void givenQueryParameter(String key, String value) { + this.queryParameters.put(key, value); + } + + private void givenQuery(String query) { + this.query = query; + } + + private void givenExpectedPlan(String expectedPlan) { + this.expectedPlan = expectedPlan; + } + + /** + * Verify that expansion occurs for a standard forward mapping. + */ + @Test + public void testBasicExpansion() { + givenQuery("COLOR == 'blue'"); + givenQuery("COLOR == 'blue' || HUE == 'blue'"); + } + + /** + * Verifies that a patterned model mapping expands the query. + */ + @Test + public void testPatternedModelExpansion() throws Exception { + givenQuery("NAM == 'amy'"); + givenExpectedPlan("NAME == 'amy' || NOME == 'amy'"); + + runTestQuery(); + } + +} diff --git a/warehouse/query-core/src/test/java/datawave/query/util/VisibilityWiseGuysIngestWithModel.java b/warehouse/query-core/src/test/java/datawave/query/util/VisibilityWiseGuysIngestWithModel.java index 0e0170ac4e7..86ad9d08652 100644 --- a/warehouse/query-core/src/test/java/datawave/query/util/VisibilityWiseGuysIngestWithModel.java +++ b/warehouse/query-core/src/test/java/datawave/query/util/VisibilityWiseGuysIngestWithModel.java @@ -805,8 +805,8 @@ public static void writeItAll(AccumuloClient client, WhatKindaRange range) throw bw = client.createBatchWriter(QueryTestTableHelper.MODEL_TABLE_NAME, bwConfig); mutation = new Mutation("NAM"); - mutation.put("DATAWAVE", "NAME" + "\u0000" + "forward", columnVisibility, timeStamp, emptyValue); - mutation.put("DATAWAVE", "NOME" + "\u0000" + "forward", columnVisibility, timeStamp, emptyValue); + // Tests pattern mapping. + mutation.put("DATAWAVE", "NAME|NOME" + "\u0000" + "forward", columnVisibility, timeStamp, emptyValue); bw.addMutation(mutation); mutation = new Mutation("AG"); From b37820f32b4c7893a5e1954bab8cfa00d1cc378b Mon Sep 17 00:00:00 2001 From: Ivan Bella <347158+ivakegg@users.noreply.github.com> Date: Fri, 29 Nov 2024 13:43:23 +0000 Subject: [PATCH 4/8] Change metadata-utils dependency to the official 4.0.7 version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 71403ac1cde..57ea487957a 100644 --- a/pom.xml +++ b/pom.xml @@ -67,7 +67,7 @@ 3.0.0 4.0.1 1.0.0 - 4.0.7-patternModelMapping + 4.0.7 3.0.0 1.0.0 4.0.7 From 209856703eeb70943bd554ea87deba8b06599f55 Mon Sep 17 00:00:00 2001 From: palindrome <31748527+hlgp@users.noreply.github.com> Date: Mon, 2 Dec 2024 09:44:50 -0500 Subject: [PATCH 5/8] Feature/composite timestamp (#2187) * The composite timestamp allows us to encode two values in the timestamp to be used in accumulo keys. The event date will take the first (right hand most) 46 bits. The last 17 bits (except for the sign bit) will be used to encode how many days after the event date that we should base the ageoff on. If the timestamp is negative, then to calculate the values the complement is taken and then the two values are extracted. The ageoff is encoded as an age off delta which is the number of days after the event date. * Added composite timestamp tests * Add get and set Timestamp on the record container. * Refined the composite timestamp to check for bounds (#2268) * Incorporated timestamp concept into the RawRecordContainer (#2297) * Override getDate to be compatible with extending classes * Updated sharded ingest to appropriately use getTimestamp vs getDate * Updated index table to use composite timestamp * Updated edge table to use composite timestamp * Added an explicit INVALID_TIMESTAMP instead of using Long.MIN_VALUE to determine if the date is not set * Updated the global index uid aggregator to not aggregate across different timestamps * Removed the "timestampIgnored" flag on the global index uid aggregator as we never turn that off and we don't want to * Added test to ensure that keys with different timestamps do not get aggregated * Updated ageoff day delta calculation to be based on the difference between the start-of-day values for the event and age off dates. --------- Co-authored-by: hlgp Co-authored-by: Ivan Bella <347158+ivakegg@users.noreply.github.com> --- .../bin/services/datawave/bootstrap-ingest.sh | 13 +- .../bin/services/datawave/install-ingest.sh | 9 +- .../bin/services/hadoop/bootstrap.sh | 6 +- .../filter/ConfigurableAgeOffFilter.java | 4 +- .../iterators/filter/RegexFilterBase.java | 3 +- .../iterators/filter/TokenFilterBase.java | 3 +- .../filter/TokenizingFilterBase.java | 3 +- .../filter/ageoff/DataTypeAgeOffFilter.java | 5 +- .../filter/ageoff/FieldAgeOffFilter.java | 3 +- .../filter/ageoff/MaximumAgeOffFilter.java | 7 +- .../filter/ColumnVisibilityLabeledFilter.java | 3 +- .../filter/ConfigurableAgeOffFilterTest.java | 6 + .../filter/ageoff/FieldAgeOffFilterTest.java | 35 ++ .../datawave/util/CompositeTimestamp.java | 212 ++++++++++ .../datawave/util/CompositeTimestampTest.java | 371 ++++++++++++++++++ .../resources/config/shard-ingest-config.xml | 9 + .../ingest/config/RawRecordContainerImpl.java | 28 +- .../ingest/data/RawRecordContainer.java | 61 ++- .../data/config/ingest/BaseIngestHelper.java | 1 + .../reader/AbstractEventRecordReader.java | 3 +- .../handler/atom/AtomDataTypeHandler.java | 16 +- .../atom/AtomErrorDataTypeHandler.java | 4 +- .../edge/ProtobufEdgeDataTypeHandler.java | 4 +- .../handler/error/ErrorDataTypeHandler.java | 2 +- .../error/ErrorShardedDataTypeHandler.java | 2 +- .../handler/shard/ShardedDataTypeHandler.java | 71 ++-- .../ContentIndexingColumnBasedHandler.java | 5 +- ...ndedContentIndexingColumnBasedHandler.java | 13 +- .../aggregator/GlobalIndexUidAggregator.java | 59 +-- .../table/aggregator/PropogatingCombiner.java | 13 + .../config/AbstractTableConfigHelper.java | 4 + .../table/config/ShardTableConfigHelper.java | 1 + .../iterators/PropogatingIterator.java | 2 +- .../datawave/iterators/ValueCombiner.java | 8 +- .../policy/ExampleIngestPolicyEnforcer.java | 2 +- .../data/RawRecordContainerImplTest.java | 12 +- .../ingest/mapreduce/EventMapperTest.java | 4 +- .../ingest/mapreduce/SimpleRawRecord.java | 15 +- .../shard/ShardedDataTypeHandlerTest.java | 69 +++- ...ContentIndexingColumnBasedHandlerTest.java | 2 +- .../GlobalIndexUidAggregatorTest.java | 100 +++-- .../config/AbstractTableConfigHelperTest.java | 2 + .../ingest/csv/mr/input/CSVRecordReader.java | 2 +- .../json/mr/input/JsonRecordReader.java | 2 +- .../wikipedia/WikipediaDataTypeHandler.java | 2 +- ...eldIndexCountingIteratorPerVisibility.java | 5 +- .../filter/GlobalIndexDateRangeFilter.java | 3 +- .../query/iterator/filter/LoadDateFilter.java | 3 +- .../iterators/FieldIndexCountingIterator.java | 6 +- .../datawave/query/predicate/TimeFilter.java | 4 +- .../shard/FieldIndexCountQueryLogic.java | 5 +- .../DocumentTransformerSupport.java | 3 +- .../java/datawave/helpers/PrintUtility.java | 13 +- ...estLimitReturnedGroupsToHitTermGroups.java | 18 +- .../util/CommonalityTokenTestDataIngest.java | 28 ++ 55 files changed, 1097 insertions(+), 192 deletions(-) create mode 100644 warehouse/core/src/main/java/datawave/util/CompositeTimestamp.java create mode 100644 warehouse/core/src/test/java/datawave/util/CompositeTimestampTest.java diff --git a/contrib/datawave-quickstart/bin/services/datawave/bootstrap-ingest.sh b/contrib/datawave-quickstart/bin/services/datawave/bootstrap-ingest.sh index bd689ec71d1..8629602bc00 100644 --- a/contrib/datawave-quickstart/bin/services/datawave/bootstrap-ingest.sh +++ b/contrib/datawave-quickstart/bin/services/datawave/bootstrap-ingest.sh @@ -184,7 +184,7 @@ function datawaveIngestWikipedia() { [ ! -f "${wikipediaRawFile}" ] && error "File not found: ${wikipediaRawFile}" && return 1 local wikipediaHdfsFile="${DW_DATAWAVE_INGEST_HDFS_BASEDIR}/$( basename ${wikipediaRawFile} )" - local putFileCommand="hdfs dfs -copyFromLocal ${wikipediaRawFile} ${wikipediaHdfsFile}" + local putFileCommand="hdfs dfs -copyFromLocal -f ${wikipediaRawFile} ${wikipediaHdfsFile}" local inputFormat="datawave.ingest.wikipedia.WikipediaEventInputFormat" local jobCommand="${DW_DATAWAVE_INGEST_HOME}/bin/ingest/live-ingest.sh ${wikipediaHdfsFile} ${DW_DATAWAVE_INGEST_NUM_SHARDS} -inputFormat ${inputFormat} -data.name.override=wikipedia ${extraOpts}" @@ -211,7 +211,7 @@ function datawaveIngestCsv() { [ ! -f "${csvRawFile}" ] && error "File not found: ${csvRawFile}" && return 1 local csvHdfsFile="${DW_DATAWAVE_INGEST_HDFS_BASEDIR}/$( basename ${csvRawFile} )" - local putFileCommand="hdfs dfs -copyFromLocal ${csvRawFile} ${csvHdfsFile}" + local putFileCommand="hdfs dfs -copyFromLocal -f ${csvRawFile} ${csvHdfsFile}" local inputFormat="datawave.ingest.csv.mr.input.CSVFileInputFormat" local jobCommand="${DW_DATAWAVE_INGEST_HOME}/bin/ingest/live-ingest.sh ${csvHdfsFile} ${DW_DATAWAVE_INGEST_NUM_SHARDS} -inputFormat ${inputFormat} -data.name.override=mycsv ${extraOpts}" @@ -232,7 +232,7 @@ function datawaveIngestJson() { [ ! -f "${jsonRawFile}" ] && error "File not found: ${jsonRawFile}" && return 1 local jsonHdfsFile="${DW_DATAWAVE_INGEST_HDFS_BASEDIR}/$( basename ${jsonRawFile} )" - local putFileCommand="hdfs dfs -copyFromLocal ${jsonRawFile} ${jsonHdfsFile}" + local putFileCommand="hdfs dfs -copyFromLocal -f ${jsonRawFile} ${jsonHdfsFile}" local inputFormat="datawave.ingest.json.mr.input.JsonInputFormat" local jobCommand="${DW_DATAWAVE_INGEST_HOME}/bin/ingest/live-ingest.sh ${jsonHdfsFile} ${DW_DATAWAVE_INGEST_NUM_SHARDS} -inputFormat ${inputFormat} -data.name.override=myjson ${extraOpts}" @@ -347,3 +347,10 @@ function datawaveIngestTarballName() { local dwVersion="$(getDataWaveVersion)" echo "$( basename "${DW_DATAWAVE_INGEST_TARBALL/-\*-/-$dwVersion-}" )" } + +function datawaveIngestExamples() { + datawaveIngestWikipedia ${DW_DATAWAVE_INGEST_TEST_FILE_WIKI} + datawaveIngestJson ${DW_DATAWAVE_INGEST_TEST_FILE_JSON} + datawaveIngestCsv ${DW_DATAWAVE_INGEST_TEST_FILE_CSV} +} + diff --git a/contrib/datawave-quickstart/bin/services/datawave/install-ingest.sh b/contrib/datawave-quickstart/bin/services/datawave/install-ingest.sh index a2cd713d960..2ab3f06fa7b 100755 --- a/contrib/datawave-quickstart/bin/services/datawave/install-ingest.sh +++ b/contrib/datawave-quickstart/bin/services/datawave/install-ingest.sh @@ -159,13 +159,6 @@ function initializeDatawaveTables() { fi } -function ingestExampleData() { - # Ingest some canned, example data files - datawaveIngestWikipedia "${DW_DATAWAVE_INGEST_TEST_FILE_WIKI}" - datawaveIngestJson "${DW_DATAWAVE_INGEST_TEST_FILE_JSON}" - datawaveIngestCsv "${DW_DATAWAVE_INGEST_TEST_FILE_CSV}" -} - initializeDatawaveTables @@ -186,4 +179,4 @@ info "See \$DW_CLOUD_HOME/bin/services/datawave/bootstrap-ingest.sh to view/edit # Ingest raw data examples, if appropriate... -[ "${DW_REDEPLOY_IN_PROGRESS}" != true ] && [ "${DW_DATAWAVE_INGEST_TEST_SKIP}" == false ] && ingestExampleData +[ "${DW_REDEPLOY_IN_PROGRESS}" != true ] && [ "${DW_DATAWAVE_INGEST_TEST_SKIP}" == false ] && datawaveIngestExamples diff --git a/contrib/datawave-quickstart/bin/services/hadoop/bootstrap.sh b/contrib/datawave-quickstart/bin/services/hadoop/bootstrap.sh index 9f191b00402..8cec3a75726 100644 --- a/contrib/datawave-quickstart/bin/services/hadoop/bootstrap.sh +++ b/contrib/datawave-quickstart/bin/services/hadoop/bootstrap.sh @@ -45,8 +45,8 @@ dfs.replication 1" DW_HADOOP_MR_HEAPDUMP_DIR="${DW_CLOUD_DATA}/heapdumps" # mapred-site.xml (Format: {}) -DW_HADOOP_MAPRED_SITE_CONF="mapreduce.jobhistory.address http://${DW_BIND_HOST}:8020 -mapreduce.jobhistory.webapp.address http://${DW_BIND_HOST}:8021 +DW_HADOOP_MAPRED_SITE_CONF="mapreduce.jobhistory.address ${DW_BIND_HOST}:8020 +mapreduce.jobhistory.webapp.address ${DW_BIND_HOST}:8021 mapreduce.jobhistory.intermediate-done-dir ${DW_HADOOP_MR_INTER_DIR} mapreduce.jobhistory.done-dir ${DW_HADOOP_MR_DONE_DIR} mapreduce.map.memory.mb 2048 @@ -72,7 +72,7 @@ yarn.nodemanager.pmem-check-enabled false yarn.nodemanager.vmem-check-enabled false yarn.nodemanager.resource.memory-mb 6144 yarn.app.mapreduce.am.resource.mb 1024 -yarn.log.server.url http://localhost:8070/jobhistory/logs" +yarn.log.server.url http://localhost:8021/jobhistory/logs" # capacity-scheduler.xml (Format: {}) DW_HADOOP_CAPACITY_SCHEDULER_CONF="yarn.scheduler.capacity.maximum-applications 10000 diff --git a/warehouse/age-off/src/main/java/datawave/iterators/filter/ConfigurableAgeOffFilter.java b/warehouse/age-off/src/main/java/datawave/iterators/filter/ConfigurableAgeOffFilter.java index 61b8f23bb08..21f1f692bb0 100644 --- a/warehouse/age-off/src/main/java/datawave/iterators/filter/ConfigurableAgeOffFilter.java +++ b/warehouse/age-off/src/main/java/datawave/iterators/filter/ConfigurableAgeOffFilter.java @@ -35,6 +35,8 @@ import datawave.ingest.util.cache.watch.FileRuleCacheValue; import datawave.iterators.filter.ageoff.AgeOffPeriod; import datawave.iterators.filter.ageoff.AppliedRule; +import datawave.iterators.filter.ageoff.FilterRule; +import datawave.util.CompositeTimestamp; /** * This class provides a subclass of the {@code org.apache.accumulo.core.iterators.Filter} class and implements the {@code Option Describer} interface. It @@ -167,7 +169,7 @@ public boolean accept(Key k, Value v) { return true; // short circuit check - long timeStamp = k.getTimestamp(); + long timeStamp = CompositeTimestamp.getAgeOffDate(k.getTimestamp()); if (timeStamp > this.shortCircuitDateMillis) return true; diff --git a/warehouse/age-off/src/main/java/datawave/iterators/filter/RegexFilterBase.java b/warehouse/age-off/src/main/java/datawave/iterators/filter/RegexFilterBase.java index ba94dcbb953..d19a9e5f670 100644 --- a/warehouse/age-off/src/main/java/datawave/iterators/filter/RegexFilterBase.java +++ b/warehouse/age-off/src/main/java/datawave/iterators/filter/RegexFilterBase.java @@ -13,6 +13,7 @@ import datawave.iterators.filter.ageoff.AgeOffPeriod; import datawave.iterators.filter.ageoff.AppliedRule; import datawave.iterators.filter.ageoff.FilterOptions; +import datawave.util.CompositeTimestamp; /** * This class provides an abstract base class to be extended to filter based on matching a REGEX to the {@code String} object that represents some portion of a @@ -69,7 +70,7 @@ public boolean accept(AgeOffPeriod period, Key k, Value v) { String keyField = getKeyField(k, v); Matcher matcher = pattern.matcher(keyField); if (matcher.find()) { - long timeStamp = k.getTimestamp(); + long timeStamp = CompositeTimestamp.getAgeOffDate(k.getTimestamp()); dtFlag = timeStamp > period.getCutOffMilliseconds(); if (log.isTraceEnabled()) { log.trace("timeStamp = " + timeStamp); diff --git a/warehouse/age-off/src/main/java/datawave/iterators/filter/TokenFilterBase.java b/warehouse/age-off/src/main/java/datawave/iterators/filter/TokenFilterBase.java index dd963b4b557..b6a59a034da 100644 --- a/warehouse/age-off/src/main/java/datawave/iterators/filter/TokenFilterBase.java +++ b/warehouse/age-off/src/main/java/datawave/iterators/filter/TokenFilterBase.java @@ -10,6 +10,7 @@ import datawave.iterators.filter.ageoff.AgeOffPeriod; import datawave.iterators.filter.ageoff.AppliedRule; import datawave.iterators.filter.ageoff.FilterOptions; +import datawave.util.CompositeTimestamp; import datawave.util.StringUtils; /** @@ -69,7 +70,7 @@ public boolean accept(AgeOffPeriod period, Key k, Value v) { dtFlag = true; } else { if (hasToken(k, v, patternBytes)) { - long timeStamp = k.getTimestamp(); + long timeStamp = CompositeTimestamp.getAgeOffDate(k.getTimestamp()); dtFlag = timeStamp > period.getCutOffMilliseconds(); if (log.isTraceEnabled()) { log.trace("timeStamp = " + timeStamp); diff --git a/warehouse/age-off/src/main/java/datawave/iterators/filter/TokenizingFilterBase.java b/warehouse/age-off/src/main/java/datawave/iterators/filter/TokenizingFilterBase.java index b2078cd0502..edb4e219f1d 100644 --- a/warehouse/age-off/src/main/java/datawave/iterators/filter/TokenizingFilterBase.java +++ b/warehouse/age-off/src/main/java/datawave/iterators/filter/TokenizingFilterBase.java @@ -10,6 +10,7 @@ import datawave.iterators.filter.ageoff.AgeOffPeriod; import datawave.iterators.filter.ageoff.AppliedRule; import datawave.iterators.filter.ageoff.FilterOptions; +import datawave.util.CompositeTimestamp; /** * TokenizingAgeoffFilter cuts a field into tokens (splitting at a specified set of delimiters), and makes ageoff decisions based on whether or not any of the @@ -119,7 +120,7 @@ public boolean accept(AgeOffPeriod period, Key k, Value V) { cutoffTimestamp -= calculatedTTL; } ruleApplied = true; - return k.getTimestamp() > cutoffTimestamp; + return CompositeTimestamp.getAgeOffDate(k.getTimestamp()) > cutoffTimestamp; } @Override diff --git a/warehouse/age-off/src/main/java/datawave/iterators/filter/ageoff/DataTypeAgeOffFilter.java b/warehouse/age-off/src/main/java/datawave/iterators/filter/ageoff/DataTypeAgeOffFilter.java index 74963cbc2b3..a9d7ee7f54b 100644 --- a/warehouse/age-off/src/main/java/datawave/iterators/filter/ageoff/DataTypeAgeOffFilter.java +++ b/warehouse/age-off/src/main/java/datawave/iterators/filter/ageoff/DataTypeAgeOffFilter.java @@ -16,6 +16,7 @@ import com.google.common.collect.Sets; import datawave.iterators.filter.AgeOffConfigParams; +import datawave.util.CompositeTimestamp; /** * Data type age off filter. Traverses through indexed tables @@ -209,11 +210,11 @@ public boolean accept(AgeOffPeriod period, Key k, Value v) { if (dataTypeCutoff == null) { if (defaultCutoffTime >= 0) { ruleApplied = true; - accept = k.getTimestamp() > defaultCutoffTime; + accept = CompositeTimestamp.getAgeOffDate(k.getTimestamp()) > defaultCutoffTime; } } else { ruleApplied = true; - accept = k.getTimestamp() > dataTypeCutoff; + accept = CompositeTimestamp.getAgeOffDate(k.getTimestamp()) > dataTypeCutoff; } // after age-off is applied check, if we are accepting this KeyValue and this is a Scan on a dataType which only accepts on timestamp // only continue to accept the KeyValue if the timestamp for the dataType matches what is configured diff --git a/warehouse/age-off/src/main/java/datawave/iterators/filter/ageoff/FieldAgeOffFilter.java b/warehouse/age-off/src/main/java/datawave/iterators/filter/ageoff/FieldAgeOffFilter.java index 9d61166f340..d0875f2176b 100644 --- a/warehouse/age-off/src/main/java/datawave/iterators/filter/ageoff/FieldAgeOffFilter.java +++ b/warehouse/age-off/src/main/java/datawave/iterators/filter/ageoff/FieldAgeOffFilter.java @@ -17,6 +17,7 @@ import datawave.iterators.filter.AgeOffConfigParams; import datawave.iterators.filter.ColumnVisibilityOrFilter; +import datawave.util.CompositeTimestamp; /** * Field age off filter. Traverses through indexed tables and non-indexed tables. Example follows. Note that any field TTL will follow the same units specified @@ -208,7 +209,7 @@ public boolean accept(AgeOffPeriod period, Key k, Value v) { Long dataTypeCutoff = (fieldTimes.containsKey(field)) ? fieldTimes.get(field) : null; if (dataTypeCutoff != null) { ruleApplied = true; - return k.getTimestamp() > dataTypeCutoff; + return CompositeTimestamp.getAgeOffDate(k.getTimestamp()) > dataTypeCutoff; } return true; diff --git a/warehouse/age-off/src/main/java/datawave/iterators/filter/ageoff/MaximumAgeOffFilter.java b/warehouse/age-off/src/main/java/datawave/iterators/filter/ageoff/MaximumAgeOffFilter.java index 257e0333ed9..643bb59accc 100644 --- a/warehouse/age-off/src/main/java/datawave/iterators/filter/ageoff/MaximumAgeOffFilter.java +++ b/warehouse/age-off/src/main/java/datawave/iterators/filter/ageoff/MaximumAgeOffFilter.java @@ -6,6 +6,7 @@ import org.apache.log4j.Logger; import datawave.iterators.filter.AgeOffConfigParams; +import datawave.util.CompositeTimestamp; /** * Data type age off filter. Traverses through indexed tables @@ -59,7 +60,11 @@ public boolean accept(AgeOffPeriod period, Key k, Value v) { // this rule determines whether to accept / deny (ageoff) a K/V // based solely on whether a timestamp is before (older than) the cutoff for aging off - return k.getTimestamp() > period.getCutOffMilliseconds(); + if (CompositeTimestamp.getAgeOffDate(k.getTimestamp()) > period.getCutOffMilliseconds()) { + return true; + } else { + return false; + } } /** diff --git a/warehouse/age-off/src/test/java/datawave/iterators/filter/ColumnVisibilityLabeledFilter.java b/warehouse/age-off/src/test/java/datawave/iterators/filter/ColumnVisibilityLabeledFilter.java index 9101cb2fe58..214381672ce 100644 --- a/warehouse/age-off/src/test/java/datawave/iterators/filter/ColumnVisibilityLabeledFilter.java +++ b/warehouse/age-off/src/test/java/datawave/iterators/filter/ColumnVisibilityLabeledFilter.java @@ -11,6 +11,7 @@ import datawave.iterators.filter.ageoff.AgeOffPeriod; import datawave.iterators.filter.ageoff.AppliedRule; import datawave.iterators.filter.ageoff.FilterOptions; +import datawave.util.CompositeTimestamp; import datawave.util.StringUtils; /** @@ -108,7 +109,7 @@ public boolean accept(AgeOffPeriod ageOffPeriod, Key k, Value V) { cutOff -= timeToLive; } this.filterRuleApplied = true; - return k.getTimestamp() > cutOff; + return CompositeTimestamp.getAgeOffDate(k.getTimestamp()) > cutOff; } } return true; diff --git a/warehouse/age-off/src/test/java/datawave/iterators/filter/ConfigurableAgeOffFilterTest.java b/warehouse/age-off/src/test/java/datawave/iterators/filter/ConfigurableAgeOffFilterTest.java index b35f976f4bf..9290f93f0ac 100644 --- a/warehouse/age-off/src/test/java/datawave/iterators/filter/ConfigurableAgeOffFilterTest.java +++ b/warehouse/age-off/src/test/java/datawave/iterators/filter/ConfigurableAgeOffFilterTest.java @@ -28,6 +28,7 @@ import datawave.iterators.filter.ageoff.ConfigurableIteratorEnvironment; import datawave.iterators.filter.ageoff.FilterOptions; import datawave.query.iterator.SortedListKeyValueIterator; +import datawave.util.CompositeTimestamp; public class ConfigurableAgeOffFilterTest { @@ -138,6 +139,9 @@ public void testAcceptKeyValue_TtlSet() throws Exception { // copy cofigs to actual filter we are testing filter.initialize(wrapper); + long tomorrow = System.currentTimeMillis() + CompositeTimestamp.MILLIS_PER_DAY; + long compositeTS = CompositeTimestamp.getCompositeTimeStamp(daysAgo(365), tomorrow); + // brand new key should be good assertThat(filter.accept(new Key(), VALUE), is(true)); // first five will hit the ttl short circuit @@ -155,6 +159,8 @@ public void testAcceptKeyValue_TtlSet() throws Exception { assertThat(filter.accept(getKey("foo", daysAgo(8)), VALUE), is(true)); // this is really old and matches so should not be accepted assertThat(filter.accept(getKey("foo", daysAgo(365)), VALUE), is(false)); + // this is really old and matches, but has a future age off date, so should be accepted + assertThat(filter.accept(getKey("foo", compositeTS), VALUE), is(true)); } diff --git a/warehouse/age-off/src/test/java/datawave/iterators/filter/ageoff/FieldAgeOffFilterTest.java b/warehouse/age-off/src/test/java/datawave/iterators/filter/ageoff/FieldAgeOffFilterTest.java index 469887d5575..36571bdaf04 100644 --- a/warehouse/age-off/src/test/java/datawave/iterators/filter/ageoff/FieldAgeOffFilterTest.java +++ b/warehouse/age-off/src/test/java/datawave/iterators/filter/ageoff/FieldAgeOffFilterTest.java @@ -14,6 +14,7 @@ import datawave.iterators.filter.AgeOffConfigParams; import datawave.iterators.filter.AgeOffTtlUnits; +import datawave.util.CompositeTimestamp; public class FieldAgeOffFilterTest { private static final String VISIBILITY_PATTERN = "MY_VIS"; @@ -84,6 +85,7 @@ public void testIndexTrueUsesDefaultWhenFieldLacksTtl() { Key key = new Key("1234", "field_z\\x00my-uuid", "field_z\u0000value", VISIBILITY_PATTERN, tenSecondsAgo); Assert.assertFalse(ageOffFilter.accept(filterOptions.getAgeOffPeriod(System.currentTimeMillis()), key, new Value())); Assert.assertTrue(ageOffFilter.isFilterRuleApplied()); + key = new Key("1234", "field_y", "field_y\u0000value", VISIBILITY_PATTERN, tenSecondsAgo); Assert.assertFalse(ageOffFilter.accept(filterOptions.getAgeOffPeriod(System.currentTimeMillis()), key, new Value())); Assert.assertTrue(ageOffFilter.isFilterRuleApplied()); @@ -236,6 +238,39 @@ public void testIgnoresDocument() { Assert.assertFalse(ageOffFilter.isFilterRuleApplied()); } + @Test + public void testCompositeTimestamp() { + EditableAccumuloConfiguration conf = new EditableAccumuloConfiguration(DefaultConfiguration.getInstance()); + conf.put("table.custom.isindextable", "true"); + iterEnv.setConf(conf); + + long tenSecondsAgo = System.currentTimeMillis() - (10L * ONE_SEC); + long tomorrow = System.currentTimeMillis() + CompositeTimestamp.MILLIS_PER_DAY; + + long compositeTS = CompositeTimestamp.getCompositeTimeStamp(tenSecondsAgo, tomorrow); + + FieldAgeOffFilter ageOffFilter = new FieldAgeOffFilter(); + FilterOptions filterOptions = createFilterOptionsWithPattern(); + // set the default to 5 seconds + filterOptions.setTTL(5L); + filterOptions.setTTLUnits(AgeOffTtlUnits.SECONDS); + // set up ttls for field_y and field_z only, deliberately exclude the ttl for field_y + filterOptions.setOption("fields", "field_y"); + filterOptions.setOption("field_y.ttl", "2"); // 2 seconds + ageOffFilter.init(filterOptions, iterEnv); + + // age off date allows this to accept + Key key = new Key("1234", "field_y", "field_y\u0000value", VISIBILITY_PATTERN, compositeTS); + Assert.assertTrue(ageOffFilter.accept(filterOptions.getAgeOffPeriod(System.currentTimeMillis()), key, new Value())); + Assert.assertTrue(ageOffFilter.isFilterRuleApplied()); + + // vanilla date does not + key = new Key("1234", "field_y", "field_y\u0000value", VISIBILITY_PATTERN, tenSecondsAgo); + Assert.assertFalse(ageOffFilter.accept(filterOptions.getAgeOffPeriod(System.currentTimeMillis()), key, new Value())); + Assert.assertTrue(ageOffFilter.isFilterRuleApplied()); + + } + @Test public void testKeepsMatchBeforeTtl() { long oneSecondAgo = System.currentTimeMillis() - (1 * ONE_SEC); diff --git a/warehouse/core/src/main/java/datawave/util/CompositeTimestamp.java b/warehouse/core/src/main/java/datawave/util/CompositeTimestamp.java new file mode 100644 index 00000000000..023ba517ed4 --- /dev/null +++ b/warehouse/core/src/main/java/datawave/util/CompositeTimestamp.java @@ -0,0 +1,212 @@ +package datawave.util; + +import java.util.Calendar; +import java.util.Comparator; +import java.util.TimeZone; + +/** + * The composite timestamp allows us to encode two values in the timestamp to be used in accumulo keys. The event date will take the first (right hand most) 46 + * bits. The last 17 bits (except for the sign bit) will be used to encode how many days after the event date that we should base the ageoff on. If the + * timestamp is negative, then to calculate the values the complement is taken and then the two values are extracted. The ageoff is encoded as an age off delta + * which is the number of days after the event date. + */ +public class CompositeTimestamp { + + // The number of bits for the event date + private static final int allocationForEventDate = 46; + // A mask for the event date + private static final long mask = -1L >>> (8 * 8 - allocationForEventDate); + // The max age off delta in days + private static final long maxDiff = (-1L << (allocationForEventDate + 1)) >>> (allocationForEventDate + 1); + // A useful constant + public static final long MILLIS_PER_DAY = 1000 * 60 * 60 * 24; + + // A constant for an invalid timestamp + public static final long INVALID_TIMESTAMP = Long.MIN_VALUE; + // The minimum event date + public static final long MIN_EVENT_DATE = (0 - mask); // also equivalent to Long.MIN_VALUE + 1 + // the maximum event date + public static final long MAX_EVENT_DATE = mask; + + /** + * Determine if the timestamp is composite (i.e. non-zero age off delta) + * + * @param ts + * @return True if composite + */ + public static boolean isCompositeTimestamp(long ts) { + validateTimestamp(ts); + return (Math.abs(ts) >>> allocationForEventDate > 0); + } + + /** + * Get the event date portion of the timestamp + * + * @param ts + * @return the event date + */ + public static long getEventDate(long ts) { + validateTimestamp(ts); + long eventTs = (Math.abs(ts) & mask); + if (ts < 0) { + eventTs = 0 - eventTs; + } + return eventTs; + } + + /** + * Determine the age off date portion of the timestamp. This is the event date plus the ageoff delta converted to milliseconds. + * + * @param ts + * @return The age off date + */ + public static long getAgeOffDate(long ts) { + validateTimestamp(ts); + long baseTs = Math.abs(ts); + long eventTs = (baseTs & mask); + long ageOffDiff = ((baseTs >>> allocationForEventDate) * MILLIS_PER_DAY); + if (ts < 0) { + eventTs = 0 - eventTs; + } + return eventTs + ageOffDiff; + } + + /** + * Determine the age off delta porton of the timestamp. This is the number of days difference from the event date. + * + * @param ts + * @return the age off delta + */ + public static int getAgeOffDeltaDays(long ts) { + validateTimestamp(ts); + long baseTs = Math.abs(ts); + long ageOffDiff = ((baseTs >>> allocationForEventDate) * MILLIS_PER_DAY); + return (int) ageOffDiff; + } + + /** + * Calculate an age off delta based on a timezone. This will calculate the begininning of the day in the given timezone for both the event date and the age + * off date, and then will return that difference in days. + * + * @param eventDate + * @param ageOffDate + * @param tz + * @return the age off delta + */ + public static int computeAgeOffDeltaDays(long eventDate, long ageOffDate, TimeZone tz) { + validateEventDate(eventDate); + Calendar c = Calendar.getInstance(tz); + c.setTimeInMillis(eventDate); + c.set(Calendar.HOUR_OF_DAY, 0); + c.set(Calendar.MINUTE, 0); + c.set(Calendar.SECOND, 0); + c.set(Calendar.MILLISECOND, 0); + long eventDateStart = c.getTimeInMillis(); + + c.setTimeInMillis(ageOffDate); + c.set(Calendar.HOUR_OF_DAY, 0); + c.set(Calendar.MINUTE, 0); + c.set(Calendar.SECOND, 0); + c.set(Calendar.MILLISECOND, 0); + long ageOffDateStart = c.getTimeInMillis(); + + long delta = (ageOffDateStart - eventDateStart) / MILLIS_PER_DAY; + validateAgeOffDelta(delta); + return (int) delta; + } + + /** + * Get the composite timestamp using the supplied event date and age off delta in days. + * + * @param eventDate + * @param ageOffDeltaDays + * @return The composite timestamp + */ + public static long getCompositeTimestamp(long eventDate, int ageOffDeltaDays) { + validateEventDate(eventDate); + validateAgeOffDelta(ageOffDeltaDays); + long eventBase = Math.abs(eventDate); + long compositeTS = ((long) ageOffDeltaDays << allocationForEventDate) | eventBase; + if (eventDate < 0) { + compositeTS = 0 - compositeTS; + } + return compositeTS; + } + + /** + * Get the composite timestamp using the supplied eventDate and an age off delta in days based on the supplied age off date and time zone. + * + * @param eventDate + * @param ageOffDate + * @param tz + * @return The composite timestamp + */ + public static long getCompositeTimestamp(long eventDate, long ageOffDate, TimeZone tz) { + return getCompositeTimestamp(eventDate, computeAgeOffDeltaDays(eventDate, ageOffDate, tz)); + } + + /** + * Get the composite timestamp using the supplied eventDate and an age off delta in days based on the supplied age off date using the GMT timezone + * + * @param eventDate + * @param ageOffDate + * @return The composite timestamp + */ + public static long getCompositeTimeStamp(long eventDate, long ageOffDate) { + return getCompositeTimestamp(eventDate, ageOffDate, TimeZone.getTimeZone("GMT")); + } + + /** + * Return a comparator for composite timestamps. Orders firstly on the event date, and if equal then on the ageoff date. Note for values with the same event + * date, the timestamps will order naturally for positive event dates, but need to be reversed for negative event dates. This should only be an issue for + * the global index and hence this comparator can be used in that case. + * + * @return a comparison of the timestamps + */ + public static Comparator comparator() { + return (o1, o2) -> { + long eventDate1 = CompositeTimestamp.getEventDate(o1); + long eventDate2 = CompositeTimestamp.getEventDate(o2); + if (eventDate1 < eventDate2) { + return -1; + } else if (eventDate1 == eventDate2) { + long ageOffDate1 = CompositeTimestamp.getAgeOffDate(o1); + long ageOffDate2 = CompositeTimestamp.getAgeOffDate(o2); + if (ageOffDate1 < ageOffDate2) { + return -1; + } else if (ageOffDate1 == ageOffDate2) { + return 0; + } else { + return 1; + } + } else { + return 1; + } + }; + } + + private static void validateTimestamp(long ts) { + if (ts == INVALID_TIMESTAMP) { + throw new IllegalArgumentException("Invalid timestamp"); + } + } + + private static void validateEventDate(long eventDate) { + if (eventDate < MIN_EVENT_DATE) { + throw new IllegalArgumentException("Event date cannot be less than " + MIN_EVENT_DATE); + } + if (eventDate > MAX_EVENT_DATE) { + throw new IllegalArgumentException("Event Date cannot be greater than " + MAX_EVENT_DATE); + } + } + + private static void validateAgeOffDelta(long ageOffDeltaDays) { + if (ageOffDeltaDays < 0) { + throw new IllegalArgumentException("Age off date must be greater to or equal to the event date"); + } + if (ageOffDeltaDays > maxDiff) { + throw new IllegalArgumentException("Difference between event date and age off date cannot be more than " + maxDiff + " days"); + } + } + +} diff --git a/warehouse/core/src/test/java/datawave/util/CompositeTimestampTest.java b/warehouse/core/src/test/java/datawave/util/CompositeTimestampTest.java new file mode 100644 index 00000000000..33cfad12281 --- /dev/null +++ b/warehouse/core/src/test/java/datawave/util/CompositeTimestampTest.java @@ -0,0 +1,371 @@ +package datawave.util; + +import java.time.Instant; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Calendar; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import org.junit.Assert; +import org.junit.Test; + +public class CompositeTimestampTest { + + @Test + public void testConversion() { + long eventDate = Instant.from(DateTimeFormatter.ISO_INSTANT.parse("2022-10-26T01:00:00Z")).toEpochMilli(); + long ageOff = eventDate + CompositeTimestamp.MILLIS_PER_DAY; + long expectedTS = 72035490177664L; + + long compositeTS = CompositeTimestamp.getCompositeTimeStamp(eventDate, ageOff); + + Assert.assertEquals(expectedTS, compositeTS); + Assert.assertTrue(CompositeTimestamp.isCompositeTimestamp(compositeTS)); + Assert.assertFalse(CompositeTimestamp.isCompositeTimestamp(eventDate)); + Assert.assertFalse(CompositeTimestamp.isCompositeTimestamp(ageOff)); + Assert.assertEquals(CompositeTimestamp.getEventDate(compositeTS), eventDate); + Assert.assertEquals(CompositeTimestamp.getAgeOffDate(compositeTS), ageOff); + } + + @Test + public void testOldDateWithOldAgeoff() { + long eventDate = Instant.from(DateTimeFormatter.ISO_INSTANT.parse("1900-01-01T00:00:00Z")).toEpochMilli(); + long ageOff = Instant.from(DateTimeFormatter.ISO_INSTANT.parse("1950-01-01T00:00:00Z")).toEpochMilli(); + long expectedTS = -1285076215161299968L; + + long compositeTS = CompositeTimestamp.getCompositeTimeStamp(eventDate, ageOff); + + Assert.assertEquals(expectedTS, compositeTS); + Assert.assertTrue(CompositeTimestamp.isCompositeTimestamp(compositeTS)); + Assert.assertFalse(CompositeTimestamp.isCompositeTimestamp(eventDate)); + Assert.assertFalse(CompositeTimestamp.isCompositeTimestamp(ageOff)); + Assert.assertEquals(CompositeTimestamp.getEventDate(compositeTS), eventDate); + Assert.assertEquals(CompositeTimestamp.getAgeOffDate(compositeTS), ageOff); + } + + @Test + public void testOldDateWithModernAgeoff() { + long eventDate = Instant.from(DateTimeFormatter.ISO_INSTANT.parse("1960-01-01T00:00:00Z")).toEpochMilli(); + long ageOff = Instant.from(DateTimeFormatter.ISO_INSTANT.parse("2025-01-01T00:00:00Z")).toEpochMilli(); + long expectedTS = -1670695039885298688L; + + long compositeTS = CompositeTimestamp.getCompositeTimeStamp(eventDate, ageOff); + + Assert.assertEquals(expectedTS, compositeTS); + Assert.assertTrue(CompositeTimestamp.isCompositeTimestamp(compositeTS)); + Assert.assertFalse(CompositeTimestamp.isCompositeTimestamp(eventDate)); + Assert.assertFalse(CompositeTimestamp.isCompositeTimestamp(ageOff)); + Assert.assertEquals(CompositeTimestamp.getEventDate(compositeTS), eventDate); + Assert.assertEquals(CompositeTimestamp.getAgeOffDate(compositeTS), ageOff); + } + + @Test + public void testOldDateWithMaxAgeoff() { + long eventDate = Instant.from(DateTimeFormatter.ISO_INSTANT.parse("1960-01-01T00:00:00Z")).toEpochMilli(); + long ageOff = eventDate + (131071L * CompositeTimestamp.MILLIS_PER_DAY); + long expectedTS = -9223301983729798144L; + + long compositeTS = CompositeTimestamp.getCompositeTimeStamp(eventDate, ageOff); + + Assert.assertEquals(expectedTS, compositeTS); + Assert.assertTrue(CompositeTimestamp.isCompositeTimestamp(compositeTS)); + Assert.assertFalse(CompositeTimestamp.isCompositeTimestamp(eventDate)); + Assert.assertFalse(CompositeTimestamp.isCompositeTimestamp(ageOff)); + Assert.assertEquals(CompositeTimestamp.getEventDate(compositeTS), eventDate); + Assert.assertEquals(CompositeTimestamp.getAgeOffDate(compositeTS), ageOff); + } + + @Test + public void testVeryOldDateWithMaxAgeoff() { + long eventDate = Instant.from(DateTimeFormatter.ISO_INSTANT.parse("1654-01-01T00:00:00Z")).toEpochMilli(); + long ageOff = eventDate + (131071L * CompositeTimestamp.MILLIS_PER_DAY); + long expectedTS = -9223311640052998144L; + + long compositeTs1 = Long.MIN_VALUE + 1; + long eventDate1 = CompositeTimestamp.getEventDate(compositeTs1); + + long compositeTS = CompositeTimestamp.getCompositeTimeStamp(eventDate, ageOff); + + Assert.assertEquals(expectedTS, compositeTS); + Assert.assertTrue(CompositeTimestamp.isCompositeTimestamp(compositeTS)); + Assert.assertFalse(CompositeTimestamp.isCompositeTimestamp(eventDate)); + Assert.assertFalse(CompositeTimestamp.isCompositeTimestamp(ageOff)); + Assert.assertEquals(CompositeTimestamp.getEventDate(compositeTS), eventDate); + Assert.assertEquals(CompositeTimestamp.getAgeOffDate(compositeTS), ageOff); + } + + @Test + public void testEventUpperDateBound() { + long eventDate = (-1L >>> 18); + long compositeTS = CompositeTimestamp.getCompositeTimeStamp(eventDate, eventDate); + Assert.assertEquals(eventDate, compositeTS); + Assert.assertFalse(CompositeTimestamp.isCompositeTimestamp(compositeTS)); + Assert.assertEquals(CompositeTimestamp.getEventDate(compositeTS), eventDate); + Assert.assertEquals(CompositeTimestamp.getAgeOffDate(compositeTS), eventDate); + + try { + CompositeTimestamp.getCompositeTimeStamp(eventDate + 1, 0); + Assert.fail("Expected event date greater than 17 bits to fail"); + } catch (IllegalArgumentException e) { + // expected + } + } + + @Test + public void testEventLowerDateBound() { + long eventDate = (0 - (-1L >>> 18)); + long compositeTS = CompositeTimestamp.getCompositeTimeStamp(eventDate, eventDate); + Assert.assertEquals(eventDate, compositeTS); + Assert.assertFalse(CompositeTimestamp.isCompositeTimestamp(compositeTS)); + Assert.assertEquals(CompositeTimestamp.getEventDate(compositeTS), eventDate); + Assert.assertEquals(CompositeTimestamp.getAgeOffDate(compositeTS), eventDate); + + try { + CompositeTimestamp.getCompositeTimeStamp(eventDate - 1, 0); + Assert.fail("Expected event date greater than 17 bits to fail"); + } catch (IllegalArgumentException e) { + // expected + } + } + + @Test + public void testAgeOffDateBounds() { + long eventDate = (0 - (-1L >>> 18)); + long ageOffEventDays = (-1L >>> 47); + long ageOffEventDelta = ageOffEventDays * 1000 * 60 * 60 * 24; + long compositeTS = CompositeTimestamp.getCompositeTimeStamp(eventDate, eventDate + ageOffEventDelta); + Assert.assertTrue(CompositeTimestamp.isCompositeTimestamp(compositeTS)); + Assert.assertEquals(CompositeTimestamp.getEventDate(compositeTS), eventDate); + Assert.assertEquals(CompositeTimestamp.getAgeOffDate(compositeTS), eventDate + ageOffEventDelta); + + try { + CompositeTimestamp.getCompositeTimeStamp(eventDate, eventDate - CompositeTimestamp.MILLIS_PER_DAY); + Assert.fail("Expected ageoff date less than event date to fail"); + } catch (IllegalArgumentException e) { + // expected + } + + try { + CompositeTimestamp.getCompositeTimeStamp(CompositeTimestamp.MILLIS_PER_DAY, 0); + Assert.fail("Expected ageoff date less than event date to fail"); + } catch (IllegalArgumentException e) { + // expected + } + + try { + CompositeTimestamp.getCompositeTimeStamp(eventDate, eventDate + ageOffEventDelta + CompositeTimestamp.MILLIS_PER_DAY); + Assert.fail("Expected age off date greater than " + ageOffEventDays + " days from event date to fail"); + } catch (IllegalArgumentException e) { + // expected + } + } + + @Test + public void testMaxEntropy() { + long eventDate = -1L; + long ageOff = -1L; + + long compositeTS = CompositeTimestamp.getCompositeTimeStamp(eventDate, ageOff); + + Assert.assertEquals(-1L, compositeTS); + // since the ageoff is equal to the event date, this is not considered a composite timestamp + Assert.assertFalse(CompositeTimestamp.isCompositeTimestamp(compositeTS)); + Assert.assertEquals(CompositeTimestamp.getEventDate(compositeTS), eventDate); + Assert.assertEquals(CompositeTimestamp.getAgeOffDate(compositeTS), ageOff); + } + + @Test + public void testDoomsday() { + long compositeTs = Long.MAX_VALUE - 5L; + + long eventDate = CompositeTimestamp.getEventDate(compositeTs); // 4199-11-24 + long now = System.currentTimeMillis(); + + Date endGame = new Date(TimeUnit.MILLISECONDS.toMillis(eventDate)); + Assert.assertTrue("Doomsday is " + endGame + ". You have less than one year before timestamps roll over. Get cracking.", + (now + (365L * CompositeTimestamp.MILLIS_PER_DAY)) < eventDate); + + } + + @Test + public void testPostDoomsday() { + long eventDate = Instant.from(DateTimeFormatter.ISO_INSTANT.parse("4199-11-25T00:00:00Z")).toEpochMilli(); + long ageOff = Instant.from(DateTimeFormatter.ISO_INSTANT.parse("4299-06-01T00:00:00Z")).toEpochMilli(); + + try { + CompositeTimestamp.getCompositeTimeStamp(eventDate, ageOff); + Assert.fail("Expected event date to fail"); + } catch (IllegalArgumentException e) { + // expected + } + + } + + @Test + public void testMin() { + long ts = CompositeTimestamp.getCompositeTimeStamp(CompositeTimestamp.MIN_EVENT_DATE, CompositeTimestamp.MIN_EVENT_DATE); + long event = CompositeTimestamp.getEventDate(ts); + long age = CompositeTimestamp.getEventDate(ts); + Assert.assertEquals(event, age); + } + + @Test + public void testInvalid() { + try { + CompositeTimestamp.getEventDate(CompositeTimestamp.INVALID_TIMESTAMP); + Assert.fail("Invalid timestamp not detected"); + } catch (IllegalArgumentException e) { + + } + try { + CompositeTimestamp.getAgeOffDate(CompositeTimestamp.INVALID_TIMESTAMP); + Assert.fail("Invalid timestamp not detected"); + } catch (IllegalArgumentException e) { + + } + try { + CompositeTimestamp.isCompositeTimestamp(CompositeTimestamp.INVALID_TIMESTAMP); + Assert.fail("Invalid timestamp not detected"); + } catch (IllegalArgumentException e) { + + } + } + + protected boolean isOrdered(Long... times) { + List list1 = new ArrayList<>(Arrays.asList(times)); + List list2 = new ArrayList<>(Arrays.asList(times)); + Collections.sort(list2); + return list1.equals(list2); + } + + protected boolean isOrdered(Comparator comparator, Long... times) { + List list1 = new ArrayList<>(Arrays.asList(times)); + List list2 = new ArrayList<>(Arrays.asList(times)); + Collections.sort(list2, comparator); + return list1.equals(list2); + } + + @Test + public void testOrderingAgeOffDates() { + Calendar cal = Calendar.getInstance(); + long twoMonthsLater = cal.getTimeInMillis(); + cal.add(Calendar.MONTH, -1); + long aMonthLater = cal.getTimeInMillis(); + cal.add(Calendar.MONTH, -1); + long aDate = cal.getTimeInMillis(); + + // test the same positive eventdate, but different ageoff dates + long t1 = CompositeTimestamp.getCompositeTimeStamp(aDate, aDate); + long t2 = CompositeTimestamp.getCompositeTimeStamp(aDate, aMonthLater); + long t3 = CompositeTimestamp.getCompositeTimeStamp(aDate, twoMonthsLater); + + // in this case the natural ordering will be correct + Assert.assertTrue(isOrdered(t1, t2, t3)); + // and the comparator will maintain that ordering + Assert.assertTrue(isOrdered(CompositeTimestamp.comparator(), t1, t2, t3)); + + cal = Calendar.getInstance(); + cal.setTimeInMillis(0); + cal.add(Calendar.MONTH, -1); + twoMonthsLater = cal.getTimeInMillis(); + cal.add(Calendar.MONTH, -1); + aMonthLater = cal.getTimeInMillis(); + cal.add(Calendar.MONTH, -1); + aDate = cal.getTimeInMillis(); + + // test the same negative eventdate, but different ageoff dates + t1 = CompositeTimestamp.getCompositeTimeStamp(aDate, aDate); + t2 = CompositeTimestamp.getCompositeTimeStamp(aDate, aMonthLater); + t3 = CompositeTimestamp.getCompositeTimeStamp(aDate, twoMonthsLater); + + // in this case the natural ordering will be incorrect ( and in fact exactly opposite ) + Assert.assertFalse(isOrdered(t1, t2, t3)); + Assert.assertTrue(isOrdered(t2, t2, t1)); + // but the comparator will maintain the correct ordering + Assert.assertTrue(isOrdered(CompositeTimestamp.comparator(), t1, t2, t3)); + } + + @Test + public void testOrderingEventDates() { + Calendar cal = Calendar.getInstance(); + long twoMonthsLater = cal.getTimeInMillis(); + cal.add(Calendar.MONTH, -1); + long aMonthLater = cal.getTimeInMillis(); + cal.add(Calendar.MONTH, -1); + long aDate = cal.getTimeInMillis(); + + // test different event dates with the equivalent ageoff dates + long t1 = CompositeTimestamp.getCompositeTimeStamp(aDate, aDate); + long t2 = CompositeTimestamp.getCompositeTimeStamp(aMonthLater, aMonthLater); + long t3 = CompositeTimestamp.getCompositeTimeStamp(twoMonthsLater, twoMonthsLater); + + // in this case the natural ordering will be correct + Assert.assertTrue(isOrdered(t1, t2, t3)); + // and the comparator will maintain that ordering + Assert.assertTrue(isOrdered(CompositeTimestamp.comparator(), t1, t2, t3)); + + cal = Calendar.getInstance(); + cal.setTimeInMillis(0); + cal.add(Calendar.MONTH, -1); + twoMonthsLater = cal.getTimeInMillis(); + cal.add(Calendar.MONTH, -1); + aMonthLater = cal.getTimeInMillis(); + cal.add(Calendar.MONTH, -1); + aDate = cal.getTimeInMillis(); + + // test different negative event dates with the equivalent ageoff dates + t1 = CompositeTimestamp.getCompositeTimeStamp(aDate, aDate); + t2 = CompositeTimestamp.getCompositeTimeStamp(aMonthLater, aMonthLater); + t3 = CompositeTimestamp.getCompositeTimeStamp(twoMonthsLater, twoMonthsLater); + + // in this case the natural ordering will be correct + Assert.assertTrue(isOrdered(t1, t2, t3)); + // and the comparator will maintain that ordering + Assert.assertTrue(isOrdered(CompositeTimestamp.comparator(), t1, t2, t3)); + } + + @Test + public void testOrderingEventAndAgeoffDates() { + Calendar cal = Calendar.getInstance(); + long twoMonthsLater = cal.getTimeInMillis(); + cal.add(Calendar.MONTH, -1); + long aMonthLater = cal.getTimeInMillis(); + cal.add(Calendar.MONTH, -1); + long aDate = cal.getTimeInMillis(); + + // a mix of ageoff dates and event dates + long t1 = CompositeTimestamp.getCompositeTimeStamp(aDate, twoMonthsLater); + long t2 = CompositeTimestamp.getCompositeTimeStamp(aMonthLater, aMonthLater); + long t3 = CompositeTimestamp.getCompositeTimeStamp(twoMonthsLater, twoMonthsLater); + + // in this case the natural ordering will be incorrect + Assert.assertFalse(isOrdered(t1, t2, t3)); + // but the comparator will maintain the correct ordering + Assert.assertTrue(isOrdered(CompositeTimestamp.comparator(), t1, t2, t3)); + + cal = Calendar.getInstance(); + cal.setTimeInMillis(0); + cal.add(Calendar.MONTH, -1); + twoMonthsLater = cal.getTimeInMillis(); + cal.add(Calendar.MONTH, -1); + aMonthLater = cal.getTimeInMillis(); + cal.add(Calendar.MONTH, -1); + aDate = cal.getTimeInMillis(); + + // a mix of negative ageoff dates and negative event dates + t1 = CompositeTimestamp.getCompositeTimeStamp(aDate, twoMonthsLater); + t2 = CompositeTimestamp.getCompositeTimeStamp(aMonthLater, aMonthLater); + t3 = CompositeTimestamp.getCompositeTimeStamp(twoMonthsLater, twoMonthsLater); + + // in this case the natural ordering will be correct (surprisingly) + Assert.assertTrue(isOrdered(t1, t2, t3)); + // and the comparator will maintain that ordering + Assert.assertTrue(isOrdered(CompositeTimestamp.comparator(), t1, t2, t3)); + } + +} diff --git a/warehouse/ingest-configuration/src/main/resources/config/shard-ingest-config.xml b/warehouse/ingest-configuration/src/main/resources/config/shard-ingest-config.xml index 05001c62094..86150eb31d4 100644 --- a/warehouse/ingest-configuration/src/main/resources/config/shard-ingest-config.xml +++ b/warehouse/ingest-configuration/src/main/resources/config/shard-ingest-config.xml @@ -70,6 +70,15 @@ datawave.ingest.table.config.ShardTableConfigHelper + + ${table.name.shardIndex}.disable.versioning.iterator + true + + + ${table.name.shardReverseIndex}.disable.versioning.iterator + true + + markings.setup.iterator.enabled false diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/config/RawRecordContainerImpl.java b/warehouse/ingest-core/src/main/java/datawave/ingest/config/RawRecordContainerImpl.java index 8e06d2c614e..96dea3dc999 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/config/RawRecordContainerImpl.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/config/RawRecordContainerImpl.java @@ -37,6 +37,7 @@ import datawave.ingest.data.config.ingest.IgnorableErrorHelperInterface; import datawave.ingest.protobuf.RawRecordContainer.Data; import datawave.marking.MarkingFunctions; +import datawave.util.CompositeTimestamp; public class RawRecordContainerImpl implements Writable, Configurable, RawRecordContainer { @@ -61,7 +62,10 @@ public class RawRecordContainerImpl implements Writable, Configurable, RawRecord private Multimap fatalErrors = HashMultimap.create(); private Multimap ignorableErrorHelpers = HashMultimap.create(); - private long eventDate = Long.MIN_VALUE; + /** + * This is the composite date for this event + */ + private long timestamp = CompositeTimestamp.INVALID_TIMESTAMP; private Type dataType = null; private UID uid = null; private UIDBuilder uidBuilder; @@ -179,13 +183,13 @@ public void setDataType(Type dataType) { } @Override - public long getDate() { - return this.eventDate; + public long getTimestamp() { + return this.timestamp; } @Override - public void setDate(long date) { - this.eventDate = date; + public void setTimestamp(long date) { + this.timestamp = date; } @Override @@ -421,7 +425,7 @@ public boolean equals(Object other) { } RawRecordContainerImpl e = (RawRecordContainerImpl) other; EqualsBuilder equals = new EqualsBuilder(); - equals.append(this.eventDate, e.eventDate); + equals.append(this.timestamp, e.timestamp); equals.append(this.dataType, e.dataType); equals.append(this.uid, e.uid); equals.append(this.errors, e.errors); @@ -439,7 +443,7 @@ public boolean equals(Object other) { @Override public int hashCode() { - int result = (int) (eventDate ^ (eventDate >>> 32)); + int result = (int) (timestamp ^ (timestamp >>> 32)); result = 31 * result + (dataType != null ? dataType.hashCode() : 0); result = 31 * result + (uid != null ? uid.hashCode() : 0); result = 31 * result + (errors != null ? errors.hashCode() : 0); @@ -465,7 +469,7 @@ public int hashCode() { protected RawRecordContainerImpl copyInto(RawRecordContainerImpl rrci) { copyConfiguration(rrci); - rrci.eventDate = this.eventDate; + rrci.timestamp = this.timestamp; rrci.dataType = this.dataType; rrci.uid = this.uid; rrci.errors = new ConcurrentSkipListSet<>(this.errors); @@ -645,7 +649,7 @@ public void reloadConfiguration() { @Override public void write(DataOutput out) throws IOException { Data.Builder builder = Data.newBuilder(); - builder.setDate(this.eventDate); + builder.setDate(this.timestamp); if (null != this.dataType) builder.setDataType(this.dataType.typeName()); if (null != this.uid) @@ -679,7 +683,7 @@ public void readFields(DataInput in) throws IOException { in.readFully(buf); Data data = Data.parseFrom(buf); - this.eventDate = data.getDate(); + this.timestamp = data.getDate(); if (data.hasDataType()) try { this.dataType = TypeRegistry.getType(data.getDataType()); @@ -715,7 +719,7 @@ public void readFields(DataInput in) throws IOException { * Resets state for re-use. */ public void clear() { - eventDate = Long.MIN_VALUE; + timestamp = CompositeTimestamp.INVALID_TIMESTAMP; dataType = null; uid = null; errors.clear(); @@ -740,7 +744,7 @@ public void clear() { @Override public String toString() { ToStringBuilder buf = new ToStringBuilder(this); - buf.append("eventDate", this.eventDate); + buf.append("timestamp", this.timestamp); buf.append("dataType", dataType.typeName()); buf.append("uid", String.valueOf(this.uid)); buf.append("errors", errors); diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/RawRecordContainer.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/RawRecordContainer.java index dde4200bc6f..189820a4853 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/RawRecordContainer.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/RawRecordContainer.java @@ -9,6 +9,7 @@ import org.apache.accumulo.core.security.ColumnVisibility; import datawave.data.hash.UID; +import datawave.util.CompositeTimestamp; /** * Generic container used to hold raw source data. It is used in various parts of the ingest framework and is typically persisted as an "event" within DW's @@ -45,7 +46,43 @@ public interface RawRecordContainer { * * @return the date for this raw record */ - long getDate(); + default long getDate() { + return CompositeTimestamp.getEventDate(getTimestamp()); + } + + /** + * Gets the ageoff date associated with the record + * + * @return the ageoff date for this raw record + */ + default long getAgeOffDate() { + return CompositeTimestamp.getAgeOffDate(getTimestamp()); + } + + /** + * Get the composite timestamp asociated with this record (@see CompositeTimestamp) + * + * @return + */ + long getTimestamp(); + + /** + * Determine if the timestamp has been set. This avoids having to compare the timestamp with an arbitrary value + */ + default boolean isTimestampSet() { + return getTimestamp() != CompositeTimestamp.INVALID_TIMESTAMP; + } + + /** + * This is synonomis with setTimestamp(date, date) + * + * @param timestamp + * primary date to be associated with the record, a.k.a. the "event date" + */ + @Deprecated + default void setDate(long timestamp) { + setTimestamp(timestamp); + } /** * In the DW data model, this date is often referred to as "event date" and represents the primary date value for the record. At ingest time, it is @@ -56,11 +93,29 @@ public interface RawRecordContainer { * Thus, this date is typically leveraged by DW's query api as the basis for 'begin' / 'end' date ranges for user queries. However, DW also has the ability * to leverage other dates within your records at query time, if needed. So, for date filtering concerns, you're not necessarily stuck with your choice of * 'event' date in this regard + *

* - * @param date + *

+ * This date is treated as a composite timestamp which includes the age off date as well. If the ageoff date is identical to the event date (which is + * usually the case), then the event date and the timestamp will be the exactly same value. See CompositeTimestamp for more information. The getDate() + * method will only return the event date portion of this date and the getAgeOffDate() will return the ageoff portion. This method should eventually be + * deprecated and setTimestamp should be used instead. + *

+ * + * @param timestamp * primary date to be associated with the record, a.k.a. the "event date" */ - void setDate(long date); + void setTimestamp(long timestamp); + + /** + * Equivalent to setTimestamp(CompositeTimestamp.getCompositeTimestamp(eventDate, ageOffDate)); + * + * @param eventDate + * @param ageOffDate + */ + default void setTimestamp(long eventDate, long ageOffDate) { + setTimestamp(CompositeTimestamp.getCompositeTimeStamp(eventDate, ageOffDate)); + } Collection getErrors(); diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/BaseIngestHelper.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/BaseIngestHelper.java index 4be57bdf85e..c3d28d3a2d8 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/BaseIngestHelper.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/BaseIngestHelper.java @@ -47,6 +47,7 @@ * fields values from the datatypes that they represent. */ public abstract class BaseIngestHelper extends AbstractIngestHelper implements CompositeIngest, VirtualIngest { + /** * Configuration parameter to specify that data should be marked for delete on ingest. */ diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/input/reader/AbstractEventRecordReader.java b/warehouse/ingest-core/src/main/java/datawave/ingest/input/reader/AbstractEventRecordReader.java index 52e0c50e864..20083dae89e 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/input/reader/AbstractEventRecordReader.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/input/reader/AbstractEventRecordReader.java @@ -35,6 +35,7 @@ import datawave.ingest.data.config.MarkingsHelper; import datawave.policy.IngestPolicyEnforcer; import datawave.policy.Policy; +import datawave.util.CompositeTimestamp; public abstract class AbstractEventRecordReader extends RecordReader implements EventRecordReader { @@ -246,7 +247,7 @@ protected void extractEventDate(final String fieldName, final String fieldValue) } } } - if (event.getDate() == Long.MIN_VALUE) { + if (!event.isTimestampSet()) { List patterns = new ArrayList<>(formatters.size()); for (SimpleDateFormat formatter : formatters) { patterns.add(formatter.toPattern()); diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomDataTypeHandler.java index b0f22fc8539..f48dfa0a9ec 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/atom/AtomDataTypeHandler.java @@ -180,20 +180,20 @@ public long process(KEYIN key, RawRecordContainer event, Multimap Long.MIN_VALUE) { + if (event.isTimestampSet()) { eventDate = DateHelper.format(event.getDate()); } Text colq = safeAppend(null, eventDate); diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java index 00fa4a0c1a1..e167321c8d7 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java @@ -311,7 +311,7 @@ record = record.copy(); // ShardId 'd' DataType\0UID\0Name for document content event using Event.Writable String colq = record.getDataType().outputName() + '\0' + record.getId() + '\0' + EVENT_CONTENT_FIELD; Key k = createKey(getShardId(record), new Text(ExtendedDataTypeHandler.FULL_CONTENT_COLUMN_FAMILY), new Text(colq), getVisibility(record, null), - record.getDate(), this.helper.getDeleteMode()); + record.getTimestamp(), this.helper.getDeleteMode()); BulkIngestKey ebKey = new BulkIngestKey(getShardTableName(), k); contextWriter.write(ebKey, value, context); diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java index dc3f27c7a94..4073a14a0f4 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java @@ -47,6 +47,7 @@ import datawave.ingest.util.DiskSpaceStarvationStrategy; import datawave.marking.MarkingFunctions; import datawave.query.model.Direction; +import datawave.util.CompositeTimestamp; import datawave.util.TextUtil; /** @@ -405,6 +406,9 @@ public Multimap processBulk(KEYIN key, RawRecordContainer e if (event.fatalError()) { return null; } else { + // create an event that returns its timestamp date + IngestHelperInterface helper = getHelper(event.getDataType()); + if (isReindexEnabled) { Multimap filteredEventFields = filterByRequestedFields(eventFields); if (filteredEventFields.isEmpty()) { @@ -739,13 +743,13 @@ protected Multimap createTermIndexColumn(RawRecordContainer } } // Create a key for the masked field value with the masked visibility. - Key k = this.createIndexKey(normalizedMaskedValue.getBytes(), colf, colq, maskedVisibility, event.getDate(), false); + Key k = this.createIndexKey(normalizedMaskedValue.getBytes(), colf, colq, maskedVisibility, event.getTimestamp(), false); BulkIngestKey bkey = new BulkIngestKey(tableName, k); values.put(bkey, indexValue); } if (!StringUtils.isEmpty(fieldValue)) { // Now create a key for the unmasked value with the original visibility - Key k = this.createIndexKey(fieldValue.getBytes(), colf, colq, visibility, event.getDate(), deleteMode); + Key k = this.createIndexKey(fieldValue.getBytes(), colf, colq, visibility, event.getTimestamp(), deleteMode); BulkIngestKey bkey = new BulkIngestKey(tableName, k); values.put(bkey, indexValue); } @@ -765,7 +769,7 @@ protected Multimap createTermIndexColumn(RawRecordContainer refVisibility = maskedVisibility; } - Key k = this.createIndexKey(fieldValue.getBytes(), colf, colq, refVisibility, event.getDate(), deleteMode); + Key k = this.createIndexKey(fieldValue.getBytes(), colf, colq, refVisibility, event.getTimestamp(), deleteMode); BulkIngestKey bkey = new BulkIngestKey(tableName, k); values.put(bkey, indexValue); @@ -851,14 +855,34 @@ protected Key createKey(byte[] row, Text colf, Text colq, byte[] vis, long ts, b * @return Accumulo Key object */ protected Key createIndexKey(byte[] row, Text colf, Text colq, byte[] vis, long ts, boolean delete) { - // Truncate the timestamp to the day - long tsToDay = (ts / MS_PER_DAY) * MS_PER_DAY; - - Key k = new Key(row, 0, row.length, colf.getBytes(), 0, colf.getLength(), colq.getBytes(), 0, colq.getLength(), vis, 0, vis.length, tsToDay); + Key k = new Key(row, 0, row.length, colf.getBytes(), 0, colf.getLength(), colq.getBytes(), 0, colq.getLength(), vis, 0, vis.length, + getIndexTimestamp(ts)); k.setDeleted(delete); return k; } + /** + * trim the event date and ageoff portions of the ts to the beginning of the day + * + * @param ts + * @return the timestamp to be used for index entries + */ + public static long getIndexTimestamp(long ts) { + long tsToDay = trimToBeginningOfDay(CompositeTimestamp.getEventDate(ts)); + long ageOffToDay = trimToBeginningOfDay(CompositeTimestamp.getAgeOffDate(ts)); + return CompositeTimestamp.getCompositeTimeStamp(tsToDay, ageOffToDay); + } + + /** + * Trim ms to the beginning of the day + * + * @param date + * @return the time at the beginning of the day + */ + public static long trimToBeginningOfDay(long date) { + return (date / MS_PER_DAY) * MS_PER_DAY; + } + /** * Creates a shard column key and does *NOT* apply masking logic * @@ -933,7 +957,7 @@ protected Multimap createShardEventColumn(RawRecordContaine // Generate a key for the original, unmasked field field value if (!StringUtils.isEmpty(fieldValue)) { // One key with the original value and original visibility - Key cbKey = createKey(shardId, colf, unmaskedColq, visibility, event.getDate(), deleteMode); + Key cbKey = createKey(shardId, colf, unmaskedColq, visibility, event.getTimestamp(), deleteMode); BulkIngestKey bKey = new BulkIngestKey(this.getShardTableName(), cbKey); values.put(bKey, NULL_VALUE); @@ -955,7 +979,7 @@ protected Multimap createShardEventColumn(RawRecordContaine } // Else create one key for the field with the original value and the masked visiblity - Key cbKey = createKey(shardId, colf, unmaskedColq, refVisibility, event.getDate(), deleteMode); + Key cbKey = createKey(shardId, colf, unmaskedColq, refVisibility, event.getTimestamp(), deleteMode); BulkIngestKey bKey = new BulkIngestKey(this.getShardTableName(), cbKey); if (log.isTraceEnabled()) log.trace("Creating bulk ingest Key " + bKey); @@ -974,7 +998,7 @@ protected void createMaskedShardEventColumn(RawRecordContainer event, Text colf, TextUtil.textAppend(maskedColq, maskedFieldValue, replaceMalformedUTF8); // Another key with masked value and masked visibility - Key cbKey = createKey(shardId, colf, maskedColq, maskedVisibility, event.getDate(), deleteMode); + Key cbKey = createKey(shardId, colf, maskedColq, maskedVisibility, event.getTimestamp(), deleteMode); BulkIngestKey bKey = new BulkIngestKey(this.getShardTableName(), cbKey); values.put(bKey, NULL_VALUE); } @@ -1032,7 +1056,7 @@ protected Multimap createShardFieldIndexColumn(RawRecordCon if (null != maskedFieldHelper && maskedFieldHelper.contains(fieldName)) { // Put unmasked colq with original visibility - Key k = createKey(shardId, colf, unmaskedColq, visibility, event.getDate(), deleteMode); + Key k = createKey(shardId, colf, unmaskedColq, visibility, event.getTimestamp(), deleteMode); BulkIngestKey bKey = new BulkIngestKey(this.getShardTableName(), k); values.put(bKey, value); @@ -1044,7 +1068,7 @@ protected Multimap createShardFieldIndexColumn(RawRecordCon TextUtil.textAppend(maskedColq, event.getId().toString(), replaceMalformedUTF8); // Put masked colq with masked visibility - Key key = createKey(shardId, colf, maskedColq, maskedVisibility, event.getDate(), deleteMode); + Key key = createKey(shardId, colf, maskedColq, maskedVisibility, event.getTimestamp(), deleteMode); BulkIngestKey bulkIngestKey = new BulkIngestKey(this.getShardTableName(), key); values.put(bulkIngestKey, value); } @@ -1059,7 +1083,7 @@ protected Multimap createShardFieldIndexColumn(RawRecordCon refVisibility = maskedVisibility; } - Key k = createKey(shardId, colf, unmaskedColq, refVisibility, event.getDate(), deleteMode); + Key k = createKey(shardId, colf, unmaskedColq, refVisibility, event.getTimestamp(), deleteMode); BulkIngestKey bKey = new BulkIngestKey(this.getShardTableName(), k); values.put(bKey, value); } @@ -1110,7 +1134,7 @@ protected void createShardFieldIndexColumn(RawRecordContainer event, Multimap context, StatusReporter reporter, Text uid, byte[] visibility, byte[] shardId, byte[] rawValue) throws IOException, InterruptedException, MutationsRejectedException { - Key k = createKey(shardId, new Text(ExtendedDataTypeHandler.FULL_CONTENT_COLUMN_FAMILY), uid, visibility, event.getDate(), + Key k = createKey(shardId, new Text(ExtendedDataTypeHandler.FULL_CONTENT_COLUMN_FAMILY), uid, visibility, event.getTimestamp(), this.ingestHelper.getDeleteMode()); ByteArrayOutputStream baos = null; @@ -754,7 +754,7 @@ protected void createShardFieldIndexColumn(RawRecordContainer event, ContextWrit value = DataTypeHandler.NULL_VALUE; } - Key k = createKey(shardId, colf, colq, visibility, event.getDate(), deleteMode); + Key k = createKey(shardId, colf, colq, visibility, event.getTimestamp(), deleteMode); BulkIngestKey bKey = new BulkIngestKey(new Text(this.getShardTableName()), k); contextWriter.write(bKey, value, context); } @@ -798,8 +798,9 @@ protected void createTermFrequencyIndex(RawRecordContainer event, ContextWriter< colq.append(this.eventDataTypeName).append('\u0000').append(this.eventUid).append('\u0000').append(nfv.getIndexedFieldValue()).append('\u0000') .append(nfv.getIndexedFieldName()); - BulkIngestKey bKey = new BulkIngestKey(new Text(this.getShardTableName()), new Key(shardId, - ExtendedDataTypeHandler.TERM_FREQUENCY_COLUMN_FAMILY.getBytes(), colq.toString().getBytes(), visibility, event.getDate(), deleteMode)); + BulkIngestKey bKey = new BulkIngestKey(new Text(this.getShardTableName()), + new Key(shardId, ExtendedDataTypeHandler.TERM_FREQUENCY_COLUMN_FAMILY.getBytes(), colq.toString().getBytes(), visibility, + event.getTimestamp(), deleteMode)); contextWriter.write(bKey, value, context); } @@ -844,7 +845,7 @@ protected void createIndexColumn(RawRecordContainer event, ContextWriter getValues(SortedKeyValueIterator iterator) { + return new ValueCombiner(iterator, PartialKey.ROW_COLFAM_COLQUAL_COLVIS_TIME); + } + + @Override public Value aggregate() { Builder builder = Uid.List.newBuilder(); @@ -229,27 +238,14 @@ public Value reduce(Key key, Iterator iter) { */ private boolean processRemovalUids(Uid.List value) { for (String uid : value.getREMOVEDUIDList()) { - // Don't remove the UID if it's in the UID list since that means a newer key - // (larger timestamp value) added the UID and we don't want to undo that add. - // If timestampsIgnored is set, then we are presuming lots of collisions on - // timestamp and the order of incoming values is non-deterministic. In that case + // The order of incoming values for the same timestamp is non-deterministic. In that case // we give precedence to a removal over an add and mark this UID as removed even // if it was in the UID list. - // - // If we're not propagating changes then don't bother adding UIDs to the - // REMOVEDUID list. This happens if either we're doing a scan or a full major - // compaction. In those cases, we're guaranteed that we'll see all of the values - // for a given key and therefore there's no need to include removed UIDs in the - // output protocol buffer since the remove have already been applied. - if (timestampsIgnored) { - uids.remove(uid); - // Even if propagate is false, the removal UID should persist until all PB lists - // in the current iteration have been seen, in case a subsequent PB has the UID - // marked for removal. - uidsToRemove.add(uid); - } else if (propogate && !uids.contains(uid)) { - uidsToRemove.add(uid); - } + uids.remove(uid); + // Even if propagate is false, the removal UID should persist until all PB lists + // in the current iteration have been seen, in case a subsequent PB has the UID + // marked for removal. + uidsToRemove.add(uid); // Avoid exceeding maxUids in the uidToRemove list, even when propogating. // Check for this condition after adding each of the remove UID entries to uidsToRemove, // which also de-duplicates uids in that Set. @@ -326,18 +322,12 @@ public boolean propogateKey() { @Override public boolean validateOptions(Map options) { boolean valid = super.validateOptions(options); - if (valid) { - if (options.containsKey(TIMESTAMPS_IGNORED)) { - timestampsIgnored = Boolean.parseBoolean(options.get(TIMESTAMPS_IGNORED)); - } - } return valid; } @Override public SortedKeyValueIterator deepCopy(IteratorEnvironment env) { GlobalIndexUidAggregator copy = (GlobalIndexUidAggregator) super.deepCopy(env); - copy.timestampsIgnored = timestampsIgnored; copy.propogate = propogate; // Not copying other fields that are all cleared in the reset() method. return copy; @@ -346,12 +336,5 @@ public SortedKeyValueIterator deepCopy(IteratorEnvironment env) { @Override public void init(SortedKeyValueIterator source, Map options, IteratorEnvironment env) throws IOException { super.init(source, options, env); - if (options.containsKey(TIMESTAMPS_IGNORED)) { - timestampsIgnored = Boolean.parseBoolean(options.get(TIMESTAMPS_IGNORED)); - } - } - - public static void setTimestampsIgnoredOpt(IteratorSetting is, boolean timestampsIgnored) { - is.addOption(TIMESTAMPS_IGNORED, Boolean.toString(timestampsIgnored)); } } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/table/aggregator/PropogatingCombiner.java b/warehouse/ingest-core/src/main/java/datawave/ingest/table/aggregator/PropogatingCombiner.java index 089a5536663..79e7a60cb0e 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/table/aggregator/PropogatingCombiner.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/table/aggregator/PropogatingCombiner.java @@ -5,8 +5,11 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.iterators.Combiner; +import org.apache.accumulo.core.iterators.SortedKeyValueIterator; import org.apache.log4j.Logger; +import datawave.iterators.ValueCombiner; + /** * * Aids in determining if a value and the corresponding keys should propogate @@ -23,6 +26,16 @@ public abstract class PropogatingCombiner extends Combiner { */ protected boolean propogate = true; + /** + * Get the iterator of values to be combined + * + * @param iterator + * @return a value iterator + */ + public Iterator getValues(SortedKeyValueIterator iterator) { + return new ValueCombiner(iterator); + } + /** * Shpuld return a thread safe value. * diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/AbstractTableConfigHelper.java b/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/AbstractTableConfigHelper.java index 479a98d2a88..41d2c811d9f 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/AbstractTableConfigHelper.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/AbstractTableConfigHelper.java @@ -82,6 +82,8 @@ public static void setPropertyIfNecessary(String tableName, String propertyName, * the aggregators that should be set on {@code tableName} * @param tops * accumulo table operations helper for configuring tables + * @param config + * the configuration * @param log * a {@link Logger} for diagnostic messages * @@ -143,6 +145,8 @@ public static Map generateInitialTableProperties(Configuration co * the aggregators to check for on {@code tableName} * @param tops * accumulo table operations helper for configuring tables + * @param config + * the configuration * @return {@code true} if {@code aggregators} are configured on {@code tableName} and {@code false} if not * * @throws TableNotFoundException diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/ShardTableConfigHelper.java b/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/ShardTableConfigHelper.java index 49ba804d89f..8100a053fc9 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/ShardTableConfigHelper.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/ShardTableConfigHelper.java @@ -170,6 +170,7 @@ protected void configureShardTable(TableOperations tops) throws AccumuloExceptio // Set a text index aggregator on the "tf" (Term Frequency) column family CombinerConfiguration tfConf = new CombinerConfiguration(new Column("tf"), new IteratorSetting(10, "TF", datawave.ingest.table.aggregator.TextIndexAggregator.class.getName())); + setAggregatorConfigurationIfNecessary(tableName, Collections.singletonList(tfConf), tops, conf, log); if (markingsSetupIteratorEnabled) { diff --git a/warehouse/ingest-core/src/main/java/datawave/iterators/PropogatingIterator.java b/warehouse/ingest-core/src/main/java/datawave/iterators/PropogatingIterator.java index d7eadac85fe..91ae66d9cb9 100644 --- a/warehouse/ingest-core/src/main/java/datawave/iterators/PropogatingIterator.java +++ b/warehouse/ingest-core/src/main/java/datawave/iterators/PropogatingIterator.java @@ -150,7 +150,7 @@ private boolean aggregateRowColumn() throws IOException { // reset the state of the combiner. aggr.reset(); - aggregatedValue = aggr.reduce(keyToAggregate, new ValueCombiner(iterator)); + aggregatedValue = aggr.reduce(keyToAggregate, aggr.getValues(iterator)); // always propagate deletes if (aggr.propogateKey() || workKey.isDeleted()) { diff --git a/warehouse/ingest-core/src/main/java/datawave/iterators/ValueCombiner.java b/warehouse/ingest-core/src/main/java/datawave/iterators/ValueCombiner.java index 0168ff42681..e4653804382 100644 --- a/warehouse/ingest-core/src/main/java/datawave/iterators/ValueCombiner.java +++ b/warehouse/ingest-core/src/main/java/datawave/iterators/ValueCombiner.java @@ -18,6 +18,7 @@ public class ValueCombiner implements Iterator { Key topKey; SortedKeyValueIterator source; boolean hasNext; + PartialKey keysToCombine = PartialKey.ROW_COLFAM_COLQUAL_COLVIS; private static final Logger log = Logger.getLogger(ValueCombiner.class); @@ -28,7 +29,12 @@ public class ValueCombiner implements Iterator { * The {@code SortedKeyValueIterator} from which to read data. */ public ValueCombiner(SortedKeyValueIterator source) { + this(source, PartialKey.ROW_COLFAM_COLQUAL_COLVIS); + } + + public ValueCombiner(SortedKeyValueIterator source, PartialKey keysToCombine) { this.source = source; + this.keysToCombine = keysToCombine; topKey = new Key(source.getTopKey()); hasNext = _hasNext(); } @@ -40,7 +46,7 @@ private boolean _hasNext() { log.trace(source.getTopKey()); } - return source.hasTop() && topKey.equals(source.getTopKey(), PartialKey.ROW_COLFAM_COLQUAL_COLVIS); + return source.hasTop() && topKey.equals(source.getTopKey(), keysToCombine); } /** diff --git a/warehouse/ingest-core/src/main/java/datawave/policy/ExampleIngestPolicyEnforcer.java b/warehouse/ingest-core/src/main/java/datawave/policy/ExampleIngestPolicyEnforcer.java index c1942d3d5be..e0c01c673e6 100644 --- a/warehouse/ingest-core/src/main/java/datawave/policy/ExampleIngestPolicyEnforcer.java +++ b/warehouse/ingest-core/src/main/java/datawave/policy/ExampleIngestPolicyEnforcer.java @@ -51,7 +51,7 @@ private void validateSecurityMarkings(RawRecordContainer event) { * an event container */ private void validateEventDate(RawRecordContainer event) { - if (event.getDate() == Long.MIN_VALUE) { + if (!event.isTimestampSet()) { event.addError(RawDataErrorNames.EVENT_DATE_MISSING); log.error("Event date missing for Event in raw file: " + event.getRawFileName() + ", record: " + event.getRawRecordNumber()); } diff --git a/warehouse/ingest-core/src/test/java/datawave/ingest/data/RawRecordContainerImplTest.java b/warehouse/ingest-core/src/test/java/datawave/ingest/data/RawRecordContainerImplTest.java index cbd546f38b7..a8f191526b3 100644 --- a/warehouse/ingest-core/src/test/java/datawave/ingest/data/RawRecordContainerImplTest.java +++ b/warehouse/ingest-core/src/test/java/datawave/ingest/data/RawRecordContainerImplTest.java @@ -22,6 +22,7 @@ import datawave.ingest.config.IngestConfigurationFactory; import datawave.ingest.config.RawRecordContainerImpl; import datawave.ingest.data.config.MarkingsHelper; +import datawave.util.CompositeTimestamp; public class RawRecordContainerImplTest { @@ -377,17 +378,18 @@ public void validate() { if (getAltIds() == null || getAltIds().isEmpty()) { addError(RawDataErrorNames.UUID_MISSING); } - if (Long.MIN_VALUE == getDate()) { + if (!isTimestampSet()) { addError(RawDataErrorNames.EVENT_DATE_MISSING); + } else { + if (0L == getRawFileTimestamp()) { + setRawFileTimestamp(getDate()); + } } + if (0 == getRawRecordNumber()) { addError(RawDataErrorNames.INVALID_RECORD_NUMBER); } - if (0L == getRawFileTimestamp()) { - setRawFileTimestamp(getDate()); - } - try { initializeMarkingsHelper(); } catch (Throwable t) { diff --git a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/EventMapperTest.java b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/EventMapperTest.java index b0248906afa..39e6046c659 100644 --- a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/EventMapperTest.java +++ b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/EventMapperTest.java @@ -79,7 +79,7 @@ public void setUp() throws Exception { record = new SimpleRawRecord(); record.setRawFileTimestamp(eventTime); record.setDataType(type); - record.setDate(eventTime); + record.setTimestamp(eventTime); record.setRawFileName("/some/filename"); record.setRawData("some data".getBytes()); record.generateId(null); @@ -87,7 +87,7 @@ record = new SimpleRawRecord(); errorRecord = new SimpleRawRecord(); errorRecord.setRawFileTimestamp(0); errorRecord.setDataType(type); - errorRecord.setDate(eventTime); + errorRecord.setTimestamp(eventTime); errorRecord.setRawFileName("/some/filename"); errorRecord.setRawData("some data".getBytes()); errorRecord.generateId(null); diff --git a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/SimpleRawRecord.java b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/SimpleRawRecord.java index 10429435564..9d8180b4021 100644 --- a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/SimpleRawRecord.java +++ b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/SimpleRawRecord.java @@ -20,6 +20,7 @@ import datawave.ingest.data.RawRecordContainer; import datawave.ingest.data.Type; import datawave.ingest.data.TypeRegistry; +import datawave.util.CompositeTimestamp; /** * A POJO implementation of the {@link datawave.ingest.data.RawRecordContainer}. @@ -31,7 +32,7 @@ public class SimpleRawRecord implements RawRecordContainer, Writable { private Map securityMarkings = new TreeMap<>(); private UID id = uidBuilder.newId(); private Type dataType; - private long date = Long.MIN_VALUE; + private long timestamp = CompositeTimestamp.INVALID_TIMESTAMP; private Collection errors = new ArrayList<>(); private Collection altIds = Collections.emptyList(); private String rawFileName = ""; @@ -89,13 +90,13 @@ public void setDataType(Type dataType) { } @Override - public long getDate() { - return date; + public long getTimestamp() { + return timestamp; } @Override - public void setDate(long date) { - this.date = date; + public void setTimestamp(long timestamp) { + this.timestamp = timestamp; } @Override @@ -247,7 +248,7 @@ public void write(DataOutput dataOutput) throws IOException { id.write(dataOutput); dataOutput.writeUTF(dataType.typeName()); - dataOutput.writeLong(date); + dataOutput.writeLong(timestamp); TestWritableUtil.writeCollection(errors, dataOutput); TestWritableUtil.writeCollection(altIds, dataOutput); @@ -274,7 +275,7 @@ public void readFields(DataInput dataInput) throws IOException { id.readFields(dataInput); dataType = TypeRegistry.getType(dataInput.readUTF()); - date = dataInput.readLong(); + timestamp = dataInput.readLong(); errors = TestWritableUtil.readCollection(dataInput); altIds = TestWritableUtil.readCollection(dataInput); diff --git a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandlerTest.java b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandlerTest.java index eac3bd65cc6..8acf41dd121 100644 --- a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandlerTest.java +++ b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandlerTest.java @@ -5,8 +5,10 @@ import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.concurrent.TimeUnit; import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.TaskAttemptID; @@ -17,8 +19,7 @@ import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; -import datawave.data.type.GeometryType; -import datawave.data.type.NumberType; +import datawave.data.hash.HashUID; import datawave.ingest.config.RawRecordContainerImpl; import datawave.ingest.data.RawRecordContainer; import datawave.ingest.data.Type; @@ -27,15 +28,14 @@ import datawave.ingest.data.config.MaskedFieldHelper; import datawave.ingest.data.config.NormalizedContentInterface; import datawave.ingest.data.config.NormalizedFieldAndValue; -import datawave.ingest.data.config.ingest.BaseIngestHelper; import datawave.ingest.data.config.ingest.ContentBaseIngestHelper; import datawave.ingest.mapreduce.job.BulkIngestKey; -import datawave.ingest.mapreduce.partition.BalancedShardPartitioner; import datawave.ingest.protobuf.Uid; import datawave.ingest.table.config.ShardTableConfigHelper; import datawave.ingest.table.config.TableConfigHelper; import datawave.policy.IngestPolicyEnforcer; import datawave.query.model.Direction; +import datawave.util.CompositeTimestamp; import datawave.util.TableName; public class ShardedDataTypeHandlerTest { @@ -48,6 +48,8 @@ public class ShardedDataTypeHandlerTest { private static final String DATA_TYPE_NAME = "wkt"; private static final String INGEST_HELPER_CLASS = TestIngestHelper.class.getName(); + private static final long MS_PER_DAY = TimeUnit.DAYS.toMillis(1); + Configuration configuration; public static class TestIngestHelper extends ContentBaseIngestHelper { @@ -140,6 +142,7 @@ public void testCreateTermIndex() { record.setRawFileName("data_" + 0 + ".dat"); record.setRawRecordNumber(1); record.setRawData(entry.getBytes(StandardCharsets.UTF_8)); + record.setDate(System.currentTimeMillis()); Uid.List uid = Uid.List.newBuilder().setIGNORE(false).setCOUNT(1).addUID("d8zay2.-3pnndm.-anolok").build(); byte[] visibility = new byte[] {65, 76, 76}; @@ -160,6 +163,7 @@ public void testCreateTermReverseIndex() { record.setRawFileName("data_" + 0 + ".dat"); record.setRawRecordNumber(1); record.setRawData(entry.getBytes(StandardCharsets.UTF_8)); + record.setDate(System.currentTimeMillis()); Uid.List uid = Uid.List.newBuilder().setIGNORE(false).setCOUNT(1).addUID("d8zay2.-3pnndm.-anolok").build(); byte[] visibility = new byte[] {65, 76, 76}; @@ -189,6 +193,7 @@ public void testMaskedForward() { record.setRawFileName("data_" + 0 + ".dat"); record.setRawRecordNumber(1); record.setRawData(entry.getBytes(StandardCharsets.UTF_8)); + record.setDate(System.currentTimeMillis()); Uid.List uid = Uid.List.newBuilder().setIGNORE(false).setCOUNT(1).addUID("d8zay2.-3pnndm.-anolok").build(); byte[] visibility = new byte[] {65, 76, 76}; @@ -218,6 +223,7 @@ public void testNonMaskedReverseIndex() { record.setRawFileName("data_" + 0 + ".dat"); record.setRawRecordNumber(1); record.setRawData(entry.getBytes(StandardCharsets.UTF_8)); + record.setDate(System.currentTimeMillis()); Uid.List uid = Uid.List.newBuilder().setIGNORE(false).setCOUNT(1).addUID("d8zay2.-3pnndm.-anolok").build(); byte[] visibility = new byte[] {65, 76, 76}; @@ -246,6 +252,7 @@ public void testNonMaskedVisibility() { record.setRawFileName("data_" + 0 + ".dat"); record.setRawRecordNumber(1); record.setRawData(entry.getBytes(StandardCharsets.UTF_8)); + record.setDate(System.currentTimeMillis()); Uid.List uid = Uid.List.newBuilder().setIGNORE(false).setCOUNT(1).addUID("d8zay2.-3pnndm.-anolok").build(); byte[] visibility = new byte[] {65, 76, 76}; @@ -262,4 +269,58 @@ public void testNonMaskedVisibility() { } } + @Test + public void testAgeOffDate() { + Type dataType = new Type(DATA_TYPE_NAME, TestIngestHelper.class, null, null, 10, null); + String entry = "testingtesting"; + RawRecordContainer record = new RawRecordContainerImpl(); + record.setDataType(dataType); + record.setRawFileName("data_" + 0 + ".dat"); + record.setRawRecordNumber(1); + record.setRawData(entry.getBytes(StandardCharsets.UTF_8)); + record.setId(HashUID.builder().newId(record.getRawData())); + record.setVisibility(new ColumnVisibility("PUBLIC")); + long expectedEventDate = System.currentTimeMillis(); + record.setDate(expectedEventDate); + Multimap fields = ingestHelper.getEventFields(record); + + assertEquals(expectedEventDate, record.getDate()); + assertEquals(expectedEventDate, record.getAgeOffDate()); + assertEquals(expectedEventDate, record.getTimestamp()); + + Multimap data = handler.processBulk(null, record, fields, null); + + long expectedTimestamp = CompositeTimestamp.getCompositeTimeStamp(expectedEventDate, expectedEventDate); + long tsToDay = (expectedEventDate / MS_PER_DAY) * MS_PER_DAY; + long expectedIndexTimestamp = CompositeTimestamp.getCompositeTimeStamp(tsToDay, tsToDay); + for (BulkIngestKey key : data.keySet()) { + if (key.getTableName().toString().toUpperCase().contains("INDEX")) { + assertEquals(key.toString(), expectedIndexTimestamp, key.getKey().getTimestamp()); + } else { + assertEquals(key.toString(), expectedTimestamp, key.getKey().getTimestamp()); + } + } + + // now get an ageoff date (must be a multiple of MS_PER_DAY past the event date) + long expectedAgeOffDate = expectedEventDate + (MS_PER_DAY * 11); + expectedTimestamp = CompositeTimestamp.getCompositeTimeStamp(expectedEventDate, expectedAgeOffDate); + record.setTimestamp(expectedTimestamp); + + assertEquals(expectedEventDate, record.getDate()); + assertEquals(expectedAgeOffDate, record.getAgeOffDate()); + assertEquals(expectedTimestamp, record.getTimestamp()); + + data = handler.processBulk(null, record, fields, null); + + long ageOffToDay = (expectedAgeOffDate / MS_PER_DAY) * MS_PER_DAY; + expectedIndexTimestamp = CompositeTimestamp.getCompositeTimeStamp(tsToDay, ageOffToDay); + for (BulkIngestKey key : data.keySet()) { + if (key.getTableName().toString().toUpperCase().contains("INDEX")) { + assertEquals(key.toString(), expectedIndexTimestamp, key.getKey().getTimestamp()); + } else { + assertEquals(key.toString(), expectedTimestamp, key.getKey().getTimestamp()); + } + } + } + } diff --git a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/tokenize/ContentIndexingColumnBasedHandlerTest.java b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/tokenize/ContentIndexingColumnBasedHandlerTest.java index d4f7d5fb748..05fb07cc75d 100644 --- a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/tokenize/ContentIndexingColumnBasedHandlerTest.java +++ b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/handler/tokenize/ContentIndexingColumnBasedHandlerTest.java @@ -155,7 +155,7 @@ private void setupMocks() { EasyMock.expect(event.getVisibility()).andReturn(colVis).anyTimes(); EasyMock.expect(event.getDataType()).andReturn(TypeRegistry.getType(TEST_TYPE)).anyTimes(); EasyMock.expect(event.getId()).andReturn(uid).anyTimes(); - EasyMock.expect(event.getDate()).andReturn(CURRENT_TIME).anyTimes(); + EasyMock.expect(event.getTimestamp()).andReturn(CURRENT_TIME).anyTimes(); EasyMock.replay(event); } catch (Exception e) { throw new RuntimeException(e); diff --git a/warehouse/ingest-core/src/test/java/datawave/ingest/table/aggregator/GlobalIndexUidAggregatorTest.java b/warehouse/ingest-core/src/test/java/datawave/ingest/table/aggregator/GlobalIndexUidAggregatorTest.java index da5c24d3dcf..226b8b7f3fd 100644 --- a/warehouse/ingest-core/src/test/java/datawave/ingest/table/aggregator/GlobalIndexUidAggregatorTest.java +++ b/warehouse/ingest-core/src/test/java/datawave/ingest/table/aggregator/GlobalIndexUidAggregatorTest.java @@ -12,25 +12,36 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import java.io.IOException; +import java.time.Instant; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.Iterator; import java.util.List; +import java.util.Set; import java.util.UUID; import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.iterators.SortedKeyValueIterator; import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.junit.Test; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import com.google.common.collect.TreeMultimap; import datawave.ingest.protobuf.Uid; import datawave.ingest.protobuf.Uid.List.Builder; +import datawave.iterators.SortedMultiMapIterator; +import datawave.util.CompositeTimestamp; public class GlobalIndexUidAggregatorTest { @@ -236,10 +247,9 @@ public void testCountWithDuplicates() throws Exception { } @Test - public void testRemoveAndReAddUUID() throws Exception { + public void testRemoveAndThenAddUUID() throws Exception { GlobalIndexUidAggregator localAgg = new GlobalIndexUidAggregator(); IteratorSetting is = new IteratorSetting(19, "test", GlobalIndexUidAggregator.class); - GlobalIndexUidAggregator.setTimestampsIgnoredOpt(is, false); GlobalIndexUidAggregator.setCombineAllColumns(is, true); localAgg.validateOptions(is.getOptions()); @@ -258,12 +268,12 @@ public void testRemoveAndReAddUUID() throws Exception { Collections.reverse(values); Value result = localAgg.reduce(new Key("key"), values.iterator()); Uid.List resultList = Uid.List.parseFrom(result.get()); - assertEquals(2, resultList.getCOUNT()); - assertEquals(2, resultList.getUIDCount()); - assertEquals(2, resultList.getUIDList().size()); - assertEquals(0, resultList.getREMOVEDUIDList().size()); + assertEquals(1, resultList.getCOUNT()); + assertEquals(1, resultList.getUIDCount()); + assertEquals(1, resultList.getUIDList().size()); + assertEquals(1, resultList.getREMOVEDUIDList().size()); assertTrue(resultList.getUIDList().contains(uuid1)); - assertTrue(resultList.getUIDList().contains(uuid2)); + assertTrue(resultList.getREMOVEDUIDList().contains(uuid2)); } @Test @@ -298,7 +308,6 @@ public void testRemoveAndReAddUUIDWithTimestampsIgnored() throws Exception { public void testNegativeCountWithPartialMajorCompaction() throws Exception { GlobalIndexUidAggregator localAgg = new GlobalIndexUidAggregator(); IteratorSetting is = new IteratorSetting(19, "test", GlobalIndexUidAggregator.class); - GlobalIndexUidAggregator.setTimestampsIgnoredOpt(is, false); GlobalIndexUidAggregator.setCombineAllColumns(is, true); localAgg.validateOptions(is.getOptions()); @@ -413,7 +422,6 @@ public void testNegativeCountWithPartialMajorCompactionAndTimestampsIgnored() th public void testRemoveAndReAddUUIDWithPartialMajorCompaction() throws Exception { GlobalIndexUidAggregator localAgg = new GlobalIndexUidAggregator(); IteratorSetting is = new IteratorSetting(19, "test", GlobalIndexUidAggregator.class); - GlobalIndexUidAggregator.setTimestampsIgnoredOpt(is, false); GlobalIndexUidAggregator.setCombineAllColumns(is, true); localAgg.validateOptions(is.getOptions()); @@ -468,12 +476,12 @@ public void testRemoveAndReAddUUIDWithPartialMajorCompaction() throws Exception result = localAgg.reduce(new Key("key"), values.iterator()); resultList = Uid.List.parseFrom(result.get()); - assertEquals(2, resultList.getCOUNT()); - assertEquals(2, resultList.getUIDCount()); - assertEquals(2, resultList.getUIDList().size()); + assertEquals(1, resultList.getCOUNT()); + assertEquals(1, resultList.getUIDCount()); + assertEquals(1, resultList.getUIDList().size()); assertTrue(resultList.getUIDList().contains(uuid1)); - assertTrue(resultList.getUIDList().contains(uuid2)); - assertEquals(0, resultList.getREMOVEDUIDList().size()); + assertFalse(resultList.getUIDList().contains(uuid2)); + assertEquals(1, resultList.getREMOVEDUIDList().size()); } @Test @@ -728,7 +736,6 @@ public void testAddDuplicateUUIDWithSeenIgnoreAndCompaction() throws Exception { public void testRemoveAndReAdd() throws Exception { GlobalIndexUidAggregator localAgg = new GlobalIndexUidAggregator(); IteratorSetting is = new IteratorSetting(19, "test", GlobalIndexUidAggregator.class); - GlobalIndexUidAggregator.setTimestampsIgnoredOpt(is, false); GlobalIndexUidAggregator.setCombineAllColumns(is, true); localAgg.validateOptions(is.getOptions()); @@ -737,20 +744,19 @@ public void testRemoveAndReAdd() throws Exception { String uuid1 = UUID.randomUUID().toString(); ArrayList values = Lists.newArrayList(); - // When we're considering timestamps, an add of a UID, followed by a removal - // of that UID, and then a re-add should result in the UID ending up in the - // UID list. + // an add of a UID, followed by a removal of that UID, and then a re-add + // should still result in the UID removed values.add(toValue(createNewUidList(uuid1))); values.add(toValue(createNewRemoveUidList(uuid1))); values.add(toValue(createNewUidList(uuid1))); Value result = localAgg.reduce(new Key("key"), values.iterator()); Uid.List resultList = Uid.List.parseFrom(result.get()); - assertEquals(1, resultList.getUIDCount()); - assertEquals(1, resultList.getUIDList().size()); - assertEquals(0, resultList.getREMOVEDUIDCount()); - assertEquals(0, resultList.getREMOVEDUIDList().size()); - assertEquals(1, resultList.getCOUNT()); + assertEquals(0, resultList.getUIDCount()); + assertEquals(0, resultList.getUIDList().size()); + assertEquals(1, resultList.getREMOVEDUIDCount()); + assertEquals(1, resultList.getREMOVEDUIDList().size()); + assertEquals(0, resultList.getCOUNT()); } @Test @@ -891,6 +897,54 @@ public void testWeKeepUidListDuringDoubleRemovals() { assertTrue(agg.propogateKey()); } + @Test + public void testCompositeTimestampsMatter() throws IOException { + TreeMultimap keyValues = TreeMultimap.create(); + + long eventDate = Instant.from(DateTimeFormatter.ISO_INSTANT.parse("1960-01-01T00:00:00Z")).toEpochMilli(); + long ageOff = eventDate + (131071L * CompositeTimestamp.MILLIS_PER_DAY); + long negativeCompositeTS = CompositeTimestamp.getCompositeTimeStamp(eventDate, ageOff); + + eventDate = Instant.from(DateTimeFormatter.ISO_INSTANT.parse("2022-10-26T01:00:00Z")).toEpochMilli(); + ageOff = eventDate + CompositeTimestamp.MILLIS_PER_DAY; + long compositeTS = CompositeTimestamp.getCompositeTimeStamp(eventDate, ageOff); + + ageOff = eventDate + (CompositeTimestamp.MILLIS_PER_DAY * 10); + long largerCompositeTS = CompositeTimestamp.getCompositeTimeStamp(eventDate, ageOff); + + List values = new ArrayList<>(); + for (int i = 0; i < 6; i++) { + String uuid = UUID.randomUUID().toString(); + values.add(toValue(createNewUidList(uuid))); + } + + Key key1 = new Key("key", "cf", "cq", "PUBLIC", negativeCompositeTS); + Key key2 = new Key("key", "cf", "cq", "PUBLIC", compositeTS); + Key key3 = new Key("key", "cf", "cq", "PUBLIC", largerCompositeTS); + + keyValues.put(key1, values.get(0)); + keyValues.put(key2, values.get(1)); + keyValues.put(key2, values.get(2)); + keyValues.put(key3, values.get(3)); + keyValues.put(key3, values.get(4)); + keyValues.put(key3, values.get(5)); + + // get an iterator of these key/value pairs (sorted) + SortedKeyValueIterator iterator = new SortedMultiMapIterator(keyValues); + iterator.seek(new Range(), Collections.emptySet(), false); + + // get a unique list of the keys (sorted + Iterator keys = keyValues.keySet().iterator(); + while (keys.hasNext()) { + Key key = keys.next(); + Set expected = Sets.newHashSet(keyValues.get(key)); + // ensure each call to getValues() returns all of the values for the next key where only the timestamp differs + Set actual = Sets.newHashSet(agg.getValues(iterator)); + assertEquals(expected, actual); + } + assertFalse(iterator.hasTop()); + } + private Value agg(List values) { agg.reset(); return agg.reduce(new Key("row"), values.iterator()); diff --git a/warehouse/ingest-core/src/test/java/datawave/ingest/table/config/AbstractTableConfigHelperTest.java b/warehouse/ingest-core/src/test/java/datawave/ingest/table/config/AbstractTableConfigHelperTest.java index dcbe818fceb..f62818acd4b 100644 --- a/warehouse/ingest-core/src/test/java/datawave/ingest/table/config/AbstractTableConfigHelperTest.java +++ b/warehouse/ingest-core/src/test/java/datawave/ingest/table/config/AbstractTableConfigHelperTest.java @@ -56,6 +56,8 @@ public static class TestAbstractTableConfigHelperImpl extends AbstractTableConfi public AbstractTableConfigHelperTest parent; + private Configuration config; + protected Logger createMockLogger() { Logger log = PowerMock.createMock(Logger.class); diff --git a/warehouse/ingest-csv/src/main/java/datawave/ingest/csv/mr/input/CSVRecordReader.java b/warehouse/ingest-csv/src/main/java/datawave/ingest/csv/mr/input/CSVRecordReader.java index dfdb4e99417..52899115f0d 100644 --- a/warehouse/ingest-csv/src/main/java/datawave/ingest/csv/mr/input/CSVRecordReader.java +++ b/warehouse/ingest-csv/src/main/java/datawave/ingest/csv/mr/input/CSVRecordReader.java @@ -202,7 +202,7 @@ else if (!StringUtils.isEmpty(this.csvHelper.getEventIdFieldName()) && fieldName } } // if we set the date with id, don't overwrite it - if (!(fieldName.equals(eventDateFieldName) && event.getDate() > Long.MIN_VALUE)) { + if (!(fieldName.equals(eventDateFieldName) && event.isTimestampSet())) { super.checkField(fieldName, fieldValue); } diff --git a/warehouse/ingest-json/src/main/java/datawave/ingest/json/mr/input/JsonRecordReader.java b/warehouse/ingest-json/src/main/java/datawave/ingest/json/mr/input/JsonRecordReader.java index 520b8f55337..73661a44fd5 100644 --- a/warehouse/ingest-json/src/main/java/datawave/ingest/json/mr/input/JsonRecordReader.java +++ b/warehouse/ingest-json/src/main/java/datawave/ingest/json/mr/input/JsonRecordReader.java @@ -229,7 +229,7 @@ public RawRecordContainer getEvent() { event.setRawData(currentJsonObj.toString().getBytes()); - if (Long.MIN_VALUE == event.getDate()) { + if (!event.isTimestampSet()) { event.setDate(System.currentTimeMillis()); } diff --git a/warehouse/ingest-wikipedia/src/main/java/datawave/ingest/wikipedia/WikipediaDataTypeHandler.java b/warehouse/ingest-wikipedia/src/main/java/datawave/ingest/wikipedia/WikipediaDataTypeHandler.java index 7acfa9ddc6e..2a8abeb05c1 100644 --- a/warehouse/ingest-wikipedia/src/main/java/datawave/ingest/wikipedia/WikipediaDataTypeHandler.java +++ b/warehouse/ingest-wikipedia/src/main/java/datawave/ingest/wikipedia/WikipediaDataTypeHandler.java @@ -352,7 +352,7 @@ protected void createShardEventColumn(RawRecordContainer event, ContextWriter key.getTimestamp()) ? maxTimeStamp : key.getTimestamp(); + this.maxTimeStamp = (this.maxTimeStamp > key.getTimestamp()) ? maxTimeStamp : CompositeTimestamp.getEventDate(key.getTimestamp()); } private boolean acceptTimestamp(Key k) { - return this.stampRange.containsLong(k.getTimestamp()); + return this.stampRange.containsLong(CompositeTimestamp.getEventDate(k.getTimestamp())); } private boolean isFieldIndexKey(Key key) { diff --git a/warehouse/query-core/src/main/java/datawave/core/iterators/filter/GlobalIndexDateRangeFilter.java b/warehouse/query-core/src/main/java/datawave/core/iterators/filter/GlobalIndexDateRangeFilter.java index 3b51a2a755e..4d1359826f6 100644 --- a/warehouse/query-core/src/main/java/datawave/core/iterators/filter/GlobalIndexDateRangeFilter.java +++ b/warehouse/query-core/src/main/java/datawave/core/iterators/filter/GlobalIndexDateRangeFilter.java @@ -15,6 +15,7 @@ import org.apache.log4j.Logger; import datawave.query.Constants; +import datawave.util.CompositeTimestamp; /** * The iterator skips entries in the global index for entries that lie outside the date range set on the BatchScanner @@ -66,7 +67,7 @@ public static void main(String[] args) { @Override public boolean accept(Key k, Value v) { - return range.containsLong(k.getTimestamp()); + return range.containsLong(CompositeTimestamp.getEventDate(k.getTimestamp())); } diff --git a/warehouse/query-core/src/main/java/datawave/query/iterator/filter/LoadDateFilter.java b/warehouse/query-core/src/main/java/datawave/query/iterator/filter/LoadDateFilter.java index 62a2630248d..5b762a4a38f 100644 --- a/warehouse/query-core/src/main/java/datawave/query/iterator/filter/LoadDateFilter.java +++ b/warehouse/query-core/src/main/java/datawave/query/iterator/filter/LoadDateFilter.java @@ -16,6 +16,7 @@ import datawave.core.iterators.ColumnRangeIterator; import datawave.edge.util.EdgeValue; +import datawave.util.CompositeTimestamp; import datawave.util.time.DateHelper; /** @@ -118,6 +119,6 @@ public boolean accept(Key k, Value V) { // event date from the timestamp value rather than the column qualifier. private String extractEventDate(Key k) { - return DateHelper.format(k.getTimestamp()); + return DateHelper.format(CompositeTimestamp.getEventDate(k.getTimestamp())); } } diff --git a/warehouse/query-core/src/main/java/datawave/query/iterators/FieldIndexCountingIterator.java b/warehouse/query-core/src/main/java/datawave/query/iterators/FieldIndexCountingIterator.java index 137cc924b0e..263dd99ee3d 100644 --- a/warehouse/query-core/src/main/java/datawave/query/iterators/FieldIndexCountingIterator.java +++ b/warehouse/query-core/src/main/java/datawave/query/iterators/FieldIndexCountingIterator.java @@ -32,6 +32,7 @@ import datawave.iterators.IteratorSettingHelper; import datawave.marking.MarkingFunctions; import datawave.query.Constants; +import datawave.util.CompositeTimestamp; import datawave.util.TextUtil; /** @@ -355,11 +356,12 @@ private void consume(Key key) { this.count += 1; // set most recent timestamp - this.maxTimeStamp = (this.maxTimeStamp > key.getTimestamp()) ? maxTimeStamp : key.getTimestamp(); + this.maxTimeStamp = (this.maxTimeStamp > CompositeTimestamp.getEventDate(key.getTimestamp())) ? maxTimeStamp + : CompositeTimestamp.getEventDate(key.getTimestamp()); } private boolean acceptTimestamp(Key k) { - return this.stampRange.containsLong(k.getTimestamp()); + return this.stampRange.containsLong(CompositeTimestamp.getEventDate(k.getTimestamp())); } private boolean isFieldIndexKey(Key key) { diff --git a/warehouse/query-core/src/main/java/datawave/query/predicate/TimeFilter.java b/warehouse/query-core/src/main/java/datawave/query/predicate/TimeFilter.java index fa11987eec8..f7094a96799 100644 --- a/warehouse/query-core/src/main/java/datawave/query/predicate/TimeFilter.java +++ b/warehouse/query-core/src/main/java/datawave/query/predicate/TimeFilter.java @@ -9,6 +9,8 @@ import com.google.common.base.Predicate; +import datawave.util.CompositeTimestamp; + /** * Excludes documents which do not fall within the given time range */ @@ -36,7 +38,7 @@ private class KeyTimeFilter implements Predicate { public boolean apply(Key input) { final long timestamp = input.getTimestamp(); - return acceptedRange.containsLong(timestamp); + return acceptedRange.containsLong(CompositeTimestamp.getEventDate(timestamp)); } } diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java index 0720f412b09..261c988c016 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/shard/FieldIndexCountQueryLogic.java @@ -42,6 +42,7 @@ import datawave.query.tables.ShardQueryLogic; import datawave.query.transformer.FieldIndexCountQueryTransformer; import datawave.query.util.MetadataHelper; +import datawave.util.CompositeTimestamp; import datawave.util.StringUtils; import datawave.webservice.query.exception.QueryException; @@ -487,8 +488,8 @@ public Tuple(MarkingFunctions mf) { public void aggregate(Key key, Value val) { uniqueVisibilities.add(key.getColumnVisibility()); count += Long.parseLong(new String(val.get())); - if (maxTimestamp < key.getTimestamp()) { - maxTimestamp = key.getTimestamp(); + if (maxTimestamp < CompositeTimestamp.getEventDate(key.getTimestamp())) { + maxTimestamp = CompositeTimestamp.getEventDate(key.getTimestamp()); } } diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformerSupport.java b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformerSupport.java index 213c1d2cf4b..4fbd5a2db1c 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformerSupport.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/DocumentTransformerSupport.java @@ -46,6 +46,7 @@ import datawave.query.iterator.QueryOptions; import datawave.query.iterator.profile.QuerySpan; import datawave.query.jexl.JexlASTHelper; +import datawave.util.CompositeTimestamp; import datawave.util.StringUtils; import datawave.util.time.DateHelper; import datawave.webservice.query.result.event.EventBase; @@ -386,7 +387,7 @@ protected void collectCardinalities(Document document, Key documentKey, String u log.trace("Document.getTimestamp() returned Log.MAX_VALUE - " + documentKey + " - computed dataDate from row: " + dataDate); } } else { - dataDate = new Date(timestamp); + dataDate = new Date(CompositeTimestamp.getEventDate(timestamp)); } resultCardinalityDocumentDate.addEntry(valueMap, eventId, dataType, dataDate); diff --git a/warehouse/query-core/src/test/java/datawave/helpers/PrintUtility.java b/warehouse/query-core/src/test/java/datawave/helpers/PrintUtility.java index 058d06d36e1..7d1d97ee4fb 100644 --- a/warehouse/query-core/src/test/java/datawave/helpers/PrintUtility.java +++ b/warehouse/query-core/src/test/java/datawave/helpers/PrintUtility.java @@ -23,6 +23,7 @@ import datawave.edge.util.EdgeValue; import datawave.edge.util.ExtendedHyperLogLogPlus; import datawave.ingest.protobuf.Uid; +import datawave.util.CompositeTimestamp; /** * A set of static methods for printing tables in mock Accumulo instance. @@ -61,8 +62,10 @@ public static void printTable(final AccumuloClient client, final Authorizations final Scanner scanner = client.createScanner(tableName, authorizations); for (final Entry e : scanner) { sb.append(e.getKey().toStringNoTime()); - sb.append(' '); - sb.append(dateFormat.format(new Date(e.getKey().getTimestamp()))); + sb.append(" EventDate "); + sb.append(dateFormat.format(new Date(CompositeTimestamp.getEventDate(e.getKey().getTimestamp())))); + sb.append(" AgeOffDate "); + sb.append(dateFormat.format(new Date(CompositeTimestamp.getAgeOffDate(e.getKey().getTimestamp())))); sb.append('\t'); sb.append(getPrintableValue(tableName, e.getKey(), e.getValue())); sb.append("\n"); @@ -85,8 +88,10 @@ public static void printTable(final AccumuloClient client, final Authorizations final Scanner scanner = client.createScanner(tableName, authorizations); for (final Entry e : scanner) { sb.append(e.getKey().toStringNoTime()); - sb.append(' '); - sb.append(dateFormat.format(new Date(e.getKey().getTimestamp()))); + sb.append(" EventDate "); + sb.append(dateFormat.format(new Date(CompositeTimestamp.getEventDate(e.getKey().getTimestamp())))); + sb.append(" AgeOffDate "); + sb.append(dateFormat.format(new Date(CompositeTimestamp.getAgeOffDate(e.getKey().getTimestamp())))); sb.append('\t'); sb.append(getPrintableValue(tableName, e.getKey(), e.getValue())); sb.append("\n"); diff --git a/warehouse/query-core/src/test/java/datawave/query/TestLimitReturnedGroupsToHitTermGroups.java b/warehouse/query-core/src/test/java/datawave/query/TestLimitReturnedGroupsToHitTermGroups.java index 3f105da01fd..2914592b8b3 100644 --- a/warehouse/query-core/src/test/java/datawave/query/TestLimitReturnedGroupsToHitTermGroups.java +++ b/warehouse/query-core/src/test/java/datawave/query/TestLimitReturnedGroupsToHitTermGroups.java @@ -249,7 +249,7 @@ public void testMultipleGroups() throws Exception { "CANINE.PET.12:bernese", "CAT.PET.12:himalayan", "BIRD.PET.12:cockatiel", "FISH.PET.12:swordtail", "CANINE.PET.13:shepherd", "CAT.PET.13:ragdoll", "BIRD.PET.13:lovebird", "FISH.PET.13:tetra", "CANINE.WILD.1:coyote", "CAT.WILD.1:tiger", "BIRD.WILD.1:hawk", "FISH.WILD.1:tuna", - "REPTILE.PET.1:snake", "DOG.WILD.1:coyote"); + "REPTILE.PET.1:snake", "DOG.WILD.1:coyote", "CAT.PET.50:sphynx", "CANINE.PET.50:doberman"); //@formatter:on runTestQuery(queryString, format.parse("20091231"), format.parse("20150101"), extraParameters, goodResults); @@ -267,7 +267,7 @@ public void testMultipleGroupsWithRegexAndReturnFields() throws Exception { // definitely should NOT include group 2 or 3 Set goodResults = Sets.newHashSet("CANINE.PET.0:beagle", "CANINE.PET.1:basset", "CANINE.PET.12:bernese", "CANINE.PET.13:shepherd", - "CANINE.WILD.1:coyote"); + "CANINE.WILD.1:coyote", "CANINE.PET.50:doberman"); runTestQuery(queryString, format.parse("20091231"), format.parse("20150101"), extraParameters, goodResults); } @@ -367,7 +367,7 @@ public void testMultipleIncludeGroupingFalse() throws Exception { String queryString = "filter:getAllMatches(CANINE,'.*e.*')"; Set goodResults = Sets.newHashSet("BIRD:parakeet", "BIRD:canary", "CANINE:beagle", "CANINE:coyote", "CANINE:basset", "CANINE:shepherd", - "CANINE:bernese"); + "CANINE:bernese", "CANINE:doberman"); runTestQuery(queryString, format.parse("20091231"), format.parse("20150101"), extraParameters, goodResults); } @@ -385,7 +385,8 @@ public void testLimiting() throws Exception { Set goodResults = Sets.newHashSet("BIRD:parakeet", "CANINE:beagle", "CANINE:coyote", "CANINE:basset", "CANINE:shepherd", "CANINE:bernese", "FISH:tuna", "CAT:tabby", "CAT:tom", "FISH:swordtail", "FISH:angelfish", "CAT:siamese", "FISH:goldfish", "CAT:himalayan", "CAT:leopard", "CAT:cougar", "CAT:calico", "CAT:tiger", "FISH:tetra", "FISH:mackerel", "FISH:shark", "CAT:puma", "CAT:ragdoll", "FISH:beta", - "FISH:guppy", "FISH:salmon", "REPTILE:snake", "DOG:coyote", "SIZE:20,12.5", "SIZE:90,26.5"); + + "FISH:guppy", "FISH:salmon", "REPTILE:snake", "DOG:coyote", "SIZE:20,12.5", "SIZE:90,26.5", "CAT:sphynx", "CANINE:doberman"); runTestQuery(queryString, format.parse("20091231"), format.parse("20150101"), extraParameters, goodResults); } @@ -403,7 +404,8 @@ public void testLimitingToZero() throws Exception { Set goodResults = Sets.newHashSet("CANINE:beagle", "CANINE:coyote", "CANINE:basset", "CANINE:shepherd", "CANINE:bernese", "FISH:tuna", "CAT:tabby", "CAT:tom", "FISH:swordtail", "FISH:angelfish", "CAT:siamese", "FISH:goldfish", "CAT:himalayan", "CAT:leopard", "CAT:cougar", "CAT:calico", "CAT:tiger", "FISH:tetra", "FISH:mackerel", "FISH:shark", "CAT:puma", "CAT:ragdoll", "FISH:beta", - "FISH:guppy", "FISH:salmon", "REPTILE:snake", "SIZE:20,12.5", "SIZE:90,26.5"); + + "FISH:guppy", "FISH:salmon", "REPTILE:snake", "SIZE:20,12.5", "SIZE:90,26.5", "CAT:sphynx", "CANINE:doberman"); runTestQuery(queryString, format.parse("20091231"), format.parse("20150101"), extraParameters, goodResults); } @@ -423,8 +425,9 @@ public void testLimitingWithGrouping() throws Exception { "BIRD.WILD.1:hawk", "FISH.PET.12:swordtail", "CAT.PET.13:ragdoll", "FISH.WILD.0:shark", "CAT.PET.1:calico", "FISH.PET.0:beta", "CAT.WILD.1:tiger", "FISH.PET.2:angelfish", "CAT.PET.0:tabby", "FISH.WILD.2:mackerel", "FISH.PET.13:tetra", "FISH.PET.1:goldfish", "FISH.PET.3:guppy", "CAT.PET.12:himalayan", "FISH.WILD.1:tuna", "FISH.WILD.3:salmon", "CAT.WILD.3:puma", "CAT.WILD.2:leopard", + "CAT.PET.3:siamese", "CAT.WILD.0:cougar", "CAT.PET.2:tom", "REPTILE.PET.1:snake", "DOG.WILD.1:coyote", "SIZE.CANINE.3:20,12.5", - "SIZE.CANINE.WILD.1:90,26.5"); + "SIZE.CANINE.WILD.1:90,26.5", "CAT.PET.50:sphynx", "CANINE.PET.50:doberman"); runTestQuery(queryString, format.parse("20091231"), format.parse("20150101"), extraParameters, goodResults); } @@ -444,8 +447,9 @@ public void testLimitingToZeroWithGrouping() throws Exception { "BIRD.WILD.1:hawk", "FISH.PET.12:swordtail", "CAT.PET.13:ragdoll", "FISH.WILD.0:shark", "CAT.PET.1:calico", "FISH.PET.0:beta", "CAT.WILD.1:tiger", "FISH.PET.2:angelfish", "CAT.PET.0:tabby", "FISH.WILD.2:mackerel", "FISH.PET.13:tetra", "FISH.PET.1:goldfish", "FISH.PET.3:guppy", "CAT.PET.12:himalayan", "FISH.WILD.1:tuna", "FISH.WILD.3:salmon", "CAT.WILD.3:puma", "CAT.WILD.2:leopard", + "CAT.PET.3:siamese", "CAT.WILD.0:cougar", "CAT.PET.2:tom", "REPTILE.PET.1:snake", "DOG.WILD.1:coyote", "SIZE.CANINE.3:20,12.5", - "SIZE.CANINE.WILD.1:90,26.5"); + "SIZE.CANINE.WILD.1:90,26.5", "CAT.PET.50:sphynx", "CANINE.PET.50:doberman"); runTestQuery(queryString, format.parse("20091231"), format.parse("20150101"), extraParameters, goodResults); } diff --git a/warehouse/query-core/src/test/java/datawave/query/util/CommonalityTokenTestDataIngest.java b/warehouse/query-core/src/test/java/datawave/query/util/CommonalityTokenTestDataIngest.java index 14397276cd6..6d6b3d03a6c 100644 --- a/warehouse/query-core/src/test/java/datawave/query/util/CommonalityTokenTestDataIngest.java +++ b/warehouse/query-core/src/test/java/datawave/query/util/CommonalityTokenTestDataIngest.java @@ -20,6 +20,7 @@ import datawave.data.type.Type; import datawave.ingest.protobuf.Uid; import datawave.query.QueryTestTableHelper; +import datawave.util.CompositeTimestamp; import datawave.util.TableName; /** @@ -49,6 +50,13 @@ public static void writeItAll(AccumuloClient client, WhatKindaRange range) throw Mutation mutation = null; String myUID = UID.builder().newId("MyUid".getBytes(), (Date) null).toString(); + String myUID2 = UID.builder().newId("MyUid2".getBytes(), (Date) null).toString(); + String myUID3 = UID.builder().newId("MyUid3".getBytes(), (Date) null).toString(); + + long ageOffTimestamp = 1699041441288l; + long timeStamp2 = CompositeTimestamp.getCompositeTimeStamp(timeStamp, ageOffTimestamp); + + long timeStamp3 = CompositeTimestamp.getCompositeTimeStamp(ageOffTimestamp, ageOffTimestamp); try { // write the shard table : @@ -113,6 +121,14 @@ public static void writeItAll(AccumuloClient client, WhatKindaRange range) throw mutation.put(datatype + "\u0000" + myUID, "DOG.WILD.1" + "\u0000" + "coyote", columnVisibility, timeStamp, emptyValue); mutation.put(datatype + "\u0000" + myUID, "REPTILE.PET.1" + "\u0000" + "snake", columnVisibility, timeStamp, emptyValue); + // add an event with a composite timestamp that should return + mutation.put(datatype + "\u0000" + myUID2, "CAT.PET.50" + "\u0000" + "sphynx", columnVisibility, timeStamp2, emptyValue); + mutation.put(datatype + "\u0000" + myUID2, "CANINE.PET.50" + "\u0000" + "doberman", columnVisibility, timeStamp2, emptyValue); + + // add an event with a composite timestamp that should NOT return + mutation.put(datatype + "\u0000" + myUID3, "CAT.PET.60" + "\u0000" + "manx", columnVisibility, timeStamp3, emptyValue); + mutation.put(datatype + "\u0000" + myUID3, "CANINE.PET.60" + "\u0000" + "doberman", columnVisibility, timeStamp3, emptyValue); + bw.addMutation(mutation); } finally { @@ -129,6 +145,10 @@ public static void writeItAll(AccumuloClient client, WhatKindaRange range) throw mutation = new Mutation(lcNoDiacriticsType.normalize("tabby")); mutation.put("CAT", shard + "\u0000" + datatype, columnVisibility, timeStamp, range == WhatKindaRange.SHARD ? getValueForNuthinAndYourHitsForFree() : getValueForBuilderFor(myUID)); + mutation.put("CAT", shard + "\u0000" + datatype, columnVisibility, timeStamp2, + range == WhatKindaRange.SHARD ? getValueForNuthinAndYourHitsForFree() : getValueForBuilderFor(myUID2)); + mutation.put("CAT", shard + "\u0000" + datatype, columnVisibility, timeStamp3, + range == WhatKindaRange.SHARD ? getValueForNuthinAndYourHitsForFree() : getValueForBuilderFor(myUID3)); bw.addMutation(mutation); mutation = new Mutation(lcNoDiacriticsType.normalize("calico")); mutation.put("CAT", shard + "\u0000" + datatype, columnVisibility, timeStamp, @@ -183,6 +203,10 @@ public static void writeItAll(AccumuloClient client, WhatKindaRange range) throw mutation = new Mutation(lcNoDiacriticsType.normalize("shepherd")); mutation.put("CANINE", shard + "\u0000" + datatype, columnVisibility, timeStamp, range == WhatKindaRange.SHARD ? getValueForNuthinAndYourHitsForFree() : getValueForBuilderFor(myUID)); + mutation.put("CANINE", shard + "\u0000" + datatype, columnVisibility, timeStamp2, + range == WhatKindaRange.SHARD ? getValueForNuthinAndYourHitsForFree() : getValueForBuilderFor(myUID2)); + mutation.put("CANINE", shard + "\u0000" + datatype, columnVisibility, timeStamp3, + range == WhatKindaRange.SHARD ? getValueForNuthinAndYourHitsForFree() : getValueForBuilderFor(myUID3)); bw.addMutation(mutation); mutation = new Mutation(lcNoDiacriticsType.normalize("wolf")); mutation.put("CANINE", shard + "\u0000" + datatype, columnVisibility, timeStamp, @@ -301,6 +325,8 @@ public static void writeItAll(AccumuloClient client, WhatKindaRange range) throw // cats mutation.put("fi\u0000" + "CAT", lcNoDiacriticsType.normalize("tabby") + "\u0000" + datatype + "\u0000" + myUID, columnVisibility, timeStamp, emptyValue); + mutation.put("fi\u0000" + "CAT", lcNoDiacriticsType.normalize("tabby") + "\u0000" + datatype + "\u0000" + myUID2, columnVisibility, timeStamp2, + emptyValue); mutation.put("fi\u0000" + "CAT", lcNoDiacriticsType.normalize("calico") + "\u0000" + datatype + "\u0000" + myUID, columnVisibility, timeStamp, emptyValue); mutation.put("fi\u0000" + "CAT", lcNoDiacriticsType.normalize("tom") + "\u0000" + datatype + "\u0000" + myUID, columnVisibility, timeStamp, @@ -328,6 +354,8 @@ public static void writeItAll(AccumuloClient client, WhatKindaRange range) throw emptyValue); mutation.put("fi\u0000" + "CANINE", lcNoDiacriticsType.normalize("shepherd") + "\u0000" + datatype + "\u0000" + myUID, columnVisibility, timeStamp, emptyValue); + mutation.put("fi\u0000" + "CANINE", lcNoDiacriticsType.normalize("shepherd") + "\u0000" + datatype + "\u0000" + myUID2, columnVisibility, + timeStamp2, emptyValue); mutation.put("fi\u0000" + "CANINE", lcNoDiacriticsType.normalize("wolf") + "\u0000" + datatype + "\u0000" + myUID, columnVisibility, timeStamp, emptyValue); mutation.put("fi\u0000" + "CANINE", lcNoDiacriticsType.normalize("coyote") + "\u0000" + datatype + "\u0000" + myUID, columnVisibility, timeStamp, From a5bc0215400a3201482da86db7873fe108465279 Mon Sep 17 00:00:00 2001 From: Ivan Bella <347158+ivakegg@users.noreply.github.com> Date: Mon, 2 Dec 2024 14:59:14 +0000 Subject: [PATCH 6/8] 7.11.1 --- common-test/pom.xml | 2 +- contrib/datawave-quickstart/docker/pom.xml | 2 +- core/cached-results/pom.xml | 2 +- core/common-util/pom.xml | 2 +- core/common/pom.xml | 2 +- core/connection-pool/pom.xml | 2 +- core/map-reduce/pom.xml | 2 +- core/modification/pom.xml | 2 +- core/pom.xml | 2 +- core/query/pom.xml | 2 +- core/utils/pom.xml | 2 +- docs/pom.xml | 2 +- microservices/pom.xml | 2 +- microservices/services/pom.xml | 2 +- microservices/starters/pom.xml | 2 +- pom.xml | 2 +- warehouse/accumulo-extensions/pom.xml | 2 +- warehouse/age-off-utils/pom.xml | 2 +- warehouse/age-off/pom.xml | 2 +- warehouse/assemble/datawave/pom.xml | 2 +- warehouse/assemble/pom.xml | 2 +- warehouse/assemble/webservice/pom.xml | 2 +- warehouse/common/pom.xml | 2 +- warehouse/core/pom.xml | 2 +- warehouse/data-dictionary-core/pom.xml | 2 +- warehouse/edge-dictionary-core/pom.xml | 2 +- warehouse/edge-model-configuration-core/pom.xml | 2 +- warehouse/index-stats/pom.xml | 2 +- warehouse/ingest-configuration/pom.xml | 2 +- warehouse/ingest-core/pom.xml | 2 +- warehouse/ingest-csv/pom.xml | 2 +- warehouse/ingest-json/pom.xml | 2 +- warehouse/ingest-nyctlc/pom.xml | 2 +- warehouse/ingest-scripts/pom.xml | 2 +- warehouse/ingest-ssdeep/pom.xml | 2 +- warehouse/ingest-wikipedia/pom.xml | 2 +- warehouse/metrics-core/pom.xml | 2 +- warehouse/ops-tools/config-compare/pom.xml | 2 +- warehouse/ops-tools/index-validation/pom.xml | 2 +- warehouse/ops-tools/pom.xml | 2 +- warehouse/pom.xml | 2 +- warehouse/query-core/pom.xml | 2 +- warehouse/regression-testing/pom.xml | 2 +- warehouse/ssdeep-common/pom.xml | 2 +- web-services/accumulo/pom.xml | 2 +- web-services/atom/pom.xml | 2 +- web-services/cached-results/pom.xml | 2 +- web-services/client/pom.xml | 2 +- web-services/common-util/pom.xml | 2 +- web-services/common/pom.xml | 2 +- web-services/deploy/application/pom.xml | 2 +- web-services/deploy/configuration/pom.xml | 2 +- web-services/deploy/docs/pom.xml | 2 +- web-services/deploy/pom.xml | 2 +- web-services/deploy/spring-framework-integration/pom.xml | 2 +- web-services/dictionary/pom.xml | 2 +- web-services/examples/client-login/pom.xml | 2 +- web-services/examples/http-client/pom.xml | 2 +- web-services/examples/jms-client/pom.xml | 2 +- web-services/examples/pom.xml | 2 +- web-services/examples/query-war/pom.xml | 2 +- web-services/map-reduce-embedded/pom.xml | 2 +- web-services/map-reduce-status/pom.xml | 2 +- web-services/map-reduce/pom.xml | 2 +- web-services/metrics/pom.xml | 2 +- web-services/model/pom.xml | 2 +- web-services/modification/pom.xml | 2 +- web-services/pom.xml | 2 +- web-services/query-websocket/pom.xml | 2 +- web-services/query/pom.xml | 2 +- web-services/rest-api/pom.xml | 2 +- web-services/security/pom.xml | 2 +- web-services/web-root/pom.xml | 2 +- 73 files changed, 73 insertions(+), 73 deletions(-) diff --git a/common-test/pom.xml b/common-test/pom.xml index 5cc4654c920..3ed59ce036c 100644 --- a/common-test/pom.xml +++ b/common-test/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-common-test ${project.artifactId} diff --git a/contrib/datawave-quickstart/docker/pom.xml b/contrib/datawave-quickstart/docker/pom.xml index 57d8b93f4bd..53d5913de5d 100644 --- a/contrib/datawave-quickstart/docker/pom.xml +++ b/contrib/datawave-quickstart/docker/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-parent - 7.11.1-SNAPSHOT + 7.11.1 ../../../pom.xml quickstart diff --git a/core/cached-results/pom.xml b/core/cached-results/pom.xml index 24b67694aa1..8858301ed39 100644 --- a/core/cached-results/pom.xml +++ b/core/cached-results/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-core-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-core-cached-results ${project.artifactId} diff --git a/core/common-util/pom.xml b/core/common-util/pom.xml index c2e944a59e6..8996f79cbcc 100644 --- a/core/common-util/pom.xml +++ b/core/common-util/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-core-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-core-common-util ${project.artifactId} diff --git a/core/common/pom.xml b/core/common/pom.xml index f1226aa84ac..2fd12eeb62e 100644 --- a/core/common/pom.xml +++ b/core/common/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-core-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-core-common ${project.artifactId} diff --git a/core/connection-pool/pom.xml b/core/connection-pool/pom.xml index fc0d1467136..f5825ed07b0 100644 --- a/core/connection-pool/pom.xml +++ b/core/connection-pool/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-core-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-core-connection-pool ${project.artifactId} diff --git a/core/map-reduce/pom.xml b/core/map-reduce/pom.xml index b192a33f1e5..d9bc9a6faf1 100644 --- a/core/map-reduce/pom.xml +++ b/core/map-reduce/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-core-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-core-map-reduce ${project.artifactId} diff --git a/core/modification/pom.xml b/core/modification/pom.xml index 44bd1a240cf..5b2909de823 100644 --- a/core/modification/pom.xml +++ b/core/modification/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-core-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-core-modification ${project.artifactId} diff --git a/core/pom.xml b/core/pom.xml index 9da92963851..bf3de9622a8 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-parent - 7.11.1-SNAPSHOT + 7.11.1 gov.nsa.datawave.core datawave-core-parent diff --git a/core/query/pom.xml b/core/query/pom.xml index eb33b299125..232bd54e237 100644 --- a/core/query/pom.xml +++ b/core/query/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-core-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-core-query ${project.artifactId} diff --git a/core/utils/pom.xml b/core/utils/pom.xml index d84924485b1..75c9f0a4e30 100644 --- a/core/utils/pom.xml +++ b/core/utils/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-core-parent - 7.11.1-SNAPSHOT + 7.11.1 gov.nsa.datawave.core datawave-utils-parent diff --git a/docs/pom.xml b/docs/pom.xml index 6f682c683c9..09c1dbfd3c9 100644 --- a/docs/pom.xml +++ b/docs/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-docs diff --git a/microservices/pom.xml b/microservices/pom.xml index 711c690c941..12ceabaf213 100644 --- a/microservices/pom.xml +++ b/microservices/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-parent - 7.11.1-SNAPSHOT + 7.11.1 gov.nsa.datawave.microservice datawave-microservice-build-parent diff --git a/microservices/services/pom.xml b/microservices/services/pom.xml index ce5e6f22874..4f36329f16a 100644 --- a/microservices/services/pom.xml +++ b/microservices/services/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-build-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-microservice-service-build-parent pom diff --git a/microservices/starters/pom.xml b/microservices/starters/pom.xml index 314c5eb26ce..13c5082dd06 100644 --- a/microservices/starters/pom.xml +++ b/microservices/starters/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-build-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-microservice-starter-build-parent pom diff --git a/pom.xml b/pom.xml index 57ea487957a..165c1b55e86 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 gov.nsa.datawave datawave-parent - 7.11.1-SNAPSHOT + 7.11.1 pom DataWave DataWave is a Java-based ingest and query framework that leverages Apache Accumulo to provide fast, secure access to your data. diff --git a/warehouse/accumulo-extensions/pom.xml b/warehouse/accumulo-extensions/pom.xml index 91fc22e54ae..b259171e6b1 100644 --- a/warehouse/accumulo-extensions/pom.xml +++ b/warehouse/accumulo-extensions/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-accumulo-extensions ${project.artifactId} diff --git a/warehouse/age-off-utils/pom.xml b/warehouse/age-off-utils/pom.xml index 9385765d00a..685593020ca 100644 --- a/warehouse/age-off-utils/pom.xml +++ b/warehouse/age-off-utils/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-age-off-utils ${project.artifactId} diff --git a/warehouse/age-off/pom.xml b/warehouse/age-off/pom.xml index 7d4826d2f12..7670fc97dc7 100644 --- a/warehouse/age-off/pom.xml +++ b/warehouse/age-off/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-age-off ${project.artifactId} diff --git a/warehouse/assemble/datawave/pom.xml b/warehouse/assemble/datawave/pom.xml index 9a416ee0c57..321a24851bd 100644 --- a/warehouse/assemble/datawave/pom.xml +++ b/warehouse/assemble/datawave/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave assemble-parent - 7.11.1-SNAPSHOT + 7.11.1 assemble-datawave jar diff --git a/warehouse/assemble/pom.xml b/warehouse/assemble/pom.xml index fe3e60f7f9c..cdadb82fe03 100644 --- a/warehouse/assemble/pom.xml +++ b/warehouse/assemble/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1-SNAPSHOT + 7.11.1 assemble-parent pom diff --git a/warehouse/assemble/webservice/pom.xml b/warehouse/assemble/webservice/pom.xml index 7bacf653df7..cbaf86158dc 100644 --- a/warehouse/assemble/webservice/pom.xml +++ b/warehouse/assemble/webservice/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave assemble-parent - 7.11.1-SNAPSHOT + 7.11.1 assemble-webservice ${project.artifactId} diff --git a/warehouse/common/pom.xml b/warehouse/common/pom.xml index 51e658f3ead..915051c30ac 100644 --- a/warehouse/common/pom.xml +++ b/warehouse/common/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-common ${project.artifactId} diff --git a/warehouse/core/pom.xml b/warehouse/core/pom.xml index 842a53ff368..cff9220d740 100644 --- a/warehouse/core/pom.xml +++ b/warehouse/core/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-core jar diff --git a/warehouse/data-dictionary-core/pom.xml b/warehouse/data-dictionary-core/pom.xml index fab616ef6a0..6b8d480da50 100644 --- a/warehouse/data-dictionary-core/pom.xml +++ b/warehouse/data-dictionary-core/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-data-dictionary-core jar diff --git a/warehouse/edge-dictionary-core/pom.xml b/warehouse/edge-dictionary-core/pom.xml index be286d72b6c..7f2b8751d3f 100644 --- a/warehouse/edge-dictionary-core/pom.xml +++ b/warehouse/edge-dictionary-core/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-edge-dictionary-core jar diff --git a/warehouse/edge-model-configuration-core/pom.xml b/warehouse/edge-model-configuration-core/pom.xml index b0d1d5eaf97..ad2a4dd2095 100644 --- a/warehouse/edge-model-configuration-core/pom.xml +++ b/warehouse/edge-model-configuration-core/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-edge-model-configuration-core jar diff --git a/warehouse/index-stats/pom.xml b/warehouse/index-stats/pom.xml index bc49ba5992b..d696d446765 100644 --- a/warehouse/index-stats/pom.xml +++ b/warehouse/index-stats/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-index-stats jar diff --git a/warehouse/ingest-configuration/pom.xml b/warehouse/ingest-configuration/pom.xml index 49849b275b4..63b28044f0c 100644 --- a/warehouse/ingest-configuration/pom.xml +++ b/warehouse/ingest-configuration/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-ingest-configuration diff --git a/warehouse/ingest-core/pom.xml b/warehouse/ingest-core/pom.xml index 6a9d9685b38..cfbed53bb16 100644 --- a/warehouse/ingest-core/pom.xml +++ b/warehouse/ingest-core/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-ingest-core jar diff --git a/warehouse/ingest-csv/pom.xml b/warehouse/ingest-csv/pom.xml index 67cd593675c..c5741b46586 100644 --- a/warehouse/ingest-csv/pom.xml +++ b/warehouse/ingest-csv/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-ingest-csv jar diff --git a/warehouse/ingest-json/pom.xml b/warehouse/ingest-json/pom.xml index 0ede156a7c9..0564522a531 100644 --- a/warehouse/ingest-json/pom.xml +++ b/warehouse/ingest-json/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-ingest-json jar diff --git a/warehouse/ingest-nyctlc/pom.xml b/warehouse/ingest-nyctlc/pom.xml index 0bc7b0e56d0..8e3fce34451 100644 --- a/warehouse/ingest-nyctlc/pom.xml +++ b/warehouse/ingest-nyctlc/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-ingest-nyctlc jar diff --git a/warehouse/ingest-scripts/pom.xml b/warehouse/ingest-scripts/pom.xml index 8f4123dd605..2f5b41e7e1b 100644 --- a/warehouse/ingest-scripts/pom.xml +++ b/warehouse/ingest-scripts/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-ingest-scripts ${project.artifactId} diff --git a/warehouse/ingest-ssdeep/pom.xml b/warehouse/ingest-ssdeep/pom.xml index 1f839fdbdf2..8ec17697576 100644 --- a/warehouse/ingest-ssdeep/pom.xml +++ b/warehouse/ingest-ssdeep/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-ingest-ssdeep diff --git a/warehouse/ingest-wikipedia/pom.xml b/warehouse/ingest-wikipedia/pom.xml index 1b8bd1b9217..05b73fd18a3 100644 --- a/warehouse/ingest-wikipedia/pom.xml +++ b/warehouse/ingest-wikipedia/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-ingest-wikipedia jar diff --git a/warehouse/metrics-core/pom.xml b/warehouse/metrics-core/pom.xml index 785a585b318..ea612c16a69 100644 --- a/warehouse/metrics-core/pom.xml +++ b/warehouse/metrics-core/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-metrics-core jar diff --git a/warehouse/ops-tools/config-compare/pom.xml b/warehouse/ops-tools/config-compare/pom.xml index 1129f5028e1..0bc8ac71898 100644 --- a/warehouse/ops-tools/config-compare/pom.xml +++ b/warehouse/ops-tools/config-compare/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-ops-tools-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-ops-tools-config-compare diff --git a/warehouse/ops-tools/index-validation/pom.xml b/warehouse/ops-tools/index-validation/pom.xml index 6ee5778933f..89349ad962b 100644 --- a/warehouse/ops-tools/index-validation/pom.xml +++ b/warehouse/ops-tools/index-validation/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-ops-tools-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-ops-tools-index-validation jar diff --git a/warehouse/ops-tools/pom.xml b/warehouse/ops-tools/pom.xml index 3ec4d2bab05..d17a6265e64 100644 --- a/warehouse/ops-tools/pom.xml +++ b/warehouse/ops-tools/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-ops-tools-parent pom diff --git a/warehouse/pom.xml b/warehouse/pom.xml index 5d689611759..3b31272b98b 100644 --- a/warehouse/pom.xml +++ b/warehouse/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-warehouse-parent pom diff --git a/warehouse/query-core/pom.xml b/warehouse/query-core/pom.xml index 230c1cc0531..c91465e37a4 100644 --- a/warehouse/query-core/pom.xml +++ b/warehouse/query-core/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-query-core jar diff --git a/warehouse/regression-testing/pom.xml b/warehouse/regression-testing/pom.xml index f3218b60e90..2c73be44ddb 100644 --- a/warehouse/regression-testing/pom.xml +++ b/warehouse/regression-testing/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-regression-testing ${project.artifactId} diff --git a/warehouse/ssdeep-common/pom.xml b/warehouse/ssdeep-common/pom.xml index a0c2333cd3b..cd078050c26 100644 --- a/warehouse/ssdeep-common/pom.xml +++ b/warehouse/ssdeep-common/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-ssdeep-common diff --git a/web-services/accumulo/pom.xml b/web-services/accumulo/pom.xml index 9b7102095e7..865a8b149b3 100644 --- a/web-services/accumulo/pom.xml +++ b/web-services/accumulo/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-ws-accumulo ejb diff --git a/web-services/atom/pom.xml b/web-services/atom/pom.xml index 1e1dac82c0d..0a78d82e172 100644 --- a/web-services/atom/pom.xml +++ b/web-services/atom/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-ws-atom ejb diff --git a/web-services/cached-results/pom.xml b/web-services/cached-results/pom.xml index 9741206d72b..a9fa4a43c51 100644 --- a/web-services/cached-results/pom.xml +++ b/web-services/cached-results/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-ws-cached-results ejb diff --git a/web-services/client/pom.xml b/web-services/client/pom.xml index 6b7e1f6a21f..191acdb4c91 100644 --- a/web-services/client/pom.xml +++ b/web-services/client/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-ws-client jar diff --git a/web-services/common-util/pom.xml b/web-services/common-util/pom.xml index 2009e5e173c..9c9c80db088 100644 --- a/web-services/common-util/pom.xml +++ b/web-services/common-util/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-ws-common-util jar diff --git a/web-services/common/pom.xml b/web-services/common/pom.xml index e896f6baaa4..3d6c531bf4c 100644 --- a/web-services/common/pom.xml +++ b/web-services/common/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-ws-common ejb diff --git a/web-services/deploy/application/pom.xml b/web-services/deploy/application/pom.xml index b7513b9213e..3dc80456d46 100644 --- a/web-services/deploy/application/pom.xml +++ b/web-services/deploy/application/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-deploy-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-ws-deploy-application ear diff --git a/web-services/deploy/configuration/pom.xml b/web-services/deploy/configuration/pom.xml index 663fab31f2a..5ed3c399b20 100644 --- a/web-services/deploy/configuration/pom.xml +++ b/web-services/deploy/configuration/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-deploy-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-ws-deploy-configuration jar diff --git a/web-services/deploy/docs/pom.xml b/web-services/deploy/docs/pom.xml index b99df6e17b4..da6e8cef869 100644 --- a/web-services/deploy/docs/pom.xml +++ b/web-services/deploy/docs/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-deploy-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-ws-deploy-docs war diff --git a/web-services/deploy/pom.xml b/web-services/deploy/pom.xml index 866a5e7369e..50039cde380 100644 --- a/web-services/deploy/pom.xml +++ b/web-services/deploy/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.11.1-SNAPSHOT + 7.11.1 gov.nsa.datawave.webservices datawave-ws-deploy-parent diff --git a/web-services/deploy/spring-framework-integration/pom.xml b/web-services/deploy/spring-framework-integration/pom.xml index d2250375c21..58b1377c205 100644 --- a/web-services/deploy/spring-framework-integration/pom.xml +++ b/web-services/deploy/spring-framework-integration/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-deploy-parent - 7.11.1-SNAPSHOT + 7.11.1 spring-framework-integration ${project.artifactId} diff --git a/web-services/dictionary/pom.xml b/web-services/dictionary/pom.xml index d7af6102f9c..4c7f3e747ad 100644 --- a/web-services/dictionary/pom.xml +++ b/web-services/dictionary/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-ws-dictionary ejb diff --git a/web-services/examples/client-login/pom.xml b/web-services/examples/client-login/pom.xml index ac4405399f6..3327fd5bf54 100644 --- a/web-services/examples/client-login/pom.xml +++ b/web-services/examples/client-login/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-examples-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-ws-examples-client-login ejb diff --git a/web-services/examples/http-client/pom.xml b/web-services/examples/http-client/pom.xml index 0b8d1b87226..488d116e7e3 100644 --- a/web-services/examples/http-client/pom.xml +++ b/web-services/examples/http-client/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-examples-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-ws-examples-http-client jar diff --git a/web-services/examples/jms-client/pom.xml b/web-services/examples/jms-client/pom.xml index e8f84fd9ae1..b08473e07bc 100644 --- a/web-services/examples/jms-client/pom.xml +++ b/web-services/examples/jms-client/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-examples-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-ws-examples-jms-client jar diff --git a/web-services/examples/pom.xml b/web-services/examples/pom.xml index 9052c091248..da23c54ef99 100644 --- a/web-services/examples/pom.xml +++ b/web-services/examples/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-ws-examples-parent pom diff --git a/web-services/examples/query-war/pom.xml b/web-services/examples/query-war/pom.xml index bc2c9af5303..3bc0de7a060 100644 --- a/web-services/examples/query-war/pom.xml +++ b/web-services/examples/query-war/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-examples-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-ws-examples-query-war war diff --git a/web-services/map-reduce-embedded/pom.xml b/web-services/map-reduce-embedded/pom.xml index 50631000167..a29f2840fe9 100644 --- a/web-services/map-reduce-embedded/pom.xml +++ b/web-services/map-reduce-embedded/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-ws-map-reduce-embedded jar diff --git a/web-services/map-reduce-status/pom.xml b/web-services/map-reduce-status/pom.xml index 45a6d366005..fb85165921a 100644 --- a/web-services/map-reduce-status/pom.xml +++ b/web-services/map-reduce-status/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-ws-map-reduce-status ejb diff --git a/web-services/map-reduce/pom.xml b/web-services/map-reduce/pom.xml index 3ab384d2a90..7e59e4170fc 100644 --- a/web-services/map-reduce/pom.xml +++ b/web-services/map-reduce/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-ws-map-reduce ejb diff --git a/web-services/metrics/pom.xml b/web-services/metrics/pom.xml index 2f3b2fdc71c..c214a61c50e 100644 --- a/web-services/metrics/pom.xml +++ b/web-services/metrics/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-ws-metrics ejb diff --git a/web-services/model/pom.xml b/web-services/model/pom.xml index c871f902597..99cba954779 100644 --- a/web-services/model/pom.xml +++ b/web-services/model/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-ws-model ejb diff --git a/web-services/modification/pom.xml b/web-services/modification/pom.xml index 24d87bd4676..78e16954192 100644 --- a/web-services/modification/pom.xml +++ b/web-services/modification/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-ws-modification ejb diff --git a/web-services/pom.xml b/web-services/pom.xml index 37ea551894c..914d0ad607b 100644 --- a/web-services/pom.xml +++ b/web-services/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-parent - 7.11.1-SNAPSHOT + 7.11.1 gov.nsa.datawave.webservices datawave-ws-parent diff --git a/web-services/query-websocket/pom.xml b/web-services/query-websocket/pom.xml index b7aaac54470..50b5015268f 100644 --- a/web-services/query-websocket/pom.xml +++ b/web-services/query-websocket/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-ws-query-websocket war diff --git a/web-services/query/pom.xml b/web-services/query/pom.xml index 1736b187c55..17f008dd5d7 100644 --- a/web-services/query/pom.xml +++ b/web-services/query/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-ws-query ejb diff --git a/web-services/rest-api/pom.xml b/web-services/rest-api/pom.xml index f4494da87c4..0caaed70bec 100644 --- a/web-services/rest-api/pom.xml +++ b/web-services/rest-api/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-ws-rest-api war diff --git a/web-services/security/pom.xml b/web-services/security/pom.xml index b817f78a969..512129bd9b2 100644 --- a/web-services/security/pom.xml +++ b/web-services/security/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-ws-security ejb diff --git a/web-services/web-root/pom.xml b/web-services/web-root/pom.xml index 8f5a474707b..4cc23b1a4cb 100644 --- a/web-services/web-root/pom.xml +++ b/web-services/web-root/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.11.1-SNAPSHOT + 7.11.1 datawave-ws-web-root war From 999ab7968145b6c50ccfed309b55f228655f3837 Mon Sep 17 00:00:00 2001 From: Ivan Bella <347158+ivakegg@users.noreply.github.com> Date: Mon, 2 Dec 2024 15:15:15 +0000 Subject: [PATCH 7/8] 7.11.2-SNAPSHOT --- common-test/pom.xml | 2 +- contrib/datawave-quickstart/docker/pom.xml | 2 +- core/cached-results/pom.xml | 2 +- core/common-util/pom.xml | 2 +- core/common/pom.xml | 2 +- core/connection-pool/pom.xml | 2 +- core/map-reduce/pom.xml | 2 +- core/modification/pom.xml | 2 +- core/pom.xml | 2 +- core/query/pom.xml | 2 +- core/utils/pom.xml | 2 +- docs/pom.xml | 2 +- microservices/pom.xml | 2 +- microservices/services/pom.xml | 2 +- microservices/starters/pom.xml | 2 +- pom.xml | 2 +- warehouse/accumulo-extensions/pom.xml | 2 +- warehouse/age-off-utils/pom.xml | 2 +- warehouse/age-off/pom.xml | 2 +- warehouse/assemble/datawave/pom.xml | 2 +- warehouse/assemble/pom.xml | 2 +- warehouse/assemble/webservice/pom.xml | 2 +- warehouse/common/pom.xml | 2 +- warehouse/core/pom.xml | 2 +- warehouse/data-dictionary-core/pom.xml | 2 +- warehouse/edge-dictionary-core/pom.xml | 2 +- warehouse/edge-model-configuration-core/pom.xml | 2 +- warehouse/index-stats/pom.xml | 2 +- warehouse/ingest-configuration/pom.xml | 2 +- warehouse/ingest-core/pom.xml | 2 +- warehouse/ingest-csv/pom.xml | 2 +- warehouse/ingest-json/pom.xml | 2 +- warehouse/ingest-nyctlc/pom.xml | 2 +- warehouse/ingest-scripts/pom.xml | 2 +- warehouse/ingest-ssdeep/pom.xml | 2 +- warehouse/ingest-wikipedia/pom.xml | 2 +- warehouse/metrics-core/pom.xml | 2 +- warehouse/ops-tools/config-compare/pom.xml | 2 +- warehouse/ops-tools/index-validation/pom.xml | 2 +- warehouse/ops-tools/pom.xml | 2 +- warehouse/pom.xml | 2 +- warehouse/query-core/pom.xml | 2 +- warehouse/regression-testing/pom.xml | 2 +- warehouse/ssdeep-common/pom.xml | 2 +- web-services/accumulo/pom.xml | 2 +- web-services/atom/pom.xml | 2 +- web-services/cached-results/pom.xml | 2 +- web-services/client/pom.xml | 2 +- web-services/common-util/pom.xml | 2 +- web-services/common/pom.xml | 2 +- web-services/deploy/application/pom.xml | 2 +- web-services/deploy/configuration/pom.xml | 2 +- web-services/deploy/docs/pom.xml | 2 +- web-services/deploy/pom.xml | 2 +- web-services/deploy/spring-framework-integration/pom.xml | 2 +- web-services/dictionary/pom.xml | 2 +- web-services/examples/client-login/pom.xml | 2 +- web-services/examples/http-client/pom.xml | 2 +- web-services/examples/jms-client/pom.xml | 2 +- web-services/examples/pom.xml | 2 +- web-services/examples/query-war/pom.xml | 2 +- web-services/map-reduce-embedded/pom.xml | 2 +- web-services/map-reduce-status/pom.xml | 2 +- web-services/map-reduce/pom.xml | 2 +- web-services/metrics/pom.xml | 2 +- web-services/model/pom.xml | 2 +- web-services/modification/pom.xml | 2 +- web-services/pom.xml | 2 +- web-services/query-websocket/pom.xml | 2 +- web-services/query/pom.xml | 2 +- web-services/rest-api/pom.xml | 2 +- web-services/security/pom.xml | 2 +- web-services/web-root/pom.xml | 2 +- 73 files changed, 73 insertions(+), 73 deletions(-) diff --git a/common-test/pom.xml b/common-test/pom.xml index 3ed59ce036c..62c3f68bc25 100644 --- a/common-test/pom.xml +++ b/common-test/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-common-test ${project.artifactId} diff --git a/contrib/datawave-quickstart/docker/pom.xml b/contrib/datawave-quickstart/docker/pom.xml index 53d5913de5d..a33b375938a 100644 --- a/contrib/datawave-quickstart/docker/pom.xml +++ b/contrib/datawave-quickstart/docker/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-parent - 7.11.1 + 7.11.2-SNAPSHOT ../../../pom.xml quickstart diff --git a/core/cached-results/pom.xml b/core/cached-results/pom.xml index 8858301ed39..cb876fc88c3 100644 --- a/core/cached-results/pom.xml +++ b/core/cached-results/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-core-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-core-cached-results ${project.artifactId} diff --git a/core/common-util/pom.xml b/core/common-util/pom.xml index 8996f79cbcc..f3cabc6fdec 100644 --- a/core/common-util/pom.xml +++ b/core/common-util/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-core-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-core-common-util ${project.artifactId} diff --git a/core/common/pom.xml b/core/common/pom.xml index 2fd12eeb62e..a52ae7f56ea 100644 --- a/core/common/pom.xml +++ b/core/common/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-core-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-core-common ${project.artifactId} diff --git a/core/connection-pool/pom.xml b/core/connection-pool/pom.xml index f5825ed07b0..0ffcd508fb4 100644 --- a/core/connection-pool/pom.xml +++ b/core/connection-pool/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-core-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-core-connection-pool ${project.artifactId} diff --git a/core/map-reduce/pom.xml b/core/map-reduce/pom.xml index d9bc9a6faf1..54f84907ccc 100644 --- a/core/map-reduce/pom.xml +++ b/core/map-reduce/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-core-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-core-map-reduce ${project.artifactId} diff --git a/core/modification/pom.xml b/core/modification/pom.xml index 5b2909de823..63385b60dd9 100644 --- a/core/modification/pom.xml +++ b/core/modification/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-core-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-core-modification ${project.artifactId} diff --git a/core/pom.xml b/core/pom.xml index bf3de9622a8..ef71a697a28 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-parent - 7.11.1 + 7.11.2-SNAPSHOT gov.nsa.datawave.core datawave-core-parent diff --git a/core/query/pom.xml b/core/query/pom.xml index 232bd54e237..8dbd1ba1581 100644 --- a/core/query/pom.xml +++ b/core/query/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-core-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-core-query ${project.artifactId} diff --git a/core/utils/pom.xml b/core/utils/pom.xml index 75c9f0a4e30..46c150659dc 100644 --- a/core/utils/pom.xml +++ b/core/utils/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-core-parent - 7.11.1 + 7.11.2-SNAPSHOT gov.nsa.datawave.core datawave-utils-parent diff --git a/docs/pom.xml b/docs/pom.xml index 09c1dbfd3c9..1ced48ae3a3 100644 --- a/docs/pom.xml +++ b/docs/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-docs diff --git a/microservices/pom.xml b/microservices/pom.xml index 12ceabaf213..739cd276f17 100644 --- a/microservices/pom.xml +++ b/microservices/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-parent - 7.11.1 + 7.11.2-SNAPSHOT gov.nsa.datawave.microservice datawave-microservice-build-parent diff --git a/microservices/services/pom.xml b/microservices/services/pom.xml index 4f36329f16a..2bff7a8c634 100644 --- a/microservices/services/pom.xml +++ b/microservices/services/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-build-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-microservice-service-build-parent pom diff --git a/microservices/starters/pom.xml b/microservices/starters/pom.xml index 13c5082dd06..7f403091085 100644 --- a/microservices/starters/pom.xml +++ b/microservices/starters/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-build-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-microservice-starter-build-parent pom diff --git a/pom.xml b/pom.xml index 165c1b55e86..232a06d9da4 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 gov.nsa.datawave datawave-parent - 7.11.1 + 7.11.2-SNAPSHOT pom DataWave DataWave is a Java-based ingest and query framework that leverages Apache Accumulo to provide fast, secure access to your data. diff --git a/warehouse/accumulo-extensions/pom.xml b/warehouse/accumulo-extensions/pom.xml index b259171e6b1..cf919346f83 100644 --- a/warehouse/accumulo-extensions/pom.xml +++ b/warehouse/accumulo-extensions/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-accumulo-extensions ${project.artifactId} diff --git a/warehouse/age-off-utils/pom.xml b/warehouse/age-off-utils/pom.xml index 685593020ca..9c88ea07a3b 100644 --- a/warehouse/age-off-utils/pom.xml +++ b/warehouse/age-off-utils/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-age-off-utils ${project.artifactId} diff --git a/warehouse/age-off/pom.xml b/warehouse/age-off/pom.xml index 7670fc97dc7..9f0536813dd 100644 --- a/warehouse/age-off/pom.xml +++ b/warehouse/age-off/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-age-off ${project.artifactId} diff --git a/warehouse/assemble/datawave/pom.xml b/warehouse/assemble/datawave/pom.xml index 321a24851bd..980166677b4 100644 --- a/warehouse/assemble/datawave/pom.xml +++ b/warehouse/assemble/datawave/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave assemble-parent - 7.11.1 + 7.11.2-SNAPSHOT assemble-datawave jar diff --git a/warehouse/assemble/pom.xml b/warehouse/assemble/pom.xml index cdadb82fe03..1fa005da797 100644 --- a/warehouse/assemble/pom.xml +++ b/warehouse/assemble/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1 + 7.11.2-SNAPSHOT assemble-parent pom diff --git a/warehouse/assemble/webservice/pom.xml b/warehouse/assemble/webservice/pom.xml index cbaf86158dc..5b5f5ceba62 100644 --- a/warehouse/assemble/webservice/pom.xml +++ b/warehouse/assemble/webservice/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave assemble-parent - 7.11.1 + 7.11.2-SNAPSHOT assemble-webservice ${project.artifactId} diff --git a/warehouse/common/pom.xml b/warehouse/common/pom.xml index 915051c30ac..55ccf0682e9 100644 --- a/warehouse/common/pom.xml +++ b/warehouse/common/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-common ${project.artifactId} diff --git a/warehouse/core/pom.xml b/warehouse/core/pom.xml index cff9220d740..8feb098669e 100644 --- a/warehouse/core/pom.xml +++ b/warehouse/core/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-core jar diff --git a/warehouse/data-dictionary-core/pom.xml b/warehouse/data-dictionary-core/pom.xml index 6b8d480da50..561a1080b23 100644 --- a/warehouse/data-dictionary-core/pom.xml +++ b/warehouse/data-dictionary-core/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-data-dictionary-core jar diff --git a/warehouse/edge-dictionary-core/pom.xml b/warehouse/edge-dictionary-core/pom.xml index 7f2b8751d3f..0b8ab81e407 100644 --- a/warehouse/edge-dictionary-core/pom.xml +++ b/warehouse/edge-dictionary-core/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-edge-dictionary-core jar diff --git a/warehouse/edge-model-configuration-core/pom.xml b/warehouse/edge-model-configuration-core/pom.xml index ad2a4dd2095..7eef80988a6 100644 --- a/warehouse/edge-model-configuration-core/pom.xml +++ b/warehouse/edge-model-configuration-core/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-edge-model-configuration-core jar diff --git a/warehouse/index-stats/pom.xml b/warehouse/index-stats/pom.xml index d696d446765..5053de8781e 100644 --- a/warehouse/index-stats/pom.xml +++ b/warehouse/index-stats/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-index-stats jar diff --git a/warehouse/ingest-configuration/pom.xml b/warehouse/ingest-configuration/pom.xml index 63b28044f0c..bc5eef06daa 100644 --- a/warehouse/ingest-configuration/pom.xml +++ b/warehouse/ingest-configuration/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-ingest-configuration diff --git a/warehouse/ingest-core/pom.xml b/warehouse/ingest-core/pom.xml index cfbed53bb16..f245572f4d5 100644 --- a/warehouse/ingest-core/pom.xml +++ b/warehouse/ingest-core/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-ingest-core jar diff --git a/warehouse/ingest-csv/pom.xml b/warehouse/ingest-csv/pom.xml index c5741b46586..4c9a49fb63f 100644 --- a/warehouse/ingest-csv/pom.xml +++ b/warehouse/ingest-csv/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-ingest-csv jar diff --git a/warehouse/ingest-json/pom.xml b/warehouse/ingest-json/pom.xml index 0564522a531..9843ec736f5 100644 --- a/warehouse/ingest-json/pom.xml +++ b/warehouse/ingest-json/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-ingest-json jar diff --git a/warehouse/ingest-nyctlc/pom.xml b/warehouse/ingest-nyctlc/pom.xml index 8e3fce34451..498f4edbba2 100644 --- a/warehouse/ingest-nyctlc/pom.xml +++ b/warehouse/ingest-nyctlc/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-ingest-nyctlc jar diff --git a/warehouse/ingest-scripts/pom.xml b/warehouse/ingest-scripts/pom.xml index 2f5b41e7e1b..9ca1955dc67 100644 --- a/warehouse/ingest-scripts/pom.xml +++ b/warehouse/ingest-scripts/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-ingest-scripts ${project.artifactId} diff --git a/warehouse/ingest-ssdeep/pom.xml b/warehouse/ingest-ssdeep/pom.xml index 8ec17697576..3ea05fd3cd8 100644 --- a/warehouse/ingest-ssdeep/pom.xml +++ b/warehouse/ingest-ssdeep/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-ingest-ssdeep diff --git a/warehouse/ingest-wikipedia/pom.xml b/warehouse/ingest-wikipedia/pom.xml index 05b73fd18a3..4c91ae53b0a 100644 --- a/warehouse/ingest-wikipedia/pom.xml +++ b/warehouse/ingest-wikipedia/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-ingest-wikipedia jar diff --git a/warehouse/metrics-core/pom.xml b/warehouse/metrics-core/pom.xml index ea612c16a69..93227245d7e 100644 --- a/warehouse/metrics-core/pom.xml +++ b/warehouse/metrics-core/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-metrics-core jar diff --git a/warehouse/ops-tools/config-compare/pom.xml b/warehouse/ops-tools/config-compare/pom.xml index 0bc8ac71898..5f4bfb6b209 100644 --- a/warehouse/ops-tools/config-compare/pom.xml +++ b/warehouse/ops-tools/config-compare/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-ops-tools-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-ops-tools-config-compare diff --git a/warehouse/ops-tools/index-validation/pom.xml b/warehouse/ops-tools/index-validation/pom.xml index 89349ad962b..4a33ad3be3d 100644 --- a/warehouse/ops-tools/index-validation/pom.xml +++ b/warehouse/ops-tools/index-validation/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-ops-tools-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-ops-tools-index-validation jar diff --git a/warehouse/ops-tools/pom.xml b/warehouse/ops-tools/pom.xml index d17a6265e64..082816d6849 100644 --- a/warehouse/ops-tools/pom.xml +++ b/warehouse/ops-tools/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-ops-tools-parent pom diff --git a/warehouse/pom.xml b/warehouse/pom.xml index 3b31272b98b..86289a4eb85 100644 --- a/warehouse/pom.xml +++ b/warehouse/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-warehouse-parent pom diff --git a/warehouse/query-core/pom.xml b/warehouse/query-core/pom.xml index c91465e37a4..d723a7cb70d 100644 --- a/warehouse/query-core/pom.xml +++ b/warehouse/query-core/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-query-core jar diff --git a/warehouse/regression-testing/pom.xml b/warehouse/regression-testing/pom.xml index 2c73be44ddb..79449c9d65a 100644 --- a/warehouse/regression-testing/pom.xml +++ b/warehouse/regression-testing/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-regression-testing ${project.artifactId} diff --git a/warehouse/ssdeep-common/pom.xml b/warehouse/ssdeep-common/pom.xml index cd078050c26..19f1b066079 100644 --- a/warehouse/ssdeep-common/pom.xml +++ b/warehouse/ssdeep-common/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-ssdeep-common diff --git a/web-services/accumulo/pom.xml b/web-services/accumulo/pom.xml index 865a8b149b3..7e5435d04bd 100644 --- a/web-services/accumulo/pom.xml +++ b/web-services/accumulo/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-ws-accumulo ejb diff --git a/web-services/atom/pom.xml b/web-services/atom/pom.xml index 0a78d82e172..edc53120945 100644 --- a/web-services/atom/pom.xml +++ b/web-services/atom/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-ws-atom ejb diff --git a/web-services/cached-results/pom.xml b/web-services/cached-results/pom.xml index a9fa4a43c51..589aa2bc527 100644 --- a/web-services/cached-results/pom.xml +++ b/web-services/cached-results/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-ws-cached-results ejb diff --git a/web-services/client/pom.xml b/web-services/client/pom.xml index 191acdb4c91..e568c551192 100644 --- a/web-services/client/pom.xml +++ b/web-services/client/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-ws-client jar diff --git a/web-services/common-util/pom.xml b/web-services/common-util/pom.xml index 9c9c80db088..4d530bd92c3 100644 --- a/web-services/common-util/pom.xml +++ b/web-services/common-util/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-ws-common-util jar diff --git a/web-services/common/pom.xml b/web-services/common/pom.xml index 3d6c531bf4c..e29fd33aeb0 100644 --- a/web-services/common/pom.xml +++ b/web-services/common/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-ws-common ejb diff --git a/web-services/deploy/application/pom.xml b/web-services/deploy/application/pom.xml index 3dc80456d46..b5c64c24dff 100644 --- a/web-services/deploy/application/pom.xml +++ b/web-services/deploy/application/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-deploy-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-ws-deploy-application ear diff --git a/web-services/deploy/configuration/pom.xml b/web-services/deploy/configuration/pom.xml index 5ed3c399b20..49aec8de082 100644 --- a/web-services/deploy/configuration/pom.xml +++ b/web-services/deploy/configuration/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-deploy-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-ws-deploy-configuration jar diff --git a/web-services/deploy/docs/pom.xml b/web-services/deploy/docs/pom.xml index da6e8cef869..4a5e99e8c74 100644 --- a/web-services/deploy/docs/pom.xml +++ b/web-services/deploy/docs/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-deploy-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-ws-deploy-docs war diff --git a/web-services/deploy/pom.xml b/web-services/deploy/pom.xml index 50039cde380..22213ced362 100644 --- a/web-services/deploy/pom.xml +++ b/web-services/deploy/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.11.1 + 7.11.2-SNAPSHOT gov.nsa.datawave.webservices datawave-ws-deploy-parent diff --git a/web-services/deploy/spring-framework-integration/pom.xml b/web-services/deploy/spring-framework-integration/pom.xml index 58b1377c205..7626b06878f 100644 --- a/web-services/deploy/spring-framework-integration/pom.xml +++ b/web-services/deploy/spring-framework-integration/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-deploy-parent - 7.11.1 + 7.11.2-SNAPSHOT spring-framework-integration ${project.artifactId} diff --git a/web-services/dictionary/pom.xml b/web-services/dictionary/pom.xml index 4c7f3e747ad..dc735f5a977 100644 --- a/web-services/dictionary/pom.xml +++ b/web-services/dictionary/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-ws-dictionary ejb diff --git a/web-services/examples/client-login/pom.xml b/web-services/examples/client-login/pom.xml index 3327fd5bf54..ebe43de3dc2 100644 --- a/web-services/examples/client-login/pom.xml +++ b/web-services/examples/client-login/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-examples-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-ws-examples-client-login ejb diff --git a/web-services/examples/http-client/pom.xml b/web-services/examples/http-client/pom.xml index 488d116e7e3..6a30bb2ee69 100644 --- a/web-services/examples/http-client/pom.xml +++ b/web-services/examples/http-client/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-examples-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-ws-examples-http-client jar diff --git a/web-services/examples/jms-client/pom.xml b/web-services/examples/jms-client/pom.xml index b08473e07bc..7e8f9a8f015 100644 --- a/web-services/examples/jms-client/pom.xml +++ b/web-services/examples/jms-client/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-examples-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-ws-examples-jms-client jar diff --git a/web-services/examples/pom.xml b/web-services/examples/pom.xml index da23c54ef99..44cf9ded60b 100644 --- a/web-services/examples/pom.xml +++ b/web-services/examples/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-ws-examples-parent pom diff --git a/web-services/examples/query-war/pom.xml b/web-services/examples/query-war/pom.xml index 3bc0de7a060..3eb7a0bc107 100644 --- a/web-services/examples/query-war/pom.xml +++ b/web-services/examples/query-war/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-examples-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-ws-examples-query-war war diff --git a/web-services/map-reduce-embedded/pom.xml b/web-services/map-reduce-embedded/pom.xml index a29f2840fe9..34bfe5476cb 100644 --- a/web-services/map-reduce-embedded/pom.xml +++ b/web-services/map-reduce-embedded/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-ws-map-reduce-embedded jar diff --git a/web-services/map-reduce-status/pom.xml b/web-services/map-reduce-status/pom.xml index fb85165921a..a2398b24e93 100644 --- a/web-services/map-reduce-status/pom.xml +++ b/web-services/map-reduce-status/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-ws-map-reduce-status ejb diff --git a/web-services/map-reduce/pom.xml b/web-services/map-reduce/pom.xml index 7e59e4170fc..b0ef004a262 100644 --- a/web-services/map-reduce/pom.xml +++ b/web-services/map-reduce/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-ws-map-reduce ejb diff --git a/web-services/metrics/pom.xml b/web-services/metrics/pom.xml index c214a61c50e..30327bea8d3 100644 --- a/web-services/metrics/pom.xml +++ b/web-services/metrics/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-ws-metrics ejb diff --git a/web-services/model/pom.xml b/web-services/model/pom.xml index 99cba954779..d674c8f1ff6 100644 --- a/web-services/model/pom.xml +++ b/web-services/model/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-ws-model ejb diff --git a/web-services/modification/pom.xml b/web-services/modification/pom.xml index 78e16954192..f12e91dcddd 100644 --- a/web-services/modification/pom.xml +++ b/web-services/modification/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-ws-modification ejb diff --git a/web-services/pom.xml b/web-services/pom.xml index 914d0ad607b..d85f30b607a 100644 --- a/web-services/pom.xml +++ b/web-services/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-parent - 7.11.1 + 7.11.2-SNAPSHOT gov.nsa.datawave.webservices datawave-ws-parent diff --git a/web-services/query-websocket/pom.xml b/web-services/query-websocket/pom.xml index 50b5015268f..6f5ff0c9131 100644 --- a/web-services/query-websocket/pom.xml +++ b/web-services/query-websocket/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-ws-query-websocket war diff --git a/web-services/query/pom.xml b/web-services/query/pom.xml index 17f008dd5d7..cfd2e00655d 100644 --- a/web-services/query/pom.xml +++ b/web-services/query/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-ws-query ejb diff --git a/web-services/rest-api/pom.xml b/web-services/rest-api/pom.xml index 0caaed70bec..044251e9fa0 100644 --- a/web-services/rest-api/pom.xml +++ b/web-services/rest-api/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-ws-rest-api war diff --git a/web-services/security/pom.xml b/web-services/security/pom.xml index 512129bd9b2..583d2b27823 100644 --- a/web-services/security/pom.xml +++ b/web-services/security/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-ws-security ejb diff --git a/web-services/web-root/pom.xml b/web-services/web-root/pom.xml index 4cc23b1a4cb..582ae8f1f85 100644 --- a/web-services/web-root/pom.xml +++ b/web-services/web-root/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 7.11.1 + 7.11.2-SNAPSHOT datawave-ws-web-root war From 3b31c164a56d058fb39e2d862a7e2009b3368011 Mon Sep 17 00:00:00 2001 From: Moriarty <22225248+apmoriarty@users.noreply.github.com> Date: Mon, 2 Dec 2024 17:25:12 +0000 Subject: [PATCH 8/8] Enable field filtering during document aggregation for standard event queries (#2650) --- .../src/main/java/datawave/query/iterator/QueryOptions.java | 4 ++++ .../test/resources/datawave/query/EventQueryLogicFactory.xml | 1 + .../src/test/resources/datawave/query/QueryLogicFactory.xml | 1 + 3 files changed, 6 insertions(+) diff --git a/warehouse/query-core/src/main/java/datawave/query/iterator/QueryOptions.java b/warehouse/query-core/src/main/java/datawave/query/iterator/QueryOptions.java index e0e58327a3e..f897293c857 100644 --- a/warehouse/query-core/src/main/java/datawave/query/iterator/QueryOptions.java +++ b/warehouse/query-core/src/main/java/datawave/query/iterator/QueryOptions.java @@ -793,6 +793,10 @@ public EventDataQueryFilter getFiEvaluationFilter() { } public EventDataQueryFilter getEventEvaluationFilter() { + if (evaluationFilter == null) { + // allows standard event queries to perform a seeking aggregation with field filtering + evaluationFilter = getEventFilter(); + } return eventEvaluationFilter != null ? eventEvaluationFilter.clone() : null; } diff --git a/warehouse/query-core/src/test/resources/datawave/query/EventQueryLogicFactory.xml b/warehouse/query-core/src/test/resources/datawave/query/EventQueryLogicFactory.xml index f652a4731b5..4a52dff3aaa 100644 --- a/warehouse/query-core/src/test/resources/datawave/query/EventQueryLogicFactory.xml +++ b/warehouse/query-core/src/test/resources/datawave/query/EventQueryLogicFactory.xml @@ -134,6 +134,7 @@ + diff --git a/warehouse/query-core/src/test/resources/datawave/query/QueryLogicFactory.xml b/warehouse/query-core/src/test/resources/datawave/query/QueryLogicFactory.xml index 7c043448b9e..db35c4b99f7 100644 --- a/warehouse/query-core/src/test/resources/datawave/query/QueryLogicFactory.xml +++ b/warehouse/query-core/src/test/resources/datawave/query/QueryLogicFactory.xml @@ -265,6 +265,7 @@ +