Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Replace joda time in ingest-common module #38088

Merged
merged 2 commits into from
Feb 1, 2019
Merged

Replace joda time in ingest-common module #38088

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5766f33
Replace joda time in ingest-common module
spinscale Jan 31, 2019
a38e8da
switch from Z back to UTC for bwc
spinscale Jan 31, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 11 additions & 17 deletions modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,6 @@

import org.elasticsearch.common.time.DateFormatter;
import org.elasticsearch.common.time.DateFormatters;
import org.elasticsearch.common.time.DateUtils;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.joda.time.format.ISODateTimeFormat;

import java.time.Instant;
import java.time.LocalDate;
Expand All @@ -48,26 +44,26 @@
enum DateFormat {
Iso8601 {
@Override
Function<String, DateTime> getFunction(String format, DateTimeZone timezone, Locale locale) {
return ISODateTimeFormat.dateTimeParser().withZone(timezone)::parseDateTime;
Function<String, ZonedDateTime> getFunction(String format, ZoneId timezone, Locale locale) {
return (date) -> DateFormatters.from(DateFormatter.forPattern("strict_date_time").parse(date)).withZoneSameInstant(timezone);
}
},
Unix {
@Override
Function<String, DateTime> getFunction(String format, DateTimeZone timezone, Locale locale) {
return (date) -> new DateTime((long)(Double.parseDouble(date) * 1000), timezone);
Function<String, ZonedDateTime> getFunction(String format, ZoneId timezone, Locale locale) {
return date -> Instant.ofEpochMilli((long) (Double.parseDouble(date) * 1000.0)).atZone(timezone);
}
},
UnixMs {
@Override
Function<String, DateTime> getFunction(String format, DateTimeZone timezone, Locale locale) {
return (date) -> new DateTime(Long.parseLong(date), timezone);
Function<String, ZonedDateTime> getFunction(String format, ZoneId timezone, Locale locale) {
return date -> Instant.ofEpochMilli(Long.parseLong(date)).atZone(timezone);
}
},
Tai64n {
@Override
Function<String, DateTime> getFunction(String format, DateTimeZone timezone, Locale locale) {
return (date) -> new DateTime(parseMillis(date), timezone);
Function<String, ZonedDateTime> getFunction(String format, ZoneId timezone, Locale locale) {
return date -> Instant.ofEpochMilli(parseMillis(date)).atZone(timezone);
}

private long parseMillis(String date) {
Expand All @@ -85,13 +81,12 @@ private long parseMillis(String date) {
Arrays.asList(NANO_OF_SECOND, SECOND_OF_DAY, MINUTE_OF_DAY, HOUR_OF_DAY, DAY_OF_MONTH, MONTH_OF_YEAR);

@Override
Function<String, DateTime> getFunction(String format, DateTimeZone timezone, Locale locale) {
Function<String, ZonedDateTime> getFunction(String format, ZoneId zoneId, Locale locale) {
// support the 6.x BWC compatible way of parsing java 8 dates
if (format.startsWith("8")) {
format = format.substring(1);
}

ZoneId zoneId = DateUtils.dateTimeZoneToZoneId(timezone);
int year = LocalDate.now(ZoneOffset.UTC).getYear();
DateFormatter formatter = DateFormatter.forPattern(format)
.withLocale(locale)
Expand All @@ -111,13 +106,12 @@ Function<String, DateTime> getFunction(String format, DateTimeZone timezone, Loc
accessor = newTime.withZoneSameLocal(zoneId);
}

long millis = DateFormatters.from(accessor).toInstant().toEpochMilli();
return new DateTime(millis, timezone);
return DateFormatters.from(accessor);
};
}
};

abstract Function<String, DateTime> getFunction(String format, DateTimeZone timezone, Locale locale);
abstract Function<String, ZonedDateTime> getFunction(String format, ZoneId timezone, Locale locale);

static DateFormat fromString(String format) {
switch (format) {
Expand Down
50 changes: 26 additions & 24 deletions ...s/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,25 +19,25 @@

package org.elasticsearch.ingest.common;

import java.util.ArrayList;
import java.util.Collections;
import java.util.IllformedLocaleException;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.function.Function;

import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.common.time.DateFormatter;
import org.elasticsearch.ingest.AbstractProcessor;
import org.elasticsearch.ingest.ConfigurationUtils;
import org.elasticsearch.ingest.IngestDocument;
import org.elasticsearch.ingest.Processor;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.script.TemplateScript;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;

import java.time.ZoneId;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.Collections;
import java.util.IllformedLocaleException;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.function.Function;

public final class DateIndexNameProcessor extends AbstractProcessor {

Expand All @@ -47,10 +47,10 @@ public final class DateIndexNameProcessor extends AbstractProcessor {
private final TemplateScript.Factory indexNamePrefixTemplate;
private final TemplateScript.Factory dateRoundingTemplate;
private final TemplateScript.Factory indexNameFormatTemplate;
private final DateTimeZone timezone;
private final List<Function<String, DateTime>> dateFormats;
private final ZoneId timezone;
private final List<Function<String, ZonedDateTime>> dateFormats;

DateIndexNameProcessor(String tag, String field, List<Function<String, DateTime>> dateFormats, DateTimeZone timezone,
DateIndexNameProcessor(String tag, String field, List<Function<String, ZonedDateTime>> dateFormats, ZoneId timezone,
TemplateScript.Factory indexNamePrefixTemplate, TemplateScript.Factory dateRoundingTemplate,
TemplateScript.Factory indexNameFormatTemplate) {
super(tag);
Expand All @@ -72,9 +72,9 @@ public IngestDocument execute(IngestDocument ingestDocument) throws Exception {
date = obj.toString();
}

DateTime dateTime = null;
ZonedDateTime dateTime = null;
Exception lastException = null;
for (Function<String, DateTime> dateParser : dateFormats) {
for (Function<String, ZonedDateTime> dateParser : dateFormats) {
try {
dateTime = dateParser.apply(date);
} catch (Exception e) {
Expand All @@ -90,13 +90,15 @@ public IngestDocument execute(IngestDocument ingestDocument) throws Exception {
String indexNameFormat = ingestDocument.renderTemplate(indexNameFormatTemplate);
String dateRounding = ingestDocument.renderTemplate(dateRoundingTemplate);

DateTimeFormatter formatter = DateTimeFormat.forPattern(indexNameFormat);
DateFormatter formatter = DateFormatter.forPattern(indexNameFormat);
// use UTC instead of Z is string representation of UTC, so behaviour is the same between 6.x and 7
String zone = timezone.equals(ZoneOffset.UTC) ? "UTC" : timezone.getId();
StringBuilder builder = new StringBuilder()
.append('<')
.append(indexNamePrefix)
.append('{')
.append(formatter.print(dateTime)).append("||/").append(dateRounding)
.append('{').append(indexNameFormat).append('|').append(timezone).append('}')
.append(formatter.format(dateTime)).append("||/").append(dateRounding)
.append('{').append(indexNameFormat).append('|').append(zone).append('}')
.append('}')
.append('>');
String dynamicIndexName = builder.toString();
Expand Down Expand Up @@ -125,11 +127,11 @@ TemplateScript.Factory getIndexNameFormatTemplate() {
return indexNameFormatTemplate;
}

DateTimeZone getTimezone() {
ZoneId getTimezone() {
return timezone;
}

List<Function<String, DateTime>> getDateFormats() {
List<Function<String, ZonedDateTime>> getDateFormats() {
return dateFormats;
}

Expand All @@ -146,7 +148,7 @@ public DateIndexNameProcessor create(Map<String, Processor.Factory> registry, St
Map<String, Object> config) throws Exception {
String localeString = ConfigurationUtils.readOptionalStringProperty(TYPE, tag, config, "locale");
String timezoneString = ConfigurationUtils.readOptionalStringProperty(TYPE, tag, config, "timezone");
DateTimeZone timezone = timezoneString == null ? DateTimeZone.UTC : DateTimeZone.forID(timezoneString);
ZoneId timezone = timezoneString == null ? ZoneOffset.UTC : ZoneId.of(timezoneString);
Locale locale = Locale.ENGLISH;
if (localeString != null) {
try {
Expand All @@ -159,7 +161,7 @@ public DateIndexNameProcessor create(Map<String, Processor.Factory> registry, St
if (dateFormatStrings == null) {
dateFormatStrings = Collections.singletonList("yyyy-MM-dd'T'HH:mm:ss.SSSXX");
}
List<Function<String, DateTime>> dateFormats = new ArrayList<>(dateFormatStrings.size());
List<Function<String, ZonedDateTime>> dateFormats = new ArrayList<>(dateFormatStrings.size());
for (String format : dateFormatStrings) {
DateFormat dateFormat = DateFormat.fromString(format);
dateFormats.add(dateFormat.getFunction(format, timezone, locale));
Expand Down
20 changes: 11 additions & 9 deletions modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,17 +21,18 @@

import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.time.DateFormatter;
import org.elasticsearch.common.util.LocaleUtils;
import org.elasticsearch.ingest.AbstractProcessor;
import org.elasticsearch.ingest.ConfigurationUtils;
import org.elasticsearch.ingest.IngestDocument;
import org.elasticsearch.ingest.Processor;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.script.TemplateScript;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.joda.time.format.ISODateTimeFormat;

import java.time.ZoneId;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
Expand All @@ -42,13 +43,14 @@ public final class DateProcessor extends AbstractProcessor {

public static final String TYPE = "date";
static final String DEFAULT_TARGET_FIELD = "@timestamp";
public static final DateFormatter FORMATTER = DateFormatter.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXXX");

private final TemplateScript.Factory timezone;
private final TemplateScript.Factory locale;
private final String field;
private final String targetField;
private final List<String> formats;
private final List<Function<Map<String, Object>, Function<String, DateTime>>> dateParsers;
private final List<Function<Map<String, Object>, Function<String, ZonedDateTime>>> dateParsers;

DateProcessor(String tag, @Nullable TemplateScript.Factory timezone, @Nullable TemplateScript.Factory locale,
String field, List<String> formats, String targetField) {
Expand All @@ -65,8 +67,8 @@ public final class DateProcessor extends AbstractProcessor {
}
}

private DateTimeZone newDateTimeZone(Map<String, Object> params) {
return timezone == null ? DateTimeZone.UTC : DateTimeZone.forID(timezone.newInstance(params).execute());
private ZoneId newDateTimeZone(Map<String, Object> params) {
return timezone == null ? ZoneOffset.UTC : ZoneId.of(timezone.newInstance(params).execute());
}

private Locale newLocale(Map<String, Object> params) {
Expand All @@ -82,9 +84,9 @@ public IngestDocument execute(IngestDocument ingestDocument) {
value = obj.toString();
}

DateTime dateTime = null;
ZonedDateTime dateTime = null;
Exception lastException = null;
for (Function<Map<String, Object>, Function<String, DateTime>> dateParser : dateParsers) {
for (Function<Map<String, Object>, Function<String, ZonedDateTime>> dateParser : dateParsers) {
try {
dateTime = dateParser.apply(ingestDocument.getSourceAndMetadata()).apply(value);
} catch (Exception e) {
Expand All @@ -97,7 +99,7 @@ public IngestDocument execute(IngestDocument ingestDocument) {
throw new IllegalArgumentException("unable to parse date [" + value + "]", lastException);
}

ingestDocument.setFieldValue(targetField, ISODateTimeFormat.dateTime().print(dateTime));
ingestDocument.setFieldValue(targetField, FORMATTER.format(dateTime));
return ingestDocument;
}

Expand Down
31 changes: 15 additions & 16 deletions modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,7 @@

import org.elasticsearch.common.time.DateUtils;
import org.elasticsearch.test.ESTestCase;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;

import java.time.Instant;
import java.time.ZoneId;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
Expand All @@ -38,43 +35,45 @@
public class DateFormatTests extends ESTestCase {

public void testParseJava() {
Function<String, DateTime> javaFunction = DateFormat.Java.getFunction("MMM dd HH:mm:ss Z",
DateTimeZone.forOffsetHours(-8), Locale.ENGLISH);
assertThat(Instant.ofEpochMilli(javaFunction.apply("Nov 24 01:29:01 -0800").getMillis())
Function<String, ZonedDateTime> javaFunction = DateFormat.Java.getFunction("MMM dd HH:mm:ss Z",
ZoneOffset.ofHours(-8), Locale.ENGLISH);
assertThat(javaFunction.apply("Nov 24 01:29:01 -0800").toInstant()
.atZone(ZoneId.of("GMT-8"))
.format(DateTimeFormatter.ofPattern("MM dd HH:mm:ss", Locale.ENGLISH)),
equalTo("11 24 01:29:01"));
}

public void testParseJavaDefaultYear() {
String format = randomFrom("8dd/MM", "dd/MM");
DateTimeZone timezone = DateUtils.zoneIdToDateTimeZone(ZoneId.of("Europe/Amsterdam"));
Function<String, DateTime> javaFunction = DateFormat.Java.getFunction(format, timezone, Locale.ENGLISH);
ZoneId timezone = DateUtils.of("Europe/Amsterdam");
Function<String, ZonedDateTime> javaFunction = DateFormat.Java.getFunction(format, timezone, Locale.ENGLISH);
int year = ZonedDateTime.now(ZoneOffset.UTC).getYear();
DateTime dateTime = javaFunction.apply("12/06");
ZonedDateTime dateTime = javaFunction.apply("12/06");
assertThat(dateTime.getYear(), is(year));
assertThat(dateTime.toString(), is(year + "-06-12T00:00:00.000+02:00"));
}

public void testParseUnixMs() {
assertThat(DateFormat.UnixMs.getFunction(null, DateTimeZone.UTC, null).apply("1000500").getMillis(), equalTo(1000500L));
assertThat(DateFormat.UnixMs.getFunction(null, ZoneOffset.UTC, null).apply("1000500").toInstant().toEpochMilli(),
equalTo(1000500L));
}

public void testParseUnix() {
assertThat(DateFormat.Unix.getFunction(null, DateTimeZone.UTC, null).apply("1000.5").getMillis(), equalTo(1000500L));
assertThat(DateFormat.Unix.getFunction(null, ZoneOffset.UTC, null).apply("1000.5").toInstant().toEpochMilli(),
equalTo(1000500L));
}

public void testParseUnixWithMsPrecision() {
assertThat(DateFormat.Unix.getFunction(null, DateTimeZone.UTC, null).apply("1495718015").getMillis(), equalTo(1495718015000L));
assertThat(DateFormat.Unix.getFunction(null, ZoneOffset.UTC, null).apply("1495718015").toInstant().toEpochMilli(),
equalTo(1495718015000L));
}

public void testParseISO8601() {
assertThat(DateFormat.Iso8601.getFunction(null, DateTimeZone.UTC, null).apply("2001-01-01T00:00:00-0800").getMillis(),
assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.UTC, null).apply("2001-01-01T00:00:00-0800").toInstant().toEpochMilli(),
equalTo(978336000000L));
}

public void testParseISO8601Failure() {
Function<String, DateTime> function = DateFormat.Iso8601.getFunction(null, DateTimeZone.UTC, null);
Function<String, ZonedDateTime> function = DateFormat.Iso8601.getFunction(null, ZoneOffset.UTC, null);
try {
function.apply("2001-01-0:00-0800");
fail("parse should have failed");
Expand All @@ -86,7 +85,7 @@ public void testParseISO8601Failure() {
public void testTAI64NParse() {
String input = "4000000050d506482dbdf024";
String expected = "2012-12-22T03:00:46.767+02:00";
assertThat(DateFormat.Tai64n.getFunction(null, DateTimeZone.forOffsetHours(2), null)
assertThat(DateFormat.Tai64n.getFunction(null, ZoneOffset.ofHours(2), null)
.apply((randomBoolean() ? "@" : "") + input).toString(), equalTo(expected));
}

Expand Down
6 changes: 3 additions & 3 deletions ...ngest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@
import org.elasticsearch.ingest.TestTemplateService;
import org.elasticsearch.test.ESTestCase;
import org.hamcrest.Matchers;
import org.joda.time.DateTimeZone;

import java.time.ZoneOffset;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
Expand All @@ -44,7 +44,7 @@ public void testDefaults() throws Exception {
assertThat(processor.getIndexNamePrefixTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo(""));
assertThat(processor.getDateRoundingTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("y"));
assertThat(processor.getIndexNameFormatTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("yyyy-MM-dd"));
assertThat(processor.getTimezone(), Matchers.equalTo(DateTimeZone.UTC));
assertThat(processor.getTimezone(), Matchers.equalTo(ZoneOffset.UTC));
}

public void testSpecifyOptionalSettings() throws Exception {
Expand Down Expand Up @@ -74,7 +74,7 @@ public void testSpecifyOptionalSettings() throws Exception {
config.put("timezone", "+02:00");

processor = factory.create(null, null, config);
assertThat(processor.getTimezone(), Matchers.equalTo(DateTimeZone.forOffsetHours(2)));
assertThat(processor.getTimezone(), Matchers.equalTo(ZoneOffset.ofHours(2)));

config = new HashMap<>();
config.put("field", "_field");
Expand Down
Loading