Skip to content

Commit

Permalink
Timestamp fractions pattern (elastic#15911)
Browse files Browse the repository at this point in the history 
* Add support for `f` in date patterns

* Use `f` pattern in timestamp encoder

* update check

* Add godoc

* typo
  • Loading branch information
Steffen Siering authored Jan 31, 2020
1 parent d9e6884 commit d51dcef
Show file tree
Hide file tree
Showing 8 changed files with 237 additions and 94 deletions.
24 changes: 22 additions & 2 deletions libbeat/common/dtfmt/builder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,26 @@ func (b *builder) nanoOfSecond(digits int) {
}
}

func (b *builder) fractNanoOfSecond(digits int) {
const fractDigits = 3

if digits <= 0 {
return
}

// cap number of digits at 9, as we do not support higher precision and
// would remove trailing zeroes anyway.
if digits > 9 {
digits = 9
}

minDigits := fractDigits
if digits < minDigits {
minDigits = digits
}
b.add(paddedNumber{ftNanoOfSecond, 9 - digits, minDigits, digits, fractDigits, false})
}

func (b *builder) secondOfMinute(digits int) {
b.appendDecimal(ftSecondOfMinute, digits, 2)
}
Expand Down Expand Up @@ -223,12 +243,12 @@ func (b *builder) appendDecimalValue(ft fieldType, minDigits, maxDigits int, sig
if minDigits <= 1 {
b.add(unpaddedNumber{ft, maxDigits, signed})
} else {
b.add(paddedNumber{ft, 0, minDigits, maxDigits, signed})
b.add(paddedNumber{ft, 0, minDigits, maxDigits, 0, signed})
}
}

func (b *builder) appendExtDecimal(ft fieldType, divExp, minDigits, maxDigits int) {
b.add(paddedNumber{ft, divExp, minDigits, maxDigits, false})
b.add(paddedNumber{ft, divExp, minDigits, maxDigits, 0, false})
}

func (b *builder) appendDecimal(ft fieldType, minDigits, maxDigits int) {
Expand Down
48 changes: 25 additions & 23 deletions libbeat/common/dtfmt/doc.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,29 +22,31 @@
//
// Symbol Meaning Type Supported Examples
// ------ ------- ------- --------- -------
// G era text no AD
// C century of era (&gt;=0) number no 20
// Y year of era (&gt;=0) year yes 1996
//
// x weekyear year yes 1996
// w week of weekyear number yes 27
// e day of week number yes 2
// E day of week text yes Tuesday; Tue
//
// y year year yes 1996
// D day of year number yes 189
// M month of year month yes July; Jul; 07
// d day of month number yes 10
//
// a halfday of day text yes PM
// K hour of halfday (0~11) number yes 0
// h clockhour of halfday (1~12) number yes 12
//
// H hour of day (0~23) number yes 0
// k clockhour of day (1~24) number yes 24
// m minute of hour number yes 30
// s second of minute number yes 55
// S fraction of second millis no 978
// G era text no AD
// C century of era (&gt;=0) number no 20
// Y year of era (&gt;=0) year yes 1996
//
// x weekyear year yes 1996
// w week of weekyear number yes 27
// e day of week number yes 2
// E day of week text yes Tuesday; Tue
//
// y year year yes 1996
// D day of year number yes 189
// M month of year month yes July; Jul; 07
// d day of month number yes 10
//
// a halfday of day text yes PM
// K hour of halfday (0~11) number yes 0
// h clockhour of halfday (1~12) number yes 12
//
// H hour of day (0~23) number yes 0
// k clockhour of day (1~24) number yes 24
// m minute of hour number yes 30
// s second of minute number yes 55
// S fraction of second nanoseconds yes 978000
// f fraction of seconds nanoseconds yes 123456789
// multiple of 3
//
// z time zone text no Pacific Standard Time; PST
// Z time zone offset/id zone no -0800; -08:00; America/Los_Angeles
Expand Down
66 changes: 48 additions & 18 deletions libbeat/common/dtfmt/dtfmt_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,10 +82,37 @@ func TestFormat(t *testing.T) {
{mkTime(8, 5, 24, 0), "kk:mm:ss aa", "09:05:24 AM"},
{mkTime(20, 5, 24, 0), "k:m:s a", "21:5:24 PM"},
{mkTime(20, 5, 24, 0), "kk:mm:ss aa", "21:05:24 PM"},
{mkTime(1, 2, 3, 123), "S", "1"},
{mkTime(1, 2, 3, 123), "SS", "12"},
{mkTime(1, 2, 3, 123), "SSS", "123"},
{mkTime(1, 2, 3, 123), "SSSS", "1230"},
{mkTime(1, 2, 3, 123*time.Millisecond), "S", "1"},
{mkTime(1, 2, 3, 123*time.Millisecond), "SS", "12"},
{mkTime(1, 2, 3, 123*time.Millisecond), "SSS", "123"},
{mkTime(1, 2, 3, 123*time.Millisecond), "SSSS", "1230"},
{mkTime(1, 2, 3, 123*time.Millisecond), "f", "1"},
{mkTime(1, 2, 3, 123*time.Millisecond), "ff", "12"},
{mkTime(1, 2, 3, 123*time.Millisecond), "fff", "123"},
{mkTime(1, 2, 3, 123*time.Millisecond), "ffff", "123"},
{mkTime(1, 2, 3, 123*time.Millisecond), "fffff", "123"},
{mkTime(1, 2, 3, 123*time.Millisecond), "ffffff", "123"},
{mkTime(1, 2, 3, 123*time.Millisecond), "fffffff", "123"},
{mkTime(1, 2, 3, 123*time.Millisecond), "ffffffff", "123"},
{mkTime(1, 2, 3, 123*time.Millisecond), "fffffffff", "123"},
{mkTime(1, 2, 3, 123*time.Microsecond), "f", "0"},
{mkTime(1, 2, 3, 123*time.Microsecond), "ff", "00"},
{mkTime(1, 2, 3, 123*time.Microsecond), "fff", "000"},
{mkTime(1, 2, 3, 123*time.Microsecond), "ffff", "0001"},
{mkTime(1, 2, 3, 123*time.Microsecond), "fffff", "00012"},
{mkTime(1, 2, 3, 123*time.Microsecond), "ffffff", "000123"},
{mkTime(1, 2, 3, 123*time.Microsecond), "fffffff", "000123"},
{mkTime(1, 2, 3, 123*time.Microsecond), "ffffffff", "000123"},
{mkTime(1, 2, 3, 123*time.Microsecond), "fffffffff", "000123"},
{mkTime(1, 2, 3, 123*time.Nanosecond), "f", "0"},
{mkTime(1, 2, 3, 123*time.Nanosecond), "ff", "00"},
{mkTime(1, 2, 3, 123*time.Nanosecond), "fff", "000"},
{mkTime(1, 2, 3, 123*time.Nanosecond), "ffff", "000"},
{mkTime(1, 2, 3, 123*time.Nanosecond), "fffff", "000"},
{mkTime(1, 2, 3, 123*time.Nanosecond), "ffffff", "000"},
{mkTime(1, 2, 3, 123*time.Nanosecond), "fffffff", "0000001"},
{mkTime(1, 2, 3, 123*time.Nanosecond), "ffffffff", "00000012"},
{mkTime(1, 2, 3, 123*time.Nanosecond), "fffffffff", "000000123"},

// literals
{time.Now(), "--=++,_!/?\\[]{}@#$%^&*()", "--=++,_!/?\\[]{}@#$%^&*()"},
Expand All @@ -94,24 +121,27 @@ func TestFormat(t *testing.T) {
{time.Now(), "'plain' '' 'text'", "plain ' text"},
{time.Now(), "'plain '' text'", "plain ' text"},

// beats timestamp
{mkDateTime(2017, 1, 2, 4, 6, 7, 123),
// timestamps with microseconds precision only
{mkDateTime(2017, 1, 2, 4, 6, 7, 123*time.Millisecond),
"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'",
"2017-01-02T04:06:07.123Z"},
{mkDateTime(2017, 1, 2, 4, 6, 7, 123456*time.Microsecond),
"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'",
"2017-01-02T04:06:07.123Z"},

// beats timestamp
{mkDateTimeWithLocation(2017, 1, 2, 4, 6, 7, 123, time.FixedZone("PST", -8*60*60)),
"yyyy-MM-dd'T'HH:mm:ss.SSSz",
{mkDateTimeWithLocation(2017, 1, 2, 4, 6, 7, 123*time.Millisecond, time.FixedZone("PST", -8*60*60)),
"yyyy-MM-dd'T'HH:mm:ss.fffffffffz",
"2017-01-02T04:06:07.123-08:00"},

// beats nanoseconds timestamp
{mkDateTime(2017, 1, 2, 4, 6, 7, 123),
"yyyy-MM-dd'T'HH:mm:ss.nnnnnnnnn'Z'",
"2017-01-02T04:06:07.123000000Z"},
{mkDateTime(2017, 1, 2, 4, 6, 7, 123*time.Nanosecond),
"yyyy-MM-dd'T'HH:mm:ss.fffffffff'Z'",
"2017-01-02T04:06:07.000000123Z"},

{mkDateTimeWithLocation(2017, 1, 2, 4, 6, 7, 123, time.FixedZone("PST", -8*60*60)),
"yyyy-MM-dd'T'HH:mm:ss.nnnnnnnnnz",
"2017-01-02T04:06:07.123000000-08:00"},
{mkDateTimeWithLocation(2017, 1, 2, 4, 6, 7, 123*time.Millisecond, time.FixedZone("PST", -8*60*60)),
"yyyy-MM-dd'T'HH:mm:ss.fffffffffz",
"2017-01-02T04:06:07.123-08:00"},
}

for i, test := range tests {
Expand All @@ -132,14 +162,14 @@ func mkDate(y, m, d int) time.Time {
return mkDateTime(y, m, d, 0, 0, 0, 0)
}

func mkTime(h, m, s, S int) time.Time {
func mkTime(h, m, s int, S time.Duration) time.Time {
return mkDateTime(2000, 1, 1, h, m, s, S)
}

func mkDateTime(y, M, d, h, m, s, S int) time.Time {
func mkDateTime(y, M, d, h, m, s int, S time.Duration) time.Time {
return mkDateTimeWithLocation(y, M, d, h, m, s, S, time.UTC)
}

func mkDateTimeWithLocation(y, M, d, h, m, s, S int, l *time.Location) time.Time {
return time.Date(y, time.Month(M), d, h, m, s, S*1000000, l)
func mkDateTimeWithLocation(y, M, d, h, m, s int, S time.Duration, l *time.Location) time.Time {
return time.Date(y, time.Month(M), d, h, m, s, int(S), l)
}
16 changes: 10 additions & 6 deletions libbeat/common/dtfmt/elems.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,10 +44,10 @@ type unpaddedNumber struct {
}

type paddedNumber struct {
ft fieldType
divExp int
minDigits, maxDigits int
signed bool
ft fieldType
divExp int
minDigits, maxDigits, fractDigits int
signed bool
}

type textField struct {
Expand Down Expand Up @@ -188,10 +188,14 @@ func (n unpaddedNumber) compile() (prog, error) {
}

func (n paddedNumber) compile() (prog, error) {
if n.divExp == 0 {
switch {
case n.fractDigits != 0:
return makeProg(opExtNumFractPadded, byte(n.ft), byte(n.divExp), byte(n.maxDigits), byte(n.fractDigits))
case n.divExp == 0:
return makeProg(opNumPadded, byte(n.ft), byte(n.maxDigits))
default:
return makeProg(opExtNumPadded, byte(n.ft), byte(n.divExp), byte(n.maxDigits))
}
return makeProg(opExtNumPadded, byte(n.ft), byte(n.divExp), byte(n.maxDigits))
}

func (n twoDigitYear) compile() (prog, error) {
Expand Down
11 changes: 5 additions & 6 deletions libbeat/common/dtfmt/fmt.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,6 @@ func releaseCtx(c *ctx) {
// If pattern is invalid an error is returned.
func NewFormatter(pattern string) (*Formatter, error) {
b := newBuilder()

// pattern: yyyy-MM-dd'T'HH:mm:ss.fffffffff'Z'
err := parsePatternTo(b, pattern)
if err != nil {
return nil, err
Expand Down Expand Up @@ -136,7 +134,6 @@ func (f *Formatter) Format(t time.Time) (string, error) {
}

func parsePatternTo(b *builder, pattern string) error {
// pattern: yyyy-MM-dd'T'HH:mm:ss.fffffffff'Z'
for i := 0; i < len(pattern); {
tok, tokText, err := parseToken(pattern, &i)
if err != nil {
Expand Down Expand Up @@ -213,8 +210,8 @@ func parsePatternTo(b *builder, pattern string) error {
case 'S': // fraction of second
b.nanoOfSecond(tokLen)

case 'z': // timezone offset
b.timeZoneOffsetText()
case 'f': // faction of second (without zeros)
b.fractNanoOfSecond(tokLen)

case 'n': // nano second
// if timestamp layout use `n`, it always return 9 digits nanoseconds.
Expand All @@ -223,6 +220,9 @@ func parsePatternTo(b *builder, pattern string) error {
}
b.nanoOfSecond(tokLen)

case 'z': // timezone offset
b.timeZoneOffsetText()

case '\'': // literal
if tokLen == 1 {
b.appendRune(rune(tokText[0]))
Expand All @@ -243,7 +243,6 @@ func parseToken(pattern string, i *int) (rune, string, error) {
start := *i
idx := start
length := len(pattern)
// pattern: yyyy-MM-dd'T'HH:mm:ss.fffffffff'Z'
r, w := utf8.DecodeRuneInString(pattern[idx:])
idx += w
if ('A' <= r && r <= 'Z') || ('a' <= r && r <= 'z') {
Expand Down
38 changes: 24 additions & 14 deletions libbeat/common/dtfmt/prog.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,20 +27,21 @@ type prog struct {
}

const (
opNone byte = iota
opCopy1 // copy next byte
opCopy2 // copy next 2 bytes
opCopy3 // copy next 3 bytes
opCopy4 // copy next 4 bytes
opCopyShort // [op, len, content[len]]
opCopyLong // [op, len1, len, content[len1<<8 + len]]
opNum // [op, ft]
opNumPadded // [op, ft, digits]
opExtNumPadded // [op, ft, divExp, digits]
opZeros // [op, count]
opTwoDigit // [op, ft]
opTextShort // [op, ft]
opTextLong // [op, ft]
opNone byte = iota
opCopy1 // copy next byte
opCopy2 // copy next 2 bytes
opCopy3 // copy next 3 bytes
opCopy4 // copy next 4 bytes
opCopyShort // [op, len, content[len]]
opCopyLong // [op, len1, len, content[len1<<8 + len]]
opNum // [op, ft]
opNumPadded // [op, ft, digits]
opExtNumPadded // [op, ft, divExp, digits]
opExtNumFractPadded // [op, ft, divExp, digits, fractDigits]
opZeros // [op, count]
opTwoDigit // [op, ft]
opTextShort // [op, ft]
opTextLong // [op, ft]
)

var pow10Table [10]int
Expand Down Expand Up @@ -108,6 +109,15 @@ func (p prog) eval(bytes []byte, ctx *ctx, t time.Time) ([]byte, error) {
return bytes, err
}
bytes = appendPadded(bytes, v/div, digits)
case opExtNumFractPadded:
ft, divExp, digits, fractDigits := fieldType(p.p[i]), int(p.p[i+1]), int(p.p[i+2]), int(p.p[i+3])
div := pow10Table[divExp]
i += 4
v, err := getIntField(ft, ctx, t)
if err != nil {
return bytes, err
}
bytes = appendFractPadded(bytes, v/div, digits, fractDigits)
case opZeros:
digits := int(p.p[i])
i++
Expand Down
Loading

0 comments on commit d51dcef

Please sign in to comment.