ignore all parse errors of date and return Epoch

parser/csv.go

@@ -115,7 +115,7 @@ func (c *CsvMetric) GetDateTime(key string, nullable bool) (val interface{}, err
 		val = Epoch
 		return
 	}
-	val, err = c.pp.ParseDateTime(key, c.values[idx])
+	val = c.pp.ParseDateTime(key, c.values[idx])
 	return
 }
 

parser/fastjson.go

@@ -122,17 +122,21 @@ func (c *FastjsonMetric) GetDateTime(key string, nullable bool) (val interface{}
 	case fastjson.TypeNumber:
 		var f float64
 		if f, err = v.Float64(); err != nil {
+			val = Epoch
+			err = nil
 			return
 		}
 		val = time.Unix(int64(f), int64(f*1e9)%1e9).In(time.UTC)
 	case fastjson.TypeString:
 		var b []byte
 		if b, err = v.StringBytes(); err != nil {
+			val = Epoch
+			err = nil
 			return
 		}
-		val, err = c.pp.ParseDateTime(key, string(b))
+		val = c.pp.ParseDateTime(key, string(b))
 	default:
-		err = errors.Errorf("value doesn't contain number nor string, it contains %s", v.Type().String())
+		val = Epoch
 	}
 	return
 }

parser/gjson.go

@@ -108,9 +108,9 @@ func (c *GjsonMetric) GetDateTime(key string, nullable bool) (val interface{}, e
 	case gjson.Number:
 		val = time.Unix(int64(r.Num), int64(r.Num*1e9)%1e9).In(time.UTC)
 	case gjson.String:
-		val, err = c.pp.ParseDateTime(key, r.Str)
+		val = c.pp.ParseDateTime(key, r.Str)
 	default:
-		err = errors.Errorf("value doesn't contain string, it contains %s", r.Type.String())
+		val = Epoch
 	}
 	return
 }

parser/parser.go

@@ -16,7 +16,6 @@ package parser
 
 import (
 	"encoding/json"
-	"strconv"
 	"sync"
 	"time"
 
@@ -130,7 +129,11 @@ func (pp *Pool) Put(p Parser) {
 	pp.pool.Put(p)
 }
 
-func (pp *Pool) ParseDateTime(key string, val string) (t time.Time, err error) {
+// Detect date format for each key at the first message.
+// Return time in UTC.
+// Return Epoch if parsing fail.
+func (pp *Pool) ParseDateTime(key string, val string) (t time.Time) {
+	var err error
 	var layout string
 	var lay interface{}
 	var ok bool
@@ -145,11 +148,12 @@ func (pp *Pool) ParseDateTime(key string, val string) (t time.Time, err error) {
 		pp.knownLayouts.Store(key, nil)
 	}
 	if lay == nil {
-		err = errors.Errorf("cannot parse time %s at field %s", strconv.Quote(val), key)
+		t = Epoch
 		return
 	}
 	layout, _ = lay.(string)
 	if t, err = time.ParseInLocation(layout, val, pp.timeZone); err != nil {
+		t = Epoch
 		return
 	}
 	t = t.In(time.UTC)

parser/parser_test.go

@@ -242,7 +242,7 @@ func TestParserDateTime(t *testing.T) {
 		{"time_sec_rfc3339_2", false, time.Date(2019, 12, 16, 12, 10, 30, 0, time.FixedZone("CST", 8*60*60)).In(time.UTC), nil},
 		{"time_sec_clickhouse_1", false, time.Date(2019, 12, 16, 12, 10, 30, 0, time.Local).In(time.UTC), nil},
 		{"not_exist", false, Epoch, nil},
-		{"array_int", false, nil, ErrParse},
+		{"array_int", false, Epoch, nil},
 	}
 	doTestSimple(t, "GetDateTime", testCases)
 }
@@ -253,7 +253,7 @@ func TestParserDateTime64(t *testing.T) {
 		{"time_ms_rfc3339_2", false, time.Date(2019, 12, 16, 12, 10, 30, 123000000, time.FixedZone("CST", 8*60*60)).In(time.UTC), nil},
 		{"time_ms_clickhouse_1", false, time.Date(2019, 12, 16, 12, 10, 30, 123000000, time.Local).In(time.UTC), nil},
 		{"not_exist", false, Epoch, nil},
-		{"array_int", false, nil, ErrParse},
+		{"array_int", false, Epoch, nil},
 	}
 	doTestSimple(t, "GetDateTime64", testCases)
 }