try fixing ninja timeout by making query more specific and starting f…

…rom year 2010 instead of 0

traffic-a22-data-quality/src/dc/sum.go

@@ -60,7 +60,7 @@ func sumJob() {
 	var res ninja.NinjaResponse[NinjaTreeData]
 	err := ninja.Latest(req, &res)
 	if err != nil {
-		slog.Error("error", err)
+		slog.Error("error getting latest records from ninja. aborting...", "err", err)
 		return
 	}
 
@@ -162,15 +162,19 @@ func getHistoryPaged(todo window, stationCode string, typeName string) ([]NinjaF
 	return ret, nil
 }
 
+// arbitraty starting point where there should be no data yet
+var minTime = time.Date(2010, 1, 1, 0, 0, 0, 0, time.UTC)
+
 func requestWindows(dt NinjaTreeData) map[string]map[string]window {
 	todos := make(map[string]map[string]window)
 	for _, stations := range dt {
 		for stationCode, station := range stations.Stations {
 			for tname, dataType := range station.Datatypes {
 				for _, m := range dataType.Measurements {
-					var firstBase time.Time
-					var lastBase time.Time
-					var lastAggregate time.Time
+					firstBase := minTime
+					lastBase := minTime
+					lastAggregate := minTime
+
 					if m.Period == uint64(basePeriod) {
 						lastBase = m.Time.Time
 						firstBase = m.Since.Time
@@ -221,10 +225,11 @@ func getRequestDates(todo window) (time.Time, time.Time) {
 func getNinjaData(stationCode string, typeName string, from time.Time, to time.Time, offset int) (*ninja.NinjaResponse[[]NinjaFlatData], error) {
 	req := ninja.DefaultNinjaRequest()
 	req.AddDataType(typeName)
+	req.AddStationType(baseStationType)
 	req.From = from
 	req.To = to
 	req.Select = "mvalue"
-	req.Where = fmt.Sprintf("and(mperiod.eq.%d,scode.eq.\"%s\")", basePeriod, stationCode)
+	req.Where = fmt.Sprintf("mperiod.eq.%d,scode.eq.\"%s\"", basePeriod, stationCode)
 	req.Limit = int64(sumRequestLimit)
 	req.Offset = uint64(offset * int(req.Limit))