tr/traffic-event-bz: refactor UUID generation, add CICD

.github/workflows/tr-traffic-event-prov-bz.yml

@@ -0,0 +1,79 @@
+name: CI/CD tr-traffic-event-prov-bz
+
+on: 
+  push:
+    paths:
+      - "transformers/traffic-event-prov-bz/infrastructure/**"
+      - "transformers/traffic-event-prov-bz/src/**"
+      - ".github/workflows/tr-traffic-event-prov-bz.yml"
+
+env:
+  WORKING_DIRECTORY: transformers/traffic-event-prov-bz
+  DOCKER_IMAGE: ghcr.io/noi-techpark/opendatahub-collectors/tr-traffic-event-prov-bz
+  DOCKER_TAG: ${{ github.sha }}
+  KUBERNETES_NAMESPACE: collector
+
+jobs:
+  test:
+    runs-on: ubuntu-24.04
+    concurrency: tr-traffic-event-prov-bz-build
+    steps:
+    - name: Checkout source code
+      uses: actions/checkout@v4
+
+    - name: Build and push images
+      uses: noi-techpark/github-actions/docker-build-and-push@v2
+      with:
+        working-directory: ${{ env.WORKING_DIRECTORY }}/infrastructure
+        docker-username: ${{ github.actor }}
+        docker-password: ${{ secrets.GITHUB_TOKEN }}
+
+  build:
+    runs-on: ubuntu-24.04
+    concurrency: tr-traffic-event-prov-bz-build
+    needs:
+      - test
+    steps:
+    - name: Checkout source code
+      uses: actions/checkout@v4
+
+    - name: Run tests
+      run: docker run -it --rm $(docker build -q . -f infrastructure/docker/Dockerfile --target test)
+
+  deploy-test:
+    if: github.ref == 'refs/heads/main'
+    needs: 
+      - build
+    runs-on: ubuntu-22.04
+    concurrency: tr-traffic-event-prov-bz-deploy-test
+    environment: test
+    env:
+      PROJECT_NAME: tr-traffic-event-prov-bz
+      VALUES_YAML: infrastructure/helm/neogy.yaml
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v4
+
+      - name: Customize values.yaml
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+        run: |
+          yq -i '
+            .image.repository="${{ env.DOCKER_IMAGE }}" |
+            .image.tag="${{ env.DOCKER_TAG }}" |
+            .image.pullPolicy="IfNotPresent" |
+
+            .env.BDP_PROVENANCE_NAME="${{ env.PROJECT_NAME }}" |
+            .env.BDP_PROVENANCE_VERSION="${{ github.sha}}"
+          ' ${{ env.VALUES_YAML }}
+
+      - name: Deploy on cluster  
+        uses: noi-techpark/github-actions/helm-deploy@v2
+        with:
+          k8s-name: ${{ env.PROJECT_NAME }}
+          k8s-namespace: collector
+          chart-path: helm/generic-collector
+          values-file: ${{ env.WORKING_DIRECTORY }}/${{ env.VALUES_YAML }}
+          aws-access-key-id: ${{ secrets[vars.AWS_KEY_ID] }}
+          aws-secret-access-key: ${{ secrets[vars.AWS_KEY_SECRET] }}
+          aws-eks-cluster-name: aws-main-eu-01
+          aws-region: eu-west-1

transformers/traffic-event-prov-bz/infrastructure/docker/Dockerfile

@@ -24,4 +24,5 @@ CMD ["go", "run", "main.go"]
 # TESTS
 FROM base as test
 WORKDIR /code
+COPY src/. .
 CMD ["go", "test", "."]

transformers/traffic-event-prov-bz/infrastructure/helm/values.yaml

@@ -4,7 +4,8 @@ image:
 
 env:
   # ODH Core Writer Connection
-  BDP_BASE_URL: https://share.opendatahub.testingmachine.eu
+  # BDP_BASE_URL: https://share.opendatahub.testingmachine.eu
+  BDP_BASE_URL: http://bdp-core.core.svc.cluster.local:8080
   BDP_PROVENANCE_NAME: 
   BDP_PROVENANCE_VERSION: 
   BDP_ORIGIN: PROVINCE_BZ

transformers/traffic-event-prov-bz/src/go.mod

@@ -3,11 +3,12 @@ module opendatahub.com/tr-traffic-event-prov-bz
 go 1.23.4
 
 require (
-	github.com/gofrs/uuid/v5 v5.3.0
+	github.com/google/uuid v1.6.0
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/noi-techpark/go-bdp-client v1.0.1-0.20241230091831-3fd536c600ac
 	github.com/noi-techpark/go-opendatahub-ingest v0.0.0-20241217152708-001fb6f116e0
 	github.com/twpayne/go-geom v1.5.7
+	github.com/wI2L/jsondiff v0.6.1
 	gotest.tools/v3 v3.5.1
 )
 
@@ -21,7 +22,6 @@ require (
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.1 // indirect
 	github.com/tidwall/sjson v1.2.5 // indirect
-	github.com/wI2L/jsondiff v0.6.1 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.1.2 // indirect
 	github.com/xdg-go/stringprep v1.0.4 // indirect

transformers/traffic-event-prov-bz/src/go.sum

@@ -4,12 +4,12 @@ github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc
 github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/gofrs/uuid/v5 v5.3.0 h1:m0mUMr+oVYUdxpMLgSYCZiXe7PuVPnI94+OMeVBNedk=
-github.com/gofrs/uuid/v5 v5.3.0/go.mod h1:CDOjlDMVAtN56jqyRUZh58JT31Tiw7/oQyEXZV+9bD8=
 github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
 github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
 github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
 github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=

transformers/traffic-event-prov-bz/src/main.go

@@ -5,14 +5,12 @@
 package main
 
 import (
-	"crypto/sha1"
 	"encoding/json"
 	"fmt"
 	"strconv"
-	"strings"
 	"time"
 
-	"github.com/gofrs/uuid/v5"
+	"github.com/google/uuid"
 	"github.com/kelseyhightower/envconfig"
 	"github.com/noi-techpark/go-bdp-client/bdplib"
 	"github.com/noi-techpark/go-opendatahub-ingest/dto"
@@ -24,6 +22,8 @@ import (
 
 var env tr.Env
 
+const UUID_NS = "traffic-events-prov-bz"
+
 func main() {
 	envconfig.MustProcess("", &env)
 	ms.InitLog(env.LOG_LEVEL)
@@ -58,11 +58,10 @@ func unmarshalRawJson(s string) ([]trafficEvent, error) {
 
 func mapEvent(d trafficEvent) (bdplib.Event, error) {
 	e := bdplib.Event{}
-	j, err := makeUUIDJson(d)
+	uuid, err := makeUUID(d)
 	if err != nil {
 		return e, err
 	}
-	uuid := makeUUID(j)
 	e.Uuid = uuid
 	e.EventSeriesUuid = uuid
 	e.Category = fmt.Sprintf("%s_%s | %s_%s", d.TycodeIt, d.SubTycodeIt, d.TycodeDe, d.SubTycodeDe)
@@ -83,10 +82,10 @@ func mapEvent(d trafficEvent) (bdplib.Event, error) {
 	}
 	e.EventStart = beginDate.UTC().UnixMilli()
 
-	if d.EndDate != nil && *d.EndDate != "" {
-		endDate, err := time.Parse(dayDateFormat, *d.EndDate)
+	if d.EndDate != "" {
+		endDate, err := time.Parse(dayDateFormat, d.EndDate)
 		if err != nil {
-			return e, fmt.Errorf("error parsing EndDate (%s): %w", *d.EndDate, err)
+			return e, fmt.Errorf("error parsing EndDate (%s): %w", d.EndDate, err)
 		}
 		e.EventEnd = endDate.UTC().UnixMilli() + 1 // +1 because we exclude the upper bound.
 	}
@@ -126,7 +125,7 @@ type trafficEvent struct {
 	JSONFeaturetype             string   `json:"json_featuretype"`
 	PublishDateTime             string   `json:"publishDateTime"`
 	BeginDate                   string   `json:"beginDate"`
-	EndDate                     *string  `json:"endDate"`
+	EndDate                     string   `json:"endDate"`
 	DescriptionDe               string   `json:"descriptionDe"`
 	DescriptionIt               string   `json:"descriptionIt"`
 	TycodeValue                 string   `json:"tycodeValue"`
@@ -166,86 +165,21 @@ func point2WKT(x float64, y float64) (string, error) {
 	return wkt.Marshal(p)
 }
 
-// All this strange UUID stuff is just there to replicate the UUID generation originally done in Java, and thus maintain primary key compatibility.
-// Unfortunately, it uses quite particular behavior that is elaborate to replicate.
-// In essence, it generates a JSON from a few fields, and then calculates a v5 UUID (with namespace = null)
-
-// TODO: make UUID generation more sane, but make sure to migrate all existing events in DB, an clear up if changing the UUIDs could lead to problems
-
 type UUIDMap struct {
-	BeginDate *UUIDDate `json:"beginDate"`
-	EndDate   *UUIDDate `json:"endDate"`
-	X         *float64  `json:"X"`
-	Y         *float64  `json:"Y"`
-}
-type UUIDDate struct {
-	Year       int    `json:"year"`
-	Month      string `json:"month"`
-	DayOfWeek  string `json:"dayOfWeek"`
-	LeapYear   bool   `json:"leapYear"`
-	DayOfMonth int    `json:"dayOfMonth"`
-	MonthValue int    `json:"monthValue"`
-	Era        string `json:"era"`
-	DayOfYear  int    `json:"dayOfYear"`
-	Chronology struct {
-		CalendarType string `json:"calendarType"`
-		ID           string `json:"id"`
-	} `json:"chronology"`
+	BeginDate string  `json:"beginDate"`
+	EndDate   string  `json:"endDate"`
+	X         float64 `json:"X"`
+	Y         float64 `json:"Y"`
 }
 
 const dayDateFormat = "2006-01-02"
 
-func toDate(s string) (UUIDDate, error) {
-	d, err := time.Parse(dayDateFormat, s)
-	if err != nil {
-		return UUIDDate{}, err
-	}
-	ret := UUIDDate{}
-	ret.Year = d.Year()
-	ret.Month = strings.ToUpper(d.Month().String())
-	ret.DayOfWeek = strings.ToUpper(d.Weekday().String())
-	ret.LeapYear = ret.Year%4 == 0 && ret.Year%100 != 0 || ret.Year%400 == 0
-	ret.DayOfMonth = d.Day()
-	ret.MonthValue = int(d.Month())
-	ret.Era = "CE"
-	ret.DayOfYear = d.YearDay()
-	ret.Chronology.CalendarType = "iso8601"
-	ret.Chronology.ID = "ISO"
-
-	return ret, nil
-}
-
-func makeUUID(name string) string {
-	// Have to do the v5 UUID ourselves, because the one from library does not support a nil namespace. The code is copied from there, sans the namespace part
-	hash := sha1.New()
-	hash.Write([]byte(name))
-	u := uuid.UUID{}
-	copy(u[:], hash.Sum(nil))
-	u.SetVersion(uuid.V5)
-	u.SetVariant(uuid.VariantRFC9562)
-	return u.String()
-}
-
-func makeUUIDJson(e trafficEvent) (string, error) {
-	u := UUIDMap{}
-	begin, err := toDate(e.BeginDate)
-	if err != nil {
-		return "", fmt.Errorf("cannot parse beginDate: %w", err)
-	}
-	u.BeginDate = &begin
-	if e.EndDate != nil && *e.EndDate != "" {
-		end, err := toDate(*e.EndDate)
-		if err != nil {
-			return "", fmt.Errorf("cannot parse endDate: %w", err)
-		}
-		u.EndDate = &end
-	}
-	u.X = e.X
-	u.Y = e.Y
-	jsonBytes, err := json.Marshal(u)
+func makeUUID(e trafficEvent) (string, error) {
+	u := UUIDMap{BeginDate: e.BeginDate, EndDate: e.EndDate}
+	json, err := json.Marshal(u)
 	if err != nil {
 		return "", fmt.Errorf("cannot marshal uuid json: %w", err)
 	}
-	jsonString := string(jsonBytes[:])
-	return jsonString, nil
+	uuid := uuid.NewSHA1(uuid.Nil, []byte(json)).String()
+	return uuid, nil
 }

transformers/traffic-event-prov-bz/src/main_test.go

@@ -14,28 +14,30 @@ import (
 	"gotest.tools/v3/assert"
 )
 
-func testDateJson(t *testing.T, bd string, ed string, x float64, y float64, expectedJson string) {
-	e := trafficEvent{}
-	e.BeginDate = bd
-	e.EndDate = &ed
+// Generate new reference file for integration testing.
+// Uncomment and run this if you've made changes that trip the integration test, and you are sure that it's all fine
 
-	e.X = &x
-	e.Y = &y
-
-	uuidJson, err := makeUUIDJson(e)
-	assert.NilError(t, err, "failed creating json")
-	assert.Equal(t, uuidJson, expectedJson)
-}
-
-func Test_makeUUIDJson(t *testing.T) {
-	testDateJson(t, "2025-01-07", "2025-02-13", 11.1893940941287, 46.6715162831429, "{\"beginDate\":{\"year\":2025,\"month\":\"JANUARY\",\"dayOfWeek\":\"TUESDAY\",\"leapYear\":false,\"dayOfMonth\":7,\"monthValue\":1,\"era\":\"CE\",\"dayOfYear\":7,\"chronology\":{\"calendarType\":\"iso8601\",\"id\":\"ISO\"}},\"endDate\":{\"year\":2025,\"month\":\"FEBRUARY\",\"dayOfWeek\":\"THURSDAY\",\"leapYear\":false,\"dayOfMonth\":13,\"monthValue\":2,\"era\":\"CE\",\"dayOfYear\":44,\"chronology\":{\"calendarType\":\"iso8601\",\"id\":\"ISO\"}},\"X\":11.1893940941287,\"Y\":46.6715162831429}")
-
-	// handle null date
-	testDateJson(t, "2024-09-30", "", 11.4555831531882, 46.4466206755139, "{\"beginDate\":{\"year\":2024,\"month\":\"SEPTEMBER\",\"dayOfWeek\":\"MONDAY\",\"leapYear\":true,\"dayOfMonth\":30,\"monthValue\":9,\"era\":\"CE\",\"dayOfYear\":274,\"chronology\":{\"calendarType\":\"iso8601\",\"id\":\"ISO\"}},\"endDate\":null,\"X\":11.4555831531882,\"Y\":46.4466206755139}")
+//	func Test_generateReference(t *testing.T) {
+//		GenerateReference(t)
+//	}
+func GenerateReference(t *testing.T) {
+	f, err := os.ReadFile("testdata/in.json")
+	assert.NilError(t, err, "failed loading source events file")
+	in, err := unmarshalRawJson(string(f))
+	assert.NilError(t, err, "failed unmarshalling testing input")
+	evs := []bdplib.Event{}
+	for _, e := range in {
+		ev, err := mapEvent(e)
+		assert.NilError(t, err, "failed mapping event")
+		evs = append(evs, ev)
+	}
+	json, _ := json.Marshal(evs)
+	os.WriteFile("testdata/out.json", json, 0644)
 }
 
-func Test_mapping(t *testing.T) {
-	// In and out JSONS are taken from the previous data collector. This is to confirm compatibility
+// End to end integration test, verifies a known input against a known output
+// Use the GenerateReference function above if you need to update the output
+func Test_integration(t *testing.T) {
 	f, err := os.ReadFile("testdata/in.json")
 	assert.NilError(t, err, "failed loading source events file")
 	in, err := unmarshalRawJson(string(f))
@@ -57,7 +59,7 @@ func Test_mapping(t *testing.T) {
 	}
 
 	for _, e := range evs {
-		diff, err := jsondiff.Compare(e, referenceMap[e.Name], jsondiff.Equivalent(), jsondiff.Ignores("/origin", "/uuid", "/eventSeriesUuid"))
+		diff, err := jsondiff.Compare(e, referenceMap[e.Name], jsondiff.Equivalent())
 		assert.NilError(t, err, "error diffing jsons")
 		if len(diff) > 0 {
 			t.Error("Unexpected difference between input and output:")