feat: improve observability for load balancer

.github/workflows/deploy-designer.yaml

@@ -107,7 +107,11 @@ jobs:
       config-chart-name: altinn-designer-config
       artifact-name: altinn-designer
       helm-set-arguments: environmentName=${{ matrix.environment == 'preapproved-prod' && 'prod' ||  matrix.environment }},chartVersion=0.1.0+${{ needs.determine-tag.outputs.tag }},imageTag=${{ needs.determine-tag.outputs.tag }},dbMigrationsTag=${{ needs.determine-tag.outputs.tag }}
+      trace-workflow: true
+      trace-team-name: 'team-studio'
     secrets:
       client-id: ${{ secrets.AZURE_CLIENT_ID_FC }}
       tenant-id: ${{ secrets.AZURE_TENANT_ID_FC }}
       subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID_FC }}
+      trace-connetion-string: ${{ secrets.APP_INSIGHTS_CONNECTION_STRING }}
+      trace-repo-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/deploy-loadbalancer.yaml

@@ -28,6 +28,7 @@ jobs:
       tag: 0.1.0+${{ needs.get-short-sha.outputs.short-sha }} # Helm version needs to be valid sematic version
       chart-name: altinn-loadbalancer
       registry-name: altinntjenestercontainerregistry.azurecr.io
+      environment: dev # dev environment has push access and doesn't require review
     secrets:
       client-id: ${{ secrets.AZURE_CLIENT_ID_FC }}
       tenant-id: ${{ secrets.AZURE_TENANT_ID_FC }}
@@ -46,7 +47,12 @@ jobs:
       config-chart-name: altinn-loadbalancer-config
       artifact-name: altinn-loadbalancer
       helm-set-arguments: environmentName=${{ matrix.environment }},chartVersion=0.1.0+${{ needs.get-short-sha.outputs.short-sha }}
+      trace-workflow: true
+      trace-team-name: 'team-studio'
     secrets:
       client-id: ${{ secrets.AZURE_CLIENT_ID_FC }}
       tenant-id: ${{ secrets.AZURE_TENANT_ID_FC }}
       subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID_FC }}
+      trace-connection-string: ${{ secrets.APP_INSIGHTS_CONNECTION_STRING }}
+      trace-repo-token: ${{ secrets.GITHUB_TOKEN }}
+

.github/workflows/template-flux-config-push.yaml

@@ -24,6 +24,15 @@ on:
       helm-set-arguments:
         required: false
         type: string
+      trace-workflow:
+        required: false
+        type: boolean
+        default: false
+      trace-team-name:
+        required: false
+        type: string
+        default: ''
+
 
     secrets:
       client-id:
@@ -32,6 +41,10 @@ on:
         required: true
       subscription-id:
         required: true
+      trace-connection-string:
+        required: true
+      trace-repo-token:
+        required: true
 
 jobs:
   config-oci-artifact-push:
@@ -86,3 +99,12 @@ jobs:
           artifact_env=${{ inputs.artifact-environment || inputs.environment }}
           flux tag artifact oci://${{ inputs.registry-name }}/configs/${{ inputs.artifact-name }}-${artifact_env}:${{ inputs.tag }} --tag latest
 
+      - name: Send Trace to Azure Monitor
+        if: ${{ inputs.trace-workflow }}
+        uses: altinn/altinn-platform/actions/send-ci-cd-trace@v1.0.1
+        with:
+            connection_string: ${{ secrets.trace-connection-string }}
+            app: "${{ inputs.artifact-name }}"
+            team: "${{ inputs.trace-team-name }}"
+            repo_token: ${{ secrets.trace-repo-token }}
+            environment: ${{ inputs.artifact-environment || inputs.environment }}

charts/altinn-loadbalancer/templates/configmap.yaml

@@ -148,4 +148,6 @@ data:
           proxy_set_header 	  X-Forwarded-For $remote_addr;
         }
       }
+    # write the log to syslog so it can be read by the sidecar
+    access_log syslog:server=127.0.0.1:5531,facility=local7,tag=nginx combined;
     }

charts/altinn-loadbalancer/templates/deployment.yaml

@@ -28,17 +28,42 @@ spec:
               mountPath: {{ $mount.mountPath }}
           {{- end }}
           {{- end }}
+        {{- if .Values.sidecar.enabled }}
+        - name: {{ .Values.sidecar.name }}
+          image: "{{ .Values.sidecar.image}}"
+          {{- if .Values.sidecar.args }}
+          args:
+          {{- range $arg := .Values.sidecar.args }}
+            - {{ $arg }}
+          {{- end }}
+          {{- end }}
+          {{- if .Values.sidecar.ports }}
+          ports:
+          {{- range $port := .Values.sidecar.ports }}
+            - containerPort: {{ $port.containerPort }}
+              name: {{ $port.name }}
+              protocol: {{ $port.protocol }}
+          {{- end }}
+          {{- end }}
+          {{- if .Values.sidecar.volumeMounts }}
+          volumeMounts:
+          {{- range $mount := .Values.sidecar.volumeMounts }}
+            - name: {{ $mount.name }}
+              mountPath: {{ $mount.mountPath }}
+          {{- end }}
+          {{- end }}
+        {{- end }}
       {{- if .Values.volumes }}
       volumes:
         {{- range $volume := .Values.volumes }}
         - name: {{ $volume.name }}
-        {{- if $volume.configMap }}
+          {{- if $volume.configMap }}
           configMap:
             name: {{ $volume.configMap.name }}
-        {{- end }}
-        {{- if $volume.secret }}
+          {{- end }}
+          {{- if $volume.secret }}
           secret:
             secretName: {{ $volume.secret.secretName }}
-        {{- end }}
+          {{- end }}
         {{- end }}
       {{- end }}

charts/altinn-loadbalancer/templates/exporter.yaml

@@ -0,0 +1,76 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: nginx-exporter-config
+  namespace: default
+data:
+  config.hcl: |
+    listen {
+      port = 4040
+    }
+
+    namespace "altinn_studio" {
+      source = {
+        syslog {
+          listen_address = "udp://127.0.0.1:5531"
+          format = "auto"
+          tags = [
+            "nginx"
+          ]
+        }
+      }
+      format = "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\""
+
+      labels {
+        app = "altinn-studio"
+      }
+
+      relabel "request_uri" {
+        from = "request"
+        split = 2
+        separator = " "
+
+        match "^/editor.*" {
+          replacement = "/editor"
+        }
+
+        match "^/dashboard.*" {
+          replacement = "/dashboard"
+        }
+
+        match "^/preview.*" {
+          replacement = "/preview"
+        }
+
+        match "^/sync-hub.*" {
+          replacement = "/sync-hub"
+        }
+
+        match "^/repos.*" {
+          replacement = "/repos"
+        }
+
+        match "^/designerapi.*" {
+          replacement = "/designerapi"
+        }
+
+        match "^/designer.*" {
+          replacement = "/designer"
+        }
+      }
+
+      relabel "status_code" {
+        from = "status"
+      }
+
+      relabel "method" {
+        from = "request"
+        split = 1
+        separator = " "
+
+        match "^(GET|POST|PUT|DELETE|HEAD|OPTIONS|PATCH)$" {
+          replacement = "$0"
+        }
+      }
+
+    }

charts/altinn-loadbalancer/templates/networkpolicy.yaml

@@ -0,0 +1,17 @@
+# the only egress should be to apps
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: deny-egress-nginx
+  namespace: default
+spec:
+  podSelector:
+    matchLabels:
+      run: altinn-loadbalancer
+  policyTypes:
+  - Egress
+  egress:
+  - to:
+    - namespaceSelector:
+        matchLabels:
+          kubernetes.io/metadata.name: "default"  

charts/altinn-loadbalancer/templates/podmonitor.yaml

@@ -0,0 +1,19 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: altinn-loadbalancer-podmonitor
+  namespace: default
+  labels:
+    app: altinn-loadbalancer
+    release: kube-prometheus-stack
+spec:
+  selector:
+    matchLabels:
+      run: altinn-loadbalancer
+  namespaceSelector:
+    matchNames:
+      - default
+  podMetricsEndpoints:
+    - port: metrics
+      path: /metrics
+      interval: 30s

charts/altinn-loadbalancer/values.yaml

@@ -33,6 +33,9 @@ volumes:
   - name: altinn-loadbalancer-ssl
     secret:
       secretName: ssl-key-secret
+  - name: nginx-exporter-config
+    configMap:
+      name: nginx-exporter-config
 
 service:
   type: LoadBalancer
@@ -51,3 +54,18 @@ loadbalancerIP:
   dev: 51.136.127.155
   staging: 52.157.218.253
   prod: 20.50.249.144
+
+sidecar:
+  enabled: true
+  name: "exporter"
+  image: "ghcr.io/martin-helmich/prometheus-nginxlog-exporter/exporter@sha256:2174507adfc841990d4c51e6b73a4b948d16a4010845c74109b6858a3d0d2242"
+  args:
+    - "-config-file"
+    - "/etc/prometheus-nginxlog-exporter/config.hcl"
+  ports:
+    - containerPort: 4040
+      name: metrics
+      protocol: TCP
+  volumeMounts:
+    - name: nginx-exporter-config
+      mountPath: /etc/prometheus-nginxlog-exporter