Merge branch 'main' into fix/gql-add-missing-search-parameters-for-pa…

…gination-orderby-limit
Altinn · Jan 13, 2025 · 8fcac7e · 8fcac7e
2 parents cd9d71a + 1a71763
commit 8fcac7e
Show file tree

Hide file tree

Showing 43 changed files with 724 additions and 269 deletions.
diff --git a/.azure/applications/graphql/main.bicep b/.azure/applications/graphql/main.bicep
@@ -46,7 +46,7 @@ param appConfigurationName string
 param environmentKeyVaultName string
 
 var namePrefix = 'dp-be-${environment}'
-var baseImageUrl = 'ghcr.io/digdir/dialogporten-'
+var baseImageUrl = 'ghcr.io/altinn/dialogporten-'
 
 var tags = {
   Environment: environment

diff --git a/.azure/applications/service/main.bicep b/.azure/applications/service/main.bicep
@@ -71,7 +71,7 @@ param scale Scale = {
 }
 
 var namePrefix = 'dp-be-${environment}'
-var baseImageUrl = 'ghcr.io/digdir/dialogporten-'
+var baseImageUrl = 'ghcr.io/altinn/dialogporten-'
 var tags = {
   Environment: environment
   Product: 'Dialogporten'

diff --git a/.azure/applications/sync-resource-policy-information-job/main.bicep b/.azure/applications/sync-resource-policy-information-job/main.bicep
@@ -32,7 +32,7 @@ param jobSchedule string
 param appInsightConnectionString string
 
 var namePrefix = 'dp-be-${environment}'
-var baseImageUrl = 'ghcr.io/digdir/dialogporten-'
+var baseImageUrl = 'ghcr.io/altinn/dialogporten-'
 var tags = {
   FullName: '${namePrefix}-sync-resource-policy-information'
   Environment: environment

diff --git a/.azure/applications/sync-subject-resource-mappings-job/main.bicep b/.azure/applications/sync-subject-resource-mappings-job/main.bicep
@@ -32,7 +32,7 @@ param jobSchedule string
 param appInsightConnectionString string
 
 var namePrefix = 'dp-be-${environment}'
-var baseImageUrl = 'ghcr.io/digdir/dialogporten-'
+var baseImageUrl = 'ghcr.io/altinn/dialogporten-'
 var tags = {
   FullName: '${namePrefix}-sync-subject-resource-mappings'
   Environment: environment

diff --git a/.azure/applications/web-api-eu/main.bicep b/.azure/applications/web-api-eu/main.bicep
@@ -46,7 +46,7 @@ param appConfigurationName string
 param environmentKeyVaultName string
 
 var namePrefix = 'dp-be-${environment}'
-var baseImageUrl = 'ghcr.io/digdir/dialogporten-'
+var baseImageUrl = 'ghcr.io/altinn/dialogporten-'
 var tags = {
   Environment: environment
   Product: 'Dialogporten'

diff --git a/.azure/applications/web-api-migration-job/main.bicep b/.azure/applications/web-api-migration-job/main.bicep
@@ -23,7 +23,7 @@ param containerAppEnvironmentName string
 param environmentKeyVaultName string
 
 var namePrefix = 'dp-be-${environment}'
-var baseImageUrl = 'ghcr.io/digdir/dialogporten-'
+var baseImageUrl = 'ghcr.io/altinn/dialogporten-'
 var tags = {
   Environment: environment
   Product: 'Dialogporten'

diff --git a/.azure/applications/web-api-so/main.bicep b/.azure/applications/web-api-so/main.bicep
@@ -74,7 +74,7 @@ param scale Scale = {
 }
 
 var namePrefix = 'dp-be-${environment}'
-var baseImageUrl = 'ghcr.io/digdir/dialogporten-'
+var baseImageUrl = 'ghcr.io/altinn/dialogporten-'
 var tags = {
   Environment: environment
   Product: 'Dialogporten'

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -1,4 +1,4 @@
-*                   @digdir/team-dialogporten-backend
-.azure/**           @digdir/team-dialogporten-backend @digdir/team-dialogporten-infra
-.github/**          @digdir/team-dialogporten-backend @digdir/team-dialogporten-infra
-.github/CODEOWNERS  @digdir/team-dialogporten-backend
+*                   @altinn/team-dialogporten-backend
+.azure/**           @altinn/team-dialogporten-backend @altinn/team-dialogporten-infra
+.github/**          @altinn/team-dialogporten-backend @altinn/team-dialogporten-infra
+.github/CODEOWNERS  @altinn/team-dialogporten-backend
diff --git a/.github/workflows/ci-cd-main.yml b/.github/workflows/ci-cd-main.yml
@@ -57,7 +57,7 @@ jobs:
     secrets:
       GCR_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
     with:
-      dockerImageBaseName: ghcr.io/digdir/dialogporten-
+      dockerImageBaseName: ghcr.io/altinn/dialogporten-
       version: ${{ needs.get-current-version.outputs.version }}-${{ needs.generate-git-short-sha.outputs.gitShortSha }}
 
   deploy-infra:

diff --git a/.github/workflows/ci-cd-staging.yml b/.github/workflows/ci-cd-staging.yml
@@ -68,7 +68,7 @@ jobs:
     secrets:
       GCR_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
     with:
-      dockerImageBaseName: ghcr.io/digdir/dialogporten-
+      dockerImageBaseName: ghcr.io/altinn/dialogporten-
       version: ${{ needs.get-current-version.outputs.version }}
 
   deploy-apps:

diff --git a/.github/workflows/ci-cd-yt01.yml b/.github/workflows/ci-cd-yt01.yml
@@ -38,7 +38,7 @@ jobs:
     secrets:
       GCR_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
     with:
-      dockerImageBaseName: ghcr.io/digdir/dialogporten-
+      dockerImageBaseName: ghcr.io/altinn/dialogporten-
       version: ${{ needs.get-current-version.outputs.version }}
 
   deploy-infra:

diff --git a/README.md b/README.md
@@ -126,11 +126,12 @@ These health checks are integrated with Azure Container Apps' health probe syste
 
 ## Observability with OpenTelemetry
 
-This project uses OpenTelemetry for distributed tracing and metrics collection. The setup includes:
+This project uses OpenTelemetry for distributed tracing, metrics collection, and logging. The setup includes:
 
 ### Core Features
 - Distributed tracing across services
 - Runtime and application metrics
+- Log aggregation and correlation
 - Integration with Azure Monitor/Application Insights
 - Support for both OTLP and Azure Monitor exporters
 - Automatic instrumentation for:
@@ -157,15 +158,72 @@ OpenTelemetry is configured through environment variables that are automatically
 ### Local Development
 
 For local development, the project includes a docker-compose setup with:
-- OpenTelemetry Collector
-- Grafana
-- Other supporting services
+- OpenTelemetry Collector (ports 4317/4318 for OTLP receivers)
+- Grafana (port 3000)
+- Jaeger (port 16686)
+- Loki (port 3100)
+- Prometheus (port 9090)
 
 To run the local observability stack:
 ```bash
 podman compose -f docker-compose-otel.yml up
 ```
 
+### Accessing Observability Tools
+
+Once the local stack is running, you can access the following tools:
+
+#### Distributed Tracing with Jaeger
+- URL: http://localhost:16686
+- Features:
+  - View distributed traces across services
+  - Search by service, operation, or trace ID
+  - Analyze timing and dependencies
+  - Debug request flows and errors
+
+#### Metrics with Prometheus
+- URL: http://localhost:9090
+- Features:
+  - Query raw metrics data
+  - View metric targets and service discovery
+  - Debug metric collection
+
+#### Log Aggregation with Loki
+- Direct URL: http://localhost:3100
+- Grafana Integration: http://localhost:3000 (preferred interface)
+- Features:
+  - Search and filter logs across all services
+  - Correlate logs with traces using trace IDs
+  - Create log-based alerts and dashboards
+  - Use LogQL to query logs:
+    ```logql
+    # Example: Find all error logs
+    {container="web-api"} |= "error"
+    
+    # Example: Find logs with specific trace ID
+    {container=~"web-api|graphql"} |~ "trace_id=([a-f0-9]{32})"
+    ```
+
+#### Metrics and Dashboards in Grafana
+- URL: http://localhost:3000
+- Features:
+  - Pre-configured dashboards for:
+    - Application metrics
+    - Runtime metrics
+    - HTTP request metrics
+  - Data sources:
+    - Prometheus (metrics)
+    - Loki (logs)
+    - Jaeger (traces)
+  - Create custom dashboards
+  - Set up alerts
+
+#### OpenTelemetry Collector Endpoints
+- OTLP gRPC receiver: localhost:4317
+- OTLP HTTP receiver: localhost:4318
+- Prometheus metrics: localhost:8888
+- Prometheus exporter metrics: localhost:8889
+
 ### Request Filtering
 
 The telemetry setup includes smart filtering to:
@@ -252,7 +310,7 @@ For pull requests, the title must follow [Conventional Commits](https://www.conv
 The title of the PR will be used as the commit message when squashing/merging the pull request, and the body of the PR will be used as the description.
 
 This title will be used to generate the changelog (using [Release Please](https://github.com/google-github-actions/release-please-action))
-Using `fix` will add to "Bug Fixes", `feat` will add to "Features". All the others,`chore`, `ci`, etc., will be ignored. ([Example release](https://github.com/digdir/dialogporten/releases/tag/v1.12.0))
+Using `fix` will add to "Bug Fixes", `feat` will add to "Features". All the others,`chore`, `ci`, etc., will be ignored. ([Example release](https://github.com/altinn/dialogporten/releases/tag/v1.12.0))
 
 ## Deployment
 

diff --git a/docker-compose-otel.yml b/docker-compose-otel.yml
@@ -22,15 +22,36 @@ services:
       - "14250:14250" # Model used by collector
     environment:
       - COLLECTOR_OTLP_ENABLED=true
+    healthcheck:
+      test: ["CMD", "wget", "--spider", "-q", "localhost:16686"]
+      interval: 3s
+      timeout: 3s
+      retries: 10
+      start_period: 10s
 
   # Prometheus for metrics
   prometheus:
-    image: prom/prometheus:v3.0.1
+    image: prom/prometheus:v3.1.0
     volumes:
       - ./local-otel-configuration/prometheus.yml:/etc/prometheus/prometheus.yml
     ports:
       - "9090:9090"
 
+  # Loki for log aggregation
+  loki:
+    image: grafana/loki:3.2.2
+    ports:
+      - "3100:3100"
+    volumes:
+      - ./local-otel-configuration/loki-config.yaml:/etc/loki/local-config.yaml
+    command: -config.file=/etc/loki/local-config.yaml
+    healthcheck:
+      test: ["CMD-SHELL", "wget -q --tries=1 -O- http://localhost:3100/ready"]
+      interval: 3s
+      timeout: 3s
+      retries: 10
+      start_period: 10s
+
   # Grafana for metrics visualization
   grafana:
     image: grafana/grafana:11.4.0
@@ -43,3 +64,5 @@ services:
       - ./local-otel-configuration/grafana-datasources.yml:/etc/grafana/provisioning/datasources/datasources.yml
       - ./local-otel-configuration/grafana-dashboards.yml:/etc/grafana/provisioning/dashboards/dashboards.yml
       - ./local-otel-configuration/dashboards:/etc/grafana/provisioning/dashboards
+    depends_on:
+      - loki
diff --git a/docs/schema/V1/package.json b/docs/schema/V1/package.json
@@ -8,7 +8,7 @@
   "author": "DigDir",
   "main": "src/index.js",
   "repository": {
-    "url": "git+https://github.com/digdir/dialogporten.git"
+    "url": "git+https://github.com/Altinn/dialogporten.git"
   },
   "scripts": {
     "build": "./gql-to-js.js",

diff --git a/docs/schema/V1/swagger.verified.json b/docs/schema/V1/swagger.verified.json
@@ -5838,7 +5838,7 @@
         ]
       },
       "post": {
-        "description": "The dialog is created with the given configuration. For more information see the documentation (link TBD).\n\nFor detailed information on validation rules, see [the source for CreateDialogCommandValidator](https://github.com/digdir/dialogporten/blob/main/src/Digdir.Domain.Dialogporten.Application/Features/V1/ServiceOwner/Dialogs/Commands/Create/CreateDialogCommandValidator.cs)",
+        "description": "The dialog is created with the given configuration. For more information see the documentation (link TBD).\n\nFor detailed information on validation rules, see [the source for CreateDialogCommandValidator](https://github.com/altinn/dialogporten/blob/main/src/Digdir.Domain.Dialogporten.Application/Features/V1/ServiceOwner/Dialogs/Commands/Create/CreateDialogCommandValidator.cs)",
         "operationId": "V1ServiceOwnerDialogsCreate_Dialog",
         "requestBody": {
           "content": {
@@ -7036,4 +7036,4 @@
       "url": "https://altinn-dev-api.azure-api.net/dialogporten"
     }
   ]
-}
+}
diff --git a/local-otel-configuration/dashboards/runtime-metrics.json b/local-otel-configuration/dashboards/runtime-metrics.json
@@ -85,13 +85,19 @@
             "type": "timeseries",
             "targets": [
                 {
-                    "datasource": {
-                        "type": "prometheus",
-                        "uid": "Prometheus"
-                    },
                     "expr": "dialogporten_process_runtime_dotnet_gc_heap_size_bytes",
                     "legendFormat": "Heap Size",
                     "refId": "A"
+                },
+                {
+                    "expr": "dialogporten_process_runtime_dotnet_gc_committed_memory_size_bytes",
+                    "legendFormat": "Committed Memory",
+                    "refId": "B"
+                },
+                {
+                    "expr": "dialogporten_dotnet_process_memory_working_set_bytes",
+                    "legendFormat": "Working Set",
+                    "refId": "C"
                 }
             ]
         },
@@ -171,13 +177,14 @@
             "type": "timeseries",
             "targets": [
                 {
-                    "datasource": {
-                        "type": "prometheus",
-                        "uid": "Prometheus"
-                    },
                     "expr": "rate(dialogporten_process_runtime_dotnet_gc_collections_count_total[5m])",
-                    "legendFormat": "Gen {{generation}}",
+                    "legendFormat": "Collections/sec",
                     "refId": "A"
+                },
+                {
+                    "expr": "rate(dialogporten_process_runtime_dotnet_gc_duration_nanoseconds_total[5m])",
+                    "legendFormat": "GC Duration/sec",
+                    "refId": "B"
                 }
             ]
         },
@@ -257,22 +264,19 @@
             "type": "timeseries",
             "targets": [
                 {
-                    "datasource": {
-                        "type": "prometheus",
-                        "uid": "Prometheus"
-                    },
                     "expr": "dialogporten_process_runtime_dotnet_thread_pool_queue_length",
                     "legendFormat": "Queue Length",
                     "refId": "A"
                 },
                 {
-                    "datasource": {
-                        "type": "prometheus",
-                        "uid": "Prometheus"
-                    },
                     "expr": "dialogporten_process_runtime_dotnet_thread_pool_threads_count",
                     "legendFormat": "Thread Count",
                     "refId": "B"
+                },
+                {
+                    "expr": "rate(dialogporten_process_runtime_dotnet_thread_pool_completed_items_count_total[5m])",
+                    "legendFormat": "Completed Items/sec",
+                    "refId": "C"
                 }
             ]
         },
@@ -352,20 +356,12 @@
             "type": "timeseries",
             "targets": [
                 {
-                    "datasource": {
-                        "type": "prometheus",
-                        "uid": "Prometheus"
-                    },
-                    "expr": "rate(dialogporten_process_runtime_dotnet_exceptions_count_total[$__rate_interval])",
+                    "expr": "rate(dialogporten_process_runtime_dotnet_exceptions_count_total[5m])",
                     "legendFormat": "Exceptions/sec",
                     "refId": "A"
                 },
                 {
-                    "datasource": {
-                        "type": "prometheus",
-                        "uid": "Prometheus"
-                    },
-                    "expr": "rate(dialogporten_process_runtime_dotnet_monitor_lock_contention_count_total[$__rate_interval])",
+                    "expr": "rate(dialogporten_process_runtime_dotnet_monitor_lock_contention_count_total[5m])",
                     "legendFormat": "Lock Contentions/sec",
                     "refId": "B"
                 }

diff --git a/local-otel-configuration/grafana-datasources.yml b/local-otel-configuration/grafana-datasources.yml
@@ -5,4 +5,11 @@ datasources:
     type: prometheus
     access: proxy
     url: http://prometheus:9090
-    isDefault: true 
+    isDefault: true
+
+  - name: Loki
+    type: loki
+    access: proxy
+    url: http://loki:3100
+    jsonData:
+      maxLines: 1000