feat(experimental): kick off inventory project

Track projects and their licenses
Docker compose configuration for dev with logging tracing and metrics (mostly done)
CRUD for mongo

experimental/license-inventory/.gitignore

@@ -0,0 +1 @@
+node_modules

experimental/license-inventory/dev/grafana/datasources.yaml

@@ -0,0 +1,84 @@
+apiVersion: 1
+
+datasources:
+  - name: Loki
+    type: loki
+    uid: loki
+    access: proxy
+    url: http://loki:3100
+    basicAuth: false
+    version: 1
+    editable: false
+    isDefault: true
+
+  - name: Prometheus
+    type: prometheus
+    uid: prometheus
+    access: proxy
+    orgId: 1
+    url: http://prometheus:9090
+    basicAuth: false
+    isDefault: false
+    version: 1
+    editable: false
+    jsonData:
+      httpMethod: GET
+
+  - name: Tempo
+    type: tempo
+    uid: tempo
+    access: proxy
+    orgId: 1
+    url: http://tempo:3200
+    basicAuth: false
+    isDefault: false
+    version: 1
+    editable: false
+    jsonData:
+      tracesToLogsV2:
+        datasourceUid: loki
+        # spanStartTimeShift: '-1h'
+        # spanEndTimeShift: '1h'
+        # tags: ['job', 'instance', 'pod', 'namespace']
+        filterByTraceID: true
+        filterBySpanID: true
+        # customQuery: true
+        # query: 'method="$${__span.tags.method}"'
+      tracesToMetrics:
+        datasourceUid: prometheus
+        # spanStartTimeShift: '1h'
+        # spanEndTimeShift: '-1h'
+        # tags: [{ key: 'service.name', value: 'service' }, { key: 'job' }]
+        # queries:
+        #   - name: 'Sample query'
+        #     query: 'sum(rate(traces_spanmetrics_latency_bucket{$$__tags}[5m]))'
+      # tracesToProfiles:
+      #   datasourceUid: 'grafana-pyroscope-datasource'
+      #   tags: ['job', 'instance', 'pod', 'namespace']
+      #   profileTypeId: 'process_cpu:cpu:nanoseconds:cpu:nanoseconds'
+      #   customQuery: true
+      #   query: 'method="$${__span.tags.method}"'
+      # serviceMap:
+      #   datasourceUid: prometheus
+      # nodeGraph:
+      #   enabled: true
+      # search:
+      #   hide: false
+      # traceQuery:
+      #   timeShiftEnabled: true
+      #   spanStartTimeShift: '1h'
+      #   spanEndTimeShift: '-1h'
+      # spanBar:
+      #   type: 'Tag'
+      #   tag: 'http.path'
+      # streamingEnabled:
+      #   search: true
+  # - name: Jaeger
+  #   type: jaeger
+  #   url: http://jaeger:16686
+  #   version: 1
+  #   editable: true
+  #   # isDefault: true
+  # - name: TestData
+  #   isDefault: false
+  #   type: testdata

experimental/license-inventory/dev/grafana/plugins.yaml

@@ -0,0 +1,12 @@
+apiVersion: 1
+
+apps:
+  - type: "grafana-lokiexplore-app"
+    org_id: 1
+    org_name: "Grafana"
+    disabled: false
+    jsonData:
+      apiUrl: http://localhost
+      isApiKeySet: true
+    secureJsonData:
+      apiKey: example-key

experimental/license-inventory/dev/loki.yaml

@@ -0,0 +1,52 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: /tmp/loki
+  storage:
+    filesystem:
+      chunks_directory: /tmp/loki/chunks
+      rules_directory: /tmp/loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+frontend:
+  max_outstanding_per_tenant: 2048
+
+pattern_ingester:
+  enabled: true
+
+limits_config:
+  max_global_streams_per_user: 0
+  ingestion_rate_mb: 50000
+  ingestion_burst_size_mb: 50000
+  volume_enabled: true
+
+query_range:
+  results_cache:
+    cache:
+      embedded_cache:
+        enabled: true
+        max_size_mb: 100
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+# ruler:
+#   alertmanager_url: http://localhost:9093
+
+analytics:
+  reporting_enabled: false

experimental/license-inventory/dev/otel-collector-config.yaml

@@ -0,0 +1,53 @@
+extensions:
+  health_check:
+  pprof:
+    endpoint: 0.0.0.0:1777
+  zpages:
+    endpoint: 0.0.0.0:55679
+
+receivers:
+  otlp:
+    protocols:
+      http:
+        endpoint: 0.0.0.0:4318
+
+  # collect own metrics
+  prometheus:
+    config:
+      scrape_configs:
+      - job_name: 'otel-collector'
+        scrape_interval: 10s
+        static_configs:
+        - targets: ['0.0.0.0:8888']
+
+exporters:
+  loki:
+    endpoint: http://loki:3100/loki/api/v1/push
+  prometheus:
+    endpoint: "0.0.0.0:8889"
+    namespace: otel-collector
+    send_timestamps: true
+    metric_expiration: 180m
+    enable_open_metrics: true
+    add_metric_suffixes: false
+    resource_to_telemetry_conversion:
+      enabled: true
+  otlphttp:
+   endpoint: http://tempo:4318
+   headers:
+     authorization: Basic <base64 data>
+
+processors:
+  batch:
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      exporters: [otlphttp]
+    metrics:
+      receivers: [otlp]
+      exporters: [prometheus]
+    logs:
+      receivers: [otlp]
+      exporters: [loki]

experimental/license-inventory/dev/prometheus.yaml

@@ -0,0 +1,20 @@
+global:
+  scrape_interval:     15s
+  evaluation_interval: 15s
+
+scrape_configs:
+  - job_name: 'prometheus'
+    static_configs:
+      - targets: [ 'localhost:9090' ]
+  - job_name: 'tempo'
+    static_configs:
+      - targets: [ 'tempo:3200' ]
+  - job_name: 'promtail'
+    static_configs:
+      - targets: [ 'promtail:9080' ]
+  - job_name: 'otel-collector'
+    static_configs:
+      - targets: [ 'otel-collector:8889' ]
+  - job_name: 'otel-collector-meta'
+    static_configs:
+      - targets: [ 'otel-collector:8888' ]

experimental/license-inventory/dev/promtail.yaml

@@ -0,0 +1,23 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+- job_name: system
+  static_configs:
+  - targets:
+      - "localhost"
+    labels:
+      job: mongo
+      __path__: /var/log-mongo/mongo.log*
+  - targets:
+      - "localhost"
+    labels:
+      job: license-inventory
+      __path__: /var/log-license-inventory/inventory.log*

experimental/license-inventory/dev/tempo.yaml

@@ -0,0 +1,50 @@
+stream_over_http_enabled: true
+server:
+  http_listen_port: 3200
+
+query_frontend:
+  search:
+    duration_slo: 5s
+    throughput_bytes_slo: 1.073741824e+09
+    metadata_slo:
+        duration_slo: 5s
+        throughput_bytes_slo: 1.073741824e+09
+  trace_by_id:
+    duration_slo: 5s
+
+distributor:
+  receivers:
+    otlp:
+      protocols:
+        http:
+
+compactor:
+  compaction:
+    block_retention: 1h
+
+metrics_generator:
+  registry:
+    external_labels:
+      source: tempo
+      cluster: docker-compose
+  storage:
+    path: /var/tempo/generator/wal
+    remote_write:
+      - url: http://prometheus:9090/api/v1/write
+        send_exemplars: true
+  traces_storage:
+    path: /var/tempo/generator/traces
+
+storage:
+  trace:
+    backend: local
+    wal:
+      path: /var/tempo/wal
+    local:
+      path: /var/tempo/blocks
+
+overrides:
+  defaults:
+    metrics_generator:
+      processors: [service-graphs, span-metrics, local-blocks]
+      generate_native_histograms: both