Add macos-observ-lib

docs/macos-observ-lib/.gitignore

@@ -0,0 +1,2 @@
+jsonnetfile.lock.json
+vendor

docs/macos-observ-lib/README.md

@@ -0,0 +1,86 @@
+# MacOS exporter observability lib
+
+This jsonnet observability lib can be used to generate observability package for node exporter(MacOS).
+
+## Import
+
+```sh
+jb init
+jb install https://github.com/grafana/node_exporter/docs/macos-observ-lib
+```
+
+## Examples
+
+### Example 1: Basic example
+
+You can use observ-lib to fill in monitoring-mixin structure:
+
+```jsonnet
+// mixin.libsonnet file
+local macoslib = import 'macos-observ-lib/main.libsonnet';
+
+local mac =
+  macoslib.new()
+  + macoslib.withConfigMixin({
+    filteringSelector: 'job=~".*mac.*"',
+    groupLabels: ['job'],
+    instanceLabels: ['instance'],
+    dashboardNamePrefix: 'MacOS / ',
+    dashboardTags: ['macos-mixin'],
+    uid: 'darwin',
+    // enable loki logs
+    enableLokiLogs: true,
+  });
+
+{
+  grafanaDashboards+:: mac.grafana.dashboards,
+  prometheusAlerts+:: mac.prometheus.alerts,
+  prometheusRules+:: mac.prometheus.recordingRules,
+}
+
+```
+For more examples see [node-observ-lib](../node-observ-lib).
+
+## Collectors used:
+
+Grafana Agent or combination of node_exporter/promtail can be used in order to collect data required.
+
+### Logs collection
+
+Loki logs are used to populate logs dashboard and also for annotations.
+
+To use logs, you need to opt-in, with setting `enableLokiLogs: true` in config.
+
+See example above.
+
+The following scrape snippet can be used in grafana-agent/promtail:
+
+```yaml
+    - job_name: integrations/node_exporter_direct_scrape
+      static_configs:
+      - targets:
+        - localhost
+        labels:
+          __path__: /var/log/*.log
+          instance: '<your-instance-name>'
+          job: integrations/macos-node
+      pipeline_stages:
+      - multiline:
+          firstline: '^([\w]{3} )?[\w]{3} +[\d]+ [\d]+:[\d]+:[\d]+|[\w]{4}-[\w]{2}-[\w]{2} [\w]{2}:[\w]{2}:[\w]{2}(?:[+-][\w]{2})?'
+      - regex:
+          expression: '(?P<timestamp>([\w]{3} )?[\w]{3} +[\d]+ [\d]+:[\d]+:[\d]+|[\w]{4}-[\w]{2}-[\w]{2} [\w]{2}:[\w]{2}:[\w]{2}(?:[+-][\w]{2})?) (?P<hostname>\S+) (?P<sender>.+?)\[(?P<pid>\d+)\]:? (?P<message>(?s:.*))$'
+      - labels:
+          sender:
+          hostname:
+          pid:
+      - match:
+          selector: '{sender!="", pid!=""}'
+          stages:
+            - template:
+                source: message
+                template: '{{ .sender }}[{{ .pid }}]: {{ .message }}'
+            - labeldrop:
+                - pid
+            - output:
+                source: message
+```

docs/macos-observ-lib/config.libsonnet

@@ -0,0 +1,53 @@
+{
+
+  //  any modular observability library should inlcude as inputs:
+  // 'dashboardNamePrefix' - Use as prefix for all Dashboards and (optional) rule groups
+  // 'filteringSelector' - Static selector to apply to ALL dashboard variables of type query, panel queries, alerts and recording rules.
+  // 'groupLabels' - one or more labels that can be used to identify 'group' of instances. In simple cases, can be 'job' or 'cluster'.
+  // 'instanceLabels' - one or more labels that can be used to identify single entity of instances. In simple cases, can be 'instance' or 'pod'.
+  // 'uid' - UID to prefix all dashboards original uids
+
+  filteringSelector: 'job="integrations/macos-node"',
+  groupLabels: ['job'],
+  instanceLabels: ['instance'],
+  dashboardNamePrefix: 'MacOS / ',
+  uid: 'darwin',
+
+  dashboardTags: [self.uid],
+
+  // Select the fstype for filesystem-related queries. If left
+  // empty, all filesystems are selected. If you have unusual
+  // filesystem you don't want to include in dashboards and
+  // alerting, you can exclude them here, e.g. 'fstype!="tmpfs"'.
+  fsSelector: 'fstype!=""',
+
+  // Select the mountpoint for filesystem-related queries. If left
+  // empty, all mountpoints are selected. For example if you have a
+  // special purpose tmpfs instance that has a fixed size and will
+  // always be 100% full, but you still want alerts and dashboards for
+  // other tmpfs instances, you can exclude those by mountpoint prefix
+  // like so: 'mountpoint!~"/var/lib/foo.*"'.
+  fsMountpointSelector: 'mountpoint!=""',
+
+  // Select the device for disk-related queries. If left empty, all
+  // devices are selected. If you have unusual devices you don't
+  // want to include in dashboards and alerting, you can exclude
+  // them here, e.g. 'device!="tmpfs"'.
+  diskDeviceSelector: 'device!=""',
+  dashboardPeriod: 'now-1h',
+  dashboardTimezone: 'default',
+  dashboardRefresh: '1m',
+
+  // Alerts to keep from node-observ-lib:
+  alertsMacKeep: ['NodeFilesystemAlmostOutOfSpace'],
+
+  // logs lib related
+  enableLokiLogs: true,
+  extraLogLabels: ['filename', 'sender'],
+  logsVolumeGroupBy: 'sender',
+  showLogsVolume: true,
+  logsFilteringSelector: self.filteringSelector,
+  logsExtraFilters: '',
+
+
+}

docs/macos-observ-lib/g.libsonnet

@@ -0,0 +1 @@
+import 'github.com/grafana/grafonnet/gen/grafonnet-v10.0.0/main.libsonnet'

docs/macos-observ-lib/jsonnetfile.json

@@ -0,0 +1,41 @@
+{
+  "version": 1,
+  "dependencies": [
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/grafana/grafonnet.git",
+          "subdir": "gen/grafonnet-v10.0.0"
+        }
+      },
+      "version": "main"
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/grafana/jsonnet-libs.git",
+          "subdir": "common-lib"
+        }
+      },
+      "version": "master"
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/grafana/jsonnet-libs.git",
+          "subdir": "logs-lib"
+        }
+      },
+      "version": "master"
+    },
+    {
+      "source": {
+        "local": {
+          "directory": "../node-observ-lib"
+        }
+      },
+      "version": ""
+    }
+  ],
+  "legacyImports": true
+}

docs/macos-observ-lib/main.libsonnet

@@ -0,0 +1,71 @@
+local config = import './config.libsonnet';
+local g = import './g.libsonnet';
+local panels = import './panels.libsonnet';
+local targets = import './targets.libsonnet';
+local nodelib = import 'node-observ-lib/main.libsonnet';
+
+
+// inherit nodelib
+nodelib
+{
+
+  new():
+    super.new()
+    + nodelib.withConfigMixin(config)
+    +
+    {
+      local this = self,
+      local parentGrafana = super.grafana,
+      local parentPrometheus = super.prometheus,
+
+      grafana+: {
+        // drop backToFleet link
+        links+: {
+          backToFleet:: {},
+        },
+        annotations: {
+          // keep only reboot annotation
+          reboot: parentGrafana.annotations.reboot,
+        },
+        // override targets (memory)
+        targets+: targets.new(this),
+        // override panels (update description and targets in panels)
+        panels+: panels.new(this),
+
+        // keep only overview and logs(optionally) dashes
+        dashboards:
+          {
+            overview: parentGrafana.dashboards.overview,
+          }
+          +
+          (
+            if this.config.enableLokiLogs
+            then
+              {
+                logs: parentGrafana.dashboards.logs,
+              }
+          ),
+      },
+      prometheus+: {
+        recordingRules: {},
+        alerts:
+          {
+            groups:
+              [
+                {
+                  name: group.name,
+                  rules: [
+                    rule
+                    for rule in group.rules
+                    if std.length(std.find(rule.alert, this.config.alertsMacKeep)) > 0
+                  ],
+                }
+                for group in parentPrometheus.alerts.groups
+              ],
+
+          },
+
+      },
+    },
+
+}

docs/macos-observ-lib/mixin.libsonnet

@@ -0,0 +1,8 @@
+local macoslib = import './main.libsonnet';
+local macos = macoslib.new();
+
+{
+  grafanaDashboards+:: macos.grafana.dashboards,
+  prometheusAlerts+:: macos.prometheus.alerts,
+  prometheusRules+:: macos.prometheus.recordingRules,
+}

docs/macos-observ-lib/panels.libsonnet

@@ -0,0 +1,38 @@
+local g = import './g.libsonnet';
+local commonlib = import 'common-lib/common/main.libsonnet';
+local utils = commonlib.utils;
+{
+  new(this):
+    {
+      local t = this.grafana.targets,
+      local table = g.panel.table,
+      local fieldOverride = g.panel.table.fieldOverride,
+      local instanceLabel = this.config.instanceLabels[0],
+
+      // override description and targets
+      memoryUsageTsBytes+:
+        g.panel.timeSeries.panelOptions.withDescription(
+          |||
+            - Physical memory: Total amount of memory installed in this computer;
+            - App memory: Physical memory allocated by apps and system processes;
+            - Wired memory: Physical memory, containing data that cannot be compressed or swapped to disk;
+            - Compressed memory: Physical memory used to store a compressed version of data that has not been used recently;
+            - Swap used: Amount of compressed data temporarily moved to disk to make room in memory for more recently used data.
+          |||
+        )
+        + g.panel.timeSeries.queryOptions.withTargets([
+          t.memoryUsedBytes,
+          t.memoryTotalBytes,
+          t.memoryAppBytes,
+          t.memoryWiredBytes,
+          t.memoryCompressedBytes,
+          t.memorySwapUsedBytes,
+        ])
+        + commonlib.panels.generic.timeSeries.threshold.stylizeByRegexp('Physical memory'),
+
+      //override reduceOption field to version
+      osInfo+:
+        g.panel.timeSeries.panelOptions.withTitle('OS version')
+        + { options+: { reduceOptions: { fields: '/^version$/' } } },
+    },
+}

docs/macos-observ-lib/targets.libsonnet

@@ -0,0 +1,87 @@
+local g = import './g.libsonnet';
+local prometheusQuery = g.query.prometheus;
+local lokiQuery = g.query.loki;
+
+{
+  new(this): {
+    local variables = this.grafana.variables,
+    local config = this.config,
+    local prometheusDatasource = '${' + variables.datasources.prometheus.name + '}',
+    local lokiDatasource = '${' + variables.datasources.loki.name + '}',
+
+    memoryTotalBytes:
+      prometheusQuery.new(
+        prometheusDatasource,
+        'node_memory_total_bytes{%(queriesSelector)s}' % variables
+      )
+      + prometheusQuery.withLegendFormat('Physical memory'),
+
+    memoryUsedBytes:
+      prometheusQuery.new(
+        prometheusDatasource,
+        |||
+          (
+            node_memory_internal_bytes{%(queriesSelector)s} -
+            node_memory_purgeable_bytes{%(queriesSelector)s} +
+            node_memory_wired_bytes{%(queriesSelector)s} +
+            node_memory_compressed_bytes{%(queriesSelector)s}
+          )
+        ||| % variables
+      )
+      + prometheusQuery.withLegendFormat('Memory used'),
+    memoryAppBytes:
+      prometheusQuery.new(
+        prometheusDatasource,
+        |||
+          (
+              node_memory_internal_bytes{%(queriesSelector)s} -
+              node_memory_purgeable_bytes{%(queriesSelector)s}
+          )
+        ||| % variables
+      )
+      + prometheusQuery.withLegendFormat('App memory'),
+    memoryWiredBytes:
+      prometheusQuery.new(
+        prometheusDatasource,
+        'node_memory_wired_bytes{%(queriesSelector)s}' % variables
+      )
+      + prometheusQuery.withLegendFormat('Wired memory'),
+    memoryCompressedBytes:
+      prometheusQuery.new(
+        prometheusDatasource,
+        'node_memory_compressed_bytes{%(queriesSelector)s}' % variables
+      )
+      + prometheusQuery.withLegendFormat('Compressed memory'),
+
+    memoryUsagePercent:
+      prometheusQuery.new(
+        prometheusDatasource,
+        |||
+          (
+              (
+                avg(node_memory_internal_bytes{%(queriesSelector)s}) -
+                avg(node_memory_purgeable_bytes{%(queriesSelector)s}) +
+                avg(node_memory_wired_bytes{%(queriesSelector)s}) +
+                avg(node_memory_compressed_bytes{%(queriesSelector)s})
+              ) /
+              avg(node_memory_total_bytes{%(queriesSelector)s})
+          )
+          *
+          100
+        |||
+        % variables,
+      ),
+    memorySwapTotal:
+      prometheusQuery.new(
+        prometheusDatasource,
+        'node_memory_swap_total_bytes{%(queriesSelector)s}' % variables
+      ),
+
+    memorySwapUsedBytes:
+      prometheusQuery.new(
+        prometheusDatasource,
+        'node_memory_swap_used_bytes{%(queriesSelector)s}' % variables
+      )
+      + prometheusQuery.withLegendFormat('Swap used'),
+  },
+}