added jsonnet and grafanet into confgenerator code

Makefile

@@ -77,11 +77,6 @@ build_code:
 .PHONY: build
 build: validate_go lint build_code docs ## Build flowlogs-pipeline executable and update the docs
 
-# Note: To change dashboards, change `dashboards.jsonnet`. Do not change manually `dashboards.json`
-.PHONY: dashboards
-dashboards: $(JB) $(JSONNET) ## Build grafana dashboards
-	./hack/generate-dashboards.sh
-
 .PHONY: docs
 docs: FORCE ## Update flowlogs-pipeline documentation
 	@./hack/update-docs.sh

README.md

@@ -864,7 +864,6 @@ General
 Develop  
   lint                  Lint the code  
   build                 Build flowlogs-pipeline executable and update the docs  
-  dashboards            Build grafana dashboards  
   docs                  Update flowlogs-pipeline documentation  
   clean                 Clean  
   tests-unit            Unit tests  

cmd/confgenerator/main.go

@@ -133,6 +133,7 @@ func initFlags() {
 	rootCmd.PersistentFlags().StringVar(&opts.DestConfFile, "destConfFile", "/tmp/flowlogs-pipeline.conf.yaml", "destination configuration file")
 	rootCmd.PersistentFlags().StringVar(&opts.DestDocFile, "destDocFile", "/tmp/metrics.md", "destination documentation file (.md)")
 	rootCmd.PersistentFlags().StringVar(&opts.DestGrafanaJsonnetFolder, "destGrafanaJsonnetFolder", "/tmp/jsonnet", "destination grafana jsonnet folder")
+	rootCmd.PersistentFlags().StringVar(&opts.DestDashboardFolder, "destDashboardFolder", "/tmp/dashboards", "destination grafana dashboard folder")
 	rootCmd.PersistentFlags().StringSliceVar(&opts.SkipWithTags, "skipWithTags", nil, "Skip definitions with Tags")
 	rootCmd.PersistentFlags().StringSliceVar(&opts.GenerateStages, "generateStages", nil, "Produce only specified stages (ingest, transform_generic, transform_network, extract_aggregate, encode_prom, write_loki")
 	rootCmd.PersistentFlags().StringVar(&opts.GlobalMetricsPrefix, "globalMetricsPrefix", "", "Common prefix for all generated metrics, including operational ones")

contrib/dashboards/dashboard_details.json

@@ -181,6 +181,91 @@
             }
          ]
       },
+      {
+         "aliasColors": { },
+         "bars": false,
+         "dashLength": 10,
+         "dashes": false,
+         "datasource": "prometheus",
+         "fill": 1,
+         "fillGradient": 0,
+         "gridPos": {
+            "h": 20,
+            "w": 25,
+            "x": 0,
+            "y": 0
+         },
+         "id": 4,
+         "legend": {
+            "alignAsTable": false,
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "rightSide": false,
+            "show": true,
+            "sideWidth": null,
+            "total": false,
+            "values": false
+         },
+         "lines": true,
+         "linewidth": 1,
+         "links": [ ],
+         "nullPointMode": "null",
+         "percentage": false,
+         "pointradius": 5,
+         "points": false,
+         "renderer": "flot",
+         "repeat": null,
+         "seriesOverrides": [ ],
+         "spaceLength": 10,
+         "stack": false,
+         "steppedLine": false,
+         "targets": [
+            {
+               "expr": "topk(10,rate(flp_bandwidth_per_source_subnet[1m]))",
+               "format": "time_series",
+               "intervalFactor": 2,
+               "legendFormat": "",
+               "refId": "A"
+            }
+         ],
+         "thresholds": [ ],
+         "timeFrom": null,
+         "timeShift": null,
+         "title": "Bandwidth per source subnet",
+         "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+         },
+         "type": "graph",
+         "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": [ ]
+         },
+         "yaxes": [
+            {
+               "format": "short",
+               "label": null,
+               "logBase": 1,
+               "max": null,
+               "min": null,
+               "show": true
+            },
+            {
+               "format": "short",
+               "label": null,
+               "logBase": 1,
+               "max": null,
+               "min": null,
+               "show": true
+            }
+         ]
+      },
       {
          "cards": {
             "cardPadding": null,
@@ -204,7 +289,7 @@
          "heatmap": { },
          "hideZeroBuckets": false,
          "highlightCards": true,
-         "id": 4,
+         "id": 5,
          "legend": {
             "show": false
          },
@@ -250,7 +335,7 @@
             "x": 0,
             "y": 0
          },
-         "id": 5,
+         "id": 6,
          "legend": {
             "alignAsTable": false,
             "avg": false,
@@ -335,7 +420,7 @@
             "x": 0,
             "y": 0
          },
-         "id": 6,
+         "id": 7,
          "legend": {
             "alignAsTable": false,
             "avg": false,
@@ -420,7 +505,7 @@
             "x": 0,
             "y": 0
          },
-         "id": 7,
+         "id": 8,
          "legend": {
             "alignAsTable": false,
             "avg": false,
@@ -505,7 +590,7 @@
             "x": 0,
             "y": 0
          },
-         "id": 8,
+         "id": 9,
          "legend": {
             "alignAsTable": false,
             "avg": false,
@@ -590,7 +675,7 @@
             "x": 0,
             "y": 0
          },
-         "id": 9,
+         "id": 10,
          "legend": {
             "alignAsTable": false,
             "avg": false,
@@ -675,7 +760,7 @@
             "x": 0,
             "y": 0
          },
-         "id": 10,
+         "id": 11,
          "legend": {
             "alignAsTable": false,
             "avg": false,
@@ -760,7 +845,7 @@
             "x": 0,
             "y": 0
          },
-         "id": 11,
+         "id": 12,
          "legend": {
             "alignAsTable": false,
             "avg": false,
@@ -845,7 +930,7 @@
             "x": 0,
             "y": 0
          },
-         "id": 12,
+         "id": 13,
          "legend": {
             "alignAsTable": false,
             "avg": false,
@@ -939,7 +1024,7 @@
          "heatmap": { },
          "hideZeroBuckets": false,
          "highlightCards": true,
-         "id": 13,
+         "id": 14,
          "legend": {
             "show": false
          },
@@ -985,7 +1070,7 @@
             "x": 0,
             "y": 0
          },
-         "id": 14,
+         "id": 15,
          "legend": {
             "alignAsTable": false,
             "avg": false,
@@ -1070,7 +1155,7 @@
             "x": 0,
             "y": 0
          },
-         "id": 15,
+         "id": 16,
          "legend": {
             "alignAsTable": false,
             "avg": false,
@@ -1155,7 +1240,7 @@
             "x": 0,
             "y": 0
          },
-         "id": 16,
+         "id": 17,
          "legend": {
             "alignAsTable": false,
             "avg": false,
@@ -1240,7 +1325,7 @@
             "x": 0,
             "y": 0
          },
-         "id": 17,
+         "id": 18,
          "legend": {
             "alignAsTable": false,
             "avg": false,

contrib/dashboards/jsonnet/dashboard_details.jsonnet

@@ -48,6 +48,22 @@ dashboard.new(
     h: 20,
   }
 )
+.addPanel(
+  graphPanel.new(
+    datasource='prometheus',
+    title="Bandwidth per source subnet",
+  )
+  .addTarget(
+    prometheus.target(
+      expr='topk(10,rate(flp_bandwidth_per_source_subnet[1m]))',
+    )
+  ), gridPos={
+    x: 0,
+    y: 0,
+    w: 25,
+    h: 20,
+  }
+)
 .addPanel(
   heatmapPanel.new(
     datasource='prometheus',

contrib/kubernetes/flowlogs-pipeline.conf.yaml

@@ -42,6 +42,7 @@ parameters:
         output: proto
       - input: Bytes
         output: bytes
+        multiplier: 1
       - input: TCPFlags
         output: TCPFlags
       - input: SrcAS
@@ -50,6 +51,7 @@ parameters:
         output: dstAS
       - input: Packets
         output: packets
+        multiplier: 1
       - input: TimeReceived
         output: timeReceived
 - name: extract_conntrack
@@ -443,7 +445,6 @@ parameters:
         - service
         - _RecordType
         buckets: []
-      address: 0.0.0.0
       port: 9102
       prefix: flp_
 - name: write_loki
@@ -453,3 +454,4 @@ parameters:
       url: http://loki.default.svc.cluster.local:3100
       staticLabels:
         job: flowlogs-pipeline
+

docs/api.md

@@ -149,11 +149,22 @@ Following is the supported API format for network transformations:
                      add_location: add output location fields from input
                      add_service: add output network service field from input port and parameters protocol field
                      add_kubernetes: add output kubernetes fields from input
+                     reinterpret_direction: reinterpret flow direction at a higher level than the interface
+                     add_ip_category: categorize IPs based on known subnets configuration
                  parameters: parameters specific to type
                  assignee: value needs to assign to output field
          kubeConfigPath: path to kubeconfig file (optional)
          servicesFile: path to services file (optional, default: /etc/services)
          protocolsFile: path to protocols file (optional, default: /etc/protocols)
+         ipCategories: configure IP categories
+                 cidrs: list of CIDRs to match a category
+                 name: name of the category
+         directionInfo: information to reinterpret flow direction (optional, to use with reinterpret_direction rule)
+             reporterIPField: field providing the reporter (agent) host IP
+             srcHostField: source host field
+             dstHostField: destination host field
+             flowDirectionField: field providing the flow direction in the input entries; it will be rewritten
+             ifDirectionField: interface-level field for flow direction, to create in output
 </pre>
 ## Write Loki API
 Following is the supported API format for writing to loki:

docs/confGenerator.md

@@ -22,6 +22,7 @@ Usage:
 Flags:
       --config string                     config file (default is $HOME/.confgen)
       --destConfFile string               destination configuration file (default "/tmp/flowlogs-pipeline.conf.yaml")
+      --destDashboardFolder string        destination grafana dashboard folder (default "/tmp/dashboards")
       --destDocFile string                destination documentation file (.md) (default "/tmp/metrics.md")
       --destGrafanaJsonnetFolder string   destination grafana jsonnet folder (default "/tmp/jsonnet")
       --generateStages strings            Produce only specified stages (ingest, transform_generic, transform_network, extract_aggregate, encode_prom, write_loki
@@ -47,6 +48,8 @@ flowlogs-pipeline pipeline steps and visualization configuration
 > Note: It is possible to activate `ConfGenrator` with the default flowlogs-pipeline configuration using the command `make generate-configuration`. 
 > This command compiles the code and generates default outputs.
 
+> Note: `Confgenerator` uses grafana libraries taken from `github.com/grafana/grafonnet-lib`, (commit 30280196507e0fe6fa978a3e0eaca3a62844f817).
+
 ## How to write a network definitions
 
 This is easy and quick:

docs/metrics.md

@@ -36,6 +36,18 @@ and the transformation to generate the exported metric.
 |||  
 
 
+### bandwidth per src subnet
+| **Description** | This metric observes the network bandwidth per source subnet | 
+|:---|:---|
+| **Details** | Sum bytes for all traffic per source subnet | 
+| **Usage** | Evaluate network usage breakdown per source subnet | 
+| **Tags** | bandwidth, graph, rate, subnet |
+| **OperationType** | aggregate by `srcSubnet, _RecordType` and `sum` field `bytes` |
+| **Exposed as** | `flp_bandwidth_per_source_subnet` of type `gauge` |
+| **Visualized as** | "Bandwidth per source subnet" on dashboard `details` |
+|||  
+
+
 ### connection length histogram
 | **Description** | A histogram of connection size in bytes | 
 |:---|:---|

go.mod

@@ -50,7 +50,6 @@ require (
 	github.com/fsnotify/fsnotify v1.5.1 // indirect
 	github.com/go-kit/log v0.2.0 // indirect
 	github.com/go-logfmt/logfmt v0.5.1 // indirect
-	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 	github.com/go-openapi/jsonreference v0.19.5 // indirect
 	github.com/go-openapi/swag v0.19.14 // indirect
@@ -67,6 +66,7 @@ require (
 	github.com/jpillora/backoff v1.0.0 // indirect
 	github.com/klauspost/compress v1.15.9 // indirect
 	github.com/klauspost/cpuid/v2 v2.1.0 // indirect
+	github.com/kr/pretty v0.3.0 // indirect
 	github.com/libp2p/go-reuseport v0.1.0 // indirect
 	github.com/magiconair/properties v1.8.5 // indirect
 	github.com/mailru/easyjson v0.7.6 // indirect
@@ -92,13 +92,12 @@ require (
 	go.uber.org/atomic v1.9.0 // indirect
 	golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect
 	golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect
-	golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect
+	golang.org/x/sys v0.1.0 // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac // indirect
-	gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.66.6 // indirect
@@ -112,4 +111,10 @@ require (
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )
 
+require (
+	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/google/go-jsonnet v0.19.1
+	gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 // indirect
+)
+
 replace github.com/prometheus/common v0.32.1 => github.com/netobserv/prometheus-common v0.31.2-0.20220720134304-43e74fd22881