chore: Introduce a new http -> pipelines -> blackhole soak (#10142)

* Introduce a new http -> pipelines -> blackhole soak

This commit introduces a new soak to investigate our pipelines in an ongoing
basis. The pipeline configuration for vector and the sample data was contributed
by work was contributed @vladimir-dd. I have expanded the soak to allow mounting
a TESTNAME/data to smuggle static data into the minikube, by which we feed
http_gen. Unfortunately this does not work as we cannot use the Virtual Box VM
for soak testing and kubernetes/minikube#12301 is
open. We will have to figure out some other way to smuggle data into the
minikube for this soak to function.

This commit depends on #10141 which
depends on DataDog/lading#119.

Signed-off-by: Brian L. Troutwine <brian@troutwine.us>

* Use configmap hack

It turns out that the problem discussed in the last commit is fixed by
kubernetes/minikube#13013. This is not present in a
minikube release yet, so we can't rely on it. That said, this commit introduces
a hack whereby the bootstrap is passed in a configmap. This limits the size of
the bootstrap with all the problems that come along with a low-entropy
experiment.

Signed-off-by: Brian L. Troutwine <brian@troutwine.us>

soaks/bin/run_experiment.sh

@@ -82,10 +82,12 @@ SOAK_CAPTURE_DIR="${CAPTURE_DIR}/${SOAK_NAME}"
 
 pushd "${__dir}"
 ./boot_minikube.sh --cpus "${SOAK_CPUS}" --memory "${SOAK_MEMORY}"
-mkdir -p "${SOAK_CAPTURE_DIR}"
+mkdir --parents "${SOAK_CAPTURE_DIR}"
+minikube image load "${IMAGE}"
+# Mount the capture directory. This is where the samples captured from inside
+# the minikube will be placed on the host.
 minikube mount "${SOAK_CAPTURE_DIR}:/captures" &
-minikube cache add "${IMAGE}"
-MOUNT_PID=$!
+CAPTURE_MOUNT_PID=$!
 popd
 
 pushd "${SOAK_ROOT}/tests/${SOAK_NAME}/terraform"
@@ -96,7 +98,7 @@ echo "[${VARIANT}] Sleeping for ${WARMUP_GRACE} seconds to allow warm-up"
 sleep "${WARMUP_GRACE}"
 echo "[${VARIANT}] Recording captures to ${SOAK_CAPTURE_DIR}"
 sleep "${TOTAL_SAMPLES}"
-kill "${MOUNT_PID}"
+kill "${CAPTURE_MOUNT_PID}"
 popd
 
 pushd "${__dir}"

soaks/common/terraform/modules/lading_http_blackhole/variables.tf

@@ -15,5 +15,5 @@ variable "http-blackhole-yaml" {
 
 variable "lading_image" {
   description = "The lading image to run"
-  type = string
+  type        = string
 }

soaks/common/terraform/modules/lading_http_gen/main.tf

@@ -1,3 +1,14 @@
+resource "kubernetes_config_map" "lading_bootstrap" {
+  metadata {
+    name      = "lading-http-gen-bootstrap"
+    namespace = var.namespace
+  }
+
+  data = {
+    "bootstrap.log" = var.http-gen-static-bootstrap
+  }
+}
+
 resource "kubernetes_config_map" "lading" {
   metadata {
     name      = "lading-http-gen"
@@ -70,6 +81,11 @@ resource "kubernetes_deployment" "http-gen" {
           name              = "http-gen"
           command           = ["/http_gen"]
 
+          volume_mount {
+            mount_path = "/data"
+            name       = "data"
+            read_only  = true
+          }
           volume_mount {
             mount_path = "/etc/lading"
             name       = "etc-lading"
@@ -106,6 +122,12 @@ resource "kubernetes_deployment" "http-gen" {
             name = kubernetes_config_map.lading.metadata[0].name
           }
         }
+        volume {
+          name = "data"
+          config_map {
+            name = kubernetes_config_map.lading_bootstrap.metadata[0].name
+          }
+        }
       }
     }
   }

soaks/common/terraform/modules/lading_http_gen/variables.tf

@@ -13,7 +13,14 @@ variable "http-gen-yaml" {
   type        = string
 }
 
+variable "http-gen-static-bootstrap" {
+  description = "Boostrap log to be used for static variant, mounted at /data/boostrap.log"
+  type        = string
+  default     = ""
+}
+
+
 variable "lading_image" {
   description = "The lading image to run"
-  type = string
+  type        = string
 }

soaks/common/terraform/modules/lading_splunk_hec_blackhole/variables.tf

@@ -15,5 +15,5 @@ variable "splunk-hec-blackhole-yaml" {
 
 variable "lading_image" {
   description = "The lading image to run"
-  type = string
+  type        = string
 }

soaks/common/terraform/modules/lading_splunk_hec_gen/variables.tf

@@ -15,5 +15,5 @@ variable "splunk-hec-gen-yaml" {
 
 variable "lading_image" {
   description = "The lading image to run"
-  type = string
+  type        = string
 }

soaks/common/terraform/modules/lading_tcp_gen/variables.tf

@@ -15,5 +15,5 @@ variable "tcp-gen-yaml" {
 
 variable "lading_image" {
   description = "The lading image to run"
-  type = string
+  type        = string
 }

soaks/tests/http_pipelines_blackhole/README.md

@@ -0,0 +1,9 @@
+# HTTP -> Pipelines -> Datadog Logs
+
+This soak tests the http sink feeding through multiple pipeline transforms down
+to blackhole sink. It is a complicated topology.
+
+## Method
+
+Lading `http_gen` is used to generate log load into vector. The vector internal
+blackhole is sink for this configuration.

soaks/tests/http_pipelines_blackhole/terraform/http_gen.yaml

@@ -0,0 +1,15 @@
+worker_threads: 1
+prometheus_addr: "0.0.0.0:9090"
+
+targets:
+  vector:
+    headers: {}
+    target_uri: "http://vector:8282/"
+    bytes_per_second: "500 Mb"
+    parallel_connections: 10
+    method:
+      post:
+        maximum_prebuild_cache_size_bytes: "256 Mb"
+        variant:
+          static:
+            static_path: "/data/bootstrap.log"

soaks/tests/http_pipelines_blackhole/terraform/main.tf

@@ -0,0 +1,60 @@
+# Set up the providers needed to run this soak and other terraform related
+# business. Here we only require 'kubernetes' to interact with the soak
+# minikube.
+terraform {
+  required_providers {
+    kubernetes = {
+      version = "~> 2.5.0"
+      source  = "hashicorp/kubernetes"
+    }
+  }
+}
+
+# Rig the kubernetes provider to communicate with minikube. The details of
+# adjusting `~/.kube/config` are addressed by the soak control scripts.
+provider "kubernetes" {
+  config_path = "~/.kube/config"
+}
+
+# Setup background monitoring details. These are needed by the soak control to
+# understand what vector et al's running behavior is.
+module "monitoring" {
+  source       = "../../../common/terraform/modules/monitoring"
+  type         = var.type
+  vector_image = var.vector_image
+}
+
+# Setup the soak pieces
+#
+# This soak config sets up a vector soak with lading/http-gen feeding into vector,
+# lading/http-blackhole receiving.
+resource "kubernetes_namespace" "soak" {
+  metadata {
+    name = "soak"
+  }
+}
+
+module "vector" {
+  source       = "../../../common/terraform/modules/vector"
+  type         = var.type
+  vector_image = var.vector_image
+  vector-toml  = file("${path.module}/vector.toml")
+  namespace    = kubernetes_namespace.soak.metadata[0].name
+  vector_cpus  = var.vector_cpus
+  depends_on   = [module.monitoring]
+}
+module "http-gen" {
+  source                    = "../../../common/terraform/modules/lading_http_gen"
+  type                      = var.type
+  http-gen-yaml             = file("${path.module}/http_gen.yaml")
+  # This is a hack. Ultimately this creates a configmap in the minikube, where
+  # we would _prefer_ to simply mount a directory into the kube. This is not
+  # possible, pending introductoin of
+  # https://github.com/kubernetes/minikube/issues/12301 into a release. Keep in
+  # mind that the bootstrap _must_ be below 1MB in size, which severely limits
+  # the entropy of our experiment.
+  http-gen-static-bootstrap = file("${path.module}/data/http_gen_bootstrap.log")
+  namespace                 = kubernetes_namespace.soak.metadata[0].name
+  lading_image              = var.lading_image
+  depends_on                = [module.monitoring, module.vector]
+}

soaks/tests/http_pipelines_blackhole/terraform/variables.tf

@@ -0,0 +1,19 @@
+variable "type" {
+  description = "The type of the vector install, whether 'baseline' or 'comparison'"
+  type        = string
+}
+
+variable "vector_image" {
+  description = "The image of vector to use in this investigation"
+  type        = string
+}
+
+variable "vector_cpus" {
+  description = "The total number of CPUs to give to vector"
+  type        = number
+}
+
+variable "lading_image" {
+  description = "The lading image to run"
+  type        = string
+}