feat(clusters): add pegboard pool type (#1152)

<!-- Please make sure there is an issue that this PR is correlated to. -->

## Changes

<!-- If there are frontend changes, please include screenshots. -->

fern/definition/admin/clusters/common.yml

@@ -6,6 +6,7 @@ types:
       - job
       - gg
       - ats
+      - pegboard
 
   Provider:
     enum:

lib/bolt/cli/src/commands/cluster/datacenter.rs

@@ -40,6 +40,7 @@ pub enum DatacenterPoolType {
 	Job,
 	Gg,
 	Ats,
+	Pegboard,
 }
 
 impl From<DatacenterPoolType> for models::AdminClustersPoolType {
@@ -48,6 +49,7 @@ impl From<DatacenterPoolType> for models::AdminClustersPoolType {
 			DatacenterPoolType::Job => models::AdminClustersPoolType::Job,
 			DatacenterPoolType::Gg => models::AdminClustersPoolType::Gg,
 			DatacenterPoolType::Ats => models::AdminClustersPoolType::Ats,
+			DatacenterPoolType::Pegboard => models::AdminClustersPoolType::Pegboard,
 		}
 	}
 }
@@ -329,6 +331,7 @@ mod render {
 			Some(models::AdminClustersPoolType::Job) => "Job".to_string(),
 			Some(models::AdminClustersPoolType::Gg) => "GG".to_string(),
 			Some(models::AdminClustersPoolType::Ats) => "ATS".to_string(),
+			Some(models::AdminClustersPoolType::Pegboard) => "Pegboard".to_string(),
 			None => String::new(),
 		}
 	}

lib/bolt/cli/src/commands/cluster/server.rs

@@ -123,6 +123,7 @@ impl SubCommand {
 						"job" => Ok(models::AdminClustersPoolType::Job),
 						"gg" => Ok(models::AdminClustersPoolType::Gg),
 						"ats" => Ok(models::AdminClustersPoolType::Ats),
+						"pb" | "pegboard" => Ok(models::AdminClustersPoolType::Pegboard),
 						_ => Err(anyhow!("invalid pool type")),
 					})
 					.transpose()?;
@@ -165,6 +166,7 @@ impl SubCommand {
 						"job" => Ok(models::AdminClustersPoolType::Job),
 						"gg" => Ok(models::AdminClustersPoolType::Gg),
 						"ats" => Ok(models::AdminClustersPoolType::Ats),
+						"pb" | "pegboard" => Ok(models::AdminClustersPoolType::Pegboard),
 						_ => Err(anyhow!("invalid pool type")),
 					})
 					.transpose()?;
@@ -203,6 +205,7 @@ impl SubCommand {
 						"job" => Ok(models::AdminClustersPoolType::Job),
 						"gg" => Ok(models::AdminClustersPoolType::Gg),
 						"ats" => Ok(models::AdminClustersPoolType::Ats),
+						"pb" | "pegboard" => Ok(models::AdminClustersPoolType::Pegboard),
 						_ => Err(anyhow!("invalid pool type")),
 					})
 					.transpose()?;
@@ -240,6 +243,7 @@ impl SubCommand {
 						"job" => Ok(models::AdminClustersPoolType::Job),
 						"gg" => Ok(models::AdminClustersPoolType::Gg),
 						"ats" => Ok(models::AdminClustersPoolType::Ats),
+						"pb" | "pegboard" => Ok(models::AdminClustersPoolType::Pegboard),
 						_ => Err(anyhow!("invalid pool type")),
 					})
 					.transpose()?;
@@ -284,6 +288,7 @@ impl SubCommand {
 						"job" => Ok(models::AdminClustersPoolType::Job),
 						"gg" => Ok(models::AdminClustersPoolType::Gg),
 						"ats" => Ok(models::AdminClustersPoolType::Ats),
+						"pb" | "pegboard" => Ok(models::AdminClustersPoolType::Pegboard),
 						_ => Err(anyhow!("invalid pool type")),
 					})
 					.transpose()?;
@@ -323,6 +328,7 @@ impl SubCommand {
 						"job" => Ok(models::AdminClustersPoolType::Job),
 						"gg" => Ok(models::AdminClustersPoolType::Gg),
 						"ats" => Ok(models::AdminClustersPoolType::Ats),
+						"pb" | "pegboard" => Ok(models::AdminClustersPoolType::Pegboard),
 						_ => Err(anyhow!("invalid pool type")),
 					})
 					.transpose()?;
@@ -392,6 +398,7 @@ mod render {
 			models::AdminClustersPoolType::Job => "Job".to_string(),
 			models::AdminClustersPoolType::Gg => "GG".to_string(),
 			models::AdminClustersPoolType::Ats => "ATS".to_string(),
+			models::AdminClustersPoolType::Pegboard => "Pegboard".to_string(),
 		}
 	}
 }

lib/bolt/core/src/context/project.rs

@@ -313,9 +313,14 @@ impl ProjectContextData {
 					.get(&config::ns::ProvisioningDatacenterPoolType::Job);
 				let job_count = job_pool.map(|pool| pool.desired_count).unwrap_or_default();
 
+				let pb_pool = datacenter
+					.pools
+					.get(&config::ns::ProvisioningDatacenterPoolType::Pegboard);
+				let job_count = pb_pool.map(|pool| pool.desired_count).unwrap_or_default();
+
 				assert_ne!(
-					job_count, 0,
-					"invalid datacenter ({}): Missing job servers",
+					job_count + pb_count, 0,
+					"invalid datacenter ({}): Missing job or pegboard servers",
 					name_id,
 				);
 				assert!(
@@ -328,6 +333,16 @@ impl ProjectContextData {
 					"invalid datacenter ({}): Job min > desired",
 					name_id,
 				);
+				assert!(
+					pb_count <= pb_pool.unwrap().max_count,
+					"invalid datacenter ({}): Pegboard desired > max",
+					name_id,
+				);
+				assert!(
+					pb_pool.unwrap().min_count <= pb_pool.unwrap().desired_count,
+					"invalid datacenter ({}): Pegboard min > desired",
+					name_id,
+				);
 
 				// Validate Linode
 				#[allow(irrefutable_let_patterns)]
@@ -353,6 +368,14 @@ impl ProjectContextData {
 							name_id,
 						);
 					}
+
+					if let Some(pb_pool) = &pb_pool {
+						assert!(
+							pb_pool.drain_timeout >= 55 * 60 * 1000,
+							"invalid datacenter ({}): Pegboard drain timeout < 55 min (Linode bills hourly, drain timeout should be close to hour intervals)",
+							name_id,
+						);
+					}
 				}
 			}
 		}

lib/convert/src/impls/admin.rs

@@ -11,6 +11,7 @@ impl ApiFrom<models::AdminClustersPoolType> for cluster::types::PoolType {
 			models::AdminClustersPoolType::Job => cluster::types::PoolType::Job,
 			models::AdminClustersPoolType::Gg => cluster::types::PoolType::Gg,
 			models::AdminClustersPoolType::Ats => cluster::types::PoolType::Ats,
+			models::AdminClustersPoolType::Pegboard => cluster::types::PoolType::Pegboard,
 		}
 	}
 }
@@ -21,6 +22,7 @@ impl ApiFrom<cluster::types::PoolType> for models::AdminClustersPoolType {
 			cluster::types::PoolType::Job => models::AdminClustersPoolType::Job,
 			cluster::types::PoolType::Gg => models::AdminClustersPoolType::Gg,
 			cluster::types::PoolType::Ats => models::AdminClustersPoolType::Ats,
+			cluster::types::PoolType::Pegboard => models::AdminClustersPoolType::Pegboard,
 		}
 	}
 }

svc/api/admin/src/route/clusters/datacenters.rs

@@ -58,7 +58,7 @@ pub async fn create(
 	// is to make sure that the datacenter starts in a valid state.
 	let pools = vec![
 		cluster::types::Pool {
-			pool_type: cluster::types::PoolType::Job,
+			pool_type: cluster::types::PoolType::Pegboard,
 			hardware: vec![cluster::types::Hardware {
 				provider_hardware: "g6-nanode-1".to_string(),
 			}],

svc/pkg/cluster/src/ops/datacenter/topology_get.rs

@@ -66,6 +66,8 @@ pub async fn cluster_datacenter_topology_get(
 	)
 	.await?;
 
+	todo!("implement for pegboard also");
+
 	// Fetch batch data from nomad
 	let (allocation_info, node_info) = tokio::try_join!(
 		async {

svc/pkg/cluster/src/types.rs

@@ -75,6 +75,7 @@ pub enum PoolType {
 	Job = 0,
 	Gg = 1,
 	Ats = 2,
+	Pegboard = 3,
 }
 
 impl std::fmt::Display for PoolType {
@@ -83,6 +84,7 @@ impl std::fmt::Display for PoolType {
 			PoolType::Job => write!(f, "job"),
 			PoolType::Gg => write!(f, "gg"),
 			PoolType::Ats => write!(f, "ats"),
+			PoolType::Pegboard => write!(f, "pegboard"),
 		}
 	}
 }

svc/pkg/cluster/src/util/mod.rs

@@ -20,13 +20,8 @@ pub fn default_cluster_id() -> Uuid {
 
 pub fn server_name(provider_datacenter_id: &str, pool_type: PoolType, server_id: Uuid) -> String {
 	let ns = util::env::namespace();
-	let pool_type_str = match pool_type {
-		PoolType::Job => "job",
-		PoolType::Gg => "gg",
-		PoolType::Ats => "ats",
-	};
 
-	format!("{ns}-{provider_datacenter_id}-{pool_type_str}-{server_id}",)
+	format!("{ns}-{provider_datacenter_id}-{pool_type}-{server_id}")
 }
 
 pub(crate) async fn cf_client(

svc/pkg/cluster/src/workflows/prebake.rs

@@ -38,7 +38,7 @@ pub async fn cluster_prebake(ctx: &mut WorkflowCtx, input: &Input) -> GlobalResu
 					api_token: dc.provider_api_token.clone(),
 					hardware: linode::util::consts::PREBAKE_HARDWARE.to_string(),
 					firewall_preset: match input.pool_type {
-						PoolType::Job => linode::types::FirewallPreset::Job,
+						PoolType::Job | PoolType::Pegboard => linode::types::FirewallPreset::Job,
 						PoolType::Gg => linode::types::FirewallPreset::Gg,
 						PoolType::Ats => linode::types::FirewallPreset::Ats,
 					},

svc/pkg/cluster/src/workflows/server/install/install_scripts/components/vector.rs

@@ -28,12 +28,6 @@ pub fn configure(config: &Config, pool_type: PoolType) -> String {
 		.collect::<Vec<_>>()
 		.join(", ");
 
-	let pool_type_str = match pool_type {
-		PoolType::Job => "job",
-		PoolType::Gg => "gg",
-		PoolType::Ats => "ats",
-	};
-
 	let mut config_str = formatdoc!(
 		r#"
 		[api]
@@ -52,7 +46,7 @@ pub fn configure(config: &Config, pool_type: PoolType) -> String {
 			.tags.server_id = "___SERVER_ID___"
 			.tags.datacenter_id = "___DATACENTER_ID___"
 			.tags.cluster_id = "___CLUSTER_ID___"
-			.tags.pool_type = "{pool_type_str}"
+			.tags.pool_type = "{pool_type}"
 			.tags.public_ip = "${{PUBLIC_IP}}"
 			'''
 

svc/pkg/cluster/src/workflows/server/install/install_scripts/files/cni_plugins.sh

@@ -11,4 +11,3 @@ tar -xz -C /opt/cni-plugins-$version -f /tmp/cni-plugins.tgz
 # Copy plugins to /opt/cni/bin
 mkdir -p /opt/cni/bin /opt/cni/config
 cp -r /opt/cni-plugins-$version/* /opt/cni/bin/
-

svc/pkg/cluster/src/workflows/server/install/install_scripts/mod.rs

@@ -51,6 +51,15 @@ pub async fn gen_install(
 			script.push(components::docker::install());
 			script.push(components::traffic_server::install());
 		}
+		PoolType::Pegboard => {
+			script.push(components::docker::install());
+			script.push(components::lz4::install());
+			script.push(components::skopeo::install());
+			script.push(components::umoci::install());
+			script.push(components::cni::tool());
+			script.push(components::cni::plugins());
+			script.push(components::pegboard::install());
+		}
 	}
 
 	// MARK: Common (post)
@@ -113,6 +122,17 @@ pub async fn gen_initialize(pool_type: PoolType, datacenter_id: Uuid) -> GlobalR
 		PoolType::Ats => {
 			script.push(components::traffic_server::configure().await?);
 		}
+		PoolType::Pegboard => {
+			script.push(components::pegboard::configure()?);
+
+			prometheus_targets.insert(
+				"pegboard".into(),
+				components::vector::PrometheusTarget {
+					endpoint: todo!(),
+					scrape_interval: 15,
+				},
+			);
+		}
 	}
 
 	// MARK: Common (post)

svc/pkg/cluster/src/workflows/server/install/mod.rs

@@ -262,11 +262,7 @@ fn insert_metrics(
 			&dc.datacenter_id.to_string(),
 			&dc.provider_datacenter_id,
 			&dc.name_id,
-			match pool_type {
-				PoolType::Job => "job",
-				PoolType::Gg => "gg",
-				PoolType::Ats => "ats",
-			},
+			&pool_type.to_string(),
 		])
 		.observe(dt);
 }

svc/pkg/cluster/src/workflows/server/mod.rs

@@ -99,7 +99,7 @@ pub(crate) async fn cluster_server(ctx: &mut WorkflowCtx, input: &Input) -> Glob
 						api_token: dc.provider_api_token.clone(),
 						hardware: hardware.provider_hardware.clone(),
 						firewall_preset: match input.pool_type {
-							PoolType::Job => linode::types::FirewallPreset::Job,
+							PoolType::Job | PoolType::Pegboard => linode::types::FirewallPreset::Job,
 							PoolType::Gg => linode::types::FirewallPreset::Gg,
 							PoolType::Ats => linode::types::FirewallPreset::Ats,
 						},
@@ -313,7 +313,7 @@ struct GetVlanIpInput {
 async fn get_vlan_ip(ctx: &ActivityCtx, input: &GetVlanIpInput) -> GlobalResult<Ipv4Addr> {
 	// Find next available vlan index
 	let mut vlan_addr_range = match input.pool_type {
-		PoolType::Job => util::net::job::vlan_addr_range(),
+		PoolType::Job | PoolType::Pegboard => util::net::job::vlan_addr_range(),
 		PoolType::Gg => util::net::gg::vlan_addr_range(),
 		PoolType::Ats => util::net::ats::vlan_addr_range(),
 	};

svc/pkg/cluster/standalone/default-update/src/lib.rs

@@ -44,13 +44,12 @@ struct Pool {
 }
 
 #[derive(Deserialize, PartialEq, Eq, Hash)]
+#[serde(rename_all = "snake_case")]
 enum PoolType {
-	#[serde(rename = "job")]
 	Job,
-	#[serde(rename = "gg")]
 	Gg,
-	#[serde(rename = "ats")]
 	Ats,
+	Pegboard,
 }
 
 impl From<PoolType> for cluster::types::PoolType {
@@ -59,6 +58,7 @@ impl From<PoolType> for cluster::types::PoolType {
 			PoolType::Job => cluster::types::PoolType::Job,
 			PoolType::Gg => cluster::types::PoolType::Gg,
 			PoolType::Ats => cluster::types::PoolType::Ats,
+			PoolType::Pegboard => cluster::types::PoolType::Pegboard,
 		}
 	}
 }
@@ -172,7 +172,7 @@ pub async fn run_from_env(use_autoscaler: bool) -> GlobalResult<()> {
 				.map(|(pool_type, pool)| {
 					let desired_count = match pool_type {
 						PoolType::Ats | PoolType::Gg => Some(pool.desired_count),
-						PoolType::Job => {
+						PoolType::Job | PoolType::Pegboard => {
 							if use_autoscaler {
 								None
 							} else {

svc/pkg/cluster/standalone/metrics-publish/src/lib.rs

@@ -127,6 +127,13 @@ fn insert_metrics(dc: &Datacenter, servers: &[Server]) -> GlobalResult<()> {
 				.filter(|s| matches!(s.pool_type, PoolType::Ats))
 				.collect::<Vec<_>>(),
 		),
+		(
+			PoolType::Pegboard,
+			servers_in_dc
+				.clone()
+				.filter(|s| matches!(s.pool_type, PoolType::Pegboard))
+				.collect::<Vec<_>>(),
+		),
 	];
 
 	// Aggregate all states per pool type
@@ -135,6 +142,7 @@ fn insert_metrics(dc: &Datacenter, servers: &[Server]) -> GlobalResult<()> {
 		let mut installing = 0;
 		let mut active = 0;
 		let mut nomad = 0;
+		let mut pegboard = 0;
 		let mut draining = 0;
 		let mut tainted = 0;
 		let mut draining_tainted = 0;
@@ -149,6 +157,10 @@ fn insert_metrics(dc: &Datacenter, servers: &[Server]) -> GlobalResult<()> {
 					if server.has_nomad_node {
 						nomad += 1;
 					}
+
+					if server.has_pegboard_client {
+						pegboard += 1;
+					}
 				} else {
 					installing += 1;
 				}
@@ -192,15 +204,27 @@ fn insert_metrics(dc: &Datacenter, servers: &[Server]) -> GlobalResult<()> {
 			.with_label_values(&labels)
 			.set(draining_tainted);
 
-		if let PoolType::Job = pool_type {
-			metrics::NOMAD_SERVERS
+		match pool_type {
+			PoolType::Job => {
+				metrics::NOMAD_SERVERS
+					.with_label_values(&[
+						&cluster_id,
+						&datacenter_id,
+						&dc.provider_datacenter_id,
+						&dc.name_id,
+					])
+					.set(nomad);
+			}
+			PoolType::Pegboard => {
+				metrics::PEGBOARD_SERVERS
 				.with_label_values(&[
 					&cluster_id,
 					&datacenter_id,
 					&dc.provider_datacenter_id,
 					&dc.name_id,
 				])
-				.set(nomad);
+				.set(pegboard);
+			}
 		}
 	}