diff --git a/CHANGELOG.md b/CHANGELOG.md index 32b59d6ab42d..d85b732d9d6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ IMPROVEMENTS: * client: Added support for Azure fingerprinting. [[GH-8979](https://github.com/hashicorp/nomad/issues/8979)] * client: Added support for fingerprinting the client node's Consul segment. [[GH-7214](https://github.com/hashicorp/nomad/issues/7214)] * client: Updated consul-template to v0.25.0 - config function_blacklist deprecated and replaced with function_denylist [[GH-8988](https://github.com/hashicorp/nomad/pull/8988)] + * client: Use ec2 CPU perf data from AWS API [[GH-7830](https://github.com/hashicorp/nomad/issues/7830)] * consul: Support Consul namespace (Consul Enterprise) in client configuration. [[GH-8849](https://github.com/hashicorp/nomad/pull/8849)] * csi: Relaxed validation requirements when checking volume capabilities with controller plugins, to accommodate existing plugin behaviors. [[GH-9049](https://github.com/hashicorp/nomad/issues/9049)] * driver/docker: Upgrade pause container and detect architecture [[GH-8957](https://github.com/hashicorp/nomad/pull/8957)] diff --git a/client/fingerprint/env_aws.go b/client/fingerprint/env_aws.go index bfe4ea24d571..6e54076de1bc 100644 --- a/client/fingerprint/env_aws.go +++ b/client/fingerprint/env_aws.go @@ -46,353 +46,6 @@ var ec2NetSpeedTable = map[*regexp.Regexp]int{ regexp.MustCompile(`.*\.32xlarge`): 10000, } -type ec2Specs struct { - mhz float64 - cores int - model string -} - -func (e ec2Specs) ticks() int { - return int(e.mhz) * e.cores -} - -func specs(ghz float64, vCores int, model string) ec2Specs { - return ec2Specs{ - mhz: ghz * 1000, - cores: vCores, - model: model, - } -} - -// Map of instance type to documented CPU speed. -// -// Most values are taken from https://aws.amazon.com/ec2/instance-types/. -// Values for a1 & m6g (Graviton) are taken from https://en.wikichip.org/wiki/annapurna_labs/alpine/al73400 -// Values for inf1 are taken from launching a inf1.xlarge and looking at /proc/cpuinfo -// -// In a few cases, AWS has upgraded the generation of CPU while keeping the same -// instance designation. Since it is possible to launch on the lower performance -// CPU, that one is used as the spec for the instance type. -// -// This table is provided as a best-effort to determine the number of CPU ticks -// available for use by Nomad tasks. If an instance type is missing, the fallback -// behavior is to use values from go-psutil, which is only capable of reading -// "current" CPU MHz. -var ec2ProcSpeedTable = map[string]ec2Specs{ - // -- General Purpose -- - - // a1 - "a1.medium": specs(2.3, 1, "AWS Graviton"), - "a1.large": specs(2.3, 2, "AWS Graviton"), - "a1.xlarge": specs(2.3, 4, "AWS Graviton"), - "a1.2xlarge": specs(2.3, 8, "AWS Graviton"), - "a1.4xlarge": specs(2.3, 16, "AWS Graviton"), - "a1.metal": specs(2.3, 16, "AWS Graviton"), - - // t3 - "t3.nano": specs(2.5, 2, "2.5 GHz Intel Scalable"), - "t3.micro": specs(2.5, 2, "2.5 GHz Intel Scalable"), - "t3.small": specs(2.5, 2, "2.5 GHz Intel Scalable"), - "t3.medium": specs(2.5, 2, "2.5 GHz Intel Scalable"), - "t3.large": specs(2.5, 2, "2.5 GHz Intel Scalable"), - "t3.xlarge": specs(2.5, 4, "2.5 GHz Intel Scalable"), - "t3.2xlarge": specs(2.5, 8, "2.5 GHz Intel Scalable"), - - // t3a - "t3a.nano": specs(2.5, 2, "2.5 GHz AMD EPYC 7000 series"), - "t3a.micro": specs(2.5, 2, "2.5 GHz AMD EPYC 7000 series"), - "t3a.small": specs(2.5, 2, "2.5 GHz AMD EPYC 7000 series"), - "t3a.medium": specs(2.5, 2, "2.5 GHz AMD EPYC 7000 series"), - "t3a.large": specs(2.5, 2, "2.5 GHz AMD EPYC 7000 series"), - "t3a.xlarge": specs(2.5, 4, "2.5 GHz AMD EPYC 7000 series"), - "t3a.2xlarge": specs(2.5, 8, "2.5 GHz AMD EPYC 7000 series"), - - // t2 - "t2.nano": specs(3.3, 1, "3.3 GHz Intel Scalable"), - "t2.micro": specs(3.3, 1, "3.3 GHz Intel Scalable"), - "t2.small": specs(3.3, 1, "3.3 GHz Intel Scalable"), - "t2.medium": specs(3.3, 2, "3.3 GHz Intel Scalable"), - "t2.large": specs(3.0, 2, "3.0 GHz Intel Scalable"), - "t2.xlarge": specs(3.0, 4, "3.0 GHz Intel Scalable"), - "t2.2xlarge": specs(3.0, 8, "3.0 GHz Intel Scalable"), - - // m6g - "m6g.medium": specs(2.3, 1, "AWS Graviton2 Neoverse"), - "m6g.large": specs(2.3, 2, "AWS Graviton2 Neoverse"), - "m6g.xlarge": specs(2.3, 4, "AWS Graviton2 Neoverse"), - "m6g.2xlarge": specs(2.3, 8, "AWS Graviton2 Neoverse"), - "m6g.4xlarge": specs(2.3, 16, "AWS Graviton2 Neoverse"), - "m6g.8xlarge": specs(2.3, 32, "AWS Graviton2 Neoverse"), - "m6g.12xlarge": specs(2.3, 48, "AWS Graviton2 Neoverse"), - "m6g.16xlarge": specs(2.3, 64, "AWS Graviton2 Neoverse"), - - // m5, m5d - "m5.large": specs(3.1, 2, "3.1 GHz Intel Xeon Platinum"), - "m5.xlarge": specs(3.1, 4, "3.1 GHz Intel Xeon Platinum"), - "m5.2xlarge": specs(3.1, 8, "3.1 GHz Intel Xeon Platinum"), - "m5.4xlarge": specs(3.1, 16, "3.1 GHz Intel Xeon Platinum"), - "m5.8xlarge": specs(3.1, 32, "3.1 GHz Intel Xeon Platinum"), - "m5.12xlarge": specs(3.1, 48, "3.1 GHz Intel Xeon Platinum"), - "m5.16xlarge": specs(3.1, 64, "3.1 GHz Intel Xeon Platinum"), - "m5.24xlarge": specs(3.1, 96, "3.1 GHz Intel Xeon Platinum"), - "m5.metal": specs(3.1, 96, "3.1 GHz Intel Xeon Platinum"), - "m5d.large": specs(3.1, 2, "3.1 GHz Intel Xeon Platinum"), - "m5d.xlarge": specs(3.1, 4, "3.1 GHz Intel Xeon Platinum"), - "m5d.2xlarge": specs(3.1, 8, "3.1 GHz Intel Xeon Platinum"), - "m5d.4xlarge": specs(3.1, 16, "3.1 GHz Intel Xeon Platinum"), - "m5d.8xlarge": specs(3.1, 32, "3.1 GHz Intel Xeon Platinum"), - "m5d.12xlarge": specs(3.1, 48, "3.1 GHz Intel Xeon Platinum"), - "m5d.16xlarge": specs(3.1, 64, "3.1 GHz Intel Xeon Platinum"), - "m5d.24xlarge": specs(3.1, 96, "3.1 GHz Intel Xeon Platinum"), - "m5d.metal": specs(3.1, 96, "3.1 GHz Intel Xeon Platinum"), - - // m5a, m5ad - "m5a.large": specs(2.5, 2, "2.5 GHz AMD EPYC 7000 series"), - "m5a.xlarge": specs(2.5, 4, "2.5 GHz AMD EPYC 7000 series"), - "m5a.2xlarge": specs(2.5, 8, "2.5 GHz AMD EPYC 7000 series"), - "m5a.4xlarge": specs(2.5, 16, "2.5 GHz AMD EPYC 7000 series"), - "m5a.8xlarge": specs(2.5, 32, "2.5 GHz AMD EPYC 7000 series"), - "m5a.12xlarge": specs(2.5, 48, "2.5 GHz AMD EPYC 7000 series"), - "m5a.16xlarge": specs(2.5, 64, "2.5 GHz AMD EPYC 7000 series"), - "m5a.24xlarge": specs(2.5, 96, "2.5 GHz AMD EPYC 7000 series"), - "m5ad.large": specs(2.5, 2, "2.5 GHz AMD EPYC 7000 series"), - "m5ad.xlarge": specs(2.5, 4, "2.5 GHz AMD EPYC 7000 series"), - "m5ad.2xlarge": specs(2.5, 8, "2.5 GHz AMD EPYC 7000 series"), - "m5ad.4xlarge": specs(2.5, 16, "2.5 GHz AMD EPYC 7000 series"), - "m5ad.12xlarge": specs(2.5, 48, "2.5 GHz AMD EPYC 7000 series"), - "m5ad.24xlarge": specs(2.5, 96, "2.5 GHz AMD EPYC 7000 series"), - - // m5n, m5dn - "m5n.large": specs(3.1, 2, "3.1 GHz Intel Xeon Scalable"), - "m5n.xlarge": specs(3.1, 4, "3.1 GHz Intel Xeon Scalable"), - "m5n.2xlarge": specs(3.1, 8, "3.1 GHz Intel Xeon Scalable"), - "m5n.4xlarge": specs(3.1, 16, "3.1 GHz Intel Xeon Scalable"), - "m5n.8xlarge": specs(3.1, 32, "3.1 GHz Intel Xeon Scalable"), - "m5n.12xlarge": specs(3.1, 48, "3.1 GHz Intel Xeon Scalable"), - "m5n.16xlarge": specs(3.1, 64, "3.1 GHz Intel Xeon Scalable"), - "m5n.24xlarge": specs(3.1, 96, "3.1 GHz Intel Xeon Scalable"), - "m5dn.large": specs(3.1, 2, "3.1 GHz Intel Xeon Scalable"), - "m5dn.xlarge": specs(3.1, 4, "3.1 GHz Intel Xeon Scalable"), - "m5dn.2xlarge": specs(3.1, 8, "3.1 GHz Intel Xeon Scalable"), - "m5dn.4xlarge": specs(3.1, 16, "3.1 GHz Intel Xeon Scalable"), - "m5dn.8xlarge": specs(3.1, 32, "3.1 GHz Intel Xeon Scalable"), - "m5dn.12xlarge": specs(3.1, 48, "3.1 GHz Intel Xeon Scalable"), - "m5dn.16xlarge": specs(3.1, 64, "3.1 GHz Intel Xeon Scalable"), - "m5dn.24xlarge": specs(3.1, 96, "3.1 GHz Intel Xeon Scalable"), - - // m4 - "m4.large": specs(2.3, 2, "2.3 GHz Intel Xeon® E5-2686 v4"), - "m4.xlarge": specs(2.3, 4, "2.3 GHz Intel Xeon® E5-2686 v4"), - "m4.2xlarge": specs(2.3, 8, "2.3 GHz Intel Xeon® E5-2686 v4"), - "m4.4xlarge": specs(2.3, 16, "2.3 GHz Intel Xeon® E5-2686 v4"), - "m4.10xlarge": specs(2.3, 40, "2.3 GHz Intel Xeon® E5-2686 v4"), - "m4.16xlarge": specs(2.3, 64, "2.3 GHz Intel Xeon® E5-2686 v4"), - - // -- Compute Optimized -- - - // c5, c5d - "c5.large": specs(3.4, 2, "3.4 GHz Intel Xeon Platinum 8000"), - "c5.xlarge": specs(3.4, 4, "3.4 GHz Intel Xeon Platinum 8000"), - "c5.2xlarge": specs(3.4, 8, "3.4 GHz Intel Xeon Platinum 8000"), - "c5.4xlarge": specs(3.4, 16, "3.4 GHz Intel Xeon Platinum 8000"), - "c5.9xlarge": specs(3.4, 36, "3.4 GHz Intel Xeon Platinum 8000"), - "c5.12xlarge": specs(3.6, 48, "3.6 GHz Intel Xeon Scalable"), - "c5.18xlarge": specs(3.6, 72, "3.6 GHz Intel Xeon Scalable"), - "c5.24xlarge": specs(3.6, 96, "3.6 GHz Intel Xeon Scalable"), - "c5.metal": specs(3.6, 96, "3.6 GHz Intel Xeon Scalable"), - "c5d.large": specs(3.4, 2, "3.4 GHz Intel Xeon Platinum 8000"), - "c5d.xlarge": specs(3.4, 4, "3.4 GHz Intel Xeon Platinum 8000"), - "c5d.2xlarge": specs(3.4, 8, "3.4 GHz Intel Xeon Platinum 8000"), - "c5d.4xlarge": specs(3.4, 16, "3.4 GHz Intel Xeon Platinum 8000"), - "c5d.9xlarge": specs(3.4, 36, "3.4 GHz Intel Xeon Platinum 8000"), - "c5d.12xlarge": specs(3.6, 48, "3.6 GHz Intel Xeon Scalable"), - "c5d.18xlarge": specs(3.6, 72, "3.6 GHz Intel Xeon Scalable"), - "c5d.24xlarge": specs(3.6, 96, "3.6 GHz Intel Xeon Scalable"), - "c5d.metal": specs(3.6, 96, "3.6 GHz Intel Xeon Scalable"), - - // c5n - "c5n.large": specs(3.0, 2, "3.0 GHz Intel Xeon Platinum"), - "c5n.xlarge": specs(3.0, 4, "3.0 GHz Intel Xeon Platinum"), - "c5n.2xlarge": specs(3.0, 8, "3.0 GHz Intel Xeon Platinum"), - "c5n.4xlarge": specs(3.0, 16, "3.0 GHz Intel Xeon Platinum"), - "c5n.9xlarge": specs(3.0, 36, "3.0 GHz Intel Xeon Platinum"), - "c5n.18xlarge": specs(3.0, 72, "3.0 GHz Intel Xeon Platinum"), - "c5n.metal": specs(3.0, 72, "3.0 GHz Intel Xeon Platinum"), - - // c4 - "c4.large": specs(2.9, 2, "2.9 GHz Intel Xeon E5-2666 v3"), - "c4.xlarge": specs(2.9, 4, "2.9 GHz Intel Xeon E5-2666 v3"), - "c4.2xlarge": specs(2.9, 8, "2.9 GHz Intel Xeon E5-2666 v3"), - "c4.4xlarge": specs(2.9, 16, "2.9 GHz Intel Xeon E5-2666 v3"), - "c4.8xlarge": specs(2.9, 36, "2.9 GHz Intel Xeon E5-2666 v3"), - - // -- Memory Optimized -- - - // r5, r5d - "r5.large": specs(3.1, 2, "3.1 GHz Intel Xeon Platinum 8175"), - "r5.xlarge": specs(3.1, 4, "3.1 GHz Intel Xeon Platinum 8175"), - "r5.2xlarge": specs(3.1, 8, "3.1 GHz Intel Xeon Platinum 8175"), - "r5.4xlarge": specs(3.1, 16, "3.1 GHz Intel Xeon Platinum 8175"), - "r5.8xlarge": specs(3.1, 32, "3.1 GHz Intel Xeon Platinum 8175"), - "r5.12xlarge": specs(3.1, 48, "3.1 GHz Intel Xeon Platinum 8175"), - "r5.16xlarge": specs(3.1, 64, "3.1 GHz Intel Xeon Platinum 8175"), - "r5.24xlarge": specs(3.1, 96, "3.1 GHz Intel Xeon Platinum 8175"), - "r5.metal": specs(3.1, 96, "3.1 GHz Intel Xeon Platinum 8175"), - "r5d.large": specs(3.1, 2, "3.1 GHz Intel Xeon Platinum 8175"), - "r5d.xlarge": specs(3.1, 4, "3.1 GHz Intel Xeon Platinum 8175"), - "r5d.2xlarge": specs(3.1, 8, "3.1 GHz Intel Xeon Platinum 8175"), - "r5d.4xlarge": specs(3.1, 16, "3.1 GHz Intel Xeon Platinum 8175"), - "r5d.8xlarge": specs(3.1, 32, "3.1 GHz Intel Xeon Platinum 8175"), - "r5d.12xlarge": specs(3.1, 48, "3.1 GHz Intel Xeon Platinum 8175"), - "r5d.16xlarge": specs(3.1, 64, "3.1 GHz Intel Xeon Platinum 8175"), - "r5d.24xlarge": specs(3.1, 96, "3.1 GHz Intel Xeon Platinum 8175"), - "r5d.metal": specs(3.1, 96, "3.1 GHz Intel Xeon Platinum 8175"), - - // r5a, r5ad - "r5a.large": specs(2.5, 2, "2.5 GHz AMD EPYC 7000 series"), - "r5a.xlarge": specs(2.5, 4, "2.5 GHz AMD EPYC 7000 series"), - "r5a.2xlarge": specs(2.5, 8, "2.5 GHz AMD EPYC 7000 series"), - "r5a.4xlarge": specs(2.5, 16, "2.5 GHz AMD EPYC 7000 series"), - "r5a.8xlarge": specs(2.5, 32, "2.5 GHz AMD EPYC 7000 series"), - "r5a.12xlarge": specs(2.5, 48, "2.5 GHz AMD EPYC 7000 series"), - "r5a.16xlarge": specs(2.5, 64, "2.5 GHz AMD EPYC 7000 series"), - "r5a.24xlarge": specs(2.5, 96, "2.5 GHz AMD EPYC 7000 series"), - "r5ad.large": specs(2.5, 2, "2.5 GHz AMD EPYC 7000 series"), - "r5ad.xlarge": specs(2.5, 4, "2.5 GHz AMD EPYC 7000 series"), - "r5ad.2xlarge": specs(2.5, 8, "2.5 GHz AMD EPYC 7000 series"), - "r5ad.4xlarge": specs(2.5, 16, "2.5 GHz AMD EPYC 7000 series"), - "r5ad.8xlarge": specs(2.5, 32, "2.5 GHz AMD EPYC 7000 series"), - "r5ad.12xlarge": specs(2.5, 48, "2.5 GHz AMD EPYC 7000 series"), - "r5ad.16xlarge": specs(2.5, 64, "2.5 GHz AMD EPYC 7000 series"), - "r5ad.24xlarge": specs(2.5, 96, "2.5 GHz AMD EPYC 7000 series"), - - // r5n - "r5n.large": specs(3.1, 2, "3.1 GHz Intel Xeon Scalable"), - "r5n.xlarge": specs(3.1, 4, "3.1 GHz Intel Xeon Scalable"), - "r5n.2xlarge": specs(3.1, 8, "3.1 GHz Intel Xeon Scalable"), - "r5n.4xlarge": specs(3.1, 16, "3.1 GHz Intel Xeon Scalable"), - "r5n.8xlarge": specs(3.1, 32, "3.1 GHz Intel Xeon Scalable"), - "r5n.12xlarge": specs(3.1, 48, "3.1 GHz Intel Xeon Scalable"), - "r5n.16xlarge": specs(3.1, 64, "3.1 GHz Intel Xeon Scalable"), - "r5n.24xlarge": specs(3.1, 96, "3.1 GHz Intel Xeon Scalable"), - "r5dn.large": specs(3.1, 2, "3.1 GHz Intel Xeon Scalable"), - "r5dn.xlarge": specs(3.1, 4, "3.1 GHz Intel Xeon Scalable"), - "r5dn.2xlarge": specs(3.1, 8, "3.1 GHz Intel Xeon Scalable"), - "r5dn.4xlarge": specs(3.1, 16, "3.1 GHz Intel Xeon Scalable"), - "r5dn.8xlarge": specs(3.1, 32, "3.1 GHz Intel Xeon Scalable"), - "r5dn.12xlarge": specs(3.1, 48, "3.1 GHz Intel Xeon Scalable"), - "r5dn.16xlarge": specs(3.1, 64, "3.1 GHz Intel Xeon Scalable"), - "r5dn.24xlarge": specs(3.1, 96, "3.1 GHz Intel Xeon Scalable"), - - // r4 - "r4.large": specs(2.3, 2, "2.3 GHz Intel Xeon E5-2686 v4"), - "r4.xlarge": specs(2.3, 4, "2.3 GHz Intel Xeon E5-2686 v4"), - "r4.2xlarge": specs(2.3, 8, "2.3 GHz Intel Xeon E5-2686 v4"), - "r4.4xlarge": specs(2.3, 16, "2.3 GHz Intel Xeon E5-2686 v4"), - "r4.8xlarge": specs(2.3, 32, "2.3 GHz Intel Xeon E5-2686 v4"), - "r4.16xlarge": specs(2.3, 64, "2.3 GHz Intel Xeon E5-2686 v4"), - - // x1e - "x1e.xlarge": specs(2.3, 4, "2.3 GHz Intel Xeon E7-8880 v3"), - "x1e.2xlarge": specs(2.3, 8, "2.3 GHz Intel Xeon E7-8880 v3"), - "x1e.4xlarge": specs(2.3, 16, "2.3 GHz Intel Xeon E7-8880 v3"), - "x1e.8xlarge": specs(2.3, 32, "2.3 GHz Intel Xeon E7-8880 v3"), - "x1e.16xlarge": specs(2.3, 64, "2.3 GHz Intel Xeon E7-8880 v3"), - "x1e.32xlarge": specs(2.3, 128, "2.3 GHz Intel Xeon E7-8880 v3"), - - // x1 - "x1.16xlarge": specs(2.3, 64, "2.3 GHz Intel Xeon E7-8880 v3"), - "x1.32xlarge": specs(2.3, 64, "2.3 GHz Intel Xeon E7-8880 v3"), - - // high-memory - "u-6tb1.metal": specs(2.1, 448, "2.1 GHz Intel Xeon Platinum 8176M"), - "u-9tb1.metal": specs(2.1, 448, "2.1 GHz Intel Xeon Platinum 8176M"), - "u-12tb1.metal": specs(2.1, 448, "2.1 GHz Intel Xeon Platinum 8176M"), - "u-18tb1.metal": specs(2.7, 448, "2.7 GHz Intel Xeon Scalable"), - "u-24tb1.metal": specs(2.7, 448, "2.7 GHz Intel Xeon Scalable"), - - // z1d - "z1d.large": specs(4.0, 2, "4.0 GHz Intel Xeon Scalable"), - "z1d.xlarge": specs(4.0, 4, "4.0 GHz Intel Xeon Scalable"), - "z1d.2xlarge": specs(4.0, 8, "4.0 GHz Intel Xeon Scalable"), - "z1d.3xlarge": specs(4.0, 12, "4.0 GHz Intel Xeon Scalable"), - "z1d.6xlarge": specs(4.0, 24, "4.0 GHz Intel Xeon Scalable"), - "z1d.12xlarge": specs(4.0, 48, "4.0 GHz Intel Xeon Scalable"), - "z1d.metal": specs(4.0, 48, "4.0 GHz Intel Xeon Scalable"), - - // -- Accelerated Computing -- - - // p3, p3dn - "p3.2xlarge": specs(2.3, 8, "2.3 GHz Intel Xeon E5-2686 v4"), - "p3.8xlarge": specs(2.3, 32, "2.3 GHz Intel Xeon E5-2686 v4"), - "p3.16xlarge": specs(2.3, 64, "2.3 GHz Intel Xeon E5-2686 v4"), - "p3dn.24xlarge": specs(2.5, 96, "2.5 GHz Intel Xeon P-8175M"), - - // p2 - "p2.xlarge": specs(2.3, 4, "2.3 GHz Intel Xeon E5-2686 v4"), - "p2.8xlarge": specs(2.3, 32, "2.3 GHz Intel Xeon E5-2686 v4"), - "p2.16xlarge": specs(2.3, 64, "2.3 GHz Intel Xeon E5-2686 v4"), - - // inf1 - "inf1.xlarge": specs(3.0, 4, "3.0 GHz Intel Xeon Platinum 8275CL"), - "inf1.2xlarge": specs(3.0, 8, "3.0 GHz Intel Xeon Platinum 8275CL"), - "inf1.6xlarge": specs(3.0, 24, "3.0 GHz Intel Xeon Platinum 8275CL"), - "inf1.24xlarge": specs(3.0, 96, "3.0 GHz Intel Xeon Platinum 8275CL"), - - // g4dn - "g4dn.xlarge": specs(2.5, 4, "2.5 GHz Cascade Lake 24C"), - "g4dn.2xlarge": specs(2.5, 8, "2.5 GHz Cascade Lake 24C"), - "g4dn.4xlarge": specs(2.5, 16, "2.5 GHz Cascade Lake 24C"), - "g4dn.8xlarge": specs(2.5, 32, "2.5 GHz Cascade Lake 24C"), - "g4dn.16xlarge": specs(2.5, 64, "2.5 GHz Cascade Lake 24C"), - "g4dn.12xlarge": specs(2.5, 48, "2.5 GHz Cascade Lake 24C"), - "g4dn.metal": specs(2.5, 96, "2.5 GHz Cascade Lake 24C"), - - // g3 - "g3s.xlarge": specs(2.3, 4, "2.3 GHz Intel Xeon E5-2686 v4"), - "g3s.4xlarge": specs(2.3, 16, "2.3 GHz Intel Xeon E5-2686 v4"), - "g3s.8xlarge": specs(2.3, 32, "2.3 GHz Intel Xeon E5-2686 v4"), - "g3s.16xlarge": specs(2.3, 64, "2.3 GHz Intel Xeon E5-2686 v4"), - - // f1 - "f1.2xlarge": specs(2.3, 8, "Intel Xeon E5-2686 v4"), - "f1.4xlarge": specs(2.3, 16, "Intel Xeon E5-2686 v4"), - "f1.16xlarge": specs(2.3, 64, "Intel Xeon E5-2686 v4"), - - // -- Storage Optimized -- - - // i3 - "i3.large": specs(2.3, 2, "2.3 GHz Intel Xeon E5 2686 v4"), - "i3.xlarge": specs(2.3, 4, "2.3 GHz Intel Xeon E5 2686 v4"), - "i3.2xlarge": specs(2.3, 8, "2.3 GHz Intel Xeon E5 2686 v4"), - "i3.4xlarge": specs(2.3, 16, "2.3 GHz Intel Xeon E5 2686 v4"), - "i3.8xlarge": specs(2.3, 32, "2.3 GHz Intel Xeon E5 2686 v4"), - "i3.16xlarge": specs(2.3, 64, "2.3 GHz Intel Xeon E5 2686 v4"), - "i3.metal": specs(2.3, 72, "2.3 GHz Intel Xeon E5 2686 v4"), - - // i3en - "i3en.large": specs(3.1, 2, "3.1 GHz Intel Xeon Scalable"), - "i3en.xlarge": specs(3.1, 4, "3.1 GHz Intel Xeon Scalable"), - "i3en.2xlarge": specs(3.1, 8, "3.1 GHz Intel Xeon Scalable"), - "i3en.3xlarge": specs(3.1, 12, "3.1 GHz Intel Xeon Scalable"), - "i3en.6xlarge": specs(3.1, 24, "3.1 GHz Intel Xeon Scalable"), - "i3en.12xlarge": specs(3.1, 48, "3.1 GHz Intel Xeon Scalable"), - "i3en.24xlarge": specs(3.1, 96, "3.1 GHz Intel Xeon Scalable"), - "i3en.metal": specs(3.1, 96, "3.1 GHz Intel Xeon Scalable"), - - // d2 - "d2.xlarge": specs(2.4, 4, "2.4 GHz Intel Xeon E5-2676 v3"), - "d2.2xlarge": specs(2.4, 8, "2.4 GHz Intel Xeon E5-2676 v3"), - "d2.4xlarge": specs(2.4, 16, "2.4 GHz Intel Xeon E5-2676 v3"), - "d2.8xlarge": specs(2.4, 36, "2.4 GHz Intel Xeon E5-2676 v3"), - - // h1 - "h1.2xlarge": specs(2.3, 8, "2.3 GHz Intel Xeon E5 2686 v4"), - "h1.4xlarge": specs(2.3, 16, "2.3 GHz Intel Xeon E5 2686 v4"), - "h1.8xlarge": specs(2.3, 32, "2.3 GHz Intel Xeon E5 2686 v4"), - "h1.16xlarge": specs(2.3, 64, "2.3 GHz Intel Xeon E5 2686 v4"), -} - // EnvAWSFingerprint is used to fingerprint AWS metadata type EnvAWSFingerprint struct { StaticFingerprinter @@ -521,14 +174,13 @@ func (f *EnvAWSFingerprint) Fingerprint(request *FingerprintRequest, response *F // copy over CPU speed information if specs := f.lookupCPU(ec2meta); specs != nil { - response.AddAttribute("cpu.modelname", specs.model) - response.AddAttribute("cpu.frequency", fmt.Sprintf("%.0f", specs.mhz)) - response.AddAttribute("cpu.numcores", fmt.Sprintf("%d", specs.cores)) - f.logger.Debug("lookup ec2 cpu", "cores", specs.cores, "MHz", log.Fmt("%.0f", specs.mhz), "model", specs.model) + response.AddAttribute("cpu.frequency", fmt.Sprintf("%d", specs.MHz)) + response.AddAttribute("cpu.numcores", fmt.Sprintf("%d", specs.Cores)) + f.logger.Debug("lookup ec2 cpu", "cores", specs.Cores, "ghz", log.Fmt("%.1f", specs.GHz())) - if ticks := specs.ticks(); request.Config.CpuCompute <= 0 { + if ticks := specs.Ticks(); request.Config.CpuCompute <= 0 { response.AddAttribute("cpu.totalcompute", fmt.Sprintf("%d", ticks)) - f.logger.Debug("setting ec2 cpu ticks", "ticks", ticks) + f.logger.Debug("setting ec2 cpu", "ticks", ticks) resources = new(structs.Resources) resources.CPU = ticks if nodeResources == nil { @@ -560,18 +212,13 @@ func (f *EnvAWSFingerprint) instanceType(ec2meta *ec2metadata.EC2Metadata) (stri return strings.TrimSpace(response), nil } -func (f *EnvAWSFingerprint) lookupCPU(ec2meta *ec2metadata.EC2Metadata) *ec2Specs { +func (f *EnvAWSFingerprint) lookupCPU(ec2meta *ec2metadata.EC2Metadata) *CPU { instanceType, err := f.instanceType(ec2meta) if err != nil { f.logger.Warn("failed to read EC2 metadata instance-type", "error", err) return nil } - for iType, specs := range ec2ProcSpeedTable { - if strings.EqualFold(iType, instanceType) { - return &specs - } - } - return nil + return LookupEC2CPU(instanceType) } func (f *EnvAWSFingerprint) throughput(request *FingerprintRequest, ec2meta *ec2metadata.EC2Metadata, ip string) int { diff --git a/client/fingerprint/env_aws_cpu.go b/client/fingerprint/env_aws_cpu.go new file mode 100644 index 000000000000..cfc5b2c04330 --- /dev/null +++ b/client/fingerprint/env_aws_cpu.go @@ -0,0 +1,379 @@ +// Code generated from hashicorp/nomad/tools/ec2info; DO NOT EDIT. + +package fingerprint + +// CPU contains virtual core count and processor baseline performance. +type CPU struct { + // use small units to reduce size of the embedded table + Cores uint32 // good for 4 billion cores + MHz uint32 // good for 4 billion MHz +} + +// Ticks computes the total number of cycles available across the virtual +// cores of a CPU. +func (c CPU) Ticks() int { + return int(c.MHz) * int(c.Cores) +} + +// GHz returns the speed of CPU in ghz. +func (c CPU) GHz() float64 { + return float64(c.MHz) / 1000.0 +} + +// newCPU create a CPUSpecs from the given virtual core count and core speed. +func newCPU(cores uint32, ghz float64) CPU { + return CPU{ + Cores: cores, + MHz: uint32(ghz * 1000), + } +} + +// LookupEC2CPU returns the virtual core count and core speed information from a +// lookup table generated from the Amazon EC2 API. +// +// If the instance type does not exist, nil is returned. +func LookupEC2CPU(instanceType string) *CPU { + specs, exists := instanceTypeCPU[instanceType] + if !exists { + return nil + } + return &specs +} + +var instanceTypeCPU = map[string]CPU{ + + "a1.2xlarge": newCPU(8, 2.3), + "a1.4xlarge": newCPU(16, 2.3), + "a1.large": newCPU(2, 2.3), + "a1.medium": newCPU(1, 2.3), + "a1.metal": newCPU(16, 2.3), + "a1.xlarge": newCPU(4, 2.3), + "c3.2xlarge": newCPU(8, 2.8), + "c3.4xlarge": newCPU(16, 2.8), + "c3.8xlarge": newCPU(32, 2.8), + "c3.large": newCPU(2, 2.8), + "c3.xlarge": newCPU(4, 2.8), + "c4.2xlarge": newCPU(8, 2.9), + "c4.4xlarge": newCPU(16, 2.9), + "c4.8xlarge": newCPU(36, 2.9), + "c4.large": newCPU(2, 2.9), + "c4.xlarge": newCPU(4, 2.9), + "c5.12xlarge": newCPU(48, 3.6), + "c5.18xlarge": newCPU(72, 3.4), + "c5.24xlarge": newCPU(96, 3.6), + "c5.2xlarge": newCPU(8, 3.4), + "c5.4xlarge": newCPU(16, 3.4), + "c5.9xlarge": newCPU(36, 3.4), + "c5.large": newCPU(2, 3.4), + "c5.metal": newCPU(96, 3.6), + "c5.xlarge": newCPU(4, 3.4), + "c5a.12xlarge": newCPU(48, 3.3), + "c5a.16xlarge": newCPU(64, 3.3), + "c5a.24xlarge": newCPU(96, 3.3), + "c5a.2xlarge": newCPU(8, 3.3), + "c5a.4xlarge": newCPU(16, 3.3), + "c5a.8xlarge": newCPU(32, 3.3), + "c5a.large": newCPU(2, 3.3), + "c5a.xlarge": newCPU(4, 3.3), + "c5ad.12xlarge": newCPU(48, 3.3), + "c5ad.16xlarge": newCPU(64, 3.3), + "c5ad.24xlarge": newCPU(96, 3.3), + "c5ad.2xlarge": newCPU(8, 3.3), + "c5ad.4xlarge": newCPU(16, 3.3), + "c5ad.8xlarge": newCPU(32, 3.3), + "c5ad.large": newCPU(2, 3.3), + "c5ad.xlarge": newCPU(4, 3.3), + "c5d.12xlarge": newCPU(48, 3.6), + "c5d.18xlarge": newCPU(72, 3.4), + "c5d.24xlarge": newCPU(96, 3.6), + "c5d.2xlarge": newCPU(8, 3.4), + "c5d.4xlarge": newCPU(16, 3.4), + "c5d.9xlarge": newCPU(36, 3.4), + "c5d.large": newCPU(2, 3.4), + "c5d.metal": newCPU(96, 3.6), + "c5d.xlarge": newCPU(4, 3.4), + "c5n.18xlarge": newCPU(72, 3.4), + "c5n.2xlarge": newCPU(8, 3.4), + "c5n.4xlarge": newCPU(16, 3.4), + "c5n.9xlarge": newCPU(36, 3.4), + "c5n.large": newCPU(2, 3.4), + "c5n.metal": newCPU(72, 3.4), + "c5n.xlarge": newCPU(4, 3.4), + "c6g.12xlarge": newCPU(48, 2.5), + "c6g.16xlarge": newCPU(64, 2.5), + "c6g.2xlarge": newCPU(8, 2.5), + "c6g.4xlarge": newCPU(16, 2.5), + "c6g.8xlarge": newCPU(32, 2.5), + "c6g.large": newCPU(2, 2.5), + "c6g.medium": newCPU(1, 2.5), + "c6g.metal": newCPU(64, 2.5), + "c6g.xlarge": newCPU(4, 2.5), + "c6gd.12xlarge": newCPU(48, 2.5), + "c6gd.16xlarge": newCPU(64, 2.5), + "c6gd.2xlarge": newCPU(8, 2.5), + "c6gd.4xlarge": newCPU(16, 2.5), + "c6gd.8xlarge": newCPU(32, 2.5), + "c6gd.large": newCPU(2, 2.5), + "c6gd.medium": newCPU(1, 2.5), + "c6gd.metal": newCPU(64, 2.5), + "c6gd.xlarge": newCPU(4, 2.5), + "cc2.8xlarge": newCPU(32, 2.6), + "d2.2xlarge": newCPU(8, 2.4), + "d2.4xlarge": newCPU(16, 2.4), + "d2.8xlarge": newCPU(36, 2.4), + "d2.xlarge": newCPU(4, 2.4), + "f1.16xlarge": newCPU(64, 2.3), + "f1.2xlarge": newCPU(8, 2.3), + "f1.4xlarge": newCPU(16, 2.3), + "g2.2xlarge": newCPU(8, 2.6), + "g2.8xlarge": newCPU(32, 2.6), + "g3.16xlarge": newCPU(64, 2.3), + "g3.4xlarge": newCPU(16, 2.7), + "g3.8xlarge": newCPU(32, 2.7), + "g3s.xlarge": newCPU(4, 2.7), + "g4dn.12xlarge": newCPU(48, 2.5), + "g4dn.16xlarge": newCPU(64, 2.5), + "g4dn.2xlarge": newCPU(8, 2.5), + "g4dn.4xlarge": newCPU(16, 2.5), + "g4dn.8xlarge": newCPU(32, 2.5), + "g4dn.metal": newCPU(96, 2.5), + "g4dn.xlarge": newCPU(4, 2.5), + "h1.16xlarge": newCPU(64, 2.3), + "h1.2xlarge": newCPU(8, 2.3), + "h1.4xlarge": newCPU(16, 2.3), + "h1.8xlarge": newCPU(32, 2.3), + "i2.2xlarge": newCPU(8, 2.5), + "i2.4xlarge": newCPU(16, 2.5), + "i2.8xlarge": newCPU(32, 2.5), + "i2.xlarge": newCPU(4, 2.5), + "i3.16xlarge": newCPU(64, 2.3), + "i3.2xlarge": newCPU(8, 2.3), + "i3.4xlarge": newCPU(16, 2.3), + "i3.8xlarge": newCPU(32, 2.3), + "i3.large": newCPU(2, 2.3), + "i3.metal": newCPU(72, 2.3), + "i3.xlarge": newCPU(4, 2.3), + "i3en.12xlarge": newCPU(48, 3.1), + "i3en.24xlarge": newCPU(96, 3.1), + "i3en.2xlarge": newCPU(8, 3.1), + "i3en.3xlarge": newCPU(12, 3.1), + "i3en.6xlarge": newCPU(24, 3.1), + "i3en.large": newCPU(2, 3.1), + "i3en.metal": newCPU(96, 3.1), + "i3en.xlarge": newCPU(4, 3.1), + "inf1.24xlarge": newCPU(96, 2.5), + "inf1.2xlarge": newCPU(8, 2.5), + "inf1.6xlarge": newCPU(24, 2.5), + "inf1.xlarge": newCPU(4, 2.5), + "m3.2xlarge": newCPU(8, 2.5), + "m3.large": newCPU(2, 2.5), + "m3.medium": newCPU(1, 2.5), + "m3.xlarge": newCPU(4, 2.5), + "m4.10xlarge": newCPU(40, 2.4), + "m4.16xlarge": newCPU(64, 2.3), + "m4.2xlarge": newCPU(8, 2.4), + "m4.4xlarge": newCPU(16, 2.4), + "m4.large": newCPU(2, 2.4), + "m4.xlarge": newCPU(4, 2.4), + "m5.12xlarge": newCPU(48, 3.1), + "m5.16xlarge": newCPU(64, 3.1), + "m5.24xlarge": newCPU(96, 3.1), + "m5.2xlarge": newCPU(8, 3.1), + "m5.4xlarge": newCPU(16, 3.1), + "m5.8xlarge": newCPU(32, 3.1), + "m5.large": newCPU(2, 3.1), + "m5.metal": newCPU(96, 3.1), + "m5.xlarge": newCPU(4, 3.1), + "m5a.12xlarge": newCPU(48, 2.5), + "m5a.16xlarge": newCPU(64, 2.5), + "m5a.24xlarge": newCPU(96, 2.5), + "m5a.2xlarge": newCPU(8, 2.5), + "m5a.4xlarge": newCPU(16, 2.5), + "m5a.8xlarge": newCPU(32, 2.5), + "m5a.large": newCPU(2, 2.5), + "m5a.xlarge": newCPU(4, 2.5), + "m5ad.12xlarge": newCPU(48, 2.2), + "m5ad.16xlarge": newCPU(64, 2.5), + "m5ad.24xlarge": newCPU(96, 2.2), + "m5ad.2xlarge": newCPU(8, 2.2), + "m5ad.4xlarge": newCPU(16, 2.2), + "m5ad.8xlarge": newCPU(32, 2.5), + "m5ad.large": newCPU(2, 2.2), + "m5ad.xlarge": newCPU(4, 2.2), + "m5d.12xlarge": newCPU(48, 3.1), + "m5d.16xlarge": newCPU(64, 3.1), + "m5d.24xlarge": newCPU(96, 3.1), + "m5d.2xlarge": newCPU(8, 3.1), + "m5d.4xlarge": newCPU(16, 3.1), + "m5d.8xlarge": newCPU(32, 3.1), + "m5d.large": newCPU(2, 3.1), + "m5d.metal": newCPU(96, 3.1), + "m5d.xlarge": newCPU(4, 3.1), + "m5dn.12xlarge": newCPU(48, 3.1), + "m5dn.16xlarge": newCPU(64, 3.1), + "m5dn.24xlarge": newCPU(96, 3.1), + "m5dn.2xlarge": newCPU(8, 3.1), + "m5dn.4xlarge": newCPU(16, 3.1), + "m5dn.8xlarge": newCPU(32, 3.1), + "m5dn.large": newCPU(2, 3.1), + "m5dn.metal": newCPU(96, 3.1), + "m5dn.xlarge": newCPU(4, 3.1), + "m5n.12xlarge": newCPU(48, 3.1), + "m5n.16xlarge": newCPU(64, 3.1), + "m5n.24xlarge": newCPU(96, 3.1), + "m5n.2xlarge": newCPU(8, 3.1), + "m5n.4xlarge": newCPU(16, 3.1), + "m5n.8xlarge": newCPU(32, 3.1), + "m5n.large": newCPU(2, 3.1), + "m5n.metal": newCPU(96, 3.1), + "m5n.xlarge": newCPU(4, 3.1), + "m6g.12xlarge": newCPU(48, 2.5), + "m6g.16xlarge": newCPU(64, 2.5), + "m6g.2xlarge": newCPU(8, 2.5), + "m6g.4xlarge": newCPU(16, 2.5), + "m6g.8xlarge": newCPU(32, 2.5), + "m6g.large": newCPU(2, 2.5), + "m6g.medium": newCPU(1, 2.5), + "m6g.metal": newCPU(64, 2.5), + "m6g.xlarge": newCPU(4, 2.5), + "m6gd.12xlarge": newCPU(48, 2.5), + "m6gd.16xlarge": newCPU(64, 2.5), + "m6gd.2xlarge": newCPU(8, 2.5), + "m6gd.4xlarge": newCPU(16, 2.5), + "m6gd.8xlarge": newCPU(32, 2.5), + "m6gd.large": newCPU(2, 2.5), + "m6gd.medium": newCPU(1, 2.5), + "m6gd.metal": newCPU(64, 2.5), + "m6gd.xlarge": newCPU(4, 2.5), + "p2.16xlarge": newCPU(64, 2.3), + "p2.8xlarge": newCPU(32, 2.7), + "p2.xlarge": newCPU(4, 2.7), + "p3.16xlarge": newCPU(64, 2.7), + "p3.2xlarge": newCPU(8, 2.7), + "p3.8xlarge": newCPU(32, 2.7), + "p3dn.24xlarge": newCPU(96, 2.5), + "r3.2xlarge": newCPU(8, 2.5), + "r3.4xlarge": newCPU(16, 2.5), + "r3.8xlarge": newCPU(32, 2.5), + "r3.large": newCPU(2, 2.5), + "r3.xlarge": newCPU(4, 2.5), + "r4.16xlarge": newCPU(64, 2.3), + "r4.2xlarge": newCPU(8, 2.3), + "r4.4xlarge": newCPU(16, 2.3), + "r4.8xlarge": newCPU(32, 2.3), + "r4.large": newCPU(2, 2.3), + "r4.xlarge": newCPU(4, 2.3), + "r5.12xlarge": newCPU(48, 3.1), + "r5.16xlarge": newCPU(64, 3.1), + "r5.24xlarge": newCPU(96, 3.1), + "r5.2xlarge": newCPU(8, 3.1), + "r5.4xlarge": newCPU(16, 3.1), + "r5.8xlarge": newCPU(32, 3.1), + "r5.large": newCPU(2, 3.1), + "r5.metal": newCPU(96, 3.1), + "r5.xlarge": newCPU(4, 3.1), + "r5a.12xlarge": newCPU(48, 2.5), + "r5a.16xlarge": newCPU(64, 2.5), + "r5a.24xlarge": newCPU(96, 2.5), + "r5a.2xlarge": newCPU(8, 2.5), + "r5a.4xlarge": newCPU(16, 2.5), + "r5a.8xlarge": newCPU(32, 2.5), + "r5a.large": newCPU(2, 2.5), + "r5a.xlarge": newCPU(4, 2.5), + "r5ad.12xlarge": newCPU(48, 2.2), + "r5ad.16xlarge": newCPU(64, 2.5), + "r5ad.24xlarge": newCPU(96, 2.2), + "r5ad.2xlarge": newCPU(8, 2.2), + "r5ad.4xlarge": newCPU(16, 2.2), + "r5ad.8xlarge": newCPU(32, 2.5), + "r5ad.large": newCPU(2, 2.2), + "r5ad.xlarge": newCPU(4, 2.2), + "r5d.12xlarge": newCPU(48, 3.1), + "r5d.16xlarge": newCPU(64, 3.1), + "r5d.24xlarge": newCPU(96, 3.1), + "r5d.2xlarge": newCPU(8, 3.1), + "r5d.4xlarge": newCPU(16, 3.1), + "r5d.8xlarge": newCPU(32, 3.1), + "r5d.large": newCPU(2, 3.1), + "r5d.metal": newCPU(96, 3.1), + "r5d.xlarge": newCPU(4, 3.1), + "r5dn.12xlarge": newCPU(48, 3.1), + "r5dn.16xlarge": newCPU(64, 3.1), + "r5dn.24xlarge": newCPU(96, 3.1), + "r5dn.2xlarge": newCPU(8, 3.1), + "r5dn.4xlarge": newCPU(16, 3.1), + "r5dn.8xlarge": newCPU(32, 3.1), + "r5dn.large": newCPU(2, 3.1), + "r5dn.metal": newCPU(96, 3.1), + "r5dn.xlarge": newCPU(4, 3.1), + "r5n.12xlarge": newCPU(48, 3.1), + "r5n.16xlarge": newCPU(64, 3.1), + "r5n.24xlarge": newCPU(96, 3.1), + "r5n.2xlarge": newCPU(8, 3.1), + "r5n.4xlarge": newCPU(16, 3.1), + "r5n.8xlarge": newCPU(32, 3.1), + "r5n.large": newCPU(2, 3.1), + "r5n.metal": newCPU(96, 3.1), + "r5n.xlarge": newCPU(4, 3.1), + "r6g.12xlarge": newCPU(48, 2.5), + "r6g.16xlarge": newCPU(64, 2.5), + "r6g.2xlarge": newCPU(8, 2.5), + "r6g.4xlarge": newCPU(16, 2.5), + "r6g.8xlarge": newCPU(32, 2.5), + "r6g.large": newCPU(2, 2.5), + "r6g.medium": newCPU(1, 2.5), + "r6g.metal": newCPU(64, 2.5), + "r6g.xlarge": newCPU(4, 2.5), + "r6gd.12xlarge": newCPU(48, 2.5), + "r6gd.16xlarge": newCPU(64, 2.5), + "r6gd.2xlarge": newCPU(8, 2.5), + "r6gd.4xlarge": newCPU(16, 2.5), + "r6gd.8xlarge": newCPU(32, 2.5), + "r6gd.large": newCPU(2, 2.5), + "r6gd.medium": newCPU(1, 2.5), + "r6gd.metal": newCPU(64, 2.5), + "r6gd.xlarge": newCPU(4, 2.5), + "t2.2xlarge": newCPU(8, 2.3), + "t2.large": newCPU(2, 2.3), + "t2.medium": newCPU(2, 2.3), + "t2.micro": newCPU(1, 2.5), + "t2.nano": newCPU(1, 2.4), + "t2.small": newCPU(1, 2.5), + "t2.xlarge": newCPU(4, 2.3), + "t3.2xlarge": newCPU(8, 2.5), + "t3.large": newCPU(2, 2.5), + "t3.medium": newCPU(2, 2.5), + "t3.micro": newCPU(2, 2.5), + "t3.nano": newCPU(2, 2.5), + "t3.small": newCPU(2, 2.5), + "t3.xlarge": newCPU(4, 2.5), + "t3a.2xlarge": newCPU(8, 2.2), + "t3a.large": newCPU(2, 2.2), + "t3a.medium": newCPU(2, 2.2), + "t3a.micro": newCPU(2, 2.2), + "t3a.nano": newCPU(2, 2.2), + "t3a.small": newCPU(2, 2.2), + "t3a.xlarge": newCPU(4, 2.2), + "t4g.2xlarge": newCPU(8, 2.5), + "t4g.large": newCPU(2, 2.5), + "t4g.medium": newCPU(2, 2.5), + "t4g.micro": newCPU(2, 2.5), + "t4g.nano": newCPU(2, 2.5), + "t4g.small": newCPU(2, 2.5), + "t4g.xlarge": newCPU(4, 2.5), + "x1.16xlarge": newCPU(64, 2.3), + "x1.32xlarge": newCPU(128, 2.3), + "x1e.16xlarge": newCPU(64, 2.3), + "x1e.2xlarge": newCPU(8, 2.3), + "x1e.32xlarge": newCPU(128, 2.3), + "x1e.4xlarge": newCPU(16, 2.3), + "x1e.8xlarge": newCPU(32, 2.3), + "x1e.xlarge": newCPU(4, 2.3), + "z1d.12xlarge": newCPU(48, 4), + "z1d.2xlarge": newCPU(8, 4), + "z1d.3xlarge": newCPU(12, 4), + "z1d.6xlarge": newCPU(24, 4), + "z1d.large": newCPU(2, 4), + "z1d.metal": newCPU(48, 4), + "z1d.xlarge": newCPU(4, 4), +} diff --git a/client/fingerprint/env_aws_test.go b/client/fingerprint/env_aws_test.go index a2b78b066cfc..60b3fe6fb644 100644 --- a/client/fingerprint/env_aws_test.go +++ b/client/fingerprint/env_aws_test.go @@ -216,12 +216,11 @@ func TestCPUFingerprint_AWS_InstanceFound(t *testing.T) { err := f.Fingerprint(request, &response) require.NoError(t, err) require.True(t, response.Detected) - require.Equal(t, "2.5 GHz AMD EPYC 7000 series", response.Attributes["cpu.modelname"]) - require.Equal(t, "2500", response.Attributes["cpu.frequency"]) + require.Equal(t, "2200", response.Attributes["cpu.frequency"]) require.Equal(t, "8", response.Attributes["cpu.numcores"]) - require.Equal(t, "20000", response.Attributes["cpu.totalcompute"]) - require.Equal(t, 20000, response.Resources.CPU) - require.Equal(t, int64(20000), response.NodeResources.Cpu.CpuShares) + require.Equal(t, "17600", response.Attributes["cpu.totalcompute"]) + require.Equal(t, 17600, response.Resources.CPU) + require.Equal(t, int64(17600), response.NodeResources.Cpu.CpuShares) } func TestCPUFingerprint_AWS_OverrideCompute(t *testing.T) { @@ -240,8 +239,7 @@ func TestCPUFingerprint_AWS_OverrideCompute(t *testing.T) { err := f.Fingerprint(request, &response) require.NoError(t, err) require.True(t, response.Detected) - require.Equal(t, "2.5 GHz AMD EPYC 7000 series", response.Attributes["cpu.modelname"]) - require.Equal(t, "2500", response.Attributes["cpu.frequency"]) + require.Equal(t, "2200", response.Attributes["cpu.frequency"]) require.Equal(t, "8", response.Attributes["cpu.numcores"]) require.NotContains(t, response.Attributes, "cpu.totalcompute") require.Nil(t, response.Resources) // defaults in cpu fingerprinter @@ -347,6 +345,28 @@ var awsStubs = []endpoint{ ContentType: "text/plain", Body: "0a:20:d2:42:b3:55", }, + { + Uri: "/latest/dynamic/instance-identity/document", + ContentType: "text/plain", + Body: ` + { + "devpayProductCodes" : null, + "marketplaceProductCodes" : [ "1abc2defghijklm3nopqrs4tu" ], + "availabilityZone" : "us-west-2a", + "privateIp" : "10.0.0.207", + "version" : "2017-09-30", + "instanceId" : "i-b3ba3875", + "billingProducts" : null, + "instanceType" : "t3a.2xlarge", + "accountId" : "123456789012", + "imageId" : "ami-1234", + "pendingTime" : "2016-11-19T16:32:11Z", + "architecture" : "x86_64", + "kernelId" : null, + "ramdiskId" : null, + "region" : "us-west-2" + }`, + }, } var unknownInstanceType = []endpoint{ diff --git a/tools/ec2info/aws.go b/tools/ec2info/aws.go new file mode 100644 index 000000000000..6382885c5f4a --- /dev/null +++ b/tools/ec2info/aws.go @@ -0,0 +1,124 @@ +package main + +import ( + "fmt" + "log" + "sort" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ec2" +) + +func clientForRegion(region string) (*ec2.EC2, error) { + sess, err := session.NewSession(&aws.Config{ + Region: ®ion, + }) + if err != nil { + return nil, err + } + return ec2.New(sess), nil +} + +func getRegions(client *ec2.EC2) ([]*ec2.Region, error) { + all := false // beyond account access + regions, err := client.DescribeRegions(&ec2.DescribeRegionsInput{ + AllRegions: &all, + }) + if err != nil { + return nil, err + } + return regions.Regions, nil +} + +type specs struct { + Cores int + Speed float64 +} + +func (s specs) String() string { + return fmt.Sprintf("(%d %.2f)", s.Cores, s.Speed) +} + +func getData(regions []*ec2.Region, verbose bool) (map[string]map[string]specs, error) { + data := make(map[string]map[string]specs) + + for _, region := range regions { + rData, rProblems, err := getDataForRegion(*region.RegionName) + if err != nil { + return nil, err + } + data[*region.RegionName] = rData + + if verbose { + log.Println("region", *region.RegionName, "got data for", len(rData), "instance types", len(rProblems), "incomplete") + instanceProblems(rProblems) + } + } + + return data, nil +} + +func instanceProblems(problems map[string]string) { + types := make([]string, 0, len(problems)) + for k := range problems { + types = append(types, k) + } + sort.Strings(types) + for _, iType := range types { + log.Println(" ->", iType, problems[iType]) + } +} + +func getDataForRegion(region string) (map[string]specs, map[string]string, error) { + client, err := clientForRegion(region) + if err != nil { + return nil, nil, err + } + + data := make(map[string]specs) + problems := make(map[string]string) + regionInfoPage(client, true, region, nil, data, problems) + return data, problems, nil +} + +func regionInfoPage(client *ec2.EC2, first bool, region string, token *string, data map[string]specs, problems map[string]string) { + if first || token != nil { + output, err := client.DescribeInstanceTypes(&ec2.DescribeInstanceTypesInput{ + NextToken: token, + }) + if err != nil { + log.Fatal(err) + } + + // recursively accumulate each page of data + regionInfoAccumulate(output, data, problems) + regionInfoPage(client, false, region, output.NextToken, data, problems) + } +} + +func regionInfoAccumulate(output *ec2.DescribeInstanceTypesOutput, data map[string]specs, problems map[string]string) { + for _, iType := range output.InstanceTypes { + switch { + + case iType.ProcessorInfo == nil: + fallthrough + case iType.ProcessorInfo.SustainedClockSpeedInGhz == nil: + problems[*iType.InstanceType] = "missing clock Speed" + continue + + case iType.VCpuInfo == nil: + fallthrough + case iType.VCpuInfo.DefaultVCpus == nil: + problems[*iType.InstanceType] = "missing virtual cpu Cores" + continue + + default: + data[*iType.InstanceType] = specs{ + Speed: *iType.ProcessorInfo.SustainedClockSpeedInGhz, + Cores: int(*iType.VCpuInfo.DefaultVCpus), + } + continue + } + } +} diff --git a/tools/ec2info/cpu_table.go.template b/tools/ec2info/cpu_table.go.template new file mode 100644 index 000000000000..87137f87f5f1 --- /dev/null +++ b/tools/ec2info/cpu_table.go.template @@ -0,0 +1,48 @@ +// Code generated from hashicorp/nomad/tools/ec2info; DO NOT EDIT. + +package {{.Package}} + +// CPU contains virtual core count and processor baseline performance. +type CPU struct { + // use small units to reduce size of the embedded table + Cores uint32 // good for 4 billion cores + MHz uint32 // good for 4 billion MHz +} + +// Ticks computes the total number of cycles available across the virtual +// cores of a CPU. +func (c CPU) Ticks() int { + return int(c.MHz) * int(c.Cores) +} + +// GHz returns the speed of CPU in ghz. +func (c CPU) GHz() float64 { + return float64(c.MHz) / 1000.0 +} + +// newCPU create a CPUSpecs from the given virtual core count and core speed. +func newCPU(cores uint32, ghz float64) CPU { + return CPU{ + Cores: cores, + MHz: uint32(ghz * 1000), + } +} + +// LookupEC2CPU returns the virtual core count and core speed information from a +// lookup table generated from the Amazon EC2 API. +// +// If the instance type does not exist, nil is returned. +func LookupEC2CPU(instanceType string) *CPU { + specs, exists := instanceTypeCPU[instanceType] + if !exists { + return nil + } + return &specs +} + +{{with .Data}} +var instanceTypeCPU = map[string]CPU { + {{ range $key, $value := . }} + "{{ $key }}": newCPU({{$value.Cores}}, {{$value.Speed}}), {{ end }} +} +{{end}} diff --git a/tools/ec2info/main.go b/tools/ec2info/main.go new file mode 100644 index 000000000000..8e2f17294a5c --- /dev/null +++ b/tools/ec2info/main.go @@ -0,0 +1,61 @@ +// Command ec2info provides a tool for generating a CPU performance lookup +// table indexed by EC2 instance types. +// +// By default the generated file will overwrite `env_aws_cpu.go` in Nomad's +// client/fingerprint package, when run from this directory. +// +// Requires AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN. +// +// Options +// --package : configure package name of generated output file +// --region : configure initial region from which to lookup all regions +// --outfile : configure filepath of generated output file +// --verbose : print log messages while running +// +// Usage +// $ go run . +package main + +import ( + "flag" + "log" +) + +func args() (string, string, string, bool) { + pkg := flag.String("package", "fingerprint", "generate package name") + region := flag.String("region", "us-west-1", "initial region for listing regions") + outfile := flag.String("output", "../../client/fingerprint/env_aws_cpu.go", "output filepath") + verbose := flag.Bool("verbose", true, "print extra information while running") + flag.Parse() + return *pkg, *region, *outfile, *verbose +} + +func check(err error) { + if err != nil { + log.Fatal(err) + } +} + +func main() { + pkg, region, output, verbose := args() + + client, err := clientForRegion(region) + check(err) + + regions, err := getRegions(client) + check(err) + + data, err := getData(regions, verbose) + check(err) + + flat := flatten(data) + + f, err := open(output) + check(err) + defer func() { + check(f.Close()) + }() + + check(write(f, flat, pkg)) + check(format(output)) +} diff --git a/tools/ec2info/output.go b/tools/ec2info/output.go new file mode 100644 index 000000000000..75d27be279ef --- /dev/null +++ b/tools/ec2info/output.go @@ -0,0 +1,48 @@ +package main + +import ( + "io" + "os" + "os/exec" + "text/template" +) + +// open the output file for writing. +func open(output string) (io.ReadWriteCloser, error) { + return os.Create(output) +} + +// flatten region data, assuming instance type is the same across regions. +func flatten(data map[string]map[string]specs) map[string]specs { + result := make(map[string]specs) + for _, m := range data { + for iType, specs := range m { + result[iType] = specs + } + } + return result +} + +type Template struct { + Package string + Data map[string]specs +} + +// write the data using the cpu_table.go.template to w. +func write(w io.Writer, data map[string]specs, pkg string) error { + tmpl, err := template.ParseFiles("cpu_table.go.template") + if err != nil { + return err + } + return tmpl.Execute(w, Template{ + Package: pkg, + Data: data, + }) +} + +// format the file using gofmt. +func format(file string) error { + cmd := exec.Command("gofmt", "-w", file) + _, err := cmd.CombinedOutput() + return err +} diff --git a/tools/go.mod b/tools/go.mod index 27ab4a59eec7..c6ffb1cbbf92 100644 --- a/tools/go.mod +++ b/tools/go.mod @@ -4,6 +4,7 @@ go 1.14 require ( github.com/a8m/tree v0.0.0-20181222104329-6a0b80129de4 + github.com/aws/aws-sdk-go v1.35.5 github.com/client9/misspell v0.3.4 github.com/elazarl/go-bindata-assetfs v1.0.1 github.com/golang/protobuf v1.3.4 @@ -20,6 +21,5 @@ require ( github.com/stretchr/testify v1.5.1 // indirect golang.org/x/tools v0.0.0-20200502202811-ed308ab3e770 // indirect gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect - gopkg.in/yaml.v2 v2.2.8 // indirect gotest.tools/gotestsum v0.4.2 ) diff --git a/tools/go.sum b/tools/go.sum index 948355c3b7e0..54a25fb165a0 100644 --- a/tools/go.sum +++ b/tools/go.sum @@ -17,6 +17,8 @@ github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/ github.com/apparentlymart/go-textseg/v12 v12.0.0 h1:bNEQyAGak9tojivJNkoqWErVCQbjdL7GzRt3F8NvfJ0= github.com/apparentlymart/go-textseg/v12 v12.0.0/go.mod h1:S/4uRK2UtaQttw1GenVJEynmyUenKwP++x/+DdGV/Ec= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/aws/aws-sdk-go v1.35.5 h1:doSEOxC0UkirPcle20Rc+1kAhJ4Ip+GSEeZ3nKl7Qlk= +github.com/aws/aws-sdk-go v1.35.5/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bombsimon/wsl/v2 v2.0.0 h1:+Vjcn+/T5lSrO8Bjzhk4v14Un/2UyCA1E3V5j9nwTkQ= @@ -159,6 +161,10 @@ github.com/jingyugao/rowserrcheck v0.0.0-20191204022205-72ab7603b68a h1:Gmsqmapf github.com/jingyugao/rowserrcheck v0.0.0-20191204022205-72ab7603b68a/go.mod h1:xRskid8CManxVta/ALEhJha/pweKBaVG6fWgc0yH25s= github.com/jirfag/go-printf-func-name v0.0.0-20191110105641-45db9963cdd3 h1:jNYPNLe3d8smommaoQlK7LOA5ESyUJJ+Wf79ZtA7Vp4= github.com/jirfag/go-printf-func-name v0.0.0-20191110105641-45db9963cdd3/go.mod h1:HEWGJkRDzjJY2sqdDwxccsGicWEf9BQOZsq2tV+xzM0= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= @@ -232,6 +238,8 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -348,6 +356,7 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= diff --git a/website/pages/docs/upgrade/upgrade-specific.mdx b/website/pages/docs/upgrade/upgrade-specific.mdx index a8d095a846e0..5679743d7509 100644 --- a/website/pages/docs/upgrade/upgrade-specific.mdx +++ b/website/pages/docs/upgrade/upgrade-specific.mdx @@ -32,6 +32,18 @@ jobs meeting this requirement should be modified before an update to v0.13.0. Si client and server config validation will prohibit either the region or the datacenter from containing null characters. +### EC2 CPU characteristics may be different + +Starting with Nomad v0.13.0, the AWS fingerprinter uses data derived from the +official AWS EC2 API to determine default CPU performance characteristics, including +core count and core speed. This data should be accurate for each instance type +per region. Previously, Nomad used a hand-made lookup table that was not region +aware and may have contained inaccurate or incomplete data. As part of this change, +the AWS fingerprinter no longer sets the `cpu.modelname` attribute. + +As before, `cpu_total_compute` can be used to override the discovered CPU resources +available to the Nomad client. + ## Nomad 0.12.0 ### `mbits` and Task Network Resource deprecation