From 87263de3528db706925f58f5dd1593c74678e585 Mon Sep 17 00:00:00 2001 From: Jie Chen Date: Mon, 29 May 2023 10:13:20 -0700 Subject: [PATCH] Rebased windows-prefix-delegation branch onto master branch (#228) * add healthz subpathes for all controllers (#201) * support arch arg in dockerfile (#207) * updated vpc limits to include fields for hypervisor type and bare metal status (#217) * enable node events when instance type is not supported (#218) * Associate primary network interface SG with the trunk ENI when SG is not specified in ENIConfig (#221) * Associate primary network interface SG with the trunk ENI when SG is not specified in ENIConfig * add a new CRD to delegate vpc resource requests (#210) * upgrade controller runtime version (#227) * rebased onto master branch * fixed merge conflict --------- Co-authored-by: Hao Zhou Co-authored-by: Sushmitha Ravikumar <58063229+sushrk@users.noreply.github.com> --- Dockerfile | 3 +- Makefile | 6 + apis/vpcresources/v1alpha1/cninode_types.go | 61 + .../v1alpha1/groupversion_info.go | 33 + .../v1alpha1/zz_generated.deepcopy.go | 117 ++ .../bases/vpcresources.k8s.aws_cninodes.yaml | 66 + config/rbac/role.yaml | 11 + .../vpcresources_v1alpha1_cninode.yaml | 10 + controllers/apps/deployment_controller.go | 12 +- controllers/core/configmap_controller.go | 16 +- controllers/core/node_controller.go | 54 +- controllers/core/pod_controller.go | 44 +- controllers/custom/builder.go | 29 +- controllers/custom/custom_controller.go | 38 + go.mod | 2 +- go.sum | 4 +- main.go | 102 +- .../pkg/aws/ec2/mock_instance.go | 28 +- .../pkg/node/mock_node.go | 25 +- .../pkg/provider/mock_provider.go | 15 + pkg/aws/ec2/api/eni_cleanup.go | 10 +- pkg/aws/ec2/instance.go | 40 +- pkg/aws/ec2/instance_test.go | 42 +- pkg/aws/vpc/limits.go | 1350 ++++++++++++++++- pkg/healthz/healthz.go | 83 + pkg/healthz/healthz_test.go | 62 + pkg/node/manager/manager.go | 42 +- pkg/node/manager/manager_test.go | 68 +- pkg/node/node.go | 34 +- pkg/node/node_test.go | 57 +- pkg/provider/branch/provider.go | 42 +- pkg/provider/branch/trunk/trunk.go | 4 +- pkg/provider/branch/trunk/trunk_test.go | 4 +- pkg/provider/ip/eni/eni.go | 7 +- pkg/provider/ip/eni/eni_test.go | 30 +- pkg/provider/ip/provider.go | 35 +- pkg/provider/provider.go | 3 + pkg/resource/introspect.go | 9 +- pkg/resource/manager.go | 30 +- pkg/resource/manager_test.go | 8 +- pkg/utils/errors.go | 7 + pkg/utils/events.go | 21 + pkg/worker/worker.go | 6 + scripts/gen_mocks.sh | 2 +- .../core/annotation_validation_webhook.go | 19 + webhooks/core/node_update_webhook.go | 19 + webhooks/core/pod_webhook.go | 21 + webhooks/core/pod_webhook_test.go | 4 +- 48 files changed, 2519 insertions(+), 216 deletions(-) create mode 100644 apis/vpcresources/v1alpha1/cninode_types.go create mode 100644 apis/vpcresources/v1alpha1/groupversion_info.go create mode 100644 apis/vpcresources/v1alpha1/zz_generated.deepcopy.go create mode 100644 config/crd/bases/vpcresources.k8s.aws_cninodes.yaml create mode 100644 config/samples/vpcresources_v1alpha1_cninode.yaml create mode 100644 pkg/healthz/healthz.go create mode 100644 pkg/healthz/healthz_test.go create mode 100644 pkg/utils/errors.go create mode 100644 pkg/utils/events.go diff --git a/Dockerfile b/Dockerfile index ca383308..286596ca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,6 @@ ARG BASE_IMAGE ARG BUILD_IMAGE +ARG ARCH=amd64 # Build the controller binary FROM $BUILD_IMAGE as builder @@ -27,7 +28,7 @@ ENV VERSION_PKG=github.com/aws/amazon-vpc-resource-controller-k8s/pkg/version RUN GIT_VERSION=$(git describe --tags --always) && \ GIT_COMMIT=$(git rev-parse HEAD) && \ BUILD_DATE=$(date +%Y-%m-%dT%H:%M:%S%z) && \ - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build \ + CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} GO111MODULE=on go build \ -ldflags="-X ${VERSION_PKG}.GitVersion=${GIT_VERSION} -X ${VERSION_PKG}.GitCommit=${GIT_COMMIT} -X ${VERSION_PKG}.BuildDate=${BUILD_DATE}" -a -o controller main.go FROM $BASE_IMAGE diff --git a/Makefile b/Makefile index 6b5db7f5..4345341b 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,8 @@ BASE_IMAGE ?= public.ecr.aws/eks-distro-build-tooling/eks-distro-minimal-base-no BUILD_IMAGE ?= public.ecr.aws/bitnami/golang:1.20.1 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion) CRD_OPTIONS ?= "crd:trivialVersions=true" +GOARCH ?= amd64 +PLATFORM ?= linux/amd64 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) @@ -68,6 +70,10 @@ vet: generate: controller-gen $(CONTROLLER_GEN) object:headerFile="scripts/templates/boilerplate.go.txt" paths="./..." +# Build the docker image with buildx +docker-buildx: check-env test + docker buildx build --platform=$(PLATFORM) -t $(IMAGE)-$(GOARCH) --build-arg BASE_IMAGE=$(BASE_IMAGE) --build-arg BUILD_IMAGE=$(BUILD_IMAGE) --build-arg $(GOARCH) --load . + # Build the docker image docker-build: check-env test docker build --build-arg BASE_IMAGE=$(BASE_IMAGE) --build-arg BUILD_IMAGE=$(BUILD_IMAGE) . -t ${IMAGE} diff --git a/apis/vpcresources/v1alpha1/cninode_types.go b/apis/vpcresources/v1alpha1/cninode_types.go new file mode 100644 index 00000000..17131a62 --- /dev/null +++ b/apis/vpcresources/v1alpha1/cninode_types.go @@ -0,0 +1,61 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// FeatureName is a type of feature name supported by AWS VPC CNI. It can be Security Group for Pods, custom networking, or others +type FeatureName string + +const ( + SecurityGroupsForPods FeatureName = "SecurityGroupsForPods" + CustomNetworking FeatureName = "CustomNetworking" +) + +// Important: Run "make" to regenerate code after modifying this file +// CNINodeSpec defines the desired state of CNINode +type CNINodeSpec struct { + Features []FeatureName `json:"features,omitempty"` +} + +// CNINodeStatus defines the managed VPC resources. +type CNINodeStatus struct { + //TODO: add VPS resources which will be managed by this CRD and its finalizer +} + +// +kubebuilder:object:root=true +// +kubebuilder:printcolumn:name="Features",type=string,JSONPath=`.spec.features`,description="The features delegated to VPC resource controller" +// +kubebuilder:resource:shortName=cnd + +// +kubebuilder:object:root=true +type CNINode struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + Spec CNINodeSpec `json:"spec,omitempty"` + Status CNINodeStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true +// CNINodeList contains a list of CNINodeList +type CNINodeList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []CNINode `json:"items"` +} + +func init() { + SchemeBuilder.Register(&CNINode{}, &CNINodeList{}) +} diff --git a/apis/vpcresources/v1alpha1/groupversion_info.go b/apis/vpcresources/v1alpha1/groupversion_info.go new file mode 100644 index 00000000..cd6dff20 --- /dev/null +++ b/apis/vpcresources/v1alpha1/groupversion_info.go @@ -0,0 +1,33 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +// Package v1beta1 contains API Schema definitions for the vpcresources v1beta1 API group +// +kubebuilder:object:generate=true +// +groupName=vpcresources.k8s.aws +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "vpcresources.k8s.aws", Version: "v1alpha1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/apis/vpcresources/v1alpha1/zz_generated.deepcopy.go b/apis/vpcresources/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 00000000..ff6b1ca6 --- /dev/null +++ b/apis/vpcresources/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,117 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CNINode) DeepCopyInto(out *CNINode) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CNINode. +func (in *CNINode) DeepCopy() *CNINode { + if in == nil { + return nil + } + out := new(CNINode) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CNINode) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CNINodeList) DeepCopyInto(out *CNINodeList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]CNINode, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CNINodeList. +func (in *CNINodeList) DeepCopy() *CNINodeList { + if in == nil { + return nil + } + out := new(CNINodeList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CNINodeList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CNINodeSpec) DeepCopyInto(out *CNINodeSpec) { + *out = *in + if in.Features != nil { + in, out := &in.Features, &out.Features + *out = make([]FeatureName, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CNINodeSpec. +func (in *CNINodeSpec) DeepCopy() *CNINodeSpec { + if in == nil { + return nil + } + out := new(CNINodeSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CNINodeStatus) DeepCopyInto(out *CNINodeStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CNINodeStatus. +func (in *CNINodeStatus) DeepCopy() *CNINodeStatus { + if in == nil { + return nil + } + out := new(CNINodeStatus) + in.DeepCopyInto(out) + return out +} diff --git a/config/crd/bases/vpcresources.k8s.aws_cninodes.yaml b/config/crd/bases/vpcresources.k8s.aws_cninodes.yaml new file mode 100644 index 00000000..aa8e0d05 --- /dev/null +++ b/config/crd/bases/vpcresources.k8s.aws_cninodes.yaml @@ -0,0 +1,66 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.6.2 + creationTimestamp: null + name: cninodes.vpcresources.k8s.aws +spec: + group: vpcresources.k8s.aws + names: + kind: CNINode + listKind: CNINodeList + plural: cninodes + shortNames: + - cnd + singular: cninode + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The features delegated to VPC resource controller + jsonPath: .spec.features + name: Features + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: 'Important: Run "make" to regenerate code after modifying + this file CNINodeSpec defines the desired state of CNINode' + properties: + features: + items: + description: FeatureName is a type of feature name supported by + AWS VPC CNI. It can be Security Group for Pods, custom networking, + or others + type: string + type: array + type: object + status: + description: CNINodeStatus defines the managed VPC resources. + type: object + type: object + served: true + storage: true + subresources: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index c59942c7..5b255939 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -64,6 +64,17 @@ rules: - patch - update - watch +- apiGroups: + - vpcresources.k8s.aws + resources: + - cninodes + verbs: + - create + - delete + - get + - list + - patch + - watch - apiGroups: - vpcresources.k8s.aws resources: diff --git a/config/samples/vpcresources_v1alpha1_cninode.yaml b/config/samples/vpcresources_v1alpha1_cninode.yaml new file mode 100644 index 00000000..5190ae37 --- /dev/null +++ b/config/samples/vpcresources_v1alpha1_cninode.yaml @@ -0,0 +1,10 @@ +# Example of a CNINode +apiVersion: vpcresources.k8s.aws/v1alpha1 +kind: CNINode +metadata: + name: cninode-example +spec: + features: + - SecurityGroupsForPods + - CustomNetworking + \ No newline at end of file diff --git a/controllers/apps/deployment_controller.go b/controllers/apps/deployment_controller.go index a33ad478..a560c573 100644 --- a/controllers/apps/deployment_controller.go +++ b/controllers/apps/deployment_controller.go @@ -16,15 +16,17 @@ package apps import ( "context" - "github.com/aws/amazon-vpc-resource-controller-k8s/controllers/core" + controllers "github.com/aws/amazon-vpc-resource-controller-k8s/controllers/core" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/condition" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config" + rcHealthz "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/healthz" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/k8s" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/node/manager" "github.com/go-logr/logr" appV1 "k8s.io/api/apps/v1" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/healthz" ) type DeploymentReconciler struct { @@ -63,7 +65,13 @@ func (r *DeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, nil } -func (r *DeploymentReconciler) SetupWithManager(mgr ctrl.Manager) error { +func (r *DeploymentReconciler) SetupWithManager(mgr ctrl.Manager, healthzHandler *rcHealthz.HealthzHandler) error { + // add health check on subpath for deployment controller + // TODO: this is a simple controller and unlikely hit blocking issue but we can revisit this after subpaths are released for a while + healthzHandler.AddControllersHealthCheckers( + map[string]healthz.Checker{"health-deploy-controller": rcHealthz.SimplePing("deployment controller", r.Log)}, + ) + return ctrl.NewControllerManagedBy(mgr). For(&appV1.Deployment{}). Complete(r) diff --git a/controllers/core/configmap_controller.go b/controllers/core/configmap_controller.go index f3e39d26..5391526d 100644 --- a/controllers/core/configmap_controller.go +++ b/controllers/core/configmap_controller.go @@ -19,6 +19,7 @@ import ( "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/condition" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config" + rcHealthz "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/healthz" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/k8s" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/node/manager" @@ -29,6 +30,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/healthz" ) // ConfigMapReconciler reconciles a ConfigMap object @@ -44,6 +46,7 @@ type ConfigMapReconciler struct { curWinPDWarmIPTarget int curWinPDMinIPTarget int curWinPDWarmPrefixTarget int + Context context.Context } //+kubebuilder:rbac:groups=core,resources=configmaps,namespace=kube-system,resourceNames=amazon-vpc-cni,verbs=get;list;watch @@ -119,7 +122,12 @@ func (r *ConfigMapReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( } // SetupWithManager sets up the controller with the Manager. -func (r *ConfigMapReconciler) SetupWithManager(mgr ctrl.Manager) error { +func (r *ConfigMapReconciler) SetupWithManager(mgr ctrl.Manager, healthzHandler *rcHealthz.HealthzHandler) error { + // add health check on subpath for CM controller + healthzHandler.AddControllersHealthCheckers( + map[string]healthz.Checker{"health-cm-controller": r.check()}, + ) + // Explicitly set MaxConcurrentReconciles to 1 to ensure concurrent reconciliation NOT supported for config map controller. // Don't change to more than 1 unless the struct is guarded against concurrency issues. return ctrl.NewControllerManagedBy(mgr). @@ -148,3 +156,9 @@ func UpdateNodesOnConfigMapChanges(k8sAPI k8s.K8sWrapper, nodeManager manager.Ma } return nil } + +func (r *ConfigMapReconciler) check() healthz.Checker { + r.Log.Info("ConfigMap controller's healthz subpath was added") + // We can revisit this to use PingWithTimeout() instead if we have concerns on this controller. + return rcHealthz.SimplePing("configmap controller", r.Log) +} diff --git a/controllers/core/node_controller.go b/controllers/core/node_controller.go index 96e17812..9e191c22 100644 --- a/controllers/core/node_controller.go +++ b/controllers/core/node_controller.go @@ -16,17 +16,22 @@ package controllers import ( "context" goErr "errors" + "net/http" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/condition" + rcHealthz "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/healthz" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/node/manager" + "github.com/google/uuid" "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/healthz" ) // MaxNodeConcurrentReconciles is the number of go routines that can invoke @@ -43,7 +48,7 @@ type NodeReconciler struct { Scheme *runtime.Scheme Manager manager.Manager Conditions condition.Conditions - // NodeEventCache *bigcache.BigCache + Context context.Context } // +kubebuilder:rbac:groups=core,resources=nodes,verbs=get;list;watch @@ -68,6 +73,7 @@ func (r *NodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. if err := r.Client.Get(ctx, req.NamespacedName, node); err != nil { if errors.IsNotFound(err) { + r.Log.V(1).Info("the requested node couldn't be found by k8s client", "Node", req.NamespacedName) _, found := r.Manager.GetNode(req.Name) // if cachedNode != nil && cachedNode.HasInstance() { // // delete the not found node instance id from node event cache for housekeeping @@ -80,7 +86,7 @@ func (r *NodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. logger.Error(err, "failed to delete node from manager") return ctrl.Result{}, nil } - logger.Info("deleted the node from manager") + logger.V(1).Info("deleted the node from manager") } } return ctrl.Result{}, client.IgnoreNotFound(err) @@ -98,17 +104,45 @@ func (r *NodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. return ctrl.Result{}, err } -// func (r *NodeReconciler) deleteNodeFromNodeEventCache(nodeId string) { -// if err := r.NodeEventCache.Delete(nodeId); err != nil { -// r.Log.V(1).Info("node controller removing node from node event cache failed", "Error", err) -// } else { -// r.Log.V(1).Info("node controller removed the node from node event cache successfully", "InstanceId", nodeId) -// } -// } +func (r *NodeReconciler) SetupWithManager(mgr ctrl.Manager, healthzHandler *rcHealthz.HealthzHandler) error { + // add health check on subpath for node controller + healthzHandler.AddControllersHealthCheckers( + map[string]healthz.Checker{"health-node-controller": r.Check()}, + ) -func (r *NodeReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&corev1.Node{}). WithOptions(controller.Options{MaxConcurrentReconciles: MaxNodeConcurrentReconciles}). Complete(r) } + +func (r *NodeReconciler) Check() healthz.Checker { + r.Log.Info("Node controller's healthz subpath was added") + return func(req *http.Request) error { + // if the reconciler is not ready, using the simple ping to test + // this can test the referenced cached pod datastore + if !r.Conditions.GetPodDataStoreSyncStatus() { + r.Log.V(1).Info("***** node controller healthz enpoint tested Simple Ping *****") + return nil + } + + err := rcHealthz.PingWithTimeout(func(c chan<- error) { + // when the reconciler is ready, testing the reconciler with a fake node request + pingRequest := &ctrl.Request{ + NamespacedName: types.NamespacedName{ + Namespace: corev1.NamespaceDefault, + Name: uuid.New().String(), + }, + } + + // expecting to 'return ctrl.Result{}, client.IgnoreNotFound(err)' + // IgnoreNotFound returns nil on NotFound errors. + // this can test the pod cached datastore and node cached datastore + _, rErr := r.Reconcile(r.Context, *pingRequest) + r.Log.V(1).Info("***** node controller healthz endpoint tested Reconcile *****") + c <- rErr + }, r.Log) + + return err + } +} diff --git a/controllers/core/pod_controller.go b/controllers/core/pod_controller.go index f4ac7364..5a0afa76 100644 --- a/controllers/core/pod_controller.go +++ b/controllers/core/pod_controller.go @@ -16,21 +16,26 @@ package controllers import ( "context" "fmt" + "net/http" "time" "github.com/aws/amazon-vpc-resource-controller-k8s/controllers/custom" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/condition" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config" + rcHealthz "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/healthz" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/k8s" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/k8s/pod" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/node/manager" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/resource" + "github.com/google/uuid" "github.com/go-logr/logr" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/healthz" ) // +kubebuilder:rbac:groups="",resources=events,verbs=create;update;patch @@ -73,7 +78,7 @@ func (r *PodReconciler) Reconcile(request custom.Request) (ctrl.Result, error) { return ctrl.Result{}, nil } if !exists { - r.Log.Info("pod doesn't exists in the cache anymore", + r.Log.V(1).Info("pod doesn't exists in the cache anymore", "namespace name", request.NamespacedName.String()) return ctrl.Result{}, nil } @@ -182,9 +187,10 @@ func getAggregateResources(pod *v1.Pod) map[string]int64 { // list of runnable. After Manager acquire the lease the pod controller runnable // will be started and the Pod events will be sent to Reconcile function func (r *PodReconciler) SetupWithManager(ctx context.Context, manager ctrl.Manager, - clientSet *kubernetes.Clientset, pageLimit int, syncPeriod time.Duration) error { + clientSet *kubernetes.Clientset, pageLimit int, syncPeriod time.Duration, healthzHandler *rcHealthz.HealthzHandler) error { r.Log.Info("The pod controller is using MaxConcurrentReconciles", "Routines", MaxPodConcurrentReconciles) - return custom.NewControllerManagedBy(ctx, manager). + + customChecker, err := custom.NewControllerManagedBy(ctx, manager). WithLogger(r.Log.WithName("custom pod controller")). UsingDataStore(r.DataStore). WithClientSet(clientSet). @@ -196,6 +202,38 @@ func (r *PodReconciler) SetupWithManager(ctx context.Context, manager ctrl.Manag ResyncPeriod: syncPeriod, MaxConcurrentReconciles: MaxPodConcurrentReconciles, }).UsingConditions(r.Condition).Complete(r) + + // add health check on subpath for pod and pod customized controllers + healthzHandler.AddControllersHealthCheckers( + map[string]healthz.Checker{ + "health-pod-controller": r.check(), + "health-custom-pod-controller": customChecker, + }, + ) + + return err +} + +func (r *PodReconciler) check() healthz.Checker { + r.Log.Info("Pod controller's healthz subpath was added") + // more meaningful ping + return func(req *http.Request) error { + err := rcHealthz.PingWithTimeout(func(c chan<- error) { + pingRequest := &custom.Request{ + NamespacedName: types.NamespacedName{ + Namespace: v1.NamespaceDefault, + Name: uuid.New().String(), + }, + DeletedObject: nil, + } + // calling reconcile will test pod cache + _, rErr := r.Reconcile(*pingRequest) + r.Log.V(1).Info("***** pod controller healthz endpoint tested reconcile *****") + c <- rErr + }, r.Log) + + return err + } } // updateResourceName updates resource name according to pod event and which IP allocation mode is enabled diff --git a/controllers/custom/builder.go b/controllers/custom/builder.go index c8e7aceb..7c2a4658 100644 --- a/controllers/custom/builder.go +++ b/controllers/custom/builder.go @@ -25,6 +25,7 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/manager" ) @@ -89,21 +90,21 @@ func NewControllerManagedBy(ctx context.Context, mgr manager.Manager) *Builder { // Complete adds the controller to manager's Runnable. The Controller // runnable will start when the manager starts -func (b *Builder) Complete(reconciler Reconciler) error { +func (b *Builder) Complete(reconciler Reconciler) (healthz.Checker, error) { // Loggr is no longer an interface // The suggestion is using LogSink to do nil check now if b.log.GetSink() == nil { - return fmt.Errorf("need to set the logger") + return nil, fmt.Errorf("need to set the logger") } if b.converter == nil { - return fmt.Errorf("converter not provided, " + + return nil, fmt.Errorf("converter not provided, " + "must use high level controller if conversion not required") } if b.clientSet == nil { - return fmt.Errorf("need to set kubernetes clienset") + return nil, fmt.Errorf("need to set kubernetes clienset") } if b.dataStore == nil { - return fmt.Errorf("need datastore to start the controller") + return nil, fmt.Errorf("need datastore to start the controller") } b.SetDefaults() @@ -172,17 +173,17 @@ func (b *Builder) Complete(reconciler Reconciler) error { }, } - controller := &CustomController{ - log: b.log, - options: b.options, - config: config, - Do: reconciler, - workQueue: workQueue, - conditions: b.conditions, - } + controller := NewCustomController( + b.log, + b.options, + config, + reconciler, + workQueue, + b.conditions, + ) // Adds the controller to the manager's Runnable - return b.mgr.Add(controller) + return controller.checker, b.mgr.Add(controller) } // SetDefaults sets the default options for controller diff --git a/controllers/custom/custom_controller.go b/controllers/custom/custom_controller.go index 7ae027f1..b3bfeee5 100644 --- a/controllers/custom/custom_controller.go +++ b/controllers/custom/custom_controller.go @@ -16,6 +16,7 @@ package custom import ( "context" "fmt" + "net/http" "time" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/condition" @@ -28,6 +29,9 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/healthz" + + rcHealthz "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/healthz" ) // Converter for converting k8s object and object list used in watches and list operation @@ -93,6 +97,8 @@ type CustomController struct { // the controller options Options conditions condition.Conditions + + checker healthz.Checker } // Request for Add/Update only contains the Namespace/Name @@ -106,6 +112,25 @@ type Request struct { DeletedObject interface{} } +func NewCustomController( + log logr.Logger, + options Options, + config *cache.Config, + reconciler Reconciler, + workQueue workqueue.RateLimitingInterface, + conditions condition.Conditions) *CustomController { + cc := &CustomController{ + log: log, + options: options, + config: config, + Do: reconciler, + workQueue: workQueue, + conditions: conditions, + } + cc.checker = cc.CustomCheck() + return cc +} + // Starts the low level controller func (c *CustomController) Start(ctx context.Context) error { // This is important to allow the data store to be synced @@ -256,3 +281,16 @@ func (c *CustomController) reconcileHandler(obj interface{}) bool { // Return true, don't take a break return true } + +func (c *CustomController) CustomCheck() healthz.Checker { + return func(req *http.Request) error { + err := rcHealthz.PingWithTimeout(func(status chan<- error) { + var ping interface{} + c.workQueue.NumRequeues(ping) + c.log.V(1).Info("***** health check on custom pod controller tested workQueue NumRequeues *****") + status <- nil + }, c.log) + + return err + } +} diff --git a/go.mod b/go.mod index 10e0589d..82d9630a 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( k8s.io/api v0.26.1 k8s.io/apimachinery v0.26.1 k8s.io/client-go v0.26.1 - sigs.k8s.io/controller-runtime v0.14.4 + sigs.k8s.io/controller-runtime v0.14.6 ) require ( diff --git a/go.sum b/go.sum index 86313395..ee3a058c 100644 --- a/go.sum +++ b/go.sum @@ -642,8 +642,8 @@ k8s.io/utils v0.0.0-20221128185143-99ec85e7a448/go.mod h1:OLgZIPagt7ERELqWJFomSt rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/controller-runtime v0.14.4 h1:Kd/Qgx5pd2XUL08eOV2vwIq3L9GhIbJ5Nxengbd4/0M= -sigs.k8s.io/controller-runtime v0.14.4/go.mod h1:WqIdsAY6JBsjfc/CqO0CORmNtoCtE4S6qbPc9s68h+0= +sigs.k8s.io/controller-runtime v0.14.6 h1:oxstGVvXGNnMvY7TAESYk+lzr6S3V5VFxQ6d92KcwQA= +sigs.k8s.io/controller-runtime v0.14.6/go.mod h1:WqIdsAY6JBsjfc/CqO0CORmNtoCtE4S6qbPc9s68h+0= sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= diff --git a/main.go b/main.go index a594d657..10462fb7 100644 --- a/main.go +++ b/main.go @@ -29,6 +29,7 @@ import ( ec2API "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/aws/ec2/api" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/condition" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config" + rcHealthz "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/healthz" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/k8s" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/k8s/pod" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/node/manager" @@ -52,7 +53,6 @@ import ( "k8s.io/client-go/tools/leaderelection/resourcelock" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" - "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/webhook" // +kubebuilder:scaffold:imports ) @@ -77,6 +77,7 @@ func init() { // +kubebuilder:rbac:groups=apps,resources=deployments,namespace=kube-system,resourceNames=vpc-resource-controller,verbs=get;list;watch // +kubebuilder:rbac:groups=crd.k8s.amazonaws.com,resources=eniconfigs,verbs=get;list;watch // +kubebuilder:rbac:groups=vpcresources.k8s.aws,resources=securitygrouppolicies,verbs=get;list;watch +// +kubebuilder:rbac:groups=vpcresources.k8s.aws,resources=cninodes,verbs=get;list;watch;create;patch;delete // Migration to leases based leader election // +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,namespace=kube-system,verbs=create @@ -96,6 +97,7 @@ func main() { var outputPath string var introspectBindAddr string var leaseOnly bool + var healthCheckTimeout int flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.") @@ -123,6 +125,8 @@ func main() { flag.IntVar(&listPageLimit, "page-limit", 100, "The page size limiting the number of response for list operation to API Server") flag.StringVar(&outputPath, "log-file", "stderr", "The path to redirect controller logs") + flag.IntVar(&healthCheckTimeout, "health-check-timeout", 10, + "How long healthz check waits before failing the attempt") flag.StringVar(&introspectBindAddr, "introspect-bind-addr", ":22775", "Port for serving the introspection API") flag.BoolVar(&leaseOnly, "lease-only", false, "Controller uses lease only for leader election") @@ -235,20 +239,16 @@ func main() { os.Exit(1) } + healthzHandler := rcHealthz.NewHealthzHandler(healthCheckTimeout) + // add root health ping on manager in general + healthzHandler.AddControllerHealthChecker("health-root-manager-ping", rcHealthz.SimplePing("root manager", setupLog)) + clientSet, err := kubernetes.NewForConfig(kubeConfig) if err != nil { setupLog.Error(err, "failed to create client set") os.Exit(1) } - // Add liveness probe - err = mgr.AddHealthzCheck("health-ping", healthz.Ping) - setupLog.Info("adding health check for controller") - if err != nil { - setupLog.Error(err, "unable add a health check") - os.Exit(1) - } - ctx := ctrl.SetupSignalHandler() ec2Wrapper, err := ec2API.NewEC2Wrapper(roleARN, setupLog) @@ -279,7 +279,8 @@ func main() { controllerConditions := condition.NewControllerConditions( ctrl.Log.WithName("controller conditions"), k8sApi) supportedResources := []string{config.ResourceNamePodENI, config.ResourceNameIPAddress, config.ResourceNameIPAddressFromPrefix} - resourceManager, err := resource.NewResourceManager(ctx, supportedResources, apiWrapper, controllerConditions) + resourceManager, err := resource.NewResourceManager( + ctx, supportedResources, apiWrapper, ctrl.Log.WithName("managers").WithName("resource"), healthzHandler, controllerConditions) if err != nil { ctrl.Log.Error(err, "failed to init resources", "resources", supportedResources) os.Exit(1) @@ -288,7 +289,8 @@ func main() { nodeManagerWorkers := asyncWorkers.NewDefaultWorkerPool("node async workers", 10, 1, ctrl.Log.WithName("node async workers"), ctx) nodeManager, err := manager.NewNodeManager(ctrl.Log.WithName("node manager"), resourceManager, - apiWrapper, nodeManagerWorkers, controllerConditions) + apiWrapper, nodeManagerWorkers, controllerConditions, healthzHandler) + if err != nil { ctrl.Log.Error(err, "failed to init node manager") os.Exit(1) @@ -296,85 +298,67 @@ func main() { // IMPORTANT: The Pod Reconciler must be the first controller to Run. The controller // will not allow any other controller to run till the cache has synced. - if err = (&corecontroller.PodReconciler{ + if err := (&corecontroller.PodReconciler{ Log: ctrl.Log.WithName("controllers").WithName("Pod Reconciler"), ResourceManager: resourceManager, NodeManager: nodeManager, K8sAPI: k8sApi, DataStore: dataStore, Condition: controllerConditions, - }).SetupWithManager(ctx, mgr, clientSet, listPageLimit, syncPeriod); err != nil { + }).SetupWithManager(ctx, mgr, clientSet, listPageLimit, syncPeriod, healthzHandler); err != nil { setupLog.Error(err, "unable to create controller", "controller", "pod") os.Exit(1) } - if err = (&ec2API.ENICleaner{ + if err := (&ec2API.ENICleaner{ EC2Wrapper: ec2Wrapper, ClusterName: clusterName, Log: ctrl.Log.WithName("eni cleaner"), - }).SetupWithManager(ctx, mgr); err != nil { + }).SetupWithManager(ctx, mgr, healthzHandler); err != nil { setupLog.Error(err, "unable to start eni cleaner") os.Exit(1) } - // config := bigcache.Config{ - // Shards: config.InstancesCacheShards, - // LifeWindow: config.InstancesCacheTTL, - // HardMaxCacheSize: config.InstancesCacheMaxSize, - // } - // nodeEventCache, err := bigcache.New(context.Background(), config) - if err != nil { - setupLog.Error(err, "Initializing node cache failed") - } - - if err = (&corecontroller.NodeReconciler{ + if err := (&corecontroller.NodeReconciler{ Client: mgr.GetClient(), Log: ctrl.Log.WithName("controllers").WithName("Node"), Scheme: mgr.GetScheme(), Manager: nodeManager, Conditions: controllerConditions, - // NodeEventCache: nodeEventCache, - }).SetupWithManager(mgr); err != nil { + Context: ctx, + }).SetupWithManager(mgr, healthzHandler); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Node") os.Exit(1) } - if err = (&corecontroller.ConfigMapReconciler{ + if err := (&corecontroller.ConfigMapReconciler{ Client: mgr.GetClient(), Log: ctrl.Log.WithName("controllers").WithName("ConfigMap"), Scheme: mgr.GetScheme(), NodeManager: nodeManager, K8sAPI: k8sApi, Condition: controllerConditions, - }).SetupWithManager(mgr); err != nil { + Context: ctx, + }).SetupWithManager(mgr, healthzHandler); err != nil { setupLog.Error(err, "unable to create controller", "controller", "ConfigMap") os.Exit(1) } - if err = (&apps.DeploymentReconciler{ + if err := (&apps.DeploymentReconciler{ Log: ctrl.Log.WithName("controllers").WithName("Deployment"), NodeManager: nodeManager, K8sAPI: k8sApi, Condition: controllerConditions, - }).SetupWithManager(mgr); err != nil { + }).SetupWithManager(mgr, healthzHandler); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Deployment") os.Exit(1) } - // Disable Event controller for now due to known possible performance impacts on etcd servers - // if err = (corecontroller.NewEventReconciler( - // ctrl.Log.WithName("Controllers").WithName("Event"), - // mgr.GetScheme(), - // k8sApi, nodeEventCache).SetupWithManager(mgr)); err != nil { - // setupLog.Error(err, "unable to create controller", "controller", "Event") - // os.Exit(1) - // } - - if err = (&resource.IntrospectHandler{ + if err := (&resource.IntrospectHandler{ Log: ctrl.Log.WithName("introspect"), BindAddress: introspectBindAddr, ResourceManager: resourceManager, - }).SetupWithManager(mgr); err != nil { + }).SetupWithManager(mgr, healthzHandler); err != nil { setupLog.Error(err, "unable to create introspect API") os.Exit(1) } @@ -384,25 +368,31 @@ func main() { webhookServer := mgr.GetWebhookServer() setupLog.Info("registering webhooks to the webhook server") + podMutationWebhook := webhookcore.NewPodMutationWebHook( + sgpAPI, ctrl.Log.WithName("resource mutating webhook"), controllerConditions, healthzHandler) webhookServer.Register("/mutate-v1-pod", &webhook.Admission{ - Handler: &webhookcore.PodMutationWebHook{ - SGPAPI: sgpAPI, - Log: ctrl.Log.WithName("resource mutation webhook"), - Condition: controllerConditions, - }}) + Handler: podMutationWebhook, + }) + nodeValidateWebhook := webhookcore.NewNodeUpdateWebhook( + controllerConditions, ctrl.Log.WithName("node validating webhook"), healthzHandler) webhookServer.Register("/validate-v1-node", &webhook.Admission{ - Handler: &webhookcore.NodeUpdateWebhook{ - Log: ctrl.Log.WithName("node validation webhook"), - Condition: controllerConditions, - }}) + Handler: nodeValidateWebhook}) // Validating webhook for pod. + annotationValidator := webhookcore.NewAnnotationValidator( + controllerConditions, ctrl.Log.WithName("annotation validating webhook"), healthzHandler) webhookServer.Register("/validate-v1-pod", &webhook.Admission{ - Handler: &webhookcore.AnnotationValidator{ - Log: ctrl.Log.WithName("annotation validation webhook"), - Condition: controllerConditions, - }}) + Handler: annotationValidator}) + + // Enabled each controllers' health check and aggregate them to endpoint /healthz + // curl localhost:61779/healthz?verbose can list all controllers' healthy status + err = healthzHandler.AddControllersHealthStatusChecksToManager(mgr) + setupLog.Info("adding health check for controllers") + if err != nil { + setupLog.Error(err, "unable add health check to all controllers") + os.Exit(1) + } setupLog.Info("starting manager") if err := mgr.Start(ctx); err != nil { diff --git a/mocks/amazon-vcp-resource-controller-k8s/pkg/aws/ec2/mock_instance.go b/mocks/amazon-vcp-resource-controller-k8s/pkg/aws/ec2/mock_instance.go index 2188a3a4..902cafb1 100644 --- a/mocks/amazon-vcp-resource-controller-k8s/pkg/aws/ec2/mock_instance.go +++ b/mocks/amazon-vcp-resource-controller-k8s/pkg/aws/ec2/mock_instance.go @@ -47,6 +47,20 @@ func (m *MockEC2Instance) EXPECT() *MockEC2InstanceMockRecorder { return m.recorder } +// CurrentInstanceSecurityGroups mocks base method. +func (m *MockEC2Instance) CurrentInstanceSecurityGroups() []string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CurrentInstanceSecurityGroups") + ret0, _ := ret[0].([]string) + return ret0 +} + +// CurrentInstanceSecurityGroups indicates an expected call of CurrentInstanceSecurityGroups. +func (mr *MockEC2InstanceMockRecorder) CurrentInstanceSecurityGroups() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CurrentInstanceSecurityGroups", reflect.TypeOf((*MockEC2Instance)(nil).CurrentInstanceSecurityGroups)) +} + // FreeDeviceIndex mocks base method. func (m *MockEC2Instance) FreeDeviceIndex(arg0 int64) { m.ctrl.T.Helper() @@ -88,20 +102,6 @@ func (mr *MockEC2InstanceMockRecorder) InstanceID() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstanceID", reflect.TypeOf((*MockEC2Instance)(nil).InstanceID)) } -// InstanceSecurityGroup mocks base method. -func (m *MockEC2Instance) InstanceSecurityGroup() []string { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "InstanceSecurityGroup") - ret0, _ := ret[0].([]string) - return ret0 -} - -// InstanceSecurityGroup indicates an expected call of InstanceSecurityGroup. -func (mr *MockEC2InstanceMockRecorder) InstanceSecurityGroup() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstanceSecurityGroup", reflect.TypeOf((*MockEC2Instance)(nil).InstanceSecurityGroup)) -} - // LoadDetails mocks base method. func (m *MockEC2Instance) LoadDetails(arg0 api.EC2APIHelper) error { m.ctrl.T.Helper() diff --git a/mocks/amazon-vcp-resource-controller-k8s/pkg/node/mock_node.go b/mocks/amazon-vcp-resource-controller-k8s/pkg/node/mock_node.go index 49c2f56e..476e145a 100644 --- a/mocks/amazon-vcp-resource-controller-k8s/pkg/node/mock_node.go +++ b/mocks/amazon-vcp-resource-controller-k8s/pkg/node/mock_node.go @@ -20,7 +20,6 @@ package mock_node import ( reflect "reflect" - api "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/aws/ec2/api" resource "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/resource" gomock "github.com/golang/mock/gomock" ) @@ -49,17 +48,17 @@ func (m *MockNode) EXPECT() *MockNodeMockRecorder { } // DeleteResources mocks base method. -func (m *MockNode) DeleteResources(arg0 resource.ResourceManager, arg1 api.EC2APIHelper) error { +func (m *MockNode) DeleteResources(arg0 resource.ResourceManager) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteResources", arg0, arg1) + ret := m.ctrl.Call(m, "DeleteResources", arg0) ret0, _ := ret[0].(error) return ret0 } // DeleteResources indicates an expected call of DeleteResources. -func (mr *MockNodeMockRecorder) DeleteResources(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockNodeMockRecorder) DeleteResources(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteResources", reflect.TypeOf((*MockNode)(nil).DeleteResources), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteResources", reflect.TypeOf((*MockNode)(nil).DeleteResources), arg0) } // GetNodeInstanceID mocks base method. @@ -91,17 +90,17 @@ func (mr *MockNodeMockRecorder) HasInstance() *gomock.Call { } // InitResources mocks base method. -func (m *MockNode) InitResources(arg0 resource.ResourceManager, arg1 api.EC2APIHelper) error { +func (m *MockNode) InitResources(arg0 resource.ResourceManager) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "InitResources", arg0, arg1) + ret := m.ctrl.Call(m, "InitResources", arg0) ret0, _ := ret[0].(error) return ret0 } // InitResources indicates an expected call of InitResources. -func (mr *MockNodeMockRecorder) InitResources(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockNodeMockRecorder) InitResources(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InitResources", reflect.TypeOf((*MockNode)(nil).InitResources), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InitResources", reflect.TypeOf((*MockNode)(nil).InitResources), arg0) } // IsManaged mocks base method. @@ -145,15 +144,15 @@ func (mr *MockNodeMockRecorder) UpdateCustomNetworkingSpecs(arg0, arg1 interface } // UpdateResources mocks base method. -func (m *MockNode) UpdateResources(arg0 resource.ResourceManager, arg1 api.EC2APIHelper) error { +func (m *MockNode) UpdateResources(arg0 resource.ResourceManager) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UpdateResources", arg0, arg1) + ret := m.ctrl.Call(m, "UpdateResources", arg0) ret0, _ := ret[0].(error) return ret0 } // UpdateResources indicates an expected call of UpdateResources. -func (mr *MockNodeMockRecorder) UpdateResources(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockNodeMockRecorder) UpdateResources(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateResources", reflect.TypeOf((*MockNode)(nil).UpdateResources), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateResources", reflect.TypeOf((*MockNode)(nil).UpdateResources), arg0) } diff --git a/mocks/amazon-vcp-resource-controller-k8s/pkg/provider/mock_provider.go b/mocks/amazon-vcp-resource-controller-k8s/pkg/provider/mock_provider.go index f28a4d57..fe60bb16 100644 --- a/mocks/amazon-vcp-resource-controller-k8s/pkg/provider/mock_provider.go +++ b/mocks/amazon-vcp-resource-controller-k8s/pkg/provider/mock_provider.go @@ -23,6 +23,7 @@ import ( ec2 "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/aws/ec2" pool "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/pool" gomock "github.com/golang/mock/gomock" + healthz "sigs.k8s.io/controller-runtime/pkg/healthz" reconcile "sigs.k8s.io/controller-runtime/pkg/reconcile" ) @@ -63,6 +64,20 @@ func (mr *MockResourceProviderMockRecorder) DeInitResource(arg0 interface{}) *go return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeInitResource", reflect.TypeOf((*MockResourceProvider)(nil).DeInitResource), arg0) } +// GetHealthChecker mocks base method. +func (m *MockResourceProvider) GetHealthChecker() healthz.Checker { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetHealthChecker") + ret0, _ := ret[0].(healthz.Checker) + return ret0 +} + +// GetHealthChecker indicates an expected call of GetHealthChecker. +func (mr *MockResourceProviderMockRecorder) GetHealthChecker() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetHealthChecker", reflect.TypeOf((*MockResourceProvider)(nil).GetHealthChecker)) +} + // GetPool mocks base method. func (m *MockResourceProvider) GetPool(arg0 string) (pool.Pool, bool) { m.ctrl.T.Helper() diff --git a/pkg/aws/ec2/api/eni_cleanup.go b/pkg/aws/ec2/api/eni_cleanup.go index d0737a13..f51e29b2 100644 --- a/pkg/aws/ec2/api/eni_cleanup.go +++ b/pkg/aws/ec2/api/eni_cleanup.go @@ -19,11 +19,13 @@ import ( "time" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config" + rcHealthz "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/healthz" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" "github.com/go-logr/logr" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/healthz" ) type ENICleaner struct { @@ -37,11 +39,17 @@ type ENICleaner struct { ctx context.Context } -func (e *ENICleaner) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error { +func (e *ENICleaner) SetupWithManager(ctx context.Context, mgr ctrl.Manager, healthzHandler *rcHealthz.HealthzHandler) error { e.clusterNameTagKey = fmt.Sprintf(config.ClusterNameTagKeyFormat, e.ClusterName) e.availableENIs = make(map[string]struct{}) e.ctx = ctx + healthzHandler.AddControllersHealthCheckers( + map[string]healthz.Checker{ + "health-interface-cleaner": rcHealthz.SimplePing("interface cleanup", e.Log), + }, + ) + return mgr.Add(e) } diff --git a/pkg/aws/ec2/instance.go b/pkg/aws/ec2/instance.go index 6edde892..698943ae 100644 --- a/pkg/aws/ec2/instance.go +++ b/pkg/aws/ec2/instance.go @@ -43,20 +43,20 @@ type ec2Instance struct { currentSubnetID string // currentSubnetCIDRBlock can either point to the Subnet CIDR block for instance subnet or subnet from ENIConfig currentSubnetCIDRBlock string - // currentInstanceSecurityGroup can either point to the instance security group or the security group in ENIConfig - currentInstanceSecurityGroup []string + // currentInstanceSecurityGroups can either point to the primary network interface security groups or the security groups in ENIConfig + currentInstanceSecurityGroups []string // subnetMask is the mask of the subnet CIDR block subnetMask string // deviceIndexes is the list of indexes used by the EC2 Instance deviceIndexes []bool - // instanceSecurityGroups is the security group used by the primary network interface - instanceSecurityGroups []string + // primaryENIGroups is the security group used by the primary network interface + primaryENISecurityGroups []string // primaryENIID is the ID of the primary network interface of the instance primaryENIID string // newCustomNetworkingSubnetID is the SubnetID from the ENIConfig newCustomNetworkingSubnetID string - // newCustomNetworkingSecurityGroup is the security group from the ENIConfig - newCustomNetworkingSecurityGroup []string + // newCustomNetworkingSecurityGroups is the security groups from the ENIConfig + newCustomNetworkingSecurityGroups []string } // EC2Instance exposes the immutable details of an ec2 instance and common operations on an EC2 Instance @@ -72,7 +72,7 @@ type EC2Instance interface { SubnetMask() string SubnetCidrBlock() string PrimaryNetworkInterfaceID() string - InstanceSecurityGroup() []string + CurrentInstanceSecurityGroups() []string SetNewCustomNetworkingSpec(subnetID string, securityGroup []string) UpdateCurrentSubnetAndCidrBlock(helper api.EC2APIHelper) error } @@ -115,7 +115,7 @@ func (i *ec2Instance) LoadDetails(ec2APIHelper api.EC2APIHelper) error { i.instanceType = *instance.InstanceType limits, ok := vpc.Limits[i.instanceType] if !ok { - return fmt.Errorf("unsupported instance type, couldn't find ENI Limit for instance %s", i.instanceType) + return fmt.Errorf("unsupported instance type, couldn't find ENI Limit for instance %s, error: %w", i.instanceType, utils.ErrNotFound) } defaultCardIdx := limits.DefaultNetworkCardIndex @@ -140,11 +140,11 @@ func (i *ec2Instance) LoadDetails(ec2APIHelper api.EC2APIHelper) error { i.deviceIndexes[*index] = true // Load the Security group of the primary network interface - if i.instanceSecurityGroups == nil && (nwInterface.PrivateIpAddress != nil && instance.PrivateIpAddress != nil && *nwInterface.PrivateIpAddress == *instance.PrivateIpAddress) { + if i.primaryENISecurityGroups == nil && (nwInterface.PrivateIpAddress != nil && instance.PrivateIpAddress != nil && *nwInterface.PrivateIpAddress == *instance.PrivateIpAddress) { i.primaryENIID = *nwInterface.NetworkInterfaceId // TODO: Group can change, should be refreshed each time we want to use this for _, group := range nwInterface.Groups { - i.instanceSecurityGroups = append(i.instanceSecurityGroups, *group.GroupId) + i.primaryENISecurityGroups = append(i.primaryENISecurityGroups, *group.GroupId) } } } @@ -192,12 +192,13 @@ func (i *ec2Instance) PrimaryNetworkInterfaceID() string { return i.primaryENIID } -// InstanceSecurityGroup returns the instance security group of the primary network interface -func (i *ec2Instance) InstanceSecurityGroup() []string { +// CurrentInstanceSecurityGroups returns the current instance security groups +// (primary network interface SG or SG specified in the ENIConfig) +func (i *ec2Instance) CurrentInstanceSecurityGroups() []string { i.lock.RLock() defer i.lock.RUnlock() - return i.currentInstanceSecurityGroup + return i.currentInstanceSecurityGroups } // GetHighestUnusedDeviceIndex assigns a free device index from the end of the list since IPAMD assigns indexes from @@ -236,7 +237,7 @@ func (i *ec2Instance) SetNewCustomNetworkingSpec(subnet string, securityGroups [ defer i.lock.Unlock() i.newCustomNetworkingSubnetID = subnet - i.newCustomNetworkingSecurityGroup = securityGroups + i.newCustomNetworkingSecurityGroups = securityGroups } // UpdateCurrentSubnetAndCidrBlock updates the subnet details under a write lock @@ -253,7 +254,12 @@ func (i *ec2Instance) updateCurrentSubnetAndCidrBlock(ec2APIHelper api.EC2APIHel // Custom networking is being used on node, point the current subnet ID, CIDR block and // instance security group to the one's present in the Custom networking spec if i.newCustomNetworkingSubnetID != "" { - i.currentInstanceSecurityGroup = i.newCustomNetworkingSecurityGroup + if i.newCustomNetworkingSecurityGroups != nil && len(i.newCustomNetworkingSecurityGroups) > 0 { + i.currentInstanceSecurityGroups = i.newCustomNetworkingSecurityGroups + } else { + // when security groups are not specified in ENIConfig, use the primary network interface SG as per custom networking documentation + i.currentInstanceSecurityGroups = i.primaryENISecurityGroups + } // Only get the subnet CIDR block again if the subnet ID has changed if i.newCustomNetworkingSubnetID != i.currentSubnetID { customSubnet, err := ec2APIHelper.GetSubnet(&i.newCustomNetworkingSubnetID) @@ -267,11 +273,11 @@ func (i *ec2Instance) updateCurrentSubnetAndCidrBlock(ec2APIHelper api.EC2APIHel i.currentSubnetCIDRBlock = *customSubnet.CidrBlock } } else { - // Custom networking in not being used, point to the instance security group and + // Custom networking in not being used, point to the primary network interface security group and // subnet details i.currentSubnetID = i.instanceSubnetID i.currentSubnetCIDRBlock = i.instanceSubnetCidrBlock - i.currentInstanceSecurityGroup = i.instanceSecurityGroups + i.currentInstanceSecurityGroups = i.primaryENISecurityGroups } return nil diff --git a/pkg/aws/ec2/instance_test.go b/pkg/aws/ec2/instance_test.go index f705d0fe..676af8ad 100644 --- a/pkg/aws/ec2/instance_test.go +++ b/pkg/aws/ec2/instance_test.go @@ -17,7 +17,8 @@ import ( "fmt" "testing" - "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/aws/ec2/api" + mock_api "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/aws/ec2/api" + "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/utils" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" @@ -118,7 +119,7 @@ func TestEc2Instance_LoadDetails(t *testing.T) { assert.Equal(t, subnetCidrBlock, ec2Instance.SubnetCidrBlock()) assert.Equal(t, instanceType, ec2Instance.Type()) assert.Equal(t, []bool{true, false, true}, ec2Instance.deviceIndexes) - assert.Equal(t, []string{securityGroup1, securityGroup2}, ec2Instance.InstanceSecurityGroup()) + assert.Equal(t, []string{securityGroup1, securityGroup2}, ec2Instance.CurrentInstanceSecurityGroups()) assert.Equal(t, primaryInterfaceID, ec2Instance.PrimaryNetworkInterfaceID()) } @@ -197,7 +198,7 @@ func TestEc2Instance_LoadDetails_SubnetPreLoaded(t *testing.T) { // Set the custom networking subnet ID and CIDR block ec2Instance.newCustomNetworkingSubnetID = customNWSubnetID - ec2Instance.newCustomNetworkingSecurityGroup = customNWSecurityGroups + ec2Instance.newCustomNetworkingSecurityGroups = customNWSecurityGroups customSubnet := &ec2.Subnet{CidrBlock: &customNWSubnetCidr} @@ -208,7 +209,7 @@ func TestEc2Instance_LoadDetails_SubnetPreLoaded(t *testing.T) { err := ec2Instance.LoadDetails(mockEC2ApiHelper) assert.NoError(t, err) assert.Equal(t, customNWSubnetID, ec2Instance.currentSubnetID) - assert.Equal(t, customNWSecurityGroups, ec2Instance.currentInstanceSecurityGroup) + assert.Equal(t, customNWSecurityGroups, ec2Instance.currentInstanceSecurityGroups) assert.Equal(t, customNWSubnetCidr, ec2Instance.currentSubnetCIDRBlock) } @@ -258,6 +259,8 @@ func TestEc2Instance_LoadDetails_InstanceENILimitNotFound(t *testing.T) { err := ec2Instance.LoadDetails(mockEC2ApiHelper) assert.NotNil(t, err) + // ensure the expected error is returned to trigger a node event + assert.ErrorIs(t, err, utils.ErrNotFound) // Clean up nwInterfaces.InstanceType = &instanceType @@ -329,3 +332,34 @@ func TestEc2Instance_E2E(t *testing.T) { ec2Instance.FreeDeviceIndex(deviceIndex0) assert.False(t, ec2Instance.deviceIndexes[deviceIndex0]) } + +// Tests instance details when custom networking is incorrectly configured- missing security groups +func TestEc2Instance_LoadDetails_InvalidCustomNetworkingConfiguration(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + ec2Instance, mockEC2ApiHelper := getMockInstance(ctrl) + + // Set the instance subnet ID and CIDR block + ec2Instance.instanceSubnetID = subnetID + ec2Instance.instanceSubnetCidrBlock = subnetCidrBlock + + // Set the custom networking subnet ID and CIDR block + customNWSubnetID := "custom-networking" + ec2Instance.newCustomNetworkingSubnetID = customNWSubnetID + ec2Instance.newCustomNetworkingSecurityGroups = []string{} + + customNWSubnetCidr := "192.2.0.0/24" + customSubnet := &ec2.Subnet{CidrBlock: &customNWSubnetCidr} + + mockEC2ApiHelper.EXPECT().GetInstanceDetails(&instanceID).Return(nwInterfaces, nil) + mockEC2ApiHelper.EXPECT().GetSubnet(&subnetID).Return(subnet, nil) + mockEC2ApiHelper.EXPECT().GetSubnet(&customNWSubnetID).Return(customSubnet, nil) + + err := ec2Instance.LoadDetails(mockEC2ApiHelper) + assert.NoError(t, err) + assert.Equal(t, customNWSubnetID, ec2Instance.currentSubnetID) + // Expect the primary network interface security groups when ENIConfig SG is missing + assert.Equal(t, []string{securityGroup1, securityGroup2}, ec2Instance.currentInstanceSecurityGroups) + assert.Equal(t, customNWSubnetCidr, ec2Instance.currentSubnetCIDRBlock) +} diff --git a/pkg/aws/vpc/limits.go b/pkg/aws/vpc/limits.go index a3228d08..f3c9d98f 100644 --- a/pkg/aws/vpc/limits.go +++ b/pkg/aws/vpc/limits.go @@ -17,7 +17,7 @@ // so we can get this information at runtime. // Code generated by go generate; DO NOT EDIT. -// This file was generated at 2023-03-22T18:08:01Z +// This file was generated at 2023-05-05T13:38:13-07:00 package vpc @@ -36,6 +36,8 @@ type VPCLimits struct { BranchInterface int NetworkCards []NetworkCard DefaultNetworkCardIndex int + Hypervisor string + IsBareMetal bool } // VPC Limits and flags for ENI and IPv4 Addresses @@ -52,6 +54,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "a1.4xlarge": { Interface: 8, @@ -65,6 +69,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "a1.large": { Interface: 3, @@ -78,6 +84,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "a1.medium": { Interface: 2, @@ -91,6 +99,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "a1.metal": { Interface: 8, @@ -104,6 +114,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "a1.xlarge": { Interface: 4, @@ -117,6 +129,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c1.medium": { Interface: 2, @@ -130,6 +144,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "c1.xlarge": { Interface: 4, @@ -143,6 +159,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "c3.2xlarge": { Interface: 4, @@ -156,6 +174,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "c3.4xlarge": { Interface: 8, @@ -169,6 +189,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "c3.8xlarge": { Interface: 8, @@ -182,6 +204,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "c3.large": { Interface: 3, @@ -195,6 +219,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "c3.xlarge": { Interface: 4, @@ -208,6 +234,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "c4.2xlarge": { Interface: 4, @@ -221,6 +249,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "c4.4xlarge": { Interface: 8, @@ -234,6 +264,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "c4.8xlarge": { Interface: 8, @@ -247,6 +279,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "c4.large": { Interface: 3, @@ -260,6 +294,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "c4.xlarge": { Interface: 4, @@ -273,6 +309,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "c5.12xlarge": { Interface: 8, @@ -286,6 +324,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c5.18xlarge": { Interface: 15, @@ -299,6 +339,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c5.24xlarge": { Interface: 15, @@ -312,6 +354,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c5.2xlarge": { Interface: 4, @@ -325,6 +369,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c5.4xlarge": { Interface: 8, @@ -338,6 +384,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c5.9xlarge": { Interface: 8, @@ -351,6 +399,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c5.large": { Interface: 3, @@ -364,6 +414,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c5.metal": { Interface: 15, @@ -377,6 +429,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "c5.xlarge": { Interface: 4, @@ -390,6 +444,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c5a.12xlarge": { Interface: 8, @@ -403,6 +459,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c5a.16xlarge": { Interface: 15, @@ -416,6 +474,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c5a.24xlarge": { Interface: 15, @@ -429,6 +489,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c5a.2xlarge": { Interface: 4, @@ -442,6 +504,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c5a.4xlarge": { Interface: 8, @@ -455,6 +519,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c5a.8xlarge": { Interface: 8, @@ -468,6 +534,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c5a.large": { Interface: 3, @@ -481,6 +549,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c5a.xlarge": { Interface: 4, @@ -494,6 +564,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c5ad.12xlarge": { Interface: 8, @@ -507,6 +579,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c5ad.16xlarge": { Interface: 15, @@ -520,6 +594,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c5ad.24xlarge": { Interface: 15, @@ -533,6 +609,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c5ad.2xlarge": { Interface: 4, @@ -546,6 +624,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c5ad.4xlarge": { Interface: 8, @@ -559,6 +639,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c5ad.8xlarge": { Interface: 8, @@ -572,6 +654,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c5ad.large": { Interface: 3, @@ -585,6 +669,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c5ad.xlarge": { Interface: 4, @@ -598,6 +684,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c5d.12xlarge": { Interface: 8, @@ -611,6 +699,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c5d.18xlarge": { Interface: 15, @@ -624,6 +714,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c5d.24xlarge": { Interface: 15, @@ -637,6 +729,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c5d.2xlarge": { Interface: 4, @@ -650,6 +744,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c5d.4xlarge": { Interface: 8, @@ -663,6 +759,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c5d.9xlarge": { Interface: 8, @@ -676,6 +774,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c5d.large": { Interface: 3, @@ -689,6 +789,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c5d.metal": { Interface: 15, @@ -702,6 +804,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "c5d.xlarge": { Interface: 4, @@ -715,6 +819,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c5n.18xlarge": { Interface: 15, @@ -728,6 +834,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c5n.2xlarge": { Interface: 4, @@ -741,6 +849,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c5n.4xlarge": { Interface: 8, @@ -754,6 +864,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c5n.9xlarge": { Interface: 8, @@ -767,6 +879,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c5n.large": { Interface: 3, @@ -780,6 +894,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c5n.metal": { Interface: 15, @@ -793,6 +909,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "c5n.xlarge": { Interface: 4, @@ -806,6 +924,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6a.12xlarge": { Interface: 8, @@ -819,6 +939,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6a.16xlarge": { Interface: 15, @@ -832,6 +954,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6a.24xlarge": { Interface: 15, @@ -845,6 +969,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6a.2xlarge": { Interface: 4, @@ -858,6 +984,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6a.32xlarge": { Interface: 15, @@ -871,6 +999,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6a.48xlarge": { Interface: 15, @@ -884,6 +1014,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6a.4xlarge": { Interface: 8, @@ -897,6 +1029,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6a.8xlarge": { Interface: 8, @@ -910,6 +1044,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6a.large": { Interface: 3, @@ -923,6 +1059,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6a.metal": { Interface: 15, @@ -936,6 +1074,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "c6a.xlarge": { Interface: 4, @@ -949,6 +1089,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6g.12xlarge": { Interface: 8, @@ -962,6 +1104,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6g.16xlarge": { Interface: 15, @@ -975,6 +1119,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6g.2xlarge": { Interface: 4, @@ -988,6 +1134,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6g.4xlarge": { Interface: 8, @@ -1001,6 +1149,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6g.8xlarge": { Interface: 8, @@ -1014,6 +1164,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6g.large": { Interface: 3, @@ -1027,6 +1179,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6g.medium": { Interface: 2, @@ -1040,6 +1194,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6g.metal": { Interface: 15, @@ -1053,6 +1209,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "c6g.xlarge": { Interface: 4, @@ -1066,6 +1224,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6gd.12xlarge": { Interface: 8, @@ -1079,6 +1239,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6gd.16xlarge": { Interface: 15, @@ -1092,6 +1254,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6gd.2xlarge": { Interface: 4, @@ -1105,6 +1269,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6gd.4xlarge": { Interface: 8, @@ -1118,6 +1284,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6gd.8xlarge": { Interface: 8, @@ -1131,6 +1299,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6gd.large": { Interface: 3, @@ -1144,6 +1314,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6gd.medium": { Interface: 2, @@ -1157,6 +1329,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6gd.metal": { Interface: 15, @@ -1170,6 +1344,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "c6gd.xlarge": { Interface: 4, @@ -1183,6 +1359,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6gn.12xlarge": { Interface: 8, @@ -1196,6 +1374,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6gn.16xlarge": { Interface: 15, @@ -1209,6 +1389,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6gn.2xlarge": { Interface: 4, @@ -1222,6 +1404,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6gn.4xlarge": { Interface: 8, @@ -1235,6 +1419,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6gn.8xlarge": { Interface: 8, @@ -1248,6 +1434,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6gn.large": { Interface: 3, @@ -1261,6 +1449,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6gn.medium": { Interface: 2, @@ -1274,6 +1464,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6gn.xlarge": { Interface: 4, @@ -1287,6 +1479,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6i.12xlarge": { Interface: 8, @@ -1300,6 +1494,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6i.16xlarge": { Interface: 15, @@ -1313,6 +1509,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6i.24xlarge": { Interface: 15, @@ -1326,6 +1524,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6i.2xlarge": { Interface: 4, @@ -1339,6 +1539,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6i.32xlarge": { Interface: 15, @@ -1352,6 +1554,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6i.4xlarge": { Interface: 8, @@ -1365,6 +1569,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6i.8xlarge": { Interface: 8, @@ -1378,6 +1584,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6i.large": { Interface: 3, @@ -1391,6 +1599,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6i.metal": { Interface: 15, @@ -1404,6 +1614,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "c6i.xlarge": { Interface: 4, @@ -1417,6 +1629,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6id.12xlarge": { Interface: 8, @@ -1430,6 +1644,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6id.16xlarge": { Interface: 15, @@ -1443,6 +1659,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6id.24xlarge": { Interface: 15, @@ -1456,6 +1674,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6id.2xlarge": { Interface: 4, @@ -1469,6 +1689,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6id.32xlarge": { Interface: 15, @@ -1482,6 +1704,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6id.4xlarge": { Interface: 8, @@ -1495,6 +1719,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6id.8xlarge": { Interface: 8, @@ -1508,6 +1734,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6id.large": { Interface: 3, @@ -1521,6 +1749,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6id.metal": { Interface: 15, @@ -1534,6 +1764,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "c6id.xlarge": { Interface: 4, @@ -1547,6 +1779,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6in.12xlarge": { Interface: 8, @@ -1560,6 +1794,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6in.16xlarge": { Interface: 15, @@ -1573,6 +1809,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6in.24xlarge": { Interface: 15, @@ -1586,6 +1824,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6in.2xlarge": { Interface: 4, @@ -1599,6 +1839,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6in.32xlarge": { Interface: 14, @@ -1617,6 +1859,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 1, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6in.4xlarge": { Interface: 8, @@ -1630,6 +1874,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6in.8xlarge": { Interface: 8, @@ -1643,6 +1889,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6in.large": { Interface: 3, @@ -1656,6 +1904,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c6in.metal": { Interface: 14, @@ -1674,6 +1924,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 1, }, }, + Hypervisor: "", + IsBareMetal: true, }, "c6in.xlarge": { Interface: 4, @@ -1687,6 +1939,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c7g.12xlarge": { Interface: 8, @@ -1700,6 +1954,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c7g.16xlarge": { Interface: 15, @@ -1713,6 +1969,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c7g.2xlarge": { Interface: 4, @@ -1726,6 +1984,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c7g.4xlarge": { Interface: 8, @@ -1739,6 +1999,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c7g.8xlarge": { Interface: 8, @@ -1752,6 +2014,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c7g.large": { Interface: 3, @@ -1765,6 +2029,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c7g.medium": { Interface: 2, @@ -1778,6 +2044,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "c7g.metal": { Interface: 15, @@ -1791,6 +2059,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "c7g.xlarge": { Interface: 4, @@ -1804,6 +2074,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "d2.2xlarge": { Interface: 4, @@ -1817,6 +2089,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "d2.4xlarge": { Interface: 8, @@ -1830,6 +2104,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "d2.8xlarge": { Interface: 8, @@ -1843,6 +2119,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "d2.xlarge": { Interface: 4, @@ -1856,6 +2134,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "d3.2xlarge": { Interface: 4, @@ -1869,6 +2149,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "d3.4xlarge": { Interface: 4, @@ -1882,6 +2164,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "d3.8xlarge": { Interface: 3, @@ -1895,6 +2179,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "d3.xlarge": { Interface: 4, @@ -1908,6 +2194,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "d3en.12xlarge": { Interface: 3, @@ -1921,6 +2209,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "d3en.2xlarge": { Interface: 4, @@ -1934,6 +2224,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "d3en.4xlarge": { Interface: 4, @@ -1947,6 +2239,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "d3en.6xlarge": { Interface: 4, @@ -1960,6 +2254,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "d3en.8xlarge": { Interface: 4, @@ -1973,6 +2269,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "d3en.xlarge": { Interface: 4, @@ -1986,6 +2284,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "dl1.24xlarge": { Interface: 60, @@ -2014,6 +2314,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 3, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "f1.16xlarge": { Interface: 8, @@ -2027,6 +2329,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "f1.2xlarge": { Interface: 4, @@ -2040,6 +2344,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "f1.4xlarge": { Interface: 8, @@ -2053,6 +2359,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "g2.2xlarge": { Interface: 4, @@ -2066,6 +2374,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "g2.8xlarge": { Interface: 8, @@ -2079,6 +2389,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "g3.16xlarge": { Interface: 15, @@ -2092,6 +2404,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "g3.4xlarge": { Interface: 8, @@ -2105,6 +2419,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "g3.8xlarge": { Interface: 8, @@ -2118,6 +2434,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "g3s.xlarge": { Interface: 4, @@ -2131,6 +2449,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "g4ad.16xlarge": { Interface: 8, @@ -2144,6 +2464,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "g4ad.2xlarge": { Interface: 2, @@ -2157,6 +2479,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "g4ad.4xlarge": { Interface: 3, @@ -2170,6 +2494,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "g4ad.8xlarge": { Interface: 4, @@ -2183,6 +2509,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "g4ad.xlarge": { Interface: 2, @@ -2196,6 +2524,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "g4dn.12xlarge": { Interface: 8, @@ -2209,6 +2539,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "g4dn.16xlarge": { Interface: 4, @@ -2222,6 +2554,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "g4dn.2xlarge": { Interface: 3, @@ -2235,6 +2569,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "g4dn.4xlarge": { Interface: 3, @@ -2248,6 +2584,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "g4dn.8xlarge": { Interface: 4, @@ -2261,6 +2599,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "g4dn.metal": { Interface: 15, @@ -2274,6 +2614,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "g4dn.xlarge": { Interface: 3, @@ -2287,6 +2629,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "g5.12xlarge": { Interface: 15, @@ -2300,6 +2644,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "g5.16xlarge": { Interface: 8, @@ -2313,6 +2659,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "g5.24xlarge": { Interface: 15, @@ -2326,6 +2674,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "g5.2xlarge": { Interface: 4, @@ -2339,6 +2689,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "g5.48xlarge": { Interface: 7, @@ -2352,6 +2704,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "g5.4xlarge": { Interface: 8, @@ -2365,6 +2719,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "g5.8xlarge": { Interface: 8, @@ -2378,6 +2734,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "g5.xlarge": { Interface: 4, @@ -2391,6 +2749,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "g5g.16xlarge": { Interface: 15, @@ -2404,6 +2764,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "g5g.2xlarge": { Interface: 4, @@ -2417,6 +2779,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "g5g.4xlarge": { Interface: 8, @@ -2430,6 +2794,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "g5g.8xlarge": { Interface: 8, @@ -2443,6 +2809,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "g5g.metal": { Interface: 15, @@ -2456,6 +2824,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "g5g.xlarge": { Interface: 4, @@ -2469,6 +2839,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "h1.16xlarge": { Interface: 15, @@ -2482,6 +2854,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "h1.2xlarge": { Interface: 4, @@ -2495,6 +2869,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "h1.4xlarge": { Interface: 8, @@ -2508,6 +2884,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "h1.8xlarge": { Interface: 8, @@ -2521,6 +2899,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "i2.2xlarge": { Interface: 4, @@ -2534,6 +2914,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "i2.4xlarge": { Interface: 8, @@ -2547,6 +2929,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "i2.8xlarge": { Interface: 8, @@ -2560,6 +2944,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "i2.xlarge": { Interface: 4, @@ -2573,6 +2959,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "i3.16xlarge": { Interface: 15, @@ -2586,6 +2974,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "i3.2xlarge": { Interface: 4, @@ -2599,6 +2989,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "i3.4xlarge": { Interface: 8, @@ -2612,6 +3004,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "i3.8xlarge": { Interface: 8, @@ -2625,6 +3019,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "i3.large": { Interface: 3, @@ -2638,6 +3034,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "i3.metal": { Interface: 15, @@ -2651,6 +3049,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "i3.xlarge": { Interface: 4, @@ -2664,6 +3064,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "i3en.12xlarge": { Interface: 8, @@ -2677,6 +3079,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "i3en.24xlarge": { Interface: 15, @@ -2690,6 +3094,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "i3en.2xlarge": { Interface: 4, @@ -2703,6 +3109,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "i3en.3xlarge": { Interface: 4, @@ -2716,6 +3124,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "i3en.6xlarge": { Interface: 8, @@ -2729,6 +3139,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "i3en.large": { Interface: 3, @@ -2742,6 +3154,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "i3en.metal": { Interface: 15, @@ -2755,6 +3169,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "i3en.xlarge": { Interface: 4, @@ -2768,6 +3184,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "i4i.16xlarge": { Interface: 15, @@ -2781,6 +3199,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "i4i.2xlarge": { Interface: 4, @@ -2794,6 +3214,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "i4i.32xlarge": { Interface: 15, @@ -2807,6 +3229,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "i4i.4xlarge": { Interface: 8, @@ -2820,6 +3244,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "i4i.8xlarge": { Interface: 8, @@ -2833,6 +3259,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "i4i.large": { Interface: 3, @@ -2846,6 +3274,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "i4i.metal": { Interface: 15, @@ -2859,6 +3289,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "i4i.xlarge": { Interface: 4, @@ -2872,6 +3304,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "im4gn.16xlarge": { Interface: 15, @@ -2885,6 +3319,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "im4gn.2xlarge": { Interface: 4, @@ -2898,6 +3334,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "im4gn.4xlarge": { Interface: 8, @@ -2911,6 +3349,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "im4gn.8xlarge": { Interface: 8, @@ -2924,6 +3364,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "im4gn.large": { Interface: 3, @@ -2937,6 +3379,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "im4gn.xlarge": { Interface: 4, @@ -2950,6 +3394,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "inf1.24xlarge": { Interface: 11, @@ -2963,6 +3409,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "inf1.2xlarge": { Interface: 4, @@ -2976,6 +3424,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "inf1.6xlarge": { Interface: 8, @@ -2989,6 +3439,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "inf1.xlarge": { Interface: 4, @@ -3002,6 +3454,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "is4gen.2xlarge": { Interface: 4, @@ -3015,6 +3469,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "is4gen.4xlarge": { Interface: 8, @@ -3028,6 +3484,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "is4gen.8xlarge": { Interface: 8, @@ -3041,6 +3499,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "is4gen.large": { Interface: 3, @@ -3054,6 +3514,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "is4gen.medium": { Interface: 2, @@ -3067,6 +3529,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "is4gen.xlarge": { Interface: 4, @@ -3080,6 +3544,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m1.large": { Interface: 3, @@ -3093,6 +3559,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "m1.medium": { Interface: 2, @@ -3106,6 +3574,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "m1.small": { Interface: 2, @@ -3119,6 +3589,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "m1.xlarge": { Interface: 4, @@ -3132,6 +3604,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "m2.2xlarge": { Interface: 4, @@ -3145,6 +3619,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "m2.4xlarge": { Interface: 8, @@ -3158,6 +3634,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "m2.xlarge": { Interface: 4, @@ -3171,6 +3649,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "m3.2xlarge": { Interface: 4, @@ -3184,6 +3664,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "m3.large": { Interface: 3, @@ -3197,6 +3679,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "m3.medium": { Interface: 2, @@ -3210,6 +3694,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "m3.xlarge": { Interface: 4, @@ -3223,6 +3709,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "m4.10xlarge": { Interface: 8, @@ -3236,6 +3724,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "m4.16xlarge": { Interface: 8, @@ -3249,6 +3739,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "m4.2xlarge": { Interface: 4, @@ -3262,6 +3754,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "m4.4xlarge": { Interface: 8, @@ -3275,6 +3769,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "m4.large": { Interface: 2, @@ -3288,6 +3784,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "m4.xlarge": { Interface: 4, @@ -3301,6 +3799,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "m5.12xlarge": { Interface: 8, @@ -3314,6 +3814,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5.16xlarge": { Interface: 15, @@ -3327,6 +3829,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5.24xlarge": { Interface: 15, @@ -3340,6 +3844,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5.2xlarge": { Interface: 4, @@ -3353,6 +3859,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5.4xlarge": { Interface: 8, @@ -3366,6 +3874,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5.8xlarge": { Interface: 8, @@ -3379,6 +3889,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5.large": { Interface: 3, @@ -3392,6 +3904,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5.metal": { Interface: 15, @@ -3405,6 +3919,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "m5.xlarge": { Interface: 4, @@ -3418,6 +3934,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5a.12xlarge": { Interface: 8, @@ -3431,6 +3949,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5a.16xlarge": { Interface: 15, @@ -3444,6 +3964,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5a.24xlarge": { Interface: 15, @@ -3457,6 +3979,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5a.2xlarge": { Interface: 4, @@ -3470,6 +3994,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5a.4xlarge": { Interface: 8, @@ -3483,6 +4009,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5a.8xlarge": { Interface: 8, @@ -3496,6 +4024,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5a.large": { Interface: 3, @@ -3509,6 +4039,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5a.xlarge": { Interface: 4, @@ -3522,6 +4054,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5ad.12xlarge": { Interface: 8, @@ -3535,6 +4069,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5ad.16xlarge": { Interface: 15, @@ -3548,6 +4084,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5ad.24xlarge": { Interface: 15, @@ -3561,6 +4099,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5ad.2xlarge": { Interface: 4, @@ -3574,6 +4114,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5ad.4xlarge": { Interface: 8, @@ -3587,6 +4129,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5ad.8xlarge": { Interface: 8, @@ -3600,6 +4144,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5ad.large": { Interface: 3, @@ -3613,6 +4159,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5ad.xlarge": { Interface: 4, @@ -3626,6 +4174,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5d.12xlarge": { Interface: 8, @@ -3639,6 +4189,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5d.16xlarge": { Interface: 15, @@ -3652,6 +4204,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5d.24xlarge": { Interface: 15, @@ -3665,6 +4219,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5d.2xlarge": { Interface: 4, @@ -3678,6 +4234,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5d.4xlarge": { Interface: 8, @@ -3691,6 +4249,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5d.8xlarge": { Interface: 8, @@ -3704,6 +4264,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5d.large": { Interface: 3, @@ -3717,6 +4279,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5d.metal": { Interface: 15, @@ -3730,6 +4294,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "m5d.xlarge": { Interface: 4, @@ -3743,6 +4309,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5dn.12xlarge": { Interface: 8, @@ -3756,6 +4324,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5dn.16xlarge": { Interface: 15, @@ -3769,6 +4339,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5dn.24xlarge": { Interface: 15, @@ -3782,6 +4354,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5dn.2xlarge": { Interface: 4, @@ -3795,6 +4369,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5dn.4xlarge": { Interface: 8, @@ -3808,6 +4384,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5dn.8xlarge": { Interface: 8, @@ -3821,6 +4399,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5dn.large": { Interface: 3, @@ -3834,6 +4414,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5dn.metal": { Interface: 15, @@ -3847,6 +4429,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "m5dn.xlarge": { Interface: 4, @@ -3860,6 +4444,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5n.12xlarge": { Interface: 8, @@ -3873,6 +4459,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5n.16xlarge": { Interface: 15, @@ -3886,6 +4474,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5n.24xlarge": { Interface: 15, @@ -3899,6 +4489,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5n.2xlarge": { Interface: 4, @@ -3912,6 +4504,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5n.4xlarge": { Interface: 8, @@ -3925,6 +4519,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5n.8xlarge": { Interface: 8, @@ -3938,6 +4534,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5n.large": { Interface: 3, @@ -3951,6 +4549,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5n.metal": { Interface: 15, @@ -3964,6 +4564,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "m5n.xlarge": { Interface: 4, @@ -3977,6 +4579,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5zn.12xlarge": { Interface: 15, @@ -3990,6 +4594,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5zn.2xlarge": { Interface: 4, @@ -4003,6 +4609,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5zn.3xlarge": { Interface: 8, @@ -4016,6 +4624,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5zn.6xlarge": { Interface: 8, @@ -4029,6 +4639,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5zn.large": { Interface: 3, @@ -4042,6 +4654,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m5zn.metal": { Interface: 15, @@ -4055,6 +4669,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "m5zn.xlarge": { Interface: 4, @@ -4068,6 +4684,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6a.12xlarge": { Interface: 8, @@ -4081,6 +4699,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6a.16xlarge": { Interface: 15, @@ -4094,6 +4714,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6a.24xlarge": { Interface: 15, @@ -4107,6 +4729,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6a.2xlarge": { Interface: 4, @@ -4120,6 +4744,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6a.32xlarge": { Interface: 15, @@ -4133,6 +4759,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6a.48xlarge": { Interface: 15, @@ -4146,6 +4774,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6a.4xlarge": { Interface: 8, @@ -4159,6 +4789,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6a.8xlarge": { Interface: 8, @@ -4172,6 +4804,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6a.large": { Interface: 3, @@ -4185,6 +4819,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6a.metal": { Interface: 15, @@ -4198,6 +4834,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "m6a.xlarge": { Interface: 4, @@ -4211,6 +4849,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6g.12xlarge": { Interface: 8, @@ -4224,6 +4864,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6g.16xlarge": { Interface: 15, @@ -4237,6 +4879,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6g.2xlarge": { Interface: 4, @@ -4250,6 +4894,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6g.4xlarge": { Interface: 8, @@ -4263,6 +4909,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6g.8xlarge": { Interface: 8, @@ -4276,6 +4924,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6g.large": { Interface: 3, @@ -4289,6 +4939,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6g.medium": { Interface: 2, @@ -4302,6 +4954,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6g.metal": { Interface: 15, @@ -4315,6 +4969,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "m6g.xlarge": { Interface: 4, @@ -4328,6 +4984,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6gd.12xlarge": { Interface: 8, @@ -4341,6 +4999,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6gd.16xlarge": { Interface: 15, @@ -4354,6 +5014,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6gd.2xlarge": { Interface: 4, @@ -4367,6 +5029,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6gd.4xlarge": { Interface: 8, @@ -4380,6 +5044,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6gd.8xlarge": { Interface: 8, @@ -4393,6 +5059,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6gd.large": { Interface: 3, @@ -4406,6 +5074,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6gd.medium": { Interface: 2, @@ -4419,6 +5089,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6gd.metal": { Interface: 15, @@ -4432,6 +5104,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "m6gd.xlarge": { Interface: 4, @@ -4445,6 +5119,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6i.12xlarge": { Interface: 8, @@ -4458,6 +5134,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6i.16xlarge": { Interface: 15, @@ -4471,6 +5149,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6i.24xlarge": { Interface: 15, @@ -4484,6 +5164,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6i.2xlarge": { Interface: 4, @@ -4497,6 +5179,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6i.32xlarge": { Interface: 15, @@ -4510,6 +5194,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6i.4xlarge": { Interface: 8, @@ -4523,6 +5209,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6i.8xlarge": { Interface: 8, @@ -4536,6 +5224,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6i.large": { Interface: 3, @@ -4549,6 +5239,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6i.metal": { Interface: 15, @@ -4562,6 +5254,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "m6i.xlarge": { Interface: 4, @@ -4575,6 +5269,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6id.12xlarge": { Interface: 8, @@ -4588,6 +5284,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6id.16xlarge": { Interface: 15, @@ -4601,6 +5299,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6id.24xlarge": { Interface: 15, @@ -4614,6 +5314,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6id.2xlarge": { Interface: 4, @@ -4627,6 +5329,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6id.32xlarge": { Interface: 15, @@ -4640,6 +5344,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6id.4xlarge": { Interface: 8, @@ -4653,6 +5359,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6id.8xlarge": { Interface: 8, @@ -4666,6 +5374,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6id.large": { Interface: 3, @@ -4679,6 +5389,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6id.metal": { Interface: 15, @@ -4692,6 +5404,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "m6id.xlarge": { Interface: 4, @@ -4705,6 +5419,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6idn.12xlarge": { Interface: 8, @@ -4718,6 +5434,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6idn.16xlarge": { Interface: 15, @@ -4731,6 +5449,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6idn.24xlarge": { Interface: 15, @@ -4744,6 +5464,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6idn.2xlarge": { Interface: 4, @@ -4757,6 +5479,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6idn.32xlarge": { Interface: 14, @@ -4775,6 +5499,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 1, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6idn.4xlarge": { Interface: 8, @@ -4788,6 +5514,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6idn.8xlarge": { Interface: 8, @@ -4801,6 +5529,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6idn.large": { Interface: 3, @@ -4814,6 +5544,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6idn.metal": { Interface: 14, @@ -4832,6 +5564,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 1, }, }, + Hypervisor: "", + IsBareMetal: true, }, "m6idn.xlarge": { Interface: 4, @@ -4845,6 +5579,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6in.12xlarge": { Interface: 8, @@ -4858,6 +5594,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6in.16xlarge": { Interface: 15, @@ -4871,6 +5609,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6in.24xlarge": { Interface: 15, @@ -4884,6 +5624,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6in.2xlarge": { Interface: 4, @@ -4897,6 +5639,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6in.32xlarge": { Interface: 14, @@ -4915,6 +5659,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 1, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6in.4xlarge": { Interface: 8, @@ -4928,6 +5674,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6in.8xlarge": { Interface: 8, @@ -4941,6 +5689,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6in.large": { Interface: 3, @@ -4954,6 +5704,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m6in.metal": { Interface: 14, @@ -4972,6 +5724,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 1, }, }, + Hypervisor: "", + IsBareMetal: true, }, "m6in.xlarge": { Interface: 4, @@ -4985,6 +5739,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m7g.12xlarge": { Interface: 8, @@ -4998,6 +5754,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m7g.16xlarge": { Interface: 15, @@ -5011,6 +5769,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m7g.2xlarge": { Interface: 4, @@ -5024,6 +5784,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m7g.4xlarge": { Interface: 8, @@ -5037,6 +5799,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m7g.8xlarge": { Interface: 8, @@ -5050,6 +5814,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m7g.large": { Interface: 3, @@ -5063,6 +5829,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m7g.medium": { Interface: 2, @@ -5076,6 +5844,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "m7g.metal": { Interface: 15, @@ -5089,6 +5859,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "m7g.xlarge": { Interface: 4, @@ -5102,6 +5874,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "mac1.metal": { Interface: 8, @@ -5115,6 +5889,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "mac2.metal": { Interface: 8, @@ -5128,6 +5904,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "p2.16xlarge": { Interface: 8, @@ -5141,6 +5919,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "p2.8xlarge": { Interface: 8, @@ -5154,6 +5934,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "p2.xlarge": { Interface: 4, @@ -5167,6 +5949,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "p3.16xlarge": { Interface: 8, @@ -5180,6 +5964,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "p3.2xlarge": { Interface: 4, @@ -5193,6 +5979,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "p3.8xlarge": { Interface: 8, @@ -5206,6 +5994,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "p3dn.24xlarge": { Interface: 15, @@ -5219,6 +6009,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "p4d.24xlarge": { Interface: 60, @@ -5247,6 +6039,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 3, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r3.2xlarge": { Interface: 4, @@ -5260,6 +6054,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "r3.4xlarge": { Interface: 8, @@ -5273,6 +6069,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "r3.8xlarge": { Interface: 8, @@ -5286,6 +6084,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "r3.large": { Interface: 3, @@ -5299,6 +6099,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "r3.xlarge": { Interface: 4, @@ -5312,6 +6114,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "r4.16xlarge": { Interface: 15, @@ -5325,6 +6129,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "r4.2xlarge": { Interface: 4, @@ -5338,6 +6144,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "r4.4xlarge": { Interface: 8, @@ -5351,6 +6159,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "r4.8xlarge": { Interface: 8, @@ -5364,6 +6174,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "r4.large": { Interface: 3, @@ -5377,6 +6189,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "r4.xlarge": { Interface: 4, @@ -5390,6 +6204,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "r5.12xlarge": { Interface: 8, @@ -5403,6 +6219,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5.16xlarge": { Interface: 15, @@ -5416,6 +6234,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5.24xlarge": { Interface: 15, @@ -5429,6 +6249,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5.2xlarge": { Interface: 4, @@ -5442,6 +6264,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5.4xlarge": { Interface: 8, @@ -5455,6 +6279,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5.8xlarge": { Interface: 8, @@ -5468,6 +6294,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5.large": { Interface: 3, @@ -5481,6 +6309,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5.metal": { Interface: 15, @@ -5494,6 +6324,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "r5.xlarge": { Interface: 4, @@ -5507,6 +6339,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5a.12xlarge": { Interface: 8, @@ -5520,6 +6354,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5a.16xlarge": { Interface: 15, @@ -5533,6 +6369,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5a.24xlarge": { Interface: 15, @@ -5546,6 +6384,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5a.2xlarge": { Interface: 4, @@ -5559,6 +6399,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5a.4xlarge": { Interface: 8, @@ -5572,6 +6414,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5a.8xlarge": { Interface: 8, @@ -5585,6 +6429,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5a.large": { Interface: 3, @@ -5598,6 +6444,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5a.xlarge": { Interface: 4, @@ -5611,6 +6459,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5ad.12xlarge": { Interface: 8, @@ -5624,6 +6474,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5ad.16xlarge": { Interface: 15, @@ -5637,6 +6489,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5ad.24xlarge": { Interface: 15, @@ -5650,6 +6504,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5ad.2xlarge": { Interface: 4, @@ -5663,6 +6519,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5ad.4xlarge": { Interface: 8, @@ -5676,6 +6534,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5ad.8xlarge": { Interface: 8, @@ -5689,6 +6549,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5ad.large": { Interface: 3, @@ -5702,6 +6564,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5ad.xlarge": { Interface: 4, @@ -5715,6 +6579,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5b.12xlarge": { Interface: 8, @@ -5728,6 +6594,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5b.16xlarge": { Interface: 15, @@ -5741,6 +6609,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5b.24xlarge": { Interface: 15, @@ -5754,6 +6624,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5b.2xlarge": { Interface: 4, @@ -5767,6 +6639,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5b.4xlarge": { Interface: 8, @@ -5780,6 +6654,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5b.8xlarge": { Interface: 8, @@ -5793,6 +6669,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5b.large": { Interface: 3, @@ -5806,6 +6684,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5b.metal": { Interface: 15, @@ -5819,6 +6699,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "r5b.xlarge": { Interface: 4, @@ -5832,6 +6714,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5d.12xlarge": { Interface: 8, @@ -5845,6 +6729,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5d.16xlarge": { Interface: 15, @@ -5858,6 +6744,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5d.24xlarge": { Interface: 15, @@ -5871,6 +6759,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5d.2xlarge": { Interface: 4, @@ -5884,6 +6774,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5d.4xlarge": { Interface: 8, @@ -5897,6 +6789,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5d.8xlarge": { Interface: 8, @@ -5910,6 +6804,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5d.large": { Interface: 3, @@ -5923,6 +6819,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5d.metal": { Interface: 15, @@ -5936,6 +6834,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "r5d.xlarge": { Interface: 4, @@ -5949,6 +6849,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5dn.12xlarge": { Interface: 8, @@ -5962,6 +6864,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5dn.16xlarge": { Interface: 15, @@ -5975,6 +6879,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5dn.24xlarge": { Interface: 15, @@ -5988,6 +6894,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5dn.2xlarge": { Interface: 4, @@ -6001,6 +6909,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5dn.4xlarge": { Interface: 8, @@ -6014,6 +6924,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5dn.8xlarge": { Interface: 8, @@ -6027,6 +6939,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5dn.large": { Interface: 3, @@ -6040,6 +6954,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5dn.metal": { Interface: 15, @@ -6053,6 +6969,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "r5dn.xlarge": { Interface: 4, @@ -6066,6 +6984,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5n.12xlarge": { Interface: 8, @@ -6079,6 +6999,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5n.16xlarge": { Interface: 15, @@ -6092,6 +7014,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5n.24xlarge": { Interface: 15, @@ -6105,6 +7029,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5n.2xlarge": { Interface: 4, @@ -6118,6 +7044,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5n.4xlarge": { Interface: 8, @@ -6131,6 +7059,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5n.8xlarge": { Interface: 8, @@ -6144,6 +7074,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5n.large": { Interface: 3, @@ -6157,6 +7089,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r5n.metal": { Interface: 15, @@ -6170,6 +7104,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "r5n.xlarge": { Interface: 4, @@ -6183,6 +7119,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6a.12xlarge": { Interface: 8, @@ -6196,6 +7134,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6a.16xlarge": { Interface: 15, @@ -6209,6 +7149,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6a.24xlarge": { Interface: 15, @@ -6222,6 +7164,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6a.2xlarge": { Interface: 4, @@ -6235,6 +7179,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6a.32xlarge": { Interface: 15, @@ -6248,6 +7194,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6a.48xlarge": { Interface: 15, @@ -6261,6 +7209,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6a.4xlarge": { Interface: 8, @@ -6274,6 +7224,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6a.8xlarge": { Interface: 8, @@ -6287,6 +7239,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6a.large": { Interface: 3, @@ -6300,6 +7254,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6a.metal": { Interface: 15, @@ -6313,6 +7269,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "r6a.xlarge": { Interface: 4, @@ -6326,6 +7284,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6g.12xlarge": { Interface: 8, @@ -6339,6 +7299,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6g.16xlarge": { Interface: 15, @@ -6352,6 +7314,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6g.2xlarge": { Interface: 4, @@ -6365,6 +7329,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6g.4xlarge": { Interface: 8, @@ -6378,6 +7344,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6g.8xlarge": { Interface: 8, @@ -6391,6 +7359,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6g.large": { Interface: 3, @@ -6404,6 +7374,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6g.medium": { Interface: 2, @@ -6417,6 +7389,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6g.metal": { Interface: 15, @@ -6430,6 +7404,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "r6g.xlarge": { Interface: 4, @@ -6443,6 +7419,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6gd.12xlarge": { Interface: 8, @@ -6456,6 +7434,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6gd.16xlarge": { Interface: 15, @@ -6469,6 +7449,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6gd.2xlarge": { Interface: 4, @@ -6482,6 +7464,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6gd.4xlarge": { Interface: 8, @@ -6495,6 +7479,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6gd.8xlarge": { Interface: 8, @@ -6508,6 +7494,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6gd.large": { Interface: 3, @@ -6521,6 +7509,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6gd.medium": { Interface: 2, @@ -6534,6 +7524,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6gd.metal": { Interface: 15, @@ -6547,6 +7539,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "r6gd.xlarge": { Interface: 4, @@ -6560,6 +7554,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6i.12xlarge": { Interface: 8, @@ -6573,6 +7569,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6i.16xlarge": { Interface: 15, @@ -6586,6 +7584,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6i.24xlarge": { Interface: 15, @@ -6599,6 +7599,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6i.2xlarge": { Interface: 4, @@ -6612,6 +7614,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6i.32xlarge": { Interface: 15, @@ -6625,6 +7629,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6i.4xlarge": { Interface: 8, @@ -6638,6 +7644,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6i.8xlarge": { Interface: 8, @@ -6651,6 +7659,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6i.large": { Interface: 3, @@ -6664,6 +7674,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6i.metal": { Interface: 15, @@ -6677,6 +7689,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "r6i.xlarge": { Interface: 4, @@ -6690,6 +7704,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6id.12xlarge": { Interface: 8, @@ -6703,6 +7719,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6id.16xlarge": { Interface: 15, @@ -6716,6 +7734,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6id.24xlarge": { Interface: 15, @@ -6729,6 +7749,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6id.2xlarge": { Interface: 4, @@ -6742,6 +7764,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6id.32xlarge": { Interface: 15, @@ -6755,6 +7779,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6id.4xlarge": { Interface: 8, @@ -6768,6 +7794,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6id.8xlarge": { Interface: 8, @@ -6781,6 +7809,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6id.large": { Interface: 3, @@ -6794,6 +7824,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6id.metal": { Interface: 15, @@ -6807,6 +7839,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "r6id.xlarge": { Interface: 4, @@ -6820,6 +7854,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6idn.12xlarge": { Interface: 8, @@ -6833,6 +7869,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6idn.16xlarge": { Interface: 15, @@ -6846,6 +7884,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6idn.24xlarge": { Interface: 15, @@ -6859,6 +7899,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6idn.2xlarge": { Interface: 4, @@ -6872,6 +7914,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6idn.32xlarge": { Interface: 14, @@ -6890,6 +7934,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 1, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6idn.4xlarge": { Interface: 8, @@ -6903,6 +7949,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6idn.8xlarge": { Interface: 8, @@ -6916,6 +7964,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6idn.large": { Interface: 3, @@ -6929,6 +7979,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6idn.metal": { Interface: 14, @@ -6947,6 +7999,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 1, }, }, + Hypervisor: "", + IsBareMetal: true, }, "r6idn.xlarge": { Interface: 4, @@ -6960,6 +8014,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6in.12xlarge": { Interface: 8, @@ -6973,6 +8029,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6in.16xlarge": { Interface: 15, @@ -6986,6 +8044,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6in.24xlarge": { Interface: 15, @@ -6999,6 +8059,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6in.2xlarge": { Interface: 4, @@ -7012,6 +8074,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6in.32xlarge": { Interface: 14, @@ -7030,6 +8094,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 1, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6in.4xlarge": { Interface: 8, @@ -7043,6 +8109,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6in.8xlarge": { Interface: 8, @@ -7056,6 +8124,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6in.large": { Interface: 3, @@ -7069,6 +8139,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r6in.metal": { Interface: 14, @@ -7087,6 +8159,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 1, }, }, + Hypervisor: "", + IsBareMetal: true, }, "r6in.xlarge": { Interface: 4, @@ -7100,6 +8174,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r7g.12xlarge": { Interface: 8, @@ -7113,6 +8189,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r7g.16xlarge": { Interface: 15, @@ -7126,6 +8204,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r7g.2xlarge": { Interface: 4, @@ -7139,6 +8219,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r7g.4xlarge": { Interface: 8, @@ -7152,6 +8234,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r7g.8xlarge": { Interface: 8, @@ -7165,6 +8249,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r7g.large": { Interface: 3, @@ -7178,6 +8264,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r7g.medium": { Interface: 2, @@ -7191,6 +8279,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "r7g.metal": { Interface: 15, @@ -7204,6 +8294,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "r7g.xlarge": { Interface: 4, @@ -7217,6 +8309,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "t1.micro": { Interface: 2, @@ -7230,6 +8324,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "t2.2xlarge": { Interface: 3, @@ -7243,6 +8339,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "t2.large": { Interface: 3, @@ -7256,6 +8354,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "t2.medium": { Interface: 3, @@ -7269,6 +8369,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "t2.micro": { Interface: 2, @@ -7282,6 +8384,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "t2.nano": { Interface: 2, @@ -7295,6 +8399,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "t2.small": { Interface: 3, @@ -7308,6 +8414,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "t2.xlarge": { Interface: 3, @@ -7321,6 +8429,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "t3.2xlarge": { Interface: 4, @@ -7334,6 +8444,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "t3.large": { Interface: 3, @@ -7347,6 +8459,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "t3.medium": { Interface: 3, @@ -7360,6 +8474,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "t3.micro": { Interface: 2, @@ -7373,6 +8489,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "t3.nano": { Interface: 2, @@ -7386,6 +8504,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "t3.small": { Interface: 3, @@ -7399,6 +8519,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "t3.xlarge": { Interface: 4, @@ -7412,6 +8534,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "t3a.2xlarge": { Interface: 4, @@ -7425,6 +8549,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "t3a.large": { Interface: 3, @@ -7438,6 +8564,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "t3a.medium": { Interface: 3, @@ -7451,6 +8579,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "t3a.micro": { Interface: 2, @@ -7464,6 +8594,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "t3a.nano": { Interface: 2, @@ -7477,6 +8609,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "t3a.small": { Interface: 2, @@ -7490,6 +8624,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "t3a.xlarge": { Interface: 4, @@ -7503,6 +8639,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "t4g.2xlarge": { Interface: 4, @@ -7516,6 +8654,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "t4g.large": { Interface: 3, @@ -7529,6 +8669,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "t4g.medium": { Interface: 3, @@ -7542,6 +8684,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "t4g.micro": { Interface: 2, @@ -7555,6 +8699,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "t4g.nano": { Interface: 2, @@ -7568,6 +8714,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "t4g.small": { Interface: 3, @@ -7581,6 +8729,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "t4g.xlarge": { Interface: 4, @@ -7594,6 +8744,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "trn1.2xlarge": { Interface: 4, @@ -7607,6 +8759,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "trn1.32xlarge": { Interface: 40, @@ -7655,6 +8809,98 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 7, }, }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "trn1n.32xlarge": { + Interface: 80, + IPv4PerInterface: 50, + IsTrunkingCompatible: true, + BranchInterface: 120, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 5, + NetworkCardIndex: 0, + }, + + { + MaximumNetworkInterfaces: 5, + NetworkCardIndex: 1, + }, + + { + MaximumNetworkInterfaces: 5, + NetworkCardIndex: 2, + }, + + { + MaximumNetworkInterfaces: 5, + NetworkCardIndex: 3, + }, + + { + MaximumNetworkInterfaces: 5, + NetworkCardIndex: 4, + }, + + { + MaximumNetworkInterfaces: 5, + NetworkCardIndex: 5, + }, + + { + MaximumNetworkInterfaces: 5, + NetworkCardIndex: 6, + }, + + { + MaximumNetworkInterfaces: 5, + NetworkCardIndex: 7, + }, + + { + MaximumNetworkInterfaces: 5, + NetworkCardIndex: 8, + }, + + { + MaximumNetworkInterfaces: 5, + NetworkCardIndex: 9, + }, + + { + MaximumNetworkInterfaces: 5, + NetworkCardIndex: 10, + }, + + { + MaximumNetworkInterfaces: 5, + NetworkCardIndex: 11, + }, + + { + MaximumNetworkInterfaces: 5, + NetworkCardIndex: 12, + }, + + { + MaximumNetworkInterfaces: 5, + NetworkCardIndex: 13, + }, + + { + MaximumNetworkInterfaces: 5, + NetworkCardIndex: 14, + }, + + { + MaximumNetworkInterfaces: 5, + NetworkCardIndex: 15, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, }, "u-12tb1.112xlarge": { Interface: 15, @@ -7668,6 +8914,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "u-18tb1.112xlarge": { Interface: 15, @@ -7681,6 +8929,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "u-3tb1.56xlarge": { Interface: 8, @@ -7694,6 +8944,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "u-6tb1.112xlarge": { Interface: 15, @@ -7707,6 +8959,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "u-6tb1.56xlarge": { Interface: 15, @@ -7720,6 +8974,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "u-9tb1.112xlarge": { Interface: 15, @@ -7733,6 +8989,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "vt1.24xlarge": { Interface: 15, @@ -7746,6 +9004,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "vt1.3xlarge": { Interface: 4, @@ -7759,6 +9019,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "vt1.6xlarge": { Interface: 8, @@ -7772,6 +9034,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "x1.16xlarge": { Interface: 8, @@ -7785,6 +9049,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "x1.32xlarge": { Interface: 8, @@ -7798,6 +9064,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "x1e.16xlarge": { Interface: 8, @@ -7811,6 +9079,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "x1e.2xlarge": { Interface: 4, @@ -7824,6 +9094,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "x1e.32xlarge": { Interface: 8, @@ -7837,6 +9109,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "x1e.4xlarge": { Interface: 4, @@ -7850,6 +9124,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "x1e.8xlarge": { Interface: 4, @@ -7863,6 +9139,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "x1e.xlarge": { Interface: 3, @@ -7876,6 +9154,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "xen", + IsBareMetal: false, }, "x2gd.12xlarge": { Interface: 8, @@ -7889,6 +9169,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "x2gd.16xlarge": { Interface: 15, @@ -7902,6 +9184,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "x2gd.2xlarge": { Interface: 4, @@ -7915,6 +9199,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "x2gd.4xlarge": { Interface: 8, @@ -7928,6 +9214,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "x2gd.8xlarge": { Interface: 8, @@ -7941,6 +9229,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "x2gd.large": { Interface: 3, @@ -7954,6 +9244,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "x2gd.medium": { Interface: 2, @@ -7967,6 +9259,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "x2gd.metal": { Interface: 15, @@ -7980,6 +9274,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "x2gd.xlarge": { Interface: 4, @@ -7993,6 +9289,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "x2idn.16xlarge": { Interface: 15, @@ -8006,6 +9304,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "x2idn.24xlarge": { Interface: 15, @@ -8019,6 +9319,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "x2idn.32xlarge": { Interface: 15, @@ -8032,6 +9334,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "x2idn.metal": { Interface: 15, @@ -8045,6 +9349,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "x2iedn.16xlarge": { Interface: 15, @@ -8058,6 +9364,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "x2iedn.24xlarge": { Interface: 15, @@ -8071,6 +9379,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "x2iedn.2xlarge": { Interface: 4, @@ -8084,6 +9394,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "x2iedn.32xlarge": { Interface: 15, @@ -8097,6 +9409,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "x2iedn.4xlarge": { Interface: 8, @@ -8110,6 +9424,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "x2iedn.8xlarge": { Interface: 8, @@ -8123,6 +9439,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "x2iedn.metal": { Interface: 15, @@ -8136,6 +9454,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "x2iedn.xlarge": { Interface: 4, @@ -8149,6 +9469,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "x2iezn.12xlarge": { Interface: 15, @@ -8162,6 +9484,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "x2iezn.2xlarge": { Interface: 4, @@ -8175,6 +9499,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "x2iezn.4xlarge": { Interface: 8, @@ -8188,6 +9514,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "x2iezn.6xlarge": { Interface: 8, @@ -8201,6 +9529,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "x2iezn.8xlarge": { Interface: 8, @@ -8214,6 +9544,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "x2iezn.metal": { Interface: 15, @@ -8227,6 +9559,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "z1d.12xlarge": { Interface: 15, @@ -8240,6 +9574,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "z1d.2xlarge": { Interface: 4, @@ -8253,6 +9589,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "z1d.3xlarge": { Interface: 8, @@ -8266,6 +9604,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "z1d.6xlarge": { Interface: 8, @@ -8279,6 +9619,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "z1d.large": { Interface: 3, @@ -8292,6 +9634,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, "z1d.metal": { Interface: 15, @@ -8305,6 +9649,8 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "", + IsBareMetal: true, }, "z1d.xlarge": { Interface: 4, @@ -8318,5 +9664,7 @@ var Limits = map[string]*VPCLimits{ NetworkCardIndex: 0, }, }, + Hypervisor: "nitro", + IsBareMetal: false, }, } diff --git a/pkg/healthz/healthz.go b/pkg/healthz/healthz.go new file mode 100644 index 00000000..afee21dc --- /dev/null +++ b/pkg/healthz/healthz.go @@ -0,0 +1,83 @@ +package healthz + +import ( + "errors" + "fmt" + "net/http" + "time" + + "github.com/go-logr/logr" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/manager" +) + +type HealthzHandler struct { + CheckersMap map[string]healthz.Checker +} + +var ( + HealthzTimeout time.Duration = 2 +) + +func NewHealthzHandler(timeout int) *HealthzHandler { + HealthzTimeout = time.Duration(timeout) + return &HealthzHandler{ + CheckersMap: make(map[string]healthz.Checker), + } +} + +func (hh *HealthzHandler) AddControllerHealthChecker(name string, controllerCheck healthz.Checker) { + // only add health check if the map doesn't already contain + if _, ok := hh.CheckersMap[name]; !ok { + hh.CheckersMap[name] = controllerCheck + } +} + +func (hh *HealthzHandler) AddControllersHealthCheckers(controllerCheckers map[string]healthz.Checker) { + for key, value := range controllerCheckers { + hh.AddControllerHealthChecker(key, value) + } +} + +func (hh *HealthzHandler) AddControllersHealthStatusChecksToManager(mgr manager.Manager) error { + if len(hh.CheckersMap) > 0 { + return hh.checkControllersHealthStatus(mgr, hh.CheckersMap) + } else { + return errors.New("couldn't find any controller's check to add for healthz endpoint") + } +} + +func (hh *HealthzHandler) checkControllersHealthStatus(mgr manager.Manager, checkers map[string]healthz.Checker) error { + var err error + for name, check := range checkers { + err = mgr.AddHealthzCheck(name, check) + fmt.Printf("Added Health check for %s with error %v\n", name, err) + if err != nil { + break + } + } + return err +} + +func SimplePing(controllerName string, log logr.Logger) healthz.Checker { + log.Info(fmt.Sprintf("%s's healthz subpath was added", controllerName)) + return func(req *http.Request) error { + log.V(1).Info(fmt.Sprintf("***** %s healthz endpoint was pinged *****", controllerName)) + return nil + } +} + +func PingWithTimeout(healthCheck func(c chan<- error), logger logr.Logger) error { + status := make(chan error, 1) + var err error + go healthCheck(status) + + select { + case err = <-status: + logger.V(1).Info("finished healthz check on controller before probing times out", "TimeoutInSecond", HealthzTimeout*time.Second) + case <-time.After(HealthzTimeout * time.Second): + err = errors.New("healthz check failed due to timeout") + logger.Error(err, "healthz check has a preset timeout to fail no responding probing", "TimeoutInSecond", HealthzTimeout*time.Second) + } + return err +} diff --git a/pkg/healthz/healthz_test.go b/pkg/healthz/healthz_test.go new file mode 100644 index 00000000..c779a321 --- /dev/null +++ b/pkg/healthz/healthz_test.go @@ -0,0 +1,62 @@ +package healthz + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/log/zap" +) + +var testTimeout = 3 + +// TestHealthzHandler tests creating a new healthz handler with timeout value passed to it +func TestHealthzHandler(t *testing.T) { + handler := NewHealthzHandler(testTimeout) + assert.True(t, handler != nil) + assert.True(t, HealthzTimeout == time.Duration(testTimeout)) +} + +// TestAddControllerHealthChecker tests adding individual healthz checker +func TestAddControllerHealthChecker(t *testing.T) { + handler := NewHealthzHandler(testTimeout) + checker := healthz.Ping + name := "test-ping" + handler.AddControllerHealthChecker(name, checker) + assert.True(t, len(handler.CheckersMap) == 1, "Should be only one healthz checker") + _, ok := handler.CheckersMap[name] + assert.True(t, ok) +} + +// TestAddControllersHealthCheckers tests adding the map of healthz checkers +func TestAddControllersHealthCheckers(t *testing.T) { + handler := NewHealthzHandler(testTimeout) + checkers := map[string]healthz.Checker{ + "test-checker-1": healthz.Ping, + "test-checker-2": SimplePing("test", zap.New()), + } + handler.AddControllersHealthCheckers(checkers) + assert.True(t, len(handler.CheckersMap) == 2, "Two checkers should be added") +} + +// TestPingWithTimeout_Success tests ping responding before timeout +func TestPingWithTimeout_Success(t *testing.T) { + err := PingWithTimeout(func(c chan<- error) { + time.Sleep(1 * time.Second) + c <- nil + }, zap.New()) + time.Sleep(5 * time.Second) + assert.NoError(t, err) +} + +// TestPingWithTimeout_Failure tests ping responding after timeout +func TestPingWithTimeout_Failure(t *testing.T) { + err := PingWithTimeout(func(c chan<- error) { + time.Sleep(4 * time.Second) + c <- nil + }, zap.New()) + time.Sleep(5 * time.Second) + assert.Error(t, err) + assert.EqualErrorf(t, err, "healthz check failed due to timeout", "Healthz check should fail due to timeout") +} diff --git a/pkg/node/manager/manager.go b/pkg/node/manager/manager.go index b6c9ad7a..62ab2768 100644 --- a/pkg/node/manager/manager.go +++ b/pkg/node/manager/manager.go @@ -15,16 +15,20 @@ package manager import ( "fmt" + "net/http" "strings" "sync" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/api" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/condition" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config" + rcHealthz "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/healthz" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/node" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/resource" asyncWorker "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/worker" + "github.com/google/uuid" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/healthz" "github.com/go-logr/logr" v1 "k8s.io/api/core/v1" @@ -84,7 +88,7 @@ type AsyncOperationJob struct { // NewNodeManager returns a new node manager func NewNodeManager(logger logr.Logger, resourceManager resource.ResourceManager, - wrapper api.Wrapper, worker asyncWorker.Worker, conditions condition.Conditions) (Manager, error) { + wrapper api.Wrapper, worker asyncWorker.Worker, conditions condition.Conditions, healthzHandler *rcHealthz.HealthzHandler) (Manager, error) { manager := &manager{ resourceManager: resourceManager, @@ -95,6 +99,11 @@ func NewNodeManager(logger logr.Logger, resourceManager resource.ResourceManager conditions: conditions, } + // add health check on subpath for node manager + healthzHandler.AddControllersHealthCheckers( + map[string]healthz.Checker{"health-node-manager": manager.check()}, + ) + return manager, worker.StartWorkerPool(manager.performAsyncOperation) } @@ -134,7 +143,7 @@ func (m *manager) AddNode(nodeName string) error { if shouldManage { newNode = node.NewManagedNode(m.Log, k8sNode.Name, GetNodeInstanceID(k8sNode), - GetNodeOS(k8sNode)) + GetNodeOS(k8sNode), m.wrapper.K8sAPI, m.wrapper.EC2API) err := m.updateSubnetIfUsingENIConfig(newNode, k8sNode) if err != nil { return err @@ -186,7 +195,7 @@ func (m *manager) UpdateNode(nodeName string) error { case UnManagedToManaged: log.Info("node was previously un-managed, will be added as managed node now") cachedNode = node.NewManagedNode(m.Log, k8sNode.Name, - GetNodeInstanceID(k8sNode), GetNodeOS(k8sNode)) + GetNodeInstanceID(k8sNode), GetNodeOS(k8sNode), m.wrapper.K8sAPI, m.wrapper.EC2API) // Update the Subnet if the node has custom networking configured err = m.updateSubnetIfUsingENIConfig(cachedNode, k8sNode) if err != nil { @@ -304,7 +313,7 @@ func (m *manager) performAsyncOperation(job interface{}) (ctrl.Result, error) { var err error switch asyncJob.op { case Init: - err = asyncJob.node.InitResources(m.resourceManager, m.wrapper.EC2API) + err = asyncJob.node.InitResources(m.resourceManager) if err != nil { log.Error(err, "removing the node from cache as it failed to initialize") m.removeNodeSafe(asyncJob.nodeName) @@ -319,9 +328,9 @@ func (m *manager) performAsyncOperation(job interface{}) (ctrl.Result, error) { asyncJob.op = Update return m.performAsyncOperation(asyncJob) case Update: - err = asyncJob.node.UpdateResources(m.resourceManager, m.wrapper.EC2API) + err = asyncJob.node.UpdateResources(m.resourceManager) case Delete: - err = asyncJob.node.DeleteResources(m.resourceManager, m.wrapper.EC2API) + err = asyncJob.node.DeleteResources(m.resourceManager) default: m.Log.V(1).Info("no operation operation requested", "node", asyncJob.nodeName) @@ -404,14 +413,19 @@ func (m *manager) removeNodeSafe(nodeName string) { delete(m.dataStore, nodeName) } -func (m *manager) updateNodeTrunkLabel(nodeName, labelKey, labelValue string) error { - if node, err := m.wrapper.K8sAPI.GetNode(nodeName); err != nil { - return err - } else { - updated, err := m.wrapper.K8sAPI.AddLabelToManageNode(node, labelKey, labelValue) - if !updated { - m.Log.Info("failed updating the node label for trunk when operating on node", "NodeName", nodeName, "LabelKey", labelKey, "LabelValue", labelValue) - } +func (m *manager) check() healthz.Checker { + // instead of using SimplePing, testing the node cache from manager makes the test more accurate + return func(req *http.Request) error { + err := rcHealthz.PingWithTimeout(func(c chan<- error) { + randomName := uuid.New().String() + _, found := m.GetNode(randomName) + m.Log.V(1).Info("health check tested ping GetNode to check on datastore cache in node manager successfully", "TesedNodeName", randomName, "NodeFound", found) + var ping interface{} + m.worker.SubmitJob(ping) + m.Log.V(1).Info("health check tested ping SubmitJob with a nil job to check on worker queue in node manager successfully") + c <- nil + }, m.Log) + return err } } diff --git a/pkg/node/manager/manager_test.go b/pkg/node/manager/manager_test.go index 10614f46..1347e108 100644 --- a/pkg/node/manager/manager_test.go +++ b/pkg/node/manager/manager_test.go @@ -26,6 +26,7 @@ import ( mock_worker "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/worker" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/api" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config" + "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/healthz" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/node" "github.com/golang/mock/gomock" @@ -37,15 +38,24 @@ import ( ) var ( - instanceID = "i-01234567890abcdef" - providerId = "aws:///us-west-2c/" + instanceID - eniConfigName = "eni-config-name" - subnetID = "subnet-id" - nodeName = "ip-192-168-55-73.us-west-2.compute.internal" + instanceID = "i-01234567890abcdef" + providerId = "aws:///us-west-2c/" + instanceID + eniConfigName = "eni-config-name" + subnetID = "subnet-id" + nodeName = "ip-192-168-55-73.us-west-2.compute.internal" + securityGroupId = "sg-1" eniConfig = &v1alpha1.ENIConfig{ Spec: v1alpha1.ENIConfigSpec{ - Subnet: subnetID, + SecurityGroups: []string{securityGroupId}, + Subnet: subnetID, + }, + } + + eniConfig_empty_sg = &v1alpha1.ENIConfig{ + Spec: v1alpha1.ENIConfigSpec{ + SecurityGroups: []string{}, + Subnet: subnetID, }, } @@ -70,7 +80,9 @@ var ( mockError = fmt.Errorf("mock error") unManagedNode = node.NewUnManagedNode(zap.New(), nodeName, instanceID, config.OSLinux) - managedNode = node.NewManagedNode(zap.New(), nodeName, instanceID, config.OSLinux) + managedNode = node.NewManagedNode(zap.New(), nodeName, instanceID, config.OSLinux, nil, nil) + + healthzHandler = healthz.NewHealthzHandler(5) ) type AsyncJobMatcher struct { @@ -144,7 +156,7 @@ func Test_GetNewManager(t *testing.T) { mock := NewMock(ctrl, map[string]node.Node{}) mock.MockWorker.EXPECT().StartWorkerPool(gomock.Any()).Return(nil) - manager, err := NewNodeManager(zap.New(), nil, api.Wrapper{}, mock.MockWorker, mock.MockConditions) + manager, err := NewNodeManager(zap.New(), nil, api.Wrapper{}, mock.MockWorker, mock.MockConditions, healthzHandler) assert.NotNil(t, manager) assert.NoError(t, err) @@ -158,7 +170,7 @@ func Test_GetNewManager_Error(t *testing.T) { mock := NewMock(ctrl, map[string]node.Node{}) mock.MockWorker.EXPECT().StartWorkerPool(gomock.Any()).Return(mockError) - manager, err := NewNodeManager(zap.New(), nil, api.Wrapper{}, mock.MockWorker, mock.MockConditions) + manager, err := NewNodeManager(zap.New(), nil, api.Wrapper{}, mock.MockWorker, mock.MockConditions, healthzHandler) assert.NotNil(t, manager) assert.Error(t, err, mockError) @@ -243,6 +255,34 @@ func Test_AddNode_CustomNetworking(t *testing.T) { assert.True(t, AreNodesEqual(mock.Manager.dataStore[nodeName], managedNode)) } +// Test adding node when custom networking is enabled but incorrect ENIConfig is defined; it should succeed +// TODO: combine with other Test_AddNode_CustomNetworking tests +func Test_AddNode_CustomNetworking_Incorrect_ENIConfig(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mock := NewMock(ctrl, map[string]node.Node{}) + + job := AsyncOperationJob{ + op: Init, + nodeName: nodeName, + node: managedNode, + } + + nodeWithENIConfig := v1Node.DeepCopy() + nodeWithENIConfig.Labels[config.CustomNetworkingLabel] = eniConfigName + + mock.MockK8sAPI.EXPECT().GetNode(nodeName).Return(nodeWithENIConfig, nil) + mock.MockK8sAPI.EXPECT().GetENIConfig(eniConfigName).Return(eniConfig_empty_sg, nil) + mock.MockWorker.EXPECT().SubmitJob(gomock.All(NewAsyncOperationMatcher(job))) + + err := mock.Manager.AddNode(nodeName) + assert.NoError(t, err) + assert.Contains(t, mock.Manager.dataStore, nodeName) + assert.True(t, AreNodesEqual(mock.Manager.dataStore[nodeName], managedNode)) + +} + func Test_AddNode_CustomNetworking_NoENIConfig(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -429,19 +469,19 @@ func Test_performAsyncOperation(t *testing.T) { job.op = Init mock.MockK8sAPI.EXPECT().AddLabelToManageNode(v1Node, config.HasTrunkAttachedLabel, config.BooleanTrue).Return(true, nil).AnyTimes() - mock.MockNode.EXPECT().InitResources(mock.MockResourceManager, mock.MockEC2API).Return(nil) - mock.MockNode.EXPECT().UpdateResources(mock.MockResourceManager, mock.MockEC2API).Return(nil) + mock.MockNode.EXPECT().InitResources(mock.MockResourceManager).Return(nil) + mock.MockNode.EXPECT().UpdateResources(mock.MockResourceManager).Return(nil) _, err := mock.Manager.performAsyncOperation(job) assert.Contains(t, mock.Manager.dataStore, nodeName) assert.NoError(t, err) job.op = Update - mock.MockNode.EXPECT().UpdateResources(mock.MockResourceManager, mock.MockEC2API).Return(nil) + mock.MockNode.EXPECT().UpdateResources(mock.MockResourceManager).Return(nil) _, err = mock.Manager.performAsyncOperation(job) assert.NoError(t, err) job.op = Delete - mock.MockNode.EXPECT().DeleteResources(mock.MockResourceManager, mock.MockEC2API).Return(nil) + mock.MockNode.EXPECT().DeleteResources(mock.MockResourceManager).Return(nil) _, err = mock.Manager.performAsyncOperation(job) assert.NoError(t, err) @@ -462,7 +502,7 @@ func Test_performAsyncOperation_fail(t *testing.T) { op: Init, } - mock.MockNode.EXPECT().InitResources(mock.MockResourceManager, mock.MockEC2API).Return(&node.ErrInitResources{}) + mock.MockNode.EXPECT().InitResources(mock.MockResourceManager).Return(&node.ErrInitResources{}) _, err := mock.Manager.performAsyncOperation(job) assert.NotContains(t, mock.Manager.dataStore, nodeName) // It should be cleared from cache diff --git a/pkg/node/node.go b/pkg/node/node.go index d95b1ad1..09d35dff 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -14,13 +14,17 @@ package node import ( + "errors" "fmt" "sync" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/aws/ec2" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/aws/ec2/api" + "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/k8s" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/provider" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/resource" + "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/utils" + v1 "k8s.io/api/core/v1" "github.com/go-logr/logr" ) @@ -36,6 +40,10 @@ type node struct { managed bool // instance stores the ec2 instance details that is shared by all the providers instance ec2.EC2Instance + // node has reference to k8s APIs + k8sAPI k8s.K8sWrapper + // node has reference to EC2 APIs + ec2API api.EC2APIHelper } // ErrInitResources to wrap error messages for all errors encountered @@ -50,9 +58,9 @@ func (e *ErrInitResources) Error() string { } type Node interface { - InitResources(resourceManager resource.ResourceManager, helper api.EC2APIHelper) error - DeleteResources(resourceManager resource.ResourceManager, helper api.EC2APIHelper) error - UpdateResources(resourceManager resource.ResourceManager, helper api.EC2APIHelper) error + InitResources(resourceManager resource.ResourceManager) error + DeleteResources(resourceManager resource.ResourceManager) error + UpdateResources(resourceManager resource.ResourceManager) error UpdateCustomNetworkingSpecs(subnetID string, securityGroup []string) IsReady() bool @@ -63,12 +71,14 @@ type Node interface { } // NewManagedNode returns node managed by the controller -func NewManagedNode(log logr.Logger, nodeName string, instanceID string, os string) Node { +func NewManagedNode(log logr.Logger, nodeName string, instanceID string, os string, k8sAPI k8s.K8sWrapper, ec2API api.EC2APIHelper) Node { return &node{ managed: true, log: log.WithName("node resource handler"). WithValues("node name", nodeName), instance: ec2.NewEC2Instance(nodeName, instanceID, os), + k8sAPI: k8sAPI, + ec2API: ec2API, } } @@ -86,7 +96,7 @@ func NewUnManagedNode(log logr.Logger, nodeName, instanceID, os string) Node { } // UpdateNode refreshes the capacity if it's reset to 0 -func (n *node) UpdateResources(resourceManager resource.ResourceManager, helper api.EC2APIHelper) error { +func (n *node) UpdateResources(resourceManager resource.ResourceManager) error { n.lock.Lock() defer n.lock.Unlock() @@ -112,7 +122,7 @@ func (n *node) UpdateResources(resourceManager resource.ResourceManager, helper return fmt.Errorf("failed to update one or more resources %v", errUpdates) } - err := n.instance.UpdateCurrentSubnetAndCidrBlock(helper) + err := n.instance.UpdateCurrentSubnetAndCidrBlock(n.ec2API) if err != nil { n.log.Error(err, "failed to update cidr block", "instance", n.instance.Name()) } @@ -121,12 +131,16 @@ func (n *node) UpdateResources(resourceManager resource.ResourceManager, helper } // InitResources initializes the resource pool and provider of all supported resources -func (n *node) InitResources(resourceManager resource.ResourceManager, helper api.EC2APIHelper) error { +func (n *node) InitResources(resourceManager resource.ResourceManager) error { n.lock.Lock() defer n.lock.Unlock() - - err := n.instance.LoadDetails(helper) + err := n.instance.LoadDetails(n.ec2API) if err != nil { + if errors.Is(err, utils.ErrNotFound) { + // Send a node event for users' visibility + msg := fmt.Sprintf("The instance type %s is not supported yet by the vpc resource controller", n.instance.Type()) + utils.SendNodeEvent(n.k8sAPI, n.instance.Name(), "Unsupported", msg, v1.EventTypeWarning, n.log) + } return &ErrInitResources{ Message: "failed to load instance details", Err: err, @@ -167,7 +181,7 @@ func (n *node) InitResources(resourceManager resource.ResourceManager, helper ap } // DeleteResources performs clean up of all the resource pools and provider of the nodes -func (n *node) DeleteResources(resourceManager resource.ResourceManager, _ api.EC2APIHelper) error { +func (n *node) DeleteResources(resourceManager resource.ResourceManager) error { n.lock.Lock() defer n.lock.Unlock() diff --git a/pkg/node/node_test.go b/pkg/node/node_test.go index 84c45cd8..552bd6c6 100644 --- a/pkg/node/node_test.go +++ b/pkg/node/node_test.go @@ -20,14 +20,17 @@ import ( mock_ec2 "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/aws/ec2" mock_api "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/aws/ec2/api" + mock_k8s "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/k8s" mock_provider "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/provider" mock_resource "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/resource" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/provider" + "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/utils" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) @@ -49,6 +52,7 @@ type Mocks struct { MockResourceManager *mock_resource.MockResourceManager MockInstance *mock_ec2.MockEC2Instance MockEC2API *mock_api.MockEC2APIHelper + MockK8sAPI *mock_k8s.MockK8sWrapper NodeWithMock node } @@ -67,17 +71,22 @@ func NewMock(ctrl *gomock.Controller, mockProviderCount int) Mocks { ResourceProvider: convertedProvider, MockResourceManager: mock_resource.NewMockResourceManager(ctrl), MockEC2API: mock_api.NewMockEC2APIHelper(ctrl), + MockK8sAPI: mock_k8s.NewMockK8sWrapper(ctrl), MockInstance: mockInstance, NodeWithMock: node{ log: zap.New(zap.UseDevMode(true)).WithName("branch provider"), instance: mockInstance, + ec2API: mock_api.NewMockEC2APIHelper(ctrl), }, } } // TestNewManagedNode tests the new node is not nil and node is managed but not ready func TestNewManagedNode(t *testing.T) { - node := NewManagedNode(zap.New(), nodeName, instanceID, linux) + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + node := NewManagedNode(zap.New(), nodeName, instanceID, linux, mock_k8s.NewMockK8sWrapper(ctrl), mock_api.NewMockEC2APIHelper(ctrl)) assert.NotNil(t, node) assert.True(t, node.GetNodeInstanceID() == instanceID) @@ -108,12 +117,12 @@ func TestNode_InitResources(t *testing.T) { mock.MockProviders["0"].EXPECT().IsInstanceSupported(mock.MockInstance).Return(true) mock.MockProviders["0"].EXPECT().InitResource(mock.MockInstance).Return(nil) - err := mock.NodeWithMock.InitResources(mock.MockResourceManager, mock.MockEC2API) + err := mock.NodeWithMock.InitResources(mock.MockResourceManager) assert.NoError(t, err) assert.True(t, mock.NodeWithMock.IsReady()) } -func TestNode_InitResources_InstanceNotSupported(t *testing.T) { +func TestNode_InitResources_InstanceNotTrunkSupported(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -124,11 +133,37 @@ func TestNode_InitResources_InstanceNotSupported(t *testing.T) { mock.MockProviders["0"].EXPECT().IsInstanceSupported(mock.MockInstance).Return(false) - err := mock.NodeWithMock.InitResources(mock.MockResourceManager, mock.MockEC2API) + err := mock.NodeWithMock.InitResources(mock.MockResourceManager) assert.NoError(t, err) assert.True(t, mock.NodeWithMock.IsReady()) } +func TestNode_InitResources_InstanceNotListed(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mock := NewMock(ctrl, 1) + + testInstanceType := "dummy.large" + nodeName = "testInstance" + node := &v1.Node{ + ObjectMeta: metaV1.ObjectMeta{Name: nodeName, UID: types.UID(nodeName)}, + } + + msg := "The instance type dummy.large is not supported yet by the vpc resource controller" + + mock.MockInstance.EXPECT().Type().Return(testInstanceType).Times(1) + mock.MockInstance.EXPECT().Name().Return(nodeName).Times(1) + mock.MockK8sAPI.EXPECT().GetNode(nodeName).Return(node, nil).Times(1) + mock.MockK8sAPI.EXPECT().BroadcastEvent(node, "Unsupported", msg, v1.EventTypeWarning).Times(1) + mock.MockInstance.EXPECT().LoadDetails(mock.MockEC2API).Return(fmt.Errorf("unsupported instance type, couldn't find ENI Limit for instance %s, error: %w", testInstanceType, utils.ErrNotFound)) + + mock.NodeWithMock.k8sAPI = mock.MockK8sAPI + err := mock.NodeWithMock.InitResources(mock.MockResourceManager) + assert.Error(t, err) + assert.False(t, mock.NodeWithMock.IsReady()) +} + // TestNode_InitResources_LoadInstanceDetails_Error tests that error is propagated when load instance details throws an error func TestNode_InitResources_LoadInstanceDetails_Error(t *testing.T) { ctrl := gomock.NewController(t) @@ -138,7 +173,7 @@ func TestNode_InitResources_LoadInstanceDetails_Error(t *testing.T) { mock.MockInstance.EXPECT().LoadDetails(mock.MockEC2API).Return(mockError) - err := mock.NodeWithMock.InitResources(mock.MockResourceManager, mock.MockEC2API) + err := mock.NodeWithMock.InitResources(mock.MockResourceManager) assert.Error(t, &ErrInitResources{Err: mockError}, err) } @@ -162,7 +197,7 @@ func TestNode_InitResources_SecondProviderInitFails(t *testing.T) { // Expect first provider to be de initialized mock.MockProviders["0"].EXPECT().DeInitResource(mock.MockInstance).Return(nil).AnyTimes() - err := mock.NodeWithMock.InitResources(mock.MockResourceManager, mock.MockEC2API) + err := mock.NodeWithMock.InitResources(mock.MockResourceManager) assert.NotNil(t, err) } @@ -181,7 +216,7 @@ func TestNode_DeleteResources(t *testing.T) { mock.MockProviders["1"].EXPECT().IsInstanceSupported(mock.MockInstance).Return(true) mock.MockProviders["1"].EXPECT().DeInitResource(mock.MockInstance).Return(nil) - err := mock.NodeWithMock.DeleteResources(mock.MockResourceManager, mock.MockEC2API) + err := mock.NodeWithMock.DeleteResources(mock.MockResourceManager) assert.NoError(t, err) } @@ -203,7 +238,7 @@ func TestNode_DeleteResources_SomeFail(t *testing.T) { mock.MockProviders["2"].EXPECT().IsInstanceSupported(mock.MockInstance).Return(true) mock.MockProviders["2"].EXPECT().DeInitResource(mock.MockInstance).Return(nil) - err := mock.NodeWithMock.DeleteResources(mock.MockResourceManager, mock.MockEC2API) + err := mock.NodeWithMock.DeleteResources(mock.MockResourceManager) assert.NotNil(t, err) } @@ -224,7 +259,7 @@ func TestNode_UpdateResources(t *testing.T) { mock.MockProviders["1"].EXPECT().IsInstanceSupported(mock.MockInstance).Return(false) - err := mock.NodeWithMock.UpdateResources(mock.MockResourceManager, mock.MockEC2API) + err := mock.NodeWithMock.UpdateResources(mock.MockResourceManager) assert.NoError(t, err) } @@ -247,7 +282,7 @@ func TestNode_UpdateResources_SomeFail(t *testing.T) { mock.MockProviders["2"].EXPECT().IsInstanceSupported(mock.MockInstance).Return(true) mock.MockProviders["2"].EXPECT().UpdateResourceCapacity(mock.MockInstance).Return(nil) - err := mock.NodeWithMock.UpdateResources(mock.MockResourceManager, mock.MockEC2API) + err := mock.NodeWithMock.UpdateResources(mock.MockResourceManager) assert.NotNil(t, err) } @@ -259,6 +294,6 @@ func TestNode_UpdateResources_NodeNotReady(t *testing.T) { mock := NewMock(ctrl, 1) - err := mock.NodeWithMock.UpdateResources(mock.MockResourceManager, mock.MockEC2API) + err := mock.NodeWithMock.UpdateResources(mock.MockResourceManager) assert.Nil(t, err) } diff --git a/pkg/provider/branch/provider.go b/pkg/provider/branch/provider.go index 4e33f5dd..cf6f5de0 100644 --- a/pkg/provider/branch/provider.go +++ b/pkg/provider/branch/provider.go @@ -17,6 +17,7 @@ import ( "context" "encoding/json" "fmt" + "net/http" "strconv" "sync" "time" @@ -25,16 +26,20 @@ import ( "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/aws/ec2" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/aws/vpc" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config" + rcHealthz "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/healthz" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/pool" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/provider" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/provider/branch/trunk" + "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/utils" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/worker" + "github.com/google/uuid" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/go-logr/logr" "github.com/prometheus/client_golang/prometheus" v1 "k8s.io/api/core/v1" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/metrics" ) @@ -94,6 +99,7 @@ type branchENIProvider struct { // apiWrapper apiWrapper api.Wrapper ctx context.Context + checker healthz.Checker } // NewBranchENIProvider returns the Branch ENI Provider for all nodes across the cluster @@ -102,13 +108,15 @@ func NewBranchENIProvider(logger logr.Logger, wrapper api.Wrapper, prometheusRegister() trunk.PrometheusRegister() - return &branchENIProvider{ + provider := &branchENIProvider{ apiWrapper: wrapper, log: logger, workerPool: worker, trunkENICache: make(map[string]trunk.TrunkENI), ctx: ctx, } + provider.checker = provider.check() + return provider } // prometheusRegister registers prometheus metrics @@ -459,8 +467,15 @@ func (b *branchENIProvider) GetPool(_ string) (pool.Pool, bool) { // IsInstanceSupported returns true for linux node as pod eni is only supported for linux worker node func (b *branchENIProvider) IsInstanceSupported(instance ec2.EC2Instance) bool { limits, found := vpc.Limits[instance.Type()] - return found && instance.Os() == config.OSLinux && limits.IsTrunkingCompatible + supported := found && instance.Os() == config.OSLinux && limits.IsTrunkingCompatible + if !supported { + // Send a node event for users' visibility + msg := fmt.Sprintf("The instance type %s is not supported for trunk interface (Security Group for Pods)", instance.Type()) + utils.SendNodeEvent(b.apiWrapper.K8sAPI, instance.Name(), "Unsupported", msg, v1.EventTypeWarning, b.log) + } + + return supported } func (b *branchENIProvider) Introspect() interface{} { @@ -486,3 +501,26 @@ func (b *branchENIProvider) IntrospectNode(nodeName string) interface{} { } return trunkENI.Introspect() } + +func (b *branchENIProvider) check() healthz.Checker { + b.log.Info("Branch provider's healthz subpath was added") + return func(req *http.Request) error { + err := rcHealthz.PingWithTimeout(func(c chan<- error) { + var ping interface{} + // check on job queue + b.SubmitAsyncJob(ping) + // check on trunk cache map + testNodeName := "test-node" + uuid.New().String() + trunk, found := b.getTrunkFromCache(testNodeName) + b.log.V(1).Info("healthz check vulnerable site on locks around trunk map", "TestTrunk", trunk, "FoundInCache", found) + b.log.V(1).Info("***** health check on branch ENI provider tested SubmitAsyncJob *****") + c <- nil + }, b.log) + + return err + } +} + +func (b *branchENIProvider) GetHealthChecker() healthz.Checker { + return b.checker +} diff --git a/pkg/provider/branch/trunk/trunk.go b/pkg/provider/branch/trunk/trunk.go index f7ddea38..1b6a6e1c 100644 --- a/pkg/provider/branch/trunk/trunk.go +++ b/pkg/provider/branch/trunk/trunk.go @@ -188,7 +188,7 @@ func (t *trunkENI) InitTrunk(instance ec2.EC2Instance, podList []v1.Pod) error { } trunk, err := t.ec2ApiHelper.CreateAndAttachNetworkInterface(&instanceID, aws.String(t.instance.SubnetID()), - t.instance.InstanceSecurityGroup(), nil, &freeIndex, &TrunkEniDescription, &InterfaceTypeTrunk, nil) + t.instance.CurrentInstanceSecurityGroups(), nil, &freeIndex, &TrunkEniDescription, &InterfaceTypeTrunk, nil) if err != nil { trunkENIOperationsErrCount.WithLabelValues("create_trunk_eni").Inc() log.Error(err, "failed to create trunk interface") @@ -311,7 +311,7 @@ func (t *trunkENI) CreateAndAssociateBranchENIs(pod *v1.Pod, securityGroups []st // If the security group is empty use the instance security group if securityGroups == nil || len(securityGroups) == 0 { - securityGroups = t.instance.InstanceSecurityGroup() + securityGroups = t.instance.CurrentInstanceSecurityGroups() } var newENIs []*ENIDetails diff --git a/pkg/provider/branch/trunk/trunk_test.go b/pkg/provider/branch/trunk/trunk_test.go index 0d51755b..01af30c7 100644 --- a/pkg/provider/branch/trunk/trunk_test.go +++ b/pkg/provider/branch/trunk/trunk_test.go @@ -597,7 +597,7 @@ func TestTrunkENI_InitTrunk_TrunkNotExists(t *testing.T) { freeIndex := int64(2) mockInstance.EXPECT().InstanceID().Return(InstanceId) - mockInstance.EXPECT().InstanceSecurityGroup().Return(SecurityGroups) + mockInstance.EXPECT().CurrentInstanceSecurityGroups().Return(SecurityGroups) mockEC2APIHelper.EXPECT().GetInstanceNetworkInterface(&InstanceId).Return([]*awsEc2.InstanceNetworkInterface{}, nil) mockInstance.EXPECT().GetHighestUnusedDeviceIndex().Return(freeIndex, nil) mockInstance.EXPECT().SubnetID().Return(SubnetId) @@ -774,7 +774,7 @@ func TestTrunkENI_CreateAndAssociateBranchENIs_InstanceSecurityGroup(t *testing. mockInstance.EXPECT().Type().Return(InstanceType) mockInstance.EXPECT().SubnetID().Return(SubnetId).Times(2) mockInstance.EXPECT().SubnetCidrBlock().Return(SubnetCidrBlock).Times(2) - mockInstance.EXPECT().InstanceSecurityGroup().Return(InstanceSecurityGroup) + mockInstance.EXPECT().CurrentInstanceSecurityGroups().Return(InstanceSecurityGroup) mockEC2APIHelper.EXPECT().CreateNetworkInterface(&BranchEniDescription, &SubnetId, InstanceSecurityGroup, vlan1Tag, nil, nil).Return(BranchInterface1, nil) diff --git a/pkg/provider/ip/eni/eni.go b/pkg/provider/ip/eni/eni.go index 32e8c080..e3fdb4bf 100644 --- a/pkg/provider/ip/eni/eni.go +++ b/pkg/provider/ip/eni/eni.go @@ -22,6 +22,7 @@ import ( "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/aws/ec2/api" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/aws/vpc" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config" + "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/utils" "github.com/aws/aws-sdk-go/aws" "github.com/go-logr/logr" @@ -77,7 +78,7 @@ func (e *eniManager) InitResources(ec2APIHelper api.EC2APIHelper) (*IPv4Resource limits, found := vpc.Limits[e.instance.Type()] if !found { - return nil, fmt.Errorf("unsupported instance type") + return nil, fmt.Errorf("unsupported instance type, error: %w", utils.ErrNotFound) } ipLimit := limits.IPv4PerInterface @@ -185,7 +186,7 @@ func (e *eniManager) CreateIPV4Resource(required int, resourceType config.Resour case config.ResourceTypeIPv4Address: ipResourceCount := &config.IPResourceCount{SecondaryIPv4Count: want} nwInterface, err := ec2APIHelper.CreateAndAttachNetworkInterface(aws.String(e.instance.InstanceID()), - aws.String(e.instance.SubnetID()), e.instance.InstanceSecurityGroup(), nil, aws.Int64(deviceIndex), + aws.String(e.instance.SubnetID()), e.instance.CurrentInstanceSecurityGroups(), nil, aws.Int64(deviceIndex), &ENIDescription, nil, ipResourceCount) if err != nil { // TODO: Check if any clean up is required here for linux nodes only? @@ -207,7 +208,7 @@ func (e *eniManager) CreateIPV4Resource(required int, resourceType config.Resour case config.ResourceTypeIPv4Prefix: ipResourceCount := &config.IPResourceCount{IPv4PrefixCount: want} nwInterface, err := ec2APIHelper.CreateAndAttachNetworkInterface(aws.String(e.instance.InstanceID()), - aws.String(e.instance.SubnetID()), e.instance.InstanceSecurityGroup(), nil, aws.Int64(deviceIndex), + aws.String(e.instance.SubnetID()), e.instance.CurrentInstanceSecurityGroups(), nil, aws.Int64(deviceIndex), &ENIDescription, nil, ipResourceCount) if err != nil { // TODO: Check if any clean up is required here for linux nodes only? diff --git a/pkg/provider/ip/eni/eni_test.go b/pkg/provider/ip/eni/eni_test.go index c1d33016..3e94bf2b 100644 --- a/pkg/provider/ip/eni/eni_test.go +++ b/pkg/provider/ip/eni/eni_test.go @@ -18,9 +18,10 @@ import ( "reflect" "testing" - "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/aws/ec2" - "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/aws/ec2/api" + mock_ec2 "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/aws/ec2" + mock_api "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/aws/ec2/api" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config" + "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/utils" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" "github.com/golang/mock/gomock" @@ -186,6 +187,23 @@ func TestEni_InitResources_Error(t *testing.T) { assert.Error(t, mockError, err) } +// TestEni_InitResources_Unsupported_Type_Error tests that error is returned if the instance type is not supported +func TestEni_InitResources_Unsupported_Type_Error(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + manager, mockInstance, mockEc2APIHelper := getMockManager(ctrl) + dummyType := "dummy.large" + mockInstance.EXPECT().InstanceID().Return(instanceID) + mockInstance.EXPECT().Type().Return(dummyType) + mockEc2APIHelper.EXPECT().GetInstanceNetworkInterface(&instanceID).Return(nwInterfaces, nil) + + _, err := manager.InitResources(mockEc2APIHelper) + + assert.Error(t, mockError, err) + assert.ErrorIs(t, err, utils.ErrNotFound) +} + // TestEniManager_CreateIPV4Resource_TypeIPV4Address_FromSingleENI tests IPs are created using a single ENI when it has the desired // capacity func TestEniManager_CreateIPV4Resource_TypeIPV4Address_FromSingleENI(t *testing.T) { @@ -309,7 +327,7 @@ func TestEniManager_CreateIPV4Resource_TypeIPV4Address_FromNewENI(t *testing.T) mockInstance.EXPECT().InstanceID().Return(instanceID).Times(2) mockInstance.EXPECT().SubnetID().Return(subnetID).Times(2) mockInstance.EXPECT().SubnetMask().Return(subnetMask).Times(4) - mockInstance.EXPECT().InstanceSecurityGroup().Return(instanceSG).Times(2) + mockInstance.EXPECT().CurrentInstanceSecurityGroups().Return(instanceSG).Times(2) gomock.InOrder( mockEc2APIHelper.EXPECT().CreateAndAttachNetworkInterface(&instanceID, &subnetID, instanceSG, nil, aws.Int64(3), @@ -345,7 +363,7 @@ func TestEniManager_CreateIPV4Resource_TypeIPV4Prefix_FromNewENI(t *testing.T) { mockInstance.EXPECT().GetHighestUnusedDeviceIndex().Return(int64(3), nil).Times(1) mockInstance.EXPECT().InstanceID().Return(instanceID).Times(1) mockInstance.EXPECT().SubnetID().Return(subnetID).Times(1) - mockInstance.EXPECT().InstanceSecurityGroup().Return(instanceSG).Times(1) + mockInstance.EXPECT().CurrentInstanceSecurityGroups().Return(instanceSG).Times(1) mockEc2APIHelper.EXPECT().CreateAndAttachNetworkInterface(&instanceID, &subnetID, instanceSG, nil, aws.Int64(3), &ENIDescription, nil, prefixCountFor1).Return(networkInterface3, nil) @@ -376,7 +394,7 @@ func TestEniManager_CreateIPV4Resource_TypeIPV4Address_InBetweenENIFail(t *testi mockInstance.EXPECT().GetHighestUnusedDeviceIndex().Return(int64(3), nil).Times(2) mockInstance.EXPECT().InstanceID().Return(instanceID).Times(2) mockInstance.EXPECT().SubnetID().Return(subnetID).Times(2) - mockInstance.EXPECT().InstanceSecurityGroup().Return(instanceSG).Times(2) + mockInstance.EXPECT().CurrentInstanceSecurityGroups().Return(instanceSG).Times(2) gomock.InOrder( mockEc2APIHelper.EXPECT().CreateAndAttachNetworkInterface(&instanceID, &subnetID, instanceSG, nil, aws.Int64(3), @@ -411,7 +429,7 @@ func TestEniManager_CreateIPV4Resource_TypeIPV4Prefix_InBetweenENIFail(t *testin mockInstance.EXPECT().GetHighestUnusedDeviceIndex().Return(int64(3), nil).Times(2) mockInstance.EXPECT().InstanceID().Return(instanceID).Times(2) mockInstance.EXPECT().SubnetID().Return(subnetID).Times(2) - mockInstance.EXPECT().InstanceSecurityGroup().Return(instanceSG).Times(2) + mockInstance.EXPECT().CurrentInstanceSecurityGroups().Return(instanceSG).Times(2) gomock.InOrder( mockEc2APIHelper.EXPECT().CreateAndAttachNetworkInterface(&instanceID, &subnetID, instanceSG, nil, aws.Int64(3), diff --git a/pkg/provider/ip/provider.go b/pkg/provider/ip/provider.go index 0fc46bf0..6d8f121d 100644 --- a/pkg/provider/ip/provider.go +++ b/pkg/provider/ip/provider.go @@ -14,7 +14,9 @@ package ip import ( + "errors" "fmt" + "net/http" "sync" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/api" @@ -22,13 +24,17 @@ import ( "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/aws/vpc" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/condition" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config" + rcHealthz "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/healthz" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/pool" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/provider" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/provider/ip/eni" + "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/utils" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/worker" + v1 "k8s.io/api/core/v1" "github.com/go-logr/logr" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/healthz" ) type ipv4Provider struct { @@ -46,6 +52,8 @@ type ipv4Provider struct { instanceProviderAndPool map[string]*ResourceProviderAndPool // conditions is used to check which IP allocation mode is enabled conditions condition.Conditions + // healthz check subpath + checker healthz.Checker } // ResourceProviderAndPool contains the instance's ENI manager and the resource pool @@ -62,7 +70,7 @@ type ResourceProviderAndPool struct { func NewIPv4Provider(log logr.Logger, apiWrapper api.Wrapper, workerPool worker.Worker, resourceConfig config.ResourceConfig, conditions condition.Conditions) provider.ResourceProvider { - return &ipv4Provider{ + provider := &ipv4Provider{ instanceProviderAndPool: make(map[string]*ResourceProviderAndPool), config: resourceConfig.WarmPoolConfig, log: log, @@ -70,6 +78,8 @@ func NewIPv4Provider(log logr.Logger, apiWrapper api.Wrapper, workerPool: workerPool, conditions: conditions, } + provider.checker = provider.check() + return provider } func (p *ipv4Provider) InitResource(instance ec2.EC2Instance) error { @@ -78,6 +88,11 @@ func (p *ipv4Provider) InitResource(instance ec2.EC2Instance) error { eniManager := eni.NewENIManager(instance) ipV4Resources, err := eniManager.InitResources(p.apiWrapper.EC2API) if err != nil || ipV4Resources == nil { + if errors.Is(err, utils.ErrNotFound) { + msg := fmt.Sprintf("The instance type %s is not supported for Windows", instance.Type()) + utils.SendNodeEvent(p.apiWrapper.K8sAPI, instance.Name(), "Unsupported", msg, v1.EventTypeWarning, p.log) + } + return err } @@ -418,3 +433,21 @@ func (p *ipv4Provider) IntrospectNode(nodeName string) interface{} { } return resource.resourcePool.Introspect() } + +func (p *ipv4Provider) check() healthz.Checker { + p.log.Info("IPv4 provider's healthz subpath was added") + return func(req *http.Request) error { + err := rcHealthz.PingWithTimeout(func(c chan<- error) { + var ping interface{} + p.SubmitAsyncJob(ping) + p.log.V(1).Info("***** health check on IPv4 provider tested SubmitAsyncJob *****") + c <- nil + }, p.log) + + return err + } +} + +func (p *ipv4Provider) GetHealthChecker() healthz.Checker { + return p.checker +} diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go index b989fdb9..ab45a5c0 100644 --- a/pkg/provider/provider.go +++ b/pkg/provider/provider.go @@ -17,6 +17,7 @@ import ( "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/aws/ec2" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/pool" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/healthz" ) // ResourceProvider is the provider interface that each resource managed by the controller has to implement @@ -39,4 +40,6 @@ type ResourceProvider interface { Introspect() interface{} // IntrospectNode allows introspection of a node for the given resource IntrospectNode(node string) interface{} + // GetHealthChecker provider a health check subpath for pinging provider + GetHealthChecker() healthz.Checker } diff --git a/pkg/resource/introspect.go b/pkg/resource/introspect.go index ebbf2592..eb7d9d14 100644 --- a/pkg/resource/introspect.go +++ b/pkg/resource/introspect.go @@ -18,8 +18,10 @@ import ( "encoding/json" "net/http" + rcHealthz "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/healthz" "github.com/go-logr/logr" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/healthz" ) const ( @@ -93,6 +95,11 @@ func (i *IntrospectHandler) NodeResourceHandler(w http.ResponseWriter, r *http.R w.Write(jsonData) } -func (i *IntrospectHandler) SetupWithManager(mgr ctrl.Manager) error { +func (i *IntrospectHandler) SetupWithManager(mgr ctrl.Manager, healthzHanlder *rcHealthz.HealthzHandler) error { + // add health check on subpath for introspect controller + healthzHanlder.AddControllersHealthCheckers( + map[string]healthz.Checker{"health-introspect-controller": rcHealthz.SimplePing("Introspect controller", i.Log)}, + ) + return mgr.Add(i) } diff --git a/pkg/resource/manager.go b/pkg/resource/manager.go index d101a78c..3e1aa83b 100644 --- a/pkg/resource/manager.go +++ b/pkg/resource/manager.go @@ -21,17 +21,27 @@ import ( "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/condition" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/handler" + rcHealthz "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/healthz" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/provider" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/provider/branch" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/provider/ip" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/provider/prefix" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/worker" + "github.com/go-logr/logr" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/healthz" +) + +var ( + managerHealthCheckSubpath = "health-resource-manager" + branchProviderHealthCheckSubpath = "health-branch-provider" + ipv4ProviderHealthCheckSubpath = "health-ipv4-provider" ) type Manager struct { resource map[string]Resource + log logr.Logger } type Resource struct { @@ -45,12 +55,18 @@ type ResourceManager interface { GetResourceHandler(resourceName string) (handler.Handler, bool) } -func NewResourceManager(ctx context.Context, resourceNames []string, wrapper api.Wrapper, conditions condition.Conditions) (ResourceManager, error) { +func NewResourceManager(ctx context.Context, resourceNames []string, wrapper api.Wrapper, log logr.Logger, + healthzHandler *rcHealthz.HealthzHandler, conditions condition.Conditions) (ResourceManager, error) { // Load that static configuration of the resource resourceConfig := config.LoadResourceConfig() resources := make(map[string]Resource) + healthCheckers := make(map[string]healthz.Checker) + + // add manager subpath into health checker map first + healthCheckers[managerHealthCheckSubpath] = rcHealthz.SimplePing("resource manager", log) + // For each supported resource, initialize the resource provider and handler for _, resourceName := range resourceNames { @@ -59,14 +75,14 @@ func NewResourceManager(ctx context.Context, resourceNames []string, wrapper api return nil, fmt.Errorf("failed to find resource configuration %s", resourceName) } - ctrl.Log.Info("initializing resource", "resource name", + log.Info("initializing resource", "resource name", resourceName, "resource count", resourceConfig.WorkerCount) workers := worker.NewDefaultWorkerPool( resourceConfig.Name, resourceConfig.WorkerCount, config.WorkQueueDefaultMaxRetries, - ctrl.Log.WithName(fmt.Sprintf("%s-%s", resourceName, "worker")), ctx) + log.WithName(fmt.Sprintf("%s-%s", resourceName, "worker")), ctx) var resourceHandler handler.Handler var resourceProvider provider.ResourceProvider @@ -74,6 +90,7 @@ func NewResourceManager(ctx context.Context, resourceNames []string, wrapper api if resourceName == config.ResourceNameIPAddress { resourceProvider = ip.NewIPv4Provider(ctrl.Log.WithName("ipv4 provider"), wrapper, workers, resourceConfig, conditions) + healthCheckers[ipv4ProviderHealthCheckSubpath] = resourceProvider.GetHealthChecker() resourceHandler = handler.NewWarmResourceHandler(ctrl.Log.WithName(resourceName), wrapper, resourceName, resourceProvider, ctx) } else if resourceName == config.ResourceNameIPAddressFromPrefix { @@ -84,6 +101,7 @@ func NewResourceManager(ctx context.Context, resourceNames []string, wrapper api } else if resourceName == config.ResourceNamePodENI { resourceProvider = branch.NewBranchENIProvider(ctrl.Log.WithName("branch eni provider"), wrapper, workers, resourceConfig, ctx) + healthCheckers[branchProviderHealthCheckSubpath] = resourceProvider.GetHealthChecker() resourceHandler = handler.NewOnDemandHandler(ctrl.Log.WithName(resourceName), resourceName, resourceProvider) } else { @@ -100,12 +118,16 @@ func NewResourceManager(ctx context.Context, resourceNames []string, wrapper api ResourceProvider: resourceProvider, } - ctrl.Log.Info("successfully initialized resource handler and provider", + log.Info("successfully initialized resource handler and provider", "resource name", resourceName) } + // add health check on subpath for resource manager which includes providers as well + healthzHandler.AddControllersHealthCheckers(healthCheckers) + return &Manager{ resource: resources, + log: log, }, nil } diff --git a/pkg/resource/manager_test.go b/pkg/resource/manager_test.go index 7e3fe8a4..109b002f 100644 --- a/pkg/resource/manager_test.go +++ b/pkg/resource/manager_test.go @@ -17,11 +17,12 @@ import ( "context" "testing" - "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/handler" + mock_handler "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/handler" mock_k8s "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/k8s" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/api" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/condition" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config" + "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/healthz" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -32,6 +33,8 @@ type Mock struct { Wrapper api.Wrapper } +var healthzHandler = healthz.NewHealthzHandler(5) + func NewMock(controller *gomock.Controller) Mock { return Mock{ Handler: mock_handler.NewMockHandler(controller), @@ -48,8 +51,7 @@ func Test_NewResourceManager(t *testing.T) { mockK8s := mock_k8s.NewMockK8sWrapper(ctrl) conditions := condition.NewControllerConditions(zap.New(), mockK8s) - - manger, err := NewResourceManager(context.TODO(), resources, mock.Wrapper, conditions) + manger, err := NewResourceManager(context.TODO(), resources, mock.Wrapper, zap.New(zap.UseDevMode(true)), healthzHandler, conditions) assert.NoError(t, err) _, ok := manger.GetResourceHandler(config.ResourceNamePodENI) diff --git a/pkg/utils/errors.go b/pkg/utils/errors.go new file mode 100644 index 00000000..5c816913 --- /dev/null +++ b/pkg/utils/errors.go @@ -0,0 +1,7 @@ +package utils + +import "errors" + +var ( + ErrNotFound = errors.New("resource was not found") +) diff --git a/pkg/utils/events.go b/pkg/utils/events.go new file mode 100644 index 00000000..01771090 --- /dev/null +++ b/pkg/utils/events.go @@ -0,0 +1,21 @@ +package utils + +import ( + "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/k8s" + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/types" +) + +var ( + unsupportedInstanceTypeReason = "Unsupported" +) + +func SendNodeEvent(client k8s.K8sWrapper, nodeName, reason, msg, eventType string, logger logr.Logger) { + if node, err := client.GetNode(nodeName); err == nil { + // set UID to node name for kubelet filter the event to node description + node.SetUID(types.UID(nodeName)) + client.BroadcastEvent(node, unsupportedInstanceTypeReason, msg, eventType) + } else { + logger.Error(err, "had an error to get the node for sending unsupported event", "Node", nodeName) + } +} diff --git a/pkg/worker/worker.go b/pkg/worker/worker.go index d4809d2e..66de88ca 100644 --- a/pkg/worker/worker.go +++ b/pkg/worker/worker.go @@ -126,6 +126,12 @@ func (w *worker) SetWorkerFunc(workerFunc func(interface{}) (ctrl.Result, error) // SubmitJob adds the job to the rate limited queue func (w *worker) SubmitJob(job interface{}) { + // in theory, only health check endpoint should send a nil job to test periodically + if job == nil { + queueLen := w.queue.Len() + w.Log.V(1).Info("For informational / health check purpose only to check worker queue availability", "WorkerQueueLen", queueLen) + return + } w.queue.Add(job) jobsSubmittedCount.WithLabelValues(w.resourceName).Inc() } diff --git a/scripts/gen_mocks.sh b/scripts/gen_mocks.sh index 90ff441b..21dbc9c2 100755 --- a/scripts/gen_mocks.sh +++ b/scripts/gen_mocks.sh @@ -24,4 +24,4 @@ mockgen -destination=../mocks/amazon-vcp-resource-controller-k8s/pkg/pool/mock_p # package resource maocks mockgen -destination=../mocks/amazon-vcp-resource-controller-k8s/pkg/resource/mock_resources.go github.com/aws/amazon-vpc-resource-controller-k8s/pkg/resource ResourceManager # package condition maocks -mockgen -destination=../mocks/amazon-vcp-resource-controller-k8s/pkg/condition/mock_condtion.go github.com/aws/amazon-vpc-resource-controller-k8s/pkg/condition Conditions +mockgen -destination=../mocks/amazon-vcp-resource-controller-k8s/pkg/condition/mock_condition.go github.com/aws/amazon-vpc-resource-controller-k8s/pkg/condition Conditions diff --git a/webhooks/core/annotation_validation_webhook.go b/webhooks/core/annotation_validation_webhook.go index cfb27354..5e160ee3 100644 --- a/webhooks/core/annotation_validation_webhook.go +++ b/webhooks/core/annotation_validation_webhook.go @@ -20,10 +20,12 @@ import ( "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/condition" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config" + rcHealthz "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/healthz" "github.com/go-logr/logr" admissionv1 "k8s.io/api/admission/v1" corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" ) @@ -35,6 +37,23 @@ type AnnotationValidator struct { decoder *admission.Decoder Condition condition.Conditions Log logr.Logger + Checker healthz.Checker +} + +func NewAnnotationValidator(condition condition.Conditions, log logr.Logger, healthzHandler *rcHealthz.HealthzHandler) *AnnotationValidator { + annotationValidator := &AnnotationValidator{ + Condition: condition, + Log: log, + } + + // add health check on subpath for pod annotation validating webhook + healthzHandler.AddControllersHealthCheckers( + map[string]healthz.Checker{ + "health-annotation-validating-webhook": rcHealthz.SimplePing("pod annotation validating webhook", log), + }, + ) + + return annotationValidator } // We are allowing multiple usernames to annotate the Windows/SGP Pod, eventually we will diff --git a/webhooks/core/node_update_webhook.go b/webhooks/core/node_update_webhook.go index 1f743ba5..c3e52f05 100644 --- a/webhooks/core/node_update_webhook.go +++ b/webhooks/core/node_update_webhook.go @@ -7,8 +7,10 @@ import ( "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/condition" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config" + rcHealthz "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/healthz" "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" ) @@ -16,6 +18,23 @@ type NodeUpdateWebhook struct { decoder *admission.Decoder Condition condition.Conditions Log logr.Logger + Checker healthz.Checker +} + +func NewNodeUpdateWebhook(condition condition.Conditions, log logr.Logger, healthzHandler *rcHealthz.HealthzHandler) *NodeUpdateWebhook { + nodeUpdateWebhook := &NodeUpdateWebhook{ + Condition: condition, + Log: log, + } + + // add health check on subpath for node validation webhook + healthzHandler.AddControllersHealthCheckers( + map[string]healthz.Checker{ + "health-node-validating-webhook": rcHealthz.SimplePing("node validating webhook", log), + }, + ) + + return nodeUpdateWebhook } const awsNodeUsername = "system:serviceaccount:kube-system:aws-node" diff --git a/webhooks/core/pod_webhook.go b/webhooks/core/pod_webhook.go index 6cfa0474..b1eb0cc8 100644 --- a/webhooks/core/pod_webhook.go +++ b/webhooks/core/pod_webhook.go @@ -22,10 +22,12 @@ import ( "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/condition" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config" + rcHealthz "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/healthz" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/utils" ) @@ -45,6 +47,25 @@ type PodMutationWebHook struct { Condition condition.Conditions } +func NewPodMutationWebHook( + sgpAPI utils.SecurityGroupForPodsAPI, + log logr.Logger, + condition condition.Conditions, + healthzHandler *rcHealthz.HealthzHandler, +) *PodMutationWebHook { + podWebhook := &PodMutationWebHook{ + SGPAPI: sgpAPI, + Log: log, + Condition: condition, + } + // add health check on subpath for pod mutation webhook + healthzHandler.AddControllersHealthCheckers( + map[string]healthz.Checker{"health-pod-mutating-webhook": rcHealthz.SimplePing("pod mutating webhook", log)}, + ) + + return podWebhook +} + type PodType string var ( diff --git a/webhooks/core/pod_webhook_test.go b/webhooks/core/pod_webhook_test.go index 5549af41..ec4c4fe1 100644 --- a/webhooks/core/pod_webhook_test.go +++ b/webhooks/core/pod_webhook_test.go @@ -20,8 +20,8 @@ import ( "strings" "testing" - "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/condition" - "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/utils" + mock_condition "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/condition" + mock_utils "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/utils" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config" "github.com/golang/mock/gomock"