diff --git a/cmd/controller/main.go b/cmd/controller/main.go index fc09b917beef..af2cd28e5d7b 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -62,6 +62,7 @@ func main() { cloudProvider, op.SubnetProvider, op.SecurityGroupProvider, + op.CapacityReservationProvider, op.InstanceProfileProvider, op.InstanceProvider, op.PricingProvider, diff --git a/go.mod b/go.mod index 013e3d8cb4f8..8bf66fa43d0c 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/aws/karpenter-provider-aws -go 1.22 +go 1.22.3 require ( github.com/Pallinder/go-randomdata v1.2.0 @@ -12,23 +12,23 @@ require ( github.com/go-logr/zapr v1.3.0 github.com/imdario/mergo v0.3.16 github.com/mitchellh/hashstructure/v2 v2.0.2 - github.com/onsi/ginkgo/v2 v2.17.2 - github.com/onsi/gomega v1.33.0 + github.com/onsi/ginkgo/v2 v2.17.3 + github.com/onsi/gomega v1.33.1 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pelletier/go-toml/v2 v2.2.1 - github.com/prometheus/client_golang v1.19.0 + github.com/prometheus/client_golang v1.19.1 github.com/samber/lo v1.39.0 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.0 golang.org/x/sync v0.7.0 golang.org/x/time v0.5.0 - k8s.io/api v0.29.3 - k8s.io/apiextensions-apiserver v0.29.3 - k8s.io/apimachinery v0.29.3 - k8s.io/client-go v0.29.3 + k8s.io/api v0.30.0 + k8s.io/apiextensions-apiserver v0.30.0 + k8s.io/apimachinery v0.30.0 + k8s.io/client-go v0.30.0 k8s.io/utils v0.0.0-20240102154912-e7106e64919e knative.dev/pkg v0.0.0-20231010144348-ca8c009405dd - sigs.k8s.io/controller-runtime v0.17.3 + sigs.k8s.io/controller-runtime v0.18.2 sigs.k8s.io/karpenter v0.36.0 sigs.k8s.io/yaml v1.4.0 ) @@ -46,7 +46,7 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch v5.7.0+incompatible // indirect - github.com/evanphx/json-patch/v5 v5.8.0 // indirect + github.com/evanphx/json-patch/v5 v5.9.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.6.0 // indirect @@ -63,7 +63,7 @@ require ( github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect - github.com/google/uuid v1.3.1 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -79,8 +79,8 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_model v0.6.0 // indirect - github.com/prometheus/common v0.48.0 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.53.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/prometheus/statsd_exporter v0.24.0 // indirect github.com/rivo/uniseg v0.4.4 // indirect @@ -91,10 +91,10 @@ require ( go.uber.org/automaxprocs v1.5.3 // indirect golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect golang.org/x/net v0.24.0 // indirect - golang.org/x/oauth2 v0.16.0 // indirect + golang.org/x/oauth2 v0.18.0 // indirect golang.org/x/sys v0.19.0 // indirect golang.org/x/term v0.19.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/text v0.15.0 // indirect golang.org/x/tools v0.20.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/api v0.146.0 // indirect @@ -107,11 +107,13 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/cloud-provider v0.29.3 // indirect - k8s.io/component-base v0.29.3 // indirect - k8s.io/csi-translation-lib v0.29.3 // indirect + k8s.io/cloud-provider v0.30.0 // indirect + k8s.io/component-base v0.30.0 // indirect + k8s.io/csi-translation-lib v0.30.0 // indirect k8s.io/klog/v2 v2.120.1 // indirect - k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect + k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect ) + +// replace sigs.k8s.io/karpenter => github.pie.apple.com/tvonhacht/karpenter v0.0.6-odcr diff --git a/go.sum b/go.sum index 5cd1895738ed..a368ffbba1d8 100644 --- a/go.sum +++ b/go.sum @@ -93,8 +93,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI= github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch/v5 v5.8.0 h1:lRj6N9Nci7MvzrXuX6HFzU8XjmhPiXPlsKEy1u0KQro= -github.com/evanphx/json-patch/v5 v5.8.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= +github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -200,8 +200,8 @@ github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 h1:k7nVchz72niMH6YLQN github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/grpc-ecosystem/grpc-gateway v1.14.6/go.mod h1:zdiPV4Yse/1gnckTHtghG4GkDEdKCRJduHpTxT3/jcw= @@ -270,10 +270,10 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= -github.com/onsi/ginkgo/v2 v2.17.2 h1:7eMhcy3GimbsA3hEnVKdw/PQM9XN9krpKVXsZdph0/g= -github.com/onsi/ginkgo/v2 v2.17.2/go.mod h1:nP2DPOQoNsQmsVyv5rDA8JkXQoCs6goXIvr/PRJ1eCc= -github.com/onsi/gomega v1.33.0 h1:snPCflnZrpMsy94p4lXVEkHo12lmPnc3vY5XBbreexE= -github.com/onsi/gomega v1.33.0/go.mod h1:+925n5YtiFsLzzafLUHzVMBpvvRAzrydIBiSIxjX3wY= +github.com/onsi/ginkgo/v2 v2.17.3 h1:oJcvKpIb7/8uLpDDtnQuf18xVnwKp8DTD7DQ6gTd/MU= +github.com/onsi/ginkgo/v2 v2.17.3/go.mod h1:nP2DPOQoNsQmsVyv5rDA8JkXQoCs6goXIvr/PRJ1eCc= +github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= +github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pelletier/go-toml/v2 v2.2.1 h1:9TA9+T8+8CUCO2+WYnDLCgrYi9+omqKXyjDtosvtEhg= @@ -293,22 +293,22 @@ github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqr github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= -github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= -github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= +github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= +github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos= -github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.35.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= -github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= -github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= +github.com/prometheus/common v0.53.0 h1:U2pL9w9nmJwJDa4qqLQ3ZaePJ6ZTwt7cMD3AG3+aLCE= +github.com/prometheus/common v0.53.0/go.mod h1:BrxBKv3FWBIGXw89Mg1AeBq7FSyRzXWI3l3e7W3RN5U= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= @@ -469,8 +469,8 @@ golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= -golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= +golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= +golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -548,8 +548,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -730,24 +730,24 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.29.3 h1:2ORfZ7+bGC3YJqGpV0KSDDEVf8hdGQ6A03/50vj8pmw= -k8s.io/api v0.29.3/go.mod h1:y2yg2NTyHUUkIoTC+phinTnEa3KFM6RZ3szxt014a80= -k8s.io/apiextensions-apiserver v0.29.3 h1:9HF+EtZaVpFjStakF4yVufnXGPRppWFEQ87qnO91YeI= -k8s.io/apiextensions-apiserver v0.29.3/go.mod h1:po0XiY5scnpJfFizNGo6puNU6Fq6D70UJY2Cb2KwAVc= -k8s.io/apimachinery v0.29.3 h1:2tbx+5L7RNvqJjn7RIuIKu9XTsIZ9Z5wX2G22XAa5EU= -k8s.io/apimachinery v0.29.3/go.mod h1:hx/S4V2PNW4OMg3WizRrHutyB5la0iCUbZym+W0EQIU= -k8s.io/client-go v0.29.3 h1:R/zaZbEAxqComZ9FHeQwOh3Y1ZUs7FaHKZdQtIc2WZg= -k8s.io/client-go v0.29.3/go.mod h1:tkDisCvgPfiRpxGnOORfkljmS+UrW+WtXAy2fTvXJB0= -k8s.io/cloud-provider v0.29.3 h1:y39hNq0lrPD1qmqQ2ykwMJGeWF9LsepVkR2a4wskwLc= -k8s.io/cloud-provider v0.29.3/go.mod h1:daDV1WkAO6pTrdsn7v8TpN/q9n75ExUC4RJDl7vlPKk= -k8s.io/component-base v0.29.3 h1:Oq9/nddUxlnrCuuR2K/jp6aflVvc0uDvxMzAWxnGzAo= -k8s.io/component-base v0.29.3/go.mod h1:Yuj33XXjuOk2BAaHsIGHhCKZQAgYKhqIxIjIr2UXYio= -k8s.io/csi-translation-lib v0.29.3 h1:GNYCE0f86K3Xkyrk7WKKwQZkJrum6QQapbOzYxZv6Mg= -k8s.io/csi-translation-lib v0.29.3/go.mod h1:snAzieA58/oiQXQZr27b0+b6/3+ZzitwI+57cUsMKKQ= +k8s.io/api v0.30.0 h1:siWhRq7cNjy2iHssOB9SCGNCl2spiF1dO3dABqZ8niA= +k8s.io/api v0.30.0/go.mod h1:OPlaYhoHs8EQ1ql0R/TsUgaRPhpKNxIMrKQfWUp8QSE= +k8s.io/apiextensions-apiserver v0.30.0 h1:jcZFKMqnICJfRxTgnC4E+Hpcq8UEhT8B2lhBcQ+6uAs= +k8s.io/apiextensions-apiserver v0.30.0/go.mod h1:N9ogQFGcrbWqAY9p2mUAL5mGxsLqwgtUce127VtRX5Y= +k8s.io/apimachinery v0.30.0 h1:qxVPsyDM5XS96NIh9Oj6LavoVFYff/Pon9cZeDIkHHA= +k8s.io/apimachinery v0.30.0/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= +k8s.io/client-go v0.30.0 h1:sB1AGGlhY/o7KCyCEQ0bPWzYDL0pwOZO4vAtTSh/gJQ= +k8s.io/client-go v0.30.0/go.mod h1:g7li5O5256qe6TYdAMyX/otJqMhIiGgTapdLchhmOaY= +k8s.io/cloud-provider v0.30.0 h1:hz1MXkFjsyO167sRZVchXEi2YYMQ6kolBi79nuICjzw= +k8s.io/cloud-provider v0.30.0/go.mod h1:iyVcGvDfmZ7m5cliI9TTHj0VTjYDNpc/K71Gp6hukjU= +k8s.io/component-base v0.30.0 h1:cj6bp38g0ainlfYtaOQuRELh5KSYjhKxM+io7AUIk4o= +k8s.io/component-base v0.30.0/go.mod h1:V9x/0ePFNaKeKYA3bOvIbrNoluTSG+fSJKjLdjOoeXQ= +k8s.io/csi-translation-lib v0.30.0 h1:pEe6jshNVE4od2AdgYlsAtiKP/MH+NcsBbUPA/dWA6U= +k8s.io/csi-translation-lib v0.30.0/go.mod h1:5TT/awOiKEX+8CcbReVYJyddT7xqlFrp3ChE9e45MyU= k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= -k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCfRziVtos3ofG/sQ= k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= knative.dev/pkg v0.0.0-20231010144348-ca8c009405dd h1:KJXBX9dOmRTUWduHg1gnWtPGIEl+GMh8UHdrBEZgOXE= @@ -755,8 +755,8 @@ knative.dev/pkg v0.0.0-20231010144348-ca8c009405dd/go.mod h1:36cYnaOVHkzmhgybmYX rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/controller-runtime v0.17.3 h1:65QmN7r3FWgTxDMz9fvGnO1kbf2nu+acg9p2R9oYYYk= -sigs.k8s.io/controller-runtime v0.17.3/go.mod h1:N0jpP5Lo7lMTF9aL56Z/B2oWBJjey6StQM0jRbKQXtY= +sigs.k8s.io/controller-runtime v0.18.2 h1:RqVW6Kpeaji67CY5nPEfRz6ZfFMk0lWQlNrLqlNpx+Q= +sigs.k8s.io/controller-runtime v0.18.2/go.mod h1:tuAt1+wbVsXIT8lPtk5RURxqAnq7xkpv2Mhttslg7Hw= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/karpenter v0.36.0 h1:i82fOsFWKwnChedKsj0Hep2yrTkAjCek/aZPSMX2dW8= diff --git a/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml b/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml index 66ec659e7747..ce815ddd44ef 100644 --- a/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml +++ b/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml @@ -216,6 +216,67 @@ spec: - message: must have only one blockDeviceMappings with rootVolume rule: self.filter(x, has(x.rootVolume)?x.rootVolume==true:false).size() <= 1 + capacityReservationSelectorTerms: + description: CapacityReservationSelectorTerms is a list of or Capacity + Reservation selector terms. The terms are ORed. + items: + description: |- + CapacityReservationSelectorTerm defines selection logic for a Capacity Reservation used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + availabilityZone: + description: The Availability Zone of the Capacity Reservation + type: string + id: + description: The platform of operating system for which the + Capacity Reservation reserves capacity + type: string + instanceMatchCriteria: + description: |- + Indicates the type of instance launches that the Capacity Reservation accepts. The options include 'open' and 'targeted'. + open - The Capacity Reservation accepts all instances that have + matching attributes (instance type, platform, and Availability + Zone). Instances that have matching attributes launch into the + Capacity Reservation automatically without specifying any + additional parameters. + targeted - The Capacity Reservation only accepts instances that + have matching attributes (instance type, platform, and + Availability Zone), and explicitly target the Capacity + Reservation. This ensures that only permitted instances can use + the reserved capacity. + type: string + instancePlatform: + description: |- + Indicates the tenancy of the Capacity Reservation. + A Capacity Reservation can have one of the following tenancy 'default' or 'dedicated' + default - The Capacity Reservation is created on hardware that is shared with other Amazon Web Services accounts. + dedicated - The Capacity Reservation is created on single-tenant hardware that is dedicated to a single Amazon Web Services account. + type: string + instanceType: + description: The type of operating system for which the Capacity + Reservation reserves capacity + type: string + ownerId: + description: The ID of the Amazon Web Services account that + owns the Capacity Reservation + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select subnets + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + tenancy: + description: ID is the Capacity Reservation id in EC2 + pattern: cr-[0-9a-z]+ + type: string + type: object + type: array context: description: |- Context is a Reserved field in EC2 APIs @@ -506,14 +567,86 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + required: + - id + - requirements + type: object + type: array + capacityReservations: + description: |- + CapacityReservations contains the current Capacity Reservations values that are available to the + cluster under the CapacityReservations selectors. + items: + description: CapacityReservation contains resolved Capacity Reservation + selector values utilized for node launch + properties: + availabilityZone: + description: AvailabilityZone of the Capacity Reservation + type: string + availableInstanceCount: + description: Available Instance Count of the Capacity Reservation + type: integer + id: + description: ID of the Capacity Reservation + type: string + instanceType: + description: InstanceType of the Capacity Reservation + type: string + requirements: + description: Requirements of the Capacity Reservation to be + utilized on an instance type + items: + description: |- + A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values + and minValues that represent the requirement to have at least that many values. + properties: + key: + description: The label key that the selector applies to. + type: string + minValues: + description: |- + This field is ALPHA and can be dropped or replaced at any time + MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. + maximum: 50 + minimum: 1 + type: integer + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + totalInstanceCount: + description: Total Instance Count of the Capacity Reservation + type: integer required: + - availabilityZone + - availableInstanceCount - id + - instanceType - requirements + - totalInstanceCount type: object type: array instanceProfile: diff --git a/pkg/apis/v1beta1/ec2nodeclass.go b/pkg/apis/v1beta1/ec2nodeclass.go index 66a2926b2ced..c51ee12fb3b1 100644 --- a/pkg/apis/v1beta1/ec2nodeclass.go +++ b/pkg/apis/v1beta1/ec2nodeclass.go @@ -55,6 +55,9 @@ type EC2NodeClassSpec struct { // +kubebuilder:validation:Enum:={AL2,AL2023,Bottlerocket,Ubuntu,Custom,Windows2019,Windows2022} // +required AMIFamily *string `json:"amiFamily"` + // CapacityReservationSelectorTerms is a list of or Capacity Reservation selector terms. The terms are ORed. + // +required + CapacityReservationSelectorTerms []CapacityReservationSelectorTerm `json:"capacityReservationSelectorTerms,omitempty" hash:"ignore"` // UserData to be applied to the provisioned nodes. // It must be in the appropriate format based on the AMIFamily in use. Karpenter will merge certain fields into // this UserData to ensure nodes are being provisioned with the correct configuration. @@ -175,6 +178,52 @@ type AMISelectorTerm struct { Owner string `json:"owner,omitempty"` } +// CapacityReservationSelectorTerm defines selection logic for a Capacity Reservation used by Karpenter to launch nodes. +// If multiple fields are used for selection, the requirements are ANDed. +type CapacityReservationSelectorTerm struct { + // The Availability Zone of the Capacity Reservation + // +optional + AvailabilityZone string `json:"availabilityZone,omitempty"` + // The platform of operating system for which the Capacity Reservation reserves capacity + // +optional + ID string `json:"id,omitempty"` + // Tags is a map of key/value tags used to select subnets + // Specifying '*' for a value selects all values for a given tag key. + // +kubebuilder:validation:XValidation:message="empty tag keys or values aren't supported",rule="self.all(k, k != '' && self[k] != '')" + // +kubebuilder:validation:MaxProperties:=20 + // +optional + Tags map[string]string `json:"tags,omitempty"` + // ID is the Capacity Reservation id in EC2 + // +kubebuilder:validation:Pattern:="cr-[0-9a-z]+" + // +optional + Tenancy string `json:"tenancy,omitempty"` + // Indicates the type of instance launches that the Capacity Reservation accepts. The options include 'open' and 'targeted'. + // open - The Capacity Reservation accepts all instances that have + // matching attributes (instance type, platform, and Availability + // Zone). Instances that have matching attributes launch into the + // Capacity Reservation automatically without specifying any + // additional parameters. + // targeted - The Capacity Reservation only accepts instances that + // have matching attributes (instance type, platform, and + // Availability Zone), and explicitly target the Capacity + // Reservation. This ensures that only permitted instances can use + // the reserved capacity. + // +optional + InstanceMatchCriteria string `json:"instanceMatchCriteria,omitempty"` + // Indicates the tenancy of the Capacity Reservation. + // A Capacity Reservation can have one of the following tenancy 'default' or 'dedicated' + // default - The Capacity Reservation is created on hardware that is shared with other Amazon Web Services accounts. + // dedicated - The Capacity Reservation is created on single-tenant hardware that is dedicated to a single Amazon Web Services account. + // +optional + InstancePlatform string `json:"instancePlatform,omitempty"` + // The type of operating system for which the Capacity Reservation reserves capacity + // +optional + InstanceType string `json:"instanceType,omitempty"` + // The ID of the Amazon Web Services account that owns the Capacity Reservation + // +optional + OwnerId string `json:"ownerId,omitempty"` +} + // MetadataOptions contains parameters for specifying the exposure of the // Instance Metadata Service to provisioned EC2 nodes. type MetadataOptions struct { diff --git a/pkg/apis/v1beta1/ec2nodeclass_status.go b/pkg/apis/v1beta1/ec2nodeclass_status.go index 611e94d62117..2af9766a8d21 100644 --- a/pkg/apis/v1beta1/ec2nodeclass_status.go +++ b/pkg/apis/v1beta1/ec2nodeclass_status.go @@ -38,6 +38,28 @@ type SecurityGroup struct { Name string `json:"name,omitempty"` } +// CapacityReservation contains resolved Capacity Reservation selector values utilized for node launch +type CapacityReservation struct { + // ID of the Capacity Reservation + // +required + ID string `json:"id"` + // AvailabilityZone of the Capacity Reservation + // +required + AvailabilityZone string `json:"availabilityZone"` + // Available Instance Count of the Capacity Reservation + // +required + AvailableInstanceCount int `json:"availableInstanceCount"` + // InstanceType of the Capacity Reservation + // +required + InstanceType string `json:"instanceType"` + // Requirements of the Capacity Reservation to be utilized on an instance type + // +required + Requirements []corev1beta1.NodeSelectorRequirementWithMinValues `json:"requirements"` + // Total Instance Count of the Capacity Reservation + // +required + TotalInstanceCount int `json:"totalInstanceCount"` +} + // AMI contains resolved AMI selector values utilized for node launch type AMI struct { // ID of the AMI @@ -53,6 +75,10 @@ type AMI struct { // EC2NodeClassStatus contains the resolved state of the EC2NodeClass type EC2NodeClassStatus struct { + // CapacityReservations contains the current Capacity Reservations values that are available to the + // cluster under the CapacityReservations selectors. + // +optional + CapacityReservations []CapacityReservation `json:"capacityReservations,omitempty"` // Subnets contains the current Subnet values that are available to the // cluster under the subnet selectors. // +optional diff --git a/pkg/apis/v1beta1/zz_generated.deepcopy.go b/pkg/apis/v1beta1/zz_generated.deepcopy.go index 781d88c876c8..6901f4019f07 100644 --- a/pkg/apis/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/v1beta1/zz_generated.deepcopy.go @@ -147,6 +147,50 @@ func (in *BlockDeviceMapping) DeepCopy() *BlockDeviceMapping { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CapacityReservation) DeepCopyInto(out *CapacityReservation) { + *out = *in + if in.Requirements != nil { + in, out := &in.Requirements, &out.Requirements + *out = make([]apisv1beta1.NodeSelectorRequirementWithMinValues, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CapacityReservation. +func (in *CapacityReservation) DeepCopy() *CapacityReservation { + if in == nil { + return nil + } + out := new(CapacityReservation) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CapacityReservationSelectorTerm) DeepCopyInto(out *CapacityReservationSelectorTerm) { + *out = *in + if in.Tags != nil { + in, out := &in.Tags, &out.Tags + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CapacityReservationSelectorTerm. +func (in *CapacityReservationSelectorTerm) DeepCopy() *CapacityReservationSelectorTerm { + if in == nil { + return nil + } + out := new(CapacityReservationSelectorTerm) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EC2NodeClass) DeepCopyInto(out *EC2NodeClass) { *out = *in @@ -240,6 +284,13 @@ func (in *EC2NodeClassSpec) DeepCopyInto(out *EC2NodeClassSpec) { *out = new(string) **out = **in } + if in.CapacityReservationSelectorTerms != nil { + in, out := &in.CapacityReservationSelectorTerms, &out.CapacityReservationSelectorTerms + *out = make([]CapacityReservationSelectorTerm, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.UserData != nil { in, out := &in.UserData, &out.UserData *out = new(string) @@ -303,6 +354,13 @@ func (in *EC2NodeClassSpec) DeepCopy() *EC2NodeClassSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EC2NodeClassStatus) DeepCopyInto(out *EC2NodeClassStatus) { *out = *in + if in.CapacityReservations != nil { + in, out := &in.CapacityReservations, &out.CapacityReservations + *out = make([]CapacityReservation, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.Subnets != nil { in, out := &in.Subnets, &out.Subnets *out = make([]Subnet, len(*in)) diff --git a/pkg/cloudprovider/cloudprovider.go b/pkg/cloudprovider/cloudprovider.go index acea3187e4d1..35923589d55c 100644 --- a/pkg/cloudprovider/cloudprovider.go +++ b/pkg/cloudprovider/cloudprovider.go @@ -96,6 +96,9 @@ func (c *CloudProvider) Create(ctx context.Context, nodeClaim *corev1beta1.NodeC } instance, err := c.instanceProvider.Create(ctx, nodeClass, nodeClaim, instanceTypes) if err != nil { + if cloudprovider.IsInsufficientCapacityError(err) { + return nil, cloudprovider.NewInsufficientCapacityError(fmt.Errorf("creating instance, %w", err)) + } return nil, fmt.Errorf("creating instance, %w", err) } instanceType, _ := lo.Find(instanceTypes, func(i *cloudprovider.InstanceType) bool { diff --git a/pkg/controllers/controllers.go b/pkg/controllers/controllers.go index 6510a522f870..ab23945200a3 100644 --- a/pkg/controllers/controllers.go +++ b/pkg/controllers/controllers.go @@ -24,6 +24,7 @@ import ( nodeclasstermination "github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclass/termination" controllersinstancetype "github.com/aws/karpenter-provider-aws/pkg/controllers/providers/instancetype" controllerspricing "github.com/aws/karpenter-provider-aws/pkg/controllers/providers/pricing" + "github.com/aws/karpenter-provider-aws/pkg/providers/capacityreservation" "github.com/aws/karpenter-provider-aws/pkg/providers/launchtemplate" "github.com/aws/aws-sdk-go/aws/session" @@ -52,12 +53,12 @@ import ( func NewControllers(ctx context.Context, sess *session.Session, clk clock.Clock, kubeClient client.Client, recorder events.Recorder, unavailableOfferings *cache.UnavailableOfferings, cloudProvider cloudprovider.CloudProvider, subnetProvider subnet.Provider, - securityGroupProvider securitygroup.Provider, instanceProfileProvider instanceprofile.Provider, instanceProvider instance.Provider, + securityGroupProvider securitygroup.Provider, capacityReservationProvider capacityreservation.Provider, instanceProfileProvider instanceprofile.Provider, instanceProvider instance.Provider, pricingProvider pricing.Provider, amiProvider amifamily.Provider, launchTemplateProvider launchtemplate.Provider, instanceTypeProvider instancetype.Provider) []controller.Controller { controllers := []controller.Controller{ nodeclasshash.NewController(kubeClient), - nodeclassstatus.NewController(kubeClient, subnetProvider, securityGroupProvider, amiProvider, instanceProfileProvider, launchTemplateProvider), + nodeclassstatus.NewController(kubeClient, subnetProvider, securityGroupProvider, capacityReservationProvider, amiProvider, instanceProfileProvider, launchTemplateProvider), nodeclasstermination.NewController(kubeClient, recorder, instanceProfileProvider, launchTemplateProvider), nodeclaimgarbagecollection.NewController(kubeClient, cloudProvider), nodeclaimtagging.NewController(kubeClient, instanceProvider), diff --git a/pkg/controllers/nodeclass/status/controller.go b/pkg/controllers/nodeclass/status/controller.go index da4aeca04dc0..37b7598322ae 100644 --- a/pkg/controllers/nodeclass/status/controller.go +++ b/pkg/controllers/nodeclass/status/controller.go @@ -34,6 +34,7 @@ import ( "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily" + "github.com/aws/karpenter-provider-aws/pkg/providers/capacityreservation" "github.com/aws/karpenter-provider-aws/pkg/providers/instanceprofile" "github.com/aws/karpenter-provider-aws/pkg/providers/launchtemplate" "github.com/aws/karpenter-provider-aws/pkg/providers/securitygroup" @@ -49,23 +50,25 @@ type nodeClassStatusReconciler interface { type Controller struct { kubeClient client.Client - ami *AMI - instanceprofile *InstanceProfile - subnet *Subnet - securitygroup *SecurityGroup - launchtemplate *LaunchTemplate + ami *AMI + instanceprofile *InstanceProfile + subnet *Subnet + securitygroup *SecurityGroup + capacityreservation *CapacityReservation + launchtemplate *LaunchTemplate } -func NewController(kubeClient client.Client, subnetProvider subnet.Provider, securityGroupProvider securitygroup.Provider, +func NewController(kubeClient client.Client, subnetProvider subnet.Provider, securityGroupProvider securitygroup.Provider, capacityReservationProvider capacityreservation.Provider, amiProvider amifamily.Provider, instanceProfileProvider instanceprofile.Provider, launchTemplateProvider launchtemplate.Provider) corecontroller.Controller { return corecontroller.Typed[*v1beta1.EC2NodeClass](kubeClient, &Controller{ kubeClient: kubeClient, - ami: &AMI{amiProvider: amiProvider}, - subnet: &Subnet{subnetProvider: subnetProvider}, - securitygroup: &SecurityGroup{securityGroupProvider: securityGroupProvider}, - instanceprofile: &InstanceProfile{instanceProfileProvider: instanceProfileProvider}, - launchtemplate: &LaunchTemplate{launchTemplateProvider: launchTemplateProvider}, + ami: &AMI{amiProvider: amiProvider}, + subnet: &Subnet{subnetProvider: subnetProvider}, + securitygroup: &SecurityGroup{securityGroupProvider: securityGroupProvider}, + capacityreservation: &CapacityReservation{capacityReservationProvider: capacityReservationProvider}, + instanceprofile: &InstanceProfile{instanceProfileProvider: instanceProfileProvider}, + launchtemplate: &LaunchTemplate{launchTemplateProvider: launchTemplateProvider}, }) } @@ -87,6 +90,7 @@ func (c *Controller) Reconcile(ctx context.Context, nodeClass *v1beta1.EC2NodeCl c.securitygroup, c.instanceprofile, c.launchtemplate, + c.capacityreservation, } { res, err := reconciler.Reconcile(ctx, nodeClass) errs = multierr.Append(errs, err) diff --git a/pkg/controllers/nodeclass/status/suite_test.go b/pkg/controllers/nodeclass/status/suite_test.go index 545c9ba6cfd5..ac6e75848639 100644 --- a/pkg/controllers/nodeclass/status/suite_test.go +++ b/pkg/controllers/nodeclass/status/suite_test.go @@ -59,6 +59,7 @@ var _ = BeforeSuite(func() { env.Client, awsEnv.SubnetProvider, awsEnv.SecurityGroupProvider, + awsEnv.CapacityReservationProvider, awsEnv.AMIProvider, awsEnv.InstanceProfileProvider, awsEnv.LaunchTemplateProvider, diff --git a/pkg/errors/errors.go b/pkg/errors/errors.go index 5267aad4672d..26de0dbb4837 100644 --- a/pkg/errors/errors.go +++ b/pkg/errors/errors.go @@ -48,6 +48,7 @@ var ( "UnfulfillableCapacity", "Unsupported", "InsufficientFreeAddressesInSubnet", + "ReservationCapacityExceeded", ) ) diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 113b863f9732..12ed5c5f06db 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -54,6 +54,7 @@ import ( awscache "github.com/aws/karpenter-provider-aws/pkg/cache" "github.com/aws/karpenter-provider-aws/pkg/operator/options" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily" + "github.com/aws/karpenter-provider-aws/pkg/providers/capacityreservation" "github.com/aws/karpenter-provider-aws/pkg/providers/instance" "github.com/aws/karpenter-provider-aws/pkg/providers/instanceprofile" "github.com/aws/karpenter-provider-aws/pkg/providers/instancetype" @@ -73,19 +74,20 @@ func init() { type Operator struct { *operator.Operator - Session *session.Session - UnavailableOfferingsCache *awscache.UnavailableOfferings - EC2API ec2iface.EC2API - SubnetProvider subnet.Provider - SecurityGroupProvider securitygroup.Provider - InstanceProfileProvider instanceprofile.Provider - AMIProvider amifamily.Provider - AMIResolver *amifamily.Resolver - LaunchTemplateProvider launchtemplate.Provider - PricingProvider pricing.Provider - VersionProvider version.Provider - InstanceTypesProvider instancetype.Provider - InstanceProvider instance.Provider + Session *session.Session + UnavailableOfferingsCache *awscache.UnavailableOfferings + EC2API ec2iface.EC2API + SubnetProvider subnet.Provider + SecurityGroupProvider securitygroup.Provider + InstanceProfileProvider instanceprofile.Provider + AMIProvider amifamily.Provider + AMIResolver *amifamily.Resolver + LaunchTemplateProvider launchtemplate.Provider + PricingProvider pricing.Provider + VersionProvider version.Provider + InstanceTypesProvider instancetype.Provider + InstanceProvider instance.Provider + CapacityReservationProvider capacityreservation.Provider } func NewOperator(ctx context.Context, operator *operator.Operator) (context.Context, *Operator) { @@ -174,22 +176,24 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont subnetProvider, launchTemplateProvider, ) + capacityReservationProvider := capacityreservation.NewDefaultProvider(ec2api, cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval)) return ctx, &Operator{ - Operator: operator, - Session: sess, - UnavailableOfferingsCache: unavailableOfferingsCache, - EC2API: ec2api, - SubnetProvider: subnetProvider, - SecurityGroupProvider: securityGroupProvider, - InstanceProfileProvider: instanceProfileProvider, - AMIProvider: amiProvider, - AMIResolver: amiResolver, - VersionProvider: versionProvider, - LaunchTemplateProvider: launchTemplateProvider, - PricingProvider: pricingProvider, - InstanceTypesProvider: instanceTypeProvider, - InstanceProvider: instanceProvider, + Operator: operator, + Session: sess, + UnavailableOfferingsCache: unavailableOfferingsCache, + EC2API: ec2api, + SubnetProvider: subnetProvider, + SecurityGroupProvider: securityGroupProvider, + InstanceProfileProvider: instanceProfileProvider, + AMIProvider: amiProvider, + AMIResolver: amiResolver, + VersionProvider: versionProvider, + LaunchTemplateProvider: launchTemplateProvider, + PricingProvider: pricingProvider, + InstanceTypesProvider: instanceTypeProvider, + InstanceProvider: instanceProvider, + CapacityReservationProvider: capacityReservationProvider, } } diff --git a/pkg/providers/amifamily/resolver.go b/pkg/providers/amifamily/resolver.go index 65adec3a9abb..65e1ed9a7f20 100644 --- a/pkg/providers/amifamily/resolver.go +++ b/pkg/providers/amifamily/resolver.go @@ -24,6 +24,7 @@ import ( "github.com/imdario/mergo" "github.com/samber/lo" core "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" @@ -56,6 +57,7 @@ type Options struct { InstanceStorePolicy *v1beta1.InstanceStorePolicy // Level-triggered fields that may change out of sync. SecurityGroups []v1beta1.SecurityGroup + CapacityReservations []v1beta1.CapacityReservation Tags map[string]string Labels map[string]string `hash:"ignore"` KubeDNSIP net.IP @@ -67,6 +69,7 @@ type Options struct { type LaunchTemplate struct { *Options UserData bootstrap.Bootstrapper + CapacityReservation *v1beta1.CapacityReservation BlockDeviceMappings []*v1beta1.BlockDeviceMapping MetadataOptions *v1beta1.MetadataOptions AMIID string @@ -154,12 +157,63 @@ func (r Resolver) Resolve(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, maxPods: int(instanceType.Capacity.Pods().Value()), } }) + + zones := scheduling.NewNodeSelectorRequirementsWithMinValues(nodeClaim.Spec.Requirements...).Get(v1.LabelTopologyZone) + capacityReservations := []v1beta1.CapacityReservation{} + if capacityType == "capacity-reservation" { + for _, capacityReservation := range nodeClass.Status.CapacityReservations { + if capacityReservation.AvailableInstanceCount == 0 { + continue + } + if !zones.Has(capacityReservation.AvailabilityZone) { + continue + } + capacityReservations = append(capacityReservations, capacityReservation) + } + if len(capacityReservations) == 0 { + return nil, cloudprovider.NewInsufficientCapacityError(fmt.Errorf("trying to resolve capacity-reservation but no available capacity reservations available")) + } + } + for params, instanceTypes := range paramsToInstanceTypes { - resolved, err := r.resolveLaunchTemplate(nodeClass, nodeClaim, instanceTypes, capacityType, amiFamily, amiID, params.maxPods, params.efaCount, options) - if err != nil { - return nil, err + + if len(capacityReservations) > 0 { + for _, capacityReservation := range capacityReservations { + resolved, err := r.resolveLaunchTemplate( + nodeClass, + nodeClaim, + &capacityReservation, + instanceTypes, + "on-demand", // capacity-type + amiFamily, + amiID, + params.maxPods, + params.efaCount, + options, + ) + if err != nil { + return nil, err + } + resolvedTemplates = append(resolvedTemplates, resolved) + } + } else { + resolved, err := r.resolveLaunchTemplate( + nodeClass, + nodeClaim, + nil, + instanceTypes, + capacityType, + amiFamily, + amiID, + params.maxPods, + params.efaCount, + options, + ) + if err != nil { + return nil, err + } + resolvedTemplates = append(resolvedTemplates, resolved) } - resolvedTemplates = append(resolvedTemplates, resolved) } } return resolvedTemplates, nil @@ -210,8 +264,18 @@ func (r Resolver) defaultClusterDNS(opts *Options, kubeletConfig *corev1beta1.Ku return newKubeletConfig } -func (r Resolver) resolveLaunchTemplate(nodeClass *v1beta1.EC2NodeClass, nodeClaim *corev1beta1.NodeClaim, instanceTypes []*cloudprovider.InstanceType, capacityType string, - amiFamily AMIFamily, amiID string, maxPods int, efaCount int, options *Options) (*LaunchTemplate, error) { +func (r Resolver) resolveLaunchTemplate( + nodeClass *v1beta1.EC2NodeClass, + nodeClaim *corev1beta1.NodeClaim, + capacityReservation *v1beta1.CapacityReservation, + instanceTypes []*cloudprovider.InstanceType, + capacityType string, + amiFamily AMIFamily, + amiID string, + maxPods int, + efaCount int, + options *Options, +) (*LaunchTemplate, error) { kubeletConfig := &corev1beta1.KubeletConfiguration{} if nodeClaim.Spec.Kubelet != nil { if err := mergo.Merge(kubeletConfig, nodeClaim.Spec.Kubelet); err != nil { @@ -232,6 +296,7 @@ func (r Resolver) resolveLaunchTemplate(nodeClass *v1beta1.EC2NodeClass, nodeCla nodeClass.Spec.UserData, options.InstanceStorePolicy, ), + CapacityReservation: capacityReservation, BlockDeviceMappings: nodeClass.Spec.BlockDeviceMappings, MetadataOptions: nodeClass.Spec.MetadataOptions, DetailedMonitoring: aws.BoolValue(nodeClass.Spec.DetailedMonitoring), diff --git a/pkg/providers/instance/instance.go b/pkg/providers/instance/instance.go index f228815563a7..7aaab3952704 100644 --- a/pkg/providers/instance/instance.go +++ b/pkg/providers/instance/instance.go @@ -96,17 +96,17 @@ func (p *DefaultProvider) Create(ctx context.Context, nodeClass *v1beta1.EC2Node instanceTypes = p.filterInstanceTypes(nodeClaim, instanceTypes) } tags := getTags(ctx, nodeClass, nodeClaim) - fleetInstance, err := p.launchInstance(ctx, nodeClass, nodeClaim, instanceTypes, tags) + fleetInstance, capacityType, err := p.launchInstance(ctx, nodeClass, nodeClaim, instanceTypes, tags) if awserrors.IsLaunchTemplateNotFound(err) { // retry once if launch template is not found. This allows karpenter to generate a new LT if the // cache was out-of-sync on the first try - fleetInstance, err = p.launchInstance(ctx, nodeClass, nodeClaim, instanceTypes, tags) + fleetInstance, capacityType, err = p.launchInstance(ctx, nodeClass, nodeClaim, instanceTypes, tags) } if err != nil { return nil, err } efaEnabled := lo.Contains(lo.Keys(nodeClaim.Spec.Resources.Requests), v1beta1.ResourceEFA) - return NewInstanceFromFleet(fleetInstance, tags, efaEnabled), nil + return NewInstanceFromFleet(fleetInstance, tags, efaEnabled, capacityType), nil } func (p *DefaultProvider) Get(ctx context.Context, id string) (*Instance, error) { @@ -193,17 +193,22 @@ func (p *DefaultProvider) CreateTags(ctx context.Context, id string, tags map[st return nil } -func (p *DefaultProvider) launchInstance(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, nodeClaim *corev1beta1.NodeClaim, instanceTypes []*cloudprovider.InstanceType, tags map[string]string) (*ec2.CreateFleetInstance, error) { +// workaround until capacity-reservation natively supported by EC2 API to return capacity type +func (p *DefaultProvider) launchInstance(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, nodeClaim *corev1beta1.NodeClaim, instanceTypes []*cloudprovider.InstanceType, tags map[string]string) (*ec2.CreateFleetInstance, string, error) { capacityType := p.getCapacityType(nodeClaim, instanceTypes) + zonalSubnets, err := p.subnetProvider.ZonalSubnetsForLaunch(ctx, nodeClass, instanceTypes, capacityType) if err != nil { - return nil, fmt.Errorf("getting subnets, %w", err) + return nil, "", fmt.Errorf("getting subnets, %w", err) } // Get Launch Template Configs, which may differ due to GPU or Architecture requirements launchTemplateConfigs, err := p.getLaunchTemplateConfigs(ctx, nodeClass, nodeClaim, instanceTypes, zonalSubnets, capacityType, tags) if err != nil { - return nil, fmt.Errorf("getting launch template configs, %w", err) + if cloudprovider.IsInsufficientCapacityError(err) { + return nil, "", cloudprovider.NewInsufficientCapacityError(fmt.Errorf("getting launch template configs, %w", err)) + } + return nil, "", fmt.Errorf("getting launch template configs, %w", err) } if err := p.checkODFallback(nodeClaim, instanceTypes, launchTemplateConfigs); err != nil { logging.FromContext(ctx).Warn(err.Error()) @@ -214,7 +219,8 @@ func (p *DefaultProvider) launchInstance(ctx context.Context, nodeClass *v1beta1 Context: nodeClass.Spec.Context, LaunchTemplateConfigs: launchTemplateConfigs, TargetCapacitySpecification: &ec2.TargetCapacitySpecificationRequest{ - DefaultTargetCapacityType: aws.String(capacityType), + // workaround until capacity-reservation natively supported by EC2 API + DefaultTargetCapacityType: lo.Ternary(capacityType == "capacity-reservation", aws.String("on-demand"), aws.String(capacityType)), TotalTargetCapacity: aws.Int64(1), }, TagSpecifications: []*ec2.TagSpecification{ @@ -226,6 +232,7 @@ func (p *DefaultProvider) launchInstance(ctx context.Context, nodeClass *v1beta1 if capacityType == corev1beta1.CapacityTypeSpot { createFleetInput.SpotOptions = &ec2.SpotOptionsRequest{AllocationStrategy: aws.String(ec2.SpotAllocationStrategyPriceCapacityOptimized)} } else { + // will handle capacity-reservation and on-demand createFleetInput.OnDemandOptions = &ec2.OnDemandOptionsRequest{AllocationStrategy: aws.String(ec2.FleetOnDemandAllocationStrategyLowestPrice)} } @@ -236,19 +243,19 @@ func (p *DefaultProvider) launchInstance(ctx context.Context, nodeClass *v1beta1 for _, lt := range launchTemplateConfigs { p.launchTemplateProvider.InvalidateCache(ctx, aws.StringValue(lt.LaunchTemplateSpecification.LaunchTemplateName), aws.StringValue(lt.LaunchTemplateSpecification.LaunchTemplateId)) } - return nil, fmt.Errorf("creating fleet %w", err) + return nil, "", fmt.Errorf("creating fleet %w", err) } var reqFailure awserr.RequestFailure if errors.As(err, &reqFailure) { - return nil, fmt.Errorf("creating fleet %w (%s)", err, reqFailure.RequestID()) + return nil, "", fmt.Errorf("creating fleet %w (%s)", err, reqFailure.RequestID()) } - return nil, fmt.Errorf("creating fleet %w", err) + return nil, "", fmt.Errorf("creating fleet %w", err) } p.updateUnavailableOfferingsCache(ctx, createFleetOutput.Errors, capacityType) if len(createFleetOutput.Instances) == 0 || len(createFleetOutput.Instances[0].InstanceIds) == 0 { - return nil, combineFleetErrors(createFleetOutput.Errors) + return nil, "", combineFleetErrors(createFleetOutput.Errors) } - return createFleetOutput.Instances[0], nil + return createFleetOutput.Instances[0], capacityType, nil } func getTags(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, nodeClaim *corev1beta1.NodeClaim) map[string]string { @@ -289,6 +296,9 @@ func (p *DefaultProvider) getLaunchTemplateConfigs(ctx context.Context, nodeClas var launchTemplateConfigs []*ec2.FleetLaunchTemplateConfigRequest launchTemplates, err := p.launchTemplateProvider.EnsureAll(ctx, nodeClass, nodeClaim, instanceTypes, capacityType, tags) if err != nil { + if cloudprovider.IsInsufficientCapacityError(err) { + return nil, cloudprovider.NewInsufficientCapacityError(fmt.Errorf("getting launch templates, %w", err)) + } return nil, fmt.Errorf("getting launch templates, %w", err) } for _, launchTemplate := range launchTemplates { @@ -361,12 +371,20 @@ func (p *DefaultProvider) updateUnavailableOfferingsCache(ctx context.Context, e } } -// getCapacityType selects spot if both constraints are flexible and there is an -// available offering. The AWS Cloud Provider defaults to [ on-demand ], so spot +// getCapacityType selects capacity-reservation or spot if both constraints are flexible and there is an +// available offering. The AWS Cloud Provider defaults to [ on-demand ], so capacity-reservation or spot // must be explicitly included in capacity type requirements. func (p *DefaultProvider) getCapacityType(nodeClaim *corev1beta1.NodeClaim, instanceTypes []*cloudprovider.InstanceType) string { - requirements := scheduling.NewNodeSelectorRequirementsWithMinValues(nodeClaim. - Spec.Requirements...) + requirements := scheduling.NewNodeSelectorRequirementsWithMinValues(nodeClaim.Spec.Requirements...) + if requirements.Get(corev1beta1.CapacityTypeLabelKey).Has("capacity-reservation") { + for _, instanceType := range instanceTypes { + for _, offering := range instanceType.Offerings.Available() { + if requirements.Get(v1.LabelTopologyZone).Has(offering.Zone) && offering.CapacityType == "capacity-reservation" { + return "capacity-reservation" + } + } + } + } if requirements.Get(corev1beta1.CapacityTypeLabelKey).Has(corev1beta1.CapacityTypeSpot) { for _, instanceType := range instanceTypes { for _, offering := range instanceType.Offerings.Available() { diff --git a/pkg/providers/instance/types.go b/pkg/providers/instance/types.go index 5f3804f2d004..6cb86273e477 100644 --- a/pkg/providers/instance/types.go +++ b/pkg/providers/instance/types.go @@ -61,7 +61,8 @@ func NewInstance(out *ec2.Instance) *Instance { } -func NewInstanceFromFleet(out *ec2.CreateFleetInstance, tags map[string]string, efaEnabled bool) *Instance { +// workaround until capacity-reservation natively supported by EC2 API to return capacity type in out.Lifecycle +func NewInstanceFromFleet(out *ec2.CreateFleetInstance, tags map[string]string, efaEnabled bool, capacityType string) *Instance { return &Instance{ LaunchTime: time.Now(), // estimate the launch time since we just launched State: ec2.StatePending, @@ -69,7 +70,7 @@ func NewInstanceFromFleet(out *ec2.CreateFleetInstance, tags map[string]string, ImageID: aws.StringValue(out.LaunchTemplateAndOverrides.Overrides.ImageId), Type: aws.StringValue(out.InstanceType), Zone: aws.StringValue(out.LaunchTemplateAndOverrides.Overrides.AvailabilityZone), - CapacityType: aws.StringValue(out.Lifecycle), + CapacityType: capacityType, SubnetID: aws.StringValue(out.LaunchTemplateAndOverrides.Overrides.SubnetId), Tags: tags, EFAEnabled: efaEnabled, diff --git a/pkg/providers/instancetype/instancetype.go b/pkg/providers/instancetype/instancetype.go index 0d86dd941e8a..178859853101 100644 --- a/pkg/providers/instancetype/instancetype.go +++ b/pkg/providers/instancetype/instancetype.go @@ -164,10 +164,28 @@ func (p *DefaultProvider) List(ctx context.Context, kc *corev1beta1.KubeletConfi // Any changes to the values passed into the NewInstanceType method will require making updates to the cache key // so that Karpenter is able to cache the set of InstanceTypes based on values that alter the set of instance types // !!! Important !!! - return NewInstanceType(ctx, i, p.region, - nodeClass.Spec.BlockDeviceMappings, nodeClass.Spec.InstanceStorePolicy, - kc.MaxPods, kc.PodsPerCore, kc.KubeReserved, kc.SystemReserved, kc.EvictionHard, kc.EvictionSoft, - amiFamily, p.createOfferings(ctx, i, p.instanceTypeOfferings[aws.StringValue(i.InstanceType)], allZones, subnetZones)) + return NewInstanceType( + ctx, + i, + p.region, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + kc.MaxPods, + kc.PodsPerCore, + kc.KubeReserved, + kc.SystemReserved, + kc.EvictionHard, + kc.EvictionSoft, + amiFamily, + p.createOfferings( + ctx, + i, + p.instanceTypeOfferings[aws.StringValue(i.InstanceType)], + allZones, + subnetZones, + nodeClass.Status.CapacityReservations, + ), + ) }) p.instanceTypesCache.SetDefault(key, result) return result, nil @@ -251,11 +269,22 @@ func (p *DefaultProvider) UpdateInstanceTypeOfferings(ctx context.Context) error return nil } -func (p *DefaultProvider) createOfferings(ctx context.Context, instanceType *ec2.InstanceTypeInfo, instanceTypeZones, zones, subnetZones sets.Set[string]) []cloudprovider.Offering { +func (p *DefaultProvider) createOfferings( + ctx context.Context, + instanceType *ec2.InstanceTypeInfo, + instanceTypeZones, + zones, + subnetZones sets.Set[string], + capacityReservations []v1beta1.CapacityReservation, +) []cloudprovider.Offering { var offerings []cloudprovider.Offering + + // workaround until ec2 supports "capacity-reservation" as a supported class natively + supportedUsageClasses := sets.NewString(append(aws.StringValueSlice(instanceType.SupportedUsageClasses), "capacity-reservation")...) + for zone := range zones { // while usage classes should be a distinct set, there's no guarantee of that - for capacityType := range sets.NewString(aws.StringValueSlice(instanceType.SupportedUsageClasses)...) { + for capacityType := range supportedUsageClasses { // exclude any offerings that have recently seen an insufficient capacity error from EC2 isUnavailable := p.unavailableOfferings.IsUnavailable(*instanceType.InstanceType, zone, capacityType) var price float64 @@ -265,6 +294,14 @@ func (p *DefaultProvider) createOfferings(ctx context.Context, instanceType *ec2 price, ok = p.pricingProvider.SpotPrice(*instanceType.InstanceType, zone) case ec2.UsageClassTypeOnDemand: price, ok = p.pricingProvider.OnDemandPrice(*instanceType.InstanceType) + case "capacity-reservation": + // logging.FromContext(ctx).Debugf("Creating offering for capacity reservation instance type %s and zone %s", *instanceType.InstanceType, zone) + price = 0.0 + ok = hasCapacityReservation( + capacityReservations, + instanceType, + zone, + ) case "capacity-block": // ignore since karpenter doesn't support it yet, but do not log an unknown capacity type error continue @@ -279,6 +316,7 @@ func (p *DefaultProvider) createOfferings(ctx context.Context, instanceType *ec2 Price: price, Available: available, }) + // TODO: Do we want to set this for capacityType capacity-reservation? instanceTypeOfferingAvailable.With(prometheus.Labels{ instanceTypeLabel: *instanceType.InstanceType, capacityTypeLabel: capacityType, @@ -294,6 +332,29 @@ func (p *DefaultProvider) createOfferings(ctx context.Context, instanceType *ec2 return offerings } +func hasCapacityReservation( + capacityReservations []v1beta1.CapacityReservation, + instanceType *ec2.InstanceTypeInfo, + zone string, +) bool { + for _, capacityReservation := range capacityReservations { + if capacityReservation.AvailableInstanceCount == 0 { + continue + } + + if capacityReservation.AvailabilityZone != zone { + continue + } + + if capacityReservation.InstanceType != aws.StringValue(instanceType.InstanceType) { + continue + } + + return true + } + return false +} + func (p *DefaultProvider) Reset() { p.instanceTypesInfo = []*ec2.InstanceTypeInfo{} p.instanceTypeOfferings = map[string]sets.Set[string]{} diff --git a/pkg/providers/launchtemplate/launchtemplate.go b/pkg/providers/launchtemplate/launchtemplate.go index b90949530f6f..1f2dc7a55ec9 100644 --- a/pkg/providers/launchtemplate/launchtemplate.go +++ b/pkg/providers/launchtemplate/launchtemplate.go @@ -174,17 +174,18 @@ func (p *DefaultProvider) createAMIOptions(ctx context.Context, nodeClass *v1bet return nil, fmt.Errorf("no security groups are present in the status") } options := &amifamily.Options{ - ClusterName: options.FromContext(ctx).ClusterName, - ClusterEndpoint: p.ClusterEndpoint, - ClusterCIDR: p.ClusterCIDR.Load(), - InstanceProfile: instanceProfile, - InstanceStorePolicy: nodeClass.Spec.InstanceStorePolicy, - SecurityGroups: nodeClass.Status.SecurityGroups, - Tags: tags, - Labels: labels, - CABundle: p.CABundle, - KubeDNSIP: p.KubeDNSIP, - NodeClassName: nodeClass.Name, + ClusterName: options.FromContext(ctx).ClusterName, + ClusterEndpoint: p.ClusterEndpoint, + ClusterCIDR: p.ClusterCIDR.Load(), + InstanceProfile: instanceProfile, + InstanceStorePolicy: nodeClass.Spec.InstanceStorePolicy, + SecurityGroups: nodeClass.Status.SecurityGroups, + CapacityReservations: nodeClass.Status.CapacityReservations, + Tags: tags, + Labels: labels, + CABundle: p.CABundle, + KubeDNSIP: p.KubeDNSIP, + NodeClassName: nodeClass.Name, } if nodeClass.Spec.AssociatePublicIPAddress != nil { options.AssociatePublicIPAddress = nodeClass.Spec.AssociatePublicIPAddress @@ -247,6 +248,7 @@ func (p *DefaultProvider) createLaunchTemplate(ctx context.Context, options *ami launchTemplateDataTags = append(launchTemplateDataTags, &ec2.LaunchTemplateTagSpecificationRequest{ResourceType: aws.String(ec2.ResourceTypeSpotInstancesRequest), Tags: utils.MergeTags(options.Tags)}) } networkInterfaces := p.generateNetworkInterfaces(options) + capacityReservationSpecification := p.generateCapacityReservationSpecification(options) output, err := p.ec2api.CreateLaunchTemplateWithContext(ctx, &ec2.CreateLaunchTemplateInput{ LaunchTemplateName: aws.String(LaunchTemplateName(options)), LaunchTemplateData: &ec2.RequestLaunchTemplateData{ @@ -258,9 +260,10 @@ func (p *DefaultProvider) createLaunchTemplate(ctx context.Context, options *ami Enabled: aws.Bool(options.DetailedMonitoring), }, // If the network interface is defined, the security groups are defined within it - SecurityGroupIds: lo.Ternary(networkInterfaces != nil, nil, lo.Map(options.SecurityGroups, func(s v1beta1.SecurityGroup, _ int) *string { return aws.String(s.ID) })), - UserData: aws.String(userData), - ImageId: aws.String(options.AMIID), + SecurityGroupIds: lo.Ternary(networkInterfaces != nil, nil, lo.Map(options.SecurityGroups, func(s v1beta1.SecurityGroup, _ int) *string { return aws.String(s.ID) })), + CapacityReservationSpecification: capacityReservationSpecification, + UserData: aws.String(userData), + ImageId: aws.String(options.AMIID), MetadataOptions: &ec2.LaunchTemplateInstanceMetadataOptionsRequest{ HttpEndpoint: options.MetadataOptions.HTTPEndpoint, HttpProtocolIpv6: options.MetadataOptions.HTTPProtocolIPv6, @@ -313,6 +316,22 @@ func (p *DefaultProvider) generateNetworkInterfaces(options *amifamily.LaunchTem return nil } +func (p *DefaultProvider) generateCapacityReservationSpecification(options *amifamily.LaunchTemplate) *ec2.LaunchTemplateCapacityReservationSpecificationRequest { + if options == nil { + return nil + } + + if options.CapacityReservation == nil { + return nil + } + + return &ec2.LaunchTemplateCapacityReservationSpecificationRequest{ + CapacityReservationTarget: &ec2.CapacityReservationTarget{ + CapacityReservationId: &options.CapacityReservation.ID, + }, + } +} + func (p *DefaultProvider) blockDeviceMappings(blockDeviceMappings []*v1beta1.BlockDeviceMapping) []*ec2.LaunchTemplateBlockDeviceMappingRequest { if len(blockDeviceMappings) == 0 { // The EC2 API fails with empty slices and expects nil. diff --git a/pkg/test/environment.go b/pkg/test/environment.go index 81aa70575470..cfa2a4af74c1 100644 --- a/pkg/test/environment.go +++ b/pkg/test/environment.go @@ -30,6 +30,7 @@ import ( awscache "github.com/aws/karpenter-provider-aws/pkg/cache" "github.com/aws/karpenter-provider-aws/pkg/fake" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily" + "github.com/aws/karpenter-provider-aws/pkg/providers/capacityreservation" "github.com/aws/karpenter-provider-aws/pkg/providers/instance" "github.com/aws/karpenter-provider-aws/pkg/providers/instanceprofile" "github.com/aws/karpenter-provider-aws/pkg/providers/instancetype" @@ -67,19 +68,21 @@ type Environment struct { AvailableIPAdressCache *cache.Cache AssociatePublicIPAddressCache *cache.Cache SecurityGroupCache *cache.Cache + CapacityReservationCache *cache.Cache InstanceProfileCache *cache.Cache // Providers - InstanceTypesProvider *instancetype.DefaultProvider - InstanceProvider *instance.DefaultProvider - SubnetProvider *subnet.DefaultProvider - SecurityGroupProvider *securitygroup.DefaultProvider - InstanceProfileProvider *instanceprofile.DefaultProvider - PricingProvider *pricing.DefaultProvider - AMIProvider *amifamily.DefaultProvider - AMIResolver *amifamily.Resolver - VersionProvider *version.DefaultProvider - LaunchTemplateProvider *launchtemplate.DefaultProvider + InstanceTypesProvider *instancetype.DefaultProvider + InstanceProvider *instance.DefaultProvider + SubnetProvider *subnet.DefaultProvider + SecurityGroupProvider *securitygroup.DefaultProvider + CapacityReservationProvider *capacityreservation.DefaultProvider + InstanceProfileProvider *instanceprofile.DefaultProvider + PricingProvider *pricing.DefaultProvider + AMIProvider *amifamily.DefaultProvider + AMIResolver *amifamily.Resolver + VersionProvider *version.DefaultProvider + LaunchTemplateProvider *launchtemplate.DefaultProvider } func NewEnvironment(ctx context.Context, env *coretest.Environment) *Environment { @@ -99,6 +102,7 @@ func NewEnvironment(ctx context.Context, env *coretest.Environment) *Environment availableIPAdressCache := cache.New(awscache.AvailableIPAddressTTL, awscache.DefaultCleanupInterval) associatePublicIPAddressCache := cache.New(awscache.AssociatePublicIPAddressTTL, awscache.DefaultCleanupInterval) securityGroupCache := cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + capacityReservationCache := cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) instanceProfileCache := cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) fakePricingAPI := &fake.PricingAPI{} @@ -106,6 +110,7 @@ func NewEnvironment(ctx context.Context, env *coretest.Environment) *Environment pricingProvider := pricing.NewDefaultProvider(ctx, fakePricingAPI, ec2api, fake.DefaultRegion) subnetProvider := subnet.NewDefaultProvider(ec2api, subnetCache, availableIPAdressCache, associatePublicIPAddressCache) securityGroupProvider := securitygroup.NewDefaultProvider(ec2api, securityGroupCache) + capacityReservationProvider := capacityreservation.NewDefaultProvider(ec2api, capacityReservationCache) versionProvider := version.NewDefaultProvider(env.KubernetesInterface, kubernetesVersionCache) instanceProfileProvider := instanceprofile.NewDefaultProvider(fake.DefaultRegion, iamapi, instanceProfileCache) amiProvider := amifamily.NewDefaultProvider(versionProvider, ssmapi, ec2api, ec2Cache) @@ -149,19 +154,21 @@ func NewEnvironment(ctx context.Context, env *coretest.Environment) *Environment AvailableIPAdressCache: availableIPAdressCache, AssociatePublicIPAddressCache: associatePublicIPAddressCache, SecurityGroupCache: securityGroupCache, + CapacityReservationCache: capacityReservationCache, InstanceProfileCache: instanceProfileCache, UnavailableOfferingsCache: unavailableOfferingsCache, - InstanceTypesProvider: instanceTypesProvider, - InstanceProvider: instanceProvider, - SubnetProvider: subnetProvider, - SecurityGroupProvider: securityGroupProvider, - LaunchTemplateProvider: launchTemplateProvider, - InstanceProfileProvider: instanceProfileProvider, - PricingProvider: pricingProvider, - AMIProvider: amiProvider, - AMIResolver: amiResolver, - VersionProvider: versionProvider, + InstanceTypesProvider: instanceTypesProvider, + InstanceProvider: instanceProvider, + SubnetProvider: subnetProvider, + SecurityGroupProvider: securityGroupProvider, + CapacityReservationProvider: capacityReservationProvider, + LaunchTemplateProvider: launchTemplateProvider, + InstanceProfileProvider: instanceProfileProvider, + PricingProvider: pricingProvider, + AMIProvider: amiProvider, + AMIResolver: amiResolver, + VersionProvider: versionProvider, } }