diff --git a/.gitignore b/.gitignore index 86994f5c..1ecddaaf 100644 --- a/.gitignore +++ b/.gitignore @@ -22,5 +22,9 @@ hack/tools/bin/ .DS_Store .vscode/ .idea/ + .run/ dist/ + +# File produced during testing +e2e/fixtures/mi-pull-image-exec.yaml diff --git a/README.md b/README.md index 8eef8dbf..c97a4a1d 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ Virtual Kubelet's ACI provider relies heavily on the feature set that ACI servic * [Exec support](https://docs.microsoft.com/azure/container-instances/container-instances-exec) for container instances * Azure Monitor integration ( aka OMS) * Support for init-containers ([use init containers](#Create-pod-with-init-containers)) +* Pull ACR image using managed identity ([acr image pull](./docs/Pull-Images-Using-Managed-Identity.md)) ### Limitations (Not supported) @@ -184,3 +185,4 @@ If it is an AKS managed virtual node, please follow the steps [here](https://do [az-container-list]: https://docs.microsoft.com/cli/azure/container?view=azure-cli-latest#az_container_list + diff --git a/docs/Pull-Images-Using-Managed-Identity.md b/docs/Pull-Images-Using-Managed-Identity.md new file mode 100644 index 00000000..87611cd1 --- /dev/null +++ b/docs/Pull-Images-Using-Managed-Identity.md @@ -0,0 +1,28 @@ +### Pulling Images Using AKS Managed identity +If your image is on a private reigstry, the AKS agent pool identity can be used to pull the images + +Attach the private acr registry to the cluster. This will give AcrPull access to the AKS agent pool managed identity +```bash +az aks update -g -n --attach-acr +``` + +Create a new pod that pulls an image from the private registry, for example +```yaml +spec: + containers: + - image: .azurecr.io/: + name: test-container +``` + +#### Optional: Use a Custom Managed Identity +To use a custom manged identity instead of the AKS agent pool identity, it must be added as a kubelet identity on the aks cluster. +```bash +az identity create -g -n +az aks update -g -n --assign-kubelet-identity +``` + +Then Attach the container registry +```bash +az aks update -g -n --attach-acr +``` + diff --git a/e2e/deployments_test.go b/e2e/deployments_test.go new file mode 100644 index 00000000..64ce3970 --- /dev/null +++ b/e2e/deployments_test.go @@ -0,0 +1,90 @@ +package e2e + +import ( + "testing" + "time" + "context" + + "github.com/virtual-kubelet/azure-aci/pkg/featureflag" +) + +func TestImagePullUsingKubeletIdentityMI(t *testing.T) { + ctx := context.TODO() + enabledFeatures := featureflag.InitFeatureFlag(ctx) + if !enabledFeatures.IsEnabled(ctx, featureflag.ManagedIdentityPullFeature) { + t.Skipf("%s feature is not enabled", featureflag.ManagedIdentityPullFeature) + } + // delete the pod first + cmd := kubectl("delete", "namespace", "vk-test", "--ignore-not-found") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatal(string(out)) + } + + // create namespace + cmd = kubectl("apply", "-f", "fixtures/namespace.yml") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatal(string(out)) + } + + // run container group pulling image from acr using MI + cmd = kubectl("apply", "-f", "fixtures/mi-pull-image-exec.yaml") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatal(string(out)) + } + + deadline, ok := t.Deadline() + timeout := time.Until(deadline) + if !ok { + timeout = 300 * time.Second + } + cmd = kubectl("wait", "--for=condition=ready", "--timeout="+timeout.String(), "pod/e2etest-acr-test-mi-container", "--namespace=vk-test") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatal(string(out)) + } + t.Log("success pulling image from ACR using managed identity") + + // query metrics + deadline = time.Now().Add(5 * time.Minute) + for { + t.Log("query metrics ....") + cmd = kubectl("get", "--raw", "/apis/metrics.k8s.io/v1beta1/namespaces/vk-test/pods/e2etest-acr-test-mi-container") + out, err := cmd.CombinedOutput() + if time.Now().After(deadline) { + t.Fatal("failed to query pod's stats from metrics server API") + } + if err == nil { + t.Logf("success query metrics %s", string(out)) + break + } + } + + // check pod status + t.Log("get pod status ....") + cmd = kubectl("get", "pod", "--field-selector=status.phase=Running", "--namespace=vk-test", "--output=jsonpath={.items..metadata.name}") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } + if string(out) != "e2etest-acr-test-mi-container" { + t.Fatal("failed to get pod's status") + } + t.Logf("success query pod status %s", string(out)) + + // check container status + t.Log("get container status ....") + cmd = kubectl("get", "pod", "e2etest-acr-test-mi-container", "--namespace=vk-test", "--output=jsonpath={.status.containerStatuses[0].ready}") + out, err = cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } + if string(out) != "true" { + t.Fatal("failed to get pod's status") + } + t.Logf("success query container status %s", string(out)) + + t.Log("clean up pod") + cmd = kubectl("delete", "namespace", "vk-test", "--ignore-not-found") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatal(string(out)) + } +} diff --git a/e2e/fixtures/mi-pull-image.yaml b/e2e/fixtures/mi-pull-image.yaml new file mode 100644 index 00000000..718ef625 --- /dev/null +++ b/e2e/fixtures/mi-pull-image.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: Pod +metadata: + name: e2etest-acr-test-mi-container + namespace: vk-test +spec: + nodeName: vk-aci-test-aks + restartPolicy: Never + containers: + - image: ${ACR_NAME}.azurecr.io/library/alpine + imagePullPolicy: Always + name: e2etest-acr-test-mi-container + command: [ + "sh", + "-c", + "sleep 1; while sleep 1; do echo pulled image using mi; done", + ] + resources: + requests: + memory: 1G + cpu: 1 + nodeSelector: + kubernetes.io/role: agent + beta.kubernetes.io/os: linux + type: virtual-kubelet + tolerations: + - key: virtual-kubelet.io/provider + operator: Exists diff --git a/e2e/fixtures_test.go b/e2e/fixtures_test.go index d87a68d2..594d283b 100644 --- a/e2e/fixtures_test.go +++ b/e2e/fixtures_test.go @@ -5,6 +5,7 @@ import ( "os/exec" ) +//execute kubectl command in terminal func kubectl(args ...string) *exec.Cmd { cmd := exec.Command("kubectl", args...) cmd.Env = os.Environ() diff --git a/go.mod b/go.mod index 9a275496..1c5bac38 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,11 @@ go 1.20 require ( contrib.go.opencensus.io/exporter/ocagent v0.7.0 - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 - github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.2 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2 v2.2.0-beta.1 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice v1.0.0 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.0.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.1.0 github.com/Azure/go-autorest/autorest v0.11.27 github.com/Azure/go-autorest/autorest/adal v0.9.20 @@ -36,12 +38,12 @@ require ( ) require ( - github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.2.0 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect - github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v0.9.0 // indirect github.com/NYTimes/gziphandler v1.1.1 // indirect github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 // indirect github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect @@ -63,7 +65,7 @@ require ( github.com/go-openapi/jsonreference v0.20.1 // indirect github.com/go-openapi/swag v0.22.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v4 v4.4.2 // indirect + github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/cel-go v0.12.6 // indirect @@ -85,7 +87,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 // indirect + github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.15.1 // indirect github.com/prometheus/common v0.42.0 // indirect @@ -108,7 +110,7 @@ require ( go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.24.0 // indirect - golang.org/x/crypto v0.1.0 // indirect + golang.org/x/crypto v0.6.0 // indirect golang.org/x/net v0.10.0 // indirect golang.org/x/oauth2 v0.8.0 // indirect golang.org/x/sys v0.9.0 // indirect diff --git a/go.sum b/go.sum index 336d34ce..d4e50ed9 100644 --- a/go.sum +++ b/go.sum @@ -47,15 +47,19 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 contrib.go.opencensus.io/exporter/ocagent v0.7.0 h1:BEfdCTXfMV30tLZD8c9n64V/tIZX5+9sXiuFLnrr1k8= contrib.go.opencensus.io/exporter/ocagent v0.7.0/go.mod h1:IshRmMJBhDfFj5Y67nVhMYTTIze91RUeT73ipWKs/GY= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 h1:pqrAR74b6EoR4kcxF7L7Wg2B8Jgil9UUZtMvxhEFqWo= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 h1:jp0dGvZ7ZK0mgqnTSClMxa5xuRL7NZgHameVYF6BurY= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 h1:rTnT/Jrcm+figWlYz4Ixzt0SJVR2cMC8lvZcimipiEY= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0/go.mod h1:ON4tFdPTwRcgWEaVDrN3584Ef+b7GgSJaXxe5fW9t4M= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.2 h1:uqM+VoHjVH6zdlkLF2b6O0ZANcHoj3rO0PoQ3jglUJA= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.2/go.mod h1:twTKAa1E6hLmSDjLhaCkbTMQKc7p/rNLU40rLxGEOCI= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.2.0 h1:leh5DwKv6Ihwi+h60uHtn6UWAxBbZ0q8DwQVMzf61zw= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.2.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2 v2.2.0-beta.1 h1:eY6fhA944YceJrJ9OGn1T5iqe5DA2rQ+O1/Gi3P4bXU= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2 v2.2.0-beta.1/go.mod h1:5Q/hN8CkM0y7bBldgIdoPMp9jyBZ1KVeexQvfY2KXw8= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice v1.0.0 h1:figxyQZXzZQIcP3njhC68bYUiTw45J8/SsHaLW8Ax0M= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice v1.0.0/go.mod h1:TmlMW4W5OvXOmOyKNnor8nlMMiO1ctIyzmHme/VHsrA= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal v1.0.0 h1:lMW1lD/17LUA5z1XTURo7LcVG2ICBPlyMHjIUrcFZNQ= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.0.0 h1:ZOt3s8LxEoRGgdD/k7Co4wGAWKmO4+jdPRCRBa8Rzc0= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.0.0/go.mod h1:ZJWUTTEMZLTJI4PPI6vuv/OCEs9YjEX9EqjCnLJ8afA= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.1.0 h1:mk57wRUA8fyjFxVcPPGv4shLcWDXPFYokTJL9zJxQtE= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.1.0/go.mod h1:mU96hbp8qJDA9OzTV1Ji7wCyPyaqC5kI6ZPsZfJ8sE4= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.0.0 h1:ECsQtyERDVz3NP3kvDOTLvbQhqWp/x9EsGKtb4ogUr8= @@ -75,8 +79,8 @@ github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+Z github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= -github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 h1:VgSJlZH5u0k2qxSpqyghcFQKmvYckj46uymKK5XzkBM= -github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0/go.mod h1:BDJ5qMFKx9DugEg3+uQSDCdbYPr5s9vBTrL9P8TpqOU= +github.com/AzureAD/microsoft-authentication-library-for-go v0.9.0 h1:UE9n9rkJF62ArLb1F3DEjRt8O3jLwMWdSoypKV4f3MU= +github.com/AzureAD/microsoft-authentication-library-for-go v0.9.0/go.mod h1:kgDmCTgBzIEPFElEF+FK0SdjAor06dRq2Go927dnQ6o= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -167,8 +171,8 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= -github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= -github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= +github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -316,8 +320,8 @@ github.com/onsi/ginkgo/v2 v2.9.1 h1:zie5Ly042PD3bsCvsSOPvRnFwyo3rKe64TJlD6nu0mk= github.com/onsi/gomega v1.27.4 h1:Z2AnStgsdSayCMDiCU42qIz+HLqEPcgiOCXjAU/w+8E= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= -github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 h1:Qj1ukM4GlMWXNdMBuXcXfz/Kw9s1qm0CLY32QxuSImI= -github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -422,8 +426,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= -golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -569,6 +573,7 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/hack/e2e/aks-addon.sh b/hack/e2e/aks-addon.sh index 401cc830..dc9ed4f6 100755 --- a/hack/e2e/aks-addon.sh +++ b/hack/e2e/aks-addon.sh @@ -84,9 +84,12 @@ if [ "$E2E_TARGET" = "pr" ]; then IMG_URL=$ACR_NAME.azurecr.io OUTPUT_TYPE=type=registry IMG_TAG=$IMG_TAG IMAGE=$IMG_URL/$IMG_REPO make docker-build-image OUTPUT_TYPE=type=registry INIT_IMG_TAG=$INIT_IMG_TAG INIT_IMAGE=$IMG_URL/$INIT_IMG_REPO make docker-build-init-image - + az acr import --name ${ACR_NAME} --source docker.io/library/alpine:latest fi +export ACR_ID="$(az acr show --resource-group ${RESOURCE_GROUP} --name ${ACR_NAME} --query id -o tsv)" +export ACR_NAME=${ACR_NAME} + TMPDIR="$(mktemp -d)" az network vnet create \ @@ -172,7 +175,7 @@ while true; do sleep 3 done -kubectl wait --for=condition=Ready --timeout=300s node "$TEST_NODE_NAME" +kubectl wait --for=condition=Ready --timeout=600s node "$TEST_NODE_NAME" export TEST_NODE_NAME @@ -213,4 +216,6 @@ CSI_DRIVER_STORAGE_ACCOUNT_KEY=$(az storage account keys list --resource-group " export CSI_DRIVER_STORAGE_ACCOUNT_NAME=$CSI_DRIVER_STORAGE_ACCOUNT_NAME export CSI_DRIVER_STORAGE_ACCOUNT_KEY=$CSI_DRIVER_STORAGE_ACCOUNT_KEY +envsubst < e2e/fixtures/mi-pull-image.yaml > e2e/fixtures/mi-pull-image-exec.yaml + $@ diff --git a/hack/e2e/aks.sh b/hack/e2e/aks.sh index dfb2459e..80f47d5d 100755 --- a/hack/e2e/aks.sh +++ b/hack/e2e/aks.sh @@ -61,7 +61,6 @@ cleanup() { } trap 'cleanup' EXIT - check_aci_registered() { az provider list --query "[?contains(namespace,'Microsoft.ContainerInstance')]" -o json | jq -r '.[0].registrationState' } @@ -74,48 +73,65 @@ if [ ! "$(check_aci_registered)" = "Registered" ]; then done fi +echo -e "\n......Creating Resource Group\n" az group create --name "$RESOURCE_GROUP" --location "$LOCATION" -if [ "$E2E_TARGET" = "pr" ]; then - az acr create --resource-group "$RESOURCE_GROUP" \ +echo -e "\n......Creating ACR\n" +az acr create --resource-group "$RESOURCE_GROUP" \ --name "$ACR_NAME" --sku Basic +if [ "$E2E_TARGET" = "pr" ]; then az acr login --name "$ACR_NAME" IMG_URL=$ACR_NAME.azurecr.io OUTPUT_TYPE=type=registry IMG_TAG=$IMG_TAG IMAGE=$IMG_URL/$IMG_REPO make docker-build-image OUTPUT_TYPE=type=registry INIT_IMG_TAG=$INIT_IMG_TAG INIT_IMAGE=$IMG_URL/$INIT_IMG_REPO make docker-build-init-image fi +echo -e "\n......Creating ACR.........[DONE]\n" KUBE_DNS_IP=10.0.0.10 +echo -e "\n......Creating vNet\n" az network vnet create \ --resource-group $RESOURCE_GROUP \ --name $VNET_NAME \ --address-prefixes $VNET_RANGE \ --subnet-name $CLUSTER_SUBNET_NAME \ --subnet-prefix $CLUSTER_SUBNET_RANGE +echo -e "\n......Creating vNet.........[DONE]\n" +echo -e "\n......Creating vNet subnet\n" aci_subnet_id="$(az network vnet subnet create \ --resource-group $RESOURCE_GROUP \ --vnet-name $VNET_NAME \ --name $ACI_SUBNET_NAME \ --address-prefix $ACI_SUBNET_RANGE \ --query id -o tsv)" +echo -e "\n......Creating vNet subnet.........[DONE]\n" vnet_id="$(az network vnet show --resource-group $RESOURCE_GROUP --name $VNET_NAME --query id -o tsv)" aks_subnet_id="$(az network vnet subnet show --resource-group $RESOURCE_GROUP --vnet-name $VNET_NAME --name $CLUSTER_SUBNET_NAME --query id -o tsv)" TMPDIR="$(mktemp -d)" +echo -e "\n......Creating managed identities for AKS and Node\n" cluster_identity="$(az identity create --name "${RESOURCE_GROUP}-aks-identity" --resource-group "${RESOURCE_GROUP}" --query principalId -o tsv)" node_identity="$(az identity create --name "${RESOURCE_GROUP}-node-identity" --resource-group "${RESOURCE_GROUP}" --query principalId -o tsv)" node_identity_id="$(az identity show --name ${RESOURCE_GROUP}-node-identity --resource-group ${RESOURCE_GROUP} --query id -o tsv)" cluster_identity_id="$(az identity show --name ${RESOURCE_GROUP}-aks-identity --resource-group ${RESOURCE_GROUP} --query id -o tsv)" +echo -e "\n......Creating managed identities for AKS and Node.........[DONE]\n" + + +az acr import --name ${ACR_NAME} --source docker.io/library/alpine:latest +export ACR_ID="$(az acr show --resource-group ${RESOURCE_GROUP} --name ${ACR_NAME} --query id -o tsv)" +export ACR_NAME=${ACR_NAME} + node_identity_client_id="$(az identity create --name "${RESOURCE_GROUP}-aks-identity" --resource-group "${RESOURCE_GROUP}" --query clientId -o tsv)" +echo -e "\n......Creating AKS Cluster.........[DONE]\n" + if [ "$E2E_TARGET" = "pr" ]; then az aks create \ -g "$RESOURCE_GROUP" \ @@ -144,9 +160,11 @@ az aks create \ --dns-service-ip "$KUBE_DNS_IP" \ --assign-kubelet-identity "$node_identity_id" \ --assign-identity "$cluster_identity_id" \ + --attach-acr $ACR_ID \ --generate-ssh-keys fi +echo -e "\n......Creating RBAC Role for Network Contributor on vNet\n" az role assignment create \ --role "Network Contributor" \ --assignee-object-id "$node_identity" \ @@ -162,21 +180,36 @@ az role assignment create \ --assignee-object-id "$node_identity" \ --assignee-principal-type "ServicePrincipal" \ --scope "$aci_subnet_id" +echo -e "\n......Creating RBAC Role for Network Contributor on vNet.........[DONE]\n" + # Make sure ACI can create containers in the AKS RG. # Note, this is not wonderful since it gives a lot of permissions to the identity which is also shared with the kubelet (which it doesn't need). # Unfortunately there is no way to scope this down (AFIACT) currently. +echo -e "\n......Creating RBAC Role for Contributor on Resource Group\n" az role assignment create \ --role "Contributor" \ --assignee-object-id "$node_identity" \ --assignee-principal-type "ServicePrincipal" \ --scope "/subscriptions/$(az account show --query id -o tsv)/resourceGroups/MC_${RESOURCE_GROUP}_${RESOURCE_GROUP}_${LOCATION}" +az role assignment create \ + --role "Contributor" \ + --assignee-object-id "$node_identity" \ + --assignee-principal-type "ServicePrincipal" \ + --scope "/subscriptions/$(az account show --query id -o tsv)/resourceGroups/${RESOURCE_GROUP}" +echo -e "\n......Creating RBAC Role for Contributor on Resource Group.........[DONE]\n" + +echo -e "\n......Set AKS Cluster context\n" az aks get-credentials -g "$RESOURCE_GROUP" -n "$CLUSTER_NAME" -f "${TMPDIR}/kubeconfig" export KUBECONFIG="${TMPDIR}/kubeconfig" +echo -e "\n......Set AKS Cluster context.........[DONE]\n" +echo -e "\n......Get AKS Cluster Master URL\n" MASTER_URI="$(kubectl cluster-info | awk '/Kubernetes control plane/{print $7}' | sed "s,\x1B\[[0-9;]*[a-zA-Z],,g")" +echo -e "\n......Get AKS Cluster Master URL.........[DONE]\n" +echo -e "\n......Install Virtual node on the AKS Cluster with ACI provider\n" ## Linux VK helm install \ --kubeconfig="${KUBECONFIG}" \ @@ -206,6 +239,7 @@ done kubectl wait --for=condition=Ready --timeout=300s node "$TEST_NODE_NAME" export TEST_NODE_NAME +echo -e "\n......Install Virtual node on the AKS Cluster with ACI provider.........[DONE]\n" ## Windows VK helm install \ @@ -234,6 +268,7 @@ kubectl wait --for=condition=Ready --timeout=300s node "$TEST_WINDOWS_NODE_NAME" export TEST_WINDOWS_NODE_NAME +echo -e "\n......Initialize environment variabled needed for E2e tests\n" ## CSI Driver test az storage account create -n $CSI_DRIVER_STORAGE_ACCOUNT_NAME -g $RESOURCE_GROUP -l $LOCATION --sku Standard_LRS export AZURE_STORAGE_CONNECTION_STRING=$(az storage account show-connection-string -n $CSI_DRIVER_STORAGE_ACCOUNT_NAME -g $RESOURCE_GROUP -o tsv) @@ -244,4 +279,8 @@ CSI_DRIVER_STORAGE_ACCOUNT_KEY=$(az storage account keys list --resource-group $ export CSI_DRIVER_STORAGE_ACCOUNT_NAME=$CSI_DRIVER_STORAGE_ACCOUNT_NAME export CSI_DRIVER_STORAGE_ACCOUNT_KEY=$CSI_DRIVER_STORAGE_ACCOUNT_KEY +envsubst < e2e/fixtures/mi-pull-image.yaml > e2e/fixtures/mi-pull-image-exec.yaml + +echo -e "\n......Initialize environment variabled needed for E2e tests.........[DONE]\n" + $@ diff --git a/pkg/client/client_apis.go b/pkg/client/client_apis.go index 7ff69a4b..cec8c4e0 100644 --- a/pkg/client/client_apis.go +++ b/pkg/client/client_apis.go @@ -15,6 +15,8 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" azaciv2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2" + armcontainerservice "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice" + armmsi "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi" "github.com/pkg/errors" "github.com/virtual-kubelet/azure-aci/pkg/auth" "github.com/virtual-kubelet/azure-aci/pkg/validation" @@ -33,12 +35,17 @@ type AzClientsInterface interface { DeleteContainerGroup(ctx context.Context, resourceGroup, cgName string) error ListLogs(ctx context.Context, resourceGroup, cgName, containerName string, opts api.ContainerLogOpts) (*string, error) ExecuteContainerCommand(ctx context.Context, resourceGroup, cgName, containerName string, containerReq azaciv2.ContainerExecRequest) (*azaciv2.ContainerExecResponse, error) + GetIdentitiesListResult(ctx context.Context, resourceGroup string) ([]*armmsi.Identity, error) + GetClusterListResult(ctx context.Context, resourceGroup string) ([]*armcontainerservice.ManagedCluster, error) + GetClusterListBySubscriptionResult(ctx context.Context) ([]*armcontainerservice.ManagedCluster, error) } type AzClientsAPIs struct { ContainersClient *azaciv2.ContainersClient ContainerGroupClient *azaciv2.ContainerGroupsClient LocationClient *azaciv2.LocationClient + MSIClient *armmsi.UserAssignedIdentitiesClient + AKSClient *armcontainerservice.ManagedClustersClient } func NewAzClientsAPIs(ctx context.Context, azConfig auth.Config) (*AzClientsAPIs, error) { @@ -90,9 +97,21 @@ func NewAzClientsAPIs(ctx context.Context, azConfig auth.Config) (*AzClientsAPIs return nil, errors.Wrap(err, "failed to create location client ") } + msiClient, err := armmsi.NewUserAssignedIdentitiesClient(azConfig.AuthConfig.SubscriptionID, credential, &options) + if err != nil { + return nil, errors.Wrap(err, "failed to create msi client ") + } + + aksClient, err := armcontainerservice.NewManagedClustersClient(azConfig.AuthConfig.SubscriptionID, credential, &options) + if err != nil { + return nil, errors.Wrap(err, "failed to create aks client ") + } + obj.ContainersClient = cClient obj.ContainerGroupClient = cgClient obj.LocationClient = lClient + obj.MSIClient = msiClient + obj.AKSClient = aksClient logger.Debug("aci clients have been initialized successfully") return &obj, nil @@ -299,3 +318,69 @@ func (a *AzClientsAPIs) ExecuteContainerCommand(ctx context.Context, resourceGro func containerGroupName(podNS, podName string) string { return fmt.Sprintf("%s-%s", podNS, podName) } + +func (a *AzClientsAPIs) GetIdentitiesListResult(ctx context.Context, resourceGroup string) ([]*armmsi.Identity, error) { + logger := log.G(ctx).WithField("method", "GetIdentitiesListResult") + ctx, span := trace.StartSpan(ctx, "client.GetIdentitiesListResult") + defer span.End() + + var rawResponse *http.Response + ctxWithResp := runtime.WithCaptureResponse(ctx, &rawResponse) + + pager := a.MSIClient.NewListByResourceGroupPager(resourceGroup, nil) + var idList []*armmsi.Identity + for pager.More() { + page, err := pager.NextPage(ctxWithResp) + if err != nil { + logger.Errorf("an error has occurred while getting list of Identities, status code %d", rawResponse.StatusCode) + return nil, err + } + idList = append(idList, page.Value...) + } + return idList, nil +} + +func (a *AzClientsAPIs) GetClusterListResult(ctx context.Context, resourceGroup string) ([]*armcontainerservice.ManagedCluster, error) { + logger := log.G(ctx).WithField("method", "GetClusterListResult") + ctx, span := trace.StartSpan(ctx, "client.GetClusterListResult") + defer span.End() + + var rawResponse *http.Response + ctxWithResp := runtime.WithCaptureResponse(ctx, &rawResponse) + + var clusterList []*armcontainerservice.ManagedCluster + pager := a.AKSClient.NewListByResourceGroupPager(resourceGroup, nil) + + for pager.More() { + page, err := pager.NextPage(ctxWithResp) + if err != nil { + logger.Errorf("an error has occurred while getting list of clusters, status code %d", rawResponse.StatusCode) + return nil, err + } + clusterList = append(clusterList, page.Value...) + } + return clusterList, nil +} + + +func (a *AzClientsAPIs) GetClusterListBySubscriptionResult(ctx context.Context) ([]*armcontainerservice.ManagedCluster, error) { + logger := log.G(ctx).WithField("method", "GetClusterListBySubscriptionResult") + ctx, span := trace.StartSpan(ctx, "client.GetClusterListBySubscriptionResult") + defer span.End() + + var rawResponse *http.Response + ctxWithResp := runtime.WithCaptureResponse(ctx, &rawResponse) + + var clusterList []*armcontainerservice.ManagedCluster + pager := a.AKSClient.NewListPager(nil) + + for pager.More() { + page, err := pager.NextPage(ctxWithResp) + if err != nil { + logger.Errorf("an error has occurred while getting list of clusters, status code %d", rawResponse.StatusCode) + return nil, err + } + clusterList = append(clusterList, page.Value...) + } + return clusterList, nil +} diff --git a/pkg/featureflag/feature_flag.go b/pkg/featureflag/feature_flag.go index 7b7fc3db..d0ce4c5c 100644 --- a/pkg/featureflag/feature_flag.go +++ b/pkg/featureflag/feature_flag.go @@ -13,6 +13,7 @@ import ( const ( InitContainerFeature = "init-container" ConfidentialComputeFeature = "confidential-compute" + ManagedIdentityPullFeature = "managed-identity-image-pull" // Events : support ACI to K8s event translation and broadcasting Events = "events" @@ -21,6 +22,7 @@ const ( var enabledFeatures = []string{ InitContainerFeature, ConfidentialComputeFeature, + ManagedIdentityPullFeature, Events, } diff --git a/pkg/provider/aci.go b/pkg/provider/aci.go index 858f1cf0..c0ceef94 100644 --- a/pkg/provider/aci.go +++ b/pkg/provider/aci.go @@ -15,6 +15,7 @@ import ( "reflect" "strings" "time" + "regexp" azaciv2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2" "github.com/gorilla/websocket" @@ -76,6 +77,7 @@ const ( const ( confidentialComputeSkuLabel = "virtual-kubelet.io/container-sku" confidentialComputeCcePolicyLabel = "virtual-kubelet.io/confidential-compute-cce-policy" + containerGroupIdentitiesLabel = "virtual-kubelet.io/container-group-identities" ) // ACIProvider implements the virtual-kubelet provider interface and communicates with Azure's ACI APIs. @@ -334,6 +336,26 @@ func (p *ACIProvider) CreatePod(ctx context.Context, pod *v1.Pod) error { } + // if no username credentials are provided use agentpool MI if for pulling images from ACR + if len(creds) == 0 && p.enabledFeatures.IsEnabled(ctx, featureflag.ManagedIdentityPullFeature) { + identityList := []string{} + agentPoolKubeletIdentity, err := p.GetAgentPoolKubeletIdentity(ctx, pod) + if err != nil { + log.G(ctx).Infof("could not find Agent pool identity %v", err) + } + + if agentPoolKubeletIdentity != nil { + identityList = append(identityList, *agentPoolKubeletIdentity) + } + + if cgIdentityString := pod.Annotations[containerGroupIdentitiesLabel]; cgIdentityString != "" { + cgIdentityURIs := strings.Split(cgIdentityString, ";") + identityList = append(identityList, cgIdentityURIs...) + } + SetContainerGroupIdentity(ctx, identityList, azaciv2.ResourceIdentityTypeUserAssigned, cg) + creds = p.getManagedIdentityImageRegistryCredentials(pod, agentPoolKubeletIdentity, cg) + } + if p.enabledFeatures.IsEnabled(ctx, featureflag.InitContainerFeature) { // get initContainers initContainers, err := p.getInitContainers(ctx, pod) @@ -406,6 +428,40 @@ func (p *ACIProvider) CreatePod(ctx context.Context, pod *v1.Pod) error { return p.azClientsAPIs.CreateContainerGroup(ctx, p.resourceGroup, pod.Namespace, pod.Name, cg) } +// get list of distinct acr servernames from pod +func (p *ACIProvider) getImageServerNames(pod *v1.Pod) []string { + serverNamesMap := map[string]int{} + acrRegexp := "[a-z0-9]+\\.azurecr\\.io" + for _, container := range pod.Spec.Containers { + re := regexp.MustCompile(`/`) + imageSplit := re.Split(container.Image, -1) + isMatch, _ := regexp.MatchString(acrRegexp, imageSplit[0]) + if len(imageSplit) > 1 && isMatch { + serverNamesMap[imageSplit[0]] = 0 + } + } + serverNames := []string{} + for k := range serverNamesMap { + serverNames = append(serverNames, k) + } + return serverNames +} + +func (p *ACIProvider) getManagedIdentityImageRegistryCredentials(pod *v1.Pod, identity *string, containerGroup *azaciv2.ContainerGroup) ([]*azaciv2.ImageRegistryCredential){ + serverNames := p.getImageServerNames(pod) + ips := make([]*azaciv2.ImageRegistryCredential, 0, len(pod.Spec.ImagePullSecrets)) + if identity != nil{ + for i := range serverNames { + cred := azaciv2.ImageRegistryCredential{ + Server: &serverNames[i], + Identity: identity, + } + ips = append(ips, &cred) + } + } + return ips +} + // setACIExtensions func (p *ACIProvider) setACIExtensions(ctx context.Context) error { masterURI := os.Getenv("MASTER_URI") diff --git a/pkg/provider/aci_mi_image_pull_test.go b/pkg/provider/aci_mi_image_pull_test.go new file mode 100644 index 00000000..ab9e0e1c --- /dev/null +++ b/pkg/provider/aci_mi_image_pull_test.go @@ -0,0 +1,283 @@ +package provider + +import ( + "fmt" + "testing" + "context" + "strings" + + azaciv2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2" + "github.com/golang/mock/gomock" + "github.com/google/uuid" + "gotest.tools/assert" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestGetImageServerNames(t *testing.T) { + podName := "pod-" + uuid.New().String() + podNamespace := "ns-" + uuid.New().String() + containerName := "mi-image-pull-container" + + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + mockSecretLister := NewMockSecretLister(mockCtrl) + + aciMocks := createNewACIMock() + provider, err := createTestProvider(aciMocks, NewMockConfigMapLister(mockCtrl), + mockSecretLister, NewMockPodLister(mockCtrl), nil) + if err != nil { + t.Fatal("Unable to create test provider", err) + } + + cases := []struct { + description string + imageNames []string + expectedLength int + }{ + { + description: "string image name with azurecr io", + imageNames: []string{ + "fakename.azurecr.io/fakeimage:faketag", + "fakename2.azurecr.io/fakeimage:faketag", + }, + expectedLength: 2, + + }, + { + description: "alphanumeric image name with azurecr.io", + imageNames: []string{"123fakename456.azurecr.io/fakerepo/fakeimage:faketag"}, + expectedLength: 1, + }, + { + description: "image name without azurecr.io", + imageNames: []string{ + "fakerepo/fakeimage:faketag", + "fakerepo2/fakeimage2:faketag", + }, + expectedLength: 0, + }, + { + description: "image name with and without azurecr.io", + imageNames: []string{ + "fakerepo.azurecr.io/fakeimage:faketag", + "fakerepo2/fakeimage2:faketag", + }, + expectedLength: 1, + }, + } + for _, tc := range cases { + t.Run(tc.description, func(t *testing.T) { + // pod spec definition with container image names + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: podNamespace, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{}, + }, + } + for i, imageName := range tc.imageNames { + pod.Spec.Containers = append(pod.Spec.Containers, v1.Container{ + Image: imageName, + Name: fmt.Sprintf("%s-%d", containerName, i), + }) + } + + serverNames := provider.getImageServerNames(pod) + assert.Equal(t, tc.expectedLength, len(serverNames)) + }) + } +} + +func TestSetContainerGroupIdentity(t *testing.T) { + fakeIdentityURI := "fakeuri" + fakeIdentityURI2 := "fakeuri2" + + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + cases := []struct { + description string + identityList []string + identityType azaciv2.ResourceIdentityType + }{ + { + description: "identity is nil", + identityList: []string{}, + identityType: azaciv2.ResourceIdentityTypeUserAssigned, + }, + { + description: "identity is not nil", + identityList: []string{fakeIdentityURI}, + identityType: azaciv2.ResourceIdentityTypeUserAssigned, + + }, + { + description: "identity is not nil", + identityList: []string{fakeIdentityURI, fakeIdentityURI2}, + identityType: azaciv2.ResourceIdentityTypeUserAssigned, + + }, + { + description: "identity type is not user assignted", + identityList: []string{fakeIdentityURI, fakeIdentityURI2}, + identityType: azaciv2.ResourceIdentityTypeSystemAssigned, + }, + } + for _, tc := range cases { + t.Run(tc.description, func(t *testing.T) { + + testContainerGroup := &azaciv2.ContainerGroup{} + SetContainerGroupIdentity(context.Background(), tc.identityList, tc.identityType, testContainerGroup) + + if tc.identityType == azaciv2.ResourceIdentityTypeUserAssigned && len(tc.identityList) > 0 { + // identity uri, clientID, principalID should match + assert.Check(t, testContainerGroup.Identity != nil, "container group identity should be populated") + assert.Equal(t, *testContainerGroup.Identity.Type, azaciv2.ResourceIdentityTypeUserAssigned, "identity type should match") + assert.Check(t, len(testContainerGroup.Identity.UserAssignedIdentities) == len(tc.identityList), "all identities should be populated in UserAssignedIdentities") + assert.Check(t, testContainerGroup.Identity.UserAssignedIdentities[tc.identityList[0]] != nil , "identity uri should be present in UserAssignedIdentities") + } else { + // identity should not be added + assert.Check(t, testContainerGroup.Identity == nil, "container group identity should not be populated") + } + }) + } +} + +func TestGetManagedIdentityImageRegistryCredentials(t *testing.T) { + fakeIdentityURI := "fakeuri" + fakeImageName := "fakeregistry.azurecr.io/fakeimage:faketag" + fakeImageName2 := "fakeregistry2.azurecr.io/fakeimage:faketag" + + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: podNamespace, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + v1.Container{ + Image: fakeImageName, + }, + v1.Container{ + Image: fakeImageName, // duplicate image server + }, + v1.Container{ + Image: fakeImageName2, + }, + }, + }, + } + + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + mockSecretLister := NewMockSecretLister(mockCtrl) + + aciMocks := createNewACIMock() + provider, err := createTestProvider(aciMocks, NewMockConfigMapLister(mockCtrl), + mockSecretLister, NewMockPodLister(mockCtrl), nil) + if err != nil { + t.Fatal("Unable to create test provider", err) + } + + cases := []struct { + description string + identity *string + }{ + { + description: "identity is nil", + identity: nil, + }, + { + description: "identity is not nil", + identity: &fakeIdentityURI, + + }, + } + for _, tc := range cases { + t.Run(tc.description, func(t *testing.T) { + testContainerGroup := &azaciv2.ContainerGroup{} + creds := provider.getManagedIdentityImageRegistryCredentials(pod, tc.identity, testContainerGroup) + + if tc.identity != nil{ + // image registry credentials should have identity + assert.Check(t, creds != nil, "image registry creds should be populated") + assert.Equal(t, len(creds), 2, "credentials for all distinct acr should be added") + assert.Equal(t, *(creds)[0].Identity, *tc.identity, "identity uri should be correct") + assert.Equal(t, *(creds)[1].Identity, *tc.identity, "identity uri should be correct") + assert.Equal(t, *(creds)[0].Server, "fakeregistry.azurecr.io", "server should be correct") + assert.Equal(t, *(creds)[1].Server, "fakeregistry2.azurecr.io", "server should be correct") + } else { + // identity should not be added to image registry credentials + assert.Check(t, len(creds) == 0, "image registry creds should not be populated") + + } + }) + } +} + +func TestSetContainerGroupIdentityFromAnnotation(t *testing.T) { + fakeIdentityURI := "fakeuri;fakeuri2" + + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: podNamespace, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + v1.Container{ + }, + }, + }, + } + + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + mockSecretLister := NewMockSecretLister(mockCtrl) + + aciMocks := createNewACIMock() + aciMocks.MockCreateContainerGroup = func(ctx context.Context, resourceGroup, podNS, podName string, cg *azaciv2.ContainerGroup) error { + ids := strings.Split(pod.Annotations[containerGroupIdentitiesLabel], ";") + if cg.Identity != nil { + assert.Check(t, len(ids) == len(cg.Identity.UserAssignedIdentities), "container group identity should be set from annotation") + } + return nil + } + + provider, err := createTestProvider(aciMocks, NewMockConfigMapLister(mockCtrl), + mockSecretLister, NewMockPodLister(mockCtrl), nil) + if err != nil { + t.Fatal("Unable to create test provider", err) + } + + cases := []struct { + description string + annotations map[string]string + }{ + { + description: "container group identity annotation is nil ", + annotations: map[string]string{}, + }, + { + description: "container group identity annotation is not nil", + annotations: map[string]string{ + containerGroupIdentitiesLabel: fakeIdentityURI, + }, + }, + } + for _, tc := range cases { + t.Run(tc.description, func(t *testing.T) { + pod.Annotations = tc.annotations + if err := provider.CreatePod(context.Background(), pod); err != nil { + t.Fatal("failed to create pod", err) + } + }) + } +} + diff --git a/pkg/provider/identity.go b/pkg/provider/identity.go new file mode 100644 index 00000000..e36f1941 --- /dev/null +++ b/pkg/provider/identity.go @@ -0,0 +1,103 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the Apache 2.0 license. +*/ +package provider + +import ( + "context" + "fmt" + "strings" + "os" + "regexp" + + azaciv2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2" + "github.com/virtual-kubelet/virtual-kubelet/log" + v1 "k8s.io/api/core/v1" +) + +func SetContainerGroupIdentity(ctx context.Context, identityList []string, identityType azaciv2.ResourceIdentityType, containerGroup *azaciv2.ContainerGroup) { + if len(identityList) == 0 || identityType != azaciv2.ResourceIdentityTypeUserAssigned { + return + } + + cgIdentity := azaciv2.ContainerGroupIdentity{ + Type: &identityType, + UserAssignedIdentities: map[string]*azaciv2.UserAssignedIdentities{}, + } + + for i := range identityList { + cgIdentity.UserAssignedIdentities[identityList[i]] = &azaciv2.UserAssignedIdentities{} + } + + log.G(ctx).Infof("setting managed identity based imageRegistryCredentials\n") + containerGroup.Identity = &cgIdentity +} + +func (p *ACIProvider) GetAgentPoolKubeletIdentity(ctx context.Context, pod *v1.Pod) (*string, error) { + + if kubeletIdentity := os.Getenv("AKS_KUBELET_IDENTITY"); kubeletIdentity != "" { + return &kubeletIdentity, nil + } + + // list identities by resource group: covers both default MC_ resource group and user defined node resource group + idList, err := p.azClientsAPIs.GetIdentitiesListResult(ctx, p.resourceGroup) + if err != nil { + log.G(ctx).Errorf("Error while listing identities, %v", err) + } + for i := range idList { + if strings.HasSuffix(*idList[i].ID, "agentpool") { + return idList[i].ID, nil + } + } + + // ACI Resource group provided by user or a user specified kubelet identity is used on the cluster + // find cluster in the resource group and get kubelet identity + rg := p.resourceGroup + if strings.HasPrefix(p.resourceGroup, "MC_") { + rg = strings.Split(p.resourceGroup, "_")[1] + } + masterURI := os.Getenv("MASTER_URI") + t := regexp.MustCompile(`[:/]`) + masterURISplit := t.Split(masterURI, -1) + fqdn := "" + if len(masterURISplit) > 1 { + fqdn = masterURISplit[3] + } + + log.G(ctx).Infof("looking for cluster in resource group: %s \n", rg) + + // List clusters in RG and filter on fqdn + clusterList, err := p.azClientsAPIs.GetClusterListResult(ctx, p.resourceGroup) + if err != nil { + log.G(ctx).Errorf("Error while listing clusters in resource group , %v", err) + } + for _, cluster := range clusterList { + // pick the cluster based on fqdn + if (*cluster.Properties.Fqdn == fqdn) { + kubeletIdentity, ok:= cluster.Properties.IdentityProfile["kubeletidentity"] + if !ok || kubeletIdentity == nil { + return nil, fmt.Errorf("could not get kubelet identity from cluster\n") + } + return kubeletIdentity.ResourceID, nil + } + } + + // if all fails + // try to find cluster in the subscription and get kubeletidentity + clusterList, err = p.azClientsAPIs.GetClusterListBySubscriptionResult(ctx) + if err != nil { + log.G(ctx).Errorf("Error while listing clusters in subscription, %v", err) + } + // pick the cluster based on fqdn + for _, cluster := range clusterList { + if (*cluster.Properties.Fqdn == fqdn) { + kubeletIdentity, ok:= cluster.Properties.IdentityProfile["kubeletidentity"] + if !ok || kubeletIdentity == nil { + return nil, fmt.Errorf("could not get kubelet identity from cluster\n") + } + return kubeletIdentity.ResourceID, nil + } + } + return nil, fmt.Errorf("could not find an agent pool identity for cluster under subscription %s\n", p.providerNetwork.VnetSubscriptionID) +} diff --git a/pkg/provider/mock_aci_test.go b/pkg/provider/mock_aci_test.go index c2c71379..3d92b848 100644 --- a/pkg/provider/mock_aci_test.go +++ b/pkg/provider/mock_aci_test.go @@ -8,6 +8,8 @@ import ( "context" azaciv2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2" + armcontainerservice "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice" + armmsi "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi" "github.com/virtual-kubelet/virtual-kubelet/node/api" ) @@ -94,3 +96,15 @@ func (m *MockACIProvider) GetContainerGroup(ctx context.Context, resourceGroup, } return nil, nil } + +func (m *MockACIProvider) GetIdentitiesListResult(ctx context.Context, resourceGroup string) ([]*armmsi.Identity, error) { + return nil, nil +} + +func (m *MockACIProvider) GetClusterListResult(ctx context.Context, resourceGroup string) ([]*armcontainerservice.ManagedCluster, error) { + return nil, nil +} + +func (m *MockACIProvider) GetClusterListBySubscriptionResult(ctx context.Context) ([]*armcontainerservice.ManagedCluster, error) { + return nil, nil +}