diff --git a/e2e/testcases/permission_test.go b/e2e/testcases/permission_test.go new file mode 100644 index 0000000000..76f6f20346 --- /dev/null +++ b/e2e/testcases/permission_test.go @@ -0,0 +1,386 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "kpt.dev/configsync/e2e" + "kpt.dev/configsync/e2e/nomostest" + "kpt.dev/configsync/e2e/nomostest/ntopts" + nomostesting "kpt.dev/configsync/e2e/nomostest/testing" + "kpt.dev/configsync/e2e/nomostest/testpredicates" + "kpt.dev/configsync/e2e/nomostest/testutils" + "kpt.dev/configsync/e2e/nomostest/workloadidentity" + "kpt.dev/configsync/pkg/api/configsync" + "kpt.dev/configsync/pkg/api/configsync/v1beta1" + "kpt.dev/configsync/pkg/core" + "kpt.dev/configsync/pkg/importer/filesystem" + "kpt.dev/configsync/pkg/kinds" + "kpt.dev/configsync/pkg/reconcilermanager/controllers" + "kpt.dev/configsync/pkg/status" + "kpt.dev/configsync/pkg/testing/fake" +) + +// gsaRestrictedEmail returns a GSA email that has no permissions and no IAM bindings as a WI user. +func gsaRestrictedEmail() string { + return fmt.Sprintf("e2e-test-restricted-user@%s.iam.gserviceaccount.com", *e2e.GCPProject) +} + +// gsaRestrictedImpersonatedEmail returns a GSA email that has no permissions, but has an IAM binding as a WI user. +func gsaRestrictedImpersonatedEmail() string { + return fmt.Sprintf("e2e-test-restricted-wi-user@%s.iam.gserviceaccount.com", *e2e.GCPProject) +} + +const ( + + // The exact error message returned for GKE WI is: + // credential refresh failed: auth URL returned status 500, body: \"error retrieveing TokenSource.Token: compute: Received 403 `Unable to generate access token; IAM returned 403 Forbidden: Permission 'iam.serviceAccounts.getAccessToken' denied on resource (or it may not exist).\\nThis error could be caused by a missing IAM policy binding on the target IAM service account.\\nFor more information, refer to the Workload Identity documentation:\\n\\thttps://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity#authenticating_to + // The exact error message returned for Fleet WI is: + // credential refresh failed: auth URL returned status 500, body: \"error retrieveing TokenSource.Token: oauth2/google: status code 403: {\\n \\\"error\\\": {\\n \\\"code\\\": 403,\\n \\\"message\\\": \\\"Permission 'iam.serviceAccounts.getAccessToken' denied on resource (or it may not exist).\\\",\\n \\\"status\\\": \\\"PERMISSION_DENIED\\\",\\n \\\"details\\\": [\\n {\\n \\\"@type\\\": \\\"type.googleapis.com/google.rpc.ErrorInfo\\\",\\n \\\"reason\\\": \\\"IAM_PERMISSION_DENIED\\\",\\n \\\"domain\\\": \\\"iam.googleapis.com\\\",\\n \\\"metadata\\\": {\\n \\\"permission\\\": \\\"iam.serviceAccounts.getAccessToken\\\"\\n }\\n }\\n ]\\n }\\n}\\n\\n\" + missingWIBindingError = `Permission 'iam.serviceAccounts.getAccessToken' denied on resource (or it may not exist).` + + // The exact error message returned by CSR is: + // remote: PERMISSION_DENIED: The caller does not have permission\\nremote: [type.googleapis.com/google.rpc.RequestInfo]\\nremote: request_id: \\\"6a2759876362432c8d3bb63ad4e45a99\\\"\\nfatal: unable to access 'https://source.developers.google.com/p/stolos-dev/r/kustomize-components/ + missingCSRPermissionError = "PERMISSION_DENIED: The caller does not have permission" + + // The exact error message returned by AR for the OCI image is: + // failed to pull image us-docker.pkg.dev/stolos-dev/config-sync-test-private/kustomize-components:v1: GET https://us-docker.pkg.dev/v2/token?scope=repository%3Astolos-dev%2Fconfig-sync-test-private%2Fkustomize-components%3Apull\u0026service=: DENIED: Permission \"artifactregistry.repositories.downloadArtifacts\" denied on resource \"projects/stolos-dev/locations/us/repositories/config-sync-test-private\" (or it may not exist) + missingOCIARPermissionError = `DENIED: Permission \"artifactregistry.repositories.downloadArtifacts\" denied on resource` + + // The exact error message returned by AR for the Helm chart is: + // unexpected error rendering chart, will retry","Err":"rendering helm chart: invoking helm: Error: failed to authorize: failed to fetch oauth token: unexpected status from GET request to https://us-docker.pkg.dev/v2/token?scope=repository%3Astolos-dev%2Fconfig-sync-e2e-test--nanyu-cluster-1%2Fcoredns-test-helm-argke-workload-identit%3Apull\u0026service=us-docker.pkg.dev: 403 Forbidden + missingHelmARPermissionError = "403 Forbidden" + + // The exact error message for missing GSA is: + // metadata: GCE metadata \"instance/service-accounts/default/token?scopes=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fcloud-platform\" not defined + missingGSAError = "not defined" +) + +func missingGSAEmail() string { + return fmt.Sprintf("e2e-test-non-existent-user@%s.iam.gserviceaccount.com", *e2e.GCPProject) +} + +func TestGSAMissingPermissions(t *testing.T) { + testCases := []struct { + name string + fwiTest bool + sourceType v1beta1.SourceType + AuthType configsync.AuthType + sourceRepo string + sourceChart string + sourceVersion string + gsaEmail string + expectedErrMsg string + }{ + { + name: "GSA - Git source missing GSA for GKE WI", + fwiTest: false, + sourceType: v1beta1.GitSource, + AuthType: configsync.AuthGCPServiceAccount, + sourceRepo: csrRepo(), + gsaEmail: missingGSAEmail(), + expectedErrMsg: missingGSAError, + }, + { + name: "GSA - OCI source missing GSA for GKE WI", + fwiTest: false, + sourceType: v1beta1.OciSource, + AuthType: configsync.AuthGCPServiceAccount, + sourceRepo: privateARImage(), + gsaEmail: missingGSAEmail(), + expectedErrMsg: missingGSAError, + }, + { + name: "GSA - Helm source missing GSA for GKE WI", + fwiTest: false, + sourceType: v1beta1.HelmSource, + AuthType: configsync.AuthGCPServiceAccount, + sourceVersion: privateCoreDNSHelmChartVersion, + sourceChart: privateCoreDNSHelmChart, + gsaEmail: missingGSAEmail(), + expectedErrMsg: missingGSAError, + }, + { + name: "GSA - Git source missing GSA for Fleet WI", + fwiTest: false, + sourceType: v1beta1.GitSource, + AuthType: configsync.AuthGCPServiceAccount, + sourceRepo: csrRepo(), + gsaEmail: missingGSAEmail(), + expectedErrMsg: missingGSAError, + }, + { + name: "GSA - OCI source missing GSA for Fleet WI", + fwiTest: false, + sourceType: v1beta1.OciSource, + AuthType: configsync.AuthGCPServiceAccount, + sourceRepo: privateARImage(), + gsaEmail: missingGSAEmail(), + expectedErrMsg: missingGSAError, + }, + { + name: "GSA - Helm source missing GSA for Fleet WI", + fwiTest: false, + sourceType: v1beta1.HelmSource, + AuthType: configsync.AuthGCPServiceAccount, + sourceVersion: privateCoreDNSHelmChartVersion, + sourceChart: privateCoreDNSHelmChart, + gsaEmail: missingGSAEmail(), + expectedErrMsg: missingGSAError, + }, + { + name: "GSA - Git source missing IAM binding for GKE WI", + fwiTest: false, + sourceType: v1beta1.GitSource, + AuthType: configsync.AuthGCPServiceAccount, + sourceRepo: csrRepo(), + gsaEmail: gsaRestrictedEmail(), + expectedErrMsg: missingWIBindingError, + }, + { + name: "GSA - OCI source missing IAM binding for GKE WI", + fwiTest: false, + sourceType: v1beta1.OciSource, + AuthType: configsync.AuthGCPServiceAccount, + sourceRepo: privateARImage(), + gsaEmail: gsaRestrictedEmail(), + expectedErrMsg: missingWIBindingError, + }, + { + name: "GSA - Helm source missing IAM binding for GKE WI", + fwiTest: false, + sourceType: v1beta1.HelmSource, + AuthType: configsync.AuthGCPServiceAccount, + sourceVersion: privateCoreDNSHelmChartVersion, + sourceChart: privateCoreDNSHelmChart, + gsaEmail: gsaRestrictedEmail(), + expectedErrMsg: missingWIBindingError, + }, + { + name: "GSA - Git source missing IAM binding for Fleet WI", + fwiTest: true, + sourceType: v1beta1.GitSource, + AuthType: configsync.AuthGCPServiceAccount, + sourceRepo: csrRepo(), + gsaEmail: gsaRestrictedEmail(), + expectedErrMsg: missingWIBindingError, + }, + { + name: "GSA - OCI source missing IAM binding for Fleet WI", + fwiTest: true, + sourceType: v1beta1.OciSource, + AuthType: configsync.AuthGCPServiceAccount, + sourceRepo: privateARImage(), + gsaEmail: gsaRestrictedEmail(), + expectedErrMsg: missingWIBindingError, + }, + { + name: "GSA - Helm source missing IAM binding for Fleet WI", + fwiTest: true, + sourceType: v1beta1.HelmSource, + AuthType: configsync.AuthGCPServiceAccount, + sourceVersion: privateCoreDNSHelmChartVersion, + sourceChart: privateCoreDNSHelmChart, + gsaEmail: gsaRestrictedEmail(), + expectedErrMsg: missingWIBindingError, + }, + { + name: "GSA - Git source with IAM binding for GKE WI, but missing reader permission", + fwiTest: false, + sourceType: v1beta1.GitSource, + AuthType: configsync.AuthGCPServiceAccount, + sourceRepo: csrRepo(), + gsaEmail: gsaRestrictedImpersonatedEmail(), + expectedErrMsg: missingCSRPermissionError, + }, + { + name: "GSA - OCI source with IAM binding for GKE WI, but missing reader permission", + fwiTest: false, + sourceType: v1beta1.OciSource, + AuthType: configsync.AuthGCPServiceAccount, + sourceRepo: privateARImage(), + gsaEmail: gsaRestrictedImpersonatedEmail(), + expectedErrMsg: missingOCIARPermissionError, + }, + { + name: "GSA - Helm source with IAM binding for GKE WI, but missing reader permission", + fwiTest: false, + sourceType: v1beta1.HelmSource, + AuthType: configsync.AuthGCPServiceAccount, + sourceVersion: privateCoreDNSHelmChartVersion, + sourceChart: privateCoreDNSHelmChart, + gsaEmail: gsaRestrictedImpersonatedEmail(), + expectedErrMsg: missingHelmARPermissionError, + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + nt := nomostest.New(t, nomostesting.WorkloadIdentity, ntopts.Unstructured, ntopts.RequireGKE(t)) + rs := fake.RootSyncObjectV1Beta1(configsync.RootSyncName) + tenant := "tenant-a" + + if err := workloadidentity.ValidateEnabled(nt); err != nil { + nt.T.Fatal(err) + } + + mustConfigureMembership(nt, tc.fwiTest, false) + + spec := &sourceSpec{ + sourceType: tc.sourceType, + sourceRepo: tc.sourceRepo, + sourceChart: tc.sourceChart, + sourceVersion: tc.sourceVersion, + gsaEmail: tc.gsaEmail, + } + mustConfigureRootSync(nt, rs, tenant, spec) + nt.WaitForRootSyncSourceError(rs.Name, status.SourceErrorCode, tc.expectedErrMsg) + }) + } + +} + +func TestKSAMissingReaderPermission(t *testing.T) { + testCases := []struct { + name string + fwiTest bool + sourceType v1beta1.SourceType + AuthType configsync.AuthType + sourceRepo string + sourceChart string + sourceVersion string + gsaEmail string + expectedErrMsg string + }{ + { + name: "WI-KSA - OCI source missing permission for GKE WI", + fwiTest: false, + sourceType: v1beta1.OciSource, + AuthType: configsync.AuthGCPServiceAccount, + sourceRepo: privateARImage(), + gsaEmail: gsaRestrictedEmail(), + expectedErrMsg: missingOCIARPermissionError, + }, + { + name: "WI-KSA - Helm source missing permission for GKE WI", + fwiTest: false, + sourceType: v1beta1.HelmSource, + AuthType: configsync.AuthGCPServiceAccount, + sourceVersion: privateCoreDNSHelmChartVersion, + sourceChart: privateCoreDNSHelmChart, + gsaEmail: gsaRestrictedEmail(), + expectedErrMsg: missingHelmARPermissionError, + }, + { + name: "WI-KSA - OCI source missing permission for Fleet WI", + fwiTest: true, + sourceType: v1beta1.OciSource, + AuthType: configsync.AuthGCPServiceAccount, + sourceRepo: privateARImage(), + gsaEmail: gsaRestrictedEmail(), + expectedErrMsg: missingOCIARPermissionError, + }, + { + name: "WI-KSA - Helm source missing permission for Fleet WI", + fwiTest: true, + sourceType: v1beta1.HelmSource, + AuthType: configsync.AuthGCPServiceAccount, + sourceVersion: privateCoreDNSHelmChartVersion, + sourceChart: privateCoreDNSHelmChart, + gsaEmail: gsaRestrictedEmail(), + expectedErrMsg: missingHelmARPermissionError, + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + nt := nomostest.New(t, nomostesting.WorkloadIdentity, + ntopts.Unstructured, + ntopts.RequireGKE(t)) + + if err := workloadidentity.ValidateEnabled(nt); err != nil { + nt.T.Fatal(err) + } + mustConfigureMembership(nt, tc.fwiTest, false) + + // Add another RootSync (rs-byoid) to the `root-sync` Rootsync. + rs := fake.RootSyncObjectV1Beta1("rs-byoid") + syncDir := "tenant" + rs.Spec.SourceFormat = string(filesystem.SourceFormatUnstructured) + rs.Spec.SourceType = string(tc.sourceType) + spec := &sourceSpec{ + sourceType: tc.sourceType, + sourceRepo: tc.sourceRepo, + sourceChart: tc.sourceChart, + sourceVersion: tc.sourceVersion, + gsaEmail: tc.gsaEmail, + } + if err := pushSource(nt, spec); err != nil { + nt.T.Fatal(err) + } + switch tc.sourceType { + case v1beta1.OciSource: + rs.Spec.Oci = &v1beta1.Oci{ + Image: spec.sourceRepo, + Dir: syncDir, + Auth: configsync.AuthK8sServiceAccount, + } + case v1beta1.HelmSource: + rs.Spec.Helm = &v1beta1.HelmRootSync{ + HelmBase: v1beta1.HelmBase{ + Repo: spec.sourceRepo, + Chart: spec.sourceChart, + Version: spec.sourceVersion, + ReleaseName: "my-coredns", + Period: metav1.Duration{}, + Auth: configsync.AuthK8sServiceAccount, + }, + Namespace: "coredns", + } + } + nt.Must(nt.RootRepos[configsync.RootSyncName].Add("acme/rs-byoid.yaml", rs)) + nt.Must(nt.RootRepos[configsync.RootSyncName].CommitAndPush("Add a RootSync with restricted k8sserviceaccount")) + + ksaRef := types.NamespacedName{ + Namespace: configsync.ControllerNamespace, + Name: core.RootReconcilerName(rs.Name), + } + require.NoError(nt.T, + nt.WatchForSync(kinds.RootSyncV1Beta1(), configsync.RootSyncName, configsync.ControllerNamespace, + nomostest.DefaultRootSha1Fn, nomostest.RootSyncHasStatusSyncCommit, nil), + ) + require.NoError(nt.T, + nt.Watcher.WatchObject(kinds.ServiceAccount(), ksaRef.Name, ksaRef.Namespace, []testpredicates.Predicate{ + testpredicates.MissingAnnotation(controllers.GCPSAAnnotationKey), + })) + if tc.fwiTest { + nt.T.Log("Validate the serviceaccount_impersonation_url is absent from the injected FWI credentials") + nomostest.Wait(nt.T, "wait for FWI credentials to exist", nt.DefaultWaitTimeout, func() error { + return testutils.ReconcilerPodHasFWICredsAnnotation(nt, nomostest.DefaultRootReconcilerName, "", configsync.AuthK8sServiceAccount) + }) + } + + nt.WaitForRootSyncSourceError(rs.Name, status.SourceErrorCode, tc.expectedErrMsg) + }) + } + +} diff --git a/e2e/testcases/workload_identity_test.go b/e2e/testcases/workload_identity_test.go index 55f82434ca..91e774bf7b 100644 --- a/e2e/testcases/workload_identity_test.go +++ b/e2e/testcases/workload_identity_test.go @@ -184,6 +184,9 @@ func TestWorkloadIdentity(t *testing.T) { tc := tc t.Run(tc.name, func(t *testing.T) { nt := nomostest.New(t, nomostesting.WorkloadIdentity, ntopts.Unstructured, ntopts.RequireGKE(t)) + + rs := fake.RootSyncObjectV1Beta1(configsync.RootSyncName) + tenant := "tenant-a" if err := workloadidentity.ValidateEnabled(nt); err != nil { nt.T.Fatal(err) } @@ -192,81 +195,17 @@ func TestWorkloadIdentity(t *testing.T) { nt.T.Fatal(err) } - // Truncate the fleetMembership length to be at most 63 characters. - fleetMembership := truncateStringByLength(fmt.Sprintf("%s-%s", truncateStringByLength(*e2e.GCPProject, 20), nt.ClusterName), 63) - gkeURI := "https://container.googleapis.com/v1/projects/" + *e2e.GCPProject - if *e2e.GCPRegion != "" { - gkeURI += fmt.Sprintf("/locations/%s/clusters/%s", *e2e.GCPRegion, nt.ClusterName) - } else { - gkeURI += fmt.Sprintf("/zones/%s/clusters/%s", *e2e.GCPZone, nt.ClusterName) - } - - testutils.ClearMembershipInfo(nt, fleetMembership, *e2e.GCPProject, gkeURI) - testutils.ClearMembershipInfo(nt, fleetMembership, testutils.TestCrossProjectFleetProjectID, gkeURI) - - rs := fake.RootSyncObjectV1Beta1(configsync.RootSyncName) - nt.T.Cleanup(func() { - testutils.ClearMembershipInfo(nt, fleetMembership, *e2e.GCPProject, gkeURI) - testutils.ClearMembershipInfo(nt, fleetMembership, testutils.TestCrossProjectFleetProjectID, gkeURI) - }) - - tenant := "tenant-a" - - // Register the cluster for fleet workload identity test - if tc.fleetWITest { - fleetProject := *e2e.GCPProject - if tc.crossProject { - fleetProject = testutils.TestCrossProjectFleetProjectID - } - nt.T.Logf("Register the cluster to a fleet in project %q", fleetProject) - if err := testutils.RegisterCluster(nt, fleetMembership, fleetProject, gkeURI); err != nil { - nt.T.Fatalf("Failed to register the cluster to project %q: %v", fleetProject, err) - exists, err := testutils.FleetHasMembership(nt, fleetMembership, fleetProject) - if err != nil { - nt.T.Fatalf("Unable to check if membership exists: %v", err) - } - if !exists { - nt.T.Fatalf("The membership wasn't created") - } - } - nt.T.Logf("Restart the reconciler-manager to pick up the Membership") - // The reconciler manager checks if the Membership CRD exists before setting - // up the RootSync and RepoSync controllers: cmd/reconciler-manager/main.go:90. - // If the CRD exists, it configures the Membership watch. - // Otherwise, the watch is not configured to prevent the controller from crashing caused by an unknown CRD. - // DeletePodByLabel deletes the current reconciler-manager Pod so that new Pod - // can set up the watch. Once the watch is configured, it can detect the - // deletion and creation of the Membership, which implies cluster unregistration and registration. - // The underlying reconciler should be updated with FWI creds after the reconciler-manager restarts. - nomostest.DeletePodByLabel(nt, "app", reconcilermanager.ManagerName, false) - } - - // For helm charts, we need to push the chart to the AR before configuring the RootSync - if tc.sourceType == v1beta1.HelmSource { - chart, err := artifactregistry.PushHelmChart(nt, tc.sourceChart, tc.sourceVersion) - if err != nil { - nt.T.Fatalf("failed to push helm chart: %v", err) - } - - tc.sourceRepo = chart.Image.RepositoryOCI() - tc.sourceChart = chart.Image.Name - tc.sourceVersion = chart.Image.Version - tc.rootCommitFn = nomostest.HelmChartVersionShaFn(chart.Image.Version) - } + mustConfigureMembership(nt, tc.fleetWITest, tc.crossProject) - // Reuse the RootSync instead of creating a new one so that testing resources can be cleaned up after the test. - nt.T.Logf("Update RootSync to sync %s from repo %s", tenant, tc.sourceRepo) - switch tc.sourceType { - case v1beta1.GitSource: - nt.MustMergePatch(rs, fmt.Sprintf(`{"spec": {"git": {"dir": "%s", "branch": "main", "repo": "%s", "auth": "gcpserviceaccount", "gcpServiceAccountEmail": "%s", "secretRef": {"name": ""}}}}`, - tenant, tc.sourceRepo, tc.gsaEmail)) - case v1beta1.OciSource: - nt.MustMergePatch(rs, fmt.Sprintf(`{"spec": {"sourceType": "%s", "oci": {"dir": "%s", "image": "%s", "auth": "gcpserviceaccount", "gcpServiceAccountEmail": "%s"}, "git": null}}`, - v1beta1.OciSource, tenant, tc.sourceRepo, tc.gsaEmail)) - case v1beta1.HelmSource: - nt.MustMergePatch(rs, fmt.Sprintf(`{"spec": {"sourceType": "%s", "helm": {"chart": "%s", "repo": "%s", "version": "%s", "auth": "gcpserviceaccount", "gcpServiceAccountEmail": "%s", "releaseName": "my-coredns", "namespace": "coredns"}, "git": null}}`, - v1beta1.HelmSource, tc.sourceChart, tc.sourceRepo, tc.sourceVersion, tc.gsaEmail)) + spec := &sourceSpec{ + sourceType: tc.sourceType, + sourceRepo: tc.sourceRepo, + sourceChart: tc.sourceChart, + sourceVersion: tc.sourceVersion, + gsaEmail: tc.gsaEmail, + rootCommitFn: tc.rootCommitFn, } + mustConfigureRootSync(nt, rs, tenant, spec) ksaRef := types.NamespacedName{ Namespace: configsync.ControllerNamespace, @@ -283,17 +222,17 @@ func TestWorkloadIdentity(t *testing.T) { }) } - if tc.sourceType == v1beta1.HelmSource { - err := nt.WatchForAllSyncs(nomostest.WithRootSha1Func(tc.rootCommitFn), - nomostest.WithSyncDirectoryMap(map[types.NamespacedName]string{nomostest.DefaultRootRepoNamespacedName: tc.sourceChart})) + if spec.sourceType == v1beta1.HelmSource { + err := nt.WatchForAllSyncs(nomostest.WithRootSha1Func(spec.rootCommitFn), + nomostest.WithSyncDirectoryMap(map[types.NamespacedName]string{nomostest.DefaultRootRepoNamespacedName: spec.sourceChart})) if err != nil { nt.T.Fatal(err) } - if err := nt.Validate(fmt.Sprintf("my-coredns-%s", tc.sourceChart), "coredns", &appsv1.Deployment{}); err != nil { + if err := nt.Validate(fmt.Sprintf("my-coredns-%s", spec.sourceChart), "coredns", &appsv1.Deployment{}); err != nil { nt.T.Error(err) } } else { - err := nt.WatchForAllSyncs(nomostest.WithRootSha1Func(tc.rootCommitFn), + err := nt.WatchForAllSyncs(nomostest.WithRootSha1Func(spec.rootCommitFn), nomostest.WithSyncDirectoryMap(map[types.NamespacedName]string{nomostest.DefaultRootRepoNamespacedName: tenant})) if err != nil { nt.T.Fatal(err) @@ -303,7 +242,7 @@ func TestWorkloadIdentity(t *testing.T) { // Migrate from gcpserviceaccount to k8sserviceaccount if tc.testKSAMigration { - if err := migrateFromGSAtoKSA(nt, rs, ksaRef, tc.fleetWITest, tc.rootCommitFn); err != nil { + if err := migrateFromGSAtoKSA(nt, rs, ksaRef, tc.fleetWITest, spec.rootCommitFn); err != nil { nt.T.Fatal(err) } } @@ -406,3 +345,101 @@ func migrateFromGSAtoKSA(nt *nomostest.NT, rs *v1beta1.RootSync, ksaRef types.Na } return nil } + +// mustConfigureMembership clears the membership before and after the test. +// When testing Fleet WI, it registers the cluster to a fleet. +func mustConfigureMembership(nt *nomostest.NT, fleetWITest, crossProject bool) { + // Truncate the fleetMembership length to be at most 63 characters. + fleetMembership := truncateStringByLength(fmt.Sprintf("%s-%s", truncateStringByLength(*e2e.GCPProject, 20), nt.ClusterName), 63) + gkeURI := "https://container.googleapis.com/v1/projects/" + *e2e.GCPProject + if *e2e.GCPRegion != "" { + gkeURI += fmt.Sprintf("/locations/%s/clusters/%s", *e2e.GCPRegion, nt.ClusterName) + } else { + gkeURI += fmt.Sprintf("/zones/%s/clusters/%s", *e2e.GCPZone, nt.ClusterName) + } + + testutils.ClearMembershipInfo(nt, fleetMembership, *e2e.GCPProject, gkeURI) + testutils.ClearMembershipInfo(nt, fleetMembership, testutils.TestCrossProjectFleetProjectID, gkeURI) + + nt.T.Cleanup(func() { + testutils.ClearMembershipInfo(nt, fleetMembership, *e2e.GCPProject, gkeURI) + testutils.ClearMembershipInfo(nt, fleetMembership, testutils.TestCrossProjectFleetProjectID, gkeURI) + }) + + // Register the cluster for fleet workload identity test + if fleetWITest { + fleetProject := *e2e.GCPProject + if crossProject { + fleetProject = testutils.TestCrossProjectFleetProjectID + } + nt.T.Logf("Register the cluster to a fleet in project %q", fleetProject) + if err := testutils.RegisterCluster(nt, fleetMembership, fleetProject, gkeURI); err != nil { + nt.T.Fatalf("Failed to register the cluster to project %q: %v", fleetProject, err) + exists, err := testutils.FleetHasMembership(nt, fleetMembership, fleetProject) + if err != nil { + nt.T.Fatalf("Unable to check if membership exists: %v", err) + } + if !exists { + nt.T.Fatalf("The membership wasn't created") + } + } + nt.T.Logf("Restart the reconciler-manager to pick up the Membership") + // The reconciler manager checks if the Membership CRD exists before setting + // up the RootSync and RepoSync controllers: cmd/reconciler-manager/main.go:90. + // If the CRD exists, it configures the Membership watch. + // Otherwise, the watch is not configured to prevent the controller from crashing caused by an unknown CRD. + // DeletePodByLabel deletes the current reconciler-manager Pod so that new Pod + // can set up the watch. Once the watch is configured, it can detect the + // deletion and creation of the Membership, which implies cluster unregistration and registration. + // The underlying reconciler should be updated with FWI creds after the reconciler-manager restarts. + nomostest.DeletePodByLabel(nt, "app", reconcilermanager.ManagerName, false) + } +} + +type sourceSpec struct { + sourceType v1beta1.SourceType + sourceRepo string + sourceChart string + sourceVersion string + gsaEmail string + rootCommitFn nomostest.Sha1Func +} + +func pushSource(nt *nomostest.NT, sourceSpec *sourceSpec) error { + // For helm charts, we need to push the chart to the AR before configuring the RootSync + if sourceSpec.sourceType == v1beta1.HelmSource { + chart, err := artifactregistry.PushHelmChart(nt, sourceSpec.sourceChart, sourceSpec.sourceVersion) + if err != nil { + return err + } + + sourceSpec.sourceRepo = chart.Image.RepositoryOCI() + sourceSpec.sourceChart = chart.Image.Name + sourceSpec.sourceVersion = chart.Image.Version + sourceSpec.rootCommitFn = nomostest.HelmChartVersionShaFn(chart.Image.Version) + } + return nil +} + +// mustConfigureRootSync updates RootSync to sync with the provided auth. +// It reuses the RootSync instead of creating a new one so that test resources +// can be cleaned up after the test. +func mustConfigureRootSync(nt *nomostest.NT, rs *v1beta1.RootSync, tenant string, sourceSpec *sourceSpec) { + if err := pushSource(nt, sourceSpec); err != nil { + nt.T.Fatal(err) + } + nt.T.Logf("Update RootSync to sync %s from repo %s", tenant, sourceSpec.sourceRepo) + switch sourceSpec.sourceType { + case v1beta1.GitSource: + nt.MustMergePatch(rs, fmt.Sprintf(`{"spec": {"git": {"dir": "%s", "branch": "main", "repo": "%s", "auth": "gcpserviceaccount", "gcpServiceAccountEmail": "%s", "secretRef": {"name": ""}}}}`, + tenant, sourceSpec.sourceRepo, sourceSpec.gsaEmail)) + case v1beta1.OciSource: + nt.MustMergePatch(rs, fmt.Sprintf(`{"spec": {"sourceType": "%s", "oci": {"dir": "%s", "image": "%s", "auth": "gcpserviceaccount", "gcpServiceAccountEmail": "%s"}, "git": null}}`, + v1beta1.OciSource, tenant, sourceSpec.sourceRepo, sourceSpec.gsaEmail)) + case v1beta1.HelmSource: + // Set the helm re-pulling duration to 5s instead of relying on the default 1h, + // because updates to IAM policy bindings doesn't trigger a reconciliation. + nt.MustMergePatch(rs, fmt.Sprintf(`{"spec": {"sourceType": "%s", "helm": {"chart": "%s", "repo": "%s", "version": "%s", "auth": "gcpserviceaccount", "gcpServiceAccountEmail": "%s", "releaseName": "my-coredns", "namespace": "coredns", "period": "5s"}, "git": null}}`, + v1beta1.HelmSource, sourceSpec.sourceChart, sourceSpec.sourceRepo, sourceSpec.sourceVersion, sourceSpec.gsaEmail)) + } +} diff --git a/e2e/testinfra/terraform/common/service_accounts.tf b/e2e/testinfra/terraform/common/service_accounts.tf index 4c2e28d8fc..4bbb78b987 100644 --- a/e2e/testinfra/terraform/common/service_accounts.tf +++ b/e2e/testinfra/terraform/common/service_accounts.tf @@ -78,3 +78,25 @@ resource "google_service_account_iam_member" "e2e_metric_writer_gke_binding" { role = "roles/iam.workloadIdentityUser" member = "serviceAccount:${data.google_project.project.project_id}.svc.id.goog[config-management-monitoring/default]" } + +// A Google service account that doesn't have any permissions. +resource "google_service_account" "e2e_restricted_user_sa" { + account_id = "e2e-test-restricted-user" + display_name = "Test service account without reader permission or WI binding" + description = "Service account without reader permission or WI binding" + project = data.google_project.project.id +} + +// A Google service account that doesn't have any permissions, but has an IAM binding as a WI user. +resource "google_service_account" "e2e_restricted_and_impersonated_user_sa" { + account_id = "e2e-test-restricted-wi-user" + display_name = "Test impersonated service account without reader permission" + description = "Service account that is impersonated but has no reader permission" + project = data.google_project.project.id +} + +resource "google_service_account_iam_member" "e2e_e2e_restricted_and_impersonated_user_gke_binding" { + service_account_id = google_service_account.e2e_restricted_and_impersonated_user_sa.name + role = "roles/iam.workloadIdentityUser" + member = "serviceAccount:${data.google_project.project.project_id}.svc.id.goog[config-management-system/root-reconciler]" +}