diff --git a/api/v1/installation_types.go b/api/v1/installation_types.go index c8ac658acf..fc15ccb747 100644 --- a/api/v1/installation_types.go +++ b/api/v1/installation_types.go @@ -202,6 +202,12 @@ type InstallationSpec struct { // Azure is used to configure azure provider specific options. // +optional Azure *Azure `json:"azure,omitempty"` + + // Proxy is used to configure the HTTP(S) proxy settings that will be applied to Tigera containers that connect + // to destinations outside the cluster. It is expected that NO_PROXY is configured such that destinations within + // the cluster (including the API server) are exempt from proxying. + // +optional + Proxy *Proxy `json:"proxy,omitempty"` } type Azure struct { @@ -1048,3 +1054,68 @@ type WindowsNodeSpec struct { // +optional VXLANAdapter string `json:"vxlanAdapter,omitempty"` } + +type Proxy struct { + // HTTPProxy defines the value of the HTTP_PROXY environment variable that will be set on Tigera containers that connect to + // destinations outside the cluster. + // +optional + HTTPProxy string `json:"httpProxy,omitempty"` + + // HTTPSProxy defines the value of the HTTPS_PROXY environment variable that will be set on Tigera containers that connect to + // destinations outside the cluster. + // +optional + HTTPSProxy string `json:"httpsProxy,omitempty"` + + // NoProxy defines the value of the NO_PROXY environment variable that will be set on Tigera containers that connect to + // destinations outside the cluster. This value must be set such that destinations within the scope of the cluster, including + // the Kubernetes API server, are exempt from being proxied. + // +optional + NoProxy string `json:"noProxy,omitempty"` +} + +func (p *Proxy) EnvVars() (envVars []v1.EnvVar) { + if p == nil { + return + } + + if p.HTTPProxy != "" { + envVars = append(envVars, []v1.EnvVar{ + { + Name: "HTTP_PROXY", + Value: p.HTTPProxy, + }, + { + Name: "http_proxy", + Value: p.HTTPProxy, + }, + }...) + } + + if p.HTTPSProxy != "" { + envVars = append(envVars, []v1.EnvVar{ + { + Name: "HTTPS_PROXY", + Value: p.HTTPSProxy, + }, + { + Name: "https_proxy", + Value: p.HTTPSProxy, + }, + }...) + } + + if p.NoProxy != "" { + envVars = append(envVars, []v1.EnvVar{ + { + Name: "NO_PROXY", + Value: p.NoProxy, + }, + { + Name: "no_proxy", + Value: p.NoProxy, + }, + }...) + } + + return envVars +} diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index a5abf3ce37..c2fbed5d47 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -4422,6 +4422,11 @@ func (in *InstallationSpec) DeepCopyInto(out *InstallationSpec) { *out = new(Azure) (*in).DeepCopyInto(*out) } + if in.Proxy != nil { + in, out := &in.Proxy, &out.Proxy + *out = new(Proxy) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InstallationSpec. @@ -6751,6 +6756,21 @@ func (in *PrometheusSpec) DeepCopy() *PrometheusSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Proxy) DeepCopyInto(out *Proxy) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Proxy. +func (in *Proxy) DeepCopy() *Proxy { + if in == nil { + return nil + } + out := new(Proxy) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Retention) DeepCopyInto(out *Retention) { *out = *in diff --git a/pkg/controller/authentication/authentication_controller.go b/pkg/controller/authentication/authentication_controller.go index da791c8c7b..833b8643f3 100644 --- a/pkg/controller/authentication/authentication_controller.go +++ b/pkg/controller/authentication/authentication_controller.go @@ -18,6 +18,9 @@ import ( "context" "fmt" + "golang.org/x/net/http/httpproxy" + v1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/kubernetes" "github.com/tigera/operator/pkg/render/common/networkpolicy" @@ -156,13 +159,15 @@ var _ reconcile.Reconciler = &ReconcileAuthentication{} // ReconcileAuthentication reconciles an Authentication object type ReconcileAuthentication struct { - client client.Client - scheme *runtime.Scheme - provider oprv1.Provider - status status.StatusManager - clusterDomain string - tierWatchReady *utils.ReadyFlag - multiTenant bool + client client.Client + scheme *runtime.Scheme + provider oprv1.Provider + status status.StatusManager + clusterDomain string + tierWatchReady *utils.ReadyFlag + multiTenant bool + resolvedPodProxies []*httpproxy.Config + lastAvailabilityTransition metav1.Time } // Reconcile the cluster state with the Authentication object that is found in the cluster. @@ -306,6 +311,78 @@ func (r *ReconcileAuthentication) Reconcile(ctx context.Context, request reconci return reconcile.Result{}, err } + // Determine the current deployment availability. + var currentAvailabilityTransition metav1.Time + var currentlyAvailable bool + dexDeployment := v1.Deployment{} + err = r.client.Get(ctx, client.ObjectKey{Name: render.DexObjectName, Namespace: render.DexNamespace}, &dexDeployment) + if err != nil && !errors.IsNotFound(err) { + r.status.SetDegraded(oprv1.ResourceReadError, "Failed to read the deployment status of Dex", err, reqLogger) + return reconcile.Result{}, nil + } else if err == nil { + for _, condition := range dexDeployment.Status.Conditions { + if condition.Type == v1.DeploymentAvailable { + currentAvailabilityTransition = condition.LastTransitionTime + if condition.Status == corev1.ConditionTrue { + currentlyAvailable = true + } + break + } + } + } + + // Resolve the proxies used by each Dex pod. We only update the resolved proxies if the availability of the + // Dex deployment has changed since our last reconcile and the deployment is currently available. We restrict + // the resolution of pod proxies in this way to limit the number of pod queries we make. + if !currentAvailabilityTransition.Equal(&r.lastAvailabilityTransition) && currentlyAvailable { + // Query dex pods. + labelSelector := labels.SelectorFromSet(map[string]string{ + "app.kubernetes.io/name": render.DexObjectName, + }) + pods := corev1.PodList{} + err := r.client.List(ctx, &pods, &client.ListOptions{ + LabelSelector: labelSelector, + Namespace: render.DexNamespace, + }) + if err != nil { + r.status.SetDegraded(oprv1.ResourceReadError, "Failed to list the pods of the Dex deployment", err, reqLogger) + return reconcile.Result{}, nil + } + + // Resolve the proxy config for each pod. Pods without a proxy will have a nil proxy config value. + var podProxies []*httpproxy.Config + for _, pod := range pods.Items { + for _, container := range pod.Spec.Containers { + if container.Name == render.DexObjectName { + var podProxyConfig *httpproxy.Config + var httpProxy, httpsProxy, noProxy string + for _, env := range container.Env { + switch env.Name { + case "http_proxy", "HTTP_PROXY": + httpProxy = env.Value + case "https_proxy", "HTTPS_PROXY": + httpsProxy = env.Value + case "no_proxy", "NO_PROXY": + noProxy = env.Value + } + } + if httpProxy != "" || httpsProxy != "" || noProxy != "" { + podProxyConfig = &httpproxy.Config{ + HTTPProxy: httpProxy, + HTTPSProxy: httpsProxy, + NoProxy: noProxy, + } + } + + podProxies = append(podProxies, podProxyConfig) + } + } + } + + r.resolvedPodProxies = podProxies + } + r.lastAvailabilityTransition = currentAvailabilityTransition + disableDex := utils.IsDexDisabled(authentication) // DexConfig adds convenience methods around dex related objects in k8s and can be used to configure Dex. @@ -324,6 +401,7 @@ func (r *ReconcileAuthentication) Reconcile(ctx context.Context, request reconci TLSKeyPair: tlsKeyPair, TrustedBundle: trustedBundle, Authentication: authentication, + PodProxies: r.resolvedPodProxies, } // Render the desired objects from the CRD and create or update them. diff --git a/pkg/controller/authentication/authentication_controller_test.go b/pkg/controller/authentication/authentication_controller_test.go index 3b363a1681..2f245a2265 100644 --- a/pkg/controller/authentication/authentication_controller_test.go +++ b/pkg/controller/authentication/authentication_controller_test.go @@ -17,6 +17,10 @@ package authentication import ( "context" "fmt" + "net" + "net/url" + "strconv" + "strings" "time" . "github.com/onsi/ginkgo" @@ -24,8 +28,6 @@ import ( . "github.com/onsi/gomega" "github.com/stretchr/testify/mock" v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" - "github.com/tigera/operator/pkg/render/common/secret" - operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/apis" "github.com/tigera/operator/pkg/common" @@ -34,7 +36,11 @@ import ( "github.com/tigera/operator/pkg/controller/status" "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/common/networkpolicy" + "github.com/tigera/operator/pkg/render/common/secret" "github.com/tigera/operator/test" + "golang.org/x/net/http/httpproxy" + "k8s.io/apimachinery/pkg/runtime/schema" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" appsv1 "k8s.io/api/apps/v1" @@ -49,14 +55,16 @@ import ( var _ = Describe("authentication controller tests", func() { var ( - cli client.Client - scheme *runtime.Scheme - ctx context.Context - mockStatus *status.MockStatus - readyFlag *utils.ReadyFlag - idpSecret *corev1.Secret - auth *operatorv1.Authentication - replicas int32 + cli client.Client + scheme *runtime.Scheme + ctx context.Context + mockStatus *status.MockStatus + readyFlag *utils.ReadyFlag + idpSecret *corev1.Secret + installation *operatorv1.Installation + auth *operatorv1.Authentication + objTrackerWithCalls test.ObjectTrackerWithCalls + replicas int32 ) BeforeEach(func() { @@ -67,7 +75,8 @@ var _ = Describe("authentication controller tests", func() { Expect(rbacv1.SchemeBuilder.AddToScheme(scheme)).ShouldNot(HaveOccurred()) ctx = context.Background() - cli = ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + objTrackerWithCalls = test.NewObjectTrackerWithCalls(scheme) + cli = ctrlrfake.DefaultFakeClientBuilder(scheme).WithObjectTracker(&objTrackerWithCalls).Build() // Set up a mock status mockStatus = &status.MockStatus{} @@ -87,7 +96,8 @@ var _ = Describe("authentication controller tests", func() { certificateManager, err := certificatemanager.Create(cli, nil, "cluster.local", common.OperatorNamespace(), certificatemanager.AllowCACreation()) Expect(err).NotTo(HaveOccurred()) Expect(cli.Create(context.Background(), certificateManager.KeyPair().Secret(common.OperatorNamespace()))).NotTo(HaveOccurred()) - Expect(cli.Create(ctx, &operatorv1.Installation{ + + installation = &operatorv1.Installation{ ObjectMeta: metav1.ObjectMeta{ Name: "default", }, @@ -100,7 +110,8 @@ var _ = Describe("authentication controller tests", func() { Variant: operatorv1.TigeraSecureEnterprise, Registry: "some.registry.org/", }, - })).To(BeNil()) + } + Expect(cli.Create(ctx, installation)).To(BeNil()) Expect(cli.Create(ctx, &operatorv1.APIServer{ ObjectMeta: metav1.ObjectMeta{Name: "tigera-secure"}, Status: operatorv1.APIServerStatus{State: operatorv1.TigeraStatusReady}, @@ -169,7 +180,7 @@ var _ = Describe("authentication controller tests", func() { }, } Expect(cli.Create(ctx, ts)).NotTo(HaveOccurred()) - r := &ReconcileAuthentication{cli, scheme, operatorv1.ProviderNone, mockStatus, "", readyFlag, false} + r := &ReconcileAuthentication{cli, scheme, operatorv1.ProviderNone, mockStatus, "", readyFlag, false, []*httpproxy.Config{}, metav1.Now()} _, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{ Name: "authentication", Namespace: "", @@ -194,7 +205,7 @@ var _ = Describe("authentication controller tests", func() { Expect(cli.Create(ctx, ts)).NotTo(HaveOccurred()) - r := &ReconcileAuthentication{cli, scheme, operatorv1.ProviderNone, mockStatus, "", readyFlag, false} + r := &ReconcileAuthentication{cli, scheme, operatorv1.ProviderNone, mockStatus, "", readyFlag, false, []*httpproxy.Config{}, metav1.Now()} _, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{ Name: "authentication", Namespace: "", @@ -235,7 +246,7 @@ var _ = Describe("authentication controller tests", func() { }, } Expect(cli.Create(ctx, ts)).NotTo(HaveOccurred()) - r := &ReconcileAuthentication{cli, scheme, operatorv1.ProviderNone, mockStatus, "", readyFlag, false} + r := &ReconcileAuthentication{cli, scheme, operatorv1.ProviderNone, mockStatus, "", readyFlag, false, []*httpproxy.Config{}, metav1.Now()} _, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{ Name: "authentication", Namespace: "", @@ -295,7 +306,7 @@ var _ = Describe("authentication controller tests", func() { }, } Expect(cli.Create(ctx, ts)).NotTo(HaveOccurred()) - r := &ReconcileAuthentication{cli, scheme, operatorv1.ProviderNone, mockStatus, "", readyFlag, false} + r := &ReconcileAuthentication{cli, scheme, operatorv1.ProviderNone, mockStatus, "", readyFlag, false, []*httpproxy.Config{}, metav1.Now()} _, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{ Name: "authentication", Namespace: "", @@ -340,7 +351,7 @@ var _ = Describe("authentication controller tests", func() { Expect(cli.Create(ctx, auth)).ToNot(HaveOccurred()) // Reconcile - r := &ReconcileAuthentication{cli, scheme, operatorv1.ProviderNone, mockStatus, "", readyFlag, false} + r := &ReconcileAuthentication{cli, scheme, operatorv1.ProviderNone, mockStatus, "", readyFlag, false, []*httpproxy.Config{}, metav1.Now()} _, err := r.Reconcile(ctx, reconcile.Request{}) Expect(err).ShouldNot(HaveOccurred()) authentication, err := utils.GetAuthentication(ctx, cli) @@ -496,7 +507,241 @@ var _ = Describe("authentication controller tests", func() { r.tierWatchReady = &utils.ReadyFlag{} test.ExpectWaitForTierWatch(ctx, r, mockStatus) }) + + Context("Proxy detection", func() { + cases := []test.ProxyTestCase{ + { + PodProxies: []*test.ProxyConfig{{ + HTTPProxy: "http://proxy.io", + }}, + }, + { + PodProxies: []*test.ProxyConfig{{ + HTTPSProxy: "https://proxy.io", + }}, + }, + { + PodProxies: []*test.ProxyConfig{{ + HTTPProxy: "http://proxy.io", + HTTPSProxy: "https://proxy.io", + }}, + }, + { + PodProxies: []*test.ProxyConfig{{ + HTTPProxy: "http://proxy.io", + HTTPSProxy: "https://192.168.0.2:9000", + }}, + }, + { + PodProxies: []*test.ProxyConfig{{ + HTTPProxy: "http://192.168.0.1:9000", + HTTPSProxy: "https://192.168.0.1:9000", + }}, + Lowercase: true, + }, + { + PodProxies: []*test.ProxyConfig{ + { + HTTPProxy: "http://proxy.io:9000", + HTTPSProxy: "https://proxy.io:9000", + }, + { + HTTPProxy: "http://proxy.io:9000", + HTTPSProxy: "https://proxy.io:9000", + }, + }, + }, + } + + for _, testCase := range cases { + Describe(fmt.Sprintf("Proxy detection when %+v", test.PrettyFormatProxyTestCase(testCase)), func() { + // Set up the test based on the test case. + BeforeEach(func() { + mockStatus.On("ReadyToMonitor") + mockStatus.On("SetMetaData", mock.Anything).Return() + mockStatus.On("AddDeployments", mock.Anything) + mockStatus.On("ClearDegraded", mock.Anything) + mockStatus.On("IsAvailable").Return(true) + + // Create the pod whichs back the deployment and have the appropriate proxy settings. + // idp-resolution: If we update the controller to resolve the specific IdP and use that for + // policy calculation, we'll need to set the IdP on the Authentication CR here. + for i, proxy := range testCase.PodProxies { + createPodWithProxy(ctx, cli, proxy, testCase.Lowercase, i) + } + }) + + It(fmt.Sprintf("detects proxy correctly when %+v", test.PrettyFormatProxyTestCase(testCase)), func() { + // First reconcile creates the dex deployment without any availability condition. + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + + // Validate that we made no calls to get Pods at this stage. + podGVR := schema.GroupVersionResource{ + Version: "v1", + Resource: "pods", + } + Expect(objTrackerWithCalls.CallCount(podGVR, test.ObjectTrackerCallList)).To(BeZero()) + + // Set the deployment to be unavailable. We need to recreate the deployment otherwise the status update is ignored. + gd := appsv1.Deployment{} + err = cli.Get(ctx, client.ObjectKey{Name: "tigera-dex", Namespace: "tigera-dex"}, &gd) + Expect(err).NotTo(HaveOccurred()) + err = cli.Delete(ctx, &gd) + Expect(err).NotTo(HaveOccurred()) + gd.ResourceVersion = "" + gd.Status.Conditions = []appsv1.DeploymentCondition{{ + Type: appsv1.DeploymentAvailable, + Status: corev1.ConditionFalse, + LastTransitionTime: metav1.Time{Time: time.Now()}, + }} + err = cli.Create(ctx, &gd) + Expect(err).NotTo(HaveOccurred()) + + // Reconcile again. We should see no calls since the deployment has not transitioned to available. + _, err = r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + Expect(objTrackerWithCalls.CallCount(podGVR, test.ObjectTrackerCallList)).To(Equal(0)) + + // Set the deployment to available. + err = cli.Delete(ctx, &gd) + Expect(err).NotTo(HaveOccurred()) + gd.ResourceVersion = "" + gd.Status.Conditions = []appsv1.DeploymentCondition{{ + Type: appsv1.DeploymentAvailable, + Status: corev1.ConditionTrue, + LastTransitionTime: metav1.Time{Time: time.Now().Add(time.Minute)}, + }} + err = cli.Create(ctx, &gd) + Expect(err).NotTo(HaveOccurred()) + + // Reconcile again. The proxy detection logic should kick in since the dex deployment is ready. + _, err = r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + Expect(objTrackerWithCalls.CallCount(podGVR, test.ObjectTrackerCallList)).To(Equal(1)) + + // Resolve the allow-tigera policy for Dex. + policies := v3.NetworkPolicyList{} + Expect(cli.List(ctx, &policies)).ToNot(HaveOccurred()) + Expect(policies.Items).To(HaveLen(2)) + Expect(policies.Items[0].Name).To(Equal("allow-tigera.allow-tigera-dex")) + policy := policies.Items[0] + + // Generate the expectation based on the test case, and compare the rendered rules to our expectations. + expectedEgressRules := getExpectedEgressDestinationRulesFromCase(testCase) + var renderedEgressRules []v3.EntityRule + for _, egressRule := range policy.Spec.Egress[2:] { + renderedEgressRules = append(renderedEgressRules, egressRule.Destination) + } + Expect(policy.Spec.Egress).To(HaveLen(2 + len(expectedEgressRules))) + Expect(renderedEgressRules).To(ContainElements(expectedEgressRules)) + + // Reconcile again. Verify that we do not cause any additional query for pods now that we have resolved the proxy. + _, err = r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + Expect(objTrackerWithCalls.CallCount(podGVR, test.ObjectTrackerCallList)).To(Equal(1)) + }) + }) + } + }) }) + + Context("Proxy setting", func() { + DescribeTable("sets the proxy", func(http, https, noProxy bool) { + // Setup valid auth configuration. + auth.Spec.OIDC = &operatorv1.AuthenticationOIDC{ + IssuerURL: "https://example.com", + UsernameClaim: "email", + GroupsClaim: "group", + GroupsPrefix: "g", + UsernamePrefix: "u", + } + Expect(cli.Create(ctx, auth)).ToNot(HaveOccurred()) + Expect(cli.Create(ctx, idpSecret)).ToNot(HaveOccurred()) + + // Set up the proxy. + installationCopy := installation.DeepCopy() + installationCopy.Spec.Proxy = &operatorv1.Proxy{} + if http { + installationCopy.Spec.Proxy.HTTPProxy = "test-http-proxy" + } + if https { + installationCopy.Spec.Proxy.HTTPSProxy = "test-https-proxy" + } + if noProxy { + installationCopy.Spec.Proxy.NoProxy = "test-no-proxy" + } + err := cli.Update(ctx, installationCopy) + Expect(err).NotTo(HaveOccurred()) + + // Reconcile to create the dex deployment. + r := ReconcileAuthentication{ + client: cli, + scheme: scheme, + provider: operatorv1.ProviderNone, + status: mockStatus, + tierWatchReady: readyFlag, + } + _, err = r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + + // Get the deployment and validate the env vars. + dd := appsv1.Deployment{} + err = cli.Get(ctx, client.ObjectKey{Name: "tigera-dex", Namespace: "tigera-dex"}, &dd) + Expect(err).NotTo(HaveOccurred()) + + var expectedEnvVars []corev1.EnvVar + if http { + expectedEnvVars = append(expectedEnvVars, + corev1.EnvVar{ + Name: "HTTP_PROXY", + Value: "test-http-proxy", + }, + corev1.EnvVar{ + Name: "http_proxy", + Value: "test-http-proxy", + }, + ) + } + + if https { + expectedEnvVars = append(expectedEnvVars, + corev1.EnvVar{ + Name: "HTTPS_PROXY", + Value: "test-https-proxy", + }, + corev1.EnvVar{ + Name: "https_proxy", + Value: "test-https-proxy", + }, + ) + } + + if noProxy { + expectedEnvVars = append(expectedEnvVars, + corev1.EnvVar{ + Name: "NO_PROXY", + Value: "test-no-proxy", + }, + corev1.EnvVar{ + Name: "no_proxy", + Value: "test-no-proxy", + }, + ) + } + + Expect(dd.Spec.Template.Spec.Containers).To(HaveLen(1)) + Expect(dd.Spec.Template.Spec.Containers[0].Env).To(ContainElements(expectedEnvVars)) + }, + Entry("http/https/noProxy", true, true, true), + Entry("http", true, false, false), + Entry("https", false, true, false), + Entry("http/https", true, true, false), + Entry("http/noProxy", true, false, true), + Entry("https/noProxy", false, true, true), + ) + }) + tls, err := secret.CreateTLSSecret(nil, "a", "a", corev1.TLSPrivateKeyKey, corev1.TLSCertKey, time.Hour, nil, "a") Expect(err).NotTo(HaveOccurred()) validCert := tls.Data[corev1.TLSCertKey] @@ -519,7 +764,7 @@ var _ = Describe("authentication controller tests", func() { } Expect(cli.Create(ctx, idpSecret)).ToNot(HaveOccurred()) Expect(cli.Create(ctx, auth)).ToNot(HaveOccurred()) - r := &ReconcileAuthentication{cli, scheme, operatorv1.ProviderNone, mockStatus, "", readyFlag, false} + r := &ReconcileAuthentication{cli, scheme, operatorv1.ProviderNone, mockStatus, "", readyFlag, false, []*httpproxy.Config{}, metav1.Now()} _, err := r.Reconcile(ctx, reconcile.Request{}) if expectReconcilePass { Expect(err).ToNot(HaveOccurred()) @@ -638,3 +883,133 @@ func copyAndAddPromptTypes(auth *operatorv1.AuthenticationOIDC, promptTypes []op copy.PromptTypes = promptTypes return copy } + +func createPodWithProxy(ctx context.Context, c client.Client, config *test.ProxyConfig, lowercase bool, replicaNum int) { + pod := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tigera-dex" + strconv.Itoa(replicaNum), + Namespace: "tigera-dex", + Labels: map[string]string{ + "k8s-app": "tigera-dex", + "app.kubernetes.io/name": "tigera-dex", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "tigera-dex", + Env: []corev1.EnvVar{}, + }}, + }, + } + + if config != nil { + // Set the env vars. + httpsProxyVarName := "HTTPS_PROXY" + httpProxyVarName := "HTTP_PROXY" + noProxyVarName := "NO_PROXY" + if lowercase { + httpsProxyVarName = strings.ToLower(httpsProxyVarName) + httpProxyVarName = strings.ToLower(httpProxyVarName) + noProxyVarName = strings.ToLower(noProxyVarName) + } + // Environment variables that are empty can be represented as an unset variable or a set variable with an empty string. + // For our tests, we'll represent them as an unset variable. + if config.HTTPProxy != "" { + pod.Spec.Containers[0].Env = append(pod.Spec.Containers[0].Env, corev1.EnvVar{ + Name: httpProxyVarName, + Value: config.HTTPProxy, + }) + } + if config.HTTPSProxy != "" { + pod.Spec.Containers[0].Env = append(pod.Spec.Containers[0].Env, corev1.EnvVar{ + Name: httpsProxyVarName, + Value: config.HTTPSProxy, + }) + } + if config.NoProxy != "" { + pod.Spec.Containers[0].Env = append(pod.Spec.Containers[0].Env, corev1.EnvVar{ + Name: noProxyVarName, + Value: config.NoProxy, + }) + } + } + + err := c.Create(ctx, &pod) + Expect(err).NotTo(HaveOccurred()) +} + +// getExpectedEgressDestinationRulesFromCase returns the expected rules based on the current test case. It assumes that +// no IdP resolution is occurring, and all potential destinations are allowed. +// idp-resolution: If the controller is updated to resolve the specific IdP destination, this function should be +// updated to return a single egress rule based on the IdP destination and the proxy/no-proxy settings. +func getExpectedEgressDestinationRulesFromCase(c test.ProxyTestCase) []v3.EntityRule { + var egressRules []v3.EntityRule + observedDestinations := map[string]bool{} + + // Generate expected proxy rules. + for _, proxy := range c.PodProxies { + var proxyURLs []string + + if proxy.HTTPProxy != "" { + proxyURLs = append(proxyURLs, proxy.HTTPProxy) + } + + if proxy.HTTPSProxy != "" { + proxyURLs = append(proxyURLs, proxy.HTTPSProxy) + } + + for _, proxyURLString := range proxyURLs { + proxyURL, err := url.ParseRequestURI(proxyURLString) + Expect(err).NotTo(HaveOccurred()) + + // Resolve host and port + var host string + var port uint16 + hostSplit := strings.Split(proxyURL.Host, ":") + switch { + case len(hostSplit) == 2: + port64, err := strconv.ParseUint(hostSplit[1], 10, 16) + Expect(err).NotTo(HaveOccurred()) + host = hostSplit[0] + port = uint16(port64) + case proxyURL.Scheme == "https": + host = proxyURL.Host + port = 443 + default: + host = proxyURL.Host + port = 80 + } + hostIsIP := net.ParseIP(host) != nil + + hostPortString := fmt.Sprintf("%v:%v", host, port) + if observedDestinations[hostPortString] { + continue + } + + if hostIsIP { + egressRules = append(egressRules, v3.EntityRule{ + Nets: []string{fmt.Sprintf("%s/32", host)}, + Ports: networkpolicy.Ports(port), + }) + } else { + egressRules = append(egressRules, v3.EntityRule{ + Domains: []string{host}, + Ports: networkpolicy.Ports(port), + }) + } + + observedDestinations[hostPortString] = true + } + } + + // Add expected target rules. + egressRules = append(egressRules, v3.EntityRule{ + Nets: []string{"0.0.0.0/0"}, + Ports: networkpolicy.Ports(443, 6443, 389, 636), + }) + egressRules = append(egressRules, v3.EntityRule{ + Nets: []string{"::/0"}, + Ports: networkpolicy.Ports(443, 6443, 389, 636), + }) + return egressRules +} diff --git a/pkg/controller/clusterconnection/clusterconnection_controller_test.go b/pkg/controller/clusterconnection/clusterconnection_controller_test.go index d3d96e708b..1c6930f113 100644 --- a/pkg/controller/clusterconnection/clusterconnection_controller_test.go +++ b/pkg/controller/clusterconnection/clusterconnection_controller_test.go @@ -24,6 +24,7 @@ import ( "time" . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" "github.com/stretchr/testify/mock" "github.com/tigera/operator/pkg/render/common/networkpolicy" @@ -58,6 +59,7 @@ var _ = Describe("ManagementClusterConnection controller tests", func() { var c client.Client var ctx context.Context var cfg *operatorv1.ManagementClusterConnection + var installation *operatorv1.Installation var r reconcile.Reconciler var clientScheme *runtime.Scheme var dpl *appsv1.Deployment @@ -135,22 +137,22 @@ var _ = Describe("ManagementClusterConnection controller tests", func() { } err = c.Create(ctx, cfg) Expect(err).NotTo(HaveOccurred()) - err = c.Create( - ctx, - &operatorv1.Installation{ - Spec: operatorv1.InstallationSpec{ - Variant: operatorv1.TigeraSecureEnterprise, - Registry: "some.registry.org/", - }, - ObjectMeta: metav1.ObjectMeta{Name: "default"}, - Status: operatorv1.InstallationStatus{ - Variant: operatorv1.TigeraSecureEnterprise, - Computed: &operatorv1.InstallationSpec{ - Registry: "my-reg", - KubernetesProvider: operatorv1.ProviderNone, - }, + + installation = &operatorv1.Installation{ + Spec: operatorv1.InstallationSpec{ + Variant: operatorv1.TigeraSecureEnterprise, + Registry: "some.registry.org/", + }, + ObjectMeta: metav1.ObjectMeta{Name: "default"}, + Status: operatorv1.InstallationStatus{ + Variant: operatorv1.TigeraSecureEnterprise, + Computed: &operatorv1.InstallationSpec{ + Registry: "my-reg", + KubernetesProvider: operatorv1.ProviderNone, }, - }) + }, + } + err = c.Create(ctx, installation) Expect(err).NotTo(HaveOccurred()) }) @@ -358,20 +360,20 @@ var _ = Describe("ManagementClusterConnection controller tests", func() { testCases := append(coreCases, multiPodCases...) for _, testCase := range testCases { - Describe(fmt.Sprintf("Proxy detection when %+v", prettyFormatTestCase(testCase)), func() { + Describe(fmt.Sprintf("Proxy detection when %+v", test.PrettyFormatProxyTestCase(testCase)), func() { // Set up the test based on the test case. BeforeEach(func() { - for i, proxy := range testCase.podProxies { - createPodWithProxy(ctx, c, proxy, testCase.lowercase, i) + for i, proxy := range testCase.PodProxies { + createPodWithProxy(ctx, c, proxy, testCase.Lowercase, i) } // Set the target - cfg.Spec.ManagementClusterAddr = testCase.target + cfg.Spec.ManagementClusterAddr = testCase.Target err := c.Update(ctx, cfg) Expect(err).NotTo(HaveOccurred()) }) - It(fmt.Sprintf("detects proxy correctly when %+v", prettyFormatTestCase(testCase)), func() { + It(fmt.Sprintf("detects proxy correctly when %+v", test.PrettyFormatProxyTestCase(testCase)), func() { // First reconcile creates the guardian deployment without any availability condition. _, err := r.Reconcile(ctx, reconcile.Request{}) Expect(err).ShouldNot(HaveOccurred()) @@ -452,6 +454,85 @@ var _ = Describe("ManagementClusterConnection controller tests", func() { }) }) + Context("Proxy setting", func() { + DescribeTable("sets the proxy", func(http, https, noProxy bool) { + installationCopy := installation.DeepCopy() + installationCopy.Spec.Proxy = &operatorv1.Proxy{} + + if http { + installationCopy.Spec.Proxy.HTTPProxy = "test-http-proxy" + } + if https { + installationCopy.Spec.Proxy.HTTPSProxy = "test-https-proxy" + } + if noProxy { + installationCopy.Spec.Proxy.NoProxy = "test-no-proxy" + } + + err := c.Update(ctx, installationCopy) + Expect(err).NotTo(HaveOccurred()) + + // Reconcile creates the guardian deployment. + _, err = r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + + // Get the deployment and validate the env vars. + gd := appsv1.Deployment{} + err = c.Get(ctx, client.ObjectKey{Name: "tigera-guardian", Namespace: "tigera-guardian"}, &gd) + Expect(err).NotTo(HaveOccurred()) + + var expectedEnvVars []v1.EnvVar + if http { + expectedEnvVars = append(expectedEnvVars, + v1.EnvVar{ + Name: "HTTP_PROXY", + Value: "test-http-proxy", + }, + v1.EnvVar{ + Name: "http_proxy", + Value: "test-http-proxy", + }, + ) + } + + if https { + expectedEnvVars = append(expectedEnvVars, + v1.EnvVar{ + Name: "HTTPS_PROXY", + Value: "test-https-proxy", + }, + v1.EnvVar{ + Name: "https_proxy", + Value: "test-https-proxy", + }, + ) + } + + if noProxy { + expectedEnvVars = append(expectedEnvVars, + v1.EnvVar{ + Name: "NO_PROXY", + Value: "test-no-proxy", + }, + v1.EnvVar{ + Name: "no_proxy", + Value: "test-no-proxy", + }, + ) + } + + Expect(gd.Spec.Template.Spec.Containers).To(HaveLen(1)) + Expect(gd.Spec.Template.Spec.Containers[0].Env).To(ContainElements(expectedEnvVars)) + }, + Entry("http/https/noProxy", true, true, true), + Entry("http", true, false, false), + Entry("https", false, true, false), + Entry("http/https", true, true, false), + Entry("http/noProxy", true, false, true), + Entry("https/noProxy", false, true, true), + ) + }) + Context("Reconcile for Condition status", func() { generation := int64(2) It("should reconcile with empty tigerastatus conditions ", func() { @@ -627,7 +708,7 @@ var _ = Describe("ManagementClusterConnection controller tests", func() { }) }) -func createPodWithProxy(ctx context.Context, c client.Client, config *proxyConfig, lowercase bool, replicaNum int) { +func createPodWithProxy(ctx context.Context, c client.Client, config *test.ProxyConfig, lowercase bool, replicaNum int) { pod := v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "tigera-guardian" + strconv.Itoa(replicaNum), @@ -657,10 +738,10 @@ func createPodWithProxy(ctx context.Context, c client.Client, config *proxyConfi } // Environment variables that are empty can be represented as an unset variable or a set variable with an empty string. // For our tests, we'll represent them as an unset variable. - if config.httpsProxy != "" { + if config.HTTPSProxy != "" { pod.Spec.Containers[0].Env = append(pod.Spec.Containers[0].Env, v1.EnvVar{ Name: httpsProxyVarName, - Value: config.httpsProxy, + Value: config.HTTPSProxy, }) // Add a static HTTP_PROXY variable to catch any scenarios where the controller picks the wrong env var (Guardian uses HTTPS). pod.Spec.Containers[0].Env = append(pod.Spec.Containers[0].Env, v1.EnvVar{ @@ -668,10 +749,10 @@ func createPodWithProxy(ctx context.Context, c client.Client, config *proxyConfi Value: "http://wrong-proxy-url.com/", }) } - if config.noProxy != "" { + if config.NoProxy != "" { pod.Spec.Containers[0].Env = append(pod.Spec.Containers[0].Env, v1.EnvVar{ Name: noProxyVarName, - Value: config.noProxy, + Value: config.NoProxy, }) } } @@ -680,17 +761,6 @@ func createPodWithProxy(ctx context.Context, c client.Client, config *proxyConfi Expect(err).NotTo(HaveOccurred()) } -type proxyTestCase struct { - lowercase bool - target string - podProxies []*proxyConfig -} - -type proxyConfig struct { - httpsProxy string - noProxy string -} - type expectedEgressRule struct { host string port uint16 @@ -698,11 +768,11 @@ type expectedEgressRule struct { isProxied bool } -func generateCoreProxyTestCases(targetDomain, targetIP, proxyDomain, proxyIP, proxyPort string) []proxyTestCase { - var cases []proxyTestCase +func generateCoreProxyTestCases(targetDomain, targetIP, proxyDomain, proxyIP, proxyPort string) []test.ProxyTestCase { + var cases []test.ProxyTestCase // We will collect the cases by target type. Targets are in the form of ip:port or domain:port. for _, target := range []string{targetDomain, targetIP} { - var casesByTargetType []proxyTestCase + var casesByTargetType []test.ProxyTestCase // Generate the proxy strings. They can be http or https, use a domain or IP host, and can optionally specify a port. var proxyStrings []string for _, scheme := range []string{"http", "https"} { @@ -759,18 +829,18 @@ func generateCoreProxyTestCases(targetDomain, targetIP, proxyDomain, proxyIP, pr for _, lowercase := range []bool{true, false} { for _, proxyString := range proxyStrings { for _, noProxyString := range noProxyStrings { - testCase := proxyTestCase{ - lowercase: lowercase, - target: target, + testCase := test.ProxyTestCase{ + Lowercase: lowercase, + Target: target, } - var podProxyConfig *proxyConfig + var podProxyConfig *test.ProxyConfig if proxyString != "" || noProxyString != "" { - podProxyConfig = &proxyConfig{ - httpsProxy: proxyString, - noProxy: noProxyString, + podProxyConfig = &test.ProxyConfig{ + HTTPSProxy: proxyString, + NoProxy: noProxyString, } } - testCase.podProxies = []*proxyConfig{podProxyConfig} + testCase.PodProxies = []*test.ProxyConfig{podProxyConfig} casesByTargetType = append(casesByTargetType, testCase) } } @@ -780,19 +850,19 @@ func generateCoreProxyTestCases(targetDomain, targetIP, proxyDomain, proxyIP, pr return cases } -func getExpectedEgressRulesFromCase(c proxyTestCase) []expectedEgressRule { +func getExpectedEgressRulesFromCase(c test.ProxyTestCase) []expectedEgressRule { var expectedEgressRules []expectedEgressRule expectedEgressRulesAdded := map[expectedEgressRule]bool{} - for _, proxy := range c.podProxies { + for _, proxy := range c.PodProxies { var isProxied bool - if proxy != nil && proxy.httpsProxy != "" { - if proxy.noProxy == "" { + if proxy != nil && proxy.HTTPSProxy != "" { + if proxy.NoProxy == "" { isProxied = true } else { var proxyIsExempt bool - for _, noProxySubstring := range strings.Split(proxy.noProxy, ",") { - if strings.Contains(c.target, noProxySubstring) { + for _, noProxySubstring := range strings.Split(proxy.NoProxy, ",") { + if strings.Contains(c.Target, noProxySubstring) { proxyIsExempt = true break } @@ -806,7 +876,7 @@ func getExpectedEgressRulesFromCase(c proxyTestCase) []expectedEgressRule { var host string var port uint16 if isProxied { - proxyURL, err := url.ParseRequestURI(proxy.httpsProxy) + proxyURL, err := url.ParseRequestURI(proxy.HTTPSProxy) Expect(err).NotTo(HaveOccurred()) // Resolve port @@ -829,7 +899,7 @@ func getExpectedEgressRulesFromCase(c proxyTestCase) []expectedEgressRule { } else { var portString string var err error - host, portString, err = net.SplitHostPort(c.target) + host, portString, err = net.SplitHostPort(c.Target) Expect(err).NotTo(HaveOccurred()) port64, err := strconv.ParseUint(portString, 10, 16) Expect(err).NotTo(HaveOccurred()) @@ -852,86 +922,73 @@ func getExpectedEgressRulesFromCase(c proxyTestCase) []expectedEgressRule { return expectedEgressRules } -func multiplePodCases() []proxyTestCase { - return []proxyTestCase{ +func multiplePodCases() []test.ProxyTestCase { + return []test.ProxyTestCase{ // Mainline case with multiple pods: both have the same proxy. { - target: "voltron:9000", - podProxies: []*proxyConfig{ + Target: "voltron:9000", + PodProxies: []*test.ProxyConfig{ { - httpsProxy: "http://proxy.io/", - noProxy: "nomatch", + HTTPSProxy: "http://proxy.io/", + NoProxy: "nomatch", }, { - httpsProxy: "http://proxy.io/", - noProxy: "nomatch", + HTTPSProxy: "http://proxy.io/", + NoProxy: "nomatch", }, }, }, // Mainline case with multiple pods: neither have a proxy. { - target: "voltron:9000", - podProxies: []*proxyConfig{nil, nil}, + Target: "voltron:9000", + PodProxies: []*test.ProxyConfig{nil, nil}, }, // One pod has a proxy, one pod does not. { - target: "voltron:9000", - podProxies: []*proxyConfig{{httpsProxy: "http://proxy.io/", noProxy: "nomatch"}, nil}, + Target: "voltron:9000", + PodProxies: []*test.ProxyConfig{{HTTPSProxy: "http://proxy.io/", NoProxy: "nomatch"}, nil}, }, // Pods have different proxies. { - target: "voltron:9000", - podProxies: []*proxyConfig{ + Target: "voltron:9000", + PodProxies: []*test.ProxyConfig{ { - httpsProxy: "http://proxy.io/", - noProxy: "nomatch", + HTTPSProxy: "http://proxy.io/", + NoProxy: "nomatch", }, { - httpsProxy: "http://proxy-number-two.io/", - noProxy: "nomatch", + HTTPSProxy: "http://proxy-number-two.io/", + NoProxy: "nomatch", }, }, }, // Pods have different proxies, but one of them is exempt from the proxy. { - target: "voltron:9000", - podProxies: []*proxyConfig{ + Target: "voltron:9000", + PodProxies: []*test.ProxyConfig{ { - httpsProxy: "http://proxy.io/", - noProxy: "nomatch", + HTTPSProxy: "http://proxy.io/", + NoProxy: "nomatch", }, { - httpsProxy: "http://proxy-number-two.io/", - noProxy: "voltron", + HTTPSProxy: "http://proxy-number-two.io/", + NoProxy: "voltron", }, }, }, // Pods have different proxies, but both of them are exempt from the proxy. { - target: "voltron:9000", - podProxies: []*proxyConfig{ + Target: "voltron:9000", + PodProxies: []*test.ProxyConfig{ { - httpsProxy: "http://proxy.io/", - noProxy: "voltron", + HTTPSProxy: "http://proxy.io/", + NoProxy: "voltron", }, { - httpsProxy: "http://proxy-number-two.io/", - noProxy: "voltron", + HTTPSProxy: "http://proxy-number-two.io/", + NoProxy: "voltron", }, }, }, } } - -func prettyFormatTestCase(testCase proxyTestCase) string { - var containerProxies []string - for _, containerProxy := range testCase.podProxies { - if containerProxy == nil { - containerProxies = append(containerProxies, "nil") - } else { - containerProxies = append(containerProxies, fmt.Sprintf("{httpsProxy: %s, noProxy: %s}", containerProxy.httpsProxy, containerProxy.noProxy)) - } - } - - return fmt.Sprintf("lowercase: %v, target: %s, containerProxies: [%s]", testCase.lowercase, testCase.target, strings.Join(containerProxies, ",")) -} diff --git a/pkg/controller/utils/merge.go b/pkg/controller/utils/merge.go index 6903a68acf..38257cdb78 100644 --- a/pkg/controller/utils/merge.go +++ b/pkg/controller/utils/merge.go @@ -214,6 +214,11 @@ func OverrideInstallationSpec(cfg, override operatorv1.InstallationSpec) operato inst.Azure = override.Azure } + switch compareFields(inst.Proxy, override.Proxy) { + case BOnlySet, Different: + inst.Proxy = override.Proxy + } + return inst } diff --git a/pkg/crds/operator/operator.tigera.io_installations.yaml b/pkg/crds/operator/operator.tigera.io_installations.yaml index d4d6dac4c5..085f068dc7 100644 --- a/pkg/crds/operator/operator.tigera.io_installations.yaml +++ b/pkg/crds/operator/operator.tigera.io_installations.yaml @@ -6635,6 +6635,29 @@ spec: description: NonPrivileged configures Calico to be run in non-privileged containers as non-root users where possible. type: string + proxy: + description: |- + Proxy is used to configure the HTTP(S) proxy settings that will be applied to Tigera containers that connect + to destinations outside the cluster. It is expected that NO_PROXY is configured such that destinations within + the cluster (including the API server) are exempt from proxying. + properties: + httpProxy: + description: |- + HTTPProxy defines the value of the HTTP_PROXY environment variable that will be set on Tigera containers that connect to + destinations outside the cluster. + type: string + httpsProxy: + description: |- + HTTPSProxy defines the value of the HTTPS_PROXY environment variable that will be set on Tigera containers that connect to + destinations outside the cluster. + type: string + noProxy: + description: |- + NoProxy defines the value of the NO_PROXY environment variable that will be set on Tigera containers that connect to + destinations outside the cluster. This value must be set such that destinations within the scope of the cluster, including + the Kubernetes API server, are exempt from being proxied. + type: string + type: object registry: description: |- Registry is the default Docker registry used for component Docker images. @@ -15054,6 +15077,29 @@ spec: description: NonPrivileged configures Calico to be run in non-privileged containers as non-root users where possible. type: string + proxy: + description: |- + Proxy is used to configure the HTTP(S) proxy settings that will be applied to Tigera containers that connect + to destinations outside the cluster. It is expected that NO_PROXY is configured such that destinations within + the cluster (including the API server) are exempt from proxying. + properties: + httpProxy: + description: |- + HTTPProxy defines the value of the HTTP_PROXY environment variable that will be set on Tigera containers that connect to + destinations outside the cluster. + type: string + httpsProxy: + description: |- + HTTPSProxy defines the value of the HTTPS_PROXY environment variable that will be set on Tigera containers that connect to + destinations outside the cluster. + type: string + noProxy: + description: |- + NoProxy defines the value of the NO_PROXY environment variable that will be set on Tigera containers that connect to + destinations outside the cluster. This value must be set such that destinations within the scope of the cluster, including + the Kubernetes API server, are exempt from being proxied. + type: string + type: object registry: description: |- Registry is the default Docker registry used for component Docker images. diff --git a/pkg/render/dex.go b/pkg/render/dex.go index 707d988d47..28fdded1e2 100644 --- a/pkg/render/dex.go +++ b/pkg/render/dex.go @@ -17,10 +17,14 @@ package render import ( "fmt" + "net" + "net/url" "strings" + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + "github.com/tigera/api/pkg/lib/numorstring" + "golang.org/x/net/http/httpproxy" "gopkg.in/yaml.v2" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -28,8 +32,6 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client" - v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" - operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" @@ -73,6 +75,11 @@ type DexComponentConfiguration struct { TrustedBundle certificatemanagement.TrustedBundle Authentication *operatorv1.Authentication + + // PodProxies represents the resolved proxy configuration for each Dex pod. + // If this slice is empty, then resolution has not yet occurred. Pods with no proxy + // configured are represented with a nil value. + PodProxies []*httpproxy.Config } type dexComponent struct { @@ -231,6 +238,9 @@ func (c *dexComponent) deployment() client.Object { tolerations = append(tolerations, rmeta.TolerateGKEARM64NoSchedule) } + envVars := c.cfg.DexConfig.RequiredEnv("") + envVars = append(envVars, c.cfg.Installation.Proxy.EnvVars()...) + d := &appsv1.Deployment{ TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{ @@ -259,7 +269,7 @@ func (c *dexComponent) deployment() client.Object { Name: DexObjectName, Image: c.image, ImagePullPolicy: ImagePullPolicy(), - Env: c.cfg.DexConfig.RequiredEnv(""), + Env: envVars, LivenessProbe: c.probe(), SecurityContext: securitycontext.NewNonRootContext(), @@ -398,25 +408,10 @@ func (c *dexComponent) allowTigeraNetworkPolicy(installationVariant operatorv1.P Protocol: &networkpolicy.TCPProtocol, Destination: networkpolicy.KubeAPIServerEntityRule, }, - - // These rules allow egress between dex and identity providers. - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: v3.EntityRule{ - Nets: []string{"0.0.0.0/0"}, - Ports: networkpolicy.Ports(443, 6443, 389, 636), - }, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: v3.EntityRule{ - Nets: []string{"::/0"}, - Ports: networkpolicy.Ports(443, 6443, 389, 636), - }, - }, }...) + for _, egressRule := range c.resolveEgressRulesByDestination() { + egressRules = append(egressRules, egressRule) + } dexIngressPortDestination := v3.EntityRule{ Ports: networkpolicy.Ports(DexPort), @@ -477,3 +472,157 @@ func (c *dexComponent) allowTigeraNetworkPolicy(installationVariant operatorv1.P }, } } + +func (c *dexComponent) resolveEgressRulesByDestination() map[string]v3.Rule { + egressRulesByDestination := make(map[string]v3.Rule) + processedPodProxies := ProcessPodProxies(c.cfg.PodProxies) + for i, podProxy := range processedPodProxies { + egressDestinations, err := resolveEgressDestinationsForPod(podProxy) + if err != nil { + log.Error(err, fmt.Sprintf("failed to resolve egress destinations for pod %d, skipping for policy rendering", i)) + continue + } + + for _, egressDestination := range egressDestinations { + egressRule, err := resolveEgressRuleForDestination(egressDestination) + if err != nil { + log.Error(err, fmt.Sprintf("failed to resolve egress rule for pod %d, skipping for policy rendering", i)) + continue + } + + egressRulesByDestination[egressDestination] = egressRule + } + } + + return egressRulesByDestination +} + +// resolveEgressDestinationsForPod collects all possible http proxy destinations and all possible IdP destinations. +// In the future, this function may return only the specific destinations it expects Dex pods to connect to given the +// current issuer configuration (in the Authentication CR) and the HTTP proxy configuration. +func resolveEgressDestinationsForPod(podProxy *httpproxy.Config) ([]string, error) { + var egressDestinations []string + + if podProxy == nil { + podProxy = &httpproxy.Config{} + } + + // From here, we resolve multiple destinations by assuming any of the configured proxies could be active, and that + // an IdP could live at any IP. + // idp-resolution: In the future, we could resolve a single destination by resolving our expected IdP + // issuer URL and using podProxy.ProxyFunc to resolve a single expected destination URL. + if podProxy.HTTPProxy != "" { + httpProxyURL, err := url.Parse(podProxy.HTTPProxy) + if err != nil { + return nil, err + } + + httpProxyDestination, err := parseHostPortFromURL(httpProxyURL) + if err != nil { + return nil, err + } + + egressDestinations = append(egressDestinations, httpProxyDestination) + } + + if podProxy.HTTPSProxy != "" { + httpsProxyURL, err := url.Parse(podProxy.HTTPSProxy) + if err != nil { + return nil, err + } + + httpsProxyDestination, err := parseHostPortFromURL(httpsProxyURL) + if err != nil { + return nil, err + } + + egressDestinations = append(egressDestinations, httpsProxyDestination) + } + + egressDestinations = append(egressDestinations, "0.0.0.0/0") + egressDestinations = append(egressDestinations, "::/0") + + return egressDestinations, nil +} + +func resolveEgressRuleForDestination(destination string) (v3.Rule, error) { + // Support "any" destinations that signify any potential IdP destination IP. + // idp-resolution: These cases can be removed if we are able to resolve specific IdP destinations based on the Authentication config. + if destination == "0.0.0.0/0" { + return v3.Rule{ + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: v3.EntityRule{ + Nets: []string{"0.0.0.0/0"}, + Ports: networkpolicy.Ports(443, 6443, 389, 636), + }, + }, nil + } + if destination == "::/0" { + return v3.Rule{ + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: v3.EntityRule{ + Nets: []string{"::/0"}, + Ports: networkpolicy.Ports(443, 6443, 389, 636), + }, + }, nil + } + + // Process specific destinations. + var egressRule v3.Rule + host, port, err := net.SplitHostPort(destination) + if err != nil { + return v3.Rule{}, err + } + parsedPort, err := numorstring.PortFromString(port) + if err != nil { + return v3.Rule{}, err + } + parsedIp := net.ParseIP(host) + if parsedIp == nil { + // Assume host is a valid hostname. + egressRule = v3.Rule{ + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: v3.EntityRule{ + Domains: []string{host}, + Ports: []numorstring.Port{parsedPort}, + }, + } + } else { + var netSuffix string + if parsedIp.To4() != nil { + netSuffix = "/32" + } else { + netSuffix = "/128" + } + + egressRule = v3.Rule{ + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: v3.EntityRule{ + Nets: []string{parsedIp.String() + netSuffix}, + Ports: []numorstring.Port{parsedPort}, + }, + } + } + + return egressRule, nil +} + +func parseHostPortFromURL(url *url.URL) (string, error) { + if url.Port() != "" { + // Host is already in host:port form. + return url.Host, nil + } + + switch url.Scheme { + case "http": + return net.JoinHostPort(url.Host, "80"), nil + case "https": + return net.JoinHostPort(url.Host, "443"), nil + default: + return "", fmt.Errorf("unexpected scheme for URL: %s", url.Scheme) + } +} diff --git a/pkg/render/guardian.go b/pkg/render/guardian.go index 9b32b7561f..8bcc3f967b 100644 --- a/pkg/render/guardian.go +++ b/pkg/render/guardian.go @@ -313,21 +313,24 @@ func (c *GuardianComponent) volumes() []corev1.Volume { } func (c *GuardianComponent) container() []corev1.Container { + envVars := []corev1.EnvVar{ + {Name: "GUARDIAN_PORT", Value: "9443"}, + {Name: "GUARDIAN_LOGLEVEL", Value: "INFO"}, + {Name: "GUARDIAN_VOLTRON_URL", Value: c.cfg.URL}, + {Name: "GUARDIAN_VOLTRON_CA_TYPE", Value: string(c.cfg.TunnelCAType)}, + {Name: "GUARDIAN_PACKET_CAPTURE_CA_BUNDLE_PATH", Value: c.cfg.TrustedCertBundle.MountPath()}, + {Name: "GUARDIAN_PROMETHEUS_CA_BUNDLE_PATH", Value: c.cfg.TrustedCertBundle.MountPath()}, + {Name: "GUARDIAN_QUERYSERVER_CA_BUNDLE_PATH", Value: c.cfg.TrustedCertBundle.MountPath()}, + } + envVars = append(envVars, c.cfg.Installation.Proxy.EnvVars()...) + return []corev1.Container{ { Name: GuardianDeploymentName, Image: c.image, ImagePullPolicy: ImagePullPolicy(), - Env: []corev1.EnvVar{ - {Name: "GUARDIAN_PORT", Value: "9443"}, - {Name: "GUARDIAN_LOGLEVEL", Value: "INFO"}, - {Name: "GUARDIAN_VOLTRON_URL", Value: c.cfg.URL}, - {Name: "GUARDIAN_VOLTRON_CA_TYPE", Value: string(c.cfg.TunnelCAType)}, - {Name: "GUARDIAN_PACKET_CAPTURE_CA_BUNDLE_PATH", Value: c.cfg.TrustedCertBundle.MountPath()}, - {Name: "GUARDIAN_PROMETHEUS_CA_BUNDLE_PATH", Value: c.cfg.TrustedCertBundle.MountPath()}, - {Name: "GUARDIAN_QUERYSERVER_CA_BUNDLE_PATH", Value: c.cfg.TrustedCertBundle.MountPath()}, - }, - VolumeMounts: c.volumeMounts(), + Env: envVars, + VolumeMounts: c.volumeMounts(), LivenessProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ diff --git a/test/utils.go b/test/utils.go index 84e114de25..b59f50394c 100644 --- a/test/utils.go +++ b/test/utils.go @@ -20,6 +20,7 @@ import ( "encoding/pem" "fmt" "reflect" + "strings" "time" "github.com/stretchr/testify/mock" @@ -327,3 +328,28 @@ func (o *ObjectTrackerWithCalls) Watch(gvr schema.GroupVersionResource, ns strin o.inc(gvr, ObjectTrackerCallWatch) return o.ObjectTracker.Watch(gvr, ns) } + +type ProxyTestCase struct { + Lowercase bool + Target string + PodProxies []*ProxyConfig +} + +type ProxyConfig struct { + HTTPProxy string + HTTPSProxy string + NoProxy string +} + +func PrettyFormatProxyTestCase(testCase ProxyTestCase) string { + var containerProxies []string + for _, containerProxy := range testCase.PodProxies { + if containerProxy == nil { + containerProxies = append(containerProxies, "nil") + } else { + containerProxies = append(containerProxies, fmt.Sprintf("{HTTPProxy: %s, HTTPSProxy: %s, NoProxy: %s}", containerProxy.HTTPProxy, containerProxy.HTTPSProxy, containerProxy.NoProxy)) + } + } + + return fmt.Sprintf("Lowercase: %v, Target: %s, containerProxies: [%s]", testCase.Lowercase, testCase.Target, strings.Join(containerProxies, ",")) +}