Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

internal/provisioner: support more than one instance per namespace #4426

Merged
merged 6 commits into from
Mar 30, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions changelogs/unreleased/4426-skriss-minor.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
## Gateway provisioner: add support for more than one Gateway/Contour instance per namespace

The Gateway provisioner now supports having more than one Gateway/Contour instance per namespace.
All resource names now include a `-<gateway-name>` suffix to avoid conflicts (cluster-scoped resources also include the namespace as part of the resource name).
Contour instances are always provisioned in the namespace of the Gateway custom resource itself.

117 changes: 117 additions & 0 deletions internal/provisioner/model/names.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
// Copyright Project Contour Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package model

import (
"fmt"
"strings"
)

// ConfigMapName returns the name of the Contour ConfigMap resource.
func (c *Contour) ConfigMapName() string {
return "contour-" + c.Name
}

// ContourServiceName returns the name of the Contour Service resource.
func (c *Contour) ContourServiceName() string {
return "contour-" + c.Name
}

// EnvoyServiceName returns the name of the Envoy Service resource.
func (c *Contour) EnvoyServiceName() string {
return "envoy-" + c.Name
}

// ContourDeploymentName returns the name of the Contour Deployment resource.
func (c *Contour) ContourDeploymentName() string {
return "contour-" + c.Name
}

// EnvoyDaemonSetName returns the name of the Envoy DaemonSet resource.
func (c *Contour) EnvoyDaemonSetName() string {
return "envoy-" + c.Name
}

// LeaderElectionLeaseName returns the name of the Contour leader election Lease resource.
func (c *Contour) LeaderElectionLeaseName() string {
return "leader-elect-" + c.Name
}

// ContourCertsSecretName returns the name of the Contour xDS TLS certs Secret resource.
func (c *Contour) ContourCertsSecretName() string {
return c.Name + "-contourcert"
}

// EnvoyCertsSecretName returns the name of the Envoy xDS TLS certs Secret resource.
func (c *Contour) EnvoyCertsSecretName() string {
return c.Name + "-envoycert"
}
Comment on lines +51 to +59
Copy link
Member Author

@skriss skriss Mar 29, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have a subsequent PR that flips these cert secrets' names around to use the Gateway name as a suffix rather than a prefix, for consistency. Will be done as part of directly generating the certs rather than using the Job.


// CertgenJobName returns the name of the certgen Job resource.
func (c *Contour) CertgenJobName(contourImage string) string {
return fmt.Sprintf("contour-certgen-%s-%s", tagFromImage(contourImage), c.Name)
}

// tagFromImage returns the tag from the provided image or an
// empty string if the image does not contain a tag.
func tagFromImage(image string) string {
if strings.Contains(image, ":") {
parsed := strings.Split(image, ":")
return parsed[1]
}
return ""
}

// ContourRBACNames returns the names of the RBAC resources for
// the Contour deployment.
func (c *Contour) ContourRBACNames() RBACNames {
return RBACNames{
ServiceAccount: fmt.Sprintf("contour-%s", c.Name),
ClusterRole: fmt.Sprintf("contour-%s-%s", c.Namespace, c.Name),
ClusterRoleBinding: fmt.Sprintf("contour-%s-%s", c.Namespace, c.Name),
Role: fmt.Sprintf("contour-%s", c.Name),

// this one has a different prefix to differentiate from the certgen role binding (see below).
RoleBinding: fmt.Sprintf("contour-rolebinding-%s", c.Name),
}
}

// EnvoyRBACNames returns the names of the RBAC resources for
// the Envoy daemonset.
func (c *Contour) EnvoyRBACNames() RBACNames {
return RBACNames{
ServiceAccount: "envoy-" + c.Name,
}
}

// CertgenRBACNames returns the names of the RBAC resources for
// the Certgen job.
func (c *Contour) CertgenRBACNames() RBACNames {
return RBACNames{
ServiceAccount: "contour-certgen-" + c.Name,
Role: "contour-certgen-" + c.Name,

// this one is name contour-<gateway-name> despite being for certgen for legacy reasons.
RoleBinding: "contour-" + c.Name,
}
}

// RBACNames holds a set of names of related RBAC resources.
type RBACNames struct {
ServiceAccount string
ClusterRole string
ClusterRoleBinding string
Role string
RoleBinding string
}
16 changes: 8 additions & 8 deletions internal/provisioner/objects/configmap/configmap.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,13 +31,6 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client"
)

const (
// ContourConfigMapName is the name of Contour's ConfigMap resource.
// [TODO] danehans: Remove and use contour.Name when
// https://github.com/projectcontour/contour/issues/2122 is fixed.
ContourConfigMapName = "contour"
)

var contourConfigMapTemplate = template.Must(template.New("contour.yaml").Parse(`#
# server:
# determine which XDS Server implementation to utilize in Contour.
Expand Down Expand Up @@ -150,6 +143,8 @@ accesslog-format: envoy
# Configure the number of additional ingress proxy hops from the
# right side of the x-forwarded-for HTTP header to trust.
# num-trusted-hops: 0
# Name of the envoy service to inspect for Ingress status details.
skriss marked this conversation as resolved.
Show resolved Hide resolved
envoy-service-name: {{ .EnvoyServiceName }}
`))

// configMapParams contains everything needed to manage a Contour ConfigMap.
Expand Down Expand Up @@ -177,18 +172,23 @@ type contourConfig struct {
// EnableExternalNameService sets whether ExternalName Services are
// allowed.
EnableExternalNameService bool

// EnvoyServiceName is the name of the envoy service to inspect for Ingress
// status details.
EnvoyServiceName string
}

// configForContour returns a configMapParams with default fields set for contour.
func configForContour(contour *model.Contour) *configMapParams {
return &configMapParams{
Namespace: contour.Namespace,
Name: ContourConfigMapName,
Name: contour.ConfigMapName(),
Labels: model.OwnerLabels(contour),
Contour: contourConfig{
GatewayNamespace: contour.Namespace,
GatewayName: contour.Name,
EnableExternalNameService: pointer.BoolDeref(contour.Spec.EnableExternalNameService, false),
EnvoyServiceName: contour.EnvoyServiceName(),
},
}
}
Expand Down
10 changes: 8 additions & 2 deletions internal/provisioner/objects/configmap/configmap_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,8 @@ accesslog-format: envoy
# Configure the number of additional ingress proxy hops from the
# right side of the x-forwarded-for HTTP header to trust.
# num-trusted-hops: 0
# Name of the envoy service to inspect for Ingress status details.
envoy-service-name: envoy-test
`

c := &model.Contour{
Expand All @@ -142,7 +144,7 @@ accesslog-format: envoy
}
cm, err := desired(configForContour(c))
require.NoError(t, err)
require.Equal(t, "contour", cm.Name)
require.Equal(t, "contour-test", cm.Name)
require.Equal(t, "test-ns", cm.Namespace)
require.Contains(t, cm.Data, "contour.yaml")
assert.Equal(t, expected, cm.Data["contour.yaml"])
Expand Down Expand Up @@ -257,6 +259,8 @@ accesslog-format: envoy
# Configure the number of additional ingress proxy hops from the
# right side of the x-forwarded-for HTTP header to trust.
# num-trusted-hops: 0
# Name of the envoy service to inspect for Ingress status details.
envoy-service-name: envoy-test
`
c := &model.Contour{
ObjectMeta: v1.ObjectMeta{
Expand Down Expand Up @@ -382,6 +386,8 @@ accesslog-format: envoy
# Configure the number of additional ingress proxy hops from the
# right side of the x-forwarded-for HTTP header to trust.
# num-trusted-hops: 0
# Name of the envoy service to inspect for Ingress status details.
envoy-service-name: envoy-test
`

c := &model.Contour{
Expand All @@ -392,7 +398,7 @@ accesslog-format: envoy
}
cm, err := desired(configForContour(c))
require.NoError(t, err)
require.Equal(t, "contour", cm.Name)
require.Equal(t, "contour-test", cm.Name)
require.Equal(t, "test-ns", cm.Namespace)
require.Contains(t, cm.Data, "contour.yaml")
assert.Equal(t, expected, cm.Data["contour.yaml"])
Expand Down
24 changes: 9 additions & 15 deletions internal/provisioner/objects/daemonset/daemonset.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,6 @@ import (
)

const (
// envoyDaemonSetName is the name of Envoy's DaemonSet resource.
// [TODO] danehans: Remove and use contour.Name + "-envoy" when
// https://github.com/projectcontour/contour/issues/2122 is fixed.
envoyDaemonSetName = "envoy"
// EnvoyContainerName is the name of the Envoy container.
EnvoyContainerName = "envoy"
// ShutdownContainerName is the name of the Shutdown Manager container.
Expand All @@ -54,8 +50,6 @@ const (
envoyCertsVolName = "envoycert"
// envoyCertsVolMntDir is the directory name of the Envoy certificates volume.
envoyCertsVolMntDir = "certs"
// envoyCertsSecretName is the name of the secret used as the certificate volume source.
envoyCertsSecretName = envoyCertsVolName
// envoyCfgVolName is the name of the Envoy configuration volume.
envoyCfgVolName = "envoy-config"
// envoyCfgVolMntDir is the directory name of the Envoy configuration volume.
Expand Down Expand Up @@ -268,7 +262,7 @@ func DesiredDaemonSet(contour *model.Contour, contourImage, envoyImage string) *
Args: []string{
"bootstrap",
filepath.Join("/", envoyCfgVolMntDir, envoyCfgFileName),
"--xds-address=contour",
fmt.Sprintf("--xds-address=%s", contour.ContourServiceName()),
fmt.Sprintf("--xds-port=%d", objcfg.XDSPort),
fmt.Sprintf("--xds-resource-version=%s", xdsResourceVersion),
fmt.Sprintf("--resources-dir=%s", filepath.Join("/", envoyCfgVolMntDir, "resources")),
Expand Down Expand Up @@ -307,13 +301,13 @@ func DesiredDaemonSet(contour *model.Contour, contourImage, envoyImage string) *
ds := &appsv1.DaemonSet{
ObjectMeta: metav1.ObjectMeta{
Namespace: contour.Namespace,
Name: envoyDaemonSetName,
Name: contour.EnvoyDaemonSetName(),
Labels: labels,
},
Spec: appsv1.DaemonSetSpec{
RevisionHistoryLimit: pointer.Int32Ptr(int32(10)),
// Ensure the deamonset adopts only its own pods.
Selector: EnvoyDaemonSetPodSelector(),
Selector: EnvoyDaemonSetPodSelector(contour),
UpdateStrategy: appsv1.DaemonSetUpdateStrategy{
Type: appsv1.RollingUpdateDaemonSetStrategyType,
RollingUpdate: &appsv1.RollingUpdateDaemonSet{
Expand All @@ -329,7 +323,7 @@ func DesiredDaemonSet(contour *model.Contour, contourImage, envoyImage string) *
"prometheus.io/port": "8002",
"prometheus.io/path": "/stats/prometheus",
},
Labels: EnvoyDaemonSetPodSelector().MatchLabels,
Labels: EnvoyDaemonSetPodSelector(contour).MatchLabels,
},
Spec: corev1.PodSpec{
Containers: containers,
Expand All @@ -340,7 +334,7 @@ func DesiredDaemonSet(contour *model.Contour, contourImage, envoyImage string) *
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
DefaultMode: pointer.Int32Ptr(int32(420)),
SecretName: envoyCertsSecretName,
SecretName: contour.EnvoyCertsSecretName(),
},
},
},
Expand All @@ -357,7 +351,7 @@ func DesiredDaemonSet(contour *model.Contour, contourImage, envoyImage string) *
},
},
},
ServiceAccountName: objutil.EnvoyRbacName,
ServiceAccountName: contour.EnvoyRBACNames().ServiceAccount,
AutomountServiceAccountToken: pointer.BoolPtr(false),
TerminationGracePeriodSeconds: pointer.Int64Ptr(int64(300)),
SecurityContext: objutil.NewUnprivilegedPodSecurity(),
Expand Down Expand Up @@ -385,7 +379,7 @@ func CurrentDaemonSet(ctx context.Context, cli client.Client, contour *model.Con
ds := &appsv1.DaemonSet{}
key := types.NamespacedName{
Namespace: contour.Namespace,
Name: envoyDaemonSetName,
Name: contour.EnvoyDaemonSetName(),
}
if err := cli.Get(ctx, key, ds); err != nil {
return nil, err
Expand Down Expand Up @@ -418,10 +412,10 @@ func updateDaemonSetIfNeeded(ctx context.Context, cli client.Client, contour *mo

// EnvoyDaemonSetPodSelector returns a label selector using "app: envoy" as the
// key/value pair.
func EnvoyDaemonSetPodSelector() *metav1.LabelSelector {
func EnvoyDaemonSetPodSelector(contour *model.Contour) *metav1.LabelSelector {
return &metav1.LabelSelector{
MatchLabels: map[string]string{
"app": "envoy",
"app": contour.EnvoyDaemonSetName(),
},
}
}
Loading