Skip to content
This repository has been archived by the owner on Jun 26, 2023. It is now read-only.

Commit

Permalink
Integrate GK cert as the default internal cert
Browse files Browse the repository at this point in the history
Fork the cert management from open-policy-agent/gatekeeper repo to serve
as the default internal cert management for HNC.

Add an "enable-internal-cert-management" flag to switch between using
the internal cert (integrated Gatekeeper cert) or the external cert
"cert-manager".

Update main.go to start the controller manager with internal certs first
and once the cert files are ready, add other controllers and webhooks if
webhook is enabled and internal certs are used.

Refactor the cert generation and validator startup into
pkg/validators/setup.go.

To use external cert-manager, update config/default/kustomization.yaml
and manager_auth_proxy_patch.yaml as instructed. Run 'make deploy-cm'
before 'make deploy'.

Tested on a GKE cluster. The manager restarts 0 times and the webhooks
are working as expected.
  • Loading branch information
yiqigao217 committed Apr 30, 2020
1 parent e37728b commit ba5c7da
Show file tree
Hide file tree
Showing 14 changed files with 1,066 additions and 83 deletions.
1 change: 1 addition & 0 deletions incubator/hnc/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ RUN go mod download
COPY api/ api/
COPY cmd/ cmd/
COPY pkg/ pkg/
COPY third_party/ third_party/

# Build
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o manager ./cmd/manager/main.go
Expand Down
10 changes: 5 additions & 5 deletions incubator/hnc/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ all: test docker-build

# Run tests
test: build
go test ./api/... ./cmd/... ./pkg/... -coverprofile cover.out
go test ./api/... ./cmd/... ./pkg/... ./third_party/... -coverprofile cover.out

# Builds all binaries (manager and kubectl) and manifests
build: generate fmt vet manifests
Expand Down Expand Up @@ -128,15 +128,15 @@ endif
# deleting the CRDs will cause all the existing hierarchy configs to be wiped
# away and b) if we don't delete the deployment, a new image won't be pulled
# unless the tag changes.
deploy: deploy-prereq docker-push kubectl manifests
deploy: docker-push kubectl manifests
-kubectl -n hnc-system delete deployment hnc-controller-manager
kubectl apply -f manifests/hnc-manager.yaml

deploy-watch:
kubectl logs -n hnc-system --follow deployment/hnc-controller-manager manager

# Installs prerequisites. See https://cert-manager.io/docs/installation/kubernetes/ for why we're not validating our YAML.
deploy-prereq:
# Installs cert-manager. See https://cert-manager.io/docs/installation/kubernetes/ for why we're not validating our YAML.
deploy-cm:
@kubectl cluster-info
-kubectl create namespace cert-manager
kubectl apply --validate=false -f https://github.com/jetstack/cert-manager/releases/download/v0.11.0/cert-manager.yaml
Expand Down Expand Up @@ -166,7 +166,7 @@ kind-reboot:
# Creates a local kind cluster, destroying the old one if necessary. It's not
# *necessary* to call this wih CONFIG=kind but it's not a bad idea either so
# the correct manifests get created.
kind-reset: kind-reboot deploy-prereq
kind-reset: kind-reboot
@echo "If this didn't work, ensure you ran 'source devenv' to point kubectl at kind'"

# Convenience target to deploy specifically for kind
Expand Down
81 changes: 35 additions & 46 deletions incubator/hnc/cmd/manager/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ import (
_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
"sigs.k8s.io/controller-runtime/pkg/webhook"

// +kubebuilder:scaffold:imports

Expand All @@ -48,6 +47,18 @@ var (
setupLog = ctrl.Log.WithName("setup")
)

var (
metricsAddr string
maxReconciles int
enableLeaderElection bool
leaderElectionId string
novalidation bool
debugLogs bool
testLog bool
internalCert bool
qps int
)

func init() {
_ = clientgoscheme.AddToScheme(scheme)

Expand All @@ -58,16 +69,6 @@ func init() {
}

func main() {
var (
metricsAddr string
maxReconciles int
enableLeaderElection bool
leaderElectionId string
novalidation bool
debugLogs bool
testLog bool
qps int
)
flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
flag.BoolVar(&enableLeaderElection, "enable-leader-election", false,
"Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
Expand All @@ -76,6 +77,7 @@ func main() {
flag.BoolVar(&novalidation, "novalidation", false, "Disables validating webhook")
flag.BoolVar(&debugLogs, "debug-logs", false, "Shows verbose logs in a human-friendly format.")
flag.BoolVar(&testLog, "enable-test-log", false, "Enables test log.")
flag.BoolVar(&internalCert, "enable-internal-cert-management", false, "Enables internal cert management.")
flag.IntVar(&maxReconciles, "max-reconciles", 1, "Number of concurrent reconciles to perform.")
flag.IntVar(&qps, "apiserver-qps-throttle", 50, "The maximum QPS to the API server.")
flag.Parse()
Expand Down Expand Up @@ -131,6 +133,26 @@ func main() {
os.Exit(1)
}

// Make sure certs are generated and valid if webhooks are enabled and internal certs are used.
setupFinished, err := validators.CreateCertsIfNeeded(mgr, novalidation, internalCert)
if err != nil {
setupLog.Error(err, "unable to set up cert rotation")
os.Exit(1)
}

go startControllers(mgr, setupFinished)

setupLog.Info("starting manager")
if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
setupLog.Error(err, "problem running manager")
os.Exit(1)
}
}

func startControllers(mgr ctrl.Manager, setupFinished chan struct{}) {
// Block until the setup finishes.
<-setupFinished

if testLog {
stats.StartLoggingActivity()
}
Expand All @@ -143,42 +165,9 @@ func main() {
os.Exit(1)
}

// Create validating admission controllers
// Create all validating admission controllers.
if !novalidation {
// Create webhook for Hierarchy
setupLog.Info("Registering validating webhook (won't work when running locally; use --novalidation)")
mgr.GetWebhookServer().Register(validators.HierarchyServingPath, &webhook.Admission{Handler: &validators.Hierarchy{
Log: ctrl.Log.WithName("validators").WithName("Hierarchy"),
Forest: f,
}})

// Create webhooks for managed objects
mgr.GetWebhookServer().Register(validators.ObjectsServingPath, &webhook.Admission{Handler: &validators.Object{
Log: ctrl.Log.WithName("validators").WithName("Object"),
Forest: f,
}})

// Create webhook for the config
mgr.GetWebhookServer().Register(validators.ConfigServingPath, &webhook.Admission{Handler: &validators.HNCConfig{
Log: ctrl.Log.WithName("validators").WithName("HNCConfig"),
}})

// Create webhook for the HierarchicalNamespaces.
mgr.GetWebhookServer().Register(validators.HierarchicalNamespaceServingPath, &webhook.Admission{Handler: &validators.HierarchicalNamespace{
Log: ctrl.Log.WithName("validators").WithName("HierarchicalNamespace"),
Forest: f,
}})

// Create webhook for the namespaces (core type).
mgr.GetWebhookServer().Register(validators.NamespaceServingPath, &webhook.Admission{Handler: &validators.Namespace{
Log: ctrl.Log.WithName("validators").WithName("Namespace"),
Forest: f,
}})
}

setupLog.Info("starting manager")
if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
setupLog.Error(err, "problem running manager")
os.Exit(1)
validators.Create(mgr, f)
}
}
69 changes: 37 additions & 32 deletions incubator/hnc/config/default/kustomization.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,10 @@ bases:
- ../rbac
- ../manager
- ../webhook
- ../certmanager
# Add cert management. Only one of the ../internalcert and the
# ../certmanager should be enabled.
- ../internalcert
#- ../certmanager

patchesStrategicMerge:
# Protect the /metrics endpoint by putting it behind auth.
Expand All @@ -32,34 +35,36 @@ patchesStrategicMerge:
#- manager_prometheus_metrics_patch.yaml

- manager_webhook_patch.yaml

- webhookcainjection_patch.yaml

# the following config is for teaching kustomize how to do var substitution
vars:
- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR
objref:
kind: Certificate
group: cert-manager.io
version: v1alpha2
name: serving-cert # this name should match the one in certificate.yaml
fieldref:
fieldpath: metadata.namespace
- name: CERTIFICATE_NAME
objref:
kind: Certificate
group: cert-manager.io
version: v1alpha2
name: serving-cert # this name should match the one in certificate.yaml
- name: SERVICE_NAMESPACE # namespace of the service
objref:
kind: Service
version: v1
name: webhook-service
fieldref:
fieldpath: metadata.namespace
- name: SERVICE_NAME
objref:
kind: Service
version: v1
name: webhook-service
# # Uncomment below if ../certmanager is enabled.
# # Add additional "cert-manager" component to inject ca into
# # the ValidatingWebhookConfiguration resource.
#- webhookcainjection_patch.yaml
#
## the following config is for teaching kustomize how to do var substitution
#vars:
# - name: CERTIFICATE_NAMESPACE # namespace of the certificate CR
# objref:
# kind: Certificate
# group: cert-manager.io
# version: v1alpha2
# name: serving-cert # this name should match the one in certificate.yaml
# fieldref:
# fieldpath: metadata.namespace
# - name: CERTIFICATE_NAME
# objref:
# kind: Certificate
# group: cert-manager.io
# version: v1alpha2
# name: serving-cert # this name should match the one in certificate.yaml
# - name: SERVICE_NAMESPACE # namespace of the service
# objref:
# kind: Service
# version: v1
# name: webhook-service
# fieldref:
# fieldpath: metadata.namespace
# - name: SERVICE_NAME
# objref:
# kind: Service
# version: v1
# name: webhook-service
1 change: 1 addition & 0 deletions incubator/hnc/config/default/manager_auth_proxy_patch.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -26,3 +26,4 @@ spec:
- "--leader-election-id=hnc-controller-leader-election-helper"
- "--max-reconciles=10"
- "--apiserver-qps-throttle=50"
- "--enable-internal-cert-management=true"
2 changes: 2 additions & 0 deletions incubator/hnc/config/internalcert/kustomization.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
resources:
- manifests.yaml
5 changes: 5 additions & 0 deletions incubator/hnc/config/internalcert/manifests.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
apiVersion: v1
kind: Secret
metadata:
name: webhook-server-cert
namespace: system
2 changes: 2 additions & 0 deletions incubator/hnc/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,9 @@ require (
github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
github.com/onsi/ginkgo v1.12.0
github.com/onsi/gomega v1.9.0
github.com/pkg/errors v0.8.1
github.com/prometheus/client_golang v1.5.0
github.com/prometheus/common v0.9.1
github.com/spf13/cobra v0.0.5
go.opencensus.io v0.22.3
go.uber.org/atomic v1.4.0 // indirect
Expand Down
3 changes: 3 additions & 0 deletions incubator/hnc/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,10 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
Expand Down Expand Up @@ -570,6 +572,7 @@ google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiq
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
google.golang.org/grpc v1.23.1 h1:q4XQuHFC6I28BKZpo6IYyb3mNO+l7lSOxRuYTCiDfXk=
google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
Expand Down
78 changes: 78 additions & 0 deletions incubator/hnc/pkg/validators/setup.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
package validators

import (
"fmt"

"k8s.io/apimachinery/pkg/types"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/webhook"

"github.com/kubernetes-sigs/multi-tenancy/incubator/hnc/pkg/forest"
cert "github.com/kubernetes-sigs/multi-tenancy/incubator/hnc/third_party/open-policy-agent/gatekeeper/pkg/webhook"
)

const (
serviceName = "hnc-webhook-service"
vwhName = "hnc-validating-webhook-configuration"
caName = "hnc-ca"
caOrganization = "hnc"
secretNamespace = "hnc-system"
secretName = "hnc-webhook-server-cert"
certDir = "/tmp/k8s-webhook-server/serving-certs"
)

// DNSName is <service name>.<namespace>.svc
var dnsName = fmt.Sprintf("%s.%s.svc", serviceName, secretNamespace)

// CreateCertsIfNeeded creates all certs for webhooks. This function is called from main.go.
func CreateCertsIfNeeded(mgr ctrl.Manager, novalidation, internalCert bool) (chan struct{}, error) {
setupFinished := make(chan struct{})
if novalidation || !internalCert {
close(setupFinished)
return setupFinished, nil
}

return setupFinished, cert.AddRotator(mgr, &cert.CertRotator{
SecretKey: types.NamespacedName{
Namespace: secretNamespace,
Name: secretName,
},
CertDir: certDir,
CAName: caName,
CaOrganization: caOrganization,
DNSName: dnsName,
CertsMounted: setupFinished,
}, vwhName)
}

// Create creates all validators. This function is called from main.go.
func Create(mgr ctrl.Manager, f *forest.Forest) {
// Create webhook for Hierarchy
mgr.GetWebhookServer().Register(HierarchyServingPath, &webhook.Admission{Handler: &Hierarchy{
Log: ctrl.Log.WithName("validators").WithName("Hierarchy"),
Forest: f,
}})

// Create webhooks for managed objects
mgr.GetWebhookServer().Register(ObjectsServingPath, &webhook.Admission{Handler: &Object{
Log: ctrl.Log.WithName("validators").WithName("Object"),
Forest: f,
}})

// Create webhook for the config
mgr.GetWebhookServer().Register(ConfigServingPath, &webhook.Admission{Handler: &HNCConfig{
Log: ctrl.Log.WithName("validators").WithName("HNCConfig"),
}})

// Create webhook for the HierarchicalNamespaces.
mgr.GetWebhookServer().Register(HierarchicalNamespaceServingPath, &webhook.Admission{Handler: &HierarchicalNamespace{
Log: ctrl.Log.WithName("validators").WithName("HierarchicalNamespace"),
Forest: f,
}})

// Create webhook for the namespaces (core type).
mgr.GetWebhookServer().Register(NamespaceServingPath, &webhook.Admission{Handler: &Namespace{
Log: ctrl.Log.WithName("validators").WithName("Namespace"),
Forest: f,
}})
}
Loading

0 comments on commit ba5c7da

Please sign in to comment.