[WIP] Add CRD and controller logic

cmd/nginx/main.go

@@ -36,6 +36,7 @@ import (
 	discovery "k8s.io/apimachinery/pkg/version"
 	"k8s.io/apiserver/pkg/server/healthz"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/klog"
 
@@ -45,6 +46,8 @@ import (
 	"k8s.io/ingress-nginx/internal/k8s"
 	"k8s.io/ingress-nginx/internal/net/ssl"
 	"k8s.io/ingress-nginx/version"
+
+	pluginclientset "k8s.io/ingress-nginx/pkg/client/clientset/versioned"
 )
 
 const (
@@ -86,6 +89,11 @@ func main() {
 		handleFatalInitError(err)
 	}
 
+	ingressNginxPluginClient, err := createApiserverPluginClient(conf.APIServerHost, conf.KubeConfigFile)
+	if err != nil {
+		handleFatalInitError(err)
+	}
+
 	if len(conf.DefaultService) > 0 {
 		defSvcNs, defSvcName, err := k8s.ParseNameNS(conf.DefaultService)
 		if err != nil {
@@ -125,6 +133,7 @@ func main() {
 	// end create default fake SSL certificates
 
 	conf.Client = kubeClient
+	conf.PluginClient = ingressNginxPluginClient
 
 	reg := prometheus.NewRegistry()
 
@@ -193,6 +202,60 @@ func handleSigterm(ngx *controller.NGINXController, exit exiter) {
 // controller runs inside Kubernetes and fallback to the in-cluster config. If
 // the in-cluster config is missing or fails, we fallback to the default config.
 func createApiserverClient(apiserverHost, kubeConfig string) (*kubernetes.Clientset, error) {
+	cfg, err := createClientConfig(apiserverHost, kubeConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	client, err := kubernetes.NewForConfig(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	var v *discovery.Info
+
+	// The client may fail to connect to the API server in the first request.
+	// https://github.com/kubernetes/ingress-nginx/issues/1968
+	defaultRetry := wait.Backoff{
+		Steps:    10,
+		Duration: 1 * time.Second,
+		Factor:   1.5,
+		Jitter:   0.1,
+	}
+
+	var lastErr error
+	retries := 0
+	klog.V(2).Info("Trying to discover Kubernetes version")
+	err = wait.ExponentialBackoff(defaultRetry, func() (bool, error) {
+		v, err = client.Discovery().ServerVersion()
+
+		if err == nil {
+			return true, nil
+		}
+
+		lastErr = err
+		klog.V(2).Infof("Unexpected error discovering Kubernetes version (attempt %v): %v", retries, err)
+		retries++
+		return false, nil
+	})
+
+	// err is returned in case of timeout in the exponential backoff (ErrWaitTimeout)
+	if err != nil {
+		return nil, lastErr
+	}
+
+	// this should not happen, warn the user
+	if retries > 0 {
+		klog.Warningf("Initial connection to the Kubernetes API server was retried %d times.", retries)
+	}
+
+	klog.Infof("Running in Kubernetes cluster version v%v.%v (%v) - git (%v) commit %v - platform %v",
+		v.Major, v.Minor, v.GitVersion, v.GitTreeState, v.GitCommit, v.Platform)
+
+	return client, nil
+}
+
+func createClientConfig(apiserverHost, kubeConfig string) (*rest.Config, error) {
 	cfg, err := clientcmd.BuildConfigFromFlags(apiserverHost, kubeConfig)
 	if err != nil {
 		return nil, err
@@ -204,7 +267,17 @@ func createApiserverClient(apiserverHost, kubeConfig string) (*kubernetes.Client
 
 	klog.Infof("Creating API client for %s", cfg.Host)
 
-	client, err := kubernetes.NewForConfig(cfg)
+	return cfg, nil
+
+}
+
+func createApiserverPluginClient(apiserverHost, kubeConfig string) (*pluginclientset.Clientset, error) {
+	cfg, err := createClientConfig(apiserverHost, kubeConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	client, err := pluginclientset.NewForConfig(cfg)
 	if err != nil {
 		return nil, err
 	}

deploy/mandatory.yaml

@@ -67,6 +67,14 @@ rules:
     verbs:
       - list
       - watch
+  - apiGroups:
+      - "nginx.ingress.kubernetes.io"
+    resources:
+      - ingressnginxplugins
+    verbs:
+      - get
+      - list
+      - watch
   - apiGroups:
       - ""
     resources:
@@ -147,6 +155,14 @@ rules:
       - endpoints
     verbs:
       - get
+  - apiGroups:
+      - "nginx.ingress.kubernetes.io"
+    resources:
+      - ingressnginxplugins
+    verbs:
+      - get
+      - list
+      - watch
 
 ---
 apiVersion: rbac.authorization.k8s.io/v1beta1
@@ -211,14 +227,16 @@ spec:
       serviceAccountName: nginx-ingress-serviceaccount
       containers:
         - name: nginx-ingress-controller
-          image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.23.0
+          image: ingress-controller/nginx-ingress-controller:dev
+          imagePullPolicy: Never
           args:
             - /nginx-ingress-controller
             - --configmap=$(POD_NAMESPACE)/nginx-configuration
             - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services
             - --udp-services-configmap=$(POD_NAMESPACE)/udp-services
             - --publish-service=$(POD_NAMESPACE)/ingress-nginx
             - --annotations-prefix=nginx.ingress.kubernetes.io
+            - --v=3
           securityContext:
             allowPrivilegeEscalation: true
             capabilities:
@@ -263,3 +281,34 @@ spec:
             timeoutSeconds: 10
 
 ---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: ingressnginxplugins.nginx.ingress.kubernetes.io
+spec:
+  group: nginx.ingress.kubernetes.io
+  versions:
+    - name: v1alpha1
+      served: true
+      storage: true
+  scope: Namespaced
+  names:
+    plural: ingressnginxplugins
+    singular: ingressnginxplugin
+    kind: IngressNginxPlugin
+    shortNames:
+    - inp
+  validation:
+    openAPIV3Schema:
+      properties:
+        spec:
+          required:
+          - archive
+          - sha256sum
+          properties:
+            archive:
+              type: string
+              pattern: '^.*\.tar.gz$'
+            sha256sum:
+              type: string
+              pattern: '^[A-Fa-f0-9]{64}$'

internal/ingress/controller/controller.go

@@ -33,6 +33,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
 	clientset "k8s.io/client-go/kubernetes"
+	pluginclientset "k8s.io/ingress-nginx/pkg/client/clientset/versioned"
 
 	"k8s.io/ingress-nginx/internal/ingress"
 	"k8s.io/ingress-nginx/internal/ingress/annotations"
@@ -52,6 +53,7 @@ type Configuration struct {
 	APIServerHost  string
 	KubeConfigFile string
 	Client         clientset.Interface
+	PluginClient   pluginclientset.Interface
 
 	ResyncPeriod time.Duration
 

internal/ingress/controller/nginx.go

@@ -127,6 +127,7 @@ func NewNGINXController(config *Configuration, mc metric.Collector, fs file.File
 		config.DefaultSSLCertificate,
 		config.ResyncPeriod,
 		config.Client,
+		config.PluginClient,
 		fs,
 		n.updateCh,
 		config.DynamicCertificatesEnabled,

internal/ingress/controller/store/ingress_nginx_plugin.go

@@ -0,0 +1,39 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package store
+
+import (
+	"k8s.io/client-go/tools/cache"
+	inp "k8s.io/ingress-nginx/pkg/apis/ingressnginxplugin/v1alpha1"
+)
+
+// IngressNginxPluginLister makes a Store that lists IngressNginxPlugins.
+type IngressNginxPluginLister struct {
+	cache.Store
+}
+
+// ByKey returns the Plugin matching key in the local IngressNginxPlugin Store.
+func (cml *IngressNginxPluginLister) ByKey(key string) (*inp.IngressNginxPlugin, error) {
+	s, exists, err := cml.GetByKey(key)
+	if err != nil {
+		return nil, err
+	}
+	if !exists {
+		return nil, NotExistsError(key)
+	}
+	return s.(*inp.IngressNginxPlugin), nil
+}