Avoid env var

controlplane/cmd/server/start.go

@@ -8,12 +8,8 @@ import (
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/client-go/rest"
 	cliflag "k8s.io/component-base/cli/flag"
-	"k8s.io/component-base/cli/globalflag"
-	"k8s.io/component-base/logs"
 	_ "k8s.io/component-base/metrics/prometheus/workqueue" // for workqueue metric registration
-	"k8s.io/component-base/term"
 	"k8s.io/component-base/version/verflag"
-	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
 
 	ocmfeature "open-cluster-management.io/api/feature"
 	"open-cluster-management.io/ocm-controlplane/pkg/apiserver"
@@ -72,21 +68,7 @@ func NewAPIServerCommand() *cobra.Command {
 	}
 
 	fs := cmd.Flags()
-	namedFlagSets := s.ServerRunOptions.Flags()
-	verflag.AddFlags(namedFlagSets.FlagSet("global"))
-	globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name(), logs.SkipLoggingConfigurationFlags())
-	options.AddCustomGlobalFlags(namedFlagSets.FlagSet("generic"))
-	// add flagset ocm global config
-	ee := namedFlagSets.FlagSet("ocm global config")
-	// add enable-embedded-etcd flag
-	ee.BoolVar(&s.Extra.EmbeddedEtcdEnabled, "enable-embedded-etcd", false, "will use embedded etcd, if set to true")
-
-	for _, f := range namedFlagSets.FlagSets {
-		fs.AddFlagSet(f)
-	}
-
-	cols, _, _ := term.TerminalSize(cmd.OutOrStdout())
-	cliflag.SetUsageAndHelpFunc(cmd, namedFlagSets, cols)
+	s.AddFlags(fs)
 
 	return cmd
 }

controlplane/config/hub/bootstrap.go

@@ -6,7 +6,6 @@ import (
 	"embed"
 	"fmt"
 	"html/template"
-	"os"
 	"time"
 
 	corev1 "k8s.io/api/core/v1"
@@ -18,21 +17,24 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/serializer/yaml"
 	"k8s.io/apimachinery/pkg/util/wait"
 	yamlutil "k8s.io/apimachinery/pkg/util/yaml"
+	genericapiserver "k8s.io/apiserver/pkg/server"
 	"k8s.io/client-go/discovery"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/restmapper"
+	"k8s.io/client-go/tools/clientcmd"
+	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
+	bootstrapapi "k8s.io/cluster-bootstrap/token/api"
 	"k8s.io/klog/v2"
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo"
+	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
 	clusteradmhelpers "open-cluster-management.io/clusteradm/pkg/helpers"
 
 	confighelpers "open-cluster-management.io/ocm-controlplane/config/helpers"
 )
 
 var HubNameSpace = "open-cluster-management-hub"
 var HubSA = "hub-sa"
-var PublicNamespace = "kube-public"
-var SystemNamespace = "kube-system"
 
 //go:embed *.yaml
 var fs embed.FS
@@ -121,11 +123,11 @@ func bootstrapTokenSecret(ctx context.Context, discoveryClient discovery.Discove
 	return nil
 }
 
-func Bootstrap(ctx context.Context, discoveryClient discovery.DiscoveryInterface, dynamicClient dynamic.Interface, kubeClient kubernetes.Interface) error {
+func Bootstrap(ctx context.Context, config genericapiserver.Config, discoveryClient discovery.DiscoveryInterface, dynamicClient dynamic.Interface, kubeClient kubernetes.Interface) error {
 	// bootstrap namespace first
 	var defaultns = &corev1.Namespace{
 		ObjectMeta: metav1.ObjectMeta{
-			Name: "default",
+			Name: metav1.NamespaceDefault,
 		},
 	}
 	_, err := kubeClient.CoreV1().Namespaces().Create(ctx, defaultns, metav1.CreateOptions{})
@@ -137,28 +139,58 @@ func Bootstrap(ctx context.Context, discoveryClient discovery.DiscoveryInterface
 
 	// poll until kube-public created
 	if err = wait.PollInfinite(1*time.Second, func() (bool, error) {
-		_, err := kubeClient.CoreV1().Namespaces().Get(ctx, PublicNamespace, metav1.GetOptions{})
+		_, err := kubeClient.CoreV1().Namespaces().Get(ctx, metav1.NamespacePublic, metav1.GetOptions{})
 		if err != nil {
 			return false, nil
 		}
 		return true, nil
 	}); err == nil {
 		// configmap cluster-info
-		// TODO(ycyaoxdu): this need to be handled
-		kubeconfigpath := os.Getenv("OCM_CONFIG_DIRECTORY") + "/cert" + "/kube-aggregator.kubeconfig"
-		err = clusterinfo.CreateBootstrapConfigMapIfNotExists(kubeClient, kubeconfigpath)
+		caData, _ := config.SecureServing.Cert.CurrentCertKeyContent()
+		kubeconfig := clientcmdapi.Config{
+			Clusters: map[string]*clientcmdapi.Cluster{
+				"": {
+					Server:                   "https://" + config.ExternalAddress,
+					CertificateAuthorityData: caData,
+				},
+			},
+		}
+
+		kubeconfigRaw, err := clientcmd.Write(kubeconfig)
+		if err != nil {
+			return err
+		}
+
+		klog.V(1).Infoln("[bootstrap-token] creating/updating ConfigMap in kube-public namespace")
+		err = apiclient.CreateOrUpdateConfigMap(kubeClient, &corev1.ConfigMap{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      bootstrapapi.ConfigMapClusterInfo,
+				Namespace: metav1.NamespacePublic,
+			},
+			Data: map[string]string{
+				bootstrapapi.KubeConfigKey: string(kubeconfigRaw),
+			},
+		})
+
 		if err != nil && !errors.IsAlreadyExists(err) {
 			// don't klog.Fatal. This only happens when context is cancelled.
 			klog.Errorf("failed to bootstrap cluster-info configmap: %v", err)
 			// nolint:nilerr
 		}
+
+		err = clusterinfo.CreateClusterInfoRBACRules(kubeClient)
+		if err != nil && !errors.IsAlreadyExists(err) {
+			// don't klog.Fatal. This only happens when context is cancelled.
+			klog.Errorf("failed to bootstrap cluster-info rbac: %v", err)
+			// nolint:nilerr
+		}
 	} else {
-		klog.Errorf("failed to get namespace %s: %w", PublicNamespace, err)
+		klog.Errorf("failed to get namespace %s: %w", metav1.NamespacePublic, err)
 		// nolint:nilerr
 	}
 
 	if err = wait.PollInfinite(1*time.Second, func() (bool, error) {
-		_, err := kubeClient.CoreV1().Namespaces().Get(ctx, SystemNamespace, metav1.GetOptions{})
+		_, err := kubeClient.CoreV1().Namespaces().Get(ctx, metav1.NamespaceSystem, metav1.GetOptions{})
 		if err != nil {
 			return false, nil
 		}
@@ -170,7 +202,7 @@ func Bootstrap(ctx context.Context, discoveryClient discovery.DiscoveryInterface
 			// nolint:nilerr
 		}
 	} else {
-		klog.Errorf("failed to get namespace %s: %w", SystemNamespace, err)
+		klog.Errorf("failed to get namespace %s: %w", metav1.NamespaceSystem, err)
 		// nolint:nilerr
 	}
 

controlplane/go.mod

@@ -12,6 +12,7 @@ require (
 	k8s.io/apimachinery v0.24.3
 	k8s.io/apiserver v0.24.3
 	k8s.io/client-go v0.24.3
+	k8s.io/cluster-bootstrap v0.0.0
 	k8s.io/component-base v0.24.3
 	k8s.io/klog/v2 v2.70.1
 	k8s.io/kube-aggregator v0.24.3
@@ -144,7 +145,6 @@ require (
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/cli-runtime v0.24.3 // indirect
 	k8s.io/cloud-provider v0.24.0 // indirect
-	k8s.io/cluster-bootstrap v0.0.0 // indirect
 	k8s.io/component-helpers v0.24.0 // indirect
 	k8s.io/csi-translation-lib v0.24.0 // indirect
 	k8s.io/kubectl v0.24.3 // indirect

controlplane/hack/deploy-ocm-controlplane.sh

@@ -144,9 +144,11 @@ function start_apiserver {
     fi
 
     cp ${CERT_DIR}/kube-aggregator.kubeconfig ${CERT_DIR}/kubeconfig
-    sed -i "s@$OCM_CONFIG_DIRECTORY@/controlplane@g" ${CERT_DIR}/kube-aggregator.kubeconfig
-    sed -i 's/9443/443/g' ${CERT_DIR}/kubeconfig
-    ${KUSTOMIZE} build hack/deploy | ${KUBECTL} apply -f -
+    sed -e 's,9443,443,g' ${CERT_DIR}/kubeconfig
+    cp hack/deploy/deployment.yaml hack/deploy/deployment.yaml.tmp
+    sed -e 's,API_HOST,'${API_HOST}',' hack/deploy/deployment.yaml
+    ${KUSTOMIZE} build hack/deploy
+    mv hack/deploy/deployment.yaml.tmp hack/deploy/deployment.yaml
 
     echo "Use '${KUBECTL} --kubeconfig=${CERT_DIR}/kubeconfig' to use the aggregated API server" 
 }

controlplane/hack/deploy/deployment.yaml

@@ -18,33 +18,32 @@ spec:
       - name: controlplane
         image: docker.io/yaoyuchen/controlplane:latest
         imagePullPolicy: Always
-        env: 
-        - name: OCM_CONFIG_DIRECTORY
-          value: "/controlplane"
         args:
           - "/ocm-controlplane"
           - "--authorization-mode=RBAC"
           - "--feature-gates=DefaultClusterSet=true"
           - "--enable-bootstrap-token-auth"
           - "--service-account-key-file=/controlplane/cert/kube-serviceaccount.key"
           - "--client-ca-file=/controlplane/cert/client-ca.crt"
+          - "--client-key-file=/controlplane/cert/client-ca.key"
           - "--enable-bootstrap-token-auth"
-          - "--enable-priority-and-fairness=false" 
-          - "--api-audiences="         
-          - "--storage-backend=etcd3" 
+          - "--enable-priority-and-fairness=false"
+          - "--api-audiences="    
+          - "--storage-backend=etcd3"
           - "--v=1"
-          - "--service-account-lookup=false" 
-          - "--service-account-signing-key-file=/controlplane/cert/kube-serviceaccount.key" 
+          - "--service-account-lookup=false"
+          - "--service-account-signing-key-file=/controlplane/cert/kube-serviceaccount.key"
           - "--enable-admission-plugins=NamespaceLifecycle,ServiceAccount,MutatingAdmissionWebhook,ValidatingAdmissionWebhook"
-          - "--disable-admission-plugins=TaintNodesByCondition,Priority,DefaultTolerationSeconds,DefaultStorageClass,PodSecurity,PersistentVolumeClaimResize,RuntimeClass,DefaultIngressClass" 
-          - "--bind-address=0.0.0.0" 
-          - "--secure-port=9443" 
-          - "--tls-cert-file=/controlplane/cert/serving-kube-apiserver.crt" 
-          - "--tls-private-key-file=/controlplane/cert/serving-kube-apiserver.key" 
-          - "--feature-gates=DefaultClusterSet=true" 
-          - "--enable-embedded-etcd=true" 
-          - "--service-cluster-ip-range=10.0.0.0/24"  
+          - "--disable-admission-plugins=TaintNodesByCondition,Priority,DefaultTolerationSeconds,DefaultStorageClass,PodSecurity,PersistentVolumeClaimResize,RuntimeClass,DefaultIngressClass"
+          - "--bind-address=0.0.0.0"
+          - "--secure-port=9443"
+          - "--tls-cert-file=/controlplane/cert/serving-kube-apiserver.crt"
+          - "--tls-private-key-file=/controlplane/cert/serving-kube-apiserver.key"
+          - "--feature-gates=DefaultClusterSet=true"
+          - "--enable-embedded-etcd=true"
+          - "--service-cluster-ip-range=10.0.0.0/24"
           - "--service-account-issuer=https://kubernetes.default.svc"
+          - "--external-hostname=API_HOST"
         securityContext:
           allowPrivilegeEscalation: false
           capabilities:

controlplane/hack/lib/util.sh

@@ -299,8 +299,8 @@ clusters:
 users:
   - user:
       token: ${token}
-      client-certificate: ${dest_dir}/client-${client_id}.crt
-      client-key: ${dest_dir}/client-${client_id}.key
+      client-certificate: client-${client_id}.crt
+      client-key: client-${client_id}.key
     name: ocm-standalone-controlplane
 contexts:
   - context:

controlplane/hack/start-ocm-controlplane.sh

@@ -6,16 +6,13 @@
 KUBE_ROOT=$(pwd)
 # export SERVING_IP=$(ifconfig en0 | grep -Eo 'inet (addr:)?([0-9]*\.){3}[0-9]*' | grep -Eo '([0-9]*\.){3}[0-9]*') # ***en0 for macOS***
 # export SERVING_IP=$(ifconfig eth0 | grep -Eo 'inet (addr:)?([0-9]*\.){3}[0-9]*' | grep -Eo '([0-9]*\.){3}[0-9]*') # ***eth0 for Linux***
-# export OCM_CONFIG_DIRECTORY=$(pwd)/.ocmconfig
 if [ ! $SERVING_IP ] ; then
     echo "SERVING_IP should be set"
     exit 1
 fi
-if [ ! $OCM_CONFIG_DIRECTORY ] ; then
-    echo "OCM_CONFIG_DIRECTORY should be set"
-    exit 1
-fi
 
+# set root dir
+OCM_CONFIG_DIRECTORY=${OCM_CONFIG_DIRECTORY:-".ocmconfig"}
 #  set port
 SERVING_PORT=9443
 # use embedded etcd if set to true
@@ -67,7 +64,7 @@ API_HOST_IP=${API_HOST_IP:-$SERVING_IP}
 API_BIND_ADDR=${API_BIND_ADDR:-"0.0.0.0"}
 
 LOG_DIR=${LOG_DIR:-"/tmp"}
-ROOT_CA_FILE=${CERT_DIR}/server-ca.crt
+ROOT_CA_FILE="server-ca.crt"
 # Reuse certs will skip generate new ca/cert files under CERT_DIR
 # it's useful with PRESERVE_ETCD=true because new ca will make existed service account secrets invalided
 REUSE_CERTS=${REUSE_CERTS:-false}
@@ -199,7 +196,9 @@ function start_apiserver {
     --enable-bootstrap-token-auth \
     --enable-priority-and-fairness="false" \
     --api-audiences="" \
+    --external-hostname="${API_HOST}" \
     --client-ca-file="${CERT_DIR}/client-ca.crt" \
+    --client-key-file="${CERT_DIR}/client-ca.key" \
     --service-account-key-file="${SERVICE_ACCOUNT_KEY}" \
     --service-account-lookup="${SERVICE_ACCOUNT_LOOKUP}" \
     --service-account-issuer="https://kubernetes.default.svc" \