Avoid any envvar setting

Signed-off-by: Jian Qiu <jqiu@redhat.com>

controlplane/cmd/server/start.go

@@ -8,12 +8,8 @@ import (
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/client-go/rest"
 	cliflag "k8s.io/component-base/cli/flag"
-	"k8s.io/component-base/cli/globalflag"
-	"k8s.io/component-base/logs"
 	_ "k8s.io/component-base/metrics/prometheus/workqueue" // for workqueue metric registration
-	"k8s.io/component-base/term"
 	"k8s.io/component-base/version/verflag"
-	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
 
 	ocmfeature "open-cluster-management.io/api/feature"
 	"open-cluster-management.io/ocm-controlplane/pkg/apiserver"
@@ -72,21 +68,7 @@ func NewAPIServerCommand() *cobra.Command {
 	}
 
 	fs := cmd.Flags()
-	namedFlagSets := s.ServerRunOptions.Flags()
-	verflag.AddFlags(namedFlagSets.FlagSet("global"))
-	globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name(), logs.SkipLoggingConfigurationFlags())
-	options.AddCustomGlobalFlags(namedFlagSets.FlagSet("generic"))
-	// add flagset ocm global config
-	ee := namedFlagSets.FlagSet("ocm global config")
-	// add enable-embedded-etcd flag
-	ee.BoolVar(&s.Extra.EmbeddedEtcdEnabled, "enable-embedded-etcd", false, "will use embedded etcd, if set to true")
-
-	for _, f := range namedFlagSets.FlagSets {
-		fs.AddFlagSet(f)
-	}
-
-	cols, _, _ := term.TerminalSize(cmd.OutOrStdout())
-	cliflag.SetUsageAndHelpFunc(cmd, namedFlagSets, cols)
+	s.AddFlags(fs)
 
 	return cmd
 }

controlplane/hack/deploy/deployment.yaml

@@ -18,16 +18,14 @@ spec:
       - name: controlplane
         image: docker.io/yaoyuchen/controlplane:latest
         imagePullPolicy: Always
-        env: 
-        - name: OCM_CONFIG_DIRECTORY
-          value: "/controlplane"
         args:
           - "/ocm-controlplane"
           - "--authorization-mode=RBAC"
           - "--feature-gates=DefaultClusterSet=true"
           - "--enable-bootstrap-token-auth"
           - "--service-account-key-file=/controlplane/cert/kube-serviceaccount.key"
           - "--client-ca-file=/controlplane/cert/client-ca.crt"
+          - "--client-key-file="/controlplane/cert/client-ca.key" \"
           - "--enable-bootstrap-token-auth"
           - "--enable-priority-and-fairness=false" 
           - "--api-audiences="         

controlplane/hack/start-ocm-controlplane.sh

@@ -6,15 +6,10 @@
 KUBE_ROOT=$(pwd)
 # export SERVING_IP=$(ifconfig en0 | grep -Eo 'inet (addr:)?([0-9]*\.){3}[0-9]*' | grep -Eo '([0-9]*\.){3}[0-9]*') # ***en0 for macOS***
 # export SERVING_IP=$(ifconfig eth0 | grep -Eo 'inet (addr:)?([0-9]*\.){3}[0-9]*' | grep -Eo '([0-9]*\.){3}[0-9]*') # ***eth0 for Linux***
-# export OCM_CONFIG_DIRECTORY=$(pwd)/.ocmconfig
 if [ ! $SERVING_IP ] ; then
     echo "SERVING_IP should be set"
     exit 1
 fi
-if [ ! $OCM_CONFIG_DIRECTORY ] ; then
-    echo "OCM_CONFIG_DIRECTORY should be set"
-    exit 1
-fi
 
 #  set port
 SERVING_PORT=9443
@@ -200,6 +195,7 @@ function start_apiserver {
     --enable-priority-and-fairness="false" \
     --api-audiences="" \
     --client-ca-file="${CERT_DIR}/client-ca.crt" \
+    --client-key-file="${CERT_DIR}/client-ca.key" \
     --service-account-key-file="${SERVICE_ACCOUNT_KEY}" \
     --service-account-lookup="${SERVICE_ACCOUNT_LOOKUP}" \
     --service-account-issuer="https://kubernetes.default.svc" \

controlplane/pkg/apiserver/controlplane.go

@@ -4,9 +4,8 @@ import (
 	"context"
 	"fmt"
 	"net"
-	"os"
-	"path/filepath"
 
+	"github.com/spf13/pflag"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	genericapiserver "k8s.io/apiserver/pkg/server"
 	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
@@ -16,25 +15,16 @@ import (
 	"open-cluster-management.io/ocm-controlplane/pkg/etcd"
 )
 
-// directory to save embedded etcd generated files
-const DefaultDirectory = ".ocmconfig"
-
-type ExtraConfig struct {
-	RootDirectory string
-	// use this filed to save the embedded etcd arg
-	EmbeddedEtcdEnabled bool
-}
-
 type Options struct {
 	ServerRunOptions *options.ServerRunOptions
 	EmbeddedEtcd     *EmbeddedEtcd
-	Extra            *ExtraConfig
+	ClientKeyFile    string
 }
 
 type completedOptions struct {
 	ServerRunOptions *kubeapiserver.CompletedServerRunOptions
 	EmbeddedEtcd     *EmbeddedEtcd
-	Extra            *ExtraConfig
+	ClientKeyFile    string
 }
 
 type CompletedOptions struct {
@@ -43,81 +33,54 @@ type CompletedOptions struct {
 
 func NewServerRunOptions() *Options {
 	o := options.NewServerRunOptions()
-	var e *EmbeddedEtcd
 
 	s := Options{
 		ServerRunOptions: o,
-		EmbeddedEtcd:     e,
-		Extra: &ExtraConfig{
-			RootDirectory: DefaultDirectory,
-		},
+		EmbeddedEtcd:     NewEmbeddedEtcd(),
 	}
 	return &s
 }
 
 func (o *Options) Validate(args []string) error {
 	errors := []error{}
 	errors = append(errors, o.ServerRunOptions.Validate()...)
-	if o.Extra.EmbeddedEtcdEnabled {
-		errors = append(errors, o.EmbeddedEtcd.Validate()...)
-	}
+	errors = append(errors, o.EmbeddedEtcd.Validate()...)
 	return utilerrors.NewAggregate(errors)
 }
 
+func (e *Options) AddFlags(fs *pflag.FlagSet) {
+	e.ServerRunOptions.SecureServing.AddFlags(fs)
+	e.ServerRunOptions.Etcd.AddFlags(fs)
+	e.ServerRunOptions.Authentication.AddFlags(fs)
+	e.ServerRunOptions.Authorization.AddFlags(fs)
+	e.ServerRunOptions.Admission.AddFlags(fs)
+	e.ServerRunOptions.GenericServerRunOptions.AddUniversalFlags(fs)
+	e.EmbeddedEtcd.AddFlags(fs)
+
+	fs.StringVar(&e.ClientKeyFile, "client-key-file", e.ClientKeyFile, "client cert key file")
+	fs.StringVar(&e.ServerRunOptions.ServiceAccountSigningKeyFile, "service-account-signing-key-file", e.ServerRunOptions.ServiceAccountSigningKeyFile, ""+
+		"Path to the file that contains the current private key of the service account token issuer. The issuer will sign issued ID tokens with this private key.")
+	fs.StringVar(&e.ServerRunOptions.ServiceClusterIPRanges, "service-cluster-ip-range", e.ServerRunOptions.ServiceClusterIPRanges, ""+
+		"A CIDR notation IP range from which to assign service cluster IPs. This must not "+
+		"overlap with any IP ranges assigned to nodes or pods. Max of two dual-stack CIDRs is allowed.")
+}
+
 func (o *Options) Complete() (*CompletedOptions, error) {
 	s, err := kubeapiserver.Complete(o.ServerRunOptions)
 	if err != nil {
 		return nil, err
 	}
 
-	// check for directory
-	if !filepath.IsAbs(o.Extra.RootDirectory) {
-		pwd, err := os.Getwd()
-		if err != nil {
-			return nil, err
-		}
-		o.Extra.RootDirectory = filepath.Join(pwd, o.Extra.RootDirectory)
-	}
-
-	// set embedded etcd if enabled
-	if o.Extra.EmbeddedEtcdEnabled {
-		o.EmbeddedEtcd = NewEmbeddedEtcd()
-		if !filepath.IsAbs(o.EmbeddedEtcd.Directory) {
-			o.EmbeddedEtcd.Directory = filepath.Join(o.Extra.RootDirectory, o.EmbeddedEtcd.Directory)
-		}
-		o.ServerRunOptions.Etcd.StorageConfig.Transport.ServerList = []string{"localhost:" + o.EmbeddedEtcd.ClientPort}
-		o.EmbeddedEtcd.Enabled = true
-	}
-
 	c := completedOptions{
 		ServerRunOptions: &s,
 		EmbeddedEtcd:     o.EmbeddedEtcd,
-		Extra:            o.Extra,
+		ClientKeyFile:    o.ClientKeyFile,
 	}
 
 	return &CompletedOptions{&c}, nil
 }
 
 func (c *CompletedOptions) Run() error {
-
-	// check for directory
-	if dir := c.Extra.RootDirectory; len(dir) != 0 {
-		if fi, err := os.Stat(dir); err != nil {
-			if !os.IsNotExist(err) {
-				return err
-			}
-			if err := os.MkdirAll(dir, 0755); err != nil {
-				return err
-			}
-		} else {
-			if !fi.IsDir() {
-				return fmt.Errorf("%q is a file, please delete or select another location", dir)
-			}
-		}
-		// set this environment viriable to help set up kube csr controllers
-		// os.Setenv("OCM_CONFIG_DIRECTORY", c.Extra.RootDirectory)
-	}
-
 	// set etcd to embeddedetcd info
 	if c.EmbeddedEtcd != nil && c.EmbeddedEtcd.Enabled {
 		es := &etcd.Server{
@@ -139,5 +102,5 @@ func (c *CompletedOptions) Run() error {
 		return fmt.Errorf("error creating self-signed certificates: %v", err)
 	}
 
-	return kubeapiserver.Run(*c.ServerRunOptions, genericapiserver.SetupSignalHandler())
+	return kubeapiserver.Run(*c.ServerRunOptions, c.ClientKeyFile, genericapiserver.SetupSignalHandler())
 }

controlplane/pkg/apiserver/embeddedetcd.go

@@ -33,13 +33,14 @@ type EmbeddedEtcd struct {
 
 func NewEmbeddedEtcd() *EmbeddedEtcd {
 	return &EmbeddedEtcd{
-		Directory:  "",
+		Directory:  ".ocmconfig",
 		PeerPort:   "2380",
 		ClientPort: "2379",
 	}
 }
 
 func (e *EmbeddedEtcd) AddFlags(fs *pflag.FlagSet) {
+	fs.BoolVar(&e.Enabled, "enable-embedded-etcd", false, "will use embedded etcd, if set to true")
 	fs.StringVar(&e.Directory, "embedded-etcd-directory", e.Directory, "Directory for embedded etcd")
 	fs.StringVar(&e.PeerPort, "embedded-etcd-peer-port", e.PeerPort, "Port for embedded etcd peer")
 	fs.StringVar(&e.ClientPort, "embedded-etcd-client-port", e.ClientPort, "Port for embedded etcd client")

controlplane/pkg/apiserver/kubeapiserver/aggregator.go

@@ -115,7 +115,7 @@ func createAggregatorConfig(
 	return aggregatorConfig, nil
 }
 
-func createAggregatorServer(aggregatorConfig *aggregatorapiserver.Config, delegateAPIServer genericapiserver.DelegationTarget, apiExtensionInformers apiextensionsinformers.SharedInformerFactory) (*aggregatorapiserver.APIAggregator, error) {
+func createAggregatorServer(aggregatorConfig *aggregatorapiserver.Config, delegateAPIServer genericapiserver.DelegationTarget, apiExtensionInformers apiextensionsinformers.SharedInformerFactory, clientCert, clientKey string) (*aggregatorapiserver.APIAggregator, error) {
 	aggregatorServer, err := aggregatorConfig.Complete().NewWithDelegate(delegateAPIServer)
 	if err != nil {
 		return nil, err
@@ -164,7 +164,7 @@ func createAggregatorServer(aggregatorConfig *aggregatorapiserver.Config, delega
 	err = aggregatorServer.GenericAPIServer.AddPostStartHook("kube-controller", func(context genericapiserver.PostStartHookContext) error {
 		controllerConfig := rest.CopyConfig(aggregatorConfig.GenericConfig.LoopbackClientConfig)
 		go func() {
-			kubecontroller.RunKubeControllers(controllerConfig)
+			kubecontroller.RunKubeControllers(controllerConfig, clientCert, clientKey)
 			klog.Infof("Finished bootstrapping kube controllers")
 		}()
 		return nil

controlplane/pkg/apiserver/kubeapiserver/server.go

@@ -78,13 +78,13 @@ func init() {
 }
 
 // Run runs the specified APIServer.  This should never exit.
-func Run(completeOptions CompletedServerRunOptions, stopCh <-chan struct{}) error {
+func Run(completeOptions CompletedServerRunOptions, clientKeyFile string, stopCh <-chan struct{}) error {
 	// To help debugging, immediately log version
 	klog.Infof("Version: %+v", version.Get())
 
 	klog.InfoS("Golang settings", "GOGC", os.Getenv("GOGC"), "GOMAXPROCS", os.Getenv("GOMAXPROCS"), "GOTRACEBACK", os.Getenv("GOTRACEBACK"))
 
-	server, err := CreateServerChain(completeOptions)
+	server, err := CreateServerChain(completeOptions, clientKeyFile)
 	if err != nil {
 		return err
 	}
@@ -98,7 +98,7 @@ func Run(completeOptions CompletedServerRunOptions, stopCh <-chan struct{}) erro
 }
 
 // CreateServerChain creates the apiservers connected via delegation.
-func CreateServerChain(completedOptions CompletedServerRunOptions) (*aggregatorapiserver.APIAggregator, error) {
+func CreateServerChain(completedOptions CompletedServerRunOptions, clientKeyFile string) (*aggregatorapiserver.APIAggregator, error) {
 	kubeAPIServerConfig, serviceResolver, pluginInitializer, err := CreateKubeAPIServerConfig(completedOptions)
 	if err != nil {
 		return nil, err
@@ -127,7 +127,11 @@ func CreateServerChain(completedOptions CompletedServerRunOptions) (*aggregatora
 	if err != nil {
 		return nil, err
 	}
-	aggregatorServer, err := createAggregatorServer(aggregatorConfig, kubeAPIServer.GenericAPIServer, apiExtensionsServer.Informers)
+	aggregatorServer, err := createAggregatorServer(
+		aggregatorConfig, kubeAPIServer.GenericAPIServer, apiExtensionsServer.Informers,
+		completedOptions.Authentication.ClientCert.ClientCA,
+		clientKeyFile,
+	)
 	if err != nil {
 		// we don't need special handling for innerStopCh because the aggregator server doesn't create any go routines
 		return nil, err

controlplane/pkg/controllers/kubecontroller/controllermanager.go

@@ -51,11 +51,15 @@ const (
 	ExternalLoops
 )
 
-func RunKubeControllers(cfg *restclient.Config) error {
+func RunKubeControllers(cfg *restclient.Config, clientCert, clientKey string) error {
 	s, err := options.NewKubeControllerManagerOptions()
 	if err != nil {
 		klog.Fatalf("unable to initialize kube options: %v", err)
 	}
+
+	s.CSRSigningController.ClusterSigningCertFile = clientCert
+	s.CSRSigningController.ClusterSigningKeyFile = clientKey
+
 	config, err := s.Config(cfg)
 	if err != nil {
 		klog.Fatalf("unable to config kube controller options: %v", err)
@@ -165,7 +169,8 @@ type ControllerContext struct {
 type InitFunc func(ctx context.Context, controllerCtx ControllerContext) (controller controller.Interface, enabled bool, err error)
 
 // ControllerInitializersFunc is used to create a collection of initializers
-//  given the loopMode.
+//
+//	given the loopMode.
 type ControllerInitializersFunc func() (initializers map[string]InitFunc)
 
 var _ ControllerInitializersFunc = NewControllerInitializers

controlplane/pkg/controllers/kubecontroller/options/options.go

@@ -1,8 +1,6 @@
 package options
 
 import (
-	"os"
-
 	v1 "k8s.io/api/core/v1"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	clientset "k8s.io/client-go/kubernetes"
@@ -46,12 +44,6 @@ func NewKubeControllerManagerOptions() (*KubeControllerManagerOptions, error) {
 		return nil, err
 	}
 
-	// TODO(ycyaoxdu): this need to be handled
-	// use environment variable to pass the directory
-	certFolder := os.Getenv("OCM_CONFIG_DIRECTORY") + "/cert"
-	componentConfig.CSRSigningController.KubeAPIServerClientSignerConfiguration.CertFile = certFolder + "/" + "client-ca.crt"
-	componentConfig.CSRSigningController.KubeAPIServerClientSignerConfiguration.KeyFile = certFolder + "/" + "client-ca.key"
-
 	s := KubeControllerManagerOptions{
 		Generic: cmoptions.NewGenericControllerManagerConfigurationOptions(&componentConfig.Generic),