Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix #4948 part of #3397 #5119

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
110 changes: 65 additions & 45 deletions e2e/advanced/operator_metrics_test.go

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion e2e/common/cli/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"strings"
"testing"

"github.com/apache/camel-k/v2/pkg/cmd"
corev1 "k8s.io/api/core/v1"

. "github.com/onsi/gomega"
Expand All @@ -36,7 +37,6 @@ import (

. "github.com/apache/camel-k/v2/e2e/support"
v1 "github.com/apache/camel-k/v2/pkg/apis/camel/v1"
"github.com/apache/camel-k/v2/pkg/cmd"
)

func TestKamelCLIConfig(t *testing.T) {
Expand Down
10 changes: 9 additions & 1 deletion e2e/support/test_support.go
Original file line number Diff line number Diff line change
Expand Up @@ -2420,6 +2420,14 @@ func ConsoleCLIDownload(t *testing.T, ctx context.Context, name string) func() *
}

func OperatorPod(t *testing.T, ctx context.Context, ns string) func() *corev1.Pod {
return componentPod(t, ctx, ns, "operator")
}

func PlatformcontrollerPod(t *testing.T, ctx context.Context, ns string) func() *corev1.Pod {
return componentPod(t, ctx, ns, "platformcontroller")
}

func componentPod(t *testing.T, ctx context.Context, ns string, componentLabelValue string) func() *corev1.Pod {
return func() *corev1.Pod {
lst := corev1.PodList{
TypeMeta: metav1.TypeMeta{
Expand All @@ -2430,7 +2438,7 @@ func OperatorPod(t *testing.T, ctx context.Context, ns string) func() *corev1.Po
if err := TestClient(t).List(ctx, &lst,
ctrl.InNamespace(ns),
ctrl.MatchingLabels{
"camel.apache.org/component": "operator",
"camel.apache.org/component": componentLabelValue,
valdar marked this conversation as resolved.
Show resolved Hide resolved
}); err != nil {
failTest(t, err)
}
Expand Down
111 changes: 111 additions & 0 deletions helm/camel-k/templates/platformcontroller.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
# ---------------------------------------------------------------------------
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ---------------------------------------------------------------------------

apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: camel-k
camel.apache.org/component: platformcontroller
{{- include "camel-k.labels" . | nindent 4 }}
{{- with .Values.operator.annotations }}
annotations:
{{ toYaml . | nindent 4 }}
{{- end }}
name: camel-k-platformcontroller
spec:
replicas: 1
selector:
matchLabels:
name: camel-k-platformcontroller
strategy:
type: Recreate
template:
metadata:
labels:
app: camel-k
camel.apache.org/component: platformcontroller
name: camel-k-platformcontroller
spec:
{{- if .Values.operator.imagePullSecrets }}
imagePullSecrets:
{{ toYaml .Values.operator.imagePullSecrets | indent 8 }}
{{- end }}

containers:
- command:
- kamel
- platformcontroller
env:
- name: WATCH_NAMESPACE
{{- if eq .Values.operator.global "false" }}
valueFrom:
fieldRef:
fieldPath: metadata.namespace
{{- else }}
value: ""
{{- end }}
- name: LOG_LEVEL
value: {{ .Values.operator.logLevel }}
- name: OPERATOR_NAME
value: camel-k-platformcontroller
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: KAMEL_OPERATOR_ID
value: {{ .Values.operator.operatorId }}
image: {{ .Values.operator.image }}
imagePullPolicy: IfNotPresent
livenessProbe:
httpGet:
path: /healthz
port: 8081
initialDelaySeconds: 20
periodSeconds: 10
name: camel-k-platformcontroller
ports:
- containerPort: 8080
name: metrics
{{- with .Values.operator.resources }}
resources:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- if .Values.operator.securityContext }}
{{- with .Values.operator.securityContext }}
securityContext:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- else }}
securityContext:
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
{{- end }}
serviceAccountName: camel-k-operator
{{- with .Values.operator.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
193 changes: 193 additions & 0 deletions pkg/cmd/manager/controller.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,193 @@
/*
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package manager

import (
"context"
"strconv"
"time"

"github.com/apache/camel-k/v2/pkg/apis"
"github.com/apache/camel-k/v2/pkg/client"
"github.com/apache/camel-k/v2/pkg/util/kubernetes"
logutil "github.com/apache/camel-k/v2/pkg/util/log"
coordination "k8s.io/api/coordination/v1"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/leaderelection/resourcelock"
"sigs.k8s.io/controller-runtime/pkg/cache"
"sigs.k8s.io/controller-runtime/pkg/client/config"
"sigs.k8s.io/controller-runtime/pkg/healthz"
"sigs.k8s.io/controller-runtime/pkg/manager"
"sigs.k8s.io/controller-runtime/pkg/manager/signals"
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
)

type Manager interface {
lburgazzoli marked this conversation as resolved.
Show resolved Hide resolved
PrintVersion()
CreateBootstrapClient(cfg *rest.Config) (client.Client, string, error)
GetControllerNamespaceAndLeaderElection(ctx context.Context, bootstrapClient client.Client, leaderElection bool) (string, bool, string, error)
GetManagerOptions(bootstrapClient client.Client) (cache.Options, string, error)
CreateManager(ctx context.Context, healthPort int32, monitoringPort int32, leaderElection bool, leaderElectionID string, cfg *rest.Config, controllerNamespace string, options cache.Options) (manager.Manager, client.Client, string, error)
ControllerPreStartResourcesInit(ctx context.Context, initCtx context.Context, bootstrapClient client.Client, controllerNamespace string, ctrlClient client.Client, mgr manager.Manager) (string, error)
}

func NewControllerCmd(controllerManager Manager, log logutil.Logger) *ControllerCmd {
return &ControllerCmd{
controllerManager: controllerManager,
log: log,
}
}

type ControllerCmd struct {
controllerManager Manager
log logutil.Logger
}

func (c ControllerCmd) Run(healthPort, monitoringPort int32, leaderElection bool, leaderElectionID string) (string, error) {
errMessage, err := setMaxprocs(c.log)
if err != nil {
return errMessage, err
}

c.controllerManager.PrintVersion()
// Will only appear if DEBUG level has been enabled using the env var LOG_LEVEL
c.log.Debug("*** DEBUG level messages will be logged ***")

cfg, err := config.GetConfig()
if err != nil {
return "cannot get client config", err
}
bootstrapClient, errMessage, err := c.controllerManager.CreateBootstrapClient(cfg)
if err != nil {
return errMessage, err
}

ctx := signals.SetupSignalHandler()

controllerNamespace, leaderElection, errMessage, err := c.controllerManager.GetControllerNamespaceAndLeaderElection(ctx, bootstrapClient, leaderElection)
if err != nil {
return errMessage, err
}
if !leaderElection {
c.log.Info("Leader election is disabled!")
}

errMessage, err = setOperatorImage(ctx, bootstrapClient, controllerNamespace)
if err != nil {
return errMessage, err
}

options, errMessage, err := c.controllerManager.GetManagerOptions(bootstrapClient)
if err != nil {
return errMessage, err
}

mgr, ctrlClient, errMessage, err := c.controllerManager.CreateManager(ctx, healthPort, monitoringPort, leaderElection, leaderElectionID, cfg, controllerNamespace, options)
if err != nil {
return errMessage, err
}

initCtx, initCancel := context.WithTimeout(ctx, 1*time.Minute)
defer initCancel()

errMessage, err = c.controllerManager.ControllerPreStartResourcesInit(ctx, initCtx, bootstrapClient, controllerNamespace, ctrlClient, mgr)
if err != nil {
return errMessage, err
}

c.log.Info("Starting the manager")
return "manager exited non-zero", mgr.Start(ctx)
}

type BaseManager struct {
Log logutil.Logger
WatchNamespace string
ControllerNamespace string
AddToManager func(ctx context.Context, manager manager.Manager, client client.Client) error
}

func (bm BaseManager) CreateBootstrapClient(cfg *rest.Config) (client.Client, string, error) {
// Increase maximum burst that is used by client-side throttling,
// to prevent the requests made to apply the bundled Kamelets
// from being throttled.
cfg.QPS = 20
cfg.Burst = 200
bootstrapClient, err := client.NewClientWithConfig(false, cfg)
if err != nil {
return nil, "cannot initialize client", err
}

return bootstrapClient, "", nil
}

func (bm BaseManager) GetControllerNamespaceAndLeaderElection(ctx context.Context, bootstrapClient client.Client, leaderElection bool) (string, bool, string, error) {
controllerNamespace := bm.ControllerNamespace
if controllerNamespace == "" {
// Fallback to using the watch namespace when the operator is not in-cluster.
// It does not support local (off-cluster) operator watching resources globally,
// in which case it's not possible to determine a namespace.
controllerNamespace = bm.WatchNamespace
if controllerNamespace == "" {
leaderElection = false
bm.Log.Info("unable to determine namespace for leader election")
}
}

if ok, err := kubernetes.CheckPermission(ctx, bootstrapClient, coordination.GroupName, "leases", controllerNamespace, "", "create"); err != nil || !ok {
valdar marked this conversation as resolved.
Show resolved Hide resolved
leaderElection = false
if err != nil {
return controllerNamespace, leaderElection, "cannot check permissions for creating Leases", err
}
bm.Log.Info("The operator is not granted permissions to create Leases")
}

return controllerNamespace, leaderElection, "", nil
}

func (bm BaseManager) CreateManager(ctx context.Context, healthPort int32, monitoringPort int32, leaderElection bool, leaderElectionID string, cfg *rest.Config, controllerNamespace string, options cache.Options) (manager.Manager, client.Client, string, error) {
mgr, err := manager.New(cfg, manager.Options{
LeaderElection: leaderElection,
LeaderElectionNamespace: controllerNamespace,
LeaderElectionID: leaderElectionID,
LeaderElectionResourceLock: resourcelock.LeasesResourceLock,
LeaderElectionReleaseOnCancel: true,
HealthProbeBindAddress: ":" + strconv.Itoa(int(healthPort)),
Metrics: metricsserver.Options{BindAddress: ":" + strconv.Itoa(int(monitoringPort))},
Cache: options,
})
if err != nil {
return nil, nil, "", err
}

bm.Log.Info("Configuring manager")
if err := mgr.AddHealthzCheck("health-probe", healthz.Ping); err != nil {
return nil, nil, "Unable add liveness check", err
}
if err := apis.AddToScheme(mgr.GetScheme()); err != nil {
return nil, nil, "", err
}
ctrlClient, err := client.FromManager(mgr)
if err != nil {
return nil, nil, "", err
}
if err := bm.AddToManager(ctx, mgr, ctrlClient); err != nil {
return nil, nil, "", err
}

return mgr, ctrlClient, "", nil
}
Loading