fix: RBAC on EKS and kustomize updates

.github/workflows/build-test-release.yml

@@ -91,23 +91,20 @@ jobs:
           make docker-buildx
       - name: Update Helm chart defaults
         run: |
-          yq e -i '.controllerManager.manager.image.repository = "${{ steps.read_version.outputs.controller_image_repository }}"' deploy/helm/values.yaml
-          yq e -i '.controllerManager.manager.image.tag = "${{ steps.read_version.outputs.version }}"' deploy/helm/values.yaml
-          yq e -i '.controllerManager.telemetryProxy.image.repository = "${{ steps.read_version.outputs.proxy_image_repository }}"' deploy/helm/values.yaml
-          yq e -i '.controllerManager.telemetryProxy.image.tag = "${{ steps.read_version.outputs.version }}"' deploy/helm/values.yaml
+          yq e -i '.controllerManager.manager.image.repository = "${{ steps.read_version.outputs.controller_image_repository }}"' charts/lumigo-operator/values.yaml
+          yq e -i '.controllerManager.manager.image.tag = "${{ steps.read_version.outputs.version }}"' charts/lumigo-operator/values.yaml
+          yq e -i '.controllerManager.telemetryProxy.image.repository = "${{ steps.read_version.outputs.proxy_image_repository }}"' charts/lumigo-operator/values.yaml
+          yq e -i '.controllerManager.telemetryProxy.image.tag = "${{ steps.read_version.outputs.version }}"' charts/lumigo-operator/values.yaml
       - name: Update Helm chart metadata
         run: |
-          yq e -i '.version = "${{ steps.read_version.outputs.version }}"' deploy/helm/Chart.yaml
-          yq e -i '.appVersion = "${{ steps.read_version.outputs.version }}"' deploy/helm/Chart.yaml
-      #
+          yq e -i '.version = "${{ steps.read_version.outputs.version }}"' charts/lumigo-operator/Chart.yaml
+          yq e -i '.appVersion = "${{ steps.read_version.outputs.version }}"' charts/lumigo-operator/Chart.yaml
       - name: Configure Git
         run: |
           git config user.name "$GITHUB_ACTOR"
           git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
       - name: Publish Helm chart version
-        uses: helm/chart-releaser-action@v1.5.0
+        uses: helm/chart-releaser-action@v1.2.1
         env:
           CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
-        with:
-          charts_dir: deploy/helm
-          install_dir: tools/
+          CR_RELEASE_NAME_TEMPLATE: 'v{{ .Version }}'

CONTRIBUTING.md

@@ -54,8 +54,8 @@ $ curl localhost:5000/v2/_catalog -v
 Deploy the Lumigo operator with:
 
 ```sh
-CONTROLLER_IMG=host.docker.internal:5000/controller:latest make docker-build docker-push
-helm install lumigo deploy/helm --namespace lumigo-system --create-namespace --set "controllerManager.manager.image.repository=host.docker.internal:5000/controller" --set "controllerManager.manager.image.tag=latest"
+make docker-build docker-push
+helm install lumigo deploy/helm/templates/lumigo-operator --namespace lumigo-system --create-namespace --set "controllerManager.manager.image.repository=host.docker.internal:5000/controller" --set "controllerManager.manager.image.tag=latest" --set "controllerManager.telemetryProxy.image.repository=host.docker.internal:5000/telemetr-proxy" --set "controllerManager.telemetryProxy.image.tag=latest"
 ```
 
 ### Troubleshooting

Makefile

@@ -6,8 +6,9 @@
 VERSION ?= 0.0.1
 
 # Image URLs to use all building/pushing image targets
-CONTROLLER_IMG ?= controller:latest
-PROXY_IMG ?= proxy:latest
+IMG_VERSION ?= latest
+CONTROLLER_IMG ?= host.docker.internal:5000/controller:$(IMG_VERSION)
+PROXY_IMG ?= host.docker.internal:5000/telemetry-proxy:$(IMG_VERSION)
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
 ENVTEST_K8S_VERSION = 1.25.0
 

deploy/helm/templates/deployment.yaml → charts/lumigo-operator/templates/deployment.yaml

@@ -80,6 +80,7 @@ spec:
             - ALL
           readOnlyRootFilesystem: true
           runAsNonRoot: true
+          runAsUser: 1234
         volumeMounts:
         - name: cert
           mountPath: /tmp/k8s-webhook-server/serving-certs

deploy/helm/templates/manager-rbac.yaml → charts/lumigo-operator/templates/manager-rbac.yaml

@@ -52,8 +52,6 @@ rules:
   - watch
 - apiGroups:
   - apps
-  apiVersions:
-  - v1
   resources:
   - daemonsets
   - deployments
@@ -66,8 +64,6 @@ rules:
   - update
 - apiGroups:
   - batch
-  apiVersions:
-  - v1
   resources:
   - cronjobs
   - jobs

deploy/helm/values.yaml → charts/lumigo-operator/values.yaml

@@ -15,7 +15,7 @@ controllerManager:
         memory: 64Mi
   manager:
     image:
-      repository: host.docker.internal:5000/controller # TODO Update default
+      repository: host.docker.internal:5000/controller
       tag: latest
     resources:
       limits:
@@ -26,7 +26,7 @@ controllerManager:
         memory: 64Mi
   telemetryProxy:
     image:
-      repository: host.docker.internal:5000/proxy # TODO Update default
+      repository: host.docker.internal:5000/telemetry-proxy
       tag: latest
     resources:
       limits:

config/certmanager/certificate.yaml

@@ -29,10 +29,9 @@ metadata:
   name: serving-cert  # this name should match the one appeared in kustomizeconfig.yaml
   namespace: system
 spec:
-  # $(SERVICE_NAME) and $(SERVICE_NAMESPACE) will be substituted by kustomize
   dnsNames:
-  - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc
-  - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc.cluster.local
+  - $(INJECTOR_SERVICE_NAME).$(INJECTOR_SERVICE_NAMESPACE).svc
+  - $(INJECTOR_SERVICE_NAME).$(INJECTOR_SERVICE_NAMESPACE).svc.cluster.local
   issuerRef:
     kind: Issuer
     name: selfsigned-issuer

config/default/kustomization.yaml

@@ -18,6 +18,7 @@ bases:
 - ../manager
 - ../webhook
 - ../certmanager
+- ../telemetry-proxy
 # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
 #- ../prometheus
 
@@ -26,6 +27,7 @@ patchesStrategicMerge:
 # If you want your controller-manager to expose the /metrics
 # endpoint w/o any authn/z, please comment the following line.
 - manager_auth_proxy_patch.yaml
+- manager_telemetry_proxy_patch.yaml
 - manager_webhook_patch.yaml
 
 - webhookcainjection_patch.yaml
@@ -46,15 +48,27 @@ vars:
     group: cert-manager.io
     version: v1
     name: serving-cert # this name should match the one in certificate.yaml
-- name: SERVICE_NAMESPACE # namespace of the service
+- name: INJECTOR_SERVICE_NAMESPACE # namespace of the service
   objref:
     kind: Service
     version: v1
     name: injector-webhook-service
   fieldref:
     fieldpath: metadata.namespace
-- name: SERVICE_NAME
+- name: INJECTOR_SERVICE_NAME
   objref:
     kind: Service
     version: v1
     name: injector-webhook-service
+- name: TELEMETRY_PROXY_SERVICE_NAMESPACE # namespace of the service
+  objref:
+    kind: Service
+    version: v1
+    name: telemetry-proxy-service
+  fieldref:
+    fieldpath: metadata.namespace
+- name: TELEMETRY_PROXY_SERVICE_NAME
+  objref:
+    kind: Service
+    version: v1
+    name: telemetry-proxy-service

config/default/manager_pin_injector_version_patch.yaml

@@ -0,0 +1,12 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: controller-manager
+spec:
+  template:
+    spec:
+      containers:
+      - name: manager
+        env:
+          - name: LUMIGO_INJECTOR_IMAGE
+            value: public.ecr.aws/lumigo/lumigo-autotrace:latest # CHANGE this to the image you want to use

config/default/manager_telemetry_proxy_patch.yaml

@@ -0,0 +1,39 @@
+# This patch inject a sidecar container which is a HTTP proxy for the
+# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: controller-manager
+  namespace: system
+spec:
+  template:
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                - key: kubernetes.io/arch
+                  operator: In
+                  values:
+                    - amd64
+                    - arm64
+                - key: kubernetes.io/os
+                  operator: In
+                  values:
+                    - linux
+      containers:
+      - name: telemetry-proxy
+        image: proxy:latest
+        ports:
+          - containerPort: 4318
+            name: otlphttp
+            protocol: TCP
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+              - ALL
+          readOnlyRootFilesystem: true
+          runAsNonRoot: true
+          runAsUser: 1234

config/manager/kustomization.yaml

@@ -4,5 +4,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 images:
 - name: controller
-  newName: localhost:5000/controller
-  newTag: v70
+  newName: host.docker.internal:5000/controller
+  newTag: latest
+- name: telemetry-proxy
+  newName: host.docker.internal:5000/telemetry-proxy
+  newTag: latest

config/manager/manager.yaml

@@ -24,6 +24,7 @@ metadata:
     app.kubernetes.io/created-by: lumigo
     app.kubernetes.io/part-of: lumigo
     app.kubernetes.io/managed-by: kustomize
+    lumigo.auto-trace: "false"
 spec:
   selector:
     matchLabels:
@@ -36,28 +37,25 @@ spec:
       labels:
         control-plane: controller-manager
     spec:
-      # TODO(user): Uncomment the following code to configure the nodeAffinity expression
-      # according to the platforms which are supported by your solution. 
-      # It is considered best practice to support multiple architectures. You can
-      # build your manager image using the makefile target docker-buildx.
-      # affinity:
-      #   nodeAffinity:
-      #     requiredDuringSchedulingIgnoredDuringExecution:
-      #       nodeSelectorTerms:
-      #         - matchExpressions:
-      #           - key: kubernetes.io/arch
-      #             operator: In
-      #             values:
-      #               - amd64
-      #               - arm64
-      #               - ppc64le
-      #               - s390x
-      #           - key: kubernetes.io/os
-      #             operator: In
-      #             values:
-      #               - linux
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                - key: kubernetes.io/arch
+                  operator: In
+                  values:
+                    - amd64
+                    - arm64
+                    # - ppc64le
+                    # - s390x
+                - key: kubernetes.io/os
+                  operator: In
+                  values:
+                    - linux
       securityContext:
         runAsNonRoot: true
+        fsGroup: 1234
         # TODO(user): For common cases that do not require escalating privileges
         # it is recommended to ensure that all your Pods/Containers are restrictive.
         # More info: https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted
@@ -72,11 +70,21 @@ spec:
         - --leader-elect
         image: controller:latest
         name: manager
+        env:
+          - name: TELEMETRY_PROXY_OTLP_SERVICE
+            value: http://$(TELEMETRY_PROXY_SERVICE).$(TELEMETRY_PROXY_SERVICE_NAMESPACE).svc.cluster.local
+          - name: LUMIGO_OPERATOR_VERSION
+            value: "1"
+          - name: LUMIGO_INJECTOR_IMAGE
+            value: public.ecr.aws/lumigo/lumigo-autotrace:latest
         securityContext:
           allowPrivilegeEscalation: false
           capabilities:
             drop:
-              - "ALL"
+              - ALL
+          readOnlyRootFilesystem: true
+          runAsNonRoot: true
+          runAsUser: 1234
         livenessProbe:
           httpGet:
             path: /healthz

config/rbac/role.yaml

@@ -31,11 +31,42 @@ rules:
   - get
   - patch
   - update
-- # The Lumigo operator will access only secrets that are referenced by a Lumigo CRD
-  apiGroups:
+- apiGroups:
   - "" # Core API group
   resources:
+  - events
+  - namespaces
+  - namespaces/status
+  - nodes
+  - nodes/spec
+  - pods
+  - pods/status
+  - replicationcontrollers
+  - replicationcontrollers/status
+  - resourcequotas
+  # The Lumigo operator will access only secrets that are referenced by a Lumigo CRD
   - secrets
+  - services
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - apps
+  resources:
+  - daemonsets
+  - deployments
+  - replicasets
+  - statefulsets
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - batch
+  resources:
+  - cronjobs
+  - jobs
   verbs:
   - get
   - list

config/samples/operator_v1alpha1_lumigo.yaml

@@ -3,7 +3,9 @@ kind: Secret
 metadata:
   name: lumigo-credentials
 stringData:
-  token: t_cdef40123435481f8c549
+  # Kubectl won't allow you to deploy this dangling anchor.
+  # Get the actual value from Lumigo following this documentation: https://docs.lumigo.io/docs/lumigo-tokens
+  token: *lumigo-token # Example: t_123456789012345678901
 ---
 apiVersion: operator.lumigo.io/v1alpha1
 kind: Lumigo

config/telemetry-proxy/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+- service.yaml

config/telemetry-proxy/service.yaml

@@ -0,0 +1,22 @@
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: telemetry-proxy-service
+  labels:
+    app.kubernetes.io/name: telemetry-proxy-service
+    app.kubernetes.io/instance: telemetry-proxy-service
+    app.kubernetes.io/component: telemetry-proxy
+    app.kubernetes.io/created-by: lumigo
+    app.kubernetes.io/part-of: lumigo
+spec:
+  type: ClusterIP
+  ports:
+    - name: otlphttp
+      protocol: TCP
+      # TODO Can we use generated certificates to make it HTTPS?
+      # If we used self-signed certs, how would we pass the CA to OTLP exporters in client apps?
+      port: 80
+      targetPort: otlphttp
+  selector:
+    control-plane: controller-manager