Merge branch 'main' into messagequeue-image-tag

nrwl · Jul 17, 2024 · 64daf36 · 64daf36
2 parents bb4ce3c + 257db7a
commit 64daf36
Show file tree

Hide file tree

Showing 12 changed files with 299 additions and 6 deletions.
diff --git a/AUTH-GUIDE.md b/AUTH-GUIDE.md
@@ -8,6 +8,8 @@ Then update your `helm-values.yaml` and `secrets.yaml`:
 ```yaml
 # helm-values.yml
 github:
+    pr:
+      apiUrl: 'if-using-self-hosted'
     auth:
         enabled: true
 
@@ -42,8 +44,10 @@ Then update your `helm-values.yaml` and `secrets.yaml`:
 ```yaml
 # helm-values.yml
 gitlab:
+    apiUrl: 'if-using-self-hosted'
     auth:
         enabled: true
+        
 
 secret:
   name: 'cloudsecret'
@@ -76,6 +80,7 @@ Then update your `helm-values.yaml` and `secrets.yaml`:
 ```yaml
 # helm-values.yml
 bitbucket:
+    apiUrl: 'if-using-self-hosted'
     auth:
         enabled: true
 

diff --git a/agents-guide/AGENTS-GUIDE.md b/agents-guide/AGENTS-GUIDE.md
@@ -0,0 +1,97 @@
+# Getting started with Nx Agents
+
+We recommend deploying Nx Agents onto a new cluster, but you can also deploy onto your existing NxCloud cluster under a different namespace.
+
+### Install Valkey
+
+Valkey is an in-memory key-value store (like Redis) that is used by the workflow controller to hold temporary state.
+
+1. Create a secret similar to `agents-guide/agents-secrets.yml` and set your valkey password in there.
+   - Important: the key `valkey-password` itself shouldn't be changed, only its value
+   - You do not need to set the secret values for the S3 bucket yet. They are there as an example. Please refer to `charts/nx-agents/values.yaml` for info on what Agent storage options we support.
+2. Apply the secret: `kubectl apply -f agents-secrets.yml`
+3. Now let's deploy Valkey:
+    ```bash
+    helm repo add bitnami https://charts.bitnami.com/bitnami
+    helm install valkey bitnami/valkey --set auth.usePassword=true --set auth.existingSecret=nx-cloud-agents-secret
+    ```
+
+### Deploy the Agents chart onto your cluster
+
+##### Configuring nx-agents.yaml
+Here is an example `nx-agents.yml` config for deployment on AWS you can use as a starting point: [`basic-agents-config.yaml`](./basic-agents-config.yaml)
+(you can also refer to the full [`values.yaml`](../charts/nx-agents/values.yaml) for full docs on all the values we support)
+
+Modify your `nx-agents.yml` values file, and make sure the secrets we created above are linked up:
+1. Ensure `secret.name: nx-cloud-agents-secret` (see [here](https://github.com/nrwl/nx-cloud-helm/blob/main/charts/nx-agents/values.yaml#L132))
+2. Ensure `secret.valkeyPassword: 'valkey-password'`. The name needs to match the exact key you declared in the secret above (example [here](https://github.com/nrwl/nx-cloud-helm/blob/main/charts/nx-agents/values.yaml#L132)).
+
+##### Deploying the Nx Agents cluster
+
+Now you can push your chart changes so your controller can connect to valkey:
+
+```bash
+helm repo add nx-cloud https://nrwl.github.io/nx-cloud-helm
+helm repo update nx-cloud
+helm upgrade --install nx-agents nx-cloud/nx-agents --values=nx-agents.yml
+```
+
+###### Custom valkey URL
+
+If you have deployed valkey in a custom location you can overwrite the default url:
+
+```yaml
+controller:
+  useDefaultValkeyAddress: false # set this to false
+  deployment:
+    port: 9000
+    env:
+      - name: VALKEY_CONNECTION_STRING # declare the custom connection string
+        valueFrom: # you can insert the value from a secret or hardcode it in the nx-agents.yml
+          secretKeyRef:
+            name: nx-cloud-k8s-secret
+            key: valkey-connection-string
+```
+---
+Note on storage:
+1. The Agents need a storage bucket for storing logs and cached items (such as `node_modules`)
+2. You do not need to use S3, we also support Azure Blob Storage and GCloud buckets
+3. To choose a storage service you can configure the below values:
+   - `NX_CLOUD_GCS_BUCKET=...`
+   - `NX_CLOUD_AWS_BUCKET=...`
+   - `AZURE_CONNECTION_STRING=...`
+4. You'll then need to make sure your pods either have the correct service account attached for interacting with the above buckets (see [the AWS example](./AWS-AGENTS-GUIDE.md#connecting-an-s3-bucket))
+
+### Connect NxCloud to your Nx Agents deployment
+
+These are the options you can use to configure how NxCloud connects to your Nx Agents cluster.
+
+Depending on how you deployed your Nx Agents cluster (which namespace you used, whether it was in the same or a different cluster etc.) you might need
+to use different combinations of the below values.
+
+Set these in your NxCloud `values.yaml` file:
+
+```yaml
+nxCloudWorkflows:
+   enabled: true
+   port: 9000
+   
+   # if you have deployed Nx Agents in the same cluster use these options
+   name: 'nx-cloud-workflow-controller-service'
+   workflowsNamespace: 'nx-cloud-workflows'
+
+   # use these options if Nx Agents was deployed in a separate cluster
+   externalName: 'external-address-or-ip.com'
+   # If you find that an externalName service is not working as expected, you can set this to true to create a headless service
+   # which will create an endpoint group as an alternative. Please continue to set `externalName` to the IP address
+   # you wish to direct traffic to as we will use it to populate the endpoint slice.
+   headless: false
+```
+
+Please see `charts/nx-cloud/values.yaml` for up to date documentation on the above options.
+
+Finally, push the updates to your NxCloud cluster:
+
+```bash
+helm upgrade --install nx-cloud nx-cloud/nx-cloud --values=nx-cloud-values.yml
+```
diff --git a/agents-guide/AWS-AGENTS-GUIDE.md b/agents-guide/AWS-AGENTS-GUIDE.md
@@ -0,0 +1,97 @@
+# Deploy an Nx Agents cluster on AWS
+
+## Create the cluster
+
+```bash
+# init the cluster
+eksctl create cluster --name nx-cloud-cluster --region us-east-1 \
+  --nodegroup-name ng-1 --node-type t3.medium --nodes 5 --managed
+
+ # associate the oidc provider
+ eksctl utils associate-iam-oidc-provider \ 
+  --cluster=nx-cloud-cluster --approve
+```
+
+## Installing the EBS CSI add-on
+
+```bash
+# create a service account for the controller to use
+eksctl create iamserviceaccount \
+--name ebs-csi-controller-sa \
+--namespace kube-system \
+--cluster ami-test-agents-cluster \
+--attach-policy-arn arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy \
+--approve \
+--override-existing-serviceaccounts
+
+# install the add-on via helm
+helm repo add aws-ebs-csi-driver https://kubernetes-sigs.github.io/aws-ebs-csi-driver
+helm repo update
+helm install aws-ebs-csi-driver aws-ebs-csi-driver/aws-ebs-csi-driver \
+--namespace kube-system \
+--set controller.serviceAccount.create=false \
+--set controller.serviceAccount.name=ebs-csi-controller-sa
+```
+
+## Install valkey
+
+1. Add a valkey password in [agents-secrets.yml](./agents-secrets.yml)
+2. Deploy valkey:
+
+```bash
+kubectl apply -f agents-secrets.yaml
+helm repo add bitnami https://charts.bitnami.com/bitnami
+helm install valkey bitnami/valkey --set auth.usePassword=true --set auth.existingSecret=nx-cloud-agents-secret
+```
+
+## Connecting an S3 bucket
+
+1. Create an S3 for the agents to store their cache and their logs
+2. Create a policy that allows access to the bucket:
+```json
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Sid": "VisualEditor0",
+            "Effect": "Allow",
+            "Action": [
+                "s3:PutObject",
+                "s3:GetObjectAcl",
+                "s3:GetObject",
+                "s3:ListBucket",
+                "s3:DeleteObject",
+                "s3:PutObjectAcl"
+            ],
+            "Resource": [
+                "arn:aws:s3:::your-bucket-name",
+                "arn:aws:s3:::your-bucket-name/*"
+            ]
+        }
+    ]
+}
+```
+3. Attach the above policy to the NodeGroup IAM Role for your EKS Cluster
+
+## Deploy Nx Agents
+
+```bash
+helm upgrade --install nx-agents nx-cloud/nx-agents \
+--values=./nx-agents.yml \
+--set controller.image.tag="2405.02.15"
+```
+
+## Copy the public URL 
+
+```bash
+# copy the EXTERNAL-IP value
+kubectl get service nx-cloud-workflow-controller-service
+```
+
+## Connect your NxCloud cluster to your Nx Agents cluster
+
+Continue following the instructions [here](./AGENTS-GUIDE.md#connect-nxcloud-to-your-nx-agents-deployment) for instructions on how to connect your NxCloud cluster to the above address.
+
+## Other resources
+
+Please also check the generic [Agents Guide](./AGENTS-GUIDE.md) for background on how why we need some of the pieces above, such as valkey.
diff --git a/agents-guide/agents-secrets.yml b/agents-guide/agents-secrets.yml
@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: nx-cloud-agents-secret
+type: Opaque
+stringData:
+  AWS_S3_ACCESS_KEY_ID: ''
+  AWS_S3_SECRET_ACCESS_KEY: ''
+  valkey-password: ''
diff --git a/agents-guide/basic-agents-config.yaml b/agents-guide/basic-agents-config.yaml
@@ -0,0 +1,52 @@
+global:
+  namespace: default
+  createNamespace: false
+
+controller:
+  deployment:
+    port: 9000
+    env:
+      - name: AWS_REGION
+        value: 'us-east-1'
+      - name: NX_CLOUD_EXECUTOR_BINARY_BUCKET
+        value: 'https://s3.amazonaws.com/your-custom-location-for-storing-the-executor'
+      - name: NX_CLOUD_AWS_BUCKET
+        value: "nx-cloud-agents-s3-bucket"
+  service:
+    port: 9000
+    type: LoadBalancer
+  image:
+    registry: 'us-east1-docker.pkg.dev/your-custom-registry'
+    imageName: nx-cloud-workflow-controller
+    repository: 'your-custom-nx-cloud-image-repo'
+
+executor:
+  env:
+    AWS_REGION: 'us-east-1'
+    NX_CLOUD_AWS_BUCKET: 'nx-cloud-agents-s3-bucket'
+
+daemonset:
+  tolerations: []
+  image:
+    registry: ''
+    imageName: ubuntu
+    repository: ''
+    tag: 22.04
+  enabled: true
+  script: |
+    #!/bin/bash
+    set -e
+
+    # change the file-watcher max-count on each node to 1048576
+
+    # insert the new value into the system config
+    sysctl -w fs.inotify.max_user_watches=1048576
+
+    # check that the new value was applied
+    cat /proc/sys/fs/inotify/max_user_watches
+
+secret:
+  name: 'nx-cloud-agents-secret'
+  valkeyPassword: 'valkey-password'
+  awsS3AccessKeyId: ''
+  awsS3SecretAccessKey: ''
diff --git a/charts/nx-agents/Chart.yaml b/charts/nx-agents/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: nx-agents
 description: Nx Cloud Agents Helm Chart
 type: application
-version: 1.2.0
+version: 1.2.2
 maintainers:
   - name: nx
     url: "https://nx.app/"

diff --git a/charts/nx-agents/templates/deployment.yaml b/charts/nx-agents/templates/deployment.yaml
@@ -91,12 +91,23 @@ spec:
             {{- toYaml .Values.controller.deployment.envFrom | nindent 12}}
           {{- end }}
           env:
+          {{- if .Values.controller.useDefaultValkeyAddress }}
+            - name: VALKEY_CONNECTION_STRING
+              value: valkey://valkey-master.{{ .Values.global.namespace }}.svc.cluster.local:6379
+          {{- end }}
           {{- if gt (len .Values.controller.deployment.env) 0 }}
             {{- toYaml .Values.controller.deployment.env | nindent 12 }}
-          {{- end }}          
+          {{- end }}
           {{- with .Values.secret }}
-          {{- if .awsS3AccessKeyId }}
           {{- if .name }}
+          {{- if .valkeyPassword }}
+            - name: VALKEY_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .name }}
+                  key: {{ .valkeyPassword }}
+          {{- end }}
+          {{- if .awsS3AccessKeyId }}
             - name: AWS_S3_ACCESS_KEY_ID
               valueFrom:
                 secretKeyRef:

diff --git a/charts/nx-agents/values.yaml b/charts/nx-agents/values.yaml
@@ -83,6 +83,7 @@ controller:
     pullPolicy: Always
   # These are our recommended resources for the controller. However we have noted that a request as low as 0.2 CPU and 0.5Gi
   # memory can work for many cases
+  useDefaultValkeyAddress: true
   resources:
     limits:
       memory: '1Gi'
@@ -128,9 +129,12 @@ daemonset:
 # If set, ensure to provide the proper ENV mappings that aws client libararies expect such as AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
 # for both the controller and executor. Additionally point the controller to the correct bucket and region with the NX_CLOUD_AWS_BUCKET variable
 secret:
+  # the name of the secret needs to match the name of the secret provided to valkey (see AGENTS-GUIDE.md)
   name: ''
   awsS3AccessKeyId: ''
   awsS3SecretAccessKey: ''
+  # this needs to remain as is. It is used by the controller to load the password for valkey from the provided secret.
+  valkeyPassword: ''
 
 
 # If you would like to manage additional resources with this helm chart you can add additional manifests here.

diff --git a/charts/nx-cloud/templates/_auth-helpers.tpl b/charts/nx-cloud/templates/_auth-helpers.tpl
@@ -44,6 +44,10 @@
     secretKeyRef:
       name: {{ .Values.secret.name }}
       key: {{ .Values.secret.bitbucketAppSecret }}
+    {{ if .Values.bitbucket.apiUrl }}
+- name: BITBUCKET_API_URL
+  value: {{ .Values.bitbucket.apiUrl }}
+    {{- end }}
 {{- end }}
 
 {{- if .Values.saml.enabled }}

diff --git a/charts/nx-cloud/templates/nx-cloud-aggregator-cron.yaml b/charts/nx-cloud/templates/nx-cloud-aggregator-cron.yaml
@@ -56,6 +56,7 @@ apiVersion: batch/v1
 kind: CronJob
 metadata:
   name: nx-cloud-aggregator
+  namespace: {{ .Values.global.namespace }}
   labels:
     {{- include "nxCloud.app.labels" . | indent 4 }}
 spec:

diff --git a/charts/nx-cloud/templates/nx-cloud-frontend-deployment.yaml b/charts/nx-cloud/templates/nx-cloud-frontend-deployment.yaml
@@ -44,6 +44,11 @@ spec:
               port: {{ .Values.frontend.deployment.port }}
             initialDelaySeconds: 5
             periodSeconds: 10
+          {{- if .Values.selfSignedCertConfigMap }}
+          volumeMounts:
+            - mountPath: /self-signed-certs
+              name: self-signed-certs-volume
+          {{- end}}
           env:
         {{- include "nxCloud.env.verboseLogging" . | indent 12 }}
         {{- include "nxCloud.env.mode" . | indent 12 }}
@@ -55,6 +60,10 @@ spec:
         {{- if .Values.frontend.deployment.env }}
           {{- toYaml .Values.frontend.deployment.env | nindent 12 }}
         {{- end }}
+        {{- if .Values.selfSignedCertConfigMap }}
+            - name: NODE_EXTRA_CA_CERTS
+              value: /self-signed-certs/self-signed-cert.crt
+        {{- end}}
         {{- with .Values.secret }}
           {{- if .name }}
             {{- if .githubPrivateKey }}
@@ -80,3 +89,9 @@ spec:
                     {{- end }}
             {{- end }}
         {{- end }}
+      {{- if .Values.selfSignedCertConfigMap }}
+      volumes:
+        - configMap:
+            name: {{ .Values.selfSignedCertConfigMap }}
+          name: self-signed-certs-volume
+      {{- end }}