From e78a61870f63ff3b1ddd75d2b9f9c36a17670acf Mon Sep 17 00:00:00 2001 From: George Antoniadis Date: Tue, 19 Apr 2022 21:39:27 +0100 Subject: [PATCH 1/6] Add proof of concept kubernetes support --- README.md | 47 ++++++++++++++ manifests/polis-file-server.yaml | 37 +++++++++++ manifests/polis-math.yaml | 51 +++++++++++++++ manifests/polis-server.yaml | 108 +++++++++++++++++++++++++++++++ manifests/postgres.yaml | 70 ++++++++++++++++++++ 5 files changed, 313 insertions(+) create mode 100644 manifests/polis-file-server.yaml create mode 100644 manifests/polis-math.yaml create mode 100644 manifests/polis-server.yaml create mode 100644 manifests/postgres.yaml diff --git a/README.md b/README.md index 8e9e4b0ec7..b551582e44 100644 --- a/README.md +++ b/README.md @@ -206,6 +206,53 @@ from the defaults found in the `docker-compose.yml` file and pass it as the seco We use Cypress for automated, end-to-end browser testing for PRs on GitHub (see badge above). Please see [`e2e/README.md`](/e2e/README.md) for more information on running these tests locally. +### Kubernetes + +**Warning:** This is work in progress and should not be used in production. + +Under the `manifests` folder you can find a first version of polis running under +Kubernetes. It uses an in-cluster postegres as a stateful set, with a persistent +volume claim, and exposes the polis server using the cluster's ingress. + +The setting of the ingress is not part of the provided resources as it will vary +between providers. + +#### Todo + +- [ ] Use official postgres image, decouple migrations +- [ ] Figure out and document the default resources for the services +- [ ] Document architecture, deployement, pitfalls, and best practices + +### Notes + +- Look for `polis.local` in the `manifests/*.yaml` and replace with your hostname + +#### Running in Minikube + +Start minikube, and enable the ingress and ingress-dns addons. + +- `minikube start` +- `minikube addons enable ingress` +- `minikube addons enable ingress-dns` + +Until public containers are available, you can build the containers locally +using docker-compose and push them to the minikube host. + +- `minikube image load docker.io/compdem/polis-server:dev` +- `minikube image load docker.io/compdem/polis-file-server:dev` +- `minikube image load docker.io/compdem/polis-math:dev` +- `minikube image load docker.io/compdem/polis-postgres:dev` + +You can then apply the manifests required for polis to work. + +- `kubectl apply -f manifests` + +The final part is to expose the nginx ingress to your local machine and connect +to it. + +- `minikube tunnel` - You need to leave this running +- `open http://polis.local` + ### Miscellaneous & troubleshooting #### Docker Problems diff --git a/manifests/polis-file-server.yaml b/manifests/polis-file-server.yaml new file mode 100644 index 0000000000..5e184a9513 --- /dev/null +++ b/manifests/polis-file-server.yaml @@ -0,0 +1,37 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: polis-file-server +spec: + selector: + matchLabels: + app: polis-file-server + template: + metadata: + labels: + app: polis-file-server + spec: + containers: + - image: docker.io/compdem/polis-file-server:dev + name: polis-file-server + ports: + - containerPort: 8080 + resources: + requests: + cpu: 100m + memory: 100Mi + limits: + cpu: 1000m + memory: 1000Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: polis-file-server +spec: + ports: + - name: http + port: 80 + targetPort: 8080 + selector: + app: polis-file-server diff --git a/manifests/polis-math.yaml b/manifests/polis-math.yaml new file mode 100644 index 0000000000..75871ca360 --- /dev/null +++ b/manifests/polis-math.yaml @@ -0,0 +1,51 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: polis-math +data: + DATABASE_URL: postgres://postgres:oiPorg3Nrz0yqDLE@postgres:5432/polis-dev + WEBSERVER_USERNAME: ws-user + WEBSERVER_PASS: ws-pass +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: polis-math +spec: + selector: + matchLabels: + app: polis-math + template: + metadata: + labels: + app: polis-math + spec: + containers: + - image: docker.io/compdem/polis-math:dev + name: polis-math + envFrom: + - configMapRef: + name: polis-math + ports: + - containerPort: 5000 + resources: + requests: + cpu: 100m + memory: 100Mi + limits: + cpu: 1000m + memory: 1000Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: polis-math +spec: + ports: + - name: http + port: 80 + targetPort: 5000 + selector: + app: polis-math + + diff --git a/manifests/polis-server.yaml b/manifests/polis-server.yaml new file mode 100644 index 0000000000..fe57147264 --- /dev/null +++ b/manifests/polis-server.yaml @@ -0,0 +1,108 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: polis-server +data: + # For admin functionality, fill this out + ADMIN_EMAILS: "[]" + ADMIN_EMAIL_DATA_EXPORT: "" + ADMIN_EMAIL_DATA_EXPORT_TEST: "" + ADMIN_EMAIL_EMAIL_TEST: "" + ADMIN_UIDS: "[]" + + DATABASE_FOR_READS_NAME: DATABASE_URL + DATABASE_URL: postgres://postgres:oiPorg3Nrz0yqDLE@postgres:5432/polis-dev + + # These may be the deprecated settings for submitting web requests to the math worker + WEBSERVER_USERNAME: ws-user + WEBSERVER_PASS: ws-pass + + # Set to `false` for production + DEV_MODE: "true" + + # Set the domain name; make sure this matches what"s running on math node + DOMAIN_OVERRIDE: polis.local + + # Options: prod, preprod, dev + MATH_ENV: dev + PORT: "5000" + + # Set to `true` and insert google creds if desired + SHOULD_USE_TRANSLATION_API: "false" + # GOOGLE_CREDENTIALS_BASE64: ... + # GOOGLE_CREDS_STRINGIFIED: ... + + # These need to be configured to point to the local file server + STATIC_FILES_HOST: polis-file-server + STATIC_FILES_ADMINDASH_PORT: "80" + STATIC_FILES_PORT: "80" + + AWS_REGION: us-east-1 + + # Options: maildev, aws-ses, mailgun + EMAIL_TRANSPORT_TYPES: maildev + POLIS_FROM_ADDRESS: "Example " + + # These pieces of functionality will likely be removed in the near future + DISABLE_INTERCOM: "true" + STRIPE_SECRET_KEY: sk_test_NFBDEThkpHCYBzXPJuBlY8TW +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: polis-server +spec: + selector: + matchLabels: + app: polis-server + template: + metadata: + labels: + app: polis-server + spec: + containers: + - image: docker.io/compdem/polis-server:dev + name: polis-server + envFrom: + - configMapRef: + name: polis-server + ports: + - containerPort: 5000 + resources: + requests: + cpu: 100m + memory: 100Mi + limits: + cpu: 1000m + memory: 1000Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: polis-server +spec: + ports: + - name: http + port: 80 + targetPort: 5000 + selector: + app: polis-server +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: polis-server +spec: + rules: + - host: polis.local + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: polis-server + port: + number: 80 + + diff --git a/manifests/postgres.yaml b/manifests/postgres.yaml new file mode 100644 index 0000000000..adec55dc6e --- /dev/null +++ b/manifests/postgres.yaml @@ -0,0 +1,70 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgres + labels: + app: postgres +data: + POSTGRES_DB: polis-dev + POSTGRES_USER: postgres + POSTGRES_PASSWORD: oiPorg3Nrz0yqDLE +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: postgres-pv-claim + labels: + app: postgres +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: postgres-statefulset + labels: + app: postgres +spec: + serviceName: postgres + replicas: 1 + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + spec: + containers: + - name: postgres + image: docker.io/compdem/polis-postgres:dev + envFrom: + - configMapRef: + name: postgres + ports: + - containerPort: 5432 + name: postgresdb + volumeMounts: + - name: pv-data + mountPath: /var/lib/postgresql/data + volumes: + - name: pv-data + persistentVolumeClaim: + claimName: postgres-pv-claim +--- +apiVersion: v1 +kind: Service +metadata: + name: postgres + labels: + app: postgres +spec: + ports: + - port: 5432 + name: postgres + selector: + app: postgres From 9b9c455d483190a8151478efe4e78547dc4e176b Mon Sep 17 00:00:00 2001 From: George Antoniadis Date: Mon, 23 May 2022 20:38:18 +0100 Subject: [PATCH 2/6] chore(manifests): add ENCRYPTION_PASSWORD_00001 env var This seems to be needed until the following TODO is resolved: https://github.com/compdemocracy/polis/blob/9a294140bbfc50d5bbb50ce4e474f4ad72541881/server/src/session.ts#L8-L10 --- manifests/polis-server.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/manifests/polis-server.yaml b/manifests/polis-server.yaml index fe57147264..b62ca1eb53 100644 --- a/manifests/polis-server.yaml +++ b/manifests/polis-server.yaml @@ -46,6 +46,10 @@ data: # These pieces of functionality will likely be removed in the near future DISABLE_INTERCOM: "true" STRIPE_SECRET_KEY: sk_test_NFBDEThkpHCYBzXPJuBlY8TW + + # Note(geoah): This seems to be needed until the following TODO is resolved: + # https://github.com/compdemocracy/polis/blob/9a294140bbfc50d5bbb50ce4e474f4ad72541881/server/src/session.ts#L8-L10 + ENCRYPTION_PASSWORD_00001: "PLEASE-CHANGE-ME" --- apiVersion: apps/v1 kind: Deployment From bd85ad5915df7ee7bde8b38c5f9970f6fdea54bd Mon Sep 17 00:00:00 2001 From: George Antoniadis Date: Mon, 23 May 2022 20:40:54 +0100 Subject: [PATCH 3/6] feat: add skaffold config and update README --- README.md | 23 +++++++++++++---------- skaffold.yaml | 31 +++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 10 deletions(-) create mode 100644 skaffold.yaml diff --git a/README.md b/README.md index b551582e44..dd3b8ae500 100644 --- a/README.md +++ b/README.md @@ -227,6 +227,15 @@ between providers. - Look for `polis.local` in the `manifests/*.yaml` and replace with your hostname +### Requirements + +- Local development: + - [Minikube](https://minikube.sigs.k8s.io/docs/) + - [Skaffold](https://skaffold.dev/) + +Skaffold deals with the local development flow, syncing updated files to their +in-cluster containers. + #### Running in Minikube Start minikube, and enable the ingress and ingress-dns addons. @@ -235,17 +244,11 @@ Start minikube, and enable the ingress and ingress-dns addons. - `minikube addons enable ingress` - `minikube addons enable ingress-dns` -Until public containers are available, you can build the containers locally -using docker-compose and push them to the minikube host. - -- `minikube image load docker.io/compdem/polis-server:dev` -- `minikube image load docker.io/compdem/polis-file-server:dev` -- `minikube image load docker.io/compdem/polis-math:dev` -- `minikube image load docker.io/compdem/polis-postgres:dev` - -You can then apply the manifests required for polis to work. +You can now build and deploy the containers in the local cluster via either: -- `kubectl apply -f manifests` +- `skaffold run` - Builds and deploys everything on demand. +- `skaffold dev` - While running will watch and automatically build and deploy + updated containers. The final part is to expose the nginx ingress to your local machine and connect to it. diff --git a/skaffold.yaml b/skaffold.yaml new file mode 100644 index 0000000000..1ecc963771 --- /dev/null +++ b/skaffold.yaml @@ -0,0 +1,31 @@ +apiVersion: skaffold/v2beta28 +kind: Config +metadata: + name: polis +build: + local: + useBuildkit: true + push: false + artifacts: + - image: docker.io/compdem/polis-file-server + docker: + dockerfile: file-server/Dockerfile + - image: docker.io/compdem/polis-math + context: math + docker: + dockerfile: math/Dockerfile + - image: docker.io/compdem/polis-postgres + context: server + docker: + dockerfile: server/Dockerfile-db + - image: docker.io/compdem/polis-server + context: server + docker: + dockerfile: server/Dockerfile +deploy: + kubectl: + manifests: + - manifests/polis-file-server.yaml + - manifests/polis-math.yaml + - manifests/postgres.yaml + - manifests/polis-server.yaml From dce27f7faae531d8d8846c6e01b6f3e30ff6a440 Mon Sep 17 00:00:00 2001 From: Sonal Ranjit Date: Fri, 22 Nov 2024 19:13:12 -0500 Subject: [PATCH 4/6] Working version of kubernetes setup with polis --- manifests/polis-server.yaml | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/manifests/polis-server.yaml b/manifests/polis-server.yaml index b62ca1eb53..35ae198d3e 100644 --- a/manifests/polis-server.yaml +++ b/manifests/polis-server.yaml @@ -3,15 +3,22 @@ kind: ConfigMap metadata: name: polis-server data: - # For admin functionality, fill this out - ADMIN_EMAILS: "[]" - ADMIN_EMAIL_DATA_EXPORT: "" - ADMIN_EMAIL_DATA_EXPORT_TEST: "" - ADMIN_EMAIL_EMAIL_TEST: "" + #General Settings ADMIN_UIDS: "[]" + GIT_HASH: "" + SERVER_ENV_FILE: ".env" - DATABASE_FOR_READS_NAME: DATABASE_URL + # Database + READ_ONLY_DATABASE_URL: "" + POSTGRES_DB: "polis-dev" + POSTGRES_PORT: "5432" + POSTGRES_HOST: "postgres:5432" + POSTGRES_USER: "postgres" + POSTGRES_PASSWORD: "oiPorg3Nrz0yqDLE" DATABASE_URL: postgres://postgres:oiPorg3Nrz0yqDLE@postgres:5432/polis-dev + POSTGRES_DOCKER: "true" + + DATABASE_FOR_READS_NAME: DATABASE_URL # These may be the deprecated settings for submitting web requests to the math worker WEBSERVER_USERNAME: ws-user @@ -21,7 +28,9 @@ data: DEV_MODE: "true" # Set the domain name; make sure this matches what"s running on math node - DOMAIN_OVERRIDE: polis.local + DOMAIN_OVERRIDE: polisserver.com + API_DEV_HOSTNAME: polisserver.com + API_PROD_HOSTNAME: polisserver.com # Options: prod, preprod, dev MATH_ENV: dev @@ -98,7 +107,7 @@ metadata: name: polis-server spec: rules: - - host: polis.local + - host: polisserver.com http: paths: - path: / From 1527a51ab5aee1c243fe02e813eff3a012012d1f Mon Sep 17 00:00:00 2001 From: Sonal Ranjit Date: Sat, 23 Nov 2024 12:53:59 -0500 Subject: [PATCH 5/6] Update skaffold apiVersion to latest --- skaffold.yaml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/skaffold.yaml b/skaffold.yaml index 1ecc963771..b33ed76db3 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -1,4 +1,4 @@ -apiVersion: skaffold/v2beta28 +apiVersion: skaffold/v4beta11 kind: Config metadata: name: polis @@ -22,10 +22,11 @@ build: context: server docker: dockerfile: server/Dockerfile +manifests: + rawYaml: + - manifests/polis-file-server.yaml + - manifests/polis-math.yaml + - manifests/postgres.yaml + - manifests/polis-server.yaml deploy: kubectl: - manifests: - - manifests/polis-file-server.yaml - - manifests/polis-math.yaml - - manifests/postgres.yaml - - manifests/polis-server.yaml From b48f94b7400ac16c246e997516f605466ac1ff8b Mon Sep 17 00:00:00 2001 From: Sonal Ranjit Date: Sat, 23 Nov 2024 13:23:18 -0500 Subject: [PATCH 6/6] Migrate secret env variables to k8s Secret object --- manifests/polis-secret.yaml | 20 +++++++++++++ manifests/polis-server.yaml | 57 ++++++++++++++++++++++++------------- manifests/postgres.yaml | 30 ++++++++++--------- skaffold.yaml | 1 + 4 files changed, 74 insertions(+), 34 deletions(-) create mode 100644 manifests/polis-secret.yaml diff --git a/manifests/polis-secret.yaml b/manifests/polis-secret.yaml new file mode 100644 index 0000000000..8691b724d9 --- /dev/null +++ b/manifests/polis-secret.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Secret +metadata: + name: polis-secret +type: Opaque +data: + POSTGRES_USER: cG9zdGdyZXM= + POSTGRES_DB: cG9saXMtZGV2 + POSTGRES_PORT: NTQzMg== + POSTGRES_HOST: cG9zdGdyZXM6NTQzMg== + POSTGRES_PASSWORD: b2lQb3JnM05yejB5cURMRQ== + STRIPE_SECRET_KEY: c2tfdGVzdF9ORkJERVRoa3BIQ1lCelhQSnVCbFk4VFc= + + # Note(geoah): This seems to be needed until the following TODO is resolved: + # https://github.com/compdemocracy/polis/blob/9a294140bbfc50d5bbb50ce4e474f4ad72541881/server/src/session.ts#L8-L10 + ENCRYPTION_PASSWORD_00001: UExFQVNFX0NIQU5HRV9NRQ== + + # These may be the deprecated settings for submitting web requests to the math worker + WEBSERVER_USERNAME: d3MtdXNlcg== + WEBSERVER_PASS: d3MtcGFzcw== diff --git a/manifests/polis-server.yaml b/manifests/polis-server.yaml index 35ae198d3e..86a441f5a9 100644 --- a/manifests/polis-server.yaml +++ b/manifests/polis-server.yaml @@ -10,27 +10,16 @@ data: # Database READ_ONLY_DATABASE_URL: "" - POSTGRES_DB: "polis-dev" - POSTGRES_PORT: "5432" - POSTGRES_HOST: "postgres:5432" - POSTGRES_USER: "postgres" - POSTGRES_PASSWORD: "oiPorg3Nrz0yqDLE" - DATABASE_URL: postgres://postgres:oiPorg3Nrz0yqDLE@postgres:5432/polis-dev POSTGRES_DOCKER: "true" - DATABASE_FOR_READS_NAME: DATABASE_URL - # These may be the deprecated settings for submitting web requests to the math worker - WEBSERVER_USERNAME: ws-user - WEBSERVER_PASS: ws-pass - # Set to `false` for production DEV_MODE: "true" # Set the domain name; make sure this matches what"s running on math node - DOMAIN_OVERRIDE: polisserver.com - API_DEV_HOSTNAME: polisserver.com - API_PROD_HOSTNAME: polisserver.com + DOMAIN_OVERRIDE: polis.local + API_DEV_HOSTNAME: polis.local + API_PROD_HOSTNAME: polis.local # Options: prod, preprod, dev MATH_ENV: dev @@ -54,11 +43,6 @@ data: # These pieces of functionality will likely be removed in the near future DISABLE_INTERCOM: "true" - STRIPE_SECRET_KEY: sk_test_NFBDEThkpHCYBzXPJuBlY8TW - - # Note(geoah): This seems to be needed until the following TODO is resolved: - # https://github.com/compdemocracy/polis/blob/9a294140bbfc50d5bbb50ce4e474f4ad72541881/server/src/session.ts#L8-L10 - ENCRYPTION_PASSWORD_00001: "PLEASE-CHANGE-ME" --- apiVersion: apps/v1 kind: Deployment @@ -76,6 +60,39 @@ spec: containers: - image: docker.io/compdem/polis-server:dev name: polis-server + env: + - name: POSTGRES_DB + valueFrom: + secretKeyRef: + name: polis-secret + key: POSTGRES_DB + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + name: polis-secret + key: POSTGRES_USER + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: polis-secret + key: POSTGRES_PASSWORD + - name: POSTGRES_HOST + valueFrom: + secretKeyRef: + name: polis-secret + key: POSTGRES_HOST + - name: DATABASE_URL + value: postgres://$(POSTGRES_USER):$(POSTGRES_PASSWORD)@$(POSTGRES_HOST)/$(POSTGRES_DB) + - name: STRIPE_SECRET_KEY + valueFrom: + secretKeyRef: + name: polis-secret + key: STRIPE_SECRET_KEY + - name: ENCRYPTION_PASSWORD_00001 + valueFrom: + secretKeyRef: + name: polis-secret + key: ENCRYPTION_PASSWORD_00001 envFrom: - configMapRef: name: polis-server @@ -107,7 +124,7 @@ metadata: name: polis-server spec: rules: - - host: polisserver.com + - host: polis.local http: paths: - path: / diff --git a/manifests/postgres.yaml b/manifests/postgres.yaml index adec55dc6e..18bbb39ead 100644 --- a/manifests/postgres.yaml +++ b/manifests/postgres.yaml @@ -1,16 +1,5 @@ apiVersion: v1 -kind: ConfigMap -metadata: - name: postgres - labels: - app: postgres -data: - POSTGRES_DB: polis-dev - POSTGRES_USER: postgres - POSTGRES_PASSWORD: oiPorg3Nrz0yqDLE ---- kind: PersistentVolumeClaim -apiVersion: v1 metadata: name: postgres-pv-claim labels: @@ -42,9 +31,22 @@ spec: containers: - name: postgres image: docker.io/compdem/polis-postgres:dev - envFrom: - - configMapRef: - name: postgres + env: + - name: POSTGRES_DB + valueFrom: + secretKeyRef: + name: polis-secret + key: POSTGRES_DB + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + name: polis-secret + key: POSTGRES_USER + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: polis-secret + key: POSTGRES_PASSWORD ports: - containerPort: 5432 name: postgresdb diff --git a/skaffold.yaml b/skaffold.yaml index b33ed76db3..3297e2276f 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -24,6 +24,7 @@ build: dockerfile: server/Dockerfile manifests: rawYaml: + - manifests/polis-secret.yaml - manifests/polis-file-server.yaml - manifests/polis-math.yaml - manifests/postgres.yaml