T321886 (#225)

* Post to phabricator on PR close (#213)

Bug: T318703

* reverting db to production db

* mount nfs into pods

Bug: T321886

* Removing ingress.class annotation

* remove nfs hostPaths

* link to spawner docs

* remove labstore entries

* updating dumps mount path

In order to simplify transition to dropping links, updating
dumps path from /public/dumps to /public/dumps/public which
resembles what is currently in place. This should allow for
a seamless transition between the current config and this
config.

* full path

manifests/psp.yaml

@@ -42,16 +42,13 @@ spec:
     - 'secret'
     - 'downwardAPI'
     - 'hostPath'
+    - 'nfs'
     - 'persistentVolumeClaim'
   allowedHostPaths:
-    - pathPrefix: '/public/dumps'
-      readOnly: true
     - pathPrefix: '/mnt/nfs'
       readOnly: true
     - pathPrefix: '/var/lib/sss/pipes'
       readOnly: false
-    - pathPrefix: '/data/project'
-      readOnly: false
 ---
 apiVersion: policy/v1beta1
 kind: PodSecurityPolicy
@@ -95,6 +92,7 @@ spec:
     - 'secret'
     - 'downwardAPI'
     - 'hostPath'
+    - 'nfs'
     - 'persistentVolumeClaim'
   allowedHostPaths:
     - pathPrefix: '/public/dumps'

paws/production.yaml

@@ -9,6 +9,27 @@ dbProxy:
     tag: latest
 jupyterhub:
   hub:
+    extraVolumes:
+      - name: homes
+        nfs:
+          server: nfs-tools-project.svc.eqiad.wmnet
+          path: /srv/misc/shared/paws/project
+      - name: dumps
+        nfs:
+          server: clouddumps1002.wikimedia.org
+          path: /
+      # Without this, dumps becomes inaccessible and can hang the host
+      - name: dumps-src1
+        nfs:
+          server: clouddumps1002.wikimedia.org
+          path: /
+      - name: dumps-src2
+        nfs:
+          server: clouddumps1001.wikimedia.org
+          path: /
+    extraConfig:
+      00-myConfig: |
+          localdev = False
     db:
       type: mysql
       upgrade: true

paws/templates/localdev.yaml

@@ -11,7 +11,7 @@ spec:
   capacity:
     storage: 1Gi
   hostPath:
-    path: /data/project/paws/userhomes
+    path: /mnt/project/paws/userhomes
 ---
 apiVersion: v1
 kind: PersistentVolume
@@ -47,7 +47,7 @@ spec:
   capacity:
     storage: 1Gi
   hostPath:
-    path: /public/dumps
+    path: /mnt/public/dumps
 ---
 apiVersion: batch/v1
 kind: Job
@@ -84,5 +84,5 @@ spec:
       volumes:
         - name: security-disaster-only-for-minikube
           hostPath:
-            path: /data/project/paws/userhomes
+            path: /mnt/project/paws/userhomes
 {{ end }}

paws/templates/public.yaml

@@ -41,8 +41,14 @@ spec:
               cpu: "50m"
 
       volumes:
+{{ if .Values.localdev.enabled }}
         - hostPath:
-            path: /data/project/paws/userhomes
+            path: /mnt/project/paws/userhomes
+{{ else }}
+        - nfs:
+            server: nfs-tools-project.svc.eqiad.wmnet
+            path: /srv/misc/shared/paws/project/paws/userhomes
+{{ end }}
           name: pawshomes
 ---
 
@@ -83,8 +89,14 @@ spec:
               memory: "1000Mi"
               cpu: {{ .Values.pawspublic.renderer.cpu }}
       volumes:
+{{ if .Values.localdev.enabled }}
         - hostPath:
-            path: /data/project/paws/userhomes
+            path: /mnt/project/paws/userhomes
+{{ else }}
+        - nfs:
+            server: nfs-tools-project.svc.eqiad.wmnet
+            path: /srv/misc/shared/paws/project/paws/userhomes
+{{ end }}
           name: pawshomes
 ---
 apiVersion: autoscaling/v1
@@ -136,6 +148,7 @@ metadata:
   labels:
     name: paws-public-custom
     ingress.paws.wmcloud.org: public
+  annotations:
   name: paws-public-custom
 spec:
   rules:

paws/values.yaml

@@ -87,10 +87,10 @@ jupyterhub:
     extraVolumes:
       - name: homes
         hostPath:
-          path: /data/project
+          path: /mnt/project
       - name: dumps
         hostPath:
-          path: /public/dumps
+          path: /mnt/public/dumps
       # Without this, dumps becomes inaccessible and can hang the host
       - name: dumps-src1
         hostPath:
@@ -119,7 +119,9 @@ jupyterhub:
             return pod
 
           c.KubeSpawner.modify_pod_hook = fix_labels
-      myConfig: |
+      00-myConfig: |
+          localdev = True
+      10-myConfig: |
           import hmac
           import hashlib
           import subprocess
@@ -153,6 +155,8 @@ jupyterhub:
                       return False  # Notebook cookies keep user logged in
 
               @gen.coroutine
+              # more information about where this comes from found here:
+              # https://jupyterhub-kubespawner.readthedocs.io/en/latest/spawner.html#kubespawner.KubeSpawner.volumes
               def pre_spawn_start(self, user, spawner):
                   auth_state = yield user.get_auth_state()
                   identity = auth_state['MEDIAWIKI_USER_IDENTITY']
@@ -164,35 +168,58 @@ jupyterhub:
                   # Set rather than use .extend!
                   # Since otherwise the volumes list will grow each time
                   # the spawner stops and starts!
+
                   homedir = '/data/project/paws/userhomes/{}'.format(identity['sub'])
+                  homenfs = '/srv/misc/shared/paws/project/paws/userhomes/{}'.format(identity['sub'])
                   # Create the homedir so docker doesn't do it as root
                   os.makedirs(homedir, mode=0o755, exist_ok=True)
-                  spawner.volumes = [
-                      {
-                          'name': 'home',
-                          'hostPath': { 'path': homedir }
-                      },
-                      {
-                          'name': 'dumps',
-                          'hostPath': { 'path': '/public/dumps' }
-                      },
-                      {
-                          'name': 'dumps-src1',
-                          'hostPath': { 'path': '/mnt/nfs/dumps-clouddumps1002.wikimedia.org' }
-                      },
-                      {
-                          'name': 'dumps-src2',
-                          'hostPath': { 'path': '/mnt/nfs/dumps-clouddumps1001.wikimedia.org' }
-                      }
-                  ]
+                  if localdev == True:
+                      spawner.volumes = [
+                          {
+                              'name': 'home',
+                              'hostPath': { 'path': homenfs }
+                          },
+                          {
+                              'name': 'dumps',
+                              'hostPath': { 'path': '/public/dumps' }
+                          },
+                          {
+                              'name': 'dumps-src1',
+                              'hostPath': { 'path': '/mnt/nfs/dumps-clouddumps1002.wikimedia.org' }
+                          },
+                          {
+                              'name': 'dumps-src2',
+                              'hostPath': { 'path': '/mnt/nfs/dumps-clouddumps1001.wikimedia.org' }
+                          }
+                      ]
+                  else:
+                      spawner.volumes = [
+                          {
+                              'name': 'home',
+                              'nfs': { 'server': 'nfs-tools-project.svc.eqiad.wmnet', 'path': homenfs }
+                          },
+                          {
+                              'name': 'dumps',
+                              'nfs': { 'server': 'clouddumps1001.wikimedia.org', 'path': '/' }
+                          },
+                          {
+                              'name': 'dumps-src1',
+                              'nfs': { 'server': 'clouddumps1002.wikimedia.org', 'path': '/' }
+                          },
+                          {
+                              'name': 'dumps-src2',
+                              'nfs': { 'server': 'clouddumps1001.wikimedia.org', 'path': '/' }
+                          }
+                      ]
+
                   spawner.volume_mounts = [
                       {
                           'name': 'home',
                           'mountPath': '/home/paws'
                       },
                       {
                           'name': 'dumps',
-                          'mountPath': '/public/dumps',
+                          'mountPath': '/public/dumps/public',
                           'readOnly': True
                       },
                       {