GH-524 staging environment (#532)

Co-authored-by: Humberto Evans <hevans66@gmail.com>
labdao · Jul 21, 2023 · 9538de2 · 9538de2
1 parent 4bb5e6e
commit 9538de2
Show file tree

Hide file tree

Showing 26 changed files with 711 additions and 94 deletions.
diff --git a/infrastructure/ansible/README.md b/infrastructure/ansible/README.md
@@ -22,10 +22,20 @@ Ansible configuration consists of playbooks you run using `ansible-playbook [pla
 
 We are using an aws plug in that provides dynamic ec2 inventory. Through configuration in `inventory.aws_ec2.yaml` we tell the dynamic inventory mechanism to group instances together based on AWS instance tags. Setting (for example) a Type tag on your aws instances of the same type (in Terraform) will allow you to target the instances you wish you perform tasks on.
 
+# Environment
+
+Use tags to limit targetting specific environment using `--limit tag_Env_staging` for Staging and `--limit tag_Env_prod` for prod.
+
+Environment related vars file are available to be run under `vars/<env.yaml>`.
+
+To pass in extra-vars do:
+
+`ansible-playbook -e "@vars/staging.yaml" ............`
+
 # Playbooks
 
 * `provision_jupyter.yaml` Targets `jupyter_notebook` instances and installs [The Littlest Jupyter Hub](https://tljh.jupyter.org/). It does not do any configuration beyond installation.
-* `set_jupyter_users.yaml` Sets the admins, users, and defines access permissions to a team folder 
+* `set_jupyter_users.yaml` Sets the admins, users, and defines access permissions to a team folder
 
 # The teams.yaml
 

diff --git a/infrastructure/ansible/files/requester.service b/infrastructure/ansible/files/requester.service
@@ -13,7 +13,8 @@ ExecStart=bacalhau serve \
 {% endif %}
   --labels owner={{ owner }} \
   --job-selection-accept-networked \
-  --job-selection-data-locality anywhere
+  --job-selection-data-locality anywhere \
+  --peer none
 
 [Install]
 WantedBy=multi-user.target
diff --git a/infrastructure/ansible/install_docker_tasks.yaml b/infrastructure/ansible/install_docker_tasks.yaml
@@ -0,0 +1,34 @@
+---
+# Docker
+- name: Add Docker GPG key
+  become: yes
+  ansible.builtin.get_url:
+    url: https://download.docker.com/linux/ubuntu/gpg
+    dest: /etc/apt/trusted.gpg.d/docker.asc
+
+- name: Add Docker Repository
+  become: yes
+  ansible.builtin.apt_repository:
+    repo: deb [arch=amd64 signed-by=/etc/apt/trusted.gpg.d/docker.asc] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable
+    state: present
+
+- name: Create the docker group
+  become: yes
+  ansible.builtin.group:
+    name: docker
+
+- name: Add ubuntu user to docker group
+  become: yes
+  ansible.builtin.user:
+    name: ubuntu
+    groups: docker
+
+- name: Install docker
+  become: yes
+  ansible.builtin.apt:
+    pkg:
+      - docker-ce
+      - docker-ce-cli
+      - containerd.io
+      - docker-compose-plugin
+    update_cache: true
diff --git a/infrastructure/ansible/install_ipfs_tasks.yaml b/infrastructure/ansible/install_ipfs_tasks.yaml
@@ -40,6 +40,8 @@
     ipfs config Addresses.Gateway /ip4/0.0.0.0/tcp/8080
     ipfs config --json API.HTTPHeaders.Access-Control-Allow-Methods '["PUT", "POST"]'
     ipfs config Pinning.Recursive true
+  environment:
+    IPFS_PATH: "{{ ipfs_path }}"
 
 - name: Install the IPFS systemd unit
   become: yes

diff --git a/infrastructure/ansible/jupyter_deploy_plex.yaml b/infrastructure/ansible/jupyter_deploy_plex.yaml
@@ -9,7 +9,7 @@
       ansible.builtin.file:
         path: "{{ plex_dir }}"
         owner: ubuntu
-        group: ubuntu 
+        group: ubuntu
         state: directory
 
     - name: Ensure all files in plex dir are owned by the user

diff --git a/infrastructure/ansible/jupyter_set_users.yaml b/infrastructure/ansible/jupyter_set_users.yaml
@@ -15,7 +15,7 @@
       ansible.builtin.command:
         cmd: tljh-config add-item users.admin {{ item }}
       loop: "{{ admins | default([])}}"
-       
+
     - name: Create teams
       include_tasks: jupyter_team_setup_tasks.yaml
       loop: "{{ teams | default([])}}"

diff --git a/infrastructure/ansible/jupyter_team_setup_tasks.yaml b/infrastructure/ansible/jupyter_team_setup_tasks.yaml
@@ -39,7 +39,7 @@
     groups: "{{ item.team }}"
   loop: "{{ item.users }}"
   loop_control:
-    loop_var: user 
+    loop_var: user
 
 - name: Link shared folder into the users home directory
   become: yes
@@ -49,10 +49,10 @@
     owner: jupyter-{{ user }}
     group: jupyter-{{ user }}
     follow: false
-    state: link 
+    state: link
   loop: "{{ item.users }}"
   loop_control:
-    loop_var: user 
+    loop_var: user
 
 - name: Ensure the symlink exists to the examples directory for every user
   become: yes

diff --git a/infrastructure/ansible/provision_canary.yaml b/infrastructure/ansible/provision_canary.yaml
@@ -24,7 +24,7 @@
       ansible.builtin.file:
         path: "{{ item }}"
         owner: ubuntu
-        group: ubuntu 
+        group: ubuntu
         state: directory
       loop:
         - "{{ repo_dir }}"

diff --git a/infrastructure/ansible/provision_compute_instance.yaml b/infrastructure/ansible/provision_compute_instance.yaml
@@ -8,6 +8,12 @@
   environment:
     IPFS_PATH: "{{ ipfs_path }}"
   tasks:
+    # Must provide limit flag to ensure running against current environment
+    - fail:
+        msg: "you must use -l or --limit"
+      when: ansible_limit is not defined
+      run_once: true
+
     # Aptitude is preferred by ansible
     - name: Install aptitude
       become: yes
@@ -50,13 +56,13 @@
       become: yes
       ansible.builtin.apt:
         deb: /tmp/cuda-keyring.deb
-          
+
     - name: Get Nvidia Container Tookit GPG key
       become: yes
       ansible.builtin.shell:
         cmd: curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | gpg --yes --dearmor -o {{ nvidia_container_toolkit_key_path }}
         creates: "{{ nvidia_container_toolkit_key_path }}"
- 
+
     - name: Add Nvidia Container Tookit Repository
       become: yes
       ansible.builtin.apt_repository:
@@ -107,8 +113,8 @@
     - name: Install IPFS
       ansible.builtin.get_url:
         url: https://dist.ipfs.tech/kubo/v0.18.0/kubo_v0.18.0_linux-amd64.tar.gz
-        dest: /tmp/ipfs.tar.gz  
-    
+        dest: /tmp/ipfs.tar.gz
+
     - name: Make a folder to put IPFS files in
       ansible.builtin.file:
         path: /tmp/ipfs
@@ -182,7 +188,6 @@
       vars:
         owner: labdao
         ipfs_connect: /ip4/127.0.0.1/tcp/5001
-        receptor_url: http://ip-172-31-82-127.ec2.internal:8080/judge
       notify:
         - Restart Bacalhau
 

diff --git a/infrastructure/ansible/provision_compute_only.yaml b/infrastructure/ansible/provision_compute_only.yaml
@@ -1,109 +1,72 @@
 - name: Provision Bacalhau Compute Instance
   remote_user: ubuntu
-  hosts: tag_Type_compute_only:&tag_Env_prod
+  hosts: tag_Type_compute_only
   vars:
     nvidia_distribution: ubuntu2004
-    nvidia_container_toolkit_key_path: /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg
     ipfs_path: /opt/local/ipfs
-    requester_peer: /ip4/172.31.90.74/tcp/1235/p2p/QmbETsVtL1sQ97KKV1jPQA5ng8RSyzPWUiDgRBQp7AcjRt
-    requester_ipfs_peer: /ip4/172.31.90.74/tcp/4001/p2p/12D3KooWAjYbsjXAQWqPRCPTTaMkDkUjhschznkLrDoKUyfQvHAP
     gpu: true
   environment:
     IPFS_PATH: "{{ ipfs_path }}"
   tasks:
+    # Must provide limit flag to ensure running against current environment
+    - fail:
+        msg: "you must use -l or --limit"
+      when: ansible_limit is not defined
+      run_once: true
+
     # Aptitude is preferred by ansible
-    - name: Install aptitude
+    - name: Install aptitude and other required system packages
       become: yes
       ansible.builtin.apt:
-        name: aptitude
-        state: latest
         update_cache: true
+        pkg:
+          - aptitude
+          - curl
+          - ca-certificates
+          - gnupg
+          - lsb-release
 
     # Docker
-    - name: Add Docker GPG key
-      become: yes
-      ansible.builtin.get_url:
-        url: https://download.docker.com/linux/ubuntu/gpg
-        dest: /etc/apt/trusted.gpg.d/docker.asc
+    - name: Install Docker
+      ansible.builtin.import_tasks: install_docker_tasks.yaml
 
-    - name: Add Docker Repository
+    # Nvidia
+    - name: Add Nvidia Keyring
       become: yes
-      ansible.builtin.apt_repository:
-        repo: deb [arch=amd64 signed-by=/etc/apt/trusted.gpg.d/docker.asc] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable
-        state: present
+      ansible.builtin.apt:
+        deb: https://developer.download.nvidia.com/compute/cuda/repos/{{ nvidia_distribution }}/x86_64/cuda-keyring_1.1-1_all.deb
 
-    - name: Create the docker group
+    - name: Get Nvidia Container Tookit GPG key
       become: yes
-      ansible.builtin.group:
-        name: docker
+      ansible.builtin.apt_key:
+        url: https://nvidia.github.io/libnvidia-container/gpgkey
 
-    - name: Add ubuntu user to docker group
+    - name: Add Nvidia Container Tookit Repository
       become: yes
-      ansible.builtin.user:
-        name: ubuntu
-        groups: docker
+      ansible.builtin.apt_repository:
+        repo: deb https://nvidia.github.io/libnvidia-container/stable/ubuntu18.04/$(ARCH) /
+        state: present
 
-    - name: Install required system packages
+    - name: Install required system packages for gpu build
       become: yes
       ansible.builtin.apt:
         pkg:
-          - ca-certificates
-          - curl
-          - gnupg
-          - lsb-release
-          - docker-ce
-          - docker-ce-cli
-          - containerd.io
-          - docker-compose-plugin
+          - cuda-drivers
         state: latest
         update_cache: true
 
-    # Nvidia
-    - name: Install Nvidia GPU drivers and packages
-      block:
-      - name: Get Nvidia drivers apt key
-        ansible.builtin.get_url:
-          url: https://developer.download.nvidia.com/compute/cuda/repos/{{ nvidia_distribution }}/x86_64/cuda-keyring_1.0-1_all.deb
-          dest: /tmp/cuda-keyring.deb
-
-      - name: Add Nvidia Keyring
-        become: yes
-        ansible.builtin.apt:
-          deb: /tmp/cuda-keyring.deb
-
-      - name: Get Nvidia Container Tookit GPG key
-        become: yes
-        ansible.builtin.shell:
-          cmd: curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | gpg --yes --dearmor -o {{ nvidia_container_toolkit_key_path }}
-          creates: "{{ nvidia_container_toolkit_key_path }}"
-
-      - name: Add Nvidia Container Tookit Repository
-        become: yes
-        ansible.builtin.apt_repository:
-          repo: deb [signed-by={{ nvidia_container_toolkit_key_path }}] https://nvidia.github.io/libnvidia-container/stable/ubuntu18.04/$(ARCH) /
-          state: present
-
-      - name: Install required system packages for gpu build
-        become: yes
-        ansible.builtin.apt:
-          pkg:
-            - cuda-drivers
-          state: latest
-          update_cache: true
-
-      - name: Install Nvidia Container Tookit
-        become: yes
-        ansible.builtin.apt:
-          pkg:
-            - nvidia-docker2
-        notify:
-          - Restart docker
-
-      - name: Ensure Nvidia persitence daemon is started
-        ansible.builtin.systemd:
-          name: nvidia-persistenced
-      when: gpu
-
+    - name: Install Nvidia Container Tookit
+      become: yes
+      ansible.builtin.apt:
+        pkg:
+          - nvidia-docker2
+      notify:
+        - Restart docker
+
+    - name: Ensure Nvidia persitence daemon is started
+      ansible.builtin.systemd:
+        name: nvidia-persistenced
+
     - name: Install Golag
       become: yes
       vars:

diff --git a/infrastructure/ansible/provision_jupyter.yaml b/infrastructure/ansible/provision_jupyter.yaml
@@ -5,6 +5,12 @@
     letsencrypt_email: "josh@labdao.xyz"
     letsencrypt_domain: "jupyter.labdao.xyz"
   tasks:
+    # Must provide limit flag to ensure running against current environment
+    - fail:
+        msg: "you must use -l or --limit"
+      when: ansible_limit is not defined
+      run_once: true
+
     # Aptitude is preferred by ansible
     - name: Install aptitude
       become: yes
@@ -89,5 +95,5 @@
       ansible.builtin.command: tljh-config reload
 
     - name: Bump system resources
-      become: yes 
+      become: yes
       ansible.builtin.command: sysctl -w net.core.rmem_max=2500000
diff --git a/infrastructure/ansible/provision_receptor.yaml b/infrastructure/ansible/provision_receptor.yaml
@@ -5,6 +5,12 @@
     plex_dir: /opt/local/plex
     receptor_dir: /opt/local/receptor
   tasks:
+    # Must provide limit flag to ensure running against current environment
+    - fail:
+        msg: "you must use -l or --limit"
+      when: ansible_limit is not defined
+      run_once: true
+
     - name: Install aptitude
       become: yes
       ansible.builtin.apt:
@@ -17,7 +23,7 @@
       ansible.builtin.file:
         path: "{{ item }}"
         owner: ubuntu
-        group: ubuntu 
+        group: ubuntu
         state: directory
       loop:
         - "{{ plex_dir }}"

diff --git a/infrastructure/ansible/provision_requester.yaml b/infrastructure/ansible/provision_requester.yaml
@@ -4,6 +4,12 @@
   vars:
     ipfs_path: /opt/local/ipfs
   tasks:
+    # Must provide limit flag to ensure running against current environment
+    - fail:
+        msg: "you must use -l or --limit"
+      when: ansible_limit is not defined
+      run_once: true
+
     # Aptitude is preferred by ansible
     - name: Install aptitude
       become: yes
@@ -31,7 +37,6 @@
       vars:
         owner: labdao
         ipfs_connect: /ip4/127.0.0.1/tcp/5001
-        receptor_url: http://ip-172-31-82-127.ec2.internal:8080/judge
       notify:
         - Restart Bacalhau
 

diff --git a/infrastructure/ansible/vars/prod.yaml b/infrastructure/ansible/vars/prod.yaml
@@ -0,0 +1,4 @@
+---
+receptor_url: http://ip-172-31-82-127.ec2.internal:8080/judge
+requester_peer: /ip4/172.31.90.74/tcp/1235/p2p/QmbETsVtL1sQ97KKV1jPQA5ng8RSyzPWUiDgRBQp7AcjRt
+requester_ipfs_peer: /ip4/172.31.90.74/tcp/4001/p2p/12D3KooWAjYbsjXAQWqPRCPTTaMkDkUjhschznkLrDoKUyfQvHAP