From f1ca8a4880ee26681de1cf756388a9bb40b085fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javi=20Mart=C3=ADn?= Date: Mon, 20 Nov 2023 18:37:07 +0100 Subject: [PATCH 01/16] Update README to use Consul Democracy 2.1.0 We're going to release version 2.1.0 with many changes that aren't compatible with Consul Democracy 2.0.1, so we're creating this branch in order to merge those changes and keep them there until we release Consul Democracy 2.1.0. --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 13025a19..7cdf848d 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,7 @@ Setup locally for your [development environment](https://docs.consuldemocracy.or Checkout the latest stable version: ``` -git checkout origin/2.0.1 -b stable +git checkout origin/2.1.0 -b stable ``` Create your `deploy-secrets.yml` @@ -205,13 +205,13 @@ Using https instead of http is an important security configuration. Before you b Once you have that setup we need to configure the Installer to use your domain in the application. -First, uncomment the `domain` variable in the [configuration file](https://github.com/consuldemocracy/installer/blob/2.0.1/group_vars/all) and update it with your domain name: +First, uncomment the `domain` variable in the [configuration file](https://github.com/consuldemocracy/installer/blob/2.1.0/group_vars/all) and update it with your domain name: ``` #domain: "your_domain.com" ``` -Next, uncomment the `letsencrypt_email` variable in the [configuration file](https://github.com/consuldemocracy/installer/blob/2.0.1/group_vars/all) and update it with a valid email address: +Next, uncomment the `letsencrypt_email` variable in the [configuration file](https://github.com/consuldemocracy/installer/blob/2.1.0/group_vars/all) and update it with a valid email address: ``` #letsencrypt_email: "your_email@example.com" @@ -258,7 +258,7 @@ If you are on Ubuntu and would like to use its default `sudo` group instead of ` deploy_group: sudo ``` -There are many more variables available check them out [here]((https://github.com/consuldemocracy/installer/blob/2.0.1/group_vars/all)) +There are many more variables available check them out [here]((https://github.com/consuldemocracy/installer/blob/2.1.0/group_vars/all)) ## Other deployment options @@ -288,7 +288,7 @@ If you do not have `root` access, you will need your system administrator to gra ## Using a different user than deploy -Change the variable [deploy_user](https://github.com/consuldemocracy/installer/blob/2.0.1/group_vars/all#L12) to the username you would like to use. +Change the variable [deploy_user](https://github.com/consuldemocracy/installer/blob/2.1.0/group_vars/all#L12) to the username you would like to use. ## Ansible Documentation From 7456871ecbf84c4cd922ba978f8d0a2c896388eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sen=C3=A9n=20Rodero=20Rodr=C3=ADguez?= <15726+Senen@users.noreply.github.com> Date: Wed, 20 Sep 2023 10:36:02 +0200 Subject: [PATCH 02/16] [TMP] Make the installer install the branch with a specific version of nodejs Note we aren't installing the `master` branch (for now) because that branch requires configuring Puma with systemd. Instead, we're using the last commit which still uses Puma as a daemon. --- roles/folder_structure/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/folder_structure/tasks/main.yml b/roles/folder_structure/tasks/main.yml index 5d17d713..bf9430ad 100644 --- a/roles/folder_structure/tasks/main.yml +++ b/roles/folder_structure/tasks/main.yml @@ -22,7 +22,7 @@ state: directory - name: Create first release - shell: "git archive 2.0.1 | /usr/bin/env tar -x -f - -C {{ first_release_dir }}" + shell: "git archive e0aee199e487 | /usr/bin/env tar -x -f - -C {{ first_release_dir }}" args: chdir: "{{ consul_dir }}/repo" From 49b5ecab182f7700132c571bcda06bfff4aebbf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sen=C3=A9n=20Rodero=20Rodr=C3=ADguez?= <15726+Senen@users.noreply.github.com> Date: Wed, 20 Sep 2023 17:29:13 +0200 Subject: [PATCH 03/16] Install a specific version of Node.js Consul Democracy has added a `.node-version` file in version 2.1.0, and now it installs the version defined there when deploying with Capistrano and installs NPM packages for that version. So we're doing the same when running the installer. --- app.yml | 1 + group_vars/all | 7 +++++++ roles/errbit/tasks/main.yml | 2 +- roles/nodejs/tasks/main.yml | 26 ++++++++++++++++++++++++++ roles/puma/tasks/main.yml | 2 +- roles/queue/tasks/main.yml | 2 +- roles/rails/tasks/main.yml | 10 +++++----- roles/system/tasks/main.yml | 16 ---------------- roles/system/tasks/tools.yml | 1 - 9 files changed, 42 insertions(+), 25 deletions(-) create mode 100644 roles/nodejs/tasks/main.yml diff --git a/app.yml b/app.yml index d4a40482..0048ded2 100644 --- a/app.yml +++ b/app.yml @@ -18,6 +18,7 @@ roles: - folder_structure - ruby + - nodejs - rails - email - queue diff --git a/group_vars/all b/group_vars/all index f0ef0165..7c2a6ff2 100644 --- a/group_vars/all +++ b/group_vars/all @@ -45,6 +45,13 @@ smtp_authentication: "plain" #LetsEncrypt #letsencrypt_email: "your_email@example.com" +# Node.js +fnm_dir: "{{ home_dir }}/.fnm" +fnm_command: "export PATH=\"{{ fnm_dir }}/:$PATH\" && eval \"$(fnm env)\"" + +# RVM +rvm_command: "source {{ home_dir }}/.rvm/scripts/rvm" + # Errbit errbit: False errbit_dir: "{{ home_dir }}/errbit" diff --git a/roles/errbit/tasks/main.yml b/roles/errbit/tasks/main.yml index beb5a428..bf6eb11a 100644 --- a/roles/errbit/tasks/main.yml +++ b/roles/errbit/tasks/main.yml @@ -146,7 +146,7 @@ replace: ' errbit_host: "https://{{ errbit_domain }}"' - name: Restart CONSUL DEMOCRACY - shell: "source /home/{{ deploy_user }}/.rvm/scripts/rvm && RAILS_ENV={{ env }} bin/rails restart" + shell: "{{ fnm_command }} && {{ rvm_command }} && fnm exec bin/rails restart RAILS_ENV={{ env }}" args: chdir: "{{ release_dir }}" executable: /bin/bash diff --git a/roles/nodejs/tasks/main.yml b/roles/nodejs/tasks/main.yml new file mode 100644 index 00000000..1b135d88 --- /dev/null +++ b/roles/nodejs/tasks/main.yml @@ -0,0 +1,26 @@ +--- +- name: Install fnm + shell: | + curl -fsSL https://fnm.vercel.app/install | bash -s -- --install-dir "{{ fnm_dir }}" + args: + chdir: "{{ home_dir }}" + executable: /bin/bash + creates: "{{ fnm_dir }}/fnm" + +- name: Read Node.js version + shell: "cat .node-version" + args: + chdir: "{{ release_dir }}" + register: node_version + +- name: Install nodejs via fnm + shell: "{{ fnm_command }} && {{ rvm_command }} && fnm install {{ node_version.stdout }}" + args: + chdir: "{{ release_dir }}" + executable: /bin/bash + +- name: Install Node packages + shell: "{{ fnm_command }} && {{ rvm_command }} && fnm exec npm install --production" + args: + chdir: "{{ release_dir }}" + executable: /bin/bash diff --git a/roles/puma/tasks/main.yml b/roles/puma/tasks/main.yml index 41831f10..f1404f61 100644 --- a/roles/puma/tasks/main.yml +++ b/roles/puma/tasks/main.yml @@ -23,7 +23,7 @@ when: puma_process.stat.exists == True - name: Start puma - shell: "source {{ home_dir }}/.rvm/scripts/rvm && bundle exec puma -C {{ release_dir }}/config/puma/{{ env }}.rb -e {{ env }} -d" + shell: "{{ fnm_command }} && {{ rvm_command }} && fnm exec bundle exec puma -C {{ release_dir }}/config/puma/{{ env }}.rb -e {{ env }} -d" args: chdir: "{{ release_dir }}" executable: /bin/bash diff --git a/roles/queue/tasks/main.yml b/roles/queue/tasks/main.yml index 3e6889b8..3686c680 100644 --- a/roles/queue/tasks/main.yml +++ b/roles/queue/tasks/main.yml @@ -1,5 +1,5 @@ - name: Start DelayedJobs queue - shell: "source /home/{{ deploy_user }}/.rvm/scripts/rvm && RAILS_ENV={{ env }} bin/delayed_job -m -n 2 restart" + shell: "{{ fnm_command }} && {{ rvm_command }} && RAILS_ENV={{ env }} fnm exec bin/delayed_job -m -n 2 restart" args: executable: /bin/bash chdir: "{{ release_dir }}" diff --git a/roles/rails/tasks/main.yml b/roles/rails/tasks/main.yml index 9e748af3..7598b93f 100644 --- a/roles/rails/tasks/main.yml +++ b/roles/rails/tasks/main.yml @@ -60,7 +60,7 @@ replace: '{{ env }}:\n # secret_key_base: ""\n server_name: "{{ server_hostname }}"' - name: Generate secret key - shell: "source {{ home_dir }}/.rvm/scripts/rvm && bin/rake secret RAILS_ENV={{ env }}" + shell: "{{ fnm_command }} && {{ rvm_command }} && fnm exec bin/rake secret RAILS_ENV={{ env }}" register: secret_key_base args: chdir: "{{ release_dir }}" @@ -80,25 +80,25 @@ when: domain is not defined - name: Create Database - shell: "source /home/{{ deploy_user }}/.rvm/scripts/rvm && bin/rake db:migrate RAILS_ENV={{ env }}" + shell: "{{ fnm_command }} && {{ rvm_command }} && fnm exec bin/rake db:migrate RAILS_ENV={{ env }}" args: chdir: "{{ release_dir }}" executable: /bin/bash - name: Load configuration seeds - shell: "source /home/{{ deploy_user}}/.rvm/scripts/rvm && bin/rake db:seed RAILS_ENV={{ env }}" + shell: "{{ fnm_command }} && {{ rvm_command }} && fnm exec bin/rake db:seed RAILS_ENV={{ env }}" args: chdir: "{{ release_dir }}" executable: /bin/bash - name: Precompile assets - shell: "source /home/{{ deploy_user }}/.rvm/scripts/rvm && bin/rake assets:precompile RAILS_ENV={{ env }}" + shell: "{{ fnm_command }} && {{ rvm_command }} && fnm exec bin/rake assets:precompile RAILS_ENV={{ env }}" args: chdir: "{{ release_dir }}" executable: /bin/bash - name: Update crontab with whenever - shell: "source /home/{{ deploy_user }}/.rvm/scripts/rvm && bundle exec whenever --update-crontab {{ app_name }} --set environment={{ env }}" + shell: "{{ fnm_command }} && {{ rvm_command }} && fnm exec bundle exec whenever --update-crontab {{ app_name }} --set environment={{ env }}" args: chdir: "{{ release_dir }}" executable: /bin/bash diff --git a/roles/system/tasks/main.yml b/roles/system/tasks/main.yml index e3cd8f51..af32c71b 100644 --- a/roles/system/tasks/main.yml +++ b/roles/system/tasks/main.yml @@ -22,22 +22,6 @@ apt: name: apt-transport-https -- name: Add Node key - become: yes - apt_key: - url: https://deb.nodesource.com/gpgkey/nodesource.gpg.key - state: present - -- name: Get distribution codename - shell: lsb_release -c --short - register: distro_codename - -- name: Add Node repository - become: true - apt_repository: - repo: "deb https://deb.nodesource.com/node_10.x {{ distro_codename.stdout }} main" - state: present - - name: Remove apache server become: true apt: diff --git a/roles/system/tasks/tools.yml b/roles/system/tasks/tools.yml index 6b2ad711..a7ea1a55 100644 --- a/roles/system/tasks/tools.yml +++ b/roles/system/tasks/tools.yml @@ -20,7 +20,6 @@ - libffi-dev - curl - libcurl4-openssl-dev - - nodejs - libpq-dev - imagemagick - ruby-dev From 5c4b72bab9b6f23649b107e5c9ba455b160444c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sen=C3=A9n=20Rodero=20Rodr=C3=ADguez?= <15726+Senen@users.noreply.github.com> Date: Wed, 20 Sep 2023 18:18:00 +0200 Subject: [PATCH 04/16] Fix deprecation warning when using `include` ``` [DEPRECATION WARNING]: "include" is deprecated, use include_tasks/import_tasks instead. See https://docs.ansible.com/ansible-core/2.15/user_guide/playbooks_reuse_includes.html for details. This feature will be removed in version 2.16. Deprecation warnings can be disabled by setting deprecation_warnings=False in ansible.cfg. ``` --- galaxy/yatesr.timezone/tasks/main.yml | 2 +- roles/system/tasks/main.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/galaxy/yatesr.timezone/tasks/main.yml b/galaxy/yatesr.timezone/tasks/main.yml index 9f4fdc6b..3b86b454 100644 --- a/galaxy/yatesr.timezone/tasks/main.yml +++ b/galaxy/yatesr.timezone/tasks/main.yml @@ -1,2 +1,2 @@ --- -- include: timezone.yml +- include_tasks: timezone.yml diff --git a/roles/system/tasks/main.yml b/roles/system/tasks/main.yml index af32c71b..f9d23644 100644 --- a/roles/system/tasks/main.yml +++ b/roles/system/tasks/main.yml @@ -28,4 +28,4 @@ name: apache2 state: absent -- include: tools.yml +- include_tasks: tools.yml From c44a6ab8b905da4e1b50d9ca3d87a5094338fc04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sen=C3=A9n=20Rodero=20Rodr=C3=ADguez?= <15726+Senen@users.noreply.github.com> Date: Mon, 16 Oct 2023 16:20:29 +0200 Subject: [PATCH 05/16] Simplify commands by using the newly added variable --- roles/errbit/tasks/main.yml | 14 +++++++------- roles/errbit/templates/errbit.service | 2 +- roles/rails/tasks/main.yml | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/roles/errbit/tasks/main.yml b/roles/errbit/tasks/main.yml index bf6eb11a..51f844bb 100644 --- a/roles/errbit/tasks/main.yml +++ b/roles/errbit/tasks/main.yml @@ -35,7 +35,7 @@ executable: /bin/bash - name: Install libv8-node for the right platform - shell: "source /home/{{ deploy_user }}/.rvm/scripts/rvm && gem install libv8-node --version '{{ libv8_version.stdout }}' --platform x86_64-linux-libc" + shell: "{{ rvm_command }} && gem install libv8-node --version '{{ libv8_version.stdout }}' --platform x86_64-linux-libc" args: chdir: "{{ errbit_dir }}" executable: /bin/bash @@ -48,13 +48,13 @@ executable: /bin/bash - name: Install the mini_racer gem - shell: "source /home/{{ deploy_user }}/.rvm/scripts/rvm && gem install mini_racer --version '{{ mini_racer_version.stdout }}'" + shell: "{{ rvm_command }} && gem install mini_racer --version '{{ mini_racer_version.stdout }}'" args: chdir: "{{ errbit_dir }}" executable: /bin/bash - name: Install Errbit dependencies - shell: "source /home/{{ deploy_user }}/.rvm/scripts/rvm && bundle install" + shell: "{{ rvm_command }} && bundle install" args: chdir: "{{ errbit_dir }}" executable: /bin/bash @@ -81,7 +81,7 @@ - when: not existing_secret_key_base.found block: - name: Generate secret key - shell: "source /home/{{ deploy_user }}/.rvm/scripts/rvm && bin/rake secret" + shell: "{{ rvm_command }} && bin/rake secret" register: secret_key_base args: chdir: "{{ errbit_dir }}" @@ -93,13 +93,13 @@ line: "SECRET_KEY_BASE={{ secret_key_base.stdout }}" - name: Setup Errbit - shell: "source /home/{{ deploy_user }}/.rvm/scripts/rvm && RAILS_ENV={{ env }} bin/rake errbit:bootstrap" + shell: "{{ rvm_command }} && RAILS_ENV={{ env }} bin/rake errbit:bootstrap" args: chdir: "{{ errbit_dir }}" executable: /bin/bash - name: Precompile Errbit assets - shell: "source /home/{{ deploy_user }}/.rvm/scripts/rvm && RAILS_ENV={{ env }} bin/rake assets:precompile" + shell: "{{ rvm_command }} && RAILS_ENV={{ env }} bin/rake assets:precompile" args: chdir: "{{ errbit_dir }}" executable: /bin/bash @@ -121,7 +121,7 @@ enabled: true - name: Create app if it does not exist - shell: 'source /home/{{ deploy_user }}/.rvm/scripts/rvm && bin/rails runner -e {{ env }} "App.create(name: \"{{ domain }}\")"' + shell: '{{ rvm_command }} && bin/rails runner -e {{ env }} "App.create(name: \"{{ domain }}\")"' args: chdir: "{{ errbit_dir }}" executable: /bin/bash diff --git a/roles/errbit/templates/errbit.service b/roles/errbit/templates/errbit.service index 8d496270..00f03365 100644 --- a/roles/errbit/templates/errbit.service +++ b/roles/errbit/templates/errbit.service @@ -7,7 +7,7 @@ After=mongodb.service network.target Type=simple WorkingDirectory={{ errbit_dir }} Environment=RAILS_ENV={{ env }} -ExecStart=/bin/bash -lc 'source {{ home_dir }}/.rvm/scripts/rvm && bundle exec puma -C {{ errbit_dir }}/config/puma.default.rb -e {{ env }}' +ExecStart=/bin/bash -lc '{{ rvm_command }} && bundle exec puma -C {{ errbit_dir }}/config/puma.default.rb -e {{ env }}' Restart=always User={{ errbit_user }} Group={{ errbit_group }} diff --git a/roles/rails/tasks/main.yml b/roles/rails/tasks/main.yml index 7598b93f..75441551 100644 --- a/roles/rails/tasks/main.yml +++ b/roles/rails/tasks/main.yml @@ -1,12 +1,12 @@ --- - name: Configure Bundler path - shell: "source /home/{{ deploy_user }}/.rvm/scripts/rvm && bundle config --local path {{ shared_dir }}/bundle" + shell: "{{ rvm_command }} && bundle config --local path {{ shared_dir }}/bundle" args: chdir: "{{ release_dir }}" executable: /bin/bash - name: Configure Bundler environments - shell: "source /home/{{ deploy_user }}/.rvm/scripts/rvm && bundle config --local without development:test" + shell: "{{ rvm_command }} && bundle config --local without development:test" args: chdir: "{{ release_dir }}" executable: /bin/bash @@ -26,7 +26,7 @@ when: not usr_bin_mkdir.stat.exists - name: Install gems (this may take a few minutes) - shell: "source /home/{{ deploy_user }}/.rvm/scripts/rvm && bundle install" + shell: "{{ rvm_command }} && bundle install" args: chdir: "{{ release_dir }}" executable: /bin/bash From 59b8bbb9f7072dd9a4299f1cf8e55e7cd795011c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sen=C3=A9n=20Rodero=20Rodr=C3=ADguez?= <15726+Senen@users.noreply.github.com> Date: Tue, 24 Oct 2023 16:28:19 +0200 Subject: [PATCH 06/16] Retry nodejs installation until succeed or reaches 10 failed attempts We found nodejs binaries from `nodejs.org/dist` sometimes are not available or takes to long to download them making the installer crash. With this change we hope to significally reduce the chance to have a failure when running the installer. --- roles/nodejs/tasks/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/roles/nodejs/tasks/main.yml b/roles/nodejs/tasks/main.yml index 1b135d88..ca3f20da 100644 --- a/roles/nodejs/tasks/main.yml +++ b/roles/nodejs/tasks/main.yml @@ -18,6 +18,10 @@ args: chdir: "{{ release_dir }}" executable: /bin/bash + register: fnm_install_result + until: "fnm_install_result is not failed" + retries: 10 + delay: 10 - name: Install Node packages shell: "{{ fnm_command }} && {{ rvm_command }} && fnm exec npm install --production" From 659732c3d2c1669f89819282c66b0c599a1d259c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javi=20Mart=C3=ADn?= Date: Sat, 16 Sep 2023 00:05:17 +0200 Subject: [PATCH 07/16] Use systemd to start Puma MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `puma.service` file included here is an adaptation of the ERB template provided by the capistrano3-puma gem. In order to transition smoothly from the systemd configuration generated by this installer to the systemd configuration provided by capistrano3-puma, the `puma_service_unit_name` must be the same during both installation and deployment. Note that, now that Puma starts with systemd, we have to wait till it creates a socket before continuing; otherwise the socket file won't be there when we execute the next command, and that command will file. Also note we have to configure the XDG_RUNTIME_DIR variable. Quoting the ansible documentation [1]: > For systemd to work with ‘user’, the executing user must have its own > instance of dbus started and accessible (systemd requirement). > > The user dbus process is normally started during normal login, but not > during the run of Ansible tasks. Otherwise you will probably get a > 'Failed to connect to bus: no such file or directory' error. > > The user must have access, normally given via setting the > XDG_RUNTIME_DIR variable. We also need to check for systemd access because ansible doesn't do it automatically [2]. In order to allow unprivileged users to enable lingering to enable systemd user units, we need to install the policykit-1 package, which is installed by default in Ubuntu but not on Debian Bookworm. Without this package the action of lingering requires sudo priveleges. Finally, we're using Consul Democracy's `master` branch, since it contains support for Puma with Systemd. [1] https://docs.ansible.com/ansible/latest/collections/ansible/builtin/systemd_service_module.html#parameter-scope [2] See issue in 72674 https://github.com/ansible/ansible/issues/ Co-Authored-By: Senén Rodero --- group_vars/all | 8 +++++ roles/folder_structure/tasks/main.yml | 2 +- roles/puma/tasks/main.yml | 48 ++++++++++++++++++--------- roles/puma/templates/puma.service | 17 ++++++++++ roles/system/tasks/tools.yml | 1 + 5 files changed, 60 insertions(+), 16 deletions(-) create mode 100644 roles/puma/templates/puma.service diff --git a/group_vars/all b/group_vars/all index 7c2a6ff2..1dcfef8c 100644 --- a/group_vars/all +++ b/group_vars/all @@ -34,6 +34,14 @@ database_user: "{{ deploy_user }}" database_password: "{{ deploy_user }}" database_hostname: "localhost" +# Puma +# If you use Capistrano to deploy, make sure the puma_service_unit_name +# variable is the same as `:puma_service_unit_name` in Capistrano +puma_service_unit_name: "puma_{{ app_name }}_{{ env }}" +puma_config_file: "{{ release_dir }}/config/puma/{{ env }}.rb" +puma_access_log: "{{ shared_dir }}/log/puma_access.log" +puma_error_log: "{{ shared_dir }}/log/puma_error.log" + #SMTP smtp_address: "smtp.example.com" smtp_port: 25 diff --git a/roles/folder_structure/tasks/main.yml b/roles/folder_structure/tasks/main.yml index bf9430ad..ae5dd085 100644 --- a/roles/folder_structure/tasks/main.yml +++ b/roles/folder_structure/tasks/main.yml @@ -22,7 +22,7 @@ state: directory - name: Create first release - shell: "git archive e0aee199e487 | /usr/bin/env tar -x -f - -C {{ first_release_dir }}" + shell: "git archive master | /usr/bin/env tar -x -f - -C {{ first_release_dir }}" args: chdir: "{{ consul_dir }}/repo" diff --git a/roles/puma/tasks/main.yml b/roles/puma/tasks/main.yml index f1404f61..aa275d82 100644 --- a/roles/puma/tasks/main.yml +++ b/roles/puma/tasks/main.yml @@ -7,26 +7,44 @@ - "pids" - "sockets" -- name: Check that puma is running +- name: Create systemd folder + file: + path: "{{ home_dir }}/.config/systemd/user" + state: directory + +- name: Copy Puma service file to the systemd folder + template: + src: "{{ playbook_dir }}/roles/puma/templates/puma.service" + dest: "{{ home_dir }}/.config/systemd/user/{{ puma_service_unit_name }}.service" + +- name: Check if user has access to systemd while running ansible tasks stat: - path: "{{ shared_dir }}/tmp/pids/puma.pid" - register: puma_process + path: "/var/lib/systemd/linger/{{ deploy_user }}" + register: linger_enabled -- name: Get running puma process - shell: "cat {{ shared_dir }}/tmp/pids/puma.pid" - register: running_process - when: puma_process.stat.exists == True +- name: Enable systemd access if needed + command: "loginctl enable-linger {{ deploy_user }}" + when: not linger_enabled.stat.exists -- name: Kill running process - shell: "kill -QUIT {{ item }}" - with_items: "{{ running_process.stdout_lines }}" - when: puma_process.stat.exists == True +- name: Get user UID + shell: "id -u" + register: current_uid - name: Start puma - shell: "{{ fnm_command }} && {{ rvm_command }} && fnm exec bundle exec puma -C {{ release_dir }}/config/puma/{{ env }}.rb -e {{ env }} -d" - args: - chdir: "{{ release_dir }}" - executable: /bin/bash + systemd: + name: "{{ puma_service_unit_name }}" + daemon_reload: true + enabled: true + state: started + scope: user + environment: + XDG_RUNTIME_DIR: "/run/user/{{ current_uid.stdout }}" + +- name: Wait until Puma has created the socket + wait_for: + path: "{{ release_dir }}/tmp/sockets/puma.sock" + state: present + msg: Puma socket is not available - name: Make sure Nginx has write access to the puma socket shell: "chmod o+w tmp/sockets/*" diff --git a/roles/puma/templates/puma.service b/roles/puma/templates/puma.service new file mode 100644 index 00000000..67c7061c --- /dev/null +++ b/roles/puma/templates/puma.service @@ -0,0 +1,17 @@ +[Unit] +Description=Puma HTTP Server for {{ app_name }} ({{ env }}) +After=network.target + +[Service] +Type=simple +WorkingDirectory={{ release_dir }} +ExecStart=fnm exec bundle exec --keep-file-descriptors puma -C {{ puma_config_file }} +ExecReload=/bin/kill -USR1 $MAINPID +StandardOutput=append:{{ puma_access_log }} +StandardError=append:{{ puma_error_log }} +Restart=always +RestartSec=1 +SyslogIdentifier=puma + +[Install] +WantedBy=default.target diff --git a/roles/system/tasks/tools.yml b/roles/system/tasks/tools.yml index a7ea1a55..1997236a 100644 --- a/roles/system/tasks/tools.yml +++ b/roles/system/tasks/tools.yml @@ -24,3 +24,4 @@ - imagemagick - ruby-dev - shared-mime-info + - policykit-1 From 480c7b1c5df73306fd9ccef864d807e5f0d2e047 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javi=20Mart=C3=ADn?= Date: Mon, 16 Oct 2023 23:07:48 +0200 Subject: [PATCH 08/16] Modify the Puma command so it works with Ansible While this isn't needed when starting the Puma service on the server or with Capistrano, we need the RVM script while using the installer. The file is now different from the one generated by capistrano3-puma, but that's alright because it will be overwritten when deploying with Capistrano (as long as the `:puma_service_unit_name` is the same). --- roles/puma/templates/puma.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/puma/templates/puma.service b/roles/puma/templates/puma.service index 67c7061c..528bcef5 100644 --- a/roles/puma/templates/puma.service +++ b/roles/puma/templates/puma.service @@ -5,7 +5,7 @@ After=network.target [Service] Type=simple WorkingDirectory={{ release_dir }} -ExecStart=fnm exec bundle exec --keep-file-descriptors puma -C {{ puma_config_file }} +ExecStart=/bin/bash -lc '{{ fnm_command }} && {{ rvm_command }} && fnm exec bundle exec puma -C {{ puma_config_file }} -e {{ env }}' ExecReload=/bin/kill -USR1 $MAINPID StandardOutput=append:{{ puma_access_log }} StandardError=append:{{ puma_error_log }} From 5af53dd0d1d79ff3b9fe0cf1cca5adeba514df8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javi=20Mart=C3=ADn?= Date: Tue, 17 Oct 2023 03:34:51 +0200 Subject: [PATCH 09/16] Don't test systemd on Debian in Github Actions The Docker images for Debian don't include systemd and, even when using an image that comes with support with systemd [1], we're getting a warning when starting Puma (the following error takes place when we skip the "Enable systemd access" step, since that step also raises an error in Debian): > Warning: : daemon-reload failed, but target is a chroot or systemd > is offline. > Continuing. Error was: 1 / Failed to connect to bus: No such file or > directory > Warning: : Target is a chroot or systemd is offline. This can > lead to false positives or prevent the init system tools from working. So Puma isn't started in this case. So now Debian support is uncertain, since we aren't able to test that Puma starts correctly and the web is accessible. We're at least testing that the rest of the installation process runs correctly. [1] https://github.com/trfore/docker-debian11-systemd/blob/main/Dockerfile --- roles/puma/tasks/main.yml | 74 ++++++++++++++++++++------------------ roles/specs/tasks/main.yml | 50 +++++++++++++------------- 2 files changed, 66 insertions(+), 58 deletions(-) diff --git a/roles/puma/tasks/main.yml b/roles/puma/tasks/main.yml index aa275d82..568eeebf 100644 --- a/roles/puma/tasks/main.yml +++ b/roles/puma/tasks/main.yml @@ -17,37 +17,43 @@ src: "{{ playbook_dir }}/roles/puma/templates/puma.service" dest: "{{ home_dir }}/.config/systemd/user/{{ puma_service_unit_name }}.service" -- name: Check if user has access to systemd while running ansible tasks - stat: - path: "/var/lib/systemd/linger/{{ deploy_user }}" - register: linger_enabled - -- name: Enable systemd access if needed - command: "loginctl enable-linger {{ deploy_user }}" - when: not linger_enabled.stat.exists - -- name: Get user UID - shell: "id -u" - register: current_uid - -- name: Start puma - systemd: - name: "{{ puma_service_unit_name }}" - daemon_reload: true - enabled: true - state: started - scope: user - environment: - XDG_RUNTIME_DIR: "/run/user/{{ current_uid.stdout }}" - -- name: Wait until Puma has created the socket - wait_for: - path: "{{ release_dir }}/tmp/sockets/puma.sock" - state: present - msg: Puma socket is not available - -- name: Make sure Nginx has write access to the puma socket - shell: "chmod o+w tmp/sockets/*" - args: - chdir: "{{ release_dir }}" - executable: /bin/bash +- name: Get distribution codename + shell: lsb_release -c --short + register: distro_codename + +- when: distro_codename.stdout == "focal" or distro_codename.stdout == "jammy" or not lookup("env", "CI") + block: + - name: Check if user has access to systemd while running ansible tasks + stat: + path: "/var/lib/systemd/linger/{{ deploy_user }}" + register: linger_enabled + + - name: Enable systemd access if needed + command: "loginctl enable-linger {{ deploy_user }}" + when: not linger_enabled.stat.exists + + - name: Get user UID + shell: "id -u" + register: current_uid + + - name: Start puma + systemd: + name: "{{ puma_service_unit_name }}" + daemon_reload: true + enabled: true + state: started + scope: user + environment: + XDG_RUNTIME_DIR: "/run/user/{{ current_uid.stdout }}" + + - name: Wait until Puma has created the socket + wait_for: + path: "{{ release_dir }}/tmp/sockets/puma.sock" + state: present + msg: Puma socket is not available + + - name: Make sure Nginx has write access to the puma socket + shell: "chmod o+w tmp/sockets/*" + args: + chdir: "{{ release_dir }}" + executable: /bin/bash diff --git a/roles/specs/tasks/main.yml b/roles/specs/tasks/main.yml index f22bf63b..70337d28 100644 --- a/roles/specs/tasks/main.yml +++ b/roles/specs/tasks/main.yml @@ -1,34 +1,36 @@ --- -- when: domain is defined +- when: distro_codename.stdout == "focal" or distro_codename.stdout == "jammy" or not lookup("env", "CI") block: - - action: uri url=https://{{ server_hostname }} return_content=yes validate_certs=False - register: webpage_https_with_domain + - when: domain is defined + block: + - action: uri url=https://{{ server_hostname }} return_content=yes validate_certs=False + register: webpage_https_with_domain - - fail: - msg: "service is not happy {{ webpage_https_with_domain.content }}" - when: "'CONSUL' not in webpage_https_with_domain.content" + - fail: + msg: "service is not happy {{ webpage_https_with_domain.content }}" + when: "'CONSUL' not in webpage_https_with_domain.content" - - name: Redirect to https - uri: - url: http://{{ server_hostname }} - follow_redirects: no - validate_certs: yes - status_code: 301 + - name: Redirect to https + uri: + url: http://{{ server_hostname }} + follow_redirects: no + validate_certs: yes + status_code: 301 -- when: domain is not defined - block: - - action: uri url=http://127.0.0.1 return_content=yes validate_certs=False - register: webpage_https + - when: domain is not defined + block: + - action: uri url=http://127.0.0.1 return_content=yes validate_certs=False + register: webpage_https - - fail: - msg: "service is not happy {{ webpage_https.content }}" - when: "'CONSUL' not in webpage_https.content" + - fail: + msg: "service is not happy {{ webpage_https.content }}" + when: "'CONSUL' not in webpage_https.content" - - name: Do not redirect to https - uri: - url: http://{{ server_hostname }} - follow_redirects: no - status_code: 200 + - name: Do not redirect to https + uri: + url: http://{{ server_hostname }} + follow_redirects: no + status_code: 200 - name: Get running delayed job processes shell: "ps -ef | grep -v grep | grep -w delayed_job | awk '{print $2}'" From cb9d2f72f849b450ff1bc8d99cbab4f282f14cc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sen=C3=A9n=20Rodero=20Rodr=C3=ADguez?= <15726+Senen@users.noreply.github.com> Date: Mon, 23 Oct 2023 11:17:12 +0200 Subject: [PATCH 10/16] Add support for Ubuntu 22.04 Jammy --- .github/workflows/ubuntu.yml | 2 +- README.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 136d2d80..9daad4d2 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -11,7 +11,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-20.04] + os: [ubuntu-20.04, ubuntu-22.04] rails_env: [staging, production] steps: - uses: actions/checkout@v2 diff --git a/README.md b/README.md index 7cdf848d..67c4b0be 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ It will also create a `deploy` user to install these libraries A remote server with one of the supported distributions: - Ubuntu 20.04 x64 +- Ubuntu 22.04 x64 - Debian Bullseye x64 Access to a remote server via public ssh key without password. From 249428f4850b23425e0acdf35efefe4c95a82009 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sen=C3=A9n=20Rodero=20Rodr=C3=ADguez?= <15726+Senen@users.noreply.github.com> Date: Mon, 23 Oct 2023 13:00:11 +0200 Subject: [PATCH 11/16] Do not install Errbit in Ubuntu 22.04 1. Errbit does not have the mongodb package available via apt. 2. The latest Errbit version uses ruby 2.7.4 which needs SSL1 while Ubuntu 22.04 supports SSL3. --- .github/workflows/ubuntu.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 9daad4d2..83f2cfa0 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -7,11 +7,14 @@ on: jobs: ubuntu: - runs-on: ${{ matrix.os }} + runs-on: ${{ matrix.platforms.os }} strategy: fail-fast: false matrix: - os: [ubuntu-20.04, ubuntu-22.04] + platforms: [ + { os: "ubuntu-20.04", errbit: "True" }, + { os: "ubuntu-22.04", errbit: "False"} + ] rails_env: [staging, production] steps: - uses: actions/checkout@v2 @@ -29,4 +32,4 @@ jobs: - name: Generate dummy SSH key run: ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa - name: Run CONSUL DEMOCRACY installer - run: ansible-playbook consul.yml -i hosts --extra-vars "env=${{ matrix.rails_env }} domain=localhost errbit=True" + run: ansible-playbook consul.yml -i hosts --extra-vars "env=${{ matrix.rails_env }} domain=localhost errbit=${{ matrix.platforms.errbit }}" From f289b712cd2c3e5f460d80e967c99591258fae9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sen=C3=A9n=20Rodero=20Rodr=C3=ADguez?= <15726+Senen@users.noreply.github.com> Date: Tue, 24 Oct 2023 15:17:31 +0200 Subject: [PATCH 12/16] Ensure deploy_user home has the correct permissions For some reason I do not know yet, when running the installer on a Ubuntu 22.04 server it creates the deploy_user home directory with 750 permissions instead of 755. We need 755 permissions so other users like `www-data`, which runs the nginx service can read from the Consul Democracy puma socket. Otherwise we get a 502 Bad Gateway error when accessing the application. --- roles/user/tasks/main.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/roles/user/tasks/main.yml b/roles/user/tasks/main.yml index afd752ce..220a9fa8 100644 --- a/roles/user/tasks/main.yml +++ b/roles/user/tasks/main.yml @@ -18,6 +18,14 @@ state: present shell: /bin/bash +- name: Ensure correct permissions of deploy user home directory + file: + path: "{{ home_dir }}" + owner: "{{ deploy_user }}" + group: "{{ deploy_group }}" + mode: 0755 + state: directory + - name: Install SSH key authorized_key: user: "{{ deploy_user }}" From 20cd557b176827544e7e5f416b778aaaab15f159 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javi=20Mart=C3=ADn?= Date: Tue, 17 Oct 2023 22:41:23 +0200 Subject: [PATCH 13/16] Add support for Debian Bookworm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since Bookworm comes with PostgreSQL 15, we need to make the `deploy_user` the database owner. Quoting the PostgreSQL 15 release notes [1]: > PostgreSQL 15 also revokes the CREATE permission from all users > except a database owner from the public (or default) schema. Now, we first create the user without any privileges, then create the database with the new user as the database owner and finally grant all privileges to the new user on the new database. [1] https://www.postgresql.org/about/news/postgresql-15-released-2526/ Co-Authored-By: Senén Rodero --- .github/workflows/debian.yml | 6 ++---- README.md | 1 + roles/postgresql/tasks/main.yml | 13 +++++++------ 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/debian.yml b/.github/workflows/debian.yml index da80f23a..f2c1a81f 100644 --- a/.github/workflows/debian.yml +++ b/.github/workflows/debian.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - image: ["debian:bullseye"] + image: ["debian:bullseye", "debian:bookworm"] rails_env: [staging, production] container: image: ${{ matrix.image }} @@ -19,9 +19,7 @@ jobs: - name: Update system packages run: apt-get update -y - name: Install needed packages - run: apt-get install -y lsb-release sudo python3-pip openssh-server - - name: Install Ansible - run: pip3 install ansible + run: apt-get install -y lsb-release sudo python3-pip openssh-server ansible - name: Create hosts file run: echo "localhost ansible_connection=local ansible_user=root" > hosts - name: Generate dummy SSH key diff --git a/README.md b/README.md index 67c4b0be..407e008e 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ A remote server with one of the supported distributions: - Ubuntu 20.04 x64 - Ubuntu 22.04 x64 - Debian Bullseye x64 +- Debian Bookworm x64 Access to a remote server via public ssh key without password. The default user is `deploy` but you can [use any user](#using-a-different-user-than-deploy) with sudo privileges. diff --git a/roles/postgresql/tasks/main.yml b/roles/postgresql/tasks/main.yml index e3aaefc3..b995f2d6 100644 --- a/roles/postgresql/tasks/main.yml +++ b/roles/postgresql/tasks/main.yml @@ -17,17 +17,18 @@ - become: true become_user: postgres block: - - name: Create PostgreSQL database - postgresql_db: - name: "{{ database_name }}" - - name: Create PostgreSQL users postgresql_user: + state: present name: "{{ database_user }}" password: "{{ database_password }}" - db: "{{ database_name }}" encrypted: yes - priv: ALL + + - name: Create PostgreSQL database + postgresql_db: + state: present + name: "{{ database_name }}" + owner: "{{ database_user }}" - name: Create the shared extensions schema postgresql_schema: From 4076398c0452b086f11c563bf261c216c1d82b99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sen=C3=A9n=20Rodero=20Rodr=C3=ADguez?= Date: Thu, 1 Feb 2024 14:43:39 +0100 Subject: [PATCH 14/16] Remove FNM dependency for puma service Now it's disabled through environment variable. --- roles/puma/templates/puma.service | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/puma/templates/puma.service b/roles/puma/templates/puma.service index 528bcef5..e7d3dc93 100644 --- a/roles/puma/templates/puma.service +++ b/roles/puma/templates/puma.service @@ -5,10 +5,11 @@ After=network.target [Service] Type=simple WorkingDirectory={{ release_dir }} -ExecStart=/bin/bash -lc '{{ fnm_command }} && {{ rvm_command }} && fnm exec bundle exec puma -C {{ puma_config_file }} -e {{ env }}' +ExecStart=/bin/bash -lc '{{ rvm_command }} && bundle exec puma -C {{ puma_config_file }} -e {{ env }}' ExecReload=/bin/kill -USR1 $MAINPID StandardOutput=append:{{ puma_access_log }} StandardError=append:{{ puma_error_log }} +Environment=EXECJS_RUNTIME=Disabled Restart=always RestartSec=1 SyslogIdentifier=puma From 5ce11526b51b79576eae55ebf3c551188a8fb5a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sen=C3=A9n=20Rodero=20Rodr=C3=ADguez?= Date: Thu, 1 Feb 2024 17:43:55 +0100 Subject: [PATCH 15/16] Enable puma socket activation --- roles/puma/tasks/main.yml | 17 ++++++++++++++++- roles/puma/templates/puma.service | 1 + roles/puma/templates/puma.socket | 20 ++++++++++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 roles/puma/templates/puma.socket diff --git a/roles/puma/tasks/main.yml b/roles/puma/tasks/main.yml index 568eeebf..5b20b38f 100644 --- a/roles/puma/tasks/main.yml +++ b/roles/puma/tasks/main.yml @@ -17,6 +17,11 @@ src: "{{ playbook_dir }}/roles/puma/templates/puma.service" dest: "{{ home_dir }}/.config/systemd/user/{{ puma_service_unit_name }}.service" +- name: Copy Puma socket file to the systemd folder + template: + src: "{{ playbook_dir }}/roles/puma/templates/puma.socket" + dest: "{{ home_dir }}/.config/systemd/user/{{ puma_service_unit_name }}.socket" + - name: Get distribution codename shell: lsb_release -c --short register: distro_codename @@ -36,9 +41,19 @@ shell: "id -u" register: current_uid + - name: Enable puma socket activation + systemd: + name: "{{ puma_service_unit_name }}.socket" + daemon_reload: true + enabled: true + state: started + scope: user + environment: + XDG_RUNTIME_DIR: "/run/user/{{ current_uid.stdout }}" + - name: Start puma systemd: - name: "{{ puma_service_unit_name }}" + name: "{{ puma_service_unit_name }}.service" daemon_reload: true enabled: true state: started diff --git a/roles/puma/templates/puma.service b/roles/puma/templates/puma.service index e7d3dc93..ecc9592f 100644 --- a/roles/puma/templates/puma.service +++ b/roles/puma/templates/puma.service @@ -1,6 +1,7 @@ [Unit] Description=Puma HTTP Server for {{ app_name }} ({{ env }}) After=network.target +Requires={{ puma_service_unit_name }}.socket [Service] Type=simple diff --git a/roles/puma/templates/puma.socket b/roles/puma/templates/puma.socket new file mode 100644 index 00000000..f5fdbf75 --- /dev/null +++ b/roles/puma/templates/puma.socket @@ -0,0 +1,20 @@ +[Unit] +Description=Puma HTTP Server Accept Sockets for {{ app_name }} ({{ env }}) + +[Socket] +ListenStream={{ shared_dir }}/tmp/sockets/puma.sock + +# Don't let systemd accept the request, wait for Puma to do that. +# Systemd will start the puma service upon first request if it wasn't started. +# +# You might also want to set your Nginx upstream to have a fail_timeout large enough to accomodate your app's +# startup time. +Accept=no + +ReusePort=true +Backlog=1024 + +SyslogIdentifier=puma_socket + +[Install] +WantedBy=sockets.target From 73cfce0010e0ff55928ba5375d29c2531f5d80de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javi=20Mart=C3=ADn?= Date: Tue, 20 Feb 2024 20:38:56 +0100 Subject: [PATCH 16/16] Install Consul Democracy 2.1.0 --- roles/folder_structure/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/folder_structure/tasks/main.yml b/roles/folder_structure/tasks/main.yml index ae5dd085..7d6a6041 100644 --- a/roles/folder_structure/tasks/main.yml +++ b/roles/folder_structure/tasks/main.yml @@ -22,7 +22,7 @@ state: directory - name: Create first release - shell: "git archive master | /usr/bin/env tar -x -f - -C {{ first_release_dir }}" + shell: "git archive 2.1.0 | /usr/bin/env tar -x -f - -C {{ first_release_dir }}" args: chdir: "{{ consul_dir }}/repo"