From 1684da5c150ebe1c3fb466e51616e6709a3689c9 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Thu, 17 Nov 2022 18:27:56 -0500 Subject: [PATCH 1/7] Don't modify venv activate to set $GRAVITY_STATE_DIR --- scripts/common_startup.sh | 1 - scripts/common_startup_functions.sh | 11 ----------- 2 files changed, 12 deletions(-) diff --git a/scripts/common_startup.sh b/scripts/common_startup.sh index c6b4cc64e52a..84f02daefb02 100755 --- a/scripts/common_startup.sh +++ b/scripts/common_startup.sh @@ -163,7 +163,6 @@ if [ $SET_VENV -eq 1 ] && [ $CREATE_VENV -eq 1 ]; then fi fi fi - setup_gravity_state_dir fi # activate virtualenv or conda env, sets $GALAXY_VIRTUAL_ENV and $GALAXY_CONDA_ENV diff --git a/scripts/common_startup_functions.sh b/scripts/common_startup_functions.sh index cd613980a677..a810e73648fb 100644 --- a/scripts/common_startup_functions.sh +++ b/scripts/common_startup_functions.sh @@ -134,17 +134,6 @@ setup_python() { "$GALAXY_PYTHON" ./scripts/check_python.py || exit 1 } -setup_gravity_state_dir() { - # $GALAXY_VIRTUAL_ENV is expected to be set, and cwd must be galaxy root - if ! grep -q '^GRAVITY_STATE_DIR=' "${GALAXY_VIRTUAL_ENV}/bin/activate"; then - echo "Setting \$GRAVITY_STATE_DIR in ${GALAXY_VIRTUAL_ENV}/bin/activate" - echo '' >> "${GALAXY_VIRTUAL_ENV}/bin/activate" - echo '# Galaxy Gravity per-instance state directory configured by Galaxy common_startup.sh' >> "${GALAXY_VIRTUAL_ENV}/bin/activate" - echo "GRAVITY_STATE_DIR=\${GRAVITY_STATE_DIR:-'$(pwd)/database/gravity'}" >> "${GALAXY_VIRTUAL_ENV}/bin/activate" - echo 'export GRAVITY_STATE_DIR' >> "${GALAXY_VIRTUAL_ENV}/bin/activate" - fi -} - set_galaxy_config_file_var() { if [ -z "$GALAXY_CONFIG_FILE" ]; then GALAXY_CONFIG_FILE=$(PYTHONPATH=lib python -c "from __future__ import print_function; from galaxy.util.properties import find_config_file; print(find_config_file(['galaxy', 'universe_wsgi']) or '')") From 619c6ae47ed145a5ed3254f0ec9b40763aa5a239 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Thu, 17 Nov 2022 18:28:45 -0500 Subject: [PATCH 2/7] Use Gravity 1.0.0 pre-release --- lib/galaxy/dependencies/pinned-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index 6b8c16b80739..f6e37396a5fb 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -68,7 +68,7 @@ funcsigs==1.0.2 ; python_version >= "3.7" and python_version < "3.11" future==0.18.2 ; python_version >= "3.7" and python_version < "3.11" galaxy-sequence-utils==1.1.5 ; python_version >= "3.7" and python_version < "3.11" galaxy2cwl==0.1.4 ; python_version >= "3.7" and python_version < "3.11" -gravity==0.13.5 ; python_version >= "3.7" and python_version < "3.11" +gravity @ git+https://github.com/galaxyproject/gravity.git@release_1.x ; python_version >= "3.7" and python_version < "3.11" greenlet==2.0.1 ; python_version >= "3.7" and (platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "AMD64" or platform_machine == "win32" or platform_machine == "WIN32") and python_version < "3.11" gunicorn==20.1.0 ; python_version >= "3.7" and python_version < "3.11" gxformat2==0.16.0 ; python_version >= "3.7" and python_version < "3.11" From 0b9c30779dee6f73a4e1d4f81a91e77ec135d616 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Thu, 17 Nov 2022 18:48:19 -0500 Subject: [PATCH 3/7] Document setting env vars on static handlers --- lib/galaxy/config/sample/job_conf.sample.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/galaxy/config/sample/job_conf.sample.yml b/lib/galaxy/config/sample/job_conf.sample.yml index 47cb9067cb83..dcfc4253e55f 100644 --- a/lib/galaxy/config/sample/job_conf.sample.yml +++ b/lib/galaxy/config/sample/job_conf.sample.yml @@ -375,8 +375,15 @@ handling: # most of them). If unset, the default is any untagged handlers plus any handlers in the `job-handlers` (no tag) pool. #default: handler0 + # Static handler configurations - Configuring these is generally not desirable, fully dynamic handlers are preferable + # unless you need to control plugin loading on a per-handler basis. See the scaling documentation linked above for + # instructions on configuring dynamic handlers. + processes: handler0: + # Set environment variables that only apply to this handler + #environment: + # FOO: foo handler1: sge_handler: # Restrict a handler to load specific runners, by default they will load all. From f329c71a9727e09f4afab8e61dd9a967748dfaf4 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Thu, 17 Nov 2022 20:01:50 -0500 Subject: [PATCH 4/7] Update gravity launch test for 1.0 --- lib/galaxy_test/driver/driver_util.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/galaxy_test/driver/driver_util.py b/lib/galaxy_test/driver/driver_util.py index 5d93997a2494..f90f562e69b1 100644 --- a/lib/galaxy_test/driver/driver_util.py +++ b/lib/galaxy_test/driver/driver_util.py @@ -762,11 +762,10 @@ def launch_gravity(port, gxit_port=None, galaxy_config=None): supervisord_socket = socket.name gravity_env = os.environ.copy() gravity_env["SUPERVISORD_SOCKET"] = supervisord_socket - subprocess.check_output(["galaxyctl", "--state-dir", state_dir, "register", config_fh.name], env=gravity_env) - subprocess.check_output(["galaxyctl", "--state-dir", state_dir, "update"], env=gravity_env) - subprocess.check_output(["galaxyctl", "--state-dir", state_dir, "start"], env=gravity_env) + subprocess.check_output(["galaxyctl", "--config-file", config_fh.name, "update"], env=gravity_env) + subprocess.check_output(["galaxyctl", "--config-file", config_fh.name, "start"], env=gravity_env) return state_dir, lambda: subprocess.check_output( - ["galaxyctl", "--state-dir", state_dir, "shutdown"], env=gravity_env + ["galaxyctl", "--config-file", config_fh.name, "shutdown"], env=gravity_env ) From 36f6e3901127b9aa0c180994f74b80ecc066f416 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Thu, 17 Nov 2022 20:04:59 -0500 Subject: [PATCH 5/7] Update gravity section of sample config for 1.0 --- lib/galaxy/config/sample/galaxy.yml.sample | 153 ++++++++++++++++++++- 1 file changed, 149 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample index bb10dfc75e1d..9e5a3c58df5a 100644 --- a/lib/galaxy/config/sample/galaxy.yml.sample +++ b/lib/galaxy/config/sample/galaxy.yml.sample @@ -20,16 +20,55 @@ # ``uwsgi:`` section will be ignored if Galaxy is started via Gravity commands (e.g ``./run.sh``, ``galaxy`` or ``galaxyctl``). gravity: + # Process manager to use. + # ``supervisor`` is the default process manager when Gravity is invoked as a non-root user. + # ``systemd`` is the default when Gravity is invoked as root. + # Valid options are: supervisor, systemd + # process_manager: + + # What command to write to the process manager configs + # `gravity` (`galaxyctl exec `) is the default + # `direct` (each service's actual command) is also supported. + # Valid options are: gravity, direct, exec + # service_command_style: gravity + + # Use the process manager's *service instance* functionality for services that can run multiple instances. + # Presently this includes services like gunicorn and Galaxy dynamic job handlers. Service instances are only supported if + # ``service_command_style`` is ``gravity``, and so this option is automatically set to ``false`` if + # ``service_command_style`` is set to ``direct``. + # use_service_instances: true + + # umask under which services should be executed. Setting ``umask`` on an individual service overrides this value. + # umask: '022' + + # Memory limit (in GB), processes exceeding the limit will be killed. Default is no limit. If set, this is default value + # for all services. Setting ``memory_limit`` on an individual service overrides this value. Ignored if ``process_manager`` + # is ``supervisor``. + # memory_limit: + + # Specify Galaxy config file (galaxy.yml), if the Gravity config is separate from the Galaxy config. Assumed to be the + # same file as the Gravity config if a ``galaxy`` key exists at the root level, otherwise, this option is required. + # galaxy_config_file: + # Specify Galaxy's root directory. # Gravity will attempt to find the root directory, but you can set the directory explicitly with this option. # galaxy_root: + # User to run Galaxy as, required when using the systemd process manager as root. + # Ignored if ``process_manager`` is ``supervisor`` or user-mode (non-root) ``systemd``. + # galaxy_user: + + # Group to run Galaxy as, optional when using the systemd process manager as root. + # Ignored if ``process_manager`` is ``supervisor`` or user-mode (non-root) ``systemd``. + # galaxy_group: + # Set to a directory that should contain log files for the processes controlled by Gravity. - # If not specified defaults to ``/log``. + # If not specified defaults to ``/gravity/log``. # log_dir: # Set to Galaxy's virtualenv directory. - # If not specified, Gravity assumes all processes are on PATH. + # If not specified, Gravity assumes all processes are on PATH. This option is required in most circumstances when using + # the ``systemd`` process manager. # virtualenv: # Select the application server. @@ -43,7 +82,7 @@ gravity: # this is hidden from you when running a single instance. # instance_name: _default_ - # Configuration for Gunicorn. + # Configuration for Gunicorn. Can be a list to run multiple gunicorns for rolling restarts. gunicorn: # Enable Galaxy gunicorn server. @@ -70,6 +109,17 @@ gravity: # Consumes less memory when multiple processes are configured. Default is ``false`` if using unicornherder, else ``true``. # preload: + # Value of supervisor startsecs, systemd TimeoutStartSec + # start_timeout: 15 + + # Value of supervisor stopwaitsecs, systemd TimeoutStopSec + # stop_timeout: 65 + + # Memory limit (in GB). If the service exceeds the limit, it will be killed. Default is no limit or the value of the + # ``memory_limit`` setting at the top level of the Gravity configuration, if set. Ignored if ``process_manager`` is + # ``supervisor``. + # memory_limit: + # Extra environment variables and their values to set when running the service. A dictionary where keys are the variable # names. # environment: {} @@ -100,6 +150,20 @@ gravity: # Extra arguments to pass to Celery command line. # extra_args: + # umask under which service should be executed + # umask: + + # Value of supervisor startsecs, systemd TimeoutStartSec + # start_timeout: 10 + + # Value of supervisor stopwaitsecs, systemd TimeoutStopSec + # stop_timeout: 10 + + # Memory limit (in GB). If the service exceeds the limit, it will be killed. Default is no limit or the value of the + # ``memory_limit`` setting at the top level of the Gravity configuration, if set. Ignored if ``process_manager`` is + # ``supervisor``. + # memory_limit: + # Extra environment variables and their values to set when running the service. A dictionary where keys are the variable # names. # environment: {} @@ -117,7 +181,8 @@ gravity: # port: 4002 # Routes file to monitor. - # Should be set to the same path as ``interactivetools_map`` in the ``galaxy:`` section. + # Should be set to the same path as ``interactivetools_map`` in the ``galaxy:`` section. This is ignored if + # ``interactivetools_map is set``. # sessions: database/interactivetools_map.sqlite # Include verbose messages in gx-it-proxy @@ -135,6 +200,20 @@ gravity: # This is an advanced option that is only needed when proxying to remote interactive tool container that cannot be reached through the local network. # reverse_proxy: false + # umask under which service should be executed + # umask: + + # Value of supervisor startsecs, systemd TimeoutStartSec + # start_timeout: 10 + + # Value of supervisor stopwaitsecs, systemd TimeoutStopSec + # stop_timeout: 10 + + # Memory limit (in GB). If the service exceeds the limit, it will be killed. Default is no limit or the value of the + # ``memory_limit`` setting at the top level of the Gravity configuration, if set. Ignored if ``process_manager`` is + # ``supervisor``. + # memory_limit: + # Extra environment variables and their values to set when running the service. A dictionary where keys are the variable # names. # environment: {} @@ -175,6 +254,72 @@ gravity: # Extra arguments to pass to tusd command line. # extra_args: + # umask under which service should be executed + # umask: + + # Value of supervisor startsecs, systemd TimeoutStartSec + # start_timeout: 10 + + # Value of supervisor stopwaitsecs, systemd TimeoutStopSec + # stop_timeout: 10 + + # Memory limit (in GB). If the service exceeds the limit, it will be killed. Default is no limit or the value of the + # ``memory_limit`` setting at the top level of the Gravity configuration, if set. Ignored if ``process_manager`` is + # ``supervisor``. + # memory_limit: + + # Extra environment variables and their values to set when running the service. A dictionary where keys are the variable + # names. + # environment: {} + + # Configuration for Galaxy Reports. + reports: + + # Enable Galaxy Reports server. + # enable: false + + # Path to reports.yml, relative to galaxy.yml if not absolute + # config_file: reports.yml + + # The socket to bind. A string of the form: ``HOST``, ``HOST:PORT``, ``unix:PATH``, ``fd://FD``. An IP is a valid HOST. + # bind: localhost:9001 + + # Controls the number of Galaxy Reports application processes Gunicorn will spawn. + # It is not generally necessary to increase this for the low-traffic Reports server. + # workers: 1 + + # Gunicorn workers silent for more than this many seconds are killed and restarted. + # Value is a positive number or 0. Setting it to 0 has the effect of infinite timeouts by disabling timeouts for all workers entirely. + # timeout: 300 + + # URL prefix to serve from. + # The corresponding nginx configuration is (replace and with the values from these options): + # + # location // { + # proxy_pass http:///; + # } + # + # If is a unix socket, you will need a ``:`` after the socket path but before the trailing slash like so: + # proxy_pass http://unix:/run/reports.sock:/; + # url_prefix: + + # Extra arguments to pass to Gunicorn command line. + # extra_args: + + # umask under which service should be executed + # umask: + + # Value of supervisor startsecs, systemd TimeoutStartSec + # start_timeout: 10 + + # Value of supervisor stopwaitsecs, systemd TimeoutStopSec + # stop_timeout: 10 + + # Memory limit (in GB). If the service exceeds the limit, it will be killed. Default is no limit or the value of the + # ``memory_limit`` setting at the top level of the Gravity configuration, if set. Ignored if ``process_manager`` is + # ``supervisor``. + # memory_limit: + # Extra environment variables and their values to set when running the service. A dictionary where keys are the variable # names. # environment: {} From 9976d9b42ebfa6179dad2985577d3390095e12b8 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Mon, 5 Dec 2022 12:42:27 -0500 Subject: [PATCH 6/7] Set gravity log dir for tests --- lib/galaxy_test/driver/driver_util.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/galaxy_test/driver/driver_util.py b/lib/galaxy_test/driver/driver_util.py index f90f562e69b1..5e6af4d10cf9 100644 --- a/lib/galaxy_test/driver/driver_util.py +++ b/lib/galaxy_test/driver/driver_util.py @@ -743,8 +743,10 @@ def launch_gravity(port, gxit_port=None, galaxy_config=None): galaxy_config["interactivetools_proxy_host"] = f"localhost:{gxit_port}" # Can't use in-memory celery broker, just fall back to sqlalchemy galaxy_config.update({"celery_conf": {"broker_url": None}}) + state_dir = tempfile.mkdtemp(suffix="state") config = { "gravity": { + "log_dir": os.path.join(state_dir, "log"), "gunicorn": {"bind": f"localhost:{port}", "preload": "false"}, "gx_it_proxy": { "enable": galaxy_config.get("interactivetools_enable", False), @@ -755,7 +757,6 @@ def launch_gravity(port, gxit_port=None, galaxy_config=None): }, "galaxy": galaxy_config, } - state_dir = tempfile.mkdtemp(suffix="state") with tempfile.NamedTemporaryFile("w", dir=state_dir, delete=False, suffix=".galaxy.yml") as config_fh: json.dump(config, config_fh) with tempfile.NamedTemporaryFile(delete=True) as socket: From fad0453e72da141ade6adfb787d91cef7af088fc Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Tue, 6 Dec 2022 10:53:21 -0500 Subject: [PATCH 7/7] Update gravity section of sample config --- lib/galaxy/config/sample/galaxy.yml.sample | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample index 9e5a3c58df5a..c7f1ea9cd4e8 100644 --- a/lib/galaxy/config/sample/galaxy.yml.sample +++ b/lib/galaxy/config/sample/galaxy.yml.sample @@ -29,7 +29,7 @@ gravity: # What command to write to the process manager configs # `gravity` (`galaxyctl exec `) is the default # `direct` (each service's actual command) is also supported. - # Valid options are: gravity, direct, exec + # Valid options are: gravity, direct # service_command_style: gravity # Use the process manager's *service instance* functionality for services that can run multiple instances. @@ -109,12 +109,18 @@ gravity: # Consumes less memory when multiple processes are configured. Default is ``false`` if using unicornherder, else ``true``. # preload: + # umask under which service should be executed + # umask: + # Value of supervisor startsecs, systemd TimeoutStartSec # start_timeout: 15 # Value of supervisor stopwaitsecs, systemd TimeoutStopSec # stop_timeout: 65 + # Amount of time to wait for a server to become alive when performing rolling restarts. + # restart_timeout: 300 + # Memory limit (in GB). If the service exceeds the limit, it will be killed. Default is no limit or the value of the # ``memory_limit`` setting at the top level of the Gravity configuration, if set. Ignored if ``process_manager`` is # ``supervisor``. @@ -240,14 +246,14 @@ gravity: # upload_dir: # Comma-separated string of enabled tusd hooks. - # + # # Leave at the default value to require authorization at upload creation time. # This means Galaxy's web process does not need to be running after creating the initial # upload request. - # + # # Set to empty string to disable all authorization. This means data can be uploaded (but not processed) # without the Galaxy web process being available. - # + # # You can find a list of available hooks at https://github.com/tus/tusd/blob/master/docs/hooks.md#list-of-available-hooks. # hooks_enabled_events: pre-create