diff --git a/docs/concepts/copying-task.md b/docs/concepts/copying-task.md index 2640e48d..b944dbb2 100644 --- a/docs/concepts/copying-task.md +++ b/docs/concepts/copying-task.md @@ -2,4 +2,61 @@ # Copying Task +While building your workflow, you might notice that some of your tasks resemble each other in many ways. + +Zrb allows you to copy a Task and modify a limited set of attributes. + +The following are the most commonly used methods when you copy a Task: + +- `copy(self)` +- `set_name(self, new_name: str)` +- `set_description(self, new_description: str)` +- `set_icon(self, new_icon: str)` +- `set_color(self, new_color: str)` +- `set_should_execute(self, should_execute: str)` +- `set_retry(self, new_retry: int)` +- `set_retry_interval(self, new_retry_interval: int)` +- `set_checking_interval(self, new_checking_retry_interval: int)` +- `insert_checker(self, *checkers: AnyTask)` +- `add_checker(self, *checkers: AnyTask)` +- `insert_upstream(self, *upstreams: AnyTask)` +- `add_upstream(self, *upstreams: AnyTask)` +- `insert_input(self, *inputs: AnyInput)` +- `add_input(self, *inputs: AnyInput)` +- `insert_env(self, *envs: Env)` +- `add_env(self, *envs: Env)` +- `insert_env_file(self, *env_files: EnvFile)` +- `add_env_file(self, *env_files: EnvFile)` + +Let's see the following example: + +```python +from zrb import runner, CmdTask, BoolInput + +dbt_run = CmdTask( + name='dbt-run', + cmd='dbt run' +) + +# Copying dbt run, make it skippable +skippable_dbt_run = dbt_run.copy() +skippable_dbt_run.add_input(BoolInput(name='dbt-run', default=True)) +skippable_dbt_run.set_should_execute('{{ input.dbt_run }}') + +# register dbt-run +runner.register(dbt_run) + +# Make dbt-test depends on skippable dbt run +dbt_test = CmdTask( + name='dbt-test', + cmd='dbt test', + upstreams=[skippable_dbt_run] +) + +``` + +# Next + +For more flexibility, you can extend [Task](extending-task.md) and [CmdTask](extending-cmd-task.md) + 🔖 [Table of Contents](../README.md) / [Concepts](README.md) diff --git a/docs/concepts/template-rendering.md b/docs/concepts/template-rendering.md index bd6d3ca5..b21b1f5c 100644 --- a/docs/concepts/template-rendering.md +++ b/docs/concepts/template-rendering.md @@ -12,7 +12,7 @@ Let's see some available objects in Zrb's Jinja template: - `time`: Python time module. - `util`: Zrb utilities. - `util.coalesce(value, *alternatives)`: Coalesce a value with the alternatives sequentially. An empty string is considered as a value. - - `util.coalesce_str(value, *alternatives)`: Coalesce a value with the altiernatives sequantially. An empty string is not considered as a value. + - `util.coalesce_str(value, *alternatives)`: Coalesce a value with the alternatives sequantially. An empty string is not considered as a value. - `util.to_camel-case(text)`: Returns a `camelCased` text. - `util.to_pascal_case(text)`: Returns a `PascalCased` text. - `util.to_kebab_case(text)`: Returns a `kebab-cased` text. @@ -29,11 +29,25 @@ Let's see some available objects in Zrb's Jinja template: - `task.get_input_map()`: Returning `input` dictionary. - `task.set_xcom(key, value)`: Returning an empty string after setting an XCom key. - `task.get_xcom(key)`: Getting an XCom value. + - `task.get_execution_id()`: Getting Execution ID # Input -Input has an attribute named `should_render` that defaults to `True`. This attribute makes Zrb renders Input's value as a Jinja template. +Input has an attribute named `should_render` that defaults to `True`. This attribute makes Zrb render Input's value as a Jinja template. + +The following objects are accessible from Input's value: + +- `datetime` +- `os` +- `platform` +- `time` +- `util` +- `input` (Only the preceding inputs's values are accessible) +- `task.get_execution_id()` + +Let's see an example. + ```python from zrb import runner, StrInput, CmdTask @@ -71,7 +85,21 @@ rendered-input 2024-01-16 08:25:27.325030 # Env -Env has an attribute named `should_render` that defaults to `True`. This attributes makes Zrb renders Env's `default` value as Jinja template. +Env has an attribute named `should_render` that defaults to `True`. This attribute makes Zrb render Env's `default` value as a Jinja template. + +The following objects are accessible from Env's value: + +- `datetime` +- `os` +- `platform` +- `time` +- `util` +- `input` +- `env` (Only the preceding Env's values are accessible) +- `task.get_execution_id()` + +Let's see an example. + ```python from zrb import runner, Env, CmdTask @@ -109,9 +137,298 @@ RENDERED_ENV crimson-metallum-07790 # EnvFile -EnvFile also has an attribute named `should_render` that defaults to `True`. +EnvFile has an attribute named `should_render` that defaults to `True`. This attribute makes Zrb render the environment variable's values in your as a Jinja template. + +The following objects are accessible from the environment variable's value: + +- `datetime` +- `os` +- `platform` +- `time` +- `util` +- `input` +- `env` (Only the preceding Env's values are accessible) +- `task.get_execution_id()` + +Let's see an example. + + +```bash +# file-name: a.env +NOT_RENDERED_ENV="{{ something }}" +``` + +```bash +# file-name: b.env +RENDRED_ENV="{{ task.get_execution_id() }}" +``` + +```python +from zrb import runner, EnvFile, CmdTask + +import os + +CURRENT_DIR = os.dirname(__file__) + +task = CmdTask( + name='task', + env_files=[ + EnvFile( + path=os.path.join(CURRENT_DIR, 'a.env'), + should_render=False + ), + EnvFile( + path=os.path.join(CURRENT_DIR, 'b.env') + should_render=True # The default value + ), + ], + cmd=[ + 'echo "NOT_RENDERED_ENV $NOT_RENDERED_ENV"', + 'echo "RENDERED_ENV $RENDERED_ENV"', + ] +) +runner.register(task) +``` + +```bash +zrb task +``` + +``` +NOT_RENDERED_ENV {{ something }} +RENDERED_ENV crimson-metallum-07790 +``` + +# Task Attributes + +All template objects are accessible from the following Task Properties. When in doubt, you can check on the properties annotation. Anything with `JinjaTemplate` are renderable. + +## BaseTask + +Zrb renders the following attributes as Jinja Template: + +- `should_execute` (`Union[bool, JinjaTemplate, Callable[..., bool]]`) + +Example: + +```python +from zrb import runner, BoolInput, Task + +task = Task( + name='task', + inputs=[ + BoolInput(name='should-execute', default=True) + ], + should_execute='{{ input.should_execute }}' +) +runner.register(task) +``` + +## CmdTask + +Zrb renders the following attributes as Jinja Template: + +- `cmd` (`CmdVal`) +- `cmd_path` (`CmdVal`) +- `should_execute` (`Union[bool, JinjaTemplate, Callable[..., bool]]`) + +Example: + +```python +from zrb import runner, BoolInput, StrInput, CmdTask + +task = CmdTask( + name='task', + inputs=[ + BoolInput(name='should-execute', default=True), + StrInput(name='name', default='World') + ], + should_execute='{{ input.should_execute }}', + cmd='echo "{{ input.name }}"' +) +runner.register(task) +``` + +## RemoteCmdTask + +Zrb renders the following attributes as Jinja Template: + +- `cmd` (`CmdVal`) +- `cmd_path` (`CmdVal`) +- `should_execute` (`Union[bool, JinjaTemplate, Callable[..., bool]]`) + +Furthermore, Zrb also renders [`remote_configs`](#remoteconfig) attributes as Jinja Template. + +Example: + +```python +from zrb import runner, BoolInput, StrInput, PasswordInput, RemoteCmdTask, RemoteConfig + +task = RemoteCmdTask( + name='task', + inputs=[ + BoolInput(name='should-execute', default=True), + StrInput(name='name', default='World'), + StrInput(name='server-1-host', default='stalchmst.com'), + StrInput(name='server-1-user', default='root'), + PasswordInput(name='server-1-pass'), + StrInput(name='server-2-host', default='contoso.com'), + StrInput(name='server-2-user', default='root'), + PasswordInput(name='server-2-pass') + ], + remote_configs=[ + RemoteConfig( + host='{{ input.server_1_host }}', + user='{{ input.server_1_user }}', + password='{{ input.server_1_pass }}' + ), + RemoteConfig( + host='{{ input.server_2_host }}' + user='{{ input.server_2_user }}', + password='{{ input.server_2_pass }}' + ), + ], + should_execute='{{ input.should_execute }}', + cmd='echo "{{ input.name }}" && uname -a' +) +runner.register(task) +``` + +## RsyncTask + +Zrb renders the following attributes as Jinja Template: + +- `src` (`JinjaTemplate`) +- `dst` (`JinjaTemplate`) + +Furthermore, Zrb also renders [`remote_configs`](#remoteconfig) attributes as Jinja Template. + +## RemoteConfig + +Zrb renders the following attributes as Jinja Template: + +- `host` (`JinjaTemplate`) +- `user` (`JinjaTemplate`) +- `password` (`JinjaTemplate`) +- `ssh_key` (`JinjaTemplate`) +- `port` (`JinjaTemplate`) +- `config_map` (`Mapping[str, JinjaTemplate]`) + + +## DockerComposeTask + +Zrb renders the following attributes as Jinja Template: + +- `compose_options` (`Mapping[JinjaTemplate, JinjaTemplate]`) +- `compose_flags` (`Iterable[JinjaTemplate]`) +- `compose_args` (`Iterable[JinjaTemplate]`) +- `should_execute` (`Union[bool, JinjaTemplate, Callable[..., bool]]`) +- `setup_cmd` (`CmdVal`) +- `setup_cmd_path` (`CmdVal`) + +Example: + +```python +from zrb import runner, BoolInput, IntInput, DockerComposeTask + +task = DockerComposeTask( + name='task', + inputs=[ + BoolInput(name='should-execute', default=True), + IntInput(name='uid', default=1000) + ], + should_execute='{{ input.should_execute }}', + compose_cmd='up', + compose_options={ + '-u': '{{ input.uid }}' + } +) +runner.register(task) +``` + +## ResourceMaker + +Zrb renders the following attributes as Jinja Template: + +- `template_path` (`JinjaTemplate`) +- `destination_path` (`JinjaTemplate`) +- `should_execute` (`Union[bool, JinjaTemplate, Callable[..., bool]]`) + +Furthermore, Zrb also renders the `replacements` attribute values as Jinja Template. + +Example: + +```python +from zrb import runner, BoolInput, StrInput, ResourceMaker +import os + +CURRENT_DIR = os.path.dirname(__file__) + +task = ResourceMaker( + name='task', + template_path=os.path.join(CURRENT_DIR, 'template'), + inputs=[ + StrInput(name='project-dir', default='.'), + StrInput(name='project-name', default='new_project') + ], + destination_path='{{ input.project_dir }}', + replacements={ + 'project_name': '{{ input.project_name }}' + } +) +runner.register(task) +``` + + +## Notifier + +Zrb renders the following attributes as Jinja Template: + +- `title` (`JinjaTemplate`) +- `message` (`JinjaTemplate`) + +Example: + +```python +from zrb import Runner, StrInput, Notifier + +task = Notifier( + name='task', + inputs=[ + StrInput(name='title', default='Notification'), + StrInput(name='message', default='Message'), + ], + title='{{ input.title }}', + message='{{ input.message }}', +) +runner.register(task) +``` + +## Checkers + +Zrb renders Checker attributes as Jinja Template. The detailed renderable attributes are as follows: -If EnvFile's `should_render` is `True`, Zrb will parse the environment values in your environment file as Jinja syntax. +- `TimeWatcher` + - `schedule` (`JinjaTemplate`) + - `should_execute` (`Union[bool, JinjaTemplate, Callable[..., bool]]`) +- `HTTPChecker` + - `host` (`JinjaTemplate`) + - `port` (`Union[JinjaTemplate, int]`) + - `timeout` (`Union[JinjaTemplate, int]`) + - `should_execute` (`Union[bool, JinjaTemplate, Callable[..., bool]]`) +- `PortChecker` + - `host` (`JinjaTemplate`) + - `port` (`Union[JinjaTemplate, int]`) + - `timeout` (`Union[JinjaTemplate, int]`) + - `should_execute` (`Union[bool, JinjaTemplate, Callable[..., bool]]`) +- `PathChecker` + - `path` (`JinjaTemplate`) + - `ignored_path` (`Union[Iterable[JinjaTemplate], JinjaTemplate]`) + - `should_execute` (`Union[bool, JinjaTemplate, Callable[..., bool]]`) +- `PathWatcher` + - `path` (`JinjaTemplate`) + - `ignored_path` (`Union[Iterable[JinjaTemplate], JinjaTemplate]`) + - `should_execute` (`Union[bool, JinjaTemplate, Callable[..., bool]]`) 🔖 [Table of Contents](../README.md) / [Concepts](README.md) diff --git a/docs/getting-started.md b/docs/getting-started.md index dd2f063a..e40e1e36 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -284,60 +284,17 @@ In the rest of this section, you will learn about Zrb project and how to make yo

-Zrb allows you to isolate your work by putting them into multiple Zrb projects. +At its basic, a Project is a directory containing a single file named `zrb_init.py`. This simple setup is already sufficient for a simple hello-world project. -At its basic, a project is a directory containing a single file named `zrb_init.py`. This simple setup is already sufficient for a simple hello-world project. - -However, to make something more than a simple hello-world, you better use `zrb project create` command. - -```bash -zrb project create --project-dir my-project --project-name my-project -``` - -Once invoked, you will see a project named `my-project` under your current directory. Let's see what this project looks like: +You can create a Project by invoking the following command: ```bash +mkdir my-project cd my-project -ls -al +touch zrb_init.py ``` -``` -total 52 -drwxr-xr-x 6 gofrendi gofrendi 4096 Nov 12 07:52 . -drwxr-xr-x 14 gofrendi gofrendi 4096 Nov 12 07:52 .. -drwxr-xr-x 7 gofrendi gofrendi 4096 Nov 12 07:52 .git -drwxr-xr-x 3 gofrendi gofrendi 4096 Nov 12 07:52 .github --rw-r--r-- 1 gofrendi gofrendi 27 Nov 12 07:52 .gitignore --rw-r--r-- 1 gofrendi gofrendi 7 Nov 12 07:52 .python-version --rw-r--r-- 1 gofrendi gofrendi 1937 Nov 12 07:52 README.md -drwxr-xr-x 3 gofrendi gofrendi 4096 Nov 12 07:52 _automate --rwxr-xr-x 1 gofrendi gofrendi 1507 Nov 12 07:52 project.sh --rw-r--r-- 1 gofrendi gofrendi 13 Nov 12 07:52 requirements.txt -drwxr-xr-x 2 gofrendi gofrendi 4096 Nov 12 07:52 src --rw-r--r-- 1 gofrendi gofrendi 118 Nov 12 07:52 template.env --rw-r--r-- 1 gofrendi gofrendi 54 Nov 12 07:52 zrb_init.py -``` - -Every Zrb project has a file named `zrb_init.py` under the top-level directory. This file is your entry point to define your Task definitions. - -By convention, a project usually contains other two sub-directories: - -- ___automate__: This folder contains all your automation scripts and task definitions -- __src__: This folder contains all your resources like Docker compose file, helm charts, and source code. - -Moreover, Zrb provides some built-in Tasks under `project` Task Group. As always, you can invoke `zrb project` to see those tasks. - -## Using `project.sh` - -When you create a project using `zrb project create` command, you will find a file named `project.sh`. This script file helps you to load the virtual environment, install requirements, and activate shell completion. - -To use the script, you need to invoke the following command: - -```bash -source project.sh -``` - -Anytime you start working on your project, you should load `project.sh`. +For a more sophisticated way to create a Project, please visit [the project section](concepts/project.md) # Creating A Task @@ -445,502 +402,6 @@ The following properties are usually available: - __runner__: Only available in `@python_task`. The valid value is `zrb.runner`. -## Task Dependencies - -
- - - - -

- - Followers are like shadows: bigger in the spotlight. - -

-
- -There are two ways to define task dependencies in Zrb. - -- Using shift-right operator (i.e., `>>`). -- Using `upstreams` parameter. - -By defining dependencies, you can ensure that Zrb will wait for your upstreams to be ready before proceeding with the main task. - -You can use `>>` operator as follows: - -```python -task_1 = CmdTask(name='task-1') -task_2 = CmdTask(name='task-2') -task_3 = CmdTask(name='task-3') -task_4 = CmdTask(name='task-4') -task_5 = CmdTask(name='task-5') -task_6 = CmdTask(name='task-6') - -task_1 >> Parallel(task_2, task_3) >> Parallel(task_4, task_5) >> task_6 -``` - -Or you can use `upstreams` parameter as follows: - -```python -task_1 = CmdTask(name='task-1') -task_2 = CmdTask(name='task-2', upstreams=[task_1]) -task_3 = CmdTask(name='task-3', upstreams=[task_1]) -task_4 = CmdTask(name='task-4', upstreams=[task_2, task_3]) -task_5 = CmdTask(name='task-5', upstreams=[task_2, task_3]) -task_6 = CmdTask(name='task-6', upstreams=[task_4, task_5]) -``` - -## Task Inputs - -
- -

- - Input: where your program politely asks, 'What's the magic word? - -

-
- - -You can define task inputs using `StrInput`, `BoolInput`, `ChoiceInput`, `FloatInput`, `IntInput`, or `PasswordInput`. -To create an input, you need to provide some parameters: - -- __name__: The name of the input. By convention, this should be kebab-cased (required). -- __default__: The default value of the input (optional, default: `None`). -- __should_render__: Whether the input should be rendered as Jinja template or not (optional, default: `True`). - -For example, here you have an input named `message` with `Hello World` as the default value: - -```python -from zrb import StrInput - -message = StrInput(name='message', default='Hello World') -``` - -When you run a task with task inputs, Zrb will prompt you to override the input values. You can press `enter` if you want to use the default values. - -### Using Task Inputs on Task Class - -To access the values of your inputs in your Task Properties, you can use Jinja template `{{ input.input_name }}`. Notice that you should use `snake_case` instead of `kebab-case` to refer to the input. Let's see the following example: - -```python -from zrb import runner, CmdTask, StrInput - -hello_cmd = CmdTask( - name='hello-cmd', - inputs=[ - StrInput(name='your-name', default='World') - ], - # Notice, we use {{input.your_name}} not {{input.your-name}} !!! - cmd='echo Hello {{input.your_name}}' -) -runner.register(hello_cmd) -``` - -You can then run the task by invoking: - -```bash -zrb hello-cmd -# or -zrb hello-cmd --your-name "John Wick" -``` - -### Using Task Inputs on `@python_task` Decorator - -As for `@python_task`, you can use `kwargs` dictionary to get the input. - -```python -from zrb import runner, python_task, StrInput - -@python_task( - name='hello-py', - inputs=[ - StrInput(name='your-name', default='World') - ], - runner=runner -) -def hello_py(*args, **kwargs): - # Notice, we use `your_name` instead of `your-name` !!! - name = kwargs.get('your_name') - return f'Hello {name}' -``` - - -You can then run the task by invoking: - -```bash -zrb hello-py -# or -zrb hello-py --your-name "John Wick" -``` - -## Task Environments - -
- -

- - Save the Earth. It's the only planet with chocolate! - -

-
- -Aside from input, you can also define the `Task`'s environment variables using `Env` and `EnvFile`. - -### Env - -You can use `Env` to define a single environment variable for your Tasks. Typically, a Task could take multiple `Env`. - -To create an `Env`, you need to provide some parameters: - -- __name__: Name of the environment variable (required). -- __os_name__: Name of OS environment (optional, default=`None`) - - if set to `None`, Zrb will link the environment variable to the OS environment. - - if set to an empty string (i.e., `''`), Zrb will not link the environment variable to the OS's environment. - - if set to a non-empty string, Zrb will link the environment variable to the OS's environment corresponding to this value. -- __default__: Default value of the environment variable (optional, default: `None`). -- __should_render__: Whether the environment variable should be rendered as a Jinja template (optional, default: `True`). - - -```python -from zrb import Env - -env = Env(name='MESSAGE') -``` - -### EnvFile - -
- -

- - An island is just a sea's attempt at a mountain peak joke. - -

-
- -`EnvFile` loads an environment file and uses its values as Task's environment variables. Typically a Task could take multiple `EnvFile`. - -To create an `EnvFile`, you need to provide some parameters: - -- __env_file__: Name of the environment file (required). -- __prefix__: Custom prefix for environment's os_name (optional, default=`None`) -- __should_render__: Whether the environment variable should be rendered as a Jinja template (optional, default: `True`). - -```python -from zrb import EnvFile -import os - -PROJECT_ENV = os.path.join(os.path.dirname(__file__), 'project.env') -env_file = EnvFile(path=PROJECT_ENV) -``` - -### Using Env and EnvFile - -To use `EnvFile` in your tasks. Let's first create an environment file named `project.env`: - -```bash -# file-name: project.env -SERVER_HOST=localhost -``` - -### Using Env and EnvFile on Task Class - -To access the values of your inputs from your `CmdTask`, you can use Jinja template `{{ env.ENV_NAME }}`. - -```python -from zrb import runner, CmdTask, Env, EnvFile -import os - -PROJECT_ENV = os.path.join(os.path.dirname(__file__), 'project.env') - -hello_cmd = CmdTask( - name='hello-cmd', - envs=[ - Env(name='MESSAGE', default='Hello world'), - ], - env_files=[ - EnvFile(path=PROJECT_ENV) - ], - cmd=[ - 'echo Message: {{env.MESSAGE}}', - 'echo Host: {{env.SERVER_HOST}}', - ] -) -runner.register(hello_cmd) -``` - -You can then run the task by invoking: - -```bash -zrb hello-cmd -``` - -It will give you the following results: - -``` -Message: Hello world -Host: localhost -``` - -### Using Env and EnvFile on `@python_task` Decorator - -As for `@python_task`, you cannot use `os.getenv` to access the task's environment. Instead, you should get the `task` instance from the `kwargs` argument and invoke `task.get_env_map()`. - -```python -from zrb import runner, AnyTask, python_task, Env, EnvFile -import os - -PROJECT_ENV = os.path.join(os.path.dirname(__file__), 'project.env') - - -@python_task( - name='hello-py', - envs=[ - Env(name='MESSAGE', default='Hello world'), - ], - env_files=[ - EnvFile(path=PROJECT_ENV) - ], - runner=runner -) -def hello_py(*args, **kwargs): - task: AnyTask = kwargs.get('_task') - env_map = task.get_env_map() - message = env_map.get('MESSAGE') - server_host = env_map.get('SERVER_HOST') - return '\n'.join([ - f'Message: {message}', - f'Host: {server_host}' - ]) -``` - -You can then run the task by invoking: - -```bash -zrb hello-cmd -``` - -It will give you the following results: - -``` -Message: Hello world -Host: localhost -``` - -## Environment Cascading - -
- -

- - Cascading: Nature's way of saying, 'Let's take this step by step, but faster!' - -

-
- -Zrb has a feature named environment-cascading. In short, it can help you to switch between `DEV`, `PROD`, or `STAGING` based on `ZRB_ENV` value. - -For example, suppose we have the following task: - -```python -show_db_url = Cmdtask( - name='show-db-url', - envs=[ - Env(name='DB_URL') - ], - cmd='echo {{ env.DB_URL }}' -) -runner.register(show_db_url) -``` - -The task is doing a simple job, showing the value of `DB_URL`. - -Now let's consider the following environment variables: - -```bash -export DB_URL=postgresql://root:toor@localhost -export PROD_DB_URL=postgresql://prod-user:somePassword@db.my-company.com -``` - -As expected, when you run `zrb show-db-url`, you will get the value of `DB_URL` (i.e., `postgresql://root:toor@localhost`) - -__Using PROD Environment__ - -Now, let's set `ZRB_ENV` to `PROD`. - -```bash -export ZRB_ENV=PROD -zrb show-db-url -``` - -You will see Zrb automatically uses the value of `PROD_DB_URL` (i.e., `postgresql://prod-user:somePassword@db.my-company.com`) - -__Using DEV Environment__ - -Let's try it again with `DEV` environment - -```bash -export ZRB_ENV=DEV -zrb show-db-url -``` - -Now, since Zrb cannot find `DB_DB_URL`, it will use the `DB_URL` instead (i.e., `postgresql://prod-user:somePassword@db.my-company.com`) - -Using this behavior, you can work on multiple environments with the same codebase. - - -## Execution ID - -
- -

- - Sharing a ticket is like sharing a dessert; everyone's happy until it's their turn to pay. - -

-
- -In Zrb, a Task and all its upstreams will share the same Execution ID. -To get the Execution ID, you can use the `get_execution_id` method or `$_ZRB_EXECUTION_ID`, depending on whether you use a TaskClass or `@python_task` decorator. - -Let's see how we can get the Execution ID on different tasks: - -```python -from zrb import runner, Parallel, CmdTask, Task, python_task - -hello_cmd = CmdTask( - name='hello-cmd', - cmd='echo "Execution ID: $_ZRB_EXECUTION_ID"' -) - -@python_task( - name='hello-py' -) -def hello_py(*args, **kwargs): - task = kwargs.get('_task') - task.print_out(f'Execution ID: {task.get_execution_id()}') - -hello = Task( - name='hello', - run=lambda *args, **kwargs: kwargs.get('_task').get_execution_id() -) - -Parallel(hello_cmd, hello_py) >> hello -runner.register(hello) -``` - -You will find that `hello-cmd`, `hello-py`, and `hello` share the same Execution ID. - -You can use ExecutionID for many cases, especially those related to Cross Task Communication (XCom). - -## XCom (Cross Task Communication) - -
- -

- - Remember when phones were dumb and people were smart? Good times. - -

-
- -All instances of BaseTask share a global `xcom` dictionary. You can think of `xcom` as in-memory key-value storage. - -The structure of `xcom` dictionary is as follows: - -```python -__xcom: Mapping[str, Mapping[str, str]] = { - 'execution-id-1': { - 'key-1': 'value-1', - 'key-2': 'value-2' - }, - 'execution-id-2': { - 'key-1': 'value-1', - 'key-2': 'value-2' - } -} -``` - -To set and get value from `xcom`, you can use `set_xcom` and `get_xcom` method. Zrb automatically handle `execution-id` so that you can focus on xcom's key and value. - -Let's see the following example: - -```python -from zrb import runner, Parallel, CmdTask, python_task, Task - -set_xcom_cmd = CmdTask( - name='set-xcom-cmd', - cmd='echo "hi{{task.set_xcom("one", "ichi")}}"' -) - -@python_task( - name='set-xcom-py' -) -def set_xcom_py(*args, **kwargs): - task: Task = kwargs.get('_task') - task.set_xcom('two', 'ni') - - -get_xcom_cmd = CmdTask( - name='get-xcom-cmd', - cmd=[ - 'echo {{task.get_xcom("one")}}', - 'echo {{task.get_xcom("two")}}', - ] -) - -@python_task( - name='get-xcom-py' -) -def get_xcom_py(*args, **kwargs): - task: Task = kwargs.get('_task') - task.print_out(task.get_xcom("one")) - task.print_out(task.get_xcom("two")) - - -test_xcom = Task(name='test-xcom') -Parallel(set_xcom_cmd, set_xcom_py) >> Parallel(get_xcom_cmd, get_xcom_py) >> test_xcom -runner.register(test_xcom) -``` - -The example shows that `set-xcom-cmd` and `set-xcom-py` set XCom values `one` and `two`, respectively. - -On the other hand, `get-xcom-cmd` and `get-xcom-py` fetch the values and print them. - -Furthermore, every Zrb Task has its return values saved as `__xcom['execution-id']['task-name']`. To have a better understanding, let's see the following example: - -```python -from zrb import runner, Parallel, CmdTask, python_task - -hello_cmd = CmdTask( - name='hello-cmd', - cmd='echo hello-cmd', -) - -@python_task( - name='hello-py' -) -def hello_py(*args, **kwargs): - return 'hello-py' - -hello = CmdTask( - name='hello', - cmd=[ - 'echo {{task.get_xcom("hello-cmd")}}', - 'echo {{task.get_xcom("hello-py")}}', - ], -) - -Parallel(hello_cmd, hello_py) >> hello -runner.register(hello) -``` - -With XCom, you can easily share your data across your tasks. - -Now, since you have already see the basic concepts, let's see some examples. - - ## Basic Example
diff --git a/project.sh b/project.sh index afdd29a6..d3664f14 100755 --- a/project.sh +++ b/project.sh @@ -25,9 +25,15 @@ reload() { if [ -z "$PROJECT_AUTO_INSTALL_PIP" ] || [ "$PROJECT_AUTO_INSTALL_PIP" = 1 ] then - echo '🤖 Install requirements' - pip install --upgrade pip - pip install -r "${PROJECT_DIR}/requirements.txt" + echo '🤖 Checking .venv and requirements.txt timestamp' + _VENV_TIMESTAMP=$(find .venv -type d -exec stat -c %Y {} \; | sort -n | tail -n 1) + _REQUIREMENTS_TIMESTAMP=$(stat -c %Y requirements.txt) + if [ "$_VENV_TIMESTAMP" -lt "$_REQUIREMENTS_TIMESTAMP" ] + then + echo '🤖 Install requirements' + pip install --upgrade pip + pip install -r "${PROJECT_DIR}/requirements.txt" + fi fi echo '🤖 Install zrb as symlink' diff --git a/src/zrb/builtin/generator/project/template/project.sh b/src/zrb/builtin/generator/project/template/project.sh index 209a032d..bf2ef92f 100755 --- a/src/zrb/builtin/generator/project/template/project.sh +++ b/src/zrb/builtin/generator/project/template/project.sh @@ -31,9 +31,15 @@ reload() { if [ -z "$PROJECT_AUTO_INSTALL_PIP" ] || [ "$PROJECT_AUTO_INSTALL_PIP" = 1 ] then - echo '🤖 Install requirements' - pip install --upgrade pip - pip install -r "${PROJECT_DIR}/requirements.txt" + echo '🤖 Checking .venv and requirements.txt timestamp' + _VENV_TIMESTAMP=$(find .venv -type d -exec stat -c %Y {} \; | sort -n | tail -n 1) + _REQUIREMENTS_TIMESTAMP=$(stat -c %Y requirements.txt) + if [ "$_VENV_TIMESTAMP" -lt "$_REQUIREMENTS_TIMESTAMP" ] + then + echo '🤖 Install requirements' + pip install --upgrade pip + pip install -r "${PROJECT_DIR}/requirements.txt" + fi fi _CURRENT_SHELL=$(ps -p $$ | awk 'NR==2 {print $4}') diff --git a/src/zrb/task/any_task.py b/src/zrb/task/any_task.py index 4dc273d5..d1255c61 100644 --- a/src/zrb/task/any_task.py +++ b/src/zrb/task/any_task.py @@ -580,7 +580,7 @@ def set_color(self, new_color: str): @abstractmethod def set_should_execute( - self, should_execute: Union[bool, str, Callable[..., bool]] + self, should_execute: Union[bool, JinjaTemplate, Callable[..., bool]] ): ''' Determines whether the task should execute. diff --git a/src/zrb/task/base_remote_cmd_task.py b/src/zrb/task/base_remote_cmd_task.py index c628d41d..08f0722c 100644 --- a/src/zrb/task/base_remote_cmd_task.py +++ b/src/zrb/task/base_remote_cmd_task.py @@ -1,5 +1,5 @@ from zrb.helper.typing import ( - Any, Callable, Iterable, Mapping, Optional, Union, TypeVar + Any, Callable, Iterable, Mapping, Optional, Union, TypeVar, JinjaTemplate ) from zrb.helper.typecheck import typechecked from zrb.helper.util import to_snake_case @@ -31,12 +31,12 @@ class RemoteConfig: def __init__( self, - host: str, - user: str = '', - password: str = '', - ssh_key: str = '', - port: int = 22, - config_map: Optional[Mapping[str, str]] = None + host: JinjaTemplate, + user: JinjaTemplate = '', + password: JinjaTemplate = '', + ssh_key: JinjaTemplate = '', + port: Union[int, JinjaTemplate] = 22, + config_map: Optional[Mapping[str, JinjaTemplate]] = None ): self.host = host self.user = user diff --git a/src/zrb/task/base_task/component/base_task_model.py b/src/zrb/task/base_task/component/base_task_model.py index 7626e910..9fb1c1b1 100644 --- a/src/zrb/task/base_task/component/base_task_model.py +++ b/src/zrb/task/base_task/component/base_task_model.py @@ -1,5 +1,5 @@ from zrb.helper.typing import ( - Any, Callable, Iterable, List, Mapping, Optional, Union + Any, Callable, Iterable, List, Mapping, Optional, Union, JinjaTemplate ) from zrb.helper.typecheck import typechecked from zrb.config.config import show_time, logging_level @@ -54,7 +54,7 @@ def __init__( on_ready: Optional[OnReady] = None, on_retry: Optional[OnRetry] = None, on_failed: Optional[OnFailed] = None, - should_execute: Union[bool, str, Callable[..., bool]] = True, + should_execute: Union[bool, JinjaTemplate, Callable[..., bool]] = True, return_upstream_result: bool = False ): self.__rjust_full_cli_name: Optional[str] = None diff --git a/src/zrb/task/base_task/component/common_task_model.py b/src/zrb/task/base_task/component/common_task_model.py index ed55d434..408e55d1 100644 --- a/src/zrb/task/base_task/component/common_task_model.py +++ b/src/zrb/task/base_task/component/common_task_model.py @@ -1,5 +1,5 @@ from zrb.helper.typing import ( - Any, Callable, Iterable, List, Mapping, Optional, Union + Any, Callable, Iterable, List, Mapping, Optional, Union, JinjaTemplate ) from zrb.helper.typecheck import typechecked from zrb.task.any_task_event_handler import ( @@ -44,7 +44,7 @@ def __init__( on_ready: Optional[OnReady] = None, on_retry: Optional[OnRetry] = None, on_failed: Optional[OnFailed] = None, - should_execute: Union[bool, str, Callable[..., bool]] = True, + should_execute: Union[bool, JinjaTemplate, Callable[..., bool]] = True, return_upstream_result: bool = False ): self._name = name @@ -130,7 +130,7 @@ def set_retry(self, new_retry: int): self._retry = new_retry def set_should_execute( - self, should_execute: Union[bool, str, Callable[..., bool]] + self, should_execute: Union[bool, JinjaTemplate, Callable[..., bool]] ): self._should_execute = should_execute diff --git a/src/zrb/task/cmd_task.py b/src/zrb/task/cmd_task.py index ecd4d22a..dcfb9a09 100644 --- a/src/zrb/task/cmd_task.py +++ b/src/zrb/task/cmd_task.py @@ -1,5 +1,5 @@ from zrb.helper.typing import ( - Any, Callable, Iterable, List, Optional, Union, TypeVar + Any, Callable, Iterable, List, Optional, Union, TypeVar, JinjaTemplate ) from zrb.helper.typecheck import typechecked from zrb.helper.string.conversion import to_variable_name @@ -39,7 +39,11 @@ def _reset_stty(): _has_stty = False -CmdVal = Union[str, Iterable[str], Callable[..., Union[Iterable[str], str]]] +CmdVal = Union[ + JinjaTemplate, + Iterable[JinjaTemplate], + Callable[..., Union[Iterable[JinjaTemplate], JinjaTemplate]] +] TCmdTask = TypeVar('TCmdTask', bound='CmdTask') @@ -357,7 +361,9 @@ def __get_rendered_cmd_path( for cmd_path_str in cmd_path ]) - def __get_rendered_cmd(self, cmd: Union[str, Iterable[str]]) -> str: + def __get_rendered_cmd( + self, cmd: Union[JinjaTemplate, Iterable[JinjaTemplate]] + ) -> str: if isinstance(cmd, str): return self.render_str(cmd) return self.render_str('\n'.join(list(cmd))) diff --git a/src/zrb/task/docker_compose_task.py b/src/zrb/task/docker_compose_task.py index 940f420b..45086137 100644 --- a/src/zrb/task/docker_compose_task.py +++ b/src/zrb/task/docker_compose_task.py @@ -1,5 +1,6 @@ from zrb.helper.typing import ( - Any, Callable, Iterable, List, Mapping, Optional, Union, TypeVar + Any, Callable, Iterable, List, Mapping, Optional, Union, TypeVar, + JinjaTemplate ) from zrb.helper.typecheck import typechecked from zrb.task.cmd_task import CmdTask, CmdResult, CmdVal @@ -77,9 +78,9 @@ def __init__( compose_service_configs: Mapping[str, ServiceConfig] = {}, compose_file: Optional[str] = None, compose_cmd: str = 'up', - compose_options: Mapping[str, str] = {}, - compose_flags: Iterable[str] = [], - compose_args: Iterable[str] = [], + compose_options: Mapping[JinjaTemplate, JinjaTemplate] = {}, + compose_flags: Iterable[JinjaTemplate] = [], + compose_args: Iterable[JinjaTemplate] = [], compose_env_prefix: str = '', setup_cmd: CmdVal = '', setup_cmd_path: CmdVal = '', diff --git a/src/zrb/task/http_checker.py b/src/zrb/task/http_checker.py index 6eb3b3ae..1cc92f3b 100644 --- a/src/zrb/task/http_checker.py +++ b/src/zrb/task/http_checker.py @@ -1,4 +1,6 @@ -from zrb.helper.typing import Any, Callable, Iterable, Optional, Union, TypeVar +from zrb.helper.typing import ( + Any, Callable, Iterable, Optional, Union, TypeVar, JinjaTemplate +) from zrb.helper.typecheck import typechecked from zrb.task.checker import Checker from http.client import HTTPConnection, HTTPSConnection @@ -58,9 +60,9 @@ def __init__( icon: Optional[str] = None, color: Optional[str] = None, description: str = '', - host: str = 'localhost', - port: Union[int, str] = 80, - timeout: Union[int, str] = 5, + host: JinjaTemplate = 'localhost', + port: Union[int, JinjaTemplate] = 80, + timeout: Union[int, JinjaTemplate] = 5, method: str = 'HEAD', url: str = '/', is_https: Union[bool, str] = False, @@ -75,7 +77,7 @@ def __init__( checking_interval: Union[int, float] = 0.1, progress_interval: Union[int, float] = 5, expected_result: bool = True, - should_execute: Union[bool, str, Callable[..., bool]] = True + should_execute: Union[bool, JinjaTemplate, Callable[..., bool]] = True ): Checker.__init__( self, diff --git a/src/zrb/task/notifier.py b/src/zrb/task/notifier.py index 6aaffe40..602da967 100644 --- a/src/zrb/task/notifier.py +++ b/src/zrb/task/notifier.py @@ -1,4 +1,6 @@ -from zrb.helper.typing import Any, Callable, Iterable, Optional, Union +from zrb.helper.typing import ( + Any, Callable, Iterable, Optional, Union, JinjaTemplate +) from zrb.helper.typecheck import typechecked from zrb.task.base_task.base_task import BaseTask from zrb.task.any_task import AnyTask @@ -33,8 +35,8 @@ def __init__( icon: Optional[str] = None, color: Optional[str] = None, description: str = '', - title: str = '', - message: str = '', + title: JinjaTemplate = '', + message: JinjaTemplate = '', show_toast: bool = True, show_stdout: bool = True, upstreams: Iterable[AnyTask] = [], diff --git a/src/zrb/task/path_checker.py b/src/zrb/task/path_checker.py index 6e04e273..be41943f 100644 --- a/src/zrb/task/path_checker.py +++ b/src/zrb/task/path_checker.py @@ -1,5 +1,5 @@ from zrb.helper.typing import ( - Any, Callable, Iterable, List, Optional, Union, TypeVar + Any, Callable, Iterable, List, Optional, Union, TypeVar, JinjaTemplate ) from zrb.helper.typecheck import typechecked from zrb.helper.file.match import get_file_names @@ -37,12 +37,12 @@ def __init__( on_ready: Optional[OnReady] = None, on_retry: Optional[OnRetry] = None, on_failed: Optional[OnFailed] = None, - path: str = '', - ignored_path: Union[str, Iterable[str]] = [], + path: JinjaTemplate = '', + ignored_path: Union[JinjaTemplate, Iterable[JinjaTemplate]] = [], checking_interval: Union[int, float] = 0.1, progress_interval: Union[int, float] = 5, expected_result: bool = True, - should_execute: Union[bool, str, Callable[..., bool]] = True + should_execute: Union[bool, JinjaTemplate, Callable[..., bool]] = True ): Checker.__init__( self, diff --git a/src/zrb/task/path_watcher.py b/src/zrb/task/path_watcher.py index 00058833..45512b6a 100644 --- a/src/zrb/task/path_watcher.py +++ b/src/zrb/task/path_watcher.py @@ -1,5 +1,6 @@ from zrb.helper.typing import ( - Any, Callable, Iterable, List, Mapping, Optional, Union, TypeVar + Any, Callable, Iterable, List, Mapping, Optional, Union, TypeVar, + JinjaTemplate ) from zrb.helper.typecheck import typechecked from zrb.helper.file.match import get_file_names @@ -49,14 +50,14 @@ def __init__( on_ready: Optional[OnReady] = None, on_retry: Optional[OnRetry] = None, on_failed: Optional[OnFailed] = None, - path: str = '', - ignored_path: Union[str, Iterable[str]] = [], + path: JinjaTemplate = '', + ignored_path: Union[JinjaTemplate, Iterable[JinjaTemplate]] = [], checking_interval: Union[int, float] = 0.1, progress_interval: Union[int, float] = 30, watch_new_files: bool = True, watch_modified_files: bool = True, watch_deleted_files: bool = True, - should_execute: Union[bool, str, Callable[..., bool]] = True + should_execute: Union[bool, JinjaTemplate, Callable[..., bool]] = True ): Checker.__init__( self, diff --git a/src/zrb/task/port_checker.py b/src/zrb/task/port_checker.py index 04e5063c..5a5493d9 100644 --- a/src/zrb/task/port_checker.py +++ b/src/zrb/task/port_checker.py @@ -1,4 +1,6 @@ -from zrb.helper.typing import Any, Callable, Iterable, Optional, Union, TypeVar +from zrb.helper.typing import ( + Any, Callable, Iterable, Optional, Union, TypeVar, JinjaTemplate +) from zrb.task.checker import Checker from zrb.helper.typecheck import typechecked from zrb.task.any_task import AnyTask @@ -37,9 +39,9 @@ def __init__( icon: Optional[str] = None, color: Optional[str] = None, description: str = '', - host: str = 'localhost', - port: Union[int, str] = 80, - timeout: Union[int, str] = 5, + host: JinjaTemplate = 'localhost', + port: Union[int, JinjaTemplate] = 80, + timeout: Union[int, JinjaTemplate] = 5, upstreams: Iterable[AnyTask] = [], on_triggered: Optional[OnTriggered] = None, on_waiting: Optional[OnWaiting] = None, diff --git a/src/zrb/task/resource_maker.py b/src/zrb/task/resource_maker.py index 7e8a4441..d521158d 100644 --- a/src/zrb/task/resource_maker.py +++ b/src/zrb/task/resource_maker.py @@ -1,5 +1,5 @@ from zrb.helper.typing import ( - Any, Callable, Iterable, Mapping, Optional, Union, TypeVar + Any, Callable, Iterable, Mapping, Optional, Union, TypeVar, JinjaTemplate ) from zrb.helper.typecheck import typechecked from zrb.task.base_task.base_task import BaseTask @@ -17,7 +17,7 @@ to_human_readable, to_capitalized_human_readable ) -Replacement = Mapping[str, str] +Replacement = Mapping[str, JinjaTemplate] ReplacementMutator = Callable[ [AnyTask, Replacement], Replacement @@ -31,8 +31,8 @@ class ResourceMaker(BaseTask): def __init__( self, name: str, - template_path: str, - destination_path: str, + template_path: JinjaTemplate, + destination_path: JinjaTemplate, replacements: Replacement = {}, replacement_mutator: Optional[ReplacementMutator] = None, excludes: Iterable[str] = [], @@ -51,7 +51,7 @@ def __init__( on_ready: Optional[OnReady] = None, on_retry: Optional[OnRetry] = None, on_failed: Optional[OnFailed] = None, - should_execute: Union[bool, str, Callable[..., bool]] = True, + should_execute: Union[bool, JinjaTemplate, Callable[..., bool]] = True, skip_parsing: Optional[Iterable[str]] = None ): BaseTask.__init__( diff --git a/src/zrb/task/rsync_task.py b/src/zrb/task/rsync_task.py index 3304a86f..7335e05e 100644 --- a/src/zrb/task/rsync_task.py +++ b/src/zrb/task/rsync_task.py @@ -1,4 +1,6 @@ -from zrb.helper.typing import Any, Callable, Iterable, Optional, Union +from zrb.helper.typing import ( + Any, Callable, Iterable, Optional, Union, JinjaTemplate +) from zrb.helper.typecheck import typechecked from zrb.task.any_task import AnyTask from zrb.task.any_task_event_handler import ( @@ -37,8 +39,8 @@ def __init__( self, name: str, remote_configs: Iterable[RemoteConfig], - src: str, - dst: str, + src: JinjaTemplate, + dst: JinjaTemplate, is_remote_src: bool = False, is_remote_dst: bool = True, group: Optional[Group] = None, diff --git a/src/zrb/task/time_watcher.py b/src/zrb/task/time_watcher.py index 64c47292..5b9dbd3a 100644 --- a/src/zrb/task/time_watcher.py +++ b/src/zrb/task/time_watcher.py @@ -1,5 +1,5 @@ from zrb.helper.typing import ( - Any, Callable, Iterable, Optional, Union, TypeVar + Any, Callable, Iterable, Optional, Union, TypeVar, JinjaTemplate ) from zrb.helper.typecheck import typechecked from zrb.task.checker import Checker @@ -45,10 +45,10 @@ def __init__( on_ready: Optional[OnReady] = None, on_retry: Optional[OnRetry] = None, on_failed: Optional[OnFailed] = None, - schedule: str = '', + schedule: JinjaTemplate = '', checking_interval: Union[int, float] = 1, progress_interval: Union[int, float] = 30, - should_execute: Union[bool, str, Callable[..., bool]] = True + should_execute: Union[bool, JinjaTemplate, Callable[..., bool]] = True ): Checker.__init__( self,