diff --git a/docs/examples/workflows/coinflip.md b/docs/examples/workflows/coinflip.md index f55698f4a..618ea28f7 100644 --- a/docs/examples/workflows/coinflip.md +++ b/docs/examples/workflows/coinflip.md @@ -72,12 +72,9 @@ import random + result = ''heads'' if random.randint(0, 1) == 0 else ''tails'' - result = "heads" if random.randint(0, 1) == 0 else "tails" - - print(result) - - ' + print(result)' - name: heads script: command: @@ -89,9 +86,7 @@ sys.path.append(os.getcwd()) - print("it was heads") - - ' + print(''it was heads'')' - name: tails script: command: @@ -103,8 +98,6 @@ sys.path.append(os.getcwd()) - print("it was tails") - - ' + print(''it was tails'')' ``` diff --git a/docs/examples/workflows/complex_deps.md b/docs/examples/workflows/complex_deps.md index 396578429..57a599778 100644 --- a/docs/examples/workflows/complex_deps.md +++ b/docs/examples/workflows/complex_deps.md @@ -78,6 +78,6 @@ image: python:3.8 source: "import os\nimport sys\nsys.path.append(os.getcwd())\nimport json\n\ try: p = json.loads(r'''{{inputs.parameters.p}}''')\nexcept: p = r'''{{inputs.parameters.p}}'''\n\ - \nif p < 0.5:\n raise Exception(p)\nprint(42)\n" + \nif p < 0.5:\n raise Exception(p)\nprint(42)" ``` diff --git a/docs/examples/workflows/dag-with-script-output-param-passing.md b/docs/examples/workflows/dag-with-script-output-param-passing.md index 26ede674d..47fe03ac2 100644 --- a/docs/examples/workflows/dag-with-script-output-param-passing.md +++ b/docs/examples/workflows/dag-with-script-output-param-passing.md @@ -68,8 +68,8 @@ command: - python image: python:3.8 - source: "import os\nimport sys\nsys.path.append(os.getcwd())\nwith open(\"/test\"\ - , \"w\") as f_out:\n f_out.write(\"test\")\n" + source: "import os\nimport sys\nsys.path.append(os.getcwd())\nwith open('/test',\ + \ 'w') as f_out:\n f_out.write('test')" - inputs: parameters: - name: a @@ -91,8 +91,6 @@ except: a = r''''''{{inputs.parameters.a}}'''''' - print(a) - - ' + print(a)' ``` diff --git a/docs/examples/workflows/dag_conditional_parameters.md b/docs/examples/workflows/dag_conditional_parameters.md index a6f28c2cb..b08e87dad 100644 --- a/docs/examples/workflows/dag_conditional_parameters.md +++ b/docs/examples/workflows/dag_conditional_parameters.md @@ -79,25 +79,18 @@ image: python:alpine3.6 source: 'import random - - print("heads" if random.randint(0, 1) == 0 else "tails") - - ' + print(''heads'' if random.randint(0, 1) == 0 else ''tails'')' - name: heads script: command: - python image: python:alpine3.6 - source: 'print("heads") - - ' + source: print('heads') - name: tails script: command: - python image: python:alpine3.6 - source: 'print("tails") - - ' + source: print('tails') ``` diff --git a/docs/examples/workflows/dag_diamond_with_callable_decorators.md b/docs/examples/workflows/dag_diamond_with_callable_decorators.md index 8318d8010..0bb2c0d02 100644 --- a/docs/examples/workflows/dag_diamond_with_callable_decorators.md +++ b/docs/examples/workflows/dag_diamond_with_callable_decorators.md @@ -84,8 +84,6 @@ except: message = r''''''{{inputs.parameters.message}}'''''' - print(message) - - ' + print(message)' ``` diff --git a/docs/examples/workflows/dag_diamond_with_callable_script.md b/docs/examples/workflows/dag_diamond_with_callable_script.md index ace829d35..3d39712e4 100644 --- a/docs/examples/workflows/dag_diamond_with_callable_script.md +++ b/docs/examples/workflows/dag_diamond_with_callable_script.md @@ -70,9 +70,7 @@ except: message = r''''''{{inputs.parameters.message}}'''''' - print(message) - - ' + print(message)' - dag: tasks: - arguments: diff --git a/docs/examples/workflows/dag_with_script_param_passing.md b/docs/examples/workflows/dag_with_script_param_passing.md index ea682f413..f0f9ba5df 100644 --- a/docs/examples/workflows/dag_with_script_param_passing.md +++ b/docs/examples/workflows/dag_with_script_param_passing.md @@ -61,9 +61,7 @@ sys.path.append(os.getcwd()) - print(42) - - ' + print(42)' - inputs: parameters: - name: a @@ -85,8 +83,6 @@ except: a = r''''''{{inputs.parameters.a}}'''''' - print(a) - - ' + print(a)' ``` diff --git a/docs/examples/workflows/dynamic_volumes.md b/docs/examples/workflows/dynamic_volumes.md index 58feadcc9..17e35c2f9 100644 --- a/docs/examples/workflows/dynamic_volumes.md +++ b/docs/examples/workflows/dynamic_volumes.md @@ -51,10 +51,7 @@ import subprocess - - print(subprocess.run("cd && /mnt && df -h", shell=True, capture_output=True).stdout.decode()) - - ' + print(subprocess.run(''cd && /mnt && df -h'', shell=True, capture_output=True).stdout.decode())' volumeMounts: - mountPath: /mnt/vol name: v diff --git a/docs/examples/workflows/global_config.md b/docs/examples/workflows/global_config.md index 41e4ac88b..61dd6eab8 100644 --- a/docs/examples/workflows/global_config.md +++ b/docs/examples/workflows/global_config.md @@ -57,8 +57,6 @@ sys.path.append(os.getcwd()) - print("hello") - - ' + print(''hello'')' ``` diff --git a/docs/examples/workflows/multi_env.md b/docs/examples/workflows/multi_env.md index 34ce7df8e..06f590676 100644 --- a/docs/examples/workflows/multi_env.md +++ b/docs/examples/workflows/multi_env.md @@ -61,15 +61,10 @@ import os + assert os.environ[''a''] == ''1'', os.environ[''a''] - # note that env params come in as strings + assert os.environ[''b''] == ''2'', os.environ[''b''] - assert os.environ["a"] == "1", os.environ["a"] - - assert os.environ["b"] == "2", os.environ["b"] - - assert os.environ["c"] == "3", os.environ["c"] - - ' + assert os.environ[''c''] == ''3'', os.environ[''c'']' ``` diff --git a/docs/examples/workflows/script_auto_infer.md b/docs/examples/workflows/script_auto_infer.md index c19739aed..e5e59583a 100644 --- a/docs/examples/workflows/script_auto_infer.md +++ b/docs/examples/workflows/script_auto_infer.md @@ -68,8 +68,8 @@ - python image: python:3.8 source: "import os\nimport sys\nsys.path.append(os.getcwd())\nimport pickle\n\ - \nresult = \"foo testing\"\nwith open(\"/tmp/result\", \"wb\") as f:\n \ - \ pickle.dump(result, f)\n" + result = 'foo testing'\nwith open('/tmp/result', 'wb') as f:\n pickle.dump(result,\ + \ f)" - inputs: artifacts: - name: i @@ -80,7 +80,6 @@ - python image: python:3.8 source: "import os\nimport sys\nsys.path.append(os.getcwd())\nimport json\n\n\ - import pickle\n\nwith open(\"/tmp/i\", \"rb\") as f:\n i = pickle.load(f)\n\ - print(i)\n" + import pickle\nwith open('/tmp/i', 'rb') as f:\n i = pickle.load(f)\nprint(i)" ``` diff --git a/docs/examples/workflows/script_variations.md b/docs/examples/workflows/script_variations.md new file mode 100644 index 000000000..29438f24c --- /dev/null +++ b/docs/examples/workflows/script_variations.md @@ -0,0 +1,82 @@ +# Script Variations + + + + + + +=== "Hera" + + ```python linenums="1" + from hera.workflows import Workflow, script + + + @script() + def hello_world(): # pragma: no cover + print("Hello World!") + + + @script() + def multiline_function( + test: str, + another_test: str, + ) -> str: # pragma: no cover + print("Hello World!") + + + with Workflow(generate_name="fv-test-", entrypoint="d") as w: + hello_world() + multiline_function() + ``` + +=== "YAML" + + ```yaml linenums="1" + apiVersion: argoproj.io/v1alpha1 + kind: Workflow + metadata: + generateName: fv-test- + spec: + entrypoint: d + templates: + - name: hello-world + script: + command: + - python + image: python:3.8 + source: 'import os + + import sys + + sys.path.append(os.getcwd()) + + print(''Hello World!'')' + - inputs: + parameters: + - name: test + - name: another_test + name: multiline-function + script: + command: + - python + image: python:3.8 + source: 'import os + + import sys + + sys.path.append(os.getcwd()) + + import json + + try: another_test = json.loads(r''''''{{inputs.parameters.another_test}}'''''') + + except: another_test = r''''''{{inputs.parameters.another_test}}'''''' + + try: test = json.loads(r''''''{{inputs.parameters.test}}'''''') + + except: test = r''''''{{inputs.parameters.test}}'''''' + + + print(''Hello World!'')' + ``` + diff --git a/docs/examples/workflows/script_with_default_params.md b/docs/examples/workflows/script_with_default_params.md index f2054aa76..5c0805ccf 100644 --- a/docs/examples/workflows/script_with_default_params.md +++ b/docs/examples/workflows/script_with_default_params.md @@ -102,8 +102,6 @@ except: c = r''''''{{inputs.parameters.c}}'''''' - print(a, b, c) - - ' + print(a, b, c)' ``` diff --git a/docs/examples/workflows/script_with_resources.md b/docs/examples/workflows/script_with_resources.md index 359b86da7..421865cb1 100644 --- a/docs/examples/workflows/script_with_resources.md +++ b/docs/examples/workflows/script_with_resources.md @@ -44,8 +44,6 @@ sys.path.append(os.getcwd()) - print("ok") - - ' + print(''ok'')' ``` diff --git a/docs/examples/workflows/upstream/coinflip.md b/docs/examples/workflows/upstream/coinflip.md index 35cd2bcf4..4b8698783 100644 --- a/docs/examples/workflows/upstream/coinflip.md +++ b/docs/examples/workflows/upstream/coinflip.md @@ -95,11 +95,8 @@ image: python:alpine3.6 source: 'import random + result = ''heads'' if random.randint(0, 1) == 0 else ''tails'' - result = "heads" if random.randint(0, 1) == 0 else "tails" - - print(result) - - ' + print(result)' ``` diff --git a/docs/examples/workflows/upstream/colored_logs.md b/docs/examples/workflows/upstream/colored_logs.md index d352d374a..3158c3dba 100644 --- a/docs/examples/workflows/upstream/colored_logs.md +++ b/docs/examples/workflows/upstream/colored_logs.md @@ -53,12 +53,11 @@ - name: PYTHONUNBUFFERED value: '1' image: python:3.7 - source: "import time # noqa: I001\nimport random\n\nmessages = [\n \"No\ - \ Color\",\n \"\\x1b[30m%s\\x1b[0m\" % \"FG Black\",\n \"\\x1b[32m%s\\\ - x1b[0m\" % \"FG Green\",\n \"\\x1b[34m%s\\x1b[0m\" % \"FG Blue\",\n \ - \ \"\\x1b[36m%s\\x1b[0m\" % \"FG Cyan\",\n \"\\x1b[41m%s\\x1b[0m\" % \"\ - BG Red\",\n \"\\x1b[43m%s\\x1b[0m\" % \"BG Yellow\",\n \"\\x1b[45m%s\\\ - x1b[0m\" % \"BG Magenta\",\n]\nfor i in range(1, 100):\n print(random.choice(messages))\n\ - \ time.sleep(1)\n" + source: "import time\nimport random\nmessages = ['No Color', '\\x1b[30m%s\\\ + x1b[0m' % 'FG Black', '\\x1b[32m%s\\x1b[0m' % 'FG Green', '\\x1b[34m%s\\x1b[0m'\ + \ % 'FG Blue', '\\x1b[36m%s\\x1b[0m' % 'FG Cyan', '\\x1b[41m%s\\x1b[0m' %\ + \ 'BG Red', '\\x1b[43m%s\\x1b[0m' % 'BG Yellow', '\\x1b[45m%s\\x1b[0m' % 'BG\ + \ Magenta']\nfor i in range(1, 100):\n print(random.choice(messages))\n\ + \ time.sleep(1)" ``` diff --git a/docs/examples/workflows/upstream/conditional_artifacts.md b/docs/examples/workflows/upstream/conditional_artifacts.md index cdf3ded8a..d4a11bcca 100644 --- a/docs/examples/workflows/upstream/conditional_artifacts.md +++ b/docs/examples/workflows/upstream/conditional_artifacts.md @@ -106,10 +106,7 @@ image: python:alpine3.6 source: 'import random - - print("heads" if random.randint(0, 1) == 0 else "tails") - - ' + print(''heads'' if random.randint(0, 1) == 0 else ''tails'')' - name: heads outputs: artifacts: @@ -119,8 +116,7 @@ command: - python image: python:alpine3.6 - source: "with open(\"result.txt\", \"w\") as f:\n f.write(\"it was heads\"\ - )\n" + source: "with open('result.txt', 'w') as f:\n f.write('it was heads')" - name: tails outputs: artifacts: @@ -130,8 +126,7 @@ command: - python image: python:alpine3.6 - source: "with open(\"result.txt\", \"w\") as f:\n f.write(\"it was tails\"\ - )\n" + source: "with open('result.txt', 'w') as f:\n f.write('it was tails')" - name: main outputs: artifacts: diff --git a/docs/examples/workflows/upstream/container_set_template__outputs_result_workflow.md b/docs/examples/workflows/upstream/container_set_template__outputs_result_workflow.md index 4f7a84e80..24bbdd663 100644 --- a/docs/examples/workflows/upstream/container_set_template__outputs_result_workflow.md +++ b/docs/examples/workflows/upstream/container_set_template__outputs_result_workflow.md @@ -80,9 +80,7 @@ command: - python image: python:alpine3.6 - source: 'assert "{{inputs.parameters.x}}" == "hi" - - ' + source: assert '{{inputs.parameters.x}}' == 'hi' - dag: tasks: - name: a diff --git a/docs/examples/workflows/upstream/dag_conditional_parameters.md b/docs/examples/workflows/upstream/dag_conditional_parameters.md index b60287a36..9ba5baec1 100644 --- a/docs/examples/workflows/upstream/dag_conditional_parameters.md +++ b/docs/examples/workflows/upstream/dag_conditional_parameters.md @@ -70,17 +70,13 @@ command: - python image: python:alpine3.6 - source: 'print("heads") - - ' + source: print('heads') - name: tails script: command: - python image: python:alpine3.6 - source: 'print("tails") - - ' + source: print('tails') - name: flip-coin script: command: @@ -88,10 +84,7 @@ image: python:alpine3.6 source: 'import random - - print("heads" if random.randint(0, 1) == 0 else "tails") - - ' + print(''heads'' if random.randint(0, 1) == 0 else ''tails'')' - dag: tasks: - name: flip-coin diff --git a/docs/examples/workflows/upstream/exit_handler_with_artifacts.md b/docs/examples/workflows/upstream/exit_handler_with_artifacts.md index 122b1691f..19876c6bf 100644 --- a/docs/examples/workflows/upstream/exit_handler_with_artifacts.md +++ b/docs/examples/workflows/upstream/exit_handler_with_artifacts.md @@ -139,7 +139,7 @@ spec: command: - python image: python:alpine3.6 - source: "with open(\"result.txt\", \"w\") as f:\n f.write(\"Welcome\")\n" + source: "with open('result.txt', 'w') as f:\n f.write('Welcome')" - container: args: - cat /tmp/message diff --git a/docs/examples/workflows/upstream/loops_param_result.md b/docs/examples/workflows/upstream/loops_param_result.md index 265935dab..dce94119f 100644 --- a/docs/examples/workflows/upstream/loops_param_result.md +++ b/docs/examples/workflows/upstream/loops_param_result.md @@ -84,9 +84,6 @@ import sys - - json.dump([i for i in range(20, 31)], sys.stdout) - - ' + json.dump([i for i in range(20, 31)], sys.stdout)' ``` diff --git a/docs/examples/workflows/upstream/retry_script.md b/docs/examples/workflows/upstream/retry_script.md index 7c111b3c3..4aa577e35 100644 --- a/docs/examples/workflows/upstream/retry_script.md +++ b/docs/examples/workflows/upstream/retry_script.md @@ -45,11 +45,8 @@ import sys - exit_code = random.choice([0, 1, 1]) - sys.exit(exit_code) - - ' + sys.exit(exit_code)' ``` diff --git a/docs/examples/workflows/volume_mounts.md b/docs/examples/workflows/volume_mounts.md index 599ecb4a0..525ca70be 100644 --- a/docs/examples/workflows/volume_mounts.md +++ b/docs/examples/workflows/volume_mounts.md @@ -94,12 +94,9 @@ import subprocess + print(os.listdir(''/mnt'')) - print(os.listdir("/mnt")) - - print(subprocess.run("cd /mnt && df -h", shell=True, capture_output=True).stdout.decode()) - - ' + print(subprocess.run(''cd /mnt && df -h'', shell=True, capture_output=True).stdout.decode())' volumeMounts: - mountPath: /mnt/vol name: '{{inputs.parameters.vol}}' diff --git a/examples/workflows/coinflip.yaml b/examples/workflows/coinflip.yaml index 7f580f5fc..2cadd019b 100644 --- a/examples/workflows/coinflip.yaml +++ b/examples/workflows/coinflip.yaml @@ -31,12 +31,9 @@ spec: import random + result = ''heads'' if random.randint(0, 1) == 0 else ''tails'' - result = "heads" if random.randint(0, 1) == 0 else "tails" - - print(result) - - ' + print(result)' - name: heads script: command: @@ -48,9 +45,7 @@ spec: sys.path.append(os.getcwd()) - print("it was heads") - - ' + print(''it was heads'')' - name: tails script: command: @@ -62,6 +57,4 @@ spec: sys.path.append(os.getcwd()) - print("it was tails") - - ' + print(''it was tails'')' diff --git a/examples/workflows/complex-deps.yaml b/examples/workflows/complex-deps.yaml index 458197787..5fd5211f7 100644 --- a/examples/workflows/complex-deps.yaml +++ b/examples/workflows/complex-deps.yaml @@ -45,4 +45,4 @@ spec: image: python:3.8 source: "import os\nimport sys\nsys.path.append(os.getcwd())\nimport json\n\ try: p = json.loads(r'''{{inputs.parameters.p}}''')\nexcept: p = r'''{{inputs.parameters.p}}'''\n\ - \nif p < 0.5:\n raise Exception(p)\nprint(42)\n" + \nif p < 0.5:\n raise Exception(p)\nprint(42)" diff --git a/examples/workflows/dag-conditional-parameters.yaml b/examples/workflows/dag-conditional-parameters.yaml index 4cf207550..5762a92e3 100644 --- a/examples/workflows/dag-conditional-parameters.yaml +++ b/examples/workflows/dag-conditional-parameters.yaml @@ -31,23 +31,16 @@ spec: image: python:alpine3.6 source: 'import random - - print("heads" if random.randint(0, 1) == 0 else "tails") - - ' + print(''heads'' if random.randint(0, 1) == 0 else ''tails'')' - name: heads script: command: - python image: python:alpine3.6 - source: 'print("heads") - - ' + source: print('heads') - name: tails script: command: - python image: python:alpine3.6 - source: 'print("tails") - - ' + source: print('tails') diff --git a/examples/workflows/dag-diamond-with-callable-decorators.yaml b/examples/workflows/dag-diamond-with-callable-decorators.yaml index 5970903ea..ec3b9bb75 100644 --- a/examples/workflows/dag-diamond-with-callable-decorators.yaml +++ b/examples/workflows/dag-diamond-with-callable-decorators.yaml @@ -50,6 +50,4 @@ spec: except: message = r''''''{{inputs.parameters.message}}'''''' - print(message) - - ' + print(message)' diff --git a/examples/workflows/dag-diamond-with-callable-script.yaml b/examples/workflows/dag-diamond-with-callable-script.yaml index 862bebca5..7274cf13d 100644 --- a/examples/workflows/dag-diamond-with-callable-script.yaml +++ b/examples/workflows/dag-diamond-with-callable-script.yaml @@ -20,9 +20,7 @@ spec: except: message = r''''''{{inputs.parameters.message}}'''''' - print(message) - - ' + print(message)' - dag: tasks: - arguments: diff --git a/examples/workflows/dag-with-script-output-param-passing.yaml b/examples/workflows/dag-with-script-output-param-passing.yaml index 8bbe8f294..3dd9857e7 100644 --- a/examples/workflows/dag-with-script-output-param-passing.yaml +++ b/examples/workflows/dag-with-script-output-param-passing.yaml @@ -27,8 +27,8 @@ spec: command: - python image: python:3.8 - source: "import os\nimport sys\nsys.path.append(os.getcwd())\nwith open(\"/test\"\ - , \"w\") as f_out:\n f_out.write(\"test\")\n" + source: "import os\nimport sys\nsys.path.append(os.getcwd())\nwith open('/test',\ + \ 'w') as f_out:\n f_out.write('test')" - inputs: parameters: - name: a @@ -50,6 +50,4 @@ spec: except: a = r''''''{{inputs.parameters.a}}'''''' - print(a) - - ' + print(a)' diff --git a/examples/workflows/dag-with-script-param-passing.yaml b/examples/workflows/dag-with-script-param-passing.yaml index 498288683..28bd0ca2a 100644 --- a/examples/workflows/dag-with-script-param-passing.yaml +++ b/examples/workflows/dag-with-script-param-passing.yaml @@ -28,9 +28,7 @@ spec: sys.path.append(os.getcwd()) - print(42) - - ' + print(42)' - inputs: parameters: - name: a @@ -52,6 +50,4 @@ spec: except: a = r''''''{{inputs.parameters.a}}'''''' - print(a) - - ' + print(a)' diff --git a/examples/workflows/dynamic-volumes.yaml b/examples/workflows/dynamic-volumes.yaml index 23a4c68ad..c2d997f16 100644 --- a/examples/workflows/dynamic-volumes.yaml +++ b/examples/workflows/dynamic-volumes.yaml @@ -23,10 +23,7 @@ spec: import subprocess - - print(subprocess.run("cd && /mnt && df -h", shell=True, capture_output=True).stdout.decode()) - - ' + print(subprocess.run(''cd && /mnt && df -h'', shell=True, capture_output=True).stdout.decode())' volumeMounts: - mountPath: /mnt/vol name: v diff --git a/examples/workflows/global-config.yaml b/examples/workflows/global-config.yaml index e1318212a..468f6cc1b 100644 --- a/examples/workflows/global-config.yaml +++ b/examples/workflows/global-config.yaml @@ -23,6 +23,4 @@ spec: sys.path.append(os.getcwd()) - print("hello") - - ' + print(''hello'')' diff --git a/examples/workflows/multi-env.yaml b/examples/workflows/multi-env.yaml index c12785d36..f01df69d3 100644 --- a/examples/workflows/multi-env.yaml +++ b/examples/workflows/multi-env.yaml @@ -30,13 +30,8 @@ spec: import os + assert os.environ[''a''] == ''1'', os.environ[''a''] - # note that env params come in as strings + assert os.environ[''b''] == ''2'', os.environ[''b''] - assert os.environ["a"] == "1", os.environ["a"] - - assert os.environ["b"] == "2", os.environ["b"] - - assert os.environ["c"] == "3", os.environ["c"] - - ' + assert os.environ[''c''] == ''3'', os.environ[''c'']' diff --git a/examples/workflows/script-auto-infer.yaml b/examples/workflows/script-auto-infer.yaml index 0ae68d39f..d87475187 100644 --- a/examples/workflows/script-auto-infer.yaml +++ b/examples/workflows/script-auto-infer.yaml @@ -27,8 +27,8 @@ spec: - python image: python:3.8 source: "import os\nimport sys\nsys.path.append(os.getcwd())\nimport pickle\n\ - \nresult = \"foo testing\"\nwith open(\"/tmp/result\", \"wb\") as f:\n \ - \ pickle.dump(result, f)\n" + result = 'foo testing'\nwith open('/tmp/result', 'wb') as f:\n pickle.dump(result,\ + \ f)" - inputs: artifacts: - name: i @@ -39,5 +39,4 @@ spec: - python image: python:3.8 source: "import os\nimport sys\nsys.path.append(os.getcwd())\nimport json\n\n\ - import pickle\n\nwith open(\"/tmp/i\", \"rb\") as f:\n i = pickle.load(f)\n\ - print(i)\n" + import pickle\nwith open('/tmp/i', 'rb') as f:\n i = pickle.load(f)\nprint(i)" diff --git a/examples/workflows/script-variations.yaml b/examples/workflows/script-variations.yaml new file mode 100644 index 000000000..224e014bc --- /dev/null +++ b/examples/workflows/script-variations.yaml @@ -0,0 +1,46 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: fv-test- +spec: + entrypoint: d + templates: + - name: hello-world + script: + command: + - python + image: python:3.8 + source: 'import os + + import sys + + sys.path.append(os.getcwd()) + + print(''Hello World!'')' + - inputs: + parameters: + - name: test + - name: another_test + name: multiline-function + script: + command: + - python + image: python:3.8 + source: 'import os + + import sys + + sys.path.append(os.getcwd()) + + import json + + try: another_test = json.loads(r''''''{{inputs.parameters.another_test}}'''''') + + except: another_test = r''''''{{inputs.parameters.another_test}}'''''' + + try: test = json.loads(r''''''{{inputs.parameters.test}}'''''') + + except: test = r''''''{{inputs.parameters.test}}'''''' + + + print(''Hello World!'')' diff --git a/examples/workflows/script-with-default-params.yaml b/examples/workflows/script-with-default-params.yaml index e820ed5f3..995531b6c 100644 --- a/examples/workflows/script-with-default-params.yaml +++ b/examples/workflows/script-with-default-params.yaml @@ -73,6 +73,4 @@ spec: except: c = r''''''{{inputs.parameters.c}}'''''' - print(a, b, c) - - ' + print(a, b, c)' diff --git a/examples/workflows/script-with-resources.yaml b/examples/workflows/script-with-resources.yaml index 9edac7573..be0724a1f 100644 --- a/examples/workflows/script-with-resources.yaml +++ b/examples/workflows/script-with-resources.yaml @@ -19,6 +19,4 @@ spec: sys.path.append(os.getcwd()) - print("ok") - - ' + print(''ok'')' diff --git a/examples/workflows/script_variations.py b/examples/workflows/script_variations.py new file mode 100644 index 000000000..53d08ede0 --- /dev/null +++ b/examples/workflows/script_variations.py @@ -0,0 +1,19 @@ +from hera.workflows import Workflow, script + + +@script() +def hello_world(): # pragma: no cover + print("Hello World!") + + +@script() +def multiline_function( + test: str, + another_test: str, +) -> str: # pragma: no cover + print("Hello World!") + + +with Workflow(generate_name="fv-test-", entrypoint="d") as w: + hello_world() + multiline_function() diff --git a/examples/workflows/upstream/coinflip.yaml b/examples/workflows/upstream/coinflip.yaml index a26540ee7..34989a83c 100644 --- a/examples/workflows/upstream/coinflip.yaml +++ b/examples/workflows/upstream/coinflip.yaml @@ -41,9 +41,6 @@ spec: image: python:alpine3.6 source: 'import random + result = ''heads'' if random.randint(0, 1) == 0 else ''tails'' - result = "heads" if random.randint(0, 1) == 0 else "tails" - - print(result) - - ' + print(result)' diff --git a/examples/workflows/upstream/colored-logs.yaml b/examples/workflows/upstream/colored-logs.yaml index bfb898548..538c547da 100644 --- a/examples/workflows/upstream/colored-logs.yaml +++ b/examples/workflows/upstream/colored-logs.yaml @@ -13,10 +13,9 @@ spec: - name: PYTHONUNBUFFERED value: '1' image: python:3.7 - source: "import time # noqa: I001\nimport random\n\nmessages = [\n \"No\ - \ Color\",\n \"\\x1b[30m%s\\x1b[0m\" % \"FG Black\",\n \"\\x1b[32m%s\\\ - x1b[0m\" % \"FG Green\",\n \"\\x1b[34m%s\\x1b[0m\" % \"FG Blue\",\n \ - \ \"\\x1b[36m%s\\x1b[0m\" % \"FG Cyan\",\n \"\\x1b[41m%s\\x1b[0m\" % \"\ - BG Red\",\n \"\\x1b[43m%s\\x1b[0m\" % \"BG Yellow\",\n \"\\x1b[45m%s\\\ - x1b[0m\" % \"BG Magenta\",\n]\nfor i in range(1, 100):\n print(random.choice(messages))\n\ - \ time.sleep(1)\n" + source: "import time\nimport random\nmessages = ['No Color', '\\x1b[30m%s\\\ + x1b[0m' % 'FG Black', '\\x1b[32m%s\\x1b[0m' % 'FG Green', '\\x1b[34m%s\\x1b[0m'\ + \ % 'FG Blue', '\\x1b[36m%s\\x1b[0m' % 'FG Cyan', '\\x1b[41m%s\\x1b[0m' %\ + \ 'BG Red', '\\x1b[43m%s\\x1b[0m' % 'BG Yellow', '\\x1b[45m%s\\x1b[0m' % 'BG\ + \ Magenta']\nfor i in range(1, 100):\n print(random.choice(messages))\n\ + \ time.sleep(1)" diff --git a/examples/workflows/upstream/conditional-artifacts.yaml b/examples/workflows/upstream/conditional-artifacts.yaml index aaeaabd31..982eec9ab 100644 --- a/examples/workflows/upstream/conditional-artifacts.yaml +++ b/examples/workflows/upstream/conditional-artifacts.yaml @@ -20,10 +20,7 @@ spec: image: python:alpine3.6 source: 'import random - - print("heads" if random.randint(0, 1) == 0 else "tails") - - ' + print(''heads'' if random.randint(0, 1) == 0 else ''tails'')' - name: heads outputs: artifacts: @@ -33,8 +30,7 @@ spec: command: - python image: python:alpine3.6 - source: "with open(\"result.txt\", \"w\") as f:\n f.write(\"it was heads\"\ - )\n" + source: "with open('result.txt', 'w') as f:\n f.write('it was heads')" - name: tails outputs: artifacts: @@ -44,8 +40,7 @@ spec: command: - python image: python:alpine3.6 - source: "with open(\"result.txt\", \"w\") as f:\n f.write(\"it was tails\"\ - )\n" + source: "with open('result.txt', 'w') as f:\n f.write('it was tails')" - name: main outputs: artifacts: diff --git a/examples/workflows/upstream/container-set-template--outputs-result-workflow.yaml b/examples/workflows/upstream/container-set-template--outputs-result-workflow.yaml index 0a980f475..393e98f65 100644 --- a/examples/workflows/upstream/container-set-template--outputs-result-workflow.yaml +++ b/examples/workflows/upstream/container-set-template--outputs-result-workflow.yaml @@ -25,9 +25,7 @@ spec: command: - python image: python:alpine3.6 - source: 'assert "{{inputs.parameters.x}}" == "hi" - - ' + source: assert '{{inputs.parameters.x}}' == 'hi' - dag: tasks: - name: a diff --git a/examples/workflows/upstream/dag-conditional-parameters.yaml b/examples/workflows/upstream/dag-conditional-parameters.yaml index 1f73cd74a..88584c6f7 100644 --- a/examples/workflows/upstream/dag-conditional-parameters.yaml +++ b/examples/workflows/upstream/dag-conditional-parameters.yaml @@ -10,17 +10,13 @@ spec: command: - python image: python:alpine3.6 - source: 'print("heads") - - ' + source: print('heads') - name: tails script: command: - python image: python:alpine3.6 - source: 'print("tails") - - ' + source: print('tails') - name: flip-coin script: command: @@ -28,10 +24,7 @@ spec: image: python:alpine3.6 source: 'import random - - print("heads" if random.randint(0, 1) == 0 else "tails") - - ' + print(''heads'' if random.randint(0, 1) == 0 else ''tails'')' - dag: tasks: - name: flip-coin diff --git a/examples/workflows/upstream/exit-handler-with-artifacts.yaml b/examples/workflows/upstream/exit-handler-with-artifacts.yaml index af7becda2..03d68b876 100644 --- a/examples/workflows/upstream/exit-handler-with-artifacts.yaml +++ b/examples/workflows/upstream/exit-handler-with-artifacts.yaml @@ -20,7 +20,7 @@ spec: command: - python image: python:alpine3.6 - source: "with open(\"result.txt\", \"w\") as f:\n f.write(\"Welcome\")\n" + source: "with open('result.txt', 'w') as f:\n f.write('Welcome')" - container: args: - cat /tmp/message diff --git a/examples/workflows/upstream/loops-param-result.yaml b/examples/workflows/upstream/loops-param-result.yaml index 2f1c2c562..7bf46d748 100644 --- a/examples/workflows/upstream/loops-param-result.yaml +++ b/examples/workflows/upstream/loops-param-result.yaml @@ -37,7 +37,4 @@ spec: import sys - - json.dump([i for i in range(20, 31)], sys.stdout) - - ' + json.dump([i for i in range(20, 31)], sys.stdout)' diff --git a/examples/workflows/upstream/retry-script.yaml b/examples/workflows/upstream/retry-script.yaml index b62e1b099..48a316984 100644 --- a/examples/workflows/upstream/retry-script.yaml +++ b/examples/workflows/upstream/retry-script.yaml @@ -16,9 +16,6 @@ spec: import sys - exit_code = random.choice([0, 1, 1]) - sys.exit(exit_code) - - ' + sys.exit(exit_code)' diff --git a/examples/workflows/volume-mounts.yaml b/examples/workflows/volume-mounts.yaml index 83f32065d..b4a869d85 100644 --- a/examples/workflows/volume-mounts.yaml +++ b/examples/workflows/volume-mounts.yaml @@ -44,12 +44,9 @@ spec: import subprocess + print(os.listdir(''/mnt'')) - print(os.listdir("/mnt")) - - print(subprocess.run("cd /mnt && df -h", shell=True, capture_output=True).stdout.decode()) - - ' + print(subprocess.run(''cd /mnt && df -h'', shell=True, capture_output=True).stdout.decode())' volumeMounts: - mountPath: /mnt/vol name: '{{inputs.parameters.vol}}' diff --git a/src/hera/workflows/_unparse.py b/src/hera/workflows/_unparse.py new file mode 100644 index 000000000..2b686beb1 --- /dev/null +++ b/src/hera/workflows/_unparse.py @@ -0,0 +1,949 @@ +"""A module for unparsing ASTs back to source code. + +NOTE: This is a verbatim copy of the code at https://github.com/python/cpython/blob/3.9/Lib/ast.py + +This is only used for backwards compatiblity for python 3.8 as unparse was added in python 3.9. +""" +import ast +import sys +from ast import AsyncFunctionDef, ClassDef, Constant, Expr, FunctionDef, If, Module, Name, Starred, Tuple +from contextlib import contextmanager, nullcontext +from enum import IntEnum, auto + +# Large float and imaginary literals get turned into infinities in the AST. +# We unparse those infinities to INFSTR. +_INFSTR = "1e" + repr(sys.float_info.max_10_exp + 1) + + +class _Precedence(IntEnum): + """Precedence table that originated from python grammar.""" + + TUPLE = auto() + YIELD = auto() # 'yield', 'yield from' + TEST = auto() # 'if'-'else', 'lambda' + OR = auto() # 'or' + AND = auto() # 'and' + NOT = auto() # 'not' + CMP = auto() # '<', '>', '==', '>=', '<=', '!=', + # 'in', 'not in', 'is', 'is not' + EXPR = auto() + BOR = EXPR # '|' + BXOR = auto() # '^' + BAND = auto() # '&' + SHIFT = auto() # '<<', '>>' + ARITH = auto() # '+', '-' + TERM = auto() # '*', '@', '/', '%', '//' + FACTOR = auto() # unary '+', '-', '~' + POWER = auto() # '**' + AWAIT = auto() # 'await' + ATOM = auto() + + def next(self): + try: + return self.__class__(self + 1) + except ValueError: + return self + + +_SINGLE_QUOTES = ("'", '"') +_MULTI_QUOTES = ('"""', "'''") +_ALL_QUOTES = (*_SINGLE_QUOTES, *_MULTI_QUOTES) + + +class _Unparser(ast.NodeVisitor): + """Methods in this class recursively traverse an AST and + output source code for the abstract syntax; original formatting + is disregarded.""" + + def __init__(self, *, _avoid_backslashes=False): + self._source = [] + self._buffer = [] + self._precedences = {} + self._type_ignores = {} + self._indent = 0 + self._avoid_backslashes = _avoid_backslashes + + def __call__(self, node): + """Outputs a source code string that, if converted back to an ast + (using ast.parse) will generate an AST equivalent to *node*""" + self._source = [] + self.traverse(node) + return "".join(self._source) + + def interleave(self, inter, f, seq): + """Call f on each item in seq, calling inter() in between.""" + seq = iter(seq) + try: + f(next(seq)) + except StopIteration: + pass + else: + for x in seq: + inter() + f(x) + + def items_view(self, traverser, items): + """Traverse and separate the given *items* with a comma and append it to + the buffer. If *items* is a single item sequence, a trailing comma + will be added.""" + if len(items) == 1: + traverser(items[0]) + self.write(",") + else: + self.interleave(lambda: self.write(", "), traverser, items) + + def maybe_newline(self): + """Adds a newline if it isn't the start of generated source""" + if self._source: + self.write("\n") + + def fill(self, text=""): + """Indent a piece of text and append it, according to the current + indentation level""" + self.maybe_newline() + self.write(" " * self._indent + text) + + def write(self, text): + """Append a piece of text""" + self._source.append(text) + + def buffer_writer(self, text): + self._buffer.append(text) + + @property + def buffer(self): + value = "".join(self._buffer) + self._buffer.clear() + return value + + @contextmanager + def block(self, *, extra=None): + """A context manager for preparing the source for blocks. It adds + the character':', increases the indentation on enter and decreases + the indentation on exit. If *extra* is given, it will be directly + appended after the colon character. + """ + self.write(":") + if extra: + self.write(extra) + self._indent += 1 + yield + self._indent -= 1 + + @contextmanager + def delimit(self, start, end): + """A context manager for preparing the source for expressions. It adds + *start* to the buffer and enters, after exit it adds *end*.""" + + self.write(start) + yield + self.write(end) + + def delimit_if(self, start, end, condition): + if condition: + return self.delimit(start, end) + else: + return nullcontext() + + def require_parens(self, precedence, node): + """Shortcut to adding precedence related parens""" + return self.delimit_if("(", ")", self.get_precedence(node) > precedence) + + def get_precedence(self, node): + return self._precedences.get(node, _Precedence.TEST) + + def set_precedence(self, precedence, *nodes): + for node in nodes: + self._precedences[node] = precedence + + def get_raw_docstring(self, node): + """If a docstring node is found in the body of the *node* parameter, + return that docstring node, None otherwise. + + Logic mirrored from ``_PyAST_GetDocString``.""" + if not isinstance(node, (AsyncFunctionDef, FunctionDef, ClassDef, Module)) or len(node.body) < 1: + return None + node = node.body[0] + if not isinstance(node, Expr): + return None + node = node.value + if isinstance(node, Constant) and isinstance(node.value, str): + return node + + def get_type_comment(self, node): + comment = self._type_ignores.get(node.lineno) or node.type_comment + if comment is not None: + return f" # type: {comment}" + + def traverse(self, node): + if isinstance(node, list): + for item in node: + self.traverse(item) + else: + super().visit(node) + + def _write_docstring_and_traverse_body(self, node): + if docstring := self.get_raw_docstring(node): + self._write_docstring(docstring) + self.traverse(node.body[1:]) + else: + self.traverse(node.body) + + def visit_Module(self, node): + self._type_ignores = {ignore.lineno: f"ignore{ignore.tag}" for ignore in node.type_ignores} + self._write_docstring_and_traverse_body(node) + self._type_ignores.clear() + + def visit_FunctionType(self, node): + with self.delimit("(", ")"): + self.interleave(lambda: self.write(", "), self.traverse, node.argtypes) + + self.write(" -> ") + self.traverse(node.returns) + + def visit_Expr(self, node): + self.fill() + self.set_precedence(_Precedence.YIELD, node.value) + self.traverse(node.value) + + def visit_NamedExpr(self, node): + with self.require_parens(_Precedence.TUPLE, node): + self.set_precedence(_Precedence.ATOM, node.target, node.value) + self.traverse(node.target) + self.write(" := ") + self.traverse(node.value) + + def visit_Import(self, node): + self.fill("import ") + self.interleave(lambda: self.write(", "), self.traverse, node.names) + + def visit_ImportFrom(self, node): + self.fill("from ") + self.write("." * node.level) + if node.module: + self.write(node.module) + self.write(" import ") + self.interleave(lambda: self.write(", "), self.traverse, node.names) + + def visit_Assign(self, node): + self.fill() + for target in node.targets: + self.traverse(target) + self.write(" = ") + self.traverse(node.value) + if type_comment := self.get_type_comment(node): + self.write(type_comment) + + def visit_AugAssign(self, node): + self.fill() + self.traverse(node.target) + self.write(" " + self.binop[node.op.__class__.__name__] + "= ") + self.traverse(node.value) + + def visit_AnnAssign(self, node): + self.fill() + with self.delimit_if("(", ")", not node.simple and isinstance(node.target, Name)): + self.traverse(node.target) + self.write(": ") + self.traverse(node.annotation) + if node.value: + self.write(" = ") + self.traverse(node.value) + + def visit_Return(self, node): + self.fill("return") + if node.value: + self.write(" ") + self.traverse(node.value) + + def visit_Pass(self, node): + self.fill("pass") + + def visit_Break(self, node): + self.fill("break") + + def visit_Continue(self, node): + self.fill("continue") + + def visit_Delete(self, node): + self.fill("del ") + self.interleave(lambda: self.write(", "), self.traverse, node.targets) + + def visit_Assert(self, node): + self.fill("assert ") + self.traverse(node.test) + if node.msg: + self.write(", ") + self.traverse(node.msg) + + def visit_Global(self, node): + self.fill("global ") + self.interleave(lambda: self.write(", "), self.write, node.names) + + def visit_Nonlocal(self, node): + self.fill("nonlocal ") + self.interleave(lambda: self.write(", "), self.write, node.names) + + def visit_Await(self, node): + with self.require_parens(_Precedence.AWAIT, node): + self.write("await") + if node.value: + self.write(" ") + self.set_precedence(_Precedence.ATOM, node.value) + self.traverse(node.value) + + def visit_Yield(self, node): + with self.require_parens(_Precedence.YIELD, node): + self.write("yield") + if node.value: + self.write(" ") + self.set_precedence(_Precedence.ATOM, node.value) + self.traverse(node.value) + + def visit_YieldFrom(self, node): + with self.require_parens(_Precedence.YIELD, node): + self.write("yield from ") + if not node.value: + raise ValueError("Node can't be used without a value attribute.") + self.set_precedence(_Precedence.ATOM, node.value) + self.traverse(node.value) + + def visit_Raise(self, node): + self.fill("raise") + if not node.exc: + if node.cause: + raise ValueError("Node can't use cause without an exception.") + return + self.write(" ") + self.traverse(node.exc) + if node.cause: + self.write(" from ") + self.traverse(node.cause) + + def visit_Try(self, node): + self.fill("try") + with self.block(): + self.traverse(node.body) + for ex in node.handlers: + self.traverse(ex) + if node.orelse: + self.fill("else") + with self.block(): + self.traverse(node.orelse) + if node.finalbody: + self.fill("finally") + with self.block(): + self.traverse(node.finalbody) + + def visit_ExceptHandler(self, node): + self.fill("except") + if node.type: + self.write(" ") + self.traverse(node.type) + if node.name: + self.write(" as ") + self.write(node.name) + with self.block(): + self.traverse(node.body) + + def visit_ClassDef(self, node): + self.maybe_newline() + for deco in node.decorator_list: + self.fill("@") + self.traverse(deco) + self.fill("class " + node.name) + with self.delimit_if("(", ")", condition=node.bases or node.keywords): + comma = False + for e in node.bases: + if comma: + self.write(", ") + else: + comma = True + self.traverse(e) + for e in node.keywords: + if comma: + self.write(", ") + else: + comma = True + self.traverse(e) + + with self.block(): + self._write_docstring_and_traverse_body(node) + + def visit_FunctionDef(self, node): + self._function_helper(node, "def") + + def visit_AsyncFunctionDef(self, node): + self._function_helper(node, "async def") + + def _function_helper(self, node, fill_suffix): + self.maybe_newline() + for deco in node.decorator_list: + self.fill("@") + self.traverse(deco) + def_str = fill_suffix + " " + node.name + self.fill(def_str) + with self.delimit("(", ")"): + self.traverse(node.args) + if node.returns: + self.write(" -> ") + self.traverse(node.returns) + with self.block(extra=self.get_type_comment(node)): + self._write_docstring_and_traverse_body(node) + + def visit_For(self, node): + self._for_helper("for ", node) + + def visit_AsyncFor(self, node): + self._for_helper("async for ", node) + + def _for_helper(self, fill, node): + self.fill(fill) + self.traverse(node.target) + self.write(" in ") + self.traverse(node.iter) + with self.block(extra=self.get_type_comment(node)): + self.traverse(node.body) + if node.orelse: + self.fill("else") + with self.block(): + self.traverse(node.orelse) + + def visit_If(self, node): + self.fill("if ") + self.traverse(node.test) + with self.block(): + self.traverse(node.body) + # collapse nested ifs into equivalent elifs. + while node.orelse and len(node.orelse) == 1 and isinstance(node.orelse[0], If): + node = node.orelse[0] + self.fill("elif ") + self.traverse(node.test) + with self.block(): + self.traverse(node.body) + # final else + if node.orelse: + self.fill("else") + with self.block(): + self.traverse(node.orelse) + + def visit_While(self, node): + self.fill("while ") + self.traverse(node.test) + with self.block(): + self.traverse(node.body) + if node.orelse: + self.fill("else") + with self.block(): + self.traverse(node.orelse) + + def visit_With(self, node): + self.fill("with ") + self.interleave(lambda: self.write(", "), self.traverse, node.items) + with self.block(extra=self.get_type_comment(node)): + self.traverse(node.body) + + def visit_AsyncWith(self, node): + self.fill("async with ") + self.interleave(lambda: self.write(", "), self.traverse, node.items) + with self.block(extra=self.get_type_comment(node)): + self.traverse(node.body) + + def _str_literal_helper(self, string, *, quote_types=_ALL_QUOTES, escape_special_whitespace=False): + """Helper for writing string literals, minimizing escapes. + Returns the tuple (string literal to write, possible quote types). + """ + + def escape_char(c): + # \n and \t are non-printable, but we only escape them if + # escape_special_whitespace is True + if not escape_special_whitespace and c in "\n\t": + return c + # Always escape backslashes and other non-printable characters + if c == "\\" or not c.isprintable(): + return c.encode("unicode_escape").decode("ascii") + return c + + escaped_string = "".join(map(escape_char, string)) + possible_quotes = quote_types + if "\n" in escaped_string: + possible_quotes = [q for q in possible_quotes if q in _MULTI_QUOTES] + possible_quotes = [q for q in possible_quotes if q not in escaped_string] + if not possible_quotes: + # If there aren't any possible_quotes, fallback to using repr + # on the original string. Try to use a quote from quote_types, + # e.g., so that we use triple quotes for docstrings. + string = repr(string) + quote = next((q for q in quote_types if string[0] in q), string[0]) + return string[1:-1], [quote] + if escaped_string: + # Sort so that we prefer '''"''' over """\"""" + possible_quotes.sort(key=lambda q: q[0] == escaped_string[-1]) + # If we're using triple quotes and we'd need to escape a final + # quote, escape it + if possible_quotes[0][0] == escaped_string[-1]: + assert len(possible_quotes[0]) == 3 + escaped_string = escaped_string[:-1] + "\\" + escaped_string[-1] + return escaped_string, possible_quotes + + def _write_str_avoiding_backslashes(self, string, *, quote_types=_ALL_QUOTES): + """Write string literal value with a best effort attempt to avoid backslashes.""" + string, quote_types = self._str_literal_helper(string, quote_types=quote_types) + quote_type = quote_types[0] + self.write(f"{quote_type}{string}{quote_type}") + + def visit_JoinedStr(self, node): + self.write("f") + if self._avoid_backslashes: + self._fstring_JoinedStr(node, self.buffer_writer) + self._write_str_avoiding_backslashes(self.buffer) + return + + # If we don't need to avoid backslashes globally (i.e., we only need + # to avoid them inside FormattedValues), it's cosmetically preferred + # to use escaped whitespace. That is, it's preferred to use backslashes + # for cases like: f"{x}\n". To accomplish this, we keep track of what + # in our buffer corresponds to FormattedValues and what corresponds to + # Constant parts of the f-string, and allow escapes accordingly. + buffer = [] + for value in node.values: + meth = getattr(self, "_fstring_" + type(value).__name__) + meth(value, self.buffer_writer) + buffer.append((self.buffer, isinstance(value, Constant))) + new_buffer = [] + quote_types = _ALL_QUOTES + for value, is_constant in buffer: + # Repeatedly narrow down the list of possible quote_types + value, quote_types = self._str_literal_helper( + value, quote_types=quote_types, escape_special_whitespace=is_constant + ) + new_buffer.append(value) + value = "".join(new_buffer) + quote_type = quote_types[0] + self.write(f"{quote_type}{value}{quote_type}") + + def visit_FormattedValue(self, node): + self.write("f") + self._fstring_FormattedValue(node, self.buffer_writer) + self._write_str_avoiding_backslashes(self.buffer) + + def _fstring_JoinedStr(self, node, write): + for value in node.values: + meth = getattr(self, "_fstring_" + type(value).__name__) + meth(value, write) + + def _fstring_Constant(self, node, write): + if not isinstance(node.value, str): + raise ValueError("Constants inside JoinedStr should be a string.") + value = node.value.replace("{", "{{").replace("}", "}}") + write(value) + + def _fstring_FormattedValue(self, node, write): + write("{") + unparser = type(self)(_avoid_backslashes=True) + unparser.set_precedence(_Precedence.TEST.next(), node.value) + expr = unparser.visit(node.value) + if expr.startswith("{"): + write(" ") # Separate pair of opening brackets as "{ {" + if "\\" in expr: + raise ValueError("Unable to avoid backslash in f-string expression part") + write(expr) + if node.conversion != -1: + conversion = chr(node.conversion) + if conversion not in "sra": + raise ValueError("Unknown f-string conversion.") + write(f"!{conversion}") + if node.format_spec: + write(":") + meth = getattr(self, "_fstring_" + type(node.format_spec).__name__) + meth(node.format_spec, write) + write("}") + + def visit_Name(self, node): + self.write(node.id) + + def _write_docstring(self, node): + self.fill() + if node.kind == "u": + self.write("u") + self._write_str_avoiding_backslashes(node.value, quote_types=_MULTI_QUOTES) + + def _write_constant(self, value): + if isinstance(value, (float, complex)): + # Substitute overflowing decimal literal for AST infinities, + # and inf - inf for NaNs. + self.write(repr(value).replace("inf", _INFSTR).replace("nan", f"({_INFSTR}-{_INFSTR})")) + elif self._avoid_backslashes and isinstance(value, str): + self._write_str_avoiding_backslashes(value) + else: + self.write(repr(value)) + + def visit_Constant(self, node): + value = node.value + if isinstance(value, tuple): + with self.delimit("(", ")"): + self.items_view(self._write_constant, value) + elif value is ...: + self.write("...") + else: + if node.kind == "u": + self.write("u") + self._write_constant(node.value) + + def visit_List(self, node): + with self.delimit("[", "]"): + self.interleave(lambda: self.write(", "), self.traverse, node.elts) + + def visit_ListComp(self, node): + with self.delimit("[", "]"): + self.traverse(node.elt) + for gen in node.generators: + self.traverse(gen) + + def visit_GeneratorExp(self, node): + with self.delimit("(", ")"): + self.traverse(node.elt) + for gen in node.generators: + self.traverse(gen) + + def visit_SetComp(self, node): + with self.delimit("{", "}"): + self.traverse(node.elt) + for gen in node.generators: + self.traverse(gen) + + def visit_DictComp(self, node): + with self.delimit("{", "}"): + self.traverse(node.key) + self.write(": ") + self.traverse(node.value) + for gen in node.generators: + self.traverse(gen) + + def visit_comprehension(self, node): + if node.is_async: + self.write(" async for ") + else: + self.write(" for ") + self.set_precedence(_Precedence.TUPLE, node.target) + self.traverse(node.target) + self.write(" in ") + self.set_precedence(_Precedence.TEST.next(), node.iter, *node.ifs) + self.traverse(node.iter) + for if_clause in node.ifs: + self.write(" if ") + self.traverse(if_clause) + + def visit_IfExp(self, node): + with self.require_parens(_Precedence.TEST, node): + self.set_precedence(_Precedence.TEST.next(), node.body, node.test) + self.traverse(node.body) + self.write(" if ") + self.traverse(node.test) + self.write(" else ") + self.set_precedence(_Precedence.TEST, node.orelse) + self.traverse(node.orelse) + + def visit_Set(self, node): + if node.elts: + with self.delimit("{", "}"): + self.interleave(lambda: self.write(", "), self.traverse, node.elts) + else: + # `{}` would be interpreted as a dictionary literal, and + # `set` might be shadowed. Thus: + self.write("{*()}") + + def visit_Dict(self, node): + def write_key_value_pair(k, v): + self.traverse(k) + self.write(": ") + self.traverse(v) + + def write_item(item): + k, v = item + if k is None: + # for dictionary unpacking operator in dicts {**{'y': 2}} + # see PEP 448 for details + self.write("**") + self.set_precedence(_Precedence.EXPR, v) + self.traverse(v) + else: + write_key_value_pair(k, v) + + with self.delimit("{", "}"): + self.interleave(lambda: self.write(", "), write_item, zip(node.keys, node.values)) + + def visit_Tuple(self, node): + with self.delimit("(", ")"): + self.items_view(self.traverse, node.elts) + + unop = {"Invert": "~", "Not": "not", "UAdd": "+", "USub": "-"} + unop_precedence = { + "not": _Precedence.NOT, + "~": _Precedence.FACTOR, + "+": _Precedence.FACTOR, + "-": _Precedence.FACTOR, + } + + def visit_UnaryOp(self, node): + operator = self.unop[node.op.__class__.__name__] + operator_precedence = self.unop_precedence[operator] + with self.require_parens(operator_precedence, node): + self.write(operator) + # factor prefixes (+, -, ~) shouldn't be seperated + # from the value they belong, (e.g: +1 instead of + 1) + if operator_precedence is not _Precedence.FACTOR: + self.write(" ") + self.set_precedence(operator_precedence, node.operand) + self.traverse(node.operand) + + binop = { + "Add": "+", + "Sub": "-", + "Mult": "*", + "MatMult": "@", + "Div": "/", + "Mod": "%", + "LShift": "<<", + "RShift": ">>", + "BitOr": "|", + "BitXor": "^", + "BitAnd": "&", + "FloorDiv": "//", + "Pow": "**", + } + + binop_precedence = { + "+": _Precedence.ARITH, + "-": _Precedence.ARITH, + "*": _Precedence.TERM, + "@": _Precedence.TERM, + "/": _Precedence.TERM, + "%": _Precedence.TERM, + "<<": _Precedence.SHIFT, + ">>": _Precedence.SHIFT, + "|": _Precedence.BOR, + "^": _Precedence.BXOR, + "&": _Precedence.BAND, + "//": _Precedence.TERM, + "**": _Precedence.POWER, + } + + binop_rassoc = frozenset(("**",)) + + def visit_BinOp(self, node): + operator = self.binop[node.op.__class__.__name__] + operator_precedence = self.binop_precedence[operator] + with self.require_parens(operator_precedence, node): + if operator in self.binop_rassoc: + left_precedence = operator_precedence.next() + right_precedence = operator_precedence + else: + left_precedence = operator_precedence + right_precedence = operator_precedence.next() + + self.set_precedence(left_precedence, node.left) + self.traverse(node.left) + self.write(f" {operator} ") + self.set_precedence(right_precedence, node.right) + self.traverse(node.right) + + cmpops = { + "Eq": "==", + "NotEq": "!=", + "Lt": "<", + "LtE": "<=", + "Gt": ">", + "GtE": ">=", + "Is": "is", + "IsNot": "is not", + "In": "in", + "NotIn": "not in", + } + + def visit_Compare(self, node): + with self.require_parens(_Precedence.CMP, node): + self.set_precedence(_Precedence.CMP.next(), node.left, *node.comparators) + self.traverse(node.left) + for o, e in zip(node.ops, node.comparators): + self.write(" " + self.cmpops[o.__class__.__name__] + " ") + self.traverse(e) + + boolops = {"And": "and", "Or": "or"} + boolop_precedence = {"and": _Precedence.AND, "or": _Precedence.OR} + + def visit_BoolOp(self, node): + operator = self.boolops[node.op.__class__.__name__] + operator_precedence = self.boolop_precedence[operator] + + def increasing_level_traverse(node): + nonlocal operator_precedence + operator_precedence = operator_precedence.next() + self.set_precedence(operator_precedence, node) + self.traverse(node) + + with self.require_parens(operator_precedence, node): + s = f" {operator} " + self.interleave(lambda: self.write(s), increasing_level_traverse, node.values) + + def visit_Attribute(self, node): + self.set_precedence(_Precedence.ATOM, node.value) + self.traverse(node.value) + # Special case: 3.__abs__() is a syntax error, so if node.value + # is an integer literal then we need to either parenthesize + # it or add an extra space to get 3 .__abs__(). + if isinstance(node.value, Constant) and isinstance(node.value.value, int): + self.write(" ") + self.write(".") + self.write(node.attr) + + def visit_Call(self, node): + self.set_precedence(_Precedence.ATOM, node.func) + self.traverse(node.func) + with self.delimit("(", ")"): + comma = False + for e in node.args: + if comma: + self.write(", ") + else: + comma = True + self.traverse(e) + for e in node.keywords: + if comma: + self.write(", ") + else: + comma = True + self.traverse(e) + + def visit_Subscript(self, node): + def is_simple_tuple(slice_value): + # when unparsing a non-empty tuple, the parentheses can be safely + # omitted if there aren't any elements that explicitly requires + # parentheses (such as starred expressions). + return ( + isinstance(slice_value, Tuple) + and slice_value.elts + and not any(isinstance(elt, Starred) for elt in slice_value.elts) + ) + + self.set_precedence(_Precedence.ATOM, node.value) + self.traverse(node.value) + with self.delimit("[", "]"): + if is_simple_tuple(node.slice): + self.items_view(self.traverse, node.slice.elts) + else: + self.traverse(node.slice) + + def visit_Starred(self, node): + self.write("*") + self.set_precedence(_Precedence.EXPR, node.value) + self.traverse(node.value) + + def visit_Ellipsis(self, node): + self.write("...") + + def visit_Slice(self, node): + if node.lower: + self.traverse(node.lower) + self.write(":") + if node.upper: + self.traverse(node.upper) + if node.step: + self.write(":") + self.traverse(node.step) + + def visit_arg(self, node): + self.write(node.arg) + if node.annotation: + self.write(": ") + self.traverse(node.annotation) + + def visit_arguments(self, node): + first = True + # normal arguments + all_args = node.posonlyargs + node.args + defaults = [None] * (len(all_args) - len(node.defaults)) + node.defaults + for index, elements in enumerate(zip(all_args, defaults), 1): + a, d = elements + if first: + first = False + else: + self.write(", ") + self.traverse(a) + if d: + self.write("=") + self.traverse(d) + if index == len(node.posonlyargs): + self.write(", /") + + # varargs, or bare '*' if no varargs but keyword-only arguments present + if node.vararg or node.kwonlyargs: + if first: + first = False + else: + self.write(", ") + self.write("*") + if node.vararg: + self.write(node.vararg.arg) + if node.vararg.annotation: + self.write(": ") + self.traverse(node.vararg.annotation) + + # keyword-only arguments + if node.kwonlyargs: + for a, d in zip(node.kwonlyargs, node.kw_defaults): + self.write(", ") + self.traverse(a) + if d: + self.write("=") + self.traverse(d) + + # kwargs + if node.kwarg: + if first: + first = False + else: + self.write(", ") + self.write("**" + node.kwarg.arg) + if node.kwarg.annotation: + self.write(": ") + self.traverse(node.kwarg.annotation) + + def visit_keyword(self, node): + if node.arg is None: + self.write("**") + else: + self.write(node.arg) + self.write("=") + self.traverse(node.value) + + def visit_Lambda(self, node): + with self.require_parens(_Precedence.TEST, node): + self.write("lambda ") + self.traverse(node.args) + self.write(": ") + self.set_precedence(_Precedence.TEST, node.body) + self.traverse(node.body) + + def visit_alias(self, node): + self.write(node.name) + if node.asname: + self.write(" as " + node.asname) + + def visit_withitem(self, node): + self.traverse(node.context_expr) + if node.optional_vars: + self.write(" as ") + self.traverse(node.optional_vars) + + +def unparse(ast_obj): + unparser = _Unparser() + return unparser(ast_obj) + + +def roundtrip(source): + tree = ast.parse(source) + if hasattr(ast, "unparse"): + return ast.unparse(tree) + return unparse(tree) diff --git a/src/hera/workflows/script.py b/src/hera/workflows/script.py index 114df8baa..e4c28f0bc 100644 --- a/src/hera/workflows/script.py +++ b/src/hera/workflows/script.py @@ -24,6 +24,7 @@ TemplateMixin, VolumeMountMixin, ) +from hera.workflows._unparse import roundtrip from hera.workflows.models import ( Inputs as ModelInputs, Lifecycle, @@ -328,18 +329,16 @@ def generate_source(self, instance: Script) -> str: script += copy.deepcopy(script_extra) script += "\n" - # content represents the function components, separated by new lines - # therefore, the actual code block occurs after the end parenthesis, which is a literal `):\n` - content = inspect.getsourcelines(instance.source)[0] - token_index, start_token = 1, ":\n" - for curr_index, curr_token in enumerate(content): - if start_token in curr_token: - # when we find the curr token we find the end of the function header. The next index is the - # starting point of the function body - token_index = curr_index + 1 + # We use ast parse/unparse to get the source code of the function + # in order to have consistent looking functions and getting rid of any comments + # parsing issues. + # See https://github.com/argoproj-labs/hera/issues/572 + content = roundtrip(inspect.getsource(instance.source)).splitlines() + for i, line in enumerate(content): + if line.startswith("def") or line.startswith("async def"): break - s = "".join(content[token_index:]) + s = "\n".join(content[i + 1 :]) script += textwrap.dedent(s) return textwrap.dedent(script)