Merge pull request #1412 from mvdbeek/fix_instance_id

Fix workflow download when using instance id

.github/workflows/ci.yaml

@@ -17,7 +17,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.7']
+        python-version: ['3.8']
         tox-action:
           - lint
           - lint_docs
@@ -30,8 +30,8 @@ jobs:
           #- unit-diagnostic-servetraining
           #- unit-diagnostic-servecmd
           #- unit-diagnostic-trainingwfcmd
-          - unit-nonredundant-noclientbuild-noshed-gx-2105
           - unit-nonredundant-noclientbuild-noshed-gx-2109
+          - unit-nonredundant-noclientbuild-noshed-gx-2205
           - unit-nonredundant-noclientbuild-noshed-gx-dev
           - unit-nonredundant-noclientbuild-noshed-nogx
           - unit-diagnostic-serveclientcmd

docs/best_practices_workflows.rst

@@ -125,7 +125,7 @@ the ``--from_invocation`` option.
 
 ::
 
-    $ planemo workflow_test_init --from_invocation <INVOCATION ID> --galaxy_url <GALAXY SERVER URL> --galaxy_user_key" <GALAXY API KEY>
+    $ planemo workflow_test_init --from_invocation <INVOCATION_ID> --galaxy_url <GALAXY SERVER URL> --galaxy_user_key" <GALAXY API KEY>
 
 You also need to specify the server URL and your API key, as Galaxy invocation IDs are
 only unique to a particular server. You can obtain the invocation ID from

docs/commands/workflow_test_on_invocation.rst

@@ -8,7 +8,7 @@ This section is auto-generated from the help text for the planemo command
 
 **Usage**::
 
-    planemo workflow_test_on_invocation [OPTIONS] TEST DEFINITION
+    planemo workflow_test_on_invocation [OPTIONS] TEST.YML INVOCATION_ID
 
 **Help**
 

planemo/commands/cmd_workflow_test_on_invocation.py

@@ -6,25 +6,28 @@
 from planemo.engine.factory import engine_context
 from planemo.galaxy.activity import invocation_to_run_response
 from planemo.galaxy.test.actions import handle_reports_and_summary
+from planemo.galaxy.workflows import GALAXY_WORKFLOW_INSTANCE_PREFIX
 from planemo.runnable import definition_to_test_case
 from planemo.runnable_resolve import for_runnable_identifier
 from planemo.test.results import StructuredData
 
 
 @click.command("workflow_test_on_invocation")
-@options.optional_tools_arg(multiple=False, allow_uris=False, metavar="TEST DEFINITION")
-@options.required_workflow_arg()
+@options.optional_tools_arg(multiple=False, allow_uris=False, metavar="TEST.YML")
+@options.required_invocation_id_arg()
 @options.galaxy_url_option(required=True)
 @options.galaxy_user_key_option(required=True)
 @options.test_index_option()
 @options.test_output_options()
 @command_function
-def cli(ctx, path, workflow_identifier, test_index, **kwds):
+def cli(ctx, path, invocation_id, test_index, **kwds):
     """Run defined tests against existing workflow invocation."""
     with engine_context(ctx, engine="external_galaxy", **kwds) as engine, engine.ensure_runnables_served([]) as config:
         user_gi = config.user_gi
-        invocation = user_gi.invocations.show_invocation(workflow_identifier)
-        runnable = for_runnable_identifier(ctx, invocation["workflow_id"], kwds)
+        invocation = user_gi.invocations.show_invocation(invocation_id)
+        runnable = for_runnable_identifier(
+            ctx, f"{GALAXY_WORKFLOW_INSTANCE_PREFIX}{invocation['workflow_id']}?runnable_path={path}", kwds
+        )
         test_cases = definition_to_test_case(path, runnable)
         assert (
             len(test_cases) >= test_index

planemo/galaxy/activity.py

@@ -257,7 +257,7 @@ def invocation_to_run_response(
         no_wait=no_wait,
         polling_backoff=polling_backoff,
     )
-    if final_invocation_state not in ("ok", "skipped"):
+    if final_invocation_state not in ("ok", "skipped", "scheduled"):
         msg = f"Failed to run workflow [{workflow_id}], at least one job is in [{final_invocation_state}] state."
         ctx.vlog(msg)
         summarize_history(ctx, user_gi, history_id)
@@ -763,10 +763,6 @@ def _history_id(gi, **kwds) -> str:
     return history_id
 
 
-def get_dict_from_workflow(gi, workflow_id):
-    return gi.workflows.export_workflow_dict(workflow_id)
-
-
 def wait_for_invocation_and_jobs(
     ctx, invocation_id: str, history_id: str, user_gi: GalaxyInstance, no_wait: bool, polling_backoff: int
 ):

planemo/galaxy/api.py

@@ -7,6 +7,10 @@
 DEFAULT_ADMIN_API_KEY = "test_key"
 
 
+def get_dict_from_workflow(gi: GalaxyInstance, workflow_id: str, instance: bool = False):
+    return gi.workflows._get(f"{workflow_id}/download", params={"instance": instance})
+
+
 def gi(port: Optional[int] = None, url: Optional[str] = None, key: Optional[str] = None) -> GalaxyInstance:
     """Return a bioblend ``GalaxyInstance`` for Galaxy on this port."""
     if key is None:

planemo/galaxy/workflows.py

@@ -25,11 +25,15 @@
     outputs_normalized,
 )
 
-from planemo.galaxy.api import gi
+from planemo.galaxy.api import (
+    get_dict_from_workflow,
+    gi,
+)
 from planemo.io import warn
 
 FAILED_REPOSITORIES_MESSAGE = "Failed to install one or more repositories."
 GALAXY_WORKFLOWS_PREFIX = "gxid://workflows/"
+GALAXY_WORKFLOW_INSTANCE_PREFIX = "gxid://workflow-instance/"
 
 
 def load_shed_repos(runnable):
@@ -156,10 +160,11 @@ def remote_runnable_to_workflow_id(runnable):
 
 def describe_outputs(runnable, gi=None):
     """Return a list of :class:`WorkflowOutput` objects for target workflow."""
-    if runnable.uri.startswith(GALAXY_WORKFLOWS_PREFIX):
+    if runnable.uri.startswith((GALAXY_WORKFLOWS_PREFIX, GALAXY_WORKFLOW_INSTANCE_PREFIX)):
         workflow_id = remote_runnable_to_workflow_id(runnable)
         assert gi is not None
-        workflow = get_dict_from_workflow(gi, workflow_id)
+        instance = runnable.uri.startswith(GALAXY_WORKFLOW_INSTANCE_PREFIX)
+        workflow = get_dict_from_workflow(gi, workflow_id, instance)
     else:
         workflow = _raw_dict(runnable.path)
 
@@ -291,10 +296,6 @@ def new_workflow_associated_path(workflow_path, suffix="tests"):
     return base + sep + suffix + "." + ext
 
 
-def get_dict_from_workflow(gi, workflow_id):
-    return gi.workflows.export_workflow_dict(workflow_id)
-
-
 def rewrite_job_file(input_file, output_file, job):
     """Rewrite a job file with galaxy_ids for upload_data subcommand"""
     with open(input_file) as f:
@@ -310,11 +311,10 @@ def rewrite_job_file(input_file, output_file, job):
 def get_workflow_from_invocation_id(invocation_id, galaxy_url, galaxy_api_key):
     user_gi = gi(url=galaxy_url, key=galaxy_api_key)
     workflow_id = user_gi.invocations.show_invocation(invocation_id)["workflow_id"]
-    workflow = user_gi.workflows._get(workflow_id, params={"instance": "true"})
+    workflow = get_dict_from_workflow(user_gi, workflow_id, instance=True)
     workflow_name = "-".join(workflow["name"].split())
-    export_dict = user_gi.workflows.export_workflow_dict(workflow_id=workflow["id"], version=workflow["version"])
     with open(f"{workflow_name}.ga", "w") as workflow_out:
-        json.dump(export_dict, workflow_out, indent=4)
+        json.dump(workflow, workflow_out, indent=4)
     return workflow_name
 
 

planemo/options.py

@@ -754,6 +754,14 @@ def required_workflow_arg():
     )
 
 
+def required_invocation_id_arg():
+    return click.argument(
+        "invocation_id",
+        metavar="INVOCATION_ID",
+        type=str,
+    )
+
+
 def split_job_and_test():
     return click.option(
         "--split_test/--no_split_test", default=False, help="Write workflow job and test definitions to separate files."

planemo/runnable.py

@@ -37,6 +37,7 @@
 )
 from planemo.galaxy.workflows import (
     describe_outputs,
+    GALAXY_WORKFLOW_INSTANCE_PREFIX,
     GALAXY_WORKFLOWS_PREFIX,
     WorkflowOutput,
 )
@@ -115,7 +116,7 @@ def has_path(self):
 
     @property
     def is_remote_workflow_uri(self) -> bool:
-        return self.uri.startswith(GALAXY_WORKFLOWS_PREFIX)
+        return self.uri.startswith((GALAXY_WORKFLOWS_PREFIX, GALAXY_WORKFLOW_INSTANCE_PREFIX))
 
     @property
     def test_data_search_path(self) -> str:

planemo/runnable_resolve.py

@@ -3,7 +3,10 @@
 import requests
 
 from planemo.galaxy.profiles import translate_alias
-from planemo.galaxy.workflows import GALAXY_WORKFLOWS_PREFIX
+from planemo.galaxy.workflows import (
+    GALAXY_WORKFLOW_INSTANCE_PREFIX,
+    GALAXY_WORKFLOWS_PREFIX,
+)
 from planemo.tools import uri_to_path
 from .runnable import (
     for_path,
@@ -17,14 +20,14 @@ def for_runnable_identifier(ctx, runnable_identifier, kwds):
     # could be a URI, path, or alias
     current_profile = kwds.get("profile")
     runnable_identifier = translate_alias(ctx, runnable_identifier, current_profile)
-    if not runnable_identifier.startswith(GALAXY_WORKFLOWS_PREFIX):
+    if not runnable_identifier.startswith((GALAXY_WORKFLOWS_PREFIX, GALAXY_WORKFLOW_INSTANCE_PREFIX)):
         runnable_identifier = uri_to_path(ctx, runnable_identifier)
     if os.path.exists(runnable_identifier):
         runnable = for_path(runnable_identifier)
     else:  # assume galaxy workflow or tool id
         if "/repos/" in runnable_identifier:
             runnable_identifier = f"{GALAXY_TOOLS_PREFIX}{runnable_identifier}"
-        elif not runnable_identifier.startswith(GALAXY_WORKFLOWS_PREFIX):
+        elif not runnable_identifier.startswith("gxid://"):
             runnable_identifier = f"{GALAXY_WORKFLOWS_PREFIX}{runnable_identifier}"
         runnable = for_uri(runnable_identifier)
     return runnable

tox.ini

@@ -61,8 +61,8 @@ setenv =
     master: PLANEMO_TEST_GALAXY_BRANCH=master
     dev: PLANEMO_TEST_GALAXY_BRANCH=dev
     2109: PLANEMO_TEST_GALAXY_BRANCH=release_21.09
-    2105: PLANEMO_TEST_GALAXY_BRANCH=release_21.05
     2101: PLANEMO_TEST_GALAXY_BRANCH=release_21.01
+    2205: PLANEMO_TEST_GALAXY_BRANCH=release_22.05
 skip_install =
     doc_test,lint,lint_docs,lint_docstrings,mypy,gxwf_test_test: True
 allowlist_externals =