diff --git a/nf_core/launch.py b/nf_core/launch.py index 7ecd99446f..978ffb29f1 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -12,45 +12,26 @@ import re import subprocess import textwrap - -import nf_core.schema - -# TODO: Would be nice to be able to capture keyboard interruptions in a nicer way -# add raise_keyboard_interrupt=True argument to PyInquirer.prompt() calls -# Requires a new release of PyInquirer. See https://github.com/CITGuru/PyInquirer/issues/90 - -def launch_pipeline(pipeline, revision=None, command_only=False, params_in=None, params_out=None, save_all=False, show_hidden=False): - - logging.info("This tool ignores any pipeline parameter defaults overwritten by Nextflow config files or profiles\n") - - # Create a pipeline launch object - launcher = Launch(pipeline, revision, command_only, params_in, params_out, show_hidden) - - # Build the schema and starting inputs - if launcher.get_pipeline_schema() is False: - return False - launcher.set_schema_inputs() - launcher.merge_nxf_flag_schema() - - # Kick off the interactive wizard to collect user inputs - launcher.prompt_schema() - - # Validate the parameters that we now have - if not launcher.schema_obj.validate_params(): - return False - - # Strip out the defaults - if not save_all: - launcher.strip_default_params() - - # Build and launch the `nextflow run` command - launcher.build_command() - launcher.launch_workflow() +import webbrowser + +import nf_core.schema, nf_core.utils + +# +# NOTE: WE ARE USING A PRE-RELEASE VERSION OF PYINQUIRER +# +# This is so that we can capture keyboard interruptions in a nicer way +# with the raise_keyboard_interrupt=True argument in the PyInquirer.prompt() calls +# It also allows list selections to have a default set. +# +# Waiting for a release of version of >1.0.3 of PyInquirer. +# See https://github.com/CITGuru/PyInquirer/issues/90 +# +# When available, update setup.py to use regular pip version class Launch(object): """ Class to hold config option to launch a pipeline """ - def __init__(self, pipeline, revision=None, command_only=False, params_in=None, params_out=None, show_hidden=False): + def __init__(self, pipeline=None, revision=None, command_only=False, params_in=None, params_out=None, save_all=False, show_hidden=False, url=None, web_id=None): """Initialise the Launcher class Args: @@ -60,18 +41,18 @@ def __init__(self, pipeline, revision=None, command_only=False, params_in=None, self.pipeline = pipeline self.pipeline_revision = revision self.schema_obj = None - self.use_params_file = True - if command_only: - self.use_params_file = False + self.use_params_file = False if command_only else True self.params_in = params_in - if params_out: - self.params_out = params_out - else: - self.params_out = os.path.join(os.getcwd(), 'nf-params.json') - self.show_hidden = False - if show_hidden: - self.show_hidden = True - + self.params_out = params_out if params_out else os.path.join(os.getcwd(), 'nf-params.json') + self.save_all = save_all + self.show_hidden = show_hidden + self.web_schema_launch_url = url if url else 'https://nf-co.re/launch' + self.web_schema_launch_web_url = None + self.web_schema_launch_api_url = None + self.web_id = web_id + if self.web_id: + self.web_schema_launch_web_url = '{}?id={}'.format(self.web_schema_launch_url, web_id) + self.web_schema_launch_api_url = '{}?id={}&api=true'.format(self.web_schema_launch_url, web_id) self.nextflow_cmd = 'nextflow run {}'.format(self.pipeline) # Prepend property names with a single hyphen in case we have parameters with the same ID @@ -79,22 +60,13 @@ def __init__(self, pipeline, revision=None, command_only=False, params_in=None, 'Nextflow command-line flags': { 'type': 'object', 'description': 'General Nextflow flags to control how the pipeline runs.', - 'help_text': """ - These are not specific to the pipeline and will not be saved - in any parameter file. They are just used when building the - `nextflow run` launch command. - """, + 'help_text': "These are not specific to the pipeline and will not be saved in any parameter file. They are just used when building the `nextflow run` launch command.", 'properties': { '-name': { 'type': 'string', 'description': 'Unique name for this nextflow run', 'pattern': '^[a-zA-Z0-9-_]+$' }, - '-revision': { - 'type': 'string', - 'description': 'Pipeline release / branch to use', - 'help_text': 'Revision of the project to run (either a git branch, tag or commit SHA number)' - }, '-profile': { 'type': 'string', 'description': 'Configuration profile' @@ -107,10 +79,7 @@ def __init__(self, pipeline, revision=None, command_only=False, params_in=None, '-resume': { 'type': 'boolean', 'description': 'Resume previous run, if found', - 'help_text': """ - Execute the script using the cached results, useful to continue - executions that was stopped by an error - """, + 'help_text': "Execute the script using the cached results, useful to continue executions that was stopped by an error", 'default': False } } @@ -119,13 +88,86 @@ def __init__(self, pipeline, revision=None, command_only=False, params_in=None, self.nxf_flags = {} self.params_user = {} + def launch_pipeline(self): + + # Check that we have everything we need + if self.pipeline is None and self.web_id is None: + logging.error("Either a pipeline name or web cache ID is required. Please see nf-core launch --help for more information.") + return False + + # Check if the output file exists already + if os.path.exists(self.params_out): + logging.warning("Parameter output file already exists! {}".format(os.path.relpath(self.params_out))) + if click.confirm(click.style('Do you want to overwrite this file? ', fg='yellow')+click.style('[y/N]', fg='red'), default=False, show_default=False): + os.remove(self.params_out) + logging.info("Deleted {}\n".format(self.params_out)) + else: + logging.info("Exiting. Use --params-out to specify a custom filename.") + return False + + + logging.info("This tool ignores any pipeline parameter defaults overwritten by Nextflow config files or profiles\n") + + # Check if we have a web ID + if self.web_id is not None: + self.schema_obj = nf_core.schema.PipelineSchema() + try: + if not self.get_web_launch_response(): + logging.info("Waiting for form to be completed in the browser. Remember to click Finished when you're done.") + logging.info("URL: {}".format(self.web_schema_launch_web_url)) + nf_core.utils.wait_cli_function(self.get_web_launch_response) + except AssertionError as e: + logging.error(click.style(e.args[0], fg='red')) + return False + + # Build the schema and starting inputs + if self.get_pipeline_schema() is False: + return False + self.set_schema_inputs() + self.merge_nxf_flag_schema() + + if self.prompt_web_gui(): + try: + self.launch_web_gui() + except AssertionError as e: + logging.error(click.style(e.args[0], fg='red')) + return False + else: + # Kick off the interactive wizard to collect user inputs + self.prompt_schema() + + # Validate the parameters that we now have + if not self.schema_obj.validate_params(): + return False + + # Strip out the defaults + if not self.save_all: + self.strip_default_params() + + # Build and launch the `nextflow run` command + self.build_command() + self.launch_workflow() + def get_pipeline_schema(self): """ Load and validate the schema from the supplied pipeline """ - # Get the schema + # Set up the schema self.schema_obj = nf_core.schema.PipelineSchema() + + # Check if this is a local directory + if os.path.exists(self.pipeline): + # Set the nextflow launch command to use full paths + self.nextflow_cmd = 'nextflow run {}'.format(os.path.abspath(self.pipeline)) + else: + # Assume nf-core if no org given + if self.pipeline.count('/') == 0: + self.nextflow_cmd = 'nextflow run nf-core/{}'.format(self.pipeline) + # Add revision flag to commands if set + if self.pipeline_revision: + self.nextflow_cmd += ' -r {}'.format(self.pipeline_revision) + + # Get schema from name, load it and lint it try: - # Get schema from name, load it and lint it self.schema_obj.get_schema_path(self.pipeline, revision=self.pipeline_revision) self.schema_obj.load_lint_schema() except AssertionError: @@ -172,6 +214,127 @@ def merge_nxf_flag_schema(self): schema_params.update(self.schema_obj.schema['properties']) self.schema_obj.schema['properties'] = schema_params + def prompt_web_gui(self): + """ Ask whether to use the web-based or cli wizard to collect params """ + + # Check whether --id was given and we're loading params from the web + if self.web_schema_launch_web_url is not None and self.web_schema_launch_api_url is not None: + return True + + click.secho("\nWould you like to enter pipeline parameters using a web-based interface or a command-line wizard?\n", fg='magenta') + question = { + 'type': 'list', + 'name': 'use_web_gui', + 'message': 'Choose launch method', + 'choices': [ + 'Web based', + 'Command line' + ] + } + answer = PyInquirer.prompt([question], raise_keyboard_interrupt=True) + return answer['use_web_gui'] == 'Web based' + + def launch_web_gui(self): + """ Send schema to nf-core website and launch input GUI """ + + # If --id given on the command line, we already know the URLs + if self.web_schema_launch_web_url is None and self.web_schema_launch_api_url is None: + content = { + 'post_content': 'json_schema_launcher', + 'api': 'true', + 'version': nf_core.__version__, + 'status': 'waiting_for_user', + 'schema': json.dumps(self.schema_obj.schema), + 'nxf_flags': json.dumps(self.nxf_flags), + 'input_params': json.dumps(self.schema_obj.input_params), + 'cli_launch': True, + 'nextflow_cmd': self.nextflow_cmd, + 'pipeline': self.pipeline, + 'revision': self.pipeline_revision + } + web_response = nf_core.utils.poll_nfcore_web_api(self.web_schema_launch_url, content) + try: + assert 'api_url' in web_response + assert 'web_url' in web_response + assert web_response['status'] == 'recieved' + except (AssertionError) as e: + logging.debug("Response content:\n{}".format(json.dumps(web_response, indent=4))) + raise AssertionError("Web launch response not recognised: {}\n See verbose log for full response (nf-core -v launch)".format(self.web_schema_launch_url)) + else: + self.web_schema_launch_web_url = web_response['web_url'] + self.web_schema_launch_api_url = web_response['api_url'] + + # ID supplied - has it been completed or not? + else: + logging.debug("ID supplied - checking status at {}".format(self.web_schema_launch_api_url)) + if self.get_web_launch_response(): + return True + + # Launch the web GUI + logging.info("Opening URL: {}".format(self.web_schema_launch_web_url)) + webbrowser.open(self.web_schema_launch_web_url) + logging.info("Waiting for form to be completed in the browser. Remember to click Finished when you're done.\n") + nf_core.utils.wait_cli_function(self.get_web_launch_response) + + def get_web_launch_response(self): + """ + Given a URL for a web-gui launch response, recursively query it until results are ready. + """ + web_response = nf_core.utils.poll_nfcore_web_api(self.web_schema_launch_api_url) + if web_response['status'] == 'error': + raise AssertionError("Got error from launch API ({})".format(web_response.get('message'))) + elif web_response['status'] == 'waiting_for_user': + return False + elif web_response['status'] == 'launch_params_complete': + logging.info("Found completed parameters from nf-core launch GUI") + try: + # Set everything that we can with the cache results + # NB: If using web builder, may have only run with --id and nothing else + if len(web_response['nxf_flags']) > 0: + self.nxf_flags = web_response['nxf_flags'] + if len(web_response['input_params']) > 0: + self.schema_obj.input_params = web_response['input_params'] + self.schema_obj.schema = web_response['schema'] + self.cli_launch = web_response['cli_launch'] + self.nextflow_cmd = web_response['nextflow_cmd'] + self.pipeline = web_response['pipeline'] + self.pipeline_revision = web_response['revision'] + # Sanitise form inputs, set proper variable types etc + self.sanitise_web_response() + except KeyError as e: + raise AssertionError("Missing return key from web API: {}".format(e)) + except Exception as e: + logging.debug(web_response) + raise AssertionError("Unknown exception ({}) - see verbose log for details. {}".format(type(e).__name__, e)) + return True + else: + logging.debug("Response content:\n{}".format(json.dumps(web_response, indent=4))) + raise AssertionError("Web launch GUI returned unexpected status ({}): {}\n See verbose log for full response".format(web_response['status'], self.web_schema_launch_api_url)) + + def sanitise_web_response(self): + """ + The web builder returns everything as strings. + Use the functions defined in the cli wizard to convert to the correct types. + """ + # Collect pyinquirer objects for each defined input_param + pyinquirer_objects = {} + for param_id, param_obj in self.schema_obj.schema['properties'].items(): + if(param_obj['type'] == 'object'): + for child_param_id, child_param_obj in param_obj['properties'].items(): + pyinquirer_objects[child_param_id] = self.single_param_to_pyinquirer(child_param_id, child_param_obj, print_help=False) + else: + pyinquirer_objects[param_id] = self.single_param_to_pyinquirer(param_id, param_obj, print_help=False) + + # Go through input params and sanitise + for params in [self.nxf_flags, self.schema_obj.input_params]: + for param_id in list(params.keys()): + # Remove if an empty string + if str(params[param_id]).strip() == '': + del params[param_id] + # Run filter function on value + filter_func = pyinquirer_objects.get(param_id, {}).get('filter') + if filter_func is not None: + params[param_id] = filter_func(params[param_id]) def prompt_schema(self): """ Go through the pipeline schema and prompt user to change defaults """ @@ -202,12 +365,12 @@ def prompt_param(self, param_id, param_obj, is_required, answers): # Print the question question = self.single_param_to_pyinquirer(param_id, param_obj, answers) - answer = PyInquirer.prompt([question]) + answer = PyInquirer.prompt([question], raise_keyboard_interrupt=True) # If required and got an empty reponse, ask again while type(answer[param_id]) is str and answer[param_id].strip() == '' and is_required: click.secho("Error - this property is required.", fg='red', err=True) - answer = PyInquirer.prompt([question]) + answer = PyInquirer.prompt([question], raise_keyboard_interrupt=True) # Don't return empty answers if answer[param_id] == '': @@ -251,7 +414,7 @@ def prompt_group(self, param_id, param_obj): answers = {} while not while_break: self.print_param_header(param_id, param_obj) - answer = PyInquirer.prompt([question]) + answer = PyInquirer.prompt([question], raise_keyboard_interrupt=True) if answer[param_id] == 'Continue >>': while_break = True # Check if there are any required parameters that don't have answers @@ -269,7 +432,7 @@ def prompt_group(self, param_id, param_obj): return answers - def single_param_to_pyinquirer(self, param_id, param_obj, answers=None): + def single_param_to_pyinquirer(self, param_id, param_obj, answers=None, print_help=True): """Convert a JSONSchema param to a PyInquirer question Args: @@ -289,17 +452,20 @@ def single_param_to_pyinquirer(self, param_id, param_obj, answers=None): } # Print the name, description & help text - nice_param_id = '--{}'.format(param_id) if not param_id.startswith('-') else param_id - self.print_param_header(nice_param_id, param_obj) + if print_help: + nice_param_id = '--{}'.format(param_id) if not param_id.startswith('-') else param_id + self.print_param_header(nice_param_id, param_obj) if param_obj.get('type') == 'boolean': - question['type'] = 'confirm' - question['default'] = False + question['type'] = 'list' + question['choices'] = ['True', 'False'] + question['default'] = 'False' # Start with the default from the param object if 'default' in param_obj: + # Boolean default is cast back to a string later - this just normalises all inputs if param_obj['type'] == 'boolean' and type(param_obj['default']) is str: - question['default'] = 'true' == param_obj['default'].lower() + question['default'] = param_obj['default'].lower() == 'true' else: question['default'] = param_obj['default'] @@ -314,10 +480,18 @@ def single_param_to_pyinquirer(self, param_id, param_obj, answers=None): if param_id in answers: question['default'] = answers[param_id] - # Coerce default to a string if not boolean - if param_obj.get('type') != 'boolean' and 'default' in question: + # Coerce default to a string + if 'default' in question: question['default'] = str(question['default']) + if param_obj.get('type') == 'boolean': + # Filter returned value + def filter_boolean(val): + if isinstance(val, bool): + return val + return val.lower() == 'true' + question['filter'] = filter_boolean + if param_obj.get('type') == 'number': # Validate number type def validate_number(val): @@ -424,10 +598,16 @@ def print_param_header(self, param_id, param_obj): def strip_default_params(self): """ Strip parameters if they have not changed from the default """ + # Schema defaults for param_id, val in self.schema_obj.schema_defaults.items(): - if self.schema_obj.input_params[param_id] == val: + if self.schema_obj.input_params.get(param_id) == val: del self.schema_obj.input_params[param_id] + # Nextflow flag defaults + for param_id, val in self.nxf_flag_schema['Nextflow command-line flags']['properties'].items(): + if param_id in self.nxf_flags and self.nxf_flags[param_id] == val.get('default'): + del self.nxf_flags[param_id] + def build_command(self): """ Build the nextflow run command based on what we know """ @@ -437,7 +617,7 @@ def build_command(self): if isinstance(val, bool) and val: self.nextflow_cmd += " {}".format(flag) # String values - else: + elif not isinstance(val, bool): self.nextflow_cmd += ' {} "{}"'.format(flag, val.replace('"', '\\"')) # Pipeline parameters @@ -457,7 +637,7 @@ def build_command(self): self.nextflow_cmd += " --{}".format(param) # everything else else: - self.nextflow_cmd += ' --{} "{}"'.format(param, val.replace('"', '\\"')) + self.nextflow_cmd += ' --{} "{}"'.format(param, str(val).replace('"', '\\"')) def launch_workflow(self): diff --git a/nf_core/schema.py b/nf_core/schema.py index 0b46308648..93a0df52a2 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -393,100 +393,42 @@ def launch_web_builder(self): 'status': 'waiting_for_user', 'schema': json.dumps(self.schema) } + web_response = nf_core.utils.poll_nfcore_web_api(self.web_schema_build_url, content) try: - response = requests.post(url=self.web_schema_build_url, data=content) - except (requests.exceptions.Timeout): - raise AssertionError("Schema builder URL timed out: {}".format(self.web_schema_build_url)) - except (requests.exceptions.ConnectionError): - raise AssertionError("Could not connect to schema builder URL: {}".format(self.web_schema_build_url)) + assert 'api_url' in web_response + assert 'web_url' in web_response + assert web_response['status'] == 'recieved' + except (AssertionError) as e: + logging.debug("Response content:\n{}".format(json.dumps(web_response, indent=4))) + raise AssertionError("JSON Schema builder response not recognised: {}\n See verbose log for full response (nf-core -v schema)".format(self.web_schema_build_url)) else: - if response.status_code != 200: - logging.debug("Response content:\n{}".format(response.content)) - raise AssertionError("Could not access remote JSON Schema builder: {} (HTML {} Error)".format(self.web_schema_build_url, response.status_code)) - else: - try: - web_response = json.loads(response.content) - assert 'status' in web_response - assert 'api_url' in web_response - assert 'web_url' in web_response - assert web_response['status'] == 'recieved' - except (json.decoder.JSONDecodeError, AssertionError) as e: - logging.debug("Response content:\n{}".format(response.content)) - raise AssertionError("JSON Schema builder response not recognised: {}\n See verbose log for full response (nf-core -v schema)".format(self.web_schema_build_url)) - else: - self.web_schema_build_web_url = web_response['web_url'] - self.web_schema_build_api_url = web_response['api_url'] - logging.info("Opening URL: {}".format(web_response['web_url'])) - webbrowser.open(web_response['web_url']) - logging.info("Waiting for form to be completed in the browser. Remember to click Finished when you're done.\n") - self.wait_web_builder_response() - - def wait_web_builder_response(self): - try: - is_saved = False - check_count = 0 - def spinning_cursor(): - while True: - for cursor in '⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏': - yield '{} Use ctrl+c to stop waiting and force exit. '.format(cursor) - spinner = spinning_cursor() - while not is_saved: - # Show the loading spinner every 0.1s - time.sleep(0.1) - loading_text = next(spinner) - sys.stdout.write(loading_text) - sys.stdout.flush() - sys.stdout.write('\b'*len(loading_text)) - # Only check every 2 seconds, but update the spinner every 0.1s - check_count += 1 - if check_count > 20: - is_saved = self.get_web_builder_response() - check_count = 0 - except KeyboardInterrupt: - raise AssertionError("Cancelled!") - + self.web_schema_build_web_url = web_response['web_url'] + self.web_schema_build_api_url = web_response['api_url'] + logging.info("Opening URL: {}".format(web_response['web_url'])) + webbrowser.open(web_response['web_url']) + logging.info("Waiting for form to be completed in the browser. Remember to click Finished when you're done.\n") + nf_core.utils.wait_cli_function(self.get_web_builder_response) def get_web_builder_response(self): """ Given a URL for a Schema build response, recursively query it until results are ready. Once ready, validate Schema and write to disk. """ - # Clear requests_cache so that we get the updated statuses - requests_cache.clear() - try: - response = requests.get(self.web_schema_build_api_url, headers={'Cache-Control': 'no-cache'}) - except (requests.exceptions.Timeout): - raise AssertionError("Schema builder URL timed out: {}".format(self.web_schema_build_api_url)) - except (requests.exceptions.ConnectionError): - raise AssertionError("Could not connect to schema builder URL: {}".format(self.web_schema_build_api_url)) - else: - if response.status_code != 200: - logging.debug("Response content:\n{}".format(response.content)) - raise AssertionError("Could not access remote JSON Schema builder results: {} (HTML {} Error)".format(self.web_schema_build_api_url, response.status_code)) + web_response = nf_core.utils.poll_nfcore_web_api(self.web_schema_build_api_url) + if web_response['status'] == 'error': + raise AssertionError("Got error from JSON Schema builder ( {} )".format(web_response.get('message'))) + elif web_response['status'] == 'waiting_for_user': + return False + elif web_response['status'] == 'web_builder_edited': + logging.info("Found saved status from nf-core JSON Schema builder") + try: + self.schema = web_response['schema'] + self.validate_schema(self.schema) + except AssertionError as e: + raise AssertionError("Response from JSON Builder did not pass validation:\n {}".format(e)) else: - try: - web_response = json.loads(response.content) - assert 'status' in web_response - except (json.decoder.JSONDecodeError, AssertionError) as e: - logging.debug("Response content:\n{}".format(response.content)) - raise AssertionError("JSON Schema builder results response not recognised: {}\n See verbose log for full response".format(self.web_schema_build_api_url)) - else: - if web_response['status'] == 'error': - raise AssertionError("Got error from JSON Schema builder ( {} )".format(click.style(web_response.get('message'), fg='red'))) - elif web_response['status'] == 'waiting_for_user': - return False - elif web_response['status'] == 'web_builder_edited': - logging.info("Found saved status from nf-core JSON Schema builder") - try: - self.schema = json.loads(web_response['schema']) - self.validate_schema(self.schema) - except json.decoder.JSONDecodeError as e: - raise AssertionError("Could not parse returned JSON:\n {}".format(e)) - except AssertionError as e: - raise AssertionError("Response from JSON Builder did not pass validation:\n {}".format(e)) - else: - self.save_schema() - return True - else: - logging.debug("Response content:\n{}".format(response.content)) - raise AssertionError("JSON Schema builder returned unexpected status ({}): {}\n See verbose log for full response".format(web_response['status'], self.web_schema_build_api_url)) + self.save_schema() + return True + else: + logging.debug("Response content:\n{}".format(json.dumps(web_response, indent=4))) + raise AssertionError("JSON Schema builder returned unexpected status ({}): {}\n See verbose log for full response".format(web_response['status'], self.web_schema_build_api_url)) diff --git a/nf_core/utils.py b/nf_core/utils.py index ed1fef3a5e..3332751434 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -10,8 +10,11 @@ import logging import os import re +import requests +import requests_cache import subprocess import sys +import time def fetch_wf_config(wf_path): """Uses Nextflow to retrieve the the configuration variables @@ -117,3 +120,72 @@ def setup_requests_cachedir(): expire_after=datetime.timedelta(hours=1), backend='sqlite', ) + +def wait_cli_function(poll_func, poll_every=20): + """ + Display a command-line spinner while calling a function repeatedly. + + Keep waiting until that function returns True + + Arguments: + poll_func (function): Function to call + poll_every (int): How many tenths of a second to wait between function calls. Default: 20. + + Returns: + None. Just sits in an infite loop until the function returns True. + """ + try: + is_finished = False + check_count = 0 + def spinning_cursor(): + while True: + for cursor in '⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏': + yield '{} Use ctrl+c to stop waiting and force exit. '.format(cursor) + spinner = spinning_cursor() + while not is_finished: + # Show the loading spinner every 0.1s + time.sleep(0.1) + loading_text = next(spinner) + sys.stdout.write(loading_text) + sys.stdout.flush() + sys.stdout.write('\b'*len(loading_text)) + # Only check every 2 seconds, but update the spinner every 0.1s + check_count += 1 + if check_count > poll_every: + is_finished = poll_func() + check_count = 0 + except KeyboardInterrupt: + raise AssertionError("Cancelled!") + +def poll_nfcore_web_api(api_url, post_data=None): + """ + Poll the nf-core website API + + Takes argument api_url for URL + + Expects API reponse to be valid JSON and contain a top-level 'status' key. + """ + # Clear requests_cache so that we get the updated statuses + requests_cache.clear() + try: + if post_data is None: + response = requests.get(api_url, headers={'Cache-Control': 'no-cache'}) + else: + response = requests.post(url=api_url, data=post_data) + except (requests.exceptions.Timeout): + raise AssertionError("URL timed out: {}".format(api_url)) + except (requests.exceptions.ConnectionError): + raise AssertionError("Could not connect to URL: {}".format(api_url)) + else: + if response.status_code != 200: + logging.debug("Response content:\n{}".format(response.content)) + raise AssertionError("Could not access remote API results: {} (HTML {} Error)".format(api_url, response.status_code)) + else: + try: + web_response = json.loads(response.content) + assert 'status' in web_response + except (json.decoder.JSONDecodeError, AssertionError) as e: + logging.debug("Response content:\n{}".format(response.content)) + raise AssertionError("nf-core website API results response not recognised: {}\n See verbose log for full response".format(api_url)) + else: + return web_response diff --git a/scripts/nf-core b/scripts/nf-core index 21461c39dd..71fd07fa06 100755 --- a/scripts/nf-core +++ b/scripts/nf-core @@ -91,9 +91,13 @@ def list(keywords, sort, json): @nf_core_cli.command(help_priority=2) @click.argument( 'pipeline', - required = True, + required = False, metavar = "" ) +@click.option( + '-i', '--id', + help = "ID for web-gui launch parameter set" +) @click.option( '-r', '--revision', help = "Release/branch/SHA of the project to run (if remote)" @@ -127,9 +131,16 @@ def list(keywords, sort, json): default = False, help = "Show hidden parameters." ) -def launch(pipeline, revision, command_only, params_in, params_out, save_all, show_hidden): +@click.option( + '--url', + type = str, + default = 'https://nf-co.re/launch', + help = 'Customise the builder URL (for development work)' +) +def launch(pipeline, id, revision, command_only, params_in, params_out, save_all, show_hidden, url): """ Run pipeline, interactive parameter prompts """ - if nf_core.launch.launch_pipeline(pipeline, revision, command_only, params_in, params_out, save_all, show_hidden) == False: + launcher = nf_core.launch.Launch(pipeline, revision, command_only, params_in, params_out, save_all, show_hidden, url, id) + if launcher.launch_pipeline() == False: sys.exit(1) # nf-core download diff --git a/setup.py b/setup.py index d3557a4627..65c2fefe22 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,9 @@ 'GitPython', 'jinja2', 'jsonschema', - 'PyInquirer', + # 'PyInquirer>1.0.3', + # Need the new release of PyInquirer, see nf_core/launch.py for details + 'PyInquirer @ https://github.com/CITGuru/PyInquirer/archive/master.zip', 'pyyaml', 'requests', 'requests_cache', diff --git a/tests/test_launch.py b/tests/test_launch.py index 88fbb5d68a..3234281cb2 100644 --- a/tests/test_launch.py +++ b/tests/test_launch.py @@ -5,6 +5,7 @@ import nf_core.launch import json +import mock import os import shutil import tempfile @@ -21,6 +22,12 @@ def setUp(self): self.nf_params_fn = os.path.join(tempfile.mkdtemp(), 'nf-params.json') self.launcher = nf_core.launch.Launch(self.template_dir, params_out = self.nf_params_fn) + @mock.patch.object(nf_core.launch.Launch, 'prompt_web_gui', side_effect=[True]) + @mock.patch.object(nf_core.launch.Launch, 'launch_web_gui') + def test_launch_pipeline(self, mock_webbrowser, mock_lauch_web_gui): + """ Test the main launch function """ + self.launcher.launch_pipeline() + def test_get_pipeline_schema(self): """ Test loading the params schema from a pipeline """ self.launcher.get_pipeline_schema() @@ -83,6 +90,133 @@ def test_ob_to_pyinquirer_string(self): 'default': 'data/*{1,2}.fastq.gz' } + @mock.patch('PyInquirer.prompt', side_effect=[{'use_web_gui': 'Web based'}]) + def test_prompt_web_gui_true(self, mock_prompt): + """ Check the prompt to launch the web schema or use the cli """ + assert self.launcher.prompt_web_gui() == True + + @mock.patch('PyInquirer.prompt', side_effect=[{'use_web_gui': 'Command line'}]) + def test_prompt_web_gui_false(self, mock_prompt): + """ Check the prompt to launch the web schema or use the cli """ + assert self.launcher.prompt_web_gui() == False + + def mocked_requests_post(**kwargs): + """ Helper function to emulate POST requests responses from the web """ + + class MockResponse: + def __init__(self, data, status_code): + self.status_code = status_code + self.content = json.dumps(data) + + if kwargs['url'] == 'https://nf-co.re/launch': + response_data = { + 'status': 'recieved', + 'api_url': 'https://nf-co.re', + 'web_url': 'https://nf-co.re', + 'status': 'recieved' + } + return MockResponse(response_data, 200) + + def mocked_requests_get(*args, **kwargs): + """ Helper function to emulate GET requests responses from the web """ + + class MockResponse: + def __init__(self, data, status_code): + self.status_code = status_code + self.content = json.dumps(data) + + if args[0] == 'valid_url_saved': + response_data = { + 'status': 'web_builder_edited', + 'message': 'testing', + 'schema': { "foo": "bar" } + } + return MockResponse(response_data, 200) + + @mock.patch('nf_core.utils.poll_nfcore_web_api', side_effect=[{}]) + def test_launch_web_gui_missing_keys(self, mock_poll_nfcore_web_api): + """ Check the code that opens the web browser """ + self.launcher.get_pipeline_schema() + self.launcher.merge_nxf_flag_schema() + try: + self.launcher.launch_web_gui() + except AssertionError as e: + assert e.args[0].startswith('Web launch response not recognised:') + + @mock.patch('nf_core.utils.poll_nfcore_web_api', side_effect=[{'api_url': 'foo', 'web_url': 'bar', 'status': 'recieved'}]) + @mock.patch('webbrowser.open') + @mock.patch('nf_core.utils.wait_cli_function') + def test_launch_web_gui(self, mock_poll_nfcore_web_api, mock_webbrowser, mock_wait_cli_function): + """ Check the code that opens the web browser """ + self.launcher.get_pipeline_schema() + self.launcher.merge_nxf_flag_schema() + assert self.launcher.launch_web_gui() == None + + @mock.patch.object(nf_core.launch.Launch, 'get_web_launch_response') + def test_launch_web_gui_id_supplied(self, mock_get_web_launch_response): + """ Check the code that opens the web browser """ + self.launcher.web_schema_launch_web_url = 'https://foo.com' + self.launcher.web_schema_launch_api_url = 'https://bar.com' + self.launcher.get_pipeline_schema() + self.launcher.merge_nxf_flag_schema() + assert self.launcher.launch_web_gui() == True + + @mock.patch('nf_core.utils.poll_nfcore_web_api', side_effect=[{'status': 'error', 'message': 'foo'}]) + def test_get_web_launch_response_error(self, mock_poll_nfcore_web_api): + """ Test polling the website for a launch response - status error """ + try: + self.launcher.get_web_launch_response() + except AssertionError as e: + assert e.args[0] == 'Got error from launch API (foo)' + + @mock.patch('nf_core.utils.poll_nfcore_web_api', side_effect=[{'status': 'foo'}]) + def test_get_web_launch_response_unexpected(self, mock_poll_nfcore_web_api): + """ Test polling the website for a launch response - status error """ + try: + self.launcher.get_web_launch_response() + except AssertionError as e: + assert e.args[0].startswith('Web launch GUI returned unexpected status (foo): ') + + @mock.patch('nf_core.utils.poll_nfcore_web_api', side_effect=[{'status': 'waiting_for_user'}]) + def test_get_web_launch_response_waiting(self, mock_poll_nfcore_web_api): + """ Test polling the website for a launch response - status waiting_for_user""" + assert self.launcher.get_web_launch_response() == False + + @mock.patch('nf_core.utils.poll_nfcore_web_api', side_effect=[{'status': 'launch_params_complete'}]) + def test_get_web_launch_response_missing_keys(self, mock_poll_nfcore_web_api): + """ Test polling the website for a launch response - complete, but missing keys """ + try: + self.launcher.get_web_launch_response() + except AssertionError as e: + assert e.args[0] == "Missing return key from web API: 'nxf_flags'" + + @mock.patch('nf_core.utils.poll_nfcore_web_api', side_effect=[{ + 'status': 'launch_params_complete', + 'nxf_flags': {'resume', 'true'}, + 'input_params': {'foo', 'bar'}, + 'schema': {}, + 'cli_launch': True, + 'nextflow_cmd': 'nextflow run foo', + 'pipeline': 'foo', + 'revision': 'bar', + }]) + @mock.patch.object(nf_core.launch.Launch, 'sanitise_web_response') + def test_get_web_launch_response_valid(self, mock_poll_nfcore_web_api, mock_sanitise): + """ Test polling the website for a launch response - complete, valid response """ + self.launcher.get_pipeline_schema() + assert self.launcher.get_web_launch_response() == True + + def test_sanitise_web_response(self): + """ Check that we can properly sanitise results from the web """ + self.launcher.get_pipeline_schema() + self.launcher.nxf_flags['-name'] = '' + self.launcher.schema_obj.input_params['single_end'] = 'true' + self.launcher.schema_obj.input_params['max_cpus'] = '12' + self.launcher.sanitise_web_response() + assert '-name' not in self.launcher.nxf_flags + assert self.launcher.schema_obj.input_params['single_end'] == True + assert self.launcher.schema_obj.input_params['max_cpus'] == 12 + def test_ob_to_pyinquirer_bool(self): """ Check converting a python dict to a pyenquirer format - booleans """ sc_obj = { @@ -90,12 +224,18 @@ def test_ob_to_pyinquirer_bool(self): "default": "True", } result = self.launcher.single_param_to_pyinquirer('single_end', sc_obj) - assert result == { - 'type': 'confirm', - 'name': 'single_end', - 'message': 'single_end', - 'default': True - } + assert result['type'] == 'list' + assert result['name'] == 'single_end' + assert result['message'] == 'single_end' + assert result['choices'] == ['True', 'False'] + assert result['default'] == 'True' + print(type(True)) + assert result['filter']('True') == True + assert result['filter']('true') == True + assert result['filter'](True) == True + assert result['filter']('False') == False + assert result['filter']('false') == False + assert result['filter'](False) == False def test_ob_to_pyinquirer_number(self): """ Check converting a python dict to a pyenquirer format - with enum """ diff --git a/tests/test_schema.py b/tests/test_schema.py index aac7284d16..41cce653f7 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -22,7 +22,10 @@ def setUp(self): """ Create a new PipelineSchema object """ self.schema_obj = nf_core.schema.PipelineSchema() self.root_repo_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) - self.template_dir = os.path.join(self.root_repo_dir, 'nf_core', 'pipeline-template', '{{cookiecutter.name_noslash}}') + # Copy the template to a temp directory so that we can use that for tests + self.template_dir = os.path.join(tempfile.mkdtemp(), 'wf') + template_dir = os.path.join(self.root_repo_dir, 'nf_core', 'pipeline-template', '{{cookiecutter.name_noslash}}') + shutil.copytree(template_dir, self.template_dir) self.template_schema = os.path.join(self.template_dir, 'nextflow_schema.json') def test_load_lint_schema(self): @@ -357,7 +360,7 @@ def test_launch_web_builder_404(self, mock_post): try: self.schema_obj.launch_web_builder() except AssertionError as e: - assert e.args[0] == 'Could not access remote JSON Schema builder: invalid_url (HTML 404 Error)' + assert e.args[0] == 'Could not access remote API results: invalid_url (HTML 404 Error)' @mock.patch('requests.post', side_effect=mocked_requests_post) def test_launch_web_builder_invalid_status(self, mock_post): @@ -378,7 +381,7 @@ def test_launch_web_builder_success(self, mock_post, mock_get, mock_webbrowser): self.schema_obj.launch_web_builder() except AssertionError as e: # Assertion error comes from get_web_builder_response() function - assert e.args[0].startswith('Could not access remote JSON Schema builder results: https://nf-co.re') + assert e.args[0].startswith('Could not access remote API results: https://nf-co.re') def mocked_requests_get(*args, **kwargs): @@ -410,7 +413,7 @@ def __init__(self, data, status_code): response_data = { 'status': 'web_builder_edited', 'message': 'testing', - 'schema': '{ "foo": "bar" }' + 'schema': { "foo": "bar" } } return MockResponse(response_data, 200) @@ -421,7 +424,7 @@ def test_get_web_builder_response_404(self, mock_post): try: self.schema_obj.get_web_builder_response() except AssertionError as e: - assert e.args[0] == "Could not access remote JSON Schema builder results: invalid_url (HTML 404 Error)" + assert e.args[0] == "Could not access remote API results: invalid_url (HTML 404 Error)" @mock.patch('requests.get', side_effect=mocked_requests_get) def test_get_web_builder_response_error(self, mock_post):