Consolidate parsing into common_functions (#211)

This commit re-factors the adding of parsed arguments into a set
of functions that add related arguments. Each script can then simply
use some of these function as parent parsers when setting up the
ArgumentParser for that script.

Note: there are still some issues with how arguments are passed through
from run_end_to_end to the other scripts. For instance, the scenario
name is required for run_scenario.py but in theory not for
run_end_to_end.py. However, run_end_to_end.py calls run_scenario.py, so
you will get an error message if you don't provide it in
run_end_to_end.py and the error message will be a somewhat confusing
combination of run_end_to_end.py and run_scenario.py

One solution could be to get rid of scenario_id and make scenario a
required argument. The value can be either the ID or the name, and we
can add some functionality to smartly determine what is provided and
get the equivalent name/id from the db (if needed).

Possible future updates: 

 - remove reverting to a hard-coded default scenario_location in gridpath.common_functions.determine_scenario_directory() and simply 
use the default value from the argument parser. We might want to get rid 
of the function entirely and simply call os.path.join(scenario_location, 
scenario) whenever needed.
 - similarly for the database, I think it might be more clear to add the 
default db_path to the argument parser definition, rather than assigning it
in db.common_functions.connect_to_database.

gridpath/common_functions.py

@@ -3,6 +3,8 @@
 
 import os.path
 
+from argparse import ArgumentParser
+
 
 def determine_scenario_directory(scenario_location, scenario_name):
     """
@@ -36,3 +38,124 @@ def create_directory_if_not_exists(directory):
     """
     if not os.path.exists(directory):
         os.makedirs(directory)
+
+
+def get_scenario_location_parser():
+    """
+    Create ArgumentParser object which has the common set of arguments for
+    accessing local scenario data.
+
+    We can then simply add 'parents=[get_scenario_location_parser()]' when we
+    create a parser for a script to inherit these common arguments.
+
+    Note that 'add_help' is set to 'False' to avoid multiple `-h/--help` options
+    (one for parent and one for each child), which will throw an error.
+    :return:
+    """
+
+    parser = ArgumentParser(add_help=False)
+    parser.add_argument("--scenario_location", default="../scenarios",
+                        help="The path to the directory in which to create "
+                             "the scenario directory. Defaults to "
+                             "'../scenarios' if not specified.")
+
+    return parser
+
+
+def get_scenario_name_parser():
+    """
+    Create ArgumentParser object which has the common set of arguments for
+    getting the scenario name
+
+    We can then simply add 'parents=[get_scenario_names_parser()]' when we
+    create a parser for a script to inherit these common arguments.
+
+    Note that 'add_help' is set to 'False' to avoid multiple `-h/--help` options
+    (one for parent and one for each child), which will throw an error.
+    :return:
+    """
+
+    parser = ArgumentParser(add_help=False)
+    parser.add_argument("--scenario", required=True, type=str,
+                        help="Name of the scenario problem to solve.")
+
+    return parser
+
+
+def get_db_parser():
+    """
+    Create ArgumentParser object which has the common set of arguments for
+    accessing scenario data from the database.
+
+    We can then simply add 'parents=[get_db_parser()]' when we create a
+    parser for a script to inherit these common arguments.
+
+    Note that 'add_help' is set to 'False' to avoid multiple `-h/--help` options
+    (one for parent and one for each child), which will throw an error.
+    :return:
+    """
+
+    parser = ArgumentParser(add_help=False)
+    parser.add_argument("--database", default="../db/io.db",
+                        help="The database file path. Defaults to ../db/io.db "
+                             "if not specified")
+    parser.add_argument("--scenario_id", type=int,
+                        help="The scenario_id from the database. Not needed "
+                             "if scenario is specified.")
+    parser.add_argument("--scenario", type=str,
+                        help="The scenario_name from the database. Not "
+                             "needed if scenario_id is specified.")
+
+    return parser
+
+
+def get_solve_parser():
+    """
+    Create ArgumentParser object which has the common set of arguments for
+    solving a scenario (see run_scenario.py and run_end_to_end.py).
+
+    We can then simply add 'parents=[get_solve_parser()]' when we create a
+    parser for a script to inherit these common arguments.
+
+    Note that 'add_help' is set to 'False' to avoid multiple `-h/--help` options
+    (one for parent and one for each child), which will throw an error.
+    :return:
+    """
+
+    parser = ArgumentParser(add_help=False)
+
+    # Output options
+    parser.add_argument("--log", default=False, action="store_true",
+                        help="Log output to a file in the scenario's 'logs' "
+                             "directory as well as the terminal.")
+    parser.add_argument("--quiet", default=False, action="store_true",
+                        help="Don't print run output.")
+    # Solver options
+    parser.add_argument("--solver", help="Name of the solver to use. "
+                                         "GridPath will use Cbc if solver is "
+                                         "not specified here and a "
+                                         "'solver_options.csv' file does not "
+                                         "exist in the scenario directory.")
+    parser.add_argument("--solver_executable",
+                        help="The path to the solver executable to use. This "
+                             "is optional; if you don't specify it, "
+                             "Pyomo will look for the solver executable in "
+                             "your PATH. The solver specified with the "
+                             "--solver option must be the same as the solver "
+                             "for which you are providing an executable.")
+    parser.add_argument("--mute_solver_output", default=False,
+                        action="store_true",
+                        help="Don't print solver output if set to true.")
+    parser.add_argument("--write_solver_files_to_logs_dir", default=False,
+                        action="store_true", help="Write the temporary "
+                                                  "solver files to the logs "
+                                                  "directory.")
+    parser.add_argument("--keepfiles", default=False, action="store_true",
+                        help="Save temporary solver files.")
+    parser.add_argument("--symbolic", default=False, action="store_true",
+                        help="Use symbolic labels in solver files.")
+    # Flag for test runs (various changes in behavior)
+    parser.add_argument("--testing", default=False, action="store_true",
+                        help="Flag for test suite runs. Results not saved.")
+
+    return parser

gridpath/get_scenario_inputs.py

@@ -12,7 +12,7 @@
 from db.common_functions import connect_to_database
 from gridpath.auxiliary.auxiliary import get_scenario_id_and_name
 from gridpath.common_functions import determine_scenario_directory, \
-    create_directory_if_not_exists
+    create_directory_if_not_exists, get_db_parser, get_scenario_location_parser
 from gridpath.auxiliary.module_list import determine_modules, load_modules
 from gridpath.auxiliary.scenario_chars import OptionalFeatures, SubScenarios, \
     SubProblems, SolverOptions
@@ -107,24 +107,16 @@ def delete_prior_inputs(inputs_directory):
 
 def parse_arguments(args):
     """
-    :param arguments: the script arguments specified by the user
+    :param args: the script arguments specified by the user
     :return: the parsed known argument values (<class 'argparse.Namespace'>
     Python object)
 
     Parse the known arguments.
     """
-    parser = ArgumentParser(add_help=True)
-    parser.add_argument("--database", help="The database file path.")
-    parser.add_argument("--scenario_id",
-                        help="The scenario_id from the database. Not needed "
-                             "if scenario_name is specified.")
-    parser.add_argument("--scenario",
-                        help="The scenario_name from the database. Not "
-                             "needed if scenario_id is specified.")
-    parser.add_argument("--scenario_location",
-                        help="The path to the directory in which to create "
-                             "the scenario directory. Defaults to "
-                             "'../scenarios' if not specified.")
+    parser = ArgumentParser(
+        add_help=True,
+        parents=[get_db_parser(), get_scenario_location_parser()]
+    )
     parsed_arguments = parser.parse_known_args(args=args)[0]
 
     return parsed_arguments

gridpath/import_scenario_results.py

@@ -9,7 +9,8 @@
 import sys
 
 from gridpath.auxiliary.auxiliary import get_scenario_id_and_name
-from gridpath.common_functions import determine_scenario_directory
+from gridpath.common_functions import determine_scenario_directory, \
+    get_db_parser, get_scenario_location_parser
 from db.common_functions import connect_to_database
 from gridpath.auxiliary.module_list import determine_modules, load_modules
 from gridpath.auxiliary.scenario_chars import SubProblems
@@ -72,22 +73,18 @@ def import_results_into_database(loaded_modules, scenario_id, subproblems,
 
 def parse_arguments(args):
     """
-    Parse arguments
+    :param args: the script arguments specified by the user
+    :return: the parsed known argument values (<class 'argparse.Namespace'>
+    Python object)
+
+    Parse the known arguments.
     :param args: 
     :return: 
     """
-    parser = ArgumentParser(add_help=True)
-    parser.add_argument("--database", help="The database file path.")
-    parser.add_argument("--scenario",
-                        help="The name of the scenario. Not needed if "
-                             "scenario_id is specified.")
-    parser.add_argument("--scenario_id",
-                        help="The scenario_id from the database. Not needed "
-                             "if scenario_name is specified.")
-    parser.add_argument("--scenario_location",
-                        help="The path to the base directory where the "
-                             "scenario directory is located. Defaults to "
-                             "'../scenarios' if not specified.")
+    parser = ArgumentParser(
+        add_help=True,
+        parents=[get_db_parser(), get_scenario_location_parser()]
+    )
     parsed_arguments = parser.parse_known_args(args=args)[0]
 
     return parsed_arguments

gridpath/process_results.py

@@ -7,7 +7,8 @@
 import sys
 
 from db.common_functions import connect_to_database
-from gridpath.common_functions import determine_scenario_directory
+from gridpath.common_functions import determine_scenario_directory, \
+    get_db_parser, get_scenario_location_parser
 from gridpath.auxiliary.auxiliary import get_scenario_id_and_name
 from gridpath.auxiliary.module_list import determine_modules, load_modules
 from gridpath.auxiliary.scenario_chars import SubScenarios
@@ -34,24 +35,16 @@ def process_results(
 
 def parse_arguments(args):
     """
-    :param arguments: the script arguments specified by the user
+    :param args: the script arguments specified by the user
     :return: the parsed known argument values (<class 'argparse.Namespace'>
     Python object)
 
     Parse the known arguments.
     """
-    parser = ArgumentParser(add_help=True)
-    parser.add_argument("--database", help="The database file path.")
-    parser.add_argument("--scenario",
-                        help="The name of the scenario. Not needed if "
-                             "scenario_id is specified.")
-    parser.add_argument("--scenario_id",
-                        help="The scenario_id from the database. Not needed "
-                             "if scenario_name is specified.")
-    parser.add_argument("--scenario_location",
-                        help="The path to the directory in which the scenario "
-                             "directory is located. Defaults to "
-                             "'../scenarios' if not specified.")
+    parser = ArgumentParser(
+        add_help=True,
+        parents=[get_db_parser(), get_scenario_location_parser()]
+    )
     parsed_arguments = parser.parse_known_args(args=args)[0]
 
     return parsed_arguments

gridpath/run_end_to_end.py

@@ -12,66 +12,28 @@
 
 # GridPath modules
 from db.common_functions import connect_to_database, spin_on_database_lock
+from gridpath.common_functions import get_db_parser, get_solve_parser, \
+    get_scenario_location_parser
 from gridpath import get_scenario_inputs, run_scenario, \
     import_scenario_results, process_results
 
 
-# TODO: can arguments be consolidated somehow with the other scripts
-def parse_arguments(arguments):
+def parse_arguments(args):
     """
-    :param arguments: the script arguments specified by the user
+    :param args: the script arguments specified by the user
     :return: the parsed known argument values (<class 'argparse.Namespace'>
     Python object)
 
     Parse the known arguments.
     """
-    parser = ArgumentParser(add_help=True)
-
-    # The database file path
-    parser.add_argument("--database", help="The database file path.")
-
-    # Scenario name and location options
-    parser.add_argument("--scenario",
-                        help="The name of the scenario (the same as "
-                             "the directory name)")
-    parser.add_argument("--scenario_location",
-                        help="The path to the directory in which to create "
-                             "the scenario directory.")
-    # Output options
-    parser.add_argument("--log", default=False, action="store_true",
-                        help="Log output to a file in the logs directory as "
-                             "well as the terminal.")
-    parser.add_argument("--quiet", default=False, action="store_true",
-                        help="Don't print run output.")
-
-    # Solve options
-    parser.add_argument("--solver", default="cbc",
-                        help="Name of the solver to use. Default is cbc.")
-    parser.add_argument("--solver_executable",
-                        help="The path to the solver executable to use. This "
-                             "is optional; if you don't specify it, "
-                             "Pyomo will look for the solver executable in "
-                             "your PATH. The solver specified with the "
-                             "--solver option must be the same as the solver "
-                             "for which you are providing an executable.")
-    parser.add_argument("--mute_solver_output", default=True,
-                        action="store_false",
-                        help="Don't print solver output if set to true.")
-    parser.add_argument("--write_solver_files_to_logs_dir", default=False,
-                        action="store_true", help="Write the temporary "
-                                                  "solver files to the logs "
-                                                  "directory.")
-    parser.add_argument("--keepfiles", default=False, action="store_true",
-                        help="Save temporary solver files.")
-    parser.add_argument("--symbolic", default=False, action="store_true",
-                        help="Use symbolic labels in solver files.")
-
-    # Flag for test runs (various changes in behavior)
-    parser.add_argument("--testing", default=False, action="store_true",
-                        help="Flag for test suite runs. Results not saved.")
-
-    # Parse arguments
-    parsed_arguments = parser.parse_known_args(args=arguments)[0]
+
+    parser = ArgumentParser(
+        add_help=True,
+        parents=[get_db_parser(), get_scenario_location_parser(),
+                 get_solve_parser()]
+    )
+
+    parsed_arguments = parser.parse_args(args=args)
 
     return parsed_arguments
 
@@ -100,6 +62,12 @@ def update_run_status(db_path, scenario, status_id):
 
 
 # TODO: add more run status types?
+# TODO: handle case where scenario_name is not specified but ID is (run_scenario
+#   will fail right now, as well as the update_run_status() calls (?)
+# TODO: handle error messages for parser: the argparser error message will refer
+#   to run_end_to_end.py, even if the parsing fails at one of the scripts
+#   being called here (e.g. run_scenario.py), while the listed arguments refer
+#   to the parser used when the script fails
 def main(args=None):
 
     if args is None: