From dc74340b907516df8a2db816af73c69961f79291 Mon Sep 17 00:00:00 2001 From: Pete Bunting Date: Wed, 19 Aug 2020 11:34:51 +0100 Subject: [PATCH] Updated documentation for version 1.5.0. --- docs/source/arcsi_ls_example.rst | 39 +++++++++----------------------- docs/source/conf.py | 4 ++-- docs/source/getstarted.rst | 29 ++++++++---------------- 3 files changed, 22 insertions(+), 50 deletions(-) diff --git a/docs/source/arcsi_ls_example.rst b/docs/source/arcsi_ls_example.rst index b6e43ad..cbbb341 100644 --- a/docs/source/arcsi_ls_example.rst +++ b/docs/source/arcsi_ls_example.rst @@ -182,25 +182,17 @@ populating the ``c_dict`` dict, which gets written to the database:: # job_time_limit='2-23:59', # module_load='module load parallel singularity\n\nexport http_proxy="http://a.pfb:proxy101019@10.212.63.246:3128"\nexport https_proxy="http://a.pfb:proxy101019@10.212.63.246:3128"\n') - def run_check_outputs(self): - process_tools_mod = 'perform_dwnld_jobs' - process_tools_cls = 'PerformScnDownload' - time_sample_str = self.generate_readable_timestamp_str() - out_err_file = 'processing_errs_{}.txt'.format(time_sample_str) - out_non_comp_file = 'non_complete_errs_{}.txt'.format(time_sample_str) - self.check_job_outputs(process_tools_mod, process_tools_cls, out_err_file, out_non_comp_file) - - def run_remove_outputs(self, all_jobs=False, error_jobs=False): - process_tools_mod = 'perform_dwnld_jobs' - process_tools_cls = 'PerformScnDownload' - self.remove_job_outputs(process_tools_mod, process_tools_cls, all_jobs, error_jobs) - if __name__ == "__main__": py_script = os.path.abspath("perform_dwnld_jobs.py") #script_cmd = "singularity exec --bind /scratch/a.pfb:/scratch/a.pfb --bind /home/a.pfb:/home/a.pfb /scratch/a.pfb/sw_imgs/au-eoed-dev.sif python {}".format(py_script) script_cmd = "python {}".format(py_script) - create_tools = FindLandsatScnsGenDwnlds(cmd=script_cmd, sqlite_db_file="dwnld_ls_scns.db") + process_tools_mod = 'perform_dwnld_jobs' + process_tools_cls = 'PerformScnDownload' + + create_tools = FindLandsatScnsGenDwnlds(cmd=script_cmd, sqlite_db_file="dwnld_ls_scns.db", + process_tools_mod=process_tools_mod, + process_tools_cls=process_tools_cls) create_tools.parse_cmds() @@ -304,26 +296,17 @@ which have been downloaded but not processed to an ARD product:: # job_time_limit='2-23:59', # module_load='module load parallel singularity\n\n') - def run_check_outputs(self): - process_tools_mod = 'perform_ard_process' - process_tools_cls = 'PerformScnARD' - time_sample_str = self.generate_readable_timestamp_str() - out_err_file = 'processing_errs_{}.txt'.format(time_sample_str) - out_non_comp_file = 'non_complete_errs_{}.txt'.format(time_sample_str) - self.check_job_outputs(process_tools_mod, process_tools_cls, out_err_file, out_non_comp_file) - - def run_remove_outputs(self, all_jobs=False, error_jobs=False): - process_tools_mod = 'perform_ard_process' - process_tools_cls = 'PerformScnARD' - self.remove_job_outputs(process_tools_mod, process_tools_cls, all_jobs, error_jobs) - if __name__ == "__main__": py_script = os.path.abspath("perform_ard_process.py") #script_cmd = "singularity exec --bind /scratch/a.pfb:/scratch/a.pfb --bind /home/a.pfb:/home/a.pfb /scratch/a.pfb/sw_imgs/au-eoed-dev.sif python {}".format(py_script) script_cmd = "python {}".format(py_script) - create_tools = FindSen2ScnsGenDwnlds(cmd=script_cmd, sqlite_db_file="ard_ls_scns.db") + process_tools_mod = 'perform_ard_process' + process_tools_cls = 'PerformScnARD' + + create_tools = FindSen2ScnsGenDwnlds(cmd=script_cmd, sqlite_db_file="ard_ls_scns.db" + process_tools_mod=process_tools_mod, process_tools_cls=process_tools_cls) create_tools.parse_cmds() diff --git a/docs/source/conf.py b/docs/source/conf.py index d116743..c6f591a 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -61,9 +61,9 @@ # built documents. # # The short X.Y version. -version = '1.4' +version = '1.5' # The full version, including alpha/beta/rc tags. -release = '1.4.0' +release = '1.5.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/source/getstarted.rst b/docs/source/getstarted.rst index 09687d6..7a83e62 100644 --- a/docs/source/getstarted.rst +++ b/docs/source/getstarted.rst @@ -48,7 +48,7 @@ This python script has been saved in a file called ``perform_processing.py``:: def outputs_present(self, **kwargs): return True, dict() - + def remove_outputs(self, **kwargs): print("No outputs to remove") @@ -88,30 +88,19 @@ This python script has been saved in a file called ``gen_process_cmds.py``:: self.pop_params_db() self.create_shell_exe("exe_analysis.sh", "cmds.sh", 4, db_info_file=None) - - def run_check_outputs(self): - process_tools_mod = 'perform_processing' - process_tools_cls = 'ProcessJob' - time_sample_str = self.generate_readable_timestamp_str() - out_err_file = 'processing_errs_{}.txt'.format(time_sample_str) - out_non_comp_file = 'non_complete_errs_{}.txt'.format(time_sample_str) - self.check_job_outputs(process_tools_mod, process_tools_cls, out_err_file, out_non_comp_file) - - def run_remove_outputs(self, all_jobs=False, error_jobs=False): - process_tools_mod = 'perform_processing' - process_tools_cls = 'ProcessJob' - self.remove_job_outputs(process_tools_mod, process_tools_cls, all_jobs, error_jobs) - - if __name__ == "__main__": py_script = os.path.abspath("perform_processing.py") script_cmd = "python {}".format(py_script) - create_tools = CreateTestCmds(cmd=script_cmd, sqlite_db_file="test_jobs.db") + process_tools_mod = 'perform_processing' + process_tools_cls = 'ProcessJob' + + create_tools = CreateTestCmds(cmd=script_cmd, sqlite_db_file="test_jobs.db", + process_tools_mod=process_tools_mod, process_tools_cls=process_tools_cls) create_tools.parse_cmds() -This class generates the individual job parameters, in this case 100 jobs. The dict for each job has to be appended to the ``self.params`` list within the ``gen_command_info`` function. The ``run_gen_commands`` function needs to call the ``gen_command_info`` function but also specifies the output format (i.e., batch processing using GNU parallel on a local system or via slurm on a cluster). In this case it is via a GNU parallel and a shell script listing the commands. The ``run_check_outputs`` function is specifying the ``PBPTQProcessTool`` class name and module so it can be imported to check for job completion. Also the output files for the checking reports are specified. It is often useful to put a timestamp into the file name so files are not overwritten each time the outputs are checked. +This class generates the individual job parameters, in this case 100 jobs. The dict for each job has to be appended to the ``self.params`` list within the ``gen_command_info`` function. The ``run_gen_commands`` function needs to call the ``gen_command_info`` function but also specifies the output format (i.e., batch processing using GNU parallel on a local system or via slurm on a cluster). In this case it is via a GNU parallel and a shell script listing the commands. When the class ``CreateTestCmds`` is instantiated, the command to be executed for processing to occur (i.e., ``python perform_processing.py``) needs to be specified and the database file name and path is required. @@ -140,11 +129,11 @@ If you want the report outputted to a file run:: python gen_process_cmds.py --check -o report.json To remove outputs where an error occurred then you can use the following:: - + python gen_process_cmds.py --rmouts --error To remove outputs for all jobs then you can use the following:: - + python gen_process_cmds.py --rmouts --all Where you have had an error occur it can be useful to run a single task in isolation without the database recording any information and any exception being returned to the console rather than captured. This can be performed by calling the processing python file. For example, to process job 20, run the following command::