From 4b73bfeb967ee9f57a0410276f7c39e784f0846f Mon Sep 17 00:00:00 2001 From: George Coulouris Date: Fri, 6 May 2022 19:09:15 -0400 Subject: [PATCH 1/6] JIRA PGAPX-1018 echo installation directory free space during update --- scripts/pgap.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/pgap.py b/scripts/pgap.py index 5628f47..c06090d 100755 --- a/scripts/pgap.py +++ b/scripts/pgap.py @@ -653,6 +653,8 @@ def str2sec(s): return str2sec(self.args.timeout) def update(self): + print(f"installation directory: {self.install_dir}") + subprocess.run(["/bin/df", "-k", self.install_dir]) self.update_self() threads = list() docker_thread = mp.Process(target = self.install_docker, name='docker image pull') From 0454b07df9acc67404a94e8f11a877132834bef5 Mon Sep 17 00:00:00 2001 From: George Coulouris Date: Tue, 7 Jun 2022 09:04:34 -0400 Subject: [PATCH 2/6] JIRA PGAPX-1025 use USERPROFILE on Windows --- scripts/pgap.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/pgap.py b/scripts/pgap.py index c06090d..6b290d4 100755 --- a/scripts/pgap.py +++ b/scripts/pgap.py @@ -486,7 +486,12 @@ def __init__(self, args): self.ani_hr_output = None self.branch = self.get_branch() self.repo = self.get_repo() - self.install_dir = os.environ.get('PGAP_INPUT_DIR',os.environ['HOME']+'/.pgap') + + if platform.system() == 'Windows': + self.install_dir = os.environ.get('PGAP_INPUT_DIR',os.environ['USERPROFILE']+'/.pgap') + else: + self.install_dir = os.environ.get('PGAP_INPUT_DIR',os.environ['HOME']+'/.pgap') + self.local_version = self.get_local_version() if self.args.no_internet: self.remote_versions = [self.local_version] From 28874bfaf2c07c70e003e181cf84e6357c8cae2f Mon Sep 17 00:00:00 2001 From: "Badretdin, Azat" Date: Wed, 22 Jun 2022 14:37:25 -0400 Subject: [PATCH 3/6] turned out -subject_allowlist is a manifest; fixed; JIRA: PGAPX-1024 --- progs/align_filter.cwl | 38 ++++++-------------------------------- 1 file changed, 6 insertions(+), 32 deletions(-) diff --git a/progs/align_filter.cwl b/progs/align_filter.cwl index 6e7b82d..bac8c89 100644 --- a/progs/align_filter.cwl +++ b/progs/align_filter.cwl @@ -4,44 +4,14 @@ label: "align_filter" class: CommandLineTool baseCommand: align_filter -# this is only one example -# -# -# -# /panfs/pan1.be-md.ncbi.nlm.nih.gov/gpipe/bacterial_pipeline/system/2018-03-13.build2663/bin/align_filter \ -# -onon-match \ -# /panfs/pan1.be-md.ncbi.nlm.nih.gov/gpipe/bacterial_pipeline/data56/Mycoplasma_genitalium_G37/Mycoplasma_genitalium_External_PGAP.4585524/4829637/align_filter_sa.455675142/out/align-nomatch.asn \ -# -asn-cache \ -# /panfs/pan1.be-md.ncbi.nlm.nih.gov/gpipe/bacterial_pipeline/data56/Mycoplasma_genitalium_G37/Mycoplasma_genitalium_External_PGAP.4585524/sequence_cache \ -# -filter \ -# 'pct_coverage \ -# >= \ -# 20' \ -# -ifmt \ -# seq-align \ -# -input-manifest \ -# /panfs/pan1.be-md.ncbi.nlm.nih.gov/gpipe/bacterial_pipeline/data56/Mycoplasma_genitalium_G37/Mycoplasma_genitalium_External_PGAP.4585524/4829637/align_filter_sa.455675142/inp/align.mft \ -# -logfile \ -# /panfs/pan1.be-md.ncbi.nlm.nih.gov/gpipe/bacterial_pipeline/data56/Mycoplasma_genitalium_G37/Mycoplasma_genitalium_External_PGAP.4585524/4829637/align_filter_sa.455675142/log/align_filter.log \ -# -o \ -# /panfs/pan1.be-md.ncbi.nlm.nih.gov/gpipe/bacterial_pipeline/data56/Mycoplasma_genitalium_G37/Mycoplasma_genitalium_External_PGAP.4585524/4829637/align_filter_sa.455675142/out/align.asn -# -# -# -# /panfs/pan1.be-md.ncbi.nlm.nih.gov/gpipe/bacterial_pipeline/system/2018-03-13.build2663/bin/gp_register_stats \ -# -ifmt \ -# seq-align \ -# -input \ -# /panfs/pan1.be-md.ncbi.nlm.nih.gov/gpipe/bacterial_pipeline/data56/Mycoplasma_genitalium_G37/Mycoplasma_genitalium_External_PGAP.4585524/4829637/align_filter_sa.455675142/out/align.asn -# -# -# requirements: - class: InlineJavascriptRequirement - class: InitialWorkDirRequirement listing: - entryname: aligns.mft entry: ${var blob = '# aligns.mft created for align_filter from input "input" Array of Files\n'; for (var i = 0; i < inputs.input.length; i++) { blob += inputs.input[i].path + '\n'; } return blob; } + - entryname: subject_allowlist.mft + entry: ${var blob = '# subject_allowlist.mft created for align_filter from input "subject_allowlist" File\n'; if ( subject_allowlist == null) { return blob; } else { blob += inputs.subject_allowlist.path + '\n'; return blob; }} inputs: asn_cache: @@ -64,6 +34,7 @@ inputs: default: aligns.mft inputBinding: prefix: -input-manifest + subject_allowlist.mft nogenbank: type: boolean default: true @@ -71,6 +42,9 @@ inputs: prefix: -nogenbank subject_allowlist: type: File? + subject_allowlist_mft: + type: string? + default: subject_allowlist.mft inputBinding: prefix: -subject-allowlist onon_match_name: From 735876c43997bd1bf1a24413ba01c5528e347431 Mon Sep 17 00:00:00 2001 From: "Badretdin, Azat" Date: Thu, 23 Jun 2022 05:19:05 -0400 Subject: [PATCH 4/6] typo; JIRA: PGAPX-1024 --- progs/align_filter.cwl | 1 - 1 file changed, 1 deletion(-) diff --git a/progs/align_filter.cwl b/progs/align_filter.cwl index bac8c89..a6e7ee8 100644 --- a/progs/align_filter.cwl +++ b/progs/align_filter.cwl @@ -34,7 +34,6 @@ inputs: default: aligns.mft inputBinding: prefix: -input-manifest - subject_allowlist.mft nogenbank: type: boolean default: true From e20f4165c932ce589fefa4704a5efda181ff1d08 Mon Sep 17 00:00:00 2001 From: "Badretdin, Azat" Date: Thu, 23 Jun 2022 09:25:47 -0400 Subject: [PATCH 5/6] typo; JIRA: PGAPX-1024 --- progs/align_filter.cwl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/progs/align_filter.cwl b/progs/align_filter.cwl index a6e7ee8..38ff64e 100644 --- a/progs/align_filter.cwl +++ b/progs/align_filter.cwl @@ -11,7 +11,7 @@ requirements: - entryname: aligns.mft entry: ${var blob = '# aligns.mft created for align_filter from input "input" Array of Files\n'; for (var i = 0; i < inputs.input.length; i++) { blob += inputs.input[i].path + '\n'; } return blob; } - entryname: subject_allowlist.mft - entry: ${var blob = '# subject_allowlist.mft created for align_filter from input "subject_allowlist" File\n'; if ( subject_allowlist == null) { return blob; } else { blob += inputs.subject_allowlist.path + '\n'; return blob; }} + entry: ${var blob = '# subject_allowlist.mft created for align_filter from input "subject_allowlist" File\n'; if ( inputs.subject_allowlist == null) { return blob; } else { blob += inputs.subject_allowlist.path + '\n'; return blob; }} inputs: asn_cache: From b17acbb4cc327b08901c2be226a2a16513e850df Mon Sep 17 00:00:00 2001 From: "Badretdin, Azat" Date: Mon, 1 Aug 2022 08:52:15 -0400 Subject: [PATCH 6/6] formally add 'apptainer' to the list of allowed things; JIRA: PGAP-1034 --- scripts/pgap.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/pgap.py b/scripts/pgap.py index 5628f47..1bb950e 100755 --- a/scripts/pgap.py +++ b/scripts/pgap.py @@ -174,9 +174,9 @@ def __init__(self, params, local_input, pipeline): else: self.submol = None self.yaml = self.create_inputfile(local_input) - if (self.params.docker_type == 'singularity'): + if self.params.docker_type in ['singularity', 'apptainer']: self.make_singularity_cmd() - elif (self.params.docker_type == 'podman'): + elif self.params.docker_type == 'podman': self.make_podman_cmd() else: self.make_docker_cmd() @@ -395,7 +395,7 @@ def check_runtime_setting(settings, value, min): if settings[value] != 'unlimited' and settings[value] < min: print('WARNING: {} is less than the recommended value of {}'.format(value, min)) - if (self.params.docker_type == 'singularity'): + if (self.params.docker_type in ['singularity', 'apptainer']): singularity_docker_image = self.params.docker_image if self.params.args.container_path else "docker://"+self.params.docker_image cmd = [self.params.docker_cmd, 'exec', '--bind', '{}:/cwd:ro'.format(os.getcwd()), singularity_docker_image, 'bash', '-c', 'df -k /cwd /tmp ; ulimit -a ; cat /proc/{meminfo,cpuinfo}'] @@ -604,7 +604,7 @@ def get_output_dir(self): return outputdir def get_docker_info(self): - docker_type_alternatives = ['docker', 'podman', 'singularity'] + docker_type_alternatives = ['docker', 'podman', 'singularity', 'apptainer'] if self.args.docker: self.docker_cmd = shutil.which(self.args.docker) else: @@ -674,7 +674,7 @@ def update(self): self.write_version() def install_docker(self): - if self.docker_type == 'singularity': + if self.docker_type in ['singularity', 'apptainer']: sif = self.docker_image.replace("ncbi/pgap:", "pgap_") + ".sif" try: subprocess.run([self.docker_cmd, 'sif', 'list', sif], @@ -847,7 +847,7 @@ def main(): If flag is set, run ANI first, then PGAP, overriding the organism name provided by the user in the input YAML with the value returned by ANI Predicted organism. Obviously both actions need to be requested for this flag to take effect ''') parser.add_argument('-D', '--docker', metavar='path', - help='Docker-compatible executable (e.g. docker, podman, singularity), which may include a full path like /usr/bin/docker') + help='Docker-compatible executable (e.g. docker, podman, apptainer), which may include a full path like /usr/bin/docker') parser.add_argument('-o', '--output', metavar='path', default='output', help='Output directory to be created, which may include a full path') parser.add_argument('-t', '--timeout', default='24:00:00', help=argparse.SUPPRESS)