Modules refactoring

CHANGELOG.md

@@ -35,6 +35,10 @@
 * Check for `modules.json` for entries of modules that are not actually installed in the pipeline [[#1141](https://github.com/nf-core/tools/issues/1141)]
 * Added `<keywords>` argument to `nf-core modules list` for filtering the listed modules. ([#1139](https://github.com/nf-core/tools/issues/1139)
 * Fixed `nf-core modules create-test-yml` so it doesn't break when the output directory is supplied [[#1148](https://github.com/nf-core/tools/issues/1148)]
+* Updated `nf-core modules lint` to work with new directory structure [[#1159](https://github.com/nf-core/tools/issues/1159)]
+* Updated `nf-core modules install` and `modules.json` to work with new directory structure ([#1159](https://github.com/nf-core/tools/issues/1159))
+* Updated `nf-core modules remove` to work with new directory structure [[#1159](https://github.com/nf-core/tools/issues/1159)]
+* Restructured code and removed old table style in `nf-core modules list`
 
 #### Sync
 

nf_core/lint/modules_json.py

@@ -25,10 +25,11 @@ def modules_json(self):
 
         all_modules_passed = True
 
-        for key in modules_json["modules"].keys():
-            if not key in modules_command.module_names:
-                failed.append(f"Entry for `{key}` found in `modules.json` but module is not installed in pipeline.")
-                all_modules_passed = False
+        for repo in modules_json["repos"].keys():
+            for key in modules_json["repos"][repo].keys():
+                if not key in modules_command.module_names[repo]:
+                    failed.append(f"Entry for `{key}` found in `modules.json` but module is not installed in pipeline.")
+                    all_modules_passed = False
 
         if all_modules_passed:
             passed.append("Only installed modules found in `modules.json`")

nf_core/modules/create.py

@@ -60,14 +60,14 @@ def create(self):
 
         If <directory> is a clone of nf-core/modules, it creates or modifies the following files:
 
-        modules/software/tool/subtool/
+        modules/modules/tool/subtool/
             * main.nf
             * meta.yml
             * functions.nf
-        modules/tests/software/tool/subtool/
+        modules/tests/modules/tool/subtool/
             * main.nf
             * test.yml
-        tests/config/pytest_software.yml
+        tests/config/pytest_modules.yml
 
         The function will attempt to automatically find a Bioconda package called <tool>
         and matching Docker / Singularity images from BioContainers.
@@ -211,7 +211,7 @@ def create(self):
                 "Where applicable all sample-specific information e.g. 'id', 'single_end', 'read_group' "
                 "MUST be provided as an input via a Groovy Map called 'meta'. "
                 "This information may [italic]not[/] be required in some instances, for example "
-                "[link=https://github.com/nf-core/modules/blob/master/software/bwa/index/main.nf]indexing reference genome files[/link]."
+                "[link=https://github.com/nf-core/modules/blob/master/modules/bwa/index/main.nf]indexing reference genome files[/link]."
             )
         while self.has_meta is None:
             self.has_meta = rich.prompt.Confirm.ask(
@@ -222,29 +222,29 @@ def create(self):
         self.render_template()
 
         if self.repo_type == "modules":
-            # Add entry to pytest_software.yml
+            # Add entry to pytest_modules.yml
             try:
-                with open(os.path.join(self.directory, "tests", "config", "pytest_software.yml"), "r") as fh:
-                    pytest_software_yml = yaml.safe_load(fh)
+                with open(os.path.join(self.directory, "tests", "config", "pytest_modules.yml"), "r") as fh:
+                    pytest_modules_yml = yaml.safe_load(fh)
                 if self.subtool:
-                    pytest_software_yml[self.tool_name] = [
-                        f"software/{self.tool}/{self.subtool}/**",
-                        f"tests/software/{self.tool}/{self.subtool}/**",
+                    pytest_modules_yml[self.tool_name] = [
+                        f"modules/{self.tool}/{self.subtool}/**",
+                        f"tests/modules/{self.tool}/{self.subtool}/**",
                     ]
                 else:
-                    pytest_software_yml[self.tool_name] = [
-                        f"software/{self.tool}/**",
-                        f"tests/software/{self.tool}/**",
+                    pytest_modules_yml[self.tool_name] = [
+                        f"modules/{self.tool}/**",
+                        f"tests/modules/{self.tool}/**",
                     ]
-                pytest_software_yml = dict(sorted(pytest_software_yml.items()))
-                with open(os.path.join(self.directory, "tests", "config", "pytest_software.yml"), "w") as fh:
-                    yaml.dump(pytest_software_yml, fh, sort_keys=True, Dumper=nf_core.utils.custom_yaml_dumper())
+                pytest_modules_yml = dict(sorted(pytest_modules_yml.items()))
+                with open(os.path.join(self.directory, "tests", "config", "pytest_modules.yml"), "w") as fh:
+                    yaml.dump(pytest_modules_yml, fh, sort_keys=True, Dumper=nf_core.utils.custom_yaml_dumper())
             except FileNotFoundError as e:
-                raise UserWarning(f"Could not open 'tests/config/pytest_software.yml' file!")
+                raise UserWarning(f"Could not open 'tests/config/pytest_modules.yml' file!")
 
         new_files = list(self.file_paths.values())
         if self.repo_type == "modules":
-            new_files.append(os.path.join(self.directory, "tests", "config", "pytest_software.yml"))
+            new_files.append(os.path.join(self.directory, "tests", "config", "pytest_modules.yml"))
         log.info("Created / edited following files:\n  " + "\n  ".join(new_files))
 
     def render_template(self):
@@ -282,7 +282,7 @@ def get_repo_type(self, directory):
         # Determine repository type
         if os.path.exists(os.path.join(directory, "main.nf")):
             return "pipeline"
-        elif os.path.exists(os.path.join(directory, "software")):
+        elif os.path.exists(os.path.join(directory, "modules")):
             return "modules"
         else:
             raise UserWarning(
@@ -318,11 +318,11 @@ def get_module_dirs(self):
                 )
 
             # Set file paths
-            file_paths[os.path.join("software", "main.nf")] = module_file
+            file_paths[os.path.join("modules", "main.nf")] = module_file
 
         if self.repo_type == "modules":
-            software_dir = os.path.join(self.directory, "software", self.tool_dir)
-            test_dir = os.path.join(self.directory, "tests", "software", self.tool_dir)
+            software_dir = os.path.join(self.directory, "modules", self.tool_dir)
+            test_dir = os.path.join(self.directory, "tests", "modules", self.tool_dir)
 
             # Check if module directories exist already
             if os.path.exists(software_dir) and not self.force_overwrite:
@@ -332,8 +332,8 @@ def get_module_dirs(self):
                 raise UserWarning(f"Module test directory exists: '{test_dir}'. Use '--force' to overwrite")
 
             # If a subtool, check if there is a module called the base tool name already
-            parent_tool_main_nf = os.path.join(self.directory, "software", self.tool, "main.nf")
-            parent_tool_test_nf = os.path.join(self.directory, "tests", "software", self.tool, "main.nf")
+            parent_tool_main_nf = os.path.join(self.directory, "modules", self.tool, "main.nf")
+            parent_tool_test_nf = os.path.join(self.directory, "tests", "modules", self.tool, "main.nf")
             if self.subtool and os.path.exists(parent_tool_main_nf):
                 raise UserWarning(
                     f"Module '{parent_tool_main_nf}' exists already, cannot make subtool '{self.tool_name}'"
@@ -344,16 +344,16 @@ def get_module_dirs(self):
                 )
 
             # If no subtool, check that there isn't already a tool/subtool
-            tool_glob = glob.glob("{}/*/main.nf".format(os.path.join(self.directory, "software", self.tool)))
+            tool_glob = glob.glob("{}/*/main.nf".format(os.path.join(self.directory, "modules", self.tool)))
             if not self.subtool and tool_glob:
                 raise UserWarning(
                     f"Module subtool '{tool_glob[0]}' exists already, cannot make tool '{self.tool_name}'"
                 )
 
             # Set file paths - can be tool/ or tool/subtool/ so can't do in template directory structure
-            file_paths[os.path.join("software", "functions.nf")] = os.path.join(software_dir, "functions.nf")
-            file_paths[os.path.join("software", "main.nf")] = os.path.join(software_dir, "main.nf")
-            file_paths[os.path.join("software", "meta.yml")] = os.path.join(software_dir, "meta.yml")
+            file_paths[os.path.join("modules", "functions.nf")] = os.path.join(software_dir, "functions.nf")
+            file_paths[os.path.join("modules", "main.nf")] = os.path.join(software_dir, "main.nf")
+            file_paths[os.path.join("modules", "meta.yml")] = os.path.join(software_dir, "meta.yml")
             file_paths[os.path.join("tests", "main.nf")] = os.path.join(test_dir, "main.nf")
             file_paths[os.path.join("tests", "test.yml")] = os.path.join(test_dir, "test.yml")
 

nf_core/modules/install.py

@@ -1,6 +1,4 @@
 import os
-import sys
-import json
 import questionary
 import logging
 
@@ -13,7 +11,7 @@
 
 
 class ModuleInstall(ModuleCommand):
-    def __init__(self, pipeline_dir, module=None, force=False, latest=False, sha=None):
+    def __init__(self, pipeline_dir, force=False, latest=False, sha=None):
         super().__init__(pipeline_dir)
         self.force = force
         self.latest = latest
@@ -49,10 +47,9 @@ def install(self, module):
             log.error("Module '{}' not found in list of available modules.".format(module))
             log.info("Use the command 'nf-core modules list' to view available software")
             return False
+
         # Set the install folder based on the repository name
-        install_folder = ["nf-core", "software"]
-        if not self.modules_repo.name == "nf-core/modules":
-            install_folder = ["external"]
+        install_folder = [self.modules_repo.owner, self.modules_repo.repo]
 
         # Compute the module directory
         module_dir = os.path.join(self.dir, "modules", *install_folder, module)
@@ -62,12 +59,15 @@ def install(self, module):
         if not modules_json:
             return False
 
-        current_entry = modules_json["modules"].get(module)
+        if self.modules_repo.name in modules_json["repos"]:
+            current_entry = modules_json["repos"][self.modules_repo.name].get(module)
+        else:
+            current_entry = None
 
         if current_entry is not None and self.sha is None:
             # Fetch the latest commit for the module
             current_version = current_entry["git_sha"]
-            git_log = get_module_git_log(module, per_page=1, page_nbr=1)
+            git_log = get_module_git_log(module, modules_repo=self.modules_repo, per_page=1, page_nbr=1)
             if len(git_log) == 0:
                 log.error(f"Was unable to fetch version of module '{module}'")
                 return False
@@ -107,7 +107,7 @@ def install(self, module):
             if self.latest:
                 # Fetch the latest commit for the module
                 if latest_version is None:
-                    git_log = get_module_git_log(module, per_page=1, page_nbr=1)
+                    git_log = get_module_git_log(module, modules_repo=self.modules_repo, per_page=1, page_nbr=1)
                     if len(git_log) == 0:
                         log.error(f"Was unable to fetch version of module '{module}'")
                         return False
@@ -122,15 +122,17 @@ def install(self, module):
                     log.error(e)
                     return False
 
-        log.info("Installing {}".format(module))
-        log.debug("Installing module '{}' at modules hash {}".format(module, self.modules_repo.modules_current_hash))
+        log.info(f"Installing {module}")
+        log.debug(
+            f"Installing module '{module}' at modules hash {self.modules_repo.modules_current_hash} from {self.modules_repo.name}"
+        )
 
         # Download module files
         if not self.download_module_file(module, version, install_folder, module_dir):
             return False
 
         # Update module.json with newly installed module
-        self.update_modules_json(modules_json, module, version)
+        self.update_modules_json(modules_json, self.modules_repo.name, module, version)
         return True
 
     def check_module_files_installed(self, module_name, module_dir):
@@ -155,10 +157,12 @@ def prompt_module_version_sha(self, module, installed_sha=None):
         )
         git_sha = ""
         page_nbr = 1
-        next_page_commits = get_module_git_log(module, per_page=10, page_nbr=page_nbr)
+        next_page_commits = get_module_git_log(module, modules_repo=self.modules_repo, per_page=10, page_nbr=page_nbr)
         while git_sha is "":
             commits = next_page_commits
-            next_page_commits = get_module_git_log(module, per_page=10, page_nbr=page_nbr + 1)
+            next_page_commits = get_module_git_log(
+                module, modules_repo=self.modules_repo, per_page=10, page_nbr=page_nbr + 1
+            )
             choices = []
             for title, sha in map(lambda commit: (commit["trunc_message"], commit["git_sha"]), commits):
 

nf_core/modules/lint/__init__.py

@@ -95,7 +95,7 @@ def lint(self, module=None, all_modules=False, print_results=True, show_passed=F
         Lint all or one specific module
 
         First gets a list of all local modules (in modules/local/process) and all modules
-        installed from nf-core (in modules/nf-core/software)
+        installed from nf-core (in modules/nf-core/modules)
 
         For all nf-core modules, the correct file structure is assured and important
         file content is verified. If directory subject to linting is a clone of 'nf-core/modules',
@@ -258,7 +258,7 @@ def get_installed_modules(self):
         local_modules = []
         nfcore_modules = []
         local_modules_dir = None
-        nfcore_modules_dir = os.path.join(self.dir, "modules", "nf-core", "software")
+        nfcore_modules_dir = os.path.join(self.dir, "modules", "nf-core", "modules")
 
         # Get local modules
         if self.repo_type == "pipeline":
@@ -271,7 +271,7 @@ def get_installed_modules(self):
 
         # nf-core/modules
         if self.repo_type == "modules":
-            nfcore_modules_dir = os.path.join(self.dir, "software")
+            nfcore_modules_dir = os.path.join(self.dir, "modules")
 
         # Get nf-core modules
         if os.path.exists(nfcore_modules_dir):

nf_core/modules/lint/functions_nf.py

@@ -11,6 +11,8 @@ def functions_nf(module_lint_object, module):
     Lint a functions.nf file
     Verifies that the file exists and contains all necessary functions
     """
+    local_copy = None
+    template_copy = None
     try:
         with open(module.function_nf, "r") as fh:
             lines = fh.readlines()
@@ -45,7 +47,10 @@ def functions_nf(module_lint_object, module):
         log.error(f"Could not open {template_copy_path}")
 
     # Compare the files
-    if local_copy != template_copy:
-        module.failed.append(("function_nf_comparison", "New version of functions.nf available", module.function_nf))
-    else:
-        module.passed.append(("function_nf_comparison", "functions.nf is up to date", module.function_nf))
+    if local_copy and template_copy:
+        if local_copy != template_copy:
+            module.failed.append(
+                ("function_nf_comparison", "New version of functions.nf available", module.function_nf)
+            )
+        else:
+            module.passed.append(("function_nf_comparison", "functions.nf is up to date", module.function_nf))

nf_core/modules/lint/module_changes.py

@@ -19,11 +19,11 @@ def module_changes(module_lint_object, module):
     files_to_check = ["main.nf", "functions.nf", "meta.yml"]
 
     # Loop over nf-core modules
-    module_base_url = f"https://raw.githubusercontent.com/{module_lint_object.modules_repo.name}/{module_lint_object.modules_repo.branch}/software/{module.module_name}/"
+    module_base_url = f"https://raw.githubusercontent.com/{module_lint_object.modules_repo.name}/{module_lint_object.modules_repo.branch}/modules/{module.module_name}/"
 
     # If module.git_sha specified, check specific commit version for changes
     if module.git_sha:
-        module_base_url = f"https://raw.githubusercontent.com/{module_lint_object.modules_repo.name}/{module.git_sha}/software/{module.module_name}/"
+        module_base_url = f"https://raw.githubusercontent.com/{module_lint_object.modules_repo.name}/{module.git_sha}/modules/{module.module_name}/"
 
     for f in files_to_check:
         # open local copy, continue if file not found (a failed message has already been issued in this case)

nf_core/modules/lint/module_tests.py

@@ -27,17 +27,17 @@ def module_tests(module_lint_object, module):
     else:
         module.failed.append(("test_main_exists", "test `main.nf` does not exist", module.test_main_nf))
 
-    # Check that entry in pytest_software.yml exists
+    # Check that entry in pytest_modules.yml exists
     try:
-        pytest_yml_path = os.path.join(module.base_dir, "tests", "config", "pytest_software.yml")
+        pytest_yml_path = os.path.join(module.base_dir, "tests", "config", "pytest_modules.yml")
         with open(pytest_yml_path, "r") as fh:
             pytest_yml = yaml.safe_load(fh)
             if module.module_name in pytest_yml.keys():
-                module.passed.append(("test_pytest_yml", "correct entry in pytest_software.yml", pytest_yml_path))
+                module.passed.append(("test_pytest_yml", "correct entry in pytest_modules.yml", pytest_yml_path))
             else:
-                module.failed.append(("test_pytest_yml", "missing entry in pytest_software.yml", pytest_yml_path))
+                module.failed.append(("test_pytest_yml", "missing entry in pytest_modules.yml", pytest_yml_path))
     except FileNotFoundError as e:
-        module.failed.append(("test_pytest_yml", f"Could not open pytest_software.yml file", pytest_yml_path))
+        module.failed.append(("test_pytest_yml", f"Could not open pytest_modules.yml file", pytest_yml_path))
 
     # Lint the test.yml file
     try:

nf_core/modules/lint/module_version.py

@@ -25,17 +25,20 @@ def module_version(module_lint_object, module):
 
     # Verify that a git_sha exists in the `modules.json` file for this module
     try:
-        module_entry = module_lint_object.modules_json["modules"][module.module_name]
+        module_entry = module_lint_object.modules_json["repos"][module_lint_object.modules_repo.name][
+            module.module_name
+        ]
         git_sha = module_entry["git_sha"]
         module.git_sha = git_sha
         module.passed.append(("git_sha", "Found git_sha entry in `modules.json`", modules_json_path))
-    except KeyError:
-        module.failed.append(("git_sha", "No git_sha entry in `modules.json`", modules_json_path))
 
-    # Check whether a new version is available
-    module_git_log = nf_core.modules.module_utils.get_module_git_log(module.module_name)
+        # Check whether a new version is available
+        module_git_log = nf_core.modules.module_utils.get_module_git_log(module.module_name)
+
+        if git_sha == module_git_log[0]["git_sha"]:
+            module.passed.append(("module_version", "Module is the latest version", module.module_dir))
+        else:
+            module.warned.append(("module_version", "New version available", module.module_dir))
 
-    if git_sha == module_git_log[0]["git_sha"]:
-        module.passed.append(("module_version", "Module is the latest version", module.module_dir))
-    else:
-        module.warned.append(("module_version", "New version available", module.module_dir))
+    except KeyError:
+        module.failed.append(("git_sha", "No git_sha entry in `modules.json`", modules_json_path))