diff --git a/pyproject.toml b/pyproject.toml index 10f1dddf..03d00ab4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ dependencies = [ "pykwalify", "setuptools", "packaging", + "joblib", ] [project.license] diff --git a/src/west/app/project.py b/src/west/app/project.py index d5f4c705..f7a158fa 100644 --- a/src/west/app/project.py +++ b/src/west/app/project.py @@ -36,6 +36,12 @@ ) from west.manifest import is_group as is_project_group +JOBLIB_PRESENT = True +try: + from joblib import Parallel, delayed +except ModuleNotFoundError: + JOBLIB_PRESENT = False + # # Project-related or multi-repo commands, like "init", "update", # "diff", etc. @@ -952,6 +958,12 @@ def do_add_parser(self, parser_adder): parser.add_argument('--stats', action='store_true', help='''print performance statistics for update operations''') + if JOBLIB_PRESENT: + parser.add_argument('-j', '--jobs', nargs='?', const=-1, + default=1, type=int, action='store', + help='''Use multiple jobs to parallelize update process. + Pass -1 to use all available jobs. + ''') group = parser.add_argument_group( title='local project clone caches', @@ -1087,18 +1099,26 @@ def update_all(self): import_flags=ImportFlag.FORCE_PROJECTS) failed = [] - for project in self.manifest.projects: + + def project_update(project): if (isinstance(project, ManifestProject) or project.name in self.updated): - continue + return try: if not self.project_is_active(project): self.dbg(f'{project.name}: skipping inactive project') - continue - self.update(project) + return self.updated.add(project.name) + self.update(project) except subprocess.CalledProcessError: failed.append(project) + + if not JOBLIB_PRESENT or self.args.jobs == 1: + for project in self.manifest.projects: + project_update(project) + else: + Parallel(n_jobs=self.args.jobs, require='sharedmem')( + delayed(project_update)(project) for project in self.manifest.projects) self._handle_failed(self.args, failed) def update_importer(self, project, path): @@ -1159,13 +1179,22 @@ def update_some(self): projects = self._projects(self.args.projects) failed = [] - for project in projects: + + def project_update_some(project): if isinstance(project, ManifestProject): - continue + return try: self.update(project) except subprocess.CalledProcessError: failed.append(project) + + if not JOBLIB_PRESENT or self.args.jobs == 1: + for project in projects: + project_update_some(project) + else: + Parallel(n_jobs=self.args.jobs, require='sharedmem')( + delayed(project_update_some)(project) for project in projects) + self._handle_failed(self.args, failed) def toplevel_projects(self): diff --git a/tests/test_project.py b/tests/test_project.py index bada3b26..ff016585 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -447,14 +447,14 @@ def test_grep(west_init_tmpdir): assert re.search('west-commands', cmd('grep -- -- -commands')) - -def test_update_projects(west_init_tmpdir): +@pytest.mark.parametrize("options", ["", "-j 1", "-j 2", "-j"]) +def test_update_projects(options, west_init_tmpdir): # Test the 'west update' command. It calls through to the same backend # functions that are used for automatic updates and 'west init' # reinitialization. # create local repositories - cmd('update') + cmd('update ' + options) # Add commits to the local repos. ur = update_helper(west_init_tmpdir) @@ -644,7 +644,8 @@ def test_update_head_0(west_init_tmpdir): assert modified_files.strip() == "M CODEOWNERS", \ 'local zephyr change not preserved' -def test_update_some_with_imports(repos_tmpdir): +@pytest.mark.parametrize("options", ["", "-j 1", "-j 2", "-j -1"]) +def test_update_some_with_imports(options, repos_tmpdir): # 'west update project1 project2' should work fine even when # imports are used, as long as the relevant projects are all # defined in the manifest repository. @@ -685,19 +686,19 @@ def test_update_some_with_imports(repos_tmpdir): # Updating unknown projects should fail as always. with pytest.raises(subprocess.CalledProcessError): - cmd('update unknown-project', cwd=ws) + cmd(f'update {options} unknown-project', cwd=ws) # Updating a list of projects when some are resolved via project # imports must fail. with pytest.raises(subprocess.CalledProcessError): - cmd('update Kconfiglib net-tools', cwd=ws) + cmd(f'update {options} Kconfiglib net-tools', cwd=ws) # Updates of projects defined in the manifest repository or all # projects must succeed, and behave the same as if no imports # existed. - cmd('update net-tools', cwd=ws) + cmd(f'update {options} net-tools', cwd=ws) with pytest.raises(ManifestImportFailed): Manifest.from_topdir(topdir=ws) manifest = Manifest.from_topdir(topdir=ws, @@ -708,10 +709,10 @@ def test_update_some_with_imports(repos_tmpdir): assert net_tools_project.is_cloned() assert not zephyr_project.is_cloned() - cmd('update zephyr', cwd=ws) + cmd(f'update {options} zephyr', cwd=ws) assert zephyr_project.is_cloned() - cmd('update', cwd=ws) + cmd(f'update {options}', cwd=ws) manifest = Manifest.from_topdir(topdir=ws) assert manifest.get_projects(['Kconfiglib'])[0].is_cloned()