diff --git a/web/client/codechecker_client/cmd/store.py b/web/client/codechecker_client/cmd/store.py index e6242cabc4..d8ded1c114 100644 --- a/web/client/codechecker_client/cmd/store.py +++ b/web/client/codechecker_client/cmd/store.py @@ -24,7 +24,7 @@ import zipfile import zlib -import concurrent.futures +from concurrent.futures import ProcessPoolExecutor from collections import namedtuple @@ -476,7 +476,7 @@ def filter_source_files_with_comments(source_file_info, main_report_positions): if sys.platform == "win32": return get_source_file_with_comments(jobs) else: - with concurrent.futures.ProcessPoolExecutor() as executor: + with ProcessPoolExecutor() as executor: return get_source_file_with_comments(jobs, executor.map) @@ -581,8 +581,13 @@ def parse_report_files(report_files: Set[str], zip_iter=map): missing_source_files) -def get_blame_info(repo: Repo, file_path: str): - """ Get blame info for the given file in the given git repo. """ +def get_blame_info(file_path: str): + """ Get blame info for the given file. """ + try: + repo = Repo(file_path, search_parent_directories=True) + except InvalidGitRepositoryError: + return + tracking_branch = None try: # If a commit is checked out, accessing the active_branch member will @@ -618,10 +623,22 @@ def get_blame_info(repo: Repo, file_path: str): 'from': b.linenos[0], 'to': b.linenos[-1], 'commit': commit.hexsha}) + + LOG.debug("Collected blame info for %s", file_path) + return res except Exception as ex: - LOG.warning("Failed to get blame information for %s: %s", - file_path, ex) + LOG.debug("Failed to get blame information for %s: %s", file_path, ex) + + +def collect_blame_info_for_files(file_paths: List[str], zip_iter=map): + """ Collect blame information for the given file paths. """ + file_blame_info = {} + for file_path, blame_info in zip(file_paths, + zip_iter(get_blame_info, file_paths)): + file_blame_info[file_path] = blame_info + + return file_blame_info def assemble_zip(inputs, zip_file, client): @@ -645,7 +662,7 @@ def assemble_zip(inputs, zip_file, client): changed_files, missing_source_files) = parse_report_files(report_files) else: - with concurrent.futures.ProcessPoolExecutor() as executor: + with ProcessPoolExecutor() as executor: (source_file_info, main_report_positions, files_to_compress, @@ -721,26 +738,39 @@ def assemble_zip(inputs, zip_file, client): zipf.write(ftc, zip_target) + collected_file_paths = [] for f, h in file_to_hash.items(): if h in necessary_hashes or h in file_hash_with_review_status: LOG.debug("File contents for '%s' needed by the server", f) file_path = os.path.join('root', f.lstrip('/')) + collected_file_paths.append(f) try: zipf.getinfo(file_path) except KeyError: zipf.write(f, file_path) - try: - repo = Repo(f, search_parent_directories=True) - blame_info = get_blame_info(repo, f) if repo else None - if blame_info: - zipf.writestr( - os.path.join('blame', f.lstrip('/')), - json.dumps(blame_info)) - except InvalidGitRepositoryError: - pass + # Currently ProcessPoolExecutor fails completely in windows. + # Reason is most likely combination of venv and fork() not + # being present in windows, so stuff like setting up + # PYTHONPATH in parent CodeChecker before store is executed + # are lost. + if sys.platform == "win32": + file_blame_info = collect_blame_info_for_files( + collected_file_paths) + else: + with ProcessPoolExecutor() as executor: + file_blame_info = collect_blame_info_for_files( + collected_file_paths, executor.map) + + # Add blame information to the zip for the files which will be sent to + # the server if exist. + for f, blame_info in file_blame_info.items(): + if blame_info: + zipf.writestr( + os.path.join('blame', f.lstrip('/')), + json.dumps(blame_info)) zipf.writestr('content_hashes.json', json.dumps(file_to_hash))